2 # Copyright (c) 2016 The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test compact blocks (BIP 152).
7 Version 1 compact blocks are pre-segwit (txids)
8 Version 2 compact blocks are post-segwit (wtxids)
11 from test_framework
.mininode
import *
12 from test_framework
.test_framework
import BitcoinTestFramework
13 from test_framework
.util
import *
14 from test_framework
.blocktools
import create_block
, create_coinbase
, add_witness_commitment
15 from test_framework
.script
import CScript
, OP_TRUE
17 # TestNode: A peer we use to send messages to bitcoind, and store responses.
18 class TestNode(NodeConnCB
):
21 self
.last_sendcmpct
= []
22 self
.block_announced
= False
23 # Store the hashes of blocks we've seen announced.
24 # This is for synchronizing the p2p message traffic,
25 # so we can eg wait until a particular block is announced.
26 self
.announced_blockhashes
= set()
28 def on_sendcmpct(self
, conn
, message
):
29 self
.last_sendcmpct
.append(message
)
31 def on_cmpctblock(self
, conn
, message
):
32 self
.block_announced
= True
33 self
.last_message
["cmpctblock"].header_and_shortids
.header
.calc_sha256()
34 self
.announced_blockhashes
.add(self
.last_message
["cmpctblock"].header_and_shortids
.header
.sha256
)
36 def on_headers(self
, conn
, message
):
37 self
.block_announced
= True
38 for x
in self
.last_message
["headers"].headers
:
40 self
.announced_blockhashes
.add(x
.sha256
)
42 def on_inv(self
, conn
, message
):
43 for x
in self
.last_message
["inv"].inv
:
45 self
.block_announced
= True
46 self
.announced_blockhashes
.add(x
.hash)
48 # Requires caller to hold mininode_lock
49 def received_block_announcement(self
):
50 return self
.block_announced
52 def clear_block_announcement(self
):
54 self
.block_announced
= False
55 self
.last_message
.pop("inv", None)
56 self
.last_message
.pop("headers", None)
57 self
.last_message
.pop("cmpctblock", None)
59 def get_headers(self
, locator
, hashstop
):
60 msg
= msg_getheaders()
61 msg
.locator
.vHave
= locator
62 msg
.hashstop
= hashstop
63 self
.connection
.send_message(msg
)
65 def send_header_for_blocks(self
, new_blocks
):
66 headers_message
= msg_headers()
67 headers_message
.headers
= [CBlockHeader(b
) for b
in new_blocks
]
68 self
.send_message(headers_message
)
70 def request_headers_and_sync(self
, locator
, hashstop
=0):
71 self
.clear_block_announcement()
72 self
.get_headers(locator
, hashstop
)
73 wait_until(self
.received_block_announcement
, timeout
=30, lock
=mininode_lock
)
74 self
.clear_block_announcement()
76 # Block until a block announcement for a particular block hash is
78 def wait_for_block_announcement(self
, block_hash
, timeout
=30):
80 return (block_hash
in self
.announced_blockhashes
)
81 wait_until(received_hash
, timeout
=timeout
, lock
=mininode_lock
)
83 def send_await_disconnect(self
, message
, timeout
=30):
84 """Sends a message to the node and wait for disconnect.
86 This is used when we want to send a message into the node that we expect
87 will get us disconnected, eg an invalid block."""
88 self
.send_message(message
)
89 wait_until(lambda: not self
.connected
, timeout
=timeout
, lock
=mininode_lock
)
91 class CompactBlocksTest(BitcoinTestFramework
):
92 def set_test_params(self
):
93 self
.setup_clean_chain
= True
94 # Node0 = pre-segwit, node1 = segwit-aware
96 self
.extra_args
= [["-vbparams=segwit:0:0"], ["-txindex"]]
99 def build_block_on_tip(self
, node
, segwit
=False):
100 height
= node
.getblockcount()
101 tip
= node
.getbestblockhash()
102 mtp
= node
.getblockheader(tip
)['mediantime']
103 block
= create_block(int(tip
, 16), create_coinbase(height
+ 1), mtp
+ 1)
106 add_witness_commitment(block
)
110 # Create 10 more anyone-can-spend utxo's for testing.
111 def make_utxos(self
):
112 # Doesn't matter which node we use, just use node0.
113 block
= self
.build_block_on_tip(self
.nodes
[0])
114 self
.test_node
.send_and_ping(msg_block(block
))
115 assert(int(self
.nodes
[0].getbestblockhash(), 16) == block
.sha256
)
116 self
.nodes
[0].generate(100)
118 total_value
= block
.vtx
[0].vout
[0].nValue
119 out_value
= total_value
// 10
121 tx
.vin
.append(CTxIn(COutPoint(block
.vtx
[0].sha256
, 0), b
''))
123 tx
.vout
.append(CTxOut(out_value
, CScript([OP_TRUE
])))
126 block2
= self
.build_block_on_tip(self
.nodes
[0])
127 block2
.vtx
.append(tx
)
128 block2
.hashMerkleRoot
= block2
.calc_merkle_root()
130 self
.test_node
.send_and_ping(msg_block(block2
))
131 assert_equal(int(self
.nodes
[0].getbestblockhash(), 16), block2
.sha256
)
132 self
.utxos
.extend([[tx
.sha256
, i
, out_value
] for i
in range(10)])
135 # Test "sendcmpct" (between peers preferring the same version):
136 # - No compact block announcements unless sendcmpct is sent.
137 # - If sendcmpct is sent with version > preferred_version, the message is ignored.
138 # - If sendcmpct is sent with boolean 0, then block announcements are not
139 # made with compact blocks.
140 # - If sendcmpct is then sent with boolean 1, then new block announcements
141 # are made with compact blocks.
142 # If old_node is passed in, request compact blocks with version=preferred-1
143 # and verify that it receives block announcements via compact block.
144 def test_sendcmpct(self
, node
, test_node
, preferred_version
, old_node
=None):
145 # Make sure we get a SENDCMPCT message from our peer
146 def received_sendcmpct():
147 return (len(test_node
.last_sendcmpct
) > 0)
148 wait_until(received_sendcmpct
, timeout
=30, lock
=mininode_lock
)
150 # Check that the first version received is the preferred one
151 assert_equal(test_node
.last_sendcmpct
[0].version
, preferred_version
)
152 # And that we receive versions down to 1.
153 assert_equal(test_node
.last_sendcmpct
[-1].version
, 1)
154 test_node
.last_sendcmpct
= []
156 tip
= int(node
.getbestblockhash(), 16)
158 def check_announcement_of_new_block(node
, peer
, predicate
):
159 peer
.clear_block_announcement()
160 block_hash
= int(node
.generate(1)[0], 16)
161 peer
.wait_for_block_announcement(block_hash
, timeout
=30)
162 assert(peer
.block_announced
)
165 assert predicate(peer
), (
166 "block_hash={!r}, cmpctblock={!r}, inv={!r}".format(
167 block_hash
, peer
.last_message
.get("cmpctblock", None), peer
.last_message
.get("inv", None)))
169 # We shouldn't get any block announcements via cmpctblock yet.
170 check_announcement_of_new_block(node
, test_node
, lambda p
: "cmpctblock" not in p
.last_message
)
172 # Try one more time, this time after requesting headers.
173 test_node
.request_headers_and_sync(locator
=[tip
])
174 check_announcement_of_new_block(node
, test_node
, lambda p
: "cmpctblock" not in p
.last_message
and "inv" in p
.last_message
)
176 # Test a few ways of using sendcmpct that should NOT
177 # result in compact block announcements.
178 # Before each test, sync the headers chain.
179 test_node
.request_headers_and_sync(locator
=[tip
])
181 # Now try a SENDCMPCT message with too-high version
182 sendcmpct
= msg_sendcmpct()
183 sendcmpct
.version
= preferred_version
+1
184 sendcmpct
.announce
= True
185 test_node
.send_and_ping(sendcmpct
)
186 check_announcement_of_new_block(node
, test_node
, lambda p
: "cmpctblock" not in p
.last_message
)
188 # Headers sync before next test.
189 test_node
.request_headers_and_sync(locator
=[tip
])
191 # Now try a SENDCMPCT message with valid version, but announce=False
192 sendcmpct
.version
= preferred_version
193 sendcmpct
.announce
= False
194 test_node
.send_and_ping(sendcmpct
)
195 check_announcement_of_new_block(node
, test_node
, lambda p
: "cmpctblock" not in p
.last_message
)
197 # Headers sync before next test.
198 test_node
.request_headers_and_sync(locator
=[tip
])
200 # Finally, try a SENDCMPCT message with announce=True
201 sendcmpct
.version
= preferred_version
202 sendcmpct
.announce
= True
203 test_node
.send_and_ping(sendcmpct
)
204 check_announcement_of_new_block(node
, test_node
, lambda p
: "cmpctblock" in p
.last_message
)
206 # Try one more time (no headers sync should be needed!)
207 check_announcement_of_new_block(node
, test_node
, lambda p
: "cmpctblock" in p
.last_message
)
209 # Try one more time, after turning on sendheaders
210 test_node
.send_and_ping(msg_sendheaders())
211 check_announcement_of_new_block(node
, test_node
, lambda p
: "cmpctblock" in p
.last_message
)
213 # Try one more time, after sending a version-1, announce=false message.
214 sendcmpct
.version
= preferred_version
-1
215 sendcmpct
.announce
= False
216 test_node
.send_and_ping(sendcmpct
)
217 check_announcement_of_new_block(node
, test_node
, lambda p
: "cmpctblock" in p
.last_message
)
219 # Now turn off announcements
220 sendcmpct
.version
= preferred_version
221 sendcmpct
.announce
= False
222 test_node
.send_and_ping(sendcmpct
)
223 check_announcement_of_new_block(node
, test_node
, lambda p
: "cmpctblock" not in p
.last_message
and "headers" in p
.last_message
)
225 if old_node
is not None:
226 # Verify that a peer using an older protocol version can receive
227 # announcements from this node.
228 sendcmpct
.version
= preferred_version
-1
229 sendcmpct
.announce
= True
230 old_node
.send_and_ping(sendcmpct
)
232 old_node
.request_headers_and_sync(locator
=[tip
])
233 check_announcement_of_new_block(node
, old_node
, lambda p
: "cmpctblock" in p
.last_message
)
235 # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last.
236 def test_invalid_cmpctblock_message(self
):
237 self
.nodes
[0].generate(101)
238 block
= self
.build_block_on_tip(self
.nodes
[0])
240 cmpct_block
= P2PHeaderAndShortIDs()
241 cmpct_block
.header
= CBlockHeader(block
)
242 cmpct_block
.prefilled_txn_length
= 1
243 # This index will be too high
244 prefilled_txn
= PrefilledTransaction(1, block
.vtx
[0])
245 cmpct_block
.prefilled_txn
= [prefilled_txn
]
246 self
.test_node
.send_await_disconnect(msg_cmpctblock(cmpct_block
))
247 assert_equal(int(self
.nodes
[0].getbestblockhash(), 16), block
.hashPrevBlock
)
249 # Compare the generated shortids to what we expect based on BIP 152, given
250 # bitcoind's choice of nonce.
251 def test_compactblock_construction(self
, node
, test_node
, version
, use_witness_address
):
252 # Generate a bunch of transactions.
254 num_transactions
= 25
255 address
= node
.getnewaddress()
256 if use_witness_address
:
257 # Want at least one segwit spend, so move all funds to
259 address
= node
.addwitnessaddress(address
)
260 value_to_send
= node
.getbalance()
261 node
.sendtoaddress(address
, satoshi_round(value_to_send
-Decimal(0.1)))
264 segwit_tx_generated
= False
265 for i
in range(num_transactions
):
266 txid
= node
.sendtoaddress(address
, 0.1)
267 hex_tx
= node
.gettransaction(txid
)["hex"]
268 tx
= FromHex(CTransaction(), hex_tx
)
269 if not tx
.wit
.is_null():
270 segwit_tx_generated
= True
272 if use_witness_address
:
273 assert(segwit_tx_generated
) # check that our test is not broken
275 # Wait until we've seen the block announcement for the resulting tip
276 tip
= int(node
.getbestblockhash(), 16)
277 test_node
.wait_for_block_announcement(tip
)
279 # Make sure we will receive a fast-announce compact block
280 self
.request_cb_announcements(test_node
, node
, version
)
282 # Now mine a block, and look at the resulting compact block.
283 test_node
.clear_block_announcement()
284 block_hash
= int(node
.generate(1)[0], 16)
286 # Store the raw block in our internal format.
287 block
= FromHex(CBlock(), node
.getblock("%02x" % block_hash
, False))
288 [tx
.calc_sha256() for tx
in block
.vtx
]
291 # Wait until the block was announced (via compact blocks)
292 wait_until(test_node
.received_block_announcement
, timeout
=30, lock
=mininode_lock
)
294 # Now fetch and check the compact block
295 header_and_shortids
= None
297 assert("cmpctblock" in test_node
.last_message
)
298 # Convert the on-the-wire representation to absolute indexes
299 header_and_shortids
= HeaderAndShortIDs(test_node
.last_message
["cmpctblock"].header_and_shortids
)
300 self
.check_compactblock_construction_from_block(version
, header_and_shortids
, block_hash
, block
)
302 # Now fetch the compact block using a normal non-announce getdata
304 test_node
.clear_block_announcement()
305 inv
= CInv(4, block_hash
) # 4 == "CompactBlock"
306 test_node
.send_message(msg_getdata([inv
]))
308 wait_until(test_node
.received_block_announcement
, timeout
=30, lock
=mininode_lock
)
310 # Now fetch and check the compact block
311 header_and_shortids
= None
313 assert("cmpctblock" in test_node
.last_message
)
314 # Convert the on-the-wire representation to absolute indexes
315 header_and_shortids
= HeaderAndShortIDs(test_node
.last_message
["cmpctblock"].header_and_shortids
)
316 self
.check_compactblock_construction_from_block(version
, header_and_shortids
, block_hash
, block
)
318 def check_compactblock_construction_from_block(self
, version
, header_and_shortids
, block_hash
, block
):
319 # Check that we got the right block!
320 header_and_shortids
.header
.calc_sha256()
321 assert_equal(header_and_shortids
.header
.sha256
, block_hash
)
323 # Make sure the prefilled_txn appears to have included the coinbase
324 assert(len(header_and_shortids
.prefilled_txn
) >= 1)
325 assert_equal(header_and_shortids
.prefilled_txn
[0].index
, 0)
327 # Check that all prefilled_txn entries match what's in the block.
328 for entry
in header_and_shortids
.prefilled_txn
:
329 entry
.tx
.calc_sha256()
330 # This checks the non-witness parts of the tx agree
331 assert_equal(entry
.tx
.sha256
, block
.vtx
[entry
.index
].sha256
)
333 # And this checks the witness
334 wtxid
= entry
.tx
.calc_sha256(True)
336 assert_equal(wtxid
, block
.vtx
[entry
.index
].calc_sha256(True))
338 # Shouldn't have received a witness
339 assert(entry
.tx
.wit
.is_null())
341 # Check that the cmpctblock message announced all the transactions.
342 assert_equal(len(header_and_shortids
.prefilled_txn
) + len(header_and_shortids
.shortids
), len(block
.vtx
))
344 # And now check that all the shortids are as expected as well.
345 # Determine the siphash keys to use.
346 [k0
, k1
] = header_and_shortids
.get_siphash_keys()
349 while index
< len(block
.vtx
):
350 if (len(header_and_shortids
.prefilled_txn
) > 0 and
351 header_and_shortids
.prefilled_txn
[0].index
== index
):
352 # Already checked prefilled transactions above
353 header_and_shortids
.prefilled_txn
.pop(0)
355 tx_hash
= block
.vtx
[index
].sha256
357 tx_hash
= block
.vtx
[index
].calc_sha256(True)
358 shortid
= calculate_shortid(k0
, k1
, tx_hash
)
359 assert_equal(shortid
, header_and_shortids
.shortids
[0])
360 header_and_shortids
.shortids
.pop(0)
363 # Test that bitcoind requests compact blocks when we announce new blocks
364 # via header or inv, and that responding to getblocktxn causes the block
365 # to be successfully reconstructed.
366 # Post-segwit: upgraded nodes would only make this request of cb-version-2,
367 # NODE_WITNESS peers. Unupgraded nodes would still make this request of
368 # any cb-version-1-supporting peer.
369 def test_compactblock_requests(self
, node
, test_node
, version
, segwit
):
370 # Try announcing a block with an inv or header, expect a compactblock
372 for announce
in ["inv", "header"]:
373 block
= self
.build_block_on_tip(node
, segwit
=segwit
)
375 test_node
.last_message
.pop("getdata", None)
377 if announce
== "inv":
378 test_node
.send_message(msg_inv([CInv(2, block
.sha256
)]))
379 wait_until(lambda: "getheaders" in test_node
.last_message
, timeout
=30, lock
=mininode_lock
)
380 test_node
.send_header_for_blocks([block
])
382 test_node
.send_header_for_blocks([block
])
383 wait_until(lambda: "getdata" in test_node
.last_message
, timeout
=30, lock
=mininode_lock
)
384 assert_equal(len(test_node
.last_message
["getdata"].inv
), 1)
385 assert_equal(test_node
.last_message
["getdata"].inv
[0].type, 4)
386 assert_equal(test_node
.last_message
["getdata"].inv
[0].hash, block
.sha256
)
388 # Send back a compactblock message that omits the coinbase
389 comp_block
= HeaderAndShortIDs()
390 comp_block
.header
= CBlockHeader(block
)
392 [k0
, k1
] = comp_block
.get_siphash_keys()
393 coinbase_hash
= block
.vtx
[0].sha256
395 coinbase_hash
= block
.vtx
[0].calc_sha256(True)
396 comp_block
.shortids
= [
397 calculate_shortid(k0
, k1
, coinbase_hash
) ]
398 test_node
.send_and_ping(msg_cmpctblock(comp_block
.to_p2p()))
399 assert_equal(int(node
.getbestblockhash(), 16), block
.hashPrevBlock
)
400 # Expect a getblocktxn message.
402 assert("getblocktxn" in test_node
.last_message
)
403 absolute_indexes
= test_node
.last_message
["getblocktxn"].block_txn_request
.to_absolute()
404 assert_equal(absolute_indexes
, [0]) # should be a coinbase request
406 # Send the coinbase, and verify that the tip advances.
408 msg
= msg_witness_blocktxn()
411 msg
.block_transactions
.blockhash
= block
.sha256
412 msg
.block_transactions
.transactions
= [block
.vtx
[0]]
413 test_node
.send_and_ping(msg
)
414 assert_equal(int(node
.getbestblockhash(), 16), block
.sha256
)
416 # Create a chain of transactions from given utxo, and add to a new block.
417 def build_block_with_transactions(self
, node
, utxo
, num_transactions
):
418 block
= self
.build_block_on_tip(node
)
420 for i
in range(num_transactions
):
422 tx
.vin
.append(CTxIn(COutPoint(utxo
[0], utxo
[1]), b
''))
423 tx
.vout
.append(CTxOut(utxo
[2] - 1000, CScript([OP_TRUE
])))
425 utxo
= [tx
.sha256
, 0, tx
.vout
[0].nValue
]
428 block
.hashMerkleRoot
= block
.calc_merkle_root()
432 # Test that we only receive getblocktxn requests for transactions that the
433 # node needs, and that responding to them causes the block to be
435 def test_getblocktxn_requests(self
, node
, test_node
, version
):
436 with_witness
= (version
==2)
438 def test_getblocktxn_response(compact_block
, peer
, expected_result
):
439 msg
= msg_cmpctblock(compact_block
.to_p2p())
440 peer
.send_and_ping(msg
)
442 assert("getblocktxn" in peer
.last_message
)
443 absolute_indexes
= peer
.last_message
["getblocktxn"].block_txn_request
.to_absolute()
444 assert_equal(absolute_indexes
, expected_result
)
446 def test_tip_after_message(node
, peer
, msg
, tip
):
447 peer
.send_and_ping(msg
)
448 assert_equal(int(node
.getbestblockhash(), 16), tip
)
450 # First try announcing compactblocks that won't reconstruct, and verify
451 # that we receive getblocktxn messages back.
452 utxo
= self
.utxos
.pop(0)
454 block
= self
.build_block_with_transactions(node
, utxo
, 5)
455 self
.utxos
.append([block
.vtx
[-1].sha256
, 0, block
.vtx
[-1].vout
[0].nValue
])
456 comp_block
= HeaderAndShortIDs()
457 comp_block
.initialize_from_block(block
, use_witness
=with_witness
)
459 test_getblocktxn_response(comp_block
, test_node
, [1, 2, 3, 4, 5])
461 msg_bt
= msg_blocktxn()
463 msg_bt
= msg_witness_blocktxn() # serialize with witnesses
464 msg_bt
.block_transactions
= BlockTransactions(block
.sha256
, block
.vtx
[1:])
465 test_tip_after_message(node
, test_node
, msg_bt
, block
.sha256
)
467 utxo
= self
.utxos
.pop(0)
468 block
= self
.build_block_with_transactions(node
, utxo
, 5)
469 self
.utxos
.append([block
.vtx
[-1].sha256
, 0, block
.vtx
[-1].vout
[0].nValue
])
471 # Now try interspersing the prefilled transactions
472 comp_block
.initialize_from_block(block
, prefill_list
=[0, 1, 5], use_witness
=with_witness
)
473 test_getblocktxn_response(comp_block
, test_node
, [2, 3, 4])
474 msg_bt
.block_transactions
= BlockTransactions(block
.sha256
, block
.vtx
[2:5])
475 test_tip_after_message(node
, test_node
, msg_bt
, block
.sha256
)
477 # Now try giving one transaction ahead of time.
478 utxo
= self
.utxos
.pop(0)
479 block
= self
.build_block_with_transactions(node
, utxo
, 5)
480 self
.utxos
.append([block
.vtx
[-1].sha256
, 0, block
.vtx
[-1].vout
[0].nValue
])
481 test_node
.send_and_ping(msg_tx(block
.vtx
[1]))
482 assert(block
.vtx
[1].hash in node
.getrawmempool())
484 # Prefill 4 out of the 6 transactions, and verify that only the one
485 # that was not in the mempool is requested.
486 comp_block
.initialize_from_block(block
, prefill_list
=[0, 2, 3, 4], use_witness
=with_witness
)
487 test_getblocktxn_response(comp_block
, test_node
, [5])
489 msg_bt
.block_transactions
= BlockTransactions(block
.sha256
, [block
.vtx
[5]])
490 test_tip_after_message(node
, test_node
, msg_bt
, block
.sha256
)
492 # Now provide all transactions to the node before the block is
493 # announced and verify reconstruction happens immediately.
494 utxo
= self
.utxos
.pop(0)
495 block
= self
.build_block_with_transactions(node
, utxo
, 10)
496 self
.utxos
.append([block
.vtx
[-1].sha256
, 0, block
.vtx
[-1].vout
[0].nValue
])
497 for tx
in block
.vtx
[1:]:
498 test_node
.send_message(msg_tx(tx
))
499 test_node
.sync_with_ping()
500 # Make sure all transactions were accepted.
501 mempool
= node
.getrawmempool()
502 for tx
in block
.vtx
[1:]:
503 assert(tx
.hash in mempool
)
505 # Clear out last request.
507 test_node
.last_message
.pop("getblocktxn", None)
510 comp_block
.initialize_from_block(block
, prefill_list
=[0], use_witness
=with_witness
)
511 test_tip_after_message(node
, test_node
, msg_cmpctblock(comp_block
.to_p2p()), block
.sha256
)
513 # Shouldn't have gotten a request for any transaction
514 assert("getblocktxn" not in test_node
.last_message
)
516 # Incorrectly responding to a getblocktxn shouldn't cause the block to be
517 # permanently failed.
518 def test_incorrect_blocktxn_response(self
, node
, test_node
, version
):
519 if (len(self
.utxos
) == 0):
521 utxo
= self
.utxos
.pop(0)
523 block
= self
.build_block_with_transactions(node
, utxo
, 10)
524 self
.utxos
.append([block
.vtx
[-1].sha256
, 0, block
.vtx
[-1].vout
[0].nValue
])
525 # Relay the first 5 transactions from the block in advance
526 for tx
in block
.vtx
[1:6]:
527 test_node
.send_message(msg_tx(tx
))
528 test_node
.sync_with_ping()
529 # Make sure all transactions were accepted.
530 mempool
= node
.getrawmempool()
531 for tx
in block
.vtx
[1:6]:
532 assert(tx
.hash in mempool
)
535 comp_block
= HeaderAndShortIDs()
536 comp_block
.initialize_from_block(block
, prefill_list
=[0], use_witness
=(version
== 2))
537 test_node
.send_and_ping(msg_cmpctblock(comp_block
.to_p2p()))
538 absolute_indexes
= []
540 assert("getblocktxn" in test_node
.last_message
)
541 absolute_indexes
= test_node
.last_message
["getblocktxn"].block_txn_request
.to_absolute()
542 assert_equal(absolute_indexes
, [6, 7, 8, 9, 10])
544 # Now give an incorrect response.
545 # Note that it's possible for bitcoind to be smart enough to know we're
546 # lying, since it could check to see if the shortid matches what we're
547 # sending, and eg disconnect us for misbehavior. If that behavior
548 # change were made, we could just modify this test by having a
549 # different peer provide the block further down, so that we're still
550 # verifying that the block isn't marked bad permanently. This is good
554 msg
= msg_witness_blocktxn()
555 msg
.block_transactions
= BlockTransactions(block
.sha256
, [block
.vtx
[5]] + block
.vtx
[7:])
556 test_node
.send_and_ping(msg
)
558 # Tip should not have updated
559 assert_equal(int(node
.getbestblockhash(), 16), block
.hashPrevBlock
)
561 # We should receive a getdata request
562 wait_until(lambda: "getdata" in test_node
.last_message
, timeout
=10, lock
=mininode_lock
)
563 assert_equal(len(test_node
.last_message
["getdata"].inv
), 1)
564 assert(test_node
.last_message
["getdata"].inv
[0].type == 2 or test_node
.last_message
["getdata"].inv
[0].type == 2|MSG_WITNESS_FLAG
)
565 assert_equal(test_node
.last_message
["getdata"].inv
[0].hash, block
.sha256
)
569 test_node
.send_and_ping(msg_witness_block(block
))
571 test_node
.send_and_ping(msg_block(block
))
572 assert_equal(int(node
.getbestblockhash(), 16), block
.sha256
)
574 def test_getblocktxn_handler(self
, node
, test_node
, version
):
575 # bitcoind will not send blocktxn responses for blocks whose height is
576 # more than 10 blocks deep.
577 MAX_GETBLOCKTXN_DEPTH
= 10
578 chain_height
= node
.getblockcount()
579 current_height
= chain_height
580 while (current_height
>= chain_height
- MAX_GETBLOCKTXN_DEPTH
):
581 block_hash
= node
.getblockhash(current_height
)
582 block
= FromHex(CBlock(), node
.getblock(block_hash
, False))
584 msg
= msg_getblocktxn()
585 msg
.block_txn_request
= BlockTransactionsRequest(int(block_hash
, 16), [])
586 num_to_request
= random
.randint(1, len(block
.vtx
))
587 msg
.block_txn_request
.from_absolute(sorted(random
.sample(range(len(block
.vtx
)), num_to_request
)))
588 test_node
.send_message(msg
)
589 wait_until(lambda: "blocktxn" in test_node
.last_message
, timeout
=10, lock
=mininode_lock
)
591 [tx
.calc_sha256() for tx
in block
.vtx
]
593 assert_equal(test_node
.last_message
["blocktxn"].block_transactions
.blockhash
, int(block_hash
, 16))
594 all_indices
= msg
.block_txn_request
.to_absolute()
595 for index
in all_indices
:
596 tx
= test_node
.last_message
["blocktxn"].block_transactions
.transactions
.pop(0)
598 assert_equal(tx
.sha256
, block
.vtx
[index
].sha256
)
600 # Witnesses should have been stripped
601 assert(tx
.wit
.is_null())
603 # Check that the witness matches
604 assert_equal(tx
.calc_sha256(True), block
.vtx
[index
].calc_sha256(True))
605 test_node
.last_message
.pop("blocktxn", None)
608 # Next request should send a full block response, as we're past the
609 # allowed depth for a blocktxn response.
610 block_hash
= node
.getblockhash(current_height
)
611 msg
.block_txn_request
= BlockTransactionsRequest(int(block_hash
, 16), [0])
613 test_node
.last_message
.pop("block", None)
614 test_node
.last_message
.pop("blocktxn", None)
615 test_node
.send_and_ping(msg
)
617 test_node
.last_message
["block"].block
.calc_sha256()
618 assert_equal(test_node
.last_message
["block"].block
.sha256
, int(block_hash
, 16))
619 assert "blocktxn" not in test_node
.last_message
621 def test_compactblocks_not_at_tip(self
, node
, test_node
):
622 # Test that requesting old compactblocks doesn't work.
623 MAX_CMPCTBLOCK_DEPTH
= 5
625 for i
in range(MAX_CMPCTBLOCK_DEPTH
+ 1):
626 test_node
.clear_block_announcement()
627 new_blocks
.append(node
.generate(1)[0])
628 wait_until(test_node
.received_block_announcement
, timeout
=30, lock
=mininode_lock
)
630 test_node
.clear_block_announcement()
631 test_node
.send_message(msg_getdata([CInv(4, int(new_blocks
[0], 16))]))
632 wait_until(lambda: "cmpctblock" in test_node
.last_message
, timeout
=30, lock
=mininode_lock
)
634 test_node
.clear_block_announcement()
636 wait_until(test_node
.received_block_announcement
, timeout
=30, lock
=mininode_lock
)
637 test_node
.clear_block_announcement()
639 test_node
.last_message
.pop("block", None)
640 test_node
.send_message(msg_getdata([CInv(4, int(new_blocks
[0], 16))]))
641 wait_until(lambda: "block" in test_node
.last_message
, timeout
=30, lock
=mininode_lock
)
643 test_node
.last_message
["block"].block
.calc_sha256()
644 assert_equal(test_node
.last_message
["block"].block
.sha256
, int(new_blocks
[0], 16))
646 # Generate an old compactblock, and verify that it's not accepted.
647 cur_height
= node
.getblockcount()
648 hashPrevBlock
= int(node
.getblockhash(cur_height
-5), 16)
649 block
= self
.build_block_on_tip(node
)
650 block
.hashPrevBlock
= hashPrevBlock
653 comp_block
= HeaderAndShortIDs()
654 comp_block
.initialize_from_block(block
)
655 test_node
.send_and_ping(msg_cmpctblock(comp_block
.to_p2p()))
657 tips
= node
.getchaintips()
660 if x
["hash"] == block
.hash:
661 assert_equal(x
["status"], "headers-only")
666 # Requesting this block via getblocktxn should silently fail
667 # (to avoid fingerprinting attacks).
668 msg
= msg_getblocktxn()
669 msg
.block_txn_request
= BlockTransactionsRequest(block
.sha256
, [0])
671 test_node
.last_message
.pop("blocktxn", None)
672 test_node
.send_and_ping(msg
)
674 assert "blocktxn" not in test_node
.last_message
676 def activate_segwit(self
, node
):
678 assert_equal(get_bip9_status(node
, "segwit")["status"], 'active')
680 def test_end_to_end_block_relay(self
, node
, listeners
):
681 utxo
= self
.utxos
.pop(0)
683 block
= self
.build_block_with_transactions(node
, utxo
, 10)
685 [l
.clear_block_announcement() for l
in listeners
]
687 # ToHex() won't serialize with witness, but this block has no witnesses
688 # anyway. TODO: repeat this test with witness tx's to a segwit node.
689 node
.submitblock(ToHex(block
))
692 wait_until(lambda: l
.received_block_announcement(), timeout
=30, lock
=mininode_lock
)
695 assert "cmpctblock" in l
.last_message
696 l
.last_message
["cmpctblock"].header_and_shortids
.header
.calc_sha256()
697 assert_equal(l
.last_message
["cmpctblock"].header_and_shortids
.header
.sha256
, block
.sha256
)
699 # Test that we don't get disconnected if we relay a compact block with valid header,
700 # but invalid transactions.
701 def test_invalid_tx_in_compactblock(self
, node
, test_node
, use_segwit
):
702 assert(len(self
.utxos
))
705 block
= self
.build_block_with_transactions(node
, utxo
, 5)
707 block
.hashMerkleRoot
= block
.calc_merkle_root()
709 # If we're testing with segwit, also drop the coinbase witness,
710 # but include the witness commitment.
711 add_witness_commitment(block
)
712 block
.vtx
[0].wit
.vtxinwit
= []
715 # Now send the compact block with all transactions prefilled, and
716 # verify that we don't get disconnected.
717 comp_block
= HeaderAndShortIDs()
718 comp_block
.initialize_from_block(block
, prefill_list
=[0, 1, 2, 3, 4], use_witness
=use_segwit
)
719 msg
= msg_cmpctblock(comp_block
.to_p2p())
720 test_node
.send_and_ping(msg
)
722 # Check that the tip didn't advance
723 assert(int(node
.getbestblockhash(), 16) is not block
.sha256
)
724 test_node
.sync_with_ping()
726 # Helper for enabling cb announcements
727 # Send the sendcmpct request and sync headers
728 def request_cb_announcements(self
, peer
, node
, version
):
729 tip
= node
.getbestblockhash()
730 peer
.get_headers(locator
=[int(tip
, 16)], hashstop
=0)
732 msg
= msg_sendcmpct()
733 msg
.version
= version
735 peer
.send_and_ping(msg
)
737 def test_compactblock_reconstruction_multiple_peers(self
, node
, stalling_peer
, delivery_peer
):
738 assert(len(self
.utxos
))
740 def announce_cmpct_block(node
, peer
):
741 utxo
= self
.utxos
.pop(0)
742 block
= self
.build_block_with_transactions(node
, utxo
, 5)
744 cmpct_block
= HeaderAndShortIDs()
745 cmpct_block
.initialize_from_block(block
)
746 msg
= msg_cmpctblock(cmpct_block
.to_p2p())
747 peer
.send_and_ping(msg
)
749 assert "getblocktxn" in peer
.last_message
750 return block
, cmpct_block
752 block
, cmpct_block
= announce_cmpct_block(node
, stalling_peer
)
754 for tx
in block
.vtx
[1:]:
755 delivery_peer
.send_message(msg_tx(tx
))
756 delivery_peer
.sync_with_ping()
757 mempool
= node
.getrawmempool()
758 for tx
in block
.vtx
[1:]:
759 assert(tx
.hash in mempool
)
761 delivery_peer
.send_and_ping(msg_cmpctblock(cmpct_block
.to_p2p()))
762 assert_equal(int(node
.getbestblockhash(), 16), block
.sha256
)
764 self
.utxos
.append([block
.vtx
[-1].sha256
, 0, block
.vtx
[-1].vout
[0].nValue
])
766 # Now test that delivering an invalid compact block won't break relay
768 block
, cmpct_block
= announce_cmpct_block(node
, stalling_peer
)
769 for tx
in block
.vtx
[1:]:
770 delivery_peer
.send_message(msg_tx(tx
))
771 delivery_peer
.sync_with_ping()
773 cmpct_block
.prefilled_txn
[0].tx
.wit
.vtxinwit
= [ CTxInWitness() ]
774 cmpct_block
.prefilled_txn
[0].tx
.wit
.vtxinwit
[0].scriptWitness
.stack
= [ser_uint256(0)]
776 cmpct_block
.use_witness
= True
777 delivery_peer
.send_and_ping(msg_cmpctblock(cmpct_block
.to_p2p()))
778 assert(int(node
.getbestblockhash(), 16) != block
.sha256
)
781 msg
.block_transactions
.blockhash
= block
.sha256
782 msg
.block_transactions
.transactions
= block
.vtx
[1:]
783 stalling_peer
.send_and_ping(msg
)
784 assert_equal(int(node
.getbestblockhash(), 16), block
.sha256
)
787 # Setup the p2p connections and start up the network thread.
788 self
.test_node
= TestNode()
789 self
.segwit_node
= TestNode()
790 self
.old_node
= TestNode() # version 1 peer <--> segwit node
793 connections
.append(NodeConn('127.0.0.1', p2p_port(0), self
.nodes
[0], self
.test_node
))
794 connections
.append(NodeConn('127.0.0.1', p2p_port(1), self
.nodes
[1],
795 self
.segwit_node
, services
=NODE_NETWORK|NODE_WITNESS
))
796 connections
.append(NodeConn('127.0.0.1', p2p_port(1), self
.nodes
[1],
797 self
.old_node
, services
=NODE_NETWORK
))
798 self
.test_node
.add_connection(connections
[0])
799 self
.segwit_node
.add_connection(connections
[1])
800 self
.old_node
.add_connection(connections
[2])
802 NetworkThread().start() # Start up network handling in another thread
804 # Test logic begins here
805 self
.test_node
.wait_for_verack()
807 # We will need UTXOs to construct transactions in later tests.
810 self
.log
.info("Running tests, pre-segwit activation:")
812 self
.log
.info("Testing SENDCMPCT p2p message... ")
813 self
.test_sendcmpct(self
.nodes
[0], self
.test_node
, 1)
814 sync_blocks(self
.nodes
)
815 self
.test_sendcmpct(self
.nodes
[1], self
.segwit_node
, 2, old_node
=self
.old_node
)
816 sync_blocks(self
.nodes
)
818 self
.log
.info("Testing compactblock construction...")
819 self
.test_compactblock_construction(self
.nodes
[0], self
.test_node
, 1, False)
820 sync_blocks(self
.nodes
)
821 self
.test_compactblock_construction(self
.nodes
[1], self
.segwit_node
, 2, False)
822 sync_blocks(self
.nodes
)
824 self
.log
.info("Testing compactblock requests... ")
825 self
.test_compactblock_requests(self
.nodes
[0], self
.test_node
, 1, False)
826 sync_blocks(self
.nodes
)
827 self
.test_compactblock_requests(self
.nodes
[1], self
.segwit_node
, 2, False)
828 sync_blocks(self
.nodes
)
830 self
.log
.info("Testing getblocktxn requests...")
831 self
.test_getblocktxn_requests(self
.nodes
[0], self
.test_node
, 1)
832 sync_blocks(self
.nodes
)
833 self
.test_getblocktxn_requests(self
.nodes
[1], self
.segwit_node
, 2)
834 sync_blocks(self
.nodes
)
836 self
.log
.info("Testing getblocktxn handler...")
837 self
.test_getblocktxn_handler(self
.nodes
[0], self
.test_node
, 1)
838 sync_blocks(self
.nodes
)
839 self
.test_getblocktxn_handler(self
.nodes
[1], self
.segwit_node
, 2)
840 self
.test_getblocktxn_handler(self
.nodes
[1], self
.old_node
, 1)
841 sync_blocks(self
.nodes
)
843 self
.log
.info("Testing compactblock requests/announcements not at chain tip...")
844 self
.test_compactblocks_not_at_tip(self
.nodes
[0], self
.test_node
)
845 sync_blocks(self
.nodes
)
846 self
.test_compactblocks_not_at_tip(self
.nodes
[1], self
.segwit_node
)
847 self
.test_compactblocks_not_at_tip(self
.nodes
[1], self
.old_node
)
848 sync_blocks(self
.nodes
)
850 self
.log
.info("Testing handling of incorrect blocktxn responses...")
851 self
.test_incorrect_blocktxn_response(self
.nodes
[0], self
.test_node
, 1)
852 sync_blocks(self
.nodes
)
853 self
.test_incorrect_blocktxn_response(self
.nodes
[1], self
.segwit_node
, 2)
854 sync_blocks(self
.nodes
)
856 # End-to-end block relay tests
857 self
.log
.info("Testing end-to-end block relay...")
858 self
.request_cb_announcements(self
.test_node
, self
.nodes
[0], 1)
859 self
.request_cb_announcements(self
.old_node
, self
.nodes
[1], 1)
860 self
.request_cb_announcements(self
.segwit_node
, self
.nodes
[1], 2)
861 self
.test_end_to_end_block_relay(self
.nodes
[0], [self
.segwit_node
, self
.test_node
, self
.old_node
])
862 self
.test_end_to_end_block_relay(self
.nodes
[1], [self
.segwit_node
, self
.test_node
, self
.old_node
])
864 self
.log
.info("Testing handling of invalid compact blocks...")
865 self
.test_invalid_tx_in_compactblock(self
.nodes
[0], self
.test_node
, False)
866 self
.test_invalid_tx_in_compactblock(self
.nodes
[1], self
.segwit_node
, False)
867 self
.test_invalid_tx_in_compactblock(self
.nodes
[1], self
.old_node
, False)
869 self
.log
.info("Testing reconstructing compact blocks from all peers...")
870 self
.test_compactblock_reconstruction_multiple_peers(self
.nodes
[1], self
.segwit_node
, self
.old_node
)
871 sync_blocks(self
.nodes
)
873 # Advance to segwit activation
874 self
.log
.info("Advancing to segwit activation")
875 self
.activate_segwit(self
.nodes
[1])
876 self
.log
.info("Running tests, post-segwit activation...")
878 self
.log
.info("Testing compactblock construction...")
879 self
.test_compactblock_construction(self
.nodes
[1], self
.old_node
, 1, True)
880 self
.test_compactblock_construction(self
.nodes
[1], self
.segwit_node
, 2, True)
881 sync_blocks(self
.nodes
)
883 self
.log
.info("Testing compactblock requests (unupgraded node)... ")
884 self
.test_compactblock_requests(self
.nodes
[0], self
.test_node
, 1, True)
886 self
.log
.info("Testing getblocktxn requests (unupgraded node)...")
887 self
.test_getblocktxn_requests(self
.nodes
[0], self
.test_node
, 1)
889 # Need to manually sync node0 and node1, because post-segwit activation,
890 # node1 will not download blocks from node0.
891 self
.log
.info("Syncing nodes...")
892 assert(self
.nodes
[0].getbestblockhash() != self
.nodes
[1].getbestblockhash())
893 while (self
.nodes
[0].getblockcount() > self
.nodes
[1].getblockcount()):
894 block_hash
= self
.nodes
[0].getblockhash(self
.nodes
[1].getblockcount()+1)
895 self
.nodes
[1].submitblock(self
.nodes
[0].getblock(block_hash
, False))
896 assert_equal(self
.nodes
[0].getbestblockhash(), self
.nodes
[1].getbestblockhash())
898 self
.log
.info("Testing compactblock requests (segwit node)... ")
899 self
.test_compactblock_requests(self
.nodes
[1], self
.segwit_node
, 2, True)
901 self
.log
.info("Testing getblocktxn requests (segwit node)...")
902 self
.test_getblocktxn_requests(self
.nodes
[1], self
.segwit_node
, 2)
903 sync_blocks(self
.nodes
)
905 self
.log
.info("Testing getblocktxn handler (segwit node should return witnesses)...")
906 self
.test_getblocktxn_handler(self
.nodes
[1], self
.segwit_node
, 2)
907 self
.test_getblocktxn_handler(self
.nodes
[1], self
.old_node
, 1)
909 # Test that if we submitblock to node1, we'll get a compact block
910 # announcement to all peers.
911 # (Post-segwit activation, blocks won't propagate from node0 to node1
912 # automatically, so don't bother testing a block announced to node0.)
913 self
.log
.info("Testing end-to-end block relay...")
914 self
.request_cb_announcements(self
.test_node
, self
.nodes
[0], 1)
915 self
.request_cb_announcements(self
.old_node
, self
.nodes
[1], 1)
916 self
.request_cb_announcements(self
.segwit_node
, self
.nodes
[1], 2)
917 self
.test_end_to_end_block_relay(self
.nodes
[1], [self
.segwit_node
, self
.test_node
, self
.old_node
])
919 self
.log
.info("Testing handling of invalid compact blocks...")
920 self
.test_invalid_tx_in_compactblock(self
.nodes
[0], self
.test_node
, False)
921 self
.test_invalid_tx_in_compactblock(self
.nodes
[1], self
.segwit_node
, True)
922 self
.test_invalid_tx_in_compactblock(self
.nodes
[1], self
.old_node
, True)
924 self
.log
.info("Testing invalid index in cmpctblock message...")
925 self
.test_invalid_cmpctblock_message()
928 if __name__
== '__main__':
929 CompactBlocksTest().main()