Merge #11743: qa: Add multiwallet prefix test
[bitcoinplatinum.git] / test / functional / sendheaders.py
blob68c0d95b4f9912b6c11a3870423269ab453e103c
1 #!/usr/bin/env python3
2 # Copyright (c) 2014-2016 The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test behavior of headers messages to announce blocks.
7 Setup:
9 - Two nodes:
10 - node0 is the node-under-test. We create two p2p connections to it. The
11 first p2p connection is a control and should only ever receive inv's. The
12 second p2p connection tests the headers sending logic.
13 - node1 is used to create reorgs.
15 test_null_locators
16 ==================
18 Sends two getheaders requests with null locator values. First request's hashstop
19 value refers to validated block, while second request's hashstop value refers to
20 a block which hasn't been validated. Verifies only the first request returns
21 headers.
23 test_nonnull_locators
24 =====================
26 Part 1: No headers announcements before "sendheaders"
27 a. node mines a block [expect: inv]
28 send getdata for the block [expect: block]
29 b. node mines another block [expect: inv]
30 send getheaders and getdata [expect: headers, then block]
31 c. node mines another block [expect: inv]
32 peer mines a block, announces with header [expect: getdata]
33 d. node mines another block [expect: inv]
35 Part 2: After "sendheaders", headers announcements should generally work.
36 a. peer sends sendheaders [expect: no response]
37 peer sends getheaders with current tip [expect: no response]
38 b. node mines a block [expect: tip header]
39 c. for N in 1, ..., 10:
40 * for announce-type in {inv, header}
41 - peer mines N blocks, announces with announce-type
42 [ expect: getheaders/getdata or getdata, deliver block(s) ]
43 - node mines a block [ expect: 1 header ]
45 Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer.
46 - For response-type in {inv, getheaders}
47 * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ]
48 * node mines an 8-block reorg [ expect: inv at tip ]
49 * peer responds with getblocks/getdata [expect: inv, blocks ]
50 * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ]
51 * node mines another block at tip [ expect: inv ]
52 * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers]
53 * peer requests block [ expect: block ]
54 * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ]
55 * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block]
56 * node mines 1 block [expect: 1 header, peer responds with getdata]
58 Part 4: Test direct fetch behavior
59 a. Announce 2 old block headers.
60 Expect: no getdata requests.
61 b. Announce 3 new blocks via 1 headers message.
62 Expect: one getdata request for all 3 blocks.
63 (Send blocks.)
64 c. Announce 1 header that forks off the last two blocks.
65 Expect: no response.
66 d. Announce 1 more header that builds on that fork.
67 Expect: one getdata request for two blocks.
68 e. Announce 16 more headers that build on that fork.
69 Expect: getdata request for 14 more blocks.
70 f. Announce 1 more header that builds on that fork.
71 Expect: no response.
73 Part 5: Test handling of headers that don't connect.
74 a. Repeat 10 times:
75 1. Announce a header that doesn't connect.
76 Expect: getheaders message
77 2. Send headers chain.
78 Expect: getdata for the missing blocks, tip update.
79 b. Then send 9 more headers that don't connect.
80 Expect: getheaders message each time.
81 c. Announce a header that does connect.
82 Expect: no response.
83 d. Announce 49 headers that don't connect.
84 Expect: getheaders message each time.
85 e. Announce one more that doesn't connect.
86 Expect: disconnect.
87 """
88 from test_framework.blocktools import create_block, create_coinbase
89 from test_framework.mininode import (
90 CBlockHeader,
91 CInv,
92 NODE_WITNESS,
93 NetworkThread,
94 NodeConnCB,
95 mininode_lock,
96 msg_block,
97 msg_getblocks,
98 msg_getdata,
99 msg_getheaders,
100 msg_headers,
101 msg_inv,
102 msg_sendheaders,
104 from test_framework.test_framework import BitcoinTestFramework
105 from test_framework.util import (
106 assert_equal,
107 sync_blocks,
108 wait_until,
111 DIRECT_FETCH_RESPONSE_TIME = 0.05
113 class BaseNode(NodeConnCB):
114 def __init__(self):
115 super().__init__()
116 self.block_announced = False
117 self.last_blockhash_announced = None
119 def send_get_data(self, block_hashes):
120 """Request data for a list of block hashes."""
121 msg = msg_getdata()
122 for x in block_hashes:
123 msg.inv.append(CInv(2, x))
124 self.connection.send_message(msg)
126 def send_get_headers(self, locator, hashstop):
127 msg = msg_getheaders()
128 msg.locator.vHave = locator
129 msg.hashstop = hashstop
130 self.connection.send_message(msg)
132 def send_block_inv(self, blockhash):
133 msg = msg_inv()
134 msg.inv = [CInv(2, blockhash)]
135 self.connection.send_message(msg)
137 def send_header_for_blocks(self, new_blocks):
138 headers_message = msg_headers()
139 headers_message.headers = [CBlockHeader(b) for b in new_blocks]
140 self.send_message(headers_message)
142 def send_getblocks(self, locator):
143 getblocks_message = msg_getblocks()
144 getblocks_message.locator.vHave = locator
145 self.send_message(getblocks_message)
147 def wait_for_getdata(self, hash_list, timeout=60):
148 if hash_list == []:
149 return
151 test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list
152 wait_until(test_function, timeout=timeout, lock=mininode_lock)
154 def wait_for_block_announcement(self, block_hash, timeout=60):
155 test_function = lambda: self.last_blockhash_announced == block_hash
156 wait_until(test_function, timeout=timeout, lock=mininode_lock)
158 def on_inv(self, conn, message):
159 self.block_announced = True
160 self.last_blockhash_announced = message.inv[-1].hash
162 def on_headers(self, conn, message):
163 if len(message.headers):
164 self.block_announced = True
165 message.headers[-1].calc_sha256()
166 self.last_blockhash_announced = message.headers[-1].sha256
168 def clear_last_announcement(self):
169 with mininode_lock:
170 self.block_announced = False
171 self.last_message.pop("inv", None)
172 self.last_message.pop("headers", None)
174 def check_last_announcement(self, headers=None, inv=None):
175 """Test whether the last announcement received had the right header or the right inv.
177 inv and headers should be lists of block hashes."""
179 test_function = lambda: self.block_announced
180 wait_until(test_function, timeout=60, lock=mininode_lock)
182 with mininode_lock:
183 self.block_announced = False
185 compare_inv = []
186 if "inv" in self.last_message:
187 compare_inv = [x.hash for x in self.last_message["inv"].inv]
188 if inv is not None:
189 assert_equal(compare_inv, inv)
191 compare_headers = []
192 if "headers" in self.last_message:
193 compare_headers = [x.sha256 for x in self.last_message["headers"].headers]
194 if headers is not None:
195 assert_equal(compare_headers, headers)
197 self.last_message.pop("inv", None)
198 self.last_message.pop("headers", None)
200 class SendHeadersTest(BitcoinTestFramework):
201 def set_test_params(self):
202 self.setup_clean_chain = True
203 self.num_nodes = 2
205 def mine_blocks(self, count):
206 """Mine count blocks and return the new tip."""
208 # Clear out last block announcement from each p2p listener
209 [x.clear_last_announcement() for x in self.nodes[0].p2ps]
210 self.nodes[0].generate(count)
211 return int(self.nodes[0].getbestblockhash(), 16)
213 def mine_reorg(self, length):
214 """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks).
216 Note: we clear the state of our p2p connections after the
217 to-be-reorged-out blocks are mined, so that we don't break later tests.
218 return the list of block hashes newly mined."""
220 self.nodes[0].generate(length) # make sure all invalidated blocks are node0's
221 sync_blocks(self.nodes, wait=0.1)
222 for x in self.nodes[0].p2ps:
223 x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16))
224 x.clear_last_announcement()
226 tip_height = self.nodes[1].getblockcount()
227 hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1))
228 self.nodes[1].invalidateblock(hash_to_invalidate)
229 all_hashes = self.nodes[1].generate(length + 1) # Must be longer than the orig chain
230 sync_blocks(self.nodes, wait=0.1)
231 return [int(x, 16) for x in all_hashes]
233 def run_test(self):
234 # Setup the p2p connections and start up the network thread.
235 inv_node = self.nodes[0].add_p2p_connection(BaseNode())
236 # Make sure NODE_NETWORK is not set for test_node, so no block download
237 # will occur outside of direct fetching
238 test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS)
240 NetworkThread().start() # Start up network handling in another thread
242 # Test logic begins here
243 inv_node.wait_for_verack()
244 test_node.wait_for_verack()
246 # Ensure verack's have been processed by our peer
247 inv_node.sync_with_ping()
248 test_node.sync_with_ping()
250 self.test_null_locators(test_node, inv_node)
251 self.test_nonnull_locators(test_node, inv_node)
253 def test_null_locators(self, test_node, inv_node):
254 tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0])
255 tip_hash = int(tip["hash"], 16)
257 inv_node.check_last_announcement(inv=[tip_hash], headers=[])
258 test_node.check_last_announcement(inv=[tip_hash], headers=[])
260 self.log.info("Verify getheaders with null locator and valid hashstop returns headers.")
261 test_node.clear_last_announcement()
262 test_node.send_get_headers(locator=[], hashstop=tip_hash)
263 test_node.check_last_announcement(headers=[tip_hash])
265 self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.")
266 block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1)
267 block.solve()
268 test_node.send_header_for_blocks([block])
269 test_node.clear_last_announcement()
270 test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16))
271 test_node.sync_with_ping()
272 assert_equal(test_node.block_announced, False)
273 inv_node.clear_last_announcement()
274 test_node.send_message(msg_block(block))
275 inv_node.check_last_announcement(inv=[int(block.hash, 16)], headers=[])
277 def test_nonnull_locators(self, test_node, inv_node):
278 tip = int(self.nodes[0].getbestblockhash(), 16)
280 # PART 1
281 # 1. Mine a block; expect inv announcements each time
282 self.log.info("Part 1: headers don't start before sendheaders message...")
283 for i in range(4):
284 old_tip = tip
285 tip = self.mine_blocks(1)
286 inv_node.check_last_announcement(inv=[tip], headers=[])
287 test_node.check_last_announcement(inv=[tip], headers=[])
288 # Try a few different responses; none should affect next announcement
289 if i == 0:
290 # first request the block
291 test_node.send_get_data([tip])
292 test_node.wait_for_block(tip)
293 elif i == 1:
294 # next try requesting header and block
295 test_node.send_get_headers(locator=[old_tip], hashstop=tip)
296 test_node.send_get_data([tip])
297 test_node.wait_for_block(tip)
298 test_node.clear_last_announcement() # since we requested headers...
299 elif i == 2:
300 # this time announce own block via headers
301 height = self.nodes[0].getblockcount()
302 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
303 block_time = last_time + 1
304 new_block = create_block(tip, create_coinbase(height + 1), block_time)
305 new_block.solve()
306 test_node.send_header_for_blocks([new_block])
307 test_node.wait_for_getdata([new_block.sha256])
308 test_node.send_message(msg_block(new_block))
309 test_node.sync_with_ping() # make sure this block is processed
310 inv_node.clear_last_announcement()
311 test_node.clear_last_announcement()
313 self.log.info("Part 1: success!")
314 self.log.info("Part 2: announce blocks with headers after sendheaders message...")
315 # PART 2
316 # 2. Send a sendheaders message and test that headers announcements
317 # commence and keep working.
318 test_node.send_message(msg_sendheaders())
319 prev_tip = int(self.nodes[0].getbestblockhash(), 16)
320 test_node.send_get_headers(locator=[prev_tip], hashstop=0)
321 test_node.sync_with_ping()
323 # Now that we've synced headers, headers announcements should work
324 tip = self.mine_blocks(1)
325 inv_node.check_last_announcement(inv=[tip], headers=[])
326 test_node.check_last_announcement(headers=[tip])
328 height = self.nodes[0].getblockcount() + 1
329 block_time += 10 # Advance far enough ahead
330 for i in range(10):
331 # Mine i blocks, and alternate announcing either via
332 # inv (of tip) or via headers. After each, new blocks
333 # mined by the node should successfully be announced
334 # with block header, even though the blocks are never requested
335 for j in range(2):
336 blocks = []
337 for b in range(i + 1):
338 blocks.append(create_block(tip, create_coinbase(height), block_time))
339 blocks[-1].solve()
340 tip = blocks[-1].sha256
341 block_time += 1
342 height += 1
343 if j == 0:
344 # Announce via inv
345 test_node.send_block_inv(tip)
346 test_node.wait_for_getheaders()
347 # Should have received a getheaders now
348 test_node.send_header_for_blocks(blocks)
349 # Test that duplicate inv's won't result in duplicate
350 # getdata requests, or duplicate headers announcements
351 [inv_node.send_block_inv(x.sha256) for x in blocks]
352 test_node.wait_for_getdata([x.sha256 for x in blocks])
353 inv_node.sync_with_ping()
354 else:
355 # Announce via headers
356 test_node.send_header_for_blocks(blocks)
357 test_node.wait_for_getdata([x.sha256 for x in blocks])
358 # Test that duplicate headers won't result in duplicate
359 # getdata requests (the check is further down)
360 inv_node.send_header_for_blocks(blocks)
361 inv_node.sync_with_ping()
362 [test_node.send_message(msg_block(x)) for x in blocks]
363 test_node.sync_with_ping()
364 inv_node.sync_with_ping()
365 # This block should not be announced to the inv node (since it also
366 # broadcast it)
367 assert "inv" not in inv_node.last_message
368 assert "headers" not in inv_node.last_message
369 tip = self.mine_blocks(1)
370 inv_node.check_last_announcement(inv=[tip], headers=[])
371 test_node.check_last_announcement(headers=[tip])
372 height += 1
373 block_time += 1
375 self.log.info("Part 2: success!")
377 self.log.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...")
379 # PART 3. Headers announcements can stop after large reorg, and resume after
380 # getheaders or inv from peer.
381 for j in range(2):
382 # First try mining a reorg that can propagate with header announcement
383 new_block_hashes = self.mine_reorg(length=7)
384 tip = new_block_hashes[-1]
385 inv_node.check_last_announcement(inv=[tip], headers=[])
386 test_node.check_last_announcement(headers=new_block_hashes)
388 block_time += 8
390 # Mine a too-large reorg, which should be announced with a single inv
391 new_block_hashes = self.mine_reorg(length=8)
392 tip = new_block_hashes[-1]
393 inv_node.check_last_announcement(inv=[tip], headers=[])
394 test_node.check_last_announcement(inv=[tip], headers=[])
396 block_time += 9
398 fork_point = self.nodes[0].getblock("%02x" % new_block_hashes[0])["previousblockhash"]
399 fork_point = int(fork_point, 16)
401 # Use getblocks/getdata
402 test_node.send_getblocks(locator=[fork_point])
403 test_node.check_last_announcement(inv=new_block_hashes, headers=[])
404 test_node.send_get_data(new_block_hashes)
405 test_node.wait_for_block(new_block_hashes[-1])
407 for i in range(3):
408 # Mine another block, still should get only an inv
409 tip = self.mine_blocks(1)
410 inv_node.check_last_announcement(inv=[tip], headers=[])
411 test_node.check_last_announcement(inv=[tip], headers=[])
412 if i == 0:
413 # Just get the data -- shouldn't cause headers announcements to resume
414 test_node.send_get_data([tip])
415 test_node.wait_for_block(tip)
416 elif i == 1:
417 # Send a getheaders message that shouldn't trigger headers announcements
418 # to resume (best header sent will be too old)
419 test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1])
420 test_node.send_get_data([tip])
421 test_node.wait_for_block(tip)
422 elif i == 2:
423 test_node.send_get_data([tip])
424 test_node.wait_for_block(tip)
425 # This time, try sending either a getheaders to trigger resumption
426 # of headers announcements, or mine a new block and inv it, also
427 # triggering resumption of headers announcements.
428 if j == 0:
429 test_node.send_get_headers(locator=[tip], hashstop=0)
430 test_node.sync_with_ping()
431 else:
432 test_node.send_block_inv(tip)
433 test_node.sync_with_ping()
434 # New blocks should now be announced with header
435 tip = self.mine_blocks(1)
436 inv_node.check_last_announcement(inv=[tip], headers=[])
437 test_node.check_last_announcement(headers=[tip])
439 self.log.info("Part 3: success!")
441 self.log.info("Part 4: Testing direct fetch behavior...")
442 tip = self.mine_blocks(1)
443 height = self.nodes[0].getblockcount() + 1
444 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
445 block_time = last_time + 1
447 # Create 2 blocks. Send the blocks, then send the headers.
448 blocks = []
449 for b in range(2):
450 blocks.append(create_block(tip, create_coinbase(height), block_time))
451 blocks[-1].solve()
452 tip = blocks[-1].sha256
453 block_time += 1
454 height += 1
455 inv_node.send_message(msg_block(blocks[-1]))
457 inv_node.sync_with_ping() # Make sure blocks are processed
458 test_node.last_message.pop("getdata", None)
459 test_node.send_header_for_blocks(blocks)
460 test_node.sync_with_ping()
461 # should not have received any getdata messages
462 with mininode_lock:
463 assert "getdata" not in test_node.last_message
465 # This time, direct fetch should work
466 blocks = []
467 for b in range(3):
468 blocks.append(create_block(tip, create_coinbase(height), block_time))
469 blocks[-1].solve()
470 tip = blocks[-1].sha256
471 block_time += 1
472 height += 1
474 test_node.send_header_for_blocks(blocks)
475 test_node.sync_with_ping()
476 test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME)
478 [test_node.send_message(msg_block(x)) for x in blocks]
480 test_node.sync_with_ping()
482 # Now announce a header that forks the last two blocks
483 tip = blocks[0].sha256
484 height -= 1
485 blocks = []
487 # Create extra blocks for later
488 for b in range(20):
489 blocks.append(create_block(tip, create_coinbase(height), block_time))
490 blocks[-1].solve()
491 tip = blocks[-1].sha256
492 block_time += 1
493 height += 1
495 # Announcing one block on fork should not trigger direct fetch
496 # (less work than tip)
497 test_node.last_message.pop("getdata", None)
498 test_node.send_header_for_blocks(blocks[0:1])
499 test_node.sync_with_ping()
500 with mininode_lock:
501 assert "getdata" not in test_node.last_message
503 # Announcing one more block on fork should trigger direct fetch for
504 # both blocks (same work as tip)
505 test_node.send_header_for_blocks(blocks[1:2])
506 test_node.sync_with_ping()
507 test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME)
509 # Announcing 16 more headers should trigger direct fetch for 14 more
510 # blocks
511 test_node.send_header_for_blocks(blocks[2:18])
512 test_node.sync_with_ping()
513 test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME)
515 # Announcing 1 more header should not trigger any response
516 test_node.last_message.pop("getdata", None)
517 test_node.send_header_for_blocks(blocks[18:19])
518 test_node.sync_with_ping()
519 with mininode_lock:
520 assert "getdata" not in test_node.last_message
522 self.log.info("Part 4: success!")
524 # Now deliver all those blocks we announced.
525 [test_node.send_message(msg_block(x)) for x in blocks]
527 self.log.info("Part 5: Testing handling of unconnecting headers")
528 # First we test that receipt of an unconnecting header doesn't prevent
529 # chain sync.
530 for i in range(10):
531 test_node.last_message.pop("getdata", None)
532 blocks = []
533 # Create two more blocks.
534 for j in range(2):
535 blocks.append(create_block(tip, create_coinbase(height), block_time))
536 blocks[-1].solve()
537 tip = blocks[-1].sha256
538 block_time += 1
539 height += 1
540 # Send the header of the second block -> this won't connect.
541 with mininode_lock:
542 test_node.last_message.pop("getheaders", None)
543 test_node.send_header_for_blocks([blocks[1]])
544 test_node.wait_for_getheaders()
545 test_node.send_header_for_blocks(blocks)
546 test_node.wait_for_getdata([x.sha256 for x in blocks])
547 [test_node.send_message(msg_block(x)) for x in blocks]
548 test_node.sync_with_ping()
549 assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256)
551 blocks = []
552 # Now we test that if we repeatedly don't send connecting headers, we
553 # don't go into an infinite loop trying to get them to connect.
554 MAX_UNCONNECTING_HEADERS = 10
555 for j in range(MAX_UNCONNECTING_HEADERS + 1):
556 blocks.append(create_block(tip, create_coinbase(height), block_time))
557 blocks[-1].solve()
558 tip = blocks[-1].sha256
559 block_time += 1
560 height += 1
562 for i in range(1, MAX_UNCONNECTING_HEADERS):
563 # Send a header that doesn't connect, check that we get a getheaders.
564 with mininode_lock:
565 test_node.last_message.pop("getheaders", None)
566 test_node.send_header_for_blocks([blocks[i]])
567 test_node.wait_for_getheaders()
569 # Next header will connect, should re-set our count:
570 test_node.send_header_for_blocks([blocks[0]])
572 # Remove the first two entries (blocks[1] would connect):
573 blocks = blocks[2:]
575 # Now try to see how many unconnecting headers we can send
576 # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS
577 for i in range(5 * MAX_UNCONNECTING_HEADERS - 1):
578 # Send a header that doesn't connect, check that we get a getheaders.
579 with mininode_lock:
580 test_node.last_message.pop("getheaders", None)
581 test_node.send_header_for_blocks([blocks[i % len(blocks)]])
582 test_node.wait_for_getheaders()
584 # Eventually this stops working.
585 test_node.send_header_for_blocks([blocks[-1]])
587 # Should get disconnected
588 test_node.wait_for_disconnect()
590 self.log.info("Part 5: success!")
592 # Finally, check that the inv node never received a getdata request,
593 # throughout the test
594 assert "getdata" not in inv_node.last_message
596 if __name__ == '__main__':
597 SendHeadersTest().main()