[tests] refactor check_last_announcement() in sendheaders.py
[bitcoinplatinum.git] / test / functional / sendheaders.py
blob1a585dd111f0d7caaf0fb5f3afb124c71c8ca191
1 #!/usr/bin/env python3
2 # Copyright (c) 2014-2016 The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test behavior of headers messages to announce blocks.
7 Setup:
9 - Two nodes, two p2p connections to node0. One p2p connection should only ever
10 receive inv's (omitted from testing description below, this is our control).
11 Second node is used for creating reorgs.
13 test_null_locators
14 ==================
16 Sends two getheaders requests with null locator values. First request's hashstop
17 value refers to validated block, while second request's hashstop value refers to
18 a block which hasn't been validated. Verifies only the first request returns
19 headers.
21 test_nonnull_locators
22 =====================
24 Part 1: No headers announcements before "sendheaders"
25 a. node mines a block [expect: inv]
26 send getdata for the block [expect: block]
27 b. node mines another block [expect: inv]
28 send getheaders and getdata [expect: headers, then block]
29 c. node mines another block [expect: inv]
30 peer mines a block, announces with header [expect: getdata]
31 d. node mines another block [expect: inv]
33 Part 2: After "sendheaders", headers announcements should generally work.
34 a. peer sends sendheaders [expect: no response]
35 peer sends getheaders with current tip [expect: no response]
36 b. node mines a block [expect: tip header]
37 c. for N in 1, ..., 10:
38 * for announce-type in {inv, header}
39 - peer mines N blocks, announces with announce-type
40 [ expect: getheaders/getdata or getdata, deliver block(s) ]
41 - node mines a block [ expect: 1 header ]
43 Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer.
44 - For response-type in {inv, getheaders}
45 * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ]
46 * node mines an 8-block reorg [ expect: inv at tip ]
47 * peer responds with getblocks/getdata [expect: inv, blocks ]
48 * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ]
49 * node mines another block at tip [ expect: inv ]
50 * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers]
51 * peer requests block [ expect: block ]
52 * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ]
53 * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block]
54 * node mines 1 block [expect: 1 header, peer responds with getdata]
56 Part 4: Test direct fetch behavior
57 a. Announce 2 old block headers.
58 Expect: no getdata requests.
59 b. Announce 3 new blocks via 1 headers message.
60 Expect: one getdata request for all 3 blocks.
61 (Send blocks.)
62 c. Announce 1 header that forks off the last two blocks.
63 Expect: no response.
64 d. Announce 1 more header that builds on that fork.
65 Expect: one getdata request for two blocks.
66 e. Announce 16 more headers that build on that fork.
67 Expect: getdata request for 14 more blocks.
68 f. Announce 1 more header that builds on that fork.
69 Expect: no response.
71 Part 5: Test handling of headers that don't connect.
72 a. Repeat 10 times:
73 1. Announce a header that doesn't connect.
74 Expect: getheaders message
75 2. Send headers chain.
76 Expect: getdata for the missing blocks, tip update.
77 b. Then send 9 more headers that don't connect.
78 Expect: getheaders message each time.
79 c. Announce a header that does connect.
80 Expect: no response.
81 d. Announce 49 headers that don't connect.
82 Expect: getheaders message each time.
83 e. Announce one more that doesn't connect.
84 Expect: disconnect.
85 """
86 from test_framework.blocktools import create_block, create_coinbase
87 from test_framework.mininode import (
88 CBlockHeader,
89 CInv,
90 NODE_WITNESS,
91 NetworkThread,
92 NodeConnCB,
93 mininode_lock,
94 msg_block,
95 msg_getblocks,
96 msg_getdata,
97 msg_getheaders,
98 msg_headers,
99 msg_inv,
100 msg_sendheaders,
102 from test_framework.test_framework import BitcoinTestFramework
103 from test_framework.util import (
104 assert_equal,
105 sync_blocks,
106 wait_until,
109 DIRECT_FETCH_RESPONSE_TIME = 0.05
111 class BaseNode(NodeConnCB):
112 def __init__(self):
113 super().__init__()
114 self.block_announced = False
115 self.last_blockhash_announced = None
117 def send_get_data(self, block_hashes):
118 """Request data for a list of block hashes."""
119 msg = msg_getdata()
120 for x in block_hashes:
121 msg.inv.append(CInv(2, x))
122 self.connection.send_message(msg)
124 def send_get_headers(self, locator, hashstop):
125 msg = msg_getheaders()
126 msg.locator.vHave = locator
127 msg.hashstop = hashstop
128 self.connection.send_message(msg)
130 def send_block_inv(self, blockhash):
131 msg = msg_inv()
132 msg.inv = [CInv(2, blockhash)]
133 self.connection.send_message(msg)
135 def send_header_for_blocks(self, new_blocks):
136 headers_message = msg_headers()
137 headers_message.headers = [CBlockHeader(b) for b in new_blocks]
138 self.send_message(headers_message)
140 def send_getblocks(self, locator):
141 getblocks_message = msg_getblocks()
142 getblocks_message.locator.vHave = locator
143 self.send_message(getblocks_message)
145 def wait_for_getdata(self, hash_list, timeout=60):
146 if hash_list != []:
147 test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list
148 wait_until(test_function, timeout=timeout, lock=mininode_lock)
150 def wait_for_block_announcement(self, block_hash, timeout=60):
151 test_function = lambda: self.last_blockhash_announced == block_hash
152 wait_until(test_function, timeout=timeout, lock=mininode_lock)
154 def on_inv(self, conn, message):
155 self.block_announced = True
156 self.last_blockhash_announced = message.inv[-1].hash
158 def on_headers(self, conn, message):
159 if len(message.headers):
160 self.block_announced = True
161 message.headers[-1].calc_sha256()
162 self.last_blockhash_announced = message.headers[-1].sha256
164 def clear_last_announcement(self):
165 with mininode_lock:
166 self.block_announced = False
167 self.last_message.pop("inv", None)
168 self.last_message.pop("headers", None)
170 def check_last_announcement(self, headers=None, inv=None):
171 """Test whether the last announcement received had the right header or the right inv.
173 inv and headers should be lists of block hashes."""
175 test_function = lambda: self.block_announced
176 wait_until(test_function, timeout=60, lock=mininode_lock)
178 with mininode_lock:
179 self.block_announced = False
181 compare_inv = []
182 if "inv" in self.last_message:
183 compare_inv = [x.hash for x in self.last_message["inv"].inv]
184 if inv is not None:
185 assert_equal(compare_inv, inv)
187 compare_headers = []
188 if "headers" in self.last_message:
189 compare_headers = [x.sha256 for x in self.last_message["headers"].headers]
190 if headers is not None:
191 assert_equal(compare_headers, headers)
193 self.last_message.pop("inv", None)
194 self.last_message.pop("headers", None)
196 class SendHeadersTest(BitcoinTestFramework):
197 def set_test_params(self):
198 self.setup_clean_chain = True
199 self.num_nodes = 2
201 def mine_blocks(self, count):
202 """Mine count blocks and return the new tip."""
204 # Clear out last block announcement from each p2p listener
205 [x.clear_last_announcement() for x in self.nodes[0].p2ps]
206 self.nodes[0].generate(count)
207 return int(self.nodes[0].getbestblockhash(), 16)
209 def mine_reorg(self, length):
210 """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks).
212 Note: we clear the state of our p2p connections after the
213 to-be-reorged-out blocks are mined, so that we don't break later tests.
214 return the list of block hashes newly mined."""
216 self.nodes[0].generate(length) # make sure all invalidated blocks are node0's
217 sync_blocks(self.nodes, wait=0.1)
218 for x in self.nodes[0].p2ps:
219 x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16))
220 x.clear_last_announcement()
222 tip_height = self.nodes[1].getblockcount()
223 hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1))
224 self.nodes[1].invalidateblock(hash_to_invalidate)
225 all_hashes = self.nodes[1].generate(length + 1) # Must be longer than the orig chain
226 sync_blocks(self.nodes, wait=0.1)
227 return [int(x, 16) for x in all_hashes]
229 def run_test(self):
230 # Setup the p2p connections and start up the network thread.
231 inv_node = self.nodes[0].add_p2p_connection(BaseNode())
232 # Set nServices to 0 for test_node, so no block download will occur outside of
233 # direct fetching
234 test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS)
236 NetworkThread().start() # Start up network handling in another thread
238 # Test logic begins here
239 inv_node.wait_for_verack()
240 test_node.wait_for_verack()
242 # Ensure verack's have been processed by our peer
243 inv_node.sync_with_ping()
244 test_node.sync_with_ping()
246 self.test_null_locators(test_node)
247 self.test_nonnull_locators(test_node, inv_node)
249 def test_null_locators(self, test_node):
250 tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0])
251 tip_hash = int(tip["hash"], 16)
253 self.log.info("Verify getheaders with null locator and valid hashstop returns headers.")
254 test_node.clear_last_announcement()
255 test_node.send_get_headers(locator=[], hashstop=tip_hash)
256 test_node.check_last_announcement(headers=[tip_hash])
258 self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.")
259 block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1)
260 block.solve()
261 test_node.send_header_for_blocks([block])
262 test_node.clear_last_announcement()
263 test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16))
264 test_node.sync_with_ping()
265 assert_equal(test_node.block_announced, False)
266 test_node.send_message(msg_block(block))
268 def test_nonnull_locators(self, test_node, inv_node):
269 tip = int(self.nodes[0].getbestblockhash(), 16)
271 # PART 1
272 # 1. Mine a block; expect inv announcements each time
273 self.log.info("Part 1: headers don't start before sendheaders message...")
274 for i in range(4):
275 old_tip = tip
276 tip = self.mine_blocks(1)
277 inv_node.check_last_announcement(inv=[tip], headers=[])
278 test_node.check_last_announcement(inv=[tip], headers=[])
279 # Try a few different responses; none should affect next announcement
280 if i == 0:
281 # first request the block
282 test_node.send_get_data([tip])
283 test_node.wait_for_block(tip)
284 elif i == 1:
285 # next try requesting header and block
286 test_node.send_get_headers(locator=[old_tip], hashstop=tip)
287 test_node.send_get_data([tip])
288 test_node.wait_for_block(tip)
289 test_node.clear_last_announcement() # since we requested headers...
290 elif i == 2:
291 # this time announce own block via headers
292 height = self.nodes[0].getblockcount()
293 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
294 block_time = last_time + 1
295 new_block = create_block(tip, create_coinbase(height + 1), block_time)
296 new_block.solve()
297 test_node.send_header_for_blocks([new_block])
298 test_node.wait_for_getdata([new_block.sha256])
299 test_node.send_message(msg_block(new_block))
300 test_node.sync_with_ping() # make sure this block is processed
301 inv_node.clear_last_announcement()
302 test_node.clear_last_announcement()
304 self.log.info("Part 1: success!")
305 self.log.info("Part 2: announce blocks with headers after sendheaders message...")
306 # PART 2
307 # 2. Send a sendheaders message and test that headers announcements
308 # commence and keep working.
309 test_node.send_message(msg_sendheaders())
310 prev_tip = int(self.nodes[0].getbestblockhash(), 16)
311 test_node.send_get_headers(locator=[prev_tip], hashstop=0)
312 test_node.sync_with_ping()
314 # Now that we've synced headers, headers announcements should work
315 tip = self.mine_blocks(1)
316 inv_node.check_last_announcement(inv=[tip], headers=[])
317 test_node.check_last_announcement(headers=[tip])
319 height = self.nodes[0].getblockcount() + 1
320 block_time += 10 # Advance far enough ahead
321 for i in range(10):
322 # Mine i blocks, and alternate announcing either via
323 # inv (of tip) or via headers. After each, new blocks
324 # mined by the node should successfully be announced
325 # with block header, even though the blocks are never requested
326 for j in range(2):
327 blocks = []
328 for b in range(i + 1):
329 blocks.append(create_block(tip, create_coinbase(height), block_time))
330 blocks[-1].solve()
331 tip = blocks[-1].sha256
332 block_time += 1
333 height += 1
334 if j == 0:
335 # Announce via inv
336 test_node.send_block_inv(tip)
337 test_node.wait_for_getheaders()
338 # Should have received a getheaders now
339 test_node.send_header_for_blocks(blocks)
340 # Test that duplicate inv's won't result in duplicate
341 # getdata requests, or duplicate headers announcements
342 [inv_node.send_block_inv(x.sha256) for x in blocks]
343 test_node.wait_for_getdata([x.sha256 for x in blocks])
344 inv_node.sync_with_ping()
345 else:
346 # Announce via headers
347 test_node.send_header_for_blocks(blocks)
348 test_node.wait_for_getdata([x.sha256 for x in blocks])
349 # Test that duplicate headers won't result in duplicate
350 # getdata requests (the check is further down)
351 inv_node.send_header_for_blocks(blocks)
352 inv_node.sync_with_ping()
353 [test_node.send_message(msg_block(x)) for x in blocks]
354 test_node.sync_with_ping()
355 inv_node.sync_with_ping()
356 # This block should not be announced to the inv node (since it also
357 # broadcast it)
358 assert "inv" not in inv_node.last_message
359 assert "headers" not in inv_node.last_message
360 tip = self.mine_blocks(1)
361 inv_node.check_last_announcement(inv=[tip], headers=[])
362 test_node.check_last_announcement(headers=[tip])
363 height += 1
364 block_time += 1
366 self.log.info("Part 2: success!")
368 self.log.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...")
370 # PART 3. Headers announcements can stop after large reorg, and resume after
371 # getheaders or inv from peer.
372 for j in range(2):
373 # First try mining a reorg that can propagate with header announcement
374 new_block_hashes = self.mine_reorg(length=7)
375 tip = new_block_hashes[-1]
376 inv_node.check_last_announcement(inv=[tip], headers=[])
377 test_node.check_last_announcement(headers=new_block_hashes)
379 block_time += 8
381 # Mine a too-large reorg, which should be announced with a single inv
382 new_block_hashes = self.mine_reorg(length=8)
383 tip = new_block_hashes[-1]
384 inv_node.check_last_announcement(inv=[tip], headers=[])
385 test_node.check_last_announcement(inv=[tip], headers=[])
387 block_time += 9
389 fork_point = self.nodes[0].getblock("%02x" % new_block_hashes[0])["previousblockhash"]
390 fork_point = int(fork_point, 16)
392 # Use getblocks/getdata
393 test_node.send_getblocks(locator=[fork_point])
394 test_node.check_last_announcement(inv=new_block_hashes, headers=[])
395 test_node.send_get_data(new_block_hashes)
396 test_node.wait_for_block(new_block_hashes[-1])
398 for i in range(3):
399 # Mine another block, still should get only an inv
400 tip = self.mine_blocks(1)
401 inv_node.check_last_announcement(inv=[tip], headers=[])
402 test_node.check_last_announcement(inv=[tip], headers=[])
403 if i == 0:
404 # Just get the data -- shouldn't cause headers announcements to resume
405 test_node.send_get_data([tip])
406 test_node.wait_for_block(tip)
407 elif i == 1:
408 # Send a getheaders message that shouldn't trigger headers announcements
409 # to resume (best header sent will be too old)
410 test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1])
411 test_node.send_get_data([tip])
412 test_node.wait_for_block(tip)
413 elif i == 2:
414 test_node.send_get_data([tip])
415 test_node.wait_for_block(tip)
416 # This time, try sending either a getheaders to trigger resumption
417 # of headers announcements, or mine a new block and inv it, also
418 # triggering resumption of headers announcements.
419 if j == 0:
420 test_node.send_get_headers(locator=[tip], hashstop=0)
421 test_node.sync_with_ping()
422 else:
423 test_node.send_block_inv(tip)
424 test_node.sync_with_ping()
425 # New blocks should now be announced with header
426 tip = self.mine_blocks(1)
427 inv_node.check_last_announcement(inv=[tip], headers=[])
428 test_node.check_last_announcement(headers=[tip])
430 self.log.info("Part 3: success!")
432 self.log.info("Part 4: Testing direct fetch behavior...")
433 tip = self.mine_blocks(1)
434 height = self.nodes[0].getblockcount() + 1
435 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
436 block_time = last_time + 1
438 # Create 2 blocks. Send the blocks, then send the headers.
439 blocks = []
440 for b in range(2):
441 blocks.append(create_block(tip, create_coinbase(height), block_time))
442 blocks[-1].solve()
443 tip = blocks[-1].sha256
444 block_time += 1
445 height += 1
446 inv_node.send_message(msg_block(blocks[-1]))
448 inv_node.sync_with_ping() # Make sure blocks are processed
449 test_node.last_message.pop("getdata", None)
450 test_node.send_header_for_blocks(blocks)
451 test_node.sync_with_ping()
452 # should not have received any getdata messages
453 with mininode_lock:
454 assert "getdata" not in test_node.last_message
456 # This time, direct fetch should work
457 blocks = []
458 for b in range(3):
459 blocks.append(create_block(tip, create_coinbase(height), block_time))
460 blocks[-1].solve()
461 tip = blocks[-1].sha256
462 block_time += 1
463 height += 1
465 test_node.send_header_for_blocks(blocks)
466 test_node.sync_with_ping()
467 test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME)
469 [test_node.send_message(msg_block(x)) for x in blocks]
471 test_node.sync_with_ping()
473 # Now announce a header that forks the last two blocks
474 tip = blocks[0].sha256
475 height -= 1
476 blocks = []
478 # Create extra blocks for later
479 for b in range(20):
480 blocks.append(create_block(tip, create_coinbase(height), block_time))
481 blocks[-1].solve()
482 tip = blocks[-1].sha256
483 block_time += 1
484 height += 1
486 # Announcing one block on fork should not trigger direct fetch
487 # (less work than tip)
488 test_node.last_message.pop("getdata", None)
489 test_node.send_header_for_blocks(blocks[0:1])
490 test_node.sync_with_ping()
491 with mininode_lock:
492 assert "getdata" not in test_node.last_message
494 # Announcing one more block on fork should trigger direct fetch for
495 # both blocks (same work as tip)
496 test_node.send_header_for_blocks(blocks[1:2])
497 test_node.sync_with_ping()
498 test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME)
500 # Announcing 16 more headers should trigger direct fetch for 14 more
501 # blocks
502 test_node.send_header_for_blocks(blocks[2:18])
503 test_node.sync_with_ping()
504 test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME)
506 # Announcing 1 more header should not trigger any response
507 test_node.last_message.pop("getdata", None)
508 test_node.send_header_for_blocks(blocks[18:19])
509 test_node.sync_with_ping()
510 with mininode_lock:
511 assert "getdata" not in test_node.last_message
513 self.log.info("Part 4: success!")
515 # Now deliver all those blocks we announced.
516 [test_node.send_message(msg_block(x)) for x in blocks]
518 self.log.info("Part 5: Testing handling of unconnecting headers")
519 # First we test that receipt of an unconnecting header doesn't prevent
520 # chain sync.
521 for i in range(10):
522 test_node.last_message.pop("getdata", None)
523 blocks = []
524 # Create two more blocks.
525 for j in range(2):
526 blocks.append(create_block(tip, create_coinbase(height), block_time))
527 blocks[-1].solve()
528 tip = blocks[-1].sha256
529 block_time += 1
530 height += 1
531 # Send the header of the second block -> this won't connect.
532 with mininode_lock:
533 test_node.last_message.pop("getheaders", None)
534 test_node.send_header_for_blocks([blocks[1]])
535 test_node.wait_for_getheaders()
536 test_node.send_header_for_blocks(blocks)
537 test_node.wait_for_getdata([x.sha256 for x in blocks])
538 [test_node.send_message(msg_block(x)) for x in blocks]
539 test_node.sync_with_ping()
540 assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256)
542 blocks = []
543 # Now we test that if we repeatedly don't send connecting headers, we
544 # don't go into an infinite loop trying to get them to connect.
545 MAX_UNCONNECTING_HEADERS = 10
546 for j in range(MAX_UNCONNECTING_HEADERS + 1):
547 blocks.append(create_block(tip, create_coinbase(height), block_time))
548 blocks[-1].solve()
549 tip = blocks[-1].sha256
550 block_time += 1
551 height += 1
553 for i in range(1, MAX_UNCONNECTING_HEADERS):
554 # Send a header that doesn't connect, check that we get a getheaders.
555 with mininode_lock:
556 test_node.last_message.pop("getheaders", None)
557 test_node.send_header_for_blocks([blocks[i]])
558 test_node.wait_for_getheaders()
560 # Next header will connect, should re-set our count:
561 test_node.send_header_for_blocks([blocks[0]])
563 # Remove the first two entries (blocks[1] would connect):
564 blocks = blocks[2:]
566 # Now try to see how many unconnecting headers we can send
567 # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS
568 for i in range(5 * MAX_UNCONNECTING_HEADERS - 1):
569 # Send a header that doesn't connect, check that we get a getheaders.
570 with mininode_lock:
571 test_node.last_message.pop("getheaders", None)
572 test_node.send_header_for_blocks([blocks[i % len(blocks)]])
573 test_node.wait_for_getheaders()
575 # Eventually this stops working.
576 test_node.send_header_for_blocks([blocks[-1]])
578 # Should get disconnected
579 test_node.wait_for_disconnect()
581 self.log.info("Part 5: success!")
583 # Finally, check that the inv node never received a getdata request,
584 # throughout the test
585 assert "getdata" not in inv_node.last_message
587 if __name__ == '__main__':
588 SendHeadersTest().main()