Merge #11580: Do not send (potentially) invalid headers in response to getheaders
[bitcoinplatinum.git] / test / functional / sendheaders.py
blob55bb80ea00555cf1835f6ec28a9652da960935b0
1 #!/usr/bin/env python3
2 # Copyright (c) 2014-2016 The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test behavior of headers messages to announce blocks.
7 Setup:
9 - Two nodes, two p2p connections to node0. One p2p connection should only ever
10 receive inv's (omitted from testing description below, this is our control).
11 Second node is used for creating reorgs.
13 test_null_locators
14 ==================
16 Sends two getheaders requests with null locator values. First request's hashstop
17 value refers to validated block, while second request's hashstop value refers to
18 a block which hasn't been validated. Verifies only the first request returns
19 headers.
21 test_nonnull_locators
22 =====================
24 Part 1: No headers announcements before "sendheaders"
25 a. node mines a block [expect: inv]
26 send getdata for the block [expect: block]
27 b. node mines another block [expect: inv]
28 send getheaders and getdata [expect: headers, then block]
29 c. node mines another block [expect: inv]
30 peer mines a block, announces with header [expect: getdata]
31 d. node mines another block [expect: inv]
33 Part 2: After "sendheaders", headers announcements should generally work.
34 a. peer sends sendheaders [expect: no response]
35 peer sends getheaders with current tip [expect: no response]
36 b. node mines a block [expect: tip header]
37 c. for N in 1, ..., 10:
38 * for announce-type in {inv, header}
39 - peer mines N blocks, announces with announce-type
40 [ expect: getheaders/getdata or getdata, deliver block(s) ]
41 - node mines a block [ expect: 1 header ]
43 Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer.
44 - For response-type in {inv, getheaders}
45 * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ]
46 * node mines an 8-block reorg [ expect: inv at tip ]
47 * peer responds with getblocks/getdata [expect: inv, blocks ]
48 * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ]
49 * node mines another block at tip [ expect: inv ]
50 * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers]
51 * peer requests block [ expect: block ]
52 * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ]
53 * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block]
54 * node mines 1 block [expect: 1 header, peer responds with getdata]
56 Part 4: Test direct fetch behavior
57 a. Announce 2 old block headers.
58 Expect: no getdata requests.
59 b. Announce 3 new blocks via 1 headers message.
60 Expect: one getdata request for all 3 blocks.
61 (Send blocks.)
62 c. Announce 1 header that forks off the last two blocks.
63 Expect: no response.
64 d. Announce 1 more header that builds on that fork.
65 Expect: one getdata request for two blocks.
66 e. Announce 16 more headers that build on that fork.
67 Expect: getdata request for 14 more blocks.
68 f. Announce 1 more header that builds on that fork.
69 Expect: no response.
71 Part 5: Test handling of headers that don't connect.
72 a. Repeat 10 times:
73 1. Announce a header that doesn't connect.
74 Expect: getheaders message
75 2. Send headers chain.
76 Expect: getdata for the missing blocks, tip update.
77 b. Then send 9 more headers that don't connect.
78 Expect: getheaders message each time.
79 c. Announce a header that does connect.
80 Expect: no response.
81 d. Announce 49 headers that don't connect.
82 Expect: getheaders message each time.
83 e. Announce one more that doesn't connect.
84 Expect: disconnect.
85 """
87 from test_framework.mininode import *
88 from test_framework.test_framework import BitcoinTestFramework
89 from test_framework.util import *
90 from test_framework.blocktools import create_block, create_coinbase
93 direct_fetch_response_time = 0.05
95 class TestNode(NodeConnCB):
96 def __init__(self):
97 super().__init__()
98 self.block_announced = False
99 self.last_blockhash_announced = None
101 def clear_last_announcement(self):
102 with mininode_lock:
103 self.block_announced = False
104 self.last_message.pop("inv", None)
105 self.last_message.pop("headers", None)
107 # Request data for a list of block hashes
108 def get_data(self, block_hashes):
109 msg = msg_getdata()
110 for x in block_hashes:
111 msg.inv.append(CInv(2, x))
112 self.connection.send_message(msg)
114 def get_headers(self, locator, hashstop):
115 msg = msg_getheaders()
116 msg.locator.vHave = locator
117 msg.hashstop = hashstop
118 self.connection.send_message(msg)
120 def send_block_inv(self, blockhash):
121 msg = msg_inv()
122 msg.inv = [CInv(2, blockhash)]
123 self.connection.send_message(msg)
125 def on_inv(self, conn, message):
126 self.block_announced = True
127 self.last_blockhash_announced = message.inv[-1].hash
129 def on_headers(self, conn, message):
130 if len(message.headers):
131 self.block_announced = True
132 message.headers[-1].calc_sha256()
133 self.last_blockhash_announced = message.headers[-1].sha256
135 # Test whether the last announcement we received had the
136 # right header or the right inv
137 # inv and headers should be lists of block hashes
138 def check_last_announcement(self, headers=None, inv=None):
139 expect_headers = headers if headers != None else []
140 expect_inv = inv if inv != None else []
141 test_function = lambda: self.block_announced
142 wait_until(test_function, timeout=60, lock=mininode_lock)
143 with mininode_lock:
144 self.block_announced = False
146 success = True
147 compare_inv = []
148 if "inv" in self.last_message:
149 compare_inv = [x.hash for x in self.last_message["inv"].inv]
150 if compare_inv != expect_inv:
151 success = False
153 hash_headers = []
154 if "headers" in self.last_message:
155 # treat headers as a list of block hashes
156 hash_headers = [ x.sha256 for x in self.last_message["headers"].headers ]
157 if hash_headers != expect_headers:
158 success = False
160 self.last_message.pop("inv", None)
161 self.last_message.pop("headers", None)
162 return success
164 def wait_for_getdata(self, hash_list, timeout=60):
165 if hash_list == []:
166 return
168 test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list
169 wait_until(test_function, timeout=timeout, lock=mininode_lock)
170 return
172 def wait_for_block_announcement(self, block_hash, timeout=60):
173 test_function = lambda: self.last_blockhash_announced == block_hash
174 wait_until(test_function, timeout=timeout, lock=mininode_lock)
175 return
177 def send_header_for_blocks(self, new_blocks):
178 headers_message = msg_headers()
179 headers_message.headers = [ CBlockHeader(b) for b in new_blocks ]
180 self.send_message(headers_message)
182 def send_getblocks(self, locator):
183 getblocks_message = msg_getblocks()
184 getblocks_message.locator.vHave = locator
185 self.send_message(getblocks_message)
187 class SendHeadersTest(BitcoinTestFramework):
188 def set_test_params(self):
189 self.setup_clean_chain = True
190 self.num_nodes = 2
192 # mine count blocks and return the new tip
193 def mine_blocks(self, count):
194 # Clear out last block announcement from each p2p listener
195 [x.clear_last_announcement() for x in self.nodes[0].p2ps]
196 self.nodes[0].generate(count)
197 return int(self.nodes[0].getbestblockhash(), 16)
199 # mine a reorg that invalidates length blocks (replacing them with
200 # length+1 blocks).
201 # Note: we clear the state of our p2p connections after the
202 # to-be-reorged-out blocks are mined, so that we don't break later tests.
203 # return the list of block hashes newly mined
204 def mine_reorg(self, length):
205 self.nodes[0].generate(length) # make sure all invalidated blocks are node0's
206 sync_blocks(self.nodes, wait=0.1)
207 for x in self.nodes[0].p2ps:
208 x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16))
209 x.clear_last_announcement()
211 tip_height = self.nodes[1].getblockcount()
212 hash_to_invalidate = self.nodes[1].getblockhash(tip_height-(length-1))
213 self.nodes[1].invalidateblock(hash_to_invalidate)
214 all_hashes = self.nodes[1].generate(length+1) # Must be longer than the orig chain
215 sync_blocks(self.nodes, wait=0.1)
216 return [int(x, 16) for x in all_hashes]
218 def run_test(self):
219 # Setup the p2p connections and start up the network thread.
220 inv_node = self.nodes[0].add_p2p_connection(TestNode())
221 # Set nServices to 0 for test_node, so no block download will occur outside of
222 # direct fetching
223 test_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_WITNESS)
225 NetworkThread().start() # Start up network handling in another thread
227 # Test logic begins here
228 inv_node.wait_for_verack()
229 test_node.wait_for_verack()
231 # Ensure verack's have been processed by our peer
232 inv_node.sync_with_ping()
233 test_node.sync_with_ping()
235 self.test_null_locators(test_node)
236 self.test_nonnull_locators(test_node, inv_node)
238 def test_null_locators(self, test_node):
239 tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0])
240 tip_hash = int(tip["hash"], 16)
242 self.log.info("Verify getheaders with null locator and valid hashstop returns headers.")
243 test_node.clear_last_announcement()
244 test_node.get_headers(locator=[], hashstop=tip_hash)
245 assert_equal(test_node.check_last_announcement(headers=[tip_hash]), True)
247 self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.")
248 block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1)
249 block.solve()
250 test_node.send_header_for_blocks([block])
251 test_node.clear_last_announcement()
252 test_node.get_headers(locator=[], hashstop=int(block.hash, 16))
253 test_node.sync_with_ping()
254 assert_equal(test_node.block_announced, False)
255 test_node.send_message(msg_block(block))
257 def test_nonnull_locators(self, test_node, inv_node):
258 tip = int(self.nodes[0].getbestblockhash(), 16)
260 # PART 1
261 # 1. Mine a block; expect inv announcements each time
262 self.log.info("Part 1: headers don't start before sendheaders message...")
263 for i in range(4):
264 old_tip = tip
265 tip = self.mine_blocks(1)
266 assert_equal(inv_node.check_last_announcement(inv=[tip]), True)
267 assert_equal(test_node.check_last_announcement(inv=[tip]), True)
268 # Try a few different responses; none should affect next announcement
269 if i == 0:
270 # first request the block
271 test_node.get_data([tip])
272 test_node.wait_for_block(tip)
273 elif i == 1:
274 # next try requesting header and block
275 test_node.get_headers(locator=[old_tip], hashstop=tip)
276 test_node.get_data([tip])
277 test_node.wait_for_block(tip)
278 test_node.clear_last_announcement() # since we requested headers...
279 elif i == 2:
280 # this time announce own block via headers
281 height = self.nodes[0].getblockcount()
282 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
283 block_time = last_time + 1
284 new_block = create_block(tip, create_coinbase(height+1), block_time)
285 new_block.solve()
286 test_node.send_header_for_blocks([new_block])
287 test_node.wait_for_getdata([new_block.sha256])
288 test_node.send_message(msg_block(new_block))
289 test_node.sync_with_ping() # make sure this block is processed
290 inv_node.clear_last_announcement()
291 test_node.clear_last_announcement()
293 self.log.info("Part 1: success!")
294 self.log.info("Part 2: announce blocks with headers after sendheaders message...")
295 # PART 2
296 # 2. Send a sendheaders message and test that headers announcements
297 # commence and keep working.
298 test_node.send_message(msg_sendheaders())
299 prev_tip = int(self.nodes[0].getbestblockhash(), 16)
300 test_node.get_headers(locator=[prev_tip], hashstop=0)
301 test_node.sync_with_ping()
303 # Now that we've synced headers, headers announcements should work
304 tip = self.mine_blocks(1)
305 assert_equal(inv_node.check_last_announcement(inv=[tip]), True)
306 assert_equal(test_node.check_last_announcement(headers=[tip]), True)
308 height = self.nodes[0].getblockcount()+1
309 block_time += 10 # Advance far enough ahead
310 for i in range(10):
311 # Mine i blocks, and alternate announcing either via
312 # inv (of tip) or via headers. After each, new blocks
313 # mined by the node should successfully be announced
314 # with block header, even though the blocks are never requested
315 for j in range(2):
316 blocks = []
317 for b in range(i+1):
318 blocks.append(create_block(tip, create_coinbase(height), block_time))
319 blocks[-1].solve()
320 tip = blocks[-1].sha256
321 block_time += 1
322 height += 1
323 if j == 0:
324 # Announce via inv
325 test_node.send_block_inv(tip)
326 test_node.wait_for_getheaders()
327 # Should have received a getheaders now
328 test_node.send_header_for_blocks(blocks)
329 # Test that duplicate inv's won't result in duplicate
330 # getdata requests, or duplicate headers announcements
331 [ inv_node.send_block_inv(x.sha256) for x in blocks ]
332 test_node.wait_for_getdata([x.sha256 for x in blocks])
333 inv_node.sync_with_ping()
334 else:
335 # Announce via headers
336 test_node.send_header_for_blocks(blocks)
337 test_node.wait_for_getdata([x.sha256 for x in blocks])
338 # Test that duplicate headers won't result in duplicate
339 # getdata requests (the check is further down)
340 inv_node.send_header_for_blocks(blocks)
341 inv_node.sync_with_ping()
342 [ test_node.send_message(msg_block(x)) for x in blocks ]
343 test_node.sync_with_ping()
344 inv_node.sync_with_ping()
345 # This block should not be announced to the inv node (since it also
346 # broadcast it)
347 assert "inv" not in inv_node.last_message
348 assert "headers" not in inv_node.last_message
349 tip = self.mine_blocks(1)
350 assert_equal(inv_node.check_last_announcement(inv=[tip]), True)
351 assert_equal(test_node.check_last_announcement(headers=[tip]), True)
352 height += 1
353 block_time += 1
355 self.log.info("Part 2: success!")
357 self.log.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...")
359 # PART 3. Headers announcements can stop after large reorg, and resume after
360 # getheaders or inv from peer.
361 for j in range(2):
362 # First try mining a reorg that can propagate with header announcement
363 new_block_hashes = self.mine_reorg(length=7)
364 tip = new_block_hashes[-1]
365 assert_equal(inv_node.check_last_announcement(inv=[tip]), True)
366 assert_equal(test_node.check_last_announcement(headers=new_block_hashes), True)
368 block_time += 8
370 # Mine a too-large reorg, which should be announced with a single inv
371 new_block_hashes = self.mine_reorg(length=8)
372 tip = new_block_hashes[-1]
373 assert_equal(inv_node.check_last_announcement(inv=[tip]), True)
374 assert_equal(test_node.check_last_announcement(inv=[tip]), True)
376 block_time += 9
378 fork_point = self.nodes[0].getblock("%02x" % new_block_hashes[0])["previousblockhash"]
379 fork_point = int(fork_point, 16)
381 # Use getblocks/getdata
382 test_node.send_getblocks(locator = [fork_point])
383 assert_equal(test_node.check_last_announcement(inv=new_block_hashes), True)
384 test_node.get_data(new_block_hashes)
385 test_node.wait_for_block(new_block_hashes[-1])
387 for i in range(3):
388 # Mine another block, still should get only an inv
389 tip = self.mine_blocks(1)
390 assert_equal(inv_node.check_last_announcement(inv=[tip]), True)
391 assert_equal(test_node.check_last_announcement(inv=[tip]), True)
392 if i == 0:
393 # Just get the data -- shouldn't cause headers announcements to resume
394 test_node.get_data([tip])
395 test_node.wait_for_block(tip)
396 elif i == 1:
397 # Send a getheaders message that shouldn't trigger headers announcements
398 # to resume (best header sent will be too old)
399 test_node.get_headers(locator=[fork_point], hashstop=new_block_hashes[1])
400 test_node.get_data([tip])
401 test_node.wait_for_block(tip)
402 elif i == 2:
403 test_node.get_data([tip])
404 test_node.wait_for_block(tip)
405 # This time, try sending either a getheaders to trigger resumption
406 # of headers announcements, or mine a new block and inv it, also
407 # triggering resumption of headers announcements.
408 if j == 0:
409 test_node.get_headers(locator=[tip], hashstop=0)
410 test_node.sync_with_ping()
411 else:
412 test_node.send_block_inv(tip)
413 test_node.sync_with_ping()
414 # New blocks should now be announced with header
415 tip = self.mine_blocks(1)
416 assert_equal(inv_node.check_last_announcement(inv=[tip]), True)
417 assert_equal(test_node.check_last_announcement(headers=[tip]), True)
419 self.log.info("Part 3: success!")
421 self.log.info("Part 4: Testing direct fetch behavior...")
422 tip = self.mine_blocks(1)
423 height = self.nodes[0].getblockcount() + 1
424 last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
425 block_time = last_time + 1
427 # Create 2 blocks. Send the blocks, then send the headers.
428 blocks = []
429 for b in range(2):
430 blocks.append(create_block(tip, create_coinbase(height), block_time))
431 blocks[-1].solve()
432 tip = blocks[-1].sha256
433 block_time += 1
434 height += 1
435 inv_node.send_message(msg_block(blocks[-1]))
437 inv_node.sync_with_ping() # Make sure blocks are processed
438 test_node.last_message.pop("getdata", None)
439 test_node.send_header_for_blocks(blocks)
440 test_node.sync_with_ping()
441 # should not have received any getdata messages
442 with mininode_lock:
443 assert "getdata" not in test_node.last_message
445 # This time, direct fetch should work
446 blocks = []
447 for b in range(3):
448 blocks.append(create_block(tip, create_coinbase(height), block_time))
449 blocks[-1].solve()
450 tip = blocks[-1].sha256
451 block_time += 1
452 height += 1
454 test_node.send_header_for_blocks(blocks)
455 test_node.sync_with_ping()
456 test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=direct_fetch_response_time)
458 [ test_node.send_message(msg_block(x)) for x in blocks ]
460 test_node.sync_with_ping()
462 # Now announce a header that forks the last two blocks
463 tip = blocks[0].sha256
464 height -= 1
465 blocks = []
467 # Create extra blocks for later
468 for b in range(20):
469 blocks.append(create_block(tip, create_coinbase(height), block_time))
470 blocks[-1].solve()
471 tip = blocks[-1].sha256
472 block_time += 1
473 height += 1
475 # Announcing one block on fork should not trigger direct fetch
476 # (less work than tip)
477 test_node.last_message.pop("getdata", None)
478 test_node.send_header_for_blocks(blocks[0:1])
479 test_node.sync_with_ping()
480 with mininode_lock:
481 assert "getdata" not in test_node.last_message
483 # Announcing one more block on fork should trigger direct fetch for
484 # both blocks (same work as tip)
485 test_node.send_header_for_blocks(blocks[1:2])
486 test_node.sync_with_ping()
487 test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=direct_fetch_response_time)
489 # Announcing 16 more headers should trigger direct fetch for 14 more
490 # blocks
491 test_node.send_header_for_blocks(blocks[2:18])
492 test_node.sync_with_ping()
493 test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=direct_fetch_response_time)
495 # Announcing 1 more header should not trigger any response
496 test_node.last_message.pop("getdata", None)
497 test_node.send_header_for_blocks(blocks[18:19])
498 test_node.sync_with_ping()
499 with mininode_lock:
500 assert "getdata" not in test_node.last_message
502 self.log.info("Part 4: success!")
504 # Now deliver all those blocks we announced.
505 [ test_node.send_message(msg_block(x)) for x in blocks ]
507 self.log.info("Part 5: Testing handling of unconnecting headers")
508 # First we test that receipt of an unconnecting header doesn't prevent
509 # chain sync.
510 for i in range(10):
511 test_node.last_message.pop("getdata", None)
512 blocks = []
513 # Create two more blocks.
514 for j in range(2):
515 blocks.append(create_block(tip, create_coinbase(height), block_time))
516 blocks[-1].solve()
517 tip = blocks[-1].sha256
518 block_time += 1
519 height += 1
520 # Send the header of the second block -> this won't connect.
521 with mininode_lock:
522 test_node.last_message.pop("getheaders", None)
523 test_node.send_header_for_blocks([blocks[1]])
524 test_node.wait_for_getheaders()
525 test_node.send_header_for_blocks(blocks)
526 test_node.wait_for_getdata([x.sha256 for x in blocks])
527 [ test_node.send_message(msg_block(x)) for x in blocks ]
528 test_node.sync_with_ping()
529 assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256)
531 blocks = []
532 # Now we test that if we repeatedly don't send connecting headers, we
533 # don't go into an infinite loop trying to get them to connect.
534 MAX_UNCONNECTING_HEADERS = 10
535 for j in range(MAX_UNCONNECTING_HEADERS+1):
536 blocks.append(create_block(tip, create_coinbase(height), block_time))
537 blocks[-1].solve()
538 tip = blocks[-1].sha256
539 block_time += 1
540 height += 1
542 for i in range(1, MAX_UNCONNECTING_HEADERS):
543 # Send a header that doesn't connect, check that we get a getheaders.
544 with mininode_lock:
545 test_node.last_message.pop("getheaders", None)
546 test_node.send_header_for_blocks([blocks[i]])
547 test_node.wait_for_getheaders()
549 # Next header will connect, should re-set our count:
550 test_node.send_header_for_blocks([blocks[0]])
552 # Remove the first two entries (blocks[1] would connect):
553 blocks = blocks[2:]
555 # Now try to see how many unconnecting headers we can send
556 # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS
557 for i in range(5*MAX_UNCONNECTING_HEADERS - 1):
558 # Send a header that doesn't connect, check that we get a getheaders.
559 with mininode_lock:
560 test_node.last_message.pop("getheaders", None)
561 test_node.send_header_for_blocks([blocks[i%len(blocks)]])
562 test_node.wait_for_getheaders()
564 # Eventually this stops working.
565 test_node.send_header_for_blocks([blocks[-1]])
567 # Should get disconnected
568 test_node.wait_for_disconnect()
570 self.log.info("Part 5: success!")
572 # Finally, check that the inv node never received a getdata request,
573 # throughout the test
574 assert "getdata" not in inv_node.last_message
576 if __name__ == '__main__':
577 SendHeadersTest().main()