2 # Copyright (c) 2014-2016 The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Test behavior of headers messages to announce blocks.
10 - node0 is the node-under-test. We create two p2p connections to it. The
11 first p2p connection is a control and should only ever receive inv's. The
12 second p2p connection tests the headers sending logic.
13 - node1 is used to create reorgs.
18 Sends two getheaders requests with null locator values. First request's hashstop
19 value refers to validated block, while second request's hashstop value refers to
20 a block which hasn't been validated. Verifies only the first request returns
26 Part 1: No headers announcements before "sendheaders"
27 a. node mines a block [expect: inv]
28 send getdata for the block [expect: block]
29 b. node mines another block [expect: inv]
30 send getheaders and getdata [expect: headers, then block]
31 c. node mines another block [expect: inv]
32 peer mines a block, announces with header [expect: getdata]
33 d. node mines another block [expect: inv]
35 Part 2: After "sendheaders", headers announcements should generally work.
36 a. peer sends sendheaders [expect: no response]
37 peer sends getheaders with current tip [expect: no response]
38 b. node mines a block [expect: tip header]
39 c. for N in 1, ..., 10:
40 * for announce-type in {inv, header}
41 - peer mines N blocks, announces with announce-type
42 [ expect: getheaders/getdata or getdata, deliver block(s) ]
43 - node mines a block [ expect: 1 header ]
45 Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer.
46 - For response-type in {inv, getheaders}
47 * node mines a 7 block reorg [ expect: headers announcement of 8 blocks ]
48 * node mines an 8-block reorg [ expect: inv at tip ]
49 * peer responds with getblocks/getdata [expect: inv, blocks ]
50 * node mines another block [ expect: inv at tip, peer sends getdata, expect: block ]
51 * node mines another block at tip [ expect: inv ]
52 * peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers]
53 * peer requests block [ expect: block ]
54 * node mines another block at tip [ expect: inv, peer sends getdata, expect: block ]
55 * peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block]
56 * node mines 1 block [expect: 1 header, peer responds with getdata]
58 Part 4: Test direct fetch behavior
59 a. Announce 2 old block headers.
60 Expect: no getdata requests.
61 b. Announce 3 new blocks via 1 headers message.
62 Expect: one getdata request for all 3 blocks.
64 c. Announce 1 header that forks off the last two blocks.
66 d. Announce 1 more header that builds on that fork.
67 Expect: one getdata request for two blocks.
68 e. Announce 16 more headers that build on that fork.
69 Expect: getdata request for 14 more blocks.
70 f. Announce 1 more header that builds on that fork.
73 Part 5: Test handling of headers that don't connect.
75 1. Announce a header that doesn't connect.
76 Expect: getheaders message
77 2. Send headers chain.
78 Expect: getdata for the missing blocks, tip update.
79 b. Then send 9 more headers that don't connect.
80 Expect: getheaders message each time.
81 c. Announce a header that does connect.
83 d. Announce 49 headers that don't connect.
84 Expect: getheaders message each time.
85 e. Announce one more that doesn't connect.
88 from test_framework
.blocktools
import create_block
, create_coinbase
89 from test_framework
.mininode
import (
104 from test_framework
.test_framework
import BitcoinTestFramework
105 from test_framework
.util
import (
111 DIRECT_FETCH_RESPONSE_TIME
= 0.05
113 class BaseNode(NodeConnCB
):
116 self
.block_announced
= False
117 self
.last_blockhash_announced
= None
119 def send_get_data(self
, block_hashes
):
120 """Request data for a list of block hashes."""
122 for x
in block_hashes
:
123 msg
.inv
.append(CInv(2, x
))
124 self
.connection
.send_message(msg
)
126 def send_get_headers(self
, locator
, hashstop
):
127 msg
= msg_getheaders()
128 msg
.locator
.vHave
= locator
129 msg
.hashstop
= hashstop
130 self
.connection
.send_message(msg
)
132 def send_block_inv(self
, blockhash
):
134 msg
.inv
= [CInv(2, blockhash
)]
135 self
.connection
.send_message(msg
)
137 def send_header_for_blocks(self
, new_blocks
):
138 headers_message
= msg_headers()
139 headers_message
.headers
= [CBlockHeader(b
) for b
in new_blocks
]
140 self
.send_message(headers_message
)
142 def send_getblocks(self
, locator
):
143 getblocks_message
= msg_getblocks()
144 getblocks_message
.locator
.vHave
= locator
145 self
.send_message(getblocks_message
)
147 def wait_for_getdata(self
, hash_list
, timeout
=60):
151 test_function
= lambda: "getdata" in self
.last_message
and [x
.hash for x
in self
.last_message
["getdata"].inv
] == hash_list
152 wait_until(test_function
, timeout
=timeout
, lock
=mininode_lock
)
154 def wait_for_block_announcement(self
, block_hash
, timeout
=60):
155 test_function
= lambda: self
.last_blockhash_announced
== block_hash
156 wait_until(test_function
, timeout
=timeout
, lock
=mininode_lock
)
158 def on_inv(self
, conn
, message
):
159 self
.block_announced
= True
160 self
.last_blockhash_announced
= message
.inv
[-1].hash
162 def on_headers(self
, conn
, message
):
163 if len(message
.headers
):
164 self
.block_announced
= True
165 message
.headers
[-1].calc_sha256()
166 self
.last_blockhash_announced
= message
.headers
[-1].sha256
168 def clear_last_announcement(self
):
170 self
.block_announced
= False
171 self
.last_message
.pop("inv", None)
172 self
.last_message
.pop("headers", None)
174 def check_last_announcement(self
, headers
=None, inv
=None):
175 """Test whether the last announcement received had the right header or the right inv.
177 inv and headers should be lists of block hashes."""
179 test_function
= lambda: self
.block_announced
180 wait_until(test_function
, timeout
=60, lock
=mininode_lock
)
183 self
.block_announced
= False
186 if "inv" in self
.last_message
:
187 compare_inv
= [x
.hash for x
in self
.last_message
["inv"].inv
]
189 assert_equal(compare_inv
, inv
)
192 if "headers" in self
.last_message
:
193 compare_headers
= [x
.sha256
for x
in self
.last_message
["headers"].headers
]
194 if headers
is not None:
195 assert_equal(compare_headers
, headers
)
197 self
.last_message
.pop("inv", None)
198 self
.last_message
.pop("headers", None)
200 class SendHeadersTest(BitcoinTestFramework
):
201 def set_test_params(self
):
202 self
.setup_clean_chain
= True
205 def mine_blocks(self
, count
):
206 """Mine count blocks and return the new tip."""
208 # Clear out last block announcement from each p2p listener
209 [x
.clear_last_announcement() for x
in self
.nodes
[0].p2ps
]
210 self
.nodes
[0].generate(count
)
211 return int(self
.nodes
[0].getbestblockhash(), 16)
213 def mine_reorg(self
, length
):
214 """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks).
216 Note: we clear the state of our p2p connections after the
217 to-be-reorged-out blocks are mined, so that we don't break later tests.
218 return the list of block hashes newly mined."""
220 self
.nodes
[0].generate(length
) # make sure all invalidated blocks are node0's
221 sync_blocks(self
.nodes
, wait
=0.1)
222 for x
in self
.nodes
[0].p2ps
:
223 x
.wait_for_block_announcement(int(self
.nodes
[0].getbestblockhash(), 16))
224 x
.clear_last_announcement()
226 tip_height
= self
.nodes
[1].getblockcount()
227 hash_to_invalidate
= self
.nodes
[1].getblockhash(tip_height
- (length
- 1))
228 self
.nodes
[1].invalidateblock(hash_to_invalidate
)
229 all_hashes
= self
.nodes
[1].generate(length
+ 1) # Must be longer than the orig chain
230 sync_blocks(self
.nodes
, wait
=0.1)
231 return [int(x
, 16) for x
in all_hashes
]
234 # Setup the p2p connections and start up the network thread.
235 inv_node
= self
.nodes
[0].add_p2p_connection(BaseNode())
236 # Make sure NODE_NETWORK is not set for test_node, so no block download
237 # will occur outside of direct fetching
238 test_node
= self
.nodes
[0].add_p2p_connection(BaseNode(), services
=NODE_WITNESS
)
240 NetworkThread().start() # Start up network handling in another thread
242 # Test logic begins here
243 inv_node
.wait_for_verack()
244 test_node
.wait_for_verack()
246 # Ensure verack's have been processed by our peer
247 inv_node
.sync_with_ping()
248 test_node
.sync_with_ping()
250 self
.test_null_locators(test_node
, inv_node
)
251 self
.test_nonnull_locators(test_node
, inv_node
)
253 def test_null_locators(self
, test_node
, inv_node
):
254 tip
= self
.nodes
[0].getblockheader(self
.nodes
[0].generate(1)[0])
255 tip_hash
= int(tip
["hash"], 16)
257 inv_node
.check_last_announcement(inv
=[tip_hash
], headers
=[])
258 test_node
.check_last_announcement(inv
=[tip_hash
], headers
=[])
260 self
.log
.info("Verify getheaders with null locator and valid hashstop returns headers.")
261 test_node
.clear_last_announcement()
262 test_node
.send_get_headers(locator
=[], hashstop
=tip_hash
)
263 test_node
.check_last_announcement(headers
=[tip_hash
])
265 self
.log
.info("Verify getheaders with null locator and invalid hashstop does not return headers.")
266 block
= create_block(int(tip
["hash"], 16), create_coinbase(tip
["height"] + 1), tip
["mediantime"] + 1)
268 test_node
.send_header_for_blocks([block
])
269 test_node
.clear_last_announcement()
270 test_node
.send_get_headers(locator
=[], hashstop
=int(block
.hash, 16))
271 test_node
.sync_with_ping()
272 assert_equal(test_node
.block_announced
, False)
273 inv_node
.clear_last_announcement()
274 test_node
.send_message(msg_block(block
))
275 inv_node
.check_last_announcement(inv
=[int(block
.hash, 16)], headers
=[])
277 def test_nonnull_locators(self
, test_node
, inv_node
):
278 tip
= int(self
.nodes
[0].getbestblockhash(), 16)
281 # 1. Mine a block; expect inv announcements each time
282 self
.log
.info("Part 1: headers don't start before sendheaders message...")
285 tip
= self
.mine_blocks(1)
286 inv_node
.check_last_announcement(inv
=[tip
], headers
=[])
287 test_node
.check_last_announcement(inv
=[tip
], headers
=[])
288 # Try a few different responses; none should affect next announcement
290 # first request the block
291 test_node
.send_get_data([tip
])
292 test_node
.wait_for_block(tip
)
294 # next try requesting header and block
295 test_node
.send_get_headers(locator
=[old_tip
], hashstop
=tip
)
296 test_node
.send_get_data([tip
])
297 test_node
.wait_for_block(tip
)
298 test_node
.clear_last_announcement() # since we requested headers...
300 # this time announce own block via headers
301 height
= self
.nodes
[0].getblockcount()
302 last_time
= self
.nodes
[0].getblock(self
.nodes
[0].getbestblockhash())['time']
303 block_time
= last_time
+ 1
304 new_block
= create_block(tip
, create_coinbase(height
+ 1), block_time
)
306 test_node
.send_header_for_blocks([new_block
])
307 test_node
.wait_for_getdata([new_block
.sha256
])
308 test_node
.send_message(msg_block(new_block
))
309 test_node
.sync_with_ping() # make sure this block is processed
310 inv_node
.clear_last_announcement()
311 test_node
.clear_last_announcement()
313 self
.log
.info("Part 1: success!")
314 self
.log
.info("Part 2: announce blocks with headers after sendheaders message...")
316 # 2. Send a sendheaders message and test that headers announcements
317 # commence and keep working.
318 test_node
.send_message(msg_sendheaders())
319 prev_tip
= int(self
.nodes
[0].getbestblockhash(), 16)
320 test_node
.send_get_headers(locator
=[prev_tip
], hashstop
=0)
321 test_node
.sync_with_ping()
323 # Now that we've synced headers, headers announcements should work
324 tip
= self
.mine_blocks(1)
325 inv_node
.check_last_announcement(inv
=[tip
], headers
=[])
326 test_node
.check_last_announcement(headers
=[tip
])
328 height
= self
.nodes
[0].getblockcount() + 1
329 block_time
+= 10 # Advance far enough ahead
331 # Mine i blocks, and alternate announcing either via
332 # inv (of tip) or via headers. After each, new blocks
333 # mined by the node should successfully be announced
334 # with block header, even though the blocks are never requested
337 for b
in range(i
+ 1):
338 blocks
.append(create_block(tip
, create_coinbase(height
), block_time
))
340 tip
= blocks
[-1].sha256
345 test_node
.send_block_inv(tip
)
346 test_node
.wait_for_getheaders()
347 # Should have received a getheaders now
348 test_node
.send_header_for_blocks(blocks
)
349 # Test that duplicate inv's won't result in duplicate
350 # getdata requests, or duplicate headers announcements
351 [inv_node
.send_block_inv(x
.sha256
) for x
in blocks
]
352 test_node
.wait_for_getdata([x
.sha256
for x
in blocks
])
353 inv_node
.sync_with_ping()
355 # Announce via headers
356 test_node
.send_header_for_blocks(blocks
)
357 test_node
.wait_for_getdata([x
.sha256
for x
in blocks
])
358 # Test that duplicate headers won't result in duplicate
359 # getdata requests (the check is further down)
360 inv_node
.send_header_for_blocks(blocks
)
361 inv_node
.sync_with_ping()
362 [test_node
.send_message(msg_block(x
)) for x
in blocks
]
363 test_node
.sync_with_ping()
364 inv_node
.sync_with_ping()
365 # This block should not be announced to the inv node (since it also
367 assert "inv" not in inv_node
.last_message
368 assert "headers" not in inv_node
.last_message
369 tip
= self
.mine_blocks(1)
370 inv_node
.check_last_announcement(inv
=[tip
], headers
=[])
371 test_node
.check_last_announcement(headers
=[tip
])
375 self
.log
.info("Part 2: success!")
377 self
.log
.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...")
379 # PART 3. Headers announcements can stop after large reorg, and resume after
380 # getheaders or inv from peer.
382 # First try mining a reorg that can propagate with header announcement
383 new_block_hashes
= self
.mine_reorg(length
=7)
384 tip
= new_block_hashes
[-1]
385 inv_node
.check_last_announcement(inv
=[tip
], headers
=[])
386 test_node
.check_last_announcement(headers
=new_block_hashes
)
390 # Mine a too-large reorg, which should be announced with a single inv
391 new_block_hashes
= self
.mine_reorg(length
=8)
392 tip
= new_block_hashes
[-1]
393 inv_node
.check_last_announcement(inv
=[tip
], headers
=[])
394 test_node
.check_last_announcement(inv
=[tip
], headers
=[])
398 fork_point
= self
.nodes
[0].getblock("%02x" % new_block_hashes
[0])["previousblockhash"]
399 fork_point
= int(fork_point
, 16)
401 # Use getblocks/getdata
402 test_node
.send_getblocks(locator
=[fork_point
])
403 test_node
.check_last_announcement(inv
=new_block_hashes
, headers
=[])
404 test_node
.send_get_data(new_block_hashes
)
405 test_node
.wait_for_block(new_block_hashes
[-1])
408 # Mine another block, still should get only an inv
409 tip
= self
.mine_blocks(1)
410 inv_node
.check_last_announcement(inv
=[tip
], headers
=[])
411 test_node
.check_last_announcement(inv
=[tip
], headers
=[])
413 # Just get the data -- shouldn't cause headers announcements to resume
414 test_node
.send_get_data([tip
])
415 test_node
.wait_for_block(tip
)
417 # Send a getheaders message that shouldn't trigger headers announcements
418 # to resume (best header sent will be too old)
419 test_node
.send_get_headers(locator
=[fork_point
], hashstop
=new_block_hashes
[1])
420 test_node
.send_get_data([tip
])
421 test_node
.wait_for_block(tip
)
423 test_node
.send_get_data([tip
])
424 test_node
.wait_for_block(tip
)
425 # This time, try sending either a getheaders to trigger resumption
426 # of headers announcements, or mine a new block and inv it, also
427 # triggering resumption of headers announcements.
429 test_node
.send_get_headers(locator
=[tip
], hashstop
=0)
430 test_node
.sync_with_ping()
432 test_node
.send_block_inv(tip
)
433 test_node
.sync_with_ping()
434 # New blocks should now be announced with header
435 tip
= self
.mine_blocks(1)
436 inv_node
.check_last_announcement(inv
=[tip
], headers
=[])
437 test_node
.check_last_announcement(headers
=[tip
])
439 self
.log
.info("Part 3: success!")
441 self
.log
.info("Part 4: Testing direct fetch behavior...")
442 tip
= self
.mine_blocks(1)
443 height
= self
.nodes
[0].getblockcount() + 1
444 last_time
= self
.nodes
[0].getblock(self
.nodes
[0].getbestblockhash())['time']
445 block_time
= last_time
+ 1
447 # Create 2 blocks. Send the blocks, then send the headers.
450 blocks
.append(create_block(tip
, create_coinbase(height
), block_time
))
452 tip
= blocks
[-1].sha256
455 inv_node
.send_message(msg_block(blocks
[-1]))
457 inv_node
.sync_with_ping() # Make sure blocks are processed
458 test_node
.last_message
.pop("getdata", None)
459 test_node
.send_header_for_blocks(blocks
)
460 test_node
.sync_with_ping()
461 # should not have received any getdata messages
463 assert "getdata" not in test_node
.last_message
465 # This time, direct fetch should work
468 blocks
.append(create_block(tip
, create_coinbase(height
), block_time
))
470 tip
= blocks
[-1].sha256
474 test_node
.send_header_for_blocks(blocks
)
475 test_node
.sync_with_ping()
476 test_node
.wait_for_getdata([x
.sha256
for x
in blocks
], timeout
=DIRECT_FETCH_RESPONSE_TIME
)
478 [test_node
.send_message(msg_block(x
)) for x
in blocks
]
480 test_node
.sync_with_ping()
482 # Now announce a header that forks the last two blocks
483 tip
= blocks
[0].sha256
487 # Create extra blocks for later
489 blocks
.append(create_block(tip
, create_coinbase(height
), block_time
))
491 tip
= blocks
[-1].sha256
495 # Announcing one block on fork should not trigger direct fetch
496 # (less work than tip)
497 test_node
.last_message
.pop("getdata", None)
498 test_node
.send_header_for_blocks(blocks
[0:1])
499 test_node
.sync_with_ping()
501 assert "getdata" not in test_node
.last_message
503 # Announcing one more block on fork should trigger direct fetch for
504 # both blocks (same work as tip)
505 test_node
.send_header_for_blocks(blocks
[1:2])
506 test_node
.sync_with_ping()
507 test_node
.wait_for_getdata([x
.sha256
for x
in blocks
[0:2]], timeout
=DIRECT_FETCH_RESPONSE_TIME
)
509 # Announcing 16 more headers should trigger direct fetch for 14 more
511 test_node
.send_header_for_blocks(blocks
[2:18])
512 test_node
.sync_with_ping()
513 test_node
.wait_for_getdata([x
.sha256
for x
in blocks
[2:16]], timeout
=DIRECT_FETCH_RESPONSE_TIME
)
515 # Announcing 1 more header should not trigger any response
516 test_node
.last_message
.pop("getdata", None)
517 test_node
.send_header_for_blocks(blocks
[18:19])
518 test_node
.sync_with_ping()
520 assert "getdata" not in test_node
.last_message
522 self
.log
.info("Part 4: success!")
524 # Now deliver all those blocks we announced.
525 [test_node
.send_message(msg_block(x
)) for x
in blocks
]
527 self
.log
.info("Part 5: Testing handling of unconnecting headers")
528 # First we test that receipt of an unconnecting header doesn't prevent
531 test_node
.last_message
.pop("getdata", None)
533 # Create two more blocks.
535 blocks
.append(create_block(tip
, create_coinbase(height
), block_time
))
537 tip
= blocks
[-1].sha256
540 # Send the header of the second block -> this won't connect.
542 test_node
.last_message
.pop("getheaders", None)
543 test_node
.send_header_for_blocks([blocks
[1]])
544 test_node
.wait_for_getheaders()
545 test_node
.send_header_for_blocks(blocks
)
546 test_node
.wait_for_getdata([x
.sha256
for x
in blocks
])
547 [test_node
.send_message(msg_block(x
)) for x
in blocks
]
548 test_node
.sync_with_ping()
549 assert_equal(int(self
.nodes
[0].getbestblockhash(), 16), blocks
[1].sha256
)
552 # Now we test that if we repeatedly don't send connecting headers, we
553 # don't go into an infinite loop trying to get them to connect.
554 MAX_UNCONNECTING_HEADERS
= 10
555 for j
in range(MAX_UNCONNECTING_HEADERS
+ 1):
556 blocks
.append(create_block(tip
, create_coinbase(height
), block_time
))
558 tip
= blocks
[-1].sha256
562 for i
in range(1, MAX_UNCONNECTING_HEADERS
):
563 # Send a header that doesn't connect, check that we get a getheaders.
565 test_node
.last_message
.pop("getheaders", None)
566 test_node
.send_header_for_blocks([blocks
[i
]])
567 test_node
.wait_for_getheaders()
569 # Next header will connect, should re-set our count:
570 test_node
.send_header_for_blocks([blocks
[0]])
572 # Remove the first two entries (blocks[1] would connect):
575 # Now try to see how many unconnecting headers we can send
576 # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS
577 for i
in range(5 * MAX_UNCONNECTING_HEADERS
- 1):
578 # Send a header that doesn't connect, check that we get a getheaders.
580 test_node
.last_message
.pop("getheaders", None)
581 test_node
.send_header_for_blocks([blocks
[i
% len(blocks
)]])
582 test_node
.wait_for_getheaders()
584 # Eventually this stops working.
585 test_node
.send_header_for_blocks([blocks
[-1]])
587 # Should get disconnected
588 test_node
.wait_for_disconnect()
590 self
.log
.info("Part 5: success!")
592 # Finally, check that the inv node never received a getdata request,
593 # throughout the test
594 assert "getdata" not in inv_node
.last_message
596 if __name__
== '__main__':
597 SendHeadersTest().main()