qa/rpc-tests/pruning.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2014-2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 #
   7 # Test pruning code
   8 # ********
   9 # WARNING:
  10 # This test uses 4GB of disk space.
  11 # This test takes 30 mins or more (up to 2 hours)
  12 # ********
  13
  14 from test_framework.test_framework import BitcoinTestFramework
  15 from test_framework.util import *
  16 import time
  17 import os
  18
  19
  20 def calc_usage(blockdir):
  21     return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(blockdir+f)) / (1024. * 1024.)
  22
  23 class PruneTest(BitcoinTestFramework):
  24
  25     def __init__(self):
  26         super().__init__()
  27         self.setup_clean_chain = True
  28         self.num_nodes = 3
  29
  30         # Cache for utxos, as the listunspent may take a long time later in the test
  31         self.utxo_cache_0 = []
  32         self.utxo_cache_1 = []
  33
  34     def setup_network(self):
  35         self.nodes = []
  36         self.is_network_split = False
  37
  38         # Create nodes 0 and 1 to mine
  39         self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900))
  40         self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900))
  41
  42         # Create node 2 to test pruning
  43         self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=900))
  44         self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/"
  45
  46         connect_nodes(self.nodes[0], 1)
  47         connect_nodes(self.nodes[1], 2)
  48         connect_nodes(self.nodes[2], 0)
  49         sync_blocks(self.nodes[0:3])
  50
  51     def create_big_chain(self):
  52         # Start by creating some coinbases we can spend later
  53         self.nodes[1].generate(200)
  54         sync_blocks(self.nodes[0:2])
  55         self.nodes[0].generate(150)
  56         # Then mine enough full blocks to create more than 550MiB of data
  57         for i in range(645):
  58             mine_large_block(self.nodes[0], self.utxo_cache_0)
  59
  60         sync_blocks(self.nodes[0:3])
  61
  62     def test_height_min(self):
  63         if not os.path.isfile(self.prunedir+"blk00000.dat"):
  64             raise AssertionError("blk00000.dat is missing, pruning too early")
  65         print("Success")
  66         print("Though we're already using more than 550MiB, current usage:", calc_usage(self.prunedir))
  67         print("Mining 25 more blocks should cause the first block file to be pruned")
  68         # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this
  69         for i in range(25):
  70             mine_large_block(self.nodes[0], self.utxo_cache_0)
  71
  72         waitstart = time.time()
  73         while os.path.isfile(self.prunedir+"blk00000.dat"):
  74             time.sleep(0.1)
  75             if time.time() - waitstart > 30:
  76                 raise AssertionError("blk00000.dat not pruned when it should be")
  77
  78         print("Success")
  79         usage = calc_usage(self.prunedir)
  80         print("Usage should be below target:", usage)
  81         if (usage > 550):
  82             raise AssertionError("Pruning target not being met")
  83
  84     def create_chain_with_staleblocks(self):
  85         # Create stale blocks in manageable sized chunks
  86         print("Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds")
  87
  88         for j in range(12):
  89             # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
  90             # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects
  91             # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine
  92             stop_node(self.nodes[0],0)
  93             self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)
  94             # Mine 24 blocks in node 1
  95             for i in range(24):
  96                 if j == 0:
  97                     mine_large_block(self.nodes[1], self.utxo_cache_1)
  98                 else:
  99                     self.nodes[1].generate(1) #tx's already in mempool from previous disconnects
 100
 101             # Reorg back with 25 block chain from node 0
 102             for i in range(25):
 103                 mine_large_block(self.nodes[0], self.utxo_cache_0)
 104
 105             # Create connections in the order so both nodes can see the reorg at the same time
 106             connect_nodes(self.nodes[1], 0)
 107             connect_nodes(self.nodes[2], 0)
 108             sync_blocks(self.nodes[0:3])
 109
 110         print("Usage can be over target because of high stale rate:", calc_usage(self.prunedir))
 111
 112     def reorg_test(self):
 113         # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip
 114         # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain
 115         # Reboot node 1 to clear its mempool (hopefully make the invalidate faster)
 116         # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks)
 117         stop_node(self.nodes[1],1)
 118         self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900)
 119
 120         height = self.nodes[1].getblockcount()
 121         print("Current block height:", height)
 122
 123         invalidheight = height-287
 124         badhash = self.nodes[1].getblockhash(invalidheight)
 125         print("Invalidating block at height:",invalidheight,badhash)
 126         self.nodes[1].invalidateblock(badhash)
 127
 128         # We've now switched to our previously mined-24 block fork on node 1, but thats not what we want
 129         # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago)
 130         mainchainhash = self.nodes[0].getblockhash(invalidheight - 1)
 131         curhash = self.nodes[1].getblockhash(invalidheight - 1)
 132         while curhash != mainchainhash:
 133             self.nodes[1].invalidateblock(curhash)
 134             curhash = self.nodes[1].getblockhash(invalidheight - 1)
 135
 136         assert(self.nodes[1].getblockcount() == invalidheight - 1)
 137         print("New best height", self.nodes[1].getblockcount())
 138
 139         # Reboot node1 to clear those giant tx's from mempool
 140         stop_node(self.nodes[1],1)
 141         self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900)
 142
 143         print("Generating new longer chain of 300 more blocks")
 144         self.nodes[1].generate(300)
 145
 146         print("Reconnect nodes")
 147         connect_nodes(self.nodes[0], 1)
 148         connect_nodes(self.nodes[2], 1)
 149         sync_blocks(self.nodes[0:3], timeout=120)
 150
 151         print("Verify height on node 2:",self.nodes[2].getblockcount())
 152         print("Usage possibly still high bc of stale blocks in block files:", calc_usage(self.prunedir))
 153
 154         print("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)")
 155         for i in range(22):
 156             # This can be slow, so do this in multiple RPC calls to avoid
 157             # RPC timeouts.
 158             self.nodes[0].generate(10) #node 0 has many large tx's in its mempool from the disconnects
 159         sync_blocks(self.nodes[0:3], timeout=300)
 160
 161         usage = calc_usage(self.prunedir)
 162         print("Usage should be below target:", usage)
 163         if (usage > 550):
 164             raise AssertionError("Pruning target not being met")
 165
 166         return invalidheight,badhash
 167
 168     def reorg_back(self):
 169         # Verify that a block on the old main chain fork has been pruned away
 170         try:
 171             self.nodes[2].getblock(self.forkhash)
 172             raise AssertionError("Old block wasn't pruned so can't test redownload")
 173         except JSONRPCException as e:
 174             print("Will need to redownload block",self.forkheight)
 175
 176         # Verify that we have enough history to reorg back to the fork point
 177         # Although this is more than 288 blocks, because this chain was written more recently
 178         # and only its other 299 small and 220 large block are in the block files after it,
 179         # its expected to still be retained
 180         self.nodes[2].getblock(self.nodes[2].getblockhash(self.forkheight))
 181
 182         first_reorg_height = self.nodes[2].getblockcount()
 183         curchainhash = self.nodes[2].getblockhash(self.mainchainheight)
 184         self.nodes[2].invalidateblock(curchainhash)
 185         goalbestheight = self.mainchainheight
 186         goalbesthash = self.mainchainhash2
 187
 188         # As of 0.10 the current block download logic is not able to reorg to the original chain created in
 189         # create_chain_with_stale_blocks because it doesn't know of any peer thats on that chain from which to
 190         # redownload its missing blocks.
 191         # Invalidate the reorg_test chain in node 0 as well, it can successfully switch to the original chain
 192         # because it has all the block data.
 193         # However it must mine enough blocks to have a more work chain than the reorg_test chain in order
 194         # to trigger node 2's block download logic.
 195         # At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg
 196         if self.nodes[2].getblockcount() < self.mainchainheight:
 197             blocks_to_mine = first_reorg_height + 1 - self.mainchainheight
 198             print("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed:", blocks_to_mine)
 199             self.nodes[0].invalidateblock(curchainhash)
 200             assert(self.nodes[0].getblockcount() == self.mainchainheight)
 201             assert(self.nodes[0].getbestblockhash() == self.mainchainhash2)
 202             goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1]
 203             goalbestheight = first_reorg_height + 1
 204
 205         print("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload")
 206         waitstart = time.time()
 207         while self.nodes[2].getblockcount() < goalbestheight:
 208             time.sleep(0.1)
 209             if time.time() - waitstart > 900:
 210                 raise AssertionError("Node 2 didn't reorg to proper height")
 211         assert(self.nodes[2].getbestblockhash() == goalbesthash)
 212         # Verify we can now have the data for a block previously pruned
 213         assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight)
 214
 215
 216     def run_test(self):
 217         print("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)")
 218         print("Mining a big blockchain of 995 blocks")
 219         self.create_big_chain()
 220         # Chain diagram key:
 221         # *   blocks on main chain
 222         # +,&,$,@ blocks on other forks
 223         # X   invalidated block
 224         # N1  Node 1
 225         #
 226         # Start by mining a simple chain that all nodes have
 227         # N0=N1=N2 **...*(995)
 228
 229         print("Check that we haven't started pruning yet because we're below PruneAfterHeight")
 230         self.test_height_min()
 231         # Extend this chain past the PruneAfterHeight
 232         # N0=N1=N2 **...*(1020)
 233
 234         print("Check that we'll exceed disk space target if we have a very high stale block rate")
 235         self.create_chain_with_staleblocks()
 236         # Disconnect N0
 237         # And mine a 24 block chain on N1 and a separate 25 block chain on N0
 238         # N1=N2 **...*+...+(1044)
 239         # N0    **...**...**(1045)
 240         #
 241         # reconnect nodes causing reorg on N1 and N2
 242         # N1=N2 **...*(1020) *...**(1045)
 243         #                   \
 244         #                    +...+(1044)
 245         #
 246         # repeat this process until you have 12 stale forks hanging off the
 247         # main chain on N1 and N2
 248         # N0    *************************...***************************(1320)
 249         #
 250         # N1=N2 **...*(1020) *...**(1045) *..         ..**(1295) *...**(1320)
 251         #                   \            \                      \
 252         #                    +...+(1044)  &..                    $...$(1319)
 253
 254         # Save some current chain state for later use
 255         self.mainchainheight = self.nodes[2].getblockcount()   #1320
 256         self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight)
 257
 258         print("Check that we can survive a 288 block reorg still")
 259         (self.forkheight,self.forkhash) = self.reorg_test() #(1033, )
 260         # Now create a 288 block reorg by mining a longer chain on N1
 261         # First disconnect N1
 262         # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain
 263         # N1   **...*(1020) **...**(1032)X..
 264         #                  \
 265         #                   ++...+(1031)X..
 266         #
 267         # Now mine 300 more blocks on N1
 268         # N1    **...*(1020) **...**(1032) @@...@(1332)
 269         #                 \               \
 270         #                  \               X...
 271         #                   \                 \
 272         #                    ++...+(1031)X..   ..
 273         #
 274         # Reconnect nodes and mine 220 more blocks on N1
 275         # N1    **...*(1020) **...**(1032) @@...@@@(1552)
 276         #                 \               \
 277         #                  \               X...
 278         #                   \                 \
 279         #                    ++...+(1031)X..   ..
 280         #
 281         # N2    **...*(1020) **...**(1032) @@...@@@(1552)
 282         #                 \               \
 283         #                  \               *...**(1320)
 284         #                   \                 \
 285         #                    ++...++(1044)     ..
 286         #
 287         # N0    ********************(1032) @@...@@@(1552)
 288         #                                 \
 289         #                                  *...**(1320)
 290
 291         print("Test that we can rerequest a block we previously pruned if needed for a reorg")
 292         self.reorg_back()
 293         # Verify that N2 still has block 1033 on current chain (@), but not on main chain (*)
 294         # Invalidate 1033 on current chain (@) on N2 and we should be able to reorg to
 295         # original main chain (*), but will require redownload of some blocks
 296         # In order to have a peer we think we can download from, must also perform this invalidation
 297         # on N0 and mine a new longest chain to trigger.
 298         # Final result:
 299         # N0    ********************(1032) **...****(1553)
 300         #                                 \
 301         #                                  X@...@@@(1552)
 302         #
 303         # N2    **...*(1020) **...**(1032) **...****(1553)
 304         #                 \               \
 305         #                  \               X@...@@@(1552)
 306         #                   \
 307         #                    +..
 308         #
 309         # N1 doesn't change because 1033 on main chain (*) is invalid
 310
 311         print("Done")
 312
 313 if __name__ == '__main__':
 314     PruneTest().main()