test/functional/smartfees.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2014-2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 """Test fee estimation code."""
   6
   7 from test_framework.test_framework import BitcoinTestFramework
   8 from test_framework.util import *
   9 from test_framework.script import CScript, OP_1, OP_DROP, OP_2, OP_HASH160, OP_EQUAL, hash160, OP_TRUE
  10 from test_framework.mininode import CTransaction, CTxIn, CTxOut, COutPoint, ToHex, COIN
  11
  12 # Construct 2 trivial P2SH's and the ScriptSigs that spend them
  13 # So we can create many transactions without needing to spend
  14 # time signing.
  15 redeem_script_1 = CScript([OP_1, OP_DROP])
  16 redeem_script_2 = CScript([OP_2, OP_DROP])
  17 P2SH_1 = CScript([OP_HASH160, hash160(redeem_script_1), OP_EQUAL])
  18 P2SH_2 = CScript([OP_HASH160, hash160(redeem_script_2), OP_EQUAL])
  19
  20 # Associated ScriptSig's to spend satisfy P2SH_1 and P2SH_2
  21 SCRIPT_SIG = [CScript([OP_TRUE, redeem_script_1]), CScript([OP_TRUE, redeem_script_2])]
  22
  23 global log
  24
  25 def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee_increment):
  26     """
  27     Create and send a transaction with a random fee.
  28     The transaction pays to a trivial P2SH script, and assumes that its inputs
  29     are of the same form.
  30     The function takes a list of confirmed outputs and unconfirmed outputs
  31     and attempts to use the confirmed list first for its inputs.
  32     It adds the newly created outputs to the unconfirmed list.
  33     Returns (raw transaction, fee)
  34     """
  35     # It's best to exponentially distribute our random fees
  36     # because the buckets are exponentially spaced.
  37     # Exponentially distributed from 1-128 * fee_increment
  38     rand_fee = float(fee_increment)*(1.1892**random.randint(0,28))
  39     # Total fee ranges from min_fee to min_fee + 127*fee_increment
  40     fee = min_fee - fee_increment + satoshi_round(rand_fee)
  41     tx = CTransaction()
  42     total_in = Decimal("0.00000000")
  43     while total_in <= (amount + fee) and len(conflist) > 0:
  44         t = conflist.pop(0)
  45         total_in += t["amount"]
  46         tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b""))
  47     if total_in <= amount + fee:
  48         while total_in <= (amount + fee) and len(unconflist) > 0:
  49             t = unconflist.pop(0)
  50             total_in += t["amount"]
  51             tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b""))
  52         if total_in <= amount + fee:
  53             raise RuntimeError("Insufficient funds: need %d, have %d"%(amount+fee, total_in))
  54     tx.vout.append(CTxOut(int((total_in - amount - fee)*COIN), P2SH_1))
  55     tx.vout.append(CTxOut(int(amount*COIN), P2SH_2))
  56     # These transactions don't need to be signed, but we still have to insert
  57     # the ScriptSig that will satisfy the ScriptPubKey.
  58     for inp in tx.vin:
  59         inp.scriptSig = SCRIPT_SIG[inp.prevout.n]
  60     txid = from_node.sendrawtransaction(ToHex(tx), True)
  61     unconflist.append({ "txid" : txid, "vout" : 0 , "amount" : total_in - amount - fee})
  62     unconflist.append({ "txid" : txid, "vout" : 1 , "amount" : amount})
  63
  64     return (ToHex(tx), fee)
  65
  66 def split_inputs(from_node, txins, txouts, initial_split = False):
  67     """
  68     We need to generate a lot of inputs so we can generate a ton of transactions.
  69     This function takes an input from txins, and creates and sends a transaction
  70     which splits the value into 2 outputs which are appended to txouts.
  71     Previously this was designed to be small inputs so they wouldn't have
  72     a high coin age when the notion of priority still existed.
  73     """
  74     prevtxout = txins.pop()
  75     tx = CTransaction()
  76     tx.vin.append(CTxIn(COutPoint(int(prevtxout["txid"], 16), prevtxout["vout"]), b""))
  77
  78     half_change = satoshi_round(prevtxout["amount"]/2)
  79     rem_change = prevtxout["amount"] - half_change  - Decimal("0.00001000")
  80     tx.vout.append(CTxOut(int(half_change*COIN), P2SH_1))
  81     tx.vout.append(CTxOut(int(rem_change*COIN), P2SH_2))
  82
  83     # If this is the initial split we actually need to sign the transaction
  84     # Otherwise we just need to insert the proper ScriptSig
  85     if (initial_split) :
  86         completetx = from_node.signrawtransaction(ToHex(tx))["hex"]
  87     else :
  88         tx.vin[0].scriptSig = SCRIPT_SIG[prevtxout["vout"]]
  89         completetx = ToHex(tx)
  90     txid = from_node.sendrawtransaction(completetx, True)
  91     txouts.append({ "txid" : txid, "vout" : 0 , "amount" : half_change})
  92     txouts.append({ "txid" : txid, "vout" : 1 , "amount" : rem_change})
  93
  94 def check_estimates(node, fees_seen, max_invalid, print_estimates = True):
  95     """
  96     This function calls estimatefee and verifies that the estimates
  97     meet certain invariants.
  98     """
  99     all_estimates = [ node.estimatefee(i) for i in range(1,26) ]
 100     if print_estimates:
 101         log.info([str(all_estimates[e-1]) for e in [1,2,3,6,15,25]])
 102     delta = 1.0e-6 # account for rounding error
 103     last_e = max(fees_seen)
 104     for e in [x for x in all_estimates if x >= 0]:
 105         # Estimates should be within the bounds of what transactions fees actually were:
 106         if float(e)+delta < min(fees_seen) or float(e)-delta > max(fees_seen):
 107             raise AssertionError("Estimated fee (%f) out of range (%f,%f)"
 108                                  %(float(e), min(fees_seen), max(fees_seen)))
 109         # Estimates should be monotonically decreasing
 110         if float(e)-delta > last_e:
 111             raise AssertionError("Estimated fee (%f) larger than last fee (%f) for lower number of confirms"
 112                                  %(float(e),float(last_e)))
 113         last_e = e
 114     valid_estimate = False
 115     invalid_estimates = 0
 116     for i,e in enumerate(all_estimates): # estimate is for i+1
 117         if e >= 0:
 118             valid_estimate = True
 119             if i >= 13:  # for n>=14 estimatesmartfee(n/2) should be at least as high as estimatefee(n)
 120                 assert(node.estimatesmartfee((i+1)//2)["feerate"] > float(e) - delta)
 121
 122         else:
 123             invalid_estimates += 1
 124
 125             # estimatesmartfee should still be valid
 126             approx_estimate = node.estimatesmartfee(i+1)["feerate"]
 127             answer_found = node.estimatesmartfee(i+1)["blocks"]
 128             assert(approx_estimate > 0)
 129             assert(answer_found > i+1)
 130
 131             # Once we're at a high enough confirmation count that we can give an estimate
 132             # We should have estimates for all higher confirmation counts
 133             if valid_estimate:
 134                 raise AssertionError("Invalid estimate appears at higher confirm count than valid estimate")
 135
 136     # Check on the expected number of different confirmation counts
 137     # that we might not have valid estimates for
 138     if invalid_estimates > max_invalid:
 139         raise AssertionError("More than (%d) invalid estimates"%(max_invalid))
 140     return all_estimates
 141
 142
 143 class EstimateFeeTest(BitcoinTestFramework):
 144
 145     def __init__(self):
 146         super().__init__()
 147         self.num_nodes = 3
 148         self.setup_clean_chain = False
 149
 150     def setup_network(self):
 151         """
 152         We'll setup the network to have 3 nodes that all mine with different parameters.
 153         But first we need to use one node to create a lot of outputs
 154         which we will use to generate our transactions.
 155         """
 156         self.nodes = []
 157         # Use node0 to mine blocks for input splitting
 158         self.nodes.append(self.start_node(0, self.options.tmpdir, ["-maxorphantx=1000",
 159                                                               "-whitelist=127.0.0.1"]))
 160
 161         self.log.info("This test is time consuming, please be patient")
 162         self.log.info("Splitting inputs so we can generate tx's")
 163         self.txouts = []
 164         self.txouts2 = []
 165         # Split a coinbase into two transaction puzzle outputs
 166         split_inputs(self.nodes[0], self.nodes[0].listunspent(0), self.txouts, True)
 167
 168         # Mine
 169         while (len(self.nodes[0].getrawmempool()) > 0):
 170             self.nodes[0].generate(1)
 171
 172         # Repeatedly split those 2 outputs, doubling twice for each rep
 173         # Use txouts to monitor the available utxo, since these won't be tracked in wallet
 174         reps = 0
 175         while (reps < 5):
 176             #Double txouts to txouts2
 177             while (len(self.txouts)>0):
 178                 split_inputs(self.nodes[0], self.txouts, self.txouts2)
 179             while (len(self.nodes[0].getrawmempool()) > 0):
 180                 self.nodes[0].generate(1)
 181             #Double txouts2 to txouts
 182             while (len(self.txouts2)>0):
 183                 split_inputs(self.nodes[0], self.txouts2, self.txouts)
 184             while (len(self.nodes[0].getrawmempool()) > 0):
 185                 self.nodes[0].generate(1)
 186             reps += 1
 187         self.log.info("Finished splitting")
 188
 189         # Now we can connect the other nodes, didn't want to connect them earlier
 190         # so the estimates would not be affected by the splitting transactions
 191         # Node1 mines small blocks but that are bigger than the expected transaction rate.
 192         # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes,
 193         # (17k is room enough for 110 or so transactions)
 194         self.nodes.append(self.start_node(1, self.options.tmpdir,
 195                                      ["-blockmaxsize=17000", "-maxorphantx=1000"]))
 196         connect_nodes(self.nodes[1], 0)
 197
 198         # Node2 is a stingy miner, that
 199         # produces too small blocks (room for only 55 or so transactions)
 200         node2args = ["-blockmaxsize=8000", "-maxorphantx=1000"]
 201
 202         self.nodes.append(self.start_node(2, self.options.tmpdir, node2args))
 203         connect_nodes(self.nodes[0], 2)
 204         connect_nodes(self.nodes[2], 1)
 205
 206         self.sync_all()
 207
 208     def transact_and_mine(self, numblocks, mining_node):
 209         min_fee = Decimal("0.00001")
 210         # We will now mine numblocks blocks generating on average 100 transactions between each block
 211         # We shuffle our confirmed txout set before each set of transactions
 212         # small_txpuzzle_randfee will use the transactions that have inputs already in the chain when possible
 213         # resorting to tx's that depend on the mempool when those run out
 214         for i in range(numblocks):
 215             random.shuffle(self.confutxo)
 216             for j in range(random.randrange(100-50,100+50)):
 217                 from_index = random.randint(1,2)
 218                 (txhex, fee) = small_txpuzzle_randfee(self.nodes[from_index], self.confutxo,
 219                                                       self.memutxo, Decimal("0.005"), min_fee, min_fee)
 220                 tx_kbytes = (len(txhex) // 2) / 1000.0
 221                 self.fees_per_kb.append(float(fee)/tx_kbytes)
 222             sync_mempools(self.nodes[0:3], wait=.1)
 223             mined = mining_node.getblock(mining_node.generate(1)[0],True)["tx"]
 224             sync_blocks(self.nodes[0:3], wait=.1)
 225             # update which txouts are confirmed
 226             newmem = []
 227             for utx in self.memutxo:
 228                 if utx["txid"] in mined:
 229                     self.confutxo.append(utx)
 230                 else:
 231                     newmem.append(utx)
 232             self.memutxo = newmem
 233
 234     def run_test(self):
 235         # Make log handler available to helper functions
 236         global log
 237         log = self.log
 238         self.fees_per_kb = []
 239         self.memutxo = []
 240         self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting
 241         self.log.info("Will output estimates for 1/2/3/6/15/25 blocks")
 242
 243         for i in range(2):
 244             self.log.info("Creating transactions and mining them with a block size that can't keep up")
 245             # Create transactions and mine 10 small blocks with node 2, but create txs faster than we can mine
 246             self.transact_and_mine(10, self.nodes[2])
 247             check_estimates(self.nodes[1], self.fees_per_kb, 14)
 248
 249             self.log.info("Creating transactions and mining them at a block size that is just big enough")
 250             # Generate transactions while mining 10 more blocks, this time with node1
 251             # which mines blocks with capacity just above the rate that transactions are being created
 252             self.transact_and_mine(10, self.nodes[1])
 253             check_estimates(self.nodes[1], self.fees_per_kb, 2)
 254
 255         # Finish by mining a normal-sized block:
 256         while len(self.nodes[1].getrawmempool()) > 0:
 257             self.nodes[1].generate(1)
 258
 259         sync_blocks(self.nodes[0:3], wait=.1)
 260         self.log.info("Final estimates after emptying mempools")
 261         check_estimates(self.nodes[1], self.fees_per_kb, 2)
 262
 263 if __name__ == '__main__':
 264     EstimateFeeTest().main()