src/txdb.cpp

   1 // Copyright (c) 2009-2010 Satoshi Nakamoto
   2 // Copyright (c) 2009-2017 The Bitcoin Core developers
   3 // Distributed under the MIT software license, see the accompanying
   4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 #include <txdb.h>
   7
   8 #include <chainparams.h>
   9 #include <hash.h>
  10 #include <random.h>
  11 #include <pow.h>
  12 #include <uint256.h>
  13 #include <util.h>
  14 #include <ui_interface.h>
  15 #include <init.h>
  16
  17 #include <stdint.h>
  18
  19 #include <boost/thread.hpp>
  20
  21 static const char DB_COIN = 'C';
  22 static const char DB_COINS = 'c';
  23 static const char DB_BLOCK_FILES = 'f';
  24 static const char DB_TXINDEX = 't';
  25 static const char DB_BLOCK_INDEX = 'b';
  26
  27 static const char DB_BEST_BLOCK = 'B';
  28 static const char DB_HEAD_BLOCKS = 'H';
  29 static const char DB_FLAG = 'F';
  30 static const char DB_REINDEX_FLAG = 'R';
  31 static const char DB_LAST_BLOCK = 'l';
  32
  33 namespace {
  34
  35 struct CoinEntry {
  36     COutPoint* outpoint;
  37     char key;
  38     explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN)  {}
  39
  40     template<typename Stream>
  41     void Serialize(Stream &s) const {
  42         s << key;
  43         s << outpoint->hash;
  44         s << VARINT(outpoint->n);
  45     }
  46
  47     template<typename Stream>
  48     void Unserialize(Stream& s) {
  49         s >> key;
  50         s >> outpoint->hash;
  51         s >> VARINT(outpoint->n);
  52     }
  53 };
  54
  55 }
  56
  57 CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true)
  58 {
  59 }
  60
  61 bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const {
  62     return db.Read(CoinEntry(&outpoint), coin);
  63 }
  64
  65 bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const {
  66     return db.Exists(CoinEntry(&outpoint));
  67 }
  68
  69 uint256 CCoinsViewDB::GetBestBlock() const {
  70     uint256 hashBestChain;
  71     if (!db.Read(DB_BEST_BLOCK, hashBestChain))
  72         return uint256();
  73     return hashBestChain;
  74 }
  75
  76 std::vector<uint256> CCoinsViewDB::GetHeadBlocks() const {
  77     std::vector<uint256> vhashHeadBlocks;
  78     if (!db.Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) {
  79         return std::vector<uint256>();
  80     }
  81     return vhashHeadBlocks;
  82 }
  83
  84 bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
  85     CDBBatch batch(db);
  86     size_t count = 0;
  87     size_t changed = 0;
  88     size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", nDefaultDbBatchSize);
  89     int crash_simulate = gArgs.GetArg("-dbcrashratio", 0);
  90     assert(!hashBlock.IsNull());
  91
  92     uint256 old_tip = GetBestBlock();
  93     if (old_tip.IsNull()) {
  94         // We may be in the middle of replaying.
  95         std::vector<uint256> old_heads = GetHeadBlocks();
  96         if (old_heads.size() == 2) {
  97             assert(old_heads[0] == hashBlock);
  98             old_tip = old_heads[1];
  99         }
 100     }
 101
 102     // In the first batch, mark the database as being in the middle of a
 103     // transition from old_tip to hashBlock.
 104     // A vector is used for future extensibility, as we may want to support
 105     // interrupting after partial writes from multiple independent reorgs.
 106     batch.Erase(DB_BEST_BLOCK);
 107     batch.Write(DB_HEAD_BLOCKS, std::vector<uint256>{hashBlock, old_tip});
 108
 109     for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
 110         if (it->second.flags & CCoinsCacheEntry::DIRTY) {
 111             CoinEntry entry(&it->first);
 112             if (it->second.coin.IsSpent())
 113                 batch.Erase(entry);
 114             else
 115                 batch.Write(entry, it->second.coin);
 116             changed++;
 117         }
 118         count++;
 119         CCoinsMap::iterator itOld = it++;
 120         mapCoins.erase(itOld);
 121         if (batch.SizeEstimate() > batch_size) {
 122             LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
 123             db.WriteBatch(batch);
 124             batch.Clear();
 125             if (crash_simulate) {
 126                 static FastRandomContext rng;
 127                 if (rng.randrange(crash_simulate) == 0) {
 128                     LogPrintf("Simulating a crash. Goodbye.\n");
 129                     _Exit(0);
 130                 }
 131             }
 132         }
 133     }
 134
 135     // In the last batch, mark the database as consistent with hashBlock again.
 136     batch.Erase(DB_HEAD_BLOCKS);
 137     batch.Write(DB_BEST_BLOCK, hashBlock);
 138
 139     LogPrint(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
 140     bool ret = db.WriteBatch(batch);
 141     LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
 142     return ret;
 143 }
 144
 145 size_t CCoinsViewDB::EstimateSize() const
 146 {
 147     return db.EstimateSize(DB_COIN, (char)(DB_COIN+1));
 148 }
 149
 150 CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
 151 }
 152
 153 bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) {
 154     return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
 155 }
 156
 157 bool CBlockTreeDB::WriteReindexing(bool fReindexing) {
 158     if (fReindexing)
 159         return Write(DB_REINDEX_FLAG, '1');
 160     else
 161         return Erase(DB_REINDEX_FLAG);
 162 }
 163
 164 bool CBlockTreeDB::ReadReindexing(bool &fReindexing) {
 165     fReindexing = Exists(DB_REINDEX_FLAG);
 166     return true;
 167 }
 168
 169 bool CBlockTreeDB::ReadLastBlockFile(int &nFile) {
 170     return Read(DB_LAST_BLOCK, nFile);
 171 }
 172
 173 CCoinsViewCursor *CCoinsViewDB::Cursor() const
 174 {
 175     CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast<CDBWrapper&>(db).NewIterator(), GetBestBlock());
 176     /* It seems that there are no "const iterators" for LevelDB.  Since we
 177        only need read operations on it, use a const-cast to get around
 178        that restriction.  */
 179     i->pcursor->Seek(DB_COIN);
 180     // Cache key of first record
 181     if (i->pcursor->Valid()) {
 182         CoinEntry entry(&i->keyTmp.second);
 183         i->pcursor->GetKey(entry);
 184         i->keyTmp.first = entry.key;
 185     } else {
 186         i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false
 187     }
 188     return i;
 189 }
 190
 191 bool CCoinsViewDBCursor::GetKey(COutPoint &key) const
 192 {
 193     // Return cached key
 194     if (keyTmp.first == DB_COIN) {
 195         key = keyTmp.second;
 196         return true;
 197     }
 198     return false;
 199 }
 200
 201 bool CCoinsViewDBCursor::GetValue(Coin &coin) const
 202 {
 203     return pcursor->GetValue(coin);
 204 }
 205
 206 unsigned int CCoinsViewDBCursor::GetValueSize() const
 207 {
 208     return pcursor->GetValueSize();
 209 }
 210
 211 bool CCoinsViewDBCursor::Valid() const
 212 {
 213     return keyTmp.first == DB_COIN;
 214 }
 215
 216 void CCoinsViewDBCursor::Next()
 217 {
 218     pcursor->Next();
 219     CoinEntry entry(&keyTmp.second);
 220     if (!pcursor->Valid() || !pcursor->GetKey(entry)) {
 221         keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false
 222     } else {
 223         keyTmp.first = entry.key;
 224     }
 225 }
 226
 227 bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
 228     CDBBatch batch(*this);
 229     for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
 230         batch.Write(std::make_pair(DB_BLOCK_FILES, it->first), *it->second);
 231     }
 232     batch.Write(DB_LAST_BLOCK, nLastFile);
 233     for (std::vector<const CBlockIndex*>::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) {
 234         batch.Write(std::make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it));
 235     }
 236     return WriteBatch(batch, true);
 237 }
 238
 239 bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) {
 240     return Read(std::make_pair(DB_TXINDEX, txid), pos);
 241 }
 242
 243 bool CBlockTreeDB::WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> >&vect) {
 244     CDBBatch batch(*this);
 245     for (std::vector<std::pair<uint256,CDiskTxPos> >::const_iterator it=vect.begin(); it!=vect.end(); it++)
 246         batch.Write(std::make_pair(DB_TXINDEX, it->first), it->second);
 247     return WriteBatch(batch);
 248 }
 249
 250 bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) {
 251     return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0');
 252 }
 253
 254 bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
 255     char ch;
 256     if (!Read(std::make_pair(DB_FLAG, name), ch))
 257         return false;
 258     fValue = ch == '1';
 259     return true;
 260 }
 261
 262 bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex)
 263 {
 264     std::unique_ptr<CDBIterator> pcursor(NewIterator());
 265
 266     pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
 267
 268     // Load mapBlockIndex
 269     while (pcursor->Valid()) {
 270         boost::this_thread::interruption_point();
 271         std::pair<char, uint256> key;
 272         if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
 273             CDiskBlockIndex diskindex;
 274             if (pcursor->GetValue(diskindex)) {
 275                 // Construct block index object
 276                 CBlockIndex* pindexNew = insertBlockIndex(diskindex.GetBlockHash());
 277                 pindexNew->pprev          = insertBlockIndex(diskindex.hashPrev);
 278                 pindexNew->nHeight        = diskindex.nHeight;
 279                 pindexNew->nFile          = diskindex.nFile;
 280                 pindexNew->nDataPos       = diskindex.nDataPos;
 281                 pindexNew->nUndoPos       = diskindex.nUndoPos;
 282                 pindexNew->nVersion       = diskindex.nVersion;
 283                 pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
 284                 pindexNew->nTime          = diskindex.nTime;
 285                 pindexNew->nBits          = diskindex.nBits;
 286                 pindexNew->nNonce         = diskindex.nNonce;
 287                 pindexNew->nStatus        = diskindex.nStatus;
 288                 pindexNew->nTx            = diskindex.nTx;
 289
 290                 if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams))
 291                     return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString());
 292
 293                 pcursor->Next();
 294             } else {
 295                 return error("%s: failed to read value", __func__);
 296             }
 297         } else {
 298             break;
 299         }
 300     }
 301
 302     return true;
 303 }
 304
 305 namespace {
 306
 307 //! Legacy class to deserialize pre-pertxout database entries without reindex.
 308 class CCoins
 309 {
 310 public:
 311     //! whether transaction is a coinbase
 312     bool fCoinBase;
 313
 314     //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped
 315     std::vector<CTxOut> vout;
 316
 317     //! at which height this transaction was included in the active block chain
 318     int nHeight;
 319
 320     //! empty constructor
 321     CCoins() : fCoinBase(false), vout(0), nHeight(0) { }
 322
 323     template<typename Stream>
 324     void Unserialize(Stream &s) {
 325         unsigned int nCode = 0;
 326         // version
 327         int nVersionDummy;
 328         ::Unserialize(s, VARINT(nVersionDummy));
 329         // header code
 330         ::Unserialize(s, VARINT(nCode));
 331         fCoinBase = nCode & 1;
 332         std::vector<bool> vAvail(2, false);
 333         vAvail[0] = (nCode & 2) != 0;
 334         vAvail[1] = (nCode & 4) != 0;
 335         unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1);
 336         // spentness bitmask
 337         while (nMaskCode > 0) {
 338             unsigned char chAvail = 0;
 339             ::Unserialize(s, chAvail);
 340             for (unsigned int p = 0; p < 8; p++) {
 341                 bool f = (chAvail & (1 << p)) != 0;
 342                 vAvail.push_back(f);
 343             }
 344             if (chAvail != 0)
 345                 nMaskCode--;
 346         }
 347         // txouts themself
 348         vout.assign(vAvail.size(), CTxOut());
 349         for (unsigned int i = 0; i < vAvail.size(); i++) {
 350             if (vAvail[i])
 351                 ::Unserialize(s, REF(CTxOutCompressor(vout[i])));
 352         }
 353         // coinbase height
 354         ::Unserialize(s, VARINT(nHeight));
 355     }
 356 };
 357
 358 }
 359
 360 /** Upgrade the database from older formats.
 361  *
 362  * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout.
 363  */
 364 bool CCoinsViewDB::Upgrade() {
 365     std::unique_ptr<CDBIterator> pcursor(db.NewIterator());
 366     pcursor->Seek(std::make_pair(DB_COINS, uint256()));
 367     if (!pcursor->Valid()) {
 368         return true;
 369     }
 370
 371     int64_t count = 0;
 372     LogPrintf("Upgrading utxo-set database...\n");
 373     LogPrintf("[0%%]...");
 374     uiInterface.ShowProgress(_("Upgrading UTXO database"), 0, true);
 375     size_t batch_size = 1 << 24;
 376     CDBBatch batch(db);
 377     int reportDone = 0;
 378     std::pair<unsigned char, uint256> key;
 379     std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()};
 380     while (pcursor->Valid()) {
 381         boost::this_thread::interruption_point();
 382         if (ShutdownRequested()) {
 383             break;
 384         }
 385         if (pcursor->GetKey(key) && key.first == DB_COINS) {
 386             if (count++ % 256 == 0) {
 387                 uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1);
 388                 int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5);
 389                 uiInterface.ShowProgress(_("Upgrading UTXO database"), percentageDone, true);
 390                 if (reportDone < percentageDone/10) {
 391                     // report max. every 10% step
 392                     LogPrintf("[%d%%]...", percentageDone);
 393                     reportDone = percentageDone/10;
 394                 }
 395             }
 396             CCoins old_coins;
 397             if (!pcursor->GetValue(old_coins)) {
 398                 return error("%s: cannot parse CCoins record", __func__);
 399             }
 400             COutPoint outpoint(key.second, 0);
 401             for (size_t i = 0; i < old_coins.vout.size(); ++i) {
 402                 if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) {
 403                     Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase);
 404                     outpoint.n = i;
 405                     CoinEntry entry(&outpoint);
 406                     batch.Write(entry, newcoin);
 407                 }
 408             }
 409             batch.Erase(key);
 410             if (batch.SizeEstimate() > batch_size) {
 411                 db.WriteBatch(batch);
 412                 batch.Clear();
 413                 db.CompactRange(prev_key, key);
 414                 prev_key = key;
 415             }
 416             pcursor->Next();
 417         } else {
 418             break;
 419         }
 420     }
 421     db.WriteBatch(batch);
 422     db.CompactRange({DB_COINS, uint256()}, key);
 423     uiInterface.ShowProgress("", 100, false);
 424     LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
 425     return !ShutdownRequested();
 426 }