src/wallet/db.cpp

   1 // Copyright (c) 2009-2010 Satoshi Nakamoto
   2 // Copyright (c) 2009-2014 The Bitcoin Core developers
   3 // Distributed under the MIT software license, see the accompanying
   4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 #include "db.h"
   7
   8 #include "addrman.h"
   9 #include "hash.h"
  10 #include "protocol.h"
  11 #include "util.h"
  12 #include "utilstrencodings.h"
  13
  14 #include <stdint.h>
  15
  16 #ifndef WIN32
  17 #include <sys/stat.h>
  18 #endif
  19
  20 #include <boost/filesystem.hpp>
  21 #include <boost/thread.hpp>
  22 #include <boost/version.hpp>
  23
  24 using namespace std;
  25
  26
  27 unsigned int nWalletDBUpdated;
  28
  29
  30 //
  31 // CDB
  32 //
  33
  34 CDBEnv bitdb;
  35
  36 void CDBEnv::EnvShutdown()
  37 {
  38     if (!fDbEnvInit)
  39         return;
  40
  41     fDbEnvInit = false;
  42     int ret = dbenv->close(0);
  43     if (ret != 0)
  44         LogPrintf("CDBEnv::EnvShutdown: Error %d shutting down database environment: %s\n", ret, DbEnv::strerror(ret));
  45     if (!fMockDb)
  46         DbEnv(0).remove(strPath.c_str(), 0);
  47 }
  48
  49 void CDBEnv::Reset()
  50 {
  51     delete dbenv;
  52     dbenv = new DbEnv(DB_CXX_NO_EXCEPTIONS);
  53     fDbEnvInit = false;
  54     fMockDb = false;
  55 }
  56
  57 CDBEnv::CDBEnv() : dbenv(NULL)
  58 {
  59     Reset();
  60 }
  61
  62 CDBEnv::~CDBEnv()
  63 {
  64     EnvShutdown();
  65     delete dbenv;
  66     dbenv = NULL;
  67 }
  68
  69 void CDBEnv::Close()
  70 {
  71     EnvShutdown();
  72 }
  73
  74 bool CDBEnv::Open(const boost::filesystem::path& pathIn)
  75 {
  76     if (fDbEnvInit)
  77         return true;
  78
  79     boost::this_thread::interruption_point();
  80
  81     strPath = pathIn.string();
  82     boost::filesystem::path pathLogDir = pathIn / "database";
  83     TryCreateDirectory(pathLogDir);
  84     boost::filesystem::path pathErrorFile = pathIn / "db.log";
  85     LogPrintf("CDBEnv::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string());
  86
  87     unsigned int nEnvFlags = 0;
  88     if (GetBoolArg("-privdb", true))
  89         nEnvFlags |= DB_PRIVATE;
  90
  91     dbenv->set_lg_dir(pathLogDir.string().c_str());
  92     dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
  93     dbenv->set_lg_bsize(0x10000);
  94     dbenv->set_lg_max(1048576);
  95     dbenv->set_lk_max_locks(40000);
  96     dbenv->set_lk_max_objects(40000);
  97     dbenv->set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug
  98     dbenv->set_flags(DB_AUTO_COMMIT, 1);
  99     dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
 100     dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
 101     int ret = dbenv->open(strPath.c_str(),
 102                          DB_CREATE |
 103                              DB_INIT_LOCK |
 104                              DB_INIT_LOG |
 105                              DB_INIT_MPOOL |
 106                              DB_INIT_TXN |
 107                              DB_THREAD |
 108                              DB_RECOVER |
 109                              nEnvFlags,
 110                          S_IRUSR | S_IWUSR);
 111     if (ret != 0)
 112         return error("CDBEnv::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret));
 113
 114     fDbEnvInit = true;
 115     fMockDb = false;
 116     return true;
 117 }
 118
 119 void CDBEnv::MakeMock()
 120 {
 121     if (fDbEnvInit)
 122         throw runtime_error("CDBEnv::MakeMock: Already initialized");
 123
 124     boost::this_thread::interruption_point();
 125
 126     LogPrint("db", "CDBEnv::MakeMock\n");
 127
 128     dbenv->set_cachesize(1, 0, 1);
 129     dbenv->set_lg_bsize(10485760 * 4);
 130     dbenv->set_lg_max(10485760);
 131     dbenv->set_lk_max_locks(10000);
 132     dbenv->set_lk_max_objects(10000);
 133     dbenv->set_flags(DB_AUTO_COMMIT, 1);
 134     dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
 135     int ret = dbenv->open(NULL,
 136                          DB_CREATE |
 137                              DB_INIT_LOCK |
 138                              DB_INIT_LOG |
 139                              DB_INIT_MPOOL |
 140                              DB_INIT_TXN |
 141                              DB_THREAD |
 142                              DB_PRIVATE,
 143                          S_IRUSR | S_IWUSR);
 144     if (ret > 0)
 145         throw runtime_error(strprintf("CDBEnv::MakeMock: Error %d opening database environment.", ret));
 146
 147     fDbEnvInit = true;
 148     fMockDb = true;
 149 }
 150
 151 CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFunc)(CDBEnv& dbenv, const std::string& strFile))
 152 {
 153     LOCK(cs_db);
 154     assert(mapFileUseCount.count(strFile) == 0);
 155
 156     Db db(dbenv, 0);
 157     int result = db.verify(strFile.c_str(), NULL, NULL, 0);
 158     if (result == 0)
 159         return VERIFY_OK;
 160     else if (recoverFunc == NULL)
 161         return RECOVER_FAIL;
 162
 163     // Try to recover:
 164     bool fRecovered = (*recoverFunc)(*this, strFile);
 165     return (fRecovered ? RECOVER_OK : RECOVER_FAIL);
 166 }
 167
 168 bool CDBEnv::Salvage(const std::string& strFile, bool fAggressive, std::vector<CDBEnv::KeyValPair>& vResult)
 169 {
 170     LOCK(cs_db);
 171     assert(mapFileUseCount.count(strFile) == 0);
 172
 173     u_int32_t flags = DB_SALVAGE;
 174     if (fAggressive)
 175         flags |= DB_AGGRESSIVE;
 176
 177     stringstream strDump;
 178
 179     Db db(dbenv, 0);
 180     int result = db.verify(strFile.c_str(), NULL, &strDump, flags);
 181     if (result == DB_VERIFY_BAD) {
 182         LogPrintf("CDBEnv::Salvage: Database salvage found errors, all data may not be recoverable.\n");
 183         if (!fAggressive) {
 184             LogPrintf("CDBEnv::Salvage: Rerun with aggressive mode to ignore errors and continue.\n");
 185             return false;
 186         }
 187     }
 188     if (result != 0 && result != DB_VERIFY_BAD) {
 189         LogPrintf("CDBEnv::Salvage: Database salvage failed with result %d.\n", result);
 190         return false;
 191     }
 192
 193     // Format of bdb dump is ascii lines:
 194     // header lines...
 195     // HEADER=END
 196     // hexadecimal key
 197     // hexadecimal value
 198     // ... repeated
 199     // DATA=END
 200
 201     string strLine;
 202     while (!strDump.eof() && strLine != "HEADER=END")
 203         getline(strDump, strLine); // Skip past header
 204
 205     std::string keyHex, valueHex;
 206     while (!strDump.eof() && keyHex != "DATA=END") {
 207         getline(strDump, keyHex);
 208         if (keyHex != "DATA_END") {
 209             getline(strDump, valueHex);
 210             vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex)));
 211         }
 212     }
 213
 214     return (result == 0);
 215 }
 216
 217
 218 void CDBEnv::CheckpointLSN(const std::string& strFile)
 219 {
 220     dbenv->txn_checkpoint(0, 0, 0);
 221     if (fMockDb)
 222         return;
 223     dbenv->lsn_reset(strFile.c_str(), 0);
 224 }
 225
 226
 227 CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL)
 228 {
 229     int ret;
 230     fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
 231     fFlushOnClose = fFlushOnCloseIn;
 232     if (strFilename.empty())
 233         return;
 234
 235     bool fCreate = strchr(pszMode, 'c') != NULL;
 236     unsigned int nFlags = DB_THREAD;
 237     if (fCreate)
 238         nFlags |= DB_CREATE;
 239
 240     {
 241         LOCK(bitdb.cs_db);
 242         if (!bitdb.Open(GetDataDir()))
 243             throw runtime_error("CDB: Failed to open database environment.");
 244
 245         strFile = strFilename;
 246         ++bitdb.mapFileUseCount[strFile];
 247         pdb = bitdb.mapDb[strFile];
 248         if (pdb == NULL) {
 249             pdb = new Db(bitdb.dbenv, 0);
 250
 251             bool fMockDb = bitdb.IsMock();
 252             if (fMockDb) {
 253                 DbMpoolFile* mpf = pdb->get_mpf();
 254                 ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
 255                 if (ret != 0)
 256                     throw runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile));
 257             }
 258
 259             ret = pdb->open(NULL,                               // Txn pointer
 260                             fMockDb ? NULL : strFile.c_str(),   // Filename
 261                             fMockDb ? strFile.c_str() : "main", // Logical db name
 262                             DB_BTREE,                           // Database type
 263                             nFlags,                             // Flags
 264                             0);
 265
 266             if (ret != 0) {
 267                 delete pdb;
 268                 pdb = NULL;
 269                 --bitdb.mapFileUseCount[strFile];
 270                 strFile = "";
 271                 throw runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFile));
 272             }
 273
 274             if (fCreate && !Exists(string("version"))) {
 275                 bool fTmp = fReadOnly;
 276                 fReadOnly = false;
 277                 WriteVersion(CLIENT_VERSION);
 278                 fReadOnly = fTmp;
 279             }
 280
 281             bitdb.mapDb[strFile] = pdb;
 282         }
 283     }
 284 }
 285
 286 void CDB::Flush()
 287 {
 288     if (activeTxn)
 289         return;
 290
 291     // Flush database activity from memory pool to disk log
 292     unsigned int nMinutes = 0;
 293     if (fReadOnly)
 294         nMinutes = 1;
 295
 296     bitdb.dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
 297 }
 298
 299 void CDB::Close()
 300 {
 301     if (!pdb)
 302         return;
 303     if (activeTxn)
 304         activeTxn->abort();
 305     activeTxn = NULL;
 306     pdb = NULL;
 307
 308     if (fFlushOnClose)
 309         Flush();
 310
 311     {
 312         LOCK(bitdb.cs_db);
 313         --bitdb.mapFileUseCount[strFile];
 314     }
 315 }
 316
 317 void CDBEnv::CloseDb(const string& strFile)
 318 {
 319     {
 320         LOCK(cs_db);
 321         if (mapDb[strFile] != NULL) {
 322             // Close the database handle
 323             Db* pdb = mapDb[strFile];
 324             pdb->close(0);
 325             delete pdb;
 326             mapDb[strFile] = NULL;
 327         }
 328     }
 329 }
 330
 331 bool CDBEnv::RemoveDb(const string& strFile)
 332 {
 333     this->CloseDb(strFile);
 334
 335     LOCK(cs_db);
 336     int rc = dbenv->dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT);
 337     return (rc == 0);
 338 }
 339
 340 bool CDB::Rewrite(const string& strFile, const char* pszSkip)
 341 {
 342     while (true) {
 343         {
 344             LOCK(bitdb.cs_db);
 345             if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0) {
 346                 // Flush log data to the dat file
 347                 bitdb.CloseDb(strFile);
 348                 bitdb.CheckpointLSN(strFile);
 349                 bitdb.mapFileUseCount.erase(strFile);
 350
 351                 bool fSuccess = true;
 352                 LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile);
 353                 string strFileRes = strFile + ".rewrite";
 354                 { // surround usage of db with extra {}
 355                     CDB db(strFile.c_str(), "r");
 356                     Db* pdbCopy = new Db(bitdb.dbenv, 0);
 357
 358                     int ret = pdbCopy->open(NULL,               // Txn pointer
 359                                             strFileRes.c_str(), // Filename
 360                                             "main",             // Logical db name
 361                                             DB_BTREE,           // Database type
 362                                             DB_CREATE,          // Flags
 363                                             0);
 364                     if (ret > 0) {
 365                         LogPrintf("CDB::Rewrite: Can't create database file %s\n", strFileRes);
 366                         fSuccess = false;
 367                     }
 368
 369                     Dbc* pcursor = db.GetCursor();
 370                     if (pcursor)
 371                         while (fSuccess) {
 372                             CDataStream ssKey(SER_DISK, CLIENT_VERSION);
 373                             CDataStream ssValue(SER_DISK, CLIENT_VERSION);
 374                             int ret = db.ReadAtCursor(pcursor, ssKey, ssValue, DB_NEXT);
 375                             if (ret == DB_NOTFOUND) {
 376                                 pcursor->close();
 377                                 break;
 378                             } else if (ret != 0) {
 379                                 pcursor->close();
 380                                 fSuccess = false;
 381                                 break;
 382                             }
 383                             if (pszSkip &&
 384                                 strncmp(&ssKey[0], pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
 385                                 continue;
 386                             if (strncmp(&ssKey[0], "\x07version", 8) == 0) {
 387                                 // Update version:
 388                                 ssValue.clear();
 389                                 ssValue << CLIENT_VERSION;
 390                             }
 391                             Dbt datKey(&ssKey[0], ssKey.size());
 392                             Dbt datValue(&ssValue[0], ssValue.size());
 393                             int ret2 = pdbCopy->put(NULL, &datKey, &datValue, DB_NOOVERWRITE);
 394                             if (ret2 > 0)
 395                                 fSuccess = false;
 396                         }
 397                     if (fSuccess) {
 398                         db.Close();
 399                         bitdb.CloseDb(strFile);
 400                         if (pdbCopy->close(0))
 401                             fSuccess = false;
 402                         delete pdbCopy;
 403                     }
 404                 }
 405                 if (fSuccess) {
 406                     Db dbA(bitdb.dbenv, 0);
 407                     if (dbA.remove(strFile.c_str(), NULL, 0))
 408                         fSuccess = false;
 409                     Db dbB(bitdb.dbenv, 0);
 410                     if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0))
 411                         fSuccess = false;
 412                 }
 413                 if (!fSuccess)
 414                     LogPrintf("CDB::Rewrite: Failed to rewrite database file %s\n", strFileRes);
 415                 return fSuccess;
 416             }
 417         }
 418         MilliSleep(100);
 419     }
 420     return false;
 421 }
 422
 423
 424 void CDBEnv::Flush(bool fShutdown)
 425 {
 426     int64_t nStart = GetTimeMillis();
 427     // Flush log data to the actual data file on all files that are not in use
 428     LogPrint("db", "CDBEnv::Flush: Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
 429     if (!fDbEnvInit)
 430         return;
 431     {
 432         LOCK(cs_db);
 433         map<string, int>::iterator mi = mapFileUseCount.begin();
 434         while (mi != mapFileUseCount.end()) {
 435             string strFile = (*mi).first;
 436             int nRefCount = (*mi).second;
 437             LogPrint("db", "CDBEnv::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
 438             if (nRefCount == 0) {
 439                 // Move log data to the dat file
 440                 CloseDb(strFile);
 441                 LogPrint("db", "CDBEnv::Flush: %s checkpoint\n", strFile);
 442                 dbenv->txn_checkpoint(0, 0, 0);
 443                 LogPrint("db", "CDBEnv::Flush: %s detach\n", strFile);
 444                 if (!fMockDb)
 445                     dbenv->lsn_reset(strFile.c_str(), 0);
 446                 LogPrint("db", "CDBEnv::Flush: %s closed\n", strFile);
 447                 mapFileUseCount.erase(mi++);
 448             } else
 449                 mi++;
 450         }
 451         LogPrint("db", "CDBEnv::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart);
 452         if (fShutdown) {
 453             char** listp;
 454             if (mapFileUseCount.empty()) {
 455                 dbenv->log_archive(&listp, DB_ARCH_REMOVE);
 456                 Close();
 457                 if (!fMockDb)
 458                     boost::filesystem::remove_all(boost::filesystem::path(strPath) / "database");
 459             }
 460         }
 461     }
 462 }