Merge #10502: scripted-diff: Remove BOOST_FOREACH, Q_FOREACH and PAIRTYPE
[bitcoinplatinum.git] / src / wallet / db.cpp
blob844d61079313cf344d574cdef1707227fdcc2236
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2016 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
6 #include "db.h"
8 #include "addrman.h"
9 #include "fs.h"
10 #include "hash.h"
11 #include "protocol.h"
12 #include "util.h"
13 #include "utilstrencodings.h"
15 #include <stdint.h>
17 #ifndef WIN32
18 #include <sys/stat.h>
19 #endif
21 #include <boost/foreach.hpp>
22 #include <boost/thread.hpp>
25 // CDB
28 CDBEnv bitdb;
30 void CDBEnv::EnvShutdown()
32 if (!fDbEnvInit)
33 return;
35 fDbEnvInit = false;
36 int ret = dbenv->close(0);
37 if (ret != 0)
38 LogPrintf("CDBEnv::EnvShutdown: Error %d shutting down database environment: %s\n", ret, DbEnv::strerror(ret));
39 if (!fMockDb)
40 DbEnv((u_int32_t)0).remove(strPath.c_str(), 0);
43 void CDBEnv::Reset()
45 delete dbenv;
46 dbenv = new DbEnv(DB_CXX_NO_EXCEPTIONS);
47 fDbEnvInit = false;
48 fMockDb = false;
51 CDBEnv::CDBEnv() : dbenv(NULL)
53 Reset();
56 CDBEnv::~CDBEnv()
58 EnvShutdown();
59 delete dbenv;
60 dbenv = NULL;
63 void CDBEnv::Close()
65 EnvShutdown();
68 bool CDBEnv::Open(const fs::path& pathIn)
70 if (fDbEnvInit)
71 return true;
73 boost::this_thread::interruption_point();
75 strPath = pathIn.string();
76 fs::path pathLogDir = pathIn / "database";
77 TryCreateDirectory(pathLogDir);
78 fs::path pathErrorFile = pathIn / "db.log";
79 LogPrintf("CDBEnv::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string());
81 unsigned int nEnvFlags = 0;
82 if (GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB))
83 nEnvFlags |= DB_PRIVATE;
85 dbenv->set_lg_dir(pathLogDir.string().c_str());
86 dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
87 dbenv->set_lg_bsize(0x10000);
88 dbenv->set_lg_max(1048576);
89 dbenv->set_lk_max_locks(40000);
90 dbenv->set_lk_max_objects(40000);
91 dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug
92 dbenv->set_flags(DB_AUTO_COMMIT, 1);
93 dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
94 dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
95 int ret = dbenv->open(strPath.c_str(),
96 DB_CREATE |
97 DB_INIT_LOCK |
98 DB_INIT_LOG |
99 DB_INIT_MPOOL |
100 DB_INIT_TXN |
101 DB_THREAD |
102 DB_RECOVER |
103 nEnvFlags,
104 S_IRUSR | S_IWUSR);
105 if (ret != 0)
106 return error("CDBEnv::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret));
108 fDbEnvInit = true;
109 fMockDb = false;
110 return true;
113 void CDBEnv::MakeMock()
115 if (fDbEnvInit)
116 throw std::runtime_error("CDBEnv::MakeMock: Already initialized");
118 boost::this_thread::interruption_point();
120 LogPrint(BCLog::DB, "CDBEnv::MakeMock\n");
122 dbenv->set_cachesize(1, 0, 1);
123 dbenv->set_lg_bsize(10485760 * 4);
124 dbenv->set_lg_max(10485760);
125 dbenv->set_lk_max_locks(10000);
126 dbenv->set_lk_max_objects(10000);
127 dbenv->set_flags(DB_AUTO_COMMIT, 1);
128 dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
129 int ret = dbenv->open(NULL,
130 DB_CREATE |
131 DB_INIT_LOCK |
132 DB_INIT_LOG |
133 DB_INIT_MPOOL |
134 DB_INIT_TXN |
135 DB_THREAD |
136 DB_PRIVATE,
137 S_IRUSR | S_IWUSR);
138 if (ret > 0)
139 throw std::runtime_error(strprintf("CDBEnv::MakeMock: Error %d opening database environment.", ret));
141 fDbEnvInit = true;
142 fMockDb = true;
145 CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename)
147 LOCK(cs_db);
148 assert(mapFileUseCount.count(strFile) == 0);
150 Db db(dbenv, 0);
151 int result = db.verify(strFile.c_str(), NULL, NULL, 0);
152 if (result == 0)
153 return VERIFY_OK;
154 else if (recoverFunc == NULL)
155 return RECOVER_FAIL;
157 // Try to recover:
158 bool fRecovered = (*recoverFunc)(strFile, out_backup_filename);
159 return (fRecovered ? RECOVER_OK : RECOVER_FAIL);
162 bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename)
164 // Recovery procedure:
165 // move wallet file to walletfilename.timestamp.bak
166 // Call Salvage with fAggressive=true to
167 // get as much data as possible.
168 // Rewrite salvaged data to fresh wallet file
169 // Set -rescan so any missing transactions will be
170 // found.
171 int64_t now = GetTime();
172 newFilename = strprintf("%s.%d.bak", filename, now);
174 int result = bitdb.dbenv->dbrename(NULL, filename.c_str(), NULL,
175 newFilename.c_str(), DB_AUTO_COMMIT);
176 if (result == 0)
177 LogPrintf("Renamed %s to %s\n", filename, newFilename);
178 else
180 LogPrintf("Failed to rename %s to %s\n", filename, newFilename);
181 return false;
184 std::vector<CDBEnv::KeyValPair> salvagedData;
185 bool fSuccess = bitdb.Salvage(newFilename, true, salvagedData);
186 if (salvagedData.empty())
188 LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename);
189 return false;
191 LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size());
193 std::unique_ptr<Db> pdbCopy(new Db(bitdb.dbenv, 0));
194 int ret = pdbCopy->open(NULL, // Txn pointer
195 filename.c_str(), // Filename
196 "main", // Logical db name
197 DB_BTREE, // Database type
198 DB_CREATE, // Flags
200 if (ret > 0)
202 LogPrintf("Cannot create database file %s\n", filename);
203 return false;
206 DbTxn* ptxn = bitdb.TxnBegin();
207 for (CDBEnv::KeyValPair& row : salvagedData)
209 if (recoverKVcallback)
211 CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION);
212 CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION);
213 if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue))
214 continue;
216 Dbt datKey(&row.first[0], row.first.size());
217 Dbt datValue(&row.second[0], row.second.size());
218 int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE);
219 if (ret2 > 0)
220 fSuccess = false;
222 ptxn->commit(0);
223 pdbCopy->close(0);
225 return fSuccess;
228 bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr)
230 LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0));
231 LogPrintf("Using wallet %s\n", walletFile);
233 // Wallet file must be a plain filename without a directory
234 if (walletFile != fs::basename(walletFile) + fs::extension(walletFile))
236 errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string());
237 return false;
240 if (!bitdb.Open(dataDir))
242 // try moving the database env out of the way
243 fs::path pathDatabase = dataDir / "database";
244 fs::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime());
245 try {
246 fs::rename(pathDatabase, pathDatabaseBak);
247 LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string());
248 } catch (const fs::filesystem_error&) {
249 // failure is ok (well, not really, but it's not worse than what we started with)
252 // try again
253 if (!bitdb.Open(dataDir)) {
254 // if it still fails, it probably means we can't even create the database env
255 errorStr = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir());
256 return false;
259 return true;
262 bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc)
264 if (fs::exists(dataDir / walletFile))
266 std::string backup_filename;
267 CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc, backup_filename);
268 if (r == CDBEnv::RECOVER_OK)
270 warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!"
271 " Original %s saved as %s in %s; if"
272 " your balance or transactions are incorrect you should"
273 " restore from a backup."),
274 walletFile, backup_filename, dataDir);
276 if (r == CDBEnv::RECOVER_FAIL)
278 errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile);
279 return false;
282 // also return true if files does not exists
283 return true;
286 /* End of headers, beginning of key/value data */
287 static const char *HEADER_END = "HEADER=END";
288 /* End of key/value data */
289 static const char *DATA_END = "DATA=END";
291 bool CDBEnv::Salvage(const std::string& strFile, bool fAggressive, std::vector<CDBEnv::KeyValPair>& vResult)
293 LOCK(cs_db);
294 assert(mapFileUseCount.count(strFile) == 0);
296 u_int32_t flags = DB_SALVAGE;
297 if (fAggressive)
298 flags |= DB_AGGRESSIVE;
300 std::stringstream strDump;
302 Db db(dbenv, 0);
303 int result = db.verify(strFile.c_str(), NULL, &strDump, flags);
304 if (result == DB_VERIFY_BAD) {
305 LogPrintf("CDBEnv::Salvage: Database salvage found errors, all data may not be recoverable.\n");
306 if (!fAggressive) {
307 LogPrintf("CDBEnv::Salvage: Rerun with aggressive mode to ignore errors and continue.\n");
308 return false;
311 if (result != 0 && result != DB_VERIFY_BAD) {
312 LogPrintf("CDBEnv::Salvage: Database salvage failed with result %d.\n", result);
313 return false;
316 // Format of bdb dump is ascii lines:
317 // header lines...
318 // HEADER=END
319 // hexadecimal key
320 // hexadecimal value
321 // ... repeated
322 // DATA=END
324 std::string strLine;
325 while (!strDump.eof() && strLine != HEADER_END)
326 getline(strDump, strLine); // Skip past header
328 std::string keyHex, valueHex;
329 while (!strDump.eof() && keyHex != DATA_END) {
330 getline(strDump, keyHex);
331 if (keyHex != DATA_END) {
332 if (strDump.eof())
333 break;
334 getline(strDump, valueHex);
335 if (valueHex == DATA_END) {
336 LogPrintf("CDBEnv::Salvage: WARNING: Number of keys in data does not match number of values.\n");
337 break;
339 vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex)));
343 if (keyHex != DATA_END) {
344 LogPrintf("CDBEnv::Salvage: WARNING: Unexpected end of file while reading salvage output.\n");
345 return false;
348 return (result == 0);
352 void CDBEnv::CheckpointLSN(const std::string& strFile)
354 dbenv->txn_checkpoint(0, 0, 0);
355 if (fMockDb)
356 return;
357 dbenv->lsn_reset(strFile.c_str(), 0);
361 CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL)
363 fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
364 fFlushOnClose = fFlushOnCloseIn;
365 env = dbw.env;
366 if (dbw.IsDummy()) {
367 return;
369 const std::string &strFilename = dbw.strFile;
371 bool fCreate = strchr(pszMode, 'c') != NULL;
372 unsigned int nFlags = DB_THREAD;
373 if (fCreate)
374 nFlags |= DB_CREATE;
377 LOCK(env->cs_db);
378 if (!env->Open(GetDataDir()))
379 throw std::runtime_error("CDB: Failed to open database environment.");
381 strFile = strFilename;
382 ++env->mapFileUseCount[strFile];
383 pdb = env->mapDb[strFile];
384 if (pdb == NULL) {
385 int ret;
386 pdb = new Db(env->dbenv, 0);
388 bool fMockDb = env->IsMock();
389 if (fMockDb) {
390 DbMpoolFile* mpf = pdb->get_mpf();
391 ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
392 if (ret != 0)
393 throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile));
396 ret = pdb->open(NULL, // Txn pointer
397 fMockDb ? NULL : strFile.c_str(), // Filename
398 fMockDb ? strFile.c_str() : "main", // Logical db name
399 DB_BTREE, // Database type
400 nFlags, // Flags
403 if (ret != 0) {
404 delete pdb;
405 pdb = NULL;
406 --env->mapFileUseCount[strFile];
407 strFile = "";
408 throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename));
411 if (fCreate && !Exists(std::string("version"))) {
412 bool fTmp = fReadOnly;
413 fReadOnly = false;
414 WriteVersion(CLIENT_VERSION);
415 fReadOnly = fTmp;
418 env->mapDb[strFile] = pdb;
423 void CDB::Flush()
425 if (activeTxn)
426 return;
428 // Flush database activity from memory pool to disk log
429 unsigned int nMinutes = 0;
430 if (fReadOnly)
431 nMinutes = 1;
433 env->dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
436 void CWalletDBWrapper::IncrementUpdateCounter()
438 ++nUpdateCounter;
441 void CDB::Close()
443 if (!pdb)
444 return;
445 if (activeTxn)
446 activeTxn->abort();
447 activeTxn = NULL;
448 pdb = NULL;
450 if (fFlushOnClose)
451 Flush();
454 LOCK(env->cs_db);
455 --env->mapFileUseCount[strFile];
459 void CDBEnv::CloseDb(const std::string& strFile)
462 LOCK(cs_db);
463 if (mapDb[strFile] != NULL) {
464 // Close the database handle
465 Db* pdb = mapDb[strFile];
466 pdb->close(0);
467 delete pdb;
468 mapDb[strFile] = NULL;
473 bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip)
475 if (dbw.IsDummy()) {
476 return true;
478 CDBEnv *env = dbw.env;
479 const std::string& strFile = dbw.strFile;
480 while (true) {
482 LOCK(env->cs_db);
483 if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) {
484 // Flush log data to the dat file
485 env->CloseDb(strFile);
486 env->CheckpointLSN(strFile);
487 env->mapFileUseCount.erase(strFile);
489 bool fSuccess = true;
490 LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile);
491 std::string strFileRes = strFile + ".rewrite";
492 { // surround usage of db with extra {}
493 CDB db(dbw, "r");
494 Db* pdbCopy = new Db(env->dbenv, 0);
496 int ret = pdbCopy->open(NULL, // Txn pointer
497 strFileRes.c_str(), // Filename
498 "main", // Logical db name
499 DB_BTREE, // Database type
500 DB_CREATE, // Flags
502 if (ret > 0) {
503 LogPrintf("CDB::Rewrite: Can't create database file %s\n", strFileRes);
504 fSuccess = false;
507 Dbc* pcursor = db.GetCursor();
508 if (pcursor)
509 while (fSuccess) {
510 CDataStream ssKey(SER_DISK, CLIENT_VERSION);
511 CDataStream ssValue(SER_DISK, CLIENT_VERSION);
512 int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue);
513 if (ret1 == DB_NOTFOUND) {
514 pcursor->close();
515 break;
516 } else if (ret1 != 0) {
517 pcursor->close();
518 fSuccess = false;
519 break;
521 if (pszSkip &&
522 strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
523 continue;
524 if (strncmp(ssKey.data(), "\x07version", 8) == 0) {
525 // Update version:
526 ssValue.clear();
527 ssValue << CLIENT_VERSION;
529 Dbt datKey(ssKey.data(), ssKey.size());
530 Dbt datValue(ssValue.data(), ssValue.size());
531 int ret2 = pdbCopy->put(NULL, &datKey, &datValue, DB_NOOVERWRITE);
532 if (ret2 > 0)
533 fSuccess = false;
535 if (fSuccess) {
536 db.Close();
537 env->CloseDb(strFile);
538 if (pdbCopy->close(0))
539 fSuccess = false;
540 delete pdbCopy;
543 if (fSuccess) {
544 Db dbA(env->dbenv, 0);
545 if (dbA.remove(strFile.c_str(), NULL, 0))
546 fSuccess = false;
547 Db dbB(env->dbenv, 0);
548 if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0))
549 fSuccess = false;
551 if (!fSuccess)
552 LogPrintf("CDB::Rewrite: Failed to rewrite database file %s\n", strFileRes);
553 return fSuccess;
556 MilliSleep(100);
558 return false;
562 void CDBEnv::Flush(bool fShutdown)
564 int64_t nStart = GetTimeMillis();
565 // Flush log data to the actual data file on all files that are not in use
566 LogPrint(BCLog::DB, "CDBEnv::Flush: Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
567 if (!fDbEnvInit)
568 return;
570 LOCK(cs_db);
571 std::map<std::string, int>::iterator mi = mapFileUseCount.begin();
572 while (mi != mapFileUseCount.end()) {
573 std::string strFile = (*mi).first;
574 int nRefCount = (*mi).second;
575 LogPrint(BCLog::DB, "CDBEnv::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
576 if (nRefCount == 0) {
577 // Move log data to the dat file
578 CloseDb(strFile);
579 LogPrint(BCLog::DB, "CDBEnv::Flush: %s checkpoint\n", strFile);
580 dbenv->txn_checkpoint(0, 0, 0);
581 LogPrint(BCLog::DB, "CDBEnv::Flush: %s detach\n", strFile);
582 if (!fMockDb)
583 dbenv->lsn_reset(strFile.c_str(), 0);
584 LogPrint(BCLog::DB, "CDBEnv::Flush: %s closed\n", strFile);
585 mapFileUseCount.erase(mi++);
586 } else
587 mi++;
589 LogPrint(BCLog::DB, "CDBEnv::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart);
590 if (fShutdown) {
591 char** listp;
592 if (mapFileUseCount.empty()) {
593 dbenv->log_archive(&listp, DB_ARCH_REMOVE);
594 Close();
595 if (!fMockDb)
596 fs::remove_all(fs::path(strPath) / "database");
602 bool CDB::PeriodicFlush(CWalletDBWrapper& dbw)
604 if (dbw.IsDummy()) {
605 return true;
607 bool ret = false;
608 CDBEnv *env = dbw.env;
609 const std::string& strFile = dbw.strFile;
610 TRY_LOCK(bitdb.cs_db,lockDb);
611 if (lockDb)
613 // Don't do this if any databases are in use
614 int nRefCount = 0;
615 std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin();
616 while (mit != env->mapFileUseCount.end())
618 nRefCount += (*mit).second;
619 mit++;
622 if (nRefCount == 0)
624 boost::this_thread::interruption_point();
625 std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile);
626 if (mi != env->mapFileUseCount.end())
628 LogPrint(BCLog::DB, "Flushing %s\n", strFile);
629 int64_t nStart = GetTimeMillis();
631 // Flush wallet file so it's self contained
632 env->CloseDb(strFile);
633 env->CheckpointLSN(strFile);
635 env->mapFileUseCount.erase(mi++);
636 LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
637 ret = true;
642 return ret;
645 bool CWalletDBWrapper::Rewrite(const char* pszSkip)
647 return CDB::Rewrite(*this, pszSkip);
650 bool CWalletDBWrapper::Backup(const std::string& strDest)
652 if (IsDummy()) {
653 return false;
655 while (true)
658 LOCK(env->cs_db);
659 if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0)
661 // Flush log data to the dat file
662 env->CloseDb(strFile);
663 env->CheckpointLSN(strFile);
664 env->mapFileUseCount.erase(strFile);
666 // Copy wallet file
667 fs::path pathSrc = GetDataDir() / strFile;
668 fs::path pathDest(strDest);
669 if (fs::is_directory(pathDest))
670 pathDest /= strFile;
672 try {
673 fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists);
674 LogPrintf("copied %s to %s\n", strFile, pathDest.string());
675 return true;
676 } catch (const fs::filesystem_error& e) {
677 LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), e.what());
678 return false;
682 MilliSleep(100);
684 return false;
687 void CWalletDBWrapper::Flush(bool shutdown)
689 if (!IsDummy()) {
690 env->Flush(shutdown);