Fix constness of ArgsManager methods
[bitcoinplatinum.git] / src / wallet / db.cpp
blobda2d1807563bcb2e89c0b1afb5239d5960744ff9
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2016 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
6 #include "db.h"
8 #include "addrman.h"
9 #include "fs.h"
10 #include "hash.h"
11 #include "protocol.h"
12 #include "util.h"
13 #include "utilstrencodings.h"
15 #include <stdint.h>
17 #ifndef WIN32
18 #include <sys/stat.h>
19 #endif
21 #include <boost/thread.hpp>
24 // CDB
27 CDBEnv bitdb;
29 void CDBEnv::EnvShutdown()
31 if (!fDbEnvInit)
32 return;
34 fDbEnvInit = false;
35 int ret = dbenv->close(0);
36 if (ret != 0)
37 LogPrintf("CDBEnv::EnvShutdown: Error %d shutting down database environment: %s\n", ret, DbEnv::strerror(ret));
38 if (!fMockDb)
39 DbEnv((u_int32_t)0).remove(strPath.c_str(), 0);
42 void CDBEnv::Reset()
44 delete dbenv;
45 dbenv = new DbEnv(DB_CXX_NO_EXCEPTIONS);
46 fDbEnvInit = false;
47 fMockDb = false;
50 CDBEnv::CDBEnv() : dbenv(NULL)
52 Reset();
55 CDBEnv::~CDBEnv()
57 EnvShutdown();
58 delete dbenv;
59 dbenv = NULL;
62 void CDBEnv::Close()
64 EnvShutdown();
67 bool CDBEnv::Open(const fs::path& pathIn)
69 if (fDbEnvInit)
70 return true;
72 boost::this_thread::interruption_point();
74 strPath = pathIn.string();
75 fs::path pathLogDir = pathIn / "database";
76 TryCreateDirectories(pathLogDir);
77 fs::path pathErrorFile = pathIn / "db.log";
78 LogPrintf("CDBEnv::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string());
80 unsigned int nEnvFlags = 0;
81 if (GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB))
82 nEnvFlags |= DB_PRIVATE;
84 dbenv->set_lg_dir(pathLogDir.string().c_str());
85 dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
86 dbenv->set_lg_bsize(0x10000);
87 dbenv->set_lg_max(1048576);
88 dbenv->set_lk_max_locks(40000);
89 dbenv->set_lk_max_objects(40000);
90 dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a")); /// debug
91 dbenv->set_flags(DB_AUTO_COMMIT, 1);
92 dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
93 dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
94 int ret = dbenv->open(strPath.c_str(),
95 DB_CREATE |
96 DB_INIT_LOCK |
97 DB_INIT_LOG |
98 DB_INIT_MPOOL |
99 DB_INIT_TXN |
100 DB_THREAD |
101 DB_RECOVER |
102 nEnvFlags,
103 S_IRUSR | S_IWUSR);
104 if (ret != 0)
105 return error("CDBEnv::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret));
107 fDbEnvInit = true;
108 fMockDb = false;
109 return true;
112 void CDBEnv::MakeMock()
114 if (fDbEnvInit)
115 throw std::runtime_error("CDBEnv::MakeMock: Already initialized");
117 boost::this_thread::interruption_point();
119 LogPrint(BCLog::DB, "CDBEnv::MakeMock\n");
121 dbenv->set_cachesize(1, 0, 1);
122 dbenv->set_lg_bsize(10485760 * 4);
123 dbenv->set_lg_max(10485760);
124 dbenv->set_lk_max_locks(10000);
125 dbenv->set_lk_max_objects(10000);
126 dbenv->set_flags(DB_AUTO_COMMIT, 1);
127 dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
128 int ret = dbenv->open(NULL,
129 DB_CREATE |
130 DB_INIT_LOCK |
131 DB_INIT_LOG |
132 DB_INIT_MPOOL |
133 DB_INIT_TXN |
134 DB_THREAD |
135 DB_PRIVATE,
136 S_IRUSR | S_IWUSR);
137 if (ret > 0)
138 throw std::runtime_error(strprintf("CDBEnv::MakeMock: Error %d opening database environment.", ret));
140 fDbEnvInit = true;
141 fMockDb = true;
144 CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename)
146 LOCK(cs_db);
147 assert(mapFileUseCount.count(strFile) == 0);
149 Db db(dbenv, 0);
150 int result = db.verify(strFile.c_str(), NULL, NULL, 0);
151 if (result == 0)
152 return VERIFY_OK;
153 else if (recoverFunc == NULL)
154 return RECOVER_FAIL;
156 // Try to recover:
157 bool fRecovered = (*recoverFunc)(strFile, out_backup_filename);
158 return (fRecovered ? RECOVER_OK : RECOVER_FAIL);
161 bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename)
163 // Recovery procedure:
164 // move wallet file to walletfilename.timestamp.bak
165 // Call Salvage with fAggressive=true to
166 // get as much data as possible.
167 // Rewrite salvaged data to fresh wallet file
168 // Set -rescan so any missing transactions will be
169 // found.
170 int64_t now = GetTime();
171 newFilename = strprintf("%s.%d.bak", filename, now);
173 int result = bitdb.dbenv->dbrename(NULL, filename.c_str(), NULL,
174 newFilename.c_str(), DB_AUTO_COMMIT);
175 if (result == 0)
176 LogPrintf("Renamed %s to %s\n", filename, newFilename);
177 else
179 LogPrintf("Failed to rename %s to %s\n", filename, newFilename);
180 return false;
183 std::vector<CDBEnv::KeyValPair> salvagedData;
184 bool fSuccess = bitdb.Salvage(newFilename, true, salvagedData);
185 if (salvagedData.empty())
187 LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename);
188 return false;
190 LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size());
192 std::unique_ptr<Db> pdbCopy(new Db(bitdb.dbenv, 0));
193 int ret = pdbCopy->open(NULL, // Txn pointer
194 filename.c_str(), // Filename
195 "main", // Logical db name
196 DB_BTREE, // Database type
197 DB_CREATE, // Flags
199 if (ret > 0)
201 LogPrintf("Cannot create database file %s\n", filename);
202 return false;
205 DbTxn* ptxn = bitdb.TxnBegin();
206 for (CDBEnv::KeyValPair& row : salvagedData)
208 if (recoverKVcallback)
210 CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION);
211 CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION);
212 if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue))
213 continue;
215 Dbt datKey(&row.first[0], row.first.size());
216 Dbt datValue(&row.second[0], row.second.size());
217 int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE);
218 if (ret2 > 0)
219 fSuccess = false;
221 ptxn->commit(0);
222 pdbCopy->close(0);
224 return fSuccess;
227 bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr)
229 LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0));
230 LogPrintf("Using wallet %s\n", walletFile);
232 // Wallet file must be a plain filename without a directory
233 if (walletFile != fs::basename(walletFile) + fs::extension(walletFile))
235 errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string());
236 return false;
239 if (!bitdb.Open(dataDir))
241 // try moving the database env out of the way
242 fs::path pathDatabase = dataDir / "database";
243 fs::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime());
244 try {
245 fs::rename(pathDatabase, pathDatabaseBak);
246 LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string());
247 } catch (const fs::filesystem_error&) {
248 // failure is ok (well, not really, but it's not worse than what we started with)
251 // try again
252 if (!bitdb.Open(dataDir)) {
253 // if it still fails, it probably means we can't even create the database env
254 errorStr = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir());
255 return false;
258 return true;
261 bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc)
263 if (fs::exists(dataDir / walletFile))
265 std::string backup_filename;
266 CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc, backup_filename);
267 if (r == CDBEnv::RECOVER_OK)
269 warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!"
270 " Original %s saved as %s in %s; if"
271 " your balance or transactions are incorrect you should"
272 " restore from a backup."),
273 walletFile, backup_filename, dataDir);
275 if (r == CDBEnv::RECOVER_FAIL)
277 errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile);
278 return false;
281 // also return true if files does not exists
282 return true;
285 /* End of headers, beginning of key/value data */
286 static const char *HEADER_END = "HEADER=END";
287 /* End of key/value data */
288 static const char *DATA_END = "DATA=END";
290 bool CDBEnv::Salvage(const std::string& strFile, bool fAggressive, std::vector<CDBEnv::KeyValPair>& vResult)
292 LOCK(cs_db);
293 assert(mapFileUseCount.count(strFile) == 0);
295 u_int32_t flags = DB_SALVAGE;
296 if (fAggressive)
297 flags |= DB_AGGRESSIVE;
299 std::stringstream strDump;
301 Db db(dbenv, 0);
302 int result = db.verify(strFile.c_str(), NULL, &strDump, flags);
303 if (result == DB_VERIFY_BAD) {
304 LogPrintf("CDBEnv::Salvage: Database salvage found errors, all data may not be recoverable.\n");
305 if (!fAggressive) {
306 LogPrintf("CDBEnv::Salvage: Rerun with aggressive mode to ignore errors and continue.\n");
307 return false;
310 if (result != 0 && result != DB_VERIFY_BAD) {
311 LogPrintf("CDBEnv::Salvage: Database salvage failed with result %d.\n", result);
312 return false;
315 // Format of bdb dump is ascii lines:
316 // header lines...
317 // HEADER=END
318 // hexadecimal key
319 // hexadecimal value
320 // ... repeated
321 // DATA=END
323 std::string strLine;
324 while (!strDump.eof() && strLine != HEADER_END)
325 getline(strDump, strLine); // Skip past header
327 std::string keyHex, valueHex;
328 while (!strDump.eof() && keyHex != DATA_END) {
329 getline(strDump, keyHex);
330 if (keyHex != DATA_END) {
331 if (strDump.eof())
332 break;
333 getline(strDump, valueHex);
334 if (valueHex == DATA_END) {
335 LogPrintf("CDBEnv::Salvage: WARNING: Number of keys in data does not match number of values.\n");
336 break;
338 vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex)));
342 if (keyHex != DATA_END) {
343 LogPrintf("CDBEnv::Salvage: WARNING: Unexpected end of file while reading salvage output.\n");
344 return false;
347 return (result == 0);
351 void CDBEnv::CheckpointLSN(const std::string& strFile)
353 dbenv->txn_checkpoint(0, 0, 0);
354 if (fMockDb)
355 return;
356 dbenv->lsn_reset(strFile.c_str(), 0);
360 CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL)
362 fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
363 fFlushOnClose = fFlushOnCloseIn;
364 env = dbw.env;
365 if (dbw.IsDummy()) {
366 return;
368 const std::string &strFilename = dbw.strFile;
370 bool fCreate = strchr(pszMode, 'c') != NULL;
371 unsigned int nFlags = DB_THREAD;
372 if (fCreate)
373 nFlags |= DB_CREATE;
376 LOCK(env->cs_db);
377 if (!env->Open(GetDataDir()))
378 throw std::runtime_error("CDB: Failed to open database environment.");
380 strFile = strFilename;
381 ++env->mapFileUseCount[strFile];
382 pdb = env->mapDb[strFile];
383 if (pdb == NULL) {
384 int ret;
385 pdb = new Db(env->dbenv, 0);
387 bool fMockDb = env->IsMock();
388 if (fMockDb) {
389 DbMpoolFile* mpf = pdb->get_mpf();
390 ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
391 if (ret != 0)
392 throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile));
395 ret = pdb->open(NULL, // Txn pointer
396 fMockDb ? NULL : strFile.c_str(), // Filename
397 fMockDb ? strFile.c_str() : "main", // Logical db name
398 DB_BTREE, // Database type
399 nFlags, // Flags
402 if (ret != 0) {
403 delete pdb;
404 pdb = NULL;
405 --env->mapFileUseCount[strFile];
406 strFile = "";
407 throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename));
410 if (fCreate && !Exists(std::string("version"))) {
411 bool fTmp = fReadOnly;
412 fReadOnly = false;
413 WriteVersion(CLIENT_VERSION);
414 fReadOnly = fTmp;
417 env->mapDb[strFile] = pdb;
422 void CDB::Flush()
424 if (activeTxn)
425 return;
427 // Flush database activity from memory pool to disk log
428 unsigned int nMinutes = 0;
429 if (fReadOnly)
430 nMinutes = 1;
432 env->dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
435 void CWalletDBWrapper::IncrementUpdateCounter()
437 ++nUpdateCounter;
440 void CDB::Close()
442 if (!pdb)
443 return;
444 if (activeTxn)
445 activeTxn->abort();
446 activeTxn = NULL;
447 pdb = NULL;
449 if (fFlushOnClose)
450 Flush();
453 LOCK(env->cs_db);
454 --env->mapFileUseCount[strFile];
458 void CDBEnv::CloseDb(const std::string& strFile)
461 LOCK(cs_db);
462 if (mapDb[strFile] != NULL) {
463 // Close the database handle
464 Db* pdb = mapDb[strFile];
465 pdb->close(0);
466 delete pdb;
467 mapDb[strFile] = NULL;
472 bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip)
474 if (dbw.IsDummy()) {
475 return true;
477 CDBEnv *env = dbw.env;
478 const std::string& strFile = dbw.strFile;
479 while (true) {
481 LOCK(env->cs_db);
482 if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) {
483 // Flush log data to the dat file
484 env->CloseDb(strFile);
485 env->CheckpointLSN(strFile);
486 env->mapFileUseCount.erase(strFile);
488 bool fSuccess = true;
489 LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile);
490 std::string strFileRes = strFile + ".rewrite";
491 { // surround usage of db with extra {}
492 CDB db(dbw, "r");
493 Db* pdbCopy = new Db(env->dbenv, 0);
495 int ret = pdbCopy->open(NULL, // Txn pointer
496 strFileRes.c_str(), // Filename
497 "main", // Logical db name
498 DB_BTREE, // Database type
499 DB_CREATE, // Flags
501 if (ret > 0) {
502 LogPrintf("CDB::Rewrite: Can't create database file %s\n", strFileRes);
503 fSuccess = false;
506 Dbc* pcursor = db.GetCursor();
507 if (pcursor)
508 while (fSuccess) {
509 CDataStream ssKey(SER_DISK, CLIENT_VERSION);
510 CDataStream ssValue(SER_DISK, CLIENT_VERSION);
511 int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue);
512 if (ret1 == DB_NOTFOUND) {
513 pcursor->close();
514 break;
515 } else if (ret1 != 0) {
516 pcursor->close();
517 fSuccess = false;
518 break;
520 if (pszSkip &&
521 strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
522 continue;
523 if (strncmp(ssKey.data(), "\x07version", 8) == 0) {
524 // Update version:
525 ssValue.clear();
526 ssValue << CLIENT_VERSION;
528 Dbt datKey(ssKey.data(), ssKey.size());
529 Dbt datValue(ssValue.data(), ssValue.size());
530 int ret2 = pdbCopy->put(NULL, &datKey, &datValue, DB_NOOVERWRITE);
531 if (ret2 > 0)
532 fSuccess = false;
534 if (fSuccess) {
535 db.Close();
536 env->CloseDb(strFile);
537 if (pdbCopy->close(0))
538 fSuccess = false;
539 delete pdbCopy;
542 if (fSuccess) {
543 Db dbA(env->dbenv, 0);
544 if (dbA.remove(strFile.c_str(), NULL, 0))
545 fSuccess = false;
546 Db dbB(env->dbenv, 0);
547 if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0))
548 fSuccess = false;
550 if (!fSuccess)
551 LogPrintf("CDB::Rewrite: Failed to rewrite database file %s\n", strFileRes);
552 return fSuccess;
555 MilliSleep(100);
557 return false;
561 void CDBEnv::Flush(bool fShutdown)
563 int64_t nStart = GetTimeMillis();
564 // Flush log data to the actual data file on all files that are not in use
565 LogPrint(BCLog::DB, "CDBEnv::Flush: Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
566 if (!fDbEnvInit)
567 return;
569 LOCK(cs_db);
570 std::map<std::string, int>::iterator mi = mapFileUseCount.begin();
571 while (mi != mapFileUseCount.end()) {
572 std::string strFile = (*mi).first;
573 int nRefCount = (*mi).second;
574 LogPrint(BCLog::DB, "CDBEnv::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
575 if (nRefCount == 0) {
576 // Move log data to the dat file
577 CloseDb(strFile);
578 LogPrint(BCLog::DB, "CDBEnv::Flush: %s checkpoint\n", strFile);
579 dbenv->txn_checkpoint(0, 0, 0);
580 LogPrint(BCLog::DB, "CDBEnv::Flush: %s detach\n", strFile);
581 if (!fMockDb)
582 dbenv->lsn_reset(strFile.c_str(), 0);
583 LogPrint(BCLog::DB, "CDBEnv::Flush: %s closed\n", strFile);
584 mapFileUseCount.erase(mi++);
585 } else
586 mi++;
588 LogPrint(BCLog::DB, "CDBEnv::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart);
589 if (fShutdown) {
590 char** listp;
591 if (mapFileUseCount.empty()) {
592 dbenv->log_archive(&listp, DB_ARCH_REMOVE);
593 Close();
594 if (!fMockDb)
595 fs::remove_all(fs::path(strPath) / "database");
601 bool CDB::PeriodicFlush(CWalletDBWrapper& dbw)
603 if (dbw.IsDummy()) {
604 return true;
606 bool ret = false;
607 CDBEnv *env = dbw.env;
608 const std::string& strFile = dbw.strFile;
609 TRY_LOCK(bitdb.cs_db,lockDb);
610 if (lockDb)
612 // Don't do this if any databases are in use
613 int nRefCount = 0;
614 std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin();
615 while (mit != env->mapFileUseCount.end())
617 nRefCount += (*mit).second;
618 mit++;
621 if (nRefCount == 0)
623 boost::this_thread::interruption_point();
624 std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile);
625 if (mi != env->mapFileUseCount.end())
627 LogPrint(BCLog::DB, "Flushing %s\n", strFile);
628 int64_t nStart = GetTimeMillis();
630 // Flush wallet file so it's self contained
631 env->CloseDb(strFile);
632 env->CheckpointLSN(strFile);
634 env->mapFileUseCount.erase(mi++);
635 LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
636 ret = true;
641 return ret;
644 bool CWalletDBWrapper::Rewrite(const char* pszSkip)
646 return CDB::Rewrite(*this, pszSkip);
649 bool CWalletDBWrapper::Backup(const std::string& strDest)
651 if (IsDummy()) {
652 return false;
654 while (true)
657 LOCK(env->cs_db);
658 if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0)
660 // Flush log data to the dat file
661 env->CloseDb(strFile);
662 env->CheckpointLSN(strFile);
663 env->mapFileUseCount.erase(strFile);
665 // Copy wallet file
666 fs::path pathSrc = GetDataDir() / strFile;
667 fs::path pathDest(strDest);
668 if (fs::is_directory(pathDest))
669 pathDest /= strFile;
671 try {
672 fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists);
673 LogPrintf("copied %s to %s\n", strFile, pathDest.string());
674 return true;
675 } catch (const fs::filesystem_error& e) {
676 LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), e.what());
677 return false;
681 MilliSleep(100);
683 return false;
686 void CWalletDBWrapper::Flush(bool shutdown)
688 if (!IsDummy()) {
689 env->Flush(shutdown);