1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2016 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
13 #include "utilstrencodings.h"
21 #include <boost/foreach.hpp>
22 #include <boost/thread.hpp>
30 void CDBEnv::EnvShutdown()
36 int ret
= dbenv
->close(0);
38 LogPrintf("CDBEnv::EnvShutdown: Error %d shutting down database environment: %s\n", ret
, DbEnv::strerror(ret
));
40 DbEnv((u_int32_t
)0).remove(strPath
.c_str(), 0);
46 dbenv
= new DbEnv(DB_CXX_NO_EXCEPTIONS
);
51 CDBEnv::CDBEnv() : dbenv(NULL
)
68 bool CDBEnv::Open(const fs::path
& pathIn
)
73 boost::this_thread::interruption_point();
75 strPath
= pathIn
.string();
76 fs::path pathLogDir
= pathIn
/ "database";
77 TryCreateDirectory(pathLogDir
);
78 fs::path pathErrorFile
= pathIn
/ "db.log";
79 LogPrintf("CDBEnv::Open: LogDir=%s ErrorFile=%s\n", pathLogDir
.string(), pathErrorFile
.string());
81 unsigned int nEnvFlags
= 0;
82 if (GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB
))
83 nEnvFlags
|= DB_PRIVATE
;
85 dbenv
->set_lg_dir(pathLogDir
.string().c_str());
86 dbenv
->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
87 dbenv
->set_lg_bsize(0x10000);
88 dbenv
->set_lg_max(1048576);
89 dbenv
->set_lk_max_locks(40000);
90 dbenv
->set_lk_max_objects(40000);
91 dbenv
->set_errfile(fsbridge::fopen(pathErrorFile
, "a")); /// debug
92 dbenv
->set_flags(DB_AUTO_COMMIT
, 1);
93 dbenv
->set_flags(DB_TXN_WRITE_NOSYNC
, 1);
94 dbenv
->log_set_config(DB_LOG_AUTO_REMOVE
, 1);
95 int ret
= dbenv
->open(strPath
.c_str(),
106 return error("CDBEnv::Open: Error %d opening database environment: %s\n", ret
, DbEnv::strerror(ret
));
113 void CDBEnv::MakeMock()
116 throw std::runtime_error("CDBEnv::MakeMock: Already initialized");
118 boost::this_thread::interruption_point();
120 LogPrint(BCLog::DB
, "CDBEnv::MakeMock\n");
122 dbenv
->set_cachesize(1, 0, 1);
123 dbenv
->set_lg_bsize(10485760 * 4);
124 dbenv
->set_lg_max(10485760);
125 dbenv
->set_lk_max_locks(10000);
126 dbenv
->set_lk_max_objects(10000);
127 dbenv
->set_flags(DB_AUTO_COMMIT
, 1);
128 dbenv
->log_set_config(DB_LOG_IN_MEMORY
, 1);
129 int ret
= dbenv
->open(NULL
,
139 throw std::runtime_error(strprintf("CDBEnv::MakeMock: Error %d opening database environment.", ret
));
145 CDBEnv::VerifyResult
CDBEnv::Verify(const std::string
& strFile
, recoverFunc_type recoverFunc
, std::string
& out_backup_filename
)
148 assert(mapFileUseCount
.count(strFile
) == 0);
151 int result
= db
.verify(strFile
.c_str(), NULL
, NULL
, 0);
154 else if (recoverFunc
== NULL
)
158 bool fRecovered
= (*recoverFunc
)(strFile
, out_backup_filename
);
159 return (fRecovered
? RECOVER_OK
: RECOVER_FAIL
);
162 bool CDB::Recover(const std::string
& filename
, void *callbackDataIn
, bool (*recoverKVcallback
)(void* callbackData
, CDataStream ssKey
, CDataStream ssValue
), std::string
& newFilename
)
164 // Recovery procedure:
165 // move wallet file to walletfilename.timestamp.bak
166 // Call Salvage with fAggressive=true to
167 // get as much data as possible.
168 // Rewrite salvaged data to fresh wallet file
169 // Set -rescan so any missing transactions will be
171 int64_t now
= GetTime();
172 newFilename
= strprintf("%s.%d.bak", filename
, now
);
174 int result
= bitdb
.dbenv
->dbrename(NULL
, filename
.c_str(), NULL
,
175 newFilename
.c_str(), DB_AUTO_COMMIT
);
177 LogPrintf("Renamed %s to %s\n", filename
, newFilename
);
180 LogPrintf("Failed to rename %s to %s\n", filename
, newFilename
);
184 std::vector
<CDBEnv::KeyValPair
> salvagedData
;
185 bool fSuccess
= bitdb
.Salvage(newFilename
, true, salvagedData
);
186 if (salvagedData
.empty())
188 LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename
);
191 LogPrintf("Salvage(aggressive) found %u records\n", salvagedData
.size());
193 std::unique_ptr
<Db
> pdbCopy(new Db(bitdb
.dbenv
, 0));
194 int ret
= pdbCopy
->open(NULL
, // Txn pointer
195 filename
.c_str(), // Filename
196 "main", // Logical db name
197 DB_BTREE
, // Database type
202 LogPrintf("Cannot create database file %s\n", filename
);
206 DbTxn
* ptxn
= bitdb
.TxnBegin();
207 BOOST_FOREACH(CDBEnv::KeyValPair
& row
, salvagedData
)
209 if (recoverKVcallback
)
211 CDataStream
ssKey(row
.first
, SER_DISK
, CLIENT_VERSION
);
212 CDataStream
ssValue(row
.second
, SER_DISK
, CLIENT_VERSION
);
213 if (!(*recoverKVcallback
)(callbackDataIn
, ssKey
, ssValue
))
216 Dbt
datKey(&row
.first
[0], row
.first
.size());
217 Dbt
datValue(&row
.second
[0], row
.second
.size());
218 int ret2
= pdbCopy
->put(ptxn
, &datKey
, &datValue
, DB_NOOVERWRITE
);
228 bool CDB::VerifyEnvironment(const std::string
& walletFile
, const fs::path
& dataDir
, std::string
& errorStr
)
230 LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0));
231 LogPrintf("Using wallet %s\n", walletFile
);
233 // Wallet file must be a plain filename without a directory
234 if (walletFile
!= fs::basename(walletFile
) + fs::extension(walletFile
))
236 errorStr
= strprintf(_("Wallet %s resides outside data directory %s"), walletFile
, dataDir
.string());
240 if (!bitdb
.Open(dataDir
))
242 // try moving the database env out of the way
243 fs::path pathDatabase
= dataDir
/ "database";
244 fs::path pathDatabaseBak
= dataDir
/ strprintf("database.%d.bak", GetTime());
246 fs::rename(pathDatabase
, pathDatabaseBak
);
247 LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase
.string(), pathDatabaseBak
.string());
248 } catch (const fs::filesystem_error
&) {
249 // failure is ok (well, not really, but it's not worse than what we started with)
253 if (!bitdb
.Open(dataDir
)) {
254 // if it still fails, it probably means we can't even create the database env
255 errorStr
= strprintf(_("Error initializing wallet database environment %s!"), GetDataDir());
262 bool CDB::VerifyDatabaseFile(const std::string
& walletFile
, const fs::path
& dataDir
, std::string
& warningStr
, std::string
& errorStr
, CDBEnv::recoverFunc_type recoverFunc
)
264 if (fs::exists(dataDir
/ walletFile
))
266 std::string backup_filename
;
267 CDBEnv::VerifyResult r
= bitdb
.Verify(walletFile
, recoverFunc
, backup_filename
);
268 if (r
== CDBEnv::RECOVER_OK
)
270 warningStr
= strprintf(_("Warning: Wallet file corrupt, data salvaged!"
271 " Original %s saved as %s in %s; if"
272 " your balance or transactions are incorrect you should"
273 " restore from a backup."),
274 walletFile
, backup_filename
, dataDir
);
276 if (r
== CDBEnv::RECOVER_FAIL
)
278 errorStr
= strprintf(_("%s corrupt, salvage failed"), walletFile
);
282 // also return true if files does not exists
286 /* End of headers, beginning of key/value data */
287 static const char *HEADER_END
= "HEADER=END";
288 /* End of key/value data */
289 static const char *DATA_END
= "DATA=END";
291 bool CDBEnv::Salvage(const std::string
& strFile
, bool fAggressive
, std::vector
<CDBEnv::KeyValPair
>& vResult
)
294 assert(mapFileUseCount
.count(strFile
) == 0);
296 u_int32_t flags
= DB_SALVAGE
;
298 flags
|= DB_AGGRESSIVE
;
300 std::stringstream strDump
;
303 int result
= db
.verify(strFile
.c_str(), NULL
, &strDump
, flags
);
304 if (result
== DB_VERIFY_BAD
) {
305 LogPrintf("CDBEnv::Salvage: Database salvage found errors, all data may not be recoverable.\n");
307 LogPrintf("CDBEnv::Salvage: Rerun with aggressive mode to ignore errors and continue.\n");
311 if (result
!= 0 && result
!= DB_VERIFY_BAD
) {
312 LogPrintf("CDBEnv::Salvage: Database salvage failed with result %d.\n", result
);
316 // Format of bdb dump is ascii lines:
325 while (!strDump
.eof() && strLine
!= HEADER_END
)
326 getline(strDump
, strLine
); // Skip past header
328 std::string keyHex
, valueHex
;
329 while (!strDump
.eof() && keyHex
!= DATA_END
) {
330 getline(strDump
, keyHex
);
331 if (keyHex
!= DATA_END
) {
334 getline(strDump
, valueHex
);
335 if (valueHex
== DATA_END
) {
336 LogPrintf("CDBEnv::Salvage: WARNING: Number of keys in data does not match number of values.\n");
339 vResult
.push_back(make_pair(ParseHex(keyHex
), ParseHex(valueHex
)));
343 if (keyHex
!= DATA_END
) {
344 LogPrintf("CDBEnv::Salvage: WARNING: Unexpected end of file while reading salvage output.\n");
348 return (result
== 0);
352 void CDBEnv::CheckpointLSN(const std::string
& strFile
)
354 dbenv
->txn_checkpoint(0, 0, 0);
357 dbenv
->lsn_reset(strFile
.c_str(), 0);
361 CDB::CDB(CWalletDBWrapper
& dbw
, const char* pszMode
, bool fFlushOnCloseIn
) : pdb(NULL
), activeTxn(NULL
)
363 fReadOnly
= (!strchr(pszMode
, '+') && !strchr(pszMode
, 'w'));
364 fFlushOnClose
= fFlushOnCloseIn
;
369 const std::string
&strFilename
= dbw
.strFile
;
371 bool fCreate
= strchr(pszMode
, 'c') != NULL
;
372 unsigned int nFlags
= DB_THREAD
;
378 if (!env
->Open(GetDataDir()))
379 throw std::runtime_error("CDB: Failed to open database environment.");
381 strFile
= strFilename
;
382 ++env
->mapFileUseCount
[strFile
];
383 pdb
= env
->mapDb
[strFile
];
386 pdb
= new Db(env
->dbenv
, 0);
388 bool fMockDb
= env
->IsMock();
390 DbMpoolFile
* mpf
= pdb
->get_mpf();
391 ret
= mpf
->set_flags(DB_MPOOL_NOFILE
, 1);
393 throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile
));
396 ret
= pdb
->open(NULL
, // Txn pointer
397 fMockDb
? NULL
: strFile
.c_str(), // Filename
398 fMockDb
? strFile
.c_str() : "main", // Logical db name
399 DB_BTREE
, // Database type
406 --env
->mapFileUseCount
[strFile
];
408 throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret
, strFilename
));
411 if (fCreate
&& !Exists(std::string("version"))) {
412 bool fTmp
= fReadOnly
;
414 WriteVersion(CLIENT_VERSION
);
418 env
->mapDb
[strFile
] = pdb
;
428 // Flush database activity from memory pool to disk log
429 unsigned int nMinutes
= 0;
433 env
->dbenv
->txn_checkpoint(nMinutes
? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE
) * 1024 : 0, nMinutes
, 0);
436 void CWalletDBWrapper::IncrementUpdateCounter()
455 --env
->mapFileUseCount
[strFile
];
459 void CDBEnv::CloseDb(const std::string
& strFile
)
463 if (mapDb
[strFile
] != NULL
) {
464 // Close the database handle
465 Db
* pdb
= mapDb
[strFile
];
468 mapDb
[strFile
] = NULL
;
473 bool CDB::Rewrite(CWalletDBWrapper
& dbw
, const char* pszSkip
)
478 CDBEnv
*env
= dbw
.env
;
479 const std::string
& strFile
= dbw
.strFile
;
483 if (!env
->mapFileUseCount
.count(strFile
) || env
->mapFileUseCount
[strFile
] == 0) {
484 // Flush log data to the dat file
485 env
->CloseDb(strFile
);
486 env
->CheckpointLSN(strFile
);
487 env
->mapFileUseCount
.erase(strFile
);
489 bool fSuccess
= true;
490 LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile
);
491 std::string strFileRes
= strFile
+ ".rewrite";
492 { // surround usage of db with extra {}
494 Db
* pdbCopy
= new Db(env
->dbenv
, 0);
496 int ret
= pdbCopy
->open(NULL
, // Txn pointer
497 strFileRes
.c_str(), // Filename
498 "main", // Logical db name
499 DB_BTREE
, // Database type
503 LogPrintf("CDB::Rewrite: Can't create database file %s\n", strFileRes
);
507 Dbc
* pcursor
= db
.GetCursor();
510 CDataStream
ssKey(SER_DISK
, CLIENT_VERSION
);
511 CDataStream
ssValue(SER_DISK
, CLIENT_VERSION
);
512 int ret1
= db
.ReadAtCursor(pcursor
, ssKey
, ssValue
);
513 if (ret1
== DB_NOTFOUND
) {
516 } else if (ret1
!= 0) {
522 strncmp(ssKey
.data(), pszSkip
, std::min(ssKey
.size(), strlen(pszSkip
))) == 0)
524 if (strncmp(ssKey
.data(), "\x07version", 8) == 0) {
527 ssValue
<< CLIENT_VERSION
;
529 Dbt
datKey(ssKey
.data(), ssKey
.size());
530 Dbt
datValue(ssValue
.data(), ssValue
.size());
531 int ret2
= pdbCopy
->put(NULL
, &datKey
, &datValue
, DB_NOOVERWRITE
);
537 env
->CloseDb(strFile
);
538 if (pdbCopy
->close(0))
544 Db
dbA(env
->dbenv
, 0);
545 if (dbA
.remove(strFile
.c_str(), NULL
, 0))
547 Db
dbB(env
->dbenv
, 0);
548 if (dbB
.rename(strFileRes
.c_str(), NULL
, strFile
.c_str(), 0))
552 LogPrintf("CDB::Rewrite: Failed to rewrite database file %s\n", strFileRes
);
562 void CDBEnv::Flush(bool fShutdown
)
564 int64_t nStart
= GetTimeMillis();
565 // Flush log data to the actual data file on all files that are not in use
566 LogPrint(BCLog::DB
, "CDBEnv::Flush: Flush(%s)%s\n", fShutdown
? "true" : "false", fDbEnvInit
? "" : " database not started");
571 std::map
<std::string
, int>::iterator mi
= mapFileUseCount
.begin();
572 while (mi
!= mapFileUseCount
.end()) {
573 std::string strFile
= (*mi
).first
;
574 int nRefCount
= (*mi
).second
;
575 LogPrint(BCLog::DB
, "CDBEnv::Flush: Flushing %s (refcount = %d)...\n", strFile
, nRefCount
);
576 if (nRefCount
== 0) {
577 // Move log data to the dat file
579 LogPrint(BCLog::DB
, "CDBEnv::Flush: %s checkpoint\n", strFile
);
580 dbenv
->txn_checkpoint(0, 0, 0);
581 LogPrint(BCLog::DB
, "CDBEnv::Flush: %s detach\n", strFile
);
583 dbenv
->lsn_reset(strFile
.c_str(), 0);
584 LogPrint(BCLog::DB
, "CDBEnv::Flush: %s closed\n", strFile
);
585 mapFileUseCount
.erase(mi
++);
589 LogPrint(BCLog::DB
, "CDBEnv::Flush: Flush(%s)%s took %15dms\n", fShutdown
? "true" : "false", fDbEnvInit
? "" : " database not started", GetTimeMillis() - nStart
);
592 if (mapFileUseCount
.empty()) {
593 dbenv
->log_archive(&listp
, DB_ARCH_REMOVE
);
596 fs::remove_all(fs::path(strPath
) / "database");
602 bool CDB::PeriodicFlush(CWalletDBWrapper
& dbw
)
608 CDBEnv
*env
= dbw
.env
;
609 const std::string
& strFile
= dbw
.strFile
;
610 TRY_LOCK(bitdb
.cs_db
,lockDb
);
613 // Don't do this if any databases are in use
615 std::map
<std::string
, int>::iterator mit
= env
->mapFileUseCount
.begin();
616 while (mit
!= env
->mapFileUseCount
.end())
618 nRefCount
+= (*mit
).second
;
624 boost::this_thread::interruption_point();
625 std::map
<std::string
, int>::iterator mi
= env
->mapFileUseCount
.find(strFile
);
626 if (mi
!= env
->mapFileUseCount
.end())
628 LogPrint(BCLog::DB
, "Flushing %s\n", strFile
);
629 int64_t nStart
= GetTimeMillis();
631 // Flush wallet file so it's self contained
632 env
->CloseDb(strFile
);
633 env
->CheckpointLSN(strFile
);
635 env
->mapFileUseCount
.erase(mi
++);
636 LogPrint(BCLog::DB
, "Flushed %s %dms\n", strFile
, GetTimeMillis() - nStart
);
645 bool CWalletDBWrapper::Rewrite(const char* pszSkip
)
647 return CDB::Rewrite(*this, pszSkip
);
650 bool CWalletDBWrapper::Backup(const std::string
& strDest
)
659 if (!env
->mapFileUseCount
.count(strFile
) || env
->mapFileUseCount
[strFile
] == 0)
661 // Flush log data to the dat file
662 env
->CloseDb(strFile
);
663 env
->CheckpointLSN(strFile
);
664 env
->mapFileUseCount
.erase(strFile
);
667 fs::path pathSrc
= GetDataDir() / strFile
;
668 fs::path
pathDest(strDest
);
669 if (fs::is_directory(pathDest
))
673 fs::copy_file(pathSrc
, pathDest
, fs::copy_option::overwrite_if_exists
);
674 LogPrintf("copied %s to %s\n", strFile
, pathDest
.string());
676 } catch (const fs::filesystem_error
& e
) {
677 LogPrintf("error copying %s to %s - %s\n", strFile
, pathDest
.string(), e
.what());
687 void CWalletDBWrapper::Flush(bool shutdown
)
690 env
->Flush(shutdown
);