mirror of
https://codeberg.org/anoncontributorxmr/monero.git
synced 2024-11-26 21:03:28 +01:00
Merge pull request #289
01076aeCheck if LMDB needs resize every 1000 blocks (Thomas Winget)b0d849enull out batch txn pointer as needed (BlockchainLMDB) (Thomas Winget)7b7ef73LMDB should now dynamically resize the mapsize (Thomas Winget)ac79502Move mdb_txn_safe implementation to cpp file (Thomas Winget)
This commit is contained in:
Riccardo Spagni
commit
634e367ff5
@ -148,20 +148,149 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi
|
||||
|
||||
namespace cryptonote
|
||||
{
|
||||
std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0};
|
||||
std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT;
|
||||
|
||||
// If m_batch_active is set, a batch transaction exists beyond this class, such
|
||||
// as a batch import with verification enabled, or possibly (later) a batch
|
||||
// network sync.
|
||||
mdb_txn_safe::mdb_txn_safe() : m_txn(NULL)
|
||||
{
|
||||
while (creation_gate.test_and_set());
|
||||
num_active_txns++;
|
||||
creation_gate.clear();
|
||||
}
|
||||
|
||||
mdb_txn_safe::~mdb_txn_safe()
|
||||
{
|
||||
LOG_PRINT_L3("mdb_txn_safe: destructor");
|
||||
if (m_txn != NULL)
|
||||
{
|
||||
if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
|
||||
{
|
||||
LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Example of when this occurs: a lookup fails, so a read-only txn is
|
||||
// aborted through this destructor. However, successful read-only txns
|
||||
// ideally should have been committed when done and not end up here.
|
||||
//
|
||||
// For some of the lookup methods, such as get_block_timestamp(), tx_exists(),
|
||||
// and get_tx(), when m_batch_active is set, the lookup uses the batch
|
||||
// transaction. This isn't only because the transaction is available, but it's
|
||||
// necessary so that lookups include the database updates only present in the
|
||||
// current batch write.
|
||||
//
|
||||
// A regular network sync without batch writes is expected to open a new read
|
||||
// transaction, as those lookups are part of the validation done prior to the
|
||||
// write for block and tx data, so no write transaction is open at the time.
|
||||
// NOTE: not sure if this is ever reached for a non-batch write
|
||||
// transaction, but it's probably not ideal if it did.
|
||||
LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
|
||||
}
|
||||
mdb_txn_abort(m_txn);
|
||||
}
|
||||
num_active_txns--;
|
||||
}
|
||||
|
||||
void mdb_txn_safe::commit(std::string message)
|
||||
{
|
||||
if (message.size() == 0)
|
||||
{
|
||||
message = "Failed to commit a transaction to the db";
|
||||
}
|
||||
|
||||
if (mdb_txn_commit(m_txn))
|
||||
{
|
||||
m_txn = NULL;
|
||||
LOG_PRINT_L0(message);
|
||||
throw DB_ERROR(message.c_str());
|
||||
}
|
||||
m_txn = NULL;
|
||||
}
|
||||
|
||||
void mdb_txn_safe::abort()
|
||||
{
|
||||
LOG_PRINT_L3("mdb_txn_safe: abort()");
|
||||
if(m_txn != NULL)
|
||||
{
|
||||
mdb_txn_abort(m_txn);
|
||||
m_txn = NULL;
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL");
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t mdb_txn_safe::num_active_tx()
|
||||
{
|
||||
return num_active_txns;
|
||||
}
|
||||
|
||||
void mdb_txn_safe::prevent_new_txns()
|
||||
{
|
||||
while (creation_gate.test_and_set());
|
||||
}
|
||||
|
||||
void mdb_txn_safe::wait_no_active_txns()
|
||||
{
|
||||
while (num_active_txns > 0);
|
||||
}
|
||||
|
||||
void mdb_txn_safe::allow_new_txns()
|
||||
{
|
||||
creation_gate.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
void BlockchainLMDB::do_resize()
|
||||
{
|
||||
MDB_envinfo mei;
|
||||
|
||||
mdb_env_info(m_env, &mei);
|
||||
|
||||
MDB_stat mst;
|
||||
|
||||
mdb_env_stat(m_env, &mst);
|
||||
|
||||
uint64_t new_mapsize = (double)mei.me_mapsize * RESIZE_FACTOR;
|
||||
|
||||
new_mapsize += (new_mapsize % mst.ms_psize);
|
||||
|
||||
mdb_txn_safe::prevent_new_txns();
|
||||
|
||||
if (m_write_txn != nullptr)
|
||||
{
|
||||
if (m_batch_active)
|
||||
{
|
||||
throw0(DB_ERROR("lmdb resizing not yet supported when batch transactions enabled!"));
|
||||
}
|
||||
else
|
||||
{
|
||||
throw0(DB_ERROR("attempting resize with write transaction in progress, this should not happen!"));
|
||||
}
|
||||
}
|
||||
|
||||
mdb_txn_safe::wait_no_active_txns();
|
||||
|
||||
mdb_env_set_mapsize(m_env, new_mapsize);
|
||||
|
||||
LOG_PRINT_L0("LMDB Mapsize increased."
|
||||
<< " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB"
|
||||
<< ", New: " << new_mapsize / (1024 * 1024) << "MiB");
|
||||
|
||||
mdb_txn_safe::allow_new_txns();
|
||||
}
|
||||
|
||||
bool BlockchainLMDB::need_resize()
|
||||
{
|
||||
MDB_envinfo mei;
|
||||
|
||||
mdb_env_info(m_env, &mei);
|
||||
|
||||
MDB_stat mst;
|
||||
|
||||
mdb_env_stat(m_env, &mst);
|
||||
|
||||
uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
|
||||
|
||||
if ((double)size_used / mei.me_mapsize > 0.8)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void BlockchainLMDB::add_block( const block& blk
|
||||
, const size_t& block_size
|
||||
@ -631,6 +760,7 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
|
||||
|
||||
m_batch_transactions = batch_transactions;
|
||||
m_write_txn = nullptr;
|
||||
m_write_batch_txn = nullptr;
|
||||
m_batch_active = false;
|
||||
m_height = 0;
|
||||
}
|
||||
@ -672,7 +802,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
|
||||
if (mdb_env_set_maxdbs(m_env, 20))
|
||||
throw0(DB_ERROR("Failed to set max number of dbs"));
|
||||
|
||||
size_t mapsize = 1LL << 34;
|
||||
size_t mapsize = DEFAULT_MAPSIZE;
|
||||
if (auto result = mdb_env_set_mapsize(m_env, mapsize))
|
||||
throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str()));
|
||||
if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644))
|
||||
@ -1660,16 +1790,21 @@ void BlockchainLMDB::batch_start()
|
||||
throw0(DB_ERROR("batch transactions not enabled"));
|
||||
if (m_batch_active)
|
||||
throw0(DB_ERROR("batch transaction already in progress"));
|
||||
if (m_write_batch_txn != nullptr)
|
||||
throw0(DB_ERROR("batch transaction already in progress"));
|
||||
if (m_write_txn)
|
||||
throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use"));
|
||||
check_open();
|
||||
|
||||
m_write_batch_txn = new mdb_txn_safe();
|
||||
|
||||
// NOTE: need to make sure it's destroyed properly when done
|
||||
if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn))
|
||||
if (mdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn))
|
||||
throw0(DB_ERROR("Failed to create a transaction for the db"));
|
||||
// indicates this transaction is for batch transactions, but not whether it's
|
||||
// active
|
||||
m_write_batch_txn.m_batch_txn = true;
|
||||
m_write_txn = &m_write_batch_txn;
|
||||
m_write_batch_txn->m_batch_txn = true;
|
||||
m_write_txn = m_write_batch_txn;
|
||||
m_batch_active = true;
|
||||
LOG_PRINT_L3("batch transaction: begin");
|
||||
}
|
||||
@ -1681,7 +1816,10 @@ void BlockchainLMDB::batch_commit()
|
||||
throw0(DB_ERROR("batch transactions not enabled"));
|
||||
if (! m_batch_active)
|
||||
throw0(DB_ERROR("batch transaction not in progress"));
|
||||
if (m_write_batch_txn == nullptr)
|
||||
throw0(DB_ERROR("batch transaction not in progress"));
|
||||
check_open();
|
||||
|
||||
LOG_PRINT_L3("batch transaction: committing...");
|
||||
TIME_MEASURE_START(time1);
|
||||
m_write_txn->commit();
|
||||
@ -1689,11 +1827,8 @@ void BlockchainLMDB::batch_commit()
|
||||
time_commit1 += time1;
|
||||
LOG_PRINT_L3("batch transaction: committed");
|
||||
|
||||
if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn))
|
||||
throw0(DB_ERROR("Failed to create a transaction for the db"));
|
||||
if (! m_write_batch_txn.m_batch_txn)
|
||||
throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction"));
|
||||
m_write_txn = &m_write_batch_txn;
|
||||
m_write_txn = nullptr;
|
||||
delete m_write_batch_txn;
|
||||
}
|
||||
|
||||
void BlockchainLMDB::batch_stop()
|
||||
@ -1703,6 +1838,8 @@ void BlockchainLMDB::batch_stop()
|
||||
throw0(DB_ERROR("batch transactions not enabled"));
|
||||
if (! m_batch_active)
|
||||
throw0(DB_ERROR("batch transaction not in progress"));
|
||||
if (m_write_batch_txn == nullptr)
|
||||
throw0(DB_ERROR("batch transaction not in progress"));
|
||||
check_open();
|
||||
LOG_PRINT_L3("batch transaction: committing...");
|
||||
TIME_MEASURE_START(time1);
|
||||
@ -1711,6 +1848,8 @@ void BlockchainLMDB::batch_stop()
|
||||
time_commit1 += time1;
|
||||
// for destruction of batch transaction
|
||||
m_write_txn = nullptr;
|
||||
delete m_write_batch_txn;
|
||||
m_write_batch_txn = nullptr;
|
||||
m_batch_active = false;
|
||||
LOG_PRINT_L3("batch transaction: end");
|
||||
}
|
||||
@ -1726,8 +1865,9 @@ void BlockchainLMDB::batch_abort()
|
||||
// for destruction of batch transaction
|
||||
m_write_txn = nullptr;
|
||||
// explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called.
|
||||
m_write_batch_txn.abort();
|
||||
m_write_batch_txn->abort();
|
||||
m_batch_active = false;
|
||||
m_write_batch_txn = nullptr;
|
||||
LOG_PRINT_L3("batch transaction: aborted");
|
||||
}
|
||||
|
||||
@ -1748,6 +1888,15 @@ uint64_t BlockchainLMDB::add_block( const block& blk
|
||||
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
|
||||
check_open();
|
||||
|
||||
if (m_height % 1000 == 0)
|
||||
{
|
||||
if (need_resize())
|
||||
{
|
||||
LOG_PRINT_L0("LMDB memory map needs resized, doing that now.");
|
||||
do_resize();
|
||||
}
|
||||
}
|
||||
|
||||
mdb_txn_safe txn;
|
||||
if (! m_batch_active)
|
||||
{
|
||||
|
||||
@ -26,6 +26,8 @@
|
||||
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
|
||||
#include "blockchain_db/blockchain_db.h"
|
||||
#include "cryptonote_protocol/blobdatatype.h" // for type blobdata
|
||||
|
||||
@ -36,63 +38,16 @@ namespace cryptonote
|
||||
|
||||
struct mdb_txn_safe
|
||||
{
|
||||
mdb_txn_safe() : m_txn(NULL) { }
|
||||
~mdb_txn_safe()
|
||||
{
|
||||
LOG_PRINT_L3("mdb_txn_safe: destructor");
|
||||
if (m_txn != NULL)
|
||||
{
|
||||
if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
|
||||
{
|
||||
LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Example of when this occurs: a lookup fails, so a read-only txn is
|
||||
// aborted through this destructor. However, successful read-only txns
|
||||
// ideally should have been committed when done and not end up here.
|
||||
//
|
||||
// NOTE: not sure if this is ever reached for a non-batch write
|
||||
// transaction, but it's probably not ideal if it did.
|
||||
LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
|
||||
}
|
||||
mdb_txn_abort(m_txn);
|
||||
}
|
||||
}
|
||||
mdb_txn_safe();
|
||||
~mdb_txn_safe();
|
||||
|
||||
void commit(std::string message = "")
|
||||
{
|
||||
if (message.size() == 0)
|
||||
{
|
||||
message = "Failed to commit a transaction to the db";
|
||||
}
|
||||
|
||||
if (mdb_txn_commit(m_txn))
|
||||
{
|
||||
m_txn = NULL;
|
||||
LOG_PRINT_L0(message);
|
||||
throw DB_ERROR(message.c_str());
|
||||
}
|
||||
m_txn = NULL;
|
||||
}
|
||||
void commit(std::string message = "");
|
||||
|
||||
// This should only be needed for batch transaction which must be ensured to
|
||||
// be aborted before mdb_env_close, not after. So we can't rely on
|
||||
// BlockchainLMDB destructor to call mdb_txn_safe destructor, as that's too late
|
||||
// to properly abort, since mdb_env_close would have been called earlier.
|
||||
void abort()
|
||||
{
|
||||
LOG_PRINT_L3("mdb_txn_safe: abort()");
|
||||
if(m_txn != NULL)
|
||||
{
|
||||
mdb_txn_abort(m_txn);
|
||||
m_txn = NULL;
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL");
|
||||
}
|
||||
}
|
||||
void abort();
|
||||
|
||||
operator MDB_txn*()
|
||||
{
|
||||
@ -104,11 +59,34 @@ struct mdb_txn_safe
|
||||
return &m_txn;
|
||||
}
|
||||
|
||||
uint64_t num_active_tx();
|
||||
|
||||
static void prevent_new_txns();
|
||||
static void wait_no_active_txns();
|
||||
static void allow_new_txns();
|
||||
|
||||
MDB_txn* m_txn;
|
||||
bool m_batch_txn = false;
|
||||
static std::atomic<uint64_t> num_active_txns;
|
||||
|
||||
// could use a mutex here, but this should be sufficient.
|
||||
static std::atomic_flag creation_gate;
|
||||
};
|
||||
|
||||
|
||||
// If m_batch_active is set, a batch transaction exists beyond this class, such
|
||||
// as a batch import with verification enabled, or possibly (later) a batch
|
||||
// network sync.
|
||||
//
|
||||
// For some of the lookup methods, such as get_block_timestamp(), tx_exists(),
|
||||
// and get_tx(), when m_batch_active is set, the lookup uses the batch
|
||||
// transaction. This isn't only because the transaction is available, but it's
|
||||
// necessary so that lookups include the database updates only present in the
|
||||
// current batch write.
|
||||
//
|
||||
// A regular network sync without batch writes is expected to open a new read
|
||||
// transaction, as those lookups are part of the validation done prior to the
|
||||
// write for block and tx data, so no write transaction is open at the time.
|
||||
class BlockchainLMDB : public BlockchainDB
|
||||
{
|
||||
public:
|
||||
@ -221,6 +199,10 @@ public:
|
||||
virtual void pop_block(block& blk, std::vector<transaction>& txs);
|
||||
|
||||
private:
|
||||
void do_resize();
|
||||
|
||||
bool need_resize();
|
||||
|
||||
virtual void add_block( const block& blk
|
||||
, const size_t& block_size
|
||||
, const difficulty_type& cumulative_difficulty
|
||||
@ -305,10 +287,14 @@ private:
|
||||
uint64_t m_num_outputs;
|
||||
std::string m_folder;
|
||||
mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
|
||||
mdb_txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
|
||||
mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
|
||||
|
||||
bool m_batch_transactions; // support for batch transactions
|
||||
bool m_batch_active; // whether batch transaction is in progress
|
||||
|
||||
constexpr static uint64_t DEFAULT_MAPSIZE = 1 << 30;
|
||||
constexpr static float RESIZE_PERCENT = 0.8f;
|
||||
constexpr static float RESIZE_FACTOR = 1.5f;
|
||||
};
|
||||
|
||||
} // namespace cryptonote
|
||||
|
||||
Loading…
Reference in New Issue
Block a user