nihilist/monero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Schema update: tx_indices

Browse Source
This commit is contained in:
warptangent 2016-03-04 09:38:15 -08:00 committed by Howard Chu
parent 8d252a4214
commit ae0854a431
2 changed files with 109 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
src/blockchain_db/lmdb/db_lmdb.cpp
View File

@ -156,6 +156,7 @@ const char* const LMDB_BLOCK_INFO = "block_info";
const char* const LMDB_BLOCK_HEIGHTS = "block_heights";
const char* const LMDB_TXS = "txs";
const char* const LMDB_TX_INDICES = "tx_indices";
const char* const LMDB_TX_UNLOCKS = "tx_unlocks";
const char* const LMDB_TX_HEIGHTS = "tx_heights";
const char* const LMDB_TX_OUTPUTS = "tx_outputs";
@ -624,54 +625,76 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr
int result = 0;
CURSOR(txs)
CURSOR(tx_indices)
CURSOR(tx_heights)
CURSOR(tx_unlocks)
MDB_val_copy<uint64_t> val_tx_index(m_num_txs);
MDB_val_copy<crypto::hash> val_h(tx_hash);
MDB_val unused;
if (mdb_cursor_get(m_cur_txs, &val_h, &unused, MDB_SET) == 0)
throw1(TX_EXISTS("Attempting to add transaction that's already in the db"));
result = mdb_cursor_get(m_cur_tx_indices, &val_h, &unused, MDB_SET);
if (result == 0)
throw1(TX_EXISTS(std::string("Attempting to add transaction that's already in the db (tx index ").append(boost::lexical_cast<std::string>(*(const uint64_t*)unused.mv_data)).append(")").c_str()));
else if (result != MDB_NOTFOUND)
throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str()));
result = mdb_cursor_put(m_cur_tx_indices, &val_h, &val_tx_index, 0);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add tx index to db transaction: ", result).c_str()));
MDB_val_copy<blobdata> blob(tx_to_blob(tx));
result = mdb_cursor_put(m_cur_txs, &val_h, &blob, 0);
result = mdb_cursor_put(m_cur_txs, &val_tx_index, &blob, 0);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str()));
MDB_val_copy<uint64_t> height(m_height);
result = mdb_cursor_put(m_cur_tx_heights, &val_h, &height, 0);
result = mdb_cursor_put(m_cur_tx_heights, &val_tx_index, &height, 0);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add tx block height to db transaction: ", result).c_str()));
MDB_val_copy<uint64_t> unlock_time(tx.unlock_time);
result = mdb_cursor_put(m_cur_tx_unlocks, &val_h, &unlock_time, 0);
result = mdb_cursor_put(m_cur_tx_unlocks, &val_tx_index, &unlock_time, 0);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add tx unlock time to db transaction: ", result).c_str()));
m_num_txs++;
}
// TODO: compare pros and cons of looking up the tx hash's tx index once and
// passing it in to functions like this
void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
MDB_val_copy<crypto::hash> val_h(tx_hash);
MDB_val unused;
if (mdb_get(*m_write_txn, m_txs, &val_h, &unused))
MDB_val val_tx_index;
if (mdb_get(*m_write_txn, m_tx_indices, &val_h, &val_tx_index))
throw1(TX_DNE("Attempting to remove transaction that isn't in the db"));
if (mdb_del(*m_write_txn, m_txs, &val_h, NULL))
if (mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL))
throw1(DB_ERROR("Failed to add removal of tx to db transaction"));
if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL))
if (mdb_del(*m_write_txn, m_tx_unlocks, &val_tx_index, NULL))
throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction"));
if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL))
if (mdb_del(*m_write_txn, m_tx_heights, &val_tx_index, NULL))
throw1(DB_ERROR("Failed to add removal of tx block height to db transaction"));
remove_tx_outputs(tx_hash, tx);
auto result = mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL);
int result = mdb_del(*m_write_txn, m_tx_outputs, &val_tx_index, NULL);
if (result == MDB_NOTFOUND)
LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash);
else if (result)
throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str()));
// Though other things could change, so long as earlier functions (like
// remove_tx_outputs) need to do the lookup of tx hash -> tx index, don't
// delete the tx_indices entry until the end.
if (mdb_del(*m_write_txn, m_tx_indices, &val_h, NULL))
throw1(DB_ERROR("Failed to add removal of tx index to db transaction"));
m_num_txs--;
}
void BlockchainLMDB::add_output(const crypto::hash& tx_hash,
@ -763,7 +786,12 @@ void BlockchainLMDB::add_amount_and_global_output_indices(const crypto::hash& tx
v.mv_size = sizeof(uint64_t) * 2 * num_outputs;
// LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size);
result = mdb_cursor_put(m_cur_tx_outputs, &k, &v, 0);
MDB_val val_tx_index;
result = mdb_cursor_get(m_cur_tx_indices, &k, &val_tx_index, MDB_SET);
if (result)
throw0(DB_ERROR(lmdb_error(std::string("Failed to get tx index for tx hash ") + epee::string_tools::pod_to_hex(tx_hash), result).c_str()));
result = mdb_cursor_put(m_cur_tx_outputs, &val_tx_index, &v, 0);
if (result)
throw0(DB_ERROR(std::string("Failed to add <tx hash, amount output index array> to db transaction: ").append(mdb_strerror(result)).c_str()));
}
@ -1072,10 +1100,11 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE, m_block_info, "Failed to open db handle for m_block_info");
lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights");
lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs");
lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks");
lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights");
lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs");
lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs");
lmdb_db_open(txn, LMDB_TX_INDICES, MDB_CREATE, m_tx_indices, "Failed to open db handle for m_tx_indices");
lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks");
lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights");
lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs");
lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs");
lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices");
@ -1092,9 +1121,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
mdb_set_compare(txn, m_spent_keys, compare_hash32);
mdb_set_compare(txn, m_block_heights, compare_hash32);
mdb_set_compare(txn, m_txs, compare_hash32);
mdb_set_compare(txn, m_tx_unlocks, compare_hash32);
mdb_set_compare(txn, m_tx_heights, compare_hash32);
mdb_set_compare(txn, m_tx_indices, compare_hash32);
mdb_set_compare(txn, m_hf_starting_heights, compare_uint8);
mdb_set_compare(txn, m_hf_versions, compare_uint64);
mdb_set_compare(txn, m_properties, compare_string);
@ -1106,6 +1133,11 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries);
m_height = db_stats.ms_entries;
// get and keep current number of txs
if (mdb_stat(txn, m_tx_indices, &db_stats))
throw0(DB_ERROR("Failed to query m_tx_indices"));
m_num_txs = db_stats.ms_entries;
// get and keep current number of outputs
if (mdb_stat(txn, m_output_indices, &db_stats))
throw0(DB_ERROR("Failed to query m_output_indices"));
@ -1659,25 +1691,37 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
TXN_PREFIX_RDONLY();
const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_indices);
RCURSOR(txs);
MDB_val_copy<crypto::hash> key(h);
MDB_val val_tx_index;
MDB_val result;
bool tx_found = false;
TIME_MEASURE_START(time1);
auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET);
auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET);
if (get_result == 0)
tx_found = true;
else if (get_result != MDB_NOTFOUND)
throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", get_result).c_str()));
get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET);
TIME_MEASURE_FINISH(time1);
time_tx_exists += time1;
TXN_POSTFIX_RDONLY();
if (get_result == MDB_NOTFOUND)
if (! tx_found)
{
LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
return false;
}
if (get_result == MDB_NOTFOUND)
throw0(DB_ERROR(std::string("transaction with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found at index").c_str()));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch transaction from hash"));
throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction ") + epee::string_tools::pod_to_hex(h) + " at index: ", get_result).c_str()));
return true;
}
@ -1689,15 +1733,23 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const
TXN_PREFIX_RDONLY();
const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_indices);
RCURSOR(tx_unlocks);
MDB_val_copy<crypto::hash> key(h);
MDB_val val_tx_index;
MDB_val result;
auto get_result = mdb_cursor_get(m_cur_tx_unlocks, &key, &result, MDB_SET);
auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET);
if (get_result == MDB_NOTFOUND)
throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
throw1(TX_DNE(lmdb_error(std::string("tx index with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str()));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash"));
throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx index from hash: ", get_result).c_str()));
get_result = mdb_cursor_get(m_cur_tx_unlocks, &val_tx_index, &result, MDB_SET);
if (get_result == MDB_NOTFOUND)
throw1(TX_DNE(lmdb_error(std::string("tx unlock time with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str()));
else if (get_result)
throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx unlock time from index: ", get_result).c_str()));
uint64_t ret = *(const uint64_t*)result.mv_data;
TXN_POSTFIX_RDONLY();
@ -1711,11 +1763,15 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
TXN_PREFIX_RDONLY();
const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_indices);
RCURSOR(txs);
MDB_val_copy<crypto::hash> key(h);
MDB_val val_tx_index;
MDB_val result;
auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET);
auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET);
if (get_result == 0)
get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET);
if (get_result == MDB_NOTFOUND)
throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
else if (get_result)
@ -1741,8 +1797,8 @@ uint64_t BlockchainLMDB::get_tx_count() const
TXN_PREFIX_RDONLY();
MDB_stat db_stats;
if (mdb_stat(m_txn, m_txs, &db_stats))
throw0(DB_ERROR("Failed to query m_txs"));
if (mdb_stat(m_txn, m_tx_indices, &db_stats))
throw0(DB_ERROR("Failed to query m_tx_indices"));
TXN_POSTFIX_RDONLY();
@ -1770,11 +1826,15 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
TXN_PREFIX_RDONLY();
const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_indices);
RCURSOR(tx_heights);
MDB_val_copy<crypto::hash> key(h);
MDB_val val_tx_index;
MDB_val result;
auto get_result = mdb_cursor_get(m_cur_tx_heights, &key, &result, MDB_SET);
auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET);
if (get_result == 0)
get_result = mdb_cursor_get(m_cur_tx_heights, &val_tx_index, &result, MDB_SET);
if (get_result == MDB_NOTFOUND)
{
throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
@ -1906,24 +1966,32 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const crypto::hash& h,
// create a new read-only txn here, which is incorrect.
TXN_PREFIX_RDONLY();
const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors;
RCURSOR(tx_indices);
RCURSOR(tx_outputs);
int result = 0;
MDB_val_copy<crypto::hash> k(h);
MDB_val val_tx_index;
MDB_val v;
result = mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_SET);
result = mdb_cursor_get(m_cur_tx_indices, &k, &val_tx_index, MDB_SET);
if (result == MDB_NOTFOUND)
throw1(OUTPUT_DNE(std::string("Failed to get tx index for tx hash ").append(epee::string_tools::pod_to_hex(h)).c_str()));
else if (result)
throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", result).c_str()));
result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_index, &v, MDB_SET);
if (result == MDB_NOTFOUND)
LOG_PRINT_L0("WARNING: Unexpected: tx has no amount and global indices stored in "
"tx_outputs, but it should have an empty entry even if it's a tx without "
"outputs");
else if (result)
throw0(DB_ERROR("DB error attempting to get data for tx_outputs[tx_hash]"));
throw0(DB_ERROR("DB error attempting to get data for tx_outputs[tx_index]"));
uint64_t* paired_indices = (uint64_t*)v.mv_data;
int num_elems = v.mv_size / sizeof(uint64_t);
if (num_elems % 2 != 0)
throw0(DB_ERROR("tx_outputs[tx_hash] does not have an even numer of indices"));
throw0(DB_ERROR("tx_outputs[tx_index] does not have an even numer of indices"));
int num_outputs = num_elems / 2;
for (int i = 0; i < num_outputs; ++i)
@ -2380,6 +2448,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c
}
}
uint64_t num_txs = m_num_txs;
uint64_t num_outputs = m_num_outputs;
try
{
@ -2391,6 +2460,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c
}
catch (...)
{
m_num_txs = num_txs;
m_num_outputs = num_outputs;
block_txn_abort();
throw;
@ -2406,6 +2476,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
block_txn_start(false);
uint64_t num_txs = m_num_txs;
uint64_t num_outputs = m_num_outputs;
try
{
@ -2414,6 +2485,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
}
catch (...)
{
m_num_txs = num_txs;
m_num_outputs = num_outputs;
block_txn_abort();
throw;

5
src/blockchain_db/lmdb/db_lmdb.h
View File

@ -51,6 +51,7 @@ typedef struct mdb_txn_cursors
MDB_cursor *m_txc_output_keys;
MDB_cursor *m_txc_txs;
MDB_cursor *m_txc_tx_indices;
MDB_cursor *m_txc_tx_heights;
MDB_cursor *m_txc_tx_unlocks;
MDB_cursor *m_txc_tx_outputs;
@ -68,6 +69,7 @@ typedef struct mdb_txn_cursors
#define m_cur_output_amounts m_cursors->m_txc_output_amounts
#define m_cur_output_keys m_cursors->m_txc_output_keys
#define m_cur_txs m_cursors->m_txc_txs
#define m_cur_tx_indices m_cursors->m_txc_tx_indices
#define m_cur_tx_heights m_cursors->m_txc_tx_heights
#define m_cur_tx_unlocks m_cursors->m_txc_tx_unlocks
#define m_cur_tx_outputs m_cursors->m_txc_tx_outputs
@ -85,6 +87,7 @@ typedef struct mdb_rflags
bool m_rf_output_amounts;
bool m_rf_output_keys;
bool m_rf_txs;
bool m_rf_tx_indices;
bool m_rf_tx_heights;
bool m_rf_tx_unlocks;
bool m_rf_tx_outputs;
@ -363,6 +366,7 @@ private:
MDB_dbi m_block_info;
MDB_dbi m_txs;
MDB_dbi m_tx_indices;
MDB_dbi m_tx_unlocks;
MDB_dbi m_tx_heights;
MDB_dbi m_tx_outputs;
@ -380,6 +384,7 @@ private:
MDB_dbi m_properties;
uint64_t m_height;
uint64_t m_num_txs;
uint64_t m_num_outputs;
mutable uint64_t m_cum_size; // used in batch size estimation
mutable int m_cum_count;