nihilist/monero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

blockchain_ancestry: faster and uses less memory

Browse Source
This commit is contained in:
moneromooo-monero 2018-07-30 21:36:38 +01:00
parent 2382484dcd
commit 628428a0df
No known key found for this signature in database
GPG Key ID: 686F07454D6CEFC3
1 changed files with 114 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
src/blockchain_utilities/blockchain_ancestry.cpp
View File

@ -73,31 +73,101 @@ namespace std
{ {
size_t operator()(const ancestor &a) const size_t operator()(const ancestor &a) const
{ {
crypto::hash h; return a.amount ^ a.offset; // not that bad, since amount almost always have a high bit set, and offset doesn't
crypto::cn_fast_hash(&a, sizeof(a), h);
return reinterpret_cast<const std::size_t &>(h);
} }
}; };
} }
struct tx_data_t
{
std::vector<std::pair<uint64_t, std::vector<uint64_t>>> vin;
std::vector<crypto::public_key> vout;
bool coinbase;
tx_data_t(): coinbase(false) {}
tx_data_t(const cryptonote::transaction &tx)
{
coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen);
if (!coinbase)
{
vin.reserve(tx.vin.size());
for (size_t ring = 0; ring < tx.vin.size(); ++ring)
{
if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key))
{
const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]);
vin.push_back(std::make_pair(txin.amount, cryptonote::relative_output_offsets_to_absolute(txin.key_offsets)));
}
else
{
LOG_PRINT_L0("Bad vin type in txid " << get_transaction_hash(tx));
throw std::runtime_error("Bad vin type");
}
}
}
vout.reserve(tx.vout.size());
for (size_t out = 0; out < tx.vout.size(); ++out)
{
if (tx.vout[out].target.type() == typeid(cryptonote::txout_to_key))
{
const auto &txout = boost::get<cryptonote::txout_to_key>(tx.vout[out].target);
vout.push_back(txout.key);
}
else
{
LOG_PRINT_L0("Bad vout type in txid " << get_transaction_hash(tx));
throw std::runtime_error("Bad vout type");
}
}
}
template <typename t_archive> void serialize(t_archive &a, const unsigned int ver)
{
a & coinbase;
a & vin;
a & vout;
}
};
struct ancestry_state_t struct ancestry_state_t
{ {
uint64_t height; uint64_t height;
std::unordered_map<crypto::hash, std::unordered_set<ancestor>> ancestry; std::unordered_map<crypto::hash, std::unordered_set<ancestor>> ancestry;
std::unordered_map<ancestor, crypto::hash> output_cache; std::unordered_map<ancestor, crypto::hash> output_cache;
std::unordered_map<crypto::hash, cryptonote::transaction> tx_cache; std::unordered_map<crypto::hash, ::tx_data_t> tx_cache;
std::unordered_map<uint64_t, cryptonote::block> block_cache; std::vector<cryptonote::block> block_cache;
template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) template <typename t_archive> void serialize(t_archive &a, const unsigned int ver)
{ {
a & height; a & height;
a & ancestry; a & ancestry;
a & output_cache; a & output_cache;
a & tx_cache; if (ver < 1)
a & block_cache; {
std::unordered_map<crypto::hash, cryptonote::transaction> old_tx_cache;
a & old_tx_cache;
for (const auto i: old_tx_cache)
tx_cache.insert(std::make_pair(i.first, ::tx_data_t(i.second)));
}
else
{
a & tx_cache;
}
if (ver < 2)
{
std::unordered_map<uint64_t, cryptonote::block> old_block_cache;
a & old_block_cache;
block_cache.resize(old_block_cache.size());
for (const auto i: old_block_cache)
block_cache[i.first] = i.second;
}
else
{
a & block_cache;
}
} }
}; };
BOOST_CLASS_VERSION(ancestry_state_t, 0) BOOST_CLASS_VERSION(ancestry_state_t, 2)
static void add_ancestor(std::unordered_map<ancestor, unsigned int> &ancestry, uint64_t amount, uint64_t offset) static void add_ancestor(std::unordered_map<ancestor, unsigned int> &ancestry, uint64_t amount, uint64_t offset)
{ {
@ -333,6 +403,7 @@ int main(int argc, char* argv[])
MINFO("Starting from height " << state.height); MINFO("Starting from height " << state.height);
const uint64_t db_height = db->height(); const uint64_t db_height = db->height();
state.block_cache.reserve(db_height);
for (uint64_t h = state.height; h < db_height; ++h) for (uint64_t h = state.height; h < db_height; ++h)
{ {
size_t block_ancestry_size = 0; size_t block_ancestry_size = 0;
@ -346,7 +417,10 @@ int main(int argc, char* argv[])
return 1; return 1;
} }
if (opt_cache_blocks) if (opt_cache_blocks)
state.block_cache.insert(std::make_pair(h, b)); {
state.block_cache.resize(h + 1);
state.block_cache[h] = b;
}
std::vector<crypto::hash> txids; std::vector<crypto::hash> txids;
txids.reserve(1 + b.tx_hashes.size()); txids.reserve(1 + b.tx_hashes.size());
if (opt_include_coinbase) if (opt_include_coinbase)
@ -357,13 +431,13 @@ int main(int argc, char* argv[])
{ {
printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height); printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height);
fflush(stdout); fflush(stdout);
cryptonote::transaction tx; ::tx_data_t tx_data;
std::unordered_map<crypto::hash, cryptonote::transaction>::const_iterator i = state.tx_cache.find(txid); std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid);
++total_txes; ++total_txes;
if (i != state.tx_cache.end()) if (i != state.tx_cache.end())
{ {
++cached_txes; ++cached_txes;
tx = i->second; tx_data = i->second;
} }
else else
{ {
@ -373,39 +447,38 @@ int main(int argc, char* argv[])
LOG_PRINT_L0("Failed to get txid " << txid << " from db"); LOG_PRINT_L0("Failed to get txid " << txid << " from db");
return 1; return 1;
} }
cryptonote::transaction tx;
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
{ {
LOG_PRINT_L0("Bad tx: " << txid); LOG_PRINT_L0("Bad tx: " << txid);
return 1; return 1;
} }
tx_data = ::tx_data_t(tx);
if (opt_cache_txes) if (opt_cache_txes)
state.tx_cache.insert(std::make_pair(txid, tx)); state.tx_cache.insert(std::make_pair(txid, tx_data));
} }
const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); if (tx_data.coinbase)
if (coinbase)
{ {
add_ancestry(state.ancestry, txid, std::unordered_set<ancestor>()); add_ancestry(state.ancestry, txid, std::unordered_set<ancestor>());
} }
else else
{ {
for (size_t ring = 0; ring < tx.vin.size(); ++ring) for (size_t ring = 0; ring < tx_data.vin.size(); ++ring)
{ {
if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) if (1)
{ {
const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); const uint64_t amount = tx_data.vin[ring].first;
const uint64_t amount = txin.amount; const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second;
auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets);
for (uint64_t offset: absolute_offsets) for (uint64_t offset: absolute_offsets)
{ {
const output_data_t od = db->get_output_key(amount, offset); const output_data_t od = db->get_output_key(amount, offset);
add_ancestry(state.ancestry, txid, ancestor{amount, offset}); add_ancestry(state.ancestry, txid, ancestor{amount, offset});
cryptonote::block b; cryptonote::block b;
auto iblock = state.block_cache.find(od.height);
++total_blocks; ++total_blocks;
if (iblock != state.block_cache.end()) if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty())
{ {
++cached_blocks; ++cached_blocks;
b = iblock->second; b = state.block_cache[od.height];
} }
else else
{ {
@ -417,7 +490,10 @@ int main(int argc, char* argv[])
return 1; return 1;
} }
if (opt_cache_blocks) if (opt_cache_blocks)
state.block_cache.insert(std::make_pair(od.height, b)); {
state.block_cache.resize(od.height + 1);
state.block_cache[od.height] = b;
}
} }
// find the tx which created this output // find the tx which created this output
bool found = false; bool found = false;
@ -453,13 +529,13 @@ int main(int argc, char* argv[])
{ {
if (found) if (found)
break; break;
cryptonote::transaction tx2; ::tx_data_t tx_data2;
std::unordered_map<crypto::hash, cryptonote::transaction>::const_iterator i = state.tx_cache.find(block_txid); std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(block_txid);
++total_txes; ++total_txes;
if (i != state.tx_cache.end()) if (i != state.tx_cache.end())
{ {
++cached_txes; ++cached_txes;
tx2 = i->second; tx_data2 = i->second;
} }
else else
{ {
@ -469,32 +545,25 @@ int main(int argc, char* argv[])
LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); LOG_PRINT_L0("Failed to get txid " << block_txid << " from db");
return 1; return 1;
} }
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) cryptonote::transaction tx;
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
{ {
LOG_PRINT_L0("Bad tx: " << block_txid); LOG_PRINT_L0("Bad tx: " << block_txid);
return 1; return 1;
} }
tx_data2 = ::tx_data_t(tx);
if (opt_cache_txes) if (opt_cache_txes)
state.tx_cache.insert(std::make_pair(block_txid, tx2)); state.tx_cache.insert(std::make_pair(block_txid, tx_data2));
} }
for (size_t out = 0; out < tx2.vout.size(); ++out) for (size_t out = 0; out < tx_data2.vout.size(); ++out)
{ {
if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) if (tx_data2.vout[out] == od.pubkey)
{ {
const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); found = true;
if (txout.key == od.pubkey) add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid));
{ if (opt_cache_outputs)
found = true; state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid));
add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); break;
if (opt_cache_outputs)
state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid));
break;
}
}
else
{
LOG_PRINT_L0("Bad vout type in txid " << block_txid);
return 1;
} }
} }
} }
@ -505,11 +574,6 @@ int main(int argc, char* argv[])
} }
} }
} }
else
{
LOG_PRINT_L0("Bad vin type in txid " << txid);
return 1;
}
} }
} }
const size_t ancestry_size = get_ancestry(state.ancestry, txid).size(); const size_t ancestry_size = get_ancestry(state.ancestry, txid).size();