nihilist/monero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #7221

Browse Source 
1eb14af1a rpc: limit the number of txes for get_blocks.bin (moneromooo-monero)
This commit is contained in:
Alexander Blair 2020-12-29 10:44:24 -08:00
commit a1eca8ca7e
No known key found for this signature in database
GPG Key ID: C64552D877C32479
11 changed files with 32 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/blockchain_db/blockchain_db.h
View File

@ -1312,17 +1312,20 @@ public:
* height. The number of blocks returned is variable, based on the max_size passed. * height. The number of blocks returned is variable, based on the max_size passed.
* *
* @param start_height the height of the first block * @param start_height the height of the first block
* @param min_count the minimum number of blocks to return, if they exist * @param min_block_count the minimum number of blocks to return, if they exist
* @param max_count the maximum number of blocks to return * @param max_block_count the maximum number of blocks to return
* @param max_tx_count the maximum number of txes to return
* @param max_size the maximum size of block/transaction data to return (will be exceeded by one blocks's worth at most, if min_count is met) * @param max_size the maximum size of block/transaction data to return (will be exceeded by one blocks's worth at most, if min_count is met)
* @param blocks the returned block/transaction data * @param blocks the returned block/transaction data
* @param pruned whether to return full or pruned tx data * @param pruned whether to return full or pruned tx data
* @param skip_coinbase whether to return or skip coinbase transactions (they're in blocks regardless) * @param skip_coinbase whether to return or skip coinbase transactions (they're in blocks regardless)
* @param get_miner_tx_hash whether to calculate and return the miner (coinbase) tx hash * @param get_miner_tx_hash whether to calculate and return the miner (coinbase) tx hash
* *
* The call will return at least min_block_count if possible, even if this contravenes max_tx_count
*
* @return true iff the blocks and transactions were found * @return true iff the blocks and transactions were found
*/ */
virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0; virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0;
/** /**
* @brief fetches the prunable transaction blob with the given hash * @brief fetches the prunable transaction blob with the given hash

11
src/blockchain_db/lmdb/db_lmdb.cpp
View File

@ -3171,7 +3171,7 @@ bool BlockchainLMDB::get_pruned_tx_blobs_from(const crypto::hash& h, size_t coun
return true; return true;
} }
bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const
{ {
LOG_PRINT_L3("BlockchainLMDB::" << __func__); LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open(); check_open();
@ -3185,14 +3185,15 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
RCURSOR(txs_prunable); RCURSOR(txs_prunable);
} }
blocks.reserve(std::min<size_t>(max_count, 10000)); // guard against very large max count if only checking bytes blocks.reserve(std::min<size_t>(max_block_count, 10000)); // guard against very large max count if only checking bytes
const uint64_t blockchain_height = height(); const uint64_t blockchain_height = height();
uint64_t size = 0; uint64_t size = 0;
size_t num_txes = 0;
MDB_val_copy<uint64_t> key(start_height); MDB_val_copy<uint64_t> key(start_height);
MDB_val k, v, val_tx_id; MDB_val k, v, val_tx_id;
uint64_t tx_id = ~0; uint64_t tx_id = ~0;
MDB_cursor_op op = MDB_SET; MDB_cursor_op op = MDB_SET;
for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_count && (size < max_size || blocks.size() < min_count); ++h) for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_block_count && (size < max_size || blocks.size() < min_block_count); ++h)
{ {
MDB_cursor_op op = h == start_height ? MDB_SET : MDB_NEXT; MDB_cursor_op op = h == start_height ? MDB_SET : MDB_NEXT;
int result = mdb_cursor_get(m_cur_blocks, &key, &v, op); int result = mdb_cursor_get(m_cur_blocks, &key, &v, op);
@ -3243,6 +3244,7 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
op = MDB_NEXT; op = MDB_NEXT;
current_block.second.reserve(b.tx_hashes.size()); current_block.second.reserve(b.tx_hashes.size());
num_txes += b.tx_hashes.size() + (skip_coinbase ? 0 : 1);
for (const auto &tx_hash: b.tx_hashes) for (const auto &tx_hash: b.tx_hashes)
{ {
// get pruned data // get pruned data
@ -3262,6 +3264,9 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
current_block.second.push_back(std::make_pair(tx_hash, std::move(tx_blob))); current_block.second.push_back(std::make_pair(tx_hash, std::move(tx_blob)));
size += current_block.second.back().second.size(); size += current_block.second.back().second.size();
} }
if (blocks.size() >= min_block_count && num_txes >= max_tx_count)
break;
} }
TXN_POSTFIX_RDONLY(); TXN_POSTFIX_RDONLY();

2
src/blockchain_db/lmdb/db_lmdb.h
View File

@ -257,7 +257,7 @@ public:
virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const; virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const;
virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const; virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const;
virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const; virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const;

2
src/blockchain_db/testdb.h
View File

@ -70,7 +70,7 @@ public:
virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const override { return false; } virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const override { return false; }
virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; } virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; }
virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; } virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; }
virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; } virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; }

3
src/cryptonote_config.h
View File

@ -124,7 +124,8 @@
#define CRYPTONOTE_MAX_FRAGMENTS 20 // ~20 * NOISE_BYTES max payload size for covert/noise send #define CRYPTONOTE_MAX_FRAGMENTS 20 // ~20 * NOISE_BYTES max payload size for covert/noise send
#define COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT 1000 #define COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT 1000
#define COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT 20000
#define P2P_LOCAL_WHITE_PEERLIST_LIMIT 1000 #define P2P_LOCAL_WHITE_PEERLIST_LIMIT 1000
#define P2P_LOCAL_GRAY_PEERLIST_LIMIT 5000 #define P2P_LOCAL_GRAY_PEERLIST_LIMIT 5000

6
src/cryptonote_core/blockchain.cpp
View File

@ -2701,7 +2701,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc
// find split point between ours and foreign blockchain (or start at // find split point between ours and foreign blockchain (or start at
// blockchain height <req_start_block>), and return up to max_count FULL // blockchain height <req_start_block>), and return up to max_count FULL
// blocks by reference. // blocks by reference.
bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const
{ {
LOG_PRINT_L3("Blockchain::" << __func__); LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock); CRITICAL_REGION_LOCAL(m_blockchain_lock);
@ -2726,8 +2726,8 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons
db_rtxn_guard rtxn_guard(m_db); db_rtxn_guard rtxn_guard(m_db);
total_height = get_current_blockchain_height(); total_height = get_current_blockchain_height();
blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height))); blocks.reserve(std::min(std::min(max_block_count, (size_t)10000), (size_t)(total_height - start_height)));
CHECK_AND_ASSERT_MES(m_db->get_blocks_from(start_height, 3, max_count, FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE, blocks, pruned, true, get_miner_tx_hash), CHECK_AND_ASSERT_MES(m_db->get_blocks_from(start_height, 3, max_block_count, max_tx_count, FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE, blocks, pruned, true, get_miner_tx_hash),
false, "Error getting blocks"); false, "Error getting blocks");
return true; return true;

5
src/cryptonote_core/blockchain.h
View File

@ -465,11 +465,12 @@ namespace cryptonote
* @param total_height return-by-reference our current blockchain height * @param total_height return-by-reference our current blockchain height
* @param start_height return-by-reference the height of the first block returned * @param start_height return-by-reference the height of the first block returned
* @param pruned whether to return full or pruned tx blobs * @param pruned whether to return full or pruned tx blobs
* @param max_count the max number of blocks to get * @param max_block_count the max number of blocks to get
* @param max_tx_count the max number of txes to get (it can get overshot by the last block's number of txes minus 1)
* *
* @return true if a block found in common or req_start_block specified, else false * @return true if a block found in common or req_start_block specified, else false
*/ */
bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const;
/** /**
* @brief retrieves a set of blocks and their transactions, and possibly other transactions * @brief retrieves a set of blocks and their transactions, and possibly other transactions

4
src/cryptonote_core/cryptonote_core.cpp
View File

@ -1406,9 +1406,9 @@ namespace cryptonote
return m_blockchain_storage.find_blockchain_supplement(qblock_ids, clip_pruned, resp); return m_blockchain_storage.find_blockchain_supplement(qblock_ids, clip_pruned, resp);
} }
//----------------------------------------------------------------------------------------------- //-----------------------------------------------------------------------------------------------
bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const
{ {
return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count); return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_block_count, max_tx_count);
} }
//----------------------------------------------------------------------------------------------- //-----------------------------------------------------------------------------------------------
bool core::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const bool core::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const

2
src/cryptonote_core/cryptonote_core.h
View File

@ -564,7 +564,7 @@ namespace cryptonote
* *
* @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<transaction> > >&, uint64_t&, uint64_t&, size_t) const * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<transaction> > >&, uint64_t&, uint64_t&, size_t) const
*/ */
bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const;
/** /**
* @copydoc Blockchain::get_tx_outputs_gindexs * @copydoc Blockchain::get_tx_outputs_gindexs

8
src/rpc/core_rpc_server.cpp
View File

@ -565,12 +565,12 @@ namespace cryptonote
} }
} }
size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT; size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
if (m_rpc_payment) if (m_rpc_payment)
{ {
max_blocks = res.credits / COST_PER_BLOCK; max_blocks = res.credits / COST_PER_BLOCK;
if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT) if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT)
max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT; max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
if (max_blocks == 0) if (max_blocks == 0)
{ {
res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED; res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED;
@ -579,7 +579,7 @@ namespace cryptonote
} }
std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs; std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs;
if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks)) if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
{ {
res.status = "Failed"; res.status = "Failed";
add_host_fail(ctx); add_host_fail(ctx);

2
src/rpc/daemon_handler.cpp
View File

@ -128,7 +128,7 @@ namespace rpc
{ {
std::vector<std::pair<std::pair<blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, blobdata> > > > blocks; std::vector<std::pair<std::pair<blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, blobdata> > > > blocks;
if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
{ {
res.status = Message::STATUS_FAILED; res.status = Message::STATUS_FAILED;
res.error_details = "core::find_blockchain_supplement() returned false"; res.error_details = "core::find_blockchain_supplement() returned false";