2018-01-07 06:05:16 +01:00
// Copyright (c) 2014-2018, The Monero Project
2014-07-23 15:03:52 +02:00
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
2014-03-03 23:07:58 +01:00
# pragma once
# include <boost/program_options/variables_map.hpp>
2017-01-26 16:07:23 +01:00
# include "cryptonote_basic/cryptonote_basic_impl.h"
# include "cryptonote_basic/verification_context.h"
2014-03-03 23:07:58 +01:00
# include <unordered_map>
namespace tests
{
struct block_index {
size_t height ;
crypto : : hash id ;
crypto : : hash longhash ;
cryptonote : : block blk ;
cryptonote : : blobdata blob ;
std : : list < cryptonote : : transaction > txes ;
2017-09-10 18:35:59 +02:00
block_index ( ) : height ( 0 ) , id ( crypto : : null_hash ) , longhash ( crypto : : null_hash ) { }
2014-03-03 23:07:58 +01:00
block_index ( size_t _height , const crypto : : hash & _id , const crypto : : hash & _longhash , const cryptonote : : block & _blk , const cryptonote : : blobdata & _blob , const std : : list < cryptonote : : transaction > & _txes )
: height ( _height ) , id ( _id ) , longhash ( _longhash ) , blk ( _blk ) , blob ( _blob ) , txes ( _txes ) { }
} ;
class proxy_core
{
cryptonote : : block m_genesis ;
std : : list < crypto : : hash > m_known_block_list ;
std : : unordered_map < crypto : : hash , block_index > m_hash2blkidx ;
crypto : : hash m_lastblk ;
std : : list < cryptonote : : transaction > txes ;
bool add_block ( const crypto : : hash & _id , const crypto : : hash & _longhash , const cryptonote : : block & _blk , const cryptonote : : blobdata & _blob ) ;
void build_short_history ( std : : list < crypto : : hash > & m_history , const crypto : : hash & m_start ) ;
public :
void on_synchronized ( ) { }
2017-08-22 15:54:18 +02:00
void safesyncmode ( const bool ) { }
2014-03-03 23:07:58 +01:00
uint64_t get_current_blockchain_height ( ) { return 1 ; }
2014-06-04 22:50:13 +02:00
void set_target_blockchain_height ( uint64_t ) { }
2014-03-03 23:07:58 +01:00
bool init ( const boost : : program_options : : variables_map & vm ) ;
bool deinit ( ) { return true ; }
bool get_short_chain_history ( std : : list < crypto : : hash > & ids ) ;
bool get_stat_info ( cryptonote : : core_stat_info & st_inf ) { return true ; }
bool have_block ( const crypto : : hash & id ) ;
2017-09-09 13:06:24 +02:00
void get_blockchain_top ( uint64_t & height , crypto : : hash & top_id ) ;
2017-01-14 14:01:21 +01:00
bool handle_incoming_tx ( const cryptonote : : blobdata & tx_blob , cryptonote : : tx_verification_context & tvc , bool keeped_by_block , bool relayed , bool do_not_relay ) ;
2018-04-16 01:16:02 +02:00
bool handle_incoming_txs ( const std : : vector < cryptonote : : blobdata > & tx_blobs , std : : vector < cryptonote : : tx_verification_context > & tvc , bool keeped_by_block , bool relayed , bool do_not_relay ) ;
2014-03-03 23:07:58 +01:00
bool handle_incoming_block ( const cryptonote : : blobdata & block_blob , cryptonote : : block_verification_context & bvc , bool update_miner_blocktemplate = true ) ;
void pause_mine ( ) { }
void resume_mine ( ) { }
bool on_idle ( ) { return true ; }
bool find_blockchain_supplement ( const std : : list < crypto : : hash > & qblock_ids , cryptonote : : NOTIFY_RESPONSE_CHAIN_ENTRY : : request & resp ) { return true ; }
bool handle_get_objects ( cryptonote : : NOTIFY_REQUEST_GET_OBJECTS : : request & arg , cryptonote : : NOTIFY_RESPONSE_GET_OBJECTS : : request & rsp , cryptonote : : cryptonote_connection_context & context ) { return true ; }
2016-05-26 22:48:18 +02:00
cryptonote : : Blockchain & get_blockchain_storage ( ) { throw std : : runtime_error ( " Called invalid member function: please never call get_blockchain_storage on the TESTING class proxy_core. " ) ; }
2015-02-24 20:12:56 +01:00
bool get_test_drop_download ( ) { return true ; }
bool get_test_drop_download_height ( ) { return true ; }
2018-04-16 01:16:02 +02:00
bool prepare_handle_incoming_blocks ( const std : : vector < cryptonote : : block_complete_entry > & blocks ) { return true ; }
** CHANGES ARE EXPERIMENTAL (FOR TESTING ONLY)
Bockchain:
1. Optim: Multi-thread long-hash computation when encountering groups of blocks.
2. Optim: Cache verified txs and return result from cache instead of re-checking whenever possible.
3. Optim: Preload output-keys when encoutering groups of blocks. Sort by amount and global-index before bulk querying database and multi-thread when possible.
4. Optim: Disable double spend check on block verification, double spend is already detected when trying to add blocks.
5. Optim: Multi-thread signature computation whenever possible.
6. Patch: Disable locking (recursive mutex) on called functions from check_tx_inputs which causes slowdowns (only seems to happen on ubuntu/VMs??? Reason: TBD)
7. Optim: Removed looped full-tx hash computation when retrieving transactions from pool (???).
8. Optim: Cache difficulty/timestamps (735 blocks) for next-difficulty calculations so that only 2 db reads per new block is needed when a new block arrives (instead of 1470 reads).
Berkeley-DB:
1. Fix: 32-bit data errors causing wrong output global indices and failure to send blocks to peers (etc).
2. Fix: Unable to pop blocks on reorganize due to transaction errors.
3. Patch: Large number of transaction aborts when running multi-threaded bulk queries.
4. Patch: Insufficient locks error when running full sync.
5. Patch: Incorrect db stats when returning from an immediate exit from "pop block" operation.
6. Optim: Add bulk queries to get output global indices.
7. Optim: Modified output_keys table to store public_key+unlock_time+height for single transaction lookup (vs 3)
8. Optim: Used output_keys table retrieve public_keys instead of going through output_amounts->output_txs+output_indices->txs->output:public_key
9. Optim: Added thread-safe buffers used when multi-threading bulk queries.
10. Optim: Added support for nosync/write_nosync options for improved performance (*see --db-sync-mode option for details)
11. Mod: Added checkpoint thread and auto-remove-logs option.
12. *Now usable on 32-bit systems like RPI2.
LMDB:
1. Optim: Added custom comparison for 256-bit key tables (minor speed-up, TBD: get actual effect)
2. Optim: Modified output_keys table to store public_key+unlock_time+height for single transaction lookup (vs 3)
3. Optim: Used output_keys table retrieve public_keys instead of going through output_amounts->output_txs+output_indices->txs->output:public_key
4. Optim: Added support for sync/writemap options for improved performance (*see --db-sync-mode option for details)
5. Mod: Auto resize to +1GB instead of multiplier x1.5
ETC:
1. Minor optimizations for slow-hash for ARM (RPI2). Incomplete.
2. Fix: 32-bit saturation bug when computing next difficulty on large blocks.
[PENDING ISSUES]
1. Berkely db has a very slow "pop-block" operation. This is very noticeable on the RPI2 as it sometimes takes > 10 MINUTES to pop a block during reorganization.
This does not happen very often however, most reorgs seem to take a few seconds but it possibly depends on the number of outputs present. TBD.
2. Berkeley db, possible bug "unable to allocate memory". TBD.
[NEW OPTIONS] (*Currently all enabled for testing purposes)
1. --fast-block-sync arg=[0:1] (default: 1)
a. 0 = Compute long hash per block (may take a while depending on CPU)
b. 1 = Skip long-hash and verify blocks based on embedded known good block hashes (faster, minimal CPU dependence)
2. --db-sync-mode arg=[[safe|fast|fastest]:[sync|async]:[nblocks_per_sync]] (default: fastest:async:1000)
a. safe = fdatasync/fsync (or equivalent) per stored block. Very slow, but safest option to protect against power-out/crash conditions.
b. fast/fastest = Enables asynchronous fdatasync/fsync (or equivalent). Useful for battery operated devices or STABLE systems with UPS and/or systems with battery backed write cache/solid state cache.
Fast - Write meta-data but defer data flush.
Fastest - Defer meta-data and data flush.
Sync - Flush data after nblocks_per_sync and wait.
Async - Flush data after nblocks_per_sync but do not wait for the operation to finish.
3. --prep-blocks-threads arg=[n] (default: 4 or system max threads, whichever is lower)
Max number of threads to use when computing long-hash in groups.
4. --show-time-stats arg=[0:1] (default: 1)
Show benchmark related time stats.
5. --db-auto-remove-logs arg=[0:1] (default: 1)
For berkeley-db only. Auto remove logs if enabled.
**Note: lmdb and berkeley-db have changes to the tables and are not compatible with official git head version.
At the moment, you need a full resync to use this optimized version.
[PERFORMANCE COMPARISON]
**Some figures are approximations only.
Using a baseline machine of an i7-2600K+SSD+(with full pow computation):
1. The optimized lmdb/blockhain core can process blocks up to 585K for ~1.25 hours + download time, so it usually takes 2.5 hours to sync the full chain.
2. The current head with memory can process blocks up to 585K for ~4.2 hours + download time, so it usually takes 5.5 hours to sync the full chain.
3. The current head with lmdb can process blocks up to 585K for ~32 hours + download time and usually takes 36 hours to sync the full chain.
Averate procesing times (with full pow computation):
lmdb-optimized:
1. tx_ave = 2.5 ms / tx
2. block_ave = 5.87 ms / block
memory-official-repo:
1. tx_ave = 8.85 ms / tx
2. block_ave = 19.68 ms / block
lmdb-official-repo (0f4a036437fd41a5498ee5e74e2422ea6177aa3e)
1. tx_ave = 47.8 ms / tx
2. block_ave = 64.2 ms / block
**Note: The following data denotes processing times only (does not include p2p download time)
lmdb-optimized processing times (with full pow computation):
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 1.25 hours processing time (--db-sync-mode=fastest:async:1000).
2. Laptop, Dual-core / 4-threads U4200 (3Mb) - 4.90 hours processing time (--db-sync-mode=fastest:async:1000).
3. Embedded, Quad-core / 4-threads Z3735F (2x1Mb) - 12.0 hours processing time (--db-sync-mode=fastest:async:1000).
lmdb-optimized processing times (with per-block-checkpoint)
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 10 minutes processing time (--db-sync-mode=fastest:async:1000).
berkeley-db optimized processing times (with full pow computation)
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 1.8 hours processing time (--db-sync-mode=fastest:async:1000).
2. RPI2. Improved from estimated 3 months(???) into 2.5 days (*Need 2AMP supply + Clock:1Ghz + [usb+ssd] to achieve this speed) (--db-sync-mode=fastest:async:1000).
berkeley-db optimized processing times (with per-block-checkpoint)
1. RPI2. 12-15 hours (*Need 2AMP supply + Clock:1Ghz + [usb+ssd] to achieve this speed) (--db-sync-mode=fastest:async:1000).
2015-07-10 22:09:32 +02:00
bool cleanup_handle_incoming_blocks ( bool force_sync = false ) { return true ; }
2015-12-14 01:36:37 +01:00
uint64_t get_target_blockchain_height ( ) const { return 1 ; }
2017-08-16 22:19:53 +02:00
size_t get_block_sync_size ( uint64_t height ) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT ; }
2016-10-22 21:46:51 +02:00
virtual void on_transaction_relayed ( const cryptonote : : blobdata & tx ) { }
2018-02-16 12:04:04 +01:00
cryptonote : : network_type get_nettype ( ) const { return cryptonote : : MAINNET ; }
2017-05-14 15:06:55 +02:00
bool get_pool_transaction ( const crypto : : hash & id , cryptonote : : blobdata & tx_blob ) const { return false ; }
bool pool_has_tx ( const crypto : : hash & txid ) const { return false ; }
2018-04-16 01:16:02 +02:00
bool get_blocks ( uint64_t start_offset , size_t count , std : : vector < std : : pair < cryptonote : : blobdata , cryptonote : : block > > & blocks , std : : vector < cryptonote : : blobdata > & txs ) const { return false ; }
bool get_transactions ( const std : : vector < crypto : : hash > & txs_ids , std : : vector < cryptonote : : transaction > & txs , std : : vector < crypto : : hash > & missed_txs ) const { return false ; }
2017-02-11 20:38:18 +01:00
bool get_block_by_hash ( const crypto : : hash & h , cryptonote : : block & blk , bool * orphan = NULL ) const { return false ; }
2018-04-16 04:02:26 +02:00
uint8_t get_ideal_hard_fork_version ( ) const { return 0 ; }
2017-08-10 11:10:04 +02:00
uint8_t get_ideal_hard_fork_version ( uint64_t height ) const { return 0 ; }
uint8_t get_hard_fork_version ( uint64_t height ) const { return 0 ; }
2018-03-20 12:30:09 +01:00
uint64_t get_earliest_ideal_height_for_version ( uint8_t version ) const { return 0 ; }
2017-08-12 16:24:39 +02:00
cryptonote : : difficulty_type get_block_cumulative_difficulty ( uint64_t height ) const { return 0 ; }
2017-09-02 18:11:20 +02:00
bool fluffy_blocks_enabled ( ) const { return false ; }
2018-04-16 01:16:02 +02:00
uint64_t prevalidate_block_hashes ( uint64_t height , const std : : vector < crypto : : hash > & hashes ) { return 0 ; }
2018-11-02 23:27:25 +01:00
bool pad_transactions ( ) const { return false ; }
2014-03-03 23:07:58 +01:00
} ;
}