Move txpool to the database

Integration could go further (ie, return_tx_to_pool calls should
not be needed anymore, possibly other things).

poolstate.bin is now obsolete.

src/blockchain_db/blockchain_db.h

@@ -124,6 +124,27 @@ struct tx_data_t
 };
 #pragma pack(pop)
 
+/**
+ * @brief a struct containing txpool per transaction metadata
+ */
+struct txpool_tx_meta_t
+{
+  crypto::hash max_used_block_id;
+  crypto::hash last_failed_id;
+  uint64_t blob_size;
+  uint64_t fee;
+  uint64_t max_used_block_height;
+  uint64_t last_failed_height;
+  uint64_t receive_time;
+  uint64_t last_relayed_time;
+  // 112 bytes
+  uint8_t kept_by_block;
+  uint8_t relayed;
+  uint8_t do_not_relay;
+
+  uint8_t padding[77]; // till 192 bytes
+};
+
 /***********************************
  * Exception Definitions
  ***********************************/
@@ -1251,6 +1272,71 @@ public:
    */
   virtual bool has_key_image(const crypto::key_image& img) const = 0;
 
+  /**
+   * @brief add a txpool transaction
+   *
+   * @param details the details of the transaction to add
+   */
+  virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) = 0;
+
+  /**
+   * @brief update a txpool transaction's metadata
+   *
+   * @param txid the txid of the transaction to update
+   * @param details the details of the transaction to update
+   */
+  virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& details) = 0;
+
+  /**
+   * @brief get the number of transactions in the txpool
+   */
+  virtual uint64_t get_txpool_tx_count() const = 0;
+
+  /**
+   * @brief check whether a txid is in the txpool
+   */
+  virtual bool txpool_has_tx(const crypto::hash &txid) const = 0;
+
+  /**
+   * @brief remove a txpool transaction
+   *
+   * @param txid the transaction id of the transation to remove
+   */
+  virtual void remove_txpool_tx(const crypto::hash& txid) = 0;
+
+  /**
+   * @brief get a txpool transaction's metadata
+   *
+   * @param txid the transaction id of the transation to lookup
+   *
+   * @return the metadata associated with that transaction
+   */
+  virtual txpool_tx_meta_t get_txpool_tx_meta(const crypto::hash& txid) const = 0;
+
+  /**
+   * @brief get a txpool transaction's blob
+   *
+   * @param txid the transaction id of the transation to lookup
+   *
+   * @return the blob for that transaction
+   */
+  virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const = 0;
+
+  /**
+   * @brief runs a function over all txpool transactions
+   *
+   * The subclass should run the passed function for each txpool tx it has
+   * stored, passing the tx id and metadata as its parameters.
+   *
+   * If any call to the function returns false, the subclass should return
+   * false.  Otherwise, the subclass returns true.
+   *
+   * @param std::function fn the function to run
+   *
+   * @return false if the function returns false for any transaction, otherwise true
+   */
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false) const = 0;
+
   /**
    * @brief runs a function over all key images stored
    *

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -169,6 +169,9 @@ int compare_string(const MDB_val *a, const MDB_val *b)
  *
  * spent_keys       input hash   -
  *
+ * txpool_meta      txn hash     txn metadata
+ * txpool_blob      txn hash     txn blob
+ *
  * Note: where the data items are of uniform size, DUPFIXED tables have
  * been used to save space. In most of these cases, a dummy "zerokval"
  * key is used when accessing the table; the Key listed above will be
@@ -189,6 +192,9 @@ const char* const LMDB_OUTPUT_TXS = "output_txs";
 const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts";
 const char* const LMDB_SPENT_KEYS = "spent_keys";
 
+const char* const LMDB_TXPOOL_META = "txpool_meta";
+const char* const LMDB_TXPOOL_BLOB = "txpool_blob";
+
 const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights";
 const char* const LMDB_HF_VERSIONS = "hf_versions";
 
@@ -1164,6 +1170,9 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
 
   lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_spent_keys, "Failed to open db handle for m_spent_keys");
 
+  lmdb_db_open(txn, LMDB_TXPOOL_META, MDB_CREATE, m_txpool_meta, "Failed to open db handle for m_txpool_meta");
+  lmdb_db_open(txn, LMDB_TXPOOL_BLOB, MDB_CREATE, m_txpool_blob, "Failed to open db handle for m_txpool_blob");
+
   // this subdb is dropped on sight, so it may not be present when we open the DB.
   // Since we use MDB_CREATE, we'll get an exception if we open read-only and it does not exist.
   // So we don't open for read-only, and also not drop below. It is not used elsewhere.
@@ -1181,6 +1190,8 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
   mdb_set_dupsort(txn, m_output_txs, compare_uint64);
   mdb_set_dupsort(txn, m_block_info, compare_uint64);
 
+  mdb_set_compare(txn, m_txpool_meta, compare_hash32);
+  mdb_set_compare(txn, m_txpool_blob, compare_hash32);
   mdb_set_compare(txn, m_properties, compare_string);
 
   if (!(mdb_flags & MDB_RDONLY))
@@ -1429,6 +1440,211 @@ void BlockchainLMDB::unlock()
       auto_txn.commit(); \
   } while(0)
 
+void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t &meta)
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  mdb_txn_cursors *m_cursors = &m_wcursors;
+
+  CURSOR(txpool_meta)
+  CURSOR(txpool_blob)
+
+  const crypto::hash txid = get_transaction_hash(tx);
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  MDB_val v = {sizeof(meta), (void *)&meta};
+  if (auto result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) {
+    if (result == MDB_KEYEXIST)
+      throw1(DB_ERROR("Attempting to add txpool tx metadata that's already in the db"));
+    else
+      throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str()));
+  }
+  MDB_val_copy<cryptonote::blobdata> blob_val(tx_to_blob(tx));
+  if (auto result = mdb_cursor_put(m_cur_txpool_blob, &k, &blob_val, MDB_NODUPDATA)) {
+    if (result == MDB_KEYEXIST)
+      throw1(DB_ERROR("Attempting to add txpool tx blob that's already in the db"));
+    else
+      throw1(DB_ERROR(lmdb_error("Error adding txpool tx blob to db transaction: ", result).c_str()));
+  }
+}
+
+void BlockchainLMDB::update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta)
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  mdb_txn_cursors *m_cursors = &m_wcursors;
+
+  CURSOR(txpool_meta)
+  CURSOR(txpool_blob)
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  MDB_val v;
+  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET);
+  if (result != 0)
+    throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta to update: ", result).c_str()));
+  result = mdb_cursor_del(m_cur_txpool_meta, 0);
+  if (result)
+    throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx metadata to db transaction: ", result).c_str()));
+  v = MDB_val({sizeof(meta), (void *)&meta});
+  if ((result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) != 0) {
+    if (result == MDB_KEYEXIST)
+      throw1(DB_ERROR("Attempting to add txpool tx metadata that's already in the db"));
+    else
+      throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str()));
+  }
+}
+
+uint64_t BlockchainLMDB::get_txpool_tx_count() const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  int result;
+
+  MDB_stat db_stats;
+  if ((result = mdb_stat(m_txn, m_txpool_meta, &db_stats)))
+    throw0(DB_ERROR(lmdb_error("Failed to query m_txpool_meta: ", result).c_str()));
+
+  TXN_POSTFIX_RDONLY();
+
+  return db_stats.ms_entries;
+}
+
+bool BlockchainLMDB::txpool_has_tx(const crypto::hash& txid) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(txpool_meta)
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, NULL, MDB_SET);
+  if (result != 0 && result != MDB_NOTFOUND)
+    throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str()));
+  TXN_POSTFIX_RDONLY();
+  return result != MDB_NOTFOUND;
+}
+
+void BlockchainLMDB::remove_txpool_tx(const crypto::hash& txid)
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  mdb_txn_cursors *m_cursors = &m_wcursors;
+
+  CURSOR(txpool_meta)
+  CURSOR(txpool_blob)
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, NULL, MDB_SET);
+  if (result != 0 && result != MDB_NOTFOUND)
+    throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta to remove: ", result).c_str()));
+  if (!result)
+  {
+    result = mdb_cursor_del(m_cur_txpool_meta, 0);
+    if (result)
+      throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx metadata to db transaction: ", result).c_str()));
+  }
+  result = mdb_cursor_get(m_cur_txpool_blob, &k, NULL, MDB_SET);
+  if (result != 0 && result != MDB_NOTFOUND)
+    throw1(DB_ERROR(lmdb_error("Error finding txpool tx blob to remove: ", result).c_str()));
+  if (!result)
+  {
+    result = mdb_cursor_del(m_cur_txpool_blob, 0);
+    if (result)
+      throw1(DB_ERROR(lmdb_error("Error adding removal of txpool tx blob to db transaction: ", result).c_str()));
+  }
+}
+
+txpool_tx_meta_t BlockchainLMDB::get_txpool_tx_meta(const crypto::hash& txid) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(txpool_meta)
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  MDB_val v;
+  auto result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, MDB_SET);
+  if (result != 0)
+      throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str()));
+
+  const txpool_tx_meta_t meta = *(const txpool_tx_meta_t*)v.mv_data;
+  TXN_POSTFIX_RDONLY();
+  return meta;
+}
+
+cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(txpool_blob)
+
+  MDB_val k = {sizeof(txid), (void *)&txid};
+  MDB_val v;
+  auto result = mdb_cursor_get(m_cur_txpool_blob, &k, &v, MDB_SET);
+  if (result != 0)
+      throw1(DB_ERROR(lmdb_error("Error finding txpool tx meta: ", result).c_str()));
+
+  blobdata bd;
+  bd.assign(reinterpret_cast<const char*>(v.mv_data), v.mv_size);
+  TXN_POSTFIX_RDONLY();
+  return bd;
+}
+
+bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(txpool_meta);
+  RCURSOR(txpool_blob);
+
+  MDB_val k;
+  MDB_val v;
+  bool ret = true;
+
+  MDB_cursor_op op = MDB_FIRST;
+  while (1)
+  {
+    int result = mdb_cursor_get(m_cur_txpool_meta, &k, &v, op);
+    op = MDB_NEXT;
+    if (result == MDB_NOTFOUND)
+      break;
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx metadata: ", result).c_str()));
+    const crypto::hash txid = *(const crypto::hash*)k.mv_data;
+    const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data;
+    const cryptonote::blobdata *passed_bd = NULL;
+    cryptonote::blobdata bd;
+    if (include_blob)
+    {
+      MDB_val b;
+      result = mdb_cursor_get(m_cur_txpool_blob, &k, &b, MDB_SET);
+      if (result == MDB_NOTFOUND)
+        throw0(DB_ERROR("Failed to find txpool tx blob to match metadata"));
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx blob: ", result).c_str()));
+      bd.assign(reinterpret_cast<const char*>(b.mv_data), b.mv_size);
+      passed_bd = &bd;
+    }
+
+    if (!f(txid, meta, passed_bd)) {
+      ret = false;
+      break;
+    }
+  }
+
+  TXN_POSTFIX_RDONLY();
+
+  return ret;
+}
+
 bool BlockchainLMDB::block_exists(const crypto::hash& h, uint64_t *height) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);

src/blockchain_db/lmdb/db_lmdb.h

@@ -55,6 +55,9 @@ typedef struct mdb_txn_cursors
 
   MDB_cursor *m_txc_spent_keys;
 
+  MDB_cursor *m_txc_txpool_meta;
+  MDB_cursor *m_txc_txpool_blob;
+
   MDB_cursor *m_txc_hf_versions;
 } mdb_txn_cursors;
 
@@ -67,6 +70,8 @@ typedef struct mdb_txn_cursors
 #define m_cur_tx_indices	m_cursors->m_txc_tx_indices
 #define m_cur_tx_outputs	m_cursors->m_txc_tx_outputs
 #define m_cur_spent_keys	m_cursors->m_txc_spent_keys
+#define m_cur_txpool_meta	m_cursors->m_txc_txpool_meta
+#define m_cur_txpool_blob	m_cursors->m_txc_txpool_blob
 #define m_cur_hf_versions	m_cursors->m_txc_hf_versions
 
 typedef struct mdb_rflags
@@ -81,6 +86,8 @@ typedef struct mdb_rflags
   bool m_rf_tx_indices;
   bool m_rf_tx_outputs;
   bool m_rf_spent_keys;
+  bool m_rf_txpool_meta;
+  bool m_rf_txpool_blob;
   bool m_rf_hf_versions;
 } mdb_rflags;
 
@@ -232,6 +239,15 @@ public:
 
   virtual bool has_key_image(const crypto::key_image& img) const;
 
+  virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& meta);
+  virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta);
+  virtual uint64_t get_txpool_tx_count() const;
+  virtual bool txpool_has_tx(const crypto::hash &txid) const;
+  virtual void remove_txpool_tx(const crypto::hash& txid);
+  virtual txpool_tx_meta_t get_txpool_tx_meta(const crypto::hash& txid) const;
+  virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const;
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false) const;
+
   virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
   virtual bool for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const;
   virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const;
@@ -364,6 +380,9 @@ private:
 
   MDB_dbi m_spent_keys;
 
+  MDB_dbi m_txpool_meta;
+  MDB_dbi m_txpool_blob;
+
   MDB_dbi m_hf_starting_heights;
   MDB_dbi m_hf_versions;
 

src/blockchain_utilities/blockchain_import.cpp

@@ -443,7 +443,7 @@ int import_from_file(FakeCore& simple_core, const std::string& import_file_path,
         // tx number 1: coinbase tx
         // tx number 2 onwards: archived_txs
         unsigned int tx_num = 1;
-        for (const transaction& tx : archived_txs)
+        for (transaction tx : archived_txs)
         {
           ++tx_num;
           // if tx is invalid

src/blockchain_utilities/fake_core.h

@@ -59,8 +59,6 @@ struct fake_core_db
   // for multi_db_runtime:
   fake_core_db(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const std::string& db_type="lmdb", const int db_flags=0) : m_pool(m_storage), m_storage(m_pool)
   {
-    m_pool.init(path.string());
-
     BlockchainDB* db = nullptr;
     if (db_type == "lmdb")
       db = new BlockchainLMDB();
@@ -98,6 +96,8 @@ struct fake_core_db
 
     m_storage.init(db, m_hardfork, use_testnet);
 
+    m_pool.init();
+
     if (do_batch)
       m_storage.get_db().set_batch_transactions(do_batch);
     support_batch = true;

src/cryptonote_core/blockchain.cpp

@@ -3047,7 +3047,7 @@ bool Blockchain::check_block_timestamp(const block& b) const
   return check_block_timestamp(timestamps, b);
 }
 //------------------------------------------------------------------
-void Blockchain::return_tx_to_pool(const std::vector<transaction> &txs)
+void Blockchain::return_tx_to_pool(std::vector<transaction> &txs)
 {
   uint8_t version = get_current_hard_fork_version();
   for (auto& tx : txs)
@@ -3565,7 +3565,7 @@ void Blockchain::block_longhash_worker(const uint64_t height, const std::vector<
 bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
 {
   MTRACE("Blockchain::" << __func__);
-  CRITICAL_REGION_LOCAL(m_blockchain_lock);
+  CRITICAL_REGION_BEGIN(m_blockchain_lock);
   TIME_MEASURE_START(t1);
 
   m_db->batch_stop();
@@ -3601,6 +3601,9 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
   m_blocks_txs_check.clear();
   m_check_txin_table.clear();
 
+  CRITICAL_REGION_END();
+  m_tx_pool.unlock();
+
   return true;
 }
 
@@ -3634,14 +3637,32 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e
   MTRACE("Blockchain::" << __func__);
   TIME_MEASURE_START(prepare);
   bool stop_batch;
-  CRITICAL_REGION_LOCAL(m_blockchain_lock);
+
+  // Order of locking must be:
+  //  m_incoming_tx_lock (optional)
+  //  m_tx_pool lock
+  //  blockchain lock
+  //
+  //  Something which takes the blockchain lock may never take the txpool lock
+  //  if it has not provably taken the txpool lock earlier
+  //
+  //  The txpool lock is now taken in prepare_handle_incoming_blocks
+  //  and released in cleanup_handle_incoming_blocks. This avoids issues
+  //  when something uses the pool, which now uses the blockchain and
+  //  needs a batch, since a batch could otherwise be active while the
+  //  txpool and blockchain locks were not held
+
+  m_tx_pool.lock();
+  CRITICAL_REGION_LOCAL1(m_blockchain_lock);
 
   if(blocks_entry.size() == 0)
     return false;
 
   while (!(stop_batch = m_db->batch_start(blocks_entry.size()))) {
     m_blockchain_lock.unlock();
+    m_tx_pool.unlock();
     epee::misc_utils::sleep_no_w(1000);
+    m_tx_pool.lock();
     m_blockchain_lock.lock();
   }
 
@@ -3957,6 +3978,41 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e
   return true;
 }
 
+void Blockchain::add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta)
+{
+  m_db->add_txpool_tx(tx, meta);
+}
+
+void Blockchain::update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta)
+{
+  m_db->update_txpool_tx(txid, meta);
+}
+
+void Blockchain::remove_txpool_tx(const crypto::hash &txid)
+{
+  m_db->remove_txpool_tx(txid);
+}
+
+uint64_t Blockchain::get_txpool_tx_count() const
+{
+  return m_db->get_txpool_tx_count();
+}
+
+txpool_tx_meta_t Blockchain::get_txpool_tx_meta(const crypto::hash& txid) const
+{
+  return m_db->get_txpool_tx_meta(txid);
+}
+
+cryptonote::blobdata Blockchain::get_txpool_tx_blob(const crypto::hash& txid) const
+{
+  return m_db->get_txpool_tx_blob(txid);
+}
+
+bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob) const
+{
+  return m_db->for_all_txpool_txes(f, include_blob);
+}
+
 void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, blockchain_db_sync_mode sync_mode, bool fast_sync)
 {
   m_db_sync_mode = sync_mode;
@@ -4071,6 +4127,8 @@ void Blockchain::load_compiled_in_block_hashes()
         // The core will not call check_tx_inputs(..) for these
         // transactions in this case. Consequently, the sanity check
         // for tx hashes will fail in handle_block_to_main_chain(..)
+        CRITICAL_REGION_LOCAL(m_tx_pool);
+
         std::list<transaction> txs;
         m_tx_pool.get_transactions(txs);
 
@@ -4089,6 +4147,16 @@ void Blockchain::load_compiled_in_block_hashes()
 }
 #endif
 
+void Blockchain::lock()
+{
+  m_blockchain_lock.lock();
+}
+
+void Blockchain::unlock()
+{
+  m_blockchain_lock.unlock();
+}
+
 bool Blockchain::for_all_key_images(std::function<bool(const crypto::key_image&)> f) const
 {
   return m_db->for_all_key_images(f);

src/cryptonote_core/blockchain.h

@@ -854,6 +854,17 @@ namespace cryptonote
      */
     std::list<std::pair<block_extended_info,uint64_t>> get_alternative_chains() const;
 
+    void add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta);
+    void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta);
+    void remove_txpool_tx(const crypto::hash &txid);
+    uint64_t get_txpool_tx_count() const;
+    txpool_tx_meta_t get_txpool_tx_meta(const crypto::hash& txid) const;
+    cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const;
+    bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false) const;
+
+    void lock();
+    void unlock();
+
     void cancel();
 
   private:
@@ -1238,7 +1249,7 @@ namespace cryptonote
      * @return true
      */
     bool update_next_cumulative_size_limit();
-    void return_tx_to_pool(const std::vector<transaction> &txs);
+    void return_tx_to_pool(std::vector<transaction> &txs);
 
     /**
      * @brief make sure a transaction isn't attempting a double-spend

src/cryptonote_core/cryptonote_core.cpp

@@ -272,9 +272,6 @@ namespace cryptonote
       m_config_folder_mempool = m_config_folder_mempool + "/" + m_port;
     }
 
-    r = m_mempool.init(m_fakechain ? std::string() : m_config_folder_mempool);
-    CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool");
-
     std::string db_type = command_line::get_arg(vm, command_line::arg_db_type);
     std::string db_sync_mode = command_line::get_arg(vm, command_line::arg_db_sync_mode);
     bool fast_sync = command_line::get_arg(vm, command_line::arg_fast_block_sync) != 0;
@@ -401,6 +398,9 @@ namespace cryptonote
 
     r = m_blockchain_storage.init(db, m_testnet, test_options);
 
+    r = m_mempool.init();
+    CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool");
+
     // now that we have a valid m_blockchain_storage, we can clean out any
     // transactions in the pool that do not conform to the current fork
     m_mempool.validate(m_blockchain_storage.get_current_hard_fork_version());
@@ -760,7 +760,7 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::add_new_tx(const transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
+  bool core::add_new_tx(transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
   {
     crypto::hash tx_hash = get_transaction_hash(tx);
     crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx);
@@ -774,7 +774,7 @@ namespace cryptonote
     return m_blockchain_storage.get_total_transactions();
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
+  bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
   {
     if(m_mempool.have_tx(tx_hash))
     {
@@ -795,17 +795,15 @@ namespace cryptonote
   bool core::relay_txpool_transactions()
   {
     // we attempt to relay txes that should be relayed, but were not
-    std::list<std::pair<crypto::hash, cryptonote::transaction>> txs;
+    std::list<std::pair<crypto::hash, cryptonote::blobdata>> txs;
     if (m_mempool.get_relayable_transactions(txs) && !txs.empty())
     {
       cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context);
       tx_verification_context tvc = AUTO_VAL_INIT(tvc);
       NOTIFY_NEW_TRANSACTIONS::request r;
-      blobdata bl;
       for (auto it = txs.begin(); it != txs.end(); ++it)
       {
-        t_serializable_object_to_blob(it->second, bl);
+        r.txs.push_back(it->second);
-        r.txs.push_back(bl);
       }
       get_protocol()->relay_transactions(r, fake_context);
       m_mempool.set_relayed(txs);
@@ -815,7 +813,7 @@ namespace cryptonote
   //-----------------------------------------------------------------------------------------------
   void core::on_transaction_relayed(const cryptonote::blobdata& tx_blob)
   {
-    std::list<std::pair<crypto::hash, cryptonote::transaction>> txs;
+    std::list<std::pair<crypto::hash, cryptonote::blobdata>> txs;
     cryptonote::transaction tx;
     crypto::hash tx_hash, tx_prefix_hash;
     if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash))
@@ -823,7 +821,7 @@ namespace cryptonote
       LOG_ERROR("Failed to parse relayed transaction");
       return;
     }
-    txs.push_back(std::make_pair(tx_hash, std::move(tx)));
+    txs.push_back(std::make_pair(tx_hash, std::move(tx_blob)));
     m_mempool.set_relayed(txs);
   }
   //-----------------------------------------------------------------------------------------------
@@ -893,9 +891,9 @@ namespace cryptonote
     bce.block = cryptonote::block_to_blob(b);
     for (const auto &tx_hash: b.tx_hashes)
     {
-      cryptonote::transaction tx;
+      cryptonote::blobdata txblob;
-      CHECK_AND_ASSERT_THROW_MES(pool.get_transaction(tx_hash, tx), "Transaction not found in pool");
+      CHECK_AND_ASSERT_THROW_MES(pool.get_transaction(tx_hash, txblob), "Transaction not found in pool");
-      bce.txs.push_back(cryptonote::tx_to_blob(tx));
+      bce.txs.push_back(txblob);
     }
     return bce;
   }
@@ -955,6 +953,7 @@ namespace cryptonote
   //-----------------------------------------------------------------------------------------------
   bool core::prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks)
   {
+    m_incoming_tx_lock.lock();
     m_blockchain_storage.prepare_handle_incoming_blocks(blocks);
     return true;
   }
@@ -962,7 +961,11 @@ namespace cryptonote
   //-----------------------------------------------------------------------------------------------
   bool core::cleanup_handle_incoming_blocks(bool force_sync)
   {
+    try {
       m_blockchain_storage.cleanup_handle_incoming_blocks(force_sync);
+    }
+    catch (...) {}
+    m_incoming_tx_lock.unlock();
     return true;
   }
 
@@ -1043,11 +1046,16 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::get_pool_transaction(const crypto::hash &id, transaction& tx) const
+  bool core::get_pool_transaction(const crypto::hash &id, cryptonote::blobdata& tx) const
   {
     return m_mempool.get_transaction(id, tx);
   }  
   //-----------------------------------------------------------------------------------------------
+  bool core::pool_has_tx(const crypto::hash &id) const
+  {
+    return m_mempool.have_tx(id);
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_pool_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const
   {
     return m_mempool.get_transactions_and_spent_keys_info(tx_infos, key_image_infos);

src/cryptonote_core/cryptonote_core.h

@@ -389,6 +389,13 @@ namespace cryptonote
       */
      void set_enforce_dns_checkpoints(bool enforce_dns);
 
+     /**
+      * @copydoc tx_memory_pool::have_tx
+      *
+      * @note see tx_memory_pool::have_tx
+      */
+     bool pool_has_tx(const crypto::hash &txid) const;
+
      /**
       * @copydoc tx_memory_pool::get_transactions
       *
@@ -408,7 +415,7 @@ namespace cryptonote
       *
       * @note see tx_memory_pool::get_transaction
       */
-     bool get_pool_transaction(const crypto::hash& id, transaction& tx) const;     
+     bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx) const;
 
      /**
       * @copydoc tx_memory_pool::get_pool_transactions_and_spent_keys_info
@@ -658,7 +665,7 @@ namespace cryptonote
    private:
 
      /**
-      * @copydoc add_new_tx(const transaction&, tx_verification_context&, bool)
+      * @copydoc add_new_tx(transaction&, tx_verification_context&, bool)
       *
       * @param tx_hash the transaction's hash
       * @param tx_prefix_hash the transaction prefix' hash
@@ -667,7 +674,7 @@ namespace cryptonote
       * @param do_not_relay whether to prevent the transaction from being relayed
       *
       */
-     bool add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
+     bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
 
      /**
       * @brief add a new transaction to the transaction pool
@@ -684,7 +691,7 @@ namespace cryptonote
       * is already in a block on the Blockchain, or is successfully added
       * to the transaction pool
       */
-     bool add_new_tx(const transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
+     bool add_new_tx(transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
 
      /**
       * @copydoc Blockchain::add_new_block

src/cryptonote_core/tx_pool.cpp

@@ -38,6 +38,7 @@
 #include "cryptonote_basic/cryptonote_boost_serialization.h"
 #include "cryptonote_config.h"
 #include "blockchain.h"
+#include "blockchain_db/blockchain_db.h"
 #include "common/boost_serialization_helper.h"
 #include "common/int-util.h"
 #include "misc_language.h"
@@ -64,7 +65,7 @@ namespace cryptonote
     float const ACCEPT_THRESHOLD = 1.0f;
 
     // a kind of increasing backoff within min/max bounds
-    time_t get_relay_delay(time_t now, time_t received)
+    uint64_t get_relay_delay(time_t now, time_t received)
     {
       time_t d = (now - received + MIN_RELAY_TIME) / MIN_RELAY_TIME * MIN_RELAY_TIME;
       if (d > MAX_RELAY_TIME)
@@ -81,6 +82,21 @@ namespace cryptonote
     {
       return get_min_block_size(version) * 125 / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
     }
+
+    // This class is meant to create a batch when none currently exists.
+    // If a batch exists, it can't be from another thread, since we can
+    // only be called with the txpool lock taken, and it is held during
+    // the whole prepare/handle/cleanup incoming block sequence.
+    class LockedTXN {
+    public:
+      LockedTXN(Blockchain &b): m_blockchain(b), m_batch(false) {
+        m_batch = m_blockchain.get_db().batch_start();
+      }
+      ~LockedTXN() { if (m_batch) { m_blockchain.get_db().batch_stop(); } }
+    private:
+      Blockchain &m_blockchain;
+      bool m_batch;
+    };
   }
   //---------------------------------------------------------------------------------
   //---------------------------------------------------------------------------------
@@ -89,7 +105,7 @@ namespace cryptonote
 
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::add_tx(const transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version)
+  bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version)
   {
     PERF_TIMER(add_tx);
     if (tx.version == 0)
@@ -187,29 +203,38 @@ namespace cryptonote
 
     crypto::hash max_used_block_id = null_hash;
     uint64_t max_used_block_height = 0;
-    tx_details txd;
+    cryptonote::txpool_tx_meta_t meta;
-    txd.tx = tx;
+    bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id, tvc, kept_by_block);
-    bool ch_inp_res = m_blockchain.check_tx_inputs(txd.tx, max_used_block_height, max_used_block_id, tvc, kept_by_block);
-    CRITICAL_REGION_LOCAL(m_transactions_lock);
     if(!ch_inp_res)
     {
       // if the transaction was valid before (kept_by_block), then it
       // may become valid again, so ignore the failed inputs check.
       if(kept_by_block)
       {
-        auto txd_p = m_transactions.insert(transactions_container::value_type(id, txd));
+        meta.blob_size = blob_size;
-        CHECK_AND_ASSERT_MES(txd_p.second, false, "transaction already exists at inserting in memory pool");
+        meta.fee = fee;
-        txd_p.first->second.blob_size = blob_size;
+        meta.max_used_block_id = null_hash;
-        txd_p.first->second.fee = fee;
+        meta.max_used_block_height = 0;
-        txd_p.first->second.max_used_block_id = null_hash;
+        meta.last_failed_height = 0;
-        txd_p.first->second.max_used_block_height = 0;
+        meta.last_failed_id = null_hash;
-        txd_p.first->second.last_failed_height = 0;
+        meta.kept_by_block = kept_by_block;
-        txd_p.first->second.last_failed_id = null_hash;
+        meta.receive_time = receive_time;
-        txd_p.first->second.kept_by_block = kept_by_block;
+        meta.last_relayed_time = time(NULL);
-        txd_p.first->second.receive_time = receive_time;
+        meta.relayed = relayed;
-        txd_p.first->second.last_relayed_time = time(NULL);
+        meta.do_not_relay = do_not_relay;
-        txd_p.first->second.relayed = relayed;
+        try
-        txd_p.first->second.do_not_relay = do_not_relay;
+        {
+          LockedTXN lock(m_blockchain);
+          m_blockchain.add_txpool_tx(tx, meta);
+          if (!insert_key_images(tx, kept_by_block))
+            return false;
+          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id);
+        }
+        catch (const std::exception &e)
+        {
+          MERROR("transaction already exists at inserting in memory pool: " << e.what());
+          return false;
+        }
         tvc.m_verifivation_impossible = true;
         tvc.m_added_to_pool = true;
       }else
@@ -221,30 +246,60 @@ namespace cryptonote
     }else
     {
       //update transactions container
-      auto txd_p = m_transactions.insert(transactions_container::value_type(id, txd));
+      meta.blob_size = blob_size;
-      CHECK_AND_ASSERT_MES(txd_p.second, false, "internal error: transaction already exists at inserting in memorypool");
+      meta.kept_by_block = kept_by_block;
-      txd_p.first->second.blob_size = blob_size;
+      meta.fee = fee;
-      txd_p.first->second.kept_by_block = kept_by_block;
+      meta.max_used_block_id = max_used_block_id;
-      txd_p.first->second.fee = fee;
+      meta.max_used_block_height = max_used_block_height;
-      txd_p.first->second.max_used_block_id = max_used_block_id;
+      meta.last_failed_height = 0;
-      txd_p.first->second.max_used_block_height = max_used_block_height;
+      meta.last_failed_id = null_hash;
-      txd_p.first->second.last_failed_height = 0;
+      meta.receive_time = receive_time;
-      txd_p.first->second.last_failed_id = null_hash;
+      meta.last_relayed_time = time(NULL);
-      txd_p.first->second.receive_time = receive_time;
+      meta.relayed = relayed;
-      txd_p.first->second.last_relayed_time = time(NULL);
+      meta.do_not_relay = do_not_relay;
-      txd_p.first->second.relayed = relayed;
+
-      txd_p.first->second.do_not_relay = do_not_relay;
+      try
+      {
+        LockedTXN lock(m_blockchain);
+        m_blockchain.remove_txpool_tx(get_transaction_hash(tx));
+        m_blockchain.add_txpool_tx(tx, meta);
+        if (!insert_key_images(tx, kept_by_block))
+          return false;
+        m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id);
+      }
+      catch (const std::exception &e)
+      {
+        MERROR("internal error: transaction already exists at inserting in memorypool: " << e.what());
+        return false;
+      }
       tvc.m_added_to_pool = true;
 
-      if(txd_p.first->second.fee > 0 && !do_not_relay)
+      if(meta.fee > 0 && !do_not_relay)
         tvc.m_should_be_relayed = true;
     }
 
     // assume failure during verification steps until success is certain
     tvc.m_verifivation_failed = true;
 
+    tvc.m_verifivation_failed = false;
+
+    MINFO("Transaction added to pool: txid " << id << " bytes: " << blob_size << " fee/byte: " << (fee / (double)blob_size));
+    return true;
+  }
+  //---------------------------------------------------------------------------------
+  bool tx_memory_pool::add_tx(transaction &tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay, uint8_t version)
+  {
+    crypto::hash h = null_hash;
+    size_t blob_size = 0;
+    get_transaction_hash(tx, h, blob_size);
+    return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version);
+  }
+  //---------------------------------------------------------------------------------
+  bool tx_memory_pool::insert_key_images(const transaction &tx, bool kept_by_block)
+  {
     for(const auto& in: tx.vin)
     {
+      const crypto::hash id = get_transaction_hash(tx);
       CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, txin, false);
       std::unordered_set<crypto::hash>& kei_image_set = m_spent_key_images[txin.k_image];
       CHECK_AND_ASSERT_MES(kept_by_block || kei_image_set.size() == 0, false, "internal error: kept_by_block=" << kept_by_block
@@ -253,29 +308,16 @@ namespace cryptonote
       auto ins_res = kei_image_set.insert(id);
       CHECK_AND_ASSERT_MES(ins_res.second, false, "internal error: try to insert duplicate iterator in key_image set");
     }
-
-    tvc.m_verifivation_failed = false;
-
-    m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id);
-
-    MINFO("Transaction added to pool: txid " << id << " bytes: " << blob_size << " fee/byte: " << (fee / (double)blob_size));
     return true;
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::add_tx(const transaction &tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay, uint8_t version)
-  {
-    crypto::hash h = null_hash;
-    size_t blob_size = 0;
-    get_transaction_hash(tx, h, blob_size);
-    return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version);
-  }
-  //---------------------------------------------------------------------------------
   //FIXME: Can return early before removal of all of the key images.
   //       At the least, need to make sure that a false return here
   //       is treated properly.  Should probably not return early, however.
   bool tx_memory_pool::remove_transaction_keyimages(const transaction& tx)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
     // ND: Speedup
     // 1. Move transaction hash calcuation outside of loop. ._.
     crypto::hash actual_hash = get_transaction_hash(tx);
@@ -306,22 +348,37 @@ namespace cryptonote
   bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    auto it = m_transactions.find(id);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
-    if(it == m_transactions.end())
-      return false;
 
     auto sorted_it = find_tx_in_sorted_container(id);
-
     if (sorted_it == m_txs_by_fee_and_receive_time.end())
       return false;
 
-    tx = it->second.tx;
+    try
-    blob_size = it->second.blob_size;
+    {
-    fee = it->second.fee;
+      LockedTXN lock(m_blockchain);
-    relayed = it->second.relayed;
+      txpool_tx_meta_t meta = m_blockchain.get_txpool_tx_meta(id);
-    do_not_relay = it->second.do_not_relay;
+      cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(id);
-    remove_transaction_keyimages(it->second.tx);
+      if (!parse_and_validate_tx_from_blob(txblob, tx))
-    m_transactions.erase(it);
+      {
+        MERROR("Failed to parse tx from txpool");
+        return false;
+      }
+      blob_size = meta.blob_size;
+      fee = meta.fee;
+      relayed = meta.relayed;
+      do_not_relay = meta.do_not_relay;
+
+      // remove first, in case this throws, so key images aren't removed
+      m_blockchain.remove_txpool_tx(id);
+      remove_transaction_keyimages(tx);
+    }
+    catch (const std::exception &e)
+    {
+      MERROR("Failed to remove tx from txpool: " << e.what());
+      return false;
+    }
+
     m_txs_by_fee_and_receive_time.erase(sorted_it);
     return true;
   }
@@ -344,68 +401,113 @@ namespace cryptonote
   bool tx_memory_pool::remove_stuck_transactions()
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    for(auto it = m_transactions.begin(); it!= m_transactions.end();)
+    CRITICAL_REGION_LOCAL1(m_blockchain);
-    {
+    std::unordered_set<crypto::hash> remove;
-      uint64_t tx_age = time(nullptr) - it->second.receive_time;
+    m_blockchain.for_all_txpool_txes([this, &remove](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) {
+      uint64_t tx_age = time(nullptr) - meta.receive_time;
 
-      if((tx_age > CRYPTONOTE_MEMPOOL_TX_LIVETIME && !it->second.kept_by_block) ||
+      if((tx_age > CRYPTONOTE_MEMPOOL_TX_LIVETIME && !meta.kept_by_block) ||
-         (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && it->second.kept_by_block) )
+         (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && meta.kept_by_block) )
       {
-        LOG_PRINT_L1("Tx " << it->first << " removed from tx pool due to outdated, age: " << tx_age );
+        LOG_PRINT_L1("Tx " << txid << " removed from tx pool due to outdated, age: " << tx_age );
-        remove_transaction_keyimages(it->second.tx);
+        auto sorted_it = find_tx_in_sorted_container(txid);
-        auto sorted_it = find_tx_in_sorted_container(it->first);
         if (sorted_it == m_txs_by_fee_and_receive_time.end())
         {
-          LOG_PRINT_L1("Removing tx " << it->first << " from tx pool, but it was not found in the sorted txs container!");
+          LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
         }
         else
         {
           m_txs_by_fee_and_receive_time.erase(sorted_it);
         }
-        m_timed_out_transactions.insert(it->first);
+        m_timed_out_transactions.insert(txid);
-        auto pit = it++;
+        remove.insert(txid);
-        m_transactions.erase(pit);
+      }
-      }else
+      return true;
-        ++it;
+    });
+
+    if (!remove.empty())
+    {
+      LockedTXN lock(m_blockchain);
+      for (const crypto::hash &txid: remove)
+      {
+        try
+        {
+          cryptonote::blobdata bd = m_blockchain.get_txpool_tx_blob(txid);
+          cryptonote::transaction tx;
+          if (!parse_and_validate_tx_from_blob(bd, tx))
+          {
+            MERROR("Failed to parse tx from txpool");
+            // continue
+          }
+          else
+          {
+            // remove first, so we only remove key images if the tx removal succeeds
+            m_blockchain.remove_txpool_tx(txid);
+            remove_transaction_keyimages(tx);
+          }
+        }
+        catch (const std::exception &e)
+        {
+          MWARNING("Failed to remove stuck transaction: " << txid);
+          // ignore error
+        }
+      }
     }
     return true;
   }
   //---------------------------------------------------------------------------------
   //TODO: investigate whether boolean return is appropriate
-  bool tx_memory_pool::get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::transaction>> &txs) const
+  bool tx_memory_pool::get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::blobdata>> &txs) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    const time_t now = time(NULL);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
-    for(auto it = m_transactions.begin(); it!= m_transactions.end();)
+    const uint64_t now = time(NULL);
-    {
+    m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){
       // 0 fee transactions are never relayed
-      if(it->second.fee > 0 && !it->second.do_not_relay && now - it->second.last_relayed_time > get_relay_delay(now, it->second.receive_time))
+      if(meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time))
       {
         // if the tx is older than half the max lifetime, we don't re-relay it, to avoid a problem
         // mentioned by smooth where nodes would flush txes at slightly different times, causing
         // flushed txes to be re-added when received from a node which was just about to flush it
-        time_t max_age = it->second.kept_by_block ? CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME : CRYPTONOTE_MEMPOOL_TX_LIVETIME;
+        uint64_t max_age = meta.kept_by_block ? CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME : CRYPTONOTE_MEMPOOL_TX_LIVETIME;
-        if (now - it->second.receive_time <= max_age / 2)
+        if (now - meta.receive_time <= max_age / 2)
         {
-          txs.push_back(std::make_pair(it->first, it->second.tx));
+          try
+          {
+            cryptonote::blobdata bd = m_blockchain.get_txpool_tx_blob(txid);
+            txs.push_back(std::make_pair(txid, bd));
+          }
+          catch (const std::exception &e)
+          {
+            MERROR("Failed to get transaction blob from db");
+            // ignore error
           }
         }
-      ++it;
       }
       return true;
+    });
+    return true;
   }
   //---------------------------------------------------------------------------------
-  void tx_memory_pool::set_relayed(const std::list<std::pair<crypto::hash, cryptonote::transaction>> &txs)
+  void tx_memory_pool::set_relayed(const std::list<std::pair<crypto::hash, cryptonote::blobdata>> &txs)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
     const time_t now = time(NULL);
+    LockedTXN lock(m_blockchain);
     for (auto it = txs.begin(); it != txs.end(); ++it)
     {
-      auto i = m_transactions.find(it->first);
+      try
-      if (i != m_transactions.end())
       {
-        i->second.relayed = true;
+        txpool_tx_meta_t meta = m_blockchain.get_txpool_tx_meta(it->first);
-        i->second.last_relayed_time = now;
+        meta.relayed = true;
+        meta.last_relayed_time = now;
+        m_blockchain.update_txpool_tx(it->first, meta);
+      }
+      catch (const std::exception &e)
+      {
+        MERROR("Failed to update txpool transaction metadata: " << e.what());
+        // continue
       }
     }
   }
@@ -413,46 +515,67 @@ namespace cryptonote
   size_t tx_memory_pool::get_transactions_count() const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    return m_transactions.size();
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+    return m_blockchain.get_txpool_tx_count();
   }
   //---------------------------------------------------------------------------------
   void tx_memory_pool::get_transactions(std::list<transaction>& txs) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    for(const auto& tx_vt: m_transactions)
+    CRITICAL_REGION_LOCAL1(m_blockchain);
-      txs.push_back(tx_vt.second.tx);
+    m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){
+      transaction tx;
+      if (!parse_and_validate_tx_from_blob(*bd, tx))
+      {
+        MERROR("Failed to parse tx from txpool");
+        // continue
+        return true;
+      }
+      txs.push_back(tx);
+      return true;
+    }, true);
   }
   //------------------------------------------------------------------
   void tx_memory_pool::get_transaction_hashes(std::vector<crypto::hash>& txs) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    for(const auto& tx_vt: m_transactions)
+    CRITICAL_REGION_LOCAL1(m_blockchain);
-      txs.push_back(get_transaction_hash(tx_vt.second.tx));
+    m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){
+      txs.push_back(txid);
+      return true;
+    });
   }
   //------------------------------------------------------------------
   //TODO: investigate whether boolean return is appropriate
   bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    for (const auto& tx_vt : m_transactions)
+    CRITICAL_REGION_LOCAL1(m_blockchain);
-    {
+    m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){
       tx_info txi;
-      const tx_details& txd = tx_vt.second;
+      txi.id_hash = epee::string_tools::pod_to_hex(txid);
-      txi.id_hash = epee::string_tools::pod_to_hex(tx_vt.first);
+      transaction tx;
-      txi.tx_json = obj_to_json_str(*const_cast<transaction*>(&txd.tx));
+      if (!parse_and_validate_tx_from_blob(*bd, tx))
-      txi.blob_size = txd.blob_size;
+      {
-      txi.fee = txd.fee;
+        MERROR("Failed to parse tx from txpool");
-      txi.kept_by_block = txd.kept_by_block;
+        // continue
-      txi.max_used_block_height = txd.max_used_block_height;
+        return true;
-      txi.max_used_block_id_hash = epee::string_tools::pod_to_hex(txd.max_used_block_id);
-      txi.last_failed_height = txd.last_failed_height;
-      txi.last_failed_id_hash = epee::string_tools::pod_to_hex(txd.last_failed_id);
-      txi.receive_time = txd.receive_time;
-      txi.relayed = txd.relayed;
-      txi.last_relayed_time = txd.last_relayed_time;
-      txi.do_not_relay = txd.do_not_relay;
-      tx_infos.push_back(txi);
       }
+      txi.tx_json = obj_to_json_str(tx);
+      txi.blob_size = meta.blob_size;
+      txi.fee = meta.fee;
+      txi.kept_by_block = meta.kept_by_block;
+      txi.max_used_block_height = meta.max_used_block_height;
+      txi.max_used_block_id_hash = epee::string_tools::pod_to_hex(meta.max_used_block_id);
+      txi.last_failed_height = meta.last_failed_height;
+      txi.last_failed_id_hash = epee::string_tools::pod_to_hex(meta.last_failed_id);
+      txi.receive_time = meta.receive_time;
+      txi.relayed = meta.relayed;
+      txi.last_relayed_time = meta.last_relayed_time;
+      txi.do_not_relay = meta.do_not_relay;
+      tx_infos.push_back(txi);
+      return true;
+    }, true);
 
     for (const key_images_container::value_type& kee : m_spent_key_images) {
       const crypto::key_image& k_image = kee.first;
@@ -468,15 +591,20 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::get_transaction(const crypto::hash& id, transaction& tx) const
+  bool tx_memory_pool::get_transaction(const crypto::hash& id, cryptonote::blobdata& txblob) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    auto it = m_transactions.find(id);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
-    if(it == m_transactions.end())
+    try
-      return false;
+    {
-    tx = it->second.tx;
+      txblob = m_blockchain.get_txpool_tx_blob(id);
       return true;
     }
+    catch (const std::exception &e)
+    {
+      return false;
+    }
+  }
   //---------------------------------------------------------------------------------
   bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id)
   {
@@ -491,14 +619,14 @@ namespace cryptonote
   bool tx_memory_pool::have_tx(const crypto::hash &id) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    if(m_transactions.count(id))
+    CRITICAL_REGION_LOCAL1(m_blockchain);
-      return true;
+    return m_blockchain.get_db().txpool_has_tx(id);
-    return false;
   }
   //---------------------------------------------------------------------------------
   bool tx_memory_pool::have_tx_keyimges_as_spent(const transaction& tx) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
     for(const auto& in: tx.vin)
     {
       CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail
@@ -524,7 +652,7 @@ namespace cryptonote
     m_transactions_lock.unlock();
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::is_transaction_ready_to_go(tx_details& txd) const
+  bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const
   {
     //not the best implementation at this time, sorry :(
     //check is ring_signature already checked ?
@@ -535,7 +663,7 @@ namespace cryptonote
         return false;//we already sure that this tx is broken for this height
 
       tx_verification_context tvc;
-      if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id, tvc))
+      if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc))
       {
         txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1;
         txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height);
@@ -552,7 +680,7 @@ namespace cryptonote
           return false;
         //check ring signature again, it is possible (with very small chance) that this transaction become again valid
         tx_verification_context tvc;
-        if(!m_blockchain.check_tx_inputs(txd.tx, txd.max_used_block_height, txd.max_used_block_id, tvc))
+        if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc))
         {
           txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1;
           txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height);
@@ -561,7 +689,7 @@ namespace cryptonote
       }
     }
     //if we here, transaction seems valid, but, anyway, check for key_images collisions with blockchain, just to be sure
-    if(m_blockchain.have_tx_keyimges_as_spent(txd.tx))
+    if(m_blockchain.have_tx_keyimges_as_spent(tx))
       return false;
 
     //transaction is ok.
@@ -594,20 +722,27 @@ namespace cryptonote
   {
     std::stringstream ss;
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    for (const transactions_container::value_type& txe : m_transactions) {
+    CRITICAL_REGION_LOCAL1(m_blockchain);
-      const tx_details& txd = txe.second;
+    m_blockchain.for_all_txpool_txes([&ss, short_format](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *txblob) {
-      ss << "id: " << txe.first << std::endl;
+      ss << "id: " << txid << std::endl;
       if (!short_format) {
-        ss << obj_to_json_str(*const_cast<transaction*>(&txd.tx)) << std::endl;
+        cryptonote::transaction tx;
+        if (!parse_and_validate_tx_from_blob(*txblob, tx))
+        {
+          MERROR("Failed to parse tx from txpool");
+          return true; // continue
         }
-      ss << "blob_size: " << txd.blob_size << std::endl
+        ss << obj_to_json_str(tx) << std::endl;
-        << "fee: " << print_money(txd.fee) << std::endl
-        << "kept_by_block: " << (txd.kept_by_block ? 'T' : 'F') << std::endl
-        << "max_used_block_height: " << txd.max_used_block_height << std::endl
-        << "max_used_block_id: " << txd.max_used_block_id << std::endl
-        << "last_failed_height: " << txd.last_failed_height << std::endl
-        << "last_failed_id: " << txd.last_failed_id << std::endl;
       }
+      ss << "blob_size: " << meta.blob_size << std::endl
+        << "fee: " << print_money(meta.fee) << std::endl
+        << "kept_by_block: " << (meta.kept_by_block ? 'T' : 'F') << std::endl
+        << "max_used_block_height: " << meta.max_used_block_height << std::endl
+        << "max_used_block_id: " << meta.max_used_block_id << std::endl
+        << "last_failed_height: " << meta.last_failed_height << std::endl
+        << "last_failed_id: " << meta.last_failed_id << std::endl;
+      return true;
+    }, !short_format);
 
     return ss.str();
   }
@@ -620,6 +755,7 @@ namespace cryptonote
     // with it.
 
     CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
 
     uint64_t best_coinbase = 0, coinbase = 0;
     total_size = 0;
@@ -638,11 +774,11 @@ namespace cryptonote
     auto sorted_it = m_txs_by_fee_and_receive_time.begin();
     while (sorted_it != m_txs_by_fee_and_receive_time.end())
     {
-      auto tx_it = m_transactions.find(sorted_it->second);
+      txpool_tx_meta_t meta = m_blockchain.get_txpool_tx_meta(sorted_it->second);
-      LOG_PRINT_L2("Considering " << tx_it->first << ", size " << tx_it->second.blob_size << ", current block size " << total_size << "/" << max_total_size << ", current coinbase " << print_money(best_coinbase));
+      LOG_PRINT_L2("Considering " << sorted_it->second << ", size " << meta.blob_size << ", current block size " << total_size << "/" << max_total_size << ", current coinbase " << print_money(best_coinbase));
 
       // Can not exceed maximum block size
-      if (max_total_size < total_size + tx_it->second.blob_size)
+      if (max_total_size < total_size + meta.blob_size)
       {
         LOG_PRINT_L2("  would exceed maximum block size");
         sorted_it++;
@@ -655,13 +791,13 @@ namespace cryptonote
         // If we're getting lower coinbase tx,
         // stop including more tx
         uint64_t block_reward;
-        if(!get_block_reward(median_size, total_size + tx_it->second.blob_size, already_generated_coins, block_reward, version))
+        if(!get_block_reward(median_size, total_size + meta.blob_size, already_generated_coins, block_reward, version))
         {
           LOG_PRINT_L2("  would exceed maximum block size");
           sorted_it++;
           continue;
         }
-        coinbase = block_reward + fee + tx_it->second.fee;
+        coinbase = block_reward + fee + meta.fee;
         if (coinbase < template_accept_threshold(best_coinbase))
         {
           LOG_PRINT_L2("  would decrease coinbase to " << print_money(coinbase));
@@ -680,27 +816,36 @@ namespace cryptonote
         }
       }
 
+      cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(sorted_it->second);
+      cryptonote::transaction tx;
+      if (!parse_and_validate_tx_from_blob(txblob, tx))
+      {
+        MERROR("Failed to parse tx from txpool");
+        sorted_it++;
+        continue;
+      }
+
       // Skip transactions that are not ready to be
       // included into the blockchain or that are
       // missing key images
-      if (!is_transaction_ready_to_go(tx_it->second))
+      if (!is_transaction_ready_to_go(meta, tx))
       {
         LOG_PRINT_L2("  not ready to go");
         sorted_it++;
         continue;
       }
-      if (have_key_images(k_images, tx_it->second.tx))
+      if (have_key_images(k_images, tx))
       {
         LOG_PRINT_L2("  key images already seen");
         sorted_it++;
         continue;
       }
 
-      bl.tx_hashes.push_back(tx_it->first);
+      bl.tx_hashes.push_back(sorted_it->second);
-      total_size += tx_it->second.blob_size;
+      total_size += meta.blob_size;
-      fee += tx_it->second.fee;
+      fee += meta.fee;
       best_coinbase = coinbase;
-      append_key_images(k_images, tx_it->second.tx);
+      append_key_images(k_images, tx);
       sorted_it++;
       LOG_PRINT_L2("  added, new block size " << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase));
     }
@@ -714,21 +859,40 @@ namespace cryptonote
   size_t tx_memory_pool::validate(uint8_t version)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    size_t n_removed = 0;
+    CRITICAL_REGION_LOCAL1(m_blockchain);
     size_t tx_size_limit = get_transaction_size_limit(version);
-    for (auto it = m_transactions.begin(); it != m_transactions.end(); ) {
+    std::unordered_set<crypto::hash> remove;
-      bool remove = false;
+
-      const crypto::hash &txid = get_transaction_hash(it->second.tx);
+    m_blockchain.for_all_txpool_txes([this, &remove, tx_size_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) {
-      if (it->second.blob_size >= tx_size_limit) {
+      if (meta.blob_size >= tx_size_limit) {
-        LOG_PRINT_L1("Transaction " << txid << " is too big (" << it->second.blob_size << " bytes), removing it from pool");
+        LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.blob_size << " bytes), removing it from pool");
-        remove = true;
+        remove.insert(txid);
       }
       else if (m_blockchain.have_tx(txid)) {
         LOG_PRINT_L1("Transaction " << txid << " is in the blockchain, removing it from pool");
-        remove = true;
+        remove.insert(txid);
       }
-      if (remove) {
+      return true;
-        remove_transaction_keyimages(it->second.tx);
+    });
+
+    size_t n_removed = 0;
+    if (!remove.empty())
+    {
+      LockedTXN lock(m_blockchain);
+      for (const crypto::hash &txid: remove)
+      {
+        try
+        {
+          cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid);
+          cryptonote::transaction tx;
+          if (!parse_and_validate_tx_from_blob(txblob, tx))
+          {
+            MERROR("Failed to parse tx from txpool");
+            continue;
+          }
+          // remove tx from db first
+          m_blockchain.remove_txpool_tx(txid);
+          remove_transaction_keyimages(tx);
           auto sorted_it = find_tx_in_sorted_container(txid);
           if (sorted_it == m_txs_by_fee_and_receive_time.end())
           {
@@ -738,79 +902,45 @@ namespace cryptonote
           {
             m_txs_by_fee_and_receive_time.erase(sorted_it);
           }
-        auto pit = it++;
-        m_transactions.erase(pit);
           ++n_removed;
-        continue;
         }
-      it++;
+        catch (const std::exception &e)
+        {
+          MERROR("Failed to remove invalid tx from pool");
+          // continue
+        }
+      }
     }
     return n_removed;
   }
   //---------------------------------------------------------------------------------
-  //TODO: investigate whether only ever returning true is correct
+  bool tx_memory_pool::init()
-  bool tx_memory_pool::init(const std::string& config_folder)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
 
-    m_config_folder = config_folder;
-    if (m_config_folder.empty())
-      return true;
-
-    std::string state_file_path = config_folder + "/" + CRYPTONOTE_POOLDATA_FILENAME;
-    boost::system::error_code ec;
-    if(!boost::filesystem::exists(state_file_path, ec))
-      return true;
-    bool res = tools::unserialize_obj_from_file(*this, state_file_path);
-    if(!res)
-    {
-      LOG_ERROR("Failed to load memory pool from file " << state_file_path);
-
-      m_transactions.clear();
     m_txs_by_fee_and_receive_time.clear();
     m_spent_key_images.clear();
-    }
+    return m_blockchain.for_all_txpool_txes([this](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd) {
-
+      cryptonote::transaction tx;
-    // no need to store queue of sorted transactions, as it's easy to generate.
+      if (!parse_and_validate_tx_from_blob(*bd, tx))
-    for (const auto& tx : m_transactions)
       {
-      m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(tx.second.fee / (double)tx.second.blob_size, tx.second.receive_time), tx.first);
+        MERROR("Failed to parse tx from txpool");
+        return false;
       }
-
+      if (!insert_key_images(tx, meta.kept_by_block))
-    // Ignore deserialization error
+      {
+        MFATAL("Failed to insert key images from txpool tx");
+        return false;
+      }
+      m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.blob_size, meta.receive_time), txid);
       return true;
+    }, true);
   }
 
   //---------------------------------------------------------------------------------
-  //TODO: investigate whether only ever returning true is correct
   bool tx_memory_pool::deinit()
   {
-    LOG_PRINT_L1("Received signal to deactivate memory pool store");
-
-    if (m_config_folder.empty())
-    {
-      LOG_PRINT_L1("Memory pool store already empty");
     return true;
   }
-
-    if (!tools::create_directories_if_necessary(m_config_folder))
-    {
-      LOG_ERROR("Failed to create memory pool data directory: " << m_config_folder);
-      return false;
-    }
-
-    std::string state_file_path = m_config_folder + "/" + CRYPTONOTE_POOLDATA_FILENAME;
-    bool res = tools::serialize_obj_to_file(*this, state_file_path);
-    if(!res)
-    {
-      LOG_ERROR("Failed to serialize memory pool to file " << state_file_path);
-      return false;
-    }
-    else
-    {
-      LOG_PRINT_L1("Memory pool store deactivated successfully");
-      return true;
-    }
-
-  }
 }

src/cryptonote_core/tx_pool.h

@@ -43,6 +43,7 @@
 #include "math_helper.h"
 #include "cryptonote_basic/cryptonote_basic_impl.h"
 #include "cryptonote_basic/verification_context.h"
+#include "blockchain_db/blockchain_db.h"
 #include "crypto/hash.h"
 #include "rpc/core_rpc_server_commands_defs.h"
 
@@ -100,12 +101,12 @@ namespace cryptonote
 
 
     /**
-     * @copydoc add_tx(const transaction&, tx_verification_context&, bool, bool, uint8_t)
+     * @copydoc add_tx(transaction&, tx_verification_context&, bool, bool, uint8_t)
      *
      * @param id the transaction's hash
      * @param blob_size the transaction's size
      */
-    bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version);
+    bool add_tx(transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version);
 
     /**
      * @brief add a transaction to the transaction pool
@@ -124,7 +125,7 @@ namespace cryptonote
      *
      * @return true if the transaction passes validations, otherwise false
      */
-    bool add_tx(const transaction &tx, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version);
+    bool add_tx(transaction &tx, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version);
 
     /**
      * @brief takes a transaction with the given hash from the pool
@@ -199,7 +200,7 @@ namespace cryptonote
      *
      * @return true
      */
-    bool init(const std::string& config_folder);
+    bool init();
 
     /**
      * @brief attempts to save the transaction pool state to disk
@@ -256,11 +257,11 @@ namespace cryptonote
      * @brief get a specific transaction from the pool
      *
      * @param h the hash of the transaction to get
-     * @param tx return-by-reference the transaction requested
+     * @param tx return-by-reference the transaction blob requested
      *
      * @return true if the transaction is found, otherwise false
      */
-    bool get_transaction(const crypto::hash& h, transaction& tx) const;
+    bool get_transaction(const crypto::hash& h, cryptonote::blobdata& txblob) const;
 
     /**
      * @brief get a list of all relayable transactions and their hashes
@@ -274,14 +275,14 @@ namespace cryptonote
      *
      * @return true
      */
-    bool get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::transaction>>& txs) const;
+    bool get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::blobdata>>& txs) const;
 
     /**
      * @brief tell the pool that certain transactions were just relayed
      *
      * @param txs the list of transactions (and their hashes)
      */
-    void set_relayed(const std::list<std::pair<crypto::hash, cryptonote::transaction>>& txs);
+    void set_relayed(const std::list<std::pair<crypto::hash, cryptonote::blobdata>>& txs);
 
     /**
      * @brief get the total number of transactions in the pool
@@ -316,28 +317,6 @@ namespace cryptonote
 #define CURRENT_MEMPOOL_ARCHIVE_VER    11
 #define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER    12
 
-    /**
-     * @brief serialize the transaction pool to/from disk
-     *
-     * If the archive version passed is older than the version compiled
-     * in, this function does nothing, as it cannot deserialize after a
-     * format change.
-     *
-     * @tparam archive_t the archive class
-     * @param a the archive to serialize to/from
-     * @param version the archive version
-     */
-    template<class archive_t>
-    void serialize(archive_t & a, const unsigned int version)
-    {
-      if(version < CURRENT_MEMPOOL_ARCHIVE_VER )
-        return;
-      CRITICAL_REGION_LOCAL(m_transactions_lock);
-      a & m_transactions;
-      a & m_spent_key_images;
-      a & m_timed_out_transactions;
-    }
-
     /**
      * @brief information about a single transaction
      */
@@ -378,6 +357,13 @@ namespace cryptonote
 
   private:
 
+    /**
+     * @brief insert key images into m_spent_key_images
+     *
+     * @return true on success, false on error
+     */
+    bool insert_key_images(const transaction &tx, bool kept_by_block);
+
     /**
      * @brief remove old transactions from the pool
      *
@@ -452,10 +438,7 @@ namespace cryptonote
      *
      * @return true if the transaction is good to go, otherwise false
      */
-    bool is_transaction_ready_to_go(tx_details& txd) const;
+    bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const;
-
-    //! map transactions (and related info) by their hashes
-    typedef std::unordered_map<crypto::hash, tx_details > transactions_container;
 
     //TODO: confirm the below comments and investigate whether or not this
     //      is the desired behavior
@@ -472,7 +455,6 @@ namespace cryptonote
 public:
 #endif
     mutable epee::critical_section m_transactions_lock;  //!< lock for the pool
-    transactions_container m_transactions;  //!< container for transactions in the pool
 #if defined(DEBUG_CREATE_BLOCK_TEMPLATE)
 private:
 #endif

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -479,7 +479,7 @@ namespace cryptonote
           
           // we might already have the tx that the peer
           // sent in our pool, so don't verify again..
-          if(!m_core.get_pool_transaction(tx_hash, tx))
+          if(!m_core.pool_has_tx(tx_hash))
           {
             MDEBUG("Incoming tx " << tx_hash << " not in pool, adding");
             cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc);                        
@@ -536,9 +536,10 @@ namespace cryptonote
       size_t tx_idx = 0;
       for(auto& tx_hash: new_block.tx_hashes)
       {
-        if(m_core.get_pool_transaction(tx_hash, tx))
+        cryptonote::blobdata txblob;
+        if(m_core.get_pool_transaction(tx_hash, txblob))
         {
-          have_tx.push_back(tx_to_blob(tx));
+          have_tx.push_back(txblob);
         }
         else
         {
@@ -731,6 +732,15 @@ namespace cryptonote
     if(context.m_state != cryptonote_connection_context::state_normal)
       return 1;
 
+    // while syncing, core will lock for a long time, so we ignore
+    // those txes as they aren't really needed anyway, and avoid a
+    // long block before replying
+    if(!is_synchronized())
+    {
+      LOG_DEBUG_CC(context, "Received new tx while syncing, ignored");
+      return 1;
+    }
+
     for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end();)
     {
       cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc);

src/rpc/core_rpc_server.cpp

@@ -1608,13 +1608,13 @@ namespace cryptonote
       }
       crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data());
 
-      cryptonote::transaction tx;
+      cryptonote::blobdata txblob;
-      bool r = m_core.get_pool_transaction(txid, tx);
+      bool r = m_core.get_pool_transaction(txid, txblob);
       if (r)
       {
         cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context);
         NOTIFY_NEW_TRANSACTIONS::request r;
-        r.txs.push_back(cryptonote::tx_to_blob(tx));
+        r.txs.push_back(txblob);
         m_core.get_protocol()->relay_transactions(r, fake_context);
         //TODO: make sure that tx has reached other nodes here, probably wait to receive reflections from other nodes
       }

tests/core_proxy/core_proxy.h

@@ -90,7 +90,8 @@ namespace tests
     size_t get_block_sync_size() const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; }
     virtual void on_transaction_relayed(const cryptonote::blobdata& tx) {}
     bool get_testnet() const { return false; }
-    bool get_pool_transaction(const crypto::hash& id, cryptonote::transaction& tx) const { return false; }
+    bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; }
+    bool pool_has_tx(const crypto::hash &txid) const { return false; }
     bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata, cryptonote::block>>& blocks, std::list<cryptonote::blobdata>& txs) const { return false; }
     bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<cryptonote::transaction>& txs, std::list<crypto::hash>& missed_txs) const { return false; }
     bool get_block_by_hash(const crypto::hash &h, cryptonote::block &blk, bool *orphan = NULL) const { return false; }

tests/core_tests/chaingen.h

@@ -472,6 +472,16 @@ inline bool do_replay_events(std::vector<test_event_entry>& events)
     MERROR("Failed to init core");
     return false;
   }
+
+  // start with a clean pool
+  std::vector<crypto::hash> pool_txs;
+  if (!c.get_pool_transaction_hashes(pool_txs))
+  {
+    MERROR("Failed to flush txpool");
+    return false;
+  }
+  c.get_blockchain_storage().flush_txes_from_pool(std::list<crypto::hash>(pool_txs.begin(), pool_txs.end()));
+
   t_test_class validator;
   bool ret = replay_events_through_core<t_test_class>(c, events, validator);
   c.deinit();

tests/unit_tests/ban.cpp

@@ -65,7 +65,8 @@ public:
   size_t get_block_sync_size() const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; }
   virtual void on_transaction_relayed(const cryptonote::blobdata& tx) {}
   bool get_testnet() const { return false; }
-  bool get_pool_transaction(const crypto::hash& id, cryptonote::transaction& tx) const { return false; }
+  bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; }
+  bool pool_has_tx(const crypto::hash &txid) const { return false; }
   bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata, cryptonote::block>>& blocks, std::list<cryptonote::blobdata>& txs) const { return false; }
   bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<cryptonote::transaction>& txs, std::list<crypto::hash>& missed_txs) const { return false; }
   bool get_block_by_hash(const crypto::hash &h, cryptonote::block &blk, bool *orphan = NULL) const { return false; }

tests/unit_tests/hardfork.cpp

@@ -112,6 +112,15 @@ public:
   virtual bool is_read_only() const { return false; }
   virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff) const { return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>(); }
 
+  virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) {}
+  virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& details) {}
+  virtual uint64_t get_txpool_tx_count() const { return 0; }
+  virtual bool txpool_has_tx(const crypto::hash &txid) const { return false; }
+  virtual void remove_txpool_tx(const crypto::hash& txid) {}
+  virtual txpool_tx_meta_t get_txpool_tx_meta(const crypto::hash& txid) const { return txpool_tx_meta_t(); }
+  virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const { return ""; }
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false) const { return false; }
+
   virtual void add_block( const block& blk
                         , const size_t& block_size
                         , const difficulty_type& cumulative_difficulty