LMDB should now dynamically resize the mapsize

Some filesystems (*cough* NTFS *cough*) aren't good with sparse files,
so this makes LMDB dynamically resize its mapsize as needed.  Note: the
check interval is currently every 10 blocks (for testing) and will
probably need to change to 1000 or something.  Default mapsize set to
1GiB.

Blockchain conversion tools using batching will probably segfault, I'll
fix that in the next commit.

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -148,74 +148,149 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi
 
 namespace cryptonote
 {
-  mdb_txn_safe::mdb_txn_safe() : m_txn(NULL) { }
+std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0};
+std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT;
 
-  mdb_txn_safe::~mdb_txn_safe()
+mdb_txn_safe::mdb_txn_safe() : m_txn(NULL)
+{
+  while (creation_gate.test_and_set());
+  num_active_txns++;
+  creation_gate.clear();
+}
+
+mdb_txn_safe::~mdb_txn_safe()
+{
+  LOG_PRINT_L3("mdb_txn_safe: destructor");
+  if (m_txn != NULL)
   {
-    LOG_PRINT_L3("mdb_txn_safe: destructor");
+    if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
-    if (m_txn != NULL)
     {
-      if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
+      LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
-      {
-        LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
-      }
-      else
-      {
-        // Example of when this occurs: a lookup fails, so a read-only txn is
-        // aborted through this destructor. However, successful read-only txns
-        // ideally should have been committed when done and not end up here.
-        //
-        // NOTE: not sure if this is ever reached for a non-batch write
-        // transaction, but it's probably not ideal if it did.
-        LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
-      }
-      mdb_txn_abort(m_txn);
-    }
-  }
-
-  void mdb_txn_safe::commit(std::string message)
-  {
-    if (message.size() == 0)
-    {
-      message = "Failed to commit a transaction to the db";
-    }
-
-    if (mdb_txn_commit(m_txn))
-    {
-      m_txn = NULL;
-      LOG_PRINT_L0(message);
-      throw DB_ERROR(message.c_str());
-    }
-    m_txn = NULL;
-  }
-
-  void mdb_txn_safe::abort()
-  {
-    LOG_PRINT_L3("mdb_txn_safe: abort()");
-    if(m_txn != NULL)
-    {
-      mdb_txn_abort(m_txn);
-      m_txn = NULL;
     }
     else
     {
-      LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL");
+      // Example of when this occurs: a lookup fails, so a read-only txn is
+      // aborted through this destructor. However, successful read-only txns
+      // ideally should have been committed when done and not end up here.
+      //
+      // NOTE: not sure if this is ever reached for a non-batch write
+      // transaction, but it's probably not ideal if it did.
+      LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
+    }
+    mdb_txn_abort(m_txn);
+  }
+  num_active_txns--;
+}
+
+void mdb_txn_safe::commit(std::string message)
+{
+  if (message.size() == 0)
+  {
+    message = "Failed to commit a transaction to the db";
+  }
+
+  if (mdb_txn_commit(m_txn))
+  {
+    m_txn = NULL;
+    LOG_PRINT_L0(message);
+    throw DB_ERROR(message.c_str());
+  }
+  m_txn = NULL;
+}
+
+void mdb_txn_safe::abort()
+{
+  LOG_PRINT_L3("mdb_txn_safe: abort()");
+  if(m_txn != NULL)
+  {
+    mdb_txn_abort(m_txn);
+    m_txn = NULL;
+  }
+  else
+  {
+    LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL");
+  }
+}
+
+uint64_t mdb_txn_safe::num_active_tx()
+{
+  return num_active_txns;
+}
+
+void mdb_txn_safe::prevent_new_txns()
+{
+  while (creation_gate.test_and_set());
+}
+
+void mdb_txn_safe::wait_no_active_txns()
+{
+  while (num_active_txns > 0);
+}
+
+void mdb_txn_safe::allow_new_txns()
+{
+  creation_gate.clear();
+}
+
+
+
+void BlockchainLMDB::do_resize()
+{
+  MDB_envinfo mei;
+
+  mdb_env_info(m_env, &mei);
+
+  MDB_stat mst;
+
+  mdb_env_stat(m_env, &mst);
+
+  uint64_t new_mapsize = (double)mei.me_mapsize * RESIZE_FACTOR;
+
+  new_mapsize += (new_mapsize % mst.ms_psize);
+
+  mdb_txn_safe::prevent_new_txns();
+
+  if (m_write_txn != nullptr)
+  {
+    if (m_batch_active)
+    {
+      throw0(DB_ERROR("lmdb resizing not yet supported when batch transactions enabled!"));
+    }
+    else
+    {
+      throw0(DB_ERROR("attempting resize with write transaction in progress, this should not happen!"));
     }
   }
 
-// If m_batch_active is set, a batch transaction exists beyond this class, such
+  mdb_txn_safe::wait_no_active_txns();
-// as a batch import with verification enabled, or possibly (later) a batch
+
-// network sync.
+  mdb_env_set_mapsize(m_env, new_mapsize);
-//
+
-// For some of the lookup methods, such as get_block_timestamp(), tx_exists(),
+  LOG_PRINT_L0("LMDB Mapsize increased."
-// and get_tx(), when m_batch_active is set, the lookup uses the batch
+      << "  Old: " << mei.me_mapsize / (1024 * 1024) << "MiB"
-// transaction. This isn't only because the transaction is available, but it's
+      << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
-// necessary so that lookups include the database updates only present in the
+
-// current batch write.
+  mdb_txn_safe::allow_new_txns();
-//
+}
-// A regular network sync without batch writes is expected to open a new read
+
-// transaction, as those lookups are part of the validation done prior to the
+bool BlockchainLMDB::need_resize()
-// write for block and tx data, so no write transaction is open at the time.
+{
+  MDB_envinfo mei;
+
+  mdb_env_info(m_env, &mei);
+
+  MDB_stat mst;
+
+  mdb_env_stat(m_env, &mst);
+
+  uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
+
+  if ((double)size_used / mei.me_mapsize  > 0.8)
+  {
+    return true;
+  }
+  return false;
+}
 
 void BlockchainLMDB::add_block( const block& blk
               , const size_t& block_size
@@ -726,7 +801,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
   if (mdb_env_set_maxdbs(m_env, 20))
     throw0(DB_ERROR("Failed to set max number of dbs"));
 
-  size_t mapsize = 1LL << 34;
+  size_t mapsize = DEFAULT_MAPSIZE;
   if (auto result = mdb_env_set_mapsize(m_env, mapsize))
     throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str()));
   if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644))
@@ -1710,16 +1785,23 @@ void BlockchainLMDB::batch_start()
     throw0(DB_ERROR("batch transactions not enabled"));
   if (m_batch_active)
     throw0(DB_ERROR("batch transaction already in progress"));
+  if (m_write_batch_txn != nullptr)
+    throw0(DB_ERROR("batch transaction already in progress"));
   if (m_write_txn)
     throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use"));
   check_open();
+
+  if (m_write_batch_txn == nullptr)
+  {
+    m_write_batch_txn = new mdb_txn_safe();
+  }
   // NOTE: need to make sure it's destroyed properly when done
-  if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn))
+  if (mdb_txn_begin(m_env, NULL, 0, *m_write_batch_txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
   // indicates this transaction is for batch transactions, but not whether it's
   // active
-  m_write_batch_txn.m_batch_txn = true;
+  m_write_batch_txn->m_batch_txn = true;
-  m_write_txn = &m_write_batch_txn;
+  m_write_txn = m_write_batch_txn;
   m_batch_active = true;
   LOG_PRINT_L3("batch transaction: begin");
 }
@@ -1731,7 +1813,10 @@ void BlockchainLMDB::batch_commit()
     throw0(DB_ERROR("batch transactions not enabled"));
   if (! m_batch_active)
     throw0(DB_ERROR("batch transaction not in progress"));
+  if (m_write_batch_txn == nullptr)
+    throw0(DB_ERROR("batch transaction not in progress"));
   check_open();
+
   LOG_PRINT_L3("batch transaction: committing...");
   TIME_MEASURE_START(time1);
   m_write_txn->commit();
@@ -1739,11 +1824,8 @@ void BlockchainLMDB::batch_commit()
   time_commit1 += time1;
   LOG_PRINT_L3("batch transaction: committed");
 
-  if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn))
+  m_write_txn = nullptr;
-    throw0(DB_ERROR("Failed to create a transaction for the db"));
+  delete m_write_batch_txn;
-  if (! m_write_batch_txn.m_batch_txn)
-    throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction"));
-  m_write_txn = &m_write_batch_txn;
 }
 
 void BlockchainLMDB::batch_stop()
@@ -1753,6 +1835,8 @@ void BlockchainLMDB::batch_stop()
     throw0(DB_ERROR("batch transactions not enabled"));
   if (! m_batch_active)
     throw0(DB_ERROR("batch transaction not in progress"));
+  if (m_write_batch_txn == nullptr)
+    throw0(DB_ERROR("batch transaction not in progress"));
   check_open();
   LOG_PRINT_L3("batch transaction: committing...");
   TIME_MEASURE_START(time1);
@@ -1761,6 +1845,7 @@ void BlockchainLMDB::batch_stop()
   time_commit1 += time1;
   // for destruction of batch transaction
   m_write_txn = nullptr;
+  delete m_write_batch_txn;
   m_batch_active = false;
   LOG_PRINT_L3("batch transaction: end");
 }
@@ -1776,7 +1861,7 @@ void BlockchainLMDB::batch_abort()
   // for destruction of batch transaction
   m_write_txn = nullptr;
   // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called.
-  m_write_batch_txn.abort();
+  m_write_batch_txn->abort();
   m_batch_active = false;
   LOG_PRINT_L3("batch transaction: aborted");
 }
@@ -1798,6 +1883,15 @@ uint64_t BlockchainLMDB::add_block( const block& blk
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
+  if (m_height % 10 == 0)
+  {
+    if (need_resize())
+    {
+      LOG_PRINT_L0("LMDB memory map needs resized, doing that now.");
+      do_resize();
+    }
+  }
+
   mdb_txn_safe txn;
   if (! m_batch_active)
   {

src/blockchain_db/lmdb/db_lmdb.h

@@ -26,6 +26,8 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #pragma once
 
+#include <atomic>
+
 #include "blockchain_db/blockchain_db.h"
 #include "cryptonote_protocol/blobdatatype.h" // for type blobdata
 
@@ -57,11 +59,34 @@ struct mdb_txn_safe
     return &m_txn;
   }
 
+  uint64_t num_active_tx();
+
+  static void prevent_new_txns();
+  static void wait_no_active_txns();
+  static void allow_new_txns();
+
   MDB_txn* m_txn;
   bool m_batch_txn = false;
+  static std::atomic<uint64_t> num_active_txns;
+
+  // could use a mutex here, but this should be sufficient.
+  static std::atomic_flag creation_gate;
 };
 
 
+// If m_batch_active is set, a batch transaction exists beyond this class, such
+// as a batch import with verification enabled, or possibly (later) a batch
+// network sync.
+//
+// For some of the lookup methods, such as get_block_timestamp(), tx_exists(),
+// and get_tx(), when m_batch_active is set, the lookup uses the batch
+// transaction. This isn't only because the transaction is available, but it's
+// necessary so that lookups include the database updates only present in the
+// current batch write.
+//
+// A regular network sync without batch writes is expected to open a new read
+// transaction, as those lookups are part of the validation done prior to the
+// write for block and tx data, so no write transaction is open at the time.
 class BlockchainLMDB : public BlockchainDB
 {
 public:
@@ -174,6 +199,10 @@ public:
   virtual void pop_block(block& blk, std::vector<transaction>& txs);
 
 private:
+  void do_resize();
+
+  bool need_resize();
+
   virtual void add_block( const block& blk
                 , const size_t& block_size
                 , const difficulty_type& cumulative_difficulty
@@ -258,10 +287,14 @@ private:
   uint64_t m_num_outputs;
   std::string m_folder;
   mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
-  mdb_txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
+  mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
 
   bool m_batch_transactions; // support for batch transactions
   bool m_batch_active; // whether batch transaction is in progress
+
+  constexpr static uint64_t DEFAULT_MAPSIZE = 1 << 30;
+  constexpr static float RESIZE_PERCENT = 0.8f;
+  constexpr static float RESIZE_FACTOR = 1.5f;
 };
 
 }  // namespace cryptonote