Merge pull request #765

d5d46e6 tests: obligatory hardfork unit build fix after interface change (moneromooo-monero)
25672d3 wallet: pass std::function by const ref, not value (moneromooo-monero)
0be6e08 wallet: do not leak owned amounts to the daemon unless --trusted-daemon (moneromooo-monero)
12146da wallet: change sweep_dust to sweep_unmixable (moneromooo-monero)
600a3cf New RPC and daemon command to get output histogram (moneromooo-monero)
f9a2fd2 wallet: handle rare case where fee adjustment can bump to the next kB (moneromooo-monero)
f26651a wallet: factor fee calculation (moneromooo-monero)

src/blockchain_db/berkeleydb/db_bdb.cpp

@@ -2180,6 +2180,12 @@ void BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const std::v
     LOG_PRINT_L3("db3: " << db3);
 }
 
+std::map<uint64_t, uint64_t>::BlockchainBDB::get_output_histogram(const std::vector<uint64_t> &amounts) const
+{
+  LOG_PRINT_L3("BlockchainBDB::" << __func__);
+  throw1(DB_ERROR("Not implemented."));
+}
+
 void BlockchainBDB::set_hard_fork_starting_height(uint8_t version, uint64_t height)
 {
     LOG_PRINT_L3("BlockchainBDB::" << __func__);

src/blockchain_db/berkeleydb/db_bdb.h

@@ -341,6 +341,15 @@ public:
   virtual bool can_thread_bulk_indices() const { return false; }
 #endif
 
+  /**
+   * @brief return a histogram of outputs on the blockchain
+   *
+   * @param amounts optional set of amounts to lookup
+   *
+   * @return a set of amount/instances
+   */
+  std::map<uint64_t, uint64_t> get_output_histogram(const std::vector<uint64_t> &amounts) const;
+
 private:
   virtual void add_block( const block& blk
                 , const size_t& block_size

src/blockchain_db/blockchain_db.h

@@ -1288,6 +1288,15 @@ public:
    */
   virtual void drop_hard_fork_info() = 0;
 
+  /**
+   * @brief return a histogram of outputs on the blockchain
+   *
+   * @param amounts optional set of amounts to lookup
+   *
+   * @return a set of amount/instances
+   */
+  virtual std::map<uint64_t, uint64_t> get_output_histogram(const std::vector<uint64_t> &amounts) const = 0;
+
   /**
    * @brief is BlockchainDB in read-only mode?
    *

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -2692,6 +2692,63 @@ void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::
   LOG_PRINT_L3("db3: " << db3);
 }
 
+std::map<uint64_t, uint64_t> BlockchainLMDB::get_output_histogram(const std::vector<uint64_t> &amounts) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(output_amounts);
+
+  std::map<uint64_t, uint64_t> histogram;
+  MDB_val k;
+  MDB_val v;
+
+  if (amounts.empty())
+  {
+    MDB_cursor_op op = MDB_FIRST;
+    while (1)
+    {
+      int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op);
+      op = MDB_NEXT_NODUP;
+      if (ret == MDB_NOTFOUND)
+        break;
+      if (ret)
+        throw0(DB_ERROR(lmdb_error("Failed to enumerate outputs: ", ret).c_str()));
+      mdb_size_t num_elems = 0;
+      mdb_cursor_count(m_cur_output_amounts, &num_elems);
+      uint64_t amount = *(const uint64_t*)k.mv_data;
+      histogram[amount] = num_elems;
+    }
+  }
+  else
+  {
+    for (const auto &amount: amounts)
+    {
+      MDB_val_copy<uint64_t> k(amount);
+      int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET);
+      if (ret == MDB_NOTFOUND)
+      {
+        histogram[amount] = 0;
+      }
+      else if (ret == MDB_SUCCESS)
+      {
+        mdb_size_t num_elems = 0;
+        mdb_cursor_count(m_cur_output_amounts, &num_elems);
+        histogram[amount] = num_elems;
+      }
+      else
+      {
+        throw0(DB_ERROR(lmdb_error("Failed to enumerate outputs: ", ret).c_str()));
+      }
+    }
+  }
+
+  TXN_POSTFIX_RDONLY();
+
+  return histogram;
+}
+
 void BlockchainLMDB::check_hard_fork_info()
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);

src/blockchain_db/lmdb/db_lmdb.h

@@ -280,6 +280,16 @@ public:
   virtual void pop_block(block& blk, std::vector<transaction>& txs);
 
   virtual bool can_thread_bulk_indices() const { return true; }
+
+  /**
+   * @brief return a histogram of outputs on the blockchain
+   *
+   * @param amounts optional set of amounts to lookup
+   *
+   * @return a set of amount/instances
+   */
+  std::map<uint64_t, uint64_t> get_output_histogram(const std::vector<uint64_t> &amounts) const;
+
 private:
   void do_resize(uint64_t size_increase=0);
 

src/cryptonote_core/blockchain.cpp

@@ -3322,6 +3322,11 @@ bool Blockchain::get_hard_fork_voting_info(uint8_t version, uint32_t &window, ui
   return m_hardfork->get_voting_info(version, window, votes, threshold, earliest_height, voting);
 }
 
+std::map<uint64_t, uint64_t> Blockchain:: get_output_histogram(const std::vector<uint64_t> &amounts) const
+{
+  return m_db->get_output_histogram(amounts);
+}
+
 void Blockchain::load_compiled_in_block_hashes()
 {
   if (m_fast_sync && get_blocks_dat_start(m_testnet) != nullptr)

src/cryptonote_core/blockchain.h

@@ -682,6 +682,15 @@ namespace cryptonote
      */
     bool flush_txes_from_pool(const std::list<crypto::hash> &txids);
 
+    /**
+     * @brief return a histogram of outputs on the blockchain
+     *
+     * @param amounts optional set of amounts to lookup
+     *
+     * @return a set of amount/instances
+     */
+    std::map<uint64_t, uint64_t> get_output_histogram(const std::vector<uint64_t> &amounts) const;
+
     /**
      * @brief perform a check on all key images in the blockchain
      *

src/daemon/command_parser_executor.cpp

@@ -439,5 +439,23 @@ bool t_command_parser_executor::flush_txpool(const std::vector<std::string>& arg
   return m_executor.flush_txpool(txid);
 }
 
+bool t_command_parser_executor::output_histogram(const std::vector<std::string>& args)
+{
+  if (args.size() > 2) return false;
+
+  uint64_t min_count = 3;
+  uint64_t max_count = 0;
+
+  if (args.size() >= 1)
+  {
+    min_count = boost::lexical_cast<uint64_t>(args[0]);
+  }
+  if (args.size() >= 2)
+  {
+    max_count = boost::lexical_cast<uint64_t>(args[1]);
+  }
+  return m_executor.output_histogram(min_count, max_count);
+}
+
 
 } // namespace daemonize

src/daemon/command_parser_executor.h

@@ -114,6 +114,8 @@ public:
   bool unban(const std::vector<std::string>& args);
 
   bool flush_txpool(const std::vector<std::string>& args);
+
+  bool output_histogram(const std::vector<std::string>& args);
 };
 
 } // namespace daemonize

src/daemon/command_server.cpp

@@ -214,6 +214,11 @@ t_command_server::t_command_server(
     , std::bind(&t_command_parser_executor::flush_txpool, &m_parser, p::_1)
     , "Flush a transaction from the tx pool by its txid, or the whole tx pool"
     );
+    m_command_lookup.set_handler(
+      "output_histogram"
+    , std::bind(&t_command_parser_executor::output_histogram, &m_parser, p::_1)
+    , "Print output histogram (amount, instances)"
+    );
 }
 
 bool t_command_server::process_command_str(const std::string& cmd)

src/daemon/rpc_command_executor.cpp

@@ -1203,5 +1203,41 @@ bool t_rpc_command_executor::flush_txpool(const std::string &txid)
     return true;
 }
 
+bool t_rpc_command_executor::output_histogram(uint64_t min_count, uint64_t max_count)
+{
+    cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req;
+    cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response res;
+    std::string fail_message = "Unsuccessful";
+    epee::json_rpc::error error_resp;
+
+    req.min_count = min_count;
+    req.max_count = max_count;
+
+    if (m_is_rpc)
+    {
+        if (!m_rpc_client->json_rpc_request(req, res, "get_output_histogram", fail_message.c_str()))
+        {
+            return true;
+        }
+    }
+    else
+    {
+        if (!m_rpc_server->on_get_output_histogram(req, res, error_resp))
+        {
+            tools::fail_msg_writer() << fail_message.c_str();
+            return true;
+        }
+    }
+
+    std::sort(res.histogram.begin(), res.histogram.end(),
+        [](const cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::entry &e1, const cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::entry &e2)->bool { return e1.instances < e2.instances; });
+    for (const auto &e: res.histogram)
+    {
+        tools::msg_writer() << e.instances << "  " << cryptonote::print_money(e.amount);
+    }
+
+    return true;
+}
+
 
 }// namespace daemonize

src/daemon/rpc_command_executor.h

@@ -132,6 +132,8 @@ public:
   bool unban(const std::string &ip);
 
   bool flush_txpool(const std::string &txid);
+
+  bool output_histogram(uint64_t min_count, uint64_t max_count);
 };
 
 } // namespace daemonize

src/rpc/core_rpc_server.cpp

@@ -1041,6 +1041,38 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp)
+  {
+    if(!check_core_busy())
+    {
+      error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY;
+      error_resp.message = "Core is busy.";
+      return false;
+    }
+
+    std::map<uint64_t, uint64_t> histogram;
+    try
+    {
+      histogram = m_core.get_blockchain_storage().get_output_histogram(req.amounts);
+    }
+    catch (const std::exception &e)
+    {
+      res.status = "Failed to get output histogram";
+      return true;
+    }
+
+    res.histogram.clear();
+    res.histogram.reserve(histogram.size());
+    for (const auto &i: histogram)
+    {
+      if (i.second >= req.min_count && (i.second <= req.max_count || req.max_count == 0))
+        res.histogram.push_back(COMMAND_RPC_GET_OUTPUT_HISTOGRAM::entry(i.first, i.second));
+    }
+
+    res.status = CORE_RPC_STATUS_OK;
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res)
   {
 	  cryptonote::core::set_fast_exit();

src/rpc/core_rpc_server.h

@@ -109,6 +109,7 @@ namespace cryptonote
         MAP_JON_RPC_WE("setbans",                on_set_bans,                   COMMAND_RPC_SETBANS)
         MAP_JON_RPC_WE("getbans",                on_get_bans,                   COMMAND_RPC_GETBANS)
         MAP_JON_RPC_WE("flush_txpool",           on_flush_txpool,               COMMAND_RPC_FLUSH_TRANSACTION_POOL)
+        MAP_JON_RPC_WE("get_output_histogram",   on_get_output_histogram,       COMMAND_RPC_GET_OUTPUT_HISTOGRAM)
       END_JSON_RPC_MAP()
     END_URI_MAP2()
 
@@ -149,6 +150,7 @@ namespace cryptonote
     bool on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp);
     bool on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp);
     bool on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp);
+    bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp);
     //-----------------------
 
 private:

src/rpc/core_rpc_server_commands_defs.h

@@ -986,5 +986,46 @@ namespace cryptonote
       END_KV_SERIALIZE_MAP()
     };
   };
+
+  struct COMMAND_RPC_GET_OUTPUT_HISTOGRAM
+  {
+    struct request
+    {
+      std::vector<uint64_t> amounts;
+      uint64_t min_count;
+      uint64_t max_count;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(amounts);
+        KV_SERIALIZE(min_count);
+        KV_SERIALIZE(max_count);
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct entry
+    {
+      uint64_t amount;
+      uint64_t instances;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(amount);
+        KV_SERIALIZE(instances);
+      END_KV_SERIALIZE_MAP()
+
+      entry(uint64_t amount, uint64_t instances): amount(amount), instances(instances) {}
+      entry() {}
+    };
+
+    struct response
+    {
+      std::string status;
+      std::vector<entry> histogram;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(status)
+        KV_SERIALIZE(histogram)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
 }
 

src/simplewallet/simplewallet.cpp

@@ -541,7 +541,7 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("bc_height", boost::bind(&simple_wallet::show_blockchain_height, this, _1), tr("Show blockchain height"));
   m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), tr("transfer [<mixin_count>] <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of extra inputs to include for untraceability (from 0 to maximum available)"));
   m_cmd_binder.set_handler("transfer_new", boost::bind(&simple_wallet::transfer_new, this, _1), tr("Same as transfer, but using a new transaction building algorithm"));
-  m_cmd_binder.set_handler("sweep_dust", boost::bind(&simple_wallet::sweep_dust, this, _1), tr("Send all dust outputs to yourself with mixin 0"));
+  m_cmd_binder.set_handler("sweep_unmixable", boost::bind(&simple_wallet::sweep_unmixable, this, _1), tr("Send all unmixable outputs to yourself with mixin 0"));
   m_cmd_binder.set_handler("set_log", boost::bind(&simple_wallet::set_log, this, _1), tr("set_log <level> - Change current log detail level, <0-4>"));
   m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), tr("Show current wallet public address"));
   m_cmd_binder.set_handler("integrated_address", boost::bind(&simple_wallet::print_integrated_address, this, _1), tr("integrated_address [PID] - Encode a payment ID into an integrated address for the current wallet public address (no argument uses a random payment ID), or decode an integrated address to standard address and payment ID"));
@@ -1722,8 +1722,7 @@ bool simple_wallet::refresh(const std::vector<std::string>& args)
 bool simple_wallet::show_balance(const std::vector<std::string>& args/* = std::vector<std::string>()*/)
 {
   success_msg_writer() << tr("Balance: ") << print_money(m_wallet->balance()) << ", "
-    << tr("unlocked balance: ") << print_money(m_wallet->unlocked_balance()) << ", "
-    << tr("including unlocked dust: ") << print_money(m_wallet->unlocked_dust_balance(tools::tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD)));
+    << tr("unlocked balance: ") << print_money(m_wallet->unlocked_balance());
   return true;
 }
 //----------------------------------------------------------------------------------------------------
@@ -2208,7 +2207,7 @@ bool simple_wallet::transfer_new(const std::vector<std::string> &args_)
 }
 
 //----------------------------------------------------------------------------------------------------
-bool simple_wallet::sweep_dust(const std::vector<std::string> &args_)
+bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_)
 {
   if (!try_connect_to_daemon())
     return true;
@@ -2221,28 +2220,37 @@ bool simple_wallet::sweep_dust(const std::vector<std::string> &args_)
 
   try
   {
-    uint64_t total_dust = m_wallet->unlocked_dust_balance(tools::tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD));
-
     // figure out what tx will be necessary
-    auto ptx_vector = m_wallet->create_dust_sweep_transactions();
+    auto ptx_vector = m_wallet->create_unmixable_sweep_transactions(m_trusted_daemon);
+
+    if (ptx_vector.empty())
+    {
+      fail_msg_writer() << tr("No unmixable outputs found");
+      return true;
+    }
 
     // give user total and fee, and prompt to confirm
-    uint64_t total_fee = 0;
+    uint64_t total_fee = 0, total_unmixable = 0;
     for (size_t n = 0; n < ptx_vector.size(); ++n)
     {
       total_fee += ptx_vector[n].fee;
+      for (const auto &vin: ptx_vector[n].tx.vin)
+      {
+        if (vin.type() == typeid(txin_to_key))
+          total_unmixable += boost::get<txin_to_key>(vin).amount;
+      }
     }
 
-    std::string prompt_str = tr("Sweeping ") + print_money(total_dust);
+    std::string prompt_str = tr("Sweeping ") + print_money(total_unmixable);
     if (ptx_vector.size() > 1) {
       prompt_str = (boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s.  Is this okay?  (Y/Yes/N/No)")) %
-        print_money(total_dust) %
+        print_money(total_unmixable) %
         ((unsigned long long)ptx_vector.size()) %
         print_money(total_fee)).str();
     }
     else {
       prompt_str = (boost::format(tr("Sweeping %s for a total fee of %s.  Is this okay?  (Y/Yes/N/No)")) %
-        print_money(total_dust) %
+        print_money(total_unmixable) %
         print_money(total_fee)).str();
     }
     std::string accepted = command_line::input_line(prompt_str);
@@ -2285,11 +2293,12 @@ bool simple_wallet::sweep_dust(const std::vector<std::string> &args_)
   }
   catch (const tools::error::not_enough_money& e)
   {
-    fail_msg_writer() << boost::format(tr("not enough money to transfer, available only %s, transaction amount %s = %s + %s (fee)")) %
+    fail_msg_writer() << boost::format(tr("not enough money to transfer, available only %s, transaction amount %s = %s + %s (fee).\n%s")) %
       print_money(e.available()) %
       print_money(e.tx_amount() + e.fee()) %
       print_money(e.tx_amount()) %
-      print_money(e.fee());
+      print_money(e.fee()) %
+      tr("This is usually due to dust which is so small it cannot pay for itself in fees");
   }
   catch (const tools::error::not_enough_outs_to_mix& e)
   {

src/simplewallet/simplewallet.h

@@ -121,7 +121,7 @@ namespace cryptonote
     bool transfer_main(bool new_algorithm, const std::vector<std::string> &args);
     bool transfer(const std::vector<std::string> &args);
     bool transfer_new(const std::vector<std::string> &args);
-    bool sweep_dust(const std::vector<std::string> &args);
+    bool sweep_unmixable(const std::vector<std::string> &args);
     std::vector<std::vector<cryptonote::tx_destination_entry>> split_amounts(
         std::vector<cryptonote::tx_destination_entry> dsts, size_t num_splits
     );

src/wallet/wallet2.cpp

@@ -92,6 +92,13 @@ void do_prepare_file_names(const std::string& file_path, std::string& keys_file,
   }
 }
 
+uint64_t calculate_fee(const cryptonote::blobdata &blob)
+{
+  uint64_t bytes = blob.size();
+  uint64_t kB = (bytes + 1023) / 1024;
+  return kB * FEE_PER_KB;
+}
+
 } //namespace
 
 namespace tools
@@ -2030,13 +2037,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<crypto
 	{
 	  transfer(dst_vector, fake_outs_count, unlock_time, needed_fee, extra, tx, ptx);
 	  auto txBlob = t_serializable_object_to_blob(ptx.tx);
-	  uint64_t txSize = txBlob.size();
-	  uint64_t numKB = txSize / 1024;
-	  if (txSize % 1024)
-	  {
-	    numKB++;
-	  }
-	  needed_fee = numKB * FEE_PER_KB;
+          needed_fee = calculate_fee(txBlob);
 	} while (ptx.fee < needed_fee);
 
         ptx_vector.push_back(ptx);
@@ -2408,15 +2409,9 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
       transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, unlock_time, needed_fee, extra,
         detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx);
       auto txBlob = t_serializable_object_to_blob(test_ptx.tx);
-      uint64_t txSize = txBlob.size();
-      uint64_t numKB = txSize / 1024;
-      if (txSize % 1024)
-      {
-        numKB++;
-      }
-      needed_fee = numKB * FEE_PER_KB;
+      needed_fee = calculate_fee(txBlob);
       available_for_fee = test_ptx.fee + test_ptx.change_dts.amount;
-      LOG_PRINT_L2("Made a " << numKB << " kB tx, with " << print_money(available_for_fee) << " available for fee (" <<
+      LOG_PRINT_L2("Made a " << txBlob.size() << " kB tx, with " << print_money(available_for_fee) << " available for fee (" <<
         print_money(needed_fee) << " needed)");
 
       if (needed_fee > available_for_fee && dsts[0].amount > 0)
@@ -2513,7 +2508,7 @@ uint64_t wallet2::unlocked_dust_balance(const tx_dust_policy &dust_policy) const
 }
 
 template<typename T>
-void wallet2::transfer_dust(size_t num_outputs, uint64_t unlock_time, uint64_t needed_fee, T destination_split_strategy, const tx_dust_policy& dust_policy, const std::vector<uint8_t> &extra, cryptonote::transaction& tx, pending_tx &ptx)
+void wallet2::transfer_from(const std::vector<size_t> &outs, size_t num_outputs, uint64_t unlock_time, uint64_t needed_fee, T destination_split_strategy, const tx_dust_policy& dust_policy, const std::vector<uint8_t> &extra, cryptonote::transaction& tx, pending_tx &ptx)
 {
   using namespace cryptonote;
 
@@ -2523,6 +2518,19 @@ void wallet2::transfer_dust(size_t num_outputs, uint64_t unlock_time, uint64_t n
   // throw if there are none
   uint64_t money = 0;
   std::list<transfer_container::iterator> selected_transfers;
+#if 1
+  for (size_t n = 0; n < outs.size(); ++n)
+  {
+    const transfer_details& td = m_transfers[outs[n]];
+    if (!td.m_spent)
+    {
+      selected_transfers.push_back (m_transfers.begin() + outs[n]);
+      money += td.amount();
+      if (selected_transfers.size() >= num_outputs)
+        break;
+    }
+  }
+#else
   for (transfer_container::iterator i = m_transfers.begin(); i != m_transfers.end(); ++i)
   {
     const transfer_details& td = *i;
@@ -2534,6 +2542,7 @@ void wallet2::transfer_dust(size_t num_outputs, uint64_t unlock_time, uint64_t n
         break;
     }
   }
+#endif
 
   // we don't allow no output to self, easier, but one may want to burn the dust if = fee
   THROW_WALLET_EXCEPTION_IF(money <= needed_fee, error::not_enough_money, money, needed_fee, needed_fee);
@@ -2627,8 +2636,8 @@ bool wallet2::use_fork_rules(uint8_t version)
   r = net_utils::invoke_http_json_remote_command2(m_daemon_address + "/json_rpc", req_t, resp_t, m_http_client);
   m_daemon_rpc_mutex.unlock();
   CHECK_AND_ASSERT_MES(r, false, "Failed to connect to daemon");
-  CHECK_AND_ASSERT_MES(res.status != CORE_RPC_STATUS_BUSY, false, "Failed to connect to daemon");
-  CHECK_AND_ASSERT_MES(res.status == CORE_RPC_STATUS_OK, false, "Failed to get hard fork status");
+  CHECK_AND_ASSERT_MES(resp_t.result.status != CORE_RPC_STATUS_BUSY, false, "Failed to connect to daemon");
+  CHECK_AND_ASSERT_MES(resp_t.result.status == CORE_RPC_STATUS_OK, false, "Failed to get hard fork status");
 
   bool close_enough = res.height >=  resp_t.result.earliest_height - 10; // start using the rules a bit beforehand
   if (close_enough)
@@ -2646,20 +2655,85 @@ uint64_t wallet2::get_upper_tranaction_size_limit()
   return ((full_reward_zone * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
 }
 //----------------------------------------------------------------------------------------------------
-std::vector<wallet2::pending_tx> wallet2::create_dust_sweep_transactions()
+std::vector<size_t> wallet2::select_available_outputs(const std::function<bool(const transfer_details &td)> &f)
+{
+  std::vector<size_t> outputs;
+  size_t n = 0;
+  for (transfer_container::const_iterator i = m_transfers.begin(); i != m_transfers.end(); ++i, ++n)
+  {
+    if (i->m_spent)
+      continue;
+    if (!is_transfer_unlocked(*i))
+      continue;
+    if (f(*i))
+      outputs.push_back(n);
+  }
+  return outputs;
+}
+//----------------------------------------------------------------------------------------------------
+std::vector<uint64_t> wallet2::get_unspent_amounts_vector()
+{
+  std::set<uint64_t> set;
+  for (const auto &td: m_transfers)
+  {
+    if (!td.m_spent)
+      set.insert(td.amount());
+  }
+  std::vector<uint64_t> vector;
+  vector.reserve(set.size());
+  for (const auto &i: set)
+  {
+    vector.push_back(i);
+  }
+  return vector;
+}
+//----------------------------------------------------------------------------------------------------
+std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_daemon)
+{
+  // request all outputs with at least 3 instances, so we can use mixin 2 with
+  epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request> req_t = AUTO_VAL_INIT(req_t);
+  epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response, std::string> resp_t = AUTO_VAL_INIT(resp_t);
+  m_daemon_rpc_mutex.lock();
+  req_t.jsonrpc = "2.0";
+  req_t.id = epee::serialization::storage_entry(0);
+  req_t.method = "get_output_histogram";
+  if (trusted_daemon)
+    req_t.params.amounts = get_unspent_amounts_vector();
+  req_t.params.min_count = 3;
+  req_t.params.max_count = 0;
+  bool r = net_utils::invoke_http_json_remote_command2(m_daemon_address + "/json_rpc", req_t, resp_t, m_http_client);
+  m_daemon_rpc_mutex.unlock();
+  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "select_available_unmixable_outputs");
+  THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram");
+  THROW_WALLET_EXCEPTION_IF(resp_t.result.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.result.status);
+
+  std::set<uint64_t> mixable;
+  for (const auto &i: resp_t.result.histogram)
+  {
+    mixable.insert(i.amount);
+  }
+
+  return select_available_outputs([mixable](const transfer_details &td) {
+    const uint64_t amount = td.amount();
+    if (mixable.find(amount) == mixable.end())
+      return true;
+    return false;
+  });
+}
+//----------------------------------------------------------------------------------------------------
+std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bool trusted_daemon)
 {
   // From hard fork 1, we don't consider small amounts to be dust anymore
   const bool hf1_rules = use_fork_rules(2); // first hard fork has version 2
   tx_dust_policy dust_policy(hf1_rules ? 0 : ::config::DEFAULT_DUST_THRESHOLD);
 
-  size_t num_dust_outputs = 0;
-  for (transfer_container::const_iterator i = m_transfers.begin(); i != m_transfers.end(); ++i)
+  // may throw
+  std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(trusted_daemon);
+  size_t num_dust_outputs = unmixable_outputs.size();
+
+  if (num_dust_outputs == 0)
   {
-    const transfer_details& td = *i;
-    if (!td.m_spent && (td.amount() < dust_policy.dust_threshold || !is_valid_decomposed_amount(td.amount())) && is_transfer_unlocked(td))
-    {
-      num_dust_outputs++;
-    }
+    return std::vector<wallet2::pending_tx>();
   }
 
   // failsafe split attempt counter
@@ -2682,23 +2756,18 @@ std::vector<wallet2::pending_tx> wallet2::create_dust_sweep_transactions()
 
 	// loop until fee is met without increasing tx size to next KB boundary.
 	uint64_t needed_fee = 0;
-	if (1)
+	do
 	{
-	  transfer_dust(num_outputs_per_tx, (uint64_t)0 /* unlock_time */, 0, detail::digit_split_strategy, dust_policy, extra, tx, ptx);
+	  transfer_from(unmixable_outputs, num_outputs_per_tx, (uint64_t)0 /* unlock_time */, 0, detail::digit_split_strategy, dust_policy, extra, tx, ptx);
 	  auto txBlob = t_serializable_object_to_blob(ptx.tx);
-	  uint64_t txSize = txBlob.size();
-	  uint64_t numKB = txSize / 1024;
-	  if (txSize % 1024)
-	  {
-	    numKB++;
-	  }
-	  needed_fee = numKB * FEE_PER_KB;
+          needed_fee = calculate_fee(txBlob);
 
           // reroll the tx with the actual amount minus the fee
           // if there's not enough for the fee, it'll throw
-	  transfer_dust(num_outputs_per_tx, (uint64_t)0 /* unlock_time */, needed_fee, detail::digit_split_strategy, dust_policy, extra, tx, ptx);
+	  transfer_from(unmixable_outputs, num_outputs_per_tx, (uint64_t)0 /* unlock_time */, needed_fee, detail::digit_split_strategy, dust_policy, extra, tx, ptx);
 	  txBlob = t_serializable_object_to_blob(ptx.tx);
-	}
+          needed_fee = calculate_fee(txBlob);
+	} while (ptx.fee < needed_fee);
 
         ptx_vector.push_back(ptx);
 

src/wallet/wallet2.h

@@ -280,7 +280,7 @@ namespace tools
     void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra);
     void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx);
     template<typename T>
-    void transfer_dust(size_t num_outputs, uint64_t unlock_time, uint64_t needed_fee, T destination_split_strategy, const tx_dust_policy& dust_policy, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx);
+    void transfer_from(const std::vector<size_t> &outs, size_t num_outputs, uint64_t unlock_time, uint64_t needed_fee, T destination_split_strategy, const tx_dust_policy& dust_policy, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx);
     template<typename T>
     void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::list<transfer_container::iterator> selected_transfers, size_t fake_outputs_count,
       uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx);
@@ -289,7 +289,7 @@ namespace tools
     void commit_tx(std::vector<pending_tx>& ptx_vector);
     std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee, const std::vector<uint8_t> extra);
     std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra);
-    std::vector<pending_tx> create_dust_sweep_transactions();
+    std::vector<pending_tx> create_unmixable_sweep_transactions(bool trusted_daemon);
     bool check_connection();
     void get_transfers(wallet2::transfer_container& incoming_transfers) const;
     void get_payments(const crypto::hash& payment_id, std::list<wallet2::payment_details>& payments, uint64_t min_height = 0) const;
@@ -402,6 +402,9 @@ namespace tools
     void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const;
     uint64_t get_upper_tranaction_size_limit();
     void check_pending_txes();
+    std::vector<uint64_t> get_unspent_amounts_vector();
+    std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f);
+    std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon);
 
     cryptonote::account_base m_account;
     std::string m_daemon_address;

src/wallet/wallet_errors.h

@@ -76,6 +76,7 @@ namespace tools
     //         daemon_busy
     //         no_connection_to_daemon
     //         is_key_image_spent_error
+    //         get_histogram_error
     //       wallet_files_doesnt_correspond
     //
     // * - class with protected ctor
@@ -600,6 +601,14 @@ namespace tools
       }
     };
     //----------------------------------------------------------------------------------------------------
+    struct get_histogram_error : public wallet_rpc_error
+    {
+      explicit get_histogram_error(std::string&& loc, const std::string& request)
+        : wallet_rpc_error(std::move(loc), "failed to get output histogram", request)
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
     struct wallet_files_doesnt_correspond : public wallet_logic_error
     {
       explicit wallet_files_doesnt_correspond(std::string&& loc, const std::string& keys_file, const std::string& wallet_file)

src/wallet/wallet_rpc_server.cpp

@@ -347,7 +347,7 @@ namespace tools
 
     try
     {
-      std::vector<wallet2::pending_tx> ptx_vector = m_wallet.create_dust_sweep_transactions();
+      std::vector<wallet2::pending_tx> ptx_vector = m_wallet.create_unmixable_sweep_transactions(req.trusted_daemon);
 
       m_wallet.commit_tx(ptx_vector);
 

src/wallet/wallet_rpc_server_commands_defs.h

@@ -178,9 +178,11 @@ namespace wallet_rpc
     struct request
     {
       bool get_tx_keys;
+      bool trusted_daemon;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(get_tx_keys)
+        KV_SERIALIZE(trusted_daemon)
       END_KV_SERIALIZE_MAP()
     };
 

tests/unit_tests/hardfork.cpp

@@ -108,6 +108,7 @@ public:
   virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const { return true; }
   virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const { return true; }
   virtual bool is_read_only() const { return false; }
+  virtual std::map<uint64_t, uint64_t> get_output_histogram(const std::vector<uint64_t> &amounts) const { return std::map<uint64_t, uint64_t>(); }
 
   virtual void add_block( const block& blk
                         , const size_t& block_size