Merge pull request #3536

c77d2bfa Add the possibility to export private view key for fast scan. (cslashm)
100b7bc1 Change mutex lock model to avoid dead lock and ensure locks are always released. (cslashm)
641dfc99 Automatic height setup when creating/restoring hw device. (cslashm)

src/device/device.hpp

@@ -80,6 +80,9 @@ namespace hw {
 
 
     class device {
+    protected:
+        std::string  name;
+
     public:
 
         device()  {}
@@ -87,12 +90,12 @@ namespace hw {
         virtual ~device()   {}
 
         explicit virtual operator bool() const = 0;
-
+        enum device_mode {
-        static const int SIGNATURE_REAL = 0;
+            NONE,
-        static const int SIGNATURE_FAKE = 1;
+            TRANSACTION_CREATE_REAL,
-
+            TRANSACTION_CREATE_FAKE,
-
+            TRANSACTION_PARSE
-        std::string  name;
+        };
 
         /* ======================================================================= */
         /*                              SETUP/TEARDOWN                             */
@@ -104,7 +107,18 @@ namespace hw {
         virtual bool release() = 0;
 
         virtual bool connect(void) = 0;
-        virtual bool disconnect() = 0;
+        virtual bool disconnect(void) = 0;
+
+        virtual bool  set_mode(device_mode mode) = 0;
+
+
+        /* ======================================================================= */
+        /*  LOCKER                                                                 */
+        /* ======================================================================= */ 
+        virtual void lock(void) = 0;
+        virtual void unlock(void) = 0;
+        virtual bool try_lock(void) = 0;
+
 
         /* ======================================================================= */
         /*                             WALLET & ADDRESS                            */
@@ -158,8 +172,6 @@ namespace hw {
 
         virtual bool  open_tx(crypto::secret_key &tx_key) = 0;
 
-        virtual bool  set_signature_mode(unsigned int sig_mode) = 0;
-
         virtual bool  encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) = 0;
         bool  decrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key)
         {
@@ -183,6 +195,12 @@ namespace hw {
         virtual bool  close_tx(void) = 0;
     } ;
 
+    struct reset_mode {
+        device& hwref;
+        reset_mode(hw::device& dev) : hwref(dev) { }
+        ~reset_mode() { hwref.set_mode(hw::device::NONE);}
+    };
+
     device& get_device(const std::string device_descriptor) ;
 }
 

src/device/device_default.cpp

@@ -81,6 +81,20 @@ namespace hw {
             dfns();
         }
 
+        bool  device_default::set_mode(device_mode mode) {
+            return true;
+        }
+
+        /* ======================================================================= */
+        /*  LOCKER                                                                 */
+        /* ======================================================================= */ 
+    
+        void device_default::lock() { }
+
+        bool device_default::try_lock() { return true; }
+
+        void device_default::unlock() { }
+
         /* ======================================================================= */
         /*                             WALLET & ADDRESS                            */
         /* ======================================================================= */
@@ -262,10 +276,6 @@ namespace hw {
             return true;
         }
 
-        bool  device_default::set_signature_mode(unsigned int sig_mode) {
-            return true;
-        }
-
         bool  device_default::encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) {
             crypto::key_derivation derivation;
             crypto::hash hash;

src/device/device_default.hpp

@@ -59,6 +59,15 @@ namespace hw {
             bool connect(void) override;
             bool disconnect() override;
  
+            bool set_mode(device_mode mode) override;
+
+            /* ======================================================================= */
+            /*  LOCKER                                                                 */
+            /* ======================================================================= */ 
+            void lock(void)  override;
+            void unlock(void) override;
+            bool try_lock(void) override;
+            
             /* ======================================================================= */
             /*                             WALLET & ADDRESS                            */
             /* ======================================================================= */
@@ -97,9 +106,6 @@ namespace hw {
 
             bool  open_tx(crypto::secret_key &tx_key) override;
 
-            //bool  get_additional_key(const bool subaddr, cryptonote::keypair &additional_txkey) override;
-            bool  set_signature_mode(unsigned int sig_mode) override;
-
             bool  encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override;
 
             bool  ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) override;

src/device/device_ledger.hpp

@@ -32,11 +32,11 @@
 
 #include <cstddef>
 #include <string>
-#include <mutex>
 #include "device.hpp"
 #include <PCSC/winscard.h>
 #include <PCSC/wintypes.h>
-
+#include <boost/thread/mutex.hpp>
+#include <boost/thread/recursive_mutex.hpp>
 
 namespace hw {
 
@@ -80,13 +80,11 @@ namespace hw {
 
     class device_ledger : public hw::device {
     private:
-        mutable std::mutex   device_locker;
+        // Locker for concurrent access
-        mutable std::mutex   tx_locker;
+        mutable boost::recursive_mutex   device_locker;
-        void lock_device() ;
+        mutable boost::mutex   command_locker;
-        void unlock_device() ;
-        void lock_tx() ;
-        void unlock_tx() ;
 
+        //PCSC management 
         std::string  full_name;
         SCARDCONTEXT hContext;
         SCARDHANDLE  hCard;
@@ -95,15 +93,21 @@ namespace hw {
         DWORD        length_recv;
         BYTE         buffer_recv[BUFFER_RECV_SIZE];
         unsigned int id;
-
-        Keymap key_map;
-
-
         void logCMD(void);
         void logRESP(void);
         unsigned int  exchange(unsigned int ok=0x9000, unsigned int mask=0xFFFF);
         void reset_buffer(void);
 
+        // hw running mode
+        device_mode mode;
+        // map public destination key to ephemeral destination key
+        Keymap key_map;
+
+        // To speed up blockchain parsing the view key maybe handle here.
+        crypto::secret_key viewkey;
+        bool has_view_key;
+        
+        //extra debug
         #ifdef DEBUG_HWDEVICE
         device *controle_device;
         #endif
@@ -130,6 +134,15 @@ namespace hw {
         bool connect(void) override;
         bool disconnect() override;
 
+        bool  set_mode(device_mode mode) override;
+
+        /* ======================================================================= */
+        /*  LOCKER                                                                 */
+        /* ======================================================================= */ 
+        void lock(void)  override;
+        void unlock(void) override;
+        bool try_lock(void) override;
+
         /* ======================================================================= */
         /*                             WALLET & ADDRESS                            */
         /* ======================================================================= */
@@ -168,8 +181,6 @@ namespace hw {
 
         bool  open_tx(crypto::secret_key &tx_key) override;
 
-        bool  set_signature_mode(unsigned int sig_mode) override;
-
         bool  encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override;
 
         bool  ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) override;
@@ -190,10 +201,9 @@ namespace hw {
     };
 
 
-
     #ifdef DEBUG_HWDEVICE
-    extern crypto::secret_key viewkey;
+    extern crypto::secret_key dbg_viewkey;
-    extern crypto::secret_key spendkey;
+    extern crypto::secret_key dbg_spendkey;
     #endif
 
     #endif  //WITH_DEVICE_LEDGER

src/device/log.cpp

@@ -56,8 +56,8 @@ namespace hw {
     }
 
     #ifdef DEBUG_HWDEVICE
-    extern crypto::secret_key viewkey;
+    extern crypto::secret_key dbg_viewkey;
-    extern crypto::secret_key spendkey;
+    extern crypto::secret_key dbg_spendkey;
 
 
     void decrypt(char* buf, size_t len) {
@@ -69,7 +69,7 @@ namespace hw {
           if (buf[i] != 0) break;
         }
         if (i == 32) {
-          memmove(buf, hw::ledger::viewkey.data, 32);
+          memmove(buf, hw::ledger::dbg_viewkey.data, 32);
           return;
         }
         //spend key?
@@ -77,7 +77,7 @@ namespace hw {
           if (buf[i] != (char)0xff) break;
         }
         if (i == 32) {
-          memmove(buf, hw::ledger::spendkey.data, 32);
+          memmove(buf, hw::ledger::dbg_spendkey.data, 32);
           return;
         }
       }

src/simplewallet/simplewallet.cpp

@@ -2974,6 +2974,22 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
       // create wallet
       bool r = new_wallet(vm, "Ledger");
       CHECK_AND_ASSERT_MES(r, false, tr("account creation failed"));
+      // if no block_height is specified, assume its a new account and start it "now"
+      if(m_wallet->get_refresh_from_block_height() == 0) {
+        {
+          tools::scoped_message_writer wrt = tools::msg_writer();
+          wrt << tr("No restore height is specified.");
+          wrt << tr("Assumed you are creating a new account, restore will be done from current estimated blockchain height.");
+          wrt << tr("Use --restore-height if you want to restore an already setup account from a specific height");
+        }
+        std::string confirm = input_line(tr("Is this okay?  (Y/Yes/N/No): "));
+        if (std::cin.eof() || !command_line::is_yes(confirm))
+          CHECK_AND_ASSERT_MES(false, false, tr("account creation aborted"));
+
+        m_wallet->set_refresh_from_block_height(m_wallet->estimate_blockchain_height()-1);
+        m_wallet->explicit_refresh_from_block_height(true);
+        m_restore_height = m_wallet->get_refresh_from_block_height();
+      }
     }
     else
     {
@@ -2990,7 +3006,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
       CHECK_AND_ASSERT_MES(r, false, tr("account creation failed"));
     }
 
-    if (m_restoring && m_generate_from_json.empty())
+    if (m_restoring && m_generate_from_json.empty() && m_generate_from_device.empty())
     {
       m_wallet->explicit_refresh_from_block_height(!command_line::is_arg_defaulted(vm, arg_restore_height));
     }
@@ -3388,7 +3404,7 @@ bool simple_wallet::new_wallet(const boost::program_options::variables_map& vm,
   try
   {
     m_wallet->restore(m_wallet_file, std::move(rc.second).password(), device_name);
-    message_writer(console_color_white, true) << tr("Generated new on device wallet: ")
+    message_writer(console_color_white, true) << tr("Generated new wallet on hw device: ")
       << m_wallet->get_account().get_public_address_str(m_wallet->nettype());
   }
   catch (const std::exception& e)

src/wallet/wallet2.cpp

@@ -1007,13 +1007,16 @@ void wallet2::set_unspent(size_t idx)
 //----------------------------------------------------------------------------------------------------
 void wallet2::check_acc_out_precomp(const tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, tx_scan_info_t &tx_scan_info) const
 {
+  hw::device &hwdev = m_account.get_device();
+  std::unique_lock<hw::device> hwdev_lock (hwdev);
+  hwdev.set_mode(hw::device::TRANSACTION_PARSE);
   if (o.target.type() !=  typeid(txout_to_key))
   {
      tx_scan_info.error = true;
      LOG_ERROR("wrong type id in transaction out");
      return;
   }
-  tx_scan_info.received = is_out_to_acc_precomp(m_subaddresses, boost::get<txout_to_key>(o.target).key, derivation, additional_derivations, i, m_account.get_device());
+  tx_scan_info.received = is_out_to_acc_precomp(m_subaddresses, boost::get<txout_to_key>(o.target).key, derivation, additional_derivations, i, hwdev);
   if(tx_scan_info.received)
   {
     tx_scan_info.money_transfered = o.amount; // may be 0 for ringct outputs
@@ -1080,9 +1083,15 @@ void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::publi
 //----------------------------------------------------------------------------------------------------
 void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen)
 {
-  // In this function, tx (probably) only contains the base information
+  //ensure device is let in NONE mode in any case
-  // (that is, the prunable stuff may or may not be included)
+  hw::device &hwdev = m_account.get_device(); 
   
+  std::unique_lock<hw::device> hwdev_lock (hwdev);
+  hw::reset_mode rst(hwdev);  
+  hwdev_lock.unlock();
+
+ // In this function, tx (probably) only contains the base information
+  // (that is, the prunable stuff may or may not be included)
   if (!miner_tx && !pool)
     process_unconfirmed(txid, tx, height);
   std::vector<size_t> outs;
@@ -1119,8 +1128,10 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote
     tools::threadpool& tpool = tools::threadpool::getInstance();
     tools::threadpool::waiter waiter;
     const cryptonote::account_keys& keys = m_account.get_keys();
-    hw::device &hwdev = m_account.get_device();
     crypto::key_derivation derivation;
+
+    hwdev_lock.lock();
+    hwdev.set_mode(hw::device::TRANSACTION_PARSE);
     if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation))
     {
       MWARNING("Failed to generate key derivation from tx pubkey, skipping");
@@ -1140,6 +1151,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote
         additional_derivations.pop_back();
       }
     }
+    hwdev_lock.unlock();
 
     if (miner_tx && m_refresh_type == RefreshNoCoinbase)
     {
@@ -1161,16 +1173,19 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote
             std::ref(tx_scan_info[i])));
         }
         waiter.wait();
-
         // then scan all outputs from 0
+        hwdev_lock.lock();
+        hwdev.set_mode(hw::device::NONE);
         for (size_t i = 0; i < tx.vout.size(); ++i)
         {
           THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys());
           if (tx_scan_info[i].received)
           {
+            hwdev.generate_key_derivation(tx_pub_key,  keys.m_view_secret_key, tx_scan_info[i].received->derivation);
             scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs);
           }
         }
+        hwdev_lock.unlock();
       }
     }
     else if (tx.vout.size() > 1 && tools::threadpool::getInstance().get_max_concurrency() > 1)
@@ -1181,14 +1196,19 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote
             std::ref(tx_scan_info[i])));
       }
       waiter.wait();
+
+      hwdev_lock.lock();
+      hwdev.set_mode(hw::device::NONE);
       for (size_t i = 0; i < tx.vout.size(); ++i)
       {
         THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys());
         if (tx_scan_info[i].received)
         {
+          hwdev.generate_key_derivation(tx_pub_key,  keys.m_view_secret_key, tx_scan_info[i].received->derivation);
           scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs);
         }
       }
+      hwdev_lock.unlock();
     }
     else
     {
@@ -1198,7 +1218,11 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote
         THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys());
         if (tx_scan_info[i].received)
         {
+          hwdev_lock.lock();
+          hwdev.set_mode(hw::device::NONE);
+          hwdev.generate_key_derivation(tx_pub_key,  keys.m_view_secret_key, tx_scan_info[i].received->derivation);
           scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs);
+          hwdev_lock.unlock();
         }
       }
     }
@@ -7259,6 +7283,11 @@ bool wallet2::light_wallet_key_image_is_ours(const crypto::key_image& key_image,
 // usable balance.
 std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon)
 {
+  //ensure device is let in NONE mode in any case
+  hw::device &hwdev = m_account.get_device();
+  std::unique_lock<hw::device> hwdev_lock (hwdev);
+  hw::reset_mode rst(hwdev);  
+
   if(m_light_wallet) {
     // Populate m_transfers
     light_wallet_get_unspent_outs();
@@ -7476,8 +7505,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
   unsigned int original_output_index = 0;
   std::vector<size_t>* unused_transfers_indices = &unused_transfers_indices_per_subaddr[0].second;
   std::vector<size_t>* unused_dust_indices      = &unused_dust_indices_per_subaddr[0].second;
-  hw::device &hwdev = m_account.get_device();
+  
-  hwdev.set_signature_mode(hw::device::SIGNATURE_FAKE);
+  hwdev.set_mode(hw::device::TRANSACTION_CREATE_FAKE);
   while ((!dsts.empty() && dsts[0].amount > 0) || adding_fee || !preferred_inputs.empty() || should_pick_a_second_output(use_rct, txes.back().selected_transfers.size(), *unused_transfers_indices, *unused_dust_indices)) {
     TX &tx = txes.back();
 
@@ -7706,7 +7735,7 @@ skip_tx:
   LOG_PRINT_L1("Done creating " << txes.size() << " transactions, " << print_money(accumulated_fee) <<
     " total fee, " << print_money(accumulated_change) << " total change");
 
-  hwdev.set_signature_mode(hw::device::SIGNATURE_REAL);
+  hwdev.set_mode(hw::device::TRANSACTION_CREATE_REAL);
   for (std::vector<TX>::iterator i = txes.begin(); i != txes.end(); ++i)
   {
     TX &tx = *i;
@@ -7837,6 +7866,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypt
 
 std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon)
 {
+  //ensure device is let in NONE mode in any case
+  hw::device &hwdev = m_account.get_device();
+  std::unique_lock<hw::device> hwdev_lock (hwdev);
+  hw::reset_mode rst(hwdev);  
+
   uint64_t accumulated_fee, accumulated_outputs, accumulated_change;
   struct TX {
     std::vector<size_t> selected_transfers;
@@ -7869,8 +7903,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton
   needed_fee = 0;
 
   // while we have something to send
-  hw::device &hwdev = m_account.get_device();
+  hwdev.set_mode(hw::device::TRANSACTION_CREATE_FAKE);
-  hwdev.set_signature_mode(hw::device::SIGNATURE_FAKE);
   while (!unused_dust_indices.empty() || !unused_transfers_indices.empty()) {
     TX &tx = txes.back();
 
@@ -7956,7 +7989,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton
   LOG_PRINT_L1("Done creating " << txes.size() << " transactions, " << print_money(accumulated_fee) <<
     " total fee, " << print_money(accumulated_change) << " total change");
  
-  hwdev.set_signature_mode(hw::device::SIGNATURE_REAL);
+  hwdev.set_mode(hw::device::TRANSACTION_CREATE_REAL);
   for (std::vector<TX>::iterator i = txes.begin(); i != txes.end(); ++i)
   {
     TX &tx = *i;