Merge pull request #1841

b553c282 rpc: fix BUILD_TAG mispelling (BUILDTAG) (moneromooo-monero) 02097c87 core: print the "new update found" message in cyan, for visibility (moneromooo-monero) 749ebace download: check available disk space before downloading (moneromooo-monero) f36c5f1e download: give download threads distinct names (moneromooo-monero) f6211322 core: make update download cancellable (moneromooo-monero) 63f0e074 download: async API (moneromooo-monero) 9bf017ed http_client: allow cancelling a download (moneromooo-monero) 0d90123c http_client: allow derived class to get headers at start (moneromooo-monero)
2017-03-06 15:17:08 +00:00 · 2017-03-06 15:17:08 +00:00 · ba0767477d
commit ba0767477d
parent ea047307da b553c282fb
6 changed files with 245 additions and 48 deletions
--- a/contrib/epee/include/net/http_client.h
+++ b/contrib/epee/include/net/http_client.h
@ -335,6 +335,11 @@ using namespace std;
 				m_response_info.m_body += piece_of_transfer;
        piece_of_transfer.clear();
 				return true;
+			}
+			//---------------------------------------------------------------------------
+			virtual bool on_header(const http_response_info &headers)
+      {
+        return true;
      }
 			//---------------------------------------------------------------------------
 			inline 
@ -505,6 +510,12 @@ using namespace std;
 					m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end());

 					analize_cached_header_and_invoke_state();
+          if (!on_header(m_response_info))
+          {
+            MDEBUG("Connection cancelled by on_header");
+            m_state = reciev_machine_state_done;
+            return false;
+          }
 					m_header_cache.clear();
 					if(!recv_buff.size() && (m_state != reciev_machine_state_error && m_state != reciev_machine_state_done))
 						need_more_data = true;
@ -527,7 +538,11 @@ using namespace std;
 				}
 				CHECK_AND_ASSERT_MES(m_len_in_remain >= recv_buff.size(), false, "m_len_in_remain >= recv_buff.size()");
 				m_len_in_remain -= recv_buff.size();
-				m_pcontent_encoding_handler->update_in(recv_buff);
+				if (!m_pcontent_encoding_handler->update_in(recv_buff))
+				{
+					m_state = reciev_machine_state_done;
+					return false;
+				}

 				if(m_len_in_remain == 0)
 					m_state = reciev_machine_state_done;
@ -700,7 +715,11 @@ using namespace std;
 								m_len_in_remain = 0;
 							}

-							m_pcontent_encoding_handler->update_in(chunk_body);
+							if (!m_pcontent_encoding_handler->update_in(chunk_body))
+							{
+								m_state = reciev_machine_state_error;
+								return false;
+							}

 							if(!m_len_in_remain)
 								m_chunked_state = http_chunked_state_chunk_head;
--- a/src/common/download.cpp
+++ b/src/common/download.cpp
@ -27,6 +27,7 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 #include <string>
+#include <atomic>
 #include <boost/filesystem.hpp>
 #include <boost/asio.hpp>
 #include <boost/thread/thread.hpp>
@ -40,27 +41,82 @@

 namespace tools
 {
-  static bool download_thread(const std::string &path, const std::string &url)
+  struct download_thread_control
  {
+    const std::string path;
+    const std::string uri;
+    std::function<void(const std::string&, const std::string&, bool)> result_cb;
+    std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb;
+    bool stop;
+    bool stopped;
+    bool success;
+    boost::thread thread;
+    boost::mutex mutex;
+
+    download_thread_control(const std::string &path, const std::string &uri, std::function<void(const std::string&, const std::string&, bool)> result_cb, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress_cb):
+        path(path), uri(uri), result_cb(result_cb), progress_cb(progress_cb), stop(false), stopped(false), success(false) {}
+    ~download_thread_control() { if (thread.joinable()) thread.detach(); }
+  };
+
+  static void download_thread(download_async_handle control)
+  {
+    static std::atomic<unsigned int> thread_id(0);
+
+    MLOG_SET_THREAD_NAME("DL" + std::to_string(thread_id++));
+
+    struct stopped_setter
+    {
+      stopped_setter(const download_async_handle &control): control(control) {}
+      ~stopped_setter() { control->stopped = true; }
+      download_async_handle control;
+    } stopped_setter(control);
+
    try
    {
-      MINFO("Downloading " << url << " to " << path);
+      boost::unique_lock<boost::mutex> lock(control->mutex);
+      MINFO("Downloading " << control->uri << " to " << control->path);
      std::ofstream f;
-      f.open(path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
+      f.open(control->path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
      if (!f.good()) {
-        MERROR("Failed to open file " << path);
-        return false;
+        MERROR("Failed to open file " << control->path);
+        control->result_cb(control->path, control->uri, control->success);
+        return;
      }
      class download_client: public epee::net_utils::http::http_simple_client
      {
      public:
-        download_client(std::ofstream &f): f(f) {}
+        download_client(download_async_handle control, std::ofstream &f):
+          control(control), f(f), content_length(-1), total(0) {}
        virtual ~download_client() { f.close(); }
+        virtual bool on_header(const epee::net_utils::http::http_response_info &headers)
+        {
+          ssize_t length;
+          if (epee::string_tools::get_xtype_from_string(length, headers.m_header_info.m_content_length) && length >= 0)
+          {
+            MINFO("Content-Length: " << length);
+            content_length = length;
+            boost::filesystem::path path(control->path);
+            boost::filesystem::space_info si = boost::filesystem::space(path);
+            if (si.available < (size_t)content_length)
+            {
+              const uint64_t avail = (si.available + 1023) / 1024, needed = (content_length + 1023) / 1024;
+              MERROR("Not enough space to download " << needed << " kB to " << path << " (" << avail << " kB available)");
+              return false;
+            }
+          }
+          return true;
+        }
        virtual bool handle_target_data(std::string &piece_of_transfer)
        {
          try
          {
+            boost::lock_guard<boost::mutex> lock(control->mutex);
+            if (control->stop)
+              return false;
            f << piece_of_transfer;
+            total += piece_of_transfer.size();
+            if (control->progress_cb && !control->progress_cb(control->path, control->uri, total, content_length))
+              return false;
            return f.good();
          }
          catch (const std::exception &e)
@ -70,69 +126,145 @@ namespace tools
          }
        }
      private:
+        download_async_handle control;
        std::ofstream &f;
-      } client(f);
+        ssize_t content_length;
+        size_t total;
+      } client(control, f);
      epee::net_utils::http::url_content u_c;
-      if (!epee::net_utils::parse_url(url, u_c))
+      if (!epee::net_utils::parse_url(control->uri, u_c))
      {
-        MWARNING("Failed to parse URL " << url);
-        return false;
+        MERROR("Failed to parse URL " << control->uri);
+        control->result_cb(control->path, control->uri, control->success);
+        return;
      }
      if (u_c.host.empty())
      {
-        MWARNING("Failed to determine address from URL " << url);
-        return false;
+        MERROR("Failed to determine address from URL " << control->uri);
+        control->result_cb(control->path, control->uri, control->success);
+        return;
      }
+
+      lock.unlock();
+
      uint16_t port = u_c.port ? u_c.port : 80;
      MDEBUG("Connecting to " << u_c.host << ":" << port);
      client.set_server(u_c.host, std::to_string(port), boost::none);
      if (!client.connect(std::chrono::seconds(30)))
      {
-        MERROR("Failed to connect to " << url);
-        return false;
+        boost::lock_guard<boost::mutex> lock(control->mutex);
+        MERROR("Failed to connect to " << control->uri);
+        control->result_cb(control->path, control->uri, control->success);
+        return;
      }
      MDEBUG("GETting " << u_c.uri);
      const epee::net_utils::http::http_response_info *info = NULL;
      if (!client.invoke_get(u_c.uri, std::chrono::seconds(30), "", &info))
      {
-        MERROR("Failed to connect to " << url);
+        boost::lock_guard<boost::mutex> lock(control->mutex);
+        MERROR("Failed to connect to " << control->uri);
        client.disconnect();
-        return false;
+        control->result_cb(control->path, control->uri, control->success);
+        return;
+      }
+      if (control->stop)
+      {
+        boost::lock_guard<boost::mutex> lock(control->mutex);
+        MDEBUG("Download cancelled");
+        client.disconnect();
+        control->result_cb(control->path, control->uri, control->success);
+        return;
      }
      if (!info)
      {
-        MERROR("Failed invoking GET command to " << url << ", no status info returned");
+        boost::lock_guard<boost::mutex> lock(control->mutex);
+        MERROR("Failed invoking GET command to " << control->uri << ", no status info returned");
        client.disconnect();
-        return false;
+        control->result_cb(control->path, control->uri, control->success);
+        return;
      }
      MDEBUG("response code: " << info->m_response_code);
+      MDEBUG("response length: " << info->m_header_info.m_content_length);
      MDEBUG("response comment: " << info->m_response_comment);
      MDEBUG("response body: " << info->m_body);
      for (const auto &f: info->m_additional_fields)
        MDEBUG("additional field: " << f.first << ": " << f.second);
      if (info->m_response_code != 200)
      {
+        boost::lock_guard<boost::mutex> lock(control->mutex);
        MERROR("Status code " << info->m_response_code);
        client.disconnect();
-        return false;
+        control->result_cb(control->path, control->uri, control->success);
+        return;
      }
      client.disconnect();
      f.close();
      MDEBUG("Download complete");
-      return true;
+      lock.lock();
+      control->success = true;
+      control->result_cb(control->path, control->uri, control->success);
+      return;
    }
    catch (const std::exception &e)
    {
      MERROR("Exception in download thread: " << e.what());
-      return false;
+      // fall through and call result_cb not from the catch block to avoid another exception
    }
+    boost::lock_guard<boost::mutex> lock(control->mutex);
+    control->result_cb(control->path, control->uri, control->success);
  }

-  bool download(const std::string &path, const std::string &url)
+  bool download(const std::string &path, const std::string &url, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> cb)
  {
-    bool success;
-    std::unique_ptr<boost::thread> thread(new boost::thread([&](){ success = download_thread(path, url); }));
-    thread->join();
+    bool success = false;
+    download_async_handle handle = download_async(path, url, [&success](const std::string&, const std::string&, bool result) {success = result;}, cb);
+    download_wait(handle);
    return success;
  }
+
+  download_async_handle download_async(const std::string &path, const std::string &url, std::function<void(const std::string&, const std::string&, bool)> result, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress)
+  {
+    download_async_handle control = std::make_shared<download_thread_control>(path, url, result, progress);
+    control->thread = boost::thread([control](){ download_thread(control); });
+    return control;
+  }
+
+  bool download_finished(const download_async_handle &control)
+  {
+    CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
+    boost::lock_guard<boost::mutex> lock(control->mutex);
+    return control->stopped;
+  }
+
+  bool download_error(const download_async_handle &control)
+  {
+    CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
+    boost::lock_guard<boost::mutex> lock(control->mutex);
+    return !control->success;
+  }
+
+  bool download_wait(const download_async_handle &control)
+  {
+    CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
+    {
+      boost::lock_guard<boost::mutex> lock(control->mutex);
+      if (control->stopped)
+        return true;
+    }
+    control->thread.join();
+    return true;
+  }
+
+  bool download_cancel(const download_async_handle &control)
+  {
+    CHECK_AND_ASSERT_MES(control != 0, false, "NULL async download handle");
+    {
+      boost::lock_guard<boost::mutex> lock(control->mutex);
+      if (control->stopped)
+        return true;
+      control->stop = true;
+    }
+    control->thread.join();
+    return true;
+  }
 }
--- a/src/common/download.h
+++ b/src/common/download.h
@ -32,5 +32,13 @@

 namespace tools
 {
-  bool download(const std::string &path, const std::string &url);
+  struct download_thread_control;
+  typedef std::shared_ptr<download_thread_control> download_async_handle;
+
+  bool download(const std::string &path, const std::string &url, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress = NULL);
+  download_async_handle download_async(const std::string &path, const std::string &url, std::function<void(const std::string&, const std::string&, bool)> result, std::function<bool(const std::string&, const std::string&, size_t, ssize_t)> progress = NULL);
+  bool download_error(const download_async_handle &h);
+  bool download_finished(const download_async_handle &h);
+  bool download_wait(const download_async_handle &h);
+  bool download_cancel(const download_async_handle &h);
 }
--- a/src/cryptonote_core/cryptonote_core.cpp
+++ b/src/cryptonote_core/cryptonote_core.cpp
@ -75,7 +75,8 @@ namespace cryptonote
              m_target_blockchain_height(0),
              m_checkpoints_path(""),
              m_last_dns_checkpoints_update(0),
-              m_last_json_checkpoints_update(0)
+              m_last_json_checkpoints_update(0),
+              m_update_download(0)
  {
    set_cryptonote_protocol(pprotocol);
  }
@ -134,6 +135,15 @@ namespace cryptonote
  void core::stop()
  {
    m_blockchain_storage.cancel();
+
+    tools::download_async_handle handle;
+    {
+      boost::lock_guard<boost::mutex> lock(m_update_mutex);
+      handle = m_update_download;
+      m_update_download = 0;
+    }
+    if (handle)
+      tools::download_cancel(handle);
  }
  //-----------------------------------------------------------------------------------
  void core::init_options(boost::program_options::options_description& desc)
@ -1118,7 +1128,7 @@ namespace cryptonote
      return true;

    std::string url = tools::get_update_url(software, subdir, buildtag, version, true);
-    MGINFO("Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash);
+    MCLOG_CYAN(el::Level::Info, "global", "Version " << version << " of " << software << " for " << buildtag << " is available: " << url << ", SHA256 hash " << hash);

    if (check_updates_level == UPDATES_NOTIFY)
      return true;
@ -1133,32 +1143,55 @@ namespace cryptonote
    boost::filesystem::path path(epee::string_tools::get_current_module_folder());
    path /= filename;

+    boost::unique_lock<boost::mutex> lock(m_update_mutex);
+
+    if (m_update_download != 0)
+    {
+      MCDEBUG("updates", "Already downloading update");
+      return true;
+    }
+
    crypto::hash file_hash;
    if (!tools::sha256sum(path.string(), file_hash) || (hash != epee::string_tools::pod_to_hex(file_hash)))
    {
      MCDEBUG("updates", "We don't have that file already, downloading");
-      if (!tools::download(path.string(), url))
+      m_last_update_length = 0;
+      m_update_download = tools::download_async(path.string(), url, [this, hash](const std::string &path, const std::string &uri, bool success) {
+        if (success)
        {
-        MCERROR("updates", "Failed to download " << url);
-        return false;
-      }
-      if (!tools::sha256sum(path.string(), file_hash))
+          crypto::hash file_hash;
+          if (!tools::sha256sum(path, file_hash))
          {
            MCERROR("updates", "Failed to hash " << path);
-        return false;
          }
          if (hash != epee::string_tools::pod_to_hex(file_hash))
          {
-        MCERROR("updates", "Download from " << url << " does not match the expected hash");
-        return false;
+            MCERROR("updates", "Download from " << uri << " does not match the expected hash");
          }
-      MGINFO("New version downloaded to " << path);
+          MCLOG_CYAN(el::Level::Info, "updates", "New version downloaded to " << path);
+        }
+        else
+        {
+          MCERROR("updates", "Failed to download " << uri);
+        }
+        boost::unique_lock<boost::mutex> lock(m_update_mutex);
+        m_update_download = 0;
+      }, [this](const std::string &path, const std::string &uri, size_t length, ssize_t content_length) {
+        if (length >= m_last_update_length + 1024 * 1024 * 10)
+        {
+          m_last_update_length = length;
+          MCDEBUG("updates", "Downloaded " << length << "/" << (content_length ? std::to_string(content_length) : "unknown"));
+        }
+        return true;
+      });
    }
    else
    {
      MCDEBUG("updates", "We already have " << path << " with expected hash");
    }

+    lock.unlock();
+
    if (check_updates_level == UPDATES_DOWNLOAD)
      return true;

--- a/src/cryptonote_core/cryptonote_core.h
+++ b/src/cryptonote_core/cryptonote_core.h
@ -39,6 +39,7 @@
 #include "p2p/net_node_common.h"
 #include "cryptonote_protocol/cryptonote_protocol_handler_common.h"
 #include "storages/portable_storage_template_helper.h"
+#include "common/download.h"
 #include "tx_pool.h"
 #include "blockchain.h"
 #include "cryptonote_basic/miner.h"
@ -844,6 +845,10 @@ namespace cryptonote
       UPDATES_DOWNLOAD,
       UPDATES_UPDATE,
     } check_updates_level;
+
+     tools::download_async_handle m_update_download;
+     size_t m_last_update_length;
+     boost::mutex m_update_mutex;
   };
 }

--- a/src/rpc/core_rpc_server.cpp
+++ b/src/rpc/core_rpc_server.cpp
@ -1491,8 +1491,8 @@ namespace cryptonote
  bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res)
  {
    static const char software[] = "monero";
-#ifdef BUILDTAG
-    static const char buildtag[] = BOOST_PP_STRINGIZE(BUILDTAG);
+#ifdef BUILD_TAG
+    static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG);
 #else
    static const char buildtag[] = "source";
 #endif