Merge pull request #2929

ae55bacd resumption support for updates using range requests (moneromooo-monero)
fe0fae50 epee: add a get_file_size function (moneromooo-monero)

contrib/epee/include/file_io_utils.h

@@ -133,6 +133,26 @@ namespace file_io_utils
 			return false;
 		}
 	}
+
+	inline
+		bool get_file_size(const std::string& path_to_file, uint64_t &size)
+	{
+		try
+		{
+			std::ifstream fstream;
+			fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+			fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate);
+			size = fstream.tellg();
+			fstream.close();
+			return true;
+		}
+
+		catch(...)
+		{
+			return false;
+		}
+	}
+
 }
 }
 

src/common/download.cpp

@@ -33,6 +33,7 @@
 #include <boost/thread/thread.hpp>
 #include "cryptonote_config.h"
 #include "include_base_utils.h"
+#include "file_io_utils.h"
 #include "net/http_client.h"
 #include "download.h"
 
@@ -74,9 +75,20 @@ namespace tools
     try
     {
       boost::unique_lock<boost::mutex> lock(control->mutex);
-      MINFO("Downloading " << control->uri << " to " << control->path);
+      std::ios_base::openmode mode = std::ios_base::out | std::ios_base::binary;
+      uint64_t existing_size = 0;
+      if (epee::file_io_utils::get_file_size(control->path, existing_size) && existing_size > 0)
+      {
+        MINFO("Resuming downloading " << control->uri << " to " << control->path << " from " << existing_size);
+        mode |= std::ios_base::app;
+      }
+      else
+      {
+        MINFO("Downloading " << control->uri << " to " << control->path);
+        mode |= std::ios_base::trunc;
+      }
       std::ofstream f;
-      f.open(control->path, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
+      f.open(control->path, mode);
       if (!f.good()) {
         MERROR("Failed to open file " << control->path);
         control->result_cb(control->path, control->uri, control->success);
@@ -85,11 +97,13 @@ namespace tools
       class download_client: public epee::net_utils::http::http_simple_client
       {
       public:
-        download_client(download_async_handle control, std::ofstream &f):
+        download_client(download_async_handle control, std::ofstream &f, uint64_t offset = 0):
-          control(control), f(f), content_length(-1), total(0) {}
+          control(control), f(f), content_length(-1), total(0), offset(offset) {}
         virtual ~download_client() { f.close(); }
         virtual bool on_header(const epee::net_utils::http::http_response_info &headers)
         {
+          for (const auto &kv: headers.m_header_info.m_etc_fields)
+            MDEBUG("Header: " << kv.first << ": " << kv.second);
           ssize_t length;
           if (epee::string_tools::get_xtype_from_string(length, headers.m_header_info.m_content_length) && length >= 0)
           {
@@ -104,6 +118,26 @@ namespace tools
               return false;
             }
           }
+          if (offset > 0)
+          {
+            // we requested a range, so check if we're getting it, otherwise truncate
+            bool got_range = false;
+            const std::string prefix = "bytes=" + std::to_string(offset) + "-";
+            for (const auto &kv: headers.m_header_info.m_etc_fields)
+            {
+              if (kv.first == "Content-Range" && strncmp(kv.second.c_str(), prefix.c_str(), prefix.size()))
+              {
+                got_range = true;
+                break;
+              }
+            }
+            if (!got_range)
+            {
+              MWARNING("We did not get the requested range, downloading from start");
+              f.close();
+              f.open(control->path, std::ios_base::out | std::ios_base::binary | std::ios_base::trunc);
+            }
+          }
           return true;
         }
         virtual bool handle_target_data(std::string &piece_of_transfer)
@@ -130,7 +164,8 @@ namespace tools
         std::ofstream &f;
         ssize_t content_length;
         size_t total;
-      } client(control, f);
+        uint64_t offset;
+      } client(control, f, existing_size);
       epee::net_utils::http::url_content u_c;
       if (!epee::net_utils::parse_url(control->uri, u_c))
       {
@@ -160,7 +195,14 @@ namespace tools
       }
       MDEBUG("GETting " << u_c.uri);
       const epee::net_utils::http::http_response_info *info = NULL;
-      if (!client.invoke_get(u_c.uri, std::chrono::seconds(30), "", &info))
+      epee::net_utils::http::fields_list fields;
+      if (existing_size > 0)
+      {
+        const std::string range = "bytes=" + std::to_string(existing_size) + "-";
+        MDEBUG("Asking for range: " << range);
+        fields.push_back(std::make_pair("Range", range));
+      }
+      if (!client.invoke_get(u_c.uri, std::chrono::seconds(30), "", &info, fields))
       {
         boost::lock_guard<boost::mutex> lock(control->mutex);
         MERROR("Failed to connect to " << control->uri);
@@ -190,7 +232,7 @@ namespace tools
       MDEBUG("response body: " << info->m_body);
       for (const auto &f: info->m_additional_fields)
         MDEBUG("additional field: " << f.first << ": " << f.second);
-      if (info->m_response_code != 200)
+      if (info->m_response_code != 200 && info->m_response_code != 206)
       {
         boost::lock_guard<boost::mutex> lock(control->mutex);
         MERROR("Status code " << info->m_response_code);

src/cryptonote_core/cryptonote_core.cpp

@@ -47,6 +47,7 @@ using namespace epee;
 #include "cryptonote_config.h"
 #include "cryptonote_tx_utils.h"
 #include "misc_language.h"
+#include "file_io_utils.h"
 #include <csignal>
 #include "checkpoints/checkpoints.h"
 #include "ringct/rctTypes.h"
@@ -1437,27 +1438,56 @@ namespace cryptonote
     if (!tools::sha256sum(path.string(), file_hash) || (hash != epee::string_tools::pod_to_hex(file_hash)))
     {
       MCDEBUG("updates", "We don't have that file already, downloading");
+      const std::string tmppath = path.string() + ".tmp";
+      if (epee::file_io_utils::is_file_exist(tmppath))
+      {
+        MCDEBUG("updates", "We have part of the file already, resuming download");
+      }
       m_last_update_length = 0;
-      m_update_download = tools::download_async(path.string(), url, [this, hash](const std::string &path, const std::string &uri, bool success) {
+      m_update_download = tools::download_async(tmppath, url, [this, hash, path](const std::string &tmppath, const std::string &uri, bool success) {
+        bool remove = false, good = true;
         if (success)
         {
           crypto::hash file_hash;
-          if (!tools::sha256sum(path, file_hash))
+          if (!tools::sha256sum(tmppath, file_hash))
           {
-            MCERROR("updates", "Failed to hash " << path);
+            MCERROR("updates", "Failed to hash " << tmppath);
+            remove = true;
+            good = false;
           }
-          if (hash != epee::string_tools::pod_to_hex(file_hash))
+          else if (hash != epee::string_tools::pod_to_hex(file_hash))
           {
             MCERROR("updates", "Download from " << uri << " does not match the expected hash");
+            remove = true;
+            good = false;
           }
-          MCLOG_CYAN(el::Level::Info, "updates", "New version downloaded to " << path);
         }
         else
         {
           MCERROR("updates", "Failed to download " << uri);
+          good = false;
         }
         boost::unique_lock<boost::mutex> lock(m_update_mutex);
         m_update_download = 0;
+        if (success && !remove)
+        {
+          std::error_code e = tools::replace_file(tmppath, path.string());
+          if (e)
+          {
+            MCERROR("updates", "Failed to rename downloaded file");
+            good = false;
+          }
+        }
+        else if (remove)
+        {
+          if (!boost::filesystem::remove(tmppath))
+          {
+            MCERROR("updates", "Failed to remove invalid downloaded file");
+            good = false;
+          }
+        }
+        if (good)
+          MCLOG_CYAN(el::Level::Info, "updates", "New version downloaded to " << path.string());
       }, [this](const std::string &path, const std::string &uri, size_t length, ssize_t content_length) {
         if (length >= m_last_update_length + 1024 * 1024 * 10)
         {