Add ref-counted buffer byte_slice. Currently used for sending TCP data.

contrib/epee/include/byte_slice.h

@@ -0,0 +1,145 @@
+// Copyright (c) 2019, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "span.h"
+
+namespace epee
+{
+  struct byte_slice_data;
+
+  struct release_byte_slice
+  {
+    void operator()(byte_slice_data*) const noexcept;
+  };
+
+  /*! Inspired by slices in golang. Storage is thread-safe reference counted,
+      allowing for cheap copies or range selection on the bytes. The bytes
+      owned by this class are always immutable.
+
+      The functions `operator=`, `take_slice` and `remove_prefix` may alter the
+      reference count for the backing store, which will invalidate pointers
+      previously returned if the reference count is zero. Be careful about
+      "caching" pointers in these circumstances. */
+  class byte_slice
+  {
+    /* A custom reference count is used instead of shared_ptr because it allows
+       for an allocation optimization for the span constructor. This also
+       reduces the size of this class by one pointer. */
+    std::unique_ptr<byte_slice_data, release_byte_slice> storage_;
+    span<const std::uint8_t> portion_; // within storage_
+
+    //! Internal use only; use to increase `storage` reference count.
+    byte_slice(byte_slice_data* storage, span<const std::uint8_t> portion) noexcept;
+
+    struct adapt_buffer{};
+
+    template<typename T>
+    explicit byte_slice(const adapt_buffer, T&& buffer);
+
+  public:
+    using value_type = std::uint8_t;
+    using size_type = std::size_t;
+    using difference_type = std::ptrdiff_t;
+    using pointer = const std::uint8_t*;
+    using const_pointer = const std::uint8_t*;
+    using reference = std::uint8_t;
+    using const_reference = std::uint8_t;
+    using iterator = pointer;
+    using const_iterator = const_pointer;
+
+    //! Construct empty slice.
+    byte_slice() noexcept
+      : storage_(nullptr), portion_()
+    {}
+
+    //! Construct empty slice
+    byte_slice(std::nullptr_t) noexcept
+      : byte_slice()
+    {}
+
+    //! Scatter-gather (copy) multiple `sources` into a single allocated slice.
+    explicit byte_slice(std::initializer_list<span<const std::uint8_t>> sources);
+
+    //! Convert `buffer` into a slice using one allocation for shared count.
+    explicit byte_slice(std::vector<std::uint8_t>&& buffer);
+
+    //! Convert `buffer` into a slice using one allocation for shared count.
+    explicit byte_slice(std::string&& buffer);
+
+    byte_slice(byte_slice&& source) noexcept;
+    ~byte_slice() noexcept = default;
+
+    //! \note May invalidate previously retrieved pointers.
+    byte_slice& operator=(byte_slice&&) noexcept;
+
+    //! \return A shallow (cheap) copy of the data from `this` slice.
+    byte_slice clone() const noexcept { return {storage_.get(), portion_}; }
+
+    iterator begin() const noexcept { return portion_.begin(); }
+    const_iterator cbegin() const noexcept { return portion_.begin(); }
+
+    iterator end() const noexcept { return portion_.end(); }
+    const_iterator cend() const noexcept { return portion_.end(); }
+
+    bool empty() const noexcept { return storage_ == nullptr; }
+    const std::uint8_t* data() const noexcept { return portion_.data(); }
+    std::size_t size() const noexcept { return portion_.size(); }
+
+    /*! Drop bytes from the beginning of `this` slice.
+
+        \note May invalidate previously retrieved pointers.
+        \post `this->size() = this->size() - std::min(this->size(), max_bytes)`
+        \post `if (this->size() <= max_bytes) this->data() = nullptr`
+        \return Number of bytes removed. */
+    std::size_t remove_prefix(std::size_t max_bytes) noexcept;
+
+    /*! "Take" bytes from the beginning of `this` slice.
+
+        \note May invalidate previously retrieved pointers.
+        \post `this->size() = this->size() - std::min(this->size(), max_bytes)`
+        \post `if (this->size() <= max_bytes) this->data() = nullptr`
+        \return Slice containing the bytes removed from `this` slice. */
+    byte_slice take_slice(std::size_t max_bytes) noexcept;
+
+    /*! Return a shallow (cheap) copy of a slice from `begin` and `end` offsets.
+
+        \throw std::out_of_range If `end < begin`.
+        \throw std::out_of_range If `size() < end`.
+        \return Slice starting at `data() + begin` of size `end - begin`. */
+    byte_slice get_slice(std::size_t begin, std::size_t end) const;
+  };
+} // epee
+

contrib/epee/include/net/abstract_tcp_server2.h

@@ -53,6 +53,7 @@
 #include <boost/enable_shared_from_this.hpp>
 #include <boost/interprocess/detail/atomic.hpp>
 #include <boost/thread/thread.hpp>
+#include "byte_slice.h"
 #include "net_utils_base.h"
 #include "syncobj.h"
 #include "connection_basic.hpp"
@@ -135,8 +136,7 @@ namespace net_utils
     
   private:
     //----------------- i_service_endpoint ---------------------
-    virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint)
+    virtual bool do_send(byte_slice message); ///< (see do_send from i_service_endpoint)
-    virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data
     virtual bool send_done();
     virtual bool close();
     virtual bool call_run_once_service_io();
@@ -145,6 +145,8 @@ namespace net_utils
     virtual bool add_ref();
     virtual bool release();
     //------------------------------------------------------
+    bool do_send_chunk(byte_slice chunk); ///< will send (or queue) a part of data. internal use only
+
     boost::shared_ptr<connection<t_protocol_handler> > safe_shared_from_this();
     bool shutdown();
     /// Handle completion of a receive operation.

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -520,7 +520,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   }
   //---------------------------------------------------------------------------------
     template<class t_protocol_handler>
-  bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb) {
+  bool connection<t_protocol_handler>::do_send(byte_slice message) {
     TRY_ENTRY();
 
     // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
@@ -529,6 +529,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     if (m_was_shutdown) return false;
 		// TODO avoid copy
 
+		std::uint8_t const* const message_data = message.data();
+		const std::size_t message_size = message.size();
+
 		const double factor = 32; // TODO config
 		typedef long long signed int t_safe; // my t_size to avoid any overunderflow in arithmetic
 		const t_safe chunksize_good = (t_safe)( 1024 * std::max(1.0,factor) );
@@ -538,13 +541,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         CHECK_AND_ASSERT_MES(! (chunksize_max<0), false, "Negative chunksize_max" ); // make sure it is unsigned before removin sign with cast:
         long long unsigned int chunksize_max_unsigned = static_cast<long long unsigned int>( chunksize_max ) ;
 
-        if (allow_split && (cb > chunksize_max_unsigned)) {
+        if (allow_split && (message_size > chunksize_max_unsigned)) {
 			{ // LOCK: chunking
     		epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical *** 
 
-				MDEBUG("do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr);
+				MDEBUG("do_send() will SPLIT into small chunks, from packet="<<message_size<<" B for ptr="<<message_data);
-				t_safe all = cb; // all bytes to send 
-				t_safe pos = 0; // current sending position
 				// 01234567890 
 				// ^^^^        (pos=0, len=4)     ;   pos:=pos+len, pos=4
 				//     ^^^^    (pos=4, len=4)     ;   pos:=pos+len, pos=8
@@ -554,40 +555,25 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 				// char* buf = new char[ bufsize ];
 
 				bool all_ok = true;
-				while (pos < all) {
+				while (!message.empty()) {
-					t_safe lenall = all-pos; // length from here to end
+					byte_slice chunk = message.take_slice(chunksize_good);
-					t_safe len = std::min( chunksize_good , lenall); // take a smaller part
-					CHECK_AND_ASSERT_MES(len<=chunksize_good, false, "len too large");
-					// pos=8; len=4; all=10;	len=3;
 
-                    CHECK_AND_ASSERT_MES(! (len<0), false, "negative len"); // check before we cast away sign:
+					MDEBUG("chunk_start="<<chunk.data()<<" ptr="<<message_data<<" pos="<<(chunk.data() - message_data));
-                    unsigned long long int len_unsigned = static_cast<long long int>( len );
+					MDEBUG("part of " << message.size() << ": pos="<<(chunk.data() - message_data) << " len="<<chunk.size());
-                    CHECK_AND_ASSERT_MES(len>0, false, "len not strictly positive"); // (redundant)
-                    CHECK_AND_ASSERT_MES(len_unsigned < std::numeric_limits<size_t>::max(), false, "Invalid len_unsigned");   // yeap we want strong < then max size, to be sure
-					
-					void *chunk_start = ((char*)ptr) + pos;
-					MDEBUG("chunk_start="<<chunk_start<<" ptr="<<ptr<<" pos="<<pos);
-					CHECK_AND_ASSERT_MES(chunk_start >= ptr, false, "Pointer wraparound"); // not wrapped around address?
-					//std::memcpy( (void*)buf, chunk_start, len);
 
-					MDEBUG("part of " << lenall << ": pos="<<pos << " len="<<len);
+					bool ok = do_send_chunk(std::move(chunk)); // <====== ***
-
-					bool ok = do_send_chunk(chunk_start, len); // <====== ***
 
 					all_ok = all_ok && ok;
 					if (!all_ok) {
-						MDEBUG("do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr);
+						MDEBUG("do_send() DONE ***FAILED*** from packet="<<message_size<<" B for ptr="<<message_data);
 						MDEBUG("do_send() SEND was aborted in middle of big package - this is mostly harmless "
-							<< " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << cb);
+							<< " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << message_size);
 						return false; // partial failure in sending
 					}
-					pos = pos+len;
-					CHECK_AND_ASSERT_MES(pos >0, false, "pos <= 0");
-
 					// (in catch block, or uniq pointer) delete buf;
 				} // each chunk
 
-				MDEBUG("do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr);
+				MDEBUG("do_send() DONE SPLIT from packet="<<message_size<<" B for ptr="<<message_data);
 
                 MDEBUG("do_send() m_connection_type = " << m_connection_type);
 
@@ -595,7 +581,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 			} // LOCK: chunking
 		} // a big block (to be chunked) - all chunks
 		else { // small block
-			return do_send_chunk(ptr,cb); // just send as 1 big chunk
+			return do_send_chunk(std::move(message)); // just send as 1 big chunk
 		}
 
     CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false);
@@ -603,7 +589,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 
   //---------------------------------------------------------------------------------
   template<class t_protocol_handler>
-  bool connection<t_protocol_handler>::do_send_chunk(const void* ptr, size_t cb)
+  bool connection<t_protocol_handler>::do_send_chunk(byte_slice chunk)
   {
     TRY_ENTRY();
     // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
@@ -623,7 +609,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 
     //_info("[sock " << socket().native_handle() << "] SEND " << cb);
     context.m_last_send = time(NULL);
-    context.m_send_cnt += cb;
+    context.m_send_cnt += chunk.size();
     //some data should be wrote to stream
     //request complete
     
@@ -644,7 +630,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         }*/
 
         long int ms = 250 + (rand()%50);
-        MDEBUG("Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep
+        MDEBUG("Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<chunk.size()); // XXX debug sleep
         m_send_que_lock.unlock();
         boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) );
         m_send_que_lock.lock();
@@ -657,12 +643,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         }
     }
 
-    m_send_que.resize(m_send_que.size()+1);
+    m_send_que.push_back(std::move(chunk));
-    m_send_que.back().assign((const char*)ptr, cb);
+
-    
     if(m_send_que.size() > 1)
     { // active operation should be in progress, nothing to do, just wait last operation callback
-        auto size_now = cb;
+        auto size_now = m_send_que.back().size();
         MDEBUG("do_send_chunk() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size());
         //do_send_handler_delayed( ptr , size_now ); // (((H))) // empty function
       
@@ -680,7 +665,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         auto size_now = m_send_que.front().size();
         MDEBUG("do_send_chunk() NOW SENSD: packet="<<size_now<<" B");
         if (speed_limit_is_enabled())
-			do_send_handler_write( ptr , size_now ); // (((H)))
+			do_send_handler_write( chunk.data(), chunk.size() ); // (((H)))
 
         CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), false, "Unexpected queue size");
         reset_timer(get_default_timeout(), false);

contrib/epee/include/net/connection_basic.hpp

@@ -49,6 +49,7 @@
 #include <boost/asio.hpp>
 #include <boost/asio/ssl.hpp>
 
+#include "byte_slice.h"
 #include "net/net_utils_base.h"
 #include "net/net_ssl.h"
 #include "syncobj.h"
@@ -108,7 +109,7 @@ class connection_basic { // not-templated base class for rapid developmet of som
     volatile uint32_t m_want_close_connection;
     std::atomic<bool> m_was_shutdown;
     critical_section m_send_que_lock;
-    std::list<std::string> m_send_que;
+    std::deque<byte_slice> m_send_que;
     volatile bool m_is_multithreaded;
     /// Strand to ensure the connection's handlers are not called concurrently.
     boost::asio::io_service::strand strand_;

contrib/epee/include/net/http_protocol_handler.inl

@@ -591,11 +591,12 @@ namespace net_utils
 		std::string response_data = get_response_header(response);
 		//LOG_PRINT_L0("HTTP_SEND: << \r\n" << response_data + response.m_body);
 
-    LOG_PRINT_L3("HTTP_RESPONSE_HEAD: << \r\n" << response_data);
+		LOG_PRINT_L3("HTTP_RESPONSE_HEAD: << \r\n" << response_data);
-		
+
-		m_psnd_hndlr->do_send((void*)response_data.data(), response_data.size());
 		if ((response.m_body.size() && (query_info.m_http_method != http::http_method_head)) || (query_info.m_http_method == http::http_method_options))
-			m_psnd_hndlr->do_send((void*)response.m_body.data(), response.m_body.size());
+			response_data += response.m_body;
+
+		m_psnd_hndlr->do_send(byte_slice{std::move(response_data)});
 		m_psnd_hndlr->send_done();
 		return res;
 	}

contrib/epee/include/net/levin_protocol_handler.h

@@ -157,10 +157,9 @@ namespace levin
 						m_current_head.m_return_code = m_config.m_pcommands_handler->invoke(m_current_head.m_command, buff_to_invoke, return_buff, m_conn_context);
 						m_current_head.m_cb = return_buff.size();
 						m_current_head.m_have_to_return_data = false;
-						std::string send_buff((const char*)&m_current_head, sizeof(m_current_head));
-						send_buff += return_buff;
 
-						if(!m_psnd_hndlr->do_send(send_buff.data(), send_buff.size()))
+						return_buff.insert(0, (const char*)&m_current_head, sizeof(m_current_head));
+						if(!m_psnd_hndlr->do_send(byte_slice{std::move(return_buff)}))
 							return false;
 
 					}

contrib/epee/include/net/levin_protocol_handler_async.h

@@ -136,7 +136,6 @@ public:
   critical_section m_local_inv_buff_lock;
   std::string m_local_inv_buff;
 
-  critical_section m_send_lock;
   critical_section m_call_lock;
 
   volatile uint32_t m_wait_count;
@@ -260,6 +259,34 @@ public:
     return handler->is_timer_started();
   }
   template<class callback_t> friend struct anvoke_handler;
+
+  static bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept
+  {
+    bucket_head2 head = {0};
+    head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
+    head.m_have_to_return_data = expect_response;
+    head.m_cb = SWAP64LE(msg_size);
+
+    head.m_command = SWAP32LE(command);
+    head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
+    head.m_flags = SWAP32LE(flags);
+    return head;
+  }
+
+  bool send_message(uint32_t command, epee::span<const uint8_t> in_buff, uint32_t flags, bool expect_response)
+  {
+    const bucket_head2 head = make_header(command, in_buff.size(), flags, expect_response);
+    if(!m_pservice_endpoint->do_send(byte_slice{as_byte_span(head), in_buff}))
+      return false;
+
+    MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb
+        << ", flags" << head.m_flags
+        << ", r?=" << head.m_have_to_return_data
+        <<", cmd = " << head.m_command
+        << ", ver=" << head.m_protocol_version);
+    return true;
+  }
+
 public:
   async_protocol_handler(net_utils::i_service_endpoint* psnd_hndlr, 
     config_type& config, 
@@ -458,37 +485,22 @@ public:
             if(m_current_head.m_have_to_return_data)
             {
               std::string return_buff;
-              m_current_head.m_return_code = m_config.m_pcommands_handler->invoke(
+              const uint32_t return_code = m_config.m_pcommands_handler->invoke(
-                                                                  m_current_head.m_command, 
+                m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context
-                                                                  buff_to_invoke, 
+              );
-                                                                  return_buff, 
+
-                                                                  m_connection_context);
+	      bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false);
-              m_current_head.m_cb = return_buff.size();
+	      head.m_return_code = SWAP32LE(return_code);
-              m_current_head.m_have_to_return_data = false;
+	      return_buff.insert(0, reinterpret_cast<const char*>(&head), sizeof(head));
-              m_current_head.m_protocol_version = LEVIN_PROTOCOL_VER_1;
+
-              m_current_head.m_flags = LEVIN_PACKET_RESPONSE;
+              if(!m_pservice_endpoint->do_send(byte_slice{std::move(return_buff)}))
-#if BYTE_ORDER == LITTLE_ENDIAN
-              std::string send_buff((const char*)&m_current_head, sizeof(m_current_head));
-#else
-              bucket_head2 head = m_current_head;
-              head.m_signature = SWAP64LE(head.m_signature);
-              head.m_cb = SWAP64LE(head.m_cb);
-              head.m_command = SWAP32LE(head.m_command);
-              head.m_return_code = SWAP32LE(head.m_return_code);
-              head.m_flags = SWAP32LE(head.m_flags);
-              head.m_protocol_version = SWAP32LE(head.m_protocol_version);
-              std::string send_buff((const char*)&head, sizeof(head));
-#endif
-              send_buff += return_buff;
-              CRITICAL_REGION_BEGIN(m_send_lock);
-              if(!m_pservice_endpoint->do_send(send_buff.data(), send_buff.size()))
                 return false;
-              CRITICAL_REGION_END();
+
-              MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << m_current_head.m_cb
+              MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb
-                << ", flags" << m_current_head.m_flags 
+                << ", flags" << head.m_flags
-                << ", r?=" << m_current_head.m_have_to_return_data 
+                << ", r?=" << head.m_have_to_return_data
-                <<", cmd = " << m_current_head.m_command 
+                <<", cmd = " << head.m_command
-                << ", ver=" << m_current_head.m_protocol_version);
+                << ", ver=" << head.m_protocol_version);
             }
             else
               m_config.m_pcommands_handler->notify(m_current_head.m_command, buff_to_invoke, m_connection_context);
@@ -584,26 +596,10 @@ public:
         break;
       }
 
-      bucket_head2 head = {0};
-      head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
-      head.m_cb = SWAP64LE(in_buff.size());
-      head.m_have_to_return_data = true;
-
-      head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
-      head.m_command = SWAP32LE(command);
-      head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
-
       boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0);
-      CRITICAL_REGION_BEGIN(m_send_lock);
+      CRITICAL_REGION_BEGIN(m_invoke_response_handlers_lock);
-      CRITICAL_REGION_LOCAL1(m_invoke_response_handlers_lock);
-      if(!m_pservice_endpoint->do_send(&head, sizeof(head)))
-      {
-        LOG_ERROR_CC(m_connection_context, "Failed to do_send");
-        err_code = LEVIN_ERROR_CONNECTION;
-        break;
-      }
 
-      if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size()))
+      if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true))
       {
         LOG_ERROR_CC(m_connection_context, "Failed to do_send");
         err_code = LEVIN_ERROR_CONNECTION;
@@ -642,36 +638,14 @@ public:
     if(m_deletion_initiated)
       return LEVIN_ERROR_CONNECTION_DESTROYED;
 
-    bucket_head2 head = {0};
-    head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
-    head.m_cb = SWAP64LE(in_buff.size());
-    head.m_have_to_return_data = true;
-
-    head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
-    head.m_command = SWAP32LE(command);
-    head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
-
     boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0);
-    CRITICAL_REGION_BEGIN(m_send_lock);
+
-    if(!m_pservice_endpoint->do_send(&head, sizeof(head)))
+    if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true))
     {
-      LOG_ERROR_CC(m_connection_context, "Failed to do_send");
+      LOG_ERROR_CC(m_connection_context, "Failed to send request");
       return LEVIN_ERROR_CONNECTION;
     }
 
-    if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size()))
-    {
-      LOG_ERROR_CC(m_connection_context, "Failed to do_send");
-      return LEVIN_ERROR_CONNECTION;
-    }
-    CRITICAL_REGION_END();
-
-    MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb
-                            << ", f=" << head.m_flags 
-                            << ", r?=" << head.m_have_to_return_data 
-                            << ", cmd = " << head.m_command 
-                            << ", ver=" << head.m_protocol_version);
-
     uint64_t ticks_start = misc_utils::get_tick_count();
     size_t prev_size = 0;
 
@@ -716,33 +690,12 @@ public:
     if(m_deletion_initiated)
       return LEVIN_ERROR_CONNECTION_DESTROYED;
 
-    bucket_head2 head = {0};
+    if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, false))
-    head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
-    head.m_have_to_return_data = false;
-    head.m_cb = SWAP64LE(in_buff.size());
-
-    head.m_command = SWAP32LE(command);
-    head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
-    head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
-    CRITICAL_REGION_BEGIN(m_send_lock);
-    if(!m_pservice_endpoint->do_send(&head, sizeof(head)))
     {
-      LOG_ERROR_CC(m_connection_context, "Failed to do_send()");
+      LOG_ERROR_CC(m_connection_context, "Failed to send notify message");
       return -1;
     }
 
-    if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size()))
-    {
-      LOG_ERROR_CC(m_connection_context, "Failed to do_send()");
-      return -1;
-    }
-    CRITICAL_REGION_END();
-    LOG_DEBUG_CC(m_connection_context, "LEVIN_PACKET_SENT. [len=" << head.m_cb <<
-      ", f=" << head.m_flags << 
-      ", r?=" << head.m_have_to_return_data <<
-      ", cmd = " << head.m_command << 
-      ", ver=" << head.m_protocol_version);
-
     return 1;
   }
   //------------------------------------------------------------------------------------------

contrib/epee/include/net/net_utils_base.h

@@ -34,9 +34,10 @@
 #include <boost/asio/ip/address_v6.hpp>
 #include <typeinfo>
 #include <type_traits>
+#include "byte_slice.h"
 #include "enums.h"
-#include "serialization/keyvalue_serialization.h"
 #include "misc_log_ex.h"
+#include "serialization/keyvalue_serialization.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net"
@@ -424,7 +425,7 @@ namespace net_utils
 	/************************************************************************/
 	struct i_service_endpoint
 	{
-		virtual bool do_send(const void* ptr, size_t cb)=0;
+		virtual bool do_send(byte_slice message)=0;
     virtual bool close()=0;
     virtual bool send_done()=0;
     virtual bool call_run_once_service_io()=0;

contrib/epee/src/CMakeLists.txt

@@ -26,8 +26,8 @@
 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp memwipe.c
+add_library(epee STATIC byte_slice.cpp hex.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp
-    connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp)
+    memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp)
 
 if (USE_READLINE AND GNU_READLINE_FOUND)
   add_library(epee_readline STATIC readline_buffer.cpp)

contrib/epee/src/byte_slice.cpp

@@ -0,0 +1,209 @@
+// Copyright (c) 2019, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <atomic>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+#include <utility>
+
+#include "byte_slice.h"
+
+namespace epee
+{
+  struct byte_slice_data
+  {
+    byte_slice_data() noexcept
+      : ref_count(1)
+    {}
+
+    virtual ~byte_slice_data() noexcept
+    {}
+
+    std::atomic<std::size_t> ref_count;
+  };
+
+  void release_byte_slice::operator()(byte_slice_data* ptr) const noexcept
+  {
+    if (ptr && --(ptr->ref_count) == 0)
+    {
+      ptr->~byte_slice_data();
+      free(ptr);
+    }
+  }
+
+  namespace
+  {
+    template<typename T>
+    struct adapted_byte_slice final : byte_slice_data
+    {
+      explicit adapted_byte_slice(T&& buffer)
+        : byte_slice_data(), buffer(std::move(buffer))
+      {}
+
+      virtual ~adapted_byte_slice() noexcept final override
+      {}
+
+      const T buffer;
+    };
+
+    // bytes "follow" this structure in memory slab
+    struct raw_byte_slice final : byte_slice_data
+    {
+      raw_byte_slice() noexcept
+        : byte_slice_data()
+      {}
+
+      virtual ~raw_byte_slice() noexcept final override
+      {}
+    };
+
+    /* This technique is not-standard, but allows for the reference count and
+       memory for the bytes (when given a list of spans) to be allocated in a
+       single call. In that situation, the dynamic sized bytes are after/behind
+       the raw_byte_slice class. The C runtime has to track the number of bytes
+       allocated regardless, so free'ing is relatively easy. */
+
+    template<typename T, typename... U>
+    std::unique_ptr<T, release_byte_slice> allocate_slice(std::size_t extra_bytes, U&&... args)
+    {
+      if (std::numeric_limits<std::size_t>::max() - sizeof(T) < extra_bytes)
+        throw std::bad_alloc{};
+
+      void* const ptr = malloc(sizeof(T) + extra_bytes);
+      if (ptr == nullptr)
+        throw std::bad_alloc{};
+
+      try
+      {
+        new (ptr) T{std::forward<U>(args)...};
+      }
+      catch (...)
+      {
+        free(ptr);
+        throw;
+      }
+      return std::unique_ptr<T, release_byte_slice>{reinterpret_cast<T*>(ptr)};
+    }
+  } // anonymous
+
+  byte_slice::byte_slice(byte_slice_data* storage, span<const std::uint8_t> portion) noexcept
+    : storage_(storage), portion_(portion)
+  {
+    if (storage_)
+      ++(storage_->ref_count);
+  }
+
+  template<typename T>
+  byte_slice::byte_slice(const adapt_buffer, T&& buffer)
+    : storage_(nullptr), portion_(to_byte_span(to_span(buffer)))
+  {
+    if (!buffer.empty())
+      storage_ = allocate_slice<adapted_byte_slice<T>>(0, std::move(buffer));
+  }
+
+  byte_slice::byte_slice(std::initializer_list<span<const std::uint8_t>> sources)
+    : byte_slice()
+  {
+    std::size_t space_needed = 0;
+    for (const auto source : sources)
+      space_needed += source.size();
+
+    if (space_needed)
+    {
+      auto storage = allocate_slice<raw_byte_slice>(space_needed);
+      span<std::uint8_t> out{reinterpret_cast<std::uint8_t*>(storage.get() + 1), space_needed};
+      portion_ = {out.data(), out.size()};
+
+      for (const auto source : sources)
+      {
+        std::memcpy(out.data(), source.data(), source.size());
+        if (out.remove_prefix(source.size()) < source.size())
+          throw std::bad_alloc{}; // size_t overflow on space_needed
+      }
+      storage_ = std::move(storage);
+    }
+  }
+
+  byte_slice::byte_slice(std::string&& buffer)
+    : byte_slice(adapt_buffer{}, std::move(buffer))
+  {}
+
+  byte_slice::byte_slice(std::vector<std::uint8_t>&& buffer)
+    : byte_slice(adapt_buffer{}, std::move(buffer))
+  {}
+
+  byte_slice::byte_slice(byte_slice&& source) noexcept
+    : storage_(std::move(source.storage_)), portion_(source.portion_)
+  {
+    source.portion_ = epee::span<const std::uint8_t>{};
+  }
+
+  byte_slice& byte_slice::operator=(byte_slice&& source) noexcept
+  {
+    storage_ = std::move(source.storage_);
+    portion_ = source.portion_;
+    if (source.storage_ == nullptr)
+      source.portion_ = epee::span<const std::uint8_t>{};
+
+    return *this;
+  }
+
+  std::size_t byte_slice::remove_prefix(std::size_t max_bytes) noexcept
+  {
+    max_bytes = portion_.remove_prefix(max_bytes);
+    if (portion_.empty())
+      storage_ = nullptr;
+    return max_bytes;
+  }
+
+  byte_slice byte_slice::take_slice(const std::size_t max_bytes) noexcept
+  {
+    byte_slice out{};
+    std::uint8_t const* const ptr = data();
+    out.portion_ = {ptr, portion_.remove_prefix(max_bytes)};
+
+    if (portion_.empty())
+      out.storage_ = std::move(storage_); // no atomic inc/dec
+    else
+      out = {storage_.get(), out.portion_};
+
+    return out;
+  }
+
+  byte_slice byte_slice::get_slice(const std::size_t begin, const std::size_t end) const
+  {
+    if (end < begin || portion_.size() < end)
+      throw std::out_of_range{"bad slice range"};
+
+    if (begin == end)
+      return {};
+    return {storage_.get(), {portion_.begin() + begin, end - begin}};
+  }
+} // epee

tests/fuzz/levin.cpp

@@ -149,11 +149,11 @@ namespace
     }
 
     // Implement epee::net_utils::i_service_endpoint interface
-    virtual bool do_send(const void* ptr, size_t cb)
+    virtual bool do_send(epee::byte_slice message)
     {
       m_send_counter.inc();
       boost::unique_lock<boost::mutex> lock(m_mutex);
-      m_last_send_data.append(reinterpret_cast<const char*>(ptr), cb);
+      m_last_send_data.append(reinterpret_cast<const char*>(message.data()), message.size());
       return m_send_return;
     }
 

tests/unit_tests/epee_levin_protocol_handler_async.cpp

@@ -140,12 +140,12 @@ namespace
     }
 
     // Implement epee::net_utils::i_service_endpoint interface
-    virtual bool do_send(const void* ptr, size_t cb)
+    virtual bool do_send(epee::byte_slice message)
     {
       //std::cout << "test_connection::do_send()" << std::endl;
       m_send_counter.inc();
       boost::unique_lock<boost::mutex> lock(m_mutex);
-      m_last_send_data.append(reinterpret_cast<const char*>(ptr), cb);
+      m_last_send_data.append(reinterpret_cast<const char*>(message.data()), message.size());
       return m_send_return;
     }
 
@@ -367,8 +367,8 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process
   // Parse send data
   std::string send_data = conn->last_send_data();
   epee::levin::bucket_head2 resp_head;
-  resp_head = *reinterpret_cast<const epee::levin::bucket_head2*>(send_data.data());
   ASSERT_LT(sizeof(resp_head), send_data.size());
+  std::memcpy(std::addressof(resp_head), send_data.data(), sizeof(resp_head));
   std::string out_data = send_data.substr(sizeof(resp_head));
 
   // Check sent response

tests/unit_tests/epee_utils.cpp

@@ -44,6 +44,7 @@
 
 #include "boost/archive/portable_binary_iarchive.hpp"
 #include "boost/archive/portable_binary_oarchive.hpp"
+#include "byte_slice.h"
 #include "hex.h"
 #include "net/net_utils_base.h"
 #include "net/local_ip.h"
@@ -375,6 +376,438 @@ TEST(Span, ToMutSpan)
   EXPECT_EQ((std::vector<unsigned>{1, 2, 3, 4}), mut);
 }
 
+TEST(ByteSlice, Construction)
+{
+  EXPECT_TRUE(std::is_default_constructible<epee::byte_slice>());
+  EXPECT_TRUE(std::is_move_constructible<epee::byte_slice>());
+  EXPECT_FALSE(std::is_copy_constructible<epee::byte_slice>());
+  EXPECT_TRUE(std::is_move_assignable<epee::byte_slice>());
+  EXPECT_FALSE(std::is_copy_assignable<epee::byte_slice>());
+}
+
+TEST(ByteSlice, NoExcept)
+{
+  EXPECT_TRUE(std::is_nothrow_default_constructible<epee::byte_slice>());
+  EXPECT_TRUE(std::is_nothrow_move_constructible<epee::byte_slice>());
+  EXPECT_TRUE(std::is_nothrow_move_assignable<epee::byte_slice>());
+
+  epee::byte_slice lvalue{};
+  const epee::byte_slice clvalue{};
+
+  EXPECT_TRUE(noexcept(lvalue.clone()));
+  EXPECT_TRUE(noexcept(clvalue.clone()));
+
+  EXPECT_TRUE(noexcept(lvalue.begin()));
+  EXPECT_TRUE(noexcept(clvalue.begin()));
+  EXPECT_TRUE(noexcept(lvalue.end()));
+  EXPECT_TRUE(noexcept(clvalue.end()));
+
+  EXPECT_TRUE(noexcept(lvalue.cbegin()));
+  EXPECT_TRUE(noexcept(clvalue.cbegin()));
+  EXPECT_TRUE(noexcept(lvalue.cend()));
+  EXPECT_TRUE(noexcept(clvalue.cend()));
+
+  EXPECT_TRUE(noexcept(lvalue.empty()));
+  EXPECT_TRUE(noexcept(clvalue.empty()));
+
+  EXPECT_TRUE(noexcept(lvalue.data()));
+  EXPECT_TRUE(noexcept(clvalue.data()));
+  EXPECT_TRUE(noexcept(lvalue.size()));
+  EXPECT_TRUE(noexcept(clvalue.size()));
+
+  EXPECT_TRUE(noexcept(lvalue.remove_prefix(0)));
+  EXPECT_TRUE(noexcept(lvalue.take_slice(0)));
+}
+
+TEST(ByteSlice, Empty)
+{
+  epee::byte_slice slice{};
+
+  EXPECT_EQ(slice.begin(), slice.end());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(slice.end(), slice.cend());
+
+  EXPECT_TRUE(slice.empty());
+  EXPECT_EQ(0u, slice.size());
+  EXPECT_EQ(slice.begin(), slice.data());
+
+  EXPECT_EQ(0u, slice.get_slice(0, 0).size());
+  EXPECT_THROW(slice.get_slice(0, 1), std::out_of_range);
+  EXPECT_EQ(0u, slice.remove_prefix(1));
+  EXPECT_EQ(0u, slice.take_slice(1).size());
+}
+
+TEST(ByteSlice, CopySpans)
+{
+  const epee::span<const std::uint8_t> part1 = epee::as_byte_span("this is part1");
+  const epee::span<const std::uint8_t> part2 = epee::as_byte_span("then part2");
+  const epee::span<const std::uint8_t> part3 = epee::as_byte_span("finally part3");
+
+  const epee::byte_slice slice{part1, part2, part3};
+
+  EXPECT_NE(nullptr, slice.begin());
+  EXPECT_NE(nullptr, slice.end());
+  EXPECT_NE(slice.begin(), slice.end());
+  EXPECT_NE(slice.cbegin(), slice.cend());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(slice.end(), slice.cend());
+  ASSERT_EQ(slice.size(), std::size_t(slice.end() - slice.begin()));
+
+  EXPECT_FALSE(slice.empty());
+  EXPECT_EQ(slice.begin(), slice.data());
+  ASSERT_EQ(part1.size() + part2.size() + part3.size(), slice.size());
+  EXPECT_TRUE(
+    boost::range::equal(
+      part1, boost::make_iterator_range(slice.begin(), slice.begin() + part1.size())
+    )
+  );
+  EXPECT_TRUE(
+    boost::range::equal(
+      part2, boost::make_iterator_range(slice.begin() + part1.size(), slice.end() - part3.size())
+    )
+  );
+  EXPECT_TRUE(
+    boost::range::equal(
+      part3, boost::make_iterator_range(slice.end() - part3.size(), slice.end())
+    )
+  );
+}
+
+TEST(ByteSlice, AdaptString)
+{
+  static constexpr const char base_string[] = "this is an example message";
+  std::string adapted = base_string;
+
+  const epee::span<const uint8_t> original = epee::to_byte_span(epee::to_span(adapted));
+  const epee::byte_slice slice{std::move(adapted)};
+
+  EXPECT_EQ(original.begin(), slice.begin());
+  EXPECT_EQ(original.cbegin(), slice.cbegin());
+  EXPECT_EQ(original.end(), slice.end());
+  EXPECT_EQ(original.cend(), slice.cend());
+
+  EXPECT_FALSE(slice.empty());
+  EXPECT_EQ(original.data(), slice.data());
+  EXPECT_EQ(original.size(), slice.size());
+  EXPECT_TRUE(boost::range::equal(boost::string_ref{base_string}, slice));
+}
+
+TEST(ByteSlice, EmptyAdaptString)
+{
+  epee::byte_slice slice{std::string{}};
+
+  EXPECT_EQ(slice.begin(), slice.end());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(slice.end(), slice.cend());
+
+  EXPECT_TRUE(slice.empty());
+  EXPECT_EQ(0u, slice.size());
+  EXPECT_EQ(slice.begin(), slice.data());
+
+  EXPECT_EQ(0u, slice.get_slice(0, 0).size());
+  EXPECT_THROW(slice.get_slice(0, 1), std::out_of_range);
+  EXPECT_EQ(0u, slice.remove_prefix(1));
+  EXPECT_EQ(0u, slice.take_slice(1).size());
+}
+
+TEST(ByteSlice, AdaptVector)
+{
+  static constexpr const char base_string[] = "this is an example message";
+  std::vector<std::uint8_t> adapted(sizeof(base_string));
+
+  ASSERT_EQ(sizeof(base_string), adapted.size());
+  std::memcpy(adapted.data(), base_string, sizeof(base_string));
+
+  const epee::span<const uint8_t> original = epee::to_span(adapted);
+  const epee::byte_slice slice{std::move(adapted)};
+
+  EXPECT_EQ(sizeof(base_string), original.size());
+
+  EXPECT_EQ(original.begin(), slice.begin());
+  EXPECT_EQ(original.cbegin(), slice.cbegin());
+  EXPECT_EQ(original.end(), slice.end());
+  EXPECT_EQ(original.cend(), slice.cend());
+
+  EXPECT_FALSE(slice.empty());
+  EXPECT_EQ(original.data(), slice.data());
+  EXPECT_EQ(original.size(), slice.size());
+  EXPECT_TRUE(boost::range::equal(base_string, slice));
+}
+
+TEST(ByteSlice, EmptyAdaptVector)
+{
+  epee::byte_slice slice{std::vector<std::uint8_t>{}};
+
+  EXPECT_EQ(slice.begin(), slice.end());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(slice.end(), slice.cend());
+
+  EXPECT_TRUE(slice.empty());
+  EXPECT_EQ(0u, slice.size());
+  EXPECT_EQ(slice.begin(), slice.data());
+
+  EXPECT_EQ(0u, slice.get_slice(0, 0).size());
+  EXPECT_THROW(slice.get_slice(0, 1), std::out_of_range);
+  EXPECT_EQ(0u, slice.remove_prefix(1));
+  EXPECT_EQ(0u, slice.take_slice(1).size());
+}
+
+TEST(ByteSlice, Move)
+{
+  static constexpr const char base_string[] = "another example message";
+
+  epee::byte_slice slice{epee::as_byte_span(base_string)};
+  EXPECT_TRUE(boost::range::equal(base_string, slice));
+
+  const epee::span<const std::uint8_t> original = epee::to_span(slice);
+  epee::byte_slice moved{std::move(slice)};
+  EXPECT_TRUE(boost::range::equal(base_string, moved));
+
+  EXPECT_EQ(slice.begin(), slice.end());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(slice.end(), slice.cend());
+
+  EXPECT_EQ(original.begin(), moved.begin());
+  EXPECT_EQ(moved.begin(), moved.cbegin());
+  EXPECT_EQ(original.end(), moved.end());
+  EXPECT_EQ(moved.end(), moved.cend());
+
+  EXPECT_TRUE(slice.empty());
+  EXPECT_EQ(slice.begin(), slice.data());
+  EXPECT_EQ(0u, slice.size());
+
+  EXPECT_FALSE(moved.empty());
+  EXPECT_EQ(moved.begin(), moved.data());
+  EXPECT_EQ(original.size(), moved.size());
+
+  slice = std::move(moved);
+  EXPECT_TRUE(boost::range::equal(base_string, slice));
+
+  EXPECT_EQ(original.begin(), slice.begin());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(original.end(), slice.end());
+  EXPECT_EQ(slice.end(), slice.cend());
+
+  EXPECT_FALSE(slice.empty());
+  EXPECT_EQ(slice.begin(), slice.data());
+  EXPECT_EQ(original.size(), slice.size());
+
+  EXPECT_TRUE(moved.empty());
+  EXPECT_EQ(moved.begin(), moved.data());
+  EXPECT_EQ(0u, moved.size());
+}
+
+TEST(ByteSlice, Clone)
+{
+  static constexpr const char base_string[] = "another example message";
+
+  const epee::byte_slice slice{epee::as_byte_span(base_string)};
+  EXPECT_TRUE(boost::range::equal(base_string, slice));
+
+  const epee::byte_slice clone{slice.clone()};
+  EXPECT_TRUE(boost::range::equal(base_string, clone));
+
+  EXPECT_EQ(slice.begin(), clone.begin());
+  EXPECT_EQ(slice.cbegin(), clone.cbegin());
+  EXPECT_EQ(slice.end(), clone.end());
+  EXPECT_EQ(slice.cend(), clone.cend());
+
+  EXPECT_FALSE(slice.empty());
+  EXPECT_FALSE(clone.empty());
+  EXPECT_EQ(slice.cbegin(), slice.data());
+  EXPECT_EQ(slice.data(), clone.data());
+  EXPECT_EQ(sizeof(base_string), slice.size());
+  EXPECT_EQ(slice.size(), clone.size());
+}
+
+TEST(ByteSlice, RemovePrefix)
+{
+  static constexpr const char base_string[] = "another example message";
+  static constexpr std::size_t remove_size = sizeof("another");
+  static constexpr std::size_t remaining = sizeof(base_string) - remove_size;
+
+  epee::byte_slice slice{epee::as_byte_span(base_string)};
+  EXPECT_TRUE(boost::range::equal(base_string, slice));
+
+  const epee::span<const std::uint8_t> original = epee::to_span(slice);
+  EXPECT_EQ(remove_size, slice.remove_prefix(remove_size));
+
+  EXPECT_EQ(original.begin() + remove_size, slice.begin());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(original.end(), slice.end());
+  EXPECT_EQ(slice.end(), slice.cend());
+
+  EXPECT_FALSE(slice.empty());
+  EXPECT_EQ(slice.cbegin(), slice.data());
+  EXPECT_EQ(remaining, slice.size());
+
+  // touch original pointers to check "free" status
+  EXPECT_TRUE(boost::range::equal(base_string, original));
+
+  EXPECT_EQ(remaining, slice.remove_prefix(remaining + 1));
+
+  EXPECT_EQ(slice.begin(), slice.end());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(slice.end(), slice.cend());
+
+  EXPECT_TRUE(slice.empty());
+  EXPECT_EQ(slice.cbegin(), slice.data());
+  EXPECT_EQ(0, slice.size());
+}
+
+TEST(ByteSlice, TakeSlice)
+{
+  static constexpr const char base_string[] = "another example message";
+  static constexpr std::size_t remove_size = sizeof("another");
+  static constexpr std::size_t remaining = sizeof(base_string) - remove_size;
+
+  epee::byte_slice slice{epee::as_byte_span(base_string)};
+  EXPECT_TRUE(boost::range::equal(base_string, slice));
+
+  const epee::span<const std::uint8_t> original = epee::to_span(slice);
+  const epee::byte_slice slice2 = slice.take_slice(remove_size);
+
+  EXPECT_EQ(original.begin() + remove_size, slice.begin());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(original.end(), slice.end());
+  EXPECT_EQ(slice.end(), slice.cend());
+
+  EXPECT_EQ(original.begin(), slice2.begin());
+  EXPECT_EQ(slice2.begin(), slice2.cbegin());
+  EXPECT_EQ(original.begin() + remove_size, slice2.end());
+  EXPECT_EQ(slice2.end(), slice2.cend());
+
+  EXPECT_FALSE(slice.empty());
+  EXPECT_EQ(slice.cbegin(), slice.data());
+  EXPECT_EQ(remaining, slice.size());
+
+  EXPECT_FALSE(slice2.empty());
+  EXPECT_EQ(slice2.cbegin(), slice2.data());
+  EXPECT_EQ(remove_size, slice2.size());
+
+  // touch original pointers to check "free" status
+  EXPECT_TRUE(boost::range::equal(base_string, original));
+
+  const epee::byte_slice slice3 = slice.take_slice(remaining + 1);
+
+  EXPECT_EQ(slice.begin(), slice.end());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(slice.end(), slice.cend());
+
+  EXPECT_EQ(original.begin(), slice2.begin());
+  EXPECT_EQ(slice2.begin(), slice2.cbegin());
+  EXPECT_EQ(original.begin() + remove_size, slice2.end());
+  EXPECT_EQ(slice2.end(), slice2.cend());
+
+  EXPECT_EQ(slice2.end(), slice3.begin());
+  EXPECT_EQ(slice3.begin(), slice3.cbegin());
+  EXPECT_EQ(original.end(), slice3.end());
+  EXPECT_EQ(slice3.end(), slice3.cend());
+
+  EXPECT_TRUE(slice.empty());
+  EXPECT_EQ(slice.cbegin(), slice.data());
+  EXPECT_EQ(0, slice.size());
+
+  EXPECT_FALSE(slice2.empty());
+  EXPECT_EQ(slice2.cbegin(), slice2.data());
+  EXPECT_EQ(remove_size, slice2.size());
+
+  EXPECT_FALSE(slice3.empty());
+  EXPECT_EQ(slice3.cbegin(), slice3.data());
+  EXPECT_EQ(remaining, slice3.size());
+
+  // touch original pointers to check "free" status
+  slice = nullptr;
+  EXPECT_TRUE(boost::range::equal(base_string, original));
+}
+
+TEST(ByteSlice, GetSlice)
+{
+  static constexpr const char base_string[] = "another example message";
+  static constexpr std::size_t get_size = sizeof("another");
+  static constexpr std::size_t get2_size = sizeof(base_string) - get_size;
+
+  epee::span<const std::uint8_t> original{};
+  epee::byte_slice slice2{};
+  epee::byte_slice slice3{};
+
+  // make sure get_slice increments ref count
+  {
+    const epee::byte_slice slice{epee::as_byte_span(base_string)};
+    EXPECT_TRUE(boost::range::equal(base_string, slice));
+
+    original = epee::to_span(slice);
+    slice2 = slice.get_slice(0, get_size);
+
+    EXPECT_EQ(original.begin(), slice.begin());
+    EXPECT_EQ(slice.begin(), slice.cbegin());
+    EXPECT_EQ(original.end(), slice.end());
+    EXPECT_EQ(slice.end(), slice.cend());
+
+    EXPECT_EQ(original.begin(), slice2.begin());
+    EXPECT_EQ(slice2.begin(), slice2.cbegin());
+    EXPECT_EQ(original.begin() + get_size, slice2.end());
+    EXPECT_EQ(slice2.end(), slice2.cend());
+
+    EXPECT_FALSE(slice.empty());
+    EXPECT_EQ(slice.cbegin(), slice.data());
+    EXPECT_EQ(original.size(), slice.size());
+
+    EXPECT_FALSE(slice2.empty());
+    EXPECT_EQ(slice2.cbegin(), slice2.data());
+    EXPECT_EQ(get_size, slice2.size());
+
+    // touch original pointers to check "free" status
+    EXPECT_TRUE(boost::range::equal(base_string, original));
+
+    slice3 = slice.get_slice(get_size, sizeof(base_string));
+
+    EXPECT_EQ(original.begin(), slice.begin());
+    EXPECT_EQ(slice.begin(), slice.cbegin());
+    EXPECT_EQ(original.end(), slice.end());
+    EXPECT_EQ(slice.end(), slice.cend());
+
+    EXPECT_EQ(original.begin(), slice2.begin());
+    EXPECT_EQ(slice2.begin(), slice2.cbegin());
+    EXPECT_EQ(original.begin() + get_size, slice2.end());
+    EXPECT_EQ(slice2.end(), slice2.cend());
+
+    EXPECT_EQ(slice2.end(), slice3.begin());
+    EXPECT_EQ(slice3.begin(), slice3.cbegin());
+    EXPECT_EQ(original.end(), slice3.end());
+    EXPECT_EQ(slice3.end(), slice3.cend());
+
+    EXPECT_FALSE(slice.empty());
+    EXPECT_EQ(slice.cbegin(), slice.data());
+    EXPECT_EQ(original.size(), slice.size());
+
+    EXPECT_FALSE(slice2.empty());
+    EXPECT_EQ(slice2.cbegin(), slice2.data());
+    EXPECT_EQ(get_size, slice2.size());
+
+    EXPECT_FALSE(slice3.empty());
+    EXPECT_EQ(slice3.cbegin(), slice3.data());
+    EXPECT_EQ(get2_size, slice3.size());
+
+    EXPECT_THROW(slice.get_slice(1, 0), std::out_of_range);
+    EXPECT_THROW(slice.get_slice(0, sizeof(base_string) + 1), std::out_of_range);
+    EXPECT_THROW(slice.get_slice(sizeof(base_string) + 1, sizeof(base_string) + 1), std::out_of_range);
+    EXPECT_TRUE(slice.get_slice(sizeof(base_string), sizeof(base_string)).empty());
+
+    EXPECT_EQ(original.begin(), slice.begin());
+    EXPECT_EQ(slice.begin(), slice.cbegin());
+    EXPECT_EQ(original.end(), slice.end());
+    EXPECT_EQ(slice.end(), slice.cend());
+
+    EXPECT_FALSE(slice.empty());
+    EXPECT_EQ(slice.cbegin(), slice.data());
+    EXPECT_EQ(original.size(), slice.size());
+  }
+
+  // touch original pointers to check "free" status
+  EXPECT_TRUE(boost::range::equal(base_string, original));
+}
+
 TEST(ToHex, String)
 {
   EXPECT_TRUE(epee::to_hex::string(nullptr).empty());