nihilist/monero
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

storages: overridable limits for loading portable_storage from binary

Browse Source
This commit is contained in:
moneromooo-monero 2021-01-06 13:19:58 +00:00
parent e144dd5b15
commit 372754a66e
No known key found for this signature in database
GPG Key ID: 686F07454D6CEFC3
5 changed files with 63 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
contrib/epee/include/storages/http_abstract_invoke.h
View File

@ -98,7 +98,12 @@ namespace epee
return false; return false;
} }
return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body)); static const constexpr epee::serialization::portable_storage::limits_t default_http_bin_limits = {
65536 * 3, // objects
65536 * 3, // fields
65536 * 3, // strings
};
return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body), &default_http_bin_limits);
} }
template<class t_request, class t_response, class t_transport> template<class t_request, class t_response, class t_transport>

15
contrib/epee/include/storages/levin_abstract_invoke2.h
View File

@ -52,6 +52,11 @@ namespace
snprintf(buf, sizeof(buf), "command-%u", command); snprintf(buf, sizeof(buf), "command-%u", command);
return on_levin_traffic(context, initiator, sent, error, bytes, buf); return on_levin_traffic(context, initiator, sent, error, bytes, buf);
} }
static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = {
8192, // objects
16384, // fields
16384, // strings
};
} }
namespace epee namespace epee
@ -77,7 +82,7 @@ namespace epee
return false; return false;
} }
serialization::portable_storage stg_ret; serialization::portable_storage stg_ret;
if(!stg_ret.load_from_binary(buff_to_recv)) if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits))
{ {
LOG_ERROR("Failed to load_from_binary on command " << command); LOG_ERROR("Failed to load_from_binary on command " << command);
return false; return false;
@ -124,7 +129,7 @@ namespace epee
return false; return false;
} }
typename serialization::portable_storage stg_ret; typename serialization::portable_storage stg_ret;
if(!stg_ret.load_from_binary(buff_to_recv)) if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits))
{ {
on_levin_traffic(context, true, false, true, buff_to_recv.size(), command); on_levin_traffic(context, true, false, true, buff_to_recv.size(), command);
LOG_ERROR("Failed to load_from_binary on command " << command); LOG_ERROR("Failed to load_from_binary on command " << command);
@ -155,7 +160,7 @@ namespace epee
return false; return false;
} }
serialization::portable_storage stg_ret; serialization::portable_storage stg_ret;
if(!stg_ret.load_from_binary(buff)) if(!stg_ret.load_from_binary(buff, &default_levin_limits))
{ {
on_levin_traffic(context, true, false, true, buff.size(), command); on_levin_traffic(context, true, false, true, buff.size(), command);
LOG_ERROR("Failed to load_from_binary on command " << command); LOG_ERROR("Failed to load_from_binary on command " << command);
@ -205,7 +210,7 @@ namespace epee
int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context ) int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context )
{ {
serialization::portable_storage strg; serialization::portable_storage strg;
if(!strg.load_from_binary(in_buff)) if(!strg.load_from_binary(in_buff, &default_levin_limits))
{ {
on_levin_traffic(context, false, false, true, in_buff.size(), command); on_levin_traffic(context, false, false, true, in_buff.size(), command);
LOG_ERROR("Failed to load_from_binary in command " << command); LOG_ERROR("Failed to load_from_binary in command " << command);
@ -239,7 +244,7 @@ namespace epee
int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context) int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context)
{ {
serialization::portable_storage strg; serialization::portable_storage strg;
if(!strg.load_from_binary(in_buff)) if(!strg.load_from_binary(in_buff, &default_levin_limits))
{ {
on_levin_traffic(context, false, false, true, in_buff.size(), command); on_levin_traffic(context, false, false, true, in_buff.size(), command);
LOG_ERROR("Failed to load_from_binary in notify " << command); LOG_ERROR("Failed to load_from_binary in notify " << command);

15
contrib/epee/include/storages/portable_storage.h
View File

@ -54,6 +54,13 @@ namespace epee
typedef epee::serialization::harray harray; typedef epee::serialization::harray harray;
typedef storage_entry meta_entry; typedef storage_entry meta_entry;
struct limits_t
{
size_t n_objects;
size_t n_fields;
size_t n_strings; // not counting field names
};
portable_storage(){} portable_storage(){}
virtual ~portable_storage(){} virtual ~portable_storage(){}
hsection open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist = false); hsection open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist = false);
@ -84,8 +91,8 @@ namespace epee
//------------------------------------------------------------------------------- //-------------------------------------------------------------------------------
bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192); bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192);
bool load_from_binary(const epee::span<const uint8_t> target); bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL);
bool load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); } bool load_from_binary(const std::string& target, const limits_t *limits = NULL) { return load_from_binary(epee::strspan<uint8_t>(target), limits); }
template<class trace_policy> template<class trace_policy>
bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_xml(std::string& targetObj, const std::string& root_name = "");
bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true);
@ -134,7 +141,7 @@ namespace epee
return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format
} }
inline inline
bool portable_storage::load_from_binary(const epee::span<const uint8_t> source) bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits)
{ {
m_root.m_entries.clear(); m_root.m_entries.clear();
if(source.size() < sizeof(storage_block_header)) if(source.size() < sizeof(storage_block_header))
@ -157,6 +164,8 @@ namespace epee
} }
TRY_ENTRY(); TRY_ENTRY();
throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header)); throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header));
if (limits)
buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings);
buf_reader.read(m_root); buf_reader.read(m_root);
return true;//TODO: return true;//TODO:
CATCH_ENTRY("portable_storage::load_from_binary", false); CATCH_ENTRY("portable_storage::load_from_binary", false);

55
contrib/epee/include/storages/portable_storage_from_bin.h
View File

@ -38,9 +38,6 @@
#else #else
#define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL 100 #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL 100
#endif #endif
#define EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL 65536
#define EPEE_PORTABLE_STORAGE_OBJECT_FIELD_LIMIT_INTERNAL 65536
#define EPEE_PORTABLE_STORAGE_STRING_LIMIT_INTERNAL 65536 // does not include field names
namespace epee namespace epee
{ {
@ -49,21 +46,20 @@ namespace epee
template<typename T> template<typename T>
struct ps_min_bytes { struct ps_min_bytes {
static constexpr const size_t strict = 4096; // actual low bound static constexpr const size_t strict = 4096; // actual low bound
static constexpr const size_t rough = 4096; // when we want to be stricter for DoS prevention
}; };
template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8, rough = 8; }; template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8; };
template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8, rough = 8; }; template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8; };
template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4, rough = 4; }; template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4; };
template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4, rough = 4; }; template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4; };
template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2, rough = 2; }; template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2; };
template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2, rough = 2; }; template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2; };
template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1, rough = 1; }; template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1; };
template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1, rough = 1; }; template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1; };
template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8, rough = 8; }; template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8; };
template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1, rough = 1; }; template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1; };
template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2, rough = 16; }; template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2; };
template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1, rough = 256; }; template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1; };
template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1, rough = 128; }; template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1; };
struct throwable_buffer_reader struct throwable_buffer_reader
{ {
@ -86,6 +82,7 @@ namespace epee
void read(array_entry &ae); void read(array_entry &ae);
template<class t_type> template<class t_type>
size_t min_bytes() const; size_t min_bytes() const;
void set_limits(size_t objects, size_t fields, size_t strings);
private: private:
struct recursuion_limitation_guard struct recursuion_limitation_guard
{ {
@ -109,6 +106,10 @@ namespace epee
size_t m_objects; size_t m_objects;
size_t m_fields; size_t m_fields;
size_t m_strings; size_t m_strings;
size_t max_objects;
size_t max_fields;
size_t max_strings;
}; };
inline throwable_buffer_reader::throwable_buffer_reader(const void* ptr, size_t sz) inline throwable_buffer_reader::throwable_buffer_reader(const void* ptr, size_t sz)
@ -123,6 +124,9 @@ namespace epee
m_objects = 0; m_objects = 0;
m_fields = 0; m_fields = 0;
m_strings = 0; m_strings = 0;
max_objects = std::numeric_limits<size_t>::max();
max_fields = std::numeric_limits<size_t>::max();
max_strings = std::numeric_limits<size_t>::max();
} }
inline inline
void throwable_buffer_reader::read(void* target, size_t count) void throwable_buffer_reader::read(void* target, size_t count)
@ -173,12 +177,12 @@ namespace epee
CHECK_AND_ASSERT_THROW_MES(size <= m_count / ps_min_bytes<type_name>::strict, "Size sanity check failed"); CHECK_AND_ASSERT_THROW_MES(size <= m_count / ps_min_bytes<type_name>::strict, "Size sanity check failed");
if (std::is_same<type_name, section>()) if (std::is_same<type_name, section>())
{ {
CHECK_AND_ASSERT_THROW_MES(size <= EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL - m_objects, "Too many objects"); CHECK_AND_ASSERT_THROW_MES(size <= max_objects - m_objects, "Too many objects");
m_objects += size; m_objects += size;
} }
else if (std::is_same<type_name, std::string>()) else if (std::is_same<type_name, std::string>())
{ {
CHECK_AND_ASSERT_THROW_MES(size <= EPEE_PORTABLE_STORAGE_STRING_LIMIT_INTERNAL - m_strings, "Too many strings"); CHECK_AND_ASSERT_THROW_MES(size <= max_strings - m_strings, "Too many strings");
m_strings += size; m_strings += size;
} }
@ -247,7 +251,7 @@ namespace epee
inline storage_entry throwable_buffer_reader::read_se<std::string>() inline storage_entry throwable_buffer_reader::read_se<std::string>()
{ {
RECURSION_LIMITATION(); RECURSION_LIMITATION();
CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= EPEE_PORTABLE_STORAGE_STRING_LIMIT_INTERNAL, "Too many strings"); CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= max_strings, "Too many strings");
m_strings += 1; m_strings += 1;
return storage_entry(read<std::string>()); return storage_entry(read<std::string>());
} }
@ -257,7 +261,7 @@ namespace epee
inline storage_entry throwable_buffer_reader::read_se<section>() inline storage_entry throwable_buffer_reader::read_se<section>()
{ {
RECURSION_LIMITATION(); RECURSION_LIMITATION();
CHECK_AND_ASSERT_THROW_MES(m_objects < EPEE_PORTABLE_STORAGE_OBJECT_LIMIT_INTERNAL, "Too many objects"); CHECK_AND_ASSERT_THROW_MES(m_objects < max_objects, "Too many objects");
++m_objects; ++m_objects;
section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio
storage_entry se(std::move(s)); storage_entry se(std::move(s));
@ -310,7 +314,7 @@ namespace epee
RECURSION_LIMITATION(); RECURSION_LIMITATION();
sec.m_entries.clear(); sec.m_entries.clear();
size_t count = read_varint(); size_t count = read_varint();
CHECK_AND_ASSERT_THROW_MES(count <= EPEE_PORTABLE_STORAGE_OBJECT_FIELD_LIMIT_INTERNAL - m_fields, "Too many object fields"); CHECK_AND_ASSERT_THROW_MES(count <= max_fields - m_fields, "Too many object fields");
m_fields += count; m_fields += count;
while(count--) while(count--)
{ {
@ -340,5 +344,12 @@ namespace epee
RECURSION_LIMITATION(); RECURSION_LIMITATION();
CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported"); CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported");
} }
inline
void throwable_buffer_reader::set_limits(size_t objects, size_t fields, size_t strings)
{
max_objects = objects;
max_fields = fields;
max_strings = strings;
}
} }
} }

4
contrib/epee/include/storages/portable_storage_template_helper.h
View File

@ -85,10 +85,10 @@ namespace epee
} }
//----------------------------------------------------------------------------------------------------------- //-----------------------------------------------------------------------------------------------------------
template<class t_struct> template<class t_struct>
bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff) bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff, const epee::serialization::portable_storage::limits_t *limits = NULL)
{ {
portable_storage ps; portable_storage ps;
bool rs = ps.load_from_binary(binary_buff); bool rs = ps.load_from_binary(binary_buff, limits);
if(!rs) if(!rs)
return false; return false;