mirror of
https://gitlab.torproject.org/tpo/core/tor.git
synced 2024-11-10 13:13:44 +01:00
parent
157af1d26e
commit
bf1c07cb07
@ -23,6 +23,7 @@
|
|||||||
#include "util.h"
|
#include "util.h"
|
||||||
#include "torlog.h"
|
#include "torlog.h"
|
||||||
#include "compress.h"
|
#include "compress.h"
|
||||||
|
#include "compress_lzma.h"
|
||||||
#include "compress_zlib.h"
|
#include "compress_zlib.h"
|
||||||
|
|
||||||
/** @{ */
|
/** @{ */
|
||||||
@ -81,6 +82,9 @@ tor_compress(char **out, size_t *out_len,
|
|||||||
if (method == GZIP_METHOD || method == ZLIB_METHOD)
|
if (method == GZIP_METHOD || method == ZLIB_METHOD)
|
||||||
return tor_zlib_compress(out, out_len, in, in_len, method);
|
return tor_zlib_compress(out, out_len, in, in_len, method);
|
||||||
|
|
||||||
|
if (method == LZMA_METHOD)
|
||||||
|
return tor_lzma_compress(out, out_len, in, in_len, method);
|
||||||
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -108,6 +112,12 @@ tor_uncompress(char **out, size_t *out_len,
|
|||||||
complete_only,
|
complete_only,
|
||||||
protocol_warn_level);
|
protocol_warn_level);
|
||||||
|
|
||||||
|
if (method == LZMA_METHOD)
|
||||||
|
return tor_lzma_uncompress(out, out_len, in, in_len,
|
||||||
|
method,
|
||||||
|
complete_only,
|
||||||
|
protocol_warn_level);
|
||||||
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -123,6 +133,9 @@ detect_compression_method(const char *in, size_t in_len)
|
|||||||
} else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
|
} else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
|
||||||
(ntohs(get_uint16(in)) % 31) == 0) {
|
(ntohs(get_uint16(in)) % 31) == 0) {
|
||||||
return ZLIB_METHOD;
|
return ZLIB_METHOD;
|
||||||
|
} else if (in_len > 3 &&
|
||||||
|
fast_memeq(in, "\x5d\x00\x00\x00", 4)) {
|
||||||
|
return LZMA_METHOD;
|
||||||
} else {
|
} else {
|
||||||
return UNKNOWN_METHOD;
|
return UNKNOWN_METHOD;
|
||||||
}
|
}
|
||||||
@ -135,6 +148,7 @@ struct tor_compress_state_t {
|
|||||||
|
|
||||||
union {
|
union {
|
||||||
tor_zlib_compress_state_t *zlib_state;
|
tor_zlib_compress_state_t *zlib_state;
|
||||||
|
tor_lzma_compress_state_t *lzma_state;
|
||||||
} u; /**< Compression backend state. */
|
} u; /**< Compression backend state. */
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -161,7 +175,16 @@ tor_compress_new(int compress, compress_method_t method,
|
|||||||
state->u.zlib_state = zlib_state;
|
state->u.zlib_state = zlib_state;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
case LZMA_METHOD: {
|
||||||
|
tor_lzma_compress_state_t *lzma_state =
|
||||||
|
tor_lzma_compress_new(compress, method, compression_level);
|
||||||
|
|
||||||
|
if (lzma_state == NULL)
|
||||||
|
goto err;
|
||||||
|
|
||||||
|
state->u.lzma_state = lzma_state;
|
||||||
|
break;
|
||||||
|
}
|
||||||
case NO_METHOD:
|
case NO_METHOD:
|
||||||
case UNKNOWN_METHOD:
|
case UNKNOWN_METHOD:
|
||||||
goto err;
|
goto err;
|
||||||
@ -199,7 +222,10 @@ tor_compress_process(tor_compress_state_t *state,
|
|||||||
return tor_zlib_compress_process(state->u.zlib_state,
|
return tor_zlib_compress_process(state->u.zlib_state,
|
||||||
out, out_len, in, in_len,
|
out, out_len, in, in_len,
|
||||||
finish);
|
finish);
|
||||||
|
case LZMA_METHOD:
|
||||||
|
return tor_lzma_compress_process(state->u.lzma_state,
|
||||||
|
out, out_len, in, in_len,
|
||||||
|
finish);
|
||||||
case NO_METHOD:
|
case NO_METHOD:
|
||||||
case UNKNOWN_METHOD:
|
case UNKNOWN_METHOD:
|
||||||
goto err;
|
goto err;
|
||||||
@ -221,7 +247,9 @@ tor_compress_free(tor_compress_state_t *state)
|
|||||||
case ZLIB_METHOD:
|
case ZLIB_METHOD:
|
||||||
tor_zlib_compress_free(state->u.zlib_state);
|
tor_zlib_compress_free(state->u.zlib_state);
|
||||||
break;
|
break;
|
||||||
|
case LZMA_METHOD:
|
||||||
|
tor_lzma_compress_free(state->u.lzma_state);
|
||||||
|
break;
|
||||||
case NO_METHOD:
|
case NO_METHOD:
|
||||||
case UNKNOWN_METHOD:
|
case UNKNOWN_METHOD:
|
||||||
break;
|
break;
|
||||||
@ -240,7 +268,8 @@ tor_compress_state_size(const tor_compress_state_t *state)
|
|||||||
case GZIP_METHOD:
|
case GZIP_METHOD:
|
||||||
case ZLIB_METHOD:
|
case ZLIB_METHOD:
|
||||||
return tor_zlib_compress_state_size(state->u.zlib_state);
|
return tor_zlib_compress_state_size(state->u.zlib_state);
|
||||||
|
case LZMA_METHOD:
|
||||||
|
return tor_lzma_compress_state_size(state->u.lzma_state);
|
||||||
case NO_METHOD:
|
case NO_METHOD:
|
||||||
case UNKNOWN_METHOD:
|
case UNKNOWN_METHOD:
|
||||||
goto err;
|
goto err;
|
||||||
|
@ -15,7 +15,11 @@
|
|||||||
* GZIP_METHOD is guaranteed to be supported by the compress/uncompress
|
* GZIP_METHOD is guaranteed to be supported by the compress/uncompress
|
||||||
* functions here. */
|
* functions here. */
|
||||||
typedef enum {
|
typedef enum {
|
||||||
NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3
|
NO_METHOD=0,
|
||||||
|
GZIP_METHOD=1,
|
||||||
|
ZLIB_METHOD=2,
|
||||||
|
LZMA_METHOD=3,
|
||||||
|
UNKNOWN_METHOD=4
|
||||||
} compress_method_t;
|
} compress_method_t;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
582
src/common/compress_lzma.c
Normal file
582
src/common/compress_lzma.c
Normal file
@ -0,0 +1,582 @@
|
|||||||
|
/* Copyright (c) 2004, Roger Dingledine.
|
||||||
|
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
|
||||||
|
* Copyright (c) 2007-2017, The Tor Project, Inc. */
|
||||||
|
/* See LICENSE for licensing information */
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \file compress_lzma.c
|
||||||
|
* \brief Compression backend for LZMA.
|
||||||
|
*
|
||||||
|
* This module should never be invoked directly. Use the compress module
|
||||||
|
* instead.
|
||||||
|
**/
|
||||||
|
|
||||||
|
#include "orconfig.h"
|
||||||
|
|
||||||
|
#include "util.h"
|
||||||
|
#include "torlog.h"
|
||||||
|
#include "compress.h"
|
||||||
|
#include "compress_lzma.h"
|
||||||
|
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
#include <lzma.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/** Total number of bytes allocated for LZMA state. */
|
||||||
|
static size_t total_lzma_allocation = 0;
|
||||||
|
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
/** Convert a given <b>error</b> to a human readable error string. */
|
||||||
|
static const char *
|
||||||
|
lzma_error_str(lzma_ret error)
|
||||||
|
{
|
||||||
|
switch (error) {
|
||||||
|
case LZMA_OK:
|
||||||
|
return "Operation completed successfully";
|
||||||
|
case LZMA_STREAM_END:
|
||||||
|
return "End of stream";
|
||||||
|
case LZMA_NO_CHECK:
|
||||||
|
return "Input stream lacks integrity check";
|
||||||
|
case LZMA_UNSUPPORTED_CHECK:
|
||||||
|
return "Unable to calculate integrity check";
|
||||||
|
case LZMA_GET_CHECK:
|
||||||
|
return "Integrity check available";
|
||||||
|
case LZMA_MEM_ERROR:
|
||||||
|
return "Unable to allocate memory";
|
||||||
|
case LZMA_MEMLIMIT_ERROR:
|
||||||
|
return "Memory limit reached";
|
||||||
|
case LZMA_FORMAT_ERROR:
|
||||||
|
return "Unknown file format";
|
||||||
|
case LZMA_OPTIONS_ERROR:
|
||||||
|
return "Unsupported options";
|
||||||
|
case LZMA_DATA_ERROR:
|
||||||
|
return "Corrupt input data";
|
||||||
|
case LZMA_BUF_ERROR:
|
||||||
|
return "Unable to progress";
|
||||||
|
case LZMA_PROG_ERROR:
|
||||||
|
return "Programming error";
|
||||||
|
default:
|
||||||
|
return "Unknown LZMA error";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif // HAVE_LZMA.
|
||||||
|
|
||||||
|
/** Return a string representation of the version of the currently running
|
||||||
|
* version of liblzma. */
|
||||||
|
const char *
|
||||||
|
tor_lzma_get_version_str(void)
|
||||||
|
{
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
return lzma_version_string();
|
||||||
|
#else
|
||||||
|
return "N/A";
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Return a string representation of the version of the version of liblzma
|
||||||
|
* used at compilation. */
|
||||||
|
const char *
|
||||||
|
tor_lzma_get_header_version_str(void)
|
||||||
|
{
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
return LZMA_VERSION_STRING;
|
||||||
|
#else
|
||||||
|
return "N/A";
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
|
||||||
|
* allocated buffer, using the LZMA method. Store the compressed string in
|
||||||
|
* *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on success, -1 on
|
||||||
|
* failure.
|
||||||
|
*/
|
||||||
|
int
|
||||||
|
tor_lzma_compress(char **out, size_t *out_len,
|
||||||
|
const char *in, size_t in_len,
|
||||||
|
compress_method_t method)
|
||||||
|
{
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
lzma_stream stream = LZMA_STREAM_INIT;
|
||||||
|
lzma_options_lzma stream_options;
|
||||||
|
lzma_ret retval;
|
||||||
|
lzma_action action;
|
||||||
|
size_t out_size, old_size;
|
||||||
|
off_t offset;
|
||||||
|
|
||||||
|
tor_assert(out);
|
||||||
|
tor_assert(out_len);
|
||||||
|
tor_assert(in);
|
||||||
|
tor_assert(in_len < UINT_MAX);
|
||||||
|
tor_assert(method == LZMA_METHOD);
|
||||||
|
|
||||||
|
stream.next_in = (unsigned char *)in;
|
||||||
|
stream.avail_in = in_len;
|
||||||
|
|
||||||
|
lzma_lzma_preset(&stream_options,
|
||||||
|
tor_compress_memory_level(HIGH_COMPRESSION));
|
||||||
|
|
||||||
|
retval = lzma_alone_encoder(&stream, &stream_options);
|
||||||
|
|
||||||
|
if (retval != LZMA_OK) {
|
||||||
|
log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).",
|
||||||
|
lzma_error_str(retval), retval);
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
out_size = in_len / 2;
|
||||||
|
if (out_size < 1024)
|
||||||
|
out_size = 1024;
|
||||||
|
|
||||||
|
*out = tor_malloc(out_size);
|
||||||
|
|
||||||
|
stream.next_out = (unsigned char *)*out;
|
||||||
|
stream.avail_out = out_size;
|
||||||
|
|
||||||
|
action = LZMA_RUN;
|
||||||
|
|
||||||
|
while (1) {
|
||||||
|
retval = lzma_code(&stream, action);
|
||||||
|
switch (retval) {
|
||||||
|
case LZMA_OK:
|
||||||
|
action = LZMA_FINISH;
|
||||||
|
break;
|
||||||
|
case LZMA_STREAM_END:
|
||||||
|
goto done;
|
||||||
|
case LZMA_BUF_ERROR:
|
||||||
|
offset = stream.next_out - ((unsigned char *)*out);
|
||||||
|
old_size = out_size;
|
||||||
|
out_size *= 2;
|
||||||
|
|
||||||
|
if (out_size < old_size) {
|
||||||
|
log_warn(LD_GENERAL, "Size overflow in LZMA compression.");
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
*out = tor_realloc(*out, out_size);
|
||||||
|
stream.next_out = (unsigned char *)(*out + offset);
|
||||||
|
if (out_size - offset > UINT_MAX) {
|
||||||
|
log_warn(LD_BUG, "Ran over unsigned int limit of LZMA while "
|
||||||
|
"compressing.");
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
stream.avail_out = (unsigned int)(out_size - offset);
|
||||||
|
break;
|
||||||
|
|
||||||
|
// We list all the possible values of `lzma_ret` here to silence the
|
||||||
|
// `switch-enum` warning and to detect if a new member was added.
|
||||||
|
case LZMA_NO_CHECK:
|
||||||
|
case LZMA_UNSUPPORTED_CHECK:
|
||||||
|
case LZMA_GET_CHECK:
|
||||||
|
case LZMA_MEM_ERROR:
|
||||||
|
case LZMA_MEMLIMIT_ERROR:
|
||||||
|
case LZMA_FORMAT_ERROR:
|
||||||
|
case LZMA_OPTIONS_ERROR:
|
||||||
|
case LZMA_DATA_ERROR:
|
||||||
|
case LZMA_PROG_ERROR:
|
||||||
|
default:
|
||||||
|
log_warn(LD_GENERAL, "LZMA compression didn't finish: %s.",
|
||||||
|
lzma_error_str(retval));
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
done:
|
||||||
|
*out_len = stream.total_out;
|
||||||
|
lzma_end(&stream);
|
||||||
|
|
||||||
|
if (tor_compress_is_compression_bomb(*out_len, in_len)) {
|
||||||
|
log_warn(LD_BUG, "We compressed something and got an insanely high "
|
||||||
|
"compression factor; other Tor instances would think "
|
||||||
|
"this is a compression bomb.");
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
err:
|
||||||
|
lzma_end(&stream);
|
||||||
|
tor_free(*out);
|
||||||
|
return -1;
|
||||||
|
#else // HAVE_LZMA.
|
||||||
|
(void)out;
|
||||||
|
(void)out_len;
|
||||||
|
(void)in;
|
||||||
|
(void)in_len;
|
||||||
|
(void)method;
|
||||||
|
|
||||||
|
return -1;
|
||||||
|
#endif // HAVE_LZMA.
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Given an LZMA compressed string of total length <b>in_len</b> bytes at
|
||||||
|
* <b>in</b>, uncompress them into a newly allocated buffer. Store the
|
||||||
|
* uncompressed string in *<b>out</b>, and its length in *<b>out_len</b>.
|
||||||
|
* Return 0 on success, -1 on failure.
|
||||||
|
*
|
||||||
|
* If <b>complete_only</b> is true, we consider a truncated input as a failure;
|
||||||
|
* otherwise we decompress as much as we can. Warn about truncated or corrupt
|
||||||
|
* inputs at <b>protocol_warn_level</b>.
|
||||||
|
*/
|
||||||
|
int
|
||||||
|
tor_lzma_uncompress(char **out, size_t *out_len,
|
||||||
|
const char *in, size_t in_len,
|
||||||
|
compress_method_t method,
|
||||||
|
int complete_only,
|
||||||
|
int protocol_warn_level)
|
||||||
|
{
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
lzma_stream stream = LZMA_STREAM_INIT;
|
||||||
|
lzma_ret retval;
|
||||||
|
lzma_action action;
|
||||||
|
size_t out_size, old_size;
|
||||||
|
off_t offset;
|
||||||
|
|
||||||
|
tor_assert(out);
|
||||||
|
tor_assert(out_len);
|
||||||
|
tor_assert(in);
|
||||||
|
tor_assert(in_len < UINT_MAX);
|
||||||
|
tor_assert(method == LZMA_METHOD);
|
||||||
|
|
||||||
|
stream.next_in = (unsigned char *)in;
|
||||||
|
stream.avail_in = in_len;
|
||||||
|
|
||||||
|
// FIXME(ahf): This should be something more sensible than
|
||||||
|
// UINT64_MAX: See #21665.
|
||||||
|
retval = lzma_alone_decoder(&stream, UINT64_MAX);
|
||||||
|
|
||||||
|
if (retval != LZMA_OK) {
|
||||||
|
log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).",
|
||||||
|
lzma_error_str(retval), retval);
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
out_size = in_len * 2;
|
||||||
|
if (out_size < 1024)
|
||||||
|
out_size = 1024;
|
||||||
|
|
||||||
|
if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
|
||||||
|
goto err;
|
||||||
|
|
||||||
|
*out = tor_malloc(out_size);
|
||||||
|
stream.next_out = (unsigned char *)*out;
|
||||||
|
stream.avail_out = out_size;
|
||||||
|
|
||||||
|
// FIXME(ahf): We should figure out how to use LZMA_FULL_FLUSH to
|
||||||
|
// make the partial string read tests.
|
||||||
|
// action = complete_only ? LZMA_FINISH : LZMA_SYNC_FLUSH. // To do this,
|
||||||
|
// it seems like we have to use LZMA using their "xz" encoder instead of just
|
||||||
|
// regular LZMA.
|
||||||
|
(void)complete_only;
|
||||||
|
action = LZMA_FINISH;
|
||||||
|
|
||||||
|
while (1) {
|
||||||
|
retval = lzma_code(&stream, action);
|
||||||
|
switch (retval) {
|
||||||
|
case LZMA_STREAM_END:
|
||||||
|
if (stream.avail_in == 0)
|
||||||
|
goto done;
|
||||||
|
|
||||||
|
// We might have more data here. Reset our stream.
|
||||||
|
lzma_end(&stream);
|
||||||
|
|
||||||
|
retval = lzma_alone_decoder(&stream, UINT64_MAX);
|
||||||
|
|
||||||
|
if (retval != LZMA_OK) {
|
||||||
|
log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).",
|
||||||
|
lzma_error_str(retval), retval);
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case LZMA_OK:
|
||||||
|
break;
|
||||||
|
case LZMA_BUF_ERROR:
|
||||||
|
if (stream.avail_out > 0) {
|
||||||
|
log_fn(protocol_warn_level, LD_PROTOCOL,
|
||||||
|
"possible truncated or corrupt LZMA data.");
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
offset = stream.next_out - (unsigned char *)*out;
|
||||||
|
old_size = out_size;
|
||||||
|
out_size *= 2;
|
||||||
|
|
||||||
|
if (out_size < old_size) {
|
||||||
|
log_warn(LD_GENERAL, "Size overflow in LZMA uncompression.");
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tor_compress_is_compression_bomb(in_len, out_size)) {
|
||||||
|
log_warn(LD_GENERAL, "Input looks like a possible LZMA compression "
|
||||||
|
"bomb. Not proceeding.");
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (out_size >= SIZE_T_CEILING) {
|
||||||
|
log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing "
|
||||||
|
"LZMA data.");
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
*out = tor_realloc(*out, out_size);
|
||||||
|
stream.next_out = (unsigned char *)(*out + offset);
|
||||||
|
|
||||||
|
if (out_size - offset > UINT_MAX) {
|
||||||
|
log_warn(LD_BUG, "Ran over unsigned int limit of LZMA while "
|
||||||
|
"uncompressing.");
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
|
||||||
|
stream.avail_out = (unsigned int)(out_size - offset);
|
||||||
|
break;
|
||||||
|
|
||||||
|
// We list all the possible values of `lzma_ret` here to silence the
|
||||||
|
// `switch-enum` warning and to detect if a new member was added.
|
||||||
|
case LZMA_NO_CHECK:
|
||||||
|
case LZMA_UNSUPPORTED_CHECK:
|
||||||
|
case LZMA_GET_CHECK:
|
||||||
|
case LZMA_MEM_ERROR:
|
||||||
|
case LZMA_MEMLIMIT_ERROR:
|
||||||
|
case LZMA_FORMAT_ERROR:
|
||||||
|
case LZMA_OPTIONS_ERROR:
|
||||||
|
case LZMA_DATA_ERROR:
|
||||||
|
case LZMA_PROG_ERROR:
|
||||||
|
default:
|
||||||
|
log_warn(LD_GENERAL, "LZMA decompression didn't finish: %s.",
|
||||||
|
lzma_error_str(retval));
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
done:
|
||||||
|
*out_len = stream.next_out - (unsigned char*)*out;
|
||||||
|
lzma_end(&stream);
|
||||||
|
|
||||||
|
// NUL-terminate our output.
|
||||||
|
if (out_size == *out_len)
|
||||||
|
*out = tor_realloc(*out, out_size + 1);
|
||||||
|
(*out)[*out_len] = '\0';
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
err:
|
||||||
|
lzma_end(&stream);
|
||||||
|
tor_free(*out);
|
||||||
|
return -1;
|
||||||
|
#else // HAVE_LZMA.
|
||||||
|
(void)out;
|
||||||
|
(void)out_len;
|
||||||
|
(void)in;
|
||||||
|
(void)in_len;
|
||||||
|
(void)method;
|
||||||
|
(void)complete_only;
|
||||||
|
(void)protocol_warn_level;
|
||||||
|
|
||||||
|
return -1;
|
||||||
|
#endif // HAVE_LZMA.
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Internal LZMA state for incremental compression/decompression.
|
||||||
|
* The body of this struct is not exposed. */
|
||||||
|
struct tor_lzma_compress_state_t {
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
lzma_stream stream; /**< The LZMA stream. */
|
||||||
|
#endif
|
||||||
|
|
||||||
|
int compress; /**< True if we are compressing; false if we are inflating */
|
||||||
|
|
||||||
|
/** Number of bytes read so far. Used to detect compression bombs. */
|
||||||
|
size_t input_so_far;
|
||||||
|
/** Number of bytes written so far. Used to detect compression bombs. */
|
||||||
|
size_t output_so_far;
|
||||||
|
|
||||||
|
/** Approximate number of bytes allocated for this object. */
|
||||||
|
size_t allocation;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Construct and return a tor_lzma_compress_state_t object using
|
||||||
|
* <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
|
||||||
|
* decompression. */
|
||||||
|
tor_lzma_compress_state_t *
|
||||||
|
tor_lzma_compress_new(int compress,
|
||||||
|
compress_method_t method,
|
||||||
|
compression_level_t compression_level)
|
||||||
|
{
|
||||||
|
tor_assert(method == LZMA_METHOD);
|
||||||
|
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
tor_lzma_compress_state_t *result;
|
||||||
|
lzma_ret retval;
|
||||||
|
lzma_options_lzma stream_options;
|
||||||
|
|
||||||
|
// Note that we do not explicitly initialize the lzma_stream object here,
|
||||||
|
// since the LZMA_STREAM_INIT "just" initializes all members to 0, which is
|
||||||
|
// also what `tor_malloc_zero()` does.
|
||||||
|
result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t));
|
||||||
|
result->compress = compress;
|
||||||
|
|
||||||
|
// FIXME(ahf): We should either try to do the pre-calculation that is done
|
||||||
|
// with the zlib backend or use a custom allocator here where we pass our
|
||||||
|
// tor_lzma_compress_state_t as the opaque value.
|
||||||
|
result->allocation = 0;
|
||||||
|
|
||||||
|
if (compress) {
|
||||||
|
lzma_lzma_preset(&stream_options,
|
||||||
|
tor_compress_memory_level(compression_level));
|
||||||
|
|
||||||
|
retval = lzma_alone_encoder(&result->stream, &stream_options);
|
||||||
|
|
||||||
|
if (retval != LZMA_OK) {
|
||||||
|
log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).",
|
||||||
|
lzma_error_str(retval), retval);
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// FIXME(ahf): This should be something more sensible than
|
||||||
|
// UINT64_MAX: See #21665.
|
||||||
|
retval = lzma_alone_decoder(&result->stream, UINT64_MAX);
|
||||||
|
|
||||||
|
if (retval != LZMA_OK) {
|
||||||
|
log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).",
|
||||||
|
lzma_error_str(retval), retval);
|
||||||
|
goto err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
|
||||||
|
err:
|
||||||
|
tor_free(result);
|
||||||
|
return NULL;
|
||||||
|
#else // HAVE_LZMA.
|
||||||
|
(void)compress;
|
||||||
|
(void)method;
|
||||||
|
(void)compression_level;
|
||||||
|
|
||||||
|
return NULL;
|
||||||
|
#endif // HAVE_LZMA.
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Compress/decompress some bytes using <b>state</b>. Read up to
|
||||||
|
* *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
|
||||||
|
* to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true,
|
||||||
|
* we've reached the end of the input.
|
||||||
|
*
|
||||||
|
* Return TOR_COMPRESS_DONE if we've finished the entire
|
||||||
|
* compression/decompression.
|
||||||
|
* Return TOR_COMPRESS_OK if we're processed everything from the input.
|
||||||
|
* Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
|
||||||
|
* Return TOR_COMPRESS_ERROR if the stream is corrupt.
|
||||||
|
*/
|
||||||
|
tor_compress_output_t
|
||||||
|
tor_lzma_compress_process(tor_lzma_compress_state_t *state,
|
||||||
|
char **out, size_t *out_len,
|
||||||
|
const char **in, size_t *in_len,
|
||||||
|
int finish)
|
||||||
|
{
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
lzma_ret retval;
|
||||||
|
lzma_action action;
|
||||||
|
|
||||||
|
tor_assert(state != NULL);
|
||||||
|
tor_assert(*in_len <= UINT_MAX);
|
||||||
|
tor_assert(*out_len <= UINT_MAX);
|
||||||
|
|
||||||
|
state->stream.next_in = (unsigned char *)*in;
|
||||||
|
state->stream.avail_in = *in_len;
|
||||||
|
state->stream.next_out = (unsigned char *)*out;
|
||||||
|
state->stream.avail_out = *out_len;
|
||||||
|
|
||||||
|
action = finish ? LZMA_FINISH : LZMA_RUN;
|
||||||
|
|
||||||
|
retval = lzma_code(&state->stream, action);
|
||||||
|
|
||||||
|
state->input_so_far += state->stream.next_in - ((unsigned char *)*in);
|
||||||
|
state->output_so_far += state->stream.next_out - ((unsigned char *)*out);
|
||||||
|
|
||||||
|
*out = (char *)state->stream.next_out;
|
||||||
|
*out_len = state->stream.avail_out;
|
||||||
|
*in = (const char *)state->stream.next_in;
|
||||||
|
*in_len = state->stream.avail_in;
|
||||||
|
|
||||||
|
if (! state->compress &&
|
||||||
|
tor_compress_is_compression_bomb(state->input_so_far,
|
||||||
|
state->output_so_far)) {
|
||||||
|
log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
|
||||||
|
return TOR_COMPRESS_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (retval) {
|
||||||
|
case LZMA_OK:
|
||||||
|
if (state->stream.avail_out == 0 || finish)
|
||||||
|
return TOR_COMPRESS_BUFFER_FULL;
|
||||||
|
|
||||||
|
return TOR_COMPRESS_OK;
|
||||||
|
|
||||||
|
case LZMA_BUF_ERROR:
|
||||||
|
if (state->stream.avail_in == 0 && !finish)
|
||||||
|
return TOR_COMPRESS_OK;
|
||||||
|
|
||||||
|
return TOR_COMPRESS_BUFFER_FULL;
|
||||||
|
|
||||||
|
case LZMA_STREAM_END:
|
||||||
|
return TOR_COMPRESS_DONE;
|
||||||
|
|
||||||
|
// We list all the possible values of `lzma_ret` here to silence the
|
||||||
|
// `switch-enum` warning and to detect if a new member was added.
|
||||||
|
case LZMA_NO_CHECK:
|
||||||
|
case LZMA_UNSUPPORTED_CHECK:
|
||||||
|
case LZMA_GET_CHECK:
|
||||||
|
case LZMA_MEM_ERROR:
|
||||||
|
case LZMA_MEMLIMIT_ERROR:
|
||||||
|
case LZMA_FORMAT_ERROR:
|
||||||
|
case LZMA_OPTIONS_ERROR:
|
||||||
|
case LZMA_DATA_ERROR:
|
||||||
|
case LZMA_PROG_ERROR:
|
||||||
|
default:
|
||||||
|
log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.",
|
||||||
|
state->compress ? "compression" : "decompression",
|
||||||
|
lzma_error_str(retval));
|
||||||
|
return TOR_COMPRESS_ERROR;
|
||||||
|
}
|
||||||
|
#else // HAVE_LZMA.
|
||||||
|
(void)state;
|
||||||
|
(void)out;
|
||||||
|
(void)out_len;
|
||||||
|
(void)in;
|
||||||
|
(void)in_len;
|
||||||
|
(void)finish;
|
||||||
|
return TOR_COMPRESS_ERROR;
|
||||||
|
#endif // HAVE_LZMA.
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Deallocate <b>state</b>. */
|
||||||
|
void
|
||||||
|
tor_lzma_compress_free(tor_lzma_compress_state_t *state)
|
||||||
|
{
|
||||||
|
if (state == NULL)
|
||||||
|
return;
|
||||||
|
|
||||||
|
total_lzma_allocation -= state->allocation;
|
||||||
|
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
lzma_end(&state->stream);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
tor_free(state);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Return the approximate number of bytes allocated for <b>state</b>. */
|
||||||
|
size_t
|
||||||
|
tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state)
|
||||||
|
{
|
||||||
|
tor_assert(state != NULL);
|
||||||
|
return state->allocation;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Return the approximate number of bytes allocated for all LZMA states. */
|
||||||
|
size_t
|
||||||
|
tor_lzma_get_total_allocation(void)
|
||||||
|
{
|
||||||
|
return total_lzma_allocation;
|
||||||
|
}
|
||||||
|
|
56
src/common/compress_lzma.h
Normal file
56
src/common/compress_lzma.h
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
/* Copyright (c) 2003, Roger Dingledine
|
||||||
|
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
|
||||||
|
* Copyright (c) 2007-2017, The Tor Project, Inc. */
|
||||||
|
/* See LICENSE for licensing information */
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \file compress_lzma.h
|
||||||
|
* \brief Header for compress_lzma.c
|
||||||
|
**/
|
||||||
|
|
||||||
|
#ifndef TOR_COMPRESS_LZMA_H
|
||||||
|
#define TOR_COMPRESS_LZMA_H
|
||||||
|
|
||||||
|
const char *
|
||||||
|
tor_lzma_get_version_str(void);
|
||||||
|
|
||||||
|
const char *
|
||||||
|
tor_lzma_get_header_version_str(void);
|
||||||
|
|
||||||
|
int
|
||||||
|
tor_lzma_compress(char **out, size_t *out_len,
|
||||||
|
const char *in, size_t in_len,
|
||||||
|
compress_method_t method);
|
||||||
|
|
||||||
|
int
|
||||||
|
tor_lzma_uncompress(char **out, size_t *out_len,
|
||||||
|
const char *in, size_t in_len,
|
||||||
|
compress_method_t method,
|
||||||
|
int complete_only,
|
||||||
|
int protocol_warn_level);
|
||||||
|
|
||||||
|
/** Internal state for an incremental LZMA compression/decompression. */
|
||||||
|
typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t;
|
||||||
|
|
||||||
|
tor_lzma_compress_state_t *
|
||||||
|
tor_lzma_compress_new(int compress,
|
||||||
|
compress_method_t method,
|
||||||
|
compression_level_t compression_level);
|
||||||
|
|
||||||
|
tor_compress_output_t
|
||||||
|
tor_lzma_compress_process(tor_lzma_compress_state_t *state,
|
||||||
|
char **out, size_t *out_len,
|
||||||
|
const char **in, size_t *in_len,
|
||||||
|
int finish);
|
||||||
|
|
||||||
|
void
|
||||||
|
tor_lzma_compress_free(tor_lzma_compress_state_t *state);
|
||||||
|
|
||||||
|
size_t
|
||||||
|
tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state);
|
||||||
|
|
||||||
|
size_t
|
||||||
|
tor_lzma_get_total_allocation(void);
|
||||||
|
|
||||||
|
#endif // TOR_COMPRESS_LZMA_H.
|
||||||
|
|
@ -106,6 +106,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \
|
|||||||
LIBOR_CRYPTO_A_SRC = \
|
LIBOR_CRYPTO_A_SRC = \
|
||||||
src/common/aes.c \
|
src/common/aes.c \
|
||||||
src/common/compress.c \
|
src/common/compress.c \
|
||||||
|
src/common/compress_lzma.c \
|
||||||
src/common/compress_zlib.c \
|
src/common/compress_zlib.c \
|
||||||
src/common/crypto.c \
|
src/common/crypto.c \
|
||||||
src/common/crypto_pwbox.c \
|
src/common/crypto_pwbox.c \
|
||||||
@ -147,6 +148,7 @@ COMMONHEADERS = \
|
|||||||
src/common/compat_threads.h \
|
src/common/compat_threads.h \
|
||||||
src/common/compat_time.h \
|
src/common/compat_time.h \
|
||||||
src/common/compress.h \
|
src/common/compress.h \
|
||||||
|
src/common/compress_lzma.h \
|
||||||
src/common/compress_zlib.h \
|
src/common/compress_zlib.h \
|
||||||
src/common/confline.h \
|
src/common/confline.h \
|
||||||
src/common/container.h \
|
src/common/container.h \
|
||||||
|
@ -2404,6 +2404,112 @@ test_util_gzip_compression_bomb(void *arg)
|
|||||||
tor_compress_free(state);
|
tor_compress_free(state);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
test_util_lzma(void *arg)
|
||||||
|
{
|
||||||
|
#ifdef HAVE_LZMA
|
||||||
|
char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2;
|
||||||
|
const char *ccp2;
|
||||||
|
size_t len1, len2;
|
||||||
|
tor_compress_state_t *state = NULL;
|
||||||
|
|
||||||
|
(void)arg;
|
||||||
|
buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ");
|
||||||
|
tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD);
|
||||||
|
|
||||||
|
tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1,
|
||||||
|
LZMA_METHOD));
|
||||||
|
tt_assert(buf2 != NULL);
|
||||||
|
tt_int_op(len1, OP_LT, strlen(buf1));
|
||||||
|
tt_int_op(detect_compression_method(buf2, len1), OP_EQ, LZMA_METHOD);
|
||||||
|
|
||||||
|
tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1,
|
||||||
|
LZMA_METHOD, 1, LOG_INFO));
|
||||||
|
tt_assert(buf3 != NULL);
|
||||||
|
tt_int_op(strlen(buf1) + 1, OP_EQ, len2);
|
||||||
|
tt_str_op(buf1, OP_EQ, buf3);
|
||||||
|
|
||||||
|
tor_free(buf1);
|
||||||
|
tor_free(buf2);
|
||||||
|
tor_free(buf3);
|
||||||
|
|
||||||
|
#if 0
|
||||||
|
/* Check whether we can uncompress concatenated, compressed strings. */
|
||||||
|
tor_free(buf3);
|
||||||
|
buf2 = tor_reallocarray(buf2, len1, 2);
|
||||||
|
memcpy(buf2+len1, buf2, len1);
|
||||||
|
tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2,
|
||||||
|
LZMA_METHOD, 1, LOG_INFO));
|
||||||
|
tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2);
|
||||||
|
tt_mem_op(buf3, OP_EQ,
|
||||||
|
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0"
|
||||||
|
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0",
|
||||||
|
(strlen(buf1)+1)*2);
|
||||||
|
|
||||||
|
tor_free(buf1);
|
||||||
|
tor_free(buf2);
|
||||||
|
tor_free(buf3);
|
||||||
|
|
||||||
|
/* Check whether we can uncompress partial strings. */
|
||||||
|
buf1 =
|
||||||
|
tor_strdup("String with low redundancy that won't be compressed much.");
|
||||||
|
tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1,
|
||||||
|
LZMA_METHOD));
|
||||||
|
tt_int_op(len1, OP_GT, 16);
|
||||||
|
/* when we allow an incomplete string, we should succeed.*/
|
||||||
|
tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16,
|
||||||
|
LZMA_METHOD, 0, LOG_INFO));
|
||||||
|
tt_assert(len2 > 5);
|
||||||
|
buf3[len2]='\0';
|
||||||
|
tt_assert(!strcmpstart(buf1, buf3));
|
||||||
|
|
||||||
|
/* when we demand a complete string, this must fail. */
|
||||||
|
tor_free(buf3);
|
||||||
|
tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16,
|
||||||
|
LZMA_METHOD, 1, LOG_INFO));
|
||||||
|
tt_assert(buf3 == NULL);
|
||||||
|
|
||||||
|
tor_free(buf1);
|
||||||
|
tor_free(buf2);
|
||||||
|
tor_free(buf3);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* Now, try streaming compression. */
|
||||||
|
state = tor_compress_new(1, LZMA_METHOD, HIGH_COMPRESSION);
|
||||||
|
tt_assert(state);
|
||||||
|
cp1 = buf1 = tor_malloc(1024);
|
||||||
|
len1 = 1024;
|
||||||
|
ccp2 = "ABCDEFGHIJABCDEFGHIJ";
|
||||||
|
len2 = 21;
|
||||||
|
tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0),
|
||||||
|
OP_EQ, TOR_COMPRESS_OK);
|
||||||
|
tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */
|
||||||
|
tt_assert(cp1 > buf1);
|
||||||
|
|
||||||
|
len2 = 0;
|
||||||
|
cp2 = cp1;
|
||||||
|
tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1),
|
||||||
|
OP_EQ, TOR_COMPRESS_DONE);
|
||||||
|
tt_int_op(0, OP_EQ, len2);
|
||||||
|
tt_assert(cp1 > cp2); /* Make sure we really added something. */
|
||||||
|
|
||||||
|
tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1,
|
||||||
|
LZMA_METHOD, 1, LOG_WARN));
|
||||||
|
/* Make sure it compressed right. */
|
||||||
|
tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ");
|
||||||
|
tt_int_op(21, OP_EQ, len2);
|
||||||
|
|
||||||
|
done:
|
||||||
|
if (state)
|
||||||
|
tor_compress_free(state);
|
||||||
|
tor_free(buf2);
|
||||||
|
tor_free(buf3);
|
||||||
|
tor_free(buf1);
|
||||||
|
#else
|
||||||
|
(void)arg;
|
||||||
|
#endif // HAVE_LZMA.
|
||||||
|
}
|
||||||
|
|
||||||
/** Run unit tests for mmap() wrapper functionality. */
|
/** Run unit tests for mmap() wrapper functionality. */
|
||||||
static void
|
static void
|
||||||
test_util_mmap(void *arg)
|
test_util_mmap(void *arg)
|
||||||
@ -5717,6 +5823,7 @@ struct testcase_t util_tests[] = {
|
|||||||
UTIL_LEGACY(pow2),
|
UTIL_LEGACY(pow2),
|
||||||
UTIL_LEGACY(gzip),
|
UTIL_LEGACY(gzip),
|
||||||
UTIL_TEST(gzip_compression_bomb, TT_FORK),
|
UTIL_TEST(gzip_compression_bomb, TT_FORK),
|
||||||
|
UTIL_LEGACY(lzma),
|
||||||
UTIL_LEGACY(datadir),
|
UTIL_LEGACY(datadir),
|
||||||
UTIL_LEGACY(memarea),
|
UTIL_LEGACY(memarea),
|
||||||
UTIL_LEGACY(control_formats),
|
UTIL_LEGACY(control_formats),
|
||||||
|
Loading…
Reference in New Issue
Block a user