nihilist/tor
1
0
Fork 0
mirror of https://gitlab.torproject.org/tpo/core/tor.git synced 2024-11-10 13:13:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

Move zlib compression code into its own module.

Browse Source 
This patch refactors the `torgzip` module to allow us to extend a common
compression API to support multiple compression backends.

Additionally we move the gzip/zlib code into its own module under the
name `compress_zlib`.

See https://bugs.torproject.org/21664
This commit is contained in:
Alexander Færøy 2017-04-18 03:14:36 +02:00 committed by Nick Mathewson
parent e6c6606a17
commit 9d5bc1a935
8 changed files with 630 additions and 418 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

524
src/common/compress_zlib.c Normal file
View File

@ -0,0 +1,524 @@
/* Copyright (c) 2004, Roger Dingledine.
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2017, The Tor Project, Inc. */
/* See LICENSE for licensing information */
/**
* \file compress_zlib.c
* \brief Compression backend for gzip and zlib.
*
* This module should never be invoked directly. Use the compress module
* instead.
**/
#include "orconfig.h"
#include "util.h"
#include "torlog.h"
#include "torgzip.h"
#include "compress_zlib.h"
/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of
saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
that nobody will care if the compile outputs a no-such-identifier warning.
Sorry, but we like -Werror over here, so I guess we need to define these.
I hope that zlib 1.2.6 doesn't break these too.
*/
#ifndef _LARGEFILE64_SOURCE
#define _LARGEFILE64_SOURCE 0
#endif
#ifndef _LFS64_LARGEFILE
#define _LFS64_LARGEFILE 0
#endif
#ifndef _FILE_OFFSET_BITS
#define _FILE_OFFSET_BITS 0
#endif
#ifndef off64_t
#define off64_t int64_t
#endif
#include <zlib.h>
#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
#error "We require zlib version 1.2 or later."
#endif
static size_t tor_zlib_state_size_precalc(int inflate,
int windowbits, int memlevel);
/** Total number of bytes allocated for zlib state */
static size_t total_zlib_allocation = 0;
/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
static inline int
method_bits(compress_method_t method, compression_level_t level)
{
/* Bits+16 means "use gzip" in zlib >= 1.2 */
const int flag = method == GZIP_METHOD ? 16 : 0;
switch (level) {
default:
case HIGH_COMPRESSION: return flag + 15;
case MEDIUM_COMPRESSION: return flag + 13;
case LOW_COMPRESSION: return flag + 11;
}
}
/** Return a string representation of the version of the currently running
* version of zlib. */
const char *
tor_zlib_get_version_str(void)
{
return zlibVersion();
}
/** Return a string representation of the version of the version of zlib
* used at compilation. */
const char *
tor_zlib_get_header_version_str(void)
{
return ZLIB_VERSION;
}
/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
* allocated buffer, using the method described in <b>method</b>. Store the
* compressed string in *<b>out</b>, and its length in *<b>out_len</b>. Return
* 0 on success, -1 on failure.
*/
int
tor_zlib_compress(char **out, size_t *out_len,
const char *in, size_t in_len,
compress_method_t method)
{
struct z_stream_s *stream = NULL;
size_t out_size, old_size;
off_t offset;
tor_assert(out);
tor_assert(out_len);
tor_assert(in);
tor_assert(in_len < UINT_MAX);
*out = NULL;
stream = tor_malloc_zero(sizeof(struct z_stream_s));
stream->zalloc = Z_NULL;
stream->zfree = Z_NULL;
stream->opaque = NULL;
stream->next_in = (unsigned char*) in;
stream->avail_in = (unsigned int)in_len;
if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
method_bits(method, HIGH_COMPRESSION),
tor_compress_memory_level(HIGH_COMPRESSION),
Z_DEFAULT_STRATEGY) != Z_OK) {
//LCOV_EXCL_START -- we can only provoke failure by giving junk arguments.
log_warn(LD_GENERAL, "Error from deflateInit2: %s",
stream->msg?stream->msg:"<no message>");
goto err;
//LCOV_EXCL_STOP
}
/* Guess 50% compression. */
out_size = in_len / 2;
if (out_size < 1024) out_size = 1024;
*out = tor_malloc(out_size);
stream->next_out = (unsigned char*)*out;
stream->avail_out = (unsigned int)out_size;
while (1) {
switch (deflate(stream, Z_FINISH))
{
case Z_STREAM_END:
goto done;
case Z_OK:
/* In case zlib doesn't work as I think .... */
if (stream->avail_out >= stream->avail_in+16)
break;
case Z_BUF_ERROR:
offset = stream->next_out - ((unsigned char*)*out);
old_size = out_size;
out_size *= 2;
if (out_size < old_size) {
log_warn(LD_GENERAL, "Size overflow in compression.");
goto err;
}
*out = tor_realloc(*out, out_size);
stream->next_out = (unsigned char*)(*out + offset);
if (out_size - offset > UINT_MAX) {
log_warn(LD_BUG, "Ran over unsigned int limit of zlib while "
"uncompressing.");
goto err;
}
stream->avail_out = (unsigned int)(out_size - offset);
break;
default:
log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
stream->msg ? stream->msg : "<no message>");
goto err;
}
}
done:
*out_len = stream->total_out;
#if defined(OpenBSD)
/* "Hey Rocky! Watch me change an unsigned field to a signed field in a
* third-party API!"
* "Oh, that trick will just make people do unsafe casts to the unsigned
* type in their cross-platform code!"
* "Don't be foolish. I'm _sure_ they'll have the good sense to make sure
* the newly unsigned field isn't negative." */
tor_assert(stream->total_out >= 0);
#endif
if (deflateEnd(stream)!=Z_OK) {
// LCOV_EXCL_START -- unreachable if we handled the zlib structure right
tor_assert_nonfatal_unreached();
log_warn(LD_BUG, "Error freeing gzip structures");
goto err;
// LCOV_EXCL_STOP
}
tor_free(stream);
if (tor_compress_is_compression_bomb(*out_len, in_len)) {
log_warn(LD_BUG, "We compressed something and got an insanely high "
"compression factor; other Tors would think this was a zlib bomb.");
goto err;
}
return 0;
err:
if (stream) {
deflateEnd(stream);
tor_free(stream);
}
tor_free(*out);
return -1;
}
/** Given an Zlib/Gzip compressed string of total length <b>in_len</b> bytes
* at <b>in</b>, uncompress them into a newly allocated buffer. Store the
* uncompressed string in *<b>out</b>, and its length in *<b>out_len</b>.
* Return 0 on success, -1 on failure.
*
* If <b>complete_only</b> is true, we consider a truncated input as a failure;
* otherwise we decompress as much as we can. Warn about truncated or corrupt
* inputs at <b>protocol_warn_level</b>.
*/
int
tor_zlib_uncompress(char **out, size_t *out_len,
const char *in, size_t in_len,
compress_method_t method,
int complete_only,
int protocol_warn_level)
{
struct z_stream_s *stream = NULL;
size_t out_size, old_size;
off_t offset;
int r;
tor_assert(out);
tor_assert(out_len);
tor_assert(in);
tor_assert(in_len < UINT_MAX);
*out = NULL;
stream = tor_malloc_zero(sizeof(struct z_stream_s));
stream->zalloc = Z_NULL;
stream->zfree = Z_NULL;
stream->opaque = NULL;
stream->next_in = (unsigned char*) in;
stream->avail_in = (unsigned int)in_len;
if (inflateInit2(stream,
method_bits(method, HIGH_COMPRESSION)) != Z_OK) {
// LCOV_EXCL_START -- can only hit this if we give bad inputs.
log_warn(LD_GENERAL, "Error from inflateInit2: %s",
stream->msg?stream->msg:"<no message>");
goto err;
// LCOV_EXCL_STOP
}
out_size = in_len * 2; /* guess 50% compression. */
if (out_size < 1024) out_size = 1024;
if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
goto err;
*out = tor_malloc(out_size);
stream->next_out = (unsigned char*)*out;
stream->avail_out = (unsigned int)out_size;
while (1) {
switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
{
case Z_STREAM_END:
if (stream->avail_in == 0)
goto done;
/* There may be more compressed data here. */
if ((r = inflateEnd(stream)) != Z_OK) {
log_warn(LD_BUG, "Error freeing gzip structures");
goto err;
}
if (inflateInit2(stream,
method_bits(method,HIGH_COMPRESSION)) != Z_OK) {
log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
stream->msg?stream->msg:"<no message>");
goto err;
}
break;
case Z_OK:
if (!complete_only && stream->avail_in == 0)
goto done;
/* In case zlib doesn't work as I think.... */
if (stream->avail_out >= stream->avail_in+16)
break;
case Z_BUF_ERROR:
if (stream->avail_out > 0) {
log_fn(protocol_warn_level, LD_PROTOCOL,
"possible truncated or corrupt zlib data");
goto err;
}
offset = stream->next_out - (unsigned char*)*out;
old_size = out_size;
out_size *= 2;
if (out_size < old_size) {
log_warn(LD_GENERAL, "Size overflow in uncompression.");
goto err;
}
if (tor_compress_is_compression_bomb(in_len, out_size)) {
log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; "
"not proceeding.");
goto err;
}
if (out_size >= SIZE_T_CEILING) {
log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing.");
goto err;
}
*out = tor_realloc(*out, out_size);
stream->next_out = (unsigned char*)(*out + offset);
if (out_size - offset > UINT_MAX) {
log_warn(LD_BUG, "Ran over unsigned int limit of zlib while "
"uncompressing.");
goto err;
}
stream->avail_out = (unsigned int)(out_size - offset);
break;
default:
log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
stream->msg ? stream->msg : "<no message>");
goto err;
}
}
done:
*out_len = stream->next_out - (unsigned char*)*out;
r = inflateEnd(stream);
tor_free(stream);
if (r != Z_OK) {
log_warn(LD_BUG, "Error freeing gzip structures");
goto err;
}
/* NUL-terminate output. */
if (out_size == *out_len)
*out = tor_realloc(*out, out_size + 1);
(*out)[*out_len] = '\0';
return 0;
err:
if (stream) {
inflateEnd(stream);
tor_free(stream);
}
if (*out) {
tor_free(*out);
}
return -1;
}
/** Internal zlib state for an incremental compression/decompression.
* The body of this struct is not exposed. */
struct tor_zlib_compress_state_t {
struct z_stream_s stream; /**< The zlib stream */
int compress; /**< True if we are compressing; false if we are inflating */
/** Number of bytes read so far. Used to detect zlib bombs. */
size_t input_so_far;
/** Number of bytes written so far. Used to detect zlib bombs. */
size_t output_so_far;
/** Approximate number of bytes allocated for this object. */
size_t allocation;
};
/** Return an approximate number of bytes used in RAM to hold a state with
* window bits <b>windowBits</b> and compression level 'memlevel' */
static size_t
tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
{
windowbits &= 15;
#define A_FEW_KILOBYTES 2048
if (inflate_) {
/* From zconf.h:
"The memory requirements for inflate are (in bytes) 1 << windowBits
that is, 32K for windowBits=15 (default value) plus a few kilobytes
for small objects."
*/
return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
(1 << 15) + A_FEW_KILOBYTES;
} else {
/* Also from zconf.h:
"The memory requirements for deflate are (in bytes):
(1 << (windowBits+2)) + (1 << (memLevel+9))
... plus a few kilobytes for small objects."
*/
return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
(1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
}
#undef A_FEW_KILOBYTES
}
/** Construct and return a tor_zlib_compress_state_t object using
* <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
* decompression. */
tor_zlib_compress_state_t *
tor_zlib_compress_new(int compress,
compress_method_t method,
compression_level_t compression_level)
{
tor_zlib_compress_state_t *out;
int bits, memlevel;
if (! compress) {
/* use this setting for decompression, since we might have the
* max number of window bits */
compression_level = HIGH_COMPRESSION;
}
out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t));
out->stream.zalloc = Z_NULL;
out->stream.zfree = Z_NULL;
out->stream.opaque = NULL;
out->compress = compress;
bits = method_bits(method, compression_level);
memlevel = tor_compress_memory_level(compression_level);
if (compress) {
if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
bits, memlevel,
Z_DEFAULT_STRATEGY) != Z_OK)
goto err; // LCOV_EXCL_LINE
} else {
if (inflateInit2(&out->stream, bits) != Z_OK)
goto err; // LCOV_EXCL_LINE
}
out->allocation = tor_zlib_state_size_precalc(!compress, bits, memlevel);
total_zlib_allocation += out->allocation;
return out;
err:
tor_free(out);
return NULL;
}
/** Compress/decompress some bytes using <b>state</b>. Read up to
* *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
* to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true,
* we've reached the end of the input.
*
* Return TOR_COMPRESS_DONE if we've finished the entire
* compression/decompression.
* Return TOR_COMPRESS_OK if we're processed everything from the input.
* Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
* Return TOR_COMPRESS_ERROR if the stream is corrupt.
*/
tor_compress_output_t
tor_zlib_compress_process(tor_zlib_compress_state_t *state,
char **out, size_t *out_len,
const char **in, size_t *in_len,
int finish)
{
int err;
tor_assert(state != NULL);
tor_assert(*in_len <= UINT_MAX);
tor_assert(*out_len <= UINT_MAX);
state->stream.next_in = (unsigned char*) *in;
state->stream.avail_in = (unsigned int)*in_len;
state->stream.next_out = (unsigned char*) *out;
state->stream.avail_out = (unsigned int)*out_len;
if (state->compress) {
err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
} else {
err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
}
state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
*out = (char*) state->stream.next_out;
*out_len = state->stream.avail_out;
*in = (const char *) state->stream.next_in;
*in_len = state->stream.avail_in;
if (! state->compress &&
tor_compress_is_compression_bomb(state->input_so_far,
state->output_so_far)) {
log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
return TOR_COMPRESS_ERROR;
}
switch (err)
{
case Z_STREAM_END:
return TOR_COMPRESS_DONE;
case Z_BUF_ERROR:
if (state->stream.avail_in == 0 && !finish)
return TOR_COMPRESS_OK;
return TOR_COMPRESS_BUFFER_FULL;
case Z_OK:
if (state->stream.avail_out == 0 || finish)
return TOR_COMPRESS_BUFFER_FULL;
return TOR_COMPRESS_OK;
default:
log_warn(LD_GENERAL, "Gzip returned an error: %s",
state->stream.msg ? state->stream.msg : "<no message>");
return TOR_COMPRESS_ERROR;
}
}
/** Deallocate <b>state</b>. */
void
tor_zlib_compress_free(tor_zlib_compress_state_t *state)
{
if (state == NULL)
return;
total_zlib_allocation -= state->allocation;
if (state->compress)
deflateEnd(&state->stream);
else
inflateEnd(&state->stream);
tor_free(state);
}
/** Return the approximate number of bytes allocated for <b>state</b>. */
size_t
tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state)
{
tor_assert(state != NULL);
return state->allocation;
}
/** Return the approximate number of bytes allocated for all zlib states. */
size_t
tor_zlib_get_total_allocation(void)
{
return total_zlib_allocation;
}

56
src/common/compress_zlib.h Normal file
View File

@ -0,0 +1,56 @@
/* Copyright (c) 2003, Roger Dingledine
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2017, The Tor Project, Inc. */
/* See LICENSE for licensing information */
/**
* \file compress_zlib.h
* \brief Header for compress_zlib.c
**/
#ifndef TOR_COMPRESS_ZLIB_H
#define TOR_COMPRESS_ZLIB_H
const char *
tor_zlib_get_version_str(void);
const char *
tor_zlib_get_header_version_str(void);
int
tor_zlib_compress(char **out, size_t *out_len,
const char *in, size_t in_len,
compress_method_t method);
int
tor_zlib_uncompress(char **out, size_t *out_len,
const char *in, size_t in_len,
compress_method_t method,
int complete_only,
int protocol_warn_level);
/** Internal state for an incremental zlib/gzip compression/decompression. */
typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t;
tor_zlib_compress_state_t *
tor_zlib_compress_new(int compress,
compress_method_t method,
compression_level_t compression_level);
tor_compress_output_t
tor_zlib_compress_process(tor_zlib_compress_state_t *state,
char **out, size_t *out_len,
const char **in, size_t *in_len,
int finish);
void
tor_zlib_compress_free(tor_zlib_compress_state_t *state);
size_t
tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state);
size_t
tor_zlib_get_total_allocation(void);
#endif // TOR_COMPRESS_ZLIB_H.

2
src/common/include.am
View File

@ -105,6 +105,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \
LIBOR_CRYPTO_A_SRC = \
src/common/aes.c \
src/common/compress_zlib.c \
src/common/crypto.c \
src/common/crypto_pwbox.c \
src/common/crypto_s2k.c \
@ -145,6 +146,7 @@ COMMONHEADERS = \
src/common/compat_openssl.h \
src/common/compat_threads.h \
src/common/compat_time.h \
src/common/compress_zlib.h \
src/common/confline.h \
src/common/container.h \
src/common/crypto.h \

452
src/common/torgzip.c
View File

@ -23,69 +23,7 @@
#include "util.h"
#include "torlog.h"
#include "torgzip.h"
/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of
saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
that nobody will care if the compile outputs a no-such-identifier warning.
Sorry, but we like -Werror over here, so I guess we need to define these.
I hope that zlib 1.2.6 doesn't break these too.
*/
#ifndef _LARGEFILE64_SOURCE
#define _LARGEFILE64_SOURCE 0
#endif
#ifndef _LFS64_LARGEFILE
#define _LFS64_LARGEFILE 0
#endif
#ifndef _FILE_OFFSET_BITS
#define _FILE_OFFSET_BITS 0
#endif
#ifndef off64_t
#define off64_t int64_t
#endif
#include <zlib.h>
#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
#error "We require zlib version 1.2 or later."
#endif
static size_t tor_zlib_state_size_precalc(int inflate,
int windowbits, int memlevel);
/** Total number of bytes allocated for zlib state */
static size_t total_zlib_allocation = 0;
/** Return a string representation of the version of the currently running
* version of zlib. */
const char *
tor_zlib_get_version_str(void)
{
return zlibVersion();
}
/** Return a string representation of the version of the version of zlib
* used at compilation. */
const char *
tor_zlib_get_header_version_str(void)
{
return ZLIB_VERSION;
}
/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
static inline int
method_bits(compress_method_t method, compression_level_t level)
{
/* Bits+16 means "use gzip" in zlib >= 1.2 */
const int flag = method == GZIP_METHOD ? 16 : 0;
switch (level) {
default:
case HIGH_COMPRESSION: return flag + 15;
case MEDIUM_COMPRESSION: return flag + 13;
case LOW_COMPRESSION: return flag + 11;
}
}
#include "compress_zlib.h"
/** @{ */
/* These macros define the maximum allowable compression factor. Anything of
@ -140,107 +78,9 @@ tor_compress(char **out, size_t *out_len,
const char *in, size_t in_len,
compress_method_t method)
{
struct z_stream_s *stream = NULL;
size_t out_size, old_size;
off_t offset;
if (method == GZIP_METHOD || method == ZLIB_METHOD)
return tor_zlib_compress(out, out_len, in, in_len, method);
tor_assert(out);
tor_assert(out_len);
tor_assert(in);
tor_assert(in_len < UINT_MAX);
*out = NULL;
stream = tor_malloc_zero(sizeof(struct z_stream_s));
stream->zalloc = Z_NULL;
stream->zfree = Z_NULL;
stream->opaque = NULL;
stream->next_in = (unsigned char*) in;
stream->avail_in = (unsigned int)in_len;
if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
method_bits(method, HIGH_COMPRESSION),
get_memlevel(HIGH_COMPRESSION),
Z_DEFAULT_STRATEGY) != Z_OK) {
//LCOV_EXCL_START -- we can only provoke failure by giving junk arguments.
log_warn(LD_GENERAL, "Error from deflateInit2: %s",
stream->msg?stream->msg:"<no message>");
goto err;
//LCOV_EXCL_STOP
}
/* Guess 50% compression. */
out_size = in_len / 2;
if (out_size < 1024) out_size = 1024;
*out = tor_malloc(out_size);
stream->next_out = (unsigned char*)*out;
stream->avail_out = (unsigned int)out_size;
while (1) {
switch (deflate(stream, Z_FINISH))
{
case Z_STREAM_END:
goto done;
case Z_OK:
/* In case zlib doesn't work as I think .... */
if (stream->avail_out >= stream->avail_in+16)
break;
case Z_BUF_ERROR:
offset = stream->next_out - ((unsigned char*)*out);
old_size = out_size;
out_size *= 2;
if (out_size < old_size) {
log_warn(LD_GENERAL, "Size overflow in compression.");
goto err;
}
*out = tor_realloc(*out, out_size);
stream->next_out = (unsigned char*)(*out + offset);
if (out_size - offset > UINT_MAX) {
log_warn(LD_BUG, "Ran over unsigned int limit of zlib while "
"uncompressing.");
goto err;
}
stream->avail_out = (unsigned int)(out_size - offset);
break;
default:
log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
stream->msg ? stream->msg : "<no message>");
goto err;
}
}
done:
*out_len = stream->total_out;
#if defined(OpenBSD)
/* "Hey Rocky! Watch me change an unsigned field to a signed field in a
* third-party API!"
* "Oh, that trick will just make people do unsafe casts to the unsigned
* type in their cross-platform code!"
* "Don't be foolish. I'm _sure_ they'll have the good sense to make sure
* the newly unsigned field isn't negative." */
tor_assert(stream->total_out >= 0);
#endif
if (deflateEnd(stream)!=Z_OK) {
// LCOV_EXCL_START -- unreachable if we handled the zlib structure right
tor_assert_nonfatal_unreached();
log_warn(LD_BUG, "Error freeing gzip structures");
goto err;
// LCOV_EXCL_STOP
}
tor_free(stream);
if (is_compression_bomb(*out_len, in_len)) {
log_warn(LD_BUG, "We compressed something and got an insanely high "
"compression factor; other Tors would think this was a zlib bomb.");
goto err;
}
return 0;
err:
if (stream) {
deflateEnd(stream);
tor_free(stream);
}
tor_free(*out);
return -1;
}
@ -262,127 +102,12 @@ tor_uncompress(char **out, size_t *out_len,
int complete_only,
int protocol_warn_level)
{
struct z_stream_s *stream = NULL;
size_t out_size, old_size;
off_t offset;
int r;
if (method == GZIP_METHOD || method == ZLIB_METHOD)
return tor_zlib_uncompress(out, out_len, in, in_len,
method,
complete_only,
protocol_warn_level);
tor_assert(out);
tor_assert(out_len);
tor_assert(in);
tor_assert(in_len < UINT_MAX);
*out = NULL;
stream = tor_malloc_zero(sizeof(struct z_stream_s));
stream->zalloc = Z_NULL;
stream->zfree = Z_NULL;
stream->opaque = NULL;
stream->next_in = (unsigned char*) in;
stream->avail_in = (unsigned int)in_len;
if (inflateInit2(stream,
method_bits(method, HIGH_COMPRESSION)) != Z_OK) {
// LCOV_EXCL_START -- can only hit this if we give bad inputs.
log_warn(LD_GENERAL, "Error from inflateInit2: %s",
stream->msg?stream->msg:"<no message>");
goto err;
// LCOV_EXCL_STOP
}
out_size = in_len * 2; /* guess 50% compression. */
if (out_size < 1024) out_size = 1024;
if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
goto err;
*out = tor_malloc(out_size);
stream->next_out = (unsigned char*)*out;
stream->avail_out = (unsigned int)out_size;
while (1) {
switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
{
case Z_STREAM_END:
if (stream->avail_in == 0)
goto done;
/* There may be more compressed data here. */
if ((r = inflateEnd(stream)) != Z_OK) {
log_warn(LD_BUG, "Error freeing gzip structures");
goto err;
}
if (inflateInit2(stream,
method_bits(method,HIGH_COMPRESSION)) != Z_OK) {
log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
stream->msg?stream->msg:"<no message>");
goto err;
}
break;
case Z_OK:
if (!complete_only && stream->avail_in == 0)
goto done;
/* In case zlib doesn't work as I think.... */
if (stream->avail_out >= stream->avail_in+16)
break;
case Z_BUF_ERROR:
if (stream->avail_out > 0) {
log_fn(protocol_warn_level, LD_PROTOCOL,
"possible truncated or corrupt zlib data");
goto err;
}
offset = stream->next_out - (unsigned char*)*out;
old_size = out_size;
out_size *= 2;
if (out_size < old_size) {
log_warn(LD_GENERAL, "Size overflow in uncompression.");
goto err;
}
if (is_compression_bomb(in_len, out_size)) {
log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; "
"not proceeding.");
goto err;
}
if (out_size >= SIZE_T_CEILING) {
log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing.");
goto err;
}
*out = tor_realloc(*out, out_size);
stream->next_out = (unsigned char*)(*out + offset);
if (out_size - offset > UINT_MAX) {
log_warn(LD_BUG, "Ran over unsigned int limit of zlib while "
"uncompressing.");
goto err;
}
stream->avail_out = (unsigned int)(out_size - offset);
break;
default:
log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
stream->msg ? stream->msg : "<no message>");
goto err;
}
}
done:
*out_len = stream->next_out - (unsigned char*)*out;
r = inflateEnd(stream);
tor_free(stream);
if (r != Z_OK) {
log_warn(LD_BUG, "Error freeing gzip structures");
goto err;
}
/* NUL-terminate output. */
if (out_size == *out_len)
*out = tor_realloc(*out, out_size + 1);
(*out)[*out_len] = '\0';
return 0;
err:
if (stream) {
inflateEnd(stream);
tor_free(stream);
}
if (*out) {
tor_free(*out);
}
return -1;
}
@ -406,58 +131,39 @@ detect_compression_method(const char *in, size_t in_len)
/** Internal state for an incremental compression/decompression. The body of
* this struct is not exposed. */
struct tor_compress_state_t {
struct z_stream_s stream; /**< The zlib stream */
int compress; /**< True if we are compressing; false if we are inflating */
compress_method_t method; /**< The compression method. */
/** Number of bytes read so far. Used to detect zlib bombs. */
size_t input_so_far;
/** Number of bytes written so far. Used to detect zlib bombs. */
size_t output_so_far;
/** Approximate number of bytes allocated for this object. */
size_t allocation;
union {
tor_zlib_compress_state_t *zlib_state;
} u; /**< Compression backend state. */
};
/** Construct and return a tor_compress_state_t object using <b>method</b>. If
* <b>compress</b>, it's for compression; otherwise it's for decompression. */
tor_compress_state_t *
tor_compress_new(int compress_, compress_method_t method,
tor_compress_new(int compress, compress_method_t method,
compression_level_t compression_level)
{
tor_compress_state_t *out;
int bits, memlevel;
tor_compress_state_t *state;
if (! compress_) {
/* use this setting for decompression, since we might have the
* max number of window bits */
compression_level = HIGH_COMPRESSION;
}
state = tor_malloc_zero(sizeof(tor_compress_state_t));
state->method = method;
out = tor_malloc_zero(sizeof(tor_compress_state_t));
out->stream.zalloc = Z_NULL;
out->stream.zfree = Z_NULL;
out->stream.opaque = NULL;
out->compress = compress_;
bits = method_bits(method, compression_level);
memlevel = get_memlevel(compression_level);
if (compress_) {
if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
bits, memlevel,
Z_DEFAULT_STRATEGY) != Z_OK)
goto err; // LCOV_EXCL_LINE
} else {
if (inflateInit2(&out->stream, bits) != Z_OK)
goto err; // LCOV_EXCL_LINE
}
out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel);
if (method == GZIP_METHOD || method == ZLIB_METHOD) {
tor_zlib_compress_state_t *zlib_state =
tor_zlib_compress_new(compress, method, compression_level);
total_zlib_allocation += out->allocation;
if (zlib_state == NULL)
goto err;
return out;
state->u.zlib_state = zlib_state;
}
return state;
err:
tor_free(out);
return NULL;
tor_free(state);
return NULL;
}
/** Compress/decompress some bytes using <b>state</b>. Read up to
@ -477,112 +183,38 @@ tor_compress_process(tor_compress_state_t *state,
const char **in, size_t *in_len,
int finish)
{
int err;
tor_assert(*in_len <= UINT_MAX);
tor_assert(*out_len <= UINT_MAX);
state->stream.next_in = (unsigned char*) *in;
state->stream.avail_in = (unsigned int)*in_len;
state->stream.next_out = (unsigned char*) *out;
state->stream.avail_out = (unsigned int)*out_len;
tor_assert(state != NULL);
if (state->compress) {
err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
} else {
err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
}
if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD)
return tor_zlib_compress_process(state->u.zlib_state,
out, out_len, in, in_len,
finish);
state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
*out = (char*) state->stream.next_out;
*out_len = state->stream.avail_out;
*in = (const char *) state->stream.next_in;
*in_len = state->stream.avail_in;
if (! state->compress &&
is_compression_bomb(state->input_so_far, state->output_so_far)) {
log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
return TOR_COMPRESS_ERROR;
}
switch (err)
{
case Z_STREAM_END:
return TOR_COMPRESS_DONE;
case Z_BUF_ERROR:
if (state->stream.avail_in == 0 && !finish)
return TOR_COMPRESS_OK;
return TOR_COMPRESS_BUFFER_FULL;
case Z_OK:
if (state->stream.avail_out == 0 || finish)
return TOR_COMPRESS_BUFFER_FULL;
return TOR_COMPRESS_OK;
default:
log_warn(LD_GENERAL, "Gzip returned an error: %s",
state->stream.msg ? state->stream.msg : "<no message>");
return TOR_COMPRESS_ERROR;
}
return TOR_COMPRESS_ERROR;
}
/** Deallocate <b>state</b>. */
void
tor_compress_free(tor_compress_state_t *state)
{
if (!state)
if (state == NULL)
return;
total_zlib_allocation -= state->allocation;
if (state->compress)
deflateEnd(&state->stream);
else
inflateEnd(&state->stream);
if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD)
tor_zlib_compress_free(state->u.zlib_state);
tor_free(state);
}
/** Return an approximate number of bytes used in RAM to hold a state with
* window bits <b>windowBits</b> and compression level 'memlevel' */
static size_t
tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
{
windowbits &= 15;
#define A_FEW_KILOBYTES 2048
if (inflate_) {
/* From zconf.h:
"The memory requirements for inflate are (in bytes) 1 << windowBits
that is, 32K for windowBits=15 (default value) plus a few kilobytes
for small objects."
*/
return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) +
(1 << 15) + A_FEW_KILOBYTES;
} else {
/* Also from zconf.h:
"The memory requirements for deflate are (in bytes):
(1 << (windowBits+2)) + (1 << (memLevel+9))
... plus a few kilobytes for small objects."
*/
return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) +
(1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
}
#undef A_FEW_KILOBYTES
}
/** Return the approximate number of bytes allocated for <b>state</b>. */
size_t
tor_compress_state_size(const tor_compress_state_t *state)
{
return state->allocation;
}
/** Return the approximate number of bytes allocated for all zlib states. */
size_t
tor_zlib_get_total_allocation(void)
{
return total_zlib_allocation;
tor_assert(state != NULL);
if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD)
return tor_zlib_compress_state_size(state->u.zlib_state);
return 0;
}

11
src/common/torgzip.h
View File

@ -38,12 +38,6 @@ tor_uncompress(char **out, size_t *out_len,
int complete_only,
int protocol_warn_level);
const char *
tor_zlib_get_version_str(void);
const char *
tor_zlib_get_header_version_str(void);
compress_method_t detect_compression_method(const char *in, size_t in_len);
int
@ -60,8 +54,10 @@ typedef enum {
TOR_COMPRESS_BUFFER_FULL,
TOR_COMPRESS_ERROR
} tor_compress_output_t;
/** Internal state for an incremental zlib compression/decompression. */
/** Internal state for an incremental compression/decompression. */
typedef struct tor_compress_state_t tor_compress_state_t;
tor_compress_state_t *tor_compress_new(int compress,
compress_method_t method,
compression_level_t level);
@ -73,7 +69,6 @@ tor_compress_output_t tor_compress_process(tor_compress_state_t *state,
void tor_compress_free(tor_compress_state_t *state);
size_t tor_compress_state_size(const tor_compress_state_t *state);
size_t tor_zlib_get_total_allocation(void);
#endif

1
src/or/config.c
View File

@ -69,6 +69,7 @@
#include "circuitmux.h"
#include "circuitmux_ewma.h"
#include "circuitstats.h"
#include "compress_zlib.h"
#include "config.h"
#include "connection.h"
#include "connection_edge.h"

1
src/or/main.c
View File

@ -58,6 +58,7 @@
#include "circuitlist.h"
#include "circuituse.h"
#include "command.h"
#include "compress_zlib.h"
#include "config.h"
#include "confparse.h"
#include "connection.h"

1
src/or/relay.c
View File

@ -54,6 +54,7 @@
#include "circuitbuild.h"
#include "circuitlist.h"
#include "circuituse.h"
#include "compress_zlib.h"
#include "config.h"
#include "connection.h"
#include "connection_edge.h"