nihilist/tor
1
0
Fork 0
mirror of https://gitlab.torproject.org/tpo/core/tor.git synced 2024-11-24 04:13:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'andrea/ticket18640_v3'

Browse Source
This commit is contained in:
Nick Mathewson 2016-08-25 14:29:06 -04:00
commit 1dfa2213a4
12 changed files with 840 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/or/config.c
View File

@ -211,6 +211,7 @@ static config_var_t option_vars_[] = {
V(CountPrivateBandwidth, BOOL, "0"),
V(DataDirectory, FILENAME, NULL),
V(DataDirectoryGroupReadable, BOOL, "0"),
V(DisableOOSCheck, BOOL, "1"),
V(DisableNetwork, BOOL, "0"),
V(DirAllowPrivateAddresses, BOOL, "0"),
V(TestingAuthDirTimeToLearnReachability, INTERVAL, "30 minutes"),
@ -1374,6 +1375,35 @@ options_act_reversible(const or_options_t *old_options, char **msg)
connection_mark_for_close(conn);
}
});
if (set_conn_limit) {
/*
* If we adjusted the conn limit, recompute the OOS threshold too
*
* How many possible sockets to keep in reserve? If we have lots of
* possible sockets, keep this below a limit and set ConnLimit_high_thresh
* very close to ConnLimit_, but if ConnLimit_ is low, shrink it in
* proportion.
*
* Somewhat arbitrarily, set socks_in_reserve to 5% of ConnLimit_, but
* cap it at 64.
*/
int socks_in_reserve = options->ConnLimit_ / 20;
if (socks_in_reserve > 64) socks_in_reserve = 64;
options->ConnLimit_high_thresh = options->ConnLimit_ - socks_in_reserve;
options->ConnLimit_low_thresh = (options->ConnLimit_ / 4) * 3;
log_info(LD_GENERAL,
"Recomputed OOS thresholds: ConnLimit %d, ConnLimit_ %d, "
"ConnLimit_high_thresh %d, ConnLimit_low_thresh %d",
options->ConnLimit, options->ConnLimit_,
options->ConnLimit_high_thresh,
options->ConnLimit_low_thresh);
/* Give the OOS handler a chance with the new thresholds */
connection_check_oos(get_n_open_sockets(), 0);
}
goto done;
rollback:

304
src/or/connection.c
View File

@ -754,9 +754,9 @@ connection_mark_for_close_(connection_t *conn, int line, const char *file)
* For all other cases, use connection_mark_and_flush() instead, which
* checks for or_connection_t properly, instead. See below.
*/
void
connection_mark_for_close_internal_(connection_t *conn,
int line, const char *file)
MOCK_IMPL(void,
connection_mark_for_close_internal_, (connection_t *conn,
int line, const char *file))
{
assert_connection_ok(conn,0);
tor_assert(line);
@ -1090,6 +1090,7 @@ connection_listener_new(const struct sockaddr *listensockaddr,
int start_reading = 0;
static int global_next_session_group = SESSION_GROUP_FIRST_AUTO;
tor_addr_t addr;
int exhaustion = 0;
if (listensockaddr->sa_family == AF_INET ||
listensockaddr->sa_family == AF_INET6) {
@ -1108,6 +1109,11 @@ connection_listener_new(const struct sockaddr *listensockaddr,
int e = tor_socket_errno(s);
if (ERRNO_IS_RESOURCE_LIMIT(e)) {
warn_too_many_conns();
/*
* We'll call the OOS handler at the error exit, so set the
* exhaustion flag for it.
*/
exhaustion = 1;
} else {
log_warn(LD_NET, "Socket creation failed: %s",
tor_socket_strerror(e));
@ -1226,6 +1232,11 @@ connection_listener_new(const struct sockaddr *listensockaddr,
int e = tor_socket_errno(s);
if (ERRNO_IS_RESOURCE_LIMIT(e)) {
warn_too_many_conns();
/*
* We'll call the OOS handler at the error exit, so set the
* exhaustion flag for it.
*/
exhaustion = 1;
} else {
log_warn(LD_NET,"Socket creation failed: %s.", strerror(e));
}
@ -1344,6 +1355,12 @@ connection_listener_new(const struct sockaddr *listensockaddr,
dnsserv_configure_listener(conn);
}
/*
* Normal exit; call the OOS handler since connection count just changed;
* the exhaustion flag will always be zero here though.
*/
connection_check_oos(get_n_open_sockets(), 0);
return conn;
err:
@ -1352,6 +1369,9 @@ connection_listener_new(const struct sockaddr *listensockaddr,
if (conn)
connection_free(conn);
/* Call the OOS handler, indicate if we saw an exhaustion-related error */
connection_check_oos(get_n_open_sockets(), exhaustion);
return NULL;
}
@ -1442,21 +1462,34 @@ connection_handle_listener_read(connection_t *conn, int new_type)
if (!SOCKET_OK(news)) { /* accept() error */
int e = tor_socket_errno(conn->s);
if (ERRNO_IS_ACCEPT_EAGAIN(e)) {
return 0; /* they hung up before we could accept(). that's fine. */
/*
* they hung up before we could accept(). that's fine.
*
* give the OOS handler a chance to run though
*/
connection_check_oos(get_n_open_sockets(), 0);
return 0;
} else if (ERRNO_IS_RESOURCE_LIMIT(e)) {
warn_too_many_conns();
/* Exhaustion; tell the OOS handler */
connection_check_oos(get_n_open_sockets(), 1);
return 0;
}
/* else there was a real error. */
log_warn(LD_NET,"accept() failed: %s. Closing listener.",
tor_socket_strerror(e));
connection_mark_for_close(conn);
/* Tell the OOS handler about this too */
connection_check_oos(get_n_open_sockets(), 0);
return -1;
}
log_debug(LD_NET,
"Connection accepted on socket %d (child of fd %d).",
(int)news,(int)conn->s);
/* We accepted a new conn; run OOS handler */
connection_check_oos(get_n_open_sockets(), 0);
if (make_socket_reuseable(news) < 0) {
if (tor_socket_errno(news) == EINVAL) {
/* This can happen on OSX if we get a badly timed shutdown. */
@ -1661,12 +1694,18 @@ connection_connect_sockaddr,(connection_t *conn,
s = tor_open_socket_nonblocking(protocol_family, SOCK_STREAM, proto);
if (! SOCKET_OK(s)) {
/*
* Early OOS handler calls; it matters if it's an exhaustion-related
* error or not.
*/
*socket_error = tor_socket_errno(s);
if (ERRNO_IS_RESOURCE_LIMIT(*socket_error)) {
warn_too_many_conns();
connection_check_oos(get_n_open_sockets(), 1);
} else {
log_warn(LD_NET,"Error creating network socket: %s",
tor_socket_strerror(*socket_error));
connection_check_oos(get_n_open_sockets(), 0);
}
return -1;
}
@ -1676,6 +1715,13 @@ connection_connect_sockaddr,(connection_t *conn,
tor_socket_strerror(errno));
}
/*
* We've got the socket open; give the OOS handler a chance to check
* against configuured maximum socket number, but tell it no exhaustion
* failure.
*/
connection_check_oos(get_n_open_sockets(), 0);
if (bindaddr && bind(s, bindaddr, bindaddr_len) < 0) {
*socket_error = tor_socket_errno(s);
log_warn(LD_NET,"Error binding network socket: %s",
@ -4454,6 +4500,256 @@ connection_reached_eof(connection_t *conn)
}
}
/** Comparator for the two-orconn case in OOS victim sort */
static int
oos_victim_comparator_for_orconns(or_connection_t *a, or_connection_t *b)
{
int a_circs, b_circs;
/* Fewer circuits == higher priority for OOS kill, sort earlier */
a_circs = connection_or_get_num_circuits(a);
b_circs = connection_or_get_num_circuits(b);
if (a_circs < b_circs) return -1;
else if (b_circs > a_circs) return 1;
else return 0;
}
/** Sort comparator for OOS victims; better targets sort before worse
* ones. */
static int
oos_victim_comparator(const void **a_v, const void **b_v)
{
connection_t *a = NULL, *b = NULL;
/* Get connection pointers out */
a = (connection_t *)(*a_v);
b = (connection_t *)(*b_v);
tor_assert(a != NULL);
tor_assert(b != NULL);
/*
* We always prefer orconns as victims currently; we won't even see
* these non-orconn cases, but if we do, sort them after orconns.
*/
if (a->type == CONN_TYPE_OR && b->type == CONN_TYPE_OR) {
return oos_victim_comparator_for_orconns(TO_OR_CONN(a), TO_OR_CONN(b));
} else {
/*
* One isn't an orconn; if one is, it goes first. We currently have no
* opinions about cases where neither is an orconn.
*/
if (a->type == CONN_TYPE_OR) return -1;
else if (b->type == CONN_TYPE_OR) return 1;
else return 0;
}
}
/** Pick n victim connections for the OOS handler and return them in a
* smartlist.
*/
MOCK_IMPL(STATIC smartlist_t *,
pick_oos_victims, (int n))
{
smartlist_t *eligible = NULL, *victims = NULL;
smartlist_t *conns;
int conn_counts_by_type[CONN_TYPE_MAX_ + 1], i;
/*
* Big damn assumption (someone improve this someday!):
*
* Socket exhaustion normally happens on high-volume relays, and so
* most of the connections involved are orconns. We should pick victims
* by assembling a list of all orconns, and sorting them in order of
* how much 'damage' by some metric we'd be doing by dropping them.
*
* If we move on from orconns, we should probably think about incoming
* directory connections next, or exit connections. Things we should
* probably never kill are controller connections and listeners.
*
* This function will count how many connections of different types
* exist and log it for purposes of gathering data on typical OOS
* situations to guide future improvements.
*/
/* First, get the connection array */
conns = get_connection_array();
/*
* Iterate it and pick out eligible connection types, and log some stats
* along the way.
*/
eligible = smartlist_new();
memset(conn_counts_by_type, 0, sizeof(conn_counts_by_type));
SMARTLIST_FOREACH_BEGIN(conns, connection_t *, c) {
/* Bump the counter */
tor_assert(c->type <= CONN_TYPE_MAX_);
++(conn_counts_by_type[c->type]);
/* Skip anything without a socket we can free */
if (!(SOCKET_OK(c->s))) {
continue;
}
/* Skip anything we would count as moribund */
if (connection_is_moribund(c)) {
continue;
}
switch (c->type) {
case CONN_TYPE_OR:
/* We've got an orconn, it's eligible to be OOSed */
smartlist_add(eligible, c);
break;
default:
/* We don't know what to do with it, ignore it */
break;
}
} SMARTLIST_FOREACH_END(c);
/* Log some stats */
if (smartlist_len(conns) > 0) {
/* At least one counter must be non-zero */
log_info(LD_NET, "Some stats on conn types seen during OOS follow");
for (i = CONN_TYPE_MIN_; i <= CONN_TYPE_MAX_; ++i) {
/* Did we see any? */
if (conn_counts_by_type[i] > 0) {
log_info(LD_NET, "%s: %d conns",
conn_type_to_string(i),
conn_counts_by_type[i]);
}
}
log_info(LD_NET, "Done with OOS conn type stats");
}
/* Did we find more eligible targets than we want to kill? */
if (smartlist_len(eligible) > n) {
/* Sort the list in order of target preference */
smartlist_sort(eligible, oos_victim_comparator);
/* Pick first n as victims */
victims = smartlist_new();
for (i = 0; i < n; ++i) {
smartlist_add(victims, smartlist_get(eligible, i));
}
/* Free the original list */
smartlist_free(eligible);
} else {
/* No, we can just call them all victims */
victims = eligible;
}
return victims;
}
/** Kill a list of connections for the OOS handler. */
MOCK_IMPL(STATIC void,
kill_conn_list_for_oos, (smartlist_t *conns))
{
if (!conns) return;
SMARTLIST_FOREACH_BEGIN(conns, connection_t *, c) {
/* Make sure the channel layer gets told about orconns */
if (c->type == CONN_TYPE_OR) {
connection_or_close_for_error(TO_OR_CONN(c), 1);
} else {
connection_mark_for_close(c);
}
} SMARTLIST_FOREACH_END(c);
log_notice(LD_NET,
"OOS handler marked %d connections",
smartlist_len(conns));
}
/** Out-of-Sockets handler; n_socks is the current number of open
* sockets, and failed is non-zero if a socket exhaustion related
* error immediately preceded this call. This is where to do
* circuit-killing heuristics as needed.
*/
void
connection_check_oos(int n_socks, int failed)
{
int target_n_socks = 0, moribund_socks, socks_to_kill;
smartlist_t *conns;
/* Early exit: is OOS checking disabled? */
if (get_options()->DisableOOSCheck) {
return;
}
/* Sanity-check args */
tor_assert(n_socks >= 0);
/*
* Make some log noise; keep it at debug level since this gets a chance
* to run on every connection attempt.
*/
log_debug(LD_NET,
"Running the OOS handler (%d open sockets, %s)",
n_socks, (failed != 0) ? "exhaustion seen" : "no exhaustion");
/*
* Check if we're really handling an OOS condition, and if so decide how
* many sockets we want to get down to. Be sure we check if the threshold
* is distinct from zero first; it's possible for this to be called a few
* times before we've finished reading the config.
*/
if (n_socks >= get_options()->ConnLimit_high_thresh &&
get_options()->ConnLimit_high_thresh != 0 &&
get_options()->ConnLimit_ != 0) {
/* Try to get down to the low threshold */
target_n_socks = get_options()->ConnLimit_low_thresh;
log_notice(LD_NET,
"Current number of sockets %d is greater than configured "
"limit %d; OOS handler trying to get down to %d",
n_socks, get_options()->ConnLimit_high_thresh,
target_n_socks);
} else if (failed) {
/*
* If we're not at the limit but we hit a socket exhaustion error, try to
* drop some (but not as aggressively as ConnLimit_low_threshold, which is
* 3/4 of ConnLimit_)
*/
target_n_socks = (n_socks * 9) / 10;
log_notice(LD_NET,
"We saw socket exhaustion at %d open sockets; OOS handler "
"trying to get down to %d",
n_socks, target_n_socks);
}
if (target_n_socks > 0) {
/*
* It's an OOS!
*
* Count moribund sockets; it's be important that anything we decide
* to get rid of here but don't immediately close get counted as moribund
* on subsequent invocations so we don't try to kill too many things if
* connection_check_oos() gets called multiple times.
*/
moribund_socks = connection_count_moribund();
if (moribund_socks < n_socks - target_n_socks) {
socks_to_kill = n_socks - target_n_socks - moribund_socks;
conns = pick_oos_victims(socks_to_kill);
if (conns) {
kill_conn_list_for_oos(conns);
log_notice(LD_NET,
"OOS handler killed %d conns", smartlist_len(conns));
smartlist_free(conns);
} else {
log_notice(LD_NET, "OOS handler failed to pick any victim conns");
}
} else {
log_notice(LD_NET,
"Not killing any sockets for OOS because there are %d "
"already moribund, and we only want to eliminate %d",
moribund_socks, n_socks - target_n_socks);
}
}
}
/** Log how many bytes are used by buffers of different kinds and sizes. */
void
connection_dump_buffer_mem_stats(int severity)

23
src/or/connection.h
View File

@ -34,8 +34,8 @@ void connection_about_to_close_connection(connection_t *conn);
void connection_close_immediate(connection_t *conn);
void connection_mark_for_close_(connection_t *conn,
int line, const char *file);
void connection_mark_for_close_internal_(connection_t *conn,
int line, const char *file);
MOCK_DECL(void, connection_mark_for_close_internal_,
(connection_t *conn, int line, const char *file));
#define connection_mark_for_close(c) \
connection_mark_for_close_((c), __LINE__, SHORT_FILE__)
@ -247,6 +247,22 @@ void clock_skew_warning(const connection_t *conn, long apparent_skew,
int trusted, log_domain_mask_t domain,
const char *received, const char *source);
/** Check if a connection is on the way out so the OOS handler doesn't try
* to kill more than it needs. */
static inline int
connection_is_moribund(connection_t *conn)
{
if (conn != NULL &&
(conn->conn_array_index < 0 ||
conn->marked_for_close)) {
return 1;
} else {
return 0;
}
}
void connection_check_oos(int n_socks, int failed);
#ifdef CONNECTION_PRIVATE
STATIC void connection_free_(connection_t *conn);
@ -265,6 +281,9 @@ MOCK_DECL(STATIC int,connection_connect_sockaddr,
const struct sockaddr *bindaddr,
socklen_t bindaddr_len,
int *socket_error));
MOCK_DECL(STATIC void, kill_conn_list_for_oos, (smartlist_t *conns));
MOCK_DECL(STATIC smartlist_t *, pick_oos_victims, (int n));
#endif
#endif

4
src/or/connection_or.c
View File

@ -394,8 +394,8 @@ connection_or_change_state(or_connection_t *conn, uint8_t state)
* be an or_connection_t field, but it got moved to channel_t and we
* shouldn't maintain two copies. */
int
connection_or_get_num_circuits(or_connection_t *conn)
MOCK_IMPL(int,
connection_or_get_num_circuits, (or_connection_t *conn))
{
tor_assert(conn);

2
src/or/connection_or.h
View File

@ -64,7 +64,7 @@ void connection_or_init_conn_from_address(or_connection_t *conn,
int connection_or_client_learned_peer_id(or_connection_t *conn,
const uint8_t *peer_id);
time_t connection_or_client_used(or_connection_t *conn);
int connection_or_get_num_circuits(or_connection_t *conn);
MOCK_DECL(int, connection_or_get_num_circuits, (or_connection_t *conn));
void or_handshake_state_free(or_handshake_state_t *state);
void or_handshake_state_record_cell(or_connection_t *conn,
or_handshake_state_t *state,

21
src/or/main.c
View File

@ -381,8 +381,8 @@ connection_in_array(connection_t *conn)
/** Set <b>*array</b> to an array of all connections. <b>*array</b> must not
* be modified.
*/
smartlist_t *
get_connection_array(void)
MOCK_IMPL(smartlist_t *,
get_connection_array, (void))
{
if (!connection_array)
connection_array = smartlist_new();
@ -651,6 +651,23 @@ close_closeable_connections(void)
}
}
/** Count moribund connections for the OOS handler */
MOCK_IMPL(int,
connection_count_moribund, (void))
{
int moribund = 0;
/*
* Count things we'll try to kill when close_closeable_connections()
* runs next.
*/
SMARTLIST_FOREACH_BEGIN(closeable_connection_lst, connection_t *, conn) {
if (SOCKET_OK(conn->s) && connection_is_moribund(conn)) ++moribund;
} SMARTLIST_FOREACH_END(conn);
return moribund;
}
/** Libevent callback: this gets invoked when (connection_t*)<b>conn</b> has
* some data to read. */
static void

4
src/or/main.h
View File

@ -25,7 +25,7 @@ int connection_in_array(connection_t *conn);
void add_connection_to_closeable_list(connection_t *conn);
int connection_is_on_closeable_list(connection_t *conn);
smartlist_t *get_connection_array(void);
MOCK_DECL(smartlist_t *, get_connection_array, (void));
MOCK_DECL(uint64_t,get_bytes_read,(void));
MOCK_DECL(uint64_t,get_bytes_written,(void));
@ -47,6 +47,8 @@ MOCK_DECL(void,connection_start_writing,(connection_t *conn));
void connection_stop_reading_from_linked_conn(connection_t *conn);
MOCK_DECL(int, connection_count_moribund, (void));
void directory_all_unreachable(time_t now);
void directory_info_has_arrived(time_t now, int from_cache, int suppress_logs);

7
src/or/or.h
View File

@ -3699,6 +3699,10 @@ typedef struct {
int ConnLimit; /**< Demanded minimum number of simultaneous connections. */
int ConnLimit_; /**< Maximum allowed number of simultaneous connections. */
int ConnLimit_high_thresh; /**< start trying to lower socket usage if we
* have this many. */
int ConnLimit_low_thresh; /**< try to get down to here after socket
* exhaustion. */
int RunAsDaemon; /**< If true, run in the background. (Unix only) */
int FascistFirewall; /**< Whether to prefer ORs reachable on open ports. */
smartlist_t *FirewallPorts; /**< Which ports our firewall allows
@ -4454,6 +4458,9 @@ typedef struct {
* participate in the protocol. If on (default), a flag is added to the
* vote indicating participation. */
int AuthDirSharedRandomness;
/** If 1, we skip all OOS checks. */
int DisableOOSCheck;
} or_options_t;
/** Persistent state for an onion router, as saved to disk. */

1
src/test/include.am
View File

@ -103,6 +103,7 @@ src_test_test_SOURCES = \
src/test/test_microdesc.c \
src/test/test_nodelist.c \
src/test/test_oom.c \
src/test/test_oos.c \
src/test/test_options.c \
src/test/test_policy.c \
src/test/test_procmon.c \

1
src/test/test.c
View File

@ -1210,6 +1210,7 @@ struct testgroup_t testgroups[] = {
{ "link-handshake/", link_handshake_tests },
{ "nodelist/", nodelist_tests },
{ "oom/", oom_tests },
{ "oos/", oos_tests },
{ "options/", options_tests },
{ "policy/" , policy_tests },
{ "procmon/", procmon_tests },

1
src/test/test.h
View File

@ -202,6 +202,7 @@ extern struct testcase_t logging_tests[];
extern struct testcase_t microdesc_tests[];
extern struct testcase_t nodelist_tests[];
extern struct testcase_t oom_tests[];
extern struct testcase_t oos_tests[];
extern struct testcase_t options_tests[];
extern struct testcase_t policy_tests[];
extern struct testcase_t procmon_tests[];

454
src/test/test_oos.c Normal file
View File

@ -0,0 +1,454 @@
/* Copyright (c) 2016, The Tor Project, Inc. */
/* See LICENSE for licensing information */
/* Unit tests for OOS handler */
#define CONNECTION_PRIVATE
#include "or.h"
#include "config.h"
#include "connection.h"
#include "connection_or.h"
#include "main.h"
#include "test.h"
static or_options_t mock_options;
static void
reset_options_mock(void)
{
memset(&mock_options, 0, sizeof(or_options_t));
}
static const or_options_t *
mock_get_options(void)
{
return &mock_options;
}
static int moribund_calls = 0;
static int moribund_conns = 0;
static int
mock_connection_count_moribund(void)
{
++moribund_calls;
return moribund_conns;
}
/*
* For unit test purposes it's sufficient to tell that
* kill_conn_list_for_oos() was called with an approximately
* sane argument; it's just the thing we returned from the
* mock for pick_oos_victims().
*/
static int kill_conn_list_calls = 0;
static int kill_conn_list_killed = 0;
static void
kill_conn_list_mock(smartlist_t *conns)
{
++kill_conn_list_calls;
tt_assert(conns != NULL);
kill_conn_list_killed += smartlist_len(conns);
done:
return;
}
static int pick_oos_mock_calls = 0;
static int pick_oos_mock_fail = 0;
static int pick_oos_mock_last_n = 0;
static smartlist_t *
pick_oos_victims_mock(int n)
{
smartlist_t *l;
int i;
++pick_oos_mock_calls;
tt_int_op(n, OP_GT, 0);
if (!pick_oos_mock_fail) {
/*
* connection_check_oos() just passes the list onto
* kill_conn_list_for_oos(); we don't need to simulate
* its content for this mock, just its existence, but
* we do need to check the parameter.
*/
l = smartlist_new();
for (i = 0; i < n; ++i) smartlist_add(l, NULL);
} else {
l = NULL;
}
pick_oos_mock_last_n = n;
done:
return l;
}
/** Unit test for the logic in connection_check_oos(), which is concerned
* with comparing thresholds and connection counts to decide if an OOS has
* occurred and if so, how many connections to try to kill, and then using
* pick_oos_victims() and kill_conn_list_for_oos() to carry out its grim
* duty.
*/
static void
test_oos_connection_check_oos(void *arg)
{
(void)arg;
/* Set up mocks */
reset_options_mock();
/* OOS handling is only sensitive to these fields */
mock_options.ConnLimit = 32;
mock_options.ConnLimit_ = 64;
mock_options.ConnLimit_high_thresh = 60;
mock_options.ConnLimit_low_thresh = 50;
MOCK(get_options, mock_get_options);
moribund_calls = 0;
moribund_conns = 0;
MOCK(connection_count_moribund, mock_connection_count_moribund);
kill_conn_list_calls = 0;
kill_conn_list_killed = 0;
MOCK(kill_conn_list_for_oos, kill_conn_list_mock);
pick_oos_mock_calls = 0;
pick_oos_mock_fail = 0;
MOCK(pick_oos_victims, pick_oos_victims_mock);
/* No OOS case */
connection_check_oos(50, 0);
tt_int_op(moribund_calls, OP_EQ, 0);
tt_int_op(pick_oos_mock_calls, OP_EQ, 0);
tt_int_op(kill_conn_list_calls, OP_EQ, 0);
/* OOS from socket count, nothing moribund */
connection_check_oos(62, 0);
tt_int_op(moribund_calls, OP_EQ, 1);
tt_int_op(pick_oos_mock_calls, OP_EQ, 1);
/* 12 == 62 - ConnLimit_low_thresh */
tt_int_op(pick_oos_mock_last_n, OP_EQ, 12);
tt_int_op(kill_conn_list_calls, OP_EQ, 1);
tt_int_op(kill_conn_list_killed, OP_EQ, 12);
/* OOS from socket count, some are moribund */
kill_conn_list_killed = 0;
moribund_conns = 5;
connection_check_oos(62, 0);
tt_int_op(moribund_calls, OP_EQ, 2);
tt_int_op(pick_oos_mock_calls, OP_EQ, 2);
/* 7 == 62 - ConnLimit_low_thresh - moribund_conns */
tt_int_op(pick_oos_mock_last_n, OP_EQ, 7);
tt_int_op(kill_conn_list_calls, OP_EQ, 2);
tt_int_op(kill_conn_list_killed, OP_EQ, 7);
/* OOS from socket count, but pick fails */
kill_conn_list_killed = 0;
moribund_conns = 0;
pick_oos_mock_fail = 1;
connection_check_oos(62, 0);
tt_int_op(moribund_calls, OP_EQ, 3);
tt_int_op(pick_oos_mock_calls, OP_EQ, 3);
tt_int_op(kill_conn_list_calls, OP_EQ, 2);
tt_int_op(kill_conn_list_killed, OP_EQ, 0);
pick_oos_mock_fail = 0;
/*
* OOS from socket count with so many moribund conns
* we have none to kill.
*/
kill_conn_list_killed = 0;
moribund_conns = 15;
connection_check_oos(62, 0);
tt_int_op(moribund_calls, OP_EQ, 4);
tt_int_op(pick_oos_mock_calls, OP_EQ, 3);
tt_int_op(kill_conn_list_calls, OP_EQ, 2);
/*
* OOS from socket exhaustion; OOS handler will try to
* kill 1/10 (5) of the connections.
*/
kill_conn_list_killed = 0;
moribund_conns = 0;
connection_check_oos(50, 1);
tt_int_op(moribund_calls, OP_EQ, 5);
tt_int_op(pick_oos_mock_calls, OP_EQ, 4);
tt_int_op(kill_conn_list_calls, OP_EQ, 3);
tt_int_op(kill_conn_list_killed, OP_EQ, 5);
/* OOS from socket exhaustion with moribund conns */
kill_conn_list_killed = 0;
moribund_conns = 2;
connection_check_oos(50, 1);
tt_int_op(moribund_calls, OP_EQ, 6);
tt_int_op(pick_oos_mock_calls, OP_EQ, 5);
tt_int_op(kill_conn_list_calls, OP_EQ, 4);
tt_int_op(kill_conn_list_killed, OP_EQ, 3);
/* OOS from socket exhaustion with many moribund conns */
kill_conn_list_killed = 0;
moribund_conns = 7;
connection_check_oos(50, 1);
tt_int_op(moribund_calls, OP_EQ, 7);
tt_int_op(pick_oos_mock_calls, OP_EQ, 5);
tt_int_op(kill_conn_list_calls, OP_EQ, 4);
/* OOS with both socket exhaustion and above-threshold */
kill_conn_list_killed = 0;
moribund_conns = 0;
connection_check_oos(62, 1);
tt_int_op(moribund_calls, OP_EQ, 8);
tt_int_op(pick_oos_mock_calls, OP_EQ, 6);
tt_int_op(kill_conn_list_calls, OP_EQ, 5);
tt_int_op(kill_conn_list_killed, OP_EQ, 12);
/*
* OOS with both socket exhaustion and above-threshold with some
* moribund conns
*/
kill_conn_list_killed = 0;
moribund_conns = 5;
connection_check_oos(62, 1);
tt_int_op(moribund_calls, OP_EQ, 9);
tt_int_op(pick_oos_mock_calls, OP_EQ, 7);
tt_int_op(kill_conn_list_calls, OP_EQ, 6);
tt_int_op(kill_conn_list_killed, OP_EQ, 7);
/*
* OOS with both socket exhaustion and above-threshold with many
* moribund conns
*/
kill_conn_list_killed = 0;
moribund_conns = 15;
connection_check_oos(62, 1);
tt_int_op(moribund_calls, OP_EQ, 10);
tt_int_op(pick_oos_mock_calls, OP_EQ, 7);
tt_int_op(kill_conn_list_calls, OP_EQ, 6);
done:
UNMOCK(pick_oos_victims);
UNMOCK(kill_conn_list_for_oos);
UNMOCK(connection_count_moribund);
UNMOCK(get_options);
return;
}
static int cfe_calls = 0;
static void
close_for_error_mock(or_connection_t *orconn, int flush)
{
(void)flush;
tt_assert(orconn != NULL);
++cfe_calls;
done:
return;
}
static int mark_calls = 0;
static void
mark_for_close_oos_mock(connection_t *conn,
int line, const char *file)
{
(void)line;
(void)file;
tt_assert(conn != NULL);
++mark_calls;
done:
return;
}
static void
test_oos_kill_conn_list(void *arg)
{
connection_t *c1, *c2;
or_connection_t *or_c1 = NULL;
dir_connection_t *dir_c2 = NULL;
smartlist_t *l = NULL;
(void)arg;
/* Set up mocks */
mark_calls = 0;
MOCK(connection_mark_for_close_internal_, mark_for_close_oos_mock);
cfe_calls = 0;
MOCK(connection_or_close_for_error, close_for_error_mock);
/* Make fake conns */
or_c1 = tor_malloc_zero(sizeof(*or_c1));
or_c1->base_.magic = OR_CONNECTION_MAGIC;
or_c1->base_.type = CONN_TYPE_OR;
c1 = TO_CONN(or_c1);
dir_c2 = tor_malloc_zero(sizeof(*dir_c2));
dir_c2->base_.magic = DIR_CONNECTION_MAGIC;
dir_c2->base_.type = CONN_TYPE_DIR;
c2 = TO_CONN(dir_c2);
tt_assert(c1 != NULL);
tt_assert(c2 != NULL);
/* Make list */
l = smartlist_new();
smartlist_add(l, c1);
smartlist_add(l, c2);
/* Run kill_conn_list_for_oos() */
kill_conn_list_for_oos(l);
/* Check call counters */
tt_int_op(mark_calls, OP_EQ, 1);
tt_int_op(cfe_calls, OP_EQ, 1);
done:
UNMOCK(connection_or_close_for_error);
UNMOCK(connection_mark_for_close_internal_);
if (l) smartlist_free(l);
tor_free(or_c1);
tor_free(dir_c2);
return;
}
static smartlist_t *conns_for_mock = NULL;
static smartlist_t *
get_conns_mock(void)
{
return conns_for_mock;
}
/*
* For this mock, we pretend all conns have either zero or one circuits,
* depending on if this appears on the list of things to say have a circuit.
*/
static smartlist_t *conns_with_circs = NULL;
static int
get_num_circuits_mock(or_connection_t *conn)
{
int circs = 0;
tt_assert(conn != NULL);
if (conns_with_circs &&
smartlist_contains(conns_with_circs, TO_CONN(conn))) {
circs = 1;
}
done:
return circs;
}
static void
test_oos_pick_oos_victims(void *arg)
{
(void)arg;
or_connection_t *ortmp;
dir_connection_t *dirtmp;
smartlist_t *picked;
/* Set up mocks */
conns_for_mock = smartlist_new();
MOCK(get_connection_array, get_conns_mock);
conns_with_circs = smartlist_new();
MOCK(connection_or_get_num_circuits, get_num_circuits_mock);
/* Make some fake connections */
ortmp = tor_malloc_zero(sizeof(*ortmp));
ortmp->base_.magic = OR_CONNECTION_MAGIC;
ortmp->base_.type = CONN_TYPE_OR;
smartlist_add(conns_for_mock, TO_CONN(ortmp));
/* We'll pretend this one has a circuit too */
smartlist_add(conns_with_circs, TO_CONN(ortmp));
/* Next one */
ortmp = tor_malloc_zero(sizeof(*ortmp));
ortmp->base_.magic = OR_CONNECTION_MAGIC;
ortmp->base_.type = CONN_TYPE_OR;
smartlist_add(conns_for_mock, TO_CONN(ortmp));
/* Next one is moribund */
ortmp = tor_malloc_zero(sizeof(*ortmp));
ortmp->base_.magic = OR_CONNECTION_MAGIC;
ortmp->base_.type = CONN_TYPE_OR;
ortmp->base_.marked_for_close = 1;
smartlist_add(conns_for_mock, TO_CONN(ortmp));
/* Last one isn't an orconn */
dirtmp = tor_malloc_zero(sizeof(*dirtmp));
dirtmp->base_.magic = DIR_CONNECTION_MAGIC;
dirtmp->base_.type = CONN_TYPE_DIR;
smartlist_add(conns_for_mock, TO_CONN(dirtmp));
/* Try picking one */
picked = pick_oos_victims(1);
/* It should be the one with circuits */
tt_assert(picked != NULL);
tt_int_op(smartlist_len(picked), OP_EQ, 1);
tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
smartlist_free(picked);
/* Try picking none */
picked = pick_oos_victims(0);
/* We should get an empty list */
tt_assert(picked != NULL);
tt_int_op(smartlist_len(picked), OP_EQ, 0);
smartlist_free(picked);
/* Try picking two */
picked = pick_oos_victims(2);
/* We should get both active orconns */
tt_assert(picked != NULL);
tt_int_op(smartlist_len(picked), OP_EQ, 2);
tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
smartlist_free(picked);
/* Try picking three - only two are eligible */
picked = pick_oos_victims(3);
tt_int_op(smartlist_len(picked), OP_EQ, 2);
tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
smartlist_free(picked);
done:
/* Free leftover stuff */
if (conns_with_circs) {
smartlist_free(conns_with_circs);
conns_with_circs = NULL;
}
UNMOCK(connection_or_get_num_circuits);
if (conns_for_mock) {
SMARTLIST_FOREACH(conns_for_mock, connection_t *, c, tor_free(c));
smartlist_free(conns_for_mock);
conns_for_mock = NULL;
}
UNMOCK(get_connection_array);
return;
}
struct testcase_t oos_tests[] = {
{ "connection_check_oos", test_oos_connection_check_oos,
TT_FORK, NULL, NULL },
{ "kill_conn_list", test_oos_kill_conn_list, TT_FORK, NULL, NULL },
{ "pick_oos_victims", test_oos_pick_oos_victims, TT_FORK, NULL, NULL },
END_OF_TESTCASES
};