Split bidi connection-stats code into a new C file.

This commit is contained in:
Nick Mathewson 2020-07-10 09:07:23 -04:00
parent 1a4e475d5d
commit 50bf2520b3
9 changed files with 266 additions and 228 deletions

View File

@ -105,6 +105,7 @@
#include "feature/relay/routermode.h"
#include "feature/rend/rendclient.h"
#include "feature/rend/rendcommon.h"
#include "feature/stats/connstats.h"
#include "feature/stats/rephist.h"
#include "feature/stats/bwhist.h"
#include "lib/crypt_ops/crypto_util.h"

View File

@ -95,6 +95,7 @@
#include "feature/rend/rendservice.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/predict_ports.h"
#include "feature/stats/connstats.h"
#include "feature/stats/rephist.h"
#include "lib/buf/buffers.h"
#include "lib/crypt_ops/crypto_rand.h"

View File

@ -36,6 +36,7 @@
#include "feature/nodelist/nickname.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/predict_ports.h"
#include "feature/stats/connstats.h"
#include "feature/stats/rephist.h"
#include "feature/dirauth/authmode.h"

View File

@ -0,0 +1,235 @@
/* Copyright (c) 2001 Matej Pfajfar.
* Copyright (c) 2001-2004, Roger Dingledine.
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2020, The Tor Project, Inc. */
/* See LICENSE for licensing information */
/**
* @file connstats.c
* @brief Count bidirectional vs one-way connections.
*
* Connection statistics, used by relays to count connections, and track
* one-way and bidirectional connections.
**/
#include "orconfig.h"
#include "core/or/or.h"
#include "feature/stats/connstats.h"
#include "app/config/config.h"
/** Start of the current connection stats interval or 0 if we're not
* collecting connection statistics. */
static time_t start_of_conn_stats_interval;
/** Initialize connection stats. */
void
rep_hist_conn_stats_init(time_t now)
{
start_of_conn_stats_interval = now;
}
/* Count connections that we read and wrote less than these many bytes
* from/to as below threshold. */
#define BIDI_THRESHOLD 20480
/* Count connections that we read or wrote at least this factor as many
* bytes from/to than we wrote or read to/from as mostly reading or
* writing. */
#define BIDI_FACTOR 10
/* Interval length in seconds for considering read and written bytes for
* connection stats. */
#define BIDI_INTERVAL 10
/** Start of next BIDI_INTERVAL second interval. */
static time_t bidi_next_interval = 0;
/** Number of connections that we read and wrote less than BIDI_THRESHOLD
* bytes from/to in BIDI_INTERVAL seconds. */
static uint32_t below_threshold = 0;
/** Number of connections that we read at least BIDI_FACTOR times more
* bytes from than we wrote to in BIDI_INTERVAL seconds. */
static uint32_t mostly_read = 0;
/** Number of connections that we wrote at least BIDI_FACTOR times more
* bytes to than we read from in BIDI_INTERVAL seconds. */
static uint32_t mostly_written = 0;
/** Number of connections that we read and wrote at least BIDI_THRESHOLD
* bytes from/to, but not BIDI_FACTOR times more in either direction in
* BIDI_INTERVAL seconds. */
static uint32_t both_read_and_written = 0;
/** Entry in a map from connection ID to the number of read and written
* bytes on this connection in a BIDI_INTERVAL second interval. */
typedef struct bidi_map_entry_t {
HT_ENTRY(bidi_map_entry_t) node;
uint64_t conn_id; /**< Connection ID */
size_t read; /**< Number of read bytes */
size_t written; /**< Number of written bytes */
} bidi_map_entry_t;
/** Map of OR connections together with the number of read and written
* bytes in the current BIDI_INTERVAL second interval. */
static HT_HEAD(bidimap, bidi_map_entry_t) bidi_map =
HT_INITIALIZER();
static int
bidi_map_ent_eq(const bidi_map_entry_t *a, const bidi_map_entry_t *b)
{
return a->conn_id == b->conn_id;
}
/* DOCDOC bidi_map_ent_hash */
static unsigned
bidi_map_ent_hash(const bidi_map_entry_t *entry)
{
return (unsigned) entry->conn_id;
}
HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
bidi_map_ent_eq);
HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_);
/** Release all storage held in connstats.c */
void
bidi_map_free_all(void)
{
bidi_map_entry_t **ptr, **next, *ent;
for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) {
ent = *ptr;
next = HT_NEXT_RMV(bidimap, &bidi_map, ptr);
tor_free(ent);
}
HT_CLEAR(bidimap, &bidi_map);
}
/** Reset counters for conn statistics. */
void
rep_hist_reset_conn_stats(time_t now)
{
start_of_conn_stats_interval = now;
below_threshold = 0;
mostly_read = 0;
mostly_written = 0;
both_read_and_written = 0;
bidi_map_free_all();
}
/** Stop collecting connection stats in a way that we can re-start doing
* so in rep_hist_conn_stats_init(). */
void
rep_hist_conn_stats_term(void)
{
rep_hist_reset_conn_stats(0);
}
/** We read <b>num_read</b> bytes and wrote <b>num_written</b> from/to OR
* connection <b>conn_id</b> in second <b>when</b>. If this is the first
* observation in a new interval, sum up the last observations. Add bytes
* for this connection. */
void
rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read,
size_t num_written, time_t when)
{
if (!start_of_conn_stats_interval)
return;
/* Initialize */
if (bidi_next_interval == 0)
bidi_next_interval = when + BIDI_INTERVAL;
/* Sum up last period's statistics */
if (when >= bidi_next_interval) {
bidi_map_entry_t **ptr, **next, *ent;
for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) {
ent = *ptr;
if (ent->read + ent->written < BIDI_THRESHOLD)
below_threshold++;
else if (ent->read >= ent->written * BIDI_FACTOR)
mostly_read++;
else if (ent->written >= ent->read * BIDI_FACTOR)
mostly_written++;
else
both_read_and_written++;
next = HT_NEXT_RMV(bidimap, &bidi_map, ptr);
tor_free(ent);
}
while (when >= bidi_next_interval)
bidi_next_interval += BIDI_INTERVAL;
log_info(LD_GENERAL, "%d below threshold, %d mostly read, "
"%d mostly written, %d both read and written.",
below_threshold, mostly_read, mostly_written,
both_read_and_written);
}
/* Add this connection's bytes. */
if (num_read > 0 || num_written > 0) {
bidi_map_entry_t *entry, lookup;
lookup.conn_id = conn_id;
entry = HT_FIND(bidimap, &bidi_map, &lookup);
if (entry) {
entry->written += num_written;
entry->read += num_read;
} else {
entry = tor_malloc_zero(sizeof(bidi_map_entry_t));
entry->conn_id = conn_id;
entry->written = num_written;
entry->read = num_read;
HT_INSERT(bidimap, &bidi_map, entry);
}
}
}
/** Return a newly allocated string containing the connection statistics
* until <b>now</b>, or NULL if we're not collecting conn stats. Caller must
* ensure start_of_conn_stats_interval is in the past. */
char *
rep_hist_format_conn_stats(time_t now)
{
char *result, written[ISO_TIME_LEN+1];
if (!start_of_conn_stats_interval)
return NULL; /* Not initialized. */
tor_assert(now >= start_of_conn_stats_interval);
format_iso_time(written, now);
tor_asprintf(&result, "conn-bi-direct %s (%d s) %d,%d,%d,%d\n",
written,
(unsigned) (now - start_of_conn_stats_interval),
below_threshold,
mostly_read,
mostly_written,
both_read_and_written);
return result;
}
/** If 24 hours have passed since the beginning of the current conn stats
* period, write conn stats to $DATADIR/stats/conn-stats (possibly
* overwriting an existing file) and reset counters. Return when we would
* next want to write conn stats or 0 if we never want to write. */
time_t
rep_hist_conn_stats_write(time_t now)
{
char *str = NULL;
if (!start_of_conn_stats_interval)
return 0; /* Not initialized. */
if (start_of_conn_stats_interval + WRITE_STATS_INTERVAL > now)
goto done; /* Not ready to write */
/* Generate history string. */
str = rep_hist_format_conn_stats(now);
/* Reset counters. */
rep_hist_reset_conn_stats(now);
/* Try to write to disk. */
if (!check_or_create_data_subdir("stats")) {
write_to_data_subdir("stats", "conn-stats", str, "connection statistics");
}
done:
tor_free(str);
return start_of_conn_stats_interval + WRITE_STATS_INTERVAL;
}

View File

@ -0,0 +1,24 @@
/* Copyright (c) 2001 Matej Pfajfar.
* Copyright (c) 2001-2004, Roger Dingledine.
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2020, The Tor Project, Inc. */
/* See LICENSE for licensing information */
/**
* @file connstats.h
* @brief Header for feature/stats/connstats.c
**/
#ifndef TOR_FEATURE_STATS_CONNSTATS_H
#define TOR_FEATURE_STATS_CONNSTATS_H
void rep_hist_conn_stats_init(time_t now);
void rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read,
size_t num_written, time_t when);
void rep_hist_reset_conn_stats(time_t now);
char *rep_hist_format_conn_stats(time_t now);
time_t rep_hist_conn_stats_write(time_t now);
void rep_hist_conn_stats_term(void);
void bidi_map_free_all(void);
#endif /* !defined(TOR_FEATURE_STATS_CONNSTATS_H) */

View File

@ -2,6 +2,7 @@
# ADD_C_FILE: INSERT SOURCES HERE.
LIBTOR_APP_A_SOURCES += \
src/feature/stats/bwhist.c \
src/feature/stats/connstats.c \
src/feature/stats/geoip_stats.c \
src/feature/stats/rephist.c \
src/feature/stats/predict_ports.c
@ -9,6 +10,7 @@ LIBTOR_APP_A_SOURCES += \
# ADD_C_FILE: INSERT HEADERS HERE.
noinst_HEADERS += \
src/feature/stats/bwhist.h \
src/feature/stats/connstats.h \
src/feature/stats/geoip_stats.h \
src/feature/stats/rephist.h \
src/feature/stats/predict_ports.h

View File

@ -43,9 +43,6 @@
* <li>Descriptor serving statistics, used by directory caches to track
* how many descriptors they've served.
*
* <li>Connection statistics, used by relays to track one-way and
* bidirectional connections.
*
* <li>Onion handshake statistics, used by relays to count how many
* TAP and ntor handshakes they've handled.
*
@ -78,6 +75,7 @@
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/nodelist.h"
#include "feature/stats/predict_ports.h"
#include "feature/stats/connstats.h"
#include "feature/stats/rephist.h"
#include "lib/container/order.h"
#include "lib/crypt_ops/crypto_rand.h"
@ -1648,223 +1646,6 @@ rep_hist_note_desc_served(const char * desc)
/*** Connection statistics ***/
/** Start of the current connection stats interval or 0 if we're not
* collecting connection statistics. */
static time_t start_of_conn_stats_interval;
/** Initialize connection stats. */
void
rep_hist_conn_stats_init(time_t now)
{
start_of_conn_stats_interval = now;
}
/* Count connections that we read and wrote less than these many bytes
* from/to as below threshold. */
#define BIDI_THRESHOLD 20480
/* Count connections that we read or wrote at least this factor as many
* bytes from/to than we wrote or read to/from as mostly reading or
* writing. */
#define BIDI_FACTOR 10
/* Interval length in seconds for considering read and written bytes for
* connection stats. */
#define BIDI_INTERVAL 10
/** Start of next BIDI_INTERVAL second interval. */
static time_t bidi_next_interval = 0;
/** Number of connections that we read and wrote less than BIDI_THRESHOLD
* bytes from/to in BIDI_INTERVAL seconds. */
static uint32_t below_threshold = 0;
/** Number of connections that we read at least BIDI_FACTOR times more
* bytes from than we wrote to in BIDI_INTERVAL seconds. */
static uint32_t mostly_read = 0;
/** Number of connections that we wrote at least BIDI_FACTOR times more
* bytes to than we read from in BIDI_INTERVAL seconds. */
static uint32_t mostly_written = 0;
/** Number of connections that we read and wrote at least BIDI_THRESHOLD
* bytes from/to, but not BIDI_FACTOR times more in either direction in
* BIDI_INTERVAL seconds. */
static uint32_t both_read_and_written = 0;
/** Entry in a map from connection ID to the number of read and written
* bytes on this connection in a BIDI_INTERVAL second interval. */
typedef struct bidi_map_entry_t {
HT_ENTRY(bidi_map_entry_t) node;
uint64_t conn_id; /**< Connection ID */
size_t read; /**< Number of read bytes */
size_t written; /**< Number of written bytes */
} bidi_map_entry_t;
/** Map of OR connections together with the number of read and written
* bytes in the current BIDI_INTERVAL second interval. */
static HT_HEAD(bidimap, bidi_map_entry_t) bidi_map =
HT_INITIALIZER();
static int
bidi_map_ent_eq(const bidi_map_entry_t *a, const bidi_map_entry_t *b)
{
return a->conn_id == b->conn_id;
}
/* DOCDOC bidi_map_ent_hash */
static unsigned
bidi_map_ent_hash(const bidi_map_entry_t *entry)
{
return (unsigned) entry->conn_id;
}
HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
bidi_map_ent_eq);
HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_);
/* DOCDOC bidi_map_free */
static void
bidi_map_free_all(void)
{
bidi_map_entry_t **ptr, **next, *ent;
for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) {
ent = *ptr;
next = HT_NEXT_RMV(bidimap, &bidi_map, ptr);
tor_free(ent);
}
HT_CLEAR(bidimap, &bidi_map);
}
/** Reset counters for conn statistics. */
void
rep_hist_reset_conn_stats(time_t now)
{
start_of_conn_stats_interval = now;
below_threshold = 0;
mostly_read = 0;
mostly_written = 0;
both_read_and_written = 0;
bidi_map_free_all();
}
/** Stop collecting connection stats in a way that we can re-start doing
* so in rep_hist_conn_stats_init(). */
void
rep_hist_conn_stats_term(void)
{
rep_hist_reset_conn_stats(0);
}
/** We read <b>num_read</b> bytes and wrote <b>num_written</b> from/to OR
* connection <b>conn_id</b> in second <b>when</b>. If this is the first
* observation in a new interval, sum up the last observations. Add bytes
* for this connection. */
void
rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read,
size_t num_written, time_t when)
{
if (!start_of_conn_stats_interval)
return;
/* Initialize */
if (bidi_next_interval == 0)
bidi_next_interval = when + BIDI_INTERVAL;
/* Sum up last period's statistics */
if (when >= bidi_next_interval) {
bidi_map_entry_t **ptr, **next, *ent;
for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) {
ent = *ptr;
if (ent->read + ent->written < BIDI_THRESHOLD)
below_threshold++;
else if (ent->read >= ent->written * BIDI_FACTOR)
mostly_read++;
else if (ent->written >= ent->read * BIDI_FACTOR)
mostly_written++;
else
both_read_and_written++;
next = HT_NEXT_RMV(bidimap, &bidi_map, ptr);
tor_free(ent);
}
while (when >= bidi_next_interval)
bidi_next_interval += BIDI_INTERVAL;
log_info(LD_GENERAL, "%d below threshold, %d mostly read, "
"%d mostly written, %d both read and written.",
below_threshold, mostly_read, mostly_written,
both_read_and_written);
}
/* Add this connection's bytes. */
if (num_read > 0 || num_written > 0) {
bidi_map_entry_t *entry, lookup;
lookup.conn_id = conn_id;
entry = HT_FIND(bidimap, &bidi_map, &lookup);
if (entry) {
entry->written += num_written;
entry->read += num_read;
} else {
entry = tor_malloc_zero(sizeof(bidi_map_entry_t));
entry->conn_id = conn_id;
entry->written = num_written;
entry->read = num_read;
HT_INSERT(bidimap, &bidi_map, entry);
}
}
}
/** Return a newly allocated string containing the connection statistics
* until <b>now</b>, or NULL if we're not collecting conn stats. Caller must
* ensure start_of_conn_stats_interval is in the past. */
char *
rep_hist_format_conn_stats(time_t now)
{
char *result, written[ISO_TIME_LEN+1];
if (!start_of_conn_stats_interval)
return NULL; /* Not initialized. */
tor_assert(now >= start_of_conn_stats_interval);
format_iso_time(written, now);
tor_asprintf(&result, "conn-bi-direct %s (%d s) %d,%d,%d,%d\n",
written,
(unsigned) (now - start_of_conn_stats_interval),
below_threshold,
mostly_read,
mostly_written,
both_read_and_written);
return result;
}
/** If 24 hours have passed since the beginning of the current conn stats
* period, write conn stats to $DATADIR/stats/conn-stats (possibly
* overwriting an existing file) and reset counters. Return when we would
* next want to write conn stats or 0 if we never want to write. */
time_t
rep_hist_conn_stats_write(time_t now)
{
char *str = NULL;
if (!start_of_conn_stats_interval)
return 0; /* Not initialized. */
if (start_of_conn_stats_interval + WRITE_STATS_INTERVAL > now)
goto done; /* Not ready to write */
/* Generate history string. */
str = rep_hist_format_conn_stats(now);
/* Reset counters. */
rep_hist_reset_conn_stats(now);
/* Try to write to disk. */
if (!check_or_create_data_subdir("stats")) {
write_to_data_subdir("stats", "conn-stats", str, "connection statistics");
}
done:
tor_free(str);
return start_of_conn_stats_interval + WRITE_STATS_INTERVAL;
}
/** Internal statistics to track how many requests of each type of
* handshake we've received, and how many we've assigned to cpuworkers.
* Useful for seeing trends in cpu load.

View File

@ -56,14 +56,6 @@ void rep_hist_note_desc_served(const char * desc);
void rep_hist_desc_stats_term(void);
time_t rep_hist_desc_stats_write(time_t now);
void rep_hist_conn_stats_init(time_t now);
void rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read,
size_t num_written, time_t when);
void rep_hist_reset_conn_stats(time_t now);
char *rep_hist_format_conn_stats(time_t now);
time_t rep_hist_conn_stats_write(time_t now);
void rep_hist_conn_stats_term(void);
void rep_hist_note_circuit_handshake_requested(uint16_t type);
void rep_hist_note_circuit_handshake_assigned(uint16_t type);
void rep_hist_log_circuit_handshake_stats(time_t now);

View File

@ -39,6 +39,7 @@
#include "test/test.h"
#include "core/mainloop/mainloop.h"
#include "lib/memarea/memarea.h"
#include "feature/stats/connstats.h"
#include "feature/stats/rephist.h"
#include "app/config/statefile.h"