Split geoip from geoip-related stats.

This commit just moves the code to two separate files. The geoip
code still has a few needless dependencies on core/* and features/*.
This commit is contained in:
Nick Mathewson 2018-09-27 09:35:47 -04:00
parent 79208ee852
commit f403af2207
21 changed files with 1674 additions and 1570 deletions

View File

@ -103,6 +103,7 @@
#include "feature/rend/rendclient.h"
#include "feature/rend/rendservice.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/predict_ports.h"
#include "feature/stats/rephist.h"
#include "lib/compress/compress.h"

View File

@ -27,6 +27,7 @@
#include "core/or/command.h"
#include "core/or/connection_edge.h"
#include "core/or/connection_or.h"
#include "core/or/dos.h"
#include "core/or/policies.h"
#include "core/or/protover.h"
#include "core/or/relay.h"
@ -61,6 +62,7 @@
#include "feature/rend/rendclient.h"
#include "feature/rend/rendservice.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/predict_ports.h"
#include "feature/stats/rephist.h"
#include "lib/compress/compress.h"
@ -755,6 +757,7 @@ tor_free_all(int postfork)
evdns_shutdown(1);
}
geoip_free_all();
geoip_stats_free_all();
dirvote_free_all();
routerlist_free_all();
networkstatus_free_all();

View File

@ -60,6 +60,7 @@ LIBTOR_APP_A_SOURCES = \
src/feature/client/transports.c \
src/feature/control/control.c \
src/feature/control/fmt_serverstatus.c \
src/feature/control/getinfo_geoip.c \
src/feature/dirauth/keypin.c \
src/feature/dircache/conscache.c \
src/feature/dircache/consdiffmgr.c \
@ -117,6 +118,7 @@ LIBTOR_APP_A_SOURCES = \
src/feature/rend/rendmid.c \
src/feature/rend/rendservice.c \
src/feature/stats/geoip.c \
src/feature/stats/geoip_stats.c \
src/feature/stats/rephist.c \
src/feature/stats/predict_ports.c
@ -254,6 +256,7 @@ noinst_HEADERS += \
src/feature/control/control.h \
src/feature/control/control_connection_st.h \
src/feature/control/fmt_serverstatus.h \
src/feature/control/getinfo_geoip.h \
src/feature/dirauth/authmode.h \
src/feature/dirauth/bwauth.h \
src/feature/dirauth/dircollate.h \
@ -348,6 +351,7 @@ noinst_HEADERS += \
src/feature/rend/rendmid.h \
src/feature/rend/rendservice.h \
src/feature/stats/geoip.h \
src/feature/stats/geoip_stats.h \
src/feature/stats/rephist.h \
src/feature/stats/predict_ports.h

View File

@ -92,7 +92,7 @@
#include "feature/relay/selftest.h"
#include "feature/rend/rendcache.h"
#include "feature/rend/rendservice.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/predict_ports.h"
#include "feature/stats/rephist.h"
#include "lib/container/buffers.h"

View File

@ -58,28 +58,29 @@
#define CHANNEL_PRIVATE_
#include "core/or/or.h"
#include "app/config/config.h"
#include "core/mainloop/mainloop.h"
#include "core/or/channel.h"
#include "core/or/channeltls.h"
#include "core/or/channelpadding.h"
#include "core/or/channeltls.h"
#include "core/or/circuitbuild.h"
#include "core/or/circuitlist.h"
#include "core/or/circuitstats.h"
#include "app/config/config.h"
#include "core/or/connection_or.h" /* For var_cell_free() */
#include "core/or/circuitmux.h"
#include "feature/client/entrynodes.h"
#include "feature/stats/geoip.h"
#include "core/mainloop/mainloop.h"
#include "feature/nodelist/nodelist.h"
#include "core/or/circuitstats.h"
#include "core/or/connection_or.h" /* For var_cell_free() */
#include "core/or/dos.h"
#include "core/or/relay.h"
#include "feature/stats/rephist.h"
#include "feature/relay/router.h"
#include "feature/nodelist/routerlist.h"
#include "core/or/scheduler.h"
#include "lib/time/compat_time.h"
#include "feature/client/entrynodes.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/nodelist.h"
#include "feature/nodelist/routerlist.h"
#include "feature/relay/router.h"
#include "feature/rend/rendservice.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/rephist.h"
#include "lib/evloop/timers.h"
#include "lib/time/compat_time.h"
#include "core/or/cell_queue_st.h"

View File

@ -9,17 +9,17 @@
#define DOS_PRIVATE
#include "core/or/or.h"
#include "core/or/channel.h"
#include "app/config/config.h"
#include "core/mainloop/connection.h"
#include "core/or/connection_or.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "feature/stats/geoip.h"
#include "core/mainloop/mainloop.h"
#include "core/or/channel.h"
#include "core/or/connection_or.h"
#include "core/or/relay.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/nodelist.h"
#include "core/or/relay.h"
#include "feature/relay/routermode.h"
#include "feature/stats/geoip_stats.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "core/or/dos.h"

View File

@ -65,7 +65,7 @@
#include "lib/crypt_ops/crypto_util.h"
#include "feature/dircommon/directory.h"
#include "feature/relay/dns.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "feature/hs/hs_cache.h"
#include "core/mainloop/mainloop.h"
#include "feature/nodelist/networkstatus.h"

View File

@ -30,13 +30,13 @@
#include "feature/hs/hs_stats.h"
#include "feature/hs/hs_service.h"
#include "core/or/dos.h"
#include "feature/stats/geoip_stats.h"
#include "app/config/or_state_st.h"
#include "feature/nodelist/routerinfo_st.h"
#include "lib/tls/tortls.h"
static void log_accounting(const time_t now, const or_options_t *options);
#include "feature/stats/geoip.h"
/** Return the total number of circuits. */
STATIC int

View File

@ -60,6 +60,7 @@
#include "feature/client/entrynodes.h"
#include "feature/control/control.h"
#include "feature/control/fmt_serverstatus.h"
#include "feature/control/getinfo_geoip.h"
#include "feature/dircache/dirserv.h"
#include "feature/dirclient/dirclient.h"
#include "feature/dirclient/dlstatus.h"
@ -83,7 +84,7 @@
#include "feature/rend/rendclient.h"
#include "feature/rend/rendcommon.h"
#include "feature/rend/rendservice.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/predict_ports.h"
#include "lib/container/buffers.h"
#include "lib/crypt_ops/crypto_rand.h"

View File

@ -0,0 +1,45 @@
#include "core/or/or.h"
#include "core/mainloop/connection.h"
#include "feature/control/control.h"
#include "feature/control/getinfo_geoip.h"
#include "feature/stats/geoip.h"
/** Helper used to implement GETINFO ip-to-country/... controller command. */
int
getinfo_helper_geoip(control_connection_t *control_conn,
const char *question, char **answer,
const char **errmsg)
{
(void)control_conn;
if (!strcmpstart(question, "ip-to-country/")) {
int c;
sa_family_t family;
tor_addr_t addr;
question += strlen("ip-to-country/");
if (!strcmp(question, "ipv4-available") ||
!strcmp(question, "ipv6-available")) {
family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6;
const int available = geoip_is_loaded(family);
tor_asprintf(answer, "%d", !! available);
return 0;
}
family = tor_addr_parse(&addr, question);
if (family != AF_INET && family != AF_INET6) {
*errmsg = "Invalid address family";
return -1;
}
if (!geoip_is_loaded(family)) {
*errmsg = "GeoIP data not loaded";
return -1;
}
if (family == AF_INET)
c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr));
else /* AF_INET6 */
c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr));
*answer = tor_strdup(geoip_get_country_name(c));
}
return 0;
}

View File

@ -0,0 +1,14 @@
/* Copyright (c) 2001 Matej Pfajfar.
* Copyright (c) 2001-2004, Roger Dingledine.
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2018, The Tor Project, Inc. */
/* See LICENSE for licensing information */
#ifndef TOR_GETINFO_GEOIP_H
#define TOR_GETINFO_GEOIP_H
int getinfo_helper_geoip(control_connection_t *control_conn,
const char *question, char **answer,
const char **errmsg);
#endif

View File

@ -25,7 +25,7 @@
#include "feature/nodelist/routerlist.h"
#include "feature/relay/routermode.h"
#include "feature/rend/rendcache.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/rephist.h"
#include "lib/compress/compress.h"

View File

@ -12,7 +12,7 @@
#include "feature/dirclient/dirclient.h"
#include "feature/dircommon/directory.h"
#include "feature/dircommon/fp_pair.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "lib/compress/compress.h"
#include "feature/dircommon/dir_connection_st.h"

View File

@ -37,6 +37,7 @@
#include "feature/relay/routermode.h"
#include "feature/relay/selftest.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "feature/stats/rephist.h"
#include "lib/crypt_ops/crypto_ed25519.h"
#include "lib/crypt_ops/crypto_format.h"

File diff suppressed because it is too large Load Diff

View File

@ -13,96 +13,22 @@
#define TOR_GEOIP_H
#include "lib/testsupport/testsupport.h"
#include "core/or/dos.h"
/** Indicates an action that we might be noting geoip statistics on.
* Note that if we're noticing CONNECT, we're a bridge, and if we're noticing
* the others, we're not.
*/
typedef enum {
/** We've noticed a connection as a bridge relay or entry guard. */
GEOIP_CLIENT_CONNECT = 0,
/** We've served a networkstatus consensus as a directory server. */
GEOIP_CLIENT_NETWORKSTATUS = 1,
} geoip_client_action_t;
/** Indicates either a positive reply or a reason for rejectng a network
* status request that will be included in geoip statistics. */
typedef enum {
/** Request is answered successfully. */
GEOIP_SUCCESS = 0,
/** V3 network status is not signed by a sufficient number of requested
* authorities. */
GEOIP_REJECT_NOT_ENOUGH_SIGS = 1,
/** Requested network status object is unavailable. */
GEOIP_REJECT_UNAVAILABLE = 2,
/** Requested network status not found. */
GEOIP_REJECT_NOT_FOUND = 3,
/** Network status has not been modified since If-Modified-Since time. */
GEOIP_REJECT_NOT_MODIFIED = 4,
/** Directory is busy. */
GEOIP_REJECT_BUSY = 5,
} geoip_ns_response_t;
#define GEOIP_NS_RESPONSE_NUM 6
/** Directory requests that we are measuring can be either direct or
* tunneled. */
typedef enum {
DIRREQ_DIRECT = 0,
DIRREQ_TUNNELED = 1,
} dirreq_type_t;
/** Possible states for either direct or tunneled directory requests that
* are relevant for determining network status download times. */
typedef enum {
/** Found that the client requests a network status; applies to both
* direct and tunneled requests; initial state of a request that we are
* measuring. */
DIRREQ_IS_FOR_NETWORK_STATUS = 0,
/** Finished writing a network status to the directory connection;
* applies to both direct and tunneled requests; completes a direct
* request. */
DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1,
/** END cell sent to circuit that initiated a tunneled request. */
DIRREQ_END_CELL_SENT = 2,
/** Flushed last cell from queue of the circuit that initiated a
* tunneled request to the outbuf of the OR connection. */
DIRREQ_CIRC_QUEUE_FLUSHED = 3,
/** Flushed last byte from buffer of the channel belonging to the
* circuit that initiated a tunneled request; completes a tunneled
* request. */
DIRREQ_CHANNEL_BUFFER_FLUSHED = 4
} dirreq_state_t;
#ifdef GEOIP_PRIVATE
STATIC int geoip_parse_entry(const char *line, sa_family_t family);
STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr);
STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr);
STATIC void clear_geoip_db(void);
#endif /* defined(GEOIP_PRIVATE) */
/** Entry in a map from IP address to the last time we've seen an incoming
* connection from that IP address. Used by bridges only to track which
* countries have them blocked, or the DoS mitigation subsystem if enabled. */
typedef struct clientmap_entry_t {
HT_ENTRY(clientmap_entry_t) node;
tor_addr_t addr;
/* Name of pluggable transport used by this client. NULL if no
pluggable transport was used. */
char *transport_name;
int geoip_get_country_by_ipv4(uint32_t ipaddr);
int geoip_get_country_by_ipv6(const struct in6_addr *addr);
/** Time when we last saw this IP address, in MINUTES since the epoch.
*
* (This will run out of space around 4011 CE. If Tor is still in use around
* 4000 CE, please remember to add more bits to last_seen_in_minutes.) */
unsigned int last_seen_in_minutes:30;
unsigned int action:2;
/** A per-country GeoIP record. */
typedef struct geoip_country_t {
char countrycode[3];
} geoip_country_t;
/* This object is used to keep some statistics per client address for the
* DoS mitigation subsystem. */
dos_client_stats_t dos_stats;
} clientmap_entry_t;
const smartlist_t *geoip_get_countries(void);
int should_record_bridge_info(const or_options_t *options);
int geoip_load_file(sa_family_t family, const char *filename);
MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
MOCK_DECL(int, geoip_get_n_countries, (void));
@ -111,47 +37,6 @@ MOCK_DECL(int, geoip_is_loaded, (sa_family_t family));
const char *geoip_db_digest(sa_family_t family);
MOCK_DECL(country_t, geoip_get_country, (const char *countrycode));
void geoip_note_client_seen(geoip_client_action_t action,
const tor_addr_t *addr, const char *transport_name,
time_t now);
void geoip_remove_old_clients(time_t cutoff);
clientmap_entry_t *geoip_lookup_client(const tor_addr_t *addr,
const char *transport_name,
geoip_client_action_t action);
size_t geoip_client_cache_total_allocation(void);
size_t geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes);
void geoip_note_ns_response(geoip_ns_response_t response);
char *geoip_get_transport_history(void);
int geoip_get_client_history(geoip_client_action_t action,
char **country_str, char **ipver_str);
char *geoip_get_request_history(void);
int getinfo_helper_geoip(control_connection_t *control_conn,
const char *question, char **answer,
const char **errmsg);
void geoip_free_all(void);
void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
dirreq_type_t type);
void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
dirreq_state_t new_state);
void geoip_dirreq_stats_init(time_t now);
void geoip_reset_dirreq_stats(time_t now);
char *geoip_format_dirreq_stats(time_t now);
time_t geoip_dirreq_stats_write(time_t now);
void geoip_dirreq_stats_term(void);
void geoip_entry_stats_init(time_t now);
time_t geoip_entry_stats_write(time_t now);
void geoip_entry_stats_term(void);
void geoip_reset_entry_stats(time_t now);
char *geoip_format_entry_stats(time_t now);
void geoip_bridge_stats_init(time_t now);
char *geoip_format_bridge_stats(time_t now);
time_t geoip_bridge_stats_write(time_t now);
void geoip_bridge_stats_term(void);
const char *geoip_get_bridge_stats_extrainfo(time_t);
char *geoip_get_bridge_stats_controller(time_t);
char *format_client_stats_heartbeat(time_t now);
#endif /* !defined(TOR_GEOIP_H) */

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,139 @@
/* Copyright (c) 2001 Matej Pfajfar.
* Copyright (c) 2001-2004, Roger Dingledine.
* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
* Copyright (c) 2007-2018, The Tor Project, Inc. */
/* See LICENSE for licensing information */
/**
* \file geoip_stats.h
* \brief Header file for geoip_stats.c.
**/
#ifndef TOR_GEOIP_STATS_H
#define TOR_GEOIP_STATS_H
#include "core/or/dos.h"
/** Indicates an action that we might be noting geoip statistics on.
* Note that if we're noticing CONNECT, we're a bridge, and if we're noticing
* the others, we're not.
*/
typedef enum {
/** We've noticed a connection as a bridge relay or entry guard. */
GEOIP_CLIENT_CONNECT = 0,
/** We've served a networkstatus consensus as a directory server. */
GEOIP_CLIENT_NETWORKSTATUS = 1,
} geoip_client_action_t;
/** Indicates either a positive reply or a reason for rejectng a network
* status request that will be included in geoip statistics. */
typedef enum {
/** Request is answered successfully. */
GEOIP_SUCCESS = 0,
/** V3 network status is not signed by a sufficient number of requested
* authorities. */
GEOIP_REJECT_NOT_ENOUGH_SIGS = 1,
/** Requested network status object is unavailable. */
GEOIP_REJECT_UNAVAILABLE = 2,
/** Requested network status not found. */
GEOIP_REJECT_NOT_FOUND = 3,
/** Network status has not been modified since If-Modified-Since time. */
GEOIP_REJECT_NOT_MODIFIED = 4,
/** Directory is busy. */
GEOIP_REJECT_BUSY = 5,
} geoip_ns_response_t;
#define GEOIP_NS_RESPONSE_NUM 6
/** Directory requests that we are measuring can be either direct or
* tunneled. */
typedef enum {
DIRREQ_DIRECT = 0,
DIRREQ_TUNNELED = 1,
} dirreq_type_t;
/** Possible states for either direct or tunneled directory requests that
* are relevant for determining network status download times. */
typedef enum {
/** Found that the client requests a network status; applies to both
* direct and tunneled requests; initial state of a request that we are
* measuring. */
DIRREQ_IS_FOR_NETWORK_STATUS = 0,
/** Finished writing a network status to the directory connection;
* applies to both direct and tunneled requests; completes a direct
* request. */
DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1,
/** END cell sent to circuit that initiated a tunneled request. */
DIRREQ_END_CELL_SENT = 2,
/** Flushed last cell from queue of the circuit that initiated a
* tunneled request to the outbuf of the OR connection. */
DIRREQ_CIRC_QUEUE_FLUSHED = 3,
/** Flushed last byte from buffer of the channel belonging to the
* circuit that initiated a tunneled request; completes a tunneled
* request. */
DIRREQ_CHANNEL_BUFFER_FLUSHED = 4
} dirreq_state_t;
/** Entry in a map from IP address to the last time we've seen an incoming
* connection from that IP address. Used by bridges only to track which
* countries have them blocked, or the DoS mitigation subsystem if enabled. */
typedef struct clientmap_entry_t {
HT_ENTRY(clientmap_entry_t) node;
tor_addr_t addr;
/* Name of pluggable transport used by this client. NULL if no
pluggable transport was used. */
char *transport_name;
/** Time when we last saw this IP address, in MINUTES since the epoch.
*
* (This will run out of space around 4011 CE. If Tor is still in use around
* 4000 CE, please remember to add more bits to last_seen_in_minutes.) */
unsigned int last_seen_in_minutes:30;
unsigned int action:2;
/* This object is used to keep some statistics per client address for the
* DoS mitigation subsystem. */
dos_client_stats_t dos_stats;
} clientmap_entry_t;
int should_record_bridge_info(const or_options_t *options);
void geoip_note_client_seen(geoip_client_action_t action,
const tor_addr_t *addr, const char *transport_name,
time_t now);
void geoip_remove_old_clients(time_t cutoff);
clientmap_entry_t *geoip_lookup_client(const tor_addr_t *addr,
const char *transport_name,
geoip_client_action_t action);
size_t geoip_client_cache_total_allocation(void);
size_t geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes);
void geoip_note_ns_response(geoip_ns_response_t response);
char *geoip_get_transport_history(void);
int geoip_get_client_history(geoip_client_action_t action,
char **country_str, char **ipver_str);
char *geoip_get_request_history(void);
void geoip_stats_free_all(void);
void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
dirreq_type_t type);
void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
dirreq_state_t new_state);
void geoip_dirreq_stats_init(time_t now);
void geoip_reset_dirreq_stats(time_t now);
char *geoip_format_dirreq_stats(time_t now);
time_t geoip_dirreq_stats_write(time_t now);
void geoip_dirreq_stats_term(void);
void geoip_entry_stats_init(time_t now);
time_t geoip_entry_stats_write(time_t now);
void geoip_entry_stats_term(void);
void geoip_reset_entry_stats(time_t now);
char *geoip_format_entry_stats(time_t now);
void geoip_bridge_stats_init(time_t now);
char *geoip_format_bridge_stats(time_t now);
time_t geoip_bridge_stats_write(time_t now);
void geoip_bridge_stats_term(void);
const char *geoip_get_bridge_stats_extrainfo(time_t);
char *geoip_get_bridge_stats_controller(time_t);
char *format_client_stats_heartbeat(time_t now);
#endif /* !defined(TOR_GEOIP_STATS_H) */

View File

@ -33,6 +33,7 @@
#include "feature/nodelist/networkstatus.h"
#include "core/proto/proto_http.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "feature/dircache/dirserv.h"
#include "feature/dirauth/dirvote.h"
#include "test/log_test_helpers.h"

View File

@ -9,7 +9,7 @@
#include "core/or/dos.h"
#include "core/or/circuitlist.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "core/or/channel.h"
#include "feature/nodelist/microdesc.h"
#include "feature/nodelist/networkstatus.h"
@ -500,4 +500,3 @@ struct testcase_t dos_tests[] = {
NULL, NULL },
END_OF_TESTCASES
};

View File

@ -11,6 +11,7 @@
#include "core/or/or.h"
#include "app/config/config.h"
#include "feature/stats/geoip.h"
#include "feature/stats/geoip_stats.h"
#include "test/test.h"
/* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs