Big commit: clients start downloading and using network-statuses.

Only caches need to get running-routers; nobody needs to parse, store,
or use it.  Same for the router-status line in the directories.  Add
many #if 0's that can get removed once I'm convinced they don't
contain anything I'm forgetting.

Start all newly-parsed routers as non-running and non-valid; update
them from the list of network statuses.

Update all routers when a new networkstatus comes in.

After 3 tries for a networkstatus, clients give up until they're told
to try again.

     "Let's get those missles ready to **DESTROY THE UNIVERSE**!"
				-TMBG


svn:r5063
This commit is contained in:
Nick Mathewson 2005-09-14 21:09:25 +00:00
parent 922cee3d0c
commit e3adcbdb95
6 changed files with 139 additions and 102 deletions

View File

@ -121,10 +121,11 @@ R - check reachability as soon as you hear about a new server
o Dirservers publish compressed network-status objects.
o Support retrieving several-at-once
o Everyone downloads network-status objects
N . Clients: from all directories, round-robin
o Clients: from all directories, round-robin
o Basic implementation: disable until 0.1.1.x is out.
o On failure, mark trusted_dir_server as having failed
- Retry, up to a point.
o Retry, up to a point.
- Launch retry immediately on failure.
o Parse them
o Cache them, reload on restart
o Serve cached directories
@ -140,9 +141,9 @@ N - Alice acts on network-status objects
- Implement reload-from-store
- Store downloaded descriptors
- Retry descriptors on failure for a while
- Alice sets descriptor status from network-status
o Alice sets descriptor status from network-status
o Implement
- Use
o Use
- Security
- Alices avoid duplicate class C nodes.

View File

@ -903,8 +903,6 @@ connection_dir_client_reached_eof(connection_t *conn)
}
if (conn->purpose == DIR_PURPOSE_FETCH_RUNNING_LIST) {
running_routers_t *rrs;
routerlist_t *rl;
/* just update our list of running routers, if this list is new info */
log_fn(LOG_INFO,"Received running-routers list (size %d)", (int)body_len);
if (status_code != 200) {
@ -913,6 +911,13 @@ connection_dir_client_reached_eof(connection_t *conn)
tor_free(body); tor_free(headers); tor_free(reason);
return -1;
}
if (router_parse_runningrouters(body)<0) {
log_fn(LOG_WARN,"Bad running-routers from server '%s:%d'. I'll try again soon.",
conn->address, conn->port);
tor_free(body); tor_free(headers); tor_free(reason);
return -1;
}
#if 0
if (!(rrs = router_parse_runningrouters(body, 1))) {
log_fn(LOG_WARN, "Can't parse runningrouters list (server '%s:%d')",
conn->address, conn->port);
@ -926,6 +931,7 @@ connection_dir_client_reached_eof(connection_t *conn)
} else {
running_routers_free(rrs);
}
#endif
}
if (conn->purpose == DIR_PURPOSE_FETCH_NETWORKSTATUS) {
@ -963,6 +969,7 @@ connection_dir_client_reached_eof(connection_t *conn)
else
break;
}
routers_update_all_from_networkstatus();
if (which) {
if (smartlist_len(which)) {
dir_networkstatus_download_failed(which);

View File

@ -480,6 +480,7 @@ void
directory_all_unreachable(time_t now)
{
connection_t *conn;
/* XXXX011 NM Update this to reflect new directories? */
has_fetched_directory=0;
stats_n_seconds_working=0; /* reset it */
@ -533,6 +534,9 @@ void
directory_has_arrived(time_t now, char *identity_digest)
{
or_options_t *options = get_options();
/* XXXX011 NM Update this to reflect new directories. In particular, we
* can't start building circuits until we have descriptors and networkstatus
* docs.*/
log_fn(LOG_INFO, "A directory has arrived.");
@ -710,7 +714,8 @@ run_scheduled_events(time_t now)
rend_cache_clean();
}
if (time_to_fetch_running_routers < now) {
/* Caches need to fetch running_routers; directory clients don't. */
if (options->DirPort && time_to_fetch_running_routers < now) {
if (!authdir_mode(options) || !options->V1AuthoritativeDir) {
directory_get_from_dirserver(DIR_PURPOSE_FETCH_RUNNING_LIST, NULL, 1);
}
@ -732,12 +737,13 @@ run_scheduled_events(time_t now)
!we_are_hibernating())
consider_testing_reachability();
/* Also, once per minute, check whether we want to download any
* networkstatus documents.
*/
if (server_mode(options) && options->DirPort)
update_networkstatus_cache_downloads(now);
/* XXXX Disabled until 0.1.1.6 is out: only servers need networkstatus.
else
update_networkstatus_client_downloads(now);
*/
}
/** 3a. Every second, we examine pending circuits and prune the
@ -980,6 +986,7 @@ do_main_loop(void)
if (router_reload_router_list()) {
return -1;
}
/* load the networkstatuses. */
if (router_reload_networkstatus()) {
return -1;
}

View File

@ -785,12 +785,14 @@ typedef struct {
int num_unreachable_notifications;
} routerinfo_t;
#if 0
/** Contents of a running-routers list */
typedef struct running_routers_t {
time_t published_on; /**< When was the list marked as published? */
/** Which ORs are on the list? Entries may be prefixed with ! and $. */
smartlist_t *running_routers;
} running_routers_t;
#endif
/** Contents of a single per-router entry in a network status object.
*/
@ -848,9 +850,11 @@ typedef struct {
/** When was the most recent directory that contributed to this list
* published? */
time_t published_on;
#if 0
time_t running_routers_updated_on;
/** What is the most recently received running_routers structure? */
running_routers_t *running_routers;
#endif
} routerlist_t;
/** Information on router used when extending a circuit. (We don't need a
@ -2100,6 +2104,7 @@ int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port,
int need_uptime);
int router_exit_policy_rejects_all(routerinfo_t *router);
#if 0
void running_routers_free(running_routers_t *rr);
void routerlist_set_runningrouters(routerlist_t *list, running_routers_t *rr);
int routers_update_status_from_entry(smartlist_t *routers,
@ -2108,12 +2113,14 @@ int routers_update_status_from_entry(smartlist_t *routers,
int router_update_status_from_smartlist(routerinfo_t *r,
time_t list_time,
smartlist_t *running_list);
#endif
void add_trusted_dir_server(const char *addr, uint16_t port,
const char *digest, int supports_v1);
void clear_trusted_dir_servers(void);
networkstatus_t *networkstatus_get_by_digest(const char *digest);
void update_networkstatus_cache_downloads(time_t now);
void update_networkstatus_client_downloads(time_t now);
void routers_update_all_from_networkstatus(void);
void routers_update_status_from_networkstatus(smartlist_t *routers);
smartlist_t *router_list_superseded(void);
@ -2147,16 +2154,13 @@ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest
crypto_pk_env_t *private_key);
int router_parse_list_from_string(const char **s,
routerlist_t **dest,
smartlist_t *good_nickname_list,
int rr_format,
time_t published);
int router_parse_routerlist_from_directory(const char *s,
routerlist_t **dest,
crypto_pk_env_t *pkey,
int check_version,
int write_to_cache);
running_routers_t *router_parse_runningrouters(const char *str,
int write_to_cache);
int router_parse_runningrouters(const char *str);
routerinfo_t *router_parse_entry_from_string(const char *s, const char *end);
int router_add_exit_policy_from_string(routerinfo_t *router, const char *s);
addr_policy_t *router_parse_addr_policy_from_string(const char *s,

View File

@ -47,6 +47,11 @@ extern int has_fetched_directory; /**< from main.c */
/** Global list of all of the current network_status documents that we know
* about. This list is kept sorted by published_on. */
static smartlist_t *networkstatus_list = NULL;
/** True iff networkstatus_list has changed since the last time we called
* routers_update_all_from_networkstatus. Set by router_set_networkstatus;
* cleared by routers_update_all_from_networkstatus.
*/
static int networkstatus_list_has_changed = 0;
/**
* Reload the most recent cached directory (if present).
@ -116,6 +121,7 @@ router_reload_networkstatus(void)
tor_free(s);
}
});
routers_update_all_from_networkstatus();
return 0;
}
@ -437,7 +443,10 @@ mark_all_trusteddirservers_up(void)
}
if (trusted_dir_servers) {
SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, dir,
dir->is_running = 1);
{
dir->is_running = 1;
dir->n_networkstatus_failures = 0;
});
}
}
@ -971,7 +980,6 @@ routerlist_free(routerlist_t *rl)
SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
routerinfo_free(r));
smartlist_free(rl->routers);
running_routers_free(rl->running_routers);
tor_free(rl->software_versions);
tor_free(rl);
}
@ -1235,12 +1243,16 @@ router_load_single_router(const char *s, const char **msg)
routerinfo_free(ri);
return 0;
}
#if 0
if (routerlist && routerlist->running_routers) {
running_routers_t *rr = routerlist->running_routers;
router_update_status_from_smartlist(ri,
rr->published_on,
rr->running_routers);
}
#endif
/* XXXX011 update router status from networkstatus!! */
if (router_add_to_routerlist(ri, msg)<0) {
log_fn(LOG_WARN, "Couldn't add router to list: %s Dropping.",
*msg?*msg:"(No message).");
@ -1422,11 +1434,13 @@ router_set_networkstatus(const char *s, time_t arrived_at,
ns->networkstatus_digest, DIGEST_LEN)) {
/* Same one we had before. */
networkstatus_free(ns);
log_fn(LOG_NOTICE, "Dropping network-status (%s); already have it.",fp);
if (old_ns->received_on < arrived_at)
/* XXXX We should touch the cache file. NM */
old_ns->received_on = arrived_at;
return 0;
} else if (old_ns->published_on >= ns->published_on) {
log_fn(LOG_INFO, "Dropping network-status; we have a newer one for this authority.");
log_fn(LOG_NOTICE, "Dropping network-status (%s); we have a newer one for this authority.", fp);
networkstatus_free(ns);
return 0;
} else {
@ -1441,6 +1455,13 @@ router_set_networkstatus(const char *s, time_t arrived_at,
if (!found)
smartlist_add(networkstatus_list, ns);
/*XXXX011 downgrade to INFO NM */
log_fn(LOG_NOTICE, "New networkstatus %s (%s).",
source == NS_FROM_CACHE?"from our cache":
(source==NS_FROM_DIR?"from a directory server":"from this authority"),
fp);
networkstatus_list_has_changed = 1;
smartlist_sort(networkstatus_list, _compare_networkstatus_published_on);
if (source != NS_FROM_CACHE && !skewed) {
@ -1561,12 +1582,14 @@ update_networkstatus_cache_downloads(time_t now)
}
/*XXXX Should these be configurable? NM*/
/** How old (in seconds) can a network-status be before we stop believing it? */
/** How old (in seconds) can a network-status be before we try replacing it? */
#define NETWORKSTATUS_MAX_VALIDITY (48*60*60)
/** How long (in seconds) does a client wait after getting a network status
* before downloading the next in sequence? */
#define NETWORKSTATUS_CLIENT_DL_INTERVAL (30*60)
/* How many times do we allow a networkstatus download to fail before we
* assume that the authority isn't publishing? */
#define NETWORKSTATUS_N_ALLOWABLE_FAILURES 3
/** We are not a directory cache or authority. Update our network-status list
* by launching a new directory fetch for enough network-status documents "as
* necessary". See function comments for implementation details.
@ -1574,7 +1597,7 @@ update_networkstatus_cache_downloads(time_t now)
void
update_networkstatus_client_downloads(time_t now)
{
int n_live = 0, needed = 0, n_dirservers, i;
int n_live = 0, needed = 0, n_running_dirservers, n_dirservers, i;
int most_recent_idx = -1;
trusted_dir_server_t *most_recent = NULL;
time_t most_recent_received = 0;
@ -1593,12 +1616,16 @@ update_networkstatus_client_downloads(time_t now)
*/
if (!trusted_dir_servers || !smartlist_len(trusted_dir_servers))
return;
n_dirservers = smartlist_len(trusted_dir_servers);
n_dirservers = n_running_dirservers = smartlist_len(trusted_dir_servers);
SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds,
{
networkstatus_t *ns = networkstatus_get_by_digest(ds->digest);
if (!ns)
continue;
if (ds->n_networkstatus_failures > NETWORKSTATUS_N_ALLOWABLE_FAILURES) {
--n_running_dirservers;
continue;
}
if (ns->published_on > now-NETWORKSTATUS_MAX_VALIDITY)
++n_live;
if (!most_recent || ns->received_on > most_recent_received) {
@ -1613,11 +1640,22 @@ update_networkstatus_client_downloads(time_t now)
*/
if (n_live < (n_dirservers/2)+1)
needed = (n_dirservers/2)+1-n_live;
if (needed > n_dirservers)
needed = n_dirservers;
if (needed > n_running_dirservers)
needed = n_running_dirservers;
if (needed)
/* XXXX001 Downgrade to info NM */
log_fn(LOG_NOTICE, "For %d/%d running directory servers, we have %d live"
" network-status documents. Downloading %d.",
n_running_dirservers, n_dirservers, n_live, needed);
/* Also, download at least 1 every NETWORKSTATUS_CLIENT_DL_INTERVAL. */
if (most_recent_received < now-NETWORKSTATUS_CLIENT_DL_INTERVAL && needed < 1)
if (n_running_dirservers &&
most_recent_received < now-NETWORKSTATUS_CLIENT_DL_INTERVAL && needed < 1) {
log_fn(LOG_NOTICE, "Our most recent network-status document is %d"
" seconds old; downloading another.", (int)(now-most_recent_received));
needed = 1;
}
if (!needed)
return;
@ -1637,6 +1675,8 @@ update_networkstatus_client_downloads(time_t now)
if (i >= n_dirservers)
i = 0;
ds = smartlist_get(trusted_dir_servers, i);
if (ds->n_networkstatus_failures > NETWORKSTATUS_N_ALLOWABLE_FAILURES)
continue;
base16_encode(cp, HEX_DIGEST_LEN+1, ds->digest, DIGEST_LEN);
cp += HEX_DIGEST_LEN;
--needed;
@ -1860,6 +1900,7 @@ router_exit_policy_rejects_all(routerinfo_t *router)
== ADDR_POLICY_REJECTED;
}
#if 0
/** Release all space held in <b>rr</b>. */
void
running_routers_free(running_routers_t *rr)
@ -2047,6 +2088,7 @@ router_update_status_from_smartlist(routerinfo_t *router,
smartlist_free(rl);
return 0;
}
#endif
/** Add to the list of authorized directory servers one at
* <b>address</b>:<b>port</b>, with identity key <b>digest</b>. If
@ -2111,6 +2153,40 @@ networkstatus_get_by_digest(const char *digest)
return NULL;
}
/** If the network-status list has changed since the last time we called this
* function, update the status of every router from the network-status list.
*/
void
routers_update_all_from_networkstatus(void)
{
static int have_warned_about_unverified_status = 0;
routerinfo_t *me;
if (!routerlist || !networkstatus_list || !networkstatus_list_has_changed)
return;
routers_update_status_from_networkstatus(routerlist->routers);
me = router_get_my_routerinfo();
if (me) {
/* We could be more sophisticated about this whole business. How many
* dirservers list us as named, valid, etc. */
smartlist_t *lst = smartlist_create();
smartlist_add(lst, me);
routers_update_status_from_networkstatus(lst);
if (me->is_verified == 0) {
log_fn(LOG_WARN, "Many directory servers list us as unverified. Please consider sending your identity fingerprint to the tor-ops.");
have_warned_about_unverified_status = 1;
} else if (me->is_named == 0) {
log_fn(LOG_WARN, "Many directory servers list us as unnamed. Please consider sending your identity fingerprint to the tor-ops.");
have_warned_about_unverified_status = 1;
}
}
helper_nodes_set_status_from_directory();
networkstatus_list_has_changed = 0;
}
/** Allow any network-status newer than this to influence our view of who's
* running. */
#define DEFAULT_RUNNING_INTERVAL 60*60
@ -2127,6 +2203,7 @@ routers_update_status_from_networkstatus(smartlist_t *routers)
n_recent;
int i;
time_t now = time(NULL);
trusted_dir_server_t *ds;
if (authdir_mode(get_options())) {
/* An authoritative directory should never believer someone else about
@ -2164,6 +2241,7 @@ routers_update_status_from_networkstatus(smartlist_t *routers)
SMARTLIST_FOREACH(routers, routerinfo_t *, router,
{
ds = router_get_trusteddirserver_by_digest(router->identity_digest);
n_listing = n_valid = n_naming = n_named = n_running = 0;
SMARTLIST_FOREACH(networkstatus_list, networkstatus_t *, ns,
@ -2194,6 +2272,10 @@ routers_update_status_from_networkstatus(smartlist_t *routers)
router->is_named = (n_named > n_naming/2);
router->is_verified = (n_valid > n_statuses/2);
router->is_running = (n_running > n_recent/2);
if (router->is_running && ds)
/*Hm. What about authorities? When do they reset n_networkstatus_failures?*/
ds->n_networkstatus_failures = 0;
});
}

View File

@ -419,9 +419,8 @@ router_parse_routerlist_from_directory(const char *str,
char digest[DIGEST_LEN];
routerlist_t *new_dir = NULL;
char *versions = NULL;
smartlist_t *good_nickname_list = NULL;
time_t published_on;
int i, r;
int r;
const char *end, *cp;
smartlist_t *tokens = NULL;
char dirnickname[MAX_NICKNAME_LEN+1];
@ -531,42 +530,17 @@ router_parse_routerlist_from_directory(const char *str,
goto err;
}
good_nickname_list = smartlist_create();
for (i=0; i<tok->n_args; ++i) {
smartlist_add(good_nickname_list, tok->args[i]);
}
tok->n_args = 0; /* Don't free the strings in good_nickname_list yet. */
/* Read the router list from s, advancing s up past the end of the last
* router. */
str = end;
if (router_parse_list_from_string(&str, &new_dir,
good_nickname_list,
tok->tp==K_RUNNING_ROUTERS,
published_on)) {
log_fn(LOG_WARN, "Error reading routers from directory");
goto err;
}
/* Determine if my routerinfo is considered verified. */
{
static int have_warned_about_unverified_status = 0;
routerinfo_t *me = router_get_my_routerinfo();
if (me) {
if (router_update_status_from_smartlist(me,
published_on, good_nickname_list)==1 &&
me->is_verified == 0 && !have_warned_about_unverified_status) {
log_fn(LOG_WARN,"Dirserver '%s' lists your server as unverified. Please consider sending your identity fingerprint to the tor-ops.", dirnickname);
have_warned_about_unverified_status = 1;
}
}
}
new_dir->software_versions = versions; versions = NULL;
new_dir->published_on = published_on;
new_dir->running_routers = tor_malloc_zero(sizeof(running_routers_t));
new_dir->running_routers->published_on = published_on;
new_dir->running_routers->running_routers = good_nickname_list;
SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok));
smartlist_free(tokens);
@ -583,10 +557,6 @@ router_parse_routerlist_from_directory(const char *str,
if (new_dir)
routerlist_free(new_dir);
tor_free(versions);
if (good_nickname_list) {
SMARTLIST_FOREACH(good_nickname_list, char *, n, tor_free(n));
smartlist_free(good_nickname_list);
}
done:
if (declared_key) crypto_free_pk_env(declared_key);
if (tokens) {
@ -596,17 +566,15 @@ router_parse_routerlist_from_directory(const char *str,
return r;
}
/** Read a signed router status statement from <b>str</b>. On
* success, return it, and cache the original string if
* <b>write_to_cache</b> is set. Otherwise, return NULL. */
running_routers_t *
router_parse_runningrouters(const char *str, int write_to_cache)
/** Read a signed router status statement from <b>str</b>. If it's well-formed,
* return 0. Otherwise, return -1. If we're a directory cache, cache it.*/
int
router_parse_runningrouters(const char *str)
{
char digest[DIGEST_LEN];
running_routers_t *new_list = NULL;
directory_token_t *tok;
time_t published_on;
int i;
int r = -1;
crypto_pk_env_t *declared_key = NULL;
smartlist_t *tokens = NULL;
@ -637,28 +605,6 @@ router_parse_runningrouters(const char *str, int write_to_cache)
if (parse_iso_time(tok->args[0], &published_on) < 0) {
goto err;
}
/* Now that we know the signature is okay, and we have a
* publication time, cache the list. */
if (!get_options()->AuthoritativeDir && write_to_cache)
dirserv_set_cached_directory(str, published_on, 1);
if (!(tok = find_first_by_keyword(tokens, K_ROUTER_STATUS))) {
if (!(tok = find_first_by_keyword(tokens, K_RUNNING_ROUTERS))) {
log_fn(LOG_WARN,
"Missing running-routers/router-status line from directory.");
goto err;
}
}
new_list = tor_malloc_zero(sizeof(running_routers_t));
new_list->published_on = published_on;
new_list->running_routers = smartlist_create();
for (i=0;i<tok->n_args;++i) {
smartlist_add(new_list->running_routers, tok->args[i]);
}
tok->n_args = 0; /* Don't free the elements of tok->args. */
if (!(tok = find_first_by_keyword(tokens, K_DIRECTORY_SIGNATURE))) {
log_fn(LOG_WARN, "Missing signature on running-routers");
goto err;
@ -667,19 +613,19 @@ router_parse_runningrouters(const char *str, int write_to_cache)
if (check_directory_signature(digest, tok, NULL, declared_key, 1) < 0)
goto err;
goto done;
/* Now that we know the signature is okay, and we have a
* publication time, cache the list. */
if (get_options()->DirPort && !get_options()->V1AuthoritativeDir)
dirserv_set_cached_directory(str, published_on, 1);
r = 0;
err:
if (new_list) {
running_routers_free(new_list);
new_list = NULL;
}
done:
if (declared_key) crypto_free_pk_env(declared_key);
if (tokens) {
SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok));
smartlist_free(tokens);
}
return new_list;
return r;
}
/** Given a directory or running-routers string in <b>str</b>, try to
@ -818,8 +764,7 @@ check_directory_signature(const char *digest,
*/
int
router_parse_list_from_string(const char **s, routerlist_t **dest,
smartlist_t *good_nickname_list,
int rr_format, time_t published_on)
time_t published_on)
{
routerinfo_t *router;
smartlist_t *routers;
@ -850,19 +795,10 @@ router_parse_list_from_string(const char **s, routerlist_t **dest,
continue;
}
if (!good_nickname_list) {
router->is_running = 1; /* start out assuming all dirservers are up */
router->is_verified = router->is_named = 1;
router->status_set_at = time(NULL);
}
smartlist_add(routers, router);
// log_fn(LOG_DEBUG,"just added router #%d.",smartlist_len(routers));
}
if (good_nickname_list) {
SMARTLIST_FOREACH(good_nickname_list, const char *, cp,
routers_update_status_from_entry(routers, published_on, cp));
}
routers_update_status_from_networkstatus(routers);
if (*dest)
routerlist_free(*dest);