From e3adcbdb953fa132d0658cf18a8b91f42b139640 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Wed, 14 Sep 2005 21:09:25 +0000 Subject: [PATCH] Big commit: clients start downloading and using network-statuses. Only caches need to get running-routers; nobody needs to parse, store, or use it. Same for the router-status line in the directories. Add many #if 0's that can get removed once I'm convinced they don't contain anything I'm forgetting. Start all newly-parsed routers as non-running and non-valid; update them from the list of network statuses. Update all routers when a new networkstatus comes in. After 3 tries for a networkstatus, clients give up until they're told to try again. "Let's get those missles ready to **DESTROY THE UNIVERSE**!" -TMBG svn:r5063 --- doc/TODO | 9 ++-- src/or/directory.c | 11 ++++- src/or/main.c | 13 ++++-- src/or/or.h | 12 +++-- src/or/routerlist.c | 102 ++++++++++++++++++++++++++++++++++++++----- src/or/routerparse.c | 94 +++++++-------------------------------- 6 files changed, 139 insertions(+), 102 deletions(-) diff --git a/doc/TODO b/doc/TODO index 7425f37852..4438f7bfb7 100644 --- a/doc/TODO +++ b/doc/TODO @@ -121,10 +121,11 @@ R - check reachability as soon as you hear about a new server o Dirservers publish compressed network-status objects. o Support retrieving several-at-once o Everyone downloads network-status objects -N . Clients: from all directories, round-robin + o Clients: from all directories, round-robin o Basic implementation: disable until 0.1.1.x is out. o On failure, mark trusted_dir_server as having failed - - Retry, up to a point. + o Retry, up to a point. + - Launch retry immediately on failure. o Parse them o Cache them, reload on restart o Serve cached directories @@ -140,9 +141,9 @@ N - Alice acts on network-status objects - Implement reload-from-store - Store downloaded descriptors - Retry descriptors on failure for a while - - Alice sets descriptor status from network-status + o Alice sets descriptor status from network-status o Implement - - Use + o Use - Security - Alices avoid duplicate class C nodes. diff --git a/src/or/directory.c b/src/or/directory.c index 47f1a936d2..4136a570d2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -903,8 +903,6 @@ connection_dir_client_reached_eof(connection_t *conn) } if (conn->purpose == DIR_PURPOSE_FETCH_RUNNING_LIST) { - running_routers_t *rrs; - routerlist_t *rl; /* just update our list of running routers, if this list is new info */ log_fn(LOG_INFO,"Received running-routers list (size %d)", (int)body_len); if (status_code != 200) { @@ -913,6 +911,13 @@ connection_dir_client_reached_eof(connection_t *conn) tor_free(body); tor_free(headers); tor_free(reason); return -1; } + if (router_parse_runningrouters(body)<0) { + log_fn(LOG_WARN,"Bad running-routers from server '%s:%d'. I'll try again soon.", + conn->address, conn->port); + tor_free(body); tor_free(headers); tor_free(reason); + return -1; + } +#if 0 if (!(rrs = router_parse_runningrouters(body, 1))) { log_fn(LOG_WARN, "Can't parse runningrouters list (server '%s:%d')", conn->address, conn->port); @@ -926,6 +931,7 @@ connection_dir_client_reached_eof(connection_t *conn) } else { running_routers_free(rrs); } +#endif } if (conn->purpose == DIR_PURPOSE_FETCH_NETWORKSTATUS) { @@ -963,6 +969,7 @@ connection_dir_client_reached_eof(connection_t *conn) else break; } + routers_update_all_from_networkstatus(); if (which) { if (smartlist_len(which)) { dir_networkstatus_download_failed(which); diff --git a/src/or/main.c b/src/or/main.c index 12025e6888..24dc5348e7 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -480,6 +480,7 @@ void directory_all_unreachable(time_t now) { connection_t *conn; + /* XXXX011 NM Update this to reflect new directories? */ has_fetched_directory=0; stats_n_seconds_working=0; /* reset it */ @@ -533,6 +534,9 @@ void directory_has_arrived(time_t now, char *identity_digest) { or_options_t *options = get_options(); + /* XXXX011 NM Update this to reflect new directories. In particular, we + * can't start building circuits until we have descriptors and networkstatus + * docs.*/ log_fn(LOG_INFO, "A directory has arrived."); @@ -710,7 +714,8 @@ run_scheduled_events(time_t now) rend_cache_clean(); } - if (time_to_fetch_running_routers < now) { + /* Caches need to fetch running_routers; directory clients don't. */ + if (options->DirPort && time_to_fetch_running_routers < now) { if (!authdir_mode(options) || !options->V1AuthoritativeDir) { directory_get_from_dirserver(DIR_PURPOSE_FETCH_RUNNING_LIST, NULL, 1); } @@ -732,12 +737,13 @@ run_scheduled_events(time_t now) !we_are_hibernating()) consider_testing_reachability(); + /* Also, once per minute, check whether we want to download any + * networkstatus documents. + */ if (server_mode(options) && options->DirPort) update_networkstatus_cache_downloads(now); - /* XXXX Disabled until 0.1.1.6 is out: only servers need networkstatus. else update_networkstatus_client_downloads(now); - */ } /** 3a. Every second, we examine pending circuits and prune the @@ -980,6 +986,7 @@ do_main_loop(void) if (router_reload_router_list()) { return -1; } + /* load the networkstatuses. */ if (router_reload_networkstatus()) { return -1; } diff --git a/src/or/or.h b/src/or/or.h index a2ac06ece6..f5f6af2a8c 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -785,12 +785,14 @@ typedef struct { int num_unreachable_notifications; } routerinfo_t; +#if 0 /** Contents of a running-routers list */ typedef struct running_routers_t { time_t published_on; /**< When was the list marked as published? */ /** Which ORs are on the list? Entries may be prefixed with ! and $. */ smartlist_t *running_routers; } running_routers_t; +#endif /** Contents of a single per-router entry in a network status object. */ @@ -848,9 +850,11 @@ typedef struct { /** When was the most recent directory that contributed to this list * published? */ time_t published_on; +#if 0 time_t running_routers_updated_on; /** What is the most recently received running_routers structure? */ running_routers_t *running_routers; +#endif } routerlist_t; /** Information on router used when extending a circuit. (We don't need a @@ -2100,6 +2104,7 @@ int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port, int need_uptime); int router_exit_policy_rejects_all(routerinfo_t *router); +#if 0 void running_routers_free(running_routers_t *rr); void routerlist_set_runningrouters(routerlist_t *list, running_routers_t *rr); int routers_update_status_from_entry(smartlist_t *routers, @@ -2108,12 +2113,14 @@ int routers_update_status_from_entry(smartlist_t *routers, int router_update_status_from_smartlist(routerinfo_t *r, time_t list_time, smartlist_t *running_list); +#endif void add_trusted_dir_server(const char *addr, uint16_t port, const char *digest, int supports_v1); void clear_trusted_dir_servers(void); networkstatus_t *networkstatus_get_by_digest(const char *digest); void update_networkstatus_cache_downloads(time_t now); void update_networkstatus_client_downloads(time_t now); +void routers_update_all_from_networkstatus(void); void routers_update_status_from_networkstatus(smartlist_t *routers); smartlist_t *router_list_superseded(void); @@ -2147,16 +2154,13 @@ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest crypto_pk_env_t *private_key); int router_parse_list_from_string(const char **s, routerlist_t **dest, - smartlist_t *good_nickname_list, - int rr_format, time_t published); int router_parse_routerlist_from_directory(const char *s, routerlist_t **dest, crypto_pk_env_t *pkey, int check_version, int write_to_cache); -running_routers_t *router_parse_runningrouters(const char *str, - int write_to_cache); +int router_parse_runningrouters(const char *str); routerinfo_t *router_parse_entry_from_string(const char *s, const char *end); int router_add_exit_policy_from_string(routerinfo_t *router, const char *s); addr_policy_t *router_parse_addr_policy_from_string(const char *s, diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 161d7959e4..f9a41d4c12 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -47,6 +47,11 @@ extern int has_fetched_directory; /**< from main.c */ /** Global list of all of the current network_status documents that we know * about. This list is kept sorted by published_on. */ static smartlist_t *networkstatus_list = NULL; +/** True iff networkstatus_list has changed since the last time we called + * routers_update_all_from_networkstatus. Set by router_set_networkstatus; + * cleared by routers_update_all_from_networkstatus. + */ +static int networkstatus_list_has_changed = 0; /** * Reload the most recent cached directory (if present). @@ -116,6 +121,7 @@ router_reload_networkstatus(void) tor_free(s); } }); + routers_update_all_from_networkstatus(); return 0; } @@ -437,7 +443,10 @@ mark_all_trusteddirservers_up(void) } if (trusted_dir_servers) { SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, dir, - dir->is_running = 1); + { + dir->is_running = 1; + dir->n_networkstatus_failures = 0; + }); } } @@ -971,7 +980,6 @@ routerlist_free(routerlist_t *rl) SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, routerinfo_free(r)); smartlist_free(rl->routers); - running_routers_free(rl->running_routers); tor_free(rl->software_versions); tor_free(rl); } @@ -1235,12 +1243,16 @@ router_load_single_router(const char *s, const char **msg) routerinfo_free(ri); return 0; } +#if 0 if (routerlist && routerlist->running_routers) { running_routers_t *rr = routerlist->running_routers; router_update_status_from_smartlist(ri, rr->published_on, rr->running_routers); } +#endif + /* XXXX011 update router status from networkstatus!! */ + if (router_add_to_routerlist(ri, msg)<0) { log_fn(LOG_WARN, "Couldn't add router to list: %s Dropping.", *msg?*msg:"(No message)."); @@ -1422,11 +1434,13 @@ router_set_networkstatus(const char *s, time_t arrived_at, ns->networkstatus_digest, DIGEST_LEN)) { /* Same one we had before. */ networkstatus_free(ns); + log_fn(LOG_NOTICE, "Dropping network-status (%s); already have it.",fp); if (old_ns->received_on < arrived_at) + /* XXXX We should touch the cache file. NM */ old_ns->received_on = arrived_at; return 0; } else if (old_ns->published_on >= ns->published_on) { - log_fn(LOG_INFO, "Dropping network-status; we have a newer one for this authority."); + log_fn(LOG_NOTICE, "Dropping network-status (%s); we have a newer one for this authority.", fp); networkstatus_free(ns); return 0; } else { @@ -1441,6 +1455,13 @@ router_set_networkstatus(const char *s, time_t arrived_at, if (!found) smartlist_add(networkstatus_list, ns); + /*XXXX011 downgrade to INFO NM */ + log_fn(LOG_NOTICE, "New networkstatus %s (%s).", + source == NS_FROM_CACHE?"from our cache": + (source==NS_FROM_DIR?"from a directory server":"from this authority"), + fp); + networkstatus_list_has_changed = 1; + smartlist_sort(networkstatus_list, _compare_networkstatus_published_on); if (source != NS_FROM_CACHE && !skewed) { @@ -1561,12 +1582,14 @@ update_networkstatus_cache_downloads(time_t now) } /*XXXX Should these be configurable? NM*/ -/** How old (in seconds) can a network-status be before we stop believing it? */ +/** How old (in seconds) can a network-status be before we try replacing it? */ #define NETWORKSTATUS_MAX_VALIDITY (48*60*60) /** How long (in seconds) does a client wait after getting a network status * before downloading the next in sequence? */ #define NETWORKSTATUS_CLIENT_DL_INTERVAL (30*60) - +/* How many times do we allow a networkstatus download to fail before we + * assume that the authority isn't publishing? */ +#define NETWORKSTATUS_N_ALLOWABLE_FAILURES 3 /** We are not a directory cache or authority. Update our network-status list * by launching a new directory fetch for enough network-status documents "as * necessary". See function comments for implementation details. @@ -1574,7 +1597,7 @@ update_networkstatus_cache_downloads(time_t now) void update_networkstatus_client_downloads(time_t now) { - int n_live = 0, needed = 0, n_dirservers, i; + int n_live = 0, needed = 0, n_running_dirservers, n_dirservers, i; int most_recent_idx = -1; trusted_dir_server_t *most_recent = NULL; time_t most_recent_received = 0; @@ -1593,12 +1616,16 @@ update_networkstatus_client_downloads(time_t now) */ if (!trusted_dir_servers || !smartlist_len(trusted_dir_servers)) return; - n_dirservers = smartlist_len(trusted_dir_servers); + n_dirservers = n_running_dirservers = smartlist_len(trusted_dir_servers); SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds, { networkstatus_t *ns = networkstatus_get_by_digest(ds->digest); if (!ns) continue; + if (ds->n_networkstatus_failures > NETWORKSTATUS_N_ALLOWABLE_FAILURES) { + --n_running_dirservers; + continue; + } if (ns->published_on > now-NETWORKSTATUS_MAX_VALIDITY) ++n_live; if (!most_recent || ns->received_on > most_recent_received) { @@ -1613,11 +1640,22 @@ update_networkstatus_client_downloads(time_t now) */ if (n_live < (n_dirservers/2)+1) needed = (n_dirservers/2)+1-n_live; - if (needed > n_dirservers) - needed = n_dirservers; + if (needed > n_running_dirservers) + needed = n_running_dirservers; + + if (needed) + /* XXXX001 Downgrade to info NM */ + log_fn(LOG_NOTICE, "For %d/%d running directory servers, we have %d live" + " network-status documents. Downloading %d.", + n_running_dirservers, n_dirservers, n_live, needed); + /* Also, download at least 1 every NETWORKSTATUS_CLIENT_DL_INTERVAL. */ - if (most_recent_received < now-NETWORKSTATUS_CLIENT_DL_INTERVAL && needed < 1) + if (n_running_dirservers && + most_recent_received < now-NETWORKSTATUS_CLIENT_DL_INTERVAL && needed < 1) { + log_fn(LOG_NOTICE, "Our most recent network-status document is %d" + " seconds old; downloading another.", (int)(now-most_recent_received)); needed = 1; + } if (!needed) return; @@ -1637,6 +1675,8 @@ update_networkstatus_client_downloads(time_t now) if (i >= n_dirservers) i = 0; ds = smartlist_get(trusted_dir_servers, i); + if (ds->n_networkstatus_failures > NETWORKSTATUS_N_ALLOWABLE_FAILURES) + continue; base16_encode(cp, HEX_DIGEST_LEN+1, ds->digest, DIGEST_LEN); cp += HEX_DIGEST_LEN; --needed; @@ -1860,6 +1900,7 @@ router_exit_policy_rejects_all(routerinfo_t *router) == ADDR_POLICY_REJECTED; } +#if 0 /** Release all space held in rr. */ void running_routers_free(running_routers_t *rr) @@ -2047,6 +2088,7 @@ router_update_status_from_smartlist(routerinfo_t *router, smartlist_free(rl); return 0; } +#endif /** Add to the list of authorized directory servers one at * address:port, with identity key digest. If @@ -2111,6 +2153,40 @@ networkstatus_get_by_digest(const char *digest) return NULL; } +/** If the network-status list has changed since the last time we called this + * function, update the status of every router from the network-status list. + */ +void +routers_update_all_from_networkstatus(void) +{ + static int have_warned_about_unverified_status = 0; + routerinfo_t *me; + if (!routerlist || !networkstatus_list || !networkstatus_list_has_changed) + return; + + routers_update_status_from_networkstatus(routerlist->routers); + + me = router_get_my_routerinfo(); + if (me) { + /* We could be more sophisticated about this whole business. How many + * dirservers list us as named, valid, etc. */ + smartlist_t *lst = smartlist_create(); + smartlist_add(lst, me); + routers_update_status_from_networkstatus(lst); + if (me->is_verified == 0) { + log_fn(LOG_WARN, "Many directory servers list us as unverified. Please consider sending your identity fingerprint to the tor-ops."); + have_warned_about_unverified_status = 1; + } else if (me->is_named == 0) { + log_fn(LOG_WARN, "Many directory servers list us as unnamed. Please consider sending your identity fingerprint to the tor-ops."); + have_warned_about_unverified_status = 1; + } + } + + helper_nodes_set_status_from_directory(); + + networkstatus_list_has_changed = 0; +} + /** Allow any network-status newer than this to influence our view of who's * running. */ #define DEFAULT_RUNNING_INTERVAL 60*60 @@ -2127,6 +2203,7 @@ routers_update_status_from_networkstatus(smartlist_t *routers) n_recent; int i; time_t now = time(NULL); + trusted_dir_server_t *ds; if (authdir_mode(get_options())) { /* An authoritative directory should never believer someone else about @@ -2164,6 +2241,7 @@ routers_update_status_from_networkstatus(smartlist_t *routers) SMARTLIST_FOREACH(routers, routerinfo_t *, router, { + ds = router_get_trusteddirserver_by_digest(router->identity_digest); n_listing = n_valid = n_naming = n_named = n_running = 0; SMARTLIST_FOREACH(networkstatus_list, networkstatus_t *, ns, @@ -2194,6 +2272,10 @@ routers_update_status_from_networkstatus(smartlist_t *routers) router->is_named = (n_named > n_naming/2); router->is_verified = (n_valid > n_statuses/2); router->is_running = (n_running > n_recent/2); + + if (router->is_running && ds) + /*Hm. What about authorities? When do they reset n_networkstatus_failures?*/ + ds->n_networkstatus_failures = 0; }); } diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 6e3a398f7c..e9f551d479 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -419,9 +419,8 @@ router_parse_routerlist_from_directory(const char *str, char digest[DIGEST_LEN]; routerlist_t *new_dir = NULL; char *versions = NULL; - smartlist_t *good_nickname_list = NULL; time_t published_on; - int i, r; + int r; const char *end, *cp; smartlist_t *tokens = NULL; char dirnickname[MAX_NICKNAME_LEN+1]; @@ -531,42 +530,17 @@ router_parse_routerlist_from_directory(const char *str, goto err; } - good_nickname_list = smartlist_create(); - for (i=0; in_args; ++i) { - smartlist_add(good_nickname_list, tok->args[i]); - } - tok->n_args = 0; /* Don't free the strings in good_nickname_list yet. */ - /* Read the router list from s, advancing s up past the end of the last * router. */ str = end; if (router_parse_list_from_string(&str, &new_dir, - good_nickname_list, - tok->tp==K_RUNNING_ROUTERS, published_on)) { log_fn(LOG_WARN, "Error reading routers from directory"); goto err; } - /* Determine if my routerinfo is considered verified. */ - { - static int have_warned_about_unverified_status = 0; - routerinfo_t *me = router_get_my_routerinfo(); - if (me) { - if (router_update_status_from_smartlist(me, - published_on, good_nickname_list)==1 && - me->is_verified == 0 && !have_warned_about_unverified_status) { - log_fn(LOG_WARN,"Dirserver '%s' lists your server as unverified. Please consider sending your identity fingerprint to the tor-ops.", dirnickname); - have_warned_about_unverified_status = 1; - } - } - } - new_dir->software_versions = versions; versions = NULL; new_dir->published_on = published_on; - new_dir->running_routers = tor_malloc_zero(sizeof(running_routers_t)); - new_dir->running_routers->published_on = published_on; - new_dir->running_routers->running_routers = good_nickname_list; SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok)); smartlist_free(tokens); @@ -583,10 +557,6 @@ router_parse_routerlist_from_directory(const char *str, if (new_dir) routerlist_free(new_dir); tor_free(versions); - if (good_nickname_list) { - SMARTLIST_FOREACH(good_nickname_list, char *, n, tor_free(n)); - smartlist_free(good_nickname_list); - } done: if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { @@ -596,17 +566,15 @@ router_parse_routerlist_from_directory(const char *str, return r; } -/** Read a signed router status statement from str. On - * success, return it, and cache the original string if - * write_to_cache is set. Otherwise, return NULL. */ -running_routers_t * -router_parse_runningrouters(const char *str, int write_to_cache) +/** Read a signed router status statement from str. If it's well-formed, + * return 0. Otherwise, return -1. If we're a directory cache, cache it.*/ +int +router_parse_runningrouters(const char *str) { char digest[DIGEST_LEN]; - running_routers_t *new_list = NULL; directory_token_t *tok; time_t published_on; - int i; + int r = -1; crypto_pk_env_t *declared_key = NULL; smartlist_t *tokens = NULL; @@ -637,28 +605,6 @@ router_parse_runningrouters(const char *str, int write_to_cache) if (parse_iso_time(tok->args[0], &published_on) < 0) { goto err; } - - /* Now that we know the signature is okay, and we have a - * publication time, cache the list. */ - if (!get_options()->AuthoritativeDir && write_to_cache) - dirserv_set_cached_directory(str, published_on, 1); - - if (!(tok = find_first_by_keyword(tokens, K_ROUTER_STATUS))) { - if (!(tok = find_first_by_keyword(tokens, K_RUNNING_ROUTERS))) { - log_fn(LOG_WARN, - "Missing running-routers/router-status line from directory."); - goto err; - } - } - - new_list = tor_malloc_zero(sizeof(running_routers_t)); - new_list->published_on = published_on; - new_list->running_routers = smartlist_create(); - for (i=0;in_args;++i) { - smartlist_add(new_list->running_routers, tok->args[i]); - } - tok->n_args = 0; /* Don't free the elements of tok->args. */ - if (!(tok = find_first_by_keyword(tokens, K_DIRECTORY_SIGNATURE))) { log_fn(LOG_WARN, "Missing signature on running-routers"); goto err; @@ -667,19 +613,19 @@ router_parse_runningrouters(const char *str, int write_to_cache) if (check_directory_signature(digest, tok, NULL, declared_key, 1) < 0) goto err; - goto done; + /* Now that we know the signature is okay, and we have a + * publication time, cache the list. */ + if (get_options()->DirPort && !get_options()->V1AuthoritativeDir) + dirserv_set_cached_directory(str, published_on, 1); + + r = 0; err: - if (new_list) { - running_routers_free(new_list); - new_list = NULL; - } - done: if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok)); smartlist_free(tokens); } - return new_list; + return r; } /** Given a directory or running-routers string in str, try to @@ -818,8 +764,7 @@ check_directory_signature(const char *digest, */ int router_parse_list_from_string(const char **s, routerlist_t **dest, - smartlist_t *good_nickname_list, - int rr_format, time_t published_on) + time_t published_on) { routerinfo_t *router; smartlist_t *routers; @@ -850,19 +795,10 @@ router_parse_list_from_string(const char **s, routerlist_t **dest, continue; } - if (!good_nickname_list) { - router->is_running = 1; /* start out assuming all dirservers are up */ - router->is_verified = router->is_named = 1; - router->status_set_at = time(NULL); - } smartlist_add(routers, router); -// log_fn(LOG_DEBUG,"just added router #%d.",smartlist_len(routers)); } - if (good_nickname_list) { - SMARTLIST_FOREACH(good_nickname_list, const char *, cp, - routers_update_status_from_entry(routers, published_on, cp)); - } + routers_update_status_from_networkstatus(routers); if (*dest) routerlist_free(*dest);