diff --git a/ChangeLog b/ChangeLog index 947f026a7f..c260d7335d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,15 @@ Changes in version 0.2.0.1-alpha - 2007-??-?? message, so when people paste just their logs, we know if it's openbsd or windows or what. + o Minor features (directory system): + - Directory authorities accept and serve "extra info" documents for + routers. These documents contain fields from router descriptors that + aren't usually needed, and that use a lot of excess bandwidth. Once + these fields are removed from router descriptors, the bandwidth savings + should be about 60%. (Limitation: servers do not yet upload extra-info + documents; authorities do not yet cache them.) [Partially implements + proposal 104.] + o Minor features (other): - More unit tests. diff --git a/doc/TODO b/doc/TODO index 924ba81a6b..921385a8a8 100644 --- a/doc/TODO +++ b/doc/TODO @@ -60,7 +60,8 @@ Things we'd like to do in 0.2.0.x: o Have routers generate extra-info documents. . Have have authorities accept them and serve them from specified URLs o Implement directory-protocol side. - - Implement storage. + o Implement storage in memory + - Implement cache on disk. - Have routers upload extra-info documents. - Implement option to download and cache extra-info documents. - Drop bandwidth history from router-descriptors diff --git a/src/common/mempool.c b/src/common/mempool.c index 673cf21e8b..91d4db07dc 100644 --- a/src/common/mempool.c +++ b/src/common/mempool.c @@ -140,7 +140,6 @@ struct mp_chunk_t { char mem[1]; /**< Storage for this chunk. (Not actual size.) */ }; - /** Number of extra bytes needed beyond mem_size to allocate a chunk. */ #define CHUNK_OVERHEAD (sizeof(mp_chunk_t)-1) diff --git a/src/or/dirserv.c b/src/or/dirserv.c index e8811c0d01..415810ebfd 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -562,9 +562,7 @@ dirserv_add_descriptor(const char *desc, const char **msg) extrainfo_free(ei); return -1; } - /* XXXX020 Eventually, we should store this. For now, we'll just - * discard it. */ - extrainfo_free(ei); + router_add_extrainfo_to_routerlist(ei, msg, 0, 0); return 2; } @@ -2115,11 +2113,11 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) routerinfo_t *ri = router_get_by_digest(fp); if (ri && ri->cache_info.published_on > publish_cutoff) { - if (extra) { - sd = extrainfo_get_by_descriptor_digest(ri->extra_info_digest); - } else { + if (extra) + sd = extrainfo_get_by_descriptor_digest( + ri->cache_info.extra_info_digest); + else sd = &ri->cache_info; - } } } } else { diff --git a/src/or/or.h b/src/or/or.h index f7ee77e505..577caf37ab 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1047,6 +1047,8 @@ typedef struct signed_descriptor_t { char identity_digest[DIGEST_LEN]; /** Declared publication time of the descriptor */ time_t published_on; + /** DOCDOC; routerinfo_t only. */ + char extra_info_digest[DIGEST_LEN]; /** Where is the descriptor saved? */ saved_location_t saved_location; /** If saved_location is SAVED_IN_CACHE or SAVED_IN_JOURNAL, the offset of @@ -1081,7 +1083,6 @@ typedef struct { smartlist_t *declared_family; /**< Nicknames of router which this router * claims are its family. */ char *contact_info; /**< Declared contact info for this router. */ - char extra_info_digest[DIGEST_LEN]; /**< DOCDOC */ unsigned int is_hibernating:1; /**< Whether the router claims to be * hibernating */ unsigned int has_old_dnsworkers:1; /**< Whether the router is using @@ -1128,6 +1129,7 @@ typedef struct { typedef struct extrainfo_t { signed_descriptor_t cache_info; char nickname[MAX_NICKNAME_LEN+1]; + unsigned int bad_sig : 1; char *pending_sig; } extrainfo_t; @@ -1245,6 +1247,9 @@ typedef struct { /** Map from server descriptor digest to a signed_descriptor_t from * routers or old_routers. */ digestmap_t *desc_digest_map; + /** Map from extra-info digest to a signed_descriptor_t. Only for + * routers in routers or old_routers. */ + digestmap_t *extra_info_map; /** List of routerinfo_t for all currently live routers we know. */ smartlist_t *routers; /** List of signed_descriptor_t for older router descriptors we're @@ -3016,6 +3021,8 @@ void routerlist_remove_old_routers(void); void networkstatus_list_clean(time_t now); int router_add_to_routerlist(routerinfo_t *router, const char **msg, int from_cache, int from_fetch); +void router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, + int from_cache, int from_fetch); int router_load_single_router(const char *s, uint8_t purpose, const char **msg); void router_load_routers_from_string(const char *s, diff --git a/src/or/rephist.c b/src/or/rephist.c index c41a334687..051e578f3c 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -630,7 +630,7 @@ rep_hist_get_bandwidth_lines(int for_extrainfo) tor_assert(b); format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); tor_snprintf(cp, len-(cp-buf), "%s%s %s (%d s) ", - for_extrainfo ? "" : "opt ", + for_extrainfo ? "" : "opt ", r ? "read-history" : "write-history", t, NUM_SECS_BW_SUM_INTERVAL); cp += strlen(cp); diff --git a/src/or/router.c b/src/or/router.c index 27c803159b..3c3255a49d 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -980,7 +980,8 @@ router_rebuild_descriptor(int force) ei->cache_info.signed_descriptor_digest); /* Now finish the router descriptor. */ - memcpy(ri->extra_info_digest, ei->cache_info.signed_descriptor_digest, + memcpy(ri->cache_info.extra_info_digest, + ei->cache_info.signed_descriptor_digest, DIGEST_LEN); ri->cache_info.signed_descriptor_body = tor_malloc(8192); if (router_dump_router_to_string(ri->cache_info.signed_descriptor_body, 8192, @@ -1258,7 +1259,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, } base16_encode(extra_info_digest, sizeof(extra_info_digest), - router->extra_info_digest, DIGEST_LEN); + router->cache_info.extra_info_digest, DIGEST_LEN); /* Generate the easy portion of the router descriptor. */ result = tor_snprintf(s, maxlen, diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 63f3eec297..0f98b3d3a4 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -1446,9 +1446,11 @@ router_get_by_descriptor_digest(const char *digest) signed_descriptor_t * extrainfo_get_by_descriptor_digest(const char *digest) { - /* XXXX020 implement me. */ - (void)digest; - return NULL; + extrainfo_t *ei; + tor_assert(digest); + if (!routerlist) return NULL; + ei = digestmap_get(routerlist->extra_info_map, digest); + return ei ? &ei->cache_info : NULL; } /** Return a pointer to the signed textual representation of a descriptor. @@ -1487,6 +1489,7 @@ router_get_routerlist(void) routerlist->old_routers = smartlist_create(); routerlist->identity_map = digestmap_new(); routerlist->desc_digest_map = digestmap_new(); + routerlist->extra_info_map = digestmap_new(); } return routerlist; } @@ -1546,6 +1549,13 @@ signed_descriptor_from_routerinfo(routerinfo_t *ri) return sd; } +/** DOCDOC */ +static void +_extrainfo_free(void *e) +{ + extrainfo_free(e); +} + /** Free all storage held by a routerlist rl */ void routerlist_free(routerlist_t *rl) @@ -1553,6 +1563,7 @@ routerlist_free(routerlist_t *rl) tor_assert(rl); digestmap_free(rl->identity_map, NULL); digestmap_free(rl->desc_digest_map, NULL); + digestmap_free(rl->extra_info_map, _extrainfo_free); SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r, routerinfo_free(r)); SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd, @@ -1642,6 +1653,44 @@ routerlist_insert(routerlist_t *rl, routerinfo_t *ri) // routerlist_assert_ok(rl); } +/**DOCDOC*/ +static void +extrainfo_insert(routerlist_t *rl, extrainfo_t *ei) +{ + routerinfo_t *ri = digestmap_get(rl->identity_map, + ei->cache_info.identity_digest); + extrainfo_t *ei_tmp; + if (!ri || routerinfo_incompatible_with_extrainfo(ri,ei)) { + int found = 0; + if (ei->pending_sig || ei->bad_sig) { + extrainfo_free(ei); + return; + } + /* The signature checks out; let's see if one of the old routers + * matches. */ + SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd, { + if (!memcmp(ei->cache_info.identity_digest, + sd->identity_digest, DIGEST_LEN) && + !memcmp(ei->cache_info.signed_descriptor_digest, + sd->extra_info_digest, DIGEST_LEN) && + sd->published_on == ei->cache_info.published_on) { + found = 1; + break; + } + }); + if (!found) { + extrainfo_free(ei); + return; + } + } + + ei_tmp = digestmap_set(rl->extra_info_map, + ei->cache_info.signed_descriptor_digest, + ei); + if (ei_tmp) + extrainfo_free(ei_tmp); +} + /** If we're a directory cache and routerlist rl doesn't have * a copy of router ri yet, add it to the list of old (not * recommended but still served) descriptors. Else free it. */ @@ -1673,6 +1722,7 @@ void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int idx, int make_old) { routerinfo_t *ri_tmp; + extrainfo_t *ei_tmp; idx = _routerlist_find_elt(rl->routers, ri, idx); if (idx < 0) return; @@ -1686,6 +1736,7 @@ routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int idx, int make_old) ri_tmp = digestmap_remove(rl->identity_map, ri->cache_info.identity_digest); router_dir_info_changed(); tor_assert(ri_tmp == ri); + if (make_old && get_options()->DirPort && ri->purpose == ROUTER_PURPOSE_GENERAL) { signed_descriptor_t *sd; @@ -1698,6 +1749,10 @@ routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int idx, int make_old) tor_assert(ri_tmp == ri); router_bytes_dropped += ri->cache_info.signed_descriptor_len; routerinfo_free(ri); + ei_tmp = digestmap_remove(rl->extra_info_map, + ri->cache_info.extra_info_digest); + if (ei_tmp) + extrainfo_free(ei_tmp); } // routerlist_assert_ok(rl); } @@ -1706,6 +1761,7 @@ static void routerlist_remove_old(routerlist_t *rl, signed_descriptor_t *sd, int idx) { signed_descriptor_t *sd_tmp; + extrainfo_t *ei_tmp; idx = _routerlist_find_elt(rl->old_routers, sd, idx); if (idx < 0) return; @@ -1715,6 +1771,12 @@ routerlist_remove_old(routerlist_t *rl, signed_descriptor_t *sd, int idx) tor_assert(sd_tmp == sd); router_bytes_dropped += sd->signed_descriptor_len; signed_descriptor_free(sd); + + ei_tmp = digestmap_remove(rl->extra_info_map, + sd->extra_info_digest); + if (ei_tmp) + extrainfo_free(ei_tmp); + // routerlist_assert_ok(rl); } @@ -1731,6 +1793,7 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, routerinfo_t *ri_new, int idx, int make_old) { routerinfo_t *ri_tmp; + extrainfo_t *ei_tmp; tor_assert(ri_old != ri_new); idx = _routerlist_find_elt(rl->routers, ri_old, idx); router_dir_info_changed(); @@ -1766,6 +1829,12 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, /* digests don't match; digestmap_set didn't replace */ digestmap_remove(rl->desc_digest_map, ri_old->cache_info.signed_descriptor_digest); + + ei_tmp = digestmap_remove(rl->extra_info_map, + ri_old->cache_info.extra_info_digest); + if (ei_tmp) { + extrainfo_free(ei_tmp); + } } routerinfo_free(ri_old); } @@ -2067,6 +2136,18 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, return 0; } +/** DOCDOC */ +void +router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, + int from_cache, int from_fetch) +{ + /* XXXX020 cache on disk */ + (void)from_cache; + (void)from_fetch; + (void)msg; + extrainfo_insert(router_get_routerlist(), ei); +} + /** Sorting helper: return <0, 0, or >0 depending on whether the * signed_descriptor_t* in *a has an identity digest preceding, equal * to, or later than that of *b. */ @@ -4438,6 +4519,9 @@ routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei) tor_assert(ri); tor_assert(ei); + if (ei->bad_sig) + return 1; + if (strcmp(ri->nickname, ei->nickname) || memcmp(ri->cache_info.identity_digest, ei->cache_info.identity_digest, DIGEST_LEN)) @@ -4448,8 +4532,11 @@ routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei) if (crypto_pk_public_checksig(ri->identity_pkey, signed_digest, ei->pending_sig, 128) != 20 || memcmp(signed_digest, ei->cache_info.signed_descriptor_digest, - DIGEST_LEN)) + DIGEST_LEN)) { + ei->bad_sig = 1; + tor_free(ei->pending_sig); return 1; /* Bad signature, or no match. */ + } tor_free(ei->pending_sig); } diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 9398d14f0e..92224183d1 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -967,8 +967,8 @@ router_parse_entry_from_string(const char *s, const char *end, if ((tok = find_first_by_keyword(tokens, K_EXTRA_INFO_DIGEST))) { tor_assert(tok->n_args >= 1); if (strlen(tok->args[0]) == HEX_DIGEST_LEN) { - base16_decode(router->extra_info_digest, DIGEST_LEN, tok->args[0], - HEX_DIGEST_LEN); + base16_decode(router->cache_info.extra_info_digest, + DIGEST_LEN, tok->args[0], HEX_DIGEST_LEN); } else { log_warn(LD_DIR, "Invalid extra info digest %s", escaped(tok->args[0])); }