From 28946069eef5cb980a7a14b4893ee531c9ebd6cb Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Thu, 27 Sep 2007 16:08:10 +0000 Subject: [PATCH] r15412@catbus: nickm | 2007-09-27 12:04:24 -0400 More annotated-store work: handle annotations in lists correctly. Add ability to prepend annotations to a routerdesc (and to every rtouredesc in a list), while verifying that the routerdesc is not already annotated. svn:r11665 --- doc/TODO | 19 +++++++-- src/or/dirserv.c | 7 +++- src/or/or.h | 6 ++- src/or/router.c | 4 +- src/or/routerlist.c | 7 ++-- src/or/routerparse.c | 94 ++++++++++++++++++++++++++++++++++++-------- src/or/test.c | 2 +- 7 files changed, 109 insertions(+), 30 deletions(-) diff --git a/doc/TODO b/doc/TODO index e08b7f43f3..f6749237d6 100644 --- a/doc/TODO +++ b/doc/TODO @@ -122,17 +122,30 @@ Things we'd like to do in 0.2.0.x: - use the bridges for dir fetches even when our dirport is open. R - drop 'authority' queries if they're to our own identity key; accept them otherwise. -N - Design/implement the "local-status" or something like it, from the + X Design/implement the "local-status" or something like it, from the "Descriptor purposes: how to tell them apart" section of http://archives.seul.org/or/dev/May-2007/msg00008.html - - cache of bridges that we've learned about and use but aren't - manually listed in the torrc. o timeout and retry schedules for fetching bridge descriptors - give extend_info_t a router_purpose again o react faster to download networkstatuses after the first bridge descriptor arrives o be more robust to bridges being marked as down and leaving us stranded without any known "running" bridges. +N . Cache for bridge descriptors + . Annotated router store + o Accept annotations before routers + o Preserve and ignore unexpected annotations + o Mechanism to add annotations when we first add a descriptor + o Don't serve annotations + o Reject annotations that appear in things we've downloaded + - Name the router store something different: cached-descriptors? + - But load from cached-routers if no cached-descriptors is + found. + - Document this. + - Use annotations to denote router purpose + - Learn purpose from annotations + - Set annotations based on purpose + - Preserve routers with unrecognized purpose. - Bridges operators (rudimentary version) - Ability to act as dir cache without a dir port. o Bridges publish to bridge authorities diff --git a/src/or/dirserv.c b/src/or/dirserv.c index c55e8cd345..0c31eef01c 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -536,11 +536,13 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, s = desc; list = smartlist_create(); - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0)) { + if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0, + NULL)) { SMARTLIST_FOREACH(list, routerinfo_t *, ri, { msg_out = NULL; /* Assign the purpose. + * * XXX020 Perhaps this should get pushed into * router_parse_list_from_string()? Also, tie it somehow into * router_load_single_router()? Lastly, does extrainfo_t want @@ -561,7 +563,8 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, smartlist_clear(list); s = desc; - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0)) { + if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0, + NULL)) { SMARTLIST_FOREACH(list, extrainfo_t *, ei, { msg_out = NULL; diff --git a/src/or/or.h b/src/or/or.h index 780f537bf0..c98ab14ad7 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -3603,7 +3603,8 @@ int router_parse_list_from_string(const char **s, const char *eos, smartlist_t *dest, saved_location_t saved_location, int is_extrainfo, - int allow_annotations); + int allow_annotations, + const char *prepend_annotations); int router_parse_routerlist_from_directory(const char *s, routerlist_t **dest, crypto_pk_env_t *pkey, @@ -3613,7 +3614,8 @@ int router_parse_runningrouters(const char *str); int router_parse_directory(const char *str); routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, int cache_copy, - int allow_annotations); + int allow_annotations, + const char *prepend_annotations); extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, int cache_copy, struct digest_ri_map_t *routermap); addr_policy_t *router_parse_addr_policy_from_string(const char *s, diff --git a/src/or/router.c b/src/or/router.c index 5358ccdf91..f3aa25299e 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -485,7 +485,7 @@ init_keys(void) return -1; } if (mydesc) { - ri = router_parse_entry_from_string(mydesc, NULL, 1, 0); + ri = router_parse_entry_from_string(mydesc, NULL, 1, 0, NULL); if (!ri) { log_err(LD_GENERAL,"Generated a routerinfo we couldn't parse."); return -1; @@ -1632,7 +1632,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, const char *cp; routerinfo_t *ri_tmp; cp = s_dup = tor_strdup(s); - ri_tmp = router_parse_entry_from_string(cp, NULL, 1, 0); + ri_tmp = router_parse_entry_from_string(cp, NULL, 1, 0, NULL); if (!ri_tmp) { log_err(LD_BUG, "We just generated a router descriptor we can't parse."); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 0ed2c0c489..41dbd126b4 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -3106,7 +3106,7 @@ router_load_single_router(const char *s, uint8_t purpose, const char **msg) tor_assert(msg); *msg = NULL; - if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0))) { + if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0, NULL))) { log_warn(LD_DIR, "Error parsing router descriptor; dropping."); *msg = "Couldn't parse router descriptor."; return -1; @@ -3167,7 +3167,7 @@ router_load_routers_from_string(const char *s, const char *eos, int allow_annotations = (saved_location != SAVED_NOWHERE); router_parse_list_from_string(&s, eos, routers, saved_location, 0, - allow_annotations); + allow_annotations, NULL); routers_update_status_from_networkstatus(routers, !from_cache); @@ -3228,7 +3228,8 @@ router_load_extrainfo_from_string(const char *s, const char *eos, const char *msg; int from_cache = (saved_location != SAVED_NOWHERE); - router_parse_list_from_string(&s, eos, extrainfo_list, saved_location, 1, 0); + router_parse_list_from_string(&s, eos, extrainfo_list, saved_location, 1, 0, + NULL); log_info(LD_DIR, "%d elements to add", smartlist_len(extrainfo_list)); diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 34111a76bc..761e6176db 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -372,10 +372,13 @@ static void token_free(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *find_first_by_keyword(smartlist_t *s, directory_keyword keyword); +#define TS_ANNOTATIONS_OK 1 +#define TS_NOCHECK 2 +#define TS_NO_NEW_ANNOTATIONS 4 static int tokenize_string(const char *start, const char *end, smartlist_t *out, token_rule_t *table, - int allow_annotations); + int flags); static directory_token_t *get_next_token(const char **s, const char *eos, token_rule_t *table); @@ -869,7 +872,8 @@ router_parse_list_from_string(const char **s, const char *eos, smartlist_t *dest, saved_location_t saved_location, int want_extrainfo, - int allow_annotations) + int allow_annotations, + const char *prepend_annotations) { routerinfo_t *router; extrainfo_t *extrainfo; @@ -893,15 +897,24 @@ router_parse_list_from_string(const char **s, const char *eos, if ((eos - *s) < 32) /* make sure it's long enough. */ break; - /* Don't start parsing the rest of *s unless it contains a router. */ + /* Don't start parsing the rest of *s unless it contains a router or + * extra-info. */ if (strcmpstart(*s, "extra-info ")==0) { have_extrainfo = 1; - } else if (strcmpstart(*s, "router ")==0) { + } else if (strcmpstart(*s, "router ")==0) { have_extrainfo = 0; } else { /* skip junk. */ - const char *ei = tor_memstr(*s, eos-*s, "\nextra-info "); - const char *ri = tor_memstr(*s, eos-*s, "\nrouter "); + const char *annotation = NULL, *ei, *ri; + if (**s == '@') { + annotation = *s; + } else { + if ((annotation = tor_memstr(*s, eos-*s, "\n@"))) + ++annotation; + } + + ei = tor_memstr(*s, eos-*s, "\nextra-info "); + ri = tor_memstr(*s, eos-*s, "\nrouter "); if (ri && (!ei || ri < ei)) { have_extrainfo = 0; *s = ri + 1; @@ -911,6 +924,8 @@ router_parse_list_from_string(const char **s, const char *eos, } else { break; } + if (annotation && annotation < *s) + *s = annotation; } end = tor_memstr(*s, eos-*s, "\nrouter-signature"); if (end) @@ -935,7 +950,8 @@ router_parse_list_from_string(const char **s, const char *eos, } else if (!have_extrainfo && !want_extrainfo) { router = router_parse_entry_from_string(*s, end, saved_location != SAVED_IN_CACHE, - allow_annotations); + allow_annotations, + prepend_annotations); if (router) { signed_desc = &router->cache_info; elt = router; @@ -986,10 +1002,12 @@ dump_distinct_digest_count(int severity) * returns NULL. If cache_copy is true, duplicate the contents of * s through end into the signed_descriptor_body of the resulting * routerinfo_t. + * DOCDOC annotations */ routerinfo_t * router_parse_entry_from_string(const char *s, const char *end, - int cache_copy, int allow_annotations) + int cache_copy, int allow_annotations, + const char *prepend_annotations) { routerinfo_t *router = NULL; char digest[128]; @@ -998,6 +1016,8 @@ router_parse_entry_from_string(const char *s, const char *end, struct in_addr in; const char *start_of_annotations, *cp; + tor_assert(!allow_annotations || !prepend_annotations); + if (!end) { end = s + strlen(s); } @@ -1006,6 +1026,15 @@ router_parse_entry_from_string(const char *s, const char *end, while (end > s+2 && *(end-1) == '\n' && *(end-2) == '\n') --end; + tokens = smartlist_create(); + if (prepend_annotations) { + if (tokenize_string(prepend_annotations,NULL,tokens, + routerdesc_token_table,TS_NOCHECK)) { + log_warn(LD_DIR, "Error tokenizing router descriptor."); + goto err; + } + } + start_of_annotations = s; cp = tor_memstr(s, end-s, "\nrouter "); if (!cp) { @@ -1021,10 +1050,17 @@ router_parse_entry_from_string(const char *s, const char *end, log_warn(LD_DIR, "Couldn't compute router hash."); return NULL; } - tokens = smartlist_create(); - if (tokenize_string(s,end,tokens,routerdesc_token_table,allow_annotations)) { - log_warn(LD_DIR, "Error tokenizing router descriptor."); - goto err; + { + int flags = 0; + if (allow_annotations) + flags |= TS_ANNOTATIONS_OK; + if (prepend_annotations) + flags |= TS_ANNOTATIONS_OK|TS_NO_NEW_ANNOTATIONS; + + if (tokenize_string(s,end,tokens,routerdesc_token_table, flags)) { + log_warn(LD_DIR, "Error tokenizing router descriptor."); + goto err; + } } if (smartlist_len(tokens) < 2) { @@ -1041,10 +1077,21 @@ router_parse_entry_from_string(const char *s, const char *end, router = tor_malloc_zero(sizeof(routerinfo_t)); router->routerlist_index = -1; - if (cache_copy) - router->cache_info.signed_descriptor_body = tor_strndup(s, end-s); - router->cache_info.signed_descriptor_len = s-start_of_annotations; + router->cache_info.annotations_len = s-start_of_annotations + + (prepend_annotations ? strlen(prepend_annotations) : 0) ; router->cache_info.signed_descriptor_len = end-s; + if (cache_copy) { + size_t len = router->cache_info.signed_descriptor_len + + router->cache_info.annotations_len; + char *cp = + router->cache_info.signed_descriptor_body = tor_malloc(len+1); + if (prepend_annotations) { + strlcpy(cp, prepend_annotations, len+1); + cp += strlen(prepend_annotations); + } + memcpy(cp, s, end-s); + cp[len] = '\0'; + } memcpy(router->cache_info.signed_descriptor_digest, digest, DIGEST_LEN); router->nickname = tor_strdup(tok->args[0]); @@ -1069,6 +1116,8 @@ router_parse_entry_from_string(const char *s, const char *end, router->bandwidthrate = tor_parse_long(tok->args[0],10,0,INT_MAX,NULL,NULL); + /* Set purpose XXXX020 NM NM*/ + if (!router->bandwidthrate) { log_warn(LD_DIR, "bandwidthrate %s unreadable or 0. Failing.", escaped(tok->args[0])); @@ -2663,16 +2712,18 @@ get_next_token(const char **s, const char *eos, token_rule_t *table) /** Read all tokens from a string between start and end, and add * them to out. Parse according to the token rules in table. + * Caller must free tokens in out. */ static int tokenize_string(const char *start, const char *end, smartlist_t *out, - token_rule_t *table, int allow_annotations) + token_rule_t *table, int flags) { const char **s; directory_token_t *tok = NULL; int counts[_NIL]; int i; int first_nonannotation; + int prev_len = smartlist_len(out); s = &start; if (!end) @@ -2691,7 +2742,10 @@ tokenize_string(const char *start, const char *end, smartlist_t *out, *s = eat_whitespace_eos(*s, end); } - if (allow_annotations) { + if (flags & TS_NOCHECK) + return 0; + + if ((flags & TS_ANNOTATIONS_OK)) { first_nonannotation = -1; for (i = 0; i < smartlist_len(out); ++i) { tok = smartlist_get(out, i); @@ -2711,6 +2765,12 @@ tokenize_string(const char *start, const char *end, smartlist_t *out, return -1; } } + if ((flags & TS_NO_NEW_ANNOTATIONS)) { + if (first_nonannotation != prev_len) { + log_warn(LD_DIR, "parse error: Unexpectd annotations."); + return -1; + } + } } else { for (i=0; i < smartlist_len(out); ++i) { tok = smartlist_get(out, i); diff --git a/src/or/test.c b/src/or/test.c index f9d817d101..9c707099b2 100644 --- a/src/or/test.c +++ b/src/or/test.c @@ -2189,7 +2189,7 @@ test_dir_format(void) test_assert(router_dump_router_to_string(buf, 2048, &r1, pk2)>0); cp = buf; - rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0); + rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); test_assert(rp1); test_streq(rp1->address, r1.address); test_eq(rp1->or_port, r1.or_port);