r12414@catbus: nickm | 2007-04-16 17:37:17 -0400

More proposal-104 stuff: actually remember extra-info stuff.


svn:r9975

ChangeLog

@@ -30,6 +30,15 @@ Changes in version 0.2.0.1-alpha - 2007-??-??
       message, so when people paste just their logs, we know if it's
       openbsd or windows or what.
 
+  o Minor features (directory system):
+    - Directory authorities accept and serve "extra info" documents for
+      routers.  These documents contain fields from router descriptors that
+      aren't usually needed, and that use a lot of excess bandwidth. Once
+      these fields are removed from router descriptors, the bandwidth savings
+      should be about 60%.  (Limitation: servers do not yet upload extra-info
+      documents; authorities do not yet cache them.)  [Partially implements
+      proposal 104.]
+
   o Minor features (other):
     - More unit tests.
 

doc/TODO

@@ -60,7 +60,8 @@ Things we'd like to do in 0.2.0.x:
       o Have routers generate extra-info documents.
       . Have have authorities accept them and serve them from specified URLs
         o Implement directory-protocol side.
-        - Implement storage.
+        o Implement storage in memory
+        - Implement cache on disk.
       - Have routers upload extra-info documents.
       - Implement option to download and cache extra-info documents.
       - Drop bandwidth history from router-descriptors

src/common/mempool.c

@@ -140,7 +140,6 @@ struct mp_chunk_t {
   char mem[1]; /**< Storage for this chunk. (Not actual size.) */
 };
 
-
 /** Number of extra bytes needed beyond mem_size to allocate a chunk. */
 #define CHUNK_OVERHEAD (sizeof(mp_chunk_t)-1)
 

src/or/dirserv.c

@@ -562,9 +562,7 @@ dirserv_add_descriptor(const char *desc, const char **msg)
       extrainfo_free(ei);
       return -1;
     }
-    /* XXXX020 Eventually, we should store this.  For now, we'll just
+    router_add_extrainfo_to_routerlist(ei, msg, 0, 0);
-     * discard it. */
-    extrainfo_free(ei);
     return 2;
   }
 
@@ -2115,11 +2113,11 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn)
         routerinfo_t *ri = router_get_by_digest(fp);
         if (ri &&
             ri->cache_info.published_on > publish_cutoff) {
-          if (extra) {
+          if (extra)
-            sd = extrainfo_get_by_descriptor_digest(ri->extra_info_digest);
+            sd = extrainfo_get_by_descriptor_digest(
-          } else {
+                                         ri->cache_info.extra_info_digest);
+          else
             sd = &ri->cache_info;
-          }
         }
       }
     } else {

src/or/or.h

@@ -1047,6 +1047,8 @@ typedef struct signed_descriptor_t {
   char identity_digest[DIGEST_LEN];
   /** Declared publication time of the descriptor */
   time_t published_on;
+ /** DOCDOC; routerinfo_t only. */
+  char extra_info_digest[DIGEST_LEN];
   /** Where is the descriptor saved? */
   saved_location_t saved_location;
   /** If saved_location is SAVED_IN_CACHE or SAVED_IN_JOURNAL, the offset of
@@ -1081,7 +1083,6 @@ typedef struct {
   smartlist_t *declared_family; /**< Nicknames of router which this router
                                  * claims are its family. */
   char *contact_info; /**< Declared contact info for this router. */
-  char extra_info_digest[DIGEST_LEN]; /**< DOCDOC */
   unsigned int is_hibernating:1; /**< Whether the router claims to be
                                   * hibernating */
   unsigned int has_old_dnsworkers:1; /**< Whether the router is using
@@ -1128,6 +1129,7 @@ typedef struct {
 typedef struct extrainfo_t {
   signed_descriptor_t cache_info;
   char nickname[MAX_NICKNAME_LEN+1];
+  unsigned int bad_sig : 1;
   char *pending_sig;
 } extrainfo_t;
 
@@ -1245,6 +1247,9 @@ typedef struct {
   /** Map from server descriptor digest to a signed_descriptor_t from
    * routers or old_routers. */
   digestmap_t *desc_digest_map;
+  /** Map from extra-info digest to a signed_descriptor_t.  Only for
+   * routers in routers or old_routers. */
+  digestmap_t *extra_info_map;
   /** List of routerinfo_t for all currently live routers we know. */
   smartlist_t *routers;
   /** List of signed_descriptor_t for older router descriptors we're
@@ -3016,6 +3021,8 @@ void routerlist_remove_old_routers(void);
 void networkstatus_list_clean(time_t now);
 int router_add_to_routerlist(routerinfo_t *router, const char **msg,
                              int from_cache, int from_fetch);
+void router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg,
+                                        int from_cache, int from_fetch);
 int router_load_single_router(const char *s, uint8_t purpose,
                               const char **msg);
 void router_load_routers_from_string(const char *s,

src/or/rephist.c

@@ -630,7 +630,7 @@ rep_hist_get_bandwidth_lines(int for_extrainfo)
     tor_assert(b);
     format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL);
     tor_snprintf(cp, len-(cp-buf), "%s%s %s (%d s) ",
-                 for_extrainfo ? "" : "opt ", 
+                 for_extrainfo ? "" : "opt ",
                  r ? "read-history" : "write-history", t,
                  NUM_SECS_BW_SUM_INTERVAL);
     cp += strlen(cp);

src/or/router.c

@@ -980,7 +980,8 @@ router_rebuild_descriptor(int force)
                             ei->cache_info.signed_descriptor_digest);
 
   /* Now finish the router descriptor. */
-  memcpy(ri->extra_info_digest, ei->cache_info.signed_descriptor_digest,
+  memcpy(ri->cache_info.extra_info_digest,
+         ei->cache_info.signed_descriptor_digest,
          DIGEST_LEN);
   ri->cache_info.signed_descriptor_body = tor_malloc(8192);
   if (router_dump_router_to_string(ri->cache_info.signed_descriptor_body, 8192,
@@ -1258,7 +1259,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router,
   }
 
   base16_encode(extra_info_digest, sizeof(extra_info_digest),
-                router->extra_info_digest, DIGEST_LEN);
+                router->cache_info.extra_info_digest, DIGEST_LEN);
 
   /* Generate the easy portion of the router descriptor. */
   result = tor_snprintf(s, maxlen,

src/or/routerlist.c

@@ -1446,9 +1446,11 @@ router_get_by_descriptor_digest(const char *digest)
 signed_descriptor_t *
 extrainfo_get_by_descriptor_digest(const char *digest)
 {
-  /* XXXX020 implement me. */
+  extrainfo_t *ei;
-  (void)digest;
+  tor_assert(digest);
-  return NULL;
+  if (!routerlist) return NULL;
+  ei = digestmap_get(routerlist->extra_info_map, digest);
+  return ei ? &ei->cache_info : NULL;
 }
 
 /** Return a pointer to the signed textual representation of a descriptor.
@@ -1487,6 +1489,7 @@ router_get_routerlist(void)
     routerlist->old_routers = smartlist_create();
     routerlist->identity_map = digestmap_new();
     routerlist->desc_digest_map = digestmap_new();
+    routerlist->extra_info_map = digestmap_new();
   }
   return routerlist;
 }
@@ -1546,6 +1549,13 @@ signed_descriptor_from_routerinfo(routerinfo_t *ri)
   return sd;
 }
 
+/** DOCDOC */
+static void
+_extrainfo_free(void *e)
+{
+  extrainfo_free(e);
+}
+
 /** Free all storage held by a routerlist <b>rl</b> */
 void
 routerlist_free(routerlist_t *rl)
@@ -1553,6 +1563,7 @@ routerlist_free(routerlist_t *rl)
   tor_assert(rl);
   digestmap_free(rl->identity_map, NULL);
   digestmap_free(rl->desc_digest_map, NULL);
+  digestmap_free(rl->extra_info_map, _extrainfo_free);
   SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
                     routerinfo_free(r));
   SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd,
@@ -1642,6 +1653,44 @@ routerlist_insert(routerlist_t *rl, routerinfo_t *ri)
   // routerlist_assert_ok(rl);
 }
 
+/**DOCDOC*/
+static void
+extrainfo_insert(routerlist_t *rl, extrainfo_t *ei)
+{
+  routerinfo_t *ri = digestmap_get(rl->identity_map,
+                                   ei->cache_info.identity_digest);
+  extrainfo_t *ei_tmp;
+  if (!ri || routerinfo_incompatible_with_extrainfo(ri,ei)) {
+    int found = 0;
+    if (ei->pending_sig || ei->bad_sig) {
+      extrainfo_free(ei);
+      return;
+    }
+    /* The signature checks out; let's see if one of the old routers
+     * matches. */
+    SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd, {
+        if (!memcmp(ei->cache_info.identity_digest,
+                    sd->identity_digest, DIGEST_LEN) &&
+            !memcmp(ei->cache_info.signed_descriptor_digest,
+                    sd->extra_info_digest, DIGEST_LEN) &&
+            sd->published_on == ei->cache_info.published_on) {
+          found = 1;
+          break;
+        }
+      });
+    if (!found) {
+      extrainfo_free(ei);
+      return;
+    }
+  }
+
+  ei_tmp = digestmap_set(rl->extra_info_map,
+                         ei->cache_info.signed_descriptor_digest,
+                         ei);
+  if (ei_tmp)
+    extrainfo_free(ei_tmp);
+}
+
 /** If we're a directory cache and routerlist <b>rl</b> doesn't have
  * a copy of router <b>ri</b> yet, add it to the list of old (not
  * recommended but still served) descriptors. Else free it. */
@@ -1673,6 +1722,7 @@ void
 routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int idx, int make_old)
 {
   routerinfo_t *ri_tmp;
+  extrainfo_t *ei_tmp;
   idx = _routerlist_find_elt(rl->routers, ri, idx);
   if (idx < 0)
     return;
@@ -1686,6 +1736,7 @@ routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int idx, int make_old)
   ri_tmp = digestmap_remove(rl->identity_map, ri->cache_info.identity_digest);
   router_dir_info_changed();
   tor_assert(ri_tmp == ri);
+
   if (make_old && get_options()->DirPort &&
       ri->purpose == ROUTER_PURPOSE_GENERAL) {
     signed_descriptor_t *sd;
@@ -1698,6 +1749,10 @@ routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int idx, int make_old)
     tor_assert(ri_tmp == ri);
     router_bytes_dropped += ri->cache_info.signed_descriptor_len;
     routerinfo_free(ri);
+    ei_tmp = digestmap_remove(rl->extra_info_map,
+                              ri->cache_info.extra_info_digest);
+    if (ei_tmp)
+      extrainfo_free(ei_tmp);
   }
   // routerlist_assert_ok(rl);
 }
@@ -1706,6 +1761,7 @@ static void
 routerlist_remove_old(routerlist_t *rl, signed_descriptor_t *sd, int idx)
 {
   signed_descriptor_t *sd_tmp;
+  extrainfo_t *ei_tmp;
   idx = _routerlist_find_elt(rl->old_routers, sd, idx);
   if (idx < 0)
     return;
@@ -1715,6 +1771,12 @@ routerlist_remove_old(routerlist_t *rl, signed_descriptor_t *sd, int idx)
   tor_assert(sd_tmp == sd);
   router_bytes_dropped += sd->signed_descriptor_len;
   signed_descriptor_free(sd);
+
+  ei_tmp = digestmap_remove(rl->extra_info_map,
+                            sd->extra_info_digest);
+  if (ei_tmp)
+    extrainfo_free(ei_tmp);
+
   // routerlist_assert_ok(rl);
 }
 
@@ -1731,6 +1793,7 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old,
                    routerinfo_t *ri_new, int idx, int make_old)
 {
   routerinfo_t *ri_tmp;
+  extrainfo_t *ei_tmp;
   tor_assert(ri_old != ri_new);
   idx = _routerlist_find_elt(rl->routers, ri_old, idx);
   router_dir_info_changed();
@@ -1766,6 +1829,12 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old,
       /* digests don't match; digestmap_set didn't replace */
       digestmap_remove(rl->desc_digest_map,
                        ri_old->cache_info.signed_descriptor_digest);
+
+      ei_tmp = digestmap_remove(rl->extra_info_map,
+                                ri_old->cache_info.extra_info_digest);
+      if (ei_tmp) {
+        extrainfo_free(ei_tmp);
+      }
     }
     routerinfo_free(ri_old);
   }
@@ -2067,6 +2136,18 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg,
   return 0;
 }
 
+/** DOCDOC */
+void
+router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg,
+                                   int from_cache, int from_fetch)
+{
+  /* XXXX020 cache on disk */
+  (void)from_cache;
+  (void)from_fetch;
+  (void)msg;
+  extrainfo_insert(router_get_routerlist(), ei);
+}
+
 /** Sorting helper: return &lt;0, 0, or &gt;0 depending on whether the
  * signed_descriptor_t* in *<b>a</b> has an identity digest preceding, equal
  * to, or later than that of *<b>b</b>. */
@@ -4438,6 +4519,9 @@ routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei)
   tor_assert(ri);
   tor_assert(ei);
 
+  if (ei->bad_sig)
+    return 1;
+
   if (strcmp(ri->nickname, ei->nickname) ||
       memcmp(ri->cache_info.identity_digest, ei->cache_info.identity_digest,
              DIGEST_LEN))
@@ -4448,8 +4532,11 @@ routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei)
     if (crypto_pk_public_checksig(ri->identity_pkey, signed_digest,
                                   ei->pending_sig, 128) != 20 ||
         memcmp(signed_digest, ei->cache_info.signed_descriptor_digest,
-               DIGEST_LEN))
+               DIGEST_LEN)) {
+      ei->bad_sig = 1;
+      tor_free(ei->pending_sig);
       return 1; /* Bad signature, or no match. */
+    }
 
     tor_free(ei->pending_sig);
   }

src/or/routerparse.c

@@ -967,8 +967,8 @@ router_parse_entry_from_string(const char *s, const char *end,
   if ((tok = find_first_by_keyword(tokens, K_EXTRA_INFO_DIGEST))) {
     tor_assert(tok->n_args >= 1);
     if (strlen(tok->args[0]) == HEX_DIGEST_LEN) {
-      base16_decode(router->extra_info_digest, DIGEST_LEN, tok->args[0],
+      base16_decode(router->cache_info.extra_info_digest,
-                    HEX_DIGEST_LEN);
+                    DIGEST_LEN, tok->args[0], HEX_DIGEST_LEN);
     } else {
       log_warn(LD_DIR, "Invalid extra info digest %s", escaped(tok->args[0]));
     }