r12586@catbus: nickm | 2007-04-30 15:43:05 -0400

More work for proposal 104: actually cache extrainfo documents to disk, and reload the cache properly.


svn:r10070

ChangeLog

@@ -60,8 +60,7 @@ Changes in version 0.2.0.1-alpha - 2007-??-??
       aren't usually needed, and that use a lot of excess bandwidth. Once
       these fields are removed from router descriptors, the bandwidth savings
       should be about 60%.  (Limitation: servers do not yet upload extra-info
-      documents; authorities do not yet cache them.)  [Partially implements
+      documents.)  [Partially implements proposal 104.]
-      proposal 104.]
     - Directory authorities allow multiple router descriptors and/or extra
       info documents to be uploaded in a single go.  This will make
       implementing proposal 104 simpler.

doc/TODO

@@ -71,10 +71,10 @@ Things we'd like to do in 0.2.0.x:
       . Finalize proposal
       o Implement parsing for extra-info documents
       o Have routers generate extra-info documents.
-      . Have have authorities accept them and serve them from specified URLs
+      o Have have authorities accept them and serve them from specified URLs
         o Implement directory-protocol side.
         o Implement storage in memory
-        - Implement cache on disk.
+        o Implement cache on disk.
       - Have routers upload extra-info documents.
       - Implement option to download and cache extra-info documents.
       - Drop bandwidth history from router-descriptors

src/or/or.h

@@ -1066,6 +1066,8 @@ typedef struct signed_descriptor_t {
   /** If saved_location is SAVED_IN_CACHE or SAVED_IN_JOURNAL, the offset of
    * this descriptor in the corresponding file. */
   off_t saved_offset;
+  /* DOCDOC */
+  unsigned int do_not_cache : 1;
 } signed_descriptor_t;
 
 /** Information about another onion router in the network. */
@@ -1137,6 +1139,9 @@ typedef struct {
   int routerlist_index;
 } routerinfo_t;
 
+/** DOCDOC */
+#define EXTRAINFO_PURPOSE_GENERAL -1
+
 /** Information needed to keep and cache a signed extra-info document. */
 typedef struct extrainfo_t {
   signed_descriptor_t cache_info;
@@ -1276,6 +1281,8 @@ typedef struct {
    * cache_info.saved_location == SAVED_IN_CACHE is stored in this file
    * starting at cache_info.saved_offset */
   tor_mmap_t *mmap_descriptors;
+  /** Mmaped file holding extra-info documents. */
+  tor_mmap_t *mmap_extrainfo;
 } routerlist_t;
 
 /** Information on router used when extending a circuit. We don't need a
@@ -3156,7 +3163,8 @@ int router_append_dirobj_signature(char *buf, size_t buf_len,
                                    crypto_pk_env_t *private_key);
 int router_parse_list_from_string(const char **s,
                                   smartlist_t *dest,
-                                  saved_location_t saved_location);
+                                  saved_location_t saved_location,
+                                  int is_extrainfo);
 int router_parse_routerlist_from_directory(const char *s,
                                            routerlist_t **dest,
                                            crypto_pk_env_t *pkey,

src/or/routerlist.c

@@ -41,6 +41,8 @@ static local_routerstatus_t *router_get_combined_status_by_nickname(
                                                 int warn_if_unnamed);
 static void update_router_have_minimum_dir_info(void);
 static void router_dir_info_changed(void);
+static void router_load_extrainfo_from_string(const char *s,
+                                              saved_location_t saved_location);
 
 /****************************************************************************/
 
@@ -168,49 +170,65 @@ router_reload_networkstatus(void)
  * On startup, we read both files.
  */
 
-/** The size of the router log, in bytes. */
+/** DOCDOC */
-static size_t router_journal_len = 0;
+typedef struct store_stats_t {
-/** The size of the router store, in bytes. */
+  /** The size of the router log, in bytes. */
-static size_t router_store_len = 0;
+  size_t journal_len;
-/** Total bytes dropped since last rebuild. */
+  /** The size of the router store, in bytes. */
-static size_t router_bytes_dropped = 0;
+  size_t store_len;
+  /** Total bytes dropped since last rebuild. */
+  size_t bytes_dropped;
+} store_stats_t;
+
+/** DOCDOC */
+static store_stats_t router_store_stats = { 0, 0, 0 };
+/** DOCDOC */
+static store_stats_t extrainfo_store_stats = { 0, 0, 0 };
 
 /** Helper: return 1 iff the router log is so big we want to rebuild the
  * store. */
 static int
-router_should_rebuild_store(void)
+router_should_rebuild_store(store_stats_t *stats)
 {
-  if (router_store_len > (1<<16))
+  if (stats->store_len > (1<<16))
-    return (router_journal_len > router_store_len / 2 ||
+    return (stats->journal_len > stats->store_len / 2 ||
-            router_bytes_dropped > router_store_len / 2);
+            stats->bytes_dropped > stats->store_len / 2);
   else
-    return router_journal_len > (1<<15);
+    return stats->journal_len > (1<<15);
 }
 
-/** Add the router descriptor in <b>desc</b> to the router
+/** Add the signed_descriptor_t in <b>desc</b> to the router
  * journal; change its saved_location to SAVED_IN_JOURNAL and set its
  * offset appropriately.
  *
- * If <b>purpose</b> isn't ROUTER_PURPOSE_GENERAL, just do nothing. */
+ * If <b>purpose</b> isn't ROUTER_PURPOSE_GENERAL or
+ * EXTRAINFO_PURPOSE_GENERAL, just do nothing. */
 static int
-router_append_to_journal(signed_descriptor_t *desc, uint8_t purpose)
+signed_desc_append_to_journal(signed_descriptor_t *desc, int purpose)
 {
   or_options_t *options = get_options();
   size_t fname_len = strlen(options->DataDirectory)+32;
   char *fname;
   const char *body = signed_descriptor_get_body(desc);
   size_t len = desc->signed_descriptor_len;
+  const char *fname_base = purpose == ROUTER_PURPOSE_GENERAL ?
+    "cached-routers" : "cached-extrainfo";
+  store_stats_t *stats;
 
   routerlist_check_bug_417();
 
-  if (purpose != ROUTER_PURPOSE_GENERAL) {
+  if (purpose == ROUTER_PURPOSE_GENERAL) {
+    stats = &router_store_stats;
+  } else if (purpose == EXTRAINFO_PURPOSE_GENERAL) {
+    stats = &extrainfo_store_stats;
+  } else {
     /* we shouldn't cache it. be happy and return. */
     return 0;
   }
 
   fname = tor_malloc(fname_len);
-  tor_snprintf(fname, fname_len, "%s"PATH_SEPARATOR"cached-routers.new",
+  tor_snprintf(fname, fname_len, "%s"PATH_SEPARATOR"%s.new",
-               options->DataDirectory);
+               options->DataDirectory, fname_base);
 
   tor_assert(len == strlen(body));
 
@@ -220,10 +238,10 @@ router_append_to_journal(signed_descriptor_t *desc, uint8_t purpose)
     return -1;
   }
   desc->saved_location = SAVED_IN_JOURNAL;
-  desc->saved_offset = router_journal_len;
-
   tor_free(fname);
-  router_journal_len += len;
+
+  desc->saved_offset = stats->journal_len;
+  stats->journal_len += len;
 
   routerlist_check_bug_417();
   return 0;
@@ -233,139 +251,147 @@ router_append_to_journal(signed_descriptor_t *desc, uint8_t purpose)
  * signed_descriptor_t* in *<b>a</b> is older, the same age as, or newer than
  * the signed_descriptor_t* in *<b>b</b>. */
 static int
-_compare_old_routers_by_age(const void **_a, const void **_b)
+_compare_signed_descriptors_by_age(const void **_a, const void **_b)
 {
   const signed_descriptor_t *r1 = *_a, *r2 = *_b;
   return r1->published_on - r2->published_on;
 }
 
-/** Sorting helper: return &lt;0, 0, or &gt;0 depending on whether the
- * routerinfo_t* in *<b>a</b> is older, the same age as, or newer than
- * the routerinfo_t* in *<b>b</b>. */
-static int
-_compare_routers_by_age(const void **_a, const void **_b)
-{
-  const routerinfo_t *r1 = *_a, *r2 = *_b;
-  return r1->cache_info.published_on - r2->cache_info.published_on;
-}
-
 /** If the journal is too long, or if <b>force</b> is true, then atomically
  * replace the router store with the routers currently in our routerlist, and
  * clear the journal.  Return 0 on success, -1 on failure.
+ *
+ * DOCDOC extrainfo
  */
 static int
-router_rebuild_store(int force)
+router_rebuild_store(int force, int extrainfo)
 {
   size_t len = 0;
   or_options_t *options;
   size_t fname_len;
   smartlist_t *chunk_list = NULL;
   char *fname = NULL, *fname_tmp = NULL;
-  int r = -1, i;
+  int r = -1;
   off_t offset = 0;
-  smartlist_t *old_routers, *routers;
+  smartlist_t *signed_descriptors = NULL;
+  store_stats_t *stats =
+    extrainfo ? &extrainfo_store_stats : &router_store_stats;
+  const char *fname_base =
+    extrainfo ? "cached-extrainfo" : "cached-routers";
+  tor_mmap_t **mmap_ptr;
 
-  if (!force && !router_should_rebuild_store())
+  if (!force && !router_should_rebuild_store(stats))
     return 0;
   if (!routerlist)
     return 0;
 
+  mmap_ptr =
+    extrainfo ? &routerlist->mmap_extrainfo : &routerlist->mmap_descriptors;
+
   routerlist_check_bug_417();
 
   /* Don't save deadweight. */
   routerlist_remove_old_routers();
 
-  log_info(LD_DIR, "Rebuilding router descriptor cache");
+  log_info(LD_DIR, "Rebuilding %s cache",
+           extrainfo ? "Extra-info" : "router descriptor");
 
   options = get_options();
   fname_len = strlen(options->DataDirectory)+32;
   fname = tor_malloc(fname_len);
   fname_tmp = tor_malloc(fname_len);
-  tor_snprintf(fname, fname_len, "%s"PATH_SEPARATOR"cached-routers",
+  tor_snprintf(fname, fname_len, "%s"PATH_SEPARATOR"%s",
-               options->DataDirectory);
+               options->DataDirectory, fname_base);
-  tor_snprintf(fname_tmp, fname_len, "%s"PATH_SEPARATOR"cached-routers.tmp",
+  tor_snprintf(fname_tmp, fname_len, "%s"PATH_SEPARATOR"%s.tmp",
-               options->DataDirectory);
+               options->DataDirectory, fname_base);
 
   chunk_list = smartlist_create();
 
   /* We sort the routers by age to enhance locality on disk. */
-  old_routers = smartlist_create();
+  signed_descriptors = smartlist_create();
-  smartlist_add_all(old_routers, routerlist->old_routers);
+  if (extrainfo) {
-  smartlist_sort(old_routers, _compare_old_routers_by_age);
+    digestmap_iter_t *iter;
-  routers = smartlist_create();
+    for (iter = digestmap_iter_init(routerlist->extra_info_map);
-  smartlist_add_all(routers, routerlist->routers);
+         !digestmap_iter_done(iter);
-  smartlist_sort(routers, _compare_routers_by_age);
+         iter = digestmap_iter_next(routerlist->extra_info_map, iter)) {
-  for (i = 0; i < 2; ++i) {
+      const char *key;
-    smartlist_t *lst = (i == 0) ? old_routers : routers;
+      void *val;
-    /* Now, add the appropriate members to chunk_list */
+      extrainfo_t *ei;
-    SMARTLIST_FOREACH(lst, void *, ptr,
+      digestmap_iter_get(iter, &key, &val);
+      ei = val;
+      smartlist_add(signed_descriptors, &ei->cache_info);
+    }
+  } else {
+    smartlist_add_all(signed_descriptors, routerlist->old_routers);
+    SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri,
+                      smartlist_add(signed_descriptors, &ri->cache_info));
+  }
+
+  smartlist_sort(signed_descriptors, _compare_signed_descriptors_by_age);
+
+  /* Now, add the appropriate members to chunk_list */
+  SMARTLIST_FOREACH(signed_descriptors, signed_descriptor_t *, sd,
     {
-      signed_descriptor_t *sd = (i==0) ?
-        ((signed_descriptor_t*)ptr): &((routerinfo_t*)ptr)->cache_info;
       sized_chunk_t *c;
       const char *body = signed_descriptor_get_body(sd);
       if (!body) {
         log_warn(LD_BUG, "No descriptor available for router.");
         goto done;
       }
-      if (i==1 && ((routerinfo_t*)ptr)->purpose != ROUTER_PURPOSE_GENERAL)
+      if (sd->do_not_cache)
         continue;
       c = tor_malloc(sizeof(sized_chunk_t));
       c->bytes = body;
       c->len = sd->signed_descriptor_len;
       smartlist_add(chunk_list, c);
     });
-  }
+
   if (write_chunks_to_file(fname_tmp, chunk_list, 1)<0) {
     log_warn(LD_FS, "Error writing router store to disk.");
     goto done;
   }
+
   /* Our mmap is now invalid. */
-  if (routerlist->mmap_descriptors) {
+  if (*mmap_ptr) {
-    tor_munmap_file(routerlist->mmap_descriptors);
+    tor_munmap_file(*mmap_ptr);
+    *mmap_ptr = NULL;
   }
+
   if (replace_file(fname_tmp, fname)<0) {
     log_warn(LD_FS, "Error replacing old router store.");
     goto done;
   }
 
-  routerlist->mmap_descriptors = tor_mmap_file(fname);
+  *mmap_ptr = tor_mmap_file(fname);
-  if (! routerlist->mmap_descriptors)
+  if (! *mmap_ptr)
     log_warn(LD_FS, "Unable to mmap new descriptor file at '%s'.",fname);
 
   offset = 0;
-  for (i = 0; i < 2; ++i) {
+  SMARTLIST_FOREACH(signed_descriptors, signed_descriptor_t *, sd,
-    smartlist_t *lst = (i == 0) ? old_routers : routers;
-    SMARTLIST_FOREACH(lst, void *, ptr,
     {
-      signed_descriptor_t *sd = (i==0) ?
+      if (sd->do_not_cache)
-        ((signed_descriptor_t*)ptr): &((routerinfo_t*)ptr)->cache_info;
-
-      if (i==1 && ((routerinfo_t*)ptr)->purpose != ROUTER_PURPOSE_GENERAL)
         continue;
-
       sd->saved_location = SAVED_IN_CACHE;
-      if (routerlist->mmap_descriptors) {
+      if (*mmap_ptr) {
         tor_free(sd->signed_descriptor_body); // sets it to null
         sd->saved_offset = offset;
       }
       offset += sd->signed_descriptor_len;
-      signed_descriptor_get_body(sd);
+      signed_descriptor_get_body(sd); /* reconstruct and assert */
     });
-  }
 
-  tor_snprintf(fname, fname_len, "%s"PATH_SEPARATOR"cached-routers.new",
+  tor_snprintf(fname, fname_len, "%s"PATH_SEPARATOR"%s.new",
-               options->DataDirectory);
+               options->DataDirectory, fname_base);
 
   write_str_to_file(fname, "", 1);
 
   r = 0;
-  router_store_len = len;
+  stats->store_len = len;
-  router_journal_len = 0;
+  stats->journal_len = 0;
-  router_bytes_dropped = 0;
+  stats->bytes_dropped = 0;
  done:
-  smartlist_free(old_routers);
+  if (signed_descriptors)
-  smartlist_free(routers);
+    smartlist_free(signed_descriptors);
   tor_free(fname);
   SMARTLIST_FOREACH(chunk_list, sized_chunk_t *, c, tor_free(c));
   smartlist_free(chunk_list);
@@ -374,52 +400,65 @@ router_rebuild_store(int force)
   return r;
 }
 
-/** Load all cached router descriptors from the store. Return 0 on success and
+/** DOCDOC */
- * -1 on failure.
+static int
- */
+router_reload_router_list_impl(int extrainfo)
-int
-router_reload_router_list(void)
 {
   or_options_t *options = get_options();
   size_t fname_len = strlen(options->DataDirectory)+32;
   char *fname = tor_malloc(fname_len), *contents = NULL;
+  store_stats_t *stats =
+    extrainfo ? &extrainfo_store_stats : &router_store_stats;
+  const char *fname_base =
+    extrainfo ? "cached-extrainfo" : "cached-routers";
+  tor_mmap_t **mmap_ptr;
 
   routerlist_check_bug_417();
 
   if (!routerlist)
     router_get_routerlist(); /* mallocs and inits it in place */
 
-  router_journal_len = router_store_len = 0;
+  mmap_ptr =
+    extrainfo ? &routerlist->mmap_extrainfo : &routerlist->mmap_descriptors;
 
-  tor_snprintf(fname, fname_len, "%s"PATH_SEPARATOR"cached-routers",
+  router_store_stats.journal_len = router_store_stats.store_len = 0;
-               options->DataDirectory);
 
-  if (routerlist->mmap_descriptors) /* get rid of it first */
+  tor_snprintf(fname, fname_len, "%s"PATH_SEPARATOR"%s",
-    tor_munmap_file(routerlist->mmap_descriptors);
+               options->DataDirectory, fname_base);
 
-  routerlist->mmap_descriptors = tor_mmap_file(fname);
+  if (*mmap_ptr) /* get rid of it first */
-  if (routerlist->mmap_descriptors) {
+    tor_munmap_file(*mmap_ptr);
-    router_store_len = routerlist->mmap_descriptors->size;
+  *mmap_ptr = NULL;
-    router_load_routers_from_string(routerlist->mmap_descriptors->data,
+
-                                    SAVED_IN_CACHE, NULL);
+  *mmap_ptr = tor_mmap_file(fname);
+  if (*mmap_ptr) {
+    stats->store_len = (*mmap_ptr)->size;
+    if (extrainfo)
+      router_load_extrainfo_from_string((*mmap_ptr)->data,
+                                      SAVED_IN_CACHE);
+    else
+      router_load_routers_from_string((*mmap_ptr)->data,
+                                      SAVED_IN_CACHE, NULL);
   }
 
-  tor_snprintf(fname, fname_len, "%s"PATH_SEPARATOR"cached-routers.new",
+  tor_snprintf(fname, fname_len, "%s"PATH_SEPARATOR"%s.new",
-               options->DataDirectory);
+               options->DataDirectory, fname_base);
   if (file_status(fname) == FN_FILE)
     contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, NULL);
   if (contents) {
-    router_load_routers_from_string(contents,
+    if (extrainfo)
-                                    SAVED_IN_JOURNAL, NULL);
+      router_load_extrainfo_from_string(contents, SAVED_IN_JOURNAL);
+    else
+      router_load_routers_from_string(contents, SAVED_IN_JOURNAL, NULL);
     tor_free(contents);
   }
 
   tor_free(fname);
 
-  if (router_journal_len) {
+  if (stats->journal_len) {
     /* Always clear the journal on startup.*/
-    router_rebuild_store(1);
+    router_rebuild_store(1, extrainfo);
-  } else {
+  } else if (!extrainfo) {
     /* Don't cache expired routers. (This is in an else because
      * router_rebuild_store() also calls remove_old_routers().) */
     routerlist_remove_old_routers();
@@ -430,6 +469,19 @@ router_reload_router_list(void)
   return 0;
 }
 
+/** Load all cached router descriptors and extra-info documents from the
+ * store. Return 0 on success and -1 on failure.
+ */
+int
+router_reload_router_list(void)
+{
+  if (router_reload_router_list_impl(0))
+    return 1;
+  if (router_reload_router_list_impl(1))
+    return 1;
+  return 0;
+}
+
 /** Return a smartlist containing a list of trusted_dir_server_t * for all
  * known trusted dirservers.  Callers must not modify the list or its
  * contents.
@@ -1679,10 +1731,12 @@ routerlist_insert(routerlist_t *rl, routerinfo_t *ri)
   routerlist_check_bug_417();
 }
 
-/**DOCDOC*/
+/** DOCDOC
-static void
+ * Returns true if actually inserted. */
+static int
 extrainfo_insert(routerlist_t *rl, extrainfo_t *ei)
 {
+  int r = 0;
   routerinfo_t *ri = digestmap_get(rl->identity_map,
                                    ei->cache_info.identity_digest);
   extrainfo_t *ei_tmp;
@@ -1714,11 +1768,13 @@ extrainfo_insert(routerlist_t *rl, extrainfo_t *ei)
   ei_tmp = digestmap_set(rl->extra_info_map,
                          ei->cache_info.signed_descriptor_digest,
                          ei);
+  r = 1;
   if (ei_tmp)
     extrainfo_free(ei_tmp);
 
  done:
   routerlist_check_bug_417();
+  return r;
 }
 
 /** If we're a directory cache and routerlist <b>rl</b> doesn't have
@@ -1780,12 +1836,15 @@ routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int idx, int make_old)
     ri_tmp = digestmap_remove(rl->desc_digest_map,
                               ri->cache_info.signed_descriptor_digest);
     tor_assert(ri_tmp == ri);
-    router_bytes_dropped += ri->cache_info.signed_descriptor_len;
+    router_store_stats.bytes_dropped += ri->cache_info.signed_descriptor_len;
     routerinfo_free(ri);
     ei_tmp = digestmap_remove(rl->extra_info_map,
                               ri->cache_info.extra_info_digest);
-    if (ei_tmp)
+    if (ei_tmp) {
+      extrainfo_store_stats.bytes_dropped +=
+        ei_tmp->cache_info.signed_descriptor_len;
       extrainfo_free(ei_tmp);
+    }
   }
   // routerlist_assert_ok(rl);
   routerlist_check_bug_417();
@@ -1805,13 +1864,16 @@ routerlist_remove_old(routerlist_t *rl, signed_descriptor_t *sd, int idx)
   sd_tmp = digestmap_remove(rl->desc_digest_map,
                             sd->signed_descriptor_digest);
   tor_assert(sd_tmp == sd);
-  router_bytes_dropped += sd->signed_descriptor_len;
+  router_store_stats.bytes_dropped += sd->signed_descriptor_len;
   signed_descriptor_free(sd);
 
   ei_tmp = digestmap_remove(rl->extra_info_map,
                             sd->extra_info_digest);
-  if (ei_tmp)
+  if (ei_tmp) {
+    extrainfo_store_stats.bytes_dropped +=
+      ei_tmp->cache_info.signed_descriptor_len;
     extrainfo_free(ei_tmp);
+  }
 
   routerlist_check_bug_417();
   // routerlist_assert_ok(rl);
@@ -1871,6 +1933,8 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old,
       ei_tmp = digestmap_remove(rl->extra_info_map,
                                 ri_old->cache_info.extra_info_digest);
       if (ei_tmp) {
+        extrainfo_store_stats.bytes_dropped +=
+          ei_tmp->cache_info.signed_descriptor_len;
         extrainfo_free(ei_tmp);
       }
     }
@@ -2089,7 +2153,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg,
 
       /* Only journal this desc if we'll be serving it. */
       if (!from_cache && get_options()->DirPort)
-        router_append_to_journal(&router->cache_info, router->purpose);
+        signed_desc_append_to_journal(&router->cache_info, router->purpose);
       routerlist_insert_old(routerlist, router);
       return -1;
     }
@@ -2120,7 +2184,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg,
                 router->nickname);
       /* Only journal this desc if we'll be serving it. */
       if (!from_cache && get_options()->DirPort)
-        router_append_to_journal(&router->cache_info, router->purpose);
+        signed_desc_append_to_journal(&router->cache_info, router->purpose);
       routerlist_insert_old(routerlist, router);
       *msg = "Router descriptor was not new.";
       return -1;
@@ -2158,7 +2222,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg,
       }
       routerlist_replace(routerlist, old_router, router, pos, 1);
       if (!from_cache) {
-        router_append_to_journal(&router->cache_info, router->purpose);
+        signed_desc_append_to_journal(&router->cache_info, router->purpose);
       }
       directory_set_dirty();
       *msg = unreachable ? "Dirserver believes your ORPort is unreachable" :
@@ -2173,7 +2237,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg,
    * the list. */
   routerlist_insert(routerlist, router);
   if (!from_cache)
-    router_append_to_journal(&router->cache_info, router->purpose);
+    signed_desc_append_to_journal(&router->cache_info, router->purpose);
   directory_set_dirty();
   return 0;
 }
@@ -2183,11 +2247,14 @@ void
 router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg,
                                    int from_cache, int from_fetch)
 {
-  /* XXXX020 cache on disk */
+  int inserted;
-  (void)from_cache;
   (void)from_fetch;
   (void)msg;
-  extrainfo_insert(router_get_routerlist(), ei);
+
+  inserted = extrainfo_insert(router_get_routerlist(), ei);
+
+  if (inserted && !from_cache)
+    signed_desc_append_to_journal(&ei->cache_info, EXTRAINFO_PURPOSE_GENERAL);
 }
 
 /** Sorting helper: return &lt;0, 0, or &gt;0 depending on whether the
@@ -2440,6 +2507,8 @@ router_load_single_router(const char *s, uint8_t purpose, const char **msg)
     return -1;
   }
   ri->purpose = purpose;
+  if (purpose != ROUTER_PURPOSE_GENERAL)
+    ri->cache_info.do_not_cache = 1;
   if (router_is_me(ri)) {
     log_warn(LD_DIR, "Router's identity key matches mine; dropping.");
     *msg = "Router's identity key matches mine.";
@@ -2485,7 +2554,7 @@ router_load_routers_from_string(const char *s, saved_location_t saved_location,
   const char *msg;
   int from_cache = (saved_location != SAVED_NOWHERE);
 
-  router_parse_list_from_string(&s, routers, saved_location);
+  router_parse_list_from_string(&s, routers, saved_location, 0);
 
   routers_update_status_from_networkstatus(routers, !from_cache);
 
@@ -2519,12 +2588,34 @@ router_load_routers_from_string(const char *s, saved_location_t saved_location,
     control_event_descriptors_changed(changed);
 
   routerlist_assert_ok(routerlist);
-  router_rebuild_store(0);
+  router_rebuild_store(0, 0);
 
   smartlist_free(routers);
   smartlist_free(changed);
 }
 
+/** DOCDOC */
+static void
+router_load_extrainfo_from_string(const char *s,
+                                  saved_location_t saved_location)
+{
+  smartlist_t *extrainfo_list = smartlist_create();
+  const char *msg;
+  int from_cache = (saved_location != SAVED_NOWHERE);
+
+  router_parse_list_from_string(&s, extrainfo_list, saved_location, 1);
+
+  log_info(LD_DIR, "%d elements to add", smartlist_len(extrainfo_list));
+
+  SMARTLIST_FOREACH(extrainfo_list, extrainfo_t *, ei,
+      router_add_extrainfo_to_routerlist(ei, &msg, from_cache, !from_cache));
+
+  routerlist_assert_ok(routerlist);
+  router_rebuild_store(0, 1);
+
+  smartlist_free(extrainfo_list);
+}
+
 /** Helper: return a newly allocated string containing the name of the filename
  * where we plan to cache the network status with the given identity digest. */
 char *

src/or/routerparse.c

@@ -693,12 +693,18 @@ check_directory_signature(const char *digest,
  * isn't SAVED_NOWHERE, remember the offset of each descriptor.
  *
  * Returns 0 on success and -1 on failure.
+ *
+ * DOCDOC is_extrainfo
  */
 int
 router_parse_list_from_string(const char **s, smartlist_t *dest,
-                              saved_location_t saved_location)
+                              saved_location_t saved_location,
+                              int is_extrainfo)
 {
   routerinfo_t *router;
+  extrainfo_t *extrainfo;
+  signed_descriptor_t *signed_desc;
+  void *elt;
   const char *end, *cp, *start;
 
   tor_assert(s);
@@ -709,9 +715,17 @@ router_parse_list_from_string(const char **s, smartlist_t *dest,
   while (1) {
     *s = eat_whitespace(*s);
     /* Don't start parsing the rest of *s unless it contains a router. */
-    if (strcmpstart(*s, "router ")!=0)
+    if (is_extrainfo) {
-      break;
+      if (strcmpstart(*s, "extra-info")!=0)
-    if ((end = strstr(*s+1, "\nrouter "))) {
+        break;
+    } else {
+      if (strcmpstart(*s, "router ")!=0)
+        break;
+    }
+    if (is_extrainfo && (end = strstr(*s+1, "\nextra-info"))) {
+      cp = end;
+      end++;
+    } else if (!is_extrainfo && (end = strstr(*s+1, "\nrouter "))) {
       cp = end;
       end++;
     } else if ((end = strstr(*s+1, "\ndirectory-signature"))) {
@@ -736,20 +750,31 @@ router_parse_list_from_string(const char **s, smartlist_t *dest,
       continue;
     }
 
-    router = router_parse_entry_from_string(*s, end,
+    if (is_extrainfo) {
-                                            saved_location != SAVED_IN_CACHE);
+      routerlist_t *rl = router_get_routerlist();
+      extrainfo = extrainfo_parse_entry_from_string(*s, end,
+                                       saved_location != SAVED_IN_CACHE,
+                                       rl->identity_map);
+      signed_desc = &extrainfo->cache_info;
+      elt = extrainfo;
+    } else {
+      router = router_parse_entry_from_string(*s, end,
+                                          saved_location != SAVED_IN_CACHE);
+      signed_desc = &router->cache_info;
+      elt = router;
+    }
 
-    if (!router) {
+    if (!elt) {
       log_warn(LD_DIR, "Error reading router; skipping");
       *s = end;
       continue;
     }
     if (saved_location != SAVED_NOWHERE) {
-      router->cache_info.saved_location = saved_location;
+      signed_desc->saved_location = saved_location;
-      router->cache_info.saved_offset = *s - start;
+      signed_desc->saved_offset = *s - start;
     }
     *s = end;
-    smartlist_add(dest, router);
+    smartlist_add(dest, elt);
   }
 
   return 0;