r12981@Kushana: nickm | 2007-05-18 14:12:19 -0400

First cut at code to download extra-info docs.  Also note a bad bug in directory.c (look for the string BUG BUG BUG).


svn:r10209

ChangeLog

@@ -87,8 +87,7 @@ Changes in version 0.2.0.1-alpha - 2007-??-??
       routers.  These documents contain fields from router descriptors
       that aren't usually needed, and that use a lot of excess
       bandwidth. Once these fields are removed from router descriptors,
-      the bandwidth savings should be about 60%.  (Limitation: servers
+      the bandwidth savings should be about 60%. [Partially implements
-      do not yet upload extra-info documents.)  [Partially implements
       proposal 104.]
     - Directory authorities allow multiple router descriptors and/or extra
       info documents to be uploaded in a single go.  This will make
@@ -96,6 +95,10 @@ Changes in version 0.2.0.1-alpha - 2007-??-??
     - New config option V2AuthoritativeDirectory that all directory
       authorities should set. This will let future authorities choose
       not to serve V2 directory information.
+    - Servers upload extra-info documents to any authority that accepts
+      them.  Authorities (and caches that have been configured to download
+      extra-info documents) download them as needed. [Partially implements
+      proposal 104.]
 
   o Minor features (controller):
     - Add a new config option __DisablePredictedCircuits designed for

doc/TODO

@@ -92,7 +92,8 @@ Things we'd like to do in 0.2.0.x:
         version 0.2.0.0-alpha-dev (r10070) or later.
         o Implement, but make it option-controlled.
         o Make it always-on once it seems to work.
-      - Implement option to download and cache extra-info documents.
+      o Implement option to download and cache extra-info documents.
+      - Improve the 'retry' logic on extra-info documents.
       - Drop bandwidth history from router-descriptors
     - 105: Version negotiation for the Tor protocol (finalize by Jun 1)
     - 108: Base "Stable" Flag on Mean Time Between Failures

src/or/config.c

@@ -155,6 +155,7 @@ static config_var_t _option_vars[] = {
   VAR("DirPort",             UINT,     DirPort,              "0"),
   OBSOLETE("DirPostPeriod"),
   VAR("DirServer",           LINELIST, DirServers,           NULL),
+  VAR("DownloadExtraInfo",   BOOL,     DownloadExtraInfo,    "0"),
   VAR("EnforceDistinctSubnets", BOOL,  EnforceDistinctSubnets,"1"),
   VAR("EntryNodes",          STRING,   EntryNodes,           NULL),
   VAR("ExcludeNodes",        STRING,   ExcludeNodes,         NULL),
@@ -2555,6 +2556,11 @@ options_validate(or_options_t *old_options, or_options_t *options,
                "UseEntryGuards. Disabling.");
       options->UseEntryGuards = 0;
     }
+    if (!options->DownloadExtraInfo) {
+      log_info(LD_CONFIG, "Authoritative directories always try to download "
+               "extra-info documents. Setting DownloadExtraInfo.");
+      options->DownloadExtraInfo = 1;
+    }
   }
 
   if (options->AuthoritativeDir && !options->DirPort)

src/or/directory.c

@@ -45,7 +45,8 @@ static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
 static void dir_networkstatus_download_failed(smartlist_t *failed,
                                               int status_code);
 static void dir_routerdesc_download_failed(smartlist_t *failed,
-                                           int status_code);
+                                           int status_code,
+                                           int was_extrainfo);
 static void note_request(const char *key, size_t bytes);
 
 /********* START VARIABLES **********/
@@ -78,7 +79,8 @@ purpose_is_private(uint8_t purpose)
       purpose == DIR_PURPOSE_UPLOAD_DIR ||
       purpose == DIR_PURPOSE_FETCH_RUNNING_LIST ||
       purpose == DIR_PURPOSE_FETCH_NETWORKSTATUS ||
-      purpose == DIR_PURPOSE_FETCH_SERVERDESC)
+      purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+      purpose == DIR_PURPOSE_FETCH_EXTRAINFO)
     return 0;
   return 1;
 }
@@ -106,6 +108,28 @@ authority_type_to_string(authority_type_t auth)
   return result;
 }
 
+/* DOCDOC  */
+int
+router_supports_extrainfo(const char *identity_digest, int is_authority)
+{
+  routerinfo_t *ri = router_get_by_digest(identity_digest);
+  local_routerstatus_t *lrs;
+
+  if (ri) {
+    if (ri->caches_extra_info)
+      return 1;
+    if (is_authority && ri->platform &&
+        tor_version_as_new_as(ri->platform, "Tor 0.2.0.0-alpha-dev (r10070)"))
+      return 1;
+  }
+  if (is_authority) {
+    lrs = router_get_combined_status_by_digest(identity_digest);
+    if (lrs && lrs->status.version_supports_extrainfo_upload)
+      return 1;
+  }
+  return 0;
+}
+
 /** Start a connection to every suitable directory server, using
  * connection purpose 'purpose' and uploading the payload 'payload'
  * (length 'payload_len').  The purpose should be one of
@@ -131,9 +155,6 @@ directory_post_to_dirservers(uint8_t purpose, authority_type_t type,
   SMARTLIST_FOREACH(dirservers, trusted_dir_server_t *, ds,
     {
       routerstatus_t *rs = &(ds->fake_status.status);
-      local_routerstatus_t *lrs = router_get_combined_status_by_digest(
-                                                ds->digest);
-      int new_enough;
       size_t upload_len = payload_len;
 
       if ((type & ds->type) == 0)
@@ -143,10 +164,7 @@ directory_post_to_dirservers(uint8_t purpose, authority_type_t type,
       if (purpose == DIR_PURPOSE_UPLOAD_DIR)
         ds->has_accepted_serverdesc = 0;
 
-      new_enough = (lrs && lrs->status.version_supports_extrainfo_upload) ||
+      if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) {
-        (router_digest_version_as_new_as(ds->digest,
-                                         "Tor 0.2.0.0-alpha-dev (r10070)"));
-      if (extrainfo_len && new_enough) {
         upload_len += extrainfo_len;
         log_info(LD_DIR, "Uploading an extrainfo (length %d)",
                  (int) extrainfo_len);
@@ -182,6 +200,9 @@ directory_get_from_dirserver(uint8_t purpose, const char *resource,
   /* FFFF we could break this switch into its own function, and call
    * it elsewhere in directory.c. -RD */
   switch (purpose) {
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      type = EXTRAINFO_CACHE | V2_AUTHORITY;
+      break;
     case DIR_PURPOSE_FETCH_NETWORKSTATUS:
     case DIR_PURPOSE_FETCH_SERVERDESC:
       type = V2_AUTHORITY;
@@ -344,7 +365,8 @@ connection_dir_request_failed(dir_connection_t *conn)
     log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
              conn->_base.address);
     connection_dir_download_networkstatus_failed(conn, -1);
-  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC) {
+  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
     log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
              conn->_base.address);
     connection_dir_download_routerdesc_failed(conn);
@@ -389,7 +411,7 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn,
 }
 
 /** Called when an attempt to download one or more router descriptors
- * on connection <b>conn</b> failed.
+ * or extra-info documents on connection <b>conn</b> failed.
  */
 static void
 connection_dir_download_routerdesc_failed(dir_connection_t *conn)
@@ -399,6 +421,8 @@ connection_dir_download_routerdesc_failed(dir_connection_t *conn)
 
   /* No need to relaunch descriptor downloads here: we already do it
    * every 10 seconds (DESCRIPTOR_RETRY_INTERVAL) in main.c. */
+  tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
 
   (void) conn;
 }
@@ -452,6 +476,9 @@ directory_initiate_command(const char *address, uint32_t addr,
     case DIR_PURPOSE_FETCH_SERVERDESC:
       log_debug(LD_DIR,"initiating server descriptor fetch");
       break;
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      log_debug(LD_DIR,"initiating extra-info fetch");
+      break;
     default:
       log_err(LD_BUG, "Unrecognized directory connection purpose.");
       tor_assert(0);
@@ -611,6 +638,12 @@ directory_send_command(dir_connection_t *conn,
       url = tor_malloc(len);
       tor_snprintf(url, len, "/tor/server/%s", resource);
       break;
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      httpcommand = "GET";
+      len = strlen(resource)+32;
+      url = tor_malloc(len);
+      tor_snprintf(url, len, "/tor/extra/%s", resource);
+      break;
     case DIR_PURPOSE_UPLOAD_DIR:
       tor_assert(!resource);
       tor_assert(payload);
@@ -912,7 +945,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
   compress_method_t compression;
   int plausible;
   int skewed=0;
-  int allow_partial = conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC;
+  int allow_partial = (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+                       conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
   int was_compressed=0;
   time_t now = time(NULL);
 
@@ -1165,11 +1199,17 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
     }
   }
 
-  if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC) {
+  if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+      conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
+    int was_ei = conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
     smartlist_t *which = NULL;
     int n_asked_for = 0;
-    log_info(LD_DIR,"Received server info (size %d) from server '%s:%d'",
+    log_info(LD_DIR,"Received %s (size %d) from server '%s:%d'",
+             was_ei ? "server info" : "extra server info",
              (int)body_len, conn->_base.address, conn->_base.port);
+    if (was_ei)
+      note_request(was_compressed?"dl/extra.z":"dl/extra", orig_len);
+    else
       note_request(was_compressed?"dl/server.z":"dl/server", orig_len);
     if (conn->requested_resource &&
         !strcmpstart(conn->requested_resource,"d/")) {
@@ -1192,7 +1232,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
       if (!which) {
         connection_dir_download_routerdesc_failed(conn);
       } else {
-        dir_routerdesc_download_failed(which, status_code);
+        dir_routerdesc_download_failed(which, status_code, was_ei);
         SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
         smartlist_free(which);
       }
@@ -1207,15 +1247,19 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
     if (which || (conn->requested_resource &&
                   !strcmpstart(conn->requested_resource, "all"))) {
       /* as we learn from them, we remove them from 'which' */
+      if (was_ei) {
+        router_load_extrainfo_from_string(body, SAVED_NOWHERE, which);
+      } else {
         router_load_routers_from_string(body, SAVED_NOWHERE, which);
         directory_info_has_arrived(now, 0);
       }
+    }
     if (which) { /* mark remaining ones as failed */
       log_info(LD_DIR, "Received %d/%d routers requested from %s:%d",
                n_asked_for-smartlist_len(which), n_asked_for,
                conn->_base.address, (int)conn->_base.port);
       if (smartlist_len(which)) {
-        dir_routerdesc_download_failed(which, status_code);
+        dir_routerdesc_download_failed(which, status_code, was_ei);
       }
       SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
       smartlist_free(which);
@@ -2078,17 +2122,23 @@ dir_networkstatus_download_failed(smartlist_t *failed, int status_code)
 }
 
 /** Called when one or more routerdesc fetches have failed (with uppercase
- * fingerprints listed in <b>failed</b>). */
+ * fingerprints listed in <b>failed</b>).
+ *
+ * DOCDOC was_extrainfo */
 static void
-dir_routerdesc_download_failed(smartlist_t *failed, int status_code)
+dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
+                               int was_extrainfo)
 {
   char digest[DIGEST_LEN];
   local_routerstatus_t *rs;
   time_t now = time(NULL);
   int server = server_mode(get_options()) && get_options()->DirPort;
+  (void) was_extrainfo;
   SMARTLIST_FOREACH(failed, const char *, cp,
   {
     base16_decode(digest, DIGEST_LEN, cp, strlen(cp));
+    /* XXXX020 BUG BUG BUG. Fails miserably when requesting by desc digest
+     * rather than by identity digest. */
     rs = router_get_combined_status_by_digest(digest);
     if (!rs || rs->n_download_failures >= MAX_ROUTERDESC_DOWNLOAD_FAILURES)
       continue;

src/or/main.c

@@ -668,6 +668,11 @@ directory_info_has_arrived(time_t now, int from_cache)
         "build a circuit.");
     update_router_descriptor_downloads(now);
     return;
+  } else {
+    /* Don't even bother trying to get extrainfo until the rest of our
+     * directory info is up-to-date */
+    if (options->DownloadExtraInfo)
+      update_extrainfo_downloads(now);
   }
 
   if (server_mode(options) && !we_are_hibernating() && !from_cache &&
@@ -862,6 +867,7 @@ run_scheduled_events(time_t now)
     /* XXXX  Maybe we should do this every 10sec when not enough info,
      * and every 60sec when we have enough info -NM */
     update_router_descriptor_downloads(now);
+    update_extrainfo_downloads(now);
     time_to_try_getting_descriptors = now + DESCRIPTOR_RETRY_INTERVAL;
   }
 

src/or/or.h

@@ -360,14 +360,17 @@ typedef enum {
 /** A connection to a directory server: download one or more server
  * descriptors. */
 #define DIR_PURPOSE_FETCH_SERVERDESC 6
+/** A connection to a directory server: download one or more extra-info
+ * documents. */
+#define DIR_PURPOSE_FETCH_EXTRAINFO 7
 /** A connection to a directory server: upload a server descriptor. */
-#define DIR_PURPOSE_UPLOAD_DIR 7
+#define DIR_PURPOSE_UPLOAD_DIR 8
 /** A connection to a directory server: upload a rendezvous
  * descriptor. */
-#define DIR_PURPOSE_UPLOAD_RENDDESC 8
+#define DIR_PURPOSE_UPLOAD_RENDDESC 9
 /** Purpose for connection at a directory server. */
-#define DIR_PURPOSE_SERVER 9
+#define DIR_PURPOSE_SERVER 10
-#define _DIR_PURPOSE_MAX 9
+#define _DIR_PURPOSE_MAX 10
 
 #define _EXIT_PURPOSE_MIN 1
 /** This exit stream wants to do an ordinary connect. */
@@ -1068,6 +1071,8 @@ typedef struct signed_descriptor_t {
   off_t saved_offset;
   /* DOCDOC */
   unsigned int do_not_cache : 1;
+  /* DOCDOC; XXXX020 replace with something smarter. */
+  unsigned int tried_downloading_extrainfo : 1;
 } signed_descriptor_t;
 
 /** Information about another onion router in the network. */
@@ -1322,6 +1327,7 @@ typedef enum {
   V2_AUTHORITY      = 1 << 1,
   HIDSERV_AUTHORITY = 1 << 2,
   BRIDGE_AUTHORITY  = 1 << 3,
+  EXTRAINFO_CACHE   = 1 << 4,  /* not precisely an authority type. */
 } authority_type_t;
 
 #define CRYPT_PATH_MAGIC 0x70127012u
@@ -1920,6 +1926,10 @@ typedef struct {
  /** If true, we try resolving hostnames with weird characters. */
   int ServerDNSAllowNonRFC953Hostnames;
 
+  /** If true, we try to download extra-info documents (and we serve them,
+   * if we are a cache).  For authorities, this is always true. */
+  int DownloadExtraInfo;
+
 } or_options_t;
 
 /** Persistent state for an onion router, as saved to disk. */
@@ -2560,6 +2570,7 @@ int dir_split_resource_into_fingerprints(const char *resource,
                                     smartlist_t *fp_out, int *compresseed_out,
                                     int decode_hex, int sort_uniq);
 char *directory_dump_request_log(void);
+int router_supports_extrainfo(const char *identity_digest, int is_authority);
 
 /********************************* dirserv.c ***************************/
 
@@ -3125,6 +3136,10 @@ int router_load_single_router(const char *s, uint8_t purpose,
 void router_load_routers_from_string(const char *s,
                                      saved_location_t saved_location,
                                      smartlist_t *requested_fingerprints);
+void router_load_extrainfo_from_string(const char *s,
+                                       saved_location_t saved_location,
+                                       smartlist_t *requested_fps);
+
 typedef enum {
   NS_FROM_CACHE, NS_FROM_DIR_BY_FP, NS_FROM_DIR_ALL, NS_GENERATED
 } networkstatus_source_t;
@@ -3147,6 +3162,7 @@ local_routerstatus_t *router_get_combined_status_by_digest(const char *digest);
 routerstatus_t *routerstatus_get_by_hexdigest(const char *hexdigest);
 void update_networkstatus_downloads(time_t now);
 void update_router_descriptor_downloads(time_t now);
+void update_extrainfo_downloads(time_t now);
 void routers_update_all_from_networkstatus(time_t now);
 void routers_update_status_from_networkstatus(smartlist_t *routers,
                                               int reset_failures);

src/or/router.c

@@ -1341,7 +1341,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router,
                     "opt fingerprint %s\n"
                     "uptime %ld\n"
                     "bandwidth %d %d %d\n"
-                    "opt extra-info-digest %s\n"
+                    "opt extra-info-digest %s\n%s"
                     "onion-key\n%s"
                     "signing-key\n%s"
                     "%s%s%s",
@@ -1357,6 +1357,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router,
     (int) router->bandwidthburst,
     (int) router->bandwidthcapacity,
     extra_info_digest,
+    options->DownloadExtraInfo ? "opt caches-extra-info 1\n" : "",
     onion_pkey, identity_pkey,
     family_line, bandwidth_usage,
     we_are_hibernating() ? "opt hibernating 1\n" : "");

src/or/routerlist.c

@@ -41,8 +41,6 @@ static local_routerstatus_t *router_get_combined_status_by_nickname(
                                                 int warn_if_unnamed);
 static void update_router_have_minimum_dir_info(void);
 static void router_dir_info_changed(void);
-static void router_load_extrainfo_from_string(const char *s,
-                                              saved_location_t saved_location);
 
 /****************************************************************************/
 
@@ -438,7 +436,7 @@ router_reload_router_list_impl(int extrainfo)
     stats->store_len = (*mmap_ptr)->size;
     if (extrainfo)
       router_load_extrainfo_from_string((*mmap_ptr)->data,
-                                      SAVED_IN_CACHE);
+                                        SAVED_IN_CACHE, NULL);
     else
       router_load_routers_from_string((*mmap_ptr)->data,
                                       SAVED_IN_CACHE, NULL);
@@ -450,7 +448,7 @@ router_reload_router_list_impl(int extrainfo)
     contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, NULL);
   if (contents) {
     if (extrainfo)
-      router_load_extrainfo_from_string(contents, SAVED_IN_JOURNAL);
+      router_load_extrainfo_from_string(contents, SAVED_IN_JOURNAL, NULL);
     else
       router_load_routers_from_string(contents, SAVED_IN_JOURNAL, NULL);
     tor_free(contents);
@@ -644,6 +642,9 @@ router_pick_directory_server_impl(int requireother, int fascistfirewall,
     is_trusted = router_digest_is_trusted_dir(status->identity_digest);
     if ((type & V2_AUTHORITY) && !(status->is_v2_dir || is_trusted))
       continue;
+    if ((type & EXTRAINFO_CACHE) &&
+        !router_supports_extrainfo(status->identity_digest, 0))
+      continue;
     if (prefer_tunnel &&
         status->version_supports_begindir &&
         (!fascistfirewall ||
@@ -714,6 +715,9 @@ router_pick_trusteddirserver_impl(authority_type_t type,
       if (!d->is_running) continue;
       if ((type & d->type) == 0)
         continue;
+      if ((type & EXTRAINFO_CACHE) &&
+          !router_supports_extrainfo(d->digest, 1))
+        continue;
       if (requireother && me && router_digest_is_me(d->digest))
           continue;
       if (prefer_tunnel &&
@@ -2594,9 +2598,10 @@ router_load_routers_from_string(const char *s, saved_location_t saved_location,
 }
 
 /** DOCDOC */
-static void
+void
 router_load_extrainfo_from_string(const char *s,
-                                  saved_location_t saved_location)
+                                  saved_location_t saved_location,
+                                  smartlist_t *requested_fingerprints)
 {
   smartlist_t *extrainfo_list = smartlist_create();
   const char *msg;
@@ -2606,8 +2611,15 @@ router_load_extrainfo_from_string(const char *s,
 
   log_info(LD_DIR, "%d elements to add", smartlist_len(extrainfo_list));
 
-  SMARTLIST_FOREACH(extrainfo_list, extrainfo_t *, ei,
+  SMARTLIST_FOREACH(extrainfo_list, extrainfo_t *, ei, {
-      router_add_extrainfo_to_routerlist(ei, &msg, from_cache, !from_cache));
+      if (requested_fingerprints) {
+        char fp[HEX_DIGEST_LEN+1];
+        base16_encode(fp, sizeof(fp), ei->cache_info.signed_descriptor_digest,
+                      DIGEST_LEN);
+        smartlist_string_remove(requested_fingerprints, fp);
+      }
+      router_add_extrainfo_to_routerlist(ei, &msg, from_cache, !from_cache);
+    });
 
   routerlist_assert_ok(routerlist);
   router_rebuild_store(0, 1);
@@ -4015,15 +4027,17 @@ routers_update_status_from_networkstatus(smartlist_t *routers,
 }
 
 /** For every router descriptor we are currently downloading by descriptor
- * digest, set result[d] to 1. */
+ * digest, set result[d] to 1. DOCDOC extrainfo */
 static void
-list_pending_descriptor_downloads(digestmap_t *result)
+list_pending_descriptor_downloads(digestmap_t *result, int extrainfo)
 {
   const char *prefix = "d/";
   size_t p_len = strlen(prefix);
   int i, n_conns;
   connection_t **carray;
   smartlist_t *tmp = smartlist_create();
+  int purpose =
+    extrainfo ? DIR_PURPOSE_FETCH_EXTRAINFO : DIR_PURPOSE_FETCH_SERVERDESC;
 
   tor_assert(result);
   get_connection_array(&carray, &n_conns);
@@ -4031,7 +4045,7 @@ list_pending_descriptor_downloads(digestmap_t *result)
   for (i = 0; i < n_conns; ++i) {
     connection_t *conn = carray[i];
     if (conn->type == CONN_TYPE_DIR &&
-        conn->purpose == DIR_PURPOSE_FETCH_SERVERDESC &&
+        conn->purpose == purpose &&
         !conn->marked_for_close) {
       const char *resource = TO_DIR_CONN(conn)->requested_resource;
       if (!strcmpstart(resource, prefix))
@@ -4054,6 +4068,7 @@ list_pending_descriptor_downloads(digestmap_t *result)
  */
 static void
 initiate_descriptor_downloads(routerstatus_t *source,
+                              int purpose,
                               smartlist_t *digests,
                               int lo, int hi)
 {
@@ -4081,14 +4096,11 @@ initiate_descriptor_downloads(routerstatus_t *source,
 
   if (source) {
     /* We know which authority we want. */
-    directory_initiate_command_routerstatus(source,
+    directory_initiate_command_routerstatus(source, purpose,
-                                            DIR_PURPOSE_FETCH_SERVERDESC,
                                             0, /* not private */
                                             resource, NULL, 0);
   } else {
-    directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,
+    directory_get_from_dirserver(purpose, resource, 1);
-                                 resource,
-                                 1);
   }
   tor_free(resource);
 }
@@ -4133,7 +4145,7 @@ router_list_client_downloadable(void)
     return downloadable;
 
   downloading = digestmap_new();
-  list_pending_descriptor_downloads(downloading);
+  list_pending_descriptor_downloads(downloading, 0);
 
   routerstatus_list_update_from_networkstatus(now);
   SMARTLIST_FOREACH(routerstatus_list, local_routerstatus_t *, rs,
@@ -4277,7 +4289,8 @@ update_router_descriptor_client_downloads(time_t now)
              req_plural, n_downloadable, rtr_plural, n_per_request);
     smartlist_sort_digests(downloadable);
     for (i=0; i < n_downloadable; i += n_per_request) {
-      initiate_descriptor_downloads(NULL, downloadable, i, i+n_per_request);
+      initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_SERVERDESC,
+                                    downloadable, i, i+n_per_request);
     }
     last_routerdesc_download_attempted = now;
   }
@@ -4313,7 +4326,7 @@ update_router_descriptor_cache_downloads(time_t now)
 
   /* Set map[d]=1 for the digest of every descriptor that we are currently
    * downloading. */
-  list_pending_descriptor_downloads(map);
+  list_pending_descriptor_downloads(map, 0);
 
   /* For the digest of every descriptor that we don't have, and that we aren't
    * downloading, add d to downloadable[i] if the i'th networkstatus knows
@@ -4411,7 +4424,8 @@ update_router_descriptor_cache_downloads(time_t now)
     log_info(LD_DIR, "Requesting %d descriptors from authority \"%s\"",
              smartlist_len(dl), ds->nickname);
     for (j=0; j < smartlist_len(dl); j += MAX_DL_PER_REQUEST) {
-      initiate_descriptor_downloads(&(ds->fake_status.status), dl, j,
+      initiate_descriptor_downloads(&(ds->fake_status.status),
+                                    DIR_PURPOSE_FETCH_SERVERDESC, dl, j,
                                     j+MAX_DL_PER_REQUEST);
     }
   }
@@ -4437,6 +4451,63 @@ update_router_descriptor_downloads(time_t now)
   }
 }
 
+/** DOCDOC */
+static INLINE int
+should_download_extrainfo(signed_descriptor_t *sd,
+                          const routerlist_t *rl,
+                          const digestmap_t *pending)
+{
+  const char *d = sd->extra_info_digest;
+  /* XXXX020 Check for failures; keep a failure count.  Don't just
+   * do this dumb "try once and give up" thing. */
+  return (!sd->tried_downloading_extrainfo &&
+          !tor_digest_is_zero(d) &&
+          !digestmap_get(rl->extra_info_map, d) &&
+          !digestmap_get(pending, d));
+}
+
+/** DOCDOC */
+void
+update_extrainfo_downloads(time_t now)
+{
+  or_options_t *options = get_options();
+  routerlist_t *rl;
+  smartlist_t *wanted;
+  digestmap_t *pending;
+  int i;
+  (void) now;
+  if (! options->DownloadExtraInfo)
+    return;
+
+  pending = digestmap_new();
+  list_pending_descriptor_downloads(pending, 1);
+  rl = router_get_routerlist();
+  wanted = smartlist_create();
+  SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, {
+      if (should_download_extrainfo(&ri->cache_info, rl, pending)) {
+        smartlist_add(wanted, ri->cache_info.extra_info_digest);
+        ri->cache_info.tried_downloading_extrainfo = 1; /*XXXX020 be smarter.*/
+      }
+    });
+  if (options->DirPort) {
+    SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd, {
+        if (should_download_extrainfo(sd, rl, pending)) {
+          smartlist_add(wanted, sd->extra_info_digest);
+          sd->tried_downloading_extrainfo = 1; /*XXXX020 be smarter. */
+        }
+      });
+  }
+  digestmap_free(pending, NULL);
+
+  smartlist_shuffle(wanted);
+  for (i = 0; i < smartlist_len(wanted); i += MAX_DL_PER_REQUEST) {
+    initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_EXTRAINFO,
+                                  wanted, i, i + MAX_DL_PER_REQUEST);
+  }
+
+  smartlist_free(wanted);
+}
+
 /** Return the number of routerstatus_t in <b>entries</b> that we'd actually
  * use. */
 static int