Merge branch 'bug3327_squashed'

changes/bug3327

@@ -0,0 +1,14 @@
+  o Major features:
+    - Relays now try regenerating and uploading their descriptor more
+      frequently if they are not listed in the consensus, or if the
+      version of their descriptor listed in the consensus is too
+      old. This fix should prevent situations where a server declines
+      to re-publish itself because it has done so too recently, even
+      though the authorities decided not to list its recent-enough
+      descriptor. Fix for bug 3327.
+
+  o Minor features:
+    - Relays now include a reason for regenerating their descriptors
+      an HTTP header when uploading to the authorities.  This will
+      make it easier to debug descriptor-upload issues in the future.
+

src/or/directory.c

@@ -1117,11 +1117,11 @@ directory_send_command(dir_connection_t *conn,
                        time_t if_modified_since)
 {
   char proxystring[256];
-  char proxyauthstring[256];
   char hoststring[128];
-  char imsstring[RFC1123_TIME_LEN+32];
+  smartlist_t *headers = smartlist_create();
   char *url;
   char request[8192];
+  char *header;
   const char *httpcommand = NULL;
   size_t len;
 
@@ -1141,12 +1141,11 @@ directory_send_command(dir_connection_t *conn,
   }
 
   /* Format if-modified-since */
-  if (!if_modified_since) {
-    imsstring[0] = '\0';
-  } else {
+  if (if_modified_since) {
     char b[RFC1123_TIME_LEN+1];
     format_rfc1123_time(b, if_modified_since);
-    tor_snprintf(imsstring, sizeof(imsstring), "\r\nIf-Modified-Since: %s", b);
+    tor_asprintf(&header, "If-Modified-Since: %s\r\n", b);
+    smartlist_add(headers, header);
   }
 
   /* come up with some proxy lines, if we're using one. */
@@ -1161,16 +1160,14 @@ directory_send_command(dir_connection_t *conn,
         log_warn(LD_BUG, "Encoding http authenticator failed");
     }
     if (base64_authenticator) {
-      tor_snprintf(proxyauthstring, sizeof(proxyauthstring),
-                   "\r\nProxy-Authorization: Basic %s",
+      tor_asprintf(&header,
+                   "Proxy-Authorization: Basic %s\r\n",
                    base64_authenticator);
       tor_free(base64_authenticator);
-    } else {
-      proxyauthstring[0] = 0;
+      smartlist_add(headers, header);
     }
   } else {
     proxystring[0] = 0;
-    proxyauthstring[0] = 0;
   }
 
   switch (purpose) {
@@ -1231,12 +1228,18 @@ directory_send_command(dir_connection_t *conn,
       httpcommand = "GET";
       tor_asprintf(&url, "/tor/micro/%s", resource);
       break;
-    case DIR_PURPOSE_UPLOAD_DIR:
+    case DIR_PURPOSE_UPLOAD_DIR: {
+      const char *why = router_get_descriptor_gen_reason();
       tor_assert(!resource);
       tor_assert(payload);
       httpcommand = "POST";
       url = tor_strdup("/tor/");
+      if (why) {
+        tor_asprintf(&header, "X-Desc-Gen-Reason: %s\r\n", why);
+        smartlist_add(headers, header);
+      }
       break;
+    }
     case DIR_PURPOSE_UPLOAD_VOTE:
       tor_assert(!resource);
       tor_assert(payload);
@@ -1287,26 +1290,26 @@ directory_send_command(dir_connection_t *conn,
   connection_write_to_buf(url, strlen(url), TO_CONN(conn));
   tor_free(url);
 
-  if (!strcmp(httpcommand, "GET") && !payload) {
-    tor_snprintf(request, sizeof(request),
-                 " HTTP/1.0\r\nHost: %s%s%s\r\n\r\n",
-                 hoststring,
-                 imsstring,
-                 proxyauthstring);
-  } else {
-    tor_snprintf(request, sizeof(request),
-                 " HTTP/1.0\r\nContent-Length: %lu\r\nHost: %s%s%s\r\n\r\n",
-                 payload ? (unsigned long)payload_len : 0,
-                 hoststring,
-                 imsstring,
-                 proxyauthstring);
+  if (!strcmp(httpcommand, "POST") || payload) {
+    tor_asprintf(&header, "Content-Length: %lu\r\n",
+                 payload ? (unsigned long)payload_len : 0);
+    smartlist_add(headers, header);
   }
+
+  header = smartlist_join_strings(headers, "", 0, NULL);
+  tor_snprintf(request, sizeof(request), " HTTP/1.0\r\nHost: %s\r\n%s\r\n",
+               hoststring, header);
+  tor_free(header);
+
   connection_write_to_buf(request, strlen(request), TO_CONN(conn));
 
   if (payload) {
     /* then send the payload afterwards too */
     connection_write_to_buf(payload, payload_len, TO_CONN(conn));
   }
+
+  SMARTLIST_FOREACH(headers, char *, h, tor_free(h));
+  smartlist_free(headers);
 }
 
 /** Parse an HTTP request string <b>headers</b> of the form

src/or/main.c

@@ -1326,11 +1326,7 @@ run_scheduled_events(time_t now)
       time_to_check_ipaddress = now + CHECK_IPADDRESS_INTERVAL;
       check_descriptor_ipaddress_changed(now);
     }
-/** If our router descriptor ever goes this long without being regenerated
- * because something changed, we force an immediate regenerate-and-upload. */
-#define FORCE_REGENERATE_DESCRIPTOR_INTERVAL (18*60*60)
-    mark_my_descriptor_dirty_if_older_than(
-                                  now - FORCE_REGENERATE_DESCRIPTOR_INTERVAL);
+    mark_my_descriptor_dirty_if_too_old(now);
     consider_publishable_server(0);
     /* also, check religiously for reachability, if it's within the first
      * 20 minutes of our uptime. */

src/or/router.c

@@ -1220,9 +1220,14 @@ router_get_advertised_dir_port(const or_options_t *options, uint16_t dirport)
 static routerinfo_t *desc_routerinfo = NULL;
 /** My extrainfo */
 static extrainfo_t *desc_extrainfo = NULL;
+/** Why did we most recently decide to regenerate our descriptor?  Used to
+ * tell the authorities why we're sending it to them. */
+static const char *desc_gen_reason = NULL;
 /** Since when has our descriptor been "clean"?  0 if we need to regenerate it
  * now. */
 static time_t desc_clean_since = 0;
+/** Why did we mark the descriptor dirty? */
+static const char *desc_dirty_reason = NULL;
 /** Boolean: do we need to regenerate the above? */
 static int desc_needs_upload = 0;
 
@@ -1389,6 +1394,14 @@ router_get_my_extrainfo(void)
   return desc_extrainfo;
 }
 
+/** Return a human-readable string describing what triggered us to generate
+ * our current descriptor, or NULL if we don't know. */
+const char *
+router_get_descriptor_gen_reason(void)
+{
+  return desc_gen_reason;
+}
+
 /** A list of nicknames that we've warned about including in our family
  * declaration verbatim rather than as digests. */
 static smartlist_t *warned_nonexistent_family = NULL;
@@ -1606,16 +1619,56 @@ router_rebuild_descriptor(int force)
 
   desc_clean_since = time(NULL);
   desc_needs_upload = 1;
+  desc_gen_reason = desc_dirty_reason;
+  desc_dirty_reason = NULL;
   control_event_my_descriptor_changed();
   return 0;
 }
 
-/** Mark descriptor out of date if it's older than <b>when</b> */
+/** If our router descriptor ever goes this long without being regenerated
+ * because something changed, we force an immediate regenerate-and-upload. */
+#define FORCE_REGENERATE_DESCRIPTOR_INTERVAL (18*60*60)
+
+/** If our router descriptor seems to be missing or unacceptable according
+ * to the authorities, regenerate and reupload it _this_ often. */
+#define FAST_RETRY_DESCRIPTOR_INTERVAL (90*60)
+
+/** Mark descriptor out of date if it's been "too long" since we last tried
+ * to upload one. */
 void
-mark_my_descriptor_dirty_if_older_than(time_t when)
+mark_my_descriptor_dirty_if_too_old(time_t now)
 {
-  if (desc_clean_since < when)
+  networkstatus_t *ns;
+  routerstatus_t *rs;
+  const char *retry_fast_reason = NULL; /* Set if we should retry frequently */
+  const time_t slow_cutoff = now - FORCE_REGENERATE_DESCRIPTOR_INTERVAL;
+  const time_t fast_cutoff = now - FAST_RETRY_DESCRIPTOR_INTERVAL;
+
+  /* If it's already dirty, don't mark it. */
+  if (! desc_clean_since)
+    return;
+
+  /* If it's older than FORCE_REGENERATE_DESCRIPTOR_INTERVAL, it's always
+   * time to rebuild it. */
+  if (desc_clean_since < slow_cutoff) {
     mark_my_descriptor_dirty("time for new descriptor");
+    return;
+  }
+  /* Now we see whether we want to be retrying frequently or no.  The
+   * rule here is that we'll retry frequently if we aren't listed in the
+   * live consensus we have, or if the publication time of the
+   * descriptor listed for us in the consensus is very old. */
+  ns = networkstatus_get_live_consensus(now);
+  if (ns) {
+    rs = networkstatus_vote_find_entry(ns, server_identitykey_digest);
+    if (rs == NULL)
+      retry_fast_reason = "not listed in consensus";
+    else if (rs->published_on < slow_cutoff)
+      retry_fast_reason = "version listed in consensus is quite old";
+  }
+
+  if (retry_fast_reason && desc_clean_since < fast_cutoff)
+    mark_my_descriptor_dirty(retry_fast_reason);
 }
 
 /** Call when the current descriptor is out of date. */
@@ -1624,6 +1677,8 @@ mark_my_descriptor_dirty(const char *reason)
 {
   desc_clean_since = 0;
   log_info(LD_OR, "Decided to publish new relay descriptor: %s", reason);
+  if (!desc_dirty_reason)
+    desc_dirty_reason = reason;
 }
 
 /** How frequently will we republish our descriptor because of large (factor

src/or/router.h

@@ -62,7 +62,7 @@ void consider_publishable_server(int force);
 int should_refuse_unknown_exits(const or_options_t *options);
 
 void router_upload_dir_desc_to_dirservers(int force);
-void mark_my_descriptor_dirty_if_older_than(time_t when);
+void mark_my_descriptor_dirty_if_too_old(time_t now);
 void mark_my_descriptor_dirty(const char *reason);
 void check_descriptor_bandwidth_changed(time_t now);
 void check_descriptor_ipaddress_changed(time_t now);
@@ -73,6 +73,7 @@ int router_my_exit_policy_is_reject_star(void);
 const routerinfo_t *router_get_my_routerinfo(void);
 extrainfo_t *router_get_my_extrainfo(void);
 const char *router_get_my_descriptor(void);
+const char *router_get_descriptor_gen_reason(void);
 int router_digest_is_me(const char *digest);
 int router_extrainfo_digest_is_me(const char *digest);
 int router_is_me(const routerinfo_t *router);