Merge branch 'bug28591_035_squashed'

changes/bug28591

@@ -0,0 +1,4 @@
+  o Minor bugfixes (client, bootstrap):
+    - When Tor's clock is behind the clocks on the authorities, allow Tor to
+      bootstrap successfully. Fixes bug 28591; bugfix on 0.2.0.9-alpha.
+

changes/bug28654

@@ -0,0 +1,3 @@
+  o Minor bugfixes (directory mirror):
+    - When Tor's clock is behind the clocks on the authorities, allow Tor to
+      serve future consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha.

src/core/mainloop/mainloop.c

@@ -2166,11 +2166,9 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options)
   (void)options;
   /* Check whether our networkstatus has expired. */
   networkstatus_t *ns = networkstatus_get_latest_consensus();
-  /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in
-   * networkstatus_get_reasonably_live_consensus(), but that value is way
-   * way too high.  Arma: is the bridge issue there resolved yet? -NM */
-#define NS_EXPIRY_SLOP (24*60*60)
-  if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) &&
+  /* Use reasonably live consensuses until they are no longer reasonably live.
+   */
+  if (ns && !networkstatus_consensus_reasonably_live(ns, now) &&
       router_have_minimum_dir_info()) {
     router_dir_info_changed();
   }

src/feature/dircache/dircache.c

@@ -489,28 +489,47 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
 }
 
 /** Warn that the cached consensus <b>consensus</b> of type
- * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
- * warning to avoid logging an entry on every request.
+ * <b>flavor</b> too new or too old, based on <b>is_too_new</b>,
+ * and will not be served to clients. Rate-limit the warning to avoid logging
+ * an entry on every request.
  */
 static void
-warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
-                          const char *flavor, time_t now)
+warn_consensus_is_not_reasonably_live(
+                          const struct consensus_cache_entry_t *consensus,
+                          const char *flavor, time_t now, bool is_too_new)
 {
-#define TOO_OLD_WARNING_INTERVAL (60*60)
-  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+#define NOT_REASONABLY_LIVE_WARNING_INTERVAL (60*60)
+  static ratelim_t warned[2] = { RATELIM_INIT(
+                                      NOT_REASONABLY_LIVE_WARNING_INTERVAL),
+                                RATELIM_INIT(
+                                      NOT_REASONABLY_LIVE_WARNING_INTERVAL) };
   char timestamp[ISO_TIME_LEN+1];
-  time_t valid_until;
-  char *dupes;
+  /* valid_after if is_too_new, valid_until if !is_too_new */
+  time_t valid_time = 0;
+  char *dupes = NULL;
 
-  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
-    return;
+  if (is_too_new) {
+    if (consensus_cache_entry_get_valid_after(consensus, &valid_time))
+      return;
+    dupes = rate_limit_log(&warned[1], now);
+  } else {
+    if (consensus_cache_entry_get_valid_until(consensus, &valid_time))
+      return;
+    dupes = rate_limit_log(&warned[0], now);
+  }
 
-  if ((dupes = rate_limit_log(&warned, now))) {
-    format_local_iso_time(timestamp, valid_until);
-    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
-             "serve it to clients. It was valid until %s local time and we "
-             "continued to serve it for up to 24 hours after it expired.%s",
-             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+  if (dupes) {
+    format_local_iso_time(timestamp, valid_time);
+    log_warn(LD_DIRSERV, "Our %s%sconsensus is too %s, so we will not "
+             "serve it to clients. It was valid %s %s local time and we "
+             "continued to serve it for up to 24 hours %s.%s",
+             flavor ? flavor : "",
+             flavor ? " " : "",
+             is_too_new ? "new" : "old",
+             is_too_new ? "after" : "until",
+             timestamp,
+             is_too_new ? "before it was valid" : "after it expired",
+             dupes);
     tor_free(dupes);
   }
 }
@@ -853,7 +872,6 @@ handle_get_current_consensus(dir_connection_t *conn,
 
   if (req.diff_only && !cached_consensus) {
     write_short_http_response(conn, 404, "No such diff available");
-    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
     geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
     goto done;
   }
@@ -864,19 +882,30 @@ handle_get_current_consensus(dir_connection_t *conn,
                                            &compression_used);
   }
 
-  time_t fresh_until, valid_until;
-  int have_fresh_until = 0, have_valid_until = 0;
+  time_t valid_after, fresh_until, valid_until;
+  int have_valid_after = 0, have_fresh_until = 0, have_valid_until = 0;
   if (cached_consensus) {
+    have_valid_after =
+      !consensus_cache_entry_get_valid_after(cached_consensus, &valid_after);
     have_fresh_until =
       !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
     have_valid_until =
       !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
   }
 
-  if (cached_consensus && have_valid_until &&
+  if (cached_consensus && have_valid_after &&
+      !networkstatus_valid_after_is_reasonably_live(valid_after, now)) {
+    write_short_http_response(conn, 404, "Consensus is too new");
+    warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
+                                          1);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  } else if (
+      cached_consensus && have_valid_until &&
       !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
     write_short_http_response(conn, 404, "Consensus is too old");
-    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+    warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
+                                          0);
     geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
     goto done;
   }

src/feature/nodelist/microdesc.c

@@ -111,8 +111,9 @@ microdesc_note_outdated_dirserver(const char *relay_digest)
 
   /* If we have a reasonably live consensus, then most of our dirservers should
    * still be caching all the microdescriptors in it. Reasonably live
-   * consensuses are up to a day old. But microdescriptors expire 7 days after
-   * the last consensus that referenced them. */
+   * consensuses are up to a day old (or a day in the future). But
+   * microdescriptors expire 7 days after the last consensus that referenced
+   * them. */
   if (!networkstatus_get_reasonably_live_consensus(approx_time(),
                                                    FLAV_MICRODESC)) {
     return;
@@ -545,8 +546,8 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
   size_t bytes_dropped = 0;
   time_t now = time(NULL);
 
-  /* If we don't know a live consensus, don't believe last_listed values: we
-   * might be starting up after being down for a while. */
+  /* If we don't know a reasonably live consensus, don't believe last_listed
+   * values: we might be starting up after being down for a while. */
   if (! force &&
       ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
       return;
@@ -971,6 +972,7 @@ update_microdesc_downloads(time_t now)
   if (directory_too_idle_to_fetch_descriptors(options, now))
     return;
 
+  /* Give up if we don't have a reasonably live consensus. */
   consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
   if (!consensus)
     return;

src/feature/nodelist/networkstatus.c

@@ -1388,7 +1388,7 @@ networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor))
 }
 
 /** Return the most recent consensus that we have downloaded, or NULL if we
- * don't have one. */
+ * don't have one. May return future or expired consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_latest_consensus,(void))
 {
@@ -1399,7 +1399,7 @@ networkstatus_get_latest_consensus,(void))
 }
 
 /** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL
- * if we don't have one. */
+ * if we don't have one. May return future or expired consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f))
 {
@@ -1433,10 +1433,11 @@ networkstatus_is_live(const networkstatus_t *ns, time_t now)
   return (ns->valid_after <= now && now <= ns->valid_until);
 }
 
-/** Determine if <b>consensus</b> is valid or expired recently enough that
- * we can still use it.
+/** Determine if <b>consensus</b> is valid, or expired recently enough, or not
+ * too far in the future, so that we can still use it.
  *
- * Return 1 if the consensus is reasonably live, or 0 if it is too old.
+ * Return 1 if the consensus is reasonably live, or 0 if it is too old or
+ * too new.
  */
 int
 networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
@@ -1445,29 +1446,42 @@ networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
   if (BUG(!consensus))
     return 0;
 
-  return networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
+  return networkstatus_valid_after_is_reasonably_live(consensus->valid_after,
+                                                      now) &&
+         networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
                                                       now);
 }
 
+#define REASONABLY_LIVE_TIME (24*60*60)
+
+/** As networkstatus_consensus_reasonably_live, but takes a valid_after
+ * time, and checks to see if it is in the past, or not too far in the future.
+ */
+int
+networkstatus_valid_after_is_reasonably_live(time_t valid_after,
+                                             time_t now)
+{
+  return (now >= valid_after - REASONABLY_LIVE_TIME);
+}
+
 /** As networkstatus_consensus_reasonably_live, but takes a valid_until
- * time rather than an entire consensus. */
+ * time, and checks to see if it is in the future, or not too far in the past.
+ */
 int
 networkstatus_valid_until_is_reasonably_live(time_t valid_until,
                                              time_t now)
 {
-#define REASONABLY_LIVE_TIME (24*60*60)
   return (now <= valid_until + REASONABLY_LIVE_TIME);
 }
 
 /** As networkstatus_get_live_consensus(), but is way more tolerant of expired
- * consensuses. */
+ *  and future consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_reasonably_live_consensus,(time_t now, int flavor))
 {
   networkstatus_t *consensus =
     networkstatus_get_latest_consensus_by_flavor(flavor);
   if (consensus &&
-      consensus->valid_after <= now &&
       networkstatus_consensus_reasonably_live(consensus, now))
     return consensus;
   else
@@ -2090,7 +2104,6 @@ networkstatus_set_current_consensus(const char *consensus,
 
     nodelist_set_consensus(c);
 
-    /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/
     update_consensus_networkstatus_fetch_time(now);
 
     /* Change the cell EWMA settings */

src/feature/nodelist/networkstatus.h

@@ -88,6 +88,8 @@ MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now));
 int networkstatus_is_live(const networkstatus_t *ns, time_t now);
 int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
                                             time_t now);
+int networkstatus_valid_after_is_reasonably_live(time_t valid_after,
+                                                 time_t now);
 int networkstatus_valid_until_is_reasonably_live(time_t valid_until,
                                                  time_t now);
 MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus,

src/test/test_entrynodes.c

@@ -127,6 +127,9 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
   return 1; /* NOP */
 }
 
+#define REASONABLY_FUTURE " reasonably-future"
+#define REASONABLY_PAST " reasonably-past"
+
 /* Unittest setup function: Setup a fake network. */
 static void *
 big_fake_network_setup(const struct testcase_t *testcase)
@@ -138,9 +141,10 @@ big_fake_network_setup(const struct testcase_t *testcase)
   const int N_NODES = 271;
 
   const char *argument = testcase->setup_data;
-  int reasonably_live_consensus = 0;
+  int reasonably_future_consensus = 0, reasonably_past_consensus = 0;
   if (argument) {
-    reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL;
+    reasonably_future_consensus = strstr(argument, REASONABLY_FUTURE) != NULL;
+    reasonably_past_consensus = strstr(argument, REASONABLY_PAST) != NULL;
   }
 
   big_fake_net_nodes = smartlist_new();
@@ -198,11 +202,15 @@ big_fake_network_setup(const struct testcase_t *testcase)
 
   dummy_state = tor_malloc_zero(sizeof(or_state_t));
   dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t));
-  if (reasonably_live_consensus) {
-    /* Make the dummy consensus valid from 4 hours ago, but expired an hour
+  if (reasonably_future_consensus) {
+    /* Make the dummy consensus valid in 6 hours, and expiring in 7 hours. */
+    dummy_consensus->valid_after = approx_time() + 6*3600;
+    dummy_consensus->valid_until = approx_time() + 7*3600;
+  } else if (reasonably_past_consensus) {
+    /* Make the dummy consensus valid from 16 hours ago, but expired 12 hours
      * ago. */
-    dummy_consensus->valid_after = approx_time() - 4*3600;
-    dummy_consensus->valid_until = approx_time() - 3600;
+    dummy_consensus->valid_after = approx_time() - 16*3600;
+    dummy_consensus->valid_until = approx_time() - 12*3600;
   } else {
     /* Make the dummy consensus valid for an hour either side of now. */
     dummy_consensus->valid_after = approx_time() - 3600;
@@ -3038,13 +3046,17 @@ static const struct testcase_setup_t upgrade_circuits = {
 
 #define BFN_TEST(name) \
   EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \
-  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
-    &big_fake_network, (void*)("reasonably-live") }
+  { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)(REASONABLY_FUTURE) }, \
+  { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)(REASONABLY_PAST) }
 
 #define UPGRADE_TEST(name, arg) \
   EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \
-  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
-    &upgrade_circuits, (void*)(arg " reasonably-live") }
+  { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg REASONABLY_FUTURE) }, \
+  { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg REASONABLY_PAST) }
 
 struct testcase_t entrynodes_tests[] = {
   NO_PREFIX_TEST(node_preferred_orport),

src/test/test_routerlist.c

@@ -305,7 +305,6 @@ test_router_pick_directory_server_impl(void *arg)
   tt_assert(!networkstatus_consensus_is_bootstrapping(con_md->valid_until
                                                       + 24*60*60));
   /* These times are outside the test validity period */
-  tt_assert(networkstatus_consensus_is_bootstrapping(now));
   tt_assert(networkstatus_consensus_is_bootstrapping(now + 2*24*60*60));
   tt_assert(networkstatus_consensus_is_bootstrapping(now - 2*24*60*60));