Merge remote-tracking branch 'teor/feature15775-fallback-v9-squashed'

.gitignore

@@ -28,6 +28,11 @@ cscope.*
 # OSX junk
 *.dSYM
 .DS_Store
+# updateFallbackDirs.py temp files
+details-*.json
+uptime-*.json
+*.full_url
+*.last_modified
 
 # /
 /Makefile

changes/feature15775-fallback

@@ -0,0 +1,19 @@
+  o Major features (directory mirrors):
+    - Include an opt-in trial list of Default Fallback Directories in
+      add_default_fallback_dir_servers().
+      "Tor has included a feature to fetch the initial consensus from nodes
+       other than the authorities for a while now. We just haven't shipped a
+       list of alternate locations for clients to go to yet.
+       Reasons why we might want to ship tor with a list of additional places
+       where clients can find the consensus is that it makes authority
+       reachability and BW less important.
+       We want them to have been around and using their current key, address,
+       and port for a while now (120 days), and have been running, a guard,
+       and a v2 directory mirror for most of that time."
+      We exclude BadExits and tor versions that aren't recommended.
+      We include an IPv6 address for each FallbackDir (#8374).
+      (Tor might not use IPv6 fallbacks until #6027 is merged.)
+      The unit test ensures that we successfully load all included
+      default fallback directories.
+      Closes ticket #15775. Patch by "teor".
+      OnionOO script by "weasel", "teor", "gsathya", and "karsten".

scripts/maint/fallback.blacklist

@@ -0,0 +1,19 @@
+# updateFallbackDirs.py directory mirror blacklist
+#
+# Format:
+# [ IPv4[:DirPort] ] [ orport=<ORPort> ] [ id=<ID> ] ...
+#   [ ipv6=<IPv6>[:<IPv6 ORPort>] ]
+#
+# If a sufficiently specific group of attributes matches, the directory mirror
+# will be excluded: (each group is listed on its own line)
+# <IPv4>, <DirPort>
+# <IPv4>, <ORPort>
+# <ID>
+# <IPv6>, <DirPort>
+# <IPv6>, <IPv6 ORPort>
+# If DirPort and ORPort are not present, the entire IP address is blacklisted.
+# (The blacklist overrides the whitelist.)
+
+# If a relay operator doesn't want their relay to be a FallbackDir,
+# enter the following information here:
+# <IPv4>:<DirPort> orport=<ORPort> id=<ID> ipv6=<IPv6>:<IPv6 ORPort>

scripts/maint/fallback.whitelist

@@ -0,0 +1,13 @@
+# updateFallbackDirs.py directory mirror whitelist
+#
+# Format:
+# IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
+#
+# All attributes must match for the directory mirror to be included.
+# If the fallback has an ipv6 key, the whitelist line must also have
+# it, and vice versa, otherwise they don't match.
+# (The blacklist overrides the whitelist.)
+
+# If a relay operator wants their relay to be a FallbackDir,
+# enter the following information here:
+# <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]

src/or/config.c

@@ -960,6 +960,7 @@ add_default_fallback_dir_servers,(void))
 {
   int i;
   const char *fallback[] = {
+#include "fallback_dirs.inc"
     NULL
   };
   for (i=0; fallback[i]; i++) {

src/or/fallback_dirs.inc

@@ -0,0 +1 @@
+/* This list will be empty until opt-ins are finalised. */

src/or/include.am

@@ -155,6 +155,7 @@ ORHEADERS = \
 	src/or/dnsserv.h				\
 	src/or/eventdns_tor.h				\
 	src/or/ext_orport.h				\
+	src/or/fallback_dirs.inc			\
 	src/or/fp_pair.h				\
 	src/or/geoip.h					\
 	src/or/entrynodes.h				\

src/or/routerlist.c

@@ -1299,8 +1299,8 @@ router_get_fallback_dir_servers(void)
 /** Try to find a running dirserver that supports operations of <b>type</b>.
  *
  * If there are no running dirservers in our routerlist and the
- * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the authoritative ones
+ * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones
- * as running again, and pick one.
+ * (including authorities) as running again, and pick one.
  *
  * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include
  * dirservers that we can't reach.
@@ -1308,8 +1308,9 @@ router_get_fallback_dir_servers(void)
  * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself
  * (if we're a dirserver).
  *
- * Don't pick an authority if any non-authority is viable; try to avoid using
+ * Don't pick a fallback directory mirror if any non-fallback is viable;
- * servers that have returned 503 recently.
+ * (the fallback directory mirrors include the authorities)
+ * try to avoid using servers that have returned 503 recently.
  */
 const routerstatus_t *
 router_pick_directory_server(dirinfo_type_t type, int flags)
@@ -1336,7 +1337,7 @@ router_pick_directory_server(dirinfo_type_t type, int flags)
   log_info(LD_DIR,
            "No reachable router entries for dirservers. "
            "Trying them all again.");
-  /* mark all authdirservers as up again */
+  /* mark all fallback directory mirrors as up again */
   mark_all_dirservers_up(fallback_dir_servers);
   /* try again */
   choice = router_pick_directory_server_impl(type, flags, NULL);

src/test/test_config.c

@@ -1692,9 +1692,9 @@ test_config_adding_dir_servers(void *arg)
     );
 
   /* We need to know if add_default_fallback_dir_servers is called,
+   * whatever the size of the list in fallback_dirs.inc,
    * so we use a version of add_default_fallback_dir_servers that adds
-   * one known default fallback directory.
+   * one known default fallback directory. */
-   * There doesn't appear to be any need to test it unmocked. */
   MOCK(add_default_fallback_dir_servers,
        add_default_fallback_dir_servers_known_default);
 
@@ -3492,6 +3492,33 @@ test_config_use_multiple_directories(void *arg)
   tor_free(options);
 }
 
+static void
+test_config_default_fallback_dirs(void *arg)
+{
+  const char *fallback[] = {
+#include "../or/fallback_dirs.inc"
+    NULL
+  };
+
+  int n_included_fallback_dirs = 0;
+  int n_added_fallback_dirs = 0;
+
+  (void)arg;
+  clear_dir_servers();
+
+  while (fallback[n_included_fallback_dirs])
+    n_included_fallback_dirs++;
+
+  add_default_fallback_dir_servers();
+
+  n_added_fallback_dirs = smartlist_len(router_get_fallback_dir_servers());
+
+  tt_assert(n_included_fallback_dirs == n_added_fallback_dirs);
+
+  done:
+  clear_dir_servers();
+}
+
 #define CONFIG_TEST(name, flags)                          \
   { #name, test_config_ ## name, flags, NULL, NULL }
 
@@ -3503,6 +3530,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(adding_default_trusted_dir_servers, TT_FORK),
   CONFIG_TEST(adding_dir_servers, TT_FORK),
   CONFIG_TEST(default_dir_servers, TT_FORK),
+  CONFIG_TEST(default_fallback_dirs, 0),
   CONFIG_TEST(resolve_my_address, TT_FORK),
   CONFIG_TEST(addressmap, 0),
   CONFIG_TEST(parse_bridge_line, 0),