Merge branch 'maint-0.2.6' into maint-0.2.7-redux

maint-0.2.7-redux is an attempt to try to re-create a plausible
maint-0.2.7 branch.  I've started from the tor-0.2.7.6, and then I
merged maint-0.2.6 into the branch.

This has produced 2 conflicts: one related to the
rendcommon->rendcache move, and one to the authority refactoring.

changes/19271

@@ -0,0 +1,2 @@
+  o Directory authority changes:
+    - Urras is no longer a directory authority. Closes ticket 19271.

changes/bifroest

@@ -0,0 +1,3 @@
+  o Directory authority changes (also in 0.2.8.7):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.

changes/buf-sentinel

@@ -0,0 +1,11 @@
+  o Major features (security fixes):
+
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string.  At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket 20384
+      (TROVE-2016-10-001).
+

changes/bug17906

@@ -0,0 +1,4 @@
+  o Minor features (authorities):
+    - Update the V3 identity key for dannenberg, it was changed on
+      18 November 2015.
+      Closes task #17906. Patch by "teor".

changes/bug18089

@@ -0,0 +1,6 @@
+  o Minor fixes (security):
+    - Make memwipe() do nothing when passed a NULL pointer
+      or zero size. Check size argument to memwipe() for underflow.
+      Closes bug #18089. Reported by "gk", patch by "teor".
+      Bugfix on 0.2.3.25 and 0.2.4.6-alpha (#7352),
+      commit 49dd5ef3 on 7 Nov 2012.

changes/bug18162

@@ -0,0 +1,7 @@
+  o Major bugfixes (security, pointers):
+
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug #18162;
+      bugfix on Tor 0.1.1.11-alpha, which fixed a related bug
+      incompletely. Reported by Guido Vranken.
+

changes/bug21018

@@ -0,0 +1,11 @@
+  o Major bugfixes (parsing, security):
+
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be
+      used to cause hardened clients (built with
+      --enable-expensive-hardening) to crash if they tried to visit
+      a hostile hidden service.  Non-hardened clients are only
+      affected depending on the details of their platform's memory
+      allocator. Fixes bug 21018; bugfix on 0.2.0.8-alpha. Found by
+      using libFuzzer. Also tracked as TROVE-2016-12-002 and as
+      CVE-2016-1254.

changes/geoip-april2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the April 5 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-august2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the August 2 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-december2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-february2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the February 2 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-january2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the January 5 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-january2017

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+

changes/geoip-july2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the July 6 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-jun2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the June 7 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-march2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the March 3 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-may2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the May 4 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-november2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-october2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the October 4 2016 Maxmind GeoLite2
+      Country database.
+

changes/geoip-september2016

@@ -0,0 +1,4 @@
+  o Minor features:
+    - Update geoip and geoip6 to the September 6 2016 Maxmind GeoLite2
+      Country database.
+

changes/rsa_init_bug

@@ -0,0 +1,7 @@
+  o Major bugfixes (key management):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling pointer
+      to the previous (uninitialized) key value. The impact here should be
+      limited to a difficult-to-trigger crash, if OpenSSL is running an
+      engine that makes key generation failures possible, or if OpenSSL runs
+      out of memory. Fixes bug 19152; bugfix on 0.2.1.10-alpha. Found by
+      Yuan Jochen Kang, Suman Jana, and Baishakhi Ray.

src/common/container.c

@@ -58,31 +58,33 @@ smartlist_clear(smartlist_t *sl)
   sl->num_used = 0;
 }
 
+#if SIZE_MAX < INT_MAX
+#error "We don't support systems where size_t is smaller than int."
+#endif
+
 /** Make sure that <b>sl</b> can hold at least <b>size</b> entries. */
 static INLINE void
-smartlist_ensure_capacity(smartlist_t *sl, int size)
+smartlist_ensure_capacity(smartlist_t *sl, size_t size)
 {
-#if SIZEOF_SIZE_T > SIZEOF_INT
+  /* Set MAX_CAPACITY to MIN(INT_MAX, SIZE_MAX / sizeof(void*)) */
+#if (SIZE_MAX/SIZEOF_VOID_P) > INT_MAX
 #define MAX_CAPACITY (INT_MAX)
 #else
 #define MAX_CAPACITY (int)((SIZE_MAX / (sizeof(void*))))
-#define ASSERT_CAPACITY
 #endif
-  if (size > sl->capacity) {
-    int higher = sl->capacity;
-    if (PREDICT_UNLIKELY(size > MAX_CAPACITY/2)) {
-#ifdef ASSERT_CAPACITY
-      /* We don't include this assertion when MAX_CAPACITY == INT_MAX,
-       * since int size; (size <= INT_MAX) makes analysis tools think we're
-       * doing something stupid. */
+
   tor_assert(size <= MAX_CAPACITY);
-#endif
+
+  if (size > (size_t) sl->capacity) {
+    size_t higher = (size_t) sl->capacity;
+    if (PREDICT_UNLIKELY(size > MAX_CAPACITY/2)) {
       higher = MAX_CAPACITY;
     } else {
       while (size > higher)
         higher *= 2;
     }
-    sl->capacity = higher;
+    tor_assert(higher <= INT_MAX); /* Redundant */
+    sl->capacity = (int) higher;
     sl->list = tor_reallocarray(sl->list, sizeof(void *),
                                 ((size_t)sl->capacity));
   }
@@ -94,7 +96,7 @@ smartlist_ensure_capacity(smartlist_t *sl, int size)
 void
 smartlist_add(smartlist_t *sl, void *element)
 {
-  smartlist_ensure_capacity(sl, sl->num_used+1);
+  smartlist_ensure_capacity(sl, ((size_t) sl->num_used)+1);
   sl->list[sl->num_used++] = element;
 }
 
@@ -102,11 +104,12 @@ smartlist_add(smartlist_t *sl, void *element)
 void
 smartlist_add_all(smartlist_t *s1, const smartlist_t *s2)
 {
-  int new_size = s1->num_used + s2->num_used;
-  tor_assert(new_size >= s1->num_used); /* check for overflow. */
+  size_t new_size = (size_t)s1->num_used + (size_t)s2->num_used;
+  tor_assert(new_size >= (size_t) s1->num_used); /* check for overflow. */
   smartlist_ensure_capacity(s1, new_size);
   memcpy(s1->list + s1->num_used, s2->list, s2->num_used*sizeof(void*));
-  s1->num_used = new_size;
+  tor_assert(new_size <= INT_MAX); /* redundant. */
+  s1->num_used = (int) new_size;
 }
 
 /** Remove all elements E from sl such that E==element.  Preserve
@@ -375,7 +378,7 @@ smartlist_insert(smartlist_t *sl, int idx, void *val)
   if (idx == sl->num_used) {
     smartlist_add(sl, val);
   } else {
-    smartlist_ensure_capacity(sl, sl->num_used+1);
+    smartlist_ensure_capacity(sl, ((size_t) sl->num_used)+1);
     /* Move other elements away */
     if (idx < sl->num_used)
       memmove(sl->list + idx + 1, sl->list + idx,

src/common/crypto.c

@@ -558,8 +558,10 @@ crypto_pk_generate_key_with_bits(crypto_pk_t *env, int bits)
 {
   tor_assert(env);
 
-  if (env->key)
+  if (env->key) {
     RSA_free(env->key);
+    env->key = NULL;
+  }
 
   {
     BIGNUM *e = BN_new();
@@ -2556,6 +2558,7 @@ smartlist_shuffle(smartlist_t *sl)
 /**
  * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to
  * the value <b>byte</b>.
+ * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens.
  *
  * This function is preferable to memset, since many compilers will happily
  * optimize out memset() when they can convince themselves that the data being
@@ -2573,6 +2576,15 @@ smartlist_shuffle(smartlist_t *sl)
 void
 memwipe(void *mem, uint8_t byte, size_t sz)
 {
+  if (sz == 0) {
+    return;
+  }
+  /* If sz is nonzero, then mem must not be NULL. */
+  tor_assert(mem != NULL);
+
+  /* Data this large is likely to be an underflow. */
+  tor_assert(sz < SIZE_T_CEILING);
+
   /* Because whole-program-optimization exists, we may not be able to just
    * have this function call "memset".  A smart compiler could inline it, then
    * eliminate dead memsets, and declare itself to be clever. */

src/or/buffers.c

@@ -69,12 +69,33 @@ static int parse_socks_client(const uint8_t *data, size_t datalen,
 
 #define CHUNK_HEADER_LEN STRUCT_OFFSET(chunk_t, mem[0])
 
+/* We leave this many NUL bytes at the end of the buffer. */
+#define SENTINEL_LEN 4
+
+/* Header size plus NUL bytes at the end */
+#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN)
+
 /** Return the number of bytes needed to allocate a chunk to hold
  * <b>memlen</b> bytes. */
-#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_HEADER_LEN + (memlen))
+#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen))
 /** Return the number of usable bytes in a chunk allocated with
  * malloc(<b>memlen</b>). */
-#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_HEADER_LEN)
+#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD)
+
+#define DEBUG_SENTINEL
+
+#ifdef DEBUG_SENTINEL
+#define DBG_S(s) s
+#else
+#define DBG_S(s) (void)0
+#endif
+
+#define CHUNK_SET_SENTINEL(chunk, alloclen) do {                        \
+    uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen];             \
+    DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]);  \
+    DBG_S(tor_assert(a == b));                                          \
+    memset(a,0,SENTINEL_LEN);                                           \
+  } while (0)
 
 /** Return the next character in <b>chunk</b> onto which data can be appended.
  * If the chunk is full, this might be off the end of chunk->mem. */
@@ -131,6 +152,7 @@ chunk_new_with_alloc_size(size_t alloc)
   ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc);
   total_bytes_allocated_in_chunks += alloc;
   ch->data = &ch->mem[0];
+  CHUNK_SET_SENTINEL(ch, alloc);
   return ch;
 }
 
@@ -140,18 +162,20 @@ static INLINE chunk_t *
 chunk_grow(chunk_t *chunk, size_t sz)
 {
   off_t offset;
-  size_t memlen_orig = chunk->memlen;
+  const size_t memlen_orig = chunk->memlen;
+  const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
+  const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
   tor_assert(sz > chunk->memlen);
   offset = chunk->data - chunk->mem;
-  chunk = tor_realloc(chunk, CHUNK_ALLOC_SIZE(sz));
+  chunk = tor_realloc(chunk, new_alloc);
   chunk->memlen = sz;
   chunk->data = chunk->mem + offset;
 #ifdef DEBUG_CHUNK_ALLOC
-  tor_assert(chunk->DBG_alloc == CHUNK_ALLOC_SIZE(memlen_orig));
-  chunk->DBG_alloc = CHUNK_ALLOC_SIZE(sz);
+  tor_assert(chunk->DBG_alloc == orig_alloc);
+  chunk->DBG_alloc = new_alloc;
 #endif
-  total_bytes_allocated_in_chunks +=
-    CHUNK_ALLOC_SIZE(sz) - CHUNK_ALLOC_SIZE(memlen_orig);
+  total_bytes_allocated_in_chunks += new_alloc - orig_alloc;
+  CHUNK_SET_SENTINEL(chunk, new_alloc);
   return chunk;
 }
 

src/or/config.c

@@ -869,17 +869,14 @@ static const char *default_authorities[] = {
   "dizum orport=443 "
     "v3ident=E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 "
     "194.109.206.212:80 7EA6 EAD6 FD83 083C 538F 4403 8BBF A077 587D D755",
-  "Tonga orport=443 bridge "
-    "82.94.251.203:80 4A0C CD2D DC79 9508 3D73 F5D6 6710 0C8A 5831 F16D",
+  "Bifroest orport=443 bridge "
+    "37.218.247.217:80 1D8F 3A91 C37C 5D1C 4C19 B1AD 1D0C FBE8 BF72 D8E1",
   "gabelmoo orport=443 "
     "v3ident=ED03BB616EB2F60BEC80151114BB25CEF515B226 "
     "131.188.40.189:80 F204 4413 DAC2 E02E 3D6B CF47 35A1 9BCA 1DE9 7281",
   "dannenberg orport=443 "
-    "v3ident=585769C78764D58426B8B52B6651A5A71137189A "
+    "v3ident=0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
     "193.23.244.244:80 7BE6 83E6 5D48 1413 21C5 ED92 F075 C553 64AC 7123",
-  "urras orport=80 "
-    "v3ident=80550987E1D626E3EBA5E5E75A458DE0626D088C "
-    "208.83.223.34:443 0AD3 FA88 4D18 F89E EA2D 89C0 1937 9E0E 7FD9 4417",
   "maatuska orport=80 "
     "v3ident=49015F787433103580E3B66A1707A00E60F2D15B "
     "171.25.193.9:443 BD6A 8292 55CB 08E6 6FBE 7D37 4836 3586 E46B 3810",

src/or/main.c

@@ -3200,7 +3200,6 @@ sandbox_init_filter(void)
     RENAME_SUFFIX2("stats", "exit-stats", ".tmp");
     RENAME_SUFFIX2("stats", "buffer-stats", ".tmp");
     RENAME_SUFFIX2("stats", "conn-stats", ".tmp");
-    RENAME_SUFFIX2("stats", "hidserv-stats", ".tmp");
     RENAME_SUFFIX("hashed-fingerprint", ".tmp");
     RENAME_SUFFIX("router-stability", ".tmp");
 

src/or/rendcache.c

@@ -801,8 +801,10 @@ rend_cache_store_v2_desc_as_client(const char *desc,
                                                     intro_size);
     if (n_intro_points <= 0) {
       log_warn(LD_REND, "Failed to parse introduction points. Either the "
-               "service has published a corrupt descriptor or you have "
-               "provided invalid authorization data.");
+               "service has published a corrupt descriptor, or you have "
+               "provided invalid authorization data, or (maybe!) the "
+               "server is deliberately serving broken data in an attempt "
+               "to crash you with bug 21018.");
       goto err;
     } else if (n_intro_points > MAX_INTRO_POINTS) {
       log_warn(LD_REND, "Found too many introduction points on a hidden "

src/or/routerparse.c

@@ -4105,7 +4105,7 @@ get_next_token(memarea_t *area,
 
   if (tok->tp == ERR_) {
     /* No keyword matched; call it an "K_opt" or "A_unrecognized" */
-    if (**s == '@')
+    if (*s < eol && **s == '@')
       tok->tp = A_UNKNOWN_;
     else
       tok->tp = K_OPT;
@@ -5154,7 +5154,7 @@ rend_decrypt_introduction_points(char **ipos_decrypted,
         crypto_cipher_free(cipher);
 
         len = ipos_encrypted_size - 2 - client_entries_len - CIPHER_IV_LEN;
-        dec = tor_malloc(len);
+        dec = tor_malloc_zero(len + 1);
         declen = crypto_cipher_decrypt_with_iv(session_key, dec, len,
             ipos_encrypted + 2 + client_entries_len,
             ipos_encrypted_size - 2 - client_entries_len);
@@ -5186,7 +5186,7 @@ rend_decrypt_introduction_points(char **ipos_decrypted,
                         "small.");
       return -1;
     }
-    dec = tor_malloc_zero(ipos_encrypted_size - CIPHER_IV_LEN - 1);
+    dec = tor_malloc_zero(ipos_encrypted_size - CIPHER_IV_LEN - 1 + 1);
 
     declen = crypto_cipher_decrypt_with_iv(descriptor_cookie, dec,
                                            ipos_encrypted_size -