more progress and cleanups

svn:r9269

doc/TODO

@@ -82,9 +82,11 @@ N - DNS improvements
         D Delay when we get a lot of 503s, rather than punting onto the
           servers that have given us 503s?
         o Add a 'BadDirectory' flag to statuses.
-      - authorities should *never* 503 a cache, and should never 503
+      o authorities should *never* 503 a cache, and should never 503
-        network status requests. They can 503 client descriptor requests
+        network status requests.
-        when they feel like it.
+      D They can 503 client descriptor requests when they feel like it.
+        How can they distinguish? Not implemented for now, maybe
+        should abandon.
       - update dir-spec with what we decided for each of these
 
   o Have a mode that doesn't write to disk much, so we can run Tor on
@@ -131,7 +133,7 @@ R   - "bandwidth classes", for incoming vs initiated-here conns,
       and to give dir conns lower priority.
     . Write limiting; separate token bucket for write
       o preemptively give a 503 to some v1 dir requests
-R     - preemptively give a 503 to some v2 dir requests
+      o preemptively give a 503 to some v2 dir requests
         o Write function to estimate bytes needed for N descriptors
           statuses
       D per-conn write buckets
@@ -258,6 +260,8 @@ M   - rewrite how libevent does select() on win32 so it's not so very slow.
   - Add an option (related to AvoidDiskWrites) to disable directory caching.
 
 Minor items for 0.1.2.x as time permits:
+  - a way to generate the website diagrams from source, so we can
+    translate them as utf-8 text rather than with gimp.
 R - add d64 and fp64 along-side d and fp so people can paste status
     entries into a url. since + is a valid base64 char, only allow one
     at a time. spec and then do.

doc/dir-spec.txt

@@ -379,9 +379,9 @@ $Id$
              (because its ISP censors it, because it is behind a restrictive
              proxy, or for some similar reason).
           "BadDirectory" if the router is believed to be useless as a
-             directory cache (because its directory port isn't working;
+             directory cache (because its directory port isn't working,
-             because its bandwidth is always throttled, or for some
+             its bandwidth is always throttled, or for some similar
-             similar reason).
+             reason).
           "Exit" if the router is useful for building general-purpose exit
              circuits.
           "Fast" if the router is suitable for high-bandwidth circuits.

src/common/tortls.c

@@ -875,6 +875,8 @@ tor_tls_get_n_raw_bytes(tor_tls_t *tls, size_t *n_read, size_t *n_written)
    * we wrapped around by more than ULONG_MAX since the last time we called
    * this function.
    */
+  tor_assert(r >= tls->last_read_count);
+  tor_assert(w >= tls->last_write_count);
   *n_read = (size_t)(r - tls->last_read_count);
   *n_written = (size_t)(w - tls->last_write_count);
   tls->last_read_count = r;

src/or/connection.c

@@ -1175,8 +1175,11 @@ connection_bucket_write_limit(connection_t *conn)
 int
 global_write_bucket_low(size_t attempt, int priority)
 {
-  if (global_write_bucket < (int)attempt) /* not enough space no matter what */
+  if (authdir_mode(get_options()) && priority>1)
-    return 1;
+    return 0; /* there's always room to answer v2 if we're an auth dir */
+
+  if (global_write_bucket < (int)attempt)
+    return 1; /* not enough space no matter the priority */
 
   if (priority == 1) { /* old-style v1 query */
     if (global_write_bucket-attempt < 2*get_options()->BandwidthRate)
@@ -1533,7 +1536,7 @@ connection_read_to_buf(connection_t *conn, int *max_to_read)
    * have reached 0 on a different conn, and this guy needs to
    * know to stop reading. */
   connection_consider_empty_read_buckets(conn);
-  if (n_written > 0)
+  if (n_written > 0 && connection_is_writing(conn))
     connection_consider_empty_write_buckets(conn);
 
   return 0;
@@ -1714,7 +1717,7 @@ connection_handle_write(connection_t *conn, int force)
     }
     if (n_read > 0) {
       rep_hist_note_bytes_read(n_read, now);
-      global_read_bucket -= n_read;
+      connection_read_bucket_decrement(conn, n_read);
     }
   }
 
@@ -1737,7 +1740,7 @@ connection_handle_write(connection_t *conn, int force)
    * have reached 0 on a different conn, and this guy needs to
    * know to stop writing. */
   connection_consider_empty_write_buckets(conn);
-  if (n_read > 0)
+  if (n_read > 0 && connection_is_reading(conn))
     connection_consider_empty_read_buckets(conn);
 
   return 0;

src/or/directory.c

@@ -1325,8 +1325,8 @@ connection_dir_process_inbuf(dir_connection_t *conn)
   }
 
   if (buf_datalen(conn->_base.inbuf) > MAX_DIRECTORY_OBJECT_SIZE) {
-    log_warn(LD_HTTP, "Too much data received from directory connection; "
+    log_warn(LD_HTTP, "Too much data received from directory connection: "
-             "DOS attempt or protocol shift.");
+             "denial of service attempt, or you need to upgrade?");
     connection_mark_for_close(TO_CONN(conn));
     return -1;
   }
@@ -1639,6 +1639,16 @@ directory_handle_command_get(dir_connection_t *conn, char *headers,
       smartlist_free(dir_fps);
       return 0;
     }
+    dlen = dirserv_estimate_data_size(conn->fingerprint_stack,
+                                      0, deflated);
+    if (global_write_bucket_low(dlen, 2)) {
+      log_info(LD_DIRSERV,
+               "Client asked for server descriptors, but we've been "
+               "writing too many bytes lately. Sending 503 Dir busy.");
+      write_http_status_line(conn, 503, "Directory busy, try again later");
+      return 0;
+    }
+
     // note_request(request_type,dlen);
     (void) request_type;
     write_http_response_header(conn, -1,
@@ -1696,6 +1706,15 @@ directory_handle_command_get(dir_connection_t *conn, char *headers,
     if (res < 0)
       write_http_status_line(conn, 404, msg);
     else {
+      dlen = dirserv_estimate_data_size(conn->fingerprint_stack,
+                                        1, deflated);
+      if (global_write_bucket_low(dlen, 2)) {
+        log_info(LD_DIRSERV,
+                 "Client asked for server descriptors, but we've been "
+                 "writing too many bytes lately. Sending 503 Dir busy.");
+        write_http_status_line(conn, 503, "Directory busy, try again later");
+        return 0;
+      }
       write_http_response_header(conn, -1,
                      deflated?"application/octet_stream":"text/plain",
                      deflated?"deflate":NULL, cache_lifetime);

src/or/dirserv.c

@@ -1687,7 +1687,7 @@ dirserv_get_networkstatus_v2(smartlist_t *result,
   SMARTLIST_FOREACH(fingerprints, const char *, fp,
     {
       if (router_digest_is_me(fp) && should_generate_v2_networkstatus())
-          generate_v2_networkstatus();
+        generate_v2_networkstatus();
       cached = digestmap_get(cached_v2_networkstatus, fp);
       if (cached) {
         smartlist_add(result, cached);
@@ -1839,8 +1839,9 @@ dirserv_orconn_tls_done(const char *address,
 
   SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, {
     if (!strcasecmp(address, ri->address) && or_port == ri->or_port &&
+        as_advertised &&
         !memcmp(ri->cache_info.identity_digest, digest_rcvd, DIGEST_LEN) &&
-        as_advertised) {
+        !strcasecmp(nickname_rcvd, ri->nickname)) {
       /* correct nickname and digest. mark this router reachable! */
       log_info(LD_DIRSERV, "Found router %s to be reachable. Yay.",
                ri->nickname);
@@ -1886,10 +1887,11 @@ dirserv_test_reachability(int try_all)
     ctr = (ctr + 1) % 128;
 }
 
-/** Return an approximate estimate of the number of bytes that will be needed
+/** Return an approximate estimate of the number of bytes that will
- * to transmit the server descriptors (if is_serverdescs) or networkstatus
+ * be needed to transmit the server descriptors (if is_serverdescs --
- * objects (if !is_serverdescs) listed in <b>fps</b>.  If <b>compressed</b> is
+ * they can be either d/ or fp/ queries) or networkstatus objects (if
- * set, we guess how large the data will be after compression.
+ * !is_serverdescs) listed in <b>fps</b>.  If <b>compressed</b> is set,
+ * we guess how large the data will be after compression.
  *
  * The return value is an estimate; it might be larger or smaller.
  **/