r13773@catbus: nickm | 2007-07-16 11:58:25 -0400

Initial "constrained socket buffers" patch from coderman. needs tweaking.


svn:r10842

doc/tor.1.in

@@ -101,6 +101,33 @@ You probably don't need to adjust this. It has no effect on
 Windows since that platform lacks getrlimit(). (Default: 1000)
 .LP
 .TP
+\fBConstrainedSockets \fR\fB0\fR|\fB1\fR\fP
+If set, Tor will attempt to shrink the recv and xmit buffers for all 
+sockets to the size specified in \fBConstrainedSockSize\fP.  This is useful 
+for virtual servers and other environments where system level TCP
+buffers may be limited.  If you encounter the "Error creating network
+socket: No buffer space available" message you are likely experiencing
+this problem.
+
+The preferred solution is to have the admin increase the buffer pool for
+the host itself via /proc/sys/net/ipv4/tcp_mem or equivalent facility. 
+
+The DirPort option should also not be used if TCP buffers are scarce.  The
+cached directory requests consume additional sockets which exacerbates the
+problem.
+
+You should \fBnot\fP enable this feature unless you encounter the no buffer
+space available issue.  Reducing the TCP buffers affects window size for
+for the TCP stream and will reduce throughput in proportion to round trip
+time on long paths.
+.LP
+.TP
+\fBConstrainedSockSize \fR\fINUM\fP
+When \fBConstrainedSockets\fP is enabled the recv and xmit buffers for
+all sockets will be set to this limit.  Must be a value between 2048
+and 262144 in 1024 byte increments.  Default of 8192 is recommended.
+.LP
+.TP
 \fBControlPort \fR\fIPort\fP
 If set, Tor will accept connections on this port and allow those
 connections to control the Tor process using the Tor Control Protocol

src/or/config.c

@@ -146,6 +146,8 @@ static config_var_t _option_vars[] = {
   VAR("CircuitIdleTimeout",  INTERVAL, CircuitIdleTimeout,   "1 hour"),
   VAR("ClientOnly",          BOOL,     ClientOnly,           "0"),
   VAR("ConnLimit",           UINT,     ConnLimit,            "1000"),
+  VAR("ConstrainedSockets",  BOOL,     ConstrainedSockets,   "0"),
+  VAR("ConstrainedSockSize", UINT,     ConstrainedSockSize,  "8192"),
   VAR("ContactInfo",         STRING,   ContactInfo,          NULL),
   VAR("ControlListenAddress",LINELIST, ControlListenAddress, NULL),
   VAR("ControlPort",         UINT,     ControlPort,          "0"),
@@ -330,6 +332,11 @@ static config_var_description_t options_description[] = {
   { "BandwidthBurst", "Limit the maximum token buffer size (also known as "
     "burst) to the given number of bytes." },
   { "ConnLimit", "Maximum number of simultaneous sockets allowed." },
+  { "ConstrainedSockets", "Shrink tx and rx buffers for sockets to avoid "
+    "system limits on vservers and related environments.  See man page for "
+    "more information regarding this option." },
+  { "ConstrainedSockSize", "Limit socket buffers to this size when "
+    "ConstrainedSockets is enabled." },
   /*  ControlListenAddress */
   { "ControlPort", "If set, Tor will accept connections from the same machine "
     "(localhost only) on this port, and allow those connections to control "
@@ -2925,6 +2932,29 @@ options_validate(or_options_t *old_options, or_options_t *options,
     }
   }
 
+  if (options->ConstrainedSockets) {
+    /* If the user wants to constrain socket buffer use, make sure the desired
+     * limit is between MIN|MAX_TCPSOCK_BUFFER in k increments. */
+    if (options->ConstrainedSockSize < MIN_TCPSOCK_BUFFER ||
+        options->ConstrainedSockSize > MAX_TCPSOCK_BUFFER ||
+        options->ConstrainedSockSize % 1024 ) {
+      r = tor_snprintf(buf, sizeof(buf),
+          "ConstrainedSockSize is invalid.  Must be a value between %d and %d "
+          "in 1024 byte increments.",
+          MIN_TCPSOCK_BUFFER, MAX_TCPSOCK_BUFFER);
+      *msg = tor_strdup(r >= 0 ? buf : "internal error");
+      return -1;
+    }
+    if (options->DirPort) {
+      /* Providing cached directory entries while system TCP buffers are scarce
+       * will exacerbate the socket errors.  Suggest that this be disabled. */
+      COMPLAIN("You have requested constrained socket buffers while also "
+               "serving directory entries via DirPort.  It is strongly "
+               "suggested that you disable serving directory requests when "
+               "system TCP buffer resources are scarce.");
+    }
+  }
+
   if (rend_config_services(options, 1) < 0)
     REJECT("Failed to configure rendezvous options. See logs for details.");
 

src/or/connection.c

@@ -28,6 +28,7 @@ static int connection_reached_eof(connection_t *conn);
 static int connection_read_to_buf(connection_t *conn, int *max_to_read);
 static int connection_process_inbuf(connection_t *conn, int package_partial);
 static void client_check_address_changed(int sock);
+static void set_constrained_socket_buffers(int sock, int size);
 
 static uint32_t last_interface_ip = 0;
 static smartlist_t *outgoing_addrs = NULL;
@@ -898,6 +899,8 @@ connection_handle_listener_read(connection_t *conn, int new_type)
   /* length of the remote address. Must be whatever accept() needs. */
   socklen_t remotelen = 256;
   char tmpbuf[INET_NTOA_BUF_LEN];
+  or_options_t *options = get_options();
+
   tor_assert((size_t)remotelen >= sizeof(struct sockaddr_in));
   memset(addrbuf, 0, sizeof(addrbuf));
 
@@ -923,6 +926,10 @@ connection_handle_listener_read(connection_t *conn, int new_type)
 
   set_socket_nonblocking(news);
 
+  if (options->ConstrainedSockets) {
+    set_constrained_socket_buffers (news, options->ConstrainedSockSize);
+  }
+
   tor_assert(((struct sockaddr*)addrbuf)->sa_family == conn->socket_family);
 
   if (conn->socket_family == AF_INET) {
@@ -1096,6 +1103,10 @@ connection_connect(connection_t *conn, const char *address,
 
   set_socket_nonblocking(s);
 
+  if (options->ConstrainedSockets) {
+    set_constrained_socket_buffers (s, options->ConstrainedSockSize);
+  }
+
   memset(&dest_addr,0,sizeof(dest_addr));
   dest_addr.sin_family = AF_INET;
   dest_addr.sin_port = htons(port);
@@ -2540,6 +2551,29 @@ client_check_address_changed(int sock)
   }
 }
 
+/** Some systems have limited system buffers for recv and xmit on
+ * sockets allocated in a virtual server or similar environment. For a Tor
+ * server this can produce the "Error creating network socket: No buffer
+ * space available" error once all available TCP buffer space is consumed.
+ * This method will attempt to constrain the buffers allocated for the socket
+ * to the desired size to stay below system TCP buffer limits.
+ */
+static void
+set_constrained_socket_buffers(int sock, int size)
+{
+  if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (const char *)&size, sizeof(size)) < 0) {
+    int e = tor_socket_errno(sock);
+    log_warn(LD_NET, "setsockopt() to constrain send buffer to %d bytes failed: %s",
+             size, tor_socket_strerror(e));
+  }
+  if (setsockopt(sock, SOL_SOCKET, SO_RCVBUF, (const char *)&size, sizeof(size)) < 0) {
+    int e = tor_socket_errno(sock);
+    log_warn(LD_NET, "setsockopt() to constrain recv buffer to %d bytes failed: %s",
+             size, tor_socket_strerror(e));
+  }
+  return;
+}
+
 /** Process new bytes that have arrived on conn-\>inbuf.
  *
  * This function just passes conn to the connection-specific

src/or/or.h

@@ -1748,6 +1748,10 @@ typedef struct exit_redirect_t {
   unsigned is_redirect:1;
 } exit_redirect_t;
 
+/* limits for TCP send and recv buffer size used for constrained sockets */
+#define MIN_TCPSOCK_BUFFER 2048
+#define MAX_TCPSOCK_BUFFER 262144  /* 256k */
+
 /** A linked list of lines in a config file. */
 typedef struct config_line_t {
   char *key;
@@ -1889,6 +1893,9 @@ typedef struct {
   config_line_t *ReachableORAddresses; /**< IP:ports for OR conns. */
   config_line_t *ReachableDirAddresses; /**< IP:ports for Dir conns. */
 
+  int ConstrainedSockets; /**< Shrink xmit and recv socket buffers. */
+  int ConstrainedSockSize; /**< Size of constrained buffers. */
+
   /** Application ports that require all nodes in circ to have sufficient
    * uptime. */
   smartlist_t *LongLivedPorts;