r12496@catbus: nickm | 2007-04-22 23:04:05 -0400

When logging memory usage, break down memory used in buffers by buffer type.


svn:r10004

ChangeLog

@@ -43,6 +43,8 @@ Changes in version 0.2.0.1-alpha - 2007-??-??
     - Put a platform string (e.g. "Linux i686") in the startup log
       message, so when people paste just their logs, we know if it's
       openbsd or windows or what.
+    - When logging memory usage, break down memory used in buffers by
+      buffer type.
 
   o Minor features (directory system):
     - Directory authorities accept and serve "extra info" documents for

src/or/buffers.c

@@ -75,11 +75,6 @@ struct buf_t {
   size_t datalen; /**< Number of bytes currently in <b>mem</b>. */
 };
 
-/** How many bytes, total, are used in all buffers? */
-uint64_t buf_total_used = 0;
-/** How many bytes, total, are allocated in all buffers? */
-uint64_t buf_total_alloc = 0;
-
 /** Size, in bytes, for newly allocated buffers.  Should be a power of 2. */
 #define INITIAL_BUF_SIZE (4*1024)
 /** Size, in bytes, for minimum 'shrink' size for buffers.  Buffers may start
@@ -228,8 +223,6 @@ buf_resize(buf_t *buf, size_t new_capacity)
     SET_GUARDS(buf->mem, new_capacity);
     buf->cur = buf->mem+offset;
   }
-  buf_total_alloc += new_capacity;
-  buf_total_alloc -= buf->len;
 
   if (offset + buf->datalen > buf->len) {
     /* We need to move data now that we are done growing.  The buffer
@@ -317,7 +310,6 @@ buf_remove_from_front(buf_t *buf, size_t n)
 {
   tor_assert(buf->datalen >= n);
   buf->datalen -= n;
-  buf_total_used -= n;
   if (buf->datalen) {
     buf->cur = _wrap_ptr(buf, buf->cur+n);
   } else {
@@ -347,7 +339,6 @@ buf_new_with_capacity(size_t size)
   SET_GUARDS(buf->mem, size);
   buf->len = buf->memsize = size;
 
-  buf_total_alloc += size;
   assert_buf_ok(buf);
   return buf;
 }
@@ -363,7 +354,6 @@ buf_new(void)
 void
 buf_clear(buf_t *buf)
 {
-  buf_total_used -= buf->datalen;
   buf->datalen = 0;
   buf->cur = buf->mem;
   buf->len = buf->memsize;
@@ -401,8 +391,6 @@ buf_free(buf_t *buf)
   buf->magic = 0xDEADBEEF;
   oldmem = RAW_MEM(buf->mem);
   tor_free(oldmem);
-  buf_total_alloc -= buf->len;
-  buf_total_used -= buf->datalen;
   tor_free(buf);
 }
 
@@ -436,7 +424,6 @@ read_to_buf_impl(int s, size_t at_most, buf_t *buf,
     return 0;
   } else { /* we read some bytes */
     buf->datalen += read_result;
-    buf_total_used += read_result;
     if (buf->datalen > buf->highwater)
       buf->highwater = buf->datalen;
     log_debug(LD_NET,"Read %d bytes. %d on inbuf.",read_result,
@@ -512,7 +499,6 @@ read_to_buf_tls_impl(tor_tls_t *tls, size_t at_most, buf_t *buf, char *next)
   if (r<0)
     return r;
   buf->datalen += r;
-  buf_total_used += r;
   if (buf->datalen > buf->highwater)
     buf->highwater = buf->datalen;
   log_debug(LD_NET,"Read %d bytes. %d on inbuf; %d pending",r,
@@ -756,13 +742,11 @@ write_to_buf(const char *string, size_t string_len, buf_t *buf)
 
   memcpy(next, string, string_len);
   buf->datalen += string_len;
-  buf_total_used += string_len;
 
   if (len2) {
     tor_assert(_buf_end(buf) == buf->mem);
     memcpy(buf->mem, string+string_len, len2);
     buf->datalen += len2;
-    buf_total_used += len2;
   }
   if (buf->datalen > buf->highwater)
     buf->highwater = buf->datalen;
@@ -1410,7 +1394,6 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state,
     buf->datalen += old_avail - avail;
     if (buf->datalen > buf->highwater)
       buf->highwater = buf->datalen;
-    buf_total_used += old_avail - avail;
   } while (!over);
   return 0;
 }

src/or/connection.c

@@ -389,7 +389,6 @@ connection_free_all(void)
   connection_t **carray;
 
   get_connection_array(&carray,&n);
-
   /* We don't want to log any messages to controllers. */
   for (i=0;i<n;i++)
     if (carray[i]->type == CONN_TYPE_CONTROL)
@@ -2503,6 +2502,55 @@ connection_reached_eof(connection_t *conn)
   }
 }
 
+/** Log how many bytes are used by buffers of different kinds and sizes. */
+void
+connection_dump_buffer_mem_stats(int severity)
+{
+  uint64_t used_by_type[_CONN_TYPE_MAX+1];
+  uint64_t alloc_by_type[_CONN_TYPE_MAX+1];
+  int n_conns_by_type[_CONN_TYPE_MAX+1];
+  uint64_t total_alloc = 0;
+  uint64_t total_used = 0;
+  int i, n;
+  connection_t **carray;
+
+  memset(used_by_type, 0, sizeof(used_by_type));
+  memset(alloc_by_type, 0, sizeof(alloc_by_type));
+  memset(n_conns_by_type, 0, sizeof(n_conns_by_type));
+
+  get_connection_array(&carray,&n);
+
+  for (i=0; i<n; ++i) {
+    connection_t *c = carray[i];
+    int tp = c->type;
+    ++n_conns_by_type[tp];
+    if (c->inbuf) {
+      used_by_type[tp] += buf_datalen(c->inbuf);
+      alloc_by_type[tp] += buf_capacity(c->inbuf);
+    }
+    if (c->outbuf) {
+      used_by_type[tp] += buf_datalen(c->outbuf);
+      alloc_by_type[tp] += buf_capacity(c->outbuf);
+    }
+  }
+  for (i=0; i <= _CONN_TYPE_MAX; ++i) {
+    total_used += used_by_type[i];
+    total_alloc += alloc_by_type[i];
+  }
+
+  log(severity, LD_GENERAL,
+     "In buffers for %d connections: "U64_FORMAT" used/"U64_FORMAT" allocated",
+      n, U64_PRINTF_ARG(total_used), U64_PRINTF_ARG(total_alloc));
+  for (i=_CONN_TYPE_MIN; i <= _CONN_TYPE_MAX; ++i) {
+    if (!n_conns_by_type[i])
+      continue;
+    log(severity, LD_GENERAL,
+        "  For %d %s connections: "U64_FORMAT" used/"U64_FORMAT" allocated",
+        n_conns_by_type[i], conn_type_to_string(i),
+        U64_PRINTF_ARG(used_by_type[i]), U64_PRINTF_ARG(alloc_by_type[i]));
+  }
+}
+
 /** Verify that connection <b>conn</b> has all of its invariants
  * correct. Trigger an assert if anything is invalid.
  */

src/or/main.c

@@ -1533,8 +1533,6 @@ signal_callback(int fd, short events, void *arg)
   }
 }
 
-extern uint64_t buf_total_used;
-extern uint64_t buf_total_alloc;
 extern uint64_t rephist_total_alloc;
 extern uint32_t rephist_total_num;
 
@@ -1544,10 +1542,7 @@ extern uint32_t rephist_total_num;
 static void
 dumpmemusage(int severity)
 {
-  log(severity, LD_GENERAL,
+  connection_dump_buffer_mem_stats(severity);
-      "In buffers: "U64_FORMAT" used/"U64_FORMAT" allocated (%d conns).",
-      U64_PRINTF_ARG(buf_total_used), U64_PRINTF_ARG(buf_total_alloc),
-      n_conns);
   log(severity, LD_GENERAL, "In rephist: "U64_FORMAT" used by %d Tors.",
       U64_PRINTF_ARG(rephist_total_alloc), rephist_total_num);
   dump_routerlist_mem_usage(severity);

src/or/or.h

@@ -2252,6 +2252,7 @@ char *alloc_http_authenticator(const char *authenticator);
 
 void assert_connection_ok(connection_t *conn, time_t now);
 int connection_or_nonopen_was_started_here(or_connection_t *conn);
+void connection_dump_buffer_mem_stats(int severity);
 
 /********************************* connection_edge.c *************************/