Merge branch 'metrics-dir-usage' into 'main'

export metrics for directory and hsdir usage

Closes #40919

See merge request tpo/core/tor!800

src/core/mainloop/connection.c

@@ -3640,10 +3640,9 @@ record_num_bytes_transferred_impl(connection_t *conn,
 {
   /* Count bytes of answering direct and tunneled directory requests */
   if (conn->type == CONN_TYPE_DIR && conn->purpose == DIR_PURPOSE_SERVER) {
-    if (num_read > 0)
-      bwhist_note_dir_bytes_read(num_read, now);
-    if (num_written > 0)
-      bwhist_note_dir_bytes_written(num_written, now);
+    dir_connection_t *dir_conn = TO_DIR_CONN(conn);
+    dir_conn->num_read += num_read;
+    dir_conn->num_written += num_written;
   }
 
   /* Linked connections and internal IPs aren't counted for statistics or

src/core/mainloop/mainloop.c

@@ -138,6 +138,18 @@ token_bucket_rw_t global_relayed_bucket;
 static uint64_t stats_n_bytes_read = 0;
 /** How many bytes have we written since we started the process? */
 static uint64_t stats_n_bytes_written = 0;
+/** How many bytes have we read for directory purpose since we started the
+ * process? */
+static uint64_t stats_n_dir_bytes_read = 0;
+/** How many bytes have we written for directory purpose since we started the
+ * process? */
+static uint64_t stats_n_dir_bytes_written = 0;
+/** How many bytes have we read for hs directory purpose since we started the
+ * process? */
+static uint64_t stats_n_hsdir_bytes_read = 0;
+/** How many bytes have we written for hs directory purpose since we started
+ * the process? */
+static uint64_t stats_n_hsdir_bytes_written = 0;
 /** What time did this process start up? */
 time_t time_of_process_start = 0;
 /** How many seconds have we been running? */
@@ -478,6 +490,48 @@ stats_increment_bytes_read_and_written(uint64_t r, uint64_t w)
   stats_n_bytes_written += w;
 }
 
+/**
+ * Return the amount of directory raffic read, in bytes, over the life of this
+ * process.
+ */
+MOCK_IMPL(uint64_t,
+get_dir_bytes_read,(bool hs))
+{
+  if (hs)
+    return stats_n_hsdir_bytes_read;
+  else
+    return stats_n_dir_bytes_read;
+}
+
+/**
+ * Return the amount of directory traffic read, in bytes, over the life of this
+ * process.
+ */
+MOCK_IMPL(uint64_t,
+get_dir_bytes_written,(bool hs))
+{
+  if (hs)
+    return stats_n_hsdir_bytes_written;
+  else
+    return stats_n_dir_bytes_written;
+}
+
+/**
+ * Increment the amount of directory traffic read and written, over the life of
+ * this process.
+ */
+void
+stats_increment_dir_bytes_read_and_written(uint64_t r, uint64_t w, bool hs)
+{
+  if (hs) {
+    stats_n_hsdir_bytes_read += r;
+    stats_n_hsdir_bytes_written += w;
+  } else {
+    stats_n_dir_bytes_read += r;
+    stats_n_dir_bytes_written += w;
+  }
+}
+
 /** Set the event mask on <b>conn</b> to <b>events</b>.  (The event
  * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT)
  */

src/core/mainloop/mainloop.h

@@ -28,7 +28,11 @@ int connection_is_on_closeable_list(connection_t *conn);
 MOCK_DECL(smartlist_t *, get_connection_array, (void));
 MOCK_DECL(uint64_t,get_bytes_read,(void));
 MOCK_DECL(uint64_t,get_bytes_written,(void));
+MOCK_DECL(uint64_t,get_dir_bytes_read,(bool hs));
+MOCK_DECL(uint64_t,get_dir_bytes_written,(bool hs));
 void stats_increment_bytes_read_and_written(uint64_t r, uint64_t w);
+void stats_increment_dir_bytes_read_and_written(uint64_t r,
+                                                uint64_t w, bool hs);
 
 /** Bitmask for events that we can turn on and off with
  * connection_watch_events. */

src/feature/dircache/dircache.c

@@ -1357,6 +1357,7 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn,
   const char *pubkey_str = NULL;
   const char *url = args->url;
 
+  conn->hs_request = 1;
   /* Reject non anonymous dir connections (which also tests if encrypted). We
    * do not allow single hop clients to query an HSDir. */
   if (!connection_dir_is_anonymous(conn)) {
@@ -1593,6 +1594,7 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
    * (which also tests for encrypted). We do not allow single-hop client to
    * post a descriptor onto an HSDir. */
   if (!strcmpstart(url, "/tor/hs/")) {
+    conn->hs_request = 1;
     if (!connection_dir_is_anonymous(conn)) {
       write_short_http_response(conn, 503,
                                 "Rejecting single hop HS descriptor post");

src/feature/dircommon/dir_connection_st.h

@@ -31,6 +31,9 @@ struct dir_connection_t {
   /** Is this dirconn direct, or via a multi-hop Tor circuit?
    * Direct connections can use the DirPort, or BEGINDIR over the ORPort. */
   unsigned int dirconn_direct:1;
+  /** Is this dirconn used to fetch/upload hs desciptor?
+   * Only set on relay side, unused on client side. */
+  unsigned int hs_request:1;
 
   /** If we're fetching descriptors, what router purpose shall we assign
    * to them? */
@@ -59,6 +62,17 @@ struct dir_connection_t {
    * needs this for the incoming side, so it's moved here. */
   uint64_t dirreq_id;
 
+  /** Number of bytes read on this connection. We track this here, and update
+   * global statistics on connection tear down so that we can differentiate
+   * between normal directory and hidden-service related request.
+   **/
+  size_t num_read;
+  /** Number of bytes written on this connection. We track this here, and
+   * update global statistics on connection tear down so that we can
+   * differentiate between normal directory and hidden-service related request.
+   **/
+  size_t num_written;
+
 #ifdef MEASUREMENTS_21206
   /** Number of RELAY_DATA cells received. */
   uint32_t data_cells_received;

src/feature/dircommon/directory.c

@@ -11,11 +11,13 @@
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
 #include "core/or/channeltls.h"
+#include "core/mainloop/mainloop.h"
 #include "feature/dircache/dircache.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/dircommon/fp_pair.h"
+#include "feature/stats/bwhist.h"
 #include "feature/stats/geoip_stats.h"
 #include "lib/compress/compress.h"
 
@@ -492,6 +494,16 @@ connection_dir_about_to_close(dir_connection_t *dir_conn)
     connection_dir_client_request_failed(dir_conn);
   }
 
+  if (!dir_conn->hs_request) {
+    const time_t now = approx_time();
+    if (dir_conn->num_read > 0)
+      bwhist_note_dir_bytes_read(dir_conn->num_read, now);
+    if (dir_conn->num_written > 0)
+      bwhist_note_dir_bytes_written(dir_conn->num_written, now);
+  }
+  stats_increment_dir_bytes_read_and_written(dir_conn->num_read,
+                  dir_conn->num_written, dir_conn->hs_request);
+
   connection_dir_client_refetch_hsdesc_if_needed(dir_conn);
 }
 

src/feature/relay/relay_metrics.c

@@ -61,6 +61,7 @@ static void fill_relay_drop_cell(void);
 static void fill_relay_flags(void);
 static void fill_tcp_exhaustion_values(void);
 static void fill_traffic_values(void);
+static void fill_traffic_directory_values(void);
 static void fill_signing_cert_expiry(void);
 
 static void fill_est_intro_cells(void);
@@ -172,6 +173,13 @@ static const relay_metrics_entry_t base_metrics[] =
     .help = "Traffic related counters",
     .fill_fn = fill_traffic_values,
   },
+  {
+    .key = RELAY_METRICS_NUM_TRAFFIC_DIRECTORY,
+    .type = METRICS_TYPE_COUNTER,
+    .name = METRICS_NAME(relay_traffic_directory_bytes),
+    .help = "Traffic counters related to locally served directory requests",
+    .fill_fn = fill_traffic_directory_values,
+  },
   {
     .key = RELAY_METRICS_RELAY_FLAGS,
     .type = METRICS_TYPE_GAUGE,
@@ -399,6 +407,47 @@ fill_traffic_values(void)
   metrics_store_entry_update(sentry, get_bytes_written());
 }
 
+/** Fill function for the RELAY_METRICS_NUM_TRAFFIC_DIRECTORY metric. */
+static void
+fill_traffic_directory_values(void)
+{
+  const relay_metrics_entry_t *rentry =
+    &base_metrics[RELAY_METRICS_NUM_TRAFFIC_DIRECTORY];
+  metrics_store_entry_t *sentry;
+
+  sentry = metrics_store_add(
+      the_store, rentry->type, rentry->name, rentry->help, 0, NULL);
+  metrics_store_entry_add_label(sentry,
+          metrics_format_label("direction", "read"));
+  metrics_store_entry_add_label(sentry,
+          metrics_format_label("kind", "directory"));
+  metrics_store_entry_update(sentry, get_dir_bytes_read(false));
+
+  sentry = metrics_store_add(the_store, rentry->type, rentry->name,
+                             rentry->help, 0, NULL);
+  metrics_store_entry_add_label(sentry,
+          metrics_format_label("direction", "written"));
+  metrics_store_entry_add_label(sentry,
+          metrics_format_label("kind", "directory"));
+  metrics_store_entry_update(sentry, get_dir_bytes_written(false));
+
+  sentry = metrics_store_add(
+      the_store, rentry->type, rentry->name, rentry->help, 0, NULL);
+  metrics_store_entry_add_label(sentry,
+          metrics_format_label("direction", "read"));
+  metrics_store_entry_add_label(sentry,
+          metrics_format_label("kind", "hsdir"));
+  metrics_store_entry_update(sentry, get_dir_bytes_read(true));
+
+  sentry = metrics_store_add(the_store, rentry->type, rentry->name,
+                             rentry->help, 0, NULL);
+  metrics_store_entry_add_label(sentry,
+          metrics_format_label("direction", "written"));
+  metrics_store_entry_add_label(sentry,
+          metrics_format_label("kind", "hsdir"));
+  metrics_store_entry_update(sentry, get_dir_bytes_written(true));
+}
+
 /** Fill function for the RELAY_METRICS_NUM_DOS metric. */
 static void
 fill_dos_values(void)

src/feature/relay/relay_metrics.h

@@ -43,6 +43,8 @@ typedef enum {
   RELAY_METRICS_NUM_DOS,
   /** Denial of Service defenses subsystem. */
   RELAY_METRICS_NUM_TRAFFIC,
+  /** Denial of Service defenses subsystem. */
+  RELAY_METRICS_NUM_TRAFFIC_DIRECTORY,
   /** Relay flags. */
   RELAY_METRICS_RELAY_FLAGS,
   /** Numer of circuits. */