Merge branch 'tor-gitlab/mr/709'

changes/ticket40593

@@ -0,0 +1,16 @@
+  o Major features (conflux):
+    - Implement Proposal 329 (conflux traffic splitting). Conflux splits
+      traffic across two circuits to Exits that support the protocol.
+      These circuits are pre-built only, which means that if the pre-built
+      conflux pool runs out, regular circuits will then be used.
+
+      When using conflux circuit pairs, clients choose the lower-latency
+      circuit to send data to the Exit. When the Exit sends data to the
+      client, it maximizes throughput, by fully utilizing both circuits in a
+      multiplexed fashion. Alternatively, clients can request that the Exit
+      optimize for latency when transmitting to them, by setting the torrc
+      option 'ConfluxClientUX latency'.
+
+      Onion services are not currently supported, but will be in arti. Many
+      other future optimizations will also be possible using this protocol.
+      Closes ticket 40593.

doc/man/tor.1.txt

@@ -354,6 +354,13 @@ forward slash (/) in the configuration file and on the command line.
     supported at the moment. Default value is set to "auto" meaning the
     consensus is used to decide unless set. (Default: auto)
 
+[[ConfluxClientUX]] **ConfluxClientUX** **throughput**|**latency**|**throughput_lowmem**|**latency_lowmem**::
+    This option configures the user experience that the client requests from
+    the exit, for data that the exit sends to the client. The default is
+    "throughput", which maximizes throughput. "Latency" will tell the exit to
+    only use the circuit with lower latency for all data. The lowmem versions
+    minimize queue usage memory at the client. (Default: "throughput")
+
 [[ConnLimit]] **ConnLimit** __NUM__::
     The minimum number of file descriptors that must be available to the Tor
     process before it will start. Tor will ask the OS for as many file

src/app/config/config.c

@@ -77,6 +77,7 @@
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitstats.h"
 #include "core/or/connection_edge.h"
+#include "trunnel/conflux.h"
 #include "core/or/dos.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
@@ -380,6 +381,8 @@ static const config_var_t option_vars_[] = {
   V(ClientUseIPv6,               BOOL,     "1"),
   V(ClientUseIPv4,               BOOL,     "1"),
   V(ConfluxEnabled,              AUTOBOOL, "auto"),
+  VAR("ConfluxClientUX",         STRING,   ConfluxClientUX_option,
+          "throughput"),
   V(ConnLimit,                   POSINT,     "1000"),
   V(ConnDirectionStatistics,     BOOL,     "0"),
   V(ConstrainedSockets,          BOOL,     "0"),
@@ -3545,6 +3548,21 @@ options_validate_cb(const void *old_options_, void *options_, char **msg)
     return -1;
   }
 
+ options->ConfluxClientUX = CONFLUX_UX_HIGH_THROUGHPUT;
+ if (options->ConfluxClientUX_option) {
+    if (!strcmp(options->ConfluxClientUX_option, "latency"))
+      options->ConfluxClientUX = CONFLUX_UX_MIN_LATENCY;
+    else if (!strcmp(options->ConfluxClientUX_option, "throughput"))
+      options->ConfluxClientUX = CONFLUX_UX_HIGH_THROUGHPUT;
+    else if (!strcmp(options->ConfluxClientUX_option, "latency_lowmem"))
+      options->ConfluxClientUX = CONFLUX_UX_LOW_MEM_LATENCY;
+    else if (!strcmp(options->ConfluxClientUX_option, "throughput_lowmem"))
+      options->ConfluxClientUX = CONFLUX_UX_LOW_MEM_THROUGHPUT;
+    else
+      REJECT("ConfluxClientUX must be 'latency', 'throughput, "
+             "'latency_lowmem', or 'throughput_lowmem'");
+  }
+
   if (options_validate_publish_server(old_options, options, msg) < 0)
     return -1;
 

src/app/config/or_options_st.h

@@ -727,6 +727,10 @@ struct or_options_t {
    * circuits which excludes onion service traffic. */
   int ConfluxEnabled;
 
+  /** Has the UX integer value that the client will request from the exit. */
+  char *ConfluxClientUX_option;
+  int ConfluxClientUX;
+
   /** The length of time that we think a consensus should be fresh. */
   int V3AuthVotingInterval;
   /** The length of time we think it will take to distribute votes. */

src/core/or/conflux.c

@@ -209,7 +209,7 @@ circuit_ready_to_send(const circuit_t *circ)
    * cwnd, because inflight is decremented before this check */
   // TODO-329-TUNING: This subtraction not be right.. It depends
   // on call order wrt decisions and sendme arrival
-  if (cc->inflight + cc->sendme_inc >= cc->cwnd) {
+  if (cc->inflight >= cc->cwnd) {
     cc_sendable = false;
   }
 
@@ -289,38 +289,6 @@ conflux_decide_circ_lowrtt(const conflux_t *cfx)
   return circ;
 }
 
-/**
- * Return the amount of congestion window we can send on
- * on_circ during in_usec. However, if we're still in
- * slow-start, send the whole window to establish the true
- * cwnd.
- */
-static inline uint64_t
-cwnd_sendable(const circuit_t *on_circ, uint64_t in_usec,
-              uint64_t our_usec)
-{
-  const congestion_control_t *cc = circuit_ccontrol(on_circ);
-
-  tor_assert(cc);
-
-  // TODO-329-TUNING: This function may want to consider inflight?
-
-  if (our_usec == 0 || in_usec == 0) {
-    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
-       "cwnd_sendable: Missing RTT data. in_usec: %" PRIu64
-       " our_usec: %" PRIu64, in_usec, our_usec);
-    return cc->cwnd;
-  }
-
-  if (cc->in_slow_start) {
-    return cc->cwnd;
-  } else {
-    uint64_t sendable =
-      conflux_params_get_send_pct()*cc->cwnd*in_usec/(100*our_usec);
-    return MIN(cc->cwnd, sendable);
-  }
-}
-
 /**
  * Returns the amount of room in a cwnd on a circuit.
  */
@@ -336,6 +304,41 @@ cwnd_available(const circuit_t *on_circ)
   return cc->cwnd - cc->inflight;
 }
 
+/**
+ * Return the amount of congestion window we can send on
+ * on_circ during in_usec. However, if we're still in
+ * slow-start, send the whole window to establish the true
+ * cwnd.
+ */
+static inline uint64_t
+cwnd_sendable(const circuit_t *on_circ, uint64_t in_usec,
+              uint64_t our_usec)
+{
+  const congestion_control_t *cc = circuit_ccontrol(on_circ);
+  tor_assert(cc);
+  uint64_t cwnd_adjusted = cwnd_available(on_circ);
+
+  if (our_usec == 0 || in_usec == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+       "cwnd_sendable: Missing RTT data. in_usec: %" PRIu64
+       " our_usec: %" PRIu64, in_usec, our_usec);
+    return cwnd_adjusted;
+  }
+
+  if (cc->in_slow_start) {
+    return cwnd_adjusted;
+  } else {
+    /* For any given leg, it has min_rtt/2 time before the 'primary'
+     * leg's acks start arriving. So, the amount of data this
+     * 'secondary' leg can send while the min_rtt leg transmits these
+     * acks is:
+     *   (cwnd_leg/(leg_rtt/2))*min_rtt/2 = cwnd_leg*min_rtt/leg_rtt.
+     */
+    uint64_t sendable = cwnd_adjusted*in_usec/our_usec;
+    return MIN(cc->cwnd, sendable);
+  }
+}
+
 /**
  * Returns true if we can switch to a new circuit, false otherwise.
  *
@@ -360,15 +363,14 @@ conflux_can_switch(const conflux_t *cfx)
      * of the congestion window, then we can switch.
      * We check the sendme_inc because there may be un-ackable
      * data in inflight as well, and we can still switch then. */
+    // TODO-329-TUNING: Should we try to switch if the prev_leg is
+    // ready to send, instead of this?
     if (ccontrol->inflight < ccontrol->sendme_inc ||
         100*ccontrol->inflight <=
         conflux_params_get_drain_pct()*ccontrol->cwnd) {
       return true;
     }
 
-    // TODO-329-TUNING: Should we try to switch if the prev_leg is
-    // ready to send?
-
     return false;
   }
 
@@ -407,14 +409,6 @@ conflux_decide_circ_cwndrtt(const conflux_t *cfx)
     return leg->circ;
   }
 
-  /* For any given leg, it has min_rtt/2 time before the 'primary'
-   * leg's acks start arriving. So, the amount of data this
-   * 'secondary' leg can send while the min_rtt leg transmits these
-   * acks is:
-   *   (cwnd_leg/(leg_rtt/2))*min_rtt/2 = cwnd_leg*min_rtt/leg_rtt.
-   * So any leg with available room below that is no good.
-   */
-
   leg = NULL;
 
   CONFLUX_FOR_EACH_LEG_BEGIN(cfx, l) {
@@ -425,8 +419,7 @@ conflux_decide_circ_cwndrtt(const conflux_t *cfx)
     /* Pick a 'min_leg' with the lowest RTT that still has
      * room in the congestion window. Note that this works for
      * min_leg itself, up to inflight. */
-    if (cwnd_sendable(l->circ, min_rtt, l->circ_rtts_usec) <=
+    if (cwnd_sendable(l->circ, min_rtt, l->circ_rtts_usec) > 0) {
-        cwnd_available(l->circ)) {
       leg = l;
     }
   } CONFLUX_FOR_EACH_LEG_END(l);
@@ -438,133 +431,6 @@ conflux_decide_circ_cwndrtt(const conflux_t *cfx)
   return leg->circ;
 }
 
-/**
- * Favor the circuit with the highest send rate.
- *
- * Only spill over to other circuits if they are still in slow start.
- * In steady-state, we only use the max throughput circuit.
- */
-static const circuit_t *
-conflux_decide_circ_maxrate(const conflux_t *cfx)
-{
-  uint64_t max_rate = 0;
-  const conflux_leg_t *leg = NULL;
-
-  /* Find the highest bandwidth leg */
-  CONFLUX_FOR_EACH_LEG_BEGIN(cfx, l) {
-    uint64_t rate;
-    const congestion_control_t *cc = circuit_ccontrol(l->circ);
-
-    rate = CELL_MAX_NETWORK_SIZE*USEC_PER_SEC *
-           cc->cwnd / l->circ_rtts_usec;
-    if (rate > max_rate) {
-      max_rate = rate;
-      leg = l;
-    }
-  } CONFLUX_FOR_EACH_LEG_END(l);
-
-  /* If the package window is has room, use it */
-  if (leg && circuit_ready_to_send(leg->circ)) {
-    return leg->circ;
-  }
-
-  leg = NULL;
-  max_rate = 0;
-
-  /* Find the circuit with the max rate where in_slow_start == 1: */
-  CONFLUX_FOR_EACH_LEG_BEGIN(cfx, l) {
-    uint64_t rate;
-    /* Ignore circuits with no room in the package window */
-    if (!circuit_ready_to_send(l->circ)) {
-      continue;
-    }
-
-    const congestion_control_t *cc = circuit_ccontrol(l->circ);
-
-    rate = CELL_MAX_NETWORK_SIZE*USEC_PER_SEC *
-                 cc->cwnd / l->circ_rtts_usec;
-
-    if (rate > max_rate && cc->in_slow_start) {
-      max_rate = rate;
-      leg = l;
-    }
-  } CONFLUX_FOR_EACH_LEG_END(l);
-
-  /* If no sendable leg was found, don't send on any circuit. */
-  if (!leg) {
-    return NULL;
-  }
-  return leg->circ;
-}
-
-/**
- * Favor the circuit with the highest send rate that still has space
- * in the congestion window, but when it is full, pick the next
- * highest.
- */
-static const circuit_t *
-conflux_decide_circ_highrate(const conflux_t *cfx)
-{
-  uint64_t max_rate = 0;
-  uint64_t primary_leg_rtt = 0;
-  const conflux_leg_t *leg = NULL;
-
-  /* Find the highest bandwidth leg */
-  CONFLUX_FOR_EACH_LEG_BEGIN(cfx, l) {
-    uint64_t rate;
-    const congestion_control_t *cc = circuit_ccontrol(l->circ);
-
-    rate = CELL_MAX_NETWORK_SIZE*USEC_PER_SEC *
-              cc->cwnd / l->circ_rtts_usec;
-
-    if (rate > max_rate) {
-      max_rate = rate;
-      primary_leg_rtt = l->circ_rtts_usec;
-      leg = l;
-    }
-  } CONFLUX_FOR_EACH_LEG_END(l);
-
-  /* If the package window is has room, use it */
-  if (leg && circuit_ready_to_send(leg->circ)) {
-    return leg->circ;
-  }
-
-  /* Reset the max rate to find a new max */
-  max_rate = 0;
-  leg = NULL;
-
-  /* For any given leg, it has primary_leg_rtt/2 time before the 'primary'
-   * leg's acks start arriving. So, the amount of data a 'secondary'
-   * leg can send while the primary leg transmits these acks is:
-   *   (cwnd_leg/(secondary_rtt/2))*primary_rtt/2
-   *     = cwnd_leg*primary_rtt/secondary_rtt.
-   * So any leg with available room below that that is no good.
-   */
-  CONFLUX_FOR_EACH_LEG_BEGIN(cfx, l) {
-    if (!circuit_ready_to_send(l->circ)) {
-      continue;
-    }
-    const congestion_control_t *cc = circuit_ccontrol(l->circ);
-
-    uint64_t rate = CELL_MAX_NETWORK_SIZE*USEC_PER_SEC *
-                    cc->cwnd / l->circ_rtts_usec;
-
-    /* Pick the leg with the highest rate that still has room */
-    if (rate > max_rate &&
-        cwnd_sendable(l->circ, primary_leg_rtt, l->circ_rtts_usec) <=
-        cwnd_available(l->circ)) {
-      leg = l;
-      max_rate = rate;
-    }
-  } CONFLUX_FOR_EACH_LEG_END(l);
-
-  /* If no sendable leg was found, don't send on any circuit. */
-  if (!leg) {
-    return NULL;
-  }
-  return leg->circ;
-}
-
 /**
  * This function is called when we want to send a relay cell on a
  * conflux, as well as when we want to compute available space in
@@ -611,9 +477,12 @@ conflux_decide_circ_for_send(conflux_t *cfx,
     tor_assert(cfx->curr_leg);
 
     if (new_circ != cfx->curr_leg->circ) {
-      cfx->cells_until_switch =
+      // TODO-329-TUNING: This is one mechanism to rate limit switching,
-        cwnd_sendable(new_circ,cfx->curr_leg->circ_rtts_usec,
+      // which should reduce the OOQ mem. However, we're not going to do that
-                                 new_leg->circ_rtts_usec);
+      // until we get some data on if the memory usage is high
+      cfx->cells_until_switch = 0;
+        //cwnd_sendable(new_circ,cfx->curr_leg->circ_rtts_usec,
+        //                         new_leg->circ_rtts_usec);
 
       conflux_validate_stream_lists(cfx);
 
@@ -686,13 +555,10 @@ conflux_pick_first_leg(conflux_t *cfx)
     tor_assert(min_leg);
   }
 
-  // TODO-329-TUNING: Does this create an edge condition by getting blocked,
+  // TODO-329-TUNING: We may want to initialize this to a cwnd, to
-  // is it possible that we get full before this point and block?
+  // minimize early switching?
-  // Esp if we switch to a new circuit that is not ready to
+  //cfx->cells_until_switch = circuit_ccontrol(min_leg->circ)->cwnd;
-  // send because it has unacked inflight data.... This might cause
+  cfx->cells_until_switch = 0;
-  // stalls?
-  // That is the thinking with this -1 here, but maybe it is not needed.
-  cfx->cells_until_switch = circuit_ccontrol(min_leg->circ)->cwnd - 1;
 
   cfx->curr_leg = min_leg;
 }
@@ -736,10 +602,6 @@ conflux_decide_next_circ(conflux_t *cfx)
       return (circuit_t*)conflux_decide_circ_lowrtt(cfx);
     case CONFLUX_ALG_CWNDRTT: // throughput (low oooq)
       return (circuit_t*)conflux_decide_circ_cwndrtt(cfx);
-    case CONFLUX_ALG_MAXRATE: // perf test (likely high ooq)
-      return (circuit_t*)conflux_decide_circ_maxrate(cfx);
-    case CONFLUX_ALG_HIGHRATE: // perf test (likely high ooq)
-      return (circuit_t*)conflux_decide_circ_highrate(cfx);
     default:
       return NULL;
   }

src/core/or/conflux_cell.c

@@ -266,22 +266,13 @@ conflux_cell_parse_linked(const cell_t *cell, const uint16_t cell_len)
 
 conflux_cell_link_t *
 conflux_cell_new_link(const uint8_t *nonce, uint64_t last_seqno_sent,
-                      uint64_t last_seqno_recv, bool is_client)
+                      uint64_t last_seqno_recv, uint8_t ux)
 {
   conflux_cell_link_t *link = tor_malloc_zero(sizeof(*link));
 
   link->version = 0x01;
-  if (is_client) {
+  link->desired_ux = ux;
-    // TODO-329-TUNING: The default should probably be high-throughput,
+
-    // but mobile clients may want to use low-memory.. We may also want
-    // to move this choice upstairs, so that torrc can control it.
-    link->desired_ux = CONFLUX_UX_HIGH_THROUGHPUT;
-  } else {
-    // TODO-329-TUNING: For exits, the default should be min-latency
-    // but we need to fix the tests and evaluate this first.
-    //link->desired_ux = CONFLUX_UX_MIN_LATENCY;
-    link->desired_ux = CONFLUX_UX_HIGH_THROUGHPUT;
-  }
   link->last_seqno_sent = last_seqno_sent;
   link->last_seqno_recv = last_seqno_recv;
   memcpy(link->nonce, nonce, sizeof(link->nonce));

src/core/or/conflux_cell.h

@@ -23,7 +23,7 @@ typedef struct conflux_cell_link_t {
 conflux_cell_link_t *conflux_cell_new_link(const uint8_t *nonce,
                                            uint64_t last_sent,
                                            uint64_t last_recv,
-                                           bool is_client);
+                                           uint8_t ux);
 
 conflux_cell_link_t *conflux_cell_parse_link(const cell_t *cell,
                                              const uint16_t cell_len);

src/core/or/conflux_pool.c

@@ -36,6 +36,7 @@
 
 #include "feature/nodelist/nodelist.h"
 #include "feature/client/bridges.h"
+#include "app/config/config.h"
 
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
@@ -130,6 +131,11 @@ get_linked_pool(bool is_client)
 }
 #endif
 
+/* For unit tests only: please treat these exactly as the defines in the
+ * code. */
+STATIC uint8_t DEFAULT_CLIENT_UX = CONFLUX_UX_HIGH_THROUGHPUT;
+STATIC uint8_t DEFAULT_EXIT_UX = CONFLUX_UX_MIN_LATENCY;
+
 /** Helper: Format at 8 bytes the nonce for logging. */
 static inline const char *
 fmt_nonce(const uint8_t *nonce)
@@ -143,18 +149,19 @@ fmt_nonce(const uint8_t *nonce)
 static uint8_t
 conflux_choose_algorithm(uint8_t desired_ux)
 {
-  /* TODO-329-TUNING: Pick better algs here*/
   switch (desired_ux) {
     case CONFLUX_UX_NO_OPINION:
       return CONFLUX_ALG_LOWRTT;
     case CONFLUX_UX_MIN_LATENCY:
       return CONFLUX_ALG_MINRTT;
-    case CONFLUX_UX_LOW_MEM_LATENCY:
-      return CONFLUX_ALG_MINRTT;
-    case CONFLUX_UX_LOW_MEM_THROUGHPUT:
-      return CONFLUX_ALG_CWNDRTT;
     case CONFLUX_UX_HIGH_THROUGHPUT:
       return CONFLUX_ALG_LOWRTT;
+    /* For now, we have no low mem algs, so use minRTT since it should
+     * switch less and thus use less mem */
+    /* TODO-329-TUNING: Pick better algs here*/
+    case CONFLUX_UX_LOW_MEM_THROUGHPUT:
+    case CONFLUX_UX_LOW_MEM_LATENCY:
+      return CONFLUX_ALG_MINRTT;
     default:
       /* Trunnel should protect us from this */
       tor_assert_nonfatal_unreached();
@@ -1016,6 +1023,24 @@ get_exit_for_nonce(const uint8_t *nonce)
   return exit;
 }
 
+/**
+ * Return the currently configured client UX.
+ */
+static uint8_t
+get_client_ux(void)
+{
+#ifdef TOR_UNIT_TESTS
+  return DEFAULT_CLIENT_UX;
+#else
+  const or_options_t *opt = get_options();
+  tor_assert(opt);
+  (void)DEFAULT_CLIENT_UX;
+
+  /* Return the UX */
+  return opt->ConfluxClientUX;
+#endif
+}
+
 /** Return true iff the given conflux object is allowed to launch a new leg. If
  * the cfx object is NULL, then it is always allowed to launch a new leg. */
 static bool
@@ -1109,7 +1134,7 @@ conflux_launch_leg(const uint8_t *nonce)
   leg_t *leg = leg_new(TO_CIRCUIT(circ),
                        conflux_cell_new_link(nonce,
                                              last_seq_sent, last_seq_recv,
-                                             true));
+                                             get_client_ux()));
 
   /* Increase the retry count for this conflux object as in this nonce. */
   unlinked->cfx->num_leg_launch++;
@@ -1760,8 +1785,10 @@ conflux_process_link(circuit_t *circ, const cell_t *cell,
     goto end;
   }
 
+  /* Exits should always request min latency from clients */
   conflux_cell_link_t *linked = conflux_cell_new_link(nonce, last_seq_sent,
-                                                      last_seq_recv, false);
+                                                      last_seq_recv,
+                                                      DEFAULT_EXIT_UX);
 
   conflux_cell_send_linked(linked, TO_OR_CIRCUIT(circ));
   tor_free(linked);

src/core/or/conflux_pool.h

@@ -40,6 +40,8 @@ void conflux_process_linked_ack(circuit_t *circ);
 bool launch_new_set(int num_legs);
 digest256map_t *get_linked_pool(bool is_client);
 digest256map_t *get_unlinked_pool(bool is_client);
+extern uint8_t DEFAULT_CLIENT_UX;
+extern uint8_t DEFAULT_EXIT_UX;
 #endif /* defined(UNIT_TESTS) */
 
 #endif /* TOR_CONFLUX_POOL_H */

src/core/or/conflux_st.h

@@ -20,8 +20,6 @@ typedef enum {
  CONFLUX_ALG_MINRTT = 0,
  CONFLUX_ALG_LOWRTT = 1,
  CONFLUX_ALG_CWNDRTT = 2,
- CONFLUX_ALG_MAXRATE = 3,
- CONFLUX_ALG_HIGHRATE = 4
 } conflux_alg_t;
 
 /**

src/test/test_conflux_pool.c

@@ -53,6 +53,7 @@
 #include "core/or/conflux_params.h"
 #include "core/or/conflux.h"
 #include "core/or/conflux_st.h"
+#include "trunnel/conflux.h"
 #include "lib/crypt_ops/crypto_rand.h"
 
 /* Start our monotime mocking at 1 second past whatever monotime_init()
@@ -1112,6 +1113,7 @@ test_conflux_switch(void *arg)
 {
    (void) arg;
   test_setup();
+  DEFAULT_EXIT_UX = CONFLUX_UX_HIGH_THROUGHPUT;
 
   launch_new_set(2);