Merge branch 'bug6816_squashed_nowarn' of git://git.torproject.org/nickm/tor

changes/bug6465

@@ -0,0 +1,12 @@
+  o Infrastructure features:
+    - Introduce new channel_t abstraction between circuits and or_connection_t
+      to allow for implementing alternate OR-to-OR transports.  A channel_t is
+      an abstract object which can either be a cell-bearing channel, which is
+      responsible for authenticating and handshaking with the remote OR and
+      transmitting cells to and from it, or a listening channel, which spawns
+      new cell-bearing channels at the request of remote ORs.
+
+    - Also new is the channel_tls_t subclass of channel_t, adapting it to the
+      existing or_connection_t code.  The V2/V3 protocol handshaking code
+      which formerly resided in command.c has been moved below the channel_t
+      abstraction layer and may be found in channeltls.c now.

changes/bug6816

@@ -0,0 +1,6 @@
+  o Infrastructure features:
+    - Introduce new circuitmux_t storing the queue of circuits for a channel;
+      this encapsulates and abstracts the queue logic and circuit selection
+      policy, and allows the latter to be overridden easily by switching out
+      a policy object.  The existing EWMA behavior is now implemented as a
+      circuitmux_policy_t.  This fixes bug 6816.

src/common/crypto.c

@@ -244,8 +244,10 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir)
     }
 
     if (SSLeay() < OPENSSL_V_SERIES(1,0,0)) {
-      log_notice(LD_CRYPTO, "Your OpenSSL version seems to be %s. We "
-                 "recommend 1.0.0 or later.", crypto_openssl_get_version_str());
+      log_notice(LD_CRYPTO,
+                 "Your OpenSSL version seems to be %s. We recommend 1.0.0 "
+                 "or later.",
+                 crypto_openssl_get_version_str());
     }
 
     if (useAccel > 0) {

src/common/log.c

@@ -907,7 +907,7 @@ log_level_to_string(int level)
 static const char *domain_list[] = {
   "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM",
   "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV",
-  "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", NULL
+  "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", NULL
 };
 
 /** Return a bitmask for the log domain for which <b>domain</b> is the name,

src/common/torlog.h

@@ -94,8 +94,10 @@
 #define LD_HANDSHAKE (1u<<19)
 /** Heartbeat messages */
 #define LD_HEARTBEAT (1u<<20)
+/** Abstract channel_t code */
+#define LD_CHANNEL   (1u<<21)
 /** Number of logging domains in the code. */
-#define N_LOGGING_DOMAINS 21
+#define N_LOGGING_DOMAINS 22
 
 /** This log message is not safe to send to a callback-based logger
  * immediately.  Used as a flag, not a log domain. */

src/or/Makefile.nmake

@@ -8,15 +8,15 @@ LIBS = ..\..\..\build-alpha\lib\libevent.a \
  ..\..\..\build-alpha\lib\libz.a \
  ws2_32.lib advapi32.lib shell32.lib
 
-LIBTOR_OBJECTS = buffers.obj circuitbuild.obj circuitlist.obj circuituse.obj \
+LIBTOR_OBJECTS = buffers.obj channel.obj channeltls.obj circuitbuild.obj \
+	circuitlist.obj circuitmux.obj circuitmux_ewma.obj circuituse.obj \
 	command.obj config.obj connection.obj connection_edge.obj \
 	connection_or.obj control.obj cpuworker.obj directory.obj \
-	dirserv.obj dirvote.obj dns.obj dnsserv.obj geoip.obj \
-	hibernate.obj main.obj microdesc.obj networkstatus.obj \
-	nodelist.obj onion.obj policies.obj reasons.obj relay.obj \
-	rendclient.obj rendcommon.obj rendmid.obj rendservice.obj \
-	rephist.obj router.obj routerlist.obj routerparse.obj status.obj \
-	config_codedigest.obj ntmain.obj
+	dirserv.obj dirvote.obj dns.obj dnsserv.obj geoip.obj hibernate.obj \
+	main.obj microdesc.obj networkstatus.obj nodelist.obj onion.obj \
+	policies.obj reasons.obj relay.obj rendclient.obj rendcommon.obj \
+	rendmid.obj rendservice.obj rephist.obj router.obj routerlist.obj \
+	routerparse.obj status.obj config_codedigest.obj ntmain.obj
 
 libtor.lib: $(LIBTOR_OBJECTS)
 	lib $(LIBTOR_OBJECTS) /out:libtor.lib

src/or/channel.h

@@ -0,0 +1,471 @@
+/* * Copyright (c) 2012, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file channel.h
+ * \brief Header file for channel.c
+ **/
+
+#ifndef _TOR_CHANNEL_H
+#define _TOR_CHANNEL_H
+
+#include "or.h"
+#include "circuitmux.h"
+
+/* Channel handler function pointer typedefs */
+typedef void (*channel_listener_fn_ptr)(channel_listener_t *, channel_t *);
+typedef void (*channel_cell_handler_fn_ptr)(channel_t *, cell_t *);
+typedef void (*channel_var_cell_handler_fn_ptr)(channel_t *, var_cell_t *);
+
+/*
+ * Channel struct; see the channel_t typedef in or.h.  A channel is an
+ * abstract interface for the OR-to-OR connection, similar to connection_or_t,
+ * but without the strong coupling to the underlying TLS implementation.  They
+ * are constructed by calling a protocol-specific function to open a channel
+ * to a particular node, and once constructed support the abstract operations
+ * defined below.
+ */
+
+struct channel_s {
+  /* Magic number for type-checking cast macros */
+  uint32_t magic;
+
+  /* Current channel state */
+  channel_state_t state;
+
+  /* Globally unique ID number for a channel over the lifetime of a Tor
+   * process.
+   */
+  uint64_t global_identifier;
+
+  /* Should we expect to see this channel in the channel lists? */
+  unsigned char registered:1;
+
+  /** Why did we close?
+   */
+  enum {
+    CHANNEL_NOT_CLOSING = 0,
+    CHANNEL_CLOSE_REQUESTED,
+    CHANNEL_CLOSE_FROM_BELOW,
+    CHANNEL_CLOSE_FOR_ERROR
+  } reason_for_closing;
+
+  /* Timestamps for both cell channels and listeners */
+  time_t timestamp_created; /* Channel created */
+  time_t timestamp_active; /* Any activity */
+
+  /* Methods implemented by the lower layer */
+
+  /* Free a channel */
+  void (*free)(channel_t *);
+  /* Close an open channel */
+  void (*close)(channel_t *);
+  /* Describe the transport subclass for this channel */
+  const char * (*describe_transport)(channel_t *);
+  /* Optional method to dump transport-specific statistics on the channel */
+  void (*dumpstats)(channel_t *, int);
+
+  /* Registered handlers for incoming cells */
+  channel_cell_handler_fn_ptr cell_handler;
+  channel_var_cell_handler_fn_ptr var_cell_handler;
+
+  /* Methods implemented by the lower layer */
+
+  /*
+   * Ask the underlying transport what the remote endpoint address is, in
+   * a tor_addr_t.  This is optional and subclasses may leave this NULL.
+   * If they implement it, they should write the address out to the
+   * provided tor_addr_t *, and return 1 if successful or 0 if no address
+   * available.
+   */
+  int (*get_remote_addr)(channel_t *, tor_addr_t *);
+  /*
+   * Get a text description of the remote endpoint; canonicalized if the
+   * arg is 0, or the one we originally connected to/received from if it's
+   * 1.
+   */
+  const char * (*get_remote_descr)(channel_t *, int);
+  /* Check if the lower layer has queued writes */
+  int (*has_queued_writes)(channel_t *);
+  /*
+   * If the second param is zero, ask the lower layer if this is
+   * 'canonical', for a transport-specific definition of canonical; if
+   * it is 1, ask if the answer to the preceding query is safe to rely
+   * on.
+   */
+  int (*is_canonical)(channel_t *, int);
+  /* Check if this channel matches a specified extend_info_t */
+  int (*matches_extend_info)(channel_t *, extend_info_t *);
+  /* Check if this channel matches a target address when extending */
+  int (*matches_target)(channel_t *, const tor_addr_t *);
+  /* Write a cell to an open channel */
+  int (*write_cell)(channel_t *, cell_t *);
+   /* Write a packed cell to an open channel */
+  int (*write_packed_cell)(channel_t *, packed_cell_t *);
+  /* Write a variable-length cell to an open channel */
+  int (*write_var_cell)(channel_t *, var_cell_t *);
+
+  /*
+   * Hash of the public RSA key for the other side's identity key, or
+   * zeroes if the other side hasn't shown us a valid identity key.
+   */
+  char identity_digest[DIGEST_LEN];
+  /* Nickname of the OR on the other side, or NULL if none. */
+  char *nickname;
+
+  /*
+   * Linked list of channels with the same identity digest, for the
+   * digest->channel map
+   */
+  channel_t *next_with_same_id, *prev_with_same_id;
+
+  /* List of incoming cells to handle */
+  smartlist_t *incoming_queue;
+
+  /* List of queued outgoing cells */
+  smartlist_t *outgoing_queue;
+
+  /* Circuit mux for circuits sending on this channel */
+  circuitmux_t *cmux;
+
+  /* Circuit ID generation stuff for use by circuitbuild.c */
+
+  /*
+   * When we send CREATE cells along this connection, which half of the
+   * space should we use?
+   */
+  circ_id_type_t circ_id_type:2;
+  /*
+   * Which circ_id do we try to use next on this connection?  This is
+   * always in the range 0..1<<15-1.
+   */
+  circid_t next_circ_id;
+
+  /* For how many circuits are we n_chan?  What about p_chan? */
+  unsigned int num_n_circuits, num_p_circuits;
+
+  /*
+   * True iff this channel shouldn't get any new circs attached to it,
+   * because the connection is too old, or because there's a better one.
+   * More generally, this flag is used to note an unhealthy connection;
+   * for example, if a bad connection fails we shouldn't assume that the
+   * router itself has a problem.
+   */
+  unsigned int is_bad_for_new_circs:1;
+
+  /** True iff we have decided that the other end of this connection
+   * is a client.  Channels with this flag set should never be used
+   * to satisfy an EXTEND request.  */
+  unsigned int is_client:1;
+
+  /** Set if the channel was initiated remotely (came from a listener) */
+  unsigned int is_incoming:1;
+
+  /** Set by lower layer if this is local; i.e., everything it communicates
+   * with for this channel returns true for is_local_addr().  This is used
+   * to decide whether to declare reachability when we receive something on
+   * this channel in circuitbuild.c
+   */
+  unsigned int is_local:1;
+
+  /** Channel timestamps for cell channels */
+  time_t timestamp_client; /* Client used this, according to relay.c */
+  time_t timestamp_drained; /* Output queue empty */
+  time_t timestamp_recv; /* Cell received from lower layer */
+  time_t timestamp_xmit; /* Cell sent to lower layer */
+
+  /* Timestamp for relay.c */
+  time_t timestamp_last_added_nonpadding;
+
+  /** Unique ID for measuring direct network status requests;vtunneled ones
+   * come over a circuit_t, which has a dirreq_id field as well, but is a
+   * distinct namespace. */
+  uint64_t dirreq_id;
+
+  /** Channel counters for cell channels */
+  uint64_t n_cells_recved;
+  uint64_t n_cells_xmitted;
+};
+
+struct channel_listener_s {
+  /* Current channel listener state */
+  channel_listener_state_t state;
+
+  /* Globally unique ID number for a channel over the lifetime of a Tor
+   * process.
+   */
+  uint64_t global_identifier;
+
+  /* Should we expect to see this channel in the channel lists? */
+  unsigned char registered:1;
+
+  /** Why did we close?
+   */
+  enum {
+    CHANNEL_LISTENER_NOT_CLOSING = 0,
+    CHANNEL_LISTENER_CLOSE_REQUESTED,
+    CHANNEL_LISTENER_CLOSE_FROM_BELOW,
+    CHANNEL_LISTENER_CLOSE_FOR_ERROR
+  } reason_for_closing;
+
+  /* Timestamps for both cell channels and listeners */
+  time_t timestamp_created; /* Channel created */
+  time_t timestamp_active; /* Any activity */
+
+  /* Methods implemented by the lower layer */
+
+  /* Free a channel */
+  void (*free)(channel_listener_t *);
+  /* Close an open channel */
+  void (*close)(channel_listener_t *);
+  /* Describe the transport subclass for this channel */
+  const char * (*describe_transport)(channel_listener_t *);
+  /* Optional method to dump transport-specific statistics on the channel */
+  void (*dumpstats)(channel_listener_t *, int);
+
+  /* Registered listen handler to call on incoming connection */
+  channel_listener_fn_ptr listener;
+
+  /* List of pending incoming connections */
+  smartlist_t *incoming_list;
+
+  /* Timestamps for listeners */
+  time_t timestamp_accepted;
+
+  /* Counters for listeners */
+  uint64_t n_accepted;
+};
+
+/* Channel state manipulations */
+
+int channel_state_is_valid(channel_state_t state);
+int channel_listener_state_is_valid(channel_listener_state_t state);
+
+int channel_state_can_transition(channel_state_t from, channel_state_t to);
+int channel_listener_state_can_transition(channel_listener_state_t from,
+                                          channel_listener_state_t to);
+
+const char * channel_state_to_string(channel_state_t state);
+const char *
+channel_listener_state_to_string(channel_listener_state_t state);
+
+/* Abstract channel operations */
+
+void channel_mark_for_close(channel_t *chan);
+void channel_write_cell(channel_t *chan, cell_t *cell);
+void channel_write_packed_cell(channel_t *chan, packed_cell_t *cell);
+void channel_write_var_cell(channel_t *chan, var_cell_t *cell);
+
+void channel_listener_mark_for_close(channel_listener_t *chan_l);
+
+/* Channel callback registrations */
+
+/* Listener callback */
+channel_listener_fn_ptr
+channel_listener_get_listener_fn(channel_listener_t *chan);
+
+void channel_listener_set_listener_fn(channel_listener_t *chan,
+                                      channel_listener_fn_ptr listener);
+
+/* Incoming cell callbacks */
+channel_cell_handler_fn_ptr channel_get_cell_handler(channel_t *chan);
+
+channel_var_cell_handler_fn_ptr
+channel_get_var_cell_handler(channel_t *chan);
+
+void channel_set_cell_handlers(channel_t *chan,
+                               channel_cell_handler_fn_ptr cell_handler,
+                               channel_var_cell_handler_fn_ptr
+                                 var_cell_handler);
+
+/* Clean up closed channels and channel listeners periodically; these are
+ * called from run_scheduled_events() in main.c.
+ */
+void channel_run_cleanup(void);
+void channel_listener_run_cleanup(void);
+
+/* Close all channels and deallocate everything */
+void channel_free_all(void);
+
+/* Dump some statistics in the log */
+void channel_dumpstats(int severity);
+void channel_listener_dumpstats(int severity);
+
+/* Set the cmux policy on all active channels */
+void channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol);
+
+#ifdef _TOR_CHANNEL_INTERNAL
+
+/* Channel operations for subclasses and internal use only */
+
+/* Initialize a newly allocated channel - do this first in subclass
+ * constructors.
+ */
+
+void channel_init(channel_t *chan);
+void channel_init_listener(channel_listener_t *chan);
+
+/* Channel registration/unregistration */
+void channel_register(channel_t *chan);
+void channel_unregister(channel_t *chan);
+
+/* Channel listener registration/unregistration */
+void channel_listener_register(channel_listener_t *chan_l);
+void channel_listener_unregister(channel_listener_t *chan_l);
+
+/* Close from below */
+void channel_close_from_lower_layer(channel_t *chan);
+void channel_close_for_error(channel_t *chan);
+void channel_closed(channel_t *chan);
+
+void channel_listener_close_from_lower_layer(channel_listener_t *chan_l);
+void channel_listener_close_for_error(channel_listener_t *chan_l);
+void channel_listener_closed(channel_listener_t *chan_l);
+
+/* Free a channel */
+void channel_free(channel_t *chan);
+void channel_listener_free(channel_listener_t *chan_l);
+
+/* State/metadata setters */
+
+void channel_change_state(channel_t *chan, channel_state_t to_state);
+void channel_clear_identity_digest(channel_t *chan);
+void channel_clear_remote_end(channel_t *chan);
+void channel_mark_local(channel_t *chan);
+void channel_mark_incoming(channel_t *chan);
+void channel_mark_outgoing(channel_t *chan);
+void channel_set_identity_digest(channel_t *chan,
+                                 const char *identity_digest);
+void channel_set_remote_end(channel_t *chan,
+                            const char *identity_digest,
+                            const char *nickname);
+
+void channel_listener_change_state(channel_listener_t *chan_l,
+                                   channel_listener_state_t to_state);
+
+/* Timestamp updates */
+void channel_timestamp_created(channel_t *chan);
+void channel_timestamp_active(channel_t *chan);
+void channel_timestamp_drained(channel_t *chan);
+void channel_timestamp_recv(channel_t *chan);
+void channel_timestamp_xmit(channel_t *chan);
+
+void channel_listener_timestamp_created(channel_listener_t *chan_l);
+void channel_listener_timestamp_active(channel_listener_t *chan_l);
+void channel_listener_timestamp_accepted(channel_listener_t *chan_l);
+
+/* Incoming channel handling */
+void channel_listener_process_incoming(channel_listener_t *listener);
+void channel_listener_queue_incoming(channel_listener_t *listener,
+                                     channel_t *incoming);
+
+/* Incoming cell handling */
+void channel_process_cells(channel_t *chan);
+void channel_queue_cell(channel_t *chan, cell_t *cell);
+void channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell);
+
+/* Outgoing cell handling */
+void channel_flush_cells(channel_t *chan);
+
+/* Request from lower layer for more cells if available */
+ssize_t channel_flush_some_cells(channel_t *chan, ssize_t num_cells);
+
+/* Query if data available on this channel */
+int channel_more_to_flush(channel_t *chan);
+
+/* Notify flushed outgoing for dirreq handling */
+void channel_notify_flushed(channel_t *chan);
+
+/* Handle stuff we need to do on open like notifying circuits */
+void channel_do_open_actions(channel_t *chan);
+
+#endif
+
+/* Helper functions to perform operations on channels */
+
+int channel_send_destroy(circid_t circ_id, channel_t *chan,
+                         int reason);
+
+/*
+ * Outside abstract interfaces that should eventually get turned into
+ * something transport/address format independent.
+ */
+
+channel_t * channel_connect(const tor_addr_t *addr, uint16_t port,
+                            const char *id_digest);
+
+channel_t * channel_get_for_extend(const char *digest,
+                                   const tor_addr_t *target_addr,
+                                   const char **msg_out,
+                                   int *launch_out);
+
+/* Ask which of two channels is better for circuit-extension purposes */
+int channel_is_better(time_t now,
+                      channel_t *a, channel_t *b,
+                      int forgive_new_connections);
+
+/** Channel lookups
+ */
+
+channel_t * channel_find_by_global_id(uint64_t global_identifier);
+channel_t * channel_find_by_remote_digest(const char *identity_digest);
+
+/** For things returned by channel_find_by_remote_digest(), walk the list.
+ */
+
+channel_t * channel_next_with_digest(channel_t *chan);
+channel_t * channel_prev_with_digest(channel_t *chan);
+
+/*
+ * Metadata queries/updates
+ */
+
+const char * channel_describe_transport(channel_t *chan);
+void channel_dump_statistics(channel_t *chan, int severity);
+void channel_dump_transport_statistics(channel_t *chan, int severity);
+const char * channel_get_actual_remote_descr(channel_t *chan);
+int channel_get_addr_if_possible(channel_t *chan, tor_addr_t *addr_out);
+const char * channel_get_canonical_remote_descr(channel_t *chan);
+int channel_has_queued_writes(channel_t *chan);
+int channel_is_bad_for_new_circs(channel_t *chan);
+void channel_mark_bad_for_new_circs(channel_t *chan);
+int channel_is_canonical(channel_t *chan);
+int channel_is_canonical_is_reliable(channel_t *chan);
+int channel_is_client(channel_t *chan);
+int channel_is_local(channel_t *chan);
+int channel_is_incoming(channel_t *chan);
+int channel_is_outgoing(channel_t *chan);
+void channel_mark_client(channel_t *chan);
+int channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info);
+int channel_matches_target_addr_for_extend(channel_t *chan,
+                                           const tor_addr_t *target);
+unsigned int channel_num_circuits(channel_t *chan);
+void channel_set_circid_type(channel_t *chan, crypto_pk_t *identity_rcvd);
+void channel_timestamp_client(channel_t *chan);
+
+const char * channel_listener_describe_transport(channel_listener_t *chan_l);
+void channel_listener_dump_statistics(channel_listener_t *chan_l,
+                                      int severity);
+void channel_listener_dump_transport_statistics(channel_listener_t *chan_l,
+                                                int severity);
+
+/* Timestamp queries */
+time_t channel_when_created(channel_t *chan);
+time_t channel_when_last_active(channel_t *chan);
+time_t channel_when_last_client(channel_t *chan);
+time_t channel_when_last_drained(channel_t *chan);
+time_t channel_when_last_recv(channel_t *chan);
+time_t channel_when_last_xmit(channel_t *chan);
+
+time_t channel_listener_when_created(channel_listener_t *chan_l);
+time_t channel_listener_when_last_active(channel_listener_t *chan_l);
+time_t channel_listener_when_last_accepted(channel_listener_t *chan_l);
+
+/* Counter queries */
+uint64_t channel_count_recved(channel_t *chan);
+uint64_t channel_count_xmitted(channel_t *chan);
+
+uint64_t channel_listener_count_accepted(channel_listener_t *chan_l);
+
+#endif
+

src/or/channeltls.h

@@ -0,0 +1,57 @@
+/* * Copyright (c) 2012, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file channeltls.h
+ * \brief Header file for channeltls.c
+ **/
+
+#ifndef _TOR_CHANNEL_TLS_H
+#define _TOR_CHANNEL_TLS_H
+
+#include "or.h"
+#include "channel.h"
+
+#define BASE_CHAN_TO_TLS(c) (channel_tls_from_base((c)))
+#define TLS_CHAN_TO_BASE(c) (channel_tls_to_base((c)))
+
+#define TLS_CHAN_MAGIC 0x8a192427U
+
+#ifdef _TOR_CHANNEL_INTERNAL
+
+struct channel_tls_s {
+  /* Base channel_t struct */
+  channel_t _base;
+  /* or_connection_t pointer */
+  or_connection_t *conn;
+};
+
+#endif /* _TOR_CHANNEL_INTERNAL */
+
+channel_t * channel_tls_connect(const tor_addr_t *addr, uint16_t port,
+                                const char *id_digest);
+channel_listener_t * channel_tls_get_listener(void);
+channel_listener_t * channel_tls_start_listener(void);
+channel_t * channel_tls_handle_incoming(or_connection_t *orconn);
+
+/* Casts */
+
+channel_t * channel_tls_to_base(channel_tls_t *tlschan);
+channel_tls_t * channel_tls_from_base(channel_t *chan);
+
+/* Things for connection_or.c to call back into */
+ssize_t channel_tls_flush_some_cells(channel_tls_t *chan, ssize_t num_cells);
+int channel_tls_more_to_flush(channel_tls_t *chan);
+void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn);
+void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
+                                               or_connection_t *conn,
+                                               uint8_t old_state,
+                                               uint8_t state);
+void channel_tls_handle_var_cell(var_cell_t *var_cell,
+                                 or_connection_t *conn);
+
+/* Cleanup at shutdown */
+void channel_tls_free_all(void);
+
+#endif
+

src/or/circuitbuild.c

@@ -12,9 +12,11 @@
 #define CIRCUIT_PRIVATE
 
 #include "or.h"
+#include "channel.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuituse.h"
+#include "command.h"
 #include "config.h"
 #include "confparse.h"
 #include "connection.h"
@@ -125,6 +127,9 @@ static int unit_tests = 0;
 
 /********* END VARIABLES ************/
 
+static channel_t * channel_connect_for_circuit(const tor_addr_t *addr,
+                                               uint16_t port,
+                                               const char *id_digest);
 static int circuit_deliver_create_cell(circuit_t *circ,
                                        uint8_t cell_type, const char *payload);
 static int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit);
@@ -141,6 +146,22 @@ static void bridge_free(bridge_info_t *bridge);
 static int entry_guard_inc_first_hop_count(entry_guard_t *guard);
 static void pathbias_count_success(origin_circuit_t *circ);
 
+/** This function tries to get a channel to the specified endpoint,
+ * and then calls command_setup_channel() to give it the right
+ * callbacks.
+ */
+static channel_t *
+channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port,
+                            const char *id_digest)
+{
+  channel_t *chan;
+
+  chan = channel_connect(addr, port, id_digest);
+  if (chan) command_setup_channel(chan);
+
+  return chan;
+}
+
 /**
  * This function decides if CBT learning should be disabled. It returns
  * true if one or more of the following four conditions are met:
@@ -1683,26 +1704,29 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt)
  * Return it, or 0 if can't get a unique circ_id.
  */
 static circid_t
-get_unique_circ_id_by_conn(or_connection_t *conn)
+get_unique_circ_id_by_chan(channel_t *chan)
 {
   circid_t test_circ_id;
   circid_t attempts=0;
   circid_t high_bit;
 
-  tor_assert(conn);
-  if (conn->circ_id_type == CIRC_ID_TYPE_NEITHER) {
-    log_warn(LD_BUG, "Trying to pick a circuit ID for a connection from "
+  tor_assert(chan);
+
+  if (chan->circ_id_type == CIRC_ID_TYPE_NEITHER) {
+    log_warn(LD_BUG,
+             "Trying to pick a circuit ID for a connection from "
              "a client with no identity.");
     return 0;
   }
-  high_bit = (conn->circ_id_type == CIRC_ID_TYPE_HIGHER) ? 1<<15 : 0;
+  high_bit =
+    (chan->circ_id_type == CIRC_ID_TYPE_HIGHER) ? 1<<15 : 0;
   do {
     /* Sequentially iterate over test_circ_id=1...1<<15-1 until we find a
      * circID such that (high_bit|test_circ_id) is not already used. */
-    test_circ_id = conn->next_circ_id++;
+    test_circ_id = chan->next_circ_id++;
     if (test_circ_id == 0 || test_circ_id >= 1<<15) {
       test_circ_id = 1;
-      conn->next_circ_id = 2;
+      chan->next_circ_id = 2;
     }
     if (++attempts > 1<<15) {
       /* Make sure we don't loop forever if all circ_id's are used. This
@@ -1712,7 +1736,7 @@ get_unique_circ_id_by_conn(or_connection_t *conn)
       return 0;
     }
     test_circ_id |= high_bit;
-  } while (circuit_id_in_use_on_orconn(test_circ_id, conn));
+  } while (circuit_id_in_use_on_channel(test_circ_id, chan));
   return test_circ_id;
 }
 
@@ -1891,9 +1915,9 @@ onion_populate_cpath(origin_circuit_t *circ)
 origin_circuit_t *
 origin_circuit_init(uint8_t purpose, int flags)
 {
-  /* sets circ->p_circ_id and circ->p_conn */
+  /* sets circ->p_circ_id and circ->p_chan */
   origin_circuit_t *circ = origin_circuit_new();
-  circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OR_WAIT);
+  circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_CHAN_WAIT);
   circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
   circ->build_state->onehop_tunnel =
     ((flags & CIRCLAUNCH_ONEHOP_TUNNEL) ? 1 : 0);
@@ -1945,7 +1969,7 @@ int
 circuit_handle_first_hop(origin_circuit_t *circ)
 {
   crypt_path_t *firsthop;
-  or_connection_t *n_conn;
+  channel_t *n_chan;
   int err_reason = 0;
   const char *msg = NULL;
   int should_launch = 0;
@@ -1959,12 +1983,12 @@ circuit_handle_first_hop(origin_circuit_t *circ)
             fmt_addr(&firsthop->extend_info->addr),
             firsthop->extend_info->port);
 
-  n_conn = connection_or_get_for_extend(firsthop->extend_info->identity_digest,
-                                        &firsthop->extend_info->addr,
-                                        &msg,
-                                        &should_launch);
+  n_chan = channel_get_for_extend(firsthop->extend_info->identity_digest,
+                                  &firsthop->extend_info->addr,
+                                  &msg,
+                                  &should_launch);
 
-  if (!n_conn) {
+  if (!n_chan) {
     /* not currently connected in a useful way. */
     log_info(LD_CIRC, "Next router is %s: %s",
              safe_str_client(extend_info_describe(firsthop->extend_info)),
@@ -1974,10 +1998,11 @@ circuit_handle_first_hop(origin_circuit_t *circ)
     if (should_launch) {
       if (circ->build_state->onehop_tunnel)
         control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
-      n_conn = connection_or_connect(&firsthop->extend_info->addr,
-                                     firsthop->extend_info->port,
-                                     firsthop->extend_info->identity_digest);
-      if (!n_conn) { /* connect failed, forget the whole thing */
+      n_chan = channel_connect_for_circuit(
+          &firsthop->extend_info->addr,
+          firsthop->extend_info->port,
+          firsthop->extend_info->identity_digest);
+      if (!n_chan) { /* connect failed, forget the whole thing */
         log_info(LD_CIRC,"connect to firsthop failed. Closing.");
         return -END_CIRC_REASON_CONNECTFAILED;
       }
@@ -1985,13 +2010,13 @@ circuit_handle_first_hop(origin_circuit_t *circ)
 
     log_debug(LD_CIRC,"connecting in progress (or finished). Good.");
     /* return success. The onion/circuit/etc will be taken care of
-     * automatically (may already have been) whenever n_conn reaches
+     * automatically (may already have been) whenever n_chan reaches
      * OR_CONN_STATE_OPEN.
      */
     return 0;
   } else { /* it's already open. use it. */
     tor_assert(!circ->_base.n_hop);
-    circ->_base.n_conn = n_conn;
+    circ->_base.n_chan = n_chan;
     log_debug(LD_CIRC,"Conn open. Delivering first onion skin.");
     if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
       log_info(LD_CIRC,"circuit_send_next_onion_skin failed.");
@@ -2007,48 +2032,49 @@ circuit_handle_first_hop(origin_circuit_t *circ)
  * Status is 1 if connect succeeded, or 0 if connect failed.
  */
 void
-circuit_n_conn_done(or_connection_t *or_conn, int status)
+circuit_n_chan_done(channel_t *chan, int status)
 {
   smartlist_t *pending_circs;
   int err_reason = 0;
 
-  log_debug(LD_CIRC,"or_conn to %s/%s, status=%d",
-            or_conn->nickname ? or_conn->nickname : "NULL",
-            or_conn->_base.address, status);
+  tor_assert(chan);
+
+  log_debug(LD_CIRC,"chan to %s/%s, status=%d",
+            chan->nickname ? chan->nickname : "NULL",
+            channel_get_canonical_remote_descr(chan), status);
 
   pending_circs = smartlist_new();
-  circuit_get_all_pending_on_or_conn(pending_circs, or_conn);
+  circuit_get_all_pending_on_channel(pending_circs, chan);
 
   SMARTLIST_FOREACH_BEGIN(pending_circs, circuit_t *, circ)
     {
       /* These checks are redundant wrt get_all_pending_on_or_conn, but I'm
        * leaving them in in case it's possible for the status of a circuit to
        * change as we're going down the list. */
-      if (circ->marked_for_close || circ->n_conn || !circ->n_hop ||
-          circ->state != CIRCUIT_STATE_OR_WAIT)
+      if (circ->marked_for_close || circ->n_chan || !circ->n_hop ||
+          circ->state != CIRCUIT_STATE_CHAN_WAIT)
         continue;
 
       if (tor_digest_is_zero(circ->n_hop->identity_digest)) {
         /* Look at addr/port. This is an unkeyed connection. */
-        if (!tor_addr_eq(&circ->n_hop->addr, &or_conn->_base.addr) ||
-            circ->n_hop->port != or_conn->_base.port)
+        if (!channel_matches_extend_info(chan, circ->n_hop))
           continue;
       } else {
         /* We expected a key. See if it's the right one. */
-        if (tor_memneq(or_conn->identity_digest,
+        if (tor_memneq(chan->identity_digest,
                    circ->n_hop->identity_digest, DIGEST_LEN))
           continue;
       }
-      if (!status) { /* or_conn failed; close circ */
-        log_info(LD_CIRC,"or_conn failed. Closing circ.");
-        circuit_mark_for_close(circ, END_CIRC_REASON_OR_CONN_CLOSED);
+      if (!status) { /* chan failed; close circ */
+        log_info(LD_CIRC,"Channel failed; closing circ.");
+        circuit_mark_for_close(circ, END_CIRC_REASON_CHANNEL_CLOSED);
         continue;
       }
       log_debug(LD_CIRC, "Found circ, sending create cell.");
       /* circuit_deliver_create_cell will set n_circ_id and add us to
-       * orconn_circuid_circuit_map, so we don't need to call
-       * set_circid_orconn here. */
-      circ->n_conn = or_conn;
+       * chan_circuid_circuit_map, so we don't need to call
+       * set_circid_chan here. */
+      circ->n_chan = chan;
       extend_info_free(circ->n_hop);
       circ->n_hop = NULL;
 
@@ -2064,13 +2090,13 @@ circuit_n_conn_done(or_connection_t *or_conn, int status)
         }
       } else {
         /* pull the create cell out of circ->onionskin, and send it */
-        tor_assert(circ->n_conn_onionskin);
+        tor_assert(circ->n_chan_onionskin);
         if (circuit_deliver_create_cell(circ,CELL_CREATE,
-                                        circ->n_conn_onionskin)<0) {
+                                        circ->n_chan_onionskin)<0) {
           circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT);
           continue;
         }
-        tor_free(circ->n_conn_onionskin);
+        tor_free(circ->n_chan_onionskin);
         circuit_set_state(circ, CIRCUIT_STATE_OPEN);
       }
     }
@@ -2079,7 +2105,7 @@ circuit_n_conn_done(or_connection_t *or_conn, int status)
   smartlist_free(pending_circs);
 }
 
-/** Find a new circid that isn't currently in use on the circ->n_conn
+/** Find a new circid that isn't currently in use on the circ->n_chan
  * for the outgoing
  * circuit <b>circ</b>, and deliver a cell of type <b>cell_type</b>
  * (either CELL_CREATE or CELL_CREATE_FAST) with payload <b>payload</b>
@@ -2094,29 +2120,29 @@ circuit_deliver_create_cell(circuit_t *circ, uint8_t cell_type,
   circid_t id;
 
   tor_assert(circ);
-  tor_assert(circ->n_conn);
+  tor_assert(circ->n_chan);
   tor_assert(payload);
   tor_assert(cell_type == CELL_CREATE || cell_type == CELL_CREATE_FAST);
 
-  id = get_unique_circ_id_by_conn(circ->n_conn);
+  id = get_unique_circ_id_by_chan(circ->n_chan);
   if (!id) {
     log_warn(LD_CIRC,"failed to get unique circID.");
     return -1;
   }
   log_debug(LD_CIRC,"Chosen circID %u.", id);
-  circuit_set_n_circid_orconn(circ, id, circ->n_conn);
+  circuit_set_n_circid_chan(circ, id, circ->n_chan);
 
   memset(&cell, 0, sizeof(cell_t));
   cell.command = cell_type;
   cell.circ_id = circ->n_circ_id;
 
   memcpy(cell.payload, payload, ONIONSKIN_CHALLENGE_LEN);
-  append_cell_to_circuit_queue(circ, circ->n_conn, &cell,
+  append_cell_to_circuit_queue(circ, circ->n_chan, &cell,
                                CELL_DIRECTION_OUT, 0);
 
   if (CIRCUIT_IS_ORIGIN(circ)) {
     /* mark it so it gets better rate limiting treatment. */
-    circ->n_conn->client_used = time(NULL);
+    channel_timestamp_client(circ->n_chan);
   }
 
   return 0;
@@ -2218,7 +2244,7 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
     else
       control_event_bootstrap(BOOTSTRAP_STATUS_CIRCUIT_CREATE, 0);
 
-    node = node_get_by_id(circ->_base.n_conn->identity_digest);
+    node = node_get_by_id(circ->_base.n_chan->identity_digest);
     fast = should_use_create_fast_for_circuit(circ);
     if (!fast) {
       /* We are an OR and we know the right onion key: we should
@@ -2386,7 +2412,7 @@ circuit_note_clock_jumped(int seconds_elapsed)
 int
 circuit_extend(cell_t *cell, circuit_t *circ)
 {
-  or_connection_t *n_conn;
+  channel_t *n_chan;
   relay_header_t rh;
   char *onionskin;
   char *id_digest=NULL;
@@ -2396,9 +2422,9 @@ circuit_extend(cell_t *cell, circuit_t *circ)
   const char *msg = NULL;
   int should_launch = 0;
 
-  if (circ->n_conn) {
+  if (circ->n_chan) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "n_conn already set. Bug/attack. Closing.");
+           "n_chan already set. Bug/attack. Closing.");
     return -1;
   }
   if (circ->n_hop) {
@@ -2457,19 +2483,20 @@ circuit_extend(cell_t *cell, circuit_t *circ)
   /* Next, check if we're being asked to connect to the hop that the
    * extend cell came from. There isn't any reason for that, and it can
    * assist circular-path attacks. */
-  if (tor_memeq(id_digest, TO_OR_CIRCUIT(circ)->p_conn->identity_digest,
-              DIGEST_LEN)) {
+  if (tor_memeq(id_digest,
+                TO_OR_CIRCUIT(circ)->p_chan->identity_digest,
+                DIGEST_LEN)) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
            "Client asked me to extend back to the previous hop.");
     return -1;
   }
 
-  n_conn = connection_or_get_for_extend(id_digest,
-                                        &n_addr,
-                                        &msg,
-                                        &should_launch);
+  n_chan = channel_get_for_extend(id_digest,
+                                  &n_addr,
+                                  &msg,
+                                  &should_launch);
 
-  if (!n_conn) {
+  if (!n_chan) {
     log_debug(LD_CIRC|LD_OR,"Next router (%s:%d): %s",
               fmt_addr(&n_addr), (int)n_port, msg?msg:"????");
 
@@ -2478,31 +2505,32 @@ circuit_extend(cell_t *cell, circuit_t *circ)
                                     NULL /*onion_key*/,
                                     &n_addr, n_port);
 
-    circ->n_conn_onionskin = tor_malloc(ONIONSKIN_CHALLENGE_LEN);
-    memcpy(circ->n_conn_onionskin, onionskin, ONIONSKIN_CHALLENGE_LEN);
-    circuit_set_state(circ, CIRCUIT_STATE_OR_WAIT);
+    circ->n_chan_onionskin = tor_malloc(ONIONSKIN_CHALLENGE_LEN);
+    memcpy(circ->n_chan_onionskin, onionskin, ONIONSKIN_CHALLENGE_LEN);
+    circuit_set_state(circ, CIRCUIT_STATE_CHAN_WAIT);
 
     if (should_launch) {
       /* we should try to open a connection */
-      n_conn = connection_or_connect(&n_addr, n_port, id_digest);
-      if (!n_conn) {
-        log_info(LD_CIRC,"Launching n_conn failed. Closing circuit.");
+      n_chan = channel_connect_for_circuit(&n_addr, n_port, id_digest);
+      if (!n_chan) {
+        log_info(LD_CIRC,"Launching n_chan failed. Closing circuit.");
         circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
         return 0;
       }
       log_debug(LD_CIRC,"connecting in progress (or finished). Good.");
     }
     /* return success. The onion/circuit/etc will be taken care of
-     * automatically (may already have been) whenever n_conn reaches
+     * automatically (may already have been) whenever n_chan reaches
      * OR_CONN_STATE_OPEN.
      */
     return 0;
   }
 
   tor_assert(!circ->n_hop); /* Connection is already established. */
-  circ->n_conn = n_conn;
-  log_debug(LD_CIRC,"n_conn is %s:%u",
-            n_conn->_base.address,n_conn->_base.port);
+  circ->n_chan = n_chan;
+  log_debug(LD_CIRC,
+            "n_chan is %s",
+            channel_get_canonical_remote_descr(n_chan));
 
   if (circuit_deliver_create_cell(circ, CELL_CREATE, onionskin) < 0)
     return -1;
@@ -2699,8 +2727,8 @@ pathbias_count_first_hop(origin_circuit_t *circ)
     if (!circ->has_opened) {
       entry_guard_t *guard;
 
-      guard = entry_guard_get_by_id_digest(
-              circ->_base.n_conn->identity_digest);
+      guard =
+        entry_guard_get_by_id_digest(circ->_base.n_chan->identity_digest);
       if (guard) {
         if (circ->path_state == PATH_STATE_NEW_CIRC) {
           circ->path_state = PATH_STATE_DID_FIRST_HOP;
@@ -2770,6 +2798,7 @@ pathbias_count_success(origin_circuit_t *circ)
   static ratelim_t success_notice_limit =
     RATELIM_INIT(SUCCESS_NOTICE_INTERVAL);
   char *rate_msg = NULL;
+  entry_guard_t *guard = NULL;
 
   /* We can't do path bias accounting without entry guards.
    * Testing and controller circuits also have no guards. */
@@ -2804,8 +2833,8 @@ pathbias_count_success(origin_circuit_t *circ)
 
   /* Don't count cannibalized/reused circs for path bias */
   if (!circ->has_opened) {
-    entry_guard_t *guard =
-      entry_guard_get_by_id_digest(circ->_base.n_conn->identity_digest);
+    guard =
+      entry_guard_get_by_id_digest(circ->_base.n_chan->identity_digest);
 
     if (guard) {
       if (circ->path_state == PATH_STATE_DID_FIRST_HOP) {
@@ -3021,7 +3050,7 @@ circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer, int reason)
    *     just give up.
    */
   circuit_mark_for_close(TO_CIRCUIT(circ),
-          END_CIRC_REASON_FLAG_REMOTE|reason);
+          END_CIRC_REASON_FLAG_REMOTE|END_CIRC_REASON_CHANNEL_CLOSED|reason);
   return 0;
 
 #if 0
@@ -3095,12 +3124,12 @@ onionskin_answer(or_circuit_t *circ, uint8_t cell_type, const char *payload,
   circ->is_first_hop = (cell_type == CELL_CREATED_FAST);
 
   append_cell_to_circuit_queue(TO_CIRCUIT(circ),
-                               circ->p_conn, &cell, CELL_DIRECTION_IN, 0);
+                               circ->p_chan, &cell, CELL_DIRECTION_IN, 0);
   log_debug(LD_CIRC,"Finished sending '%s' cell.",
             circ->is_first_hop ? "created_fast" : "created");
 
-  if (!is_local_addr(&circ->p_conn->_base.addr) &&
-      !connection_or_nonopen_was_started_here(circ->p_conn)) {
+  if (!channel_is_local(circ->p_chan) &&
+      !channel_is_outgoing(circ->p_chan)) {
     /* record that we could process create cells from a non-local conn
      * that we didn't initiate; presumably this means that create cells
      * can reach us too. */

src/or/circuitbuild.h

@@ -22,7 +22,7 @@ origin_circuit_t *circuit_establish_circuit(uint8_t purpose,
                                             extend_info_t *exit,
                                             int flags);
 int circuit_handle_first_hop(origin_circuit_t *circ);
-void circuit_n_conn_done(or_connection_t *or_conn, int status);
+void circuit_n_chan_done(channel_t *chan, int status);
 int inform_testing_reachability(void);
 int circuit_timeout_want_to_count_circ(origin_circuit_t *circ);
 int circuit_send_next_onion_skin(origin_circuit_t *circ);

src/or/circuitlist.c

@@ -10,6 +10,7 @@
  **/
 
 #include "or.h"
+#include "channel.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuituse.h"
@@ -34,8 +35,8 @@
 /** A global list of all circuits at this hop. */
 circuit_t *global_circuitlist=NULL;
 
-/** A list of all the circuits in CIRCUIT_STATE_OR_WAIT. */
-static smartlist_t *circuits_pending_or_conns=NULL;
+/** A list of all the circuits in CIRCUIT_STATE_CHAN_WAIT. */
+static smartlist_t *circuits_pending_chans = NULL;
 
 static void circuit_free(circuit_t *circ);
 static void circuit_free_cpath(crypt_path_t *cpath);
@@ -44,154 +45,190 @@ static void cpath_ref_decref(crypt_path_reference_t *cpath_ref);
 
 /********* END VARIABLES ************/
 
-/** A map from OR connection and circuit ID to circuit.  (Lookup performance is
+/** A map from channel and circuit ID to circuit.  (Lookup performance is
  * very important here, since we need to do it every time a cell arrives.) */
-typedef struct orconn_circid_circuit_map_t {
-  HT_ENTRY(orconn_circid_circuit_map_t) node;
-  or_connection_t *or_conn;
+typedef struct chan_circid_circuit_map_t {
+  HT_ENTRY(chan_circid_circuit_map_t) node;
+  channel_t *chan;
   circid_t circ_id;
   circuit_t *circuit;
-} orconn_circid_circuit_map_t;
+} chan_circid_circuit_map_t;
 
-/** Helper for hash tables: compare the OR connection and circuit ID for a and
+/** Helper for hash tables: compare the channel and circuit ID for a and
  * b, and return less than, equal to, or greater than zero appropriately.
  */
 static INLINE int
-_orconn_circid_entries_eq(orconn_circid_circuit_map_t *a,
-                          orconn_circid_circuit_map_t *b)
+_chan_circid_entries_eq(chan_circid_circuit_map_t *a,
+                        chan_circid_circuit_map_t *b)
 {
-  return a->or_conn == b->or_conn && a->circ_id == b->circ_id;
+  return a->chan == b->chan && a->circ_id == b->circ_id;
 }
 
 /** Helper: return a hash based on circuit ID and the pointer value of
- * or_conn in <b>a</b>. */
+ * chan in <b>a</b>. */
 static INLINE unsigned int
-_orconn_circid_entry_hash(orconn_circid_circuit_map_t *a)
+_chan_circid_entry_hash(chan_circid_circuit_map_t *a)
 {
-  return (((unsigned)a->circ_id)<<8) ^ (unsigned)(uintptr_t)(a->or_conn);
+  return (((unsigned)a->circ_id)<<8) ^ (unsigned)(uintptr_t)(a->chan);
 }
 
-/** Map from [orconn,circid] to circuit. */
-static HT_HEAD(orconn_circid_map, orconn_circid_circuit_map_t)
-     orconn_circid_circuit_map = HT_INITIALIZER();
-HT_PROTOTYPE(orconn_circid_map, orconn_circid_circuit_map_t, node,
-             _orconn_circid_entry_hash, _orconn_circid_entries_eq)
-HT_GENERATE(orconn_circid_map, orconn_circid_circuit_map_t, node,
-            _orconn_circid_entry_hash, _orconn_circid_entries_eq, 0.6,
+/** Map from [chan,circid] to circuit. */
+static HT_HEAD(chan_circid_map, chan_circid_circuit_map_t)
+     chan_circid_map = HT_INITIALIZER();
+HT_PROTOTYPE(chan_circid_map, chan_circid_circuit_map_t, node,
+             _chan_circid_entry_hash, _chan_circid_entries_eq)
+HT_GENERATE(chan_circid_map, chan_circid_circuit_map_t, node,
+            _chan_circid_entry_hash, _chan_circid_entries_eq, 0.6,
             malloc, realloc, free)
 
-/** The most recently returned entry from circuit_get_by_circid_orconn;
+/** The most recently returned entry from circuit_get_by_circid_chan;
  * used to improve performance when many cells arrive in a row from the
  * same circuit.
  */
-orconn_circid_circuit_map_t *_last_circid_orconn_ent = NULL;
+chan_circid_circuit_map_t *_last_circid_chan_ent = NULL;
 
-/** Implementation helper for circuit_set_{p,n}_circid_orconn: A circuit ID
- * and/or or_connection for circ has just changed from <b>old_conn, old_id</b>
- * to <b>conn, id</b>.  Adjust the conn,circid map as appropriate, removing
+/** Implementation helper for circuit_set_{p,n}_circid_channel: A circuit ID
+ * and/or channel for circ has just changed from <b>old_chan, old_id</b>
+ * to <b>chan, id</b>.  Adjust the chan,circid map as appropriate, removing
  * the old entry (if any) and adding a new one. */
 static void
-circuit_set_circid_orconn_helper(circuit_t *circ, int direction,
-                                 circid_t id,
-                                 or_connection_t *conn)
+circuit_set_circid_chan_helper(circuit_t *circ, int direction,
+                               circid_t id,
+                               channel_t *chan)
 {
-  orconn_circid_circuit_map_t search;
-  orconn_circid_circuit_map_t *found;
-  or_connection_t *old_conn, **conn_ptr;
+  chan_circid_circuit_map_t search;
+  chan_circid_circuit_map_t *found;
+  channel_t *old_chan, **chan_ptr;
   circid_t old_id, *circid_ptr;
-  int was_active, make_active;
+  int make_active, attached = 0;
 
   if (direction == CELL_DIRECTION_OUT) {
-    conn_ptr = &circ->n_conn;
+    chan_ptr = &circ->n_chan;
     circid_ptr = &circ->n_circ_id;
-    was_active = circ->next_active_on_n_conn != NULL;
-    make_active = circ->n_conn_cells.n > 0;
+    make_active = circ->n_chan_cells.n > 0;
   } else {
     or_circuit_t *c = TO_OR_CIRCUIT(circ);
-    conn_ptr = &c->p_conn;
+    chan_ptr = &c->p_chan;
     circid_ptr = &c->p_circ_id;
-    was_active = c->next_active_on_p_conn != NULL;
-    make_active = c->p_conn_cells.n > 0;
+    make_active = c->p_chan_cells.n > 0;
   }
-  old_conn = *conn_ptr;
+  old_chan = *chan_ptr;
   old_id = *circid_ptr;
 
-  if (id == old_id && conn == old_conn)
+  if (id == old_id && chan == old_chan)
     return;
 
-  if (_last_circid_orconn_ent &&
-      ((old_id == _last_circid_orconn_ent->circ_id &&
-        old_conn == _last_circid_orconn_ent->or_conn) ||
-       (id == _last_circid_orconn_ent->circ_id &&
-        conn == _last_circid_orconn_ent->or_conn))) {
-    _last_circid_orconn_ent = NULL;
+  if (_last_circid_chan_ent &&
+      ((old_id == _last_circid_chan_ent->circ_id &&
+        old_chan == _last_circid_chan_ent->chan) ||
+       (id == _last_circid_chan_ent->circ_id &&
+        chan == _last_circid_chan_ent->chan))) {
+    _last_circid_chan_ent = NULL;
   }
 
-  if (old_conn) { /* we may need to remove it from the conn-circid map */
-    tor_assert(old_conn->_base.magic == OR_CONNECTION_MAGIC);
+  if (old_chan) {
+    /*
+     * If we're changing channels or ID and had an old channel and a non
+     * zero old ID and weren't marked for close (i.e., we should have been
+     * attached), detach the circuit. ID changes require this because
+     * circuitmux hashes on (channel_id, circuit_id).
+     */
+    if (old_id != 0 && (old_chan != chan || old_id != id) &&
+        !(circ->marked_for_close)) {
+      tor_assert(old_chan->cmux);
+      circuitmux_detach_circuit(old_chan->cmux, circ);
+    }
+
+    /* we may need to remove it from the conn-circid map */
     search.circ_id = old_id;
-    search.or_conn = old_conn;
-    found = HT_REMOVE(orconn_circid_map, &orconn_circid_circuit_map, &search);
+    search.chan = old_chan;
+    found = HT_REMOVE(chan_circid_map, &chan_circid_map, &search);
     if (found) {
       tor_free(found);
-      --old_conn->n_circuits;
+      if (direction == CELL_DIRECTION_OUT) {
+        /* One fewer circuits use old_chan as n_chan */
+        --(old_chan->num_n_circuits);
+      } else {
+        /* One fewer circuits use old_chan as p_chan */
+        --(old_chan->num_p_circuits);
+      }
     }
-    if (was_active && old_conn != conn)
-      make_circuit_inactive_on_conn(circ,old_conn);
   }
 
   /* Change the values only after we have possibly made the circuit inactive
-   * on the previous conn. */
-  *conn_ptr = conn;
+   * on the previous chan. */
+  *chan_ptr = chan;
   *circid_ptr = id;
 
-  if (conn == NULL)
+  if (chan == NULL)
     return;
 
   /* now add the new one to the conn-circid map */
   search.circ_id = id;
-  search.or_conn = conn;
-  found = HT_FIND(orconn_circid_map, &orconn_circid_circuit_map, &search);
+  search.chan = chan;
+  found = HT_FIND(chan_circid_map, &chan_circid_map, &search);
   if (found) {
     found->circuit = circ;
   } else {
-    found = tor_malloc_zero(sizeof(orconn_circid_circuit_map_t));
+    found = tor_malloc_zero(sizeof(chan_circid_circuit_map_t));
     found->circ_id = id;
-    found->or_conn = conn;
+    found->chan = chan;
     found->circuit = circ;
-    HT_INSERT(orconn_circid_map, &orconn_circid_circuit_map, found);
+    HT_INSERT(chan_circid_map, &chan_circid_map, found);
   }
-  if (make_active && old_conn != conn)
-    make_circuit_active_on_conn(circ,conn);
 
-  ++conn->n_circuits;
+  /*
+   * Attach to the circuitmux if we're changing channels or IDs and
+   * have a new channel and ID to use and the circuit is not marked for
+   * close.
+   */
+  if (chan && id != 0 && (old_chan != chan || old_id != id) &&
+      !(circ->marked_for_close)) {
+    tor_assert(chan->cmux);
+    circuitmux_attach_circuit(chan->cmux, circ, direction);
+    attached = 1;
+  }
+
+  /*
+   * This is a no-op if we have no cells, but if we do it marks us active to
+   * the circuitmux
+   */
+  if (make_active && attached)
+    update_circuit_on_cmux(circ, direction);
+
+  /* Adjust circuit counts on new channel */
+  if (direction == CELL_DIRECTION_OUT) {
+    ++chan->num_n_circuits;
+  } else {
+    ++chan->num_p_circuits;
+  }
 }
 
 /** Set the p_conn field of a circuit <b>circ</b>, along
  * with the corresponding circuit ID, and add the circuit as appropriate
- * to the (orconn,id)-\>circuit map. */
+ * to the (chan,id)-\>circuit map. */
 void
-circuit_set_p_circid_orconn(or_circuit_t *circ, circid_t id,
-                            or_connection_t *conn)
+circuit_set_p_circid_chan(or_circuit_t *circ, circid_t id,
+                          channel_t *chan)
 {
-  circuit_set_circid_orconn_helper(TO_CIRCUIT(circ), CELL_DIRECTION_IN,
-                                   id, conn);
+  circuit_set_circid_chan_helper(TO_CIRCUIT(circ), CELL_DIRECTION_IN,
+                                 id, chan);
 
-  if (conn)
-    tor_assert(bool_eq(circ->p_conn_cells.n, circ->next_active_on_p_conn));
+  if (chan)
+    tor_assert(bool_eq(circ->p_chan_cells.n, circ->next_active_on_p_chan));
 }
 
 /** Set the n_conn field of a circuit <b>circ</b>, along
  * with the corresponding circuit ID, and add the circuit as appropriate
- * to the (orconn,id)-\>circuit map. */
+ * to the (chan,id)-\>circuit map. */
 void
-circuit_set_n_circid_orconn(circuit_t *circ, circid_t id,
-                            or_connection_t *conn)
+circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
+                          channel_t *chan)
 {
-  circuit_set_circid_orconn_helper(circ, CELL_DIRECTION_OUT, id, conn);
+  circuit_set_circid_chan_helper(circ, CELL_DIRECTION_OUT, id, chan);
 
-  if (conn)
-    tor_assert(bool_eq(circ->n_conn_cells.n, circ->next_active_on_n_conn));
+  if (chan)
+    tor_assert(bool_eq(circ->n_chan_cells.n, circ->next_active_on_n_chan));
 }
 
 /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing
@@ -202,18 +239,18 @@ circuit_set_state(circuit_t *circ, uint8_t state)
   tor_assert(circ);
   if (state == circ->state)
     return;
-  if (!circuits_pending_or_conns)
-    circuits_pending_or_conns = smartlist_new();
-  if (circ->state == CIRCUIT_STATE_OR_WAIT) {
+  if (!circuits_pending_chans)
+    circuits_pending_chans = smartlist_new();
+  if (circ->state == CIRCUIT_STATE_CHAN_WAIT) {
     /* remove from waiting-circuit list. */
-    smartlist_remove(circuits_pending_or_conns, circ);
+    smartlist_remove(circuits_pending_chans, circ);
   }
-  if (state == CIRCUIT_STATE_OR_WAIT) {
+  if (state == CIRCUIT_STATE_CHAN_WAIT) {
     /* add to waiting-circuit list. */
-    smartlist_add(circuits_pending_or_conns, circ);
+    smartlist_add(circuits_pending_chans, circ);
   }
   if (state == CIRCUIT_STATE_OPEN)
-    tor_assert(!circ->n_conn_onionskin);
+    tor_assert(!circ->n_chan_onionskin);
   circ->state = state;
 }
 
@@ -232,51 +269,53 @@ circuit_add(circuit_t *circ)
   }
 }
 
-/** Append to <b>out</b> all circuits in state OR_WAIT waiting for
+/** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for
  * the given connection. */
 void
-circuit_get_all_pending_on_or_conn(smartlist_t *out, or_connection_t *or_conn)
+circuit_get_all_pending_on_channel(smartlist_t *out, channel_t *chan)
 {
   tor_assert(out);
-  tor_assert(or_conn);
+  tor_assert(chan);
 
-  if (!circuits_pending_or_conns)
+  if (!circuits_pending_chans)
     return;
 
-  SMARTLIST_FOREACH_BEGIN(circuits_pending_or_conns, circuit_t *, circ) {
+  SMARTLIST_FOREACH_BEGIN(circuits_pending_chans, circuit_t *, circ) {
     if (circ->marked_for_close)
       continue;
     if (!circ->n_hop)
       continue;
-    tor_assert(circ->state == CIRCUIT_STATE_OR_WAIT);
+    tor_assert(circ->state == CIRCUIT_STATE_CHAN_WAIT);
     if (tor_digest_is_zero(circ->n_hop->identity_digest)) {
       /* Look at addr/port. This is an unkeyed connection. */
-      if (!tor_addr_eq(&circ->n_hop->addr, &or_conn->_base.addr) ||
-          circ->n_hop->port != or_conn->_base.port)
+      if (!channel_matches_extend_info(chan, circ->n_hop))
         continue;
     } else {
       /* We expected a key. See if it's the right one. */
-      if (tor_memneq(or_conn->identity_digest,
-                 circ->n_hop->identity_digest, DIGEST_LEN))
+      if (tor_memneq(chan->identity_digest,
+                     circ->n_hop->identity_digest, DIGEST_LEN))
         continue;
     }
     smartlist_add(out, circ);
   } SMARTLIST_FOREACH_END(circ);
 }
 
-/** Return the number of circuits in state OR_WAIT, waiting for the given
- * connection. */
+/** Return the number of circuits in state CHAN_WAIT, waiting for the given
+ * channel. */
 int
-circuit_count_pending_on_or_conn(or_connection_t *or_conn)
+circuit_count_pending_on_channel(channel_t *chan)
 {
   int cnt;
   smartlist_t *sl = smartlist_new();
-  circuit_get_all_pending_on_or_conn(sl, or_conn);
+
+  tor_assert(chan);
+
+  circuit_get_all_pending_on_channel(sl, chan);
   cnt = smartlist_len(sl);
   smartlist_free(sl);
   log_debug(LD_CIRC,"or_conn to %s at %s, %d pending circs",
-            or_conn->nickname ? or_conn->nickname : "NULL",
-            or_conn->_base.address,
+            chan->nickname ? chan->nickname : "NULL",
+            channel_get_canonical_remote_descr(chan),
             cnt);
   return cnt;
 }
@@ -324,7 +363,7 @@ circuit_state_to_string(int state)
   switch (state) {
     case CIRCUIT_STATE_BUILDING: return "doing handshakes";
     case CIRCUIT_STATE_ONIONSKIN_PENDING: return "processing the onion";
-    case CIRCUIT_STATE_OR_WAIT: return "connecting to server";
+    case CIRCUIT_STATE_CHAN_WAIT: return "connecting to server";
     case CIRCUIT_STATE_OPEN: return "open";
     default:
       log_warn(LD_BUG, "Unknown circuit state %d", state);
@@ -518,12 +557,6 @@ init_circuit_base(circuit_t *circ)
   circ->package_window = circuit_initial_package_window();
   circ->deliver_window = CIRCWINDOW_START;
 
-  /* Initialize the cell_ewma_t structure */
-  circ->n_cell_ewma.last_adjusted_tick = cell_ewma_get_tick();
-  circ->n_cell_ewma.cell_count = 0.0;
-  circ->n_cell_ewma.heap_index = -1;
-  circ->n_cell_ewma.is_for_p_conn = 0;
-
   circuit_add(circ);
 }
 
@@ -556,7 +589,7 @@ origin_circuit_new(void)
 /** Allocate a new or_circuit_t, connected to <b>p_conn</b> as
  * <b>p_circ_id</b>.  If <b>p_conn</b> is NULL, the circuit is unattached. */
 or_circuit_t *
-or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn)
+or_circuit_new(circid_t p_circ_id, channel_t *p_chan)
 {
   /* CircIDs */
   or_circuit_t *circ;
@@ -564,23 +597,13 @@ or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn)
   circ = tor_malloc_zero(sizeof(or_circuit_t));
   circ->_base.magic = OR_CIRCUIT_MAGIC;
 
-  if (p_conn)
-    circuit_set_p_circid_orconn(circ, p_circ_id, p_conn);
+  if (p_chan)
+    circuit_set_p_circid_chan(circ, p_circ_id, p_chan);
 
   circ->remaining_relay_early_cells = MAX_RELAY_EARLY_CELLS_PER_CIRCUIT;
 
   init_circuit_base(TO_CIRCUIT(circ));
 
-  /* Initialize the cell_ewma_t structure */
-
-  /* Initialize the cell counts to 0 */
-  circ->p_cell_ewma.cell_count = 0.0;
-  circ->p_cell_ewma.last_adjusted_tick = cell_ewma_get_tick();
-  circ->p_cell_ewma.is_for_p_conn = 1;
-
-  /* It's not in any heap yet. */
-  circ->p_cell_ewma.heap_index = -1;
-
   return circ;
 }
 
@@ -641,22 +664,22 @@ circuit_free(circuit_t *circ)
     }
 
     /* remove from map. */
-    circuit_set_p_circid_orconn(ocirc, 0, NULL);
+    circuit_set_p_circid_chan(ocirc, 0, NULL);
 
     /* Clear cell queue _after_ removing it from the map.  Otherwise our
      * "active" checks will be violated. */
-    cell_queue_clear(&ocirc->p_conn_cells);
+    cell_queue_clear(&ocirc->p_chan_cells);
   }
 
   extend_info_free(circ->n_hop);
-  tor_free(circ->n_conn_onionskin);
+  tor_free(circ->n_chan_onionskin);
 
   /* Remove from map. */
-  circuit_set_n_circid_orconn(circ, 0, NULL);
+  circuit_set_n_circid_chan(circ, 0, NULL);
 
   /* Clear cell queue _after_ removing it from the map.  Otherwise our
    * "active" checks will be violated. */
-  cell_queue_clear(&circ->n_conn_cells);
+  cell_queue_clear(&circ->n_chan_cells);
 
   memset(mem, 0xAA, memlen); /* poison memory */
   tor_free(mem);
@@ -702,10 +725,10 @@ circuit_free_all(void)
     global_circuitlist = next;
   }
 
-  smartlist_free(circuits_pending_or_conns);
-  circuits_pending_or_conns = NULL;
+  smartlist_free(circuits_pending_chans);
+  circuits_pending_chans = NULL;
 
-  HT_CLEAR(orconn_circid_map, &orconn_circid_circuit_map);
+  HT_CLEAR(chan_circid_map, &chan_circid_map);
 }
 
 /** Deallocate space associated with the cpath node <b>victim</b>. */
@@ -742,8 +765,12 @@ cpath_ref_decref(crypt_path_reference_t *cpath_ref)
  * of information about circuit <b>circ</b>.
  */
 static void
-circuit_dump_details(int severity, circuit_t *circ, int conn_array_index,
-                     const char *type, int this_circid, int other_circid)
+circuit_dump_conn_details(int severity,
+                          circuit_t *circ,
+                          int conn_array_index,
+                          const char *type,
+                          int this_circid,
+                          int other_circid)
 {
   log(severity, LD_CIRC, "Conn %d has %s circuit: circID %d (other side %d), "
       "state %d (%s), born %ld:",
@@ -764,50 +791,101 @@ circuit_dump_by_conn(connection_t *conn, int severity)
   circuit_t *circ;
   edge_connection_t *tmpconn;
 
-  for (circ=global_circuitlist;circ;circ = circ->next) {
+  for (circ = global_circuitlist; circ; circ = circ->next) {
     circid_t n_circ_id = circ->n_circ_id, p_circ_id = 0;
-    if (circ->marked_for_close)
+
+    if (circ->marked_for_close) {
       continue;
+    }
 
-    if (! CIRCUIT_IS_ORIGIN(circ))
+    if (!CIRCUIT_IS_ORIGIN(circ)) {
       p_circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
+    }
 
-    if (! CIRCUIT_IS_ORIGIN(circ) && TO_OR_CIRCUIT(circ)->p_conn &&
-        TO_CONN(TO_OR_CIRCUIT(circ)->p_conn) == conn)
-      circuit_dump_details(severity, circ, conn->conn_array_index, "App-ward",
-                           p_circ_id, n_circ_id);
     if (CIRCUIT_IS_ORIGIN(circ)) {
       for (tmpconn=TO_ORIGIN_CIRCUIT(circ)->p_streams; tmpconn;
            tmpconn=tmpconn->next_stream) {
         if (TO_CONN(tmpconn) == conn) {
-          circuit_dump_details(severity, circ, conn->conn_array_index,
-                               "App-ward", p_circ_id, n_circ_id);
+          circuit_dump_conn_details(severity, circ, conn->conn_array_index,
+                                    "App-ward", p_circ_id, n_circ_id);
         }
       }
     }
-    if (circ->n_conn && TO_CONN(circ->n_conn) == conn)
-      circuit_dump_details(severity, circ, conn->conn_array_index, "Exit-ward",
-                           n_circ_id, p_circ_id);
+
     if (! CIRCUIT_IS_ORIGIN(circ)) {
       for (tmpconn=TO_OR_CIRCUIT(circ)->n_streams; tmpconn;
            tmpconn=tmpconn->next_stream) {
         if (TO_CONN(tmpconn) == conn) {
-          circuit_dump_details(severity, circ, conn->conn_array_index,
-                               "Exit-ward", n_circ_id, p_circ_id);
+          circuit_dump_conn_details(severity, circ, conn->conn_array_index,
+                                    "Exit-ward", n_circ_id, p_circ_id);
         }
       }
     }
-    if (!circ->n_conn && circ->n_hop &&
-        tor_addr_eq(&circ->n_hop->addr, &conn->addr) &&
-        circ->n_hop->port == conn->port &&
-        conn->type == CONN_TYPE_OR &&
-        tor_memeq(TO_OR_CONN(conn)->identity_digest,
-                circ->n_hop->identity_digest, DIGEST_LEN)) {
-      circuit_dump_details(severity, circ, conn->conn_array_index,
-                           (circ->state == CIRCUIT_STATE_OPEN &&
-                            !CIRCUIT_IS_ORIGIN(circ)) ?
-                             "Endpoint" : "Pending",
-                           n_circ_id, p_circ_id);
+  }
+}
+
+/** A helper function for circuit_dump_by_chan() below. Log a bunch
+ * of information about circuit <b>circ</b>.
+ */
+static void
+circuit_dump_chan_details(int severity,
+                          circuit_t *circ,
+                          channel_t *chan,
+                          const char *type,
+                          int this_circid,
+                          int other_circid)
+{
+  log(severity, LD_CIRC, "Conn %p has %s circuit: circID %d (other side %d), "
+      "state %d (%s), born %ld:",
+      chan, type, this_circid, other_circid, circ->state,
+      circuit_state_to_string(circ->state),
+      (long)circ->timestamp_created.tv_sec);
+  if (CIRCUIT_IS_ORIGIN(circ)) { /* circ starts at this node */
+    circuit_log_path(severity, LD_CIRC, TO_ORIGIN_CIRCUIT(circ));
+  }
+}
+
+/** Log, at severity <b>severity</b>, information about each circuit
+ * that is connected to <b>chan</b>.
+ */
+void
+circuit_dump_by_chan(channel_t *chan, int severity)
+{
+  circuit_t *circ;
+
+  tor_assert(chan);
+
+  for (circ = global_circuitlist; circ; circ = circ->next) {
+    circid_t n_circ_id = circ->n_circ_id, p_circ_id = 0;
+
+    if (circ->marked_for_close) {
+      continue;
+    }
+
+    if (!CIRCUIT_IS_ORIGIN(circ)) {
+      p_circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
+    }
+
+    if (! CIRCUIT_IS_ORIGIN(circ) && TO_OR_CIRCUIT(circ)->p_chan &&
+        TO_OR_CIRCUIT(circ)->p_chan == chan) {
+      circuit_dump_chan_details(severity, circ, chan, "App-ward",
+                                p_circ_id, n_circ_id);
+    }
+
+    if (circ->n_chan && circ->n_chan == chan) {
+      circuit_dump_chan_details(severity, circ, chan, "Exit-ward",
+                                n_circ_id, p_circ_id);
+    }
+
+    if (!circ->n_chan && circ->n_hop &&
+        channel_matches_extend_info(chan, circ->n_hop) &&
+        tor_memeq(chan->identity_digest,
+                  circ->n_hop->identity_digest, DIGEST_LEN)) {
+      circuit_dump_chan_details(severity, circ, chan,
+                                (circ->state == CIRCUIT_STATE_OPEN &&
+                                 !CIRCUIT_IS_ORIGIN(circ)) ?
+                                "Endpoint" : "Pending",
+                                n_circ_id, p_circ_id);
     }
   }
 }
@@ -832,27 +910,39 @@ circuit_get_by_global_id(uint32_t id)
 
 /** Return a circ such that:
  *  - circ-\>n_circ_id or circ-\>p_circ_id is equal to <b>circ_id</b>, and
- *  - circ is attached to <b>conn</b>, either as p_conn or n_conn.
+ *  - circ is attached to <b>chan</b>, either as p_chan or n_chan.
  * Return NULL if no such circuit exists.
  */
 static INLINE circuit_t *
-circuit_get_by_circid_orconn_impl(circid_t circ_id, or_connection_t *conn)
+circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan)
 {
-  orconn_circid_circuit_map_t search;
-  orconn_circid_circuit_map_t *found;
+  chan_circid_circuit_map_t search;
+  chan_circid_circuit_map_t *found;
 
-  if (_last_circid_orconn_ent &&
-      circ_id == _last_circid_orconn_ent->circ_id &&
-      conn == _last_circid_orconn_ent->or_conn) {
-    found = _last_circid_orconn_ent;
+  if (_last_circid_chan_ent &&
+      circ_id == _last_circid_chan_ent->circ_id &&
+      chan == _last_circid_chan_ent->chan) {
+    found = _last_circid_chan_ent;
   } else {
     search.circ_id = circ_id;
-    search.or_conn = conn;
-    found = HT_FIND(orconn_circid_map, &orconn_circid_circuit_map, &search);
-    _last_circid_orconn_ent = found;
+    search.chan = chan;
+    found = HT_FIND(chan_circid_map, &chan_circid_map, &search);
+    _last_circid_chan_ent = found;
   }
-  if (found && found->circuit)
+  if (found && found->circuit) {
+    log_debug(LD_CIRC,
+              "circuit_get_by_circid_channel_impl() returning circuit %p for"
+              " circ_id %d, channel ID " U64_FORMAT " (%p)",
+              found->circuit, circ_id,
+              U64_PRINTF_ARG(chan->global_identifier), chan);
     return found->circuit;
+  }
+
+  log_debug(LD_CIRC,
+            "circuit_get_by_circid_channel_impl() found nothing for"
+            " circ_id %d, channel ID " U64_FORMAT " (%p)",
+            circ_id,
+            U64_PRINTF_ARG(chan->global_identifier), chan);
 
   return NULL;
   /* The rest of this checks for bugs. Disabled by default. */
@@ -862,15 +952,15 @@ circuit_get_by_circid_orconn_impl(circid_t circ_id, or_connection_t *conn)
     for (circ=global_circuitlist;circ;circ = circ->next) {
       if (! CIRCUIT_IS_ORIGIN(circ)) {
         or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
-        if (or_circ->p_conn == conn && or_circ->p_circ_id == circ_id) {
+        if (or_circ->p_chan == chan && or_circ->p_circ_id == circ_id) {
           log_warn(LD_BUG,
-                   "circuit matches p_conn, but not in hash table (Bug!)");
+                   "circuit matches p_chan, but not in hash table (Bug!)");
           return circ;
         }
       }
-      if (circ->n_conn == conn && circ->n_circ_id == circ_id) {
+      if (circ->n_chan == chan && circ->n_circ_id == circ_id) {
         log_warn(LD_BUG,
-                 "circuit matches n_conn, but not in hash table (Bug!)");
+                 "circuit matches n_chan, but not in hash table (Bug!)");
         return circ;
       }
     }
@@ -880,26 +970,38 @@ circuit_get_by_circid_orconn_impl(circid_t circ_id, or_connection_t *conn)
 
 /** Return a circ such that:
  *  - circ-\>n_circ_id or circ-\>p_circ_id is equal to <b>circ_id</b>, and
- *  - circ is attached to <b>conn</b>, either as p_conn or n_conn.
+ *  - circ is attached to <b>chan</b>, either as p_chan or n_chan.
  *  - circ is not marked for close.
  * Return NULL if no such circuit exists.
  */
 circuit_t *
-circuit_get_by_circid_orconn(circid_t circ_id, or_connection_t *conn)
+circuit_get_by_circid_channel(circid_t circ_id, channel_t *chan)
 {
-  circuit_t *circ = circuit_get_by_circid_orconn_impl(circ_id, conn);
+  circuit_t *circ = circuit_get_by_circid_channel_impl(circ_id, chan);
   if (!circ || circ->marked_for_close)
     return NULL;
   else
     return circ;
 }
 
-/** Return true iff the circuit ID <b>circ_id</b> is currently used by a
- * circuit, marked or not, on <b>conn</b>. */
-int
-circuit_id_in_use_on_orconn(circid_t circ_id, or_connection_t *conn)
+/** Return a circ such that:
+ *  - circ-\>n_circ_id or circ-\>p_circ_id is equal to <b>circ_id</b>, and
+ *  - circ is attached to <b>chan</b>, either as p_chan or n_chan.
+ * Return NULL if no such circuit exists.
+ */
+circuit_t *
+circuit_get_by_circid_channel_even_if_marked(circid_t circ_id,
+                                             channel_t *chan)
 {
-  return circuit_get_by_circid_orconn_impl(circ_id, conn) != NULL;
+  return circuit_get_by_circid_channel_impl(circ_id, chan);
+}
+
+/** Return true iff the circuit ID <b>circ_id</b> is currently used by a
+ * circuit, marked or not, on <b>chan</b>. */
+int
+circuit_id_in_use_on_channel(circid_t circ_id, channel_t *chan)
+{
+  return circuit_get_by_circid_channel_impl(circ_id, chan) != NULL;
 }
 
 /** Return the circuit that a given edge connection is using. */
@@ -916,27 +1018,27 @@ circuit_get_by_edge_conn(edge_connection_t *conn)
   return circ;
 }
 
-/** For each circuit that has <b>conn</b> as n_conn or p_conn, unlink the
- * circuit from the orconn,circid map, and mark it for close if it hasn't
+/** For each circuit that has <b>chan</b> as n_chan or p_chan, unlink the
+ * circuit from the chan,circid map, and mark it for close if it hasn't
  * been marked already.
  */
 void
-circuit_unlink_all_from_or_conn(or_connection_t *conn, int reason)
+circuit_unlink_all_from_channel(channel_t *chan, int reason)
 {
   circuit_t *circ;
 
-  connection_or_unlink_all_active_circs(conn);
+  channel_unlink_all_circuits(chan);
 
   for (circ = global_circuitlist; circ; circ = circ->next) {
     int mark = 0;
-    if (circ->n_conn == conn) {
-        circuit_set_n_circid_orconn(circ, 0, NULL);
+    if (circ->n_chan == chan) {
+        circuit_set_n_circid_chan(circ, 0, NULL);
         mark = 1;
     }
     if (! CIRCUIT_IS_ORIGIN(circ)) {
       or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
-      if (or_circ->p_conn == conn) {
-        circuit_set_p_circid_orconn(or_circ, 0, NULL);
+      if (or_circ->p_chan == chan) {
+        circuit_set_p_circid_chan(or_circ, 0, NULL);
         mark = 1;
       }
     }
@@ -1267,9 +1369,9 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line,
       circuit_rep_hist_note_result(ocirc);
     }
   }
-  if (circ->state == CIRCUIT_STATE_OR_WAIT) {
-    if (circuits_pending_or_conns)
-      smartlist_remove(circuits_pending_or_conns, circ);
+  if (circ->state == CIRCUIT_STATE_CHAN_WAIT) {
+    if (circuits_pending_chans)
+      smartlist_remove(circuits_pending_chans, circ);
   }
   if (CIRCUIT_IS_ORIGIN(circ)) {
     control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ),
@@ -1306,9 +1408,10 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line,
                                              INTRO_POINT_FAILURE_UNREACHABLE);
     }
   }
-  if (circ->n_conn) {
-    circuit_clear_cell_queue(circ, circ->n_conn);
-    connection_or_send_destroy(circ->n_circ_id, circ->n_conn, reason);
+  if (circ->n_chan) {
+    circuit_clear_cell_queue(circ, circ->n_chan);
+    channel_send_destroy(circ->n_circ_id, circ->n_chan, reason);
+    circuitmux_detach_circuit(circ->n_chan->cmux, circ);
   }
 
   if (! CIRCUIT_IS_ORIGIN(circ)) {
@@ -1333,9 +1436,10 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line,
       conn->on_circuit = NULL;
     }
 
-    if (or_circ->p_conn) {
-      circuit_clear_cell_queue(circ, or_circ->p_conn);
-      connection_or_send_destroy(or_circ->p_circ_id, or_circ->p_conn, reason);
+    if (or_circ->p_chan) {
+      circuit_clear_cell_queue(circ, or_circ->p_chan);
+      channel_send_destroy(or_circ->p_circ_id, or_circ->p_chan, reason);
+      circuitmux_detach_circuit(or_circ->p_chan->cmux, circ);
     }
   } else {
     origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
@@ -1438,22 +1542,22 @@ assert_circuit_ok(const circuit_t *c)
       or_circ = TO_OR_CIRCUIT(nonconst_circ);
   }
 
-  if (c->n_conn) {
+  if (c->n_chan) {
     tor_assert(!c->n_hop);
 
     if (c->n_circ_id) {
       /* We use the _impl variant here to make sure we don't fail on marked
        * circuits, which would not be returned by the regular function. */
-      circuit_t *c2 = circuit_get_by_circid_orconn_impl(c->n_circ_id,
-                                                        c->n_conn);
+      circuit_t *c2 = circuit_get_by_circid_channel_impl(c->n_circ_id,
+                                                         c->n_chan);
       tor_assert(c == c2);
     }
   }
-  if (or_circ && or_circ->p_conn) {
+  if (or_circ && or_circ->p_chan) {
     if (or_circ->p_circ_id) {
       /* ibid */
-      circuit_t *c2 = circuit_get_by_circid_orconn_impl(or_circ->p_circ_id,
-                                                        or_circ->p_conn);
+      circuit_t *c2 = circuit_get_by_circid_channel_impl(or_circ->p_circ_id,
+                                                         or_circ->p_chan);
       tor_assert(c == c2);
     }
   }
@@ -1464,7 +1568,7 @@ assert_circuit_ok(const circuit_t *c)
   tor_assert(c->deliver_window >= 0);
   tor_assert(c->package_window >= 0);
   if (c->state == CIRCUIT_STATE_OPEN) {
-    tor_assert(!c->n_conn_onionskin);
+    tor_assert(!c->n_chan_onionskin);
     if (or_circ) {
       tor_assert(or_circ->n_crypto);
       tor_assert(or_circ->p_crypto);
@@ -1472,12 +1576,12 @@ assert_circuit_ok(const circuit_t *c)
       tor_assert(or_circ->p_digest);
     }
   }
-  if (c->state == CIRCUIT_STATE_OR_WAIT && !c->marked_for_close) {
-    tor_assert(circuits_pending_or_conns &&
-               smartlist_isin(circuits_pending_or_conns, c));
+  if (c->state == CIRCUIT_STATE_CHAN_WAIT && !c->marked_for_close) {
+    tor_assert(circuits_pending_chans &&
+               smartlist_isin(circuits_pending_chans, c));
   } else {
-    tor_assert(!circuits_pending_or_conns ||
-               !smartlist_isin(circuits_pending_or_conns, c));
+    tor_assert(!circuits_pending_chans ||
+               !smartlist_isin(circuits_pending_chans, c));
   }
   if (origin_circ && origin_circ->cpath) {
     assert_cpath_ok(origin_circ->cpath);

src/or/circuitlist.h

@@ -18,20 +18,24 @@ const char *circuit_purpose_to_controller_string(uint8_t purpose);
 const char *circuit_purpose_to_controller_hs_state_string(uint8_t purpose);
 const char *circuit_purpose_to_string(uint8_t purpose);
 void circuit_dump_by_conn(connection_t *conn, int severity);
-void circuit_set_p_circid_orconn(or_circuit_t *circ, circid_t id,
-                                 or_connection_t *conn);
-void circuit_set_n_circid_orconn(circuit_t *circ, circid_t id,
-                                 or_connection_t *conn);
+void circuit_dump_by_chan(channel_t *chan, int severity);
+void circuit_set_p_circid_chan(or_circuit_t *circ, circid_t id,
+                               channel_t *chan);
+void circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
+                               channel_t *chan);
 void circuit_set_state(circuit_t *circ, uint8_t state);
 void circuit_close_all_marked(void);
 int32_t circuit_initial_package_window(void);
 origin_circuit_t *origin_circuit_new(void);
-or_circuit_t *or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn);
-circuit_t *circuit_get_by_circid_orconn(circid_t circ_id,
-                                        or_connection_t *conn);
-int circuit_id_in_use_on_orconn(circid_t circ_id, or_connection_t *conn);
+or_circuit_t *or_circuit_new(circid_t p_circ_id, channel_t *p_chan);
+circuit_t *circuit_get_by_circid_channel(circid_t circ_id,
+                                         channel_t *chan);
+circuit_t *
+circuit_get_by_circid_channel_even_if_marked(circid_t circ_id,
+                                             channel_t *chan);
+int circuit_id_in_use_on_channel(circid_t circ_id, channel_t *chan);
 circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn);
-void circuit_unlink_all_from_or_conn(or_connection_t *conn, int reason);
+void circuit_unlink_all_from_channel(channel_t *chan, int reason);
 origin_circuit_t *circuit_get_by_global_id(uint32_t id);
 origin_circuit_t *circuit_get_ready_rend_circ_by_rend_data(
   const rend_data_t *rend_data);
@@ -47,9 +51,9 @@ void _circuit_mark_for_close(circuit_t *circ, int reason,
                              int line, const char *file);
 int circuit_get_cpath_len(origin_circuit_t *circ);
 crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum);
-void circuit_get_all_pending_on_or_conn(smartlist_t *out,
-                                        or_connection_t *or_conn);
-int circuit_count_pending_on_or_conn(or_connection_t *or_conn);
+void circuit_get_all_pending_on_channel(smartlist_t *out,
+                                        channel_t *chan);
+int circuit_count_pending_on_channel(channel_t *chan);
 
 #define circuit_mark_for_close(c, reason)                               \
   _circuit_mark_for_close((c), (reason), __LINE__, _SHORT_FILE_)

src/or/circuitmux.h

@@ -0,0 +1,136 @@
+/* * Copyright (c) 2012, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitmux.h
+ * \brief Header file for circuitmux.c
+ **/
+
+#ifndef _TOR_CIRCUITMUX_H
+#define _TOR_CIRCUITMUX_H
+
+#include "or.h"
+
+typedef struct circuitmux_policy_s circuitmux_policy_t;
+typedef struct circuitmux_policy_data_s circuitmux_policy_data_t;
+typedef struct circuitmux_policy_circ_data_s circuitmux_policy_circ_data_t;
+
+struct circuitmux_policy_s {
+  /* Allocate cmux-wide policy-specific data */
+  circuitmux_policy_data_t * (*alloc_cmux_data)(circuitmux_t *cmux);
+  /* Free cmux-wide policy-specific data */
+  void (*free_cmux_data)(circuitmux_t *cmux,
+                         circuitmux_policy_data_t *pol_data);
+  /* Allocate circuit policy-specific data for a newly attached circuit */
+  circuitmux_policy_circ_data_t *
+    (*alloc_circ_data)(circuitmux_t *cmux,
+                       circuitmux_policy_data_t *pol_data,
+                       circuit_t *circ,
+                       cell_direction_t direction,
+                       unsigned int cell_count);
+  /* Free circuit policy-specific data */
+  void (*free_circ_data)(circuitmux_t *cmux,
+                         circuitmux_policy_data_t *pol_data,
+                         circuit_t *circ,
+                         circuitmux_policy_circ_data_t *pol_circ_data);
+  /* Notify that a circuit has become active/inactive */
+  void (*notify_circ_active)(circuitmux_t *cmux,
+                             circuitmux_policy_data_t *pol_data,
+                             circuit_t *circ,
+                             circuitmux_policy_circ_data_t *pol_circ_data);
+  void (*notify_circ_inactive)(circuitmux_t *cmux,
+                               circuitmux_policy_data_t *pol_data,
+                               circuit_t *circ,
+                               circuitmux_policy_circ_data_t *pol_circ_data);
+  /* Notify of arriving/transmitted cells on a circuit */
+  void (*notify_set_n_cells)(circuitmux_t *cmux,
+                             circuitmux_policy_data_t *pol_data,
+                             circuit_t *circ,
+                             circuitmux_policy_circ_data_t *pol_circ_data,
+                             unsigned int n_cells);
+  void (*notify_xmit_cells)(circuitmux_t *cmux,
+                            circuitmux_policy_data_t *pol_data,
+                            circuit_t *circ,
+                            circuitmux_policy_circ_data_t *pol_circ_data,
+                            unsigned int n_cells);
+  /* Choose a circuit */
+  circuit_t * (*pick_active_circuit)(circuitmux_t *cmux,
+                                     circuitmux_policy_data_t *pol_data);
+};
+
+/*
+ * Circuitmux policy implementations can subclass this to store circuitmux-
+ * wide data; it just has the magic number in the base struct.
+ */
+
+struct circuitmux_policy_data_s {
+  uint32_t magic;
+};
+
+/*
+ * Circuitmux policy implementations can subclass this to store circuit-
+ * specific data; it just has the magic number in the base struct.
+ */
+
+struct circuitmux_policy_circ_data_s {
+  uint32_t magic;
+};
+
+/*
+ * Upcast #defines for the above types
+ */
+
+/**
+ * Convert a circuitmux_policy_data_t subtype to a circuitmux_policy_data_t.
+ */
+
+#define TO_CMUX_POL_DATA(x)  (&((x)->_base))
+
+/**
+ * Convert a circuitmux_policy_circ_data_t subtype to a
+ * circuitmux_policy_circ_data_t.
+ */
+
+#define TO_CMUX_POL_CIRC_DATA(x)  (&((x)->_base))
+
+/* Consistency check */
+void circuitmux_assert_okay(circuitmux_t *cmux);
+
+/* Create/destroy */
+circuitmux_t * circuitmux_alloc(void);
+void circuitmux_detach_all_circuits(circuitmux_t *cmux);
+void circuitmux_free(circuitmux_t *cmux);
+
+/* Policy control */
+void circuitmux_clear_policy(circuitmux_t *cmux);
+const circuitmux_policy_t * circuitmux_get_policy(circuitmux_t *cmux);
+void circuitmux_set_policy(circuitmux_t *cmux,
+                           const circuitmux_policy_t *pol);
+
+/* Status inquiries */
+cell_direction_t circuitmux_attached_circuit_direction(
+    circuitmux_t *cmux,
+    circuit_t *circ);
+int circuitmux_is_circuit_attached(circuitmux_t *cmux, circuit_t *circ);
+int circuitmux_is_circuit_active(circuitmux_t *cmux, circuit_t *circ);
+unsigned int circuitmux_num_cells_for_circuit(circuitmux_t *cmux,
+                                              circuit_t *circ);
+unsigned int circuitmux_num_cells(circuitmux_t *cmux);
+unsigned int circuitmux_num_circuits(circuitmux_t *cmux);
+unsigned int circuitmux_num_active_circuits(circuitmux_t *cmux);
+
+/* Channel interface */
+circuit_t * circuitmux_get_first_active_circuit(circuitmux_t *cmux);
+void circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ,
+                                  unsigned int n_cells);
+
+/* Circuit interface */
+void circuitmux_attach_circuit(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction);
+void circuitmux_detach_circuit(circuitmux_t *cmux, circuit_t *circ);
+void circuitmux_clear_num_cells(circuitmux_t *cmux, circuit_t *circ);
+void circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ,
+                              unsigned int n_cells);
+
+#endif /* _TOR_CIRCUITMUX_H */
+

src/or/circuitmux_ewma.c

@@ -0,0 +1,683 @@
+/* * Copyright (c) 2012, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitmux_ewma.c
+ * \brief EWMA circuit selection as a circuitmux_t policy
+ **/
+
+#define _TOR_CIRCUITMUX_EWMA_C
+
+#include <math.h>
+
+#include "or.h"
+#include "circuitmux.h"
+#include "circuitmux_ewma.h"
+#include "networkstatus.h"
+
+/*** EWMA parameter #defines ***/
+
+/** How long does a tick last (seconds)? */
+#define EWMA_TICK_LEN 10
+
+/** The default per-tick scale factor, if it hasn't been overridden by a
+ * consensus or a configuration setting.  zero means "disabled". */
+#define EWMA_DEFAULT_HALFLIFE 0.0
+
+/*** Some useful constant #defines ***/
+
+/*DOCDOC*/
+#define EPSILON 0.00001
+/*DOCDOC*/
+#define LOG_ONEHALF -0.69314718055994529
+
+/*** EWMA structures ***/
+
+typedef struct cell_ewma_s cell_ewma_t;
+typedef struct ewma_policy_data_s ewma_policy_data_t;
+typedef struct ewma_policy_circ_data_s ewma_policy_circ_data_t;
+
+/**
+ * The cell_ewma_t structure keeps track of how many cells a circuit has
+ * transferred recently.  It keeps an EWMA (exponentially weighted moving
+ * average) of the number of cells flushed from the circuit queue onto a
+ * connection in channel_flush_from_first_active_circuit().
+ */
+
+struct cell_ewma_s {
+  /** The last 'tick' at which we recalibrated cell_count.
+   *
+   * A cell sent at exactly the start of this tick has weight 1.0. Cells sent
+   * since the start of this tick have weight greater than 1.0; ones sent
+   * earlier have less weight. */
+  unsigned int last_adjusted_tick;
+  /** The EWMA of the cell count. */
+  double cell_count;
+  /** True iff this is the cell count for a circuit's previous
+   * channel. */
+  unsigned int is_for_p_chan : 1;
+  /** The position of the circuit within the OR connection's priority
+   * queue. */
+  int heap_index;
+};
+
+struct ewma_policy_data_s {
+  circuitmux_policy_data_t _base;
+
+  /**
+   * Priority queue of cell_ewma_t for circuits with queued cells waiting
+   * for room to free up on the channel that owns this circuitmux.  Kept
+   * in heap order according to EWMA.  This was formerly in channel_t, and
+   * in or_connection_t before that.
+   */
+  smartlist_t *active_circuit_pqueue;
+
+  /**
+   * The tick on which the cell_ewma_ts in active_circuit_pqueue last had
+   * their ewma values rescaled.  This was formerly in channel_t, and in
+   * or_connection_t before that.
+   */
+  unsigned int active_circuit_pqueue_last_recalibrated;
+};
+
+struct ewma_policy_circ_data_s {
+  circuitmux_policy_circ_data_t _base;
+
+  /**
+   * The EWMA count for the number of cells flushed from this circuit
+   * onto this circuitmux.  Used to determine which circuit to flush
+   * from next.  This was formerly in circuit_t and or_circuit_t.
+   */
+  cell_ewma_t cell_ewma;
+
+  /**
+   * Pointer back to the circuit_t this is for; since we're separating
+   * out circuit selection policy like this, we can't attach cell_ewma_t
+   * to the circuit_t any more, so we can't use SUBTYPE_P directly to a
+   * circuit_t like before; instead get it here.
+   */
+  circuit_t *circ;
+};
+
+#define EWMA_POL_DATA_MAGIC 0x2fd8b16aU
+#define EWMA_POL_CIRC_DATA_MAGIC 0x761e7747U
+
+/*** Downcasts for the above types ***/
+
+static ewma_policy_data_t *
+TO_EWMA_POL_DATA(circuitmux_policy_data_t *);
+
+static ewma_policy_circ_data_t *
+TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *);
+
+/**
+ * Downcast a circuitmux_policy_data_t to an ewma_policy_data_t and assert
+ * if the cast is impossible.
+ */
+
+static INLINE ewma_policy_data_t *
+TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol)
+{
+  if (!pol) return NULL;
+  else {
+    tor_assert(pol->magic == EWMA_POL_DATA_MAGIC);
+    return DOWNCAST(ewma_policy_data_t, pol);
+  }
+}
+
+/**
+ * Downcast a circuitmux_policy_circ_data_t to an ewma_policy_circ_data_t
+ * and assert if the cast is impossible.
+ */
+
+static INLINE ewma_policy_circ_data_t *
+TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol)
+{
+  if (!pol) return NULL;
+  else {
+    tor_assert(pol->magic == EWMA_POL_CIRC_DATA_MAGIC);
+    return DOWNCAST(ewma_policy_circ_data_t, pol);
+  }
+}
+
+/*** Static declarations for circuitmux_ewma.c ***/
+
+static void add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma);
+static int compare_cell_ewma_counts(const void *p1, const void *p2);
+static unsigned cell_ewma_tick_from_timeval(const struct timeval *now,
+                                            double *remainder_out);
+static circuit_t * cell_ewma_to_circuit(cell_ewma_t *ewma);
+static INLINE double get_scale_factor(unsigned from_tick, unsigned to_tick);
+static cell_ewma_t * pop_first_cell_ewma(ewma_policy_data_t *pol);
+static void remove_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma);
+static void scale_single_cell_ewma(cell_ewma_t *ewma, unsigned cur_tick);
+static void scale_active_circuits(ewma_policy_data_t *pol,
+                                  unsigned cur_tick);
+
+/*** Circuitmux policy methods ***/
+
+static circuitmux_policy_data_t * ewma_alloc_cmux_data(circuitmux_t *cmux);
+static void ewma_free_cmux_data(circuitmux_t *cmux,
+                                circuitmux_policy_data_t *pol_data);
+static circuitmux_policy_circ_data_t *
+ewma_alloc_circ_data(circuitmux_t *cmux, circuitmux_policy_data_t *pol_data,
+                     circuit_t *circ, cell_direction_t direction,
+                     unsigned int cell_count);
+static void
+ewma_free_circ_data(circuitmux_t *cmux,
+                    circuitmux_policy_data_t *pol_data,
+                    circuit_t *circ,
+                    circuitmux_policy_circ_data_t *pol_circ_data);
+static void
+ewma_notify_circ_active(circuitmux_t *cmux,
+                        circuitmux_policy_data_t *pol_data,
+                        circuit_t *circ,
+                        circuitmux_policy_circ_data_t *pol_circ_data);
+static void
+ewma_notify_circ_inactive(circuitmux_t *cmux,
+                          circuitmux_policy_data_t *pol_data,
+                          circuit_t *circ,
+                          circuitmux_policy_circ_data_t *pol_circ_data);
+static void
+ewma_notify_xmit_cells(circuitmux_t *cmux,
+                       circuitmux_policy_data_t *pol_data,
+                       circuit_t *circ,
+                       circuitmux_policy_circ_data_t *pol_circ_data,
+                       unsigned int n_cells);
+static circuit_t *
+ewma_pick_active_circuit(circuitmux_t *cmux,
+                         circuitmux_policy_data_t *pol_data);
+
+/*** EWMA global variables ***/
+
+/** The per-tick scale factor to be used when computing cell-count EWMA
+ * values.  (A cell sent N ticks before the start of the current tick
+ * has value ewma_scale_factor ** N.)
+ */
+static double ewma_scale_factor = 0.1;
+/* DOCDOC ewma_enabled */
+static int ewma_enabled = 0;
+
+/*** EWMA circuitmux_policy_t method table ***/
+
+circuitmux_policy_t ewma_policy = {  .alloc_cmux_data = ewma_alloc_cmux_data,
+  .free_cmux_data = ewma_free_cmux_data,
+  .alloc_circ_data = ewma_alloc_circ_data,
+  .free_circ_data = ewma_free_circ_data,
+  .notify_circ_active = ewma_notify_circ_active,
+  .notify_circ_inactive = ewma_notify_circ_inactive,
+  .notify_set_n_cells = NULL, /* EWMA doesn't need this */
+  .notify_xmit_cells = ewma_notify_xmit_cells,
+  .pick_active_circuit = ewma_pick_active_circuit
+};
+
+/*** EWMA method implementations using the below EWMA helper functions ***/
+
+/**
+ * Allocate an ewma_policy_data_t and upcast it to a circuitmux_policy_data_t;
+ * this is called when setting the policy on a circuitmux_t to ewma_policy.
+ */
+
+static circuitmux_policy_data_t *
+ewma_alloc_cmux_data(circuitmux_t *cmux)
+{
+  ewma_policy_data_t *pol = NULL;
+
+  tor_assert(cmux);
+
+  pol = tor_malloc_zero(sizeof(*pol));
+  pol->_base.magic = EWMA_POL_DATA_MAGIC;
+  pol->active_circuit_pqueue = smartlist_new();
+  pol->active_circuit_pqueue_last_recalibrated = cell_ewma_get_tick();
+
+  return TO_CMUX_POL_DATA(pol);
+}
+
+/**
+ * Free an ewma_policy_data_t allocated with ewma_alloc_cmux_data()
+ */
+
+static void
+ewma_free_cmux_data(circuitmux_t *cmux,
+                    circuitmux_policy_data_t *pol_data)
+{
+  ewma_policy_data_t *pol = NULL;
+
+  tor_assert(cmux);
+  if (!pol_data) return;
+
+  pol = TO_EWMA_POL_DATA(pol_data);
+
+  smartlist_free(pol->active_circuit_pqueue);
+  tor_free(pol);
+}
+
+/**
+ * Allocate an ewma_policy_circ_data_t and upcast it to a
+ * circuitmux_policy_data_t; this is called when attaching a circuit to a
+ * circuitmux_t with ewma_policy.
+ */
+
+static circuitmux_policy_circ_data_t *
+ewma_alloc_circ_data(circuitmux_t *cmux,
+                     circuitmux_policy_data_t *pol_data,
+                     circuit_t *circ,
+                     cell_direction_t direction,
+                     unsigned int cell_count)
+{
+  ewma_policy_circ_data_t *cdata = NULL;
+
+  tor_assert(cmux);
+  tor_assert(pol_data);
+  tor_assert(circ);
+  tor_assert(direction == CELL_DIRECTION_OUT ||
+             direction == CELL_DIRECTION_IN);
+  /* Shut the compiler up */
+  tor_assert(cell_count == cell_count);
+
+  cdata = tor_malloc_zero(sizeof(*cdata));
+  cdata->_base.magic = EWMA_POL_CIRC_DATA_MAGIC;
+  cdata->circ = circ;
+
+  /*
+   * Initialize the cell_ewma_t structure (formerly in
+   * init_circuit_base())
+   */
+  cdata->cell_ewma.last_adjusted_tick = cell_ewma_get_tick();
+  cdata->cell_ewma.cell_count = 0.0;
+  cdata->cell_ewma.heap_index = -1;
+  if (direction == CELL_DIRECTION_IN) {
+    cdata->cell_ewma.is_for_p_chan = 1;
+  } else {
+    cdata->cell_ewma.is_for_p_chan = 0;
+  }
+
+  return TO_CMUX_POL_CIRC_DATA(cdata);
+}
+
+/**
+ * Free an ewma_policy_circ_data_t allocated with ewma_alloc_circ_data()
+ */
+
+static void
+ewma_free_circ_data(circuitmux_t *cmux,
+                    circuitmux_policy_data_t *pol_data,
+                    circuit_t *circ,
+                    circuitmux_policy_circ_data_t *pol_circ_data)
+
+{
+  ewma_policy_circ_data_t *cdata = NULL;
+
+  tor_assert(cmux);
+  tor_assert(circ);
+  tor_assert(pol_data);
+
+  if (!pol_circ_data) return;
+
+  cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
+
+  tor_free(cdata);
+}
+
+/**
+ * Handle circuit activation; this inserts the circuit's cell_ewma into
+ * the active_circuits_pqueue.
+ */
+
+static void
+ewma_notify_circ_active(circuitmux_t *cmux,
+                        circuitmux_policy_data_t *pol_data,
+                        circuit_t *circ,
+                        circuitmux_policy_circ_data_t *pol_circ_data)
+{
+  ewma_policy_data_t *pol = NULL;
+  ewma_policy_circ_data_t *cdata = NULL;
+
+  tor_assert(cmux);
+  tor_assert(pol_data);
+  tor_assert(circ);
+  tor_assert(pol_circ_data);
+
+  pol = TO_EWMA_POL_DATA(pol_data);
+  cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
+
+  add_cell_ewma(pol, &(cdata->cell_ewma));
+}
+
+/**
+ * Handle circuit deactivation; this removes the circuit's cell_ewma from
+ * the active_circuits_pqueue.
+ */
+
+static void
+ewma_notify_circ_inactive(circuitmux_t *cmux,
+                          circuitmux_policy_data_t *pol_data,
+                          circuit_t *circ,
+                          circuitmux_policy_circ_data_t *pol_circ_data)
+{
+  ewma_policy_data_t *pol = NULL;
+  ewma_policy_circ_data_t *cdata = NULL;
+
+  tor_assert(cmux);
+  tor_assert(pol_data);
+  tor_assert(circ);
+  tor_assert(pol_circ_data);
+
+  pol = TO_EWMA_POL_DATA(pol_data);
+  cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
+
+  remove_cell_ewma(pol, &(cdata->cell_ewma));
+}
+
+/**
+ * Update cell_ewma for this circuit after we've sent some cells, and
+ * remove/reinsert it in the queue.  This used to be done (brokenly,
+ * see bug 6816) in channel_flush_from_first_active_circuit().
+ */
+
+static void
+ewma_notify_xmit_cells(circuitmux_t *cmux,
+                       circuitmux_policy_data_t *pol_data,
+                       circuit_t *circ,
+                       circuitmux_policy_circ_data_t *pol_circ_data,
+                       unsigned int n_cells)
+{
+  ewma_policy_data_t *pol = NULL;
+  ewma_policy_circ_data_t *cdata = NULL;
+  unsigned int tick;
+  double fractional_tick, ewma_increment;
+  /* The current (hi-res) time */
+  struct timeval now_hires;
+  cell_ewma_t *cell_ewma, *tmp;
+
+  tor_assert(cmux);
+  tor_assert(pol_data);
+  tor_assert(circ);
+  tor_assert(pol_circ_data);
+  tor_assert(n_cells > 0);
+
+  pol = TO_EWMA_POL_DATA(pol_data);
+  cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
+
+  /* Rescale the EWMAs if needed */
+  tor_gettimeofday_cached(&now_hires);
+  tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick);
+
+  if (tick != pol->active_circuit_pqueue_last_recalibrated) {
+    scale_active_circuits(pol, tick);
+  }
+
+  /* How much do we adjust the cell count in cell_ewma by? */
+  ewma_increment =
+    ((double)(n_cells)) * pow(ewma_scale_factor, -fractional_tick);
+
+  /* Do the adjustment */
+  cell_ewma = &(cdata->cell_ewma);
+  cell_ewma->cell_count += ewma_increment;
+
+  /*
+   * Since we just sent on this circuit, it should be at the head of
+   * the queue.  Pop the head, assert that it matches, then re-add.
+   */
+  tmp = pop_first_cell_ewma(pol);
+  tor_assert(tmp == cell_ewma);
+  add_cell_ewma(pol, cell_ewma);
+}
+
+/**
+ * Pick the preferred circuit to send from; this will be the one with
+ * the lowest EWMA value in the priority queue.  This used to be done
+ * in channel_flush_from_first_active_circuit().
+ */
+
+static circuit_t *
+ewma_pick_active_circuit(circuitmux_t *cmux,
+                         circuitmux_policy_data_t *pol_data)
+{
+  ewma_policy_data_t *pol = NULL;
+  circuit_t *circ = NULL;
+  cell_ewma_t *cell_ewma = NULL;
+
+  tor_assert(cmux);
+  tor_assert(pol_data);
+
+  pol = TO_EWMA_POL_DATA(pol_data);
+
+  if (smartlist_len(pol->active_circuit_pqueue) > 0) {
+    /* Get the head of the queue */
+    cell_ewma = smartlist_get(pol->active_circuit_pqueue, 0);
+    circ = cell_ewma_to_circuit(cell_ewma);
+  }
+
+  return circ;
+}
+
+/** Helper for sorting cell_ewma_t values in their priority queue. */
+static int
+compare_cell_ewma_counts(const void *p1, const void *p2)
+{
+  const cell_ewma_t *e1 = p1, *e2 = p2;
+
+  if (e1->cell_count < e2->cell_count)
+    return -1;
+  else if (e1->cell_count > e2->cell_count)
+    return 1;
+  else
+    return 0;
+}
+
+/** Given a cell_ewma_t, return a pointer to the circuit containing it. */
+static circuit_t *
+cell_ewma_to_circuit(cell_ewma_t *ewma)
+{
+  ewma_policy_circ_data_t *cdata = NULL;
+
+  tor_assert(ewma);
+  cdata = SUBTYPE_P(ewma, ewma_policy_circ_data_t, cell_ewma);
+  tor_assert(cdata);
+
+  return cdata->circ;
+}
+
+/* ==== Functions for scaling cell_ewma_t ====
+
+   When choosing which cells to relay first, we favor circuits that have been
+   quiet recently.  This gives better latency on connections that aren't
+   pushing lots of data, and makes the network feel more interactive.
+
+   Conceptually, we take an exponentially weighted mean average of the number
+   of cells a circuit has sent, and allow active circuits (those with cells to
+   relay) to send cells in reverse order of their exponentially-weighted mean
+   average (EWMA) cell count.  [That is, a cell sent N seconds ago 'counts'
+   F^N times as much as a cell sent now, for 0<F<1.0, and we favor the
+   circuit that has sent the fewest cells]
+
+   If 'double' had infinite precision, we could do this simply by counting a
+   cell sent at startup as having weight 1.0, and a cell sent N seconds later
+   as having weight F^-N.  This way, we would never need to re-scale
+   any already-sent cells.
+
+   To prevent double from overflowing, we could count a cell sent now as
+   having weight 1.0 and a cell sent N seconds ago as having weight F^N.
+   This, however, would mean we'd need to re-scale *ALL* old circuits every
+   time we wanted to send a cell.
+
+   So as a compromise, we divide time into 'ticks' (currently, 10-second
+   increments) and say that a cell sent at the start of a current tick is
+   worth 1.0, a cell sent N seconds before the start of the current tick is
+   worth F^N, and a cell sent N seconds after the start of the current tick is
+   worth F^-N.  This way we don't overflow, and we don't need to constantly
+   rescale.
+ */
+
+/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs
+ * and the fraction of the tick that has elapsed between the start of the tick
+ * and <b>now</b>.  Return the former and store the latter in
+ * *<b>remainder_out</b>.
+ *
+ * These tick values are not meant to be shared between Tor instances, or used
+ * for other purposes. */
+
+static unsigned
+cell_ewma_tick_from_timeval(const struct timeval *now,
+                            double *remainder_out)
+{
+  unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN);
+  /* rem */
+  double rem = (now->tv_sec % EWMA_TICK_LEN) +
+    ((double)(now->tv_usec)) / 1.0e6;
+  *remainder_out = rem / EWMA_TICK_LEN;
+  return res;
+}
+
+/** Tell the caller whether ewma_enabled is set */
+int
+cell_ewma_enabled(void)
+{
+  return ewma_enabled;
+}
+
+/** Compute and return the current cell_ewma tick. */
+unsigned int
+cell_ewma_get_tick(void)
+{
+  return ((unsigned)approx_time() / EWMA_TICK_LEN);
+}
+
+/** Adjust the global cell scale factor based on <b>options</b> */
+void
+cell_ewma_set_scale_factor(const or_options_t *options,
+                           const networkstatus_t *consensus)
+{
+  int32_t halflife_ms;
+  double halflife;
+  const char *source;
+  if (options && options->CircuitPriorityHalflife >= -EPSILON) {
+    halflife = options->CircuitPriorityHalflife;
+    source = "CircuitPriorityHalflife in configuration";
+  } else if (consensus && (halflife_ms = networkstatus_get_param(
+                 consensus, "CircuitPriorityHalflifeMsec",
+                 -1, -1, INT32_MAX)) >= 0) {
+    halflife = ((double)halflife_ms)/1000.0;
+    source = "CircuitPriorityHalflifeMsec in consensus";
+  } else {
+    halflife = EWMA_DEFAULT_HALFLIFE;
+    source = "Default value";
+  }
+
+  if (halflife <= EPSILON) {
+    /* The cell EWMA algorithm is disabled. */
+    ewma_scale_factor = 0.1;
+    ewma_enabled = 0;
+    log_info(LD_OR,
+             "Disabled cell_ewma algorithm because of value in %s",
+             source);
+  } else {
+    /* convert halflife into halflife-per-tick. */
+    halflife /= EWMA_TICK_LEN;
+    /* compute per-tick scale factor. */
+    ewma_scale_factor = exp( LOG_ONEHALF / halflife );
+    ewma_enabled = 1;
+    log_info(LD_OR,
+             "Enabled cell_ewma algorithm because of value in %s; "
+             "scale factor is %f per %d seconds",
+             source, ewma_scale_factor, EWMA_TICK_LEN);
+  }
+}
+
+/** Return the multiplier necessary to convert the value of a cell sent in
+ * 'from_tick' to one sent in 'to_tick'. */
+static INLINE double
+get_scale_factor(unsigned from_tick, unsigned to_tick)
+{
+  /* This math can wrap around, but that's okay: unsigned overflow is
+     well-defined */
+  int diff = (int)(to_tick - from_tick);
+  return pow(ewma_scale_factor, diff);
+}
+
+/** Adjust the cell count of <b>ewma</b> so that it is scaled with respect to
+ * <b>cur_tick</b> */
+static void
+scale_single_cell_ewma(cell_ewma_t *ewma, unsigned cur_tick)
+{
+  double factor = get_scale_factor(ewma->last_adjusted_tick, cur_tick);
+  ewma->cell_count *= factor;
+  ewma->last_adjusted_tick = cur_tick;
+}
+
+/** Adjust the cell count of every active circuit on <b>chan</b> so
+ * that they are scaled with respect to <b>cur_tick</b> */
+static void
+scale_active_circuits(ewma_policy_data_t *pol, unsigned cur_tick)
+{
+  double factor;
+
+  tor_assert(pol);
+  tor_assert(pol->active_circuit_pqueue);
+
+  factor =
+    get_scale_factor(
+      pol->active_circuit_pqueue_last_recalibrated,
+      cur_tick);
+  /** Ordinarily it isn't okay to change the value of an element in a heap,
+   * but it's okay here, since we are preserving the order. */
+  SMARTLIST_FOREACH_BEGIN(
+      pol->active_circuit_pqueue,
+      cell_ewma_t *, e) {
+    tor_assert(e->last_adjusted_tick ==
+               pol->active_circuit_pqueue_last_recalibrated);
+    e->cell_count *= factor;
+    e->last_adjusted_tick = cur_tick;
+  } SMARTLIST_FOREACH_END(e);
+  pol->active_circuit_pqueue_last_recalibrated = cur_tick;
+}
+
+/** Rescale <b>ewma</b> to the same scale as <b>pol</b>, and add it to
+ * <b>pol</b>'s priority queue of active circuits */
+static void
+add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma)
+{
+  tor_assert(pol);
+  tor_assert(pol->active_circuit_pqueue);
+  tor_assert(ewma);
+  tor_assert(ewma->heap_index == -1);
+
+  scale_single_cell_ewma(
+      ewma,
+      pol->active_circuit_pqueue_last_recalibrated);
+
+  smartlist_pqueue_add(pol->active_circuit_pqueue,
+                       compare_cell_ewma_counts,
+                       STRUCT_OFFSET(cell_ewma_t, heap_index),
+                       ewma);
+}
+
+/** Remove <b>ewma</b> from <b>pol</b>'s priority queue of active circuits */
+static void
+remove_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma)
+{
+  tor_assert(pol);
+  tor_assert(pol->active_circuit_pqueue);
+  tor_assert(ewma);
+  tor_assert(ewma->heap_index != -1);
+
+  smartlist_pqueue_remove(pol->active_circuit_pqueue,
+                          compare_cell_ewma_counts,
+                          STRUCT_OFFSET(cell_ewma_t, heap_index),
+                          ewma);
+}
+
+/** Remove and return the first cell_ewma_t from pol's priority queue of
+ * active circuits.  Requires that the priority queue is nonempty. */
+static cell_ewma_t *
+pop_first_cell_ewma(ewma_policy_data_t *pol)
+{
+  tor_assert(pol);
+  tor_assert(pol->active_circuit_pqueue);
+
+  return smartlist_pqueue_pop(pol->active_circuit_pqueue,
+                              compare_cell_ewma_counts,
+                              STRUCT_OFFSET(cell_ewma_t, heap_index));
+}
+

src/or/circuitmux_ewma.h

@@ -0,0 +1,29 @@
+/* * Copyright (c) 2012, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitmux_ewma.h
+ * \brief Header file for circuitmux_ewma.c
+ **/
+
+#ifndef _TOR_CIRCUITMUX_EWMA_H
+#define _TOR_CIRCUITMUX_EWMA_H
+
+#include "or.h"
+#include "circuitmux.h"
+
+/* Everything but circuitmux_ewma.c should see this extern */
+#ifndef _TOR_CIRCUITMUX_EWMA_C
+
+extern circuitmux_policy_t ewma_policy;
+
+#endif /* !(_TOR_CIRCUITMUX_EWMA_C) */
+
+/* Externally visible EWMA functions */
+int cell_ewma_enabled(void);
+unsigned int cell_ewma_get_tick(void);
+void cell_ewma_set_scale_factor(const or_options_t *options,
+                                const networkstatus_t *consensus);
+
+#endif /* _TOR_CIRCUITMUX_EWMA_H */
+

src/or/circuituse.c

@@ -10,6 +10,7 @@
  **/
 
 #include "or.h"
+#include "channel.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuituse.h"
@@ -53,7 +54,7 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
   tor_assert(conn);
   tor_assert(conn->socks_request);
 
-  if (must_be_open && (circ->state != CIRCUIT_STATE_OPEN || !circ->n_conn))
+  if (must_be_open && (circ->state != CIRCUIT_STATE_OPEN || !circ->n_chan))
     return 0; /* ignore non-open circs */
   if (circ->marked_for_close)
     return 0;
@@ -565,9 +566,9 @@ circuit_expire_building(void)
       continue;
     }
 
-    if (victim->n_conn)
-      log_info(LD_CIRC,"Abandoning circ %s:%d:%d (state %d:%s, purpose %d)",
-               victim->n_conn->_base.address, victim->n_conn->_base.port,
+    if (victim->n_chan)
+      log_info(LD_CIRC,"Abandoning circ %s:%d (state %d:%s, purpose %d)",
+               channel_get_canonical_remote_descr(victim->n_chan),
                victim->n_circ_id,
                victim->state, circuit_state_to_string(victim->state),
                victim->purpose);
@@ -977,13 +978,13 @@ circuit_expire_old_circuits_serverside(time_t now)
     /* If the circuit has been idle for too long, and there are no streams
      * on it, and it ends here, and it used a create_fast, mark it for close.
      */
-    if (or_circ->is_first_hop && !circ->n_conn &&
+    if (or_circ->is_first_hop && !circ->n_chan &&
         !or_circ->n_streams && !or_circ->resolving_streams &&
-        or_circ->p_conn &&
-        or_circ->p_conn->timestamp_last_added_nonpadding <= cutoff) {
+        or_circ->p_chan &&
+        channel_when_last_xmit(or_circ->p_chan) <= cutoff) {
       log_info(LD_CIRC, "Closing circ_id %d (empty %d secs ago)",
                or_circ->p_circ_id,
-               (int)(now - or_circ->p_conn->timestamp_last_added_nonpadding));
+               (int)(now - channel_when_last_xmit(or_circ->p_chan)));
       circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
     }
   }
@@ -1163,6 +1164,7 @@ circuit_try_attaching_streams(origin_circuit_t *circ)
 void
 circuit_build_failed(origin_circuit_t *circ)
 {
+  channel_t *n_chan = NULL;
   /* we should examine circ and see if it failed because of
    * the last hop or an earlier hop. then use this info below.
    */
@@ -1179,11 +1181,12 @@ circuit_build_failed(origin_circuit_t *circ)
     /* We failed at the first hop. If there's an OR connection
      * to blame, blame it. Also, avoid this relay for a while, and
      * fail any one-hop directory fetches destined for it. */
-    const char *n_conn_id = circ->cpath->extend_info->identity_digest;
+    const char *n_chan_id = circ->cpath->extend_info->identity_digest;
     int already_marked = 0;
-    if (circ->_base.n_conn) {
-      or_connection_t *n_conn = circ->_base.n_conn;
-      if (n_conn->is_bad_for_new_circs) {
+    if (circ->_base.n_chan) {
+      n_chan = circ->_base.n_chan;
+
+      if (n_chan->is_bad_for_new_circs) {
         /* We only want to blame this router when a fresh healthy
          * connection fails. So don't mark this router as newly failed,
          * since maybe this was just an old circuit attempt that's
@@ -1195,18 +1198,18 @@ circuit_build_failed(origin_circuit_t *circ)
       }
       log_info(LD_OR,
                "Our circuit failed to get a response from the first hop "
-               "(%s:%d). I'm going to try to rotate to a better connection.",
-               n_conn->_base.address, n_conn->_base.port);
-      n_conn->is_bad_for_new_circs = 1;
+               "(%s). I'm going to try to rotate to a better connection.",
+               channel_get_canonical_remote_descr(n_chan));
+      n_chan->is_bad_for_new_circs = 1;
     } else {
       log_info(LD_OR,
                "Our circuit died before the first hop with no connection");
     }
-    if (n_conn_id && !already_marked) {
-      entry_guard_register_connect_status(n_conn_id, 0, 1, time(NULL));
+    if (n_chan_id && !already_marked) {
+      entry_guard_register_connect_status(n_chan_id, 0, 1, time(NULL));
       /* if there are any one-hop streams waiting on this circuit, fail
        * them now so they can retry elsewhere. */
-      connection_ap_fail_onehop(n_conn_id, circ->build_state);
+      connection_ap_fail_onehop(n_chan_id, circ->build_state);
     }
   }
 

src/or/command.h

@@ -12,8 +12,12 @@
 #ifndef _TOR_COMMAND_H
 #define _TOR_COMMAND_H
 
-void command_process_cell(cell_t *cell, or_connection_t *conn);
-void command_process_var_cell(var_cell_t *cell, or_connection_t *conn);
+#include "channel.h"
+
+void command_process_cell(channel_t *chan, cell_t *cell);
+void command_process_var_cell(channel_t *chan, var_cell_t *cell);
+void command_setup_channel(channel_t *chan);
+void command_setup_listener(channel_listener_t *chan_l);
 
 extern uint64_t stats_n_padding_cells_processed;
 extern uint64_t stats_n_create_cells_processed;

src/or/config.c

@@ -12,8 +12,11 @@
 #define CONFIG_PRIVATE
 
 #include "or.h"
+#include "channel.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
+#include "circuitmux.h"
+#include "circuitmux_ewma.h"
 #include "config.h"
 #include "connection.h"
 #include "connection_edge.h"
@@ -1167,6 +1170,7 @@ options_act(const or_options_t *old_options)
   char *msg=NULL;
   const int transition_affects_workers =
     old_options && options_transition_affects_workers(old_options, options);
+  int old_ewma_enabled;
 
   /* disable ptrace and later, other basic debugging techniques */
   {
@@ -1374,8 +1378,16 @@ options_act(const or_options_t *old_options)
     connection_bucket_init();
 #endif
 
+  old_ewma_enabled = cell_ewma_enabled();
   /* Change the cell EWMA settings */
   cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus());
+  /* If we just enabled ewma, set the cmux policy on all active channels */
+  if (cell_ewma_enabled() && !old_ewma_enabled) {
+    channel_set_cmux_policy_everywhere(&ewma_policy);
+  } else if (!cell_ewma_enabled() && old_ewma_enabled) {
+    /* Turn it off everywhere */
+    channel_set_cmux_policy_everywhere(NULL);
+  }
 
   /* Update the BridgePassword's hashed version as needed.  We store this as a
    * digest so that we can do side-channel-proof comparisons on it.

src/or/connection.c

@@ -12,6 +12,13 @@
 
 #include "or.h"
 #include "buffers.h"
+/*
+ * Define this so we get channel internal functions, since we're implementing
+ * part of a subclass (channel_tls_t).
+ */
+#define _TOR_CHANNEL_INTERNAL
+#include "channel.h"
+#include "channeltls.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuituse.h"
@@ -257,10 +264,6 @@ or_connection_new(int socket_family)
   connection_init(now, TO_CONN(or_conn), CONN_TYPE_OR, socket_family);
 
   or_conn->timestamp_last_added_nonpadding = time(NULL);
-  or_conn->next_circ_id = crypto_rand_int(1<<15);
-
-  or_conn->active_circuit_pqueue = smartlist_new();
-  or_conn->active_circuit_pqueue_last_recalibrated = cell_ewma_get_tick();
 
   return or_conn;
 }
@@ -502,7 +505,6 @@ _connection_free(connection_t *conn)
     or_conn->tls = NULL;
     or_handshake_state_free(or_conn->handshake_state);
     or_conn->handshake_state = NULL;
-    smartlist_free(or_conn->active_circuit_pqueue);
     tor_free(or_conn->nickname);
   }
   if (conn->type == CONN_TYPE_AP) {
@@ -693,6 +695,16 @@ _connection_mark_for_close(connection_t *conn, int line, const char *file)
     return;
   }
 
+  if (conn->type == CONN_TYPE_OR) {
+    /*
+     * Bad news if this happens without telling the controlling channel; do
+     * this so we can find things that call this wrongly when the asserts hit.
+     */
+    log_debug(LD_CHANNEL,
+              "Calling connection_mark_for_close on an OR conn at %s:%d",
+              file, line);
+  }
+
   conn->marked_for_close = line;
   conn->marked_for_close_file = file;
   add_connection_to_closeable_list(conn);
@@ -1281,12 +1293,19 @@ static int
 connection_init_accepted_conn(connection_t *conn,
                               const listener_connection_t *listener)
 {
+  int rv;
+
   connection_start_reading(conn);
 
   switch (conn->type) {
     case CONN_TYPE_OR:
       control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0);
-      return connection_tls_start_handshake(TO_OR_CONN(conn), 1);
+      rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1);
+      if (rv < 0) {
+        connection_or_close_for_error(TO_OR_CONN(conn), 0);
+      }
+      return rv;
+      break;
     case CONN_TYPE_AP:
       TO_ENTRY_CONN(conn)->isolation_flags = listener->isolation_flags;
       TO_ENTRY_CONN(conn)->session_group = listener->session_group;
@@ -2091,7 +2110,8 @@ static int
 connection_counts_as_relayed_traffic(connection_t *conn, time_t now)
 {
   if (conn->type == CONN_TYPE_OR &&
-      TO_OR_CONN(conn)->client_used + CLIENT_IDLE_TIME_FOR_PRIORITY < now)
+      connection_or_client_used(TO_OR_CONN(conn)) +
+                                CLIENT_IDLE_TIME_FOR_PRIORITY < now)
     return 1;
   if (conn->type == CONN_TYPE_DIR && DIR_CONN_IS_SERVER(conn))
     return 1;
@@ -2688,11 +2708,14 @@ connection_handle_read_impl(connection_t *conn)
   before = buf_datalen(conn->inbuf);
   if (connection_read_to_buf(conn, &max_to_read, &socket_error) < 0) {
     /* There's a read error; kill the connection.*/
-    if (conn->type == CONN_TYPE_OR &&
-        conn->state == OR_CONN_STATE_CONNECTING) {
-      connection_or_connect_failed(TO_OR_CONN(conn),
-                                   errno_to_orconn_end_reason(socket_error),
-                                   tor_socket_strerror(socket_error));
+    if (conn->type == CONN_TYPE_OR) {
+      connection_or_notify_error(TO_OR_CONN(conn),
+                                 socket_error != 0 ?
+                                   errno_to_orconn_end_reason(socket_error) :
+                                   END_OR_CONN_REASON_CONNRESET,
+                                 socket_error != 0 ?
+                                   tor_socket_strerror(socket_error) :
+                                   "(unknown, errno was 0)");
     }
     if (CONN_IS_EDGE(conn)) {
       edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
@@ -3214,9 +3237,9 @@ connection_handle_write_impl(connection_t *conn, int force)
         if (CONN_IS_EDGE(conn))
           connection_edge_end_errno(TO_EDGE_CONN(conn));
         if (conn->type == CONN_TYPE_OR)
-          connection_or_connect_failed(TO_OR_CONN(conn),
-                                       errno_to_orconn_end_reason(e),
-                                       tor_socket_strerror(e));
+          connection_or_notify_error(TO_OR_CONN(conn),
+                                     errno_to_orconn_end_reason(e),
+                                     tor_socket_strerror(e));
 
         connection_close_immediate(conn);
         connection_mark_for_close(conn);
@@ -3241,6 +3264,10 @@ connection_handle_write_impl(connection_t *conn, int force)
       connection_stop_writing(conn);
       if (connection_tls_continue_handshake(or_conn) < 0) {
         /* Don't flush; connection is dead. */
+        connection_or_notify_error(or_conn,
+                                   END_OR_CONN_REASON_MISC,
+                                   "TLS error in connection_tls_"
+                                   "continue_handshake()");
         connection_close_immediate(conn);
         connection_mark_for_close(conn);
         return -1;
@@ -3254,19 +3281,23 @@ connection_handle_write_impl(connection_t *conn, int force)
     result = flush_buf_tls(or_conn->tls, conn->outbuf,
                            max_to_write, &conn->outbuf_flushlen);
 
-    /* If we just flushed the last bytes, check if this tunneled dir
-     * request is done. */
+    /* If we just flushed the last bytes, tell the channel on the
+     * or_conn to check if it needs to geoip_change_dirreq_state() */
     /* XXXX move this to flushed_some or finished_flushing -NM */
-    if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id)
-      geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED,
-                                DIRREQ_OR_CONN_BUFFER_FLUSHED);
+    if (buf_datalen(conn->outbuf) == 0 && or_conn->chan)
+      channel_notify_flushed(TLS_CHAN_TO_BASE(or_conn->chan));
 
     switch (result) {
       CASE_TOR_TLS_ERROR_ANY:
       case TOR_TLS_CLOSE:
-        log_info(LD_NET,result!=TOR_TLS_CLOSE?
+        log_info(LD_NET, result != TOR_TLS_CLOSE ?
                  "tls error. breaking.":"TLS connection closed on flush");
         /* Don't flush; connection is dead. */
+        connection_or_notify_error(or_conn,
+                                   END_OR_CONN_REASON_MISC,
+                                   result != TOR_TLS_CLOSE ?
+                                     "TLS error in during flush" :
+                                     "TLS closed during flush");
         connection_close_immediate(conn);
         connection_mark_for_close(conn);
         return -1;
@@ -3325,8 +3356,16 @@ connection_handle_write_impl(connection_t *conn, int force)
   if (result > 0) {
     /* If we wrote any bytes from our buffer, then call the appropriate
      * functions. */
-    if (connection_flushed_some(conn) < 0)
+    if (connection_flushed_some(conn) < 0) {
+      if (connection_speaks_cells(conn)) {
+        connection_or_notify_error(TO_OR_CONN(conn),
+                                   END_OR_CONN_REASON_MISC,
+                                   "Got error back from "
+                                   "connection_flushed_some()");
+      }
+
       connection_mark_for_close(conn);
+    }
   }
 
   if (!connection_wants_to_flush(conn)) { /* it's done flushing */
@@ -4125,7 +4164,6 @@ assert_connection_ok(connection_t *conn, time_t now)
     case CONN_TYPE_OR:
       tor_assert(conn->state >= _OR_CONN_STATE_MIN);
       tor_assert(conn->state <= _OR_CONN_STATE_MAX);
-      tor_assert(TO_OR_CONN(conn)->n_circuits >= 0);
       break;
     case CONN_TYPE_EXIT:
       tor_assert(conn->state >= _EXIT_CONN_STATE_MIN);

src/or/connection_edge.c

@@ -11,6 +11,7 @@
 
 #include "or.h"
 #include "buffers.h"
+#include "channel.h"
 #include "circuitlist.h"
 #include "circuituse.h"
 #include "config.h"
@@ -3068,27 +3069,29 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
       tor_free(address);
       return 0;
     }
-    if (or_circ && or_circ->p_conn && !options->AllowSingleHopExits &&
-        (or_circ->is_first_hop ||
-         (!connection_or_digest_is_known_relay(
-                                       or_circ->p_conn->identity_digest) &&
+    if (or_circ && or_circ->p_chan) {
+      if (!options->AllowSingleHopExits &&
+           (or_circ->is_first_hop ||
+            (!connection_or_digest_is_known_relay(
+                or_circ->p_chan->identity_digest) &&
           should_refuse_unknown_exits(options)))) {
-      /* Don't let clients use us as a single-hop proxy, unless the user
-       * has explicitly allowed that in the config. It attracts attackers
-       * and users who'd be better off with, well, single-hop proxies.
-       */
-      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-             "Attempt by %s to open a stream %s. Closing.",
-             safe_str(or_circ->p_conn->_base.address),
-             or_circ->is_first_hop ? "on first hop of circuit" :
-                                     "from unknown relay");
-      relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                    or_circ->is_first_hop ?
-                                      END_STREAM_REASON_TORPROTOCOL :
-                                      END_STREAM_REASON_MISC,
-                                    NULL);
-      tor_free(address);
-      return 0;
+        /* Don't let clients use us as a single-hop proxy, unless the user
+         * has explicitly allowed that in the config. It attracts attackers
+         * and users who'd be better off with, well, single-hop proxies.
+         */
+        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+               "Attempt by %s to open a stream %s. Closing.",
+               safe_str(channel_get_canonical_remote_descr(or_circ->p_chan)),
+               or_circ->is_first_hop ? "on first hop of circuit" :
+                                       "from unknown relay");
+        relay_send_end_cell_from_edge(rh.stream_id, circ,
+                                      or_circ->is_first_hop ?
+                                        END_STREAM_REASON_TORPROTOCOL :
+                                        END_STREAM_REASON_MISC,
+                                      NULL);
+        tor_free(address);
+        return 0;
+      }
     }
   } else if (rh.command == RELAY_COMMAND_BEGIN_DIR) {
     if (!directory_permits_begindir_requests(options) ||
@@ -3101,8 +3104,8 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
      * caller might want to know whether his IP address has changed, and
      * we might already have corrected _base.addr[ess] for the relay's
      * canonical IP address. */
-    if (or_circ && or_circ->p_conn)
-      address = tor_dup_addr(&or_circ->p_conn->real_addr);
+    if (or_circ && or_circ->p_chan)
+      address = tor_strdup(channel_get_actual_remote_descr(or_circ->p_chan));
     else
       address = tor_strdup("127.0.0.1");
     port = 1; /* XXXX This value is never actually used anywhere, and there
@@ -3120,7 +3123,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
 
   /* Remember the tunneled request ID in the new edge connection, so that
    * we can measure download times. */
-  TO_CONN(n_stream)->dirreq_id = circ->dirreq_id;
+  n_stream->dirreq_id = circ->dirreq_id;
 
   n_stream->_base.purpose = EXIT_PURPOSE_CONNECT;
 
@@ -3178,8 +3181,6 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
 
   if (rh.command == RELAY_COMMAND_BEGIN_DIR) {
     tor_assert(or_circ);
-    if (or_circ->p_conn && !tor_addr_is_null(&or_circ->p_conn->real_addr))
-      tor_addr_copy(&n_stream->_base.addr, &or_circ->p_conn->real_addr);
     return connection_exit_connect_dir(n_stream);
   }
 
@@ -3364,7 +3365,7 @@ connection_exit_connect_dir(edge_connection_t *exitconn)
 
   /* Note that the new dir conn belongs to the same tunneled request as
    * the edge conn, so that we can measure download times. */
-  TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id;
+  dirconn->dirreq_id = exitconn->dirreq_id;
 
   connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn));
 

src/or/connection_or.c

@@ -12,6 +12,13 @@
 
 #include "or.h"
 #include "buffers.h"
+/*
+ * Define this so we get channel internal functions, since we're implementing
+ * part of a subclass (channel_tls_t).
+ */
+#define _TOR_CHANNEL_INTERNAL
+#include "channel.h"
+#include "channeltls.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "command.h"
@@ -43,6 +50,17 @@ static int connection_or_check_valid_tls_handshake(or_connection_t *conn,
 
 static void connection_or_tls_renegotiated_cb(tor_tls_t *tls, void *_conn);
 
+static unsigned int
+connection_or_is_bad_for_new_circs(or_connection_t *or_conn);
+static void connection_or_mark_bad_for_new_circs(or_connection_t *or_conn);
+
+/*
+ * Call this when changing connection state, so notifications to the owning
+ * channel can be handled.
+ */
+
+static void connection_or_change_state(or_connection_t *conn, uint8_t state);
+
 #ifdef USE_BUFFEREVENTS
 static void connection_or_handle_event_cb(struct bufferevent *bufev,
                                           short event, void *arg);
@@ -127,8 +145,11 @@ connection_or_set_identity_digest(or_connection_t *conn, const char *digest)
     return;
 
   /* If the identity was set previously, remove the old mapping. */
-  if (! tor_digest_is_zero(conn->identity_digest))
+  if (! tor_digest_is_zero(conn->identity_digest)) {
     connection_or_remove_from_identity_map(conn);
+    if (conn->chan)
+      channel_clear_identity_digest(TLS_CHAN_TO_BASE(conn->chan));
+  }
 
   memcpy(conn->identity_digest, digest, DIGEST_LEN);
 
@@ -139,6 +160,10 @@ connection_or_set_identity_digest(or_connection_t *conn, const char *digest)
   tmp = digestmap_set(orconn_identity_map, digest, conn);
   conn->next_with_same_id = tmp;
 
+  /* Deal with channels */
+  if (conn->chan)
+    channel_set_identity_digest(TLS_CHAN_TO_BASE(conn->chan), digest);
+
 #if 1
   /* Testing code to check for bugs in representation. */
   for (; tmp; tmp = tmp->next_with_same_id) {
@@ -282,6 +307,39 @@ connection_or_report_broken_states(int severity, int domain)
   smartlist_free(items);
 }
 
+/** Call this to change or_connection_t states, so the owning channel_tls_t can
+ * be notified.
+ */
+
+static void
+connection_or_change_state(or_connection_t *conn, uint8_t state)
+{
+  uint8_t old_state;
+
+  tor_assert(conn);
+
+  old_state = conn->_base.state;
+  conn->_base.state = state;
+
+  if (conn->chan)
+    channel_tls_handle_state_change_on_orconn(conn->chan, conn,
+                                              old_state, state);
+}
+
+/** Return the number of circuits using an or_connection_t; this used to
+ * be an or_connection_t field, but it got moved to channel_t and we
+ * shouldn't maintain two copies. */
+
+int
+connection_or_get_num_circuits(or_connection_t *conn)
+{
+  tor_assert(conn);
+
+  if (conn->chan) {
+    return channel_num_circuits(TLS_CHAN_TO_BASE(conn->chan));
+  } else return 0;
+}
+
 /**************************************************************/
 
 /** Pack the cell_t host-order structure <b>src</b> into network-order
@@ -345,8 +403,11 @@ var_cell_free(var_cell_t *cell)
 int
 connection_or_reached_eof(or_connection_t *conn)
 {
+  tor_assert(conn);
+
   log_info(LD_OR,"OR connection reached EOF. Closing.");
-  connection_mark_for_close(TO_CONN(conn));
+  connection_or_close_normally(conn, 1);
+
   return 0;
 }
 
@@ -375,9 +436,12 @@ connection_or_process_inbuf(or_connection_t *conn)
         tor_assert(TO_CONN(conn)->proxy_state == PROXY_CONNECTED);
         if (connection_tls_start_handshake(conn, 0) < 0)
           ret = -1;
+        /* Touch the channel's active timestamp if there is one */
+        if (conn->chan)
+          channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
       }
       if (ret < 0) {
-        connection_mark_for_close(TO_CONN(conn));
+        connection_or_close_for_error(conn, 0);
       }
 
       return ret;
@@ -410,7 +474,7 @@ connection_or_process_inbuf(or_connection_t *conn)
            connection_or_nonopen_was_started_here(conn) ? "to" : "from",
            conn->_base.address, conn->_base.port,
            conn_state_to_string(conn->_base.type, conn->_base.state));
-    connection_mark_for_close(TO_CONN(conn));
+    connection_or_close_for_error(conn, 0);
     ret = -1;
   }
 
@@ -430,18 +494,31 @@ connection_or_process_inbuf(or_connection_t *conn)
 int
 connection_or_flushed_some(or_connection_t *conn)
 {
-  size_t datalen = connection_get_outbuf_len(TO_CONN(conn));
+  size_t datalen, temp;
+  ssize_t n, flushed;
+
   /* If we're under the low water mark, add cells until we're just over the
    * high water mark. */
+  datalen = connection_get_outbuf_len(TO_CONN(conn));
   if (datalen < OR_CONN_LOWWATER) {
-    ssize_t n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, CELL_NETWORK_SIZE);
-    time_t now = approx_time();
-    while (conn->active_circuits && n > 0) {
-      int flushed;
-      flushed = connection_or_flush_from_first_active_circuit(conn, 1, now);
-      n -= flushed;
+    while ((conn->chan) && channel_tls_more_to_flush(conn->chan)) {
+      /* Compute how many more cells we want at most */
+      n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, CELL_NETWORK_SIZE);
+      /* Bail out if we don't want any more */
+      if (n <= 0) break;
+      /* We're still here; try to flush some more cells */
+      flushed = channel_tls_flush_some_cells(conn->chan, n);
+      /* Bail out if it says it didn't flush anything */
+      if (flushed <= 0) break;
+      /* How much in the outbuf now? */
+      temp = connection_get_outbuf_len(TO_CONN(conn));
+      /* Bail out if we didn't actually increase the outbuf size */
+      if (temp <= datalen) break;
+      /* Update datalen for the next iteration */
+      datalen = temp;
     }
   }
+
   return 0;
 }
 
@@ -480,6 +557,7 @@ connection_or_finished_connecting(or_connection_t *or_conn)
 {
   const int proxy_type = or_conn->proxy_type;
   connection_t *conn;
+
   tor_assert(or_conn);
   conn = TO_CONN(or_conn);
   tor_assert(conn->state == OR_CONN_STATE_CONNECTING);
@@ -491,18 +569,18 @@ connection_or_finished_connecting(or_connection_t *or_conn)
   if (proxy_type != PROXY_NONE) {
     /* start proxy handshake */
     if (connection_proxy_connect(conn, proxy_type) < 0) {
-      connection_mark_for_close(conn);
+      connection_or_close_for_error(or_conn, 0);
       return -1;
     }
 
     connection_start_reading(conn);
-    conn->state = OR_CONN_STATE_PROXY_HANDSHAKING;
+    connection_or_change_state(or_conn, OR_CONN_STATE_PROXY_HANDSHAKING);
     return 0;
   }
 
   if (connection_tls_start_handshake(or_conn, 0) < 0) {
     /* TLS handshaking error of some kind. */
-    connection_mark_for_close(conn);
+    connection_or_close_for_error(or_conn, 0);
     return -1;
   }
   return 0;
@@ -516,11 +594,14 @@ connection_or_about_to_close(or_connection_t *or_conn)
   time_t now = time(NULL);
   connection_t *conn = TO_CONN(or_conn);
 
+  /* Tell the controlling channel we're closed */
+  if (or_conn->chan) {
+    channel_closed(TLS_CHAN_TO_BASE(or_conn->chan));
+    or_conn->chan = NULL;
+  }
+
   /* Remember why we're closing this connection. */
   if (conn->state != OR_CONN_STATE_OPEN) {
-    /* Inform any pending (not attached) circs that they should
-     * give up. */
-    circuit_n_conn_done(TO_OR_CONN(conn), 0);
     /* now mark things down as needed */
     if (connection_or_nonopen_was_started_here(or_conn)) {
       const or_options_t *options = get_options();
@@ -548,9 +629,6 @@ connection_or_about_to_close(or_connection_t *or_conn)
     control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   }
-  /* Now close all the attached circuits on it. */
-  circuit_unlink_all_from_or_conn(TO_OR_CONN(conn),
-                                  END_CIRC_REASON_OR_CONN_CLOSED);
 }
 
 /** Return 1 if identity digest <b>id_digest</b> is known to be a
@@ -708,152 +786,26 @@ connection_or_init_conn_from_address(or_connection_t *conn,
   }
 }
 
-/** Return true iff <b>a</b> is "better" than <b>b</b> for new circuits.
- *
- * A more canonical connection is always better than a less canonical
- * connection.  That aside, a connection is better if it has circuits and the
- * other does not, or if it was created more recently.
- *
- * Requires that both input connections are open; not is_bad_for_new_circs,
- * and not impossibly non-canonical.
- *
- * If <b>forgive_new_connections</b> is true, then we do not call
- * <b>a</b>better than <b>b</b> simply because b has no circuits,
- * unless b is also relatively old.
- */
-static int
-connection_or_is_better(time_t now,
-                        const or_connection_t *a,
-                        const or_connection_t *b,
-                        int forgive_new_connections)
+/** These just pass all the is_bad_for_new_circs manipulation on to
+ * channel_t */
+
+static unsigned int
+connection_or_is_bad_for_new_circs(or_connection_t *or_conn)
 {
-  int newer;
-/** Do not definitively deprecate a new connection with no circuits on it
- * until this much time has passed. */
-#define NEW_CONN_GRACE_PERIOD (15*60)
+  tor_assert(or_conn);
 
-  if (b->is_canonical && !a->is_canonical)
-    return 0; /* A canonical connection is better than a non-canonical
-               * one, no matter how new it is or which has circuits. */
-
-  newer = b->_base.timestamp_created < a->_base.timestamp_created;
-
-  if (
-      /* We prefer canonical connections regardless of newness. */
-      (!b->is_canonical && a->is_canonical) ||
-      /* If both have circuits we prefer the newer: */
-      (b->n_circuits && a->n_circuits && newer) ||
-      /* If neither has circuits we prefer the newer: */
-      (!b->n_circuits && !a->n_circuits && newer))
-    return 1;
-
-  /* If one has no circuits and the other does... */
-  if (!b->n_circuits && a->n_circuits) {
-    /* Then it's bad, unless it's in its grace period and we're forgiving. */
-    if (forgive_new_connections &&
-        now < b->_base.timestamp_created + NEW_CONN_GRACE_PERIOD)
-      return 0;
-    else
-      return 1;
-  }
-
-  return 0;
+  if (or_conn->chan)
+    return channel_is_bad_for_new_circs(TLS_CHAN_TO_BASE(or_conn->chan));
+  else return 0;
 }
 
-/** Return the OR connection we should use to extend a circuit to the router
- * whose identity is <b>digest</b>, and whose address we believe (or have been
- * told in an extend cell) is <b>target_addr</b>.  If there is no good
- * connection, set *<b>msg_out</b> to a message describing the connection's
- * state and our next action, and set <b>launch_out</b> to a boolean for
- * whether we should launch a new connection or not.
- */
-or_connection_t *
-connection_or_get_for_extend(const char *digest,
-                             const tor_addr_t *target_addr,
-                             const char **msg_out,
-                             int *launch_out)
+static void
+connection_or_mark_bad_for_new_circs(or_connection_t *or_conn)
 {
-  or_connection_t *conn, *best=NULL;
-  int n_inprogress_goodaddr = 0, n_old = 0, n_noncanonical = 0, n_possible = 0;
-  time_t now = approx_time();
+  tor_assert(or_conn);
 
-  tor_assert(msg_out);
-  tor_assert(launch_out);
-
-  if (!orconn_identity_map) {
-    *msg_out = "Router not connected (nothing is).  Connecting.";
-    *launch_out = 1;
-    return NULL;
-  }
-
-  conn = digestmap_get(orconn_identity_map, digest);
-
-  for (; conn; conn = conn->next_with_same_id) {
-    tor_assert(conn->_base.magic == OR_CONNECTION_MAGIC);
-    tor_assert(conn->_base.type == CONN_TYPE_OR);
-    tor_assert(tor_memeq(conn->identity_digest, digest, DIGEST_LEN));
-    if (conn->_base.marked_for_close)
-      continue;
-    /* Never return a connection on which the other end appears to be
-     * a client. */
-    if (conn->is_connection_with_client) {
-      continue;
-    }
-    /* Never return a non-open connection. */
-    if (conn->_base.state != OR_CONN_STATE_OPEN) {
-      /* If the address matches, don't launch a new connection for this
-       * circuit. */
-      if (!tor_addr_compare(&conn->real_addr, target_addr, CMP_EXACT))
-        ++n_inprogress_goodaddr;
-      continue;
-    }
-    /* Never return a connection that shouldn't be used for circs. */
-    if (conn->is_bad_for_new_circs) {
-      ++n_old;
-      continue;
-    }
-    /* Never return a non-canonical connection using a recent link protocol
-     * if the address is not what we wanted.
-     *
-     * (For old link protocols, we can't rely on is_canonical getting
-     * set properly if we're talking to the right address, since we might
-     * have an out-of-date descriptor, and we will get no NETINFO cell to
-     * tell us about the right address.) */
-    if (!conn->is_canonical && conn->link_proto >= 2 &&
-        tor_addr_compare(&conn->real_addr, target_addr, CMP_EXACT)) {
-      ++n_noncanonical;
-      continue;
-    }
-
-    ++n_possible;
-
-    if (!best) {
-      best = conn; /* If we have no 'best' so far, this one is good enough. */
-      continue;
-    }
-
-    if (connection_or_is_better(now, conn, best, 0))
-      best = conn;
-  }
-
-  if (best) {
-    *msg_out = "Connection is fine; using it.";
-    *launch_out = 0;
-    return best;
-  } else if (n_inprogress_goodaddr) {
-    *msg_out = "Connection in progress; waiting.";
-    *launch_out = 0;
-    return NULL;
-  } else if (n_old || n_noncanonical) {
-    *msg_out = "Connections all too old, or too non-canonical. "
-      " Launching a new one.";
-    *launch_out = 1;
-    return NULL;
-  } else {
-    *msg_out = "Not connected. Connecting.";
-    *launch_out = 1;
-    return NULL;
-  }
+  if (or_conn->chan)
+    channel_mark_bad_for_new_circs(TLS_CHAN_TO_BASE(or_conn->chan));
 }
 
 /** How old do we let a connection to an OR get before deciding it's
@@ -874,8 +826,8 @@ connection_or_get_for_extend(const char *digest,
  *    - all open non-canonical connections for which a 'better' non-canonical
  *      connection exists to the same router at the same address.
  *
- * See connection_or_is_better() for our idea of what makes one OR connection
- * better than another.
+ * See channel_is_better() in channel.c for our idea of what makes one OR
+ * connection better than another.
  */
 static void
 connection_or_group_set_badness(or_connection_t *head, int force)
@@ -888,7 +840,7 @@ connection_or_group_set_badness(or_connection_t *head, int force)
    * everything else is. */
   for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
     if (or_conn->_base.marked_for_close ||
-        or_conn->is_bad_for_new_circs)
+        connection_or_is_bad_for_new_circs(or_conn))
       continue;
     if (force ||
         or_conn->_base.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD
@@ -898,10 +850,10 @@ connection_or_group_set_badness(or_connection_t *head, int force)
                "(fd %d, %d secs old).",
                or_conn->_base.address, or_conn->_base.port, or_conn->_base.s,
                (int)(now - or_conn->_base.timestamp_created));
-      or_conn->is_bad_for_new_circs = 1;
+      connection_or_mark_bad_for_new_circs(or_conn);
     }
 
-    if (or_conn->is_bad_for_new_circs) {
+    if (connection_or_is_bad_for_new_circs(or_conn)) {
       ++n_old;
     } else if (or_conn->_base.state != OR_CONN_STATE_OPEN) {
       ++n_inprogress;
@@ -916,7 +868,7 @@ connection_or_group_set_badness(or_connection_t *head, int force)
    * expire everything that's worse, and find the very best if we can. */
   for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
     if (or_conn->_base.marked_for_close ||
-        or_conn->is_bad_for_new_circs)
+        connection_or_is_bad_for_new_circs(or_conn))
       continue; /* This one doesn't need to be marked bad. */
     if (or_conn->_base.state != OR_CONN_STATE_OPEN)
       continue; /* Don't mark anything bad until we have seen what happens
@@ -930,12 +882,17 @@ connection_or_group_set_badness(or_connection_t *head, int force)
                "another connection to that OR that is.",
                or_conn->_base.address, or_conn->_base.port, or_conn->_base.s,
                (int)(now - or_conn->_base.timestamp_created));
-      or_conn->is_bad_for_new_circs = 1;
+      connection_or_mark_bad_for_new_circs(or_conn);
       continue;
     }
 
-    if (!best || connection_or_is_better(now, or_conn, best, 0))
+    if (!best ||
+        channel_is_better(now,
+                          TLS_CHAN_TO_BASE(or_conn->chan),
+                          TLS_CHAN_TO_BASE(best->chan),
+                          0)) {
       best = or_conn;
+    }
   }
 
   if (!best)
@@ -957,10 +914,13 @@ connection_or_group_set_badness(or_connection_t *head, int force)
    */
   for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
     if (or_conn->_base.marked_for_close ||
-        or_conn->is_bad_for_new_circs ||
+        connection_or_is_bad_for_new_circs(or_conn) ||
         or_conn->_base.state != OR_CONN_STATE_OPEN)
       continue;
-    if (or_conn != best && connection_or_is_better(now, best, or_conn, 1)) {
+    if (or_conn != best &&
+        channel_is_better(now,
+                          TLS_CHAN_TO_BASE(best->chan),
+                          TLS_CHAN_TO_BASE(or_conn->chan), 1)) {
       /* This isn't the best conn, _and_ the best conn is better than it,
          even when we're being forgiving. */
       if (best->is_canonical) {
@@ -971,7 +931,7 @@ connection_or_group_set_badness(or_connection_t *head, int force)
                  or_conn->_base.address, or_conn->_base.port, or_conn->_base.s,
                  (int)(now - or_conn->_base.timestamp_created),
                  best->_base.s, (int)(now - best->_base.timestamp_created));
-        or_conn->is_bad_for_new_circs = 1;
+        connection_or_mark_bad_for_new_circs(or_conn);
       } else if (!tor_addr_compare(&or_conn->real_addr,
                                    &best->real_addr, CMP_EXACT)) {
         log_info(LD_OR,
@@ -981,7 +941,7 @@ connection_or_group_set_badness(or_connection_t *head, int force)
                  or_conn->_base.address, or_conn->_base.port, or_conn->_base.s,
                  (int)(now - or_conn->_base.timestamp_created),
                  best->_base.s, (int)(now - best->_base.timestamp_created));
-        or_conn->is_bad_for_new_circs = 1;
+        connection_or_mark_bad_for_new_circs(or_conn);
       }
     }
   }
@@ -1019,8 +979,41 @@ connection_or_connect_failed(or_connection_t *conn,
     control_event_bootstrap_problem(msg, reason);
 }
 
+/** <b>conn</b> got an error in connection_handle_read_impl() or
+ * connection_handle_write_impl() and is going to die soon.
+ *
+ * <b>reason</b> specifies the or_conn_end_reason for the failure;
+ * <b>msg</b> specifies the strerror-style error message.
+ */
+void
+connection_or_notify_error(or_connection_t *conn,
+                           int reason, const char *msg)
+{
+  channel_t *chan;
+
+  tor_assert(conn);
+
+  /* If we're connecting, call connect_failed() too */
+  if (TO_CONN(conn)->state == OR_CONN_STATE_CONNECTING)
+    connection_or_connect_failed(conn, reason, msg);
+
+  /* Tell the controlling channel if we have one */
+  if (conn->chan) {
+    chan = TLS_CHAN_TO_BASE(conn->chan);
+    /* Don't transition if we're already in closing, closed or error */
+    if (!(chan->state == CHANNEL_STATE_CLOSING ||
+          chan->state == CHANNEL_STATE_CLOSED ||
+          chan->state == CHANNEL_STATE_ERROR)) {
+      channel_close_for_error(chan);
+    }
+  }
+
+  /* No need to mark for error because connection.c is about to do that */
+}
+
 /** Launch a new OR connection to <b>addr</b>:<b>port</b> and expect to
- * handshake with an OR with identity digest <b>id_digest</b>.
+ * handshake with an OR with identity digest <b>id_digest</b>.  Optionally,
+ * pass in a pointer to a channel using this connection.
  *
  * If <b>id_digest</b> is me, do nothing. If we're already connected to it,
  * return that connection. If the connect() is in progress, set the
@@ -1035,7 +1028,8 @@ connection_or_connect_failed(or_connection_t *conn,
  */
 or_connection_t *
 connection_or_connect(const tor_addr_t *_addr, uint16_t port,
-                      const char *id_digest)
+                      const char *id_digest,
+                      channel_tls_t *chan)
 {
   or_connection_t *conn;
   const or_options_t *options = get_options();
@@ -1058,9 +1052,17 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port,
 
   conn = or_connection_new(tor_addr_family(&addr));
 
-  /* set up conn so it's got all the data we need to remember */
+  /*
+   * Set up conn so it's got all the data we need to remember for channels
+   *
+   * This stuff needs to happen before connection_or_init_conn_from_address()
+   * so connection_or_set_identity_digest() and such know where to look to
+   * keep the channel up to date.
+   */
+  conn->chan = chan;
+  chan->conn = conn;
   connection_or_init_conn_from_address(conn, &addr, port, id_digest, 1);
-  conn->_base.state = OR_CONN_STATE_CONNECTING;
+  connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
   control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
 
   conn->is_outgoing = 1;
@@ -1129,6 +1131,52 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port,
   return conn;
 }
 
+/** Mark orconn for close and transition the associated channel, if any, to
+ * the closing state.
+ */
+
+void
+connection_or_close_normally(or_connection_t *orconn, int flush)
+{
+  channel_t *chan = NULL;
+
+  tor_assert(orconn);
+  if (flush) connection_mark_and_flush(TO_CONN(orconn));
+  else connection_mark_for_close(TO_CONN(orconn));
+  if (orconn->chan) {
+    chan = TLS_CHAN_TO_BASE(orconn->chan);
+    /* Don't transition if we're already in closing, closed or error */
+    if (!(chan->state == CHANNEL_STATE_CLOSING ||
+          chan->state == CHANNEL_STATE_CLOSED ||
+          chan->state == CHANNEL_STATE_ERROR)) {
+      channel_close_from_lower_layer(chan);
+    }
+  }
+}
+
+/** Mark orconn for close and transition the associated channel, if any, to
+ * the error state.
+ */
+
+void
+connection_or_close_for_error(or_connection_t *orconn, int flush)
+{
+  channel_t *chan = NULL;
+
+  tor_assert(orconn);
+  if (flush) connection_mark_and_flush(TO_CONN(orconn));
+  else connection_mark_for_close(TO_CONN(orconn));
+  if (orconn->chan) {
+    chan = TLS_CHAN_TO_BASE(orconn->chan);
+    /* Don't transition if we're already in closing, closed or error */
+    if (!(chan->state == CHANNEL_STATE_CLOSING ||
+          chan->state == CHANNEL_STATE_CLOSED ||
+          chan->state == CHANNEL_STATE_ERROR)) {
+      channel_close_for_error(chan);
+    }
+  }
+}
+
 /** Begin the tls handshake with <b>conn</b>. <b>receiving</b> is 0 if
  * we initiated the connection, else it's 1.
  *
@@ -1140,7 +1188,24 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port,
 int
 connection_tls_start_handshake(or_connection_t *conn, int receiving)
 {
-  conn->_base.state = OR_CONN_STATE_TLS_HANDSHAKING;
+  channel_listener_t *chan_listener;
+  channel_t *chan;
+
+  /* Incoming connections will need a new channel passed to the
+   * channel_tls_listener */
+  if (receiving) {
+    /* It shouldn't already be set */
+    tor_assert(!(conn->chan));
+    chan_listener = channel_tls_get_listener();
+    if (!chan_listener) {
+      chan_listener = channel_tls_start_listener();
+      command_setup_listener(chan_listener);
+    }
+    chan = channel_tls_handle_incoming(conn);
+    channel_listener_queue_incoming(chan_listener, chan);
+  }
+
+  connection_or_change_state(conn, OR_CONN_STATE_TLS_HANDSHAKING);
   tor_assert(!conn->tls);
   conn->tls = tor_tls_new(conn->_base.s, receiving);
   if (!conn->tls) {
@@ -1201,7 +1266,7 @@ connection_or_tls_renegotiated_cb(tor_tls_t *tls, void *_conn)
   if (connection_tls_finish_handshake(conn) < 0) {
     /* XXXX_TLS double-check that it's ok to do this from inside read. */
     /* XXXX_TLS double-check that this verifies certificates. */
-    connection_mark_for_close(TO_CONN(conn));
+    connection_or_close_for_error(conn, 0);
   }
 }
 
@@ -1242,7 +1307,8 @@ connection_tls_continue_handshake(or_connection_t *conn)
             } else {
               log_debug(LD_OR, "Done with initial SSL handshake (client-side)."
                         " Requesting renegotiation.");
-              conn->_base.state = OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING;
+              connection_or_change_state(conn,
+                  OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING);
               goto again;
             }
           }
@@ -1254,7 +1320,8 @@ connection_tls_continue_handshake(or_connection_t *conn)
           tor_tls_set_renegotiate_callback(conn->tls,
                                            connection_or_tls_renegotiated_cb,
                                            conn);
-          conn->_base.state = OR_CONN_STATE_TLS_SERVER_RENEGOTIATING;
+          connection_or_change_state(conn,
+              OR_CONN_STATE_TLS_SERVER_RENEGOTIATING);
           connection_stop_writing(TO_CONN(conn));
           connection_start_reading(TO_CONN(conn));
           return 0;
@@ -1287,7 +1354,7 @@ connection_or_handle_event_cb(struct bufferevent *bufev, short event,
     if (conn->_base.state == OR_CONN_STATE_TLS_HANDSHAKING) {
       if (tor_tls_finish_handshake(conn->tls) < 0) {
         log_warn(LD_OR, "Problem finishing handshake");
-        connection_mark_for_close(TO_CONN(conn));
+        connection_or_close_for_error(conn, 0);
         return;
       }
     }
@@ -1298,14 +1365,15 @@ connection_or_handle_event_cb(struct bufferevent *bufev, short event,
           if (tor_tls_received_v3_certificate(conn->tls)) {
             log_info(LD_OR, "Client got a v3 cert!");
             if (connection_or_launch_v3_or_handshake(conn) < 0)
-              connection_mark_for_close(TO_CONN(conn));
+              connection_or_close_for_error(conn, 0);
             return;
           } else {
-            conn->_base.state = OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING;
+            connection_or_change_state(conn,
+                OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING);
             tor_tls_unblock_renegotiation(conn->tls);
             if (bufferevent_ssl_renegotiate(conn->_base.bufev)<0) {
               log_warn(LD_OR, "Start_renegotiating went badly.");
-              connection_mark_for_close(TO_CONN(conn));
+              connection_or_close_for_error(conn, 0);
             }
             tor_tls_unblock_renegotiation(conn->tls);
             return; /* ???? */
@@ -1320,7 +1388,8 @@ connection_or_handle_event_cb(struct bufferevent *bufev, short event,
           tor_tls_set_renegotiate_callback(conn->tls,
                                            connection_or_tls_renegotiated_cb,
                                            conn);
-          conn->_base.state = OR_CONN_STATE_TLS_SERVER_RENEGOTIATING;
+          connection_or_change_state(conn,
+              OR_CONN_STATE_TLS_SERVER_RENEGOTIATING);
         } else if (handshakes == 2) {
           /* v2 handshake, as a server.  Two handshakes happened already,
            * so we treat renegotiation as done.
@@ -1329,18 +1398,18 @@ connection_or_handle_event_cb(struct bufferevent *bufev, short event,
         } else if (handshakes > 2) {
           log_warn(LD_OR, "More than two handshakes done on connection. "
                    "Closing.");
-          connection_mark_for_close(TO_CONN(conn));
+          connection_or_close_for_error(conn, 0);
         } else {
           log_warn(LD_BUG, "We were unexpectedly told that a connection "
                    "got %d handshakes. Closing.", handshakes);
-          connection_mark_for_close(TO_CONN(conn));
+          connection_or_close_for_error(conn, 0);
         }
         return;
       }
     }
     connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
     if (connection_tls_finish_handshake(conn) < 0)
-      connection_mark_for_close(TO_CONN(conn)); /* ???? */
+      connection_or_close_for_error(conn, 0); /* ???? */
     return;
   }
 
@@ -1370,29 +1439,6 @@ connection_or_nonopen_was_started_here(or_connection_t *conn)
   return !tor_tls_is_server(conn->tls);
 }
 
-/** Set the circid_type field of <b>conn</b> (which determines which part of
- * the circuit ID space we're willing to use) based on comparing our ID to
- * <b>identity_rcvd</b> */
-void
-connection_or_set_circid_type(or_connection_t *conn,
-                              crypto_pk_t *identity_rcvd)
-{
-  const int started_here = connection_or_nonopen_was_started_here(conn);
-  crypto_pk_t *our_identity =
-    started_here ? get_tlsclient_identity_key() :
-                   get_server_identity_key();
-
-  if (identity_rcvd) {
-    if (crypto_pk_cmp_keys(our_identity, identity_rcvd)<0) {
-      conn->circ_id_type = CIRC_ID_TYPE_LOWER;
-    } else {
-      conn->circ_id_type = CIRC_ID_TYPE_HIGHER;
-    }
-  } else {
-    conn->circ_id_type = CIRC_ID_TYPE_NEITHER;
-  }
-}
-
 /** <b>Conn</b> just completed its handshake. Return 0 if all is well, and
  * return -1 if he is lying, broken, or otherwise something is wrong.
  *
@@ -1470,7 +1516,8 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn,
     memset(digest_rcvd_out, 0, DIGEST_LEN);
   }
 
-  connection_or_set_circid_type(conn, identity_rcvd);
+  tor_assert(conn->chan);
+  channel_set_circid_type(TLS_CHAN_TO_BASE(conn->chan), identity_rcvd);
   crypto_pk_free(identity_rcvd);
 
   if (started_here)
@@ -1547,6 +1594,19 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
   return 0;
 }
 
+/** Return when a client used this, for connection.c, since client_used
+ * is now one of the timestamps of channel_t */
+
+time_t
+connection_or_client_used(or_connection_t *conn)
+{
+  tor_assert(conn);
+
+  if (conn->chan) {
+    return channel_when_last_client(TLS_CHAN_TO_BASE(conn->chan));
+  } else return 0;
+}
+
 /** The v1/v2 TLS handshake is finished.
  *
  * Make sure we are happy with the person we just handshaked with.
@@ -1588,7 +1648,7 @@ connection_tls_finish_handshake(or_connection_t *conn)
     tor_tls_block_renegotiation(conn->tls);
     return connection_or_set_state_open(conn);
   } else {
-    conn->_base.state = OR_CONN_STATE_OR_HANDSHAKING_V2;
+    connection_or_change_state(conn, OR_CONN_STATE_OR_HANDSHAKING_V2);
     if (connection_init_or_handshake_state(conn, started_here) < 0)
       return -1;
     if (!started_here) {
@@ -1613,7 +1673,7 @@ connection_or_launch_v3_or_handshake(or_connection_t *conn)
 
   circuit_build_times_network_is_live(&circ_times);
 
-  conn->_base.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
+  connection_or_change_state(conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
   if (connection_init_or_handshake_state(conn, 1) < 0)
     return -1;
 
@@ -1732,35 +1792,9 @@ or_handshake_state_record_var_cell(or_handshake_state_t *state,
 int
 connection_or_set_state_open(or_connection_t *conn)
 {
-  int started_here = connection_or_nonopen_was_started_here(conn);
-  time_t now = time(NULL);
-  conn->_base.state = OR_CONN_STATE_OPEN;
+  connection_or_change_state(conn, OR_CONN_STATE_OPEN);
   control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0);
 
-  if (started_here) {
-    circuit_build_times_network_is_live(&circ_times);
-    rep_hist_note_connect_succeeded(conn->identity_digest, now);
-    if (entry_guard_register_connect_status(conn->identity_digest,
-                                            1, 0, now) < 0) {
-      /* Close any circuits pending on this conn. We leave it in state
-       * 'open' though, because it didn't actually *fail* -- we just
-       * chose not to use it. (Otherwise
-       * connection_about_to_close_connection() will call a big pile of
-       * functions to indicate we shouldn't try it again.) */
-      log_debug(LD_OR, "New entry guard was reachable, but closing this "
-                "connection so we can retry the earlier entry guards.");
-      circuit_n_conn_done(conn, 0);
-      return -1;
-    }
-    router_set_status(conn->identity_digest, 1);
-  } else {
-    /* only report it to the geoip module if it's not a known router */
-    if (!router_get_by_id_digest(conn->identity_digest)) {
-      geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &TO_CONN(conn)->addr,
-                             now);
-    }
-  }
-
   or_handshake_state_free(conn->handshake_state);
   conn->handshake_state = NULL;
   IF_HAS_BUFFEREVENT(TO_CONN(conn), {
@@ -1769,8 +1803,6 @@ connection_or_set_state_open(or_connection_t *conn)
     connection_start_reading(TO_CONN(conn));
   }
 
-  circuit_n_conn_done(conn, 1); /* send the pending creates, if any. */
-
   return 0;
 }
 
@@ -1790,6 +1822,10 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
 
   connection_write_to_buf(networkcell.body, CELL_NETWORK_SIZE, TO_CONN(conn));
 
+  /* Touch the channel's active timestamp if there is one */
+  if (conn->chan)
+    channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
+
   if (conn->_base.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_cell(conn->handshake_state, cell, 0);
 
@@ -1816,6 +1852,10 @@ connection_or_write_var_cell_to_buf(const var_cell_t *cell,
     or_handshake_state_record_var_cell(conn->handshake_state, cell, 0);
   if (cell->command != CELL_PADDING)
     conn->timestamp_last_added_nonpadding = approx_time();
+
+  /* Touch the channel's active timestamp if there is one */
+  if (conn->chan)
+    channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 }
 
 /** See whether there's a variable-length cell waiting on <b>or_conn</b>'s
@@ -1852,8 +1892,13 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn)
     if (connection_fetch_var_cell_from_buf(conn, &var_cell)) {
       if (!var_cell)
         return 0; /* not yet. */
+
+      /* Touch the channel's active timestamp if there is one */
+      if (conn->chan)
+        channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
+
       circuit_build_times_network_is_live(&circ_times);
-      command_process_var_cell(var_cell, conn);
+      channel_tls_handle_var_cell(var_cell, conn);
       var_cell_free(var_cell);
     } else {
       char buf[CELL_NETWORK_SIZE];
@@ -1862,6 +1907,10 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn)
           < CELL_NETWORK_SIZE) /* whole response available? */
         return 0; /* not yet */
 
+      /* Touch the channel's active timestamp if there is one */
+      if (conn->chan)
+        channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
+
       circuit_build_times_network_is_live(&circ_times);
       connection_fetch_from_buf(buf, CELL_NETWORK_SIZE, TO_CONN(conn));
 
@@ -1869,34 +1918,11 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn)
        * network-order string) */
       cell_unpack(&cell, buf);
 
-      command_process_cell(&cell, conn);
+      channel_tls_handle_cell(&cell, conn);
     }
   }
 }
 
-/** Write a destroy cell with circ ID <b>circ_id</b> and reason <b>reason</b>
- * onto OR connection <b>conn</b>.  Don't perform range-checking on reason:
- * we may want to propagate reasons from other cells.
- *
- * Return 0.
- */
-int
-connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, int reason)
-{
-  cell_t cell;
-
-  tor_assert(conn);
-
-  memset(&cell, 0, sizeof(cell_t));
-  cell.circ_id = circ_id;
-  cell.command = CELL_DESTROY;
-  cell.payload[0] = (uint8_t) reason;
-  log_debug(LD_OR,"Sending destroy (circID %d).", circ_id);
-
-  connection_or_write_cell_to_buf(&cell, conn);
-  return 0;
-}
-
 /** Array of recognized link protocol versions. */
 static const uint16_t or_protocol_versions[] = { 1, 2, 3 };
 /** Number of versions in <b>or_protocol_versions</b>. */

src/or/connection_or.h

@@ -33,8 +33,14 @@ void connection_or_update_token_buckets(smartlist_t *conns,
 
 void connection_or_connect_failed(or_connection_t *conn,
                                   int reason, const char *msg);
+void connection_or_notify_error(or_connection_t *conn,
+                                int reason, const char *msg);
 or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port,
-                                       const char *id_digest);
+                                       const char *id_digest,
+                                       channel_tls_t *chan);
+
+void connection_or_close_normally(or_connection_t *orconn, int flush);
+void connection_or_close_for_error(or_connection_t *orconn, int flush);
 
 void connection_or_report_broken_states(int severity, int domain);
 
@@ -50,8 +56,8 @@ void connection_or_init_conn_from_address(or_connection_t *conn,
                                           int started_here);
 int connection_or_client_learned_peer_id(or_connection_t *conn,
                                          const uint8_t *peer_id);
-void connection_or_set_circid_type(or_connection_t *conn,
-                                   crypto_pk_t *identity_rcvd);
+time_t connection_or_client_used(or_connection_t *conn);
+int connection_or_get_num_circuits(or_connection_t *conn);
 void or_handshake_state_free(or_handshake_state_t *state);
 void or_handshake_state_record_cell(or_handshake_state_t *state,
                                     const cell_t *cell,
@@ -65,8 +71,6 @@ void connection_or_write_cell_to_buf(const cell_t *cell,
                                      or_connection_t *conn);
 void connection_or_write_var_cell_to_buf(const var_cell_t *cell,
                                          or_connection_t *conn);
-int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn,
-                               int reason);
 int connection_or_send_versions(or_connection_t *conn, int v3_plus);
 int connection_or_send_netinfo(or_connection_t *conn);
 int connection_or_send_certs_cell(or_connection_t *conn);

src/or/control.c

@@ -12,6 +12,8 @@
 
 #include "or.h"
 #include "buffers.h"
+#include "channel.h"
+#include "channeltls.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuituse.h"
@@ -3822,8 +3824,12 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
       log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
       return 0;
     }
-  ncircs = circuit_count_pending_on_or_conn(conn);
-  ncircs += conn->n_circuits;
+  if (conn->chan) {
+    ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan));
+  } else {
+    ncircs = 0;
+  }
+  ncircs += connection_or_get_num_circuits(conn);
   if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) {
     tor_snprintf(ncircs_buf, sizeof(ncircs_buf), "%sNCIRCS=%d",
                  reason ? " " : "", ncircs);

src/or/cpuworker.c

@@ -14,6 +14,8 @@
 
 #include "or.h"
 #include "buffers.h"
+#include "channel.h"
+#include "channeltls.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "config.h"
@@ -68,19 +70,20 @@ connection_cpu_finished_flushing(connection_t *conn)
 /** Pack global_id and circ_id; set *tag to the result. (See note on
  * cpuworker_main for wire format.) */
 static void
-tag_pack(char *tag, uint64_t conn_id, circid_t circ_id)
+tag_pack(char *tag, uint64_t chan_id, circid_t circ_id)
 {
   /*XXXX RETHINK THIS WHOLE MESS !!!! !NM NM NM NM*/
-  set_uint64(tag, conn_id);
+  /*XXXX DOUBLEPLUSTHIS!!!! AS AS AS AS*/
+  set_uint64(tag, chan_id);
   set_uint16(tag+8, circ_id);
 }
 
 /** Unpack <b>tag</b> into addr, port, and circ_id.
  */
 static void
-tag_unpack(const char *tag, uint64_t *conn_id, circid_t *circ_id)
+tag_unpack(const char *tag, uint64_t *chan_id, circid_t *circ_id)
 {
-  *conn_id = get_uint64(tag);
+  *chan_id = get_uint64(tag);
   *circ_id = get_uint16(tag+8);
 }
 
@@ -131,10 +134,9 @@ connection_cpu_process_inbuf(connection_t *conn)
 {
   char success;
   char buf[LEN_ONION_RESPONSE];
-  uint64_t conn_id;
+  uint64_t chan_id;
   circid_t circ_id;
-  connection_t *tmp_conn;
-  or_connection_t *p_conn = NULL;
+  channel_t *p_chan = NULL;
   circuit_t *circ;
 
   tor_assert(conn);
@@ -152,15 +154,16 @@ connection_cpu_process_inbuf(connection_t *conn)
     connection_fetch_from_buf(buf,LEN_ONION_RESPONSE-1,conn);
 
     /* parse out the circ it was talking about */
-    tag_unpack(buf, &conn_id, &circ_id);
+    tag_unpack(buf, &chan_id, &circ_id);
     circ = NULL;
-    tmp_conn = connection_get_by_global_id(conn_id);
-    if (tmp_conn && !tmp_conn->marked_for_close &&
-        tmp_conn->type == CONN_TYPE_OR)
-      p_conn = TO_OR_CONN(tmp_conn);
+    log_debug(LD_OR,
+              "Unpacking cpuworker reply, chan_id is " U64_FORMAT
+              ", circ_id is %d",
+              U64_PRINTF_ARG(chan_id), circ_id);
+    p_chan = channel_find_by_global_id(chan_id);
 
-    if (p_conn)
-      circ = circuit_get_by_circid_orconn(circ_id, p_conn);
+    if (p_chan)
+      circ = circuit_get_by_circid_channel(circ_id, p_chan);
 
     if (success == 0) {
       log_debug(LD_OR,
@@ -475,12 +478,12 @@ assign_onionskin_to_cpuworker(connection_t *cpuworker,
 
     tor_assert(cpuworker);
 
-    if (!circ->p_conn) {
-      log_info(LD_OR,"circ->p_conn gone. Failing circ.");
+    if (!circ->p_chan) {
+      log_info(LD_OR,"circ->p_chan gone. Failing circ.");
       tor_free(onionskin);
       return -1;
     }
-    tag_pack(tag, circ->p_conn->_base.global_identifier,
+    tag_pack(tag, circ->p_chan->global_identifier,
              circ->p_circ_id);
 
     cpuworker->state = CPUWORKER_STATE_BUSY_ONION;

src/or/directory.c

@@ -2855,8 +2855,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
         geoip_note_ns_response(act, GEOIP_SUCCESS);
         /* Note that a request for a network status has started, so that we
          * can measure the download time later on. */
-        if (TO_CONN(conn)->dirreq_id)
-          geoip_start_dirreq(TO_CONN(conn)->dirreq_id, dlen, act,
+        if (conn->dirreq_id)
+          geoip_start_dirreq(conn->dirreq_id, dlen, act,
                              DIRREQ_TUNNELED);
         else
           geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act,
@@ -3529,8 +3529,8 @@ connection_dir_finished_flushing(dir_connection_t *conn)
   /* Note that we have finished writing the directory response. For direct
    * connections this means we're done, for tunneled connections its only
    * an intermediate step. */
-  if (TO_CONN(conn)->dirreq_id)
-    geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED,
+  if (conn->dirreq_id)
+    geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED,
                               DIRREQ_FLUSHING_DIR_CONN_FINISHED);
   else
     geoip_change_dirreq_state(TO_CONN(conn)->global_identifier,

src/or/dirserv.c

@@ -8,6 +8,8 @@
 #include "buffers.h"
 #include "config.h"
 #include "confparse.h"
+#include "channel.h"
+#include "channeltls.h"
 #include "connection.h"
 #include "connection_or.h"
 #include "control.h"
@@ -3410,8 +3412,8 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router)
   log_debug(LD_OR,"Testing reachability of %s at %s:%u.",
             router->nickname, router->address, router->or_port);
   tor_addr_from_ipv4h(&router_addr, router->addr);
-  connection_or_connect(&router_addr, router->or_port,
-                        router->cache_info.identity_digest);
+  channel_tls_connect(&router_addr, router->or_port,
+                      router->cache_info.identity_digest);
 
   /* Possible IPv6. */
   if (get_options()->AuthDirHasIPv6Connectivity == 1 &&
@@ -3421,8 +3423,8 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router)
               router->nickname,
               tor_addr_to_str(addrstr, &router->ipv6_addr, sizeof(addrstr), 1),
               router->ipv6_orport);
-    connection_or_connect(&router->ipv6_addr, router->ipv6_orport,
-                          router->cache_info.identity_digest);
+    channel_tls_connect(&router->ipv6_addr, router->ipv6_orport,
+                        router->cache_info.identity_digest);
   }
 }
 

src/or/geoip.c

@@ -578,7 +578,7 @@ _c_hist_compare(const void **_a, const void **_b)
  * failed, the others as still running. */
 #define DIRREQ_TIMEOUT (10*60)
 
-/** Entry in a map from either conn->global_identifier for direct requests
+/** Entry in a map from either chan->global_identifier for direct requests
  * or a unique circuit identifier for tunneled requests to request time,
  * response size, and completion time of a network status request. Used to
  * measure download times of requests to derive average client
@@ -586,7 +586,7 @@ _c_hist_compare(const void **_a, const void **_b)
 typedef struct dirreq_map_entry_t {
   HT_ENTRY(dirreq_map_entry_t) node;
   /** Unique identifier for this network status request; this is either the
-   * conn->global_identifier of the dir conn (direct request) or a new
+   * chan->global_identifier of the dir channel (direct request) or a new
    * locally unique identifier of a circuit (tunneled request). This ID is
    * only unique among other direct or tunneled requests, respectively. */
   uint64_t dirreq_id;
@@ -705,7 +705,7 @@ geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
   if ((type == DIRREQ_DIRECT &&
          new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
       (type == DIRREQ_TUNNELED &&
-         new_state == DIRREQ_OR_CONN_BUFFER_FLUSHED)) {
+         new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) {
     tor_gettimeofday(&ent->completion_time);
     ent->completed = 1;
   }

src/or/include.am

@@ -17,8 +17,12 @@ endif
 
 src_or_libtor_a_SOURCES = \
 	src/or/buffers.c				\
+	src/or/channel.c				\
+	src/or/channeltls.c				\
 	src/or/circuitbuild.c				\
 	src/or/circuitlist.c				\
+	src/or/circuitmux.c				\
+	src/or/circuitmux_ewma.c			\
 	src/or/circuituse.c				\
 	src/or/command.c				\
 	src/or/config.c					\
@@ -86,8 +90,12 @@ src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-crypto.a
 
 ORHEADERS = \
 	src/or/buffers.h				\
+	src/or/channel.h				\
+	src/or/channeltls.h				\
 	src/or/circuitbuild.h				\
 	src/or/circuitlist.h				\
+	src/or/circuitmux.h				\
+	src/or/circuitmux_ewma.h			\
 	src/or/circuituse.h				\
 	src/or/command.h				\
 	src/or/config.h					\

src/or/main.c

@@ -13,6 +13,8 @@
 #define MAIN_PRIVATE
 #include "or.h"
 #include "buffers.h"
+#include "channel.h"
+#include "channeltls.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuituse.h"
@@ -398,6 +400,18 @@ connection_unlink(connection_t *conn)
   if (conn->type == CONN_TYPE_OR) {
     if (!tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest))
       connection_or_remove_from_identity_map(TO_OR_CONN(conn));
+    /* connection_unlink() can only get called if the connection
+     * was already on the closeable list, and it got there by
+     * connection_mark_for_close(), which was called from
+     * connection_or_close_normally() or
+     * connection_or_close_for_error(), so the channel should
+     * already be in CHANNEL_STATE_CLOSING, and then the
+     * connection_about_to_close_connection() goes to
+     * connection_or_about_to_close(), which calls channel_closed()
+     * to notify the channel_t layer, and closed the channel, so
+     * nothing more to do here to deal with the channel associated
+     * with an orconn.
+     */
   }
   connection_free(conn);
 }
@@ -1046,7 +1060,8 @@ run_connection_housekeeping(int i, time_t now)
   tor_assert(conn->outbuf);
 #endif
 
-  if (or_conn->is_bad_for_new_circs && !or_conn->n_circuits) {
+  if (channel_is_bad_for_new_circs(TLS_CHAN_TO_BASE(or_conn->chan)) &&
+      !connection_or_get_num_circuits(or_conn)) {
     /* It's bad for new circuits, and has no unmarked circuits on it:
      * mark it now. */
     log_info(LD_OR,
@@ -1056,28 +1071,29 @@ run_connection_housekeeping(int i, time_t now)
       connection_or_connect_failed(TO_OR_CONN(conn),
                                    END_OR_CONN_REASON_TIMEOUT,
                                    "Tor gave up on the connection");
-    connection_mark_and_flush(conn);
+    connection_or_close_normally(TO_OR_CONN(conn), 1);
   } else if (!connection_state_is_open(conn)) {
     if (past_keepalive) {
       /* We never managed to actually get this connection open and happy. */
       log_info(LD_OR,"Expiring non-open OR connection to fd %d (%s:%d).",
                (int)conn->s,conn->address, conn->port);
-      connection_mark_for_close(conn);
+      connection_or_close_normally(TO_OR_CONN(conn), 0);
     }
-  } else if (we_are_hibernating() && !or_conn->n_circuits &&
+  } else if (we_are_hibernating() &&
+             !connection_or_get_num_circuits(or_conn) &&
              !connection_get_outbuf_len(conn)) {
     /* We're hibernating, there's no circuits, and nothing to flush.*/
     log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) "
              "[Hibernating or exiting].",
              (int)conn->s,conn->address, conn->port);
-    connection_mark_and_flush(conn);
-  } else if (!or_conn->n_circuits &&
+    connection_or_close_normally(TO_OR_CONN(conn), 1);
+  } else if (!connection_or_get_num_circuits(or_conn) &&
              now >= or_conn->timestamp_last_added_nonpadding +
                                          IDLE_OR_CONN_TIMEOUT) {
     log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) "
              "[idle %d].", (int)conn->s,conn->address, conn->port,
              (int)(now - or_conn->timestamp_last_added_nonpadding));
-    connection_mark_for_close(conn);
+    connection_or_close_normally(TO_OR_CONN(conn), 0);
   } else if (
       now >= or_conn->timestamp_lastempty + options->KeepalivePeriod*10 &&
       now >= conn->timestamp_lastwritten + options->KeepalivePeriod*10) {
@@ -1087,7 +1103,7 @@ run_connection_housekeeping(int i, time_t now)
            (int)conn->s, conn->address, conn->port,
            (int)connection_get_outbuf_len(conn),
            (int)(now-conn->timestamp_lastwritten));
-    connection_mark_for_close(conn);
+    connection_or_close_normally(TO_OR_CONN(conn), 0);
   } else if (past_keepalive && !connection_get_outbuf_len(conn)) {
     /* send a padding cell */
     log_fn(LOG_DEBUG,LD_OR,"Sending keepalive to (%s:%d)",
@@ -1521,6 +1537,10 @@ run_scheduled_events(time_t now)
    * flush it. */
   or_state_save(now);
 
+  /** 8c. Do channel cleanup just like for connections */
+  channel_run_cleanup();
+  channel_listener_run_cleanup();
+
   /** 9. and if we're a server, check whether our DNS is telling stories to
    * us. */
   if (!net_is_disabled() &&
@@ -2151,6 +2171,10 @@ dumpstats(int severity)
     circuit_dump_by_conn(conn, severity); /* dump info about all the circuits
                                            * using this conn */
   } SMARTLIST_FOREACH_END(conn);
+
+  channel_dumpstats(severity);
+  channel_listener_dumpstats(severity);
+
   log(severity, LD_NET,
       "Cells processed: "U64_FORMAT" padding\n"
       "                 "U64_FORMAT" create\n"
@@ -2456,6 +2480,8 @@ tor_free_all(int postfork)
   circuit_free_all();
   entry_guards_free_all();
   pt_free_all();
+  channel_tls_free_all();
+  channel_free_all();
   connection_free_all();
   buf_shrink_freelists(1);
   memarea_clear_freelist();

src/or/networkstatus.c

@@ -11,7 +11,10 @@
  */
 
 #include "or.h"
+#include "channel.h"
 #include "circuitbuild.h"
+#include "circuitmux.h"
+#include "circuitmux_ewma.h"
 #include "config.h"
 #include "connection.h"
 #include "connection_or.h"
@@ -1634,6 +1637,7 @@ networkstatus_set_current_consensus(const char *consensus,
   consensus_waiting_for_certs_t *waiting = NULL;
   time_t current_valid_after = 0;
   int free_consensus = 1; /* Free 'c' at the end of the function */
+  int old_ewma_enabled;
 
   if (flav < 0) {
     /* XXXX we don't handle unrecognized flavors yet. */
@@ -1827,7 +1831,18 @@ networkstatus_set_current_consensus(const char *consensus,
 
     dirvote_recalculate_timing(options, now);
     routerstatus_list_update_named_server_map();
-    cell_ewma_set_scale_factor(options, current_consensus);
+
+    /* Update ewma and adjust policy if needed; first cache the old value */
+    old_ewma_enabled = cell_ewma_enabled();
+    /* Change the cell EWMA settings */
+    cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus());
+    /* If we just enabled ewma, set the cmux policy on all active channels */
+    if (cell_ewma_enabled() && !old_ewma_enabled) {
+      channel_set_cmux_policy_everywhere(&ewma_policy);
+    } else if (!cell_ewma_enabled() && old_ewma_enabled) {
+      /* Turn it off everywhere */
+      channel_set_cmux_policy_everywhere(NULL);
+    }
 
     /* XXXX024 this call might be unnecessary here: can changing the
      * current consensus really alter our view of any OR's rate limits? */

src/or/onion.c

@@ -106,7 +106,7 @@ onion_next_task(char **onionskin_out)
     return NULL; /* no onions pending, we're done */
 
   tor_assert(ol_list->circ);
-  tor_assert(ol_list->circ->p_conn); /* make sure it's still valid */
+  tor_assert(ol_list->circ->p_chan); /* make sure it's still valid */
   tor_assert(ol_length > 0);
   circ = ol_list->circ;
   *onionskin_out = ol_list->onionskin;

src/or/or.h

@@ -445,9 +445,9 @@ typedef enum {
 #define CIRCUIT_STATE_BUILDING 0
 /** Circuit state: Waiting to process the onionskin. */
 #define CIRCUIT_STATE_ONIONSKIN_PENDING 1
-/** Circuit state: I'd like to deliver a create, but my n_conn is still
+/** Circuit state: I'd like to deliver a create, but my n_chan is still
  * connecting. */
-#define CIRCUIT_STATE_OR_WAIT 2
+#define CIRCUIT_STATE_CHAN_WAIT 2
 /** Circuit state: onionskin(s) processed, ready to send/receive cells. */
 #define CIRCUIT_STATE_OPEN 3
 
@@ -674,7 +674,7 @@ typedef enum {
 #define END_CIRC_REASON_RESOURCELIMIT   5
 #define END_CIRC_REASON_CONNECTFAILED   6
 #define END_CIRC_REASON_OR_IDENTITY     7
-#define END_CIRC_REASON_OR_CONN_CLOSED  8
+#define END_CIRC_REASON_CHANNEL_CLOSED  8
 #define END_CIRC_REASON_FINISHED        9
 #define END_CIRC_REASON_TIMEOUT         10
 #define END_CIRC_REASON_DESTROYED       11
@@ -879,6 +879,147 @@ typedef uint16_t circid_t;
 /** Identifies a stream on a circuit */
 typedef uint16_t streamid_t;
 
+/* channel_t typedef; struct channel_s is in channel.h */
+
+typedef struct channel_s channel_t;
+
+/* channel_listener_t typedef; struct channel_listener_s is in channel.h */
+
+typedef struct channel_listener_s channel_listener_t;
+
+/* channel states for channel_t */
+
+typedef enum {
+  /*
+   * Closed state - channel is inactive
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_CLOSING
+   * Permitted transitions to:
+   *   - CHANNEL_STATE_OPENING
+   */
+  CHANNEL_STATE_CLOSED = 0,
+  /*
+   * Opening state - channel is trying to connect
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_CLOSED
+   * Permitted transitions to:
+   *   - CHANNEL_STATE_CLOSING
+   *   - CHANNEL_STATE_ERROR
+   *   - CHANNEL_STATE_OPEN
+   */
+  CHANNEL_STATE_OPENING,
+  /*
+   * Open state - channel is active and ready for use
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_MAINT
+   *   - CHANNEL_STATE_OPENING
+   * Permitted transitions to:
+   *   - CHANNEL_STATE_CLOSING
+   *   - CHANNEL_STATE_ERROR
+   *   - CHANNEL_STATE_MAINT
+   */
+  CHANNEL_STATE_OPEN,
+  /*
+   * Maintenance state - channel is temporarily offline for subclass specific
+   *   maintenance activities such as TLS renegotiation.
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_OPEN
+   * Permitted transitions to:
+   *   - CHANNEL_STATE_CLOSING
+   *   - CHANNEL_STATE_ERROR
+   *   - CHANNEL_STATE_OPEN
+   */
+  CHANNEL_STATE_MAINT,
+  /*
+   * Closing state - channel is shutting down
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_MAINT
+   *   - CHANNEL_STATE_OPEN
+   * Permitted transitions to:
+   *   - CHANNEL_STATE_CLOSED,
+   *   - CHANNEL_STATE_ERROR
+   */
+  CHANNEL_STATE_CLOSING,
+  /*
+   * Error state - channel has experienced a permanent error
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_CLOSING
+   *   - CHANNEL_STATE_MAINT
+   *   - CHANNEL_STATE_OPENING
+   *   - CHANNEL_STATE_OPEN
+   * Permitted transitions to:
+   *   - None
+   */
+  CHANNEL_STATE_ERROR,
+  /*
+   * Placeholder for maximum state value
+   */
+  CHANNEL_STATE_LAST
+} channel_state_t;
+
+/* channel listener states for channel_listener_t */
+
+typedef enum {
+  /*
+   * Closed state - channel listener is inactive
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_LISTENER_STATE_CLOSING
+   * Permitted transitions to:
+   *   - CHANNEL_LISTENER_STATE_LISTENING
+   */
+  CHANNEL_LISTENER_STATE_CLOSED = 0,
+  /*
+   * Listening state - channel listener is listening for incoming
+   * connections
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_LISTENER_STATE_CLOSED
+   * Permitted transitions to:
+   *   - CHANNEL_LISTENER_STATE_CLOSING
+   *   - CHANNEL_LISTENER_STATE_ERROR
+   */
+  CHANNEL_LISTENER_STATE_LISTENING,
+  /*
+   * Closing state - channel listener is shutting down
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_LISTENER_STATE_LISTENING
+   * Permitted transitions to:
+   *   - CHANNEL_LISTENER_STATE_CLOSED,
+   *   - CHANNEL_LISTENER_STATE_ERROR
+   */
+  CHANNEL_LISTENER_STATE_CLOSING,
+  /*
+   * Error state - channel listener has experienced a permanent error
+   *
+   * Permitted transitions from:
+   *   - CHANNEL_STATE_CLOSING
+   *   - CHANNEL_STATE_LISTENING
+   * Permitted transitions to:
+   *   - None
+   */
+  CHANNEL_LISTENER_STATE_ERROR,
+  /*
+   * Placeholder for maximum state value
+   */
+  CHANNEL_LISTENER_STATE_LAST
+} channel_listener_state_t;
+
+/* TLS channel stuff */
+
+typedef struct channel_tls_s channel_tls_t;
+
+/* circuitmux_t typedef; struct circuitmux_s is in circuitmux.h */
+
+typedef struct circuitmux_s circuitmux_t;
+
 /** Parsed onion routing cell.  All communication between nodes
  * is via cells. */
 typedef struct cell_t {
@@ -1061,9 +1202,6 @@ typedef struct connection_t {
 
   /** Unique identifier for this connection on this Tor instance. */
   uint64_t global_identifier;
-
-  /** Unique ID for measuring tunneled network status requests. */
-  uint64_t dirreq_id;
 } connection_t;
 
 /** Subtype of connection_t; used for a listener socket. */
@@ -1202,29 +1340,22 @@ typedef struct or_connection_t {
   int tls_error; /**< Last tor_tls error code. */
   /** When we last used this conn for any client traffic. If not
    * recent, we can rate limit it further. */
-  time_t client_used;
+
+  /* Channel using this connection */
+  channel_tls_t *chan;
 
   tor_addr_t real_addr; /**< The actual address that this connection came from
                        * or went to.  The <b>addr</b> field is prone to
                        * getting overridden by the address from the router
                        * descriptor matching <b>identity_digest</b>. */
 
-  circ_id_type_t circ_id_type:2; /**< When we send CREATE cells along this
-                                  * connection, which half of the space should
-                                  * we use? */
   /** Should this connection be used for extending circuits to the server
    * matching the <b>identity_digest</b> field?  Set to true if we're pretty
    * sure we aren't getting MITMed, either because we're connected to an
    * address listed in a server descriptor, or because an authenticated
    * NETINFO cell listed the address we're connected to as recognized. */
   unsigned int is_canonical:1;
-  /** True iff this connection shouldn't get any new circs attached to it,
-   * because the connection is too old, or because there's a better one.
-   * More generally, this flag is used to note an unhealthy connection;
-   * for example, if a bad connection fails we shouldn't assume that the
-   * router itself has a problem.
-   */
-  unsigned int is_bad_for_new_circs:1;
+
   /** True iff we have decided that the other end of this connection
    * is a client.  Connections with this flag set should never be used
    * to satisfy an EXTEND request.  */
@@ -1234,9 +1365,6 @@ typedef struct or_connection_t {
   unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */
   uint8_t link_proto; /**< What protocol version are we using? 0 for
                        * "none negotiated yet." */
-  circid_t next_circ_id; /**< Which circ_id do we try to use next on
-                          * this connection?  This is always in the
-                          * range 0..1<<15-1. */
 
   or_handshake_state_t *handshake_state; /**< If we are setting this connection
                                           * up, state information to do so. */
@@ -1258,24 +1386,7 @@ typedef struct or_connection_t {
   /* XXXX we could share this among all connections. */
   struct ev_token_bucket_cfg *bucket_cfg;
 #endif
-  int n_circuits; /**< How many circuits use this connection as p_conn or
-                   * n_conn ? */
 
-  /** Double-linked ring of circuits with queued cells waiting for room to
-   * free up on this connection's outbuf.  Every time we pull cells from a
-   * circuit, we advance this pointer to the next circuit in the ring. */
-  struct circuit_t *active_circuits;
-  /** Priority queue of cell_ewma_t for circuits with queued cells waiting for
-   * room to free up on this connection's outbuf.  Kept in heap order
-   * according to EWMA.
-   *
-   * This is redundant with active_circuits; if we ever decide only to use the
-   * cell_ewma algorithm for choosing circuits, we can remove active_circuits.
-   */
-  smartlist_t *active_circuit_pqueue;
-  /** The tick on which the cell_ewma_ts in active_circuit_pqueue last had
-   * their ewma values rescaled. */
-  unsigned active_circuit_pqueue_last_recalibrated;
   struct or_connection_t *next_with_same_id; /**< Next connection with same
                                               * identity digest as this one. */
 } or_connection_t;
@@ -1327,6 +1438,10 @@ typedef struct edge_connection_t {
    * cells. */
   unsigned int edge_blocked_on_circ:1;
 
+  /** Unique ID for directory requests; this used to be in connection_t, but
+   * that's going away and being used on channels instead.  We still tag
+   * edge connections with dirreq_id from circuits, so it's copied here. */
+  uint64_t dirreq_id;
 } edge_connection_t;
 
 /** Subtype of edge_connection_t for an "entry connection" -- that is, a SOCKS
@@ -1449,6 +1564,10 @@ typedef struct dir_connection_t {
   char identity_digest[DIGEST_LEN]; /**< Hash of the public RSA key for
                                      * the directory server's signing key. */
 
+  /** Unique ID for directory requests; this used to be in connection_t, but
+   * that's going away and being used on channels instead.  The dirserver still
+   * needs this for the incoming side, so it's moved here. */
+  uint64_t dirreq_id;
 } dir_connection_t;
 
 /** Subtype of connection_t for an connection to a controller. */
@@ -2462,29 +2581,6 @@ typedef struct {
   time_t expiry_time;
 } cpath_build_state_t;
 
-/**
- * The cell_ewma_t structure keeps track of how many cells a circuit has
- * transferred recently.  It keeps an EWMA (exponentially weighted moving
- * average) of the number of cells flushed from the circuit queue onto a
- * connection in connection_or_flush_from_first_active_circuit().
- */
-typedef struct {
-  /** The last 'tick' at which we recalibrated cell_count.
-   *
-   * A cell sent at exactly the start of this tick has weight 1.0. Cells sent
-   * since the start of this tick have weight greater than 1.0; ones sent
-   * earlier have less weight. */
-  unsigned last_adjusted_tick;
-  /** The EWMA of the cell count. */
-  double cell_count;
-  /** True iff this is the cell count for a circuit's previous
-   * connection. */
-  unsigned int is_for_p_conn : 1;
-  /** The position of the circuit within the OR connection's priority
-   * queue. */
-  int heap_index;
-} cell_ewma_t;
-
 #define ORIGIN_CIRCUIT_MAGIC 0x35315243u
 #define OR_CIRCUIT_MAGIC 0x98ABC04Fu
 
@@ -2515,23 +2611,39 @@ typedef struct circuit_t {
   uint32_t magic; /**< For memory and type debugging: must equal
                    * ORIGIN_CIRCUIT_MAGIC or OR_CIRCUIT_MAGIC. */
 
-  /** Queue of cells waiting to be transmitted on n_conn. */
-  cell_queue_t n_conn_cells;
-  /** The OR connection that is next in this circuit. */
-  or_connection_t *n_conn;
-  /** The circuit_id used in the next (forward) hop of this circuit. */
+  /** The channel that is next in this circuit. */
+  channel_t *n_chan;
+
+  /**
+   * The circuit_id used in the next (forward) hop of this circuit;
+   * this is unique to n_chan, but this ordered pair is globally
+   * unique:
+   *
+   * (n_chan->global_identifier, n_circ_id)
+   */
   circid_t n_circ_id;
 
-  /** The hop to which we want to extend this circuit.  Should be NULL if
-   * the circuit has attached to a connection. */
+  /**
+   * Circuit mux associated with n_chan to which this circuit is attached;
+   * NULL if we have no n_chan.
+   */
+  circuitmux_t *n_mux;
+
+  /** Queue of cells waiting to be transmitted on n_chan */
+  cell_queue_t n_chan_cells;
+
+  /**
+   * The hop to which we want to extend this circuit.  Should be NULL if
+   * the circuit has attached to a channel.
+   */
   extend_info_t *n_hop;
 
-  /** True iff we are waiting for n_conn_cells to become less full before
+  /** True iff we are waiting for n_chan_cells to become less full before
    * allowing p_streams to add any more cells. (Origin circuit only.) */
-  unsigned int streams_blocked_on_n_conn : 1;
-  /** True iff we are waiting for p_conn_cells to become less full before
+  unsigned int streams_blocked_on_n_chan : 1;
+  /** True iff we are waiting for p_chan_cells to become less full before
    * allowing n_streams to add any more cells. (OR circuit only.) */
-  unsigned int streams_blocked_on_p_conn : 1;
+  unsigned int streams_blocked_on_p_chan : 1;
 
   uint8_t state; /**< Current status of this circuit. */
   uint8_t purpose; /**< Why are we creating this circuit? */
@@ -2546,10 +2658,10 @@ typedef struct circuit_t {
    * more. */
   int deliver_window;
 
-  /** For storage while n_conn is pending
-    * (state CIRCUIT_STATE_OR_WAIT). When defined, it is always
+  /** For storage while n_chan is pending
+    * (state CIRCUIT_STATE_CHAN_WAIT). When defined, it is always
     * length ONIONSKIN_CHALLENGE_LEN. */
-  char *n_conn_onionskin;
+  char *n_chan_onionskin;
 
   /** When was this circuit created?  We keep this timestamp with a higher
    * resolution than most so that the circuit-build-time tracking code can
@@ -2575,23 +2687,19 @@ typedef struct circuit_t {
   const char *marked_for_close_file; /**< For debugging: in which file was this
                                       * circuit marked for close? */
 
-  /** Next circuit in the doubly-linked ring of circuits waiting to add
-   * cells to n_conn.  NULL if we have no cells pending, or if we're not
-   * linked to an OR connection. */
-  struct circuit_t *next_active_on_n_conn;
-  /** Previous circuit in the doubly-linked ring of circuits waiting to add
-   * cells to n_conn.  NULL if we have no cells pending, or if we're not
-   * linked to an OR connection. */
-  struct circuit_t *prev_active_on_n_conn;
-  struct circuit_t *next; /**< Next circuit in linked list of all circuits. */
-
   /** Unique ID for measuring tunneled network status requests. */
   uint64_t dirreq_id;
 
-  /** The EWMA count for the number of cells flushed from the
-   * n_conn_cells queue.  Used to determine which circuit to flush from next.
-   */
-  cell_ewma_t n_cell_ewma;
+  struct circuit_t *next; /**< Next circuit in linked list of all circuits. */
+
+  /** Next circuit in the doubly-linked ring of circuits waiting to add
+   * cells to n_conn.  NULL if we have no cells pending, or if we're not
+   * linked to an OR connection. */
+  struct circuit_t *next_active_on_n_chan;
+  /** Previous circuit in the doubly-linked ring of circuits waiting to add
+   * cells to n_conn.  NULL if we have no cells pending, or if we're not
+   * linked to an OR connection. */
+  struct circuit_t *prev_active_on_n_chan;
 } circuit_t;
 
 /** Largest number of relay_early cells that we can send on a given
@@ -2754,20 +2862,25 @@ typedef struct or_circuit_t {
   circuit_t _base;
 
   /** Next circuit in the doubly-linked ring of circuits waiting to add
-   * cells to p_conn.  NULL if we have no cells pending, or if we're not
+   * cells to p_chan.  NULL if we have no cells pending, or if we're not
    * linked to an OR connection. */
-  struct circuit_t *next_active_on_p_conn;
+  struct circuit_t *next_active_on_p_chan;
   /** Previous circuit in the doubly-linked ring of circuits waiting to add
-   * cells to p_conn.  NULL if we have no cells pending, or if we're not
+   * cells to p_chan.  NULL if we have no cells pending, or if we're not
    * linked to an OR connection. */
-  struct circuit_t *prev_active_on_p_conn;
+  struct circuit_t *prev_active_on_p_chan;
 
   /** The circuit_id used in the previous (backward) hop of this circuit. */
   circid_t p_circ_id;
   /** Queue of cells waiting to be transmitted on p_conn. */
-  cell_queue_t p_conn_cells;
-  /** The OR connection that is previous in this circuit. */
-  or_connection_t *p_conn;
+  cell_queue_t p_chan_cells;
+  /** The channel that is previous in this circuit. */
+  channel_t *p_chan;
+  /**
+   * Circuit mux associated with p_chan to which this circuit is attached;
+   * NULL if we have no p_chan.
+   */
+  circuitmux_t *p_mux;
   /** Linked list of Exit streams associated with this circuit. */
   edge_connection_t *n_streams;
   /** Linked list of Exit streams associated with this circuit that are
@@ -2824,10 +2937,6 @@ typedef struct or_circuit_t {
    * exit-ward queues of this circuit; reset every time when writing
    * buffer stats to disk. */
   uint64_t total_cell_waiting_time;
-
-  /** The EWMA count for the number of cells flushed from the
-   * p_conn_cells queue. */
-  cell_ewma_t p_cell_ewma;
 } or_circuit_t;
 
 /** Convert a circuit subtype to a circuit_t. */
@@ -4135,10 +4244,10 @@ typedef enum {
   /** Flushed last cell from queue of the circuit that initiated a
     * tunneled request to the outbuf of the OR connection. */
   DIRREQ_CIRC_QUEUE_FLUSHED = 3,
-  /** Flushed last byte from buffer of the OR connection belonging to the
+  /** Flushed last byte from buffer of the channel belonging to the
     * circuit that initiated a tunneled request; completes a tunneled
     * request. */
-  DIRREQ_OR_CONN_BUFFER_FLUSHED = 4
+  DIRREQ_CHANNEL_BUFFER_FLUSHED = 4
 } dirreq_state_t;
 
 #define WRITE_STATS_INTERVAL (24*60*60)

src/or/reasons.c

@@ -323,8 +323,8 @@ circuit_end_reason_to_control_string(int reason)
       return "CONNECTFAILED";
     case END_CIRC_REASON_OR_IDENTITY:
       return "OR_IDENTITY";
-    case END_CIRC_REASON_OR_CONN_CLOSED:
-      return "OR_CONN_CLOSED";
+    case END_CIRC_REASON_CHANNEL_CLOSED:
+      return "CHANNEL_CLOSED";
     case END_CIRC_REASON_FINISHED:
       return "FINISHED";
     case END_CIRC_REASON_TIMEOUT:

src/or/relay.c

@@ -10,10 +10,10 @@
  *    receiving from circuits, plus queuing on circuits.
  **/
 
-#include <math.h>
 #define RELAY_PRIVATE
 #include "or.h"
 #include "buffers.h"
+#include "channel.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "config.h"
@@ -166,7 +166,7 @@ int
 circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
                            cell_direction_t cell_direction)
 {
-  or_connection_t *or_conn=NULL;
+  channel_t *chan = NULL;
   crypt_path_t *layer_hint=NULL;
   char recognized=0;
   int reason;
@@ -213,17 +213,17 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
   /* not recognized. pass it on. */
   if (cell_direction == CELL_DIRECTION_OUT) {
     cell->circ_id = circ->n_circ_id; /* switch it */
-    or_conn = circ->n_conn;
+    chan = circ->n_chan;
   } else if (! CIRCUIT_IS_ORIGIN(circ)) {
     cell->circ_id = TO_OR_CIRCUIT(circ)->p_circ_id; /* switch it */
-    or_conn = TO_OR_CIRCUIT(circ)->p_conn;
+    chan = TO_OR_CIRCUIT(circ)->p_chan;
   } else {
     log_fn(LOG_PROTOCOL_WARN, LD_OR,
            "Dropping unrecognized inbound cell on origin circuit.");
     return 0;
   }
 
-  if (!or_conn) {
+  if (!chan) {
     // XXXX Can this splice stuff be done more cleanly?
     if (! CIRCUIT_IS_ORIGIN(circ) &&
         TO_OR_CIRCUIT(circ)->rend_splice &&
@@ -254,7 +254,7 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
                                   * we might kill the circ before we relay
                                   * the cells. */
 
-  append_cell_to_circuit_queue(circ, or_conn, cell, cell_direction, 0);
+  append_cell_to_circuit_queue(circ, chan, cell, cell_direction, 0);
   return 0;
 }
 
@@ -353,13 +353,13 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
                            cell_direction_t cell_direction,
                            crypt_path_t *layer_hint, streamid_t on_stream)
 {
-  or_connection_t *conn; /* where to send the cell */
+  channel_t *chan; /* where to send the cell */
 
   if (cell_direction == CELL_DIRECTION_OUT) {
     crypt_path_t *thishop; /* counter for repeated crypts */
-    conn = circ->n_conn;
-    if (!CIRCUIT_IS_ORIGIN(circ) || !conn) {
-      log_warn(LD_BUG,"outgoing relay cell has n_conn==NULL. Dropping.");
+    chan = circ->n_chan;
+    if (!CIRCUIT_IS_ORIGIN(circ) || !chan) {
+      log_warn(LD_BUG,"outgoing relay cell has n_chan==NULL. Dropping.");
       return 0; /* just drop it */
     }
 
@@ -388,14 +388,14 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
       return 0; /* just drop it */
     }
     or_circ = TO_OR_CIRCUIT(circ);
-    conn = or_circ->p_conn;
+    chan = or_circ->p_chan;
     relay_set_digest(or_circ->p_digest, cell);
     if (relay_crypt_one_payload(or_circ->p_crypto, cell->payload, 1) < 0)
       return -1;
   }
   ++stats_n_relay_cells_relayed;
 
-  append_cell_to_circuit_queue(circ, conn, cell, cell_direction, on_stream);
+  append_cell_to_circuit_queue(circ, chan, cell, cell_direction, on_stream);
   return 0;
 }
 
@@ -561,9 +561,9 @@ relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ,
     geoip_change_dirreq_state(circ->dirreq_id, DIRREQ_TUNNELED,
                               DIRREQ_END_CELL_SENT);
 
-  if (cell_direction == CELL_DIRECTION_OUT && circ->n_conn) {
+  if (cell_direction == CELL_DIRECTION_OUT && circ->n_chan) {
     /* if we're using relaybandwidthrate, this conn wants priority */
-    circ->n_conn->client_used = approx_time();
+    channel_timestamp_client(circ->n_chan);
   }
 
   if (cell_direction == CELL_DIRECTION_OUT) {
@@ -1095,7 +1095,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
          * and linked. */
         static uint64_t next_id = 0;
         circ->dirreq_id = ++next_id;
-        TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->dirreq_id = circ->dirreq_id;
+        TO_OR_CIRCUIT(circ)->p_chan->dirreq_id = circ->dirreq_id;
       }
 
       return connection_exit_begin_conn(cell, circ);
@@ -1230,12 +1230,12 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "'truncate' unsupported at origin. Dropping.");
         return 0;
       }
-      if (circ->n_conn) {
+      if (circ->n_chan) {
         uint8_t trunc_reason = get_uint8(cell->payload + RELAY_HEADER_SIZE);
-        circuit_clear_cell_queue(circ, circ->n_conn);
-        connection_or_send_destroy(circ->n_circ_id, circ->n_conn,
-                                   trunc_reason);
-        circuit_set_n_circid_orconn(circ, 0, NULL);
+        circuit_clear_cell_queue(circ, circ->n_chan);
+        channel_send_destroy(circ->n_circ_id, circ->n_chan,
+                             trunc_reason);
+        circuit_set_n_circid_chan(circ, 0, NULL);
       }
       log_debug(LD_EXIT, "Processed 'truncate', replying.");
       {
@@ -1594,10 +1594,10 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn,
    * needed to fill the cell queue. */
   int max_to_package = circ->package_window;
   if (CIRCUIT_IS_ORIGIN(circ)) {
-    cells_on_queue = circ->n_conn_cells.n;
+    cells_on_queue = circ->n_chan_cells.n;
   } else {
     or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
-    cells_on_queue = or_circ->p_conn_cells.n;
+    cells_on_queue = or_circ->p_chan_cells.n;
   }
   if (CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue < max_to_package)
     max_to_package = CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue;
@@ -1778,10 +1778,10 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint)
 }
 
 #ifdef ACTIVE_CIRCUITS_PARANOIA
-#define assert_active_circuits_ok_paranoid(conn) \
-     assert_active_circuits_ok(conn)
+#define assert_cmux_ok_paranoid(chan) \
+     assert_circuit_mux_okay(chan)
 #else
-#define assert_active_circuits_ok_paranoid(conn)
+#define assert_cmux_ok_paranoid(chan)
 #endif
 
 /** The total number of cells we have allocated from the memory pool. */
@@ -1842,6 +1842,13 @@ packed_cell_new(void)
   return mp_pool_get(cell_pool);
 }
 
+/** Return a packed cell used outside by channel_t lower layer */
+void
+packed_cell_free(packed_cell_t *cell)
+{
+  packed_cell_free_unchecked(cell);
+}
+
 /** Log current statistics for cell pool allocation at log level
  * <b>severity</b>. */
 void
@@ -1851,9 +1858,9 @@ dump_cell_pool_usage(int severity)
   int n_circs = 0;
   int n_cells = 0;
   for (c = _circuit_get_global_list(); c; c = c->next) {
-    n_cells += c->n_conn_cells.n;
+    n_cells += c->n_chan_cells.n;
     if (!CIRCUIT_IS_ORIGIN(c))
-      n_cells += TO_OR_CIRCUIT(c)->p_conn_cells.n;
+      n_cells += TO_OR_CIRCUIT(c)->p_chan_cells.n;
     ++n_circs;
   }
   log(severity, LD_MM, "%d cells allocated on %d circuits. %d cells leaked.",
@@ -1964,363 +1971,63 @@ cell_queue_pop(cell_queue_t *queue)
   return cell;
 }
 
-/** Return a pointer to the "next_active_on_{n,p}_conn" pointer of <b>circ</b>,
- * depending on whether <b>conn</b> matches n_conn or p_conn. */
-static INLINE circuit_t **
-next_circ_on_conn_p(circuit_t *circ, or_connection_t *conn)
-{
-  tor_assert(circ);
-  tor_assert(conn);
-  if (conn == circ->n_conn) {
-    return &circ->next_active_on_n_conn;
-  } else {
-    or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
-    tor_assert(conn == orcirc->p_conn);
-    return &orcirc->next_active_on_p_conn;
-  }
-}
-
-/** Return a pointer to the "prev_active_on_{n,p}_conn" pointer of <b>circ</b>,
- * depending on whether <b>conn</b> matches n_conn or p_conn. */
-static INLINE circuit_t **
-prev_circ_on_conn_p(circuit_t *circ, or_connection_t *conn)
-{
-  tor_assert(circ);
-  tor_assert(conn);
-  if (conn == circ->n_conn) {
-    return &circ->prev_active_on_n_conn;
-  } else {
-    or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
-    tor_assert(conn == orcirc->p_conn);
-    return &orcirc->prev_active_on_p_conn;
-  }
-}
-
-/** Helper for sorting cell_ewma_t values in their priority queue. */
-static int
-compare_cell_ewma_counts(const void *p1, const void *p2)
-{
-  const cell_ewma_t *e1=p1, *e2=p2;
-  if (e1->cell_count < e2->cell_count)
-    return -1;
-  else if (e1->cell_count > e2->cell_count)
-    return 1;
-  else
-    return 0;
-}
-
-/** Given a cell_ewma_t, return a pointer to the circuit containing it. */
-static circuit_t *
-cell_ewma_to_circuit(cell_ewma_t *ewma)
-{
-  if (ewma->is_for_p_conn) {
-    /* This is an or_circuit_t's p_cell_ewma. */
-    or_circuit_t *orcirc = SUBTYPE_P(ewma, or_circuit_t, p_cell_ewma);
-    return TO_CIRCUIT(orcirc);
-  } else {
-    /* This is some circuit's n_cell_ewma. */
-    return SUBTYPE_P(ewma, circuit_t, n_cell_ewma);
-  }
-}
-
-/* ==== Functions for scaling cell_ewma_t ====
-
-   When choosing which cells to relay first, we favor circuits that have been
-   quiet recently.  This gives better latency on connections that aren't
-   pushing lots of data, and makes the network feel more interactive.
-
-   Conceptually, we take an exponentially weighted mean average of the number
-   of cells a circuit has sent, and allow active circuits (those with cells to
-   relay) to send cells in reverse order of their exponentially-weighted mean
-   average (EWMA) cell count.  [That is, a cell sent N seconds ago 'counts'
-   F^N times as much as a cell sent now, for 0<F<1.0, and we favor the
-   circuit that has sent the fewest cells]
-
-   If 'double' had infinite precision, we could do this simply by counting a
-   cell sent at startup as having weight 1.0, and a cell sent N seconds later
-   as having weight F^-N.  This way, we would never need to re-scale
-   any already-sent cells.
-
-   To prevent double from overflowing, we could count a cell sent now as
-   having weight 1.0 and a cell sent N seconds ago as having weight F^N.
-   This, however, would mean we'd need to re-scale *ALL* old circuits every
-   time we wanted to send a cell.
-
-   So as a compromise, we divide time into 'ticks' (currently, 10-second
-   increments) and say that a cell sent at the start of a current tick is
-   worth 1.0, a cell sent N seconds before the start of the current tick is
-   worth F^N, and a cell sent N seconds after the start of the current tick is
-   worth F^-N.  This way we don't overflow, and we don't need to constantly
-   rescale.
+/**
+ * Update the number of cells available on the circuit's n_chan or p_chan's
+ * circuit mux.
  */
-
-/** How long does a tick last (seconds)? */
-#define EWMA_TICK_LEN 10
-
-/** The default per-tick scale factor, if it hasn't been overridden by a
- * consensus or a configuration setting.  zero means "disabled". */
-#define EWMA_DEFAULT_HALFLIFE 0.0
-
-/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs
- * and the fraction of the tick that has elapsed between the start of the tick
- * and <b>now</b>.  Return the former and store the latter in
- * *<b>remainder_out</b>.
- *
- * These tick values are not meant to be shared between Tor instances, or used
- * for other purposes. */
-static unsigned
-cell_ewma_tick_from_timeval(const struct timeval *now,
-                            double *remainder_out)
-{
-  unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN);
-  /* rem */
-  double rem = (now->tv_sec % EWMA_TICK_LEN) +
-    ((double)(now->tv_usec)) / 1.0e6;
-  *remainder_out = rem / EWMA_TICK_LEN;
-  return res;
-}
-
-/** Compute and return the current cell_ewma tick. */
-unsigned
-cell_ewma_get_tick(void)
-{
-  return ((unsigned)approx_time() / EWMA_TICK_LEN);
-}
-
-/** The per-tick scale factor to be used when computing cell-count EWMA
- * values.  (A cell sent N ticks before the start of the current tick
- * has value ewma_scale_factor ** N.)
- */
-static double ewma_scale_factor = 0.1;
-/* DOCDOC ewma_enabled */
-static int ewma_enabled = 0;
-
-/*DOCDOC*/
-#define EPSILON 0.00001
-/*DOCDOC*/
-#define LOG_ONEHALF -0.69314718055994529
-
-/** Adjust the global cell scale factor based on <b>options</b> */
 void
-cell_ewma_set_scale_factor(const or_options_t *options,
-                           const networkstatus_t *consensus)
+update_circuit_on_cmux(circuit_t *circ, cell_direction_t direction)
 {
-  int32_t halflife_ms;
-  double halflife;
-  const char *source;
-  if (options && options->CircuitPriorityHalflife >= -EPSILON) {
-    halflife = options->CircuitPriorityHalflife;
-    source = "CircuitPriorityHalflife in configuration";
-  } else if (consensus && (halflife_ms = networkstatus_get_param(
-                 consensus, "CircuitPriorityHalflifeMsec",
-                 -1, -1, INT32_MAX)) >= 0) {
-    halflife = ((double)halflife_ms)/1000.0;
-    source = "CircuitPriorityHalflifeMsec in consensus";
+  channel_t *chan = NULL;
+  or_circuit_t *or_circ = NULL;
+  circuitmux_t *cmux = NULL;
+
+  tor_assert(circ);
+
+  /* Okay, get the channel */
+  if (direction == CELL_DIRECTION_OUT) {
+    chan = circ->n_chan;
   } else {
-    halflife = EWMA_DEFAULT_HALFLIFE;
-    source = "Default value";
+    or_circ = TO_OR_CIRCUIT(circ);
+    chan = or_circ->p_chan;
   }
 
-  if (halflife <= EPSILON) {
-    /* The cell EWMA algorithm is disabled. */
-    ewma_scale_factor = 0.1;
-    ewma_enabled = 0;
-    log_info(LD_OR,
-             "Disabled cell_ewma algorithm because of value in %s",
-             source);
+  tor_assert(chan);
+  tor_assert(chan->cmux);
+
+  /* Now get the cmux */
+  cmux = chan->cmux;
+
+  /* Cmux sanity check */
+  tor_assert(circuitmux_is_circuit_attached(cmux, circ));
+  tor_assert(circuitmux_attached_circuit_direction(cmux, circ) == direction);
+
+  assert_cmux_ok_paranoid(chan);
+
+  /* Update the number of cells we have for the circuit mux */
+  if (direction == CELL_DIRECTION_OUT) {
+    circuitmux_set_num_cells(cmux, circ, circ->n_chan_cells.n);
   } else {
-    /* convert halflife into halflife-per-tick. */
-    halflife /= EWMA_TICK_LEN;
-    /* compute per-tick scale factor. */
-    ewma_scale_factor = exp( LOG_ONEHALF / halflife );
-    ewma_enabled = 1;
-    log_info(LD_OR,
-             "Enabled cell_ewma algorithm because of value in %s; "
-             "scale factor is %f per %d seconds",
-             source, ewma_scale_factor, EWMA_TICK_LEN);
+    circuitmux_set_num_cells(cmux, circ, or_circ->p_chan_cells.n);
   }
+
+  assert_cmux_ok_paranoid(chan);
 }
 
-/** Return the multiplier necessary to convert the value of a cell sent in
- * 'from_tick' to one sent in 'to_tick'. */
-static INLINE double
-get_scale_factor(unsigned from_tick, unsigned to_tick)
-{
-  /* This math can wrap around, but that's okay: unsigned overflow is
-     well-defined */
-  int diff = (int)(to_tick - from_tick);
-  return pow(ewma_scale_factor, diff);
-}
-
-/** Adjust the cell count of <b>ewma</b> so that it is scaled with respect to
- * <b>cur_tick</b> */
-static void
-scale_single_cell_ewma(cell_ewma_t *ewma, unsigned cur_tick)
-{
-  double factor = get_scale_factor(ewma->last_adjusted_tick, cur_tick);
-  ewma->cell_count *= factor;
-  ewma->last_adjusted_tick = cur_tick;
-}
-
-/** Adjust the cell count of every active circuit on <b>conn</b> so
- * that they are scaled with respect to <b>cur_tick</b> */
-static void
-scale_active_circuits(or_connection_t *conn, unsigned cur_tick)
-{
-
-  double factor = get_scale_factor(
-              conn->active_circuit_pqueue_last_recalibrated,
-              cur_tick);
-  /** Ordinarily it isn't okay to change the value of an element in a heap,
-   * but it's okay here, since we are preserving the order. */
-  SMARTLIST_FOREACH(conn->active_circuit_pqueue, cell_ewma_t *, e, {
-      tor_assert(e->last_adjusted_tick ==
-                 conn->active_circuit_pqueue_last_recalibrated);
-      e->cell_count *= factor;
-      e->last_adjusted_tick = cur_tick;
-  });
-  conn->active_circuit_pqueue_last_recalibrated = cur_tick;
-}
-
-/** Rescale <b>ewma</b> to the same scale as <b>conn</b>, and add it to
- * <b>conn</b>'s priority queue of active circuits */
-static void
-add_cell_ewma_to_conn(or_connection_t *conn, cell_ewma_t *ewma)
-{
-  tor_assert(ewma->heap_index == -1);
-  scale_single_cell_ewma(ewma,
-                         conn->active_circuit_pqueue_last_recalibrated);
-
-  smartlist_pqueue_add(conn->active_circuit_pqueue,
-                       compare_cell_ewma_counts,
-                       STRUCT_OFFSET(cell_ewma_t, heap_index),
-                       ewma);
-}
-
-/** Remove <b>ewma</b> from <b>conn</b>'s priority queue of active circuits */
-static void
-remove_cell_ewma_from_conn(or_connection_t *conn, cell_ewma_t *ewma)
-{
-  tor_assert(ewma->heap_index != -1);
-  smartlist_pqueue_remove(conn->active_circuit_pqueue,
-                          compare_cell_ewma_counts,
-                          STRUCT_OFFSET(cell_ewma_t, heap_index),
-                          ewma);
-}
-
-/** Remove and return the first cell_ewma_t from conn's priority queue of
- * active circuits.  Requires that the priority queue is nonempty. */
-static cell_ewma_t *
-pop_first_cell_ewma_from_conn(or_connection_t *conn)
-{
-  return smartlist_pqueue_pop(conn->active_circuit_pqueue,
-                              compare_cell_ewma_counts,
-                              STRUCT_OFFSET(cell_ewma_t, heap_index));
-}
-
-/** Add <b>circ</b> to the list of circuits with pending cells on
- * <b>conn</b>.  No effect if <b>circ</b> is already linked. */
+/** Remove all circuits from the cmux on <b>chan</b>. */
 void
-make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn)
+channel_unlink_all_circuits(channel_t *chan)
 {
-  circuit_t **nextp = next_circ_on_conn_p(circ, conn);
-  circuit_t **prevp = prev_circ_on_conn_p(circ, conn);
+  tor_assert(chan);
+  tor_assert(chan->cmux);
 
-  if (*nextp && *prevp) {
-    /* Already active. */
-    return;
-  }
-
-  assert_active_circuits_ok_paranoid(conn);
-
-  if (! conn->active_circuits) {
-    conn->active_circuits = circ;
-    *prevp = *nextp = circ;
-  } else {
-    circuit_t *head = conn->active_circuits;
-    circuit_t *old_tail = *prev_circ_on_conn_p(head, conn);
-    *next_circ_on_conn_p(old_tail, conn) = circ;
-    *nextp = head;
-    *prev_circ_on_conn_p(head, conn) = circ;
-    *prevp = old_tail;
-  }
-
-  if (circ->n_conn == conn) {
-    add_cell_ewma_to_conn(conn, &circ->n_cell_ewma);
-  } else {
-    or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
-    tor_assert(conn == orcirc->p_conn);
-    add_cell_ewma_to_conn(conn, &orcirc->p_cell_ewma);
-  }
-
-  assert_active_circuits_ok_paranoid(conn);
-}
-
-/** Remove <b>circ</b> from the list of circuits with pending cells on
- * <b>conn</b>.  No effect if <b>circ</b> is already unlinked. */
-void
-make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn)
-{
-  circuit_t **nextp = next_circ_on_conn_p(circ, conn);
-  circuit_t **prevp = prev_circ_on_conn_p(circ, conn);
-  circuit_t *next = *nextp, *prev = *prevp;
-
-  if (!next && !prev) {
-    /* Already inactive. */
-    return;
-  }
-
-  assert_active_circuits_ok_paranoid(conn);
-
-  tor_assert(next && prev);
-  tor_assert(*prev_circ_on_conn_p(next, conn) == circ);
-  tor_assert(*next_circ_on_conn_p(prev, conn) == circ);
-
-  if (next == circ) {
-    conn->active_circuits = NULL;
-  } else {
-    *prev_circ_on_conn_p(next, conn) = prev;
-    *next_circ_on_conn_p(prev, conn) = next;
-    if (conn->active_circuits == circ)
-      conn->active_circuits = next;
-  }
-  *prevp = *nextp = NULL;
-
-  if (circ->n_conn == conn) {
-    remove_cell_ewma_from_conn(conn, &circ->n_cell_ewma);
-  } else {
-    or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
-    tor_assert(conn == orcirc->p_conn);
-    remove_cell_ewma_from_conn(conn, &orcirc->p_cell_ewma);
-  }
-
-  assert_active_circuits_ok_paranoid(conn);
-}
-
-/** Remove all circuits from the list of circuits with pending cells on
- * <b>conn</b>. */
-void
-connection_or_unlink_all_active_circs(or_connection_t *orconn)
-{
-  circuit_t *head = orconn->active_circuits;
-  circuit_t *cur = head;
-  if (! head)
-    return;
-  do {
-    circuit_t *next = *next_circ_on_conn_p(cur, orconn);
-    *prev_circ_on_conn_p(cur, orconn) = NULL;
-    *next_circ_on_conn_p(cur, orconn) = NULL;
-    cur = next;
-  } while (cur != head);
-  orconn->active_circuits = NULL;
-
-  SMARTLIST_FOREACH(orconn->active_circuit_pqueue, cell_ewma_t *, e,
-                    e->heap_index = -1);
-  smartlist_clear(orconn->active_circuit_pqueue);
+  circuitmux_detach_all_circuits(chan->cmux);
+  chan->num_n_circuits = 0;
+  chan->num_p_circuits = 0;
 }
 
 /** Block (if <b>block</b> is true) or unblock (if <b>block</b> is false)
- * every edge connection that is using <b>circ</b> to write to <b>orconn</b>,
+ * every edge connection that is using <b>circ</b> to write to <b>chan</b>,
  * and start or stop reading as appropriate.
  *
  * If <b>stream_id</b> is nonzero, block only the edge connection whose
@@ -2329,17 +2036,17 @@ connection_or_unlink_all_active_circs(or_connection_t *orconn)
  * Returns the number of streams whose status we changed.
  */
 static int
-set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn,
+set_streams_blocked_on_circ(circuit_t *circ, channel_t *chan,
                             int block, streamid_t stream_id)
 {
   edge_connection_t *edge = NULL;
   int n = 0;
-  if (circ->n_conn == orconn) {
-    circ->streams_blocked_on_n_conn = block;
+  if (circ->n_chan == chan) {
+    circ->streams_blocked_on_n_chan = block;
     if (CIRCUIT_IS_ORIGIN(circ))
       edge = TO_ORIGIN_CIRCUIT(circ)->p_streams;
   } else {
-    circ->streams_blocked_on_p_conn = block;
+    circ->streams_blocked_on_p_chan = block;
     tor_assert(!CIRCUIT_IS_ORIGIN(circ));
     edge = TO_OR_CIRCUIT(circ)->n_streams;
   }
@@ -2374,58 +2081,51 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn,
 }
 
 /** Pull as many cells as possible (but no more than <b>max</b>) from the
- * queue of the first active circuit on <b>conn</b>, and write them to
- * <b>conn</b>-&gt;outbuf.  Return the number of cells written.  Advance
+ * queue of the first active circuit on <b>chan</b>, and write them to
+ * <b>chan</b>-&gt;outbuf.  Return the number of cells written.  Advance
  * the active circuit pointer to the next active circuit in the ring. */
 int
-connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max,
-                                              time_t now)
+channel_flush_from_first_active_circuit(channel_t *chan, int max)
 {
-  int n_flushed;
+  circuitmux_t *cmux = NULL;
+  int n_flushed = 0;
   cell_queue_t *queue;
   circuit_t *circ;
+  or_circuit_t *or_circ;
   int streams_blocked;
+  packed_cell_t *cell;
 
-  /* The current (hi-res) time */
-  struct timeval now_hires;
+  /* Get the cmux */
+  tor_assert(chan);
+  tor_assert(chan->cmux);
+  cmux = chan->cmux;
 
-  /* The EWMA cell counter for the circuit we're flushing. */
-  cell_ewma_t *cell_ewma = NULL;
-  double ewma_increment = -1;
+  /* Main loop: pick a circuit, send a cell, update the cmux */
+  while (n_flushed < max) {
+    circ = circuitmux_get_first_active_circuit(cmux);
+    /* If it returns NULL, no cells left to send */
+    if (!circ) break;
+    assert_cmux_ok_paranoid(chan);
 
-  circ = conn->active_circuits;
-  if (!circ) return 0;
-  assert_active_circuits_ok_paranoid(conn);
-
-  /* See if we're doing the ewma circuit selection algorithm. */
-  if (ewma_enabled) {
-    unsigned tick;
-    double fractional_tick;
-    tor_gettimeofday_cached(&now_hires);
-    tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick);
-
-    if (tick != conn->active_circuit_pqueue_last_recalibrated) {
-      scale_active_circuits(conn, tick);
+    if (circ->n_chan == chan) {
+      queue = &circ->n_chan_cells;
+      streams_blocked = circ->streams_blocked_on_n_chan;
+    } else {
+      or_circ = TO_OR_CIRCUIT(circ);
+      tor_assert(or_circ->p_chan == chan);
+      queue = &TO_OR_CIRCUIT(circ)->p_chan_cells;
+      streams_blocked = circ->streams_blocked_on_p_chan;
     }
 
-    ewma_increment = pow(ewma_scale_factor, -fractional_tick);
+    /* Circuitmux told us this was active, so it should have cells */
+    tor_assert(queue->n > 0);
 
-    cell_ewma = smartlist_get(conn->active_circuit_pqueue, 0);
-    circ = cell_ewma_to_circuit(cell_ewma);
-  }
-
-  if (circ->n_conn == conn) {
-    queue = &circ->n_conn_cells;
-    streams_blocked = circ->streams_blocked_on_n_conn;
-  } else {
-    queue = &TO_OR_CIRCUIT(circ)->p_conn_cells;
-    streams_blocked = circ->streams_blocked_on_p_conn;
-  }
-  tor_assert(*next_circ_on_conn_p(circ,conn));
-
-  for (n_flushed = 0; n_flushed < max && queue->head; ) {
-    packed_cell_t *cell = cell_queue_pop(queue);
-    tor_assert(*next_circ_on_conn_p(circ,conn));
+    /*
+     * Get just one cell here; once we've sent it, that can change the circuit
+     * selection, so we have to loop around for another even if this circuit
+     * has more than one.
+     */
+    cell = cell_queue_pop(queue);
 
     /* Calculate the exact time that this cell has spent in the queue. */
     if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) {
@@ -2441,8 +2141,8 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max,
                              "Looks like the CellStatistics option was "
                              "recently enabled.");
       } else {
-        or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
         insertion_time_elem_t *elem = it_queue->first;
+        or_circ = TO_OR_CIRCUIT(circ);
         cell_waiting_time =
             (uint32_t)((flushed * 10L + SECONDS_IN_A_DAY * 1000L -
                         elem->insertion_time * 10L) %
@@ -2455,66 +2155,58 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max,
             it_queue->last = NULL;
           mp_pool_release(elem);
         }
-        orcirc->total_cell_waiting_time += cell_waiting_time;
-        orcirc->processed_cells++;
+        or_circ->total_cell_waiting_time += cell_waiting_time;
+        or_circ->processed_cells++;
       }
     }
 
     /* If we just flushed our queue and this circuit is used for a
      * tunneled directory request, possibly advance its state. */
-    if (queue->n == 0 && TO_CONN(conn)->dirreq_id)
-      geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id,
+    if (queue->n == 0 && chan->dirreq_id)
+      geoip_change_dirreq_state(chan->dirreq_id,
                                 DIRREQ_TUNNELED,
                                 DIRREQ_CIRC_QUEUE_FLUSHED);
 
-    connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn));
+    /* Now send the cell */
+    channel_write_packed_cell(chan, cell);
+    cell = NULL;
 
-    packed_cell_free_unchecked(cell);
+    /*
+     * Don't packed_cell_free_unchecked(cell) here because the channel will
+     * do so when it gets out of the channel queue (probably already did, in
+     * which case that was an immediate double-free bug).
+     */
+
+    /* Update the counter */
     ++n_flushed;
-    if (cell_ewma) {
-      cell_ewma_t *tmp;
-      cell_ewma->cell_count += ewma_increment;
-      /* We pop and re-add the cell_ewma_t here, not above, since we need to
-       * re-add it immediately to keep the priority queue consistent with
-       * the linked-list implementation */
-      tmp = pop_first_cell_ewma_from_conn(conn);
-      tor_assert(tmp == cell_ewma);
-      add_cell_ewma_to_conn(conn, cell_ewma);
-    }
-    if (!ewma_enabled && circ != conn->active_circuits) {
-      /* If this happens, the current circuit just got made inactive by
-       * a call in connection_write_to_buf().  That's nothing to worry about:
-       * circuit_make_inactive_on_conn() already advanced conn->active_circuits
-       * for us.
-       */
-      assert_active_circuits_ok_paranoid(conn);
-      goto done;
-    }
-  }
-  tor_assert(*next_circ_on_conn_p(circ,conn));
-  assert_active_circuits_ok_paranoid(conn);
-  conn->active_circuits = *next_circ_on_conn_p(circ, conn);
 
-  /* Is the cell queue low enough to unblock all the streams that are waiting
-   * to write to this circuit? */
-  if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE)
-    set_streams_blocked_on_circ(circ, conn, 0, 0); /* unblock streams */
+    /*
+     * Now update the cmux; tell it we've just sent a cell, and how many
+     * we have left.
+     */
+    circuitmux_notify_xmit_cells(cmux, circ, 1);
+    circuitmux_set_num_cells(cmux, circ, queue->n);
+    if (queue->n == 0)
+      log_debug(LD_GENERAL, "Made a circuit inactive.");
 
-  /* Did we just run out of cells on this circuit's queue? */
-  if (queue->n == 0) {
-    log_debug(LD_GENERAL, "Made a circuit inactive.");
-    make_circuit_inactive_on_conn(circ, conn);
+    /* Is the cell queue low enough to unblock all the streams that are waiting
+     * to write to this circuit? */
+    if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE)
+      set_streams_blocked_on_circ(circ, chan, 0, 0); /* unblock streams */
+
+    /* If n_flushed < max still, loop around and pick another circuit */
   }
- done:
-  if (n_flushed)
-    conn->timestamp_last_added_nonpadding = now;
+
+  /* Okay, we're done sending now */
+  assert_cmux_ok_paranoid(chan);
+
   return n_flushed;
 }
 
-/** Add <b>cell</b> to the queue of <b>circ</b> writing to <b>orconn</b>
+/** Add <b>cell</b> to the queue of <b>circ</b> writing to <b>chan</b>
  * transmitting in <b>direction</b>. */
 void
-append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn,
+append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan,
                              cell_t *cell, cell_direction_t direction,
                              streamid_t fromstream)
 {
@@ -2524,12 +2216,12 @@ append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn,
     return;
 
   if (direction == CELL_DIRECTION_OUT) {
-    queue = &circ->n_conn_cells;
-    streams_blocked = circ->streams_blocked_on_n_conn;
+    queue = &circ->n_chan_cells;
+    streams_blocked = circ->streams_blocked_on_n_chan;
   } else {
     or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
-    queue = &orcirc->p_conn_cells;
-    streams_blocked = circ->streams_blocked_on_p_conn;
+    queue = &orcirc->p_chan_cells;
+    streams_blocked = circ->streams_blocked_on_p_chan;
   }
 
   cell_queue_append_packed_copy(queue, cell);
@@ -2537,27 +2229,27 @@ append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn,
   /* If we have too many cells on the circuit, we should stop reading from
    * the edge streams for a while. */
   if (!streams_blocked && queue->n >= CELL_QUEUE_HIGHWATER_SIZE)
-    set_streams_blocked_on_circ(circ, orconn, 1, 0); /* block streams */
+    set_streams_blocked_on_circ(circ, chan, 1, 0); /* block streams */
 
   if (streams_blocked && fromstream) {
     /* This edge connection is apparently not blocked; block it. */
-    set_streams_blocked_on_circ(circ, orconn, 1, fromstream);
+    set_streams_blocked_on_circ(circ, chan, 1, fromstream);
   }
 
+  update_circuit_on_cmux(circ, direction);
   if (queue->n == 1) {
-    /* This was the first cell added to the queue.  We need to make this
+    /* This was the first cell added to the queue.  We just made this
      * circuit active. */
     log_debug(LD_GENERAL, "Made a circuit active.");
-    make_circuit_active_on_conn(circ, orconn);
   }
 
-  if (! connection_get_outbuf_len(TO_CONN(orconn))) {
+  if (!channel_has_queued_writes(chan)) {
     /* There is no data at all waiting to be sent on the outbuf.  Add a
      * cell, so that we can notice when it gets flushed, flushed_some can
      * get called, and we can start putting more data onto the buffer then.
      */
     log_debug(LD_GENERAL, "Primed a buffer.");
-    connection_or_flush_from_first_active_circuit(orconn, 1, approx_time());
+    channel_flush_from_first_active_circuit(chan, 1);
   }
 }
 
@@ -2621,58 +2313,39 @@ decode_address_from_payload(tor_addr_t *addr_out, const uint8_t *payload,
   return payload + 2 + payload[1];
 }
 
-/** Remove all the cells queued on <b>circ</b> for <b>orconn</b>. */
+/** Remove all the cells queued on <b>circ</b> for <b>chan</b>. */
 void
-circuit_clear_cell_queue(circuit_t *circ, or_connection_t *orconn)
+circuit_clear_cell_queue(circuit_t *circ, channel_t *chan)
 {
   cell_queue_t *queue;
-  if (circ->n_conn == orconn) {
-    queue = &circ->n_conn_cells;
+  cell_direction_t direction;
+
+  if (circ->n_chan == chan) {
+    queue = &circ->n_chan_cells;
+    direction = CELL_DIRECTION_OUT;
   } else {
     or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
-    tor_assert(orcirc->p_conn == orconn);
-    queue = &orcirc->p_conn_cells;
+    tor_assert(orcirc->p_chan == chan);
+    queue = &orcirc->p_chan_cells;
+    direction = CELL_DIRECTION_IN;
   }
 
-  if (queue->n)
-    make_circuit_inactive_on_conn(circ,orconn);
-
+  /* Clear the queue */
   cell_queue_clear(queue);
+
+  /* Update the cell counter in the cmux */
+  update_circuit_on_cmux(circ, direction);
 }
 
-/** Fail with an assert if the active circuits ring on <b>orconn</b> is
- * corrupt.  */
+/** Fail with an assert if the circuit mux on chan is corrupt
+ */
 void
-assert_active_circuits_ok(or_connection_t *orconn)
+assert_circuit_mux_okay(channel_t *chan)
 {
-  circuit_t *head = orconn->active_circuits;
-  circuit_t *cur = head;
-  int n = 0;
-  if (! head)
-    return;
-  do {
-    circuit_t *next = *next_circ_on_conn_p(cur, orconn);
-    circuit_t *prev = *prev_circ_on_conn_p(cur, orconn);
-    cell_ewma_t *ewma;
-    tor_assert(next);
-    tor_assert(prev);
-    tor_assert(*next_circ_on_conn_p(prev, orconn) == cur);
-    tor_assert(*prev_circ_on_conn_p(next, orconn) == cur);
-    if (orconn == cur->n_conn) {
-      ewma = &cur->n_cell_ewma;
-      tor_assert(!ewma->is_for_p_conn);
-    } else {
-      ewma = &TO_OR_CIRCUIT(cur)->p_cell_ewma;
-      tor_assert(ewma->is_for_p_conn);
-    }
-    tor_assert(ewma->heap_index != -1);
-    tor_assert(ewma == smartlist_get(orconn->active_circuit_pqueue,
-                                     ewma->heap_index));
-    n++;
-    cur = next;
-  } while (cur != head);
+  tor_assert(chan);
+  tor_assert(chan->cmux);
 
-  tor_assert(n == smartlist_len(orconn->active_circuit_pqueue));
+  circuitmux_assert_okay(chan->cmux);
 }
 
 /** Return 1 if we shouldn't restart reading on this circuit, even if
@@ -2682,9 +2355,9 @@ static int
 circuit_queue_streams_are_blocked(circuit_t *circ)
 {
   if (CIRCUIT_IS_ORIGIN(circ)) {
-    return circ->streams_blocked_on_n_conn;
+    return circ->streams_blocked_on_n_chan;
   } else {
-    return circ->streams_blocked_on_p_conn;
+    return circ->streams_blocked_on_p_chan;
   }
 }
 

src/or/relay.h

@@ -41,28 +41,26 @@ void free_cell_pool(void);
 void clean_cell_pool(void);
 void dump_cell_pool_usage(int severity);
 
+/* For channeltls.c */
+void packed_cell_free(packed_cell_t *cell);
+
 void cell_queue_clear(cell_queue_t *queue);
 void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell);
 void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell);
 
-void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn,
+void append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan,
                                   cell_t *cell, cell_direction_t direction,
                                   streamid_t fromstream);
-void connection_or_unlink_all_active_circs(or_connection_t *conn);
-int connection_or_flush_from_first_active_circuit(or_connection_t *conn,
-                                                  int max, time_t now);
-void assert_active_circuits_ok(or_connection_t *orconn);
-void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn);
-void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn);
+void channel_unlink_all_circuits(channel_t *chan);
+int channel_flush_from_first_active_circuit(channel_t *chan, int max);
+void assert_circuit_mux_okay(channel_t *chan);
+void update_circuit_on_cmux(circuit_t *circ, cell_direction_t direction);
 
 int append_address_to_payload(uint8_t *payload_out, const tor_addr_t *addr);
 const uint8_t *decode_address_from_payload(tor_addr_t *addr_out,
                                         const uint8_t *payload,
                                         int payload_len);
-unsigned cell_ewma_get_tick(void);
-void cell_ewma_set_scale_factor(const or_options_t *options,
-                                const networkstatus_t *consensus);
-void circuit_clear_cell_queue(circuit_t *circ, or_connection_t *orconn);
+void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan);
 
 #ifdef RELAY_PRIVATE
 int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction,

src/or/rendmid.c

@@ -35,7 +35,7 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
            "Received an ESTABLISH_INTRO request on circuit %d",
            circ->p_circ_id);
 
-  if (circ->_base.purpose != CIRCUIT_PURPOSE_OR || circ->_base.n_conn) {
+  if (circ->_base.purpose != CIRCUIT_PURPOSE_OR || circ->_base.n_chan) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
          "Rejecting ESTABLISH_INTRO on non-OR or non-edge circuit.");
     reason = END_CIRC_REASON_TORPROTOCOL;
@@ -142,7 +142,7 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
   log_info(LD_REND, "Received an INTRODUCE1 request on circuit %d",
            circ->p_circ_id);
 
-  if (circ->_base.purpose != CIRCUIT_PURPOSE_OR || circ->_base.n_conn) {
+  if (circ->_base.purpose != CIRCUIT_PURPOSE_OR || circ->_base.n_chan) {
     log_warn(LD_PROTOCOL,
              "Rejecting INTRODUCE1 on non-OR or non-edge circuit %d.",
              circ->p_circ_id);
@@ -224,7 +224,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
   log_info(LD_REND, "Received an ESTABLISH_RENDEZVOUS request on circuit %d",
            circ->p_circ_id);
 
-  if (circ->_base.purpose != CIRCUIT_PURPOSE_OR || circ->_base.n_conn) {
+  if (circ->_base.purpose != CIRCUIT_PURPOSE_OR || circ->_base.n_chan) {
     log_warn(LD_PROTOCOL,
              "Tried to establish rendezvous on non-OR or non-edge circuit.");
     goto err;
@@ -277,7 +277,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
   char hexid[9];
   int reason = END_CIRC_REASON_INTERNAL;
 
-  if (circ->_base.purpose != CIRCUIT_PURPOSE_OR || circ->_base.n_conn) {
+  if (circ->_base.purpose != CIRCUIT_PURPOSE_OR || circ->_base.n_chan) {
     log_info(LD_REND,
              "Tried to complete rendezvous on non-OR or non-edge circuit %d.",
              circ->p_circ_id);

src/or/router.h

@@ -86,13 +86,18 @@ int router_pick_published_address(const or_options_t *options, uint32_t *addr);
 int router_rebuild_descriptor(int force);
 int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router,
                                  crypto_pk_t *ident_key);
-int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo,
-                             crypto_pk_t *ident_key);
 void router_get_prim_orport(const routerinfo_t *router,
-                            tor_addr_port_t *ap_out);
+                            tor_addr_port_t *addr_port_out);
+void router_get_pref_orport(const routerinfo_t *router,
+                            tor_addr_port_t *addr_port_out);
+void router_get_pref_ipv6_orport(const routerinfo_t *router,
+                                 tor_addr_port_t *addr_port_out);
+int router_ipv6_preferred(const routerinfo_t *router);
 int router_has_addr(const routerinfo_t *router, const tor_addr_t *addr);
 int router_has_orport(const routerinfo_t *router,
                       const tor_addr_port_t *orport);
+int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo,
+                             crypto_pk_t *ident_key);
 int is_legal_nickname(const char *s);
 int is_legal_nickname_or_hexdigest(const char *s);
 int is_legal_hexdigest(const char *s);