compute the client-side pow in a cpuworker thread

We mark the intro circuit with a new flag saying that the pow is
in the cpuworker queue. When the cpuworker comes back, it either
has a solution, in which case we proceed with sending the intro1
cell, or it has no solution, in which case we unmark the intro
circuit and let the whole process restart on the next iteration of
connection_ap_handshake_attach_circuit().

src/core/mainloop/cpuworker.c

@@ -14,7 +14,8 @@
  * Right now, we use this infrastructure
  *  <ul><li>for processing onionskins in onion.c
  *      <li>for compressing consensuses in consdiffmgr.c,
- *      <li>and for calculating diffs and compressing them in consdiffmgr.c.
+ *      <li>for calculating diffs and compressing them in consdiffmgr.c.
+ *      <li>and for solving onion service PoW challenges in pow.c.
  *  </ul>
  **/
 #include "core/or/or.h"

src/core/or/circuituse.c

@@ -3043,8 +3043,8 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       if (introcirc->base_.state == CIRCUIT_STATE_OPEN) {
         int ret;
         log_info(LD_REND, "Found open intro circ %u (id: %" PRIu32 "). "
-                          "Rend circuit %u (id: %" PRIu32 "); Sending "
-                          "introduction. (stream %d sec old)",
+                          "Rend circuit %u (id: %" PRIu32 "); Considering "
+                          "sending introduction. (stream %d sec old)",
                  (unsigned) TO_CIRCUIT(introcirc)->n_circ_id,
                  introcirc->global_identifier,
                  (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id,

src/core/or/origin_circuit_st.h

@@ -218,6 +218,10 @@ struct origin_circuit_t {
    * requests. */
   unsigned int hs_with_pow_circ : 1;
 
+  /** Set iff this intro circ required a pow, and it has already queued
+   * the pow with the cpuworker and is awaiting a reply. */
+  unsigned int hs_currently_solving_pow : 1;
+
   /** Set iff this circuit has been given a relaxed timeout because
    * no circuits have opened. Used to prevent spamming logs. */
   unsigned int relaxed_timeout : 1;

src/feature/hs/hs_client.c

@@ -541,7 +541,7 @@ intro_circ_is_ok(const origin_circuit_t *circ)
 
 /** Find a descriptor intro point object that matches the given ident in the
  * given descriptor desc. Return NULL if not found. */
-static const hs_desc_intro_point_t *
+const hs_desc_intro_point_t *
 find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident,
                                const hs_descriptor_t *desc)
 {
@@ -670,7 +670,6 @@ consider_sending_introduce1(origin_circuit_t *intro_circ,
   char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
   const ed25519_public_key_t *service_identity_pk = NULL;
   const hs_desc_intro_point_t *ip;
-  hs_pow_solution_t *pow_solution = NULL;
 
   tor_assert(rend_circ);
   if (intro_circ_is_ok(intro_circ) < 0) {
@@ -682,9 +681,15 @@ consider_sending_introduce1(origin_circuit_t *intro_circ,
    * version number but for now there is none because it's all v3. */
   hs_build_address(service_identity_pk, HS_VERSION_THREE, onion_address);
 
-  log_info(LD_REND, "Sending INTRODUCE1 cell to service %s on circuit %u",
+  log_info(LD_REND, "Considering sending INTRODUCE1 cell to service %s "
+           "on circuit %u",
            safe_str_client(onion_address), TO_CIRCUIT(intro_circ)->n_circ_id);
 
+  /* if it's already waiting on the cpuworker farm, don't queue it again */
+  if (intro_circ->hs_currently_solving_pow) {
+    goto tran_err;
+  }
+
   /* 1) Get descriptor from our cache. */
   const hs_descriptor_t *desc =
     hs_cache_lookup_as_client(service_identity_pk);
@@ -731,7 +736,6 @@ consider_sending_introduce1(origin_circuit_t *intro_circ,
   if (desc->encrypted_data.pow_params &&
       desc->encrypted_data.pow_params->suggested_effort > 0) {
     log_debug(LD_REND, "PoW params present in descriptor.");
-    pow_solution = tor_malloc_zero(sizeof(hs_pow_solution_t));
 
     /* make sure we can't be tricked into hopeless quests */
     if (desc->encrypted_data.pow_params->suggested_effort >
@@ -746,15 +750,15 @@ consider_sending_introduce1(origin_circuit_t *intro_circ,
         CLIENT_MAX_POW_EFFORT;
     }
 
-    if (hs_pow_solve(desc->encrypted_data.pow_params, pow_solution)) {
-      log_warn(LD_REND, "Haven't solved the PoW yet.");
-      goto tran_err;
-    }
-    log_notice(LD_REND, "Got a PoW solution we like! Shipping it!");
-    /* Set flag to reflect that the HS we are attempting to rendezvous has PoW
-     * defenses enabled, and as such we will need to be more lenient with
-     * timing out while waiting for the circuit to be built. */
-    rend_circ->hs_with_pow_circ = 1;
+    /* send it to the client-side pow cpuworker for solving. */
+    intro_circ->hs_currently_solving_pow = 1;
+    pow_queue_work(intro_circ->global_identifier,
+                   rend_circ->global_identifier,
+                   desc->encrypted_data.pow_params);
+
+    /* can't proceed with the intro1 cell yet, so yield back to the
+     * main loop */
+    goto tran_err;
   }
 
   /* move on to the next phase: actually try to send it */
@@ -781,7 +785,6 @@ consider_sending_introduce1(origin_circuit_t *intro_circ,
 
  end:
   memwipe(onion_address, 0, sizeof(onion_address));
-  tor_free(pow_solution);
   return status;
 }
 

src/feature/hs/hs_client.h

@@ -78,6 +78,10 @@ typedef struct hs_client_service_authorization_t {
   int flags;
 } hs_client_service_authorization_t;
 
+const hs_desc_intro_point_t *
+find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident,
+                               const hs_descriptor_t *desc);
+
 hs_client_register_auth_status_t
 hs_client_register_auth_credentials(hs_client_service_authorization_t *creds);
 

src/feature/hs/hs_pow.c

@@ -13,9 +13,15 @@ typedef unsigned __int128 uint128_t;
 #include <stdio.h>
 
 #include "ext/ht.h"
+#include "core/or/circuitlist.h"
+#include "core/or/origin_circuit_st.h"
+#include "feature/hs/hs_cache.h"
 #include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_pow.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "core/mainloop/cpuworker.h"
+#include "lib/evloop/workqueue.h"
 
 /** Replay cache set up */
 /** Cache entry for (nonce, seed) replay protection. */
@@ -144,7 +150,7 @@ hs_pow_solve(const hs_pow_desc_params_t *pow_params,
 
   /* We'll do a maximum of the nonce size iterations here which is the maximum
    * number of nonce we can try in an attempt to find a valid solution. */
-  log_notice(LD_REND, "Solving proof of work");
+  log_notice(LD_REND, "Solving proof of work (effort %u)", effort);
   for (uint64_t i = 0; i < UINT64_MAX; i++) {
     /* Calculate S = equix_solve(C || N || E) */
     if (!equix_solve(ctx, challenge, HS_POW_CHALLENGE_LEN, solution)) {
@@ -290,3 +296,150 @@ hs_pow_free_service_state(hs_pow_service_state_t *state)
   mainloop_event_free(state->pop_pqueue_ev);
   tor_free(state);
 }
+
+/* =====
+   Thread workers
+   =====*/
+
+/**
+ * An object passed to a worker thread that will try to solve the pow.
+ */
+typedef struct pow_worker_job_t {
+
+  /** Input: The pow challenge we need to solve. */
+  hs_pow_desc_params_t *pow_params;
+
+  /** State: we'll look these up to figure out how to proceed after. */
+  uint32_t intro_circ_identifier;
+  uint32_t rend_circ_identifier;
+
+  /** Output: The worker thread will malloc and write its answer here,
+   * or set it to NULL if it produced no useful answer. */
+  hs_pow_solution_t *pow_solution_out;
+
+} pow_worker_job_t;
+
+/**
+ * Worker function. This function runs inside a worker thread and receives
+ * a pow_worker_job_t as its input.
+ */
+static workqueue_reply_t
+pow_worker_threadfn(void *state_, void *work_)
+{
+  (void)state_;
+  pow_worker_job_t *job = work_;
+  job->pow_solution_out = tor_malloc_zero(sizeof(hs_pow_solution_t));
+
+  if (hs_pow_solve(job->pow_params, job->pow_solution_out)) {
+    log_info(LD_REND, "Haven't solved the PoW yet. Returning.");
+    tor_free(job->pow_solution_out);
+    job->pow_solution_out = NULL; /* how we signal that we came up empty */
+    return WQ_RPL_REPLY;
+  }
+
+  /* we have a winner! */
+  log_info(LD_REND, "cpuworker pow: we have a winner!");
+  return WQ_RPL_REPLY;
+}
+
+/**
+ * Helper: release all storage held in <b>job</b>.
+ */
+static void
+pow_worker_job_free(pow_worker_job_t *job)
+{
+  if (!job)
+    return;
+  tor_free(job->pow_params);
+  tor_free(job->pow_solution_out);
+  tor_free(job);
+}
+
+/**
+ * Worker function: This function runs in the main thread, and receives
+ * a pow_worker_job_t that the worker thread has already processed.
+ */
+static void
+pow_worker_replyfn(void *work_)
+{
+  tor_assert(in_main_thread());
+  tor_assert(work_);
+
+  pow_worker_job_t *job = work_;
+
+  // look up the circuits that we're going to use this pow in
+  origin_circuit_t *intro_circ =
+    circuit_get_by_global_id(job->intro_circ_identifier);
+  origin_circuit_t *rend_circ =
+    circuit_get_by_global_id(job->rend_circ_identifier);
+
+  /* try to re-create desc and ip */
+  const ed25519_public_key_t *service_identity_pk = NULL;
+  const hs_descriptor_t *desc = NULL;
+  const hs_desc_intro_point_t *ip = NULL;
+  if (intro_circ)
+    service_identity_pk = &intro_circ->hs_ident->identity_pk;
+  if (service_identity_pk)
+    desc = hs_cache_lookup_as_client(service_identity_pk);
+  if (desc)
+    ip = find_desc_intro_point_by_ident(intro_circ->hs_ident, desc);
+
+  if (intro_circ && rend_circ && service_identity_pk && desc && ip &&
+      job->pow_solution_out) { /* successful pow solve, and circs still here */
+
+    log_notice(LD_REND, "Got a PoW solution we like! Shipping it!");
+    /* Set flag to reflect that the HS we are attempting to rendezvous has PoW
+     * defenses enabled, and as such we will need to be more lenient with
+     * timing out while waiting for the service-side circuit to be built. */
+    rend_circ->hs_with_pow_circ = 1;
+
+    // and then send that intro cell
+    if (send_introduce1(intro_circ, rend_circ,
+                        desc, job->pow_solution_out, ip) < 0) {
+      /* if it failed, mark the intro point as ready to start over */
+      intro_circ->hs_currently_solving_pow = 0;
+    }
+
+  } else { /* unsuccessful pow solve. put it back on the queue. */
+    log_notice(LD_REND,
+               "PoW cpuworker returned with no solution. Will retry soon.");
+    if (intro_circ) {
+      intro_circ->hs_currently_solving_pow = 0;
+    }
+    /* We could imagine immediately re-launching a follow-up worker
+     * here too, but for now just let the main intro loop find the
+     * not-being-serviced request and it can start everything again. For
+     * the sake of complexity, maybe that's the best long-term solution
+     * too, and we can tune the cpuworker job to try for longer if we want
+     * to improve efficiency. */
+  }
+
+  pow_worker_job_free(job);
+}
+
+/**
+ * Queue the job of solving the pow in a worker thread.
+ */
+int
+pow_queue_work(uint32_t intro_circ_identifier,
+               uint32_t rend_circ_identifier,
+               const hs_pow_desc_params_t *pow_params)
+{
+  tor_assert(in_main_thread());
+
+  pow_worker_job_t *job = tor_malloc_zero(sizeof(*job));
+  job->intro_circ_identifier = intro_circ_identifier;
+  job->rend_circ_identifier = rend_circ_identifier;
+  job->pow_params = tor_memdup(pow_params, sizeof(hs_pow_desc_params_t));
+
+  workqueue_entry_t *work;
+  work = cpuworker_queue_work(WQ_PRI_LOW,
+                              pow_worker_threadfn,
+                              pow_worker_replyfn,
+                              job);
+  if (!work) {
+    pow_worker_job_free(job);
+    return -1;
+  }
+  return 0;
+}

src/feature/hs/hs_pow.h

@@ -130,4 +130,8 @@ int hs_pow_verify(const hs_pow_service_state_t *pow_state,
 void hs_pow_remove_seed_from_cache(uint32_t seed);
 void hs_pow_free_service_state(hs_pow_service_state_t *state);
 
+int pow_queue_work(uint32_t intro_circ_identifier,
+                   uint32_t rend_circ_identifier,
+                   const hs_pow_desc_params_t *pow_params);
+
 #endif /* !defined(TOR_HS_POW_H) */

src/lib/evloop/workqueue.c

@@ -20,7 +20,10 @@
  * The main thread can also queue an "update" that will be handled by all the
  * workers.  This is useful for updating state that all the workers share.
  *
- * In Tor today, there is currently only one thread pool, used in cpuworker.c.
+ * In Tor today, there is currently only one thread pool, managed
+ * in cpuworker.c and handling a variety of types of work, from the original
+ * "onion skin" circuit handshakes, to consensus diff computation, to
+ * client-side onion service PoW generation.
  */
 
 #include "orconfig.h"