bug#22143/prop#140: in consdiffmgr, store and use digest-as-signed

We need to index diffs by the digest-as-signed of their source
consensus, so that we can find them even from consensuses whose
signatures are encoded differently.

src/or/consdiffmgr.c

@@ -19,6 +19,7 @@
 #include "consdiffmgr.h"
 #include "cpuworker.h"
 #include "networkstatus.h"
+#include "routerparse.h"
 #include "workqueue.h"
 
 /**
@@ -35,11 +36,13 @@
 #define LABEL_SHA3_DIGEST "sha3-digest"
 /* A hex encoded SHA3 digest of the object before compression. */
 #define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
+/* A hex encoded SHA3 digest-as-signed of a consensus */
+#define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
 /* The flavor of the consensus or consensuses diff */
 #define LABEL_FLAVOR "consensus-flavor"
-/* Diff only: the SHA3 digest of the source consensus. */
+/* Diff only: the SHA3 digest-as-signed of the source consensus. */
 #define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
-/* Diff only: the SHA3 digest of the target consensus. */
+/* Diff only: the SHA3 digest-in-full of the target consensus. */
 #define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
 /* Diff only: the valid-after date of the source consensus. */
 #define LABEL_FROM_VALID_AFTER "from-valid-after"
@@ -466,6 +469,17 @@ consdiffmgr_add_consensus(const char *consensus,
 
     cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
                             (const uint8_t *)consensus, bodylen);
+    {
+      const char *start, *end;
+      if (router_get_networkstatus_v3_signed_boundaries(consensus,
+                                                        &start, &end) < 0) {
+        start = consensus;
+        end = consensus+bodylen;
+      }
+      cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
+                              (const uint8_t *)start,
+                              end - start);
+    }
 
     char *body_compressed = NULL;
     size_t size_compressed = 0;
@@ -845,7 +859,7 @@ consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
 
     uint8_t this_sha3[DIGEST256_LEN];
     if (BUG(cdm_entry_get_sha3_value(this_sha3, c,
-                                     LABEL_SHA3_DIGEST_UNCOMPRESSED)<0))
+                                     LABEL_SHA3_DIGEST_AS_SIGNED)<0))
       continue; // LCOV_EXCL_LINE
     if (cdm_diff_ht_check_and_note_pending(flavor,
                                            this_sha3, most_recent_sha3)) {
@@ -1131,7 +1145,7 @@ consensus_diff_worker_threadfn(void *state_, void *work_)
     consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
   const char *lv_from_digest =
     consensus_cache_entry_get_value(job->diff_from,
-                                    LABEL_SHA3_DIGEST_UNCOMPRESSED);
+                                    LABEL_SHA3_DIGEST_AS_SIGNED);
   const char *lv_from_flavor =
     consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
   const char *lv_to_flavor =
@@ -1140,10 +1154,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_)
     consensus_cache_entry_get_value(job->diff_to,
                                     LABEL_SHA3_DIGEST_UNCOMPRESSED);
 
+  if (! lv_from_digest) {
+    /* This isn't a bug right now, since it can happen if you're migrating
+     * from an older version of master to a newer one.  The older ones didn't
+     * annotate their stored consensus objects with sha3-digest-as-signed.
+    */
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+
   /* All these values are mandatory on the input */
   if (BUG(!lv_to_valid_after) ||
       BUG(!lv_from_valid_after) ||
-      BUG(!lv_from_digest) ||
       BUG(!lv_from_flavor) ||
       BUG(!lv_to_flavor)) {
     return WQ_RPL_REPLY; // LCOV_EXCL_LINE
@@ -1267,7 +1288,7 @@ consensus_diff_worker_replyfn(void *work_)
 
   const char *lv_from_digest =
     consensus_cache_entry_get_value(job->diff_from,
-                                    LABEL_SHA3_DIGEST_UNCOMPRESSED);
+                                    LABEL_SHA3_DIGEST_AS_SIGNED);
   const char *lv_to_digest =
     consensus_cache_entry_get_value(job->diff_to,
                                     LABEL_SHA3_DIGEST_UNCOMPRESSED);
@@ -1283,7 +1304,7 @@ consensus_diff_worker_replyfn(void *work_)
   int flav = -1;
   int cache = 1;
   if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
-                                   LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
+                                   LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
     cache = 0;
   if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
                                    LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))

src/test/test_consdiffmgr.c

@@ -10,6 +10,7 @@
 #include "consdiffmgr.h"
 #include "cpuworker.h"
 #include "networkstatus.h"
+#include "routerparse.h"
 #include "workqueue.h"
 
 #include "test.h"
@@ -66,6 +67,7 @@ fake_ns_body_new(consensus_flavor_t flav, time_t valid_after)
 
   format_iso_time(valid_after_string, valid_after);
   char *random_stuff = crypto_random_hostname(3, 25, "junk ", "");
+  char *random_stuff2 = crypto_random_hostname(3, 10, "", "");
 
   char *consensus;
   tor_asprintf(&consensus,
@@ -74,11 +76,15 @@ fake_ns_body_new(consensus_flavor_t flav, time_t valid_after)
                "valid-after %s\n"
                "r name ccccccccccccccccc etc\nsample\n"
                "r name eeeeeeeeeeeeeeeee etc\nbar\n"
-               "%s\n",
+               "%s\n"
+               "directory-signature hello-there\n"
+               "directory-signature %s\n",
                flavor_string,
                valid_after_string,
-               random_stuff);
+               random_stuff,
+               random_stuff2);
   tor_free(random_stuff);
+  tor_free(random_stuff2);
   return consensus;
 }
 
@@ -139,7 +145,10 @@ lookup_diff_from(consensus_cache_entry_t **out,
                  const char *str1)
 {
   uint8_t digest[DIGEST256_LEN];
-  crypto_digest256((char*)digest, str1, strlen(str1), DIGEST_SHA3_256);
+  if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) {
+    TT_FAIL(("Unable to compute sha3-as-signed"));
+    return CONSDIFF_NOT_FOUND;
+  }
   return consdiffmgr_find_diff_from(out, flav,
                                     DIGEST_SHA3_256, digest, sizeof(digest),
                                     NO_METHOD);
@@ -152,8 +161,9 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav,
 {
   consensus_cache_entry_t *ent = NULL;
   consdiff_status_t status = lookup_diff_from(&ent, flav, str1);
-  if (ent == NULL || status != CONSDIFF_AVAILABLE)
+  if (ent == NULL || status != CONSDIFF_AVAILABLE) {
     return -1;
+  }
 
   consensus_cache_entry_incref(ent);
   size_t size;
@@ -299,7 +309,7 @@ test_consdiffmgr_add(void *arg)
   ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */
   r = consdiffmgr_add_consensus(dummy, ns_tmp);
   tt_int_op(r, OP_EQ, -1);
-  expect_single_log_msg_containing("it's too old.");
+  expect_log_msg_containing("it's too old.");
 
   /* Try looking up a consensuses. */
   ent = cdm_cache_lookup_consensus(FLAV_NS, now-60);
@@ -352,8 +362,7 @@ test_consdiffmgr_make_diffs(void *arg)
   ns = fake_ns_new(FLAV_MICRODESC, now-3600);
   md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
   r = consdiffmgr_add_consensus(md_ns_body, ns);
-  crypto_digest256((char*)md_ns_sha3, md_ns_body, strlen(md_ns_body),
+  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body);
-                   DIGEST_SHA3_256);
   networkstatus_vote_free(ns);
   tt_int_op(r, OP_EQ, 0);