Merge branch 'tor-github/pr/1022'

.travis.yml

@@ -42,7 +42,7 @@ matrix:
   ## include creates builds with gcc, linux
   include:
     ## We include a single coverage build with the best options for coverage
-    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS=""
+    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" TOR_TEST_RNG_SEED="636f766572616765"
     ## We only want to check these build option combinations once
     ## (they shouldn't vary by compiler or OS)
     ## We run rust and coverage with hardening off, which seems like enough

changes/ticket28878

@@ -0,0 +1,11 @@
+  o Minor features (testing):
+    - The circuitpadding tests now use a reproducible RNG implementation,
+      so that if a test fails, we can learn why. Part of ticket 28878.
+    - Tor's tests now support an environment variable, TOR_TEST_RNG_SEED,
+      to set the RNG seed for tests that use a reproducible RNG.
+      Part of ticket 28878.
+
+  o Minor features (continuous integration):
+    - When running coverage builds on Travis, we now set TOR_TEST_RNG_SEED,
+      to avoid RNG-based coverage differences.
+      Part of ticket 28878.

src/ext/tinytest.c

@@ -492,6 +492,12 @@ tinytest_set_test_skipped_(void)
 		cur_test_outcome = SKIP;
 }
 
+int
+tinytest_cur_test_has_failed(void)
+{
+	return (cur_test_outcome == FAIL);
+}
+
 char *
 tinytest_format_hex_(const void *val_, unsigned long len)
 {

src/ext/tinytest.h

@@ -72,6 +72,9 @@ struct testlist_alias_t {
 };
 #define END_OF_ALIASES { NULL, NULL }
 
+/** Return true iff the current test has failed. */
+int tinytest_cur_test_has_failed(void);
+
 /** Implementation: called from a test to indicate failure, before logging. */
 void tinytest_set_test_failed_(void);
 /** Implementation: called from a test to indicate that we're skipping. */

src/test/rng_test_helpers.c

@@ -17,6 +17,7 @@
 #include "core/or/or.h"
 
 #include "lib/crypt_ops/crypto_rand.h"
+#include "ext/tinytest.h"
 
 #include "test/rng_test_helpers.h"
 
@@ -54,7 +55,8 @@ static uint8_t rng_seed[16];
 static crypto_xof_t *rng_xof = NULL;
 
 /**
- * Print the seed for our PRNG to stdout.  We use this when we're
+ * Print the seed for our PRNG to stdout.  We use this when we're failed
+ * test that had a reproducible RNG set.
  **/
 void
 testing_dump_reproducible_rng_seed(void)
@@ -122,10 +124,23 @@ enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len)
 void
 testing_enable_reproducible_rng(void)
 {
+  const char *provided_seed = getenv("TOR_TEST_RNG_SEED");
+  if (provided_seed) {
+    size_t hexlen = strlen(provided_seed);
+    size_t seedlen = hexlen / 2;
+    uint8_t *seed = tor_malloc(hexlen / 2);
+    if (base16_decode((char*)seed, seedlen, provided_seed, hexlen) < 0) {
+      puts("Cannot decode value in TOR_TEST_RNG_SEED");
+      exit(1);
+    }
+    enable_deterministic_rng_impl(seed, seedlen);
+    tor_free(seed);
+  } else {
     uint8_t seed[16];
     crypto_rand((char*)seed, sizeof(seed));
     enable_deterministic_rng_impl(seed, sizeof(seed));
   }
+}
 
 /**
  * Replace our get_thread_fast_rng(), crypto_rand() and
@@ -228,3 +243,16 @@ testing_disable_rng_override(void)
 
   rng_is_replaced = false;
 }
+
+/**
+ * As testing_disable_rng_override(), but dump the seed if the current
+ * test has failed.
+ */
+void
+testing_disable_reproducible_rng(void)
+{
+  if (tinytest_cur_test_has_failed()) {
+    testing_dump_reproducible_rng_seed();
+  }
+  testing_disable_rng_override();
+}

src/test/rng_test_helpers.h

@@ -14,8 +14,7 @@ void testing_prefilled_rng_reset(void);
 
 void testing_disable_rng_override(void);
 
-#define testing_disable_reproducible_rng() \
-  testing_disable_rng_override()
+void testing_disable_reproducible_rng(void);
 #define testing_disable_deterministic_rng() \
   testing_disable_rng_override()
 #define testing_disable_prefilled_rng() \

src/test/test_circuitpadding.c

@@ -36,6 +36,8 @@
 #include "core/or/or_circuit_st.h"
 #include "core/or/origin_circuit_st.h"
 
+#include "test/rng_test_helpers.h"
+
 /* Start our monotime mocking at 1 second past whatever monotime_init()
  * thought the actual wall clock time was, for platforms with bad resolution
  * and weird timevalues during monotime_init() before mocking. */
@@ -313,6 +315,7 @@ test_circuitpadding_rtt(void *arg)
 
   MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
   MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+  testing_enable_reproducible_rng();
 
   dummy_channel.cmux = circuitmux_alloc();
   relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
@@ -416,6 +419,7 @@ test_circuitpadding_rtt(void *arg)
   UNMOCK(circuit_package_relay_cell);
   UNMOCK(circuitmux_attach_circuit);
   tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
 
   return;
 }
@@ -540,6 +544,7 @@ test_circuitpadding_token_removal_higher(void *arg)
   /* Mock it up */
   MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
   MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
 
   /* Setup test environment (time etc.) */
   client_side = TO_CIRCUIT(origin_circuit_new());
@@ -633,6 +638,7 @@ test_circuitpadding_token_removal_higher(void *arg)
   free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
   monotime_disable_test_mocking();
   tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
 }
 
 /** Test lower token removal strategy by bin  */
@@ -645,6 +651,7 @@ test_circuitpadding_token_removal_lower(void *arg)
   /* Mock it up */
   MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
   MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
 
   /* Setup test environment (time etc.) */
   client_side = TO_CIRCUIT(origin_circuit_new());
@@ -731,6 +738,7 @@ test_circuitpadding_token_removal_lower(void *arg)
   free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
   monotime_disable_test_mocking();
   tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
 }
 
 /** Test closest token removal strategy by bin  */
@@ -743,6 +751,7 @@ test_circuitpadding_closest_token_removal(void *arg)
   /* Mock it up */
   MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
   MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
 
   /* Setup test environment (time etc.) */
   client_side = TO_CIRCUIT(origin_circuit_new());
@@ -837,6 +846,7 @@ test_circuitpadding_closest_token_removal(void *arg)
   free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
   monotime_disable_test_mocking();
   tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
 }
 
 /** Test closest token removal strategy with usec  */
@@ -849,6 +859,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg)
   /* Mock it up */
   MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
   MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
 
   /* Setup test environment (time etc.) */
   client_side = TO_CIRCUIT(origin_circuit_new());
@@ -948,6 +959,7 @@ test_circuitpadding_closest_token_removal_usec(void *arg)
   free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
   monotime_disable_test_mocking();
   tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
 }
 
 /** Test closest token removal strategy with usec  */
@@ -960,6 +972,7 @@ test_circuitpadding_token_removal_exact(void *arg)
   /* Mock it up */
   MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
   MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
 
   /* Setup test environment (time etc.) */
   client_side = TO_CIRCUIT(origin_circuit_new());
@@ -1007,6 +1020,7 @@ test_circuitpadding_token_removal_exact(void *arg)
   free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
   monotime_disable_test_mocking();
   tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
 }
 
 #undef BIG_HISTOGRAM_LEN
@@ -1019,6 +1033,8 @@ test_circuitpadding_tokens(void *arg)
   int64_t actual_mocked_monotime_start;
   (void)arg;
 
+  testing_enable_reproducible_rng();
+
   /** Test plan:
    *
    * 1. Test symmetry between bin_to_usec and usec_to_bin
@@ -1272,6 +1288,7 @@ test_circuitpadding_tokens(void *arg)
   free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
   monotime_disable_test_mocking();
   tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
 }
 
 void
@@ -1299,6 +1316,7 @@ test_circuitpadding_wronghop(void *arg)
   /* Mock this function so that our cell counting tests don't get confused by
    * padding that gets sent by scheduled timers. */
   MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
 
   client_side = TO_CIRCUIT(origin_circuit_new());
   dummy_channel.cmux = circuitmux_alloc();
@@ -1472,6 +1490,7 @@ test_circuitpadding_wronghop(void *arg)
   UNMOCK(circuit_package_relay_cell);
   UNMOCK(circuitmux_attach_circuit);
   nodes_free();
+  testing_disable_reproducible_rng();
 }
 
 void
@@ -1952,6 +1971,7 @@ test_circuitpadding_conditions(void *arg)
   int64_t actual_mocked_monotime_start;
   (void)arg;
   MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  testing_enable_reproducible_rng();
 
   nodes_init();
   dummy_channel.cmux = circuitmux_alloc();
@@ -2056,6 +2076,7 @@ test_circuitpadding_conditions(void *arg)
 
  done:
   /* XXX: Free everything */
+  testing_disable_reproducible_rng();
   return;
 }
 
@@ -2385,6 +2406,7 @@ test_circuitpadding_sample_distribution(void *arg)
   /* mock this function so that we dont actually schedule any padding */
   MOCK(circpad_machine_schedule_padding,
        circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
 
   /* Initialize a machine with multiple probability distributions */
   circpad_machines_init();
@@ -2417,6 +2439,7 @@ test_circuitpadding_sample_distribution(void *arg)
  done:
   free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
   UNMOCK(circpad_machine_schedule_padding);
+  testing_disable_reproducible_rng();
 }
 
 static circpad_decision_t
@@ -2442,6 +2465,7 @@ test_circuitpadding_machine_rate_limiting(void *arg)
    * really care about padding counts */
   MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock);
   MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+  testing_enable_reproducible_rng();
 
   /* Setup machine and circuits */
   client_side = TO_CIRCUIT(origin_circuit_new());
@@ -2495,6 +2519,7 @@ test_circuitpadding_machine_rate_limiting(void *arg)
 
  done:
   free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  testing_disable_reproducible_rng();
 }
 
 /* Test global padding rate limits */
@@ -2514,6 +2539,7 @@ test_circuitpadding_global_rate_limiting(void *arg)
   MOCK(circuit_package_relay_cell,
        circuit_package_relay_cell_mock);
   MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  testing_enable_reproducible_rng();
 
   monotime_init();
   monotime_enable_test_mocking();
@@ -2593,6 +2619,7 @@ test_circuitpadding_global_rate_limiting(void *arg)
   circuitmux_free(dummy_channel.cmux);
   SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
   smartlist_free(vote1.net_params);
+  testing_disable_reproducible_rng();
 }
 
 /* Test reduced and disabled padding */
@@ -2603,6 +2630,7 @@ test_circuitpadding_reduce_disable(void *arg)
   int64_t actual_mocked_monotime_start;
 
   MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  testing_enable_reproducible_rng();
 
   nodes_init();
   dummy_channel.cmux = circuitmux_alloc();
@@ -2742,6 +2770,7 @@ test_circuitpadding_reduce_disable(void *arg)
   free_fake_orcirc(relay_side);
   circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
   circuitmux_free(dummy_channel.cmux);
+  testing_disable_reproducible_rng();
 }
 
 /** Just a basic machine whose whole purpose is to reach the END state */

src/test/test_prob_distr.c

@@ -1119,13 +1119,14 @@ test_psi_dist_sample(const struct dist *dist)
 }
 
 static void
-dump_seed(void)
+write_stochastic_warning(void)
 {
+  if (tinytest_cur_test_has_failed()) {
     printf("\n"
          "NOTE: This is a stochastic test, and we expect it to fail from\n"
          "time to time, with some low probability. If you see it fail more\n"
          "than one trial in 100, though, please tell us.\n\n");
-  testing_dump_reproducible_rng_seed();
+  }
 }
 
 static void
@@ -1180,7 +1181,7 @@ test_stochastic_uniform(void *arg)
 
  done:
   if (tests_failed) {
-    dump_seed();
+    write_stochastic_warning();
   }
   testing_disable_reproducible_rng();
 }
@@ -1273,7 +1274,7 @@ test_stochastic_genpareto(void *arg)
 
  done:
   if (tests_failed) {
-    dump_seed();
+    write_stochastic_warning();
   }
   testing_disable_reproducible_rng();
 }
@@ -1301,7 +1302,7 @@ test_stochastic_geometric(void *arg)
 
  done:
   if (tests_failed) {
-    dump_seed();
+    write_stochastic_warning();
   }
   testing_disable_reproducible_rng();
 }
@@ -1328,7 +1329,7 @@ test_stochastic_logistic(void *arg)
 
  done:
   if (tests_failed) {
-    dump_seed();
+    write_stochastic_warning();
   }
   testing_disable_reproducible_rng();
 }
@@ -1337,7 +1338,6 @@ static void
 test_stochastic_log_logistic(void *arg)
 {
   bool ok = 0;
-  bool tests_failed = true;
   (void) arg;
 
   testing_enable_reproducible_rng();
@@ -1351,12 +1351,8 @@ test_stochastic_log_logistic(void *arg)
   ok = test_stochastic_log_logistic_impl(exp(-10), 1e-2);
   tt_assert(ok);
 
-  tests_failed = false;
-
  done:
-  if (tests_failed) {
-    dump_seed();
-  }
+  write_stochastic_warning();
   testing_disable_reproducible_rng();
 }
 
@@ -1364,7 +1360,6 @@ static void
 test_stochastic_weibull(void *arg)
 {
   bool ok = 0;
-  bool tests_failed = true;
   (void) arg;
 
   testing_enable_reproducible_rng();
@@ -1380,12 +1375,8 @@ test_stochastic_weibull(void *arg)
   ok = test_stochastic_weibull_impl(10, 1);
   tt_assert(ok);
 
-  tests_failed = false;
-
  done:
-  if (tests_failed) {
-    dump_seed();
-  }
+  write_stochastic_warning();
   testing_disable_reproducible_rng();
   UNMOCK(crypto_rand);
 }