Merge pull request #7620

fe63256 cryptonote_protocol_handler: fix race condition (anon)
ded2f3a cryptonote_protocol_handler: add race condition demo (anon)

src/cryptonote_basic/connection_context.h

@@ -77,6 +77,8 @@ namespace cryptonote
     int m_expect_response;
     uint64_t m_expect_height;
     size_t m_num_requested;
+    epee::copyable_atomic m_new_stripe_notification{0};
+    epee::copyable_atomic m_idle_peer_notification{0};
   };
 
   inline std::string get_protocol_state_string(cryptonote_connection_context::state s)

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -137,6 +137,41 @@ namespace cryptonote
     CHECK_AND_ASSERT_MES_CC( context.m_callback_request_count > 0, false, "false callback fired, but context.m_callback_request_count=" << context.m_callback_request_count);
     --context.m_callback_request_count;
 
+    uint32_t notified = true;
+    if (context.m_idle_peer_notification.compare_exchange_strong(notified, not notified))
+    {
+      if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time)
+      {
+        const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time();
+        const boost::posix_time::time_duration dt = now - context.m_last_request_time;
+        const auto ms = dt.total_microseconds();
+        if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME))
+        {
+          if (context.m_score-- >= 0)
+          {
+            MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response);
+            context.m_last_request_time = boost::date_time::not_a_date_time;
+            context.m_expect_response = 0;
+            context.m_expect_height = 0;
+            context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download
+          }
+          else
+          {
+            MINFO(context << "dropping idle peer with negative score");
+            drop_connection_with_score(context, context.m_expect_response == 0 ? 1 : 5, false);
+            return false;
+          }
+        }
+      }
+    }
+
+    notified = true;
+    if (context.m_new_stripe_notification.compare_exchange_strong(notified, not notified))
+    {
+      if (context.m_state == cryptonote_connection_context::state_normal)
+        context.m_state = cryptonote_connection_context::state_synchronizing;
+    }
+
     if(context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time == boost::posix_time::not_a_date_time)
     {
       NOTIFY_REQUEST_CHAIN::request r = {};
@@ -1687,7 +1722,7 @@ skip:
         const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed);
         if (stripe && peer_stripe && peer_stripe != stripe)
           return true;
-        context.m_state = cryptonote_connection_context::state_synchronizing;
+        context.m_new_stripe_notification = true;
         LOG_PRINT_CCONTEXT_L2("requesting callback");
         ++context.m_callback_request_count;
         m_p2p->request_callback(context);
@@ -1710,7 +1745,6 @@ skip:
   bool t_cryptonote_protocol_handler<t_core>::kick_idle_peers()
   {
     MTRACE("Checking for idle peers...");
-    std::vector<std::pair<boost::uuids::uuid, unsigned>> idle_peers;
     m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool
     {
       if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time)
@@ -1720,36 +1754,16 @@ skip:
         const auto ms = dt.total_microseconds();
         if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME))
         {
-          if (context.m_score-- >= 0)
+          context.m_idle_peer_notification = true;
-          {
+          LOG_PRINT_CCONTEXT_L2("requesting callback");
-            MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response);
+          ++context.m_callback_request_count;
-            LOG_PRINT_CCONTEXT_L2("requesting callback");
+          m_p2p->request_callback(context);
-            context.m_last_request_time = boost::date_time::not_a_date_time;
+          MLOG_PEER_STATE("requesting callback");
-            context.m_expect_response = 0;
-            context.m_expect_height = 0;
-            context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download
-            ++context.m_callback_request_count;
-            m_p2p->request_callback(context);
-          }
-          else
-          {
-            idle_peers.push_back(std::make_pair(context.m_connection_id, context.m_expect_response == 0 ? 1 : 5));
-          }
         }
       }
       return true;
     });
 
-    for (const auto &e: idle_peers)
-    {
-      const auto &uuid = e.first;
-      m_p2p->for_connection(uuid, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{
-        MINFO(ctx << "dropping idle peer with negative score");
-        drop_connection_with_score(ctx, e.second, false);
-        return true;
-      });
-    }
-
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------

tests/unit_tests/node_server.cpp

@@ -310,5 +310,597 @@ TEST(node_server, bind_same_p2p_port)
   EXPECT_TRUE(init(new_node(), port_another));
 }
 
+TEST(cryptonote_protocol_handler, race_condition)
+{
+  struct contexts {
+    using basic = epee::net_utils::connection_context_base;
+    using cryptonote = cryptonote::cryptonote_connection_context;
+    using p2p = nodetool::p2p_connection_context_t<cryptonote>;
+  };
+  using context_t = contexts::p2p;
+  using handler_t = epee::levin::async_protocol_handler<context_t>;
+  using connection_t = epee::net_utils::connection<handler_t>;
+  using connection_ptr = boost::shared_ptr<connection_t>;
+  using connections_t = std::vector<connection_ptr>;
+  using shared_state_t = typename connection_t::shared_state;
+  using shared_state_ptr = std::shared_ptr<shared_state_t>;
+  using io_context_t = boost::asio::io_service;
+  using event_t = epee::simple_event;
+  using ec_t = boost::system::error_code;
+  auto create_conn_pair = [](connection_ptr in, connection_ptr out) {
+    using endpoint_t = boost::asio::ip::tcp::endpoint;
+    using acceptor_t = boost::asio::ip::tcp::acceptor;
+    io_context_t io_context;
+    endpoint_t endpoint(boost::asio::ip::address::from_string("127.0.0.1"), 5262);
+    acceptor_t acceptor(io_context);
+    ec_t ec;
+    acceptor.open(endpoint.protocol(), ec);
+    EXPECT_EQ(ec.value(), 0);
+    acceptor.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true));
+    acceptor.bind(endpoint, ec);
+    EXPECT_EQ(ec.value(), 0);
+    acceptor.listen(boost::asio::socket_base::max_listen_connections, ec);
+    EXPECT_EQ(ec.value(), 0);
+    out->socket().open(endpoint.protocol(), ec);
+    EXPECT_EQ(ec.value(), 0);
+    acceptor.async_accept(in->socket(), [](const ec_t &ec){});
+    out->socket().async_connect(endpoint, [](const ec_t &ec){});
+    io_context.run();
+    acceptor.close(ec);
+    EXPECT_EQ(ec.value(), 0);
+    EXPECT_TRUE(in->start(true, true));
+    EXPECT_TRUE(out->start(false, true));
+    return std::make_pair<>(std::move(in), std::move(out));
+  };
+  auto get_conn_tag = [](connection_t &conn){
+    context_t context;
+    conn.get_context(context);
+    return context.m_connection_id;
+  };
+  using work_t = boost::asio::io_service::work;
+  using work_ptr = std::shared_ptr<work_t>;
+  using workers_t = std::vector<std::thread>;
+  using commands_handler_t = epee::levin::levin_commands_handler<context_t>;
+  using p2p_endpoint_t = nodetool::i_p2p_endpoint<contexts::cryptonote>;
+  using core_t = cryptonote::core;
+  using core_ptr = std::unique_ptr<core_t>;
+  using core_protocol_t = cryptonote::t_cryptonote_protocol_handler<core_t>;
+  using core_protocol_ptr = std::shared_ptr<core_protocol_t>;
+  using block_t = cryptonote::block;
+  using diff_t = cryptonote::difficulty_type;
+  using reward_t = uint64_t;
+  using height_t = uint64_t;
+  struct span {
+    using blocks = epee::span<const block_t>;
+  };
+  auto get_block_template = [](
+    core_t &core,
+    block_t &block,
+    diff_t &diff,
+    reward_t &reward
+  ){
+    auto &storage = core.get_blockchain_storage();
+    const auto height = storage.get_current_blockchain_height();
+    const auto hardfork = storage.get_current_hard_fork_version();
+    block.major_version = hardfork;
+    block.minor_version = storage.get_ideal_hard_fork_version();
+    block.prev_id = storage.get_tail_id();
+    auto &db = storage.get_db();
+    block.timestamp = db.get_top_block_timestamp();
+    block.nonce = 0xACAB;
+    block.miner_tx.vin.clear();
+    block.miner_tx.vout.clear();
+    block.miner_tx.extra.clear();
+    block.miner_tx.version = hardfork >= 4 ? 2 : 1;
+    block.miner_tx.unlock_time = height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW;
+    block.miner_tx.vin.push_back(cryptonote::txin_gen{height});
+    cryptonote::add_tx_pub_key_to_extra(block.miner_tx, {});
+    cryptonote::get_block_reward(
+      db.get_block_weight(height - 1),
+      {},
+      db.get_block_already_generated_coins(height - 1),
+      reward,
+      hardfork
+    );
+    block.miner_tx.vout.push_back(cryptonote::tx_out{reward, cryptonote::txout_to_key{}});
+    diff = storage.get_difficulty_for_next_block();
+  };
+  struct stat {
+    struct chain {
+      diff_t diff;
+      reward_t reward;
+    };
+  };
+  auto add_block = [](
+    core_t &core,
+    const block_t &block,
+    const stat::chain &stat
+  ){
+    core.get_blockchain_storage().get_db().batch_start({}, {});
+    core.get_blockchain_storage().get_db().add_block(
+      {block, cryptonote::block_to_blob(block)},
+      cryptonote::get_transaction_weight(block.miner_tx),
+      core.get_blockchain_storage().get_next_long_term_block_weight(
+        cryptonote::get_transaction_weight(block.miner_tx)
+      ),
+      stat.diff,
+      stat.reward,
+      {}
+    );
+    core.get_blockchain_storage().get_db().batch_stop();
+  };
+  struct messages {
+    struct core {
+      using sync = cryptonote::CORE_SYNC_DATA;
+    };
+    using handshake = nodetool::COMMAND_HANDSHAKE_T<core::sync>;
+  };
+  struct net_node_t: commands_handler_t, p2p_endpoint_t {
+    using span_t = epee::span<const uint8_t>;
+    using string_t = std::string;
+    using zone_t = epee::net_utils::zone;
+    using uuid_t = boost::uuids::uuid;
+    using relay_t = cryptonote::relay_method;
+    using blobs_t = std::vector<cryptonote::blobdata>;
+    using id_t = nodetool::peerid_type;
+    using callback_t = std::function<bool(contexts::cryptonote &, id_t, uint32_t)>;
+    using address_t = epee::net_utils::network_address;
+    using connections_t = std::vector<std::pair<zone_t, uuid_t>>;
+    struct bans {
+      using subnets = std::map<epee::net_utils::ipv4_network_subnet, time_t>;
+      using hosts = std::map<std::string, time_t>;
+    };
+    struct slice {
+      using bytes = epee::byte_slice;
+    };
+    shared_state_ptr shared_state;
+    core_protocol_ptr core_protocol;
+    virtual int invoke(int command, const span_t in, slice::bytes &out, context_t &context) override {
+      if (core_protocol) {
+        if (command == messages::handshake::ID) {
+          return epee::net_utils::buff_to_t_adapter<void, typename messages::handshake::request, typename messages::handshake::response>(
+            command,
+            in,
+            out,
+            [this](int command, typename messages::handshake::request &in, typename messages::handshake::response &out, context_t &context){
+              core_protocol->process_payload_sync_data(in.payload_data, context, true);
+              core_protocol->get_payload_sync_data(out.payload_data);
+              return 1;
+            },
+            context
+          );
+        }
+        bool handled;
+        return core_protocol->handle_invoke_map(false, command, in, out, context, handled);
+      }
+      else
+        return {};
+    }
+    virtual int notify(int command, const span_t in, context_t &context) override {
+      if (core_protocol) {
+        bool handled;
+        slice::bytes out;
+        return core_protocol->handle_invoke_map(true, command, in, out, context, handled);
+      }
+      else
+        return {};
+    }
+    virtual void callback(context_t &context) override {
+      if (core_protocol)
+        core_protocol->on_callback(context);
+    }
+    virtual void on_connection_new(context_t&) override {}
+    virtual void on_connection_close(context_t &context) override {
+      if (core_protocol)
+        core_protocol->on_connection_close(context);
+    }
+    virtual ~net_node_t() override {}
+    virtual bool add_host_fail(const address_t&, unsigned int = {}) override {
+      return {};
+    }
+    virtual bool block_host(address_t address, time_t = {}, bool = {}) override {
+      return {};
+    }
+    virtual bool drop_connection(const contexts::basic& context) override {
+      if (shared_state)
+        return shared_state->close(context.m_connection_id);
+      else
+        return {};
+    }
+    virtual bool for_connection(const uuid_t& uuid, callback_t f) override {
+      if (shared_state)
+        return shared_state->for_connection(uuid,[&f](context_t &context){
+          return f(context, context.peer_id, context.support_flags);
+        });
+      else
+        return {};
+    }
+    virtual bool invoke_command_to_peer(int command, const span_t in, string_t& out, const contexts::basic& context) override {
+      if (shared_state)
+        return shared_state->invoke(command, in, out, context.m_connection_id);
+      else
+        return {};
+    }
+    virtual bool invoke_notify_to_peer(int command, const span_t in, const contexts::basic& context) override {
+      if (shared_state)
+        return shared_state->notify(command, in, context.m_connection_id);
+      else
+        return {};
+    }
+    virtual bool relay_notify_to_list(int command, const span_t in, connections_t connections) override {
+      if (shared_state) {
+        for (auto &e: connections)
+          shared_state->notify(command, in, e.second);
+      }
+      return {};
+    }
+    virtual bool unblock_host(const address_t&) override {
+      return {};
+    }
+    virtual zone_t send_txs(blobs_t, const zone_t, const uuid_t&, relay_t) override {
+      return {};
+    }
+    virtual bans::subnets get_blocked_subnets() override {
+      return {};
+    }
+    virtual bans::hosts get_blocked_hosts() override {
+      return {};
+    }
+    virtual uint64_t get_public_connections_count() override {
+      if (shared_state)
+        return shared_state->get_connections_count();
+      else
+        return {};
+    }
+    virtual void add_used_stripe_peer(const contexts::cryptonote&) override {}
+    virtual void clear_used_stripe_peers() override {}
+    virtual void remove_used_stripe_peer(const contexts::cryptonote&) override {}
+    virtual void for_each_connection(callback_t f) override {
+      if (shared_state)
+        shared_state->foreach_connection([&f](context_t &context){
+          return f(context, context.peer_id, context.support_flags);
+        });
+    }
+    virtual void request_callback(const contexts::basic &context) override {
+      if (shared_state)
+        shared_state->request_callback(context.m_connection_id);
+    }
+  };
+  auto conduct_handshake = [get_conn_tag](net_node_t &net_node, connection_ptr conn){
+    event_t handshaked;
+    net_node.shared_state->for_connection(
+      get_conn_tag(*conn),
+      [&handshaked, &net_node](context_t &context){
+        typename messages::handshake::request msg;
+        net_node.core_protocol->get_payload_sync_data(msg.payload_data);
+        epee::net_utils::async_invoke_remote_command2<typename messages::handshake::response>(
+          context,
+          messages::handshake::ID,
+          msg,
+          *net_node.shared_state,
+          [&handshaked, &net_node](int code, const typename messages::handshake::response &msg, context_t &context){
+            EXPECT_TRUE(code >= 0);
+            net_node.core_protocol->process_payload_sync_data(msg.payload_data, context, true);
+            handshaked.raise();
+          },
+          P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT
+        );
+        return true;
+      }
+    );
+    handshaked.wait();
+  };
+  using path_t = boost::filesystem::path;
+  auto create_dir = []{
+    ec_t ec;
+    path_t path = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path("daemon-%%%%%%%%%%%%%%%%", ec);
+    if (ec)
+      return path_t{};
+    auto success = boost::filesystem::create_directory(path, ec);
+    if (not ec && success)
+      return path;
+    return path_t{};
+  };
+  auto remove_tree = [](const path_t &path){
+    ec_t ec;
+    boost::filesystem::remove_all(path, ec);
+  };
+  using options_t = boost::program_options::variables_map;
+  struct daemon_t {
+    options_t options;
+    core_ptr core;
+    core_protocol_ptr core_protocol;
+    net_node_t net_node;
+    shared_state_ptr shared_state;
+    connections_t conn;
+  };
+  struct daemons_t {
+    daemon_t main;
+    daemon_t alt;
+  };
+  using options_description_t = boost::program_options::options_description;
+
+  const auto dir = create_dir();
+  ASSERT_TRUE(not dir.empty());
+
+  daemons_t daemon{
+    {
+      [&dir]{
+        options_t options;
+        boost::program_options::store(
+          boost::program_options::command_line_parser({
+            "--data-dir",
+            (dir / "main").string(),
+            "--disable-dns-checkpoints",
+            "--check-updates=disabled",
+            "--fixed-difficulty=1",
+            "--block-sync-size=1",
+            "--db-sync-mode=fastest:async:50000",
+          }).options([]{
+            options_description_t options_description{};
+            cryptonote::core::init_options(options_description);
+            return options_description;
+          }()).run(),
+          options
+        );
+        return options;
+      }(),
+      {},
+      {},
+      {},
+      {},
+      {},
+    },
+    {
+      [&dir]{
+        options_t options;
+        boost::program_options::store(
+          boost::program_options::command_line_parser({
+            "--data-dir",
+            (dir / "alt").string(),
+            "--disable-dns-checkpoints",
+            "--check-updates=disabled",
+            "--fixed-difficulty=1",
+            "--block-sync-size=1",
+            "--db-sync-mode=fastest:async:50000",
+          }).options([]{
+            options_description_t options_description{};
+            cryptonote::core::init_options(options_description);
+            return options_description;
+          }()).run(),
+          options
+        );
+        return options;
+      }(),
+      {},
+      {},
+      {},
+      {},
+      {},
+    },
+  };
+
+  io_context_t io_context;
+  work_ptr work = std::make_shared<work_t>(io_context);
+  workers_t workers;
+  while (workers.size() < 4) {
+    workers.emplace_back([&io_context]{
+      io_context.run();
+    });
+  }
+
+  connection_t::set_rate_up_limit(std::numeric_limits<int64_t>::max());
+  connection_t::set_rate_down_limit(std::numeric_limits<int64_t>::max());
+
+  {
+    daemon.main.core = core_ptr(new core_t(nullptr));
+    daemon.main.core->init(daemon.main.options, nullptr, nullptr);
+    daemon.main.net_node.core_protocol = daemon.main.core_protocol = core_protocol_ptr(new core_protocol_t(
+      *daemon.main.core, &daemon.main.net_node, {}
+    ));
+    daemon.main.core->set_cryptonote_protocol(daemon.main.core_protocol.get());
+    daemon.main.core_protocol->init(daemon.main.options);
+    daemon.main.net_node.shared_state = daemon.main.shared_state = std::make_shared<shared_state_t>();
+    daemon.main.shared_state->set_handler(&daemon.main.net_node);
+    daemon.alt.shared_state = std::make_shared<shared_state_t>();
+    daemon.alt.shared_state->set_handler(&daemon.alt.net_node);
+
+    struct {
+      event_t prepare;
+      event_t check;
+      event_t finish;
+    } events;
+    auto connections = create_conn_pair(
+      connection_ptr(new connection_t(io_context, daemon.main.shared_state, {}, {})),
+      connection_ptr(new connection_t(io_context, daemon.alt.shared_state, {}, {}))
+    );
+    {
+      auto conn = connections.first;
+      auto shared_state = daemon.main.shared_state;
+      const auto tag = get_conn_tag(*conn);
+      conn->strand_.post([tag, conn, shared_state, &events]{
+        shared_state->for_connection(tag, [](context_t &context){
+          context.m_expect_height = -1;
+          context.m_expect_response = -1;
+          context.m_last_request_time = boost::date_time::min_date_time;
+          context.m_score = 0;
+          context.m_state = contexts::cryptonote::state_synchronizing;
+          return true;
+        });
+        events.prepare.raise();
+        events.check.wait();
+        shared_state->for_connection(tag, [](context_t &context){
+          EXPECT_TRUE(context.m_expect_height == -1);
+          EXPECT_TRUE(context.m_expect_response == -1);
+          EXPECT_TRUE(context.m_last_request_time == boost::date_time::min_date_time);
+          EXPECT_TRUE(context.m_score == 0);
+          EXPECT_TRUE(context.m_state == contexts::cryptonote::state_synchronizing);
+          return true;
+        });
+        events.finish.raise();
+      });
+    }
+    events.prepare.wait();
+    daemon.main.core_protocol->on_idle();
+    events.check.raise();
+    events.finish.wait();
+
+    connections.first->strand_.post([connections]{
+      connections.first->cancel();
+    });
+    connections.second->strand_.post([connections]{
+      connections.second->cancel();
+    });
+    connections.first.reset();
+    connections.second.reset();
+    while (daemon.main.shared_state->sock_count);
+    while (daemon.alt.shared_state->sock_count);
+    daemon.main.core_protocol->deinit();
+    daemon.main.core->stop();
+    daemon.main.core->deinit();
+    daemon.main.net_node.shared_state.reset();
+    daemon.main.shared_state.reset();
+    daemon.main.core_protocol.reset();
+    daemon.main.core.reset();
+    daemon.alt.shared_state.reset();
+  }
+
+  {
+    daemon.main.core = core_ptr(new core_t(nullptr));
+    daemon.main.core->init(daemon.main.options, nullptr, nullptr);
+    daemon.main.net_node.core_protocol = daemon.main.core_protocol = core_protocol_ptr(new core_protocol_t(
+      *daemon.main.core, &daemon.main.net_node, {}
+    ));
+    daemon.main.core->set_cryptonote_protocol(daemon.main.core_protocol.get());
+    daemon.main.core->set_checkpoints({});
+    daemon.main.core_protocol->init(daemon.main.options);
+    daemon.main.net_node.shared_state = daemon.main.shared_state = std::make_shared<shared_state_t>();
+    daemon.main.shared_state->set_handler(&daemon.main.net_node);
+    daemon.alt.core = core_ptr(new core_t(nullptr));
+    daemon.alt.core->init(daemon.alt.options, nullptr, nullptr);
+    daemon.alt.net_node.core_protocol = daemon.alt.core_protocol = core_protocol_ptr(new core_protocol_t(
+      *daemon.alt.core, &daemon.alt.net_node, {}
+    ));
+    daemon.alt.core->set_cryptonote_protocol(daemon.alt.core_protocol.get());
+    daemon.alt.core->set_checkpoints({});
+    daemon.alt.core_protocol->init(daemon.alt.options);
+    daemon.alt.net_node.shared_state = daemon.alt.shared_state = std::make_shared<shared_state_t>();
+    daemon.alt.shared_state->set_handler(&daemon.alt.net_node);
+
+    struct {
+      io_context_t io_context;
+      work_ptr work;
+      workers_t workers;
+    } check;
+    check.work = std::make_shared<work_t>(check.io_context);
+    check.workers.emplace_back([&check]{
+      check.io_context.run();
+    });
+    while (daemon.main.conn.size() < 1) {
+      daemon.main.conn.emplace_back(new connection_t(check.io_context, daemon.main.shared_state, {}, {}));
+      daemon.alt.conn.emplace_back(new connection_t(io_context, daemon.alt.shared_state, {}, {}));
+      create_conn_pair(daemon.main.conn.back(), daemon.alt.conn.back());
+      conduct_handshake(daemon.alt.net_node, daemon.alt.conn.back());
+    }
+    struct {
+      event_t prepare;
+      event_t sync;
+      event_t finish;
+    } events;
+    {
+      auto conn = daemon.main.conn.back();
+      auto shared_state = daemon.main.shared_state;
+      const auto tag = get_conn_tag(*conn);
+      conn->strand_.post([tag, conn, shared_state, &events]{
+        shared_state->for_connection(tag, [](context_t &context){
+          EXPECT_TRUE(context.m_state == contexts::cryptonote::state_normal);
+          return true;
+        });
+        events.prepare.raise();
+        events.sync.wait();
+        shared_state->for_connection(tag, [](context_t &context){
+          EXPECT_TRUE(context.m_state == contexts::cryptonote::state_normal);
+          return true;
+        });
+        events.finish.raise();
+      });
+    }
+    events.prepare.wait();
+    daemon.main.core->get_blockchain_storage().add_block_notify(
+      [&events](height_t height, span::blocks blocks){
+        if (height >= CRYPTONOTE_PRUNING_STRIPE_SIZE)
+          events.sync.raise();
+      }
+    );
+    {
+      stat::chain stat{
+        daemon.alt.core->get_blockchain_storage().get_db().get_block_cumulative_difficulty(
+          daemon.alt.core->get_current_blockchain_height() - 1
+        ),
+        daemon.alt.core->get_blockchain_storage().get_db().get_block_already_generated_coins(
+          daemon.alt.core->get_current_blockchain_height() - 1
+        ),
+      };
+      while (daemon.alt.core->get_current_blockchain_height() < CRYPTONOTE_PRUNING_STRIPE_SIZE + CRYPTONOTE_PRUNING_TIP_BLOCKS) {
+        block_t block;
+        diff_t diff;
+        reward_t reward;
+        get_block_template(*daemon.alt.core, block, diff, reward);
+        stat.diff += diff;
+        stat.reward = stat.reward < (MONEY_SUPPLY - stat.reward) ? stat.reward + reward : MONEY_SUPPLY;
+        add_block(*daemon.alt.core, block, stat);
+        if (daemon.main.core->get_current_blockchain_height() + 1 < CRYPTONOTE_PRUNING_STRIPE_SIZE)
+          add_block(*daemon.main.core, block, stat);
+      }
+    }
+    while (daemon.main.conn.size() < 2) {
+      daemon.main.conn.emplace_back(new connection_t(io_context, daemon.main.shared_state, {}, {}));
+      daemon.alt.conn.emplace_back(new connection_t(io_context, daemon.alt.shared_state, {}, {}));
+      create_conn_pair(daemon.main.conn.back(), daemon.alt.conn.back());
+      conduct_handshake(daemon.alt.net_node, daemon.alt.conn.back());
+    }
+    events.finish.wait();
+
+    for (;daemon.main.conn.size(); daemon.main.conn.pop_back()) {
+      auto conn = daemon.main.conn.back();
+      conn->strand_.post([conn]{
+        conn->cancel();
+      });
+    }
+    for (;daemon.alt.conn.size(); daemon.alt.conn.pop_back()) {
+      auto conn = daemon.alt.conn.back();
+      conn->strand_.post([conn]{
+        conn->cancel();
+      });
+    }
+    while (daemon.main.shared_state->sock_count);
+    while (daemon.alt.shared_state->sock_count);
+    daemon.main.core_protocol->deinit();
+    daemon.main.core->stop();
+    daemon.main.core->deinit();
+    daemon.main.net_node.shared_state.reset();
+    daemon.main.shared_state.reset();
+    daemon.main.core_protocol.reset();
+    daemon.main.core.reset();
+    daemon.alt.core_protocol->deinit();
+    daemon.alt.core->stop();
+    daemon.alt.core->deinit();
+    daemon.alt.net_node.shared_state.reset();
+    daemon.alt.shared_state.reset();
+    daemon.alt.core_protocol.reset();
+    daemon.alt.core.reset();
+    check.work.reset();
+    for (auto& w: check.workers) {
+      w.join();
+    }
+  }
+
+  work.reset();
+  for (auto& w: workers) {
+    w.join();
+  }
+  remove_tree(dir);
+}
+
 namespace nodetool { template class node_server<cryptonote::t_cryptonote_protocol_handler<test_core>>; }
 namespace cryptonote { template class t_cryptonote_protocol_handler<test_core>; }