From 7129343ea18493a1814563c55fd501f7f0231d11 Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 12 Dec 2023 16:13:56 -0500 Subject: [PATCH] some debugging for bootstrap and route purge --- veilid-core/src/network_manager/mod.rs | 2 +- veilid-core/src/routing_table/mod.rs | 3 +++ veilid-core/src/routing_table/route_spec_store/mod.rs | 4 ++++ veilid-core/src/routing_table/tasks/bootstrap.rs | 3 ++- veilid-core/src/routing_table/tasks/mod.rs | 2 +- .../src/routing_table/tasks/private_route_management.rs | 7 +++---- veilid-core/src/veilid_api/debug.rs | 2 +- veilid-python/tests/test_routing_context.py | 6 +++--- veilid-server/Cargo.toml | 1 + veilid-server/src/client_api.rs | 6 ++++++ 10 files changed, 25 insertions(+), 11 deletions(-) diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index 9c0f951a..9415512a 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -1096,7 +1096,7 @@ impl NetworkManager { Ok(true) } - pub fn debug_restart_network(&self) { + pub fn restart_network(&self) { self.net().restart_network(); } } diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 729f22c1..10574d80 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -38,6 +38,9 @@ pub use types::*; ////////////////////////////////////////////////////////////////////////// +/// How many nodes in our routing table we require for a functional PublicInternet RoutingDomain +pub const MIN_PUBLIC_INTERNET_ROUTING_DOMAIN_NODE_COUNT: usize = 4; + /// How frequently we tick the relay management routine pub const RELAY_MANAGEMENT_INTERVAL_SECS: u32 = 1; diff --git a/veilid-core/src/routing_table/route_spec_store/mod.rs b/veilid-core/src/routing_table/route_spec_store/mod.rs index 32fd3d79..81b58933 100644 --- a/veilid-core/src/routing_table/route_spec_store/mod.rs +++ b/veilid-core/src/routing_table/route_spec_store/mod.rs @@ -152,6 +152,10 @@ impl RouteSpecStore { /// Purge the route spec store pub async fn purge(&self) -> VeilidAPIResult<()> { + // Briefly pause routing table ticker while changes are made + let _tick_guard = self.unlocked_inner.routing_table.pause_tasks().await; + self.unlocked_inner.routing_table.cancel_tasks().await; + { let inner = &mut *self.inner.lock(); inner.content = Default::default(); diff --git a/veilid-core/src/routing_table/tasks/bootstrap.rs b/veilid-core/src/routing_table/tasks/bootstrap.rs index ceebe7a8..1cb9f43e 100644 --- a/veilid-core/src/routing_table/tasks/bootstrap.rs +++ b/veilid-core/src/routing_table/tasks/bootstrap.rs @@ -285,7 +285,8 @@ impl RoutingTable { { Ok(NodeContactMethod::Direct(v)) => v, Ok(v) => { - log_rtab!(warn "invalid contact method for bootstrap: {:?}", v); + log_rtab!(warn "invalid contact method for bootstrap, restarting network: {:?}", v); + routing_table.network_manager().restart_network(); return; } Err(e) => { diff --git a/veilid-core/src/routing_table/tasks/mod.rs b/veilid-core/src/routing_table/tasks/mod.rs index 0a599b26..a61e5194 100644 --- a/veilid-core/src/routing_table/tasks/mod.rs +++ b/veilid-core/src/routing_table/tasks/mod.rs @@ -157,7 +157,7 @@ impl RoutingTable { for ck in VALID_CRYPTO_KINDS { let eckey = (RoutingDomain::PublicInternet, ck); let cnt = entry_counts.get(&eckey).copied().unwrap_or_default(); - if cnt == 0 { + if cnt < MIN_PUBLIC_INTERNET_ROUTING_DOMAIN_NODE_COUNT { needs_bootstrap = true; } else if cnt < min_peer_count { needs_peer_minimum_refresh = true; diff --git a/veilid-core/src/routing_table/tasks/private_route_management.rs b/veilid-core/src/routing_table/tasks/private_route_management.rs index 467a7856..f0cf4420 100644 --- a/veilid-core/src/routing_table/tasks/private_route_management.rs +++ b/veilid-core/src/routing_table/tasks/private_route_management.rs @@ -2,7 +2,6 @@ use super::*; use futures_util::stream::{FuturesUnordered, StreamExt}; use futures_util::FutureExt; -use stop_token::future::FutureExt as StopFutureExt; const BACKGROUND_SAFETY_ROUTE_COUNT: usize = 2; @@ -103,10 +102,10 @@ impl RoutingTable { } /// Test set of routes and remove the ones that don't test clean - #[instrument(level = "trace", skip(self, stop_token), err)] + #[instrument(level = "trace", skip(self, _stop_token), err)] async fn test_route_set( &self, - stop_token: StopToken, + _stop_token: StopToken, routes_needing_testing: Vec, ) -> EyreResult<()> { if routes_needing_testing.is_empty() { @@ -158,7 +157,7 @@ impl RoutingTable { } // Wait for test_route futures to complete in parallel - while let Ok(Some(_)) = unord.next().timeout_at(stop_token.clone()).await {} + while unord.next().await.is_some() {} } // Process failed routes diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index e7a4f10a..0fc13136 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -809,7 +809,7 @@ impl VeilidAPI { } let netman = self.network_manager()?; - netman.debug_restart_network(); + netman.restart_network(); Ok("Network restarted".to_owned()) } else { diff --git a/veilid-python/tests/test_routing_context.py b/veilid-python/tests/test_routing_context.py index 3266e4bb..0dca6319 100644 --- a/veilid-python/tests/test_routing_context.py +++ b/veilid-python/tests/test_routing_context.py @@ -166,8 +166,8 @@ async def test_routing_context_app_message_loopback_big_packets(): # import it as a remote route as well so we can send to it prr = await api.import_remote_private_route(blob) - # do this test 1000 times - for _ in range(1000): + # do this test 100 times + for _ in range(100): # send a random sized random app message to our own private route message = random.randbytes(random.randint(0, 32768)) await rc.app_message(prr, message) @@ -230,7 +230,7 @@ async def test_routing_context_app_call_loopback_big_packets(): # import it as a remote route as well so we can send to it prr = await api.import_remote_private_route(blob) - + # do this test 10 times for _ in range(10): # send a random sized random app message to our own private route diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index 2dde4fc9..b983835f 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -35,6 +35,7 @@ rt-tokio = [ tracking = ["veilid-core/tracking"] network-result-extra = ["veilid-core/network-result-extra"] verbose-tracing = ["veilid-core/verbose-tracing"] +debug-json-api = [] [dependencies] veilid-core = { path = "../veilid-core", default-features = false } diff --git a/veilid-server/src/client_api.rs b/veilid-server/src/client_api.rs index 403cf957..4eb9c39e 100644 --- a/veilid-server/src/client_api.rs +++ b/veilid-server/src/client_api.rs @@ -217,6 +217,9 @@ impl ClientApi { // (trim all whitespace around input lines just to make things more permissive for API users) let request: json_api::Request = deserialize_json(&sanitized_line)?; + #[cfg(feature = "debug-json-api")] + debug!("JSONAPI: Request: {:?}", request); + // See if this is a control message or a veilid-core message let response = if let json_api::RequestOp::Control { args } = request.op { // Process control messages @@ -231,6 +234,9 @@ impl ClientApi { jrp.clone().process_request(request).await }; + #[cfg(feature = "debug-json-api")] + debug!("JSONAPI: Response: {:?}", response); + // Marshal json + newline => NDJSON let response_string = serialize_json(json_api::RecvMessage::Response(response)) + "\n"; if let Err(e) = responses_tx.send_async(response_string).await {