From e9a95f9cd4c18d25c41939361e0e4ef86aba9f48 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 15 Aug 2026 18:17:44 -0400 Subject: [PATCH] Bugfixes for 0.5.7 --- CHANGELOG.md | 29 +- scripts/ios_build.sh | 4 +- veilid-core/public-api.txt | 4 + veilid-core/smoketest/Cargo.lock | 228 ++--- veilid-core/src/attachment_manager/mod.rs | 4 + .../src/network_manager/address_check.rs | 24 +- .../bootstrap/direct_bootstrap/mod.rs | 22 +- .../bootstrap/direct_bootstrap/v0.rs | 8 +- .../bootstrap/direct_bootstrap/v1.rs | 8 +- .../src/network_manager/connection_handle.rs | 12 +- .../src/network_manager/connection_manager.rs | 29 +- veilid-core/src/network_manager/mod.rs | 64 +- .../network/address_family_confirm_logic.rs | 108 +++ .../src/network_manager/network/mod.rs | 13 + .../network/native/discovery_context.rs | 128 +-- .../network/native/low_level_protocol_tcp.rs | 6 +- .../network/native/low_level_protocol_udp.rs | 3 +- .../src/network_manager/network/native/mod.rs | 458 ++++++---- .../network/native/network_state.rs | 22 +- .../network/native/protocol/udp.rs | 166 ++-- .../network/native/tasks/confirm_dial_info.rs | 33 +- .../network/platform_network.rs | 14 +- .../network/wasm/address_family_confirm.rs | 138 +-- .../src/network_manager/network/wasm/mod.rs | 249 ++++-- .../src/network_manager/network_connection.rs | 3 + .../node_contact_method_cache.rs | 4 + .../src/network_manager/online_detector.rs | 394 ++++++++- veilid-core/src/network_manager/send_data.rs | 242 +++-- veilid-core/src/network_manager/stats.rs | 7 +- veilid-core/src/network_manager/tasks/mod.rs | 21 + .../tests_network_manager/mod.rs | 6 + .../test_address_family_confirm.rs | 148 ++++ .../test_connection_table.rs | 4 +- .../test_online_detector.rs | 167 ++++ .../test_transport_stats.rs | 216 +++++ .../src/network_manager/transport_stats.rs | 553 ++++++++++++ veilid-core/src/network_manager/types/flow.rs | 3 + .../types/node_contact_method_result.rs | 4 + .../network_manager/types/send_data_result.rs | 12 + .../network_manager/types/transport_type.rs | 26 + .../src/routing_table/bucket_entry/mod.rs | 99 ++- .../bucket_entry/state_reason.rs | 301 ++++--- veilid-core/src/routing_table/mod.rs | 46 +- .../src/routing_table/node_ref/traits.rs | 34 +- .../route_spec_store/route_test.rs | 7 +- .../routing_domains/local_network/editor.rs | 3 + .../controller/relay_management.rs | 132 ++- .../routing_domains/public_internet/detail.rs | 91 +- .../routing_domains/public_internet/editor.rs | 3 + .../routing_domain_detail_common.rs | 2 + .../routing_domains/routing_domain_relay.rs | 18 +- .../src/routing_table/routing_table_inner.rs | 14 +- veilid-core/src/routing_table/tasks/mod.rs | 21 + .../src/routing_table/tasks/ping_validator.rs | 80 +- .../tasks/update_cm_viability.rs | 141 +++ .../routing_table/tests_routing_table/mod.rs | 3 + .../test_bucket_entry_state.rs | 448 +++++++--- .../src/rpc_processor/fanout/fanout_call.rs | 137 ++- .../src/rpc_processor/fanout/fanout_queue.rs | 20 +- veilid-core/src/rpc_processor/mod.rs | 156 +++- veilid-core/src/rpc_processor/rpc_status.rs | 62 +- .../rpc_processor/rpc_validate_dial_info.rs | 25 +- .../src/rpc_processor/waitable_reply.rs | 23 +- veilid-core/src/storage_manager/constants.rs | 7 + veilid-core/src/storage_manager/get_value.rs | 3 + .../src/storage_manager/inspect_record.rs | 86 +- veilid-core/src/storage_manager/mod.rs | 14 + .../outbound_transaction_manager/mod.rs | 81 +- .../terminal_delivery.rs | 83 ++ .../outbound_watch_manager/mod.rs | 4 + .../outbound_watch_state.rs | 46 + .../src/storage_manager/record_encryption.rs | 8 + .../inbound_transactions_interface.rs | 9 + veilid-core/src/storage_manager/set_value.rs | 66 +- .../tasks/check_outbound_watches.rs | 8 + .../tests_storage_manager/mod.rs | 2 + .../tests_storage_manager/test_min_seqnum.rs | 134 +++ .../src/storage_manager/transaction.rs | 52 +- .../src/storage_manager/transaction_begin.rs | 117 ++- .../storage_manager/transaction_command.rs | 11 +- .../src/storage_manager/watch_value.rs | 197 +++-- veilid-core/src/tests/common/test_dht.rs | 5 + veilid-core/src/tests/mocks/mod.rs | 1 + veilid-core/src/veilid_api/api.rs | 9 + veilid-core/src/veilid_api/debug.rs | 267 +++++- .../src/veilid_api/tests_veilid_api/mod.rs | 6 + .../tests_veilid_api/test_value_seq_num.rs | 44 + .../dht/dht_transaction_set_value_options.rs | 7 + .../types/dht/set_dht_value_options.rs | 8 + .../src/veilid_api/types/dht/value_seq_num.rs | 11 +- veilid-core/src/veilid_config.rs | 4 + .../example/integration_test/app_test.dart | 392 ++++---- .../example/integration_test/test_crypto.dart | 3 +- .../example/integration_test/test_dht.dart | 55 ++ .../test_dht_transactions_full.dart | 6 +- .../test_protected_store.dart | 10 +- veilid-flutter/example/test/widget_test.dart | 3 +- veilid-flutter/hook/build.dart | 1 + veilid-flutter/lib/default_config.dart | 77 +- veilid-flutter/lib/processor_interface.dart | 10 +- veilid-flutter/lib/retry.dart | 77 +- veilid-flutter/lib/veilid_api_exception.dart | 32 +- veilid-flutter/lib/veilid_config.dart | 34 +- veilid-flutter/lib/veilid_config.freezed.dart | 43 +- veilid-flutter/lib/veilid_config.g.dart | 2 + .../lib/veilid_dht_transaction.dart | 12 +- .../lib/veilid_dht_transaction.freezed.dart | 43 +- .../lib/veilid_dht_transaction.g.dart | 6 +- veilid-flutter/lib/veilid_ffi.dart | 22 +- veilid-flutter/lib/veilid_js.dart | 49 +- .../lib/veilid_plugin_stub_web.dart | 2 +- veilid-flutter/lib/veilid_stub.dart | 2 +- veilid-flutter/lib/veilid_types.dart | 20 +- veilid-flutter/lib/veilid_types.freezed.dart | 43 +- veilid-flutter/lib/veilid_types.g.dart | 2 + .../lib/src/integration_log.dart | 145 ++- .../veilid_test/lib/src/log_fixture.dart | 3 +- .../lib/src/periodic_ticker_web.dart | 6 +- .../veilid_test/lib/src/unit_fixture.dart | 83 ++ .../lib/src/update_processor_fixture.dart | 3 +- .../packages/veilid_test/lib/veilid_test.dart | 1 + .../packages/veilid_test/pubspec.yaml | 7 + veilid-flutter/run_integration_tests.sh | 4 +- veilid-flutter/test/test_encoding.dart | 2 +- veilid-flutter/test/test_native_asset.dart | 4 +- .../test/test_value_subkey_range.dart | 24 +- veilid-python/tests/test_dht.py | 213 ++++- veilid-python/tests/test_dht_groups.py | 835 ++++++++++++++++++ veilid-python/tests/test_dht_transactions.py | 77 ++ veilid-python/veilid/api.py | 30 +- veilid-python/veilid/config.py | 1 + veilid-python/veilid/schema/RecvMessage.json | 7 + veilid-python/veilid/schema/Request.json | 31 + veilid-python/veilid/types.py | 30 +- veilid-remote-api/smoketest/Cargo.lock | 252 +++--- veilid-tools/Cargo.toml | 2 + veilid-tools/public-api.txt | 27 + veilid-tools/smoketest/Cargo.lock | 183 ++-- veilid-tools/src/lib.rs | 4 + veilid-tools/src/network_attempt.rs | 43 + veilid-tools/src/network_interfaces/mod.rs | 32 +- veilid-tools/src/resolver/non_windows/mod.rs | 114 ++- veilid-tools/src/socket_tools.rs | 150 ++++ veilid-tools/src/tests/native/mod.rs | 11 + .../src/tests/native/test_udp_ttl_send.rs | 63 ++ veilid-wasm/src/veilid_client_js.rs | 7 + veilid-wasm/src/wasm_helpers.rs | 19 +- veilid-wasm/tests/src/suites/dht.ts | 34 + 148 files changed, 7923 insertions(+), 2210 deletions(-) create mode 100644 veilid-core/src/network_manager/network/address_family_confirm_logic.rs create mode 100644 veilid-core/src/network_manager/tests_network_manager/test_address_family_confirm.rs create mode 100644 veilid-core/src/network_manager/tests_network_manager/test_online_detector.rs create mode 100644 veilid-core/src/network_manager/tests_network_manager/test_transport_stats.rs create mode 100644 veilid-core/src/network_manager/transport_stats.rs create mode 100644 veilid-core/src/routing_table/tasks/update_cm_viability.rs create mode 100644 veilid-core/src/storage_manager/outbound_transaction_manager/terminal_delivery.rs create mode 100644 veilid-core/src/storage_manager/tests_storage_manager/test_min_seqnum.rs create mode 100644 veilid-core/src/veilid_api/tests_veilid_api/test_value_seq_num.rs create mode 100644 veilid-flutter/packages/veilid_test/lib/src/unit_fixture.dart create mode 100644 veilid-python/tests/test_dht_groups.py create mode 100644 veilid-tools/src/network_attempt.rs create mode 100644 veilid-tools/src/tests/native/test_udp_ttl_send.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e14f6734..6052d997 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,24 +2,35 @@ - veilid-core: - Stats collection, aggregation, and analysis for all subsystems. Robot-generated python script for reviewing the effect of optimizations. - - Fix OnlineDetector spurious flap - - Properly drop TCP connections when offline condition is detected so they can be recreated in the event of NAT state loss, as is common on cellular - - Fix UDP stalls on cellular/CLAT464 - - Correct timeout for native WS connects - - Timeout-bound public internet dial info detection - - AIMD experiment for DHT operations. Off by default for now behind `aimd-dht` feature - - Fix background transaction pre-empt on foreground record open + - Fix OnlineDetector spurious flap. + - Properly drop TCP connections when offline condition is detected so they can be recreated in the event of NAT state loss, as is common on cellular. + - Fix UDP stalls on cellular/CLAT464. + - Correct timeout for native WS connects. + - Timeout-bound public internet dial info detection. + - AIMD experiment for DHT operations. Off by default for now behind `aimd-dht` feature. + - Fix background transaction pre-empt on foreground record open. - ProtectedStore is now exported correctly through the VeilidAPI object and the old reust interface is removed. Switch to accessing ProtectedStore through VeilidAPI. + - Loosen IPv6 detection for environments where the default route might not be enumerable due to sandboxing. + - Fix `inspect_dht_record` excessive search duration. + - `set_dht_value` and `DHTTransaction::set` options now expose `min_seqnum`, which raises the written sequence number after the automatic increment. `ValueSeqNum::MAX` minus `n` leaves `n` writes on the subkey. `DHTReportScope::UpdateSet` no longer fails on a subkey at the maximum sequence number. + - Eliminate locks from holepunching, using best-effort TTL instead. + - Fix per-transport statistical suppression so broken protocols stop being selected constantly. + - Backoff on 'outbound-only' address change detection frequency. + - Rejected/disqualified nodes do not contribute to inspect_dht_record consensus width cap. + - Transaction post-consensus grace period reduced to avoid waiting for unnecessary slow additional-consensus nodes. + - OnlineDetector now tracks local stalls, and local degradation is no longer blamed on specific nodes. + - Better watch reporting coalescing. - veilid-flutter: - - Full conversion to Native Assets and rust build hooks. Your apps will need to remove any extra build steps for building veilid-flutter and linking to the output, it is fully handled by flutter now. - - CocoaPods support removed + - Full conversion to Native Assets and rust build hooks. Apps must drop any extra build steps for building veilid-flutter and linking its output; Flutter handles it. + - CocoaPods support removed. - veilid-python: - More tests - veilid-tools: - AIMD congestion control system (behind `aimd` feature) + - socket_tools UDP best-effort send-with-ttl implementation **Changed in Veilid 0.5.7** diff --git a/scripts/ios_build.sh b/scripts/ios_build.sh index af196090..67cbf363 100755 --- a/scripts/ios_build.sh +++ b/scripts/ios_build.sh @@ -68,8 +68,8 @@ do HOMEBREW_DIR=$(dirname `which brew`) fi - # staticlib alone: built alongside cdylib, LTO internalizes the #[no_mangle] FFI exports out of the archive - # (rustc needs one package, so -p selects it out of the virtual workspace manifest; crate name uses dashes) + # Build staticlib alone: alongside cdylib, LTO internalizes the #[no_mangle] FFI exports out of the archive + # -p picks the one package rustc needs out of the virtual workspace manifest, by its dashed crate name env -i PATH=/usr/bin:/bin:$HOMEBREW_DIR:$CARGO_DIR HOME="$HOME" USER="$USER" IPHONEOS_DEPLOYMENT_TARGET="$IPHONEOS_DEPLOYMENT_TARGET" ${RUST_ENV[@]+"${RUST_ENV[@]}"} cargo $CARGO_TOOLCHAIN rustc $EXTRA_CARGO_OPTIONS -p "${PACKAGE_NAME//_/-}" --lib --crate-type staticlib --target $CARGO_TARGET --manifest-path $CARGO_MANIFEST_PATH LIPOS="$LIPOS $TARGET_PATH/$CARGO_TARGET/$BUILD_MODE/lib$PACKAGE_NAME.a" diff --git a/veilid-core/public-api.txt b/veilid-core/public-api.txt index 9d540286..116420da 100644 --- a/veilid-core/public-api.txt +++ b/veilid-core/public-api.txt @@ -1326,6 +1326,7 @@ pub async fn veilid_core::DHTTransaction::set(&self, veilid_core::RecordKey, vei impl core::fmt::Debug for veilid_core::DHTTransaction pub fn veilid_core::DHTTransaction::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct veilid_core::DHTTransactionSetValueOptions +pub veilid_core::DHTTransactionSetValueOptions::min_seqnum: core::option::Option pub veilid_core::DHTTransactionSetValueOptions::writer: core::option::Option pub struct veilid_core::DecapsulationKey impl veilid_core::DecapsulationKey @@ -2562,6 +2563,7 @@ pub fn veilid_core::SecretKeyGroup::get_heap_size(&self) -> usize pub fn veilid_core::SecretKeyGroup::get_heap_size_with_tracker(&self, TRACKER) -> (usize, TRACKER) pub struct veilid_core::SetDHTValueOptions pub veilid_core::SetDHTValueOptions::allow_offline: core::option::Option +pub veilid_core::SetDHTValueOptions::min_seqnum: core::option::Option pub veilid_core::SetDHTValueOptions::writer: core::option::Option impl core::default::Default for veilid_core::SetDHTValueOptions pub fn veilid_core::SetDHTValueOptions::default() -> Self @@ -2908,6 +2910,7 @@ pub const fn veilid_core::ValueSeqNum::is_max(&self) -> bool pub const fn veilid_core::ValueSeqNum::is_none(&self) -> bool pub const fn veilid_core::ValueSeqNum::is_some(&self) -> bool pub fn veilid_core::ValueSeqNum::next(&self) -> veilid_core::VeilidAPIResult +pub fn veilid_core::ValueSeqNum::saturating_next(&self) -> Self pub const fn veilid_core::ValueSeqNum::to_option(&self) -> core::option::Option impl core::convert::From> for veilid_core::ValueSeqNum pub fn veilid_core::ValueSeqNum::from(core::option::Option) -> Self @@ -3106,6 +3109,7 @@ pub veilid_core::VeilidConfigInternalNetwork::protocol: veilid_core::VeilidConfi pub veilid_core::VeilidConfigInternalNetwork::restricted_nat_retries: u32 pub veilid_core::VeilidConfigInternalNetwork::reverse_connection_receipt_time_ms: u32 pub veilid_core::VeilidConfigInternalNetwork::rpc: veilid_core::VeilidConfigInternalRPC +pub veilid_core::VeilidConfigInternalNetwork::single_send_timeout_ms: u32 impl core::default::Default for veilid_core::VeilidConfigInternalNetwork pub fn veilid_core::VeilidConfigInternalNetwork::default() -> Self pub struct veilid_core::VeilidConfigInternalProtocol diff --git a/veilid-core/smoketest/Cargo.lock b/veilid-core/smoketest/Cargo.lock index f8e12d22..44726453 100644 --- a/veilid-core/smoketest/Cargo.lock +++ b/veilid-core/smoketest/Cargo.lock @@ -41,9 +41,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -312,9 +312,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", @@ -446,9 +446,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" dependencies = [ "arrayref", "arrayvec", @@ -550,9 +550,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" [[package]] name = "capnp" @@ -589,9 +589,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", "shlex", @@ -684,9 +684,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -694,9 +694,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -946,9 +946,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "dbus" @@ -1200,10 +1200,11 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.12" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" dependencies = [ + "autocfg", "indenter", "once_cell", ] @@ -1237,9 +1238,9 @@ checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "flume" @@ -1297,9 +1298,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -1312,9 +1313,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -1322,15 +1323,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -1339,9 +1340,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-lite" @@ -1358,13 +1359,13 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -1380,15 +1381,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -1402,9 +1403,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -1616,7 +1617,7 @@ dependencies = [ "once_cell", "rand 0.9.5", "ring", - "thiserror 2.0.19", + "thiserror 2.0.20", "tinyvec", "tokio", "tracing", @@ -1639,7 +1640,7 @@ dependencies = [ "rand 0.9.5", "resolv-conf", "smallvec", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -1724,9 +1725,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -1738,9 +1739,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -1751,9 +1752,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1765,16 +1766,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -1785,15 +1787,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" dependencies = [ "displaydoc", "icu_locale_core", @@ -1883,9 +1885,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "is_terminal_polyfill" @@ -1920,7 +1922,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.19", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -2074,9 +2076,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.18" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" dependencies = [ "libc", ] @@ -2100,9 +2102,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -2176,9 +2178,9 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "minicov" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" +checksum = "c3aa3aa12b448ac225b3102217d1ac5cc717908f02722926524b0599c933c7a0" dependencies = [ "cc", "walkdir", @@ -2212,9 +2214,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.15" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -2229,9 +2231,9 @@ dependencies = [ [[package]] name = "netlink-packet-core" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3463cbb78394cb0141e2c926b93fc2197e473394b761986eca3b9da2c63ae0f4" +checksum = "b897d7bd4f0af82e68d40d0344cf37e97f9c97ddf74a098de3e4da05e96ca395" dependencies = [ "paste", ] @@ -2250,9 +2252,9 @@ dependencies = [ [[package]] name = "netlink-proto" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6f7398dddf5f152d2a91a2921a134c6097056e292c0d4b9906007855e7cece6" +checksum = "93af8261786086024cd5e96e0a991dd65ced07bbf7c233a487bbc96b971d5539" dependencies = [ "bytes", "futures-channel", @@ -2260,7 +2262,7 @@ dependencies = [ "log", "netlink-packet-core", "netlink-sys", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -2351,9 +2353,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -2623,9 +2625,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polling" @@ -2653,15 +2655,15 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -2838,7 +2840,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -2855,9 +2857,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -2897,7 +2899,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" dependencies = [ "hashbrown", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -2988,9 +2990,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" dependencies = [ "ring", "rustls-pki-types", @@ -3352,11 +3354,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -3372,9 +3374,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", @@ -3392,9 +3394,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.54" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", "num-conv", @@ -3422,9 +3424,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -3596,7 +3598,7 @@ dependencies = [ "log", "rand 0.9.5", "sha1", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -3659,9 +3661,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -3741,7 +3743,7 @@ dependencies = [ "serde_ignored", "serde_json", "sha2 0.11.0", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", "tracing-subscriber", @@ -3805,7 +3807,7 @@ dependencies = [ "serde", "shell-words", "socket2", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -4219,9 +4221,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "ws_stream_wasm" @@ -4236,7 +4238,7 @@ dependencies = [ "pharos", "rustc_version", "send_wrapper", - "thiserror 2.0.19", + "thiserror 2.0.20", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -4278,18 +4280,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", @@ -4339,9 +4341,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -4350,9 +4352,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" dependencies = [ "yoke", "zerofrom", @@ -4361,13 +4363,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "47402523226a02bfe5230160dc3ccc089aa6f6f19e7fcbb4e6f824bbb1b4aa62" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] diff --git a/veilid-core/src/attachment_manager/mod.rs b/veilid-core/src/attachment_manager/mod.rs index 4cf89249..1f9c1e7a 100644 --- a/veilid-core/src/attachment_manager/mod.rs +++ b/veilid-core/src/attachment_manager/mod.rs @@ -351,6 +351,10 @@ impl AttachmentManager { if let Some(lag) = last_tick_ts.map(|x| cur_tick_ts.duration_since(x)) { if lag > TimestampDuration::new_ms(2 * (TICK_INTERVAL_MSEC as u64)) { veilid_log!(self debug "tick lag: {}", lag); + // A stalled executor stalls every in-flight RPC; their timeouts are about us + self.network_manager() + .online_detector() + .record_local_stall(lag); } } diff --git a/veilid-core/src/network_manager/address_check.rs b/veilid-core/src/network_manager/address_check.rs index 1b79252f..e2da40fa 100644 --- a/veilid-core/src/network_manager/address_check.rs +++ b/veilid-core/src/network_manager/address_check.rs @@ -134,9 +134,10 @@ impl AddressCheck { // Clear our history as well now so we start fresh when we get a new peer info self.address_inconsistency_table.remove(&acck); - self.last_inbound_triggered_address.remove(&acck); self.address_consistency_table.remove(&acck); - self.last_outbound_triggered_address.remove(&acck); + + // Leave the last-triggered dedups alone: re-arming detection on every + // publish loops forever on a symmetric NAT's stable address } } @@ -244,6 +245,18 @@ impl AddressCheck { return; } + // sender_info is untrusted, and a report naming a different address family + // than its flow cannot describe our address, so counting it re-detects forever + if socket_address.address_type() != flow.address_type() { + veilid_log!(self debug target:"network_result", + "ignoring address report {} on {} flow from {}: address family mismatch", + socket_address, + flow.address_type(), + reporting_peer + ); + return; + } + // Get the ip(block) this report is coming from let reporting_ipblock = ip_to_ipblock(self.config.ip6_prefix_size, flow.remote_address().ip_addr()); @@ -281,6 +294,8 @@ impl AddressCheck { "{:?} address has changed, detecting dial info", routing_domain ); + // Our external address moved, so transport evidence gathered at the old one is stale + self.routing_table().clear_all_transport_evidence(); } } @@ -425,9 +440,8 @@ impl AddressCheck { .and_modify(|e| *e += 1) .or_insert(1); if count >= ADDRESS_CONSISTENCY_DETECTION_COUNT { - // Suppress repeated re-fires for the same stable address. Only fire when - // the consistent address actually changes, or after a peer info change - // resets state via report_peer_info_change. + // Fire once per distinct address: a symmetric NAT reports the same + // stable address forever, and detection must conclude exactly once if self.last_outbound_triggered_address.get(&acckey) == Some(&address) { return false; } diff --git a/veilid-core/src/network_manager/bootstrap/direct_bootstrap/mod.rs b/veilid-core/src/network_manager/bootstrap/direct_bootstrap/mod.rs index d2d3ff49..3d10b9bb 100644 --- a/veilid-core/src/network_manager/bootstrap/direct_bootstrap/mod.rs +++ b/veilid-core/src/network_manager/bootstrap/direct_bootstrap/mod.rs @@ -44,10 +44,15 @@ impl NetworkManager { // Send boot magic to requested peer address let data = Bytes::copy_from_slice(BOOT_MAGIC); - let out_data = network_result_value_or_log!(self self + // Our own transport is suppressed, so this says nothing about the bootstrap node + let NetworkAttempt::Performed(attempt) = self .net() - .send_recv_data_unbound_to_dial_info(dial_info, data, timeout_ms) - .await? => [ format!(": dial_info={}, data.len={}", dial_info, data.len()) ] + .send_recv_data_unbound_to_dial_info(dial_info.clone(), data.clone(), timeout_ms) + .await? + else { + return Ok(Vec::new()); + }; + let out_data = network_result_value_or_log!(self attempt => [ format!(": dial_info={}, data.len={}", dial_info, data.len()) ] { return Ok(Vec::new()); }); @@ -77,10 +82,15 @@ impl NetworkManager { // Send boot magic to requested peer address let data = Bytes::copy_from_slice(B01T_MAGIC); - let out_data = network_result_value_or_log!(self self + // Our own transport is suppressed, so this says nothing about the bootstrap node + let NetworkAttempt::Performed(attempt) = self .net() - .send_recv_data_unbound_to_dial_info(dial_info, data, timeout_ms) - .await? => [ format!(": dial_info={}, data.len={}", dial_info, data.len()) ] + .send_recv_data_unbound_to_dial_info(dial_info.clone(), data.clone(), timeout_ms) + .await? + else { + return Ok(Vec::new()); + }; + let out_data = network_result_value_or_log!(self attempt => [ format!(": dial_info={}, data.len={}", dial_info, data.len()) ] { return Ok(Vec::new()); }); diff --git a/veilid-core/src/network_manager/bootstrap/direct_bootstrap/v0.rs b/veilid-core/src/network_manager/bootstrap/direct_bootstrap/v0.rs index 05b891f5..81bc1caf 100644 --- a/veilid-core/src/network_manager/bootstrap/direct_bootstrap/v0.rs +++ b/veilid-core/src/network_manager/bootstrap/direct_bootstrap/v0.rs @@ -24,11 +24,9 @@ impl NetworkManager { // Reply with a chunk of signed routing table let net = self.net(); match pin_future_closure!(net.send_data_to_existing_flow(flow, json_bytes)).await? { - SendDataToExistingFlowResult::Sent(_) => { - // Bootstrap reply was sent - Ok(NetworkResult::value(())) - } - SendDataToExistingFlowResult::NotSent(_) => Ok(NetworkResult::no_connection_other( + NetworkAttempt::Performed(NetworkResult::Value(_)) => Ok(NetworkResult::value(())), + NetworkAttempt::Performed(NetworkResult::Timeout) => Ok(NetworkResult::timeout()), + _ => Ok(NetworkResult::no_connection_other( "bootstrap reply could not be sent", )), } diff --git a/veilid-core/src/network_manager/bootstrap/direct_bootstrap/v1.rs b/veilid-core/src/network_manager/bootstrap/direct_bootstrap/v1.rs index 7c5f3f42..59757ab7 100644 --- a/veilid-core/src/network_manager/bootstrap/direct_bootstrap/v1.rs +++ b/veilid-core/src/network_manager/bootstrap/direct_bootstrap/v1.rs @@ -101,11 +101,9 @@ impl NetworkManager { // Reply with bootstrap records let net = self.net(); match pin_future_closure!(net.send_data_to_existing_flow(flow, json_bytes)).await? { - SendDataToExistingFlowResult::Sent(_) => { - // Bootstrap reply was sent - Ok(NetworkResult::value(())) - } - SendDataToExistingFlowResult::NotSent(_) => Ok(NetworkResult::no_connection_other( + NetworkAttempt::Performed(NetworkResult::Value(_)) => Ok(NetworkResult::value(())), + NetworkAttempt::Performed(NetworkResult::Timeout) => Ok(NetworkResult::timeout()), + _ => Ok(NetworkResult::no_connection_other( "bootstrap reply could not be sent", )), } diff --git a/veilid-core/src/network_manager/connection_handle.rs b/veilid-core/src/network_manager/connection_handle.rs index ab68c6fc..f5804cd6 100644 --- a/veilid-core/src/network_manager/connection_handle.rs +++ b/veilid-core/src/network_manager/connection_handle.rs @@ -7,12 +7,13 @@ pub struct ConnectionHandle { flow: Flow, channel: flume::Sender<(Option, Bytes)>, dead: Arc, + dialed: bool, } #[derive(Debug)] pub enum ConnectionHandleSendResult { Sent, - NotSent(Bytes), + NotSent, } impl ConnectionHandle { @@ -21,12 +22,14 @@ impl ConnectionHandle { flow: Flow, channel: flume::Sender<(Option, Bytes)>, dead: Arc, + dialed: bool, ) -> Self { Self { connection_id, flow, channel, dead, + dialed, } } @@ -44,6 +47,7 @@ impl ConnectionHandle { UniqueFlow { flow: self.flow, connection_id: Some(self.connection_id), + dialed: self.dialed, } } @@ -51,7 +55,7 @@ impl ConnectionHandle { // pub fn send(&self, message: Bytes) -> ConnectionHandleSendResult { // match self.channel.send((Span::current().id(), message)) { // Ok(()) => ConnectionHandleSendResult::Sent, - // Err(e) => ConnectionHandleSendResult::NotSent(e.0 .1), + // Err(_) => ConnectionHandleSendResult::NotSent, // } // } @@ -59,7 +63,7 @@ impl ConnectionHandle { pub async fn send_async(&self, message: Bytes) -> ConnectionHandleSendResult { // A dead connection's processor no longer drains the channel; queueing would silently drop if self.dead.load(Ordering::Relaxed) { - return ConnectionHandleSendResult::NotSent(message); + return ConnectionHandleSendResult::NotSent; } match self .channel @@ -67,7 +71,7 @@ impl ConnectionHandle { .await { Ok(()) => ConnectionHandleSendResult::Sent, - Err(e) => ConnectionHandleSendResult::NotSent(e.0 .1), + Err(_) => ConnectionHandleSendResult::NotSent, } } } diff --git a/veilid-core/src/network_manager/connection_manager.rs b/veilid-core/src/network_manager/connection_manager.rs index 57351b54..7d2b7afb 100644 --- a/veilid-core/src/network_manager/connection_manager.rs +++ b/veilid-core/src/network_manager/connection_manager.rs @@ -189,7 +189,13 @@ impl ConnectionManager { Ok(()) } - pub fn commit_event_handler(&self, _evt: Arc) { + pub fn commit_event_handler(&self, evt: Arc) { + // We are somewhere else now, so our own transport send history no longer applies + if evt.dial_info_changed { + self.network_manager() + .transport_stats() + .reset_routing_domain(evt.routing_domain, "dial info changed"); + } self.update_protections(); } @@ -750,16 +756,14 @@ impl ConnectionManager { // In-use connections will already get reported because they will cause a 'lost_question' stat on the remote node if let Some(protect_nr) = conn.protected_node_ref() { // Lifetime: only counts remote-closed protected connections. - // Skip recording when we're offline ourselves so peer-side - // stats don't get poisoned by our own connectivity loss. + // Our own connectivity loss must not poison peer stats let cur_ts = Timestamp::now_non_decreasing(); let lifetime = cur_ts.duration_since(conn.established_time()); - let we_are_online = self + let can_blame_peer = !self .network_manager() .online_detector() - .online_state(RoutingDomain::PublicInternet) - == OnlineState::Online; - if we_are_online { + .blame_suppressed(RoutingDomain::PublicInternet); + if can_blame_peer { protect_nr.record_protected_connection_drop(lifetime); if let Some(penalty) = self .arc @@ -783,9 +787,12 @@ impl ConnectionManager { veilid_log!(self debug "== Protected connection dropped (count={}, lifetime={}): {} -> {} for node {}", pa.drops_in_span, lifetime, conn.connection_id(), conn.debug_print(Timestamp::now()), protect_nr); if pa.drops_in_span >= PROTECTED_CONNECTION_DROP_COUNT { - protect_nr.record_protected_connection_dead( - conn.flow().transport_type(), - ); + // While degraded the drops are ours: back off, but blame nobody + if can_blame_peer { + protect_nr.record_protected_connection_dead( + conn.flow().transport_type(), + ); + } reconnect = false; pa.drops_in_span = 0; @@ -823,7 +830,7 @@ impl ConnectionManager { .network_manager() .online_detector() .online_state(RoutingDomain::PublicInternet) - == OnlineState::Online + != OnlineState::Offline { veilid_log!(self debug "Connection table is empty, triggering dial info confirmation"); self.network_manager() diff --git a/veilid-core/src/network_manager/mod.rs b/veilid-core/src/network_manager/mod.rs index daa96617..846294e3 100644 --- a/veilid-core/src/network_manager/mod.rs +++ b/veilid-core/src/network_manager/mod.rs @@ -16,6 +16,7 @@ mod relay_worker; mod send_data; mod stats; mod tasks; +mod transport_stats; mod types; #[cfg(any(test, feature = "test-util"))] @@ -33,6 +34,7 @@ pub use stats::*; pub(crate) use bootstrap::*; pub(crate) use node_contact_method_cache::*; pub(crate) use online_detector::*; +pub(crate) use transport_stats::*; pub(crate) use types::*; //////////////////////////////////////////////////////////////////////////////////////// @@ -138,12 +140,16 @@ pub(crate) struct NetworkManager { // Per-routing-domain online detector online_detector: OnlineDetector, + // Our own outbound send health, per routing domain and transport + transport_stats: TransportStats, + // Accessors components: RwLock>, // Background processes rolling_transfers_task: TickTask, address_filter_task: TickTask, + transport_stats_log_task: TickTask, // Network key network_key: Option, @@ -247,6 +253,7 @@ impl NetworkManager { let inner = Self::new_inner(); let address_filter = AddressFilter::new(registry.clone()); let online_detector = OnlineDetector::new(registry.clone()); + let transport_stats = TransportStats::new(registry.clone()); let this = Self { registry, @@ -254,6 +261,7 @@ impl NetworkManager { accounting: Arc::new(RwLock::new(NetworkManagerAccounting::default())), address_filter, online_detector, + transport_stats, components: RwLock::new(None), rolling_transfers_task: TickTask::new( "rolling_transfers_task", @@ -263,6 +271,10 @@ impl NetworkManager { "address_filter_task", ADDRESS_FILTER_TASK_INTERVAL_SECS, ), + transport_stats_log_task: TickTask::new( + "transport_stats_log_task", + TRANSPORT_STATS_LOG_INTERVAL_SECS, + ), network_key, startup_context, concurrency, @@ -282,6 +294,10 @@ impl NetworkManager { &self.online_detector } + pub fn transport_stats(&self) -> &TransportStats { + &self.transport_stats + } + fn net(&self) -> Network { self.components.read().as_ref().unwrap_or_log().net.clone() } @@ -934,13 +950,11 @@ impl NetworkManager { envelope .to_encrypted_data(&crypto, body, &secret_key, &network_key) - .measure_limit( - TimestampDuration::new_ms(500), - veilid_log_dbg!( - registry, - "NetworkManager::build_envelope to_encrypted_data slow" - ), - ) + .measure_limit(TimestampDuration::new_ms(500), |duration| { + veilid_log!(self debug "NetworkManager::build_envelope to_encrypted_data slow: {:#}", duration); + // Encryption this slow ate the timeout budget of everything in flight + self.online_detector().record_local_stall(duration); + }) .await .wrap_err("envelope failed to encode") } @@ -997,10 +1011,10 @@ impl NetworkManager { let timestamp = Timestamp::now_increasing(); let out = match self .build_envelope(timestamp, best_node_id, envelope_version, body) - .measure_limit( - TimestampDuration::new_ms(1000), - veilid_log_dbg!(self, "NetworkManager::build_envelope slow"), - ) + .measure_limit(TimestampDuration::new_ms(1000), |duration| { + veilid_log!(self debug "NetworkManager::build_envelope slow: {:#}", duration); + self.online_detector().record_local_stall(duration); + }) .timeout_at(stop_token.clone()) .await? { @@ -1066,10 +1080,15 @@ impl NetworkManager { // should not be subject to our ability to decode it // Send receipt directly - network_result_value_or_log!(self self + let NetworkAttempt::Performed(attempt) = self .net() - .send_data_unbound_to_dial_info(dial_info, rcpt_data) - .await? => [ format!(": dial_info={}, rcpt_data.len={}", dial_info, rcpt_data.len()) ] { + .send_data_unbound_to_dial_info(dial_info.clone(), rcpt_data.clone()) + .await? + else { + veilid_log!(self debug "not sending out-of-band receipt to {} because the transport is suppressed", dial_info); + return Ok(()); + }; + network_result_value_or_log!(self attempt => [ format!(": dial_info={}, rcpt_data.len={}", dial_info, rcpt_data.len()) ] { return Ok(()); } ); @@ -1351,6 +1370,23 @@ impl NetworkManager { Ok(true) } + /// Record the last flow for a peer after one of OUR sends succeeded. + /// + /// Skips UDP: a flow we only sent on says nothing the dial info didn't, and remembering + /// it lets the existing-flow shortcuts bypass contact method selection. The receive path + /// calls `set_last_flow` directly, where a UDP flow does carry information. + pub(super) fn set_last_flow_from_send( + &self, + node_ref: NodeRef, + flow: Flow, + timestamp: Timestamp, + ) { + if flow.protocol_type().low_level_protocol_type() == LowLevelProtocolType::UDP { + return; + } + self.set_last_flow(node_ref, flow, timestamp); + } + /// Record the last flow for a peer in the routing table and the connection table appropriately pub(super) fn set_last_flow(&self, node_ref: NodeRef, flow: Flow, timestamp: Timestamp) { // Set the last flow on the routing table entry diff --git a/veilid-core/src/network_manager/network/address_family_confirm_logic.rs b/veilid-core/src/network_manager/network/address_family_confirm_logic.rs new file mode 100644 index 00000000..8ac5e6de --- /dev/null +++ b/veilid-core/src/network_manager/network/address_family_confirm_logic.rs @@ -0,0 +1,108 @@ +//! Decision logic for empirical address-family confirmation, used by the wasm +//! network and unit-tested on every platform. +//! +//! Invariant: a probe round that condemns every family defers rather than +//! confirming the empty set. An empty confirmed set stops all traffic, and with +//! no traffic no network-change event re-arms confirmation, so the node stays +//! dark until restart. Only a domain with no configured address types confirms +//! empty. + +use super::*; + +/// Minimum number of candidate nodes to probe an address family +pub(crate) const ADDR_FAMILY_MIN_NODES_TO_CONFIRM: usize = 4; + +/// Address family probe result +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum FamilyProbeDisposition { + /// A probe got an answer over this family + Supported, + /// Enough probe failures to condemn this family + Unsupported, + /// Probes ran out before either conclusion + Inconclusive, + /// The stop token fired mid-round + Cancelled, +} + +/// Address family confirmation state +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum AddressFamilyConfirmState { + /// No address types are configured for this routing domain + NoConfiguredAddressTypes, + /// All configured address types have been confirmed + AllConfiguredConfirmed, + /// Not enough candidates to verify unconfirmed address type + NotEnoughCandidates, + /// Confirm candidates to determine final set of address types + ConfirmCandidates, +} + +/// Outcome of one confirmation pass +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ConfirmationPassOutcome { + /// Publish this set and mark the domain confirmed + Confirm(AddressTypeSet), + /// Not enough evidence to conclude; leave unconfirmed so the worker retries + Deferred, +} + +/// Candidate inventory for one confirmation pass, generic over the candidate +/// handle so the decision logic tests off-network +pub(crate) struct AddressFamilyConfirmCandidates { + /// All address types configured for this routing domain + pub(crate) configured_address_types: AddressTypeSet, + /// All address types that have been positively seen since we started the confirmation process + pub(crate) confirmed_address_types: AddressTypeSet, + /// Nodes that can confirm an unconfirmed address type + pub(crate) candidates: BTreeMap>, +} + +impl AddressFamilyConfirmCandidates { + /// Determine the current state of the address family confirmation process + pub fn state(&self) -> AddressFamilyConfirmState { + if self.configured_address_types.is_empty() { + AddressFamilyConfirmState::NoConfiguredAddressTypes + } else if self.confirmed_address_types == self.configured_address_types { + AddressFamilyConfirmState::AllConfiguredConfirmed + } else { + let missing_address_types = + self.configured_address_types - self.confirmed_address_types; + for mat in missing_address_types { + let candidates = self + .candidates + .get(&mat) + .as_ref() + .map(|c| c.len()) + .unwrap_or(0); + if candidates < ADDR_FAMILY_MIN_NODES_TO_CONFIRM { + return AddressFamilyConfirmState::NotEnoughCandidates; + } + } + AddressFamilyConfirmState::ConfirmCandidates + } + } +} + +/// Fold a probe round into a pass outcome. `seen_live` holds families already +/// confirmed by real traffic since the confirm request. +pub(crate) fn resolve_probe_dispositions( + seen_live: AddressTypeSet, + dispositions: &[(AddressType, FamilyProbeDisposition)], +) -> ConfirmationPassOutcome { + let mut confirmed_address_types = seen_live; + for (at, disposition) in dispositions { + match disposition { + FamilyProbeDisposition::Cancelled => return ConfirmationPassOutcome::Deferred, + FamilyProbeDisposition::Supported => { + confirmed_address_types.insert(*at); + } + FamilyProbeDisposition::Unsupported | FamilyProbeDisposition::Inconclusive => {} + } + } + // All-condemned must defer, never confirm empty (see module doc) + if confirmed_address_types.is_empty() { + return ConfirmationPassOutcome::Deferred; + } + ConfirmationPassOutcome::Confirm(confirmed_address_types) +} diff --git a/veilid-core/src/network_manager/network/mod.rs b/veilid-core/src/network_manager/network/mod.rs index 4b366e32..f89b521b 100644 --- a/veilid-core/src/network_manager/network/mod.rs +++ b/veilid-core/src/network_manager/network/mod.rs @@ -1,3 +1,10 @@ +// Wasm-only in production, but unit-tested on every platform +#[cfg(any( + all(target_arch = "wasm32", target_os = "unknown"), + test, + feature = "test-util" +))] +mod address_family_confirm_logic; mod platform_network; mod platform_protocol_network_connection; @@ -15,6 +22,12 @@ cfg_if! { use super::*; +#[cfg(any( + all(target_arch = "wasm32", target_os = "unknown"), + test, + feature = "test-util" +))] +pub(crate) use address_family_confirm_logic::*; pub use platform_network::*; pub use platform_protocol_network_connection::*; diff --git a/veilid-core/src/network_manager/network/native/discovery_context.rs b/veilid-core/src/network_manager/network/native/discovery_context.rs index f9b48a70..3e2204d4 100644 --- a/veilid-core/src/network_manager/network/native/discovery_context.rs +++ b/veilid-core/src/network_manager/network/native/discovery_context.rs @@ -122,8 +122,12 @@ struct DiscoveryContextInner { } pub(super) struct DiscoveryContextUnlockedInner { + /// Protocol, address family and port this context probes config: DiscoveryContextConfig, + /// Publication-quality addresses (privacy/CLAT/CGNAT excluded) intf_addrs: BTreeSet
, + /// Interface address state, including the addresses we send from but never publish + interface_address_state: Arc, } #[derive(Clone)] @@ -154,10 +158,12 @@ impl DiscoveryContext { config: DiscoveryContextConfig, stop_token: StopToken, ) -> Self { - let intf_addrs = network + let interface_address_state = network .last_network_state() .unwrap_or_log() .interface_address_state + .clone(); + let intf_addrs = interface_address_state .interface_addresses .iter() .map(|x| Address::from_ip_addr(x.ip())) @@ -165,7 +171,11 @@ impl DiscoveryContext { Self { network, - unlocked_inner: Arc::new(DiscoveryContextUnlockedInner { config, intf_addrs }), + unlocked_inner: Arc::new(DiscoveryContextUnlockedInner { + config, + intf_addrs, + interface_address_state, + }), inner: Arc::new(Mutex::new(DiscoveryContextInner { external_infos: Vec::new(), mapped_dial_info: None, @@ -193,7 +203,7 @@ impl DiscoveryContext { ) .await { - Ok(StatusResult::Answer { answer, .. }) => answer, + Ok(StatusResult::Sent(status_answer)) => status_answer.answer, Ok(other) => { veilid_log!(self debug target: "network_result", "request_public_address {:?}: {:?}", @@ -400,13 +410,32 @@ impl DiscoveryContext { true } + /// Publication-set interface addresses of this context's family: what we can offer + /// peers, not what is ours. + fn family_intf_addrs(&self) -> impl Iterator { + let address_type = self.config.address_type; + self.intf_addrs + .iter() + .filter(move |a| a.address_type() == address_type) + } + + /// True when the address is ours and in this context's family. Uses the identity set, + /// so it matches the RFC 4941 privacy addresses we send from but never publish, which + /// is what a peer reports back as our external address. + fn is_own_family_address(&self, address: Address) -> bool { + address.address_type() == self.config.address_type + && self + .interface_address_state + .is_own_address(address.ip_addr()) + } + #[cfg_attr(feature = "instrument", instrument(level = "trace", skip(self), ret, fields(__VEILID_LOG_KEY = self.log_key())))] async fn validate_dial_info( &self, node_ref: NodeRef, dial_info: DialInfo, redirect: bool, - ) -> bool { + ) -> DialInfoValidationOutcome { // ask the node to send us a dial info validation receipt // no need to clear_last_flows here, because the dial_info is always returned via the // send_out_of_band_receipt mechanism, which will always create a new flow @@ -417,8 +446,9 @@ impl DiscoveryContext { .await { Err(e) => { + // Our own RPC failure; says nothing about whether peers can reach us veilid_log!(self trace "failed to send validate_dial_info to {:?}: {}", node_ref, e); - false + DialInfoValidationOutcome::Inconclusive } Ok(v) => v, } @@ -503,6 +533,7 @@ impl DiscoveryContext { true, ) .await + == DialInfoValidationOutcome::Reachable { veilid_log!(this debug "successfully validated mapped dial info {:?} on node {:?}", mapped_dial_info, external_info.node); @@ -554,8 +585,7 @@ impl DiscoveryContext { let this = self.clone(); let do_no_nat_fut: PinBoxFutureStatic = Box::pin(async move { - // Do a validate_dial_info on the external address from a redirected node - if this + let detected_class = match this .validate_dial_info( external_info.node.clone(), external_info.dial_info.clone(), @@ -563,35 +593,25 @@ impl DiscoveryContext { ) .await { - // Add public dial info with Direct dialinfo class - DetectionResultKind::Result { - possibilities, - result: DetectionResult { - config: this.config, - ddi: DetectedDialInfo::Detected(DialInfoDetail { - dial_info: external_info.dial_info.clone(), - class: DialInfoClass::Direct, - }), - external_address_types: AddressTypeSet::only( - external_info.address.address_type(), - ), - }, - } - } else { - // Add public dial info with Blocked dialinfo class - DetectionResultKind::Result { - possibilities, - result: DetectionResult { - config: this.config, - ddi: DetectedDialInfo::Detected(DialInfoDetail { - dial_info: external_info.dial_info.clone(), - class: DialInfoClass::Blocked, - }), - external_address_types: AddressTypeSet::only( - external_info.address.address_type(), - ), - }, + DialInfoValidationOutcome::Reachable => DialInfoClass::Direct, + DialInfoValidationOutcome::Unreachable => DialInfoClass::Blocked, + // Blocked here would blame a firewall for our own send failing + DialInfoValidationOutcome::Inconclusive => { + return DetectionResultKind::Failure { possibilities }; } + }; + DetectionResultKind::Result { + possibilities, + result: DetectionResult { + config: this.config, + ddi: DetectedDialInfo::Detected(DialInfoDetail { + dial_info: external_info.dial_info.clone(), + class: detected_class, + }), + external_address_types: AddressTypeSet::only( + external_info.address.address_type(), + ), + }, } }); @@ -616,11 +636,8 @@ impl DiscoveryContext { // If any local interface address of this context's family is a carrier-grade NAT or // DS-Lite indicator, classify as Symmetric for this family only. Other families may // still be globally reachable. - let family_intf_addrs_indicate_cgnat = self - .intf_addrs - .iter() - .filter(|a| a.address_type() == self.config.address_type) - .any(|a| a.is_cgnat_indicator()); + let family_intf_addrs_indicate_cgnat = + self.family_intf_addrs().any(|a| a.is_cgnat_indicator()); if family_intf_addrs_indicate_cgnat { let external_address_types = external_info.iter().fold(AddressTypeSet::new(), |acc, ei| { @@ -745,6 +762,7 @@ impl DiscoveryContext { true, ) .await + == DialInfoValidationOutcome::Reachable { // Add public dial info with Direct dialinfo class return DetectionResultKind::Result { @@ -791,14 +809,18 @@ impl DiscoveryContext { // Let's see what kind of NAT we have // Does a redirected dial info validation from a different address and a random port find us? loop { - if c_this + let outcome = c_this .validate_dial_info( c_external_1.node.clone(), c_external_1.dial_info.clone(), true, ) - .await - { + .await; + // A request that never got out can't discriminate NAT class + if outcome == DialInfoValidationOutcome::Inconclusive { + return DetectionResultKind::Failure { possibilities }; + } + if outcome == DialInfoValidationOutcome::Reachable { // Yes, another machine can use the dial info directly, so Full Cone // Add public dial info with full cone NAT network class @@ -846,14 +868,18 @@ impl DiscoveryContext { loop { // Do a validate_dial_info on the external address from a random port - if c_this + let outcome = c_this .validate_dial_info( c_external_2.node.clone(), c_external_1.dial_info.clone(), false, ) - .await - { + .await; + // Without a request going out we can't tell address- from port-restricted + if outcome == DialInfoValidationOutcome::Inconclusive { + return DetectionResultKind::Failure { possibilities }; + } + if outcome == DialInfoValidationOutcome::Reachable { // Got a reply from a non-default port, which means we're only address restricted return DetectionResultKind::Result { possibilities, @@ -934,18 +960,14 @@ impl DiscoveryContext { // NAT Detection /////////// - // If our local interface list contains any of the external addresses then there is no NAT in place + // An external address that is one of our own means no NAT for this family + // Family-scoped so a CGNAT'd IPv4 doesn't drag a routable IPv6 onto the NAT path let local_address_in_external_info = self .inner .lock() .external_infos .iter() - .find_map(|ei| { - self.intf_addrs - .contains(&ei.address.address()) - .then_some(true) - }) - .unwrap_or_default(); + .any(|ei| self.is_own_family_address(ei.address.address())); if local_address_in_external_info { self.protocol_process_no_nat(&mut all_possibilities, &mut unord); diff --git a/veilid-core/src/network_manager/network/native/low_level_protocol_tcp.rs b/veilid-core/src/network_manager/network/native/low_level_protocol_tcp.rs index 12a31283..5ef4621d 100644 --- a/veilid-core/src/network_manager/network/native/low_level_protocol_tcp.rs +++ b/veilid-core/src/network_manager/network/native/low_level_protocol_tcp.rs @@ -228,12 +228,14 @@ impl NativeNetwork { } Ok(None) => { // No protocol handlers matched? drop it. + #[cfg(feature = "verbose-tracing")] veilid_log!(self debug "no protocol handler for connection from {:?}", peer_addr); return; } - Err(e) => { + Err(_e) => { // Failed to negotiate connection? drop it. - veilid_log!(self debug "failed to negotiate connection from {:?}: {}", peer_addr, e); + #[cfg(feature = "verbose-tracing")] + veilid_log!(self debug "failed to negotiate connection from {:?}: {}", peer_addr, _e); return; } }; diff --git a/veilid-core/src/network_manager/network/native/low_level_protocol_udp.rs b/veilid-core/src/network_manager/network/native/low_level_protocol_udp.rs index c0353c03..9203fd58 100644 --- a/veilid-core/src/network_manager/network/native/low_level_protocol_udp.rs +++ b/veilid-core/src/network_manager/network/native/low_level_protocol_udp.rs @@ -185,8 +185,7 @@ impl NativeNetwork { let socket_arc = Arc::new(udp_socket); // Create protocol handler - let protocol_handler = - RawUdpProtocolHandler::new(self.registry(), socket_arc, addr.is_ipv6()); + let protocol_handler = RawUdpProtocolHandler::new(self.registry(), socket_arc); // Record protocol handler let mut inner = self.inner.lock(); diff --git a/veilid-core/src/network_manager/network/native/mod.rs b/veilid-core/src/network_manager/network/native/mod.rs index df2e1295..0ff7c85e 100644 --- a/veilid-core/src/network_manager/network/native/mod.rs +++ b/veilid-core/src/network_manager/network/native/mod.rs @@ -38,6 +38,7 @@ impl_veilid_log_facility!("net"); pub const MAX_DIAL_INFO_CONFIRM_FAILURE_COUNT: usize = 3; pub const UPDATE_OUTBOUND_ONLY_DIAL_INFO_PERIOD_SECS: u32 = 10; +pub const UPDATE_OUTBOUND_ONLY_DIAL_INFO_MAX_PERIOD_SECS: u32 = 300; pub const UPDATE_DIAL_INFO_TASK_TICK_PERIOD_SECS: u32 = 1; pub const NETWORK_INTERFACES_TASK_TICK_PERIOD_SECS: u32 = 1; pub const UPNP_TASK_TICK_PERIOD_SECS: u32 = 1; @@ -57,6 +58,9 @@ struct NetworkInner { routing_domain_detect_address_changes: BTreeSet, /// the next time we are allowed to check for better dialinfo when we are OutboundOnly next_outbound_only_dial_info_check: Timestamp, + /// current outbound-only recheck period; doubles when a confirmation finds no dial info, + /// resets when one does or when a reconfirmation is requested + outbound_only_dial_info_check_period_secs: u32, /// join handles for all the low level network background tasks join_handles: Vec>, /// stop source for shutting down the low level network background tasks @@ -208,6 +212,13 @@ impl PlatformNetwork for NativeNetwork { if address_filter.is_ip_addr_punished(dial_info.address().ip_addr()) { return Ok(NetworkResult::no_connection_other("punished")); } + // Debug availability override: fail like the network broke it + if network_manager.transport_stats().is_transport_disabled( + dial_info.protocol_type().low_level_protocol_type(), + dial_info.address_type(), + ) { + return Ok(NetworkResult::no_connection_other("transport disabled")); + } NativeProtocolNetworkConnection::connect( self.registry(), local_address, @@ -224,7 +235,7 @@ impl PlatformNetwork for NativeNetwork { &self, dial_info: DialInfo, data: Bytes, - ) -> PinBoxFuture<'_, EyreResult>> { + ) -> PinBoxFuture<'_, EyreResult>> { Box::pin(async move { let _guard = self.startup_lock.enter()?; @@ -242,64 +253,94 @@ impl PlatformNetwork for NativeNetwork { .address_filter() .is_ip_addr_punished(ip_addr) { - return Ok(NetworkResult::no_connection_other("punished")); + return Ok(NetworkAttempt::Performed( + NetworkResult::no_connection_other("punished"), + )); } - match dial_info.protocol_type() { - ProtocolType::UDP => { - let peer_socket_addr = dial_info.to_socket_addr(); - let h = RawUdpProtocolHandler::new_unspecified_bound_handler( - self.registry(), - &peer_socket_addr, - ) - .wrap_err("create socket failure")?; - let _ = network_result_try!(h - .send_message(data, peer_socket_addr) - .await - .map(NetworkResult::Value) - .wrap_err("send message failure")?); - } - ProtocolType::TCP => { - let peer_socket_addr = dial_info.to_socket_addr(); - let pnc = network_result_try!(RawTcpProtocolHandler::connect( - self.registry(), - None, - peer_socket_addr, - connect_timeout_ms - ) - .await - .wrap_err("connect failure")?); - network_result_try!(pnc.send(data).await.wrap_err("send failure")?); - } - ProtocolType::WS => { - let pnc = network_result_try!(WebsocketProtocolHandler::connect( - self.registry(), - None, - dial_info, - connect_timeout_ms - ) - .await - .wrap_err("connect failure")?); - network_result_try!(pnc.send(data).await.wrap_err("send failure")?); - } - #[cfg(feature = "enable-protocol-wss")] - ProtocolType::WSS => { - let pnc = network_result_try!(WebsocketProtocolHandler::connect( - self.registry(), - None, - dial_info, - connect_timeout_ms - ) - .await - .wrap_err("connect failure")?); - network_result_try!(pnc.send(data).await.wrap_err("send failure")?); - } + // Unbound sends have no contact method selection to steer around a suppressed transport + if self + .network_manager() + .transport_stats() + .unbound_send_deprioritized(&dial_info) + { + return Ok(NetworkAttempt::Skipped); } + + // The address names the routing domain, so nodeless traffic still counts + let stats_dial_info = dial_info.clone(); + let send_result: EyreResult> = async { + match dial_info.protocol_type() { + ProtocolType::UDP => { + let peer_socket_addr = dial_info.to_socket_addr(); + let h = RawUdpProtocolHandler::new_unspecified_bound_handler( + self.registry(), + &peer_socket_addr, + ) + .wrap_err("create socket failure")?; + network_result_try!(h + .send_message(data, peer_socket_addr, "unbound") + .await + .wrap_err("send message failure")?); + } + ProtocolType::TCP => { + let peer_socket_addr = dial_info.to_socket_addr(); + let pnc = network_result_try!(RawTcpProtocolHandler::connect( + self.registry(), + None, + peer_socket_addr, + connect_timeout_ms + ) + .await + .wrap_err("connect failure")?); + network_result_try!(pnc.send(data).await.wrap_err("send failure")?); + } + ProtocolType::WS => { + let pnc = network_result_try!(WebsocketProtocolHandler::connect( + self.registry(), + None, + dial_info, + connect_timeout_ms + ) + .await + .wrap_err("connect failure")?); + network_result_try!(pnc.send(data).await.wrap_err("send failure")?); + } + #[cfg(feature = "enable-protocol-wss")] + ProtocolType::WSS => { + let pnc = network_result_try!(WebsocketProtocolHandler::connect( + self.registry(), + None, + dial_info, + connect_timeout_ms + ) + .await + .wrap_err("connect failure")?); + network_result_try!(pnc.send(data).await.wrap_err("send failure")?); + } + } + Ok(NetworkResult::Value(())) + } + .await; + + if let Ok(nr) = &send_result { + self.network_manager() + .transport_stats() + .record_unbound_send( + &stats_dial_info, + TransportSendOutcome::from_network_result(nr), + ); + } + let nr = send_result?; + if !matches!(nr, NetworkResult::Value(_)) { + return Ok(NetworkAttempt::Performed(nr)); + } + // Network accounting self.network_manager() .stats_packet_sent(ip_addr, ByteCount::new(data_len as u64)); - Ok(NetworkResult::Value(())) + Ok(NetworkAttempt::Performed(NetworkResult::Value(()))) }) } @@ -309,7 +350,7 @@ impl PlatformNetwork for NativeNetwork { dial_info: DialInfo, data: Bytes, timeout_ms: u32, - ) -> PinBoxFuture<'_, EyreResult>> { + ) -> PinBoxFuture<'_, EyreResult>> { Box::pin(async move { let _guard = self.startup_lock.enter()?; @@ -326,101 +367,137 @@ impl PlatformNetwork for NativeNetwork { .address_filter() .is_ip_addr_punished(ip_addr) { - return Ok(NetworkResult::no_connection_other("punished")); + return Ok(NetworkAttempt::Performed( + NetworkResult::no_connection_other("punished"), + )); } - match dial_info.protocol_type() { - // Connectionless protocols - ProtocolType::UDP => { - let peer_socket_addr = dial_info.to_socket_addr(); - let h = RawUdpProtocolHandler::new_unspecified_bound_handler( - self.registry(), - &peer_socket_addr, - ) - .wrap_err("create socket failure")?; - network_result_try!(h - .send_message(data, peer_socket_addr) + // Unbound sends have no contact method selection to steer around a suppressed transport + if network_manager + .transport_stats() + .unbound_send_deprioritized(&dial_info) + { + return Ok(NetworkAttempt::Skipped); + } + + let result: EyreResult> = async { + match dial_info.protocol_type() { + // Connectionless protocols + ProtocolType::UDP => { + let peer_socket_addr = dial_info.to_socket_addr(); + let h = RawUdpProtocolHandler::new_unspecified_bound_handler( + self.registry(), + &peer_socket_addr, + ) + .wrap_err("create socket failure")?; + // Attribute the send only; a missing reply is the peer's silence + let send_res = h + .send_message(data, peer_socket_addr, "unbound_recv") + .await + .wrap_err("send message failure")?; + network_manager.transport_stats().record_unbound_send( + &dial_info, + TransportSendOutcome::from_network_result(&send_res), + ); + network_result_try!(send_res); + network_manager.stats_packet_sent(ip_addr, ByteCount::new(data_len as u64)); + + // receive single response + let mut out = BytesMut::zeroed(MAX_MESSAGE_SIZE); + let (recv_len, recv_addr) = network_result_try!(timeout( + timeout_ms, + h.recv_message(&mut out).in_current_span() + ) .await - .wrap_err("send message failure")?); - network_manager.stats_packet_sent(ip_addr, ByteCount::new(data_len as u64)); + .into_network_result()) + .wrap_err("recv_message failure")?; - // receive single response - let mut out = BytesMut::zeroed(MAX_MESSAGE_SIZE); - let (recv_len, recv_addr) = network_result_try!(timeout( - timeout_ms, - h.recv_message(&mut out).in_current_span() - ) - .await - .into_network_result()) - .wrap_err("recv_message failure")?; + let recv_socket_addr = recv_addr.remote_address().socket_addr(); + network_manager.stats_packet_rcvd( + recv_socket_addr.ip(), + ByteCount::new(recv_len as u64), + ); - let recv_socket_addr = recv_addr.remote_address().socket_addr(); - network_manager - .stats_packet_rcvd(recv_socket_addr.ip(), ByteCount::new(recv_len as u64)); - - // if the from address is not the same as the one we sent to, then drop this - if recv_socket_addr != peer_socket_addr { - bail!("wrong address"); + // if the from address is not the same as the one we sent to, then drop this + if recv_socket_addr != peer_socket_addr { + bail!("wrong address"); + } + out.resize(recv_len, 0u8); + Ok(NetworkResult::Value(out.into())) } - out.resize(recv_len, 0u8); - Ok(NetworkResult::Value(out.into())) - } - // Connection-oriented protocols - _ => { - let pnc = network_result_try!(match dial_info.protocol_type() { - ProtocolType::UDP => { - bail!("UDP handled by connectionless arm"); - } - ProtocolType::TCP => { - let peer_socket_addr = dial_info.to_socket_addr(); - RawTcpProtocolHandler::connect( - self.registry(), - None, - peer_socket_addr, - connect_timeout_ms, - ) - .await - .wrap_err("connect failure")? - } - ProtocolType::WS => { - WebsocketProtocolHandler::connect( + // Connection-oriented protocols + _ => { + let stats_dial_info = dial_info.clone(); + let connect_res = match dial_info.protocol_type() { + ProtocolType::UDP => { + bail!("UDP handled by connectionless arm"); + } + ProtocolType::TCP => { + let peer_socket_addr = dial_info.to_socket_addr(); + RawTcpProtocolHandler::connect( + self.registry(), + None, + peer_socket_addr, + connect_timeout_ms, + ) + .await + .wrap_err("connect failure")? + } + ProtocolType::WS => WebsocketProtocolHandler::connect( self.registry(), None, dial_info, connect_timeout_ms, ) .await - .wrap_err("connect failure")? - } - #[cfg(feature = "enable-protocol-wss")] - ProtocolType::WSS => { - WebsocketProtocolHandler::connect( + .wrap_err("connect failure")?, + #[cfg(feature = "enable-protocol-wss")] + ProtocolType::WSS => WebsocketProtocolHandler::connect( self.registry(), None, dial_info, connect_timeout_ms, ) .await - .wrap_err("connect failure")? - } - }); + .wrap_err("connect failure")?, + }; + // Attribute connect+send only; a missing reply is the peer's silence + let pnc = match connect_res { + NetworkResult::Value(pnc) => pnc, + nr => { + network_manager.transport_stats().record_unbound_send( + &stats_dial_info, + TransportSendOutcome::from_network_result(&nr), + ); + return Ok(nr.map(|_| unreachable!())); + } + }; + let send_res = pnc.send(data).await.wrap_err("send failure")?; + network_manager.transport_stats().record_unbound_send( + &stats_dial_info, + TransportSendOutcome::from_network_result(&send_res), + ); + network_result_try!(send_res); + network_manager.stats_packet_sent(ip_addr, ByteCount::new(data_len as u64)); - network_result_try!(pnc.send(data).await.wrap_err("send failure")?); - network_manager.stats_packet_sent(ip_addr, ByteCount::new(data_len as u64)); + let out = network_result_try!(network_result_try!(timeout( + timeout_ms, + pnc.recv().in_current_span() + ) + .await + .into_network_result()) + .wrap_err("recv failure")?); - let out = network_result_try!(network_result_try!(timeout( - timeout_ms, - pnc.recv().in_current_span() - ) - .await - .into_network_result()) - .wrap_err("recv failure")?); + network_manager + .stats_packet_rcvd(ip_addr, ByteCount::new(out.len() as u64)); - network_manager.stats_packet_rcvd(ip_addr, ByteCount::new(out.len() as u64)); - - Ok(NetworkResult::Value(out)) + Ok(NetworkResult::Value(out)) + } } } + .await; + + Ok(NetworkAttempt::Performed(result?)) }) } @@ -429,7 +506,7 @@ impl PlatformNetwork for NativeNetwork { &self, flow: Flow, data: Bytes, - ) -> PinBoxFuture<'_, EyreResult> { + ) -> PinBoxFuture<'_, EyreResult>> { Box::pin(async move { let _guard = self.startup_lock.enter()?; let network_manager = self.network_manager(); @@ -444,22 +521,37 @@ impl PlatformNetwork for NativeNetwork { &peer_socket_addr, &flow.local().map(|sa| sa.socket_addr()), ) { - network_result_value_or_log!(self ph.clone() - .send_message(data.clone(), peer_socket_addr) + match ph + .clone() + .send_message(data.clone(), peer_socket_addr, "existing_flow") .await - .wrap_err("sending data to existing flow")? => [ format!(": data.len={}, flow={:?}", data.len(), flow) ] - { return Ok(SendDataToExistingFlowResult::NotSent(data)); } ); + .wrap_err("sending data to existing flow")? + { + NetworkResult::Value(_) => {} + // Wedged transport; dialing would repeat the identical send + NetworkResult::Timeout => { + return Ok(NetworkAttempt::Performed(NetworkResult::Timeout)); + } + nres => { + veilid_log!(self debug target:"network_result", + "send_data_to_existing_flow: {:?}: data.len={}, flow={:?}", + nres, data.len(), flow + ); + return Ok(NetworkAttempt::Skipped); + } + } // Network accounting network_manager .stats_packet_sent(peer_socket_addr.ip(), ByteCount::new(data_len as u64)); - // Data was consumed + // Data was consumed; UDP is connectionless so this is a dial let unique_flow = UniqueFlow { flow, connection_id: None, + dialed: true, }; - return Ok(SendDataToExistingFlowResult::Sent(unique_flow)); + return Ok(NetworkAttempt::value(unique_flow)); } } @@ -467,9 +559,8 @@ impl PlatformNetwork for NativeNetwork { // Try to send to the exact existing connection if one exists if let Some(conn) = network_manager.connection_manager().get_connection(flow) { - // connection exists, send over it - match conn.send_async(data).await { - ConnectionHandleSendResult::Sent => { + match timeout(self.single_send_timeout_ms(), conn.send_async(data)).await { + Ok(ConnectionHandleSendResult::Sent) => { // Network accounting network_manager.stats_packet_sent( flow.remote().socket_addr().ip(), @@ -477,18 +568,19 @@ impl PlatformNetwork for NativeNetwork { ); // Data was consumed - return Ok(SendDataToExistingFlowResult::Sent(conn.unique_flow())); + return Ok(NetworkAttempt::value(conn.unique_flow())); } - ConnectionHandleSendResult::NotSent(data) => { - // Couldn't send - // Pass the data back out so we don't own it any more - return Ok(SendDataToExistingFlowResult::NotSent(data)); + Ok(ConnectionHandleSendResult::NotSent) => { + return Ok(NetworkAttempt::Skipped); + } + Err(_) => { + veilid_log!(self debug "connection send timed out: flow={:?}", flow); + return Ok(NetworkAttempt::Performed(NetworkResult::Timeout)); } } } // Connection didn't exist - // Pass the data back out so we don't own it any more - Ok(SendDataToExistingFlowResult::NotSent(data)) + Ok(NetworkAttempt::Skipped) }) } @@ -531,23 +623,31 @@ impl PlatformNetwork for NativeNetwork { } }; let flow = network_result_try!(ph - .send_message(data, peer_socket_addr) + .send_message(data, peer_socket_addr, "dial_info") .await .wrap_err("failed to send data to dial info")?); unique_flow = UniqueFlow { flow, connection_id: None, + dialed: true, }; } else { // Handle connection-oriented protocols let connmgr = network_manager.connection_manager(); let conn = network_result_try!(connmgr.get_or_create_connection(dial_info).await?); - if let ConnectionHandleSendResult::NotSent(_) = conn.send_async(data).await { - return Ok(NetworkResult::NoConnection(io::Error::new( - io::ErrorKind::ConnectionReset, - "failed to send", - ))); + match timeout(self.single_send_timeout_ms(), conn.send_async(data)).await { + Ok(ConnectionHandleSendResult::Sent) => {} + Ok(ConnectionHandleSendResult::NotSent) => { + return Ok(NetworkResult::NoConnection(io::Error::new( + io::ErrorKind::ConnectionReset, + "failed to send", + ))); + } + Err(_) => { + veilid_log!(self debug "connection send timed out: dial_info={:?}", conn.unique_flow().flow); + return Ok(NetworkResult::timeout()); + } } unique_flow = conn.unique_flow(); } @@ -573,36 +673,34 @@ impl PlatformNetwork for NativeNetwork { let network_manager = self.network_manager(); let ip_addr = dial_info.address().ip_addr(); - let unique_flow; - if dial_info.protocol_type().low_level_protocol_type() == LowLevelProtocolType::UDP { - // Handle connectionless protocol - let peer_socket_addr = dial_info.to_socket_addr(); - // Prefer a source-bound handler for PublicInternet sends - let preferred_local = self.preferred_outbound_source_addr(&dial_info); - let ph = match self - .find_best_udp_protocol_handler(&peer_socket_addr, &preferred_local) - { - Some(ph) => ph, - None => { - return Ok(NetworkResult::no_connection_other( - "no appropriate UDP protocol handler for dial_info", - )); - } - }; - let flow = network_result_try!(ph - .send_hole_punch(peer_socket_addr, HOLE_PUNCH_TTL) - .await - .wrap_err("failed to send hole punch to dial info")?); - unique_flow = UniqueFlow { - flow, - connection_id: None, - }; - } else { + if dial_info.protocol_type().low_level_protocol_type() != LowLevelProtocolType::UDP { return Ok(NetworkResult::ServiceUnavailable( "unimplemented for this protocol".to_owned(), )); } + // Handle connectionless protocol + let peer_socket_addr = dial_info.to_socket_addr(); + // Prefer a source-bound handler for PublicInternet sends + let preferred_local = self.preferred_outbound_source_addr(&dial_info); + let ph = match self.find_best_udp_protocol_handler(&peer_socket_addr, &preferred_local) + { + Some(ph) => ph, + None => { + return Ok(NetworkResult::no_connection_other( + "no appropriate UDP protocol handler for dial_info", + )); + } + }; + let flow = network_result_try!(ph + .send_hole_punch(peer_socket_addr, HOLE_PUNCH_TTL) + .wrap_err("failed to send hole punch to dial info")?); + let unique_flow = UniqueFlow { + flow, + connection_id: None, + dialed: true, + }; + // Network accounting network_manager.stats_packet_sent(ip_addr, ByteCount::new(0)); @@ -682,6 +780,9 @@ impl PlatformNetwork for NativeNetwork { if !rdc.read_dyn().confirmed() { return false; } + // Real evidence of change makes the outbound-only recheck eager again + self.inner.lock().outbound_only_dial_info_check_period_secs = + UPDATE_OUTBOUND_ONLY_DIAL_INFO_PERIOD_SECS; let mut editor = rdc.edit_dyn(); editor.set_confirmed(false); editor.commit(); @@ -690,12 +791,23 @@ impl PlatformNetwork for NativeNetwork { } impl NativeNetwork { + /// Upper bound on one send; a stalled connection parks channel sends forever otherwise + fn single_send_timeout_ms(&self) -> u32 { + let config = self.config(); + let net_config = &config.internal().network; + net_config + .rpc + .timeout_ms + .min(net_config.single_send_timeout_ms) + } + fn new_inner() -> NetworkInner { NetworkInner { network_needs_restart: false, dial_info_confirm_failure_count: BTreeMap::new(), routing_domain_detect_address_changes: BTreeSet::new(), next_outbound_only_dial_info_check: Timestamp::default(), + outbound_only_dial_info_check_period_secs: UPDATE_OUTBOUND_ONLY_DIAL_INFO_PERIOD_SECS, join_handles: Vec::new(), stop_source: None, bound_address_per_protocol: BTreeMap::new(), diff --git a/veilid-core/src/network_manager/network/native/network_state.rs b/veilid-core/src/network_manager/network/native/network_state.rs index 71661de7..c043eda5 100644 --- a/veilid-core/src/network_manager/network/native/network_state.rs +++ b/veilid-core/src/network_manager/network/native/network_state.rs @@ -102,12 +102,11 @@ impl NetworkState { } } + /// True for any address on our interfaces, including the privacy and CLAT addresses + /// the published interface address set excludes. #[expect(dead_code)] pub fn is_interface_ip_addr(&self, addr: IpAddr) -> bool { - self.interface_address_state - .interface_addresses - .iter() - .any(|x| x.ip() == addr) + self.interface_address_state.is_own_address(addr) } pub fn translate_unspecified_socket_addr(&self, from: SocketAddr) -> Vec { @@ -284,6 +283,9 @@ impl NativeNetwork { let routable_addresses_changed = last_routable_addresses != new_routable_addresses; if routable_addresses_changed { + // Our addresses moved, so transport failures we blamed on peers may have been ours + self.routing_table().clear_all_transport_evidence(); + let added_addresses = new_routable_addresses .difference(&last_routable_addresses) .copied() @@ -460,16 +462,12 @@ impl NativeNetwork { .gateway_addresses .iter() .any(|g| matches!(g, IpAddr::V4(_))); - let has_v6_default_route = ias - .gateway_addresses - .iter() - .any(|g| matches!(g, IpAddr::V6(_))); let ipv4_local = allow_v4 && Self::can_bind_v4() && raw.has_non_loopback_ipv4; let ipv4_global = ipv4_local && has_v4_default_route; let ipv6_local = allow_v6 && Self::can_bind_v6() && raw.has_non_loopback_ipv6; - let ipv6_global = ipv6_local && has_v6_default_route && raw.has_unicast_global_ipv6; + let ipv6_global = ipv6_local && raw.has_unicast_global_ipv6; // TODO: better ipv6 detection for NAT should happen if ipv4_local { veilid_log!(self debug "Local IPV4 detected (interface address present)"); @@ -481,11 +479,7 @@ impl NativeNetwork { veilid_log!(self debug "Local IPV6 detected (interface address present)"); } if ipv6_global { - veilid_log!(self debug "Global IPV6 detected (unicast-global address + default route)"); - } else if ipv6_local && !raw.has_unicast_global_ipv6 { - veilid_log!(self debug "Skipping global IPV6: no unicast-global address (only link-local/ULA present)"); - } else if ipv6_local && !has_v6_default_route { - veilid_log!(self debug "Skipping global IPV6: no IPv6 default route"); + veilid_log!(self debug "Global IPV6 detected (unicast-global address)"); } AddressConfig { diff --git a/veilid-core/src/network_manager/network/native/protocol/udp.rs b/veilid-core/src/network_manager/network/native/protocol/udp.rs index 4441e4d6..67674dc6 100644 --- a/veilid-core/src/network_manager/network/native/protocol/udp.rs +++ b/veilid-core/src/network_manager/network/native/protocol/udp.rs @@ -11,12 +11,6 @@ pub struct RawUdpProtocolHandler { socket: Arc, /// Reassembly buffer for fragmented UDP messages assembly_buffer: AssemblyBuffer, - /// True if `socket` is IPv6, false for IPv4 - is_ipv6: bool, - /// TTL the socket was created with; restored after holepunch sends - default_ttl: u32, - /// Current TTL on the socket; mutated transiently by `send_hole_punch` - current_ttl: Arc>, /// Drop to signal cancellation; taken out by `stop()` stop_source: Arc>>, /// Cloned by recv-loops that follow this handler's stop @@ -26,16 +20,7 @@ pub struct RawUdpProtocolHandler { impl_veilid_component_accessors!(RawUdpProtocolHandler); impl RawUdpProtocolHandler { - pub fn new(registry: VeilidComponentRegistry, socket: Arc, is_ipv6: bool) -> Self { - // Get original TTL - let default_ttl = if is_ipv6 { - socket2_operation(socket.as_ref(), |s| s.unicast_hops_v6()) - .expect_or_log("getting IPV6_UNICAST_HOPS should not fail") - } else { - socket2_operation(socket.as_ref(), |s| s.ttl_v4()) - .expect_or_log("getting IP_TTL should not fail") - }; - + pub fn new(registry: VeilidComponentRegistry, socket: Arc) -> Self { let stop_source = StopSource::new(); let stop_token = stop_source.token(); @@ -43,9 +28,6 @@ impl RawUdpProtocolHandler { registry, socket, assembly_buffer: AssemblyBuffer::new(), - is_ipv6, - default_ttl, - current_ttl: Arc::new(AsyncMutex::new(default_ttl)), stop_source: Arc::new(Mutex::new(Some(stop_source))), stop_token, } @@ -61,6 +43,18 @@ impl RawUdpProtocolHandler { *self.stop_source.lock() = None; } + /// Debug availability override: a suppressed transport must fail like the network broke it + fn is_transport_disabled(&self, remote_addr: SocketAddr) -> bool { + let address_type = if remote_addr.is_ipv6() { + AddressType::IPV6 + } else { + AddressType::IPV4 + }; + self.network_manager() + .transport_stats() + .is_transport_disabled(LowLevelProtocolType::UDP, address_type) + } + #[cfg_attr(feature = "instrument", instrument(level = "trace", target = "protocol", err, skip(self, data), fields(self.socket = ?self.socket, data.len = data.len(), ret.len, ret.flow)))] pub async fn recv_message(&self, data: &mut [u8]) -> io::Result<(usize, Flow)> { let (message_len, flow) = loop { @@ -126,11 +120,13 @@ impl RawUdpProtocolHandler { Ok((message_len, flow)) } + /// Fragment and send a message. `purpose` names the call site in the send-timeout log. #[cfg_attr(feature = "instrument", instrument(level = "trace", target = "protocol", err, skip(self, data), fields(self.socket = ?self.socket, data.len = data.len(), ret.flow)))] pub async fn send_message( &self, data: Bytes, remote_addr: SocketAddr, + purpose: &'static str, ) -> io::Result> { if data.len() > MAX_MESSAGE_SIZE { bail_io_error_other!("sending too large UDP message"); @@ -145,46 +141,44 @@ impl RawUdpProtocolHandler { return Ok(NetworkResult::no_connection_other("punished")); } - // Ensure the TTL for sent packets is the default, - // then fragment and send the packets - { - let current_ttl = self.current_ttl.lock().await; - if *current_ttl != self.default_ttl { - veilid_log!(self error "Incorrect TTL on sent UDP packet ({} != {}): len={}, remote_addr={:?}", *current_ttl, self.default_ttl, data.len(), remote_addr); + if self.is_transport_disabled(remote_addr) { + return Ok(NetworkResult::no_connection_other("transport disabled")); + } + + let sender = |framed_chunk: Bytes, remote_addr: SocketAddr| async move { + let len = network_result_try!(self + .socket + .send_to(&framed_chunk, remote_addr) + .await + .into_network_result()?); + if len != framed_chunk.len() { + bail_io_error_other!("UDP partial send") } - // Fragment and send - let sender = |framed_chunk: Bytes, remote_addr: SocketAddr| async move { - let len = network_result_try!(self - .socket - .send_to(&framed_chunk, remote_addr) - .await - .into_network_result()?); - if len != framed_chunk.len() { - bail_io_error_other!("UDP partial send") - } + veilid_log!(self trace "udp::send_message:chunk(len={}) {:?}", len, remote_addr); + Ok(NetworkResult::value(())) + }; - veilid_log!(self trace "udp::send_message:chunk(len={}) {:?}", len, remote_addr); - Ok(NetworkResult::value(())) - }; - - // A stalled interface parks send_to forever while the ttl lock is held, - // serializing every UDP send behind it; bound the fragmented send so it fails - let timeout_ms = self.registry.config().internal().network.rpc.timeout_ms; - network_result_try!(match timeout( - timeout_ms, - self.assembly_buffer - .split_message(data, remote_addr, sender) - ) - .await - { - Ok(res) => res?, - Err(_) => { - veilid_log!(self debug "UDP send timed out: remote_addr={:?}", remote_addr); - return Ok(NetworkResult::timeout()); - } - }); - } + // A stalled interface parks send_to forever; bound the fragmented send so it fails + let config = self.registry.config(); + let net_config = &config.internal().network; + let timeout_ms = net_config + .rpc + .timeout_ms + .min(net_config.single_send_timeout_ms); + network_result_try!(match timeout( + timeout_ms, + self.assembly_buffer + .split_message(data, remote_addr, sender) + ) + .await + { + Ok(res) => res?, + Err(_) => { + veilid_log!(self debug "UDP send timed out: purpose={} remote_addr={:?}", purpose, remote_addr); + return Ok(NetworkResult::timeout()); + } + }); // Return a flow for the sent message let peer_addr = PeerAddress::new( @@ -206,7 +200,7 @@ impl RawUdpProtocolHandler { } #[cfg_attr(feature = "instrument", instrument(level = "trace", target = "protocol", err, skip(self), fields(self.socket = ?self.socket, ret.flow)))] - pub async fn send_hole_punch( + pub fn send_hole_punch( &self, remote_addr: SocketAddr, ttl: u32, @@ -220,43 +214,17 @@ impl RawUdpProtocolHandler { return Ok(NetworkResult::no_connection_other("punished")); } - // Ensure the TTL for sent packets is the default, - // then fragment and send the packets - let res = { - let mut current_ttl = self.current_ttl.lock().await; - if *current_ttl != self.default_ttl { - veilid_log!(self error "Incorrect TTL before sending holepunch UDP packet ({} != {}): remote_addr={:?}", *current_ttl, self.default_ttl, remote_addr); + if self.is_transport_disabled(remote_addr) { + return Ok(NetworkResult::no_connection_other("transport disabled")); + } + + // Zero length packet with a best-effort per-packet TTL + let res = match send_udp_datagram_with_ttl(self.socket.as_ref(), &[], remote_addr, ttl) { + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + veilid_log!(self debug "UDP hole punch send would block: remote_addr={:?}", remote_addr); + return Ok(NetworkResult::timeout()); } - - // Get synchronous socket - socket2_operation(self.socket.as_ref(), |s| { - // Set TTL - let ttl_res = if self.is_ipv6 { - s.set_unicast_hops_v6(ttl) - } else { - s.set_ttl_v4(ttl) - }; - ttl_res.inspect_err(|e| { - veilid_log!(self error "Failed to set TTL on holepunch UDP socket: {} remote_addr={:?}", e, remote_addr); - })?; - *current_ttl = ttl; - - // Send zero length packet - let res = s.send_to(&[], &remote_addr.into()); - - // Restore TTL immediately - let ttl_res = if self.is_ipv6 { - s.set_unicast_hops_v6(self.default_ttl) - } else { - s.set_ttl_v4(self.default_ttl) - }; - ttl_res.inspect_err(|e| { - veilid_log!(self error "Failed to reset TTL on holepunch UDP socket: {} remote_addr={:?}", e, remote_addr); - })?; - *current_ttl = self.default_ttl; - - res - }) + res => res, }; // Check for errors @@ -296,11 +264,7 @@ impl RawUdpProtocolHandler { let local_socket_addr = compatible_unspecified_socket_addr(socket_addr); let socket = bind_async_udp_socket(local_socket_addr)? .ok_or(io::Error::from(io::ErrorKind::AddrInUse))?; - Ok(RawUdpProtocolHandler::new( - registry, - Arc::new(socket), - local_socket_addr.is_ipv6(), - )) + Ok(RawUdpProtocolHandler::new(registry, Arc::new(socket))) } /// Bind a UDP socket to a specific local source address @@ -314,10 +278,6 @@ impl RawUdpProtocolHandler { ) -> io::Result { let socket = bind_async_udp_socket(local_socket_addr)? .ok_or(io::Error::from(io::ErrorKind::AddrInUse))?; - Ok(RawUdpProtocolHandler::new( - registry, - Arc::new(socket), - local_socket_addr.is_ipv6(), - )) + Ok(RawUdpProtocolHandler::new(registry, Arc::new(socket))) } } diff --git a/veilid-core/src/network_manager/network/native/tasks/confirm_dial_info.rs b/veilid-core/src/network_manager/network/native/tasks/confirm_dial_info.rs index 9355d2bf..a62dd4e3 100644 --- a/veilid-core/src/network_manager/network/native/tasks/confirm_dial_info.rs +++ b/veilid-core/src/network_manager/network/native/tasks/confirm_dial_info.rs @@ -174,20 +174,10 @@ impl NativeNetwork { return Ok(()); } - let confirmed = self - .do_dial_info_confirm_public_internet(stop_token, l, t) + // The next outbound-only recheck is scheduled inside, where the conclusion is known + self.do_dial_info_confirm_public_internet(stop_token, l, t) .await?; - // Done with public dial info check - if confirmed { - let mut inner = self.inner.lock(); - - // Don't try to re-do OutboundOnly dialinfo for another 10 seconds - inner.next_outbound_only_dial_info_check = Timestamp::now().later( - TimestampDuration::new_secs(UPDATE_OUTBOUND_ONLY_DIAL_INFO_PERIOD_SECS), - ) - } - Ok(()) } @@ -461,6 +451,8 @@ impl NativeNetwork { confirmation_result.external_address_types, protocol_config.public_internet_capabilities, )); + let has_dial_info = !static_dial_info_details.is_empty() + || !confirmation_result.dial_info_details.is_empty(); editor.clear_dial_info_details(None, None); for did in static_dial_info_details { editor.add_dial_info_detail(did); @@ -475,6 +467,23 @@ impl NativeNetwork { // Publish the peer info (idempotent if it hasn't changed) rdc.publish_peer_info(); + // A stable symmetric NAT needs no re-proving, so back off while there is no dial info + { + let mut inner = self.inner.lock(); + let period_secs = if has_dial_info { + inner.outbound_only_dial_info_check_period_secs = + UPDATE_OUTBOUND_ONLY_DIAL_INFO_PERIOD_SECS; + UPDATE_OUTBOUND_ONLY_DIAL_INFO_PERIOD_SECS + } else { + let period = inner.outbound_only_dial_info_check_period_secs; + inner.outbound_only_dial_info_check_period_secs = + (period * 2).min(UPDATE_OUTBOUND_ONLY_DIAL_INFO_MAX_PERIOD_SECS); + period + }; + inner.next_outbound_only_dial_info_check = + Timestamp::now().later(TimestampDuration::new_secs(period_secs)); + } + Ok(true) } diff --git a/veilid-core/src/network_manager/network/platform_network.rs b/veilid-core/src/network_manager/network/platform_network.rs index 425de99a..79847643 100644 --- a/veilid-core/src/network_manager/network/platform_network.rs +++ b/veilid-core/src/network_manager/network/platform_network.rs @@ -14,14 +14,6 @@ pub fn new_platform_network(registry: VeilidComponentRegistry) -> Network { } } -/// Result of sending data to an existing flow -pub enum SendDataToExistingFlowResult { - /// Data was sent successfully, returning the unique flow used to send the data - Sent(UniqueFlow), - /// Data was not sent successfully, returning the data that was not sent - NotSent(Bytes), -} - /// Network base implementation trait /// /// Defines the minimum set of operations required for the NetworkManager component to operate @@ -84,7 +76,7 @@ pub trait PlatformNetwork: VeilidComponentRegistryAccessor + Send + Sync { &self, dial_info: DialInfo, data: Bytes, - ) -> PinBoxFuture<'_, EyreResult>>; + ) -> PinBoxFuture<'_, EyreResult>>; /// Send and receive data to/from a dial info, unbound, using a new connection from a random port /// @@ -98,7 +90,7 @@ pub trait PlatformNetwork: VeilidComponentRegistryAccessor + Send + Sync { dial_info: DialInfo, data: Bytes, timeout_ms: u32, - ) -> PinBoxFuture<'_, EyreResult>>; + ) -> PinBoxFuture<'_, EyreResult>>; /// Send data to a flow that already exists /// @@ -107,7 +99,7 @@ pub trait PlatformNetwork: VeilidComponentRegistryAccessor + Send + Sync { &self, flow: Flow, data: Bytes, - ) -> PinBoxFuture<'_, EyreResult>; + ) -> PinBoxFuture<'_, EyreResult>>; /// Send data directly to a dial info, possibly without knowing which node it is going to /// diff --git a/veilid-core/src/network_manager/network/wasm/address_family_confirm.rs b/veilid-core/src/network_manager/network/wasm/address_family_confirm.rs index 17366bf7..e4346e2b 100644 --- a/veilid-core/src/network_manager/network/wasm/address_family_confirm.rs +++ b/veilid-core/src/network_manager/network/wasm/address_family_confirm.rs @@ -6,69 +6,11 @@ use super::*; /// Interval between polling for address family confirmation const ADDR_FAMILY_POLL_INTERVAL_MS: u32 = 1000; -/// Minimum number of nodes to confirm an address family -const ADDR_FAMILY_MIN_NODES_TO_CONFIRM: usize = 4; /// Maximum number of probes per family const ADDR_FAMILY_MAX_PROBES: usize = 8; /// Maximum number of failures to tolerate before declaring an address family unsupported const ADDR_FAMILY_FAILURE_THRESHOLD: usize = 3; -/// Address family probe result -enum FamilyProbeDisposition { - Supported, - Unsupported, - Inconclusive, - Cancelled, -} - -/// Address family confirmation state -enum AddressFamilyConfirmState { - /// No address types are configured for this routing domain - NoConfiguredAddressTypes, - /// All configured address types have been confirmed - AllConfiguredConfirmed, - /// Not enough candidates to verify unconfirmed address type - NotEnoughCandidates, - /// Confirm candidates to determine final set of address types - ConfirmCandidates, -} - -/// Address family confirmation state -struct AddressFamilyConfirmCandidates { - /// All address types configured for this routing domain - configured_address_types: AddressTypeSet, - /// All address types that have been positively seen since we started the confirmation process - confirmed_address_types: AddressTypeSet, - /// Nodes that can confirm an unconfirmed address type - candidates: BTreeMap>, -} - -impl AddressFamilyConfirmCandidates { - /// Determine the current state of the address family confirmation process - pub fn state(&self) -> AddressFamilyConfirmState { - if self.configured_address_types.is_empty() { - AddressFamilyConfirmState::NoConfiguredAddressTypes - } else if self.confirmed_address_types == self.configured_address_types { - AddressFamilyConfirmState::AllConfiguredConfirmed - } else { - let missing_address_types = - self.configured_address_types - self.confirmed_address_types; - for mat in missing_address_types { - let candidates = self - .candidates - .get(&mat) - .as_ref() - .map(|c| c.len()) - .unwrap_or(0); - if candidates < ADDR_FAMILY_MIN_NODES_TO_CONFIRM { - return AddressFamilyConfirmState::NotEnoughCandidates; - } - } - AddressFamilyConfirmState::ConfirmCandidates - } - } -} - impl WasmNetwork { // Reset address family confirmation state pub(super) fn reset_address_family_confirm_state(&self, routing_domain: RoutingDomain) -> bool { @@ -226,7 +168,8 @@ impl WasmNetwork { } /// One confirmation pass: probe each allowed family, drop conclusively-unreachable ones, - /// then set confirmed=true (mirrors native confirm completion) + /// then set confirmed=true. A deferred outcome leaves the domain unconfirmed so the + /// worker retries next tick. async fn perform_address_family_confirmation_pass( &self, routing_domain: RoutingDomain, @@ -236,64 +179,47 @@ impl WasmNetwork { let address_family_confirm_candidates = self.collect_family_probe_candidates(routing_domain)?; - match address_family_confirm_candidates.state() { + let outcome = match address_family_confirm_candidates.state() { AddressFamilyConfirmState::NoConfiguredAddressTypes => { veilid_log!(self debug "no address types configured for routing domain {:#}; confirming no address types", routing_domain); - self.confirm_address_families( - routing_domain, - address_family_confirm_candidates.confirmed_address_types, - ); - return Ok(()); + ConfirmationPassOutcome::Confirm(AddressTypeSet::empty()) } AddressFamilyConfirmState::AllConfiguredConfirmed => { - veilid_log!(self debug "all address types confirmed for routing domain {:#}; enabling all configuredaddress types", routing_domain); - self.confirm_address_families( - routing_domain, + veilid_log!(self debug "all address types confirmed for routing domain {:#}; enabling all configured address types", routing_domain); + ConfirmationPassOutcome::Confirm( address_family_confirm_candidates.confirmed_address_types, - ); - return Ok(()); + ) } AddressFamilyConfirmState::NotEnoughCandidates => { veilid_log!(self debug "not enough candidates to confirm address types for routing domain {:#}; skipping confirmation", routing_domain); - return Ok(()); + ConfirmationPassOutcome::Deferred } AddressFamilyConfirmState::ConfirmCandidates => { veilid_log!(self debug "Need to confirm address types {:?} for routing domain {:#}", address_family_confirm_candidates.candidates.keys(), routing_domain); - } - } - // Keep optimistic; remove only families that fail conclusively (no candidates = keep) - let mut confirmed_address_types = address_family_confirm_candidates.confirmed_address_types; - for (at, cands) in address_family_confirm_candidates.candidates { - if cands.len() < ADDR_FAMILY_MIN_NODES_TO_CONFIRM { - veilid_log!(self error "not enough candidates to confirm address type {:#} for routing domain {:#}; skipping confirmation", at, routing_domain); - continue; + let seen_live = address_family_confirm_candidates.confirmed_address_types; + let mut dispositions = Vec::new(); + for (at, cands) in address_family_confirm_candidates.candidates { + let disposition = self + .probe_address_family(routing_domain, at, cands, stop_token) + .await; + if matches!(disposition, FamilyProbeDisposition::Inconclusive) { + // Unreachable when state() guaranteed enough candidates + veilid_log!(self error "inconclusive result confirming address type {:#} for routing domain {:#}", at, routing_domain); + } + let cancelled = matches!(disposition, FamilyProbeDisposition::Cancelled); + dispositions.push((at, disposition)); + if cancelled { + break; + } + } + resolve_probe_dispositions(seen_live, &dispositions) } - match self - .probe_address_family(routing_domain, at, cands.clone(), stop_token) - .await - { - FamilyProbeDisposition::Unsupported => { - // Definitely not supported, don't add to confirmed set - } - FamilyProbeDisposition::Cancelled => { - // Bail out immediately because we got the stop token - return Ok(()); - } - FamilyProbeDisposition::Inconclusive => { - // Shouldn't happen because we only get here if there weren't enough candidates - veilid_log!(self error "Inconculsive results confirming address type {:#} for routing domain {:#}; skipping confirmation", at, routing_domain); - continue; - } - FamilyProbeDisposition::Supported => { - // Record as confirmed - confirmed_address_types.insert(at); - } - } - } + }; - // Confirm whatever we got from the probes - self.confirm_address_families(routing_domain, confirmed_address_types); + if let ConfirmationPassOutcome::Confirm(confirmed_address_types) = outcome { + self.confirm_address_families(routing_domain, confirmed_address_types); + } Ok(()) } @@ -303,8 +229,8 @@ impl WasmNetwork { fn collect_family_probe_candidates( &self, routing_domain: RoutingDomain, - ) -> EyreResult { - // All possible configures address types we can confirm + ) -> EyreResult> { + // All possible configured address types we can confirm let configured_address_types = configured_address_type_set(&self.config()); // List of candidates that can be pinged for positive confirmation of missing address types @@ -425,7 +351,7 @@ impl WasmNetwork { Err(_) => return FamilyProbeDisposition::Cancelled, }; match res { - Ok(StatusResult::Answer { .. }) => return FamilyProbeDisposition::Supported, + Ok(StatusResult::Sent(_)) => return FamilyProbeDisposition::Supported, _ => { failures += 1; if failures >= ADDR_FAMILY_FAILURE_THRESHOLD { diff --git a/veilid-core/src/network_manager/network/wasm/mod.rs b/veilid-core/src/network_manager/network/wasm/mod.rs index c52c0adb..fb7e53df 100644 --- a/veilid-core/src/network_manager/network/wasm/mod.rs +++ b/veilid-core/src/network_manager/network/wasm/mod.rs @@ -176,6 +176,13 @@ impl PlatformNetwork for WasmNetwork { if address_filter.is_ip_addr_punished(dial_info.address().ip_addr()) { return Ok(NetworkResult::no_connection_other("punished")); } + // Debug availability override: fail like the network broke it + if network_manager.transport_stats().is_transport_disabled( + dial_info.protocol_type().low_level_protocol_type(), + dial_info.address_type(), + ) { + return Ok(NetworkResult::no_connection_other("transport disabled")); + } WasmProtocolNetworkConnection::connect(self.registry(), dial_info, timeout_ms) .await .wrap_err("io error in connect") @@ -187,7 +194,7 @@ impl PlatformNetwork for WasmNetwork { &self, dial_info: DialInfo, data: Bytes, - ) -> PinBoxFuture<'_, EyreResult>> { + ) -> PinBoxFuture<'_, EyreResult>> { Box::pin(async move { let _guard = self.unlocked_inner.startup_lock.enter()?; @@ -204,44 +211,74 @@ impl PlatformNetwork for WasmNetwork { .address_filter() .is_ip_addr_punished(ip_addr) { - return Ok(NetworkResult::no_connection_other("punished")); + return Ok(NetworkAttempt::Performed( + NetworkResult::no_connection_other("punished"), + )); } - match dial_info.protocol_type() { - ProtocolType::UDP => { - bail!("no support for UDP protocol") - } - ProtocolType::TCP => { - bail!("no support for TCP protocol") - } - ProtocolType::WS => { - let pnc = network_result_try!(ws::WebsocketProtocolHandler::connect( - self.registry(), - dial_info, - timeout_ms - ) - .await - .wrap_err("connect failure")?); - network_result_try!(pnc.send(data).await.wrap_err("send failure")?); - } - #[cfg(feature = "enable-protocol-wss")] - ProtocolType::WSS => { - let pnc = network_result_try!(ws::WebsocketProtocolHandler::connect( - self.registry(), - dial_info, - timeout_ms - ) - .await - .wrap_err("connect failure")?); - network_result_try!(pnc.send(data).await.wrap_err("send failure")?); - } - }; + // No contact method selection here to steer around a suppressed transport + if self + .network_manager() + .transport_stats() + .unbound_send_deprioritized(&dial_info) + { + return Ok(NetworkAttempt::Skipped); + } + + // The address names the routing domain, so nodeless traffic still counts + let stats_dial_info = dial_info.clone(); + let send_result: EyreResult> = async { + match dial_info.protocol_type() { + ProtocolType::UDP => { + bail!("no support for UDP protocol") + } + ProtocolType::TCP => { + bail!("no support for TCP protocol") + } + ProtocolType::WS => { + let pnc = network_result_try!(ws::WebsocketProtocolHandler::connect( + self.registry(), + dial_info, + timeout_ms + ) + .await + .wrap_err("connect failure")?); + network_result_try!(pnc.send(data).await.wrap_err("send failure")?); + } + #[cfg(feature = "enable-protocol-wss")] + ProtocolType::WSS => { + let pnc = network_result_try!(ws::WebsocketProtocolHandler::connect( + self.registry(), + dial_info, + timeout_ms + ) + .await + .wrap_err("connect failure")?); + network_result_try!(pnc.send(data).await.wrap_err("send failure")?); + } + }; + Ok(NetworkResult::Value(())) + } + .await; + + if let Ok(nr) = &send_result { + self.network_manager() + .transport_stats() + .record_unbound_send( + &stats_dial_info, + TransportSendOutcome::from_network_result(nr), + ); + } + let nr = send_result?; + if !matches!(nr, NetworkResult::Value(_)) { + return Ok(NetworkAttempt::Performed(nr)); + } // Network accounting self.network_manager() .stats_packet_sent(ip_addr, ByteCount::new(data_len as u64)); - Ok(NetworkResult::Value(())) + Ok(NetworkAttempt::Performed(NetworkResult::Value(()))) }) } @@ -250,7 +287,7 @@ impl PlatformNetwork for WasmNetwork { dial_info: DialInfo, data: Bytes, timeout_ms: u32, - ) -> PinBoxFuture<'_, EyreResult>> { + ) -> PinBoxFuture<'_, EyreResult>> { Box::pin(async move { let _guard = self.startup_lock.enter()?; @@ -267,55 +304,85 @@ impl PlatformNetwork for WasmNetwork { .address_filter() .is_ip_addr_punished(ip_addr) { - return Ok(NetworkResult::no_connection_other("punished")); + return Ok(NetworkAttempt::Performed( + NetworkResult::no_connection_other("punished"), + )); } - match dial_info.protocol_type() { - ProtocolType::UDP => { - bail!("no support for UDP protocol") - } - ProtocolType::TCP => { - bail!("no support for TCP protocol") - } - _ => { - let pnc = network_result_try!(match dial_info.protocol_type() { - ProtocolType::UDP => bail!("no support for UDP protocol"), - ProtocolType::TCP => bail!("no support for TCP protocol"), - ProtocolType::WS => { - ws::WebsocketProtocolHandler::connect( + // No contact method selection here to steer around a suppressed transport + if network_manager + .transport_stats() + .unbound_send_deprioritized(&dial_info) + { + return Ok(NetworkAttempt::Skipped); + } + + let result: EyreResult> = async { + match dial_info.protocol_type() { + ProtocolType::UDP => { + bail!("no support for UDP protocol") + } + ProtocolType::TCP => { + bail!("no support for TCP protocol") + } + _ => { + let stats_dial_info = dial_info.clone(); + let connect_res = match dial_info.protocol_type() { + ProtocolType::UDP => bail!("no support for UDP protocol"), + ProtocolType::TCP => bail!("no support for TCP protocol"), + ProtocolType::WS => ws::WebsocketProtocolHandler::connect( self.registry(), dial_info, connect_timeout_ms, ) .await - .wrap_err("connect failure")? - } - #[cfg(feature = "enable-protocol-wss")] - ProtocolType::WSS => { - ws::WebsocketProtocolHandler::connect( + .wrap_err("connect failure")?, + #[cfg(feature = "enable-protocol-wss")] + ProtocolType::WSS => ws::WebsocketProtocolHandler::connect( self.registry(), dial_info, connect_timeout_ms, ) .await - .wrap_err("connect failure")? - } - }); + .wrap_err("connect failure")?, + }; + // Attribute connect+send only; a missing reply is the peer's silence + let pnc = match connect_res { + NetworkResult::Value(pnc) => pnc, + nr => { + network_manager.transport_stats().record_unbound_send( + &stats_dial_info, + TransportSendOutcome::from_network_result(&nr), + ); + return Ok(nr.map(|_| unreachable!())); + } + }; + let send_res = pnc.send(data).await.wrap_err("send failure")?; + network_manager.transport_stats().record_unbound_send( + &stats_dial_info, + TransportSendOutcome::from_network_result(&send_res), + ); + network_result_try!(send_res); + network_manager.stats_packet_sent(ip_addr, ByteCount::new(data_len as u64)); - network_result_try!(pnc.send(data).await.wrap_err("send failure")?); - network_manager.stats_packet_sent(ip_addr, ByteCount::new(data_len as u64)); - - let out = - network_result_try!(network_result_try!(timeout(timeout_ms, pnc.recv()) - .await - .into_network_result()) + let out = network_result_try!(network_result_try!(timeout( + timeout_ms, + pnc.recv() + ) + .await + .into_network_result()) .wrap_err("recv failure")?); - network_manager.stats_packet_rcvd(ip_addr, ByteCount::new(out.len() as u64)); + network_manager + .stats_packet_rcvd(ip_addr, ByteCount::new(out.len() as u64)); - Ok(NetworkResult::Value(out)) + Ok(NetworkResult::Value(out)) + } } } + .await; + + Ok(NetworkAttempt::Performed(result?)) }) } @@ -324,7 +391,7 @@ impl PlatformNetwork for WasmNetwork { &self, flow: Flow, data: Bytes, - ) -> PinBoxFuture<'_, EyreResult> { + ) -> PinBoxFuture<'_, EyreResult>> { Box::pin(async move { let _guard = self.startup_lock.enter()?; @@ -346,25 +413,31 @@ impl PlatformNetwork for WasmNetwork { // Try to send to the exact existing connection if one exists if let Some(conn) = network_manager.connection_manager().get_connection(flow) { - // connection exists, send over it - match conn.send_async(data).await { - ConnectionHandleSendResult::Sent => { + // A stalled connection parks channel sends; bound it so it fails + let config = self.config(); + let net_config = &config.internal().network; + let timeout_ms = net_config + .rpc + .timeout_ms + .min(net_config.single_send_timeout_ms); + match timeout(timeout_ms, conn.send_async(data)).await { + Ok(ConnectionHandleSendResult::Sent) => { // Network accounting network_manager.stats_packet_sent(ip_addr, ByteCount::new(data_len as u64)); - // Data was consumed - return Ok(SendDataToExistingFlowResult::Sent(conn.unique_flow())); + return Ok(NetworkAttempt::value(conn.unique_flow())); } - ConnectionHandleSendResult::NotSent(data) => { - // Couldn't send - // Pass the data back out so we don't own it any more - return Ok(SendDataToExistingFlowResult::NotSent(data)); + Ok(ConnectionHandleSendResult::NotSent) => { + return Ok(NetworkAttempt::Skipped); + } + Err(_) => { + veilid_log!(self debug "connection send timed out: flow={:?}", flow); + return Ok(NetworkAttempt::Performed(NetworkResult::Timeout)); } } } // Connection didn't exist - // Pass the data back out so we don't own it any more - Ok(SendDataToExistingFlowResult::NotSent(data)) + Ok(NetworkAttempt::Skipped) }) } @@ -396,11 +469,25 @@ impl PlatformNetwork for WasmNetwork { .await? ); - if let ConnectionHandleSendResult::NotSent(_) = conn.send_async(data).await { - return Ok(NetworkResult::NoConnection(io::Error::new( - io::ErrorKind::ConnectionReset, - "failed to send", - ))); + // A stalled connection parks channel sends; bound it so it fails + let config = self.config(); + let net_config = &config.internal().network; + let timeout_ms = net_config + .rpc + .timeout_ms + .min(net_config.single_send_timeout_ms); + match timeout(timeout_ms, conn.send_async(data)).await { + Ok(ConnectionHandleSendResult::Sent) => {} + Ok(ConnectionHandleSendResult::NotSent) => { + return Ok(NetworkResult::NoConnection(io::Error::new( + io::ErrorKind::ConnectionReset, + "failed to send", + ))); + } + Err(_) => { + veilid_log!(self debug "connection send timed out: dial_info={:?}", conn.unique_flow().flow); + return Ok(NetworkResult::timeout()); + } } let unique_flow = conn.unique_flow(); diff --git a/veilid-core/src/network_manager/network_connection.rs b/veilid-core/src/network_manager/network_connection.rs index 372f4259..6463223b 100644 --- a/veilid-core/src/network_manager/network_connection.rs +++ b/veilid-core/src/network_manager/network_connection.rs @@ -255,6 +255,7 @@ impl NetworkConnection { UniqueFlow { flow: self.flow, connection_id: Some(self.connection_id), + dialed: self.opt_dial_info.is_some(), } } @@ -264,6 +265,8 @@ impl NetworkConnection { self.flow, self.sender.clone(), self.dead.clone(), + // Outbound-initiated connections carry the dial info they used + self.opt_dial_info.is_some(), ) } diff --git a/veilid-core/src/network_manager/node_contact_method_cache.rs b/veilid-core/src/network_manager/node_contact_method_cache.rs index dd9a5e74..50c46152 100644 --- a/veilid-core/src/network_manager/node_contact_method_cache.rs +++ b/veilid-core/src/network_manager/node_contact_method_cache.rs @@ -29,6 +29,7 @@ enum NodeContactMethodKind { NoRoutingDomain, NoPeerInfo, NoContactMethod, + TransportSuppressed, Punished, Existing, Direct, @@ -43,6 +44,9 @@ impl From<&NodeContactMethodResult> for NodeContactMethodKind { NodeContactMethodResult::NoRoutingDomain => NodeContactMethodKind::NoRoutingDomain, NodeContactMethodResult::NoPeerInfo => NodeContactMethodKind::NoPeerInfo, NodeContactMethodResult::NoContactMethod => NodeContactMethodKind::NoContactMethod, + NodeContactMethodResult::TransportSuppressed => { + NodeContactMethodKind::TransportSuppressed + } NodeContactMethodResult::Punished => NodeContactMethodKind::Punished, NodeContactMethodResult::Resolved(ContactMethod::Existing) => { NodeContactMethodKind::Existing diff --git a/veilid-core/src/network_manager/online_detector.rs b/veilid-core/src/network_manager/online_detector.rs index 60779ae7..4454090e 100644 --- a/veilid-core/src/network_manager/online_detector.rs +++ b/veilid-core/src/network_manager/online_detector.rs @@ -10,11 +10,47 @@ pub const OFFLINE_DETECTION_FAILURE_THRESHOLD_MESSAGE: usize = 4; #[derive(Copy, Clone, Debug, Eq, PartialEq, Default)] pub enum OnlineState { + /// Round trips complete and failure evidence is trustworthy #[default] Online, + /// Round trips complete but failure evidence is about us; blame writers hold fire + Degraded, + /// Nothing gets through; blame and recovery behaviors both gate Offline, } +impl OnlineState { + /// Whether blame writers should hold fire: in Degraded or Offline the failure + /// evidence is about us, not the peer it names. + pub fn suppresses_blame(&self) -> bool { + !matches!(self, OnlineState::Online) + } +} + +/// How long ping-derived offline must persist before it counts as a real outage. +/// Sub-second blips (missed ping deadlines under load, idle-node sparse probes) carry +/// no transport information, and the carrier-NAT-flush wedge it must still catch lasts +/// minutes. +const OFFLINE_SUSTAINED_SECS: u32 = 5; + +/// Sliding window that failure breadth accumulates over for degradation detection (~2x rpc timeout) +const DEGRADED_EVIDENCE_WINDOW_SECS: u32 = 10; + +/// Base clean time after the last degradation evidence before Degraded can end +pub(crate) const DEGRADED_EXIT_CLEAN_SECS: u32 = 5; + +/// Ceiling on the escalated clean window +pub(crate) const DEGRADED_EXIT_CLEAN_MAX_SECS: u32 = 20; + +/// Recovering and degrading again within this is one marginal period, not two +const DEGRADED_REENTRY_SECS: u32 = 30; + +/// Degraded decays after this long without a post-evidence success +pub(crate) const DEGRADED_MAX_HOLD_SECS: u32 = 60; + +/// Local pipeline stalls at or above this consume enough of the rpc timeout to void peer evidence +const LOCAL_STALL_MIN_MS: u64 = 2500; + /// Failure count threshold for a socket class. fn failure_threshold(framing_type: FramingType) -> usize { match framing_type { @@ -23,6 +59,46 @@ fn failure_threshold(framing_type: FramingType) -> usize { } } +/// Clean window required to leave Degraded, doubled per consecutive rapid re-entry. +/// +/// A marginal period is evidence with gaps in it, and at the base window those gaps open +/// brief Online windows for blame to land in. Escalating closes them on a network that +/// keeps re-degrading; a recovered one stays on the fast path. +pub(crate) fn degraded_exit_clean_secs(reentries: u32) -> u32 { + DEGRADED_EXIT_CLEAN_SECS + .saturating_mul(1u32 << reentries.min(16)) + .min(DEGRADED_EXIT_CLEAN_MAX_SECS) +} + +/// The three-state answer, derived from the stored two-state machine plus +/// degradation evidence. Pure so the timing rules are testable without a network. +/// +/// Entry is fast, exit is slow: fresh evidence holds Degraded, and leaving requires a +/// success AFTER the evidence (an interleaved success is what keeps the offline detector +/// blind), with the max hold bounding the idle case. +pub(crate) fn effective_online_state( + stored: OnlineState, + last_evidence_ts: Option, + last_success_ts: Option, + clean_secs: u32, + now: Timestamp, +) -> OnlineState { + if stored == OnlineState::Offline { + return OnlineState::Offline; + } + let Some(ev) = last_evidence_ts else { + return OnlineState::Online; + }; + if now < ev.later(TimestampDuration::new_secs(clean_secs)) { + return OnlineState::Degraded; + } + let recovered = last_success_ts.is_some_and(|s| s > ev); + if recovered || now >= ev.later(TimestampDuration::new_secs(DEGRADED_MAX_HOLD_SECS)) { + return OnlineState::Online; + } + OnlineState::Degraded +} + /// Events that drive the online/offline state machine. enum OnlineEvent { /// Received something from outside. @@ -44,6 +120,18 @@ enum OnlineEvent { struct OnlineDetectionPerDomain { /// Previously-live nodes that failed on each framing type since the last success. pending_failures: BTreeMap>, + /// The same breadth over a time window, which an interleaved success cannot erase + windowed_failures: BTreeMap>, + /// Last time the windowed breadth crossed threshold on every enabled framing type + last_degraded_evidence_ts: Option, + /// Last completed round trip; exiting Degraded requires one after the evidence + last_success_ts: Option, + /// Last committed three-state answer, updated on events and lazily on reads + effective_state: OnlineState, + /// Consecutive Degraded re-entries within `DEGRADED_REENTRY_SECS`; escalates the exit window + degraded_reentries: u32, + /// When Degraded last ended, for the rapid-re-entry test + degraded_recovered_ts: Option, /// When we last switched online/offline. last_transition_ts: Timestamp, /// Current state. @@ -52,46 +140,68 @@ struct OnlineDetectionPerDomain { flap_detector: FlapDetector, /// Start of the current offline period, if currently offline. offline_started_ts: Option, + /// Whether this offline period's connection purge has already run. + offline_purged: bool, /// Recently-completed offline intervals (start, end), pruned to /// `RELIABLE_PING_INTERVAL_MAX` old. Credits node reliability for time we /// couldn't test. recent_offline_intervals: VecDeque<(Timestamp, Timestamp)>, + /// Start of the current degraded period, if effectively degraded + degraded_started_ts: Option, + /// The same for degraded periods, where we could test but must not believe the answer. + recent_degraded_intervals: VecDeque<(Timestamp, Timestamp)>, /// Cumulative online<->offline transition count #[cfg(feature = "stats")] total_transitions: u64, /// Cumulative time spent offline #[cfg(feature = "stats")] total_time_offline: TimestampDuration, + /// Cumulative time spent degraded + #[cfg(feature = "stats")] + total_time_degraded: TimestampDuration, } impl Default for OnlineDetectionPerDomain { fn default() -> Self { Self { pending_failures: BTreeMap::new(), + windowed_failures: BTreeMap::new(), + last_degraded_evidence_ts: None, + last_success_ts: None, + effective_state: OnlineState::default(), + degraded_reentries: 0, + degraded_recovered_ts: None, last_transition_ts: Timestamp::new(0), online_state: OnlineState::default(), // 4 online/offline flips within ~60s = flapping flap_detector: FlapDetector::new_secs(4.0, 60), offline_started_ts: None, + offline_purged: false, recent_offline_intervals: VecDeque::new(), + degraded_started_ts: None, + recent_degraded_intervals: VecDeque::new(), #[cfg(feature = "stats")] total_transitions: 0, #[cfg(feature = "stats")] total_time_offline: TimestampDuration::new(0), + #[cfg(feature = "stats")] + total_time_degraded: TimestampDuration::new(0), } } } /// Detects if our node is online or offline. /// -/// In order to determine if network errors are 'our fault 'or 'the other side's fault', we need to -/// generally know if we are online or offline, before assigning blame to another node and changing -/// its state. This +/// To determine if network errors are 'our fault' or 'the other side's fault', we need to +/// generally know if we are online or offline, before assigning blame to another node and +/// changing its state. pub struct OnlineDetector { registry: VeilidComponentRegistry, inner: Mutex>, /// Framing types the network uses; only these gate offline detection. enabled_framing_types: Mutex, + /// Last local pipeline stall long enough to void timeout evidence; process-wide + last_local_stall: Mutex>, } impl_veilid_component_accessors!(OnlineDetector); @@ -112,6 +222,7 @@ impl OnlineDetector { inner: Mutex::new(BTreeMap::new()), // Require both classes until the network reports its protocol config. enabled_framing_types: Mutex::new(FramingTypeSet::all()), + last_local_stall: Mutex::new(None), } } @@ -195,17 +306,52 @@ impl OnlineDetector { } } + /// Report a measured local pipeline stall (executor tick lag, envelope build). + /// + /// A peer cannot answer inside a timeout we spent ourselves, so long stalls void + /// timeout evidence. Short ones are filtered here, so call sites report unconditionally + /// from their slow branches. + pub fn record_local_stall(&self, duration: TimestampDuration) { + if duration < TimestampDuration::new_ms(LOCAL_STALL_MIN_MS) { + return; + } + let now = Timestamp::now(); + *self.last_local_stall.lock() = Some(now); + // Commit promptly so gates and the transition log fire before the next read + for routing_domain in RoutingDomain::all() { + if !Self::uses_ping_detection(routing_domain) { + continue; + } + let opt_transition = { + let mut inner = self.inner.lock(); + let pd = inner.entry(routing_domain).or_default(); + Self::commit_effective(pd, Some(now), now) + }; + self.log_effective_transition(routing_domain, opt_transition, "local stall"); + } + } + + /// Whether blame writers should hold fire for this routing domain. + pub fn blame_suppressed(&self, routing_domain: RoutingDomain) -> bool { + self.online_state(routing_domain).suppresses_blame() + } + /// Current online state for a routing domain. pub fn online_state(&self, routing_domain: RoutingDomain) -> OnlineState { // Domains without ping detection are always considered online. if !Self::uses_ping_detection(routing_domain) { return OnlineState::Online; } - self.inner - .lock() - .get(&routing_domain) - .map(|pd| pd.online_state) - .unwrap_or_default() + let local_stall_ts = *self.last_local_stall.lock(); + let now = Timestamp::now(); + let (state, opt_transition) = { + let mut inner = self.inner.lock(); + let pd = inner.entry(routing_domain).or_default(); + let opt_transition = Self::commit_effective(pd, local_stall_ts, now); + (pd.effective_state, opt_transition) + }; + self.log_effective_transition(routing_domain, opt_transition, "evidence aged"); + state } /// When this domain last came back online, if within the reliable ping span. @@ -234,6 +380,30 @@ impl OnlineDetector { routing_domain: RoutingDomain, from: Timestamp, to: Timestamp, + ) -> TimestampDuration { + self.interval_overlap(routing_domain, from, to, false) + } + + /// The same, plus degraded time, where the blame gates discarded what came back. + /// + /// Only for spans that condemn. Degraded time still carries positive evidence + /// (successes are recorded throughout), so it must not be deducted from how long a + /// node has been answering. + pub fn blame_suppressed_overlap( + &self, + routing_domain: RoutingDomain, + from: Timestamp, + to: Timestamp, + ) -> TimestampDuration { + self.interval_overlap(routing_domain, from, to, true) + } + + fn interval_overlap( + &self, + routing_domain: RoutingDomain, + from: Timestamp, + to: Timestamp, + include_degraded: bool, ) -> TimestampDuration { if !Self::uses_ping_detection(routing_domain) || to <= from { return TimestampDuration::new(0); @@ -242,13 +412,21 @@ impl OnlineDetector { let Some(pd) = inner.get(&routing_domain) else { return TimestampDuration::new(0); }; + let empty: VecDeque<(Timestamp, Timestamp)> = VecDeque::new(); + let (degraded_intervals, opt_degraded_started) = if include_degraded { + (&pd.recent_degraded_intervals, pd.degraded_started_ts) + } else { + (&empty, None) + }; let mut overlap = 0u64; - // Completed intervals plus any ongoing offline period extended to `to`. + // Offline and degraded never overlap: entering one closes the other for (s, e) in pd .recent_offline_intervals .iter() + .chain(degraded_intervals.iter()) .copied() .chain(pd.offline_started_ts.map(|s| (s, to))) + .chain(opt_degraded_started.map(|s| (s, to))) { let lo = s.max(from); let hi = e.min(to); @@ -259,13 +437,13 @@ impl OnlineDetector { TimestampDuration::new(overlap) } - /// Drive the state machine with an event, dispatching to the current state's handler. + /// Stats snapshot: (online, degraded, transitions, time offline, time degraded). #[cfg(feature = "stats")] pub(crate) fn online_stats( &self, routing_domain: RoutingDomain, now: Timestamp, - ) -> (bool, u64, TimestampDuration) { + ) -> (bool, bool, u64, TimestampDuration, TimestampDuration) { let inner = self.inner.lock(); match inner.get(&routing_domain) { Some(pd) => { @@ -273,38 +451,144 @@ impl OnlineDetector { if let Some(start) = pd.offline_started_ts { time_offline = time_offline.saturating_add(now.duration_since(start)); } + let mut time_degraded = pd.total_time_degraded; + if let Some(start) = pd.degraded_started_ts { + time_degraded = time_degraded.saturating_add(now.duration_since(start)); + } ( matches!(pd.online_state, OnlineState::Online), + matches!(pd.effective_state, OnlineState::Degraded), pd.total_transitions, time_offline, + time_degraded, ) } - None => (true, 0, TimestampDuration::new(0)), + None => ( + true, + false, + 0, + TimestampDuration::new(0), + TimestampDuration::new(0), + ), } } + /// Recompute and commit the effective three-state answer; returns a + /// transition for the caller to log outside the lock. + fn commit_effective( + pd: &mut OnlineDetectionPerDomain, + local_stall_ts: Option, + now: Timestamp, + ) -> Option<(OnlineState, OnlineState, u32)> { + let evidence = pd.last_degraded_evidence_ts.max(local_stall_ts); + let clean_secs = degraded_exit_clean_secs(pd.degraded_reentries); + let new_state = effective_online_state( + pd.online_state, + evidence, + pd.last_success_ts, + clean_secs, + now, + ); + let old_state = pd.effective_state; + if new_state == old_state { + return None; + } + // Decided on entry, held for the whole degraded period + match (old_state, new_state) { + (OnlineState::Degraded, _) => pd.degraded_recovered_ts = Some(now), + (_, OnlineState::Degraded) => { + let rapid = pd.degraded_recovered_ts.is_some_and(|ts| { + now.duration_since(ts) <= TimestampDuration::new_secs(DEGRADED_REENTRY_SECS) + }); + pd.degraded_reentries = if rapid { + pd.degraded_reentries.saturating_add(1) + } else { + 0 + }; + } + _ => {} + } + pd.effective_state = new_state; + if new_state == OnlineState::Degraded { + pd.degraded_started_ts = Some(now); + } else if let Some(start) = pd.degraded_started_ts.take() { + veilid_stats!( + pd.total_time_degraded = pd + .total_time_degraded + .saturating_add(now.duration_since(start)) + ); + pd.recent_degraded_intervals.push_back((start, now)); + while pd.recent_degraded_intervals.front().is_some_and(|(_, e)| { + now.duration_since(*e) > crate::routing_table::RELIABLE_PING_INTERVAL_MAX + }) { + pd.recent_degraded_intervals.pop_front(); + } + } + Some(( + old_state, + new_state, + degraded_exit_clean_secs(pd.degraded_reentries), + )) + } + + /// Degraded transitions log here; online and offline log from the stored machine. + fn log_effective_transition( + &self, + routing_domain: RoutingDomain, + opt_transition: Option<(OnlineState, OnlineState, u32)>, + why: &str, + ) { + let Some((old_state, new_state, clean_secs)) = opt_transition else { + return; + }; + match (old_state, new_state) { + (_, OnlineState::Degraded) => { + veilid_log!(self info "{:?} degraded detected ({}): blame suppressed, exit needs {}s clean", routing_domain, why, clean_secs); + } + (OnlineState::Degraded, OnlineState::Online) => { + veilid_log!(self info "{:?} recovered from degraded", routing_domain); + } + _ => {} + } + } + + /// Drive the state machine with an event, dispatching to the current state's handler. fn process_event(&self, routing_domain: RoutingDomain, event: OnlineEvent) { if !Self::uses_ping_detection(routing_domain) { return; } let enabled_framing_types = *self.enabled_framing_types.lock(); + let local_stall_ts = *self.last_local_stall.lock(); let now = Timestamp::now(); - let (opt_new_state, opt_flap_penalty) = { + let (opt_new_state, opt_flap_penalty, do_purge, purge_reason, do_wipe, opt_eff_transition) = { let mut inner = self.inner.lock(); let pd = inner.entry(routing_domain).or_default(); + // Exiting Degraded needs a round trip after the evidence, so stamp in every state + if matches!(event, OnlineEvent::Success) { + pd.last_success_ts = Some(now); + } let opt_new_state = match pd.online_state { OnlineState::Offline => Self::process_offline_event(&event), - OnlineState::Online => { - Self::process_online_event(pd, &event, enabled_framing_types) + OnlineState::Online | OnlineState::Degraded => { + Self::process_online_event(pd, &event, enabled_framing_types, now) } }; let mut opt_flap_penalty = None; + let mut do_wipe = false; if let Some(new_state) = opt_new_state { match new_state { - OnlineState::Offline => pd.offline_started_ts = Some(now), + // The stored machine is two-state; Degraded is the derived overlay + OnlineState::Degraded => unreachable!(), + OnlineState::Offline => { + pd.offline_started_ts = Some(now); + pd.offline_purged = false; + } OnlineState::Online => { if let Some(start) = pd.offline_started_ts.take() { + // Only a sustained outage invalidates transport evidence + do_wipe = now.duration_since(start) + >= TimestampDuration::new_secs(OFFLINE_SUSTAINED_SECS); pd.recent_offline_intervals.push_back((start, now)); veilid_stats!( pd.total_time_offline = pd @@ -332,21 +616,60 @@ impl OnlineDetector { opt_flap_penalty = pd.flap_detector.record(now.as_u64(), new_state); } } - (opt_new_state, opt_flap_penalty) + + // Purge policy: a NAT rebind black-holes established flows, so a real outage + // must close everything and a blip must not. An interface change is a confirmed + // move (purge at once); ping-derived offline must persist first, rechecked + // on each event while offline. + let mut do_purge = false; + let mut purge_reason = ""; + if pd.online_state == OnlineState::Offline && !matches!(event, OnlineEvent::Detached) { + // A move purges even mid-outage: recovery attempts may have opened + // fresh connections on the old interface since the last purge + let confirmed_move = matches!(event, OnlineEvent::NetworkChange); + let sustained = !pd.offline_purged + && pd.offline_started_ts.is_some_and(|start| { + now.duration_since(start) + >= TimestampDuration::new_secs(OFFLINE_SUSTAINED_SECS) + }); + if confirmed_move || sustained { + pd.offline_purged = true; + do_purge = true; + purge_reason = if confirmed_move { + "interface change" + } else { + "sustained offline" + }; + } + } + let opt_eff_transition = Self::commit_effective(pd, local_stall_ts, now); + ( + opt_new_state, + opt_flap_penalty, + do_purge, + purge_reason, + do_wipe, + opt_eff_transition, + ) }; // Log the transition (and any flapping) outside the lock. if let Some(new_state) = opt_new_state { veilid_log!(self info "{:?} {} detected", routing_domain, if new_state == OnlineState::Online { "online" } else { "offline" }); - - // A NAT rebind kills established flows silently; close them so recovery pings - // reconnect instead of riding dead ones - if new_state == OnlineState::Offline && !matches!(event, OnlineEvent::Detached) { - self.network_manager() - .connection_manager() - .close_all_connections(); - } + } + self.log_effective_transition(routing_domain, opt_eff_transition, "broad failures"); + if do_purge { + // close_all_connections is silent on an empty table; log the decision + veilid_log!(self debug "{:?} purging connections: {}", routing_domain, purge_reason); + self.network_manager() + .connection_manager() + .close_all_connections(); + } + if do_wipe { + self.network_manager() + .transport_stats() + .reset_routing_domain(routing_domain, "sustained offline ended"); } if let Some(penalty) = opt_flap_penalty { veilid_log!(self debugwarn "{:?} online state FLAPPING (penalty={:.1})", routing_domain, penalty); @@ -369,6 +692,7 @@ impl OnlineDetector { pd: &mut OnlineDetectionPerDomain, event: &OnlineEvent, enabled_framing_types: FramingTypeSet, + now: Timestamp, ) -> Option { match event { OnlineEvent::Success => { @@ -385,8 +709,23 @@ impl OnlineDetector { return None; } let bucket = pd.pending_failures.entry(*framing_type).or_default(); + let wbucket = pd.windowed_failures.entry(*framing_type).or_default(); for node_id in live_node_ids { bucket.insert(node_id.clone()); + wbucket.insert(node_id.clone(), now); + } + // Windowed breadth survives interleaved successes: the degraded signature + let window = TimestampDuration::new_secs(DEGRADED_EVIDENCE_WINDOW_SECS); + for wb in pd.windowed_failures.values_mut() { + wb.retain(|_, ts| now.duration_since(*ts) <= window); + } + let all_windowed = !enabled_framing_types.is_empty() + && enabled_framing_types.iter().all(|ft| { + pd.windowed_failures.get(&ft).map_or(0, BTreeMap::len) + >= failure_threshold(ft) + }); + if all_windowed { + pd.last_degraded_evidence_ts = Some(now); } // Offline only when every enabled socket class has crossed its threshold, so // localized trouble on one class doesn't drive offline while another works. @@ -403,6 +742,11 @@ impl OnlineDetector { } OnlineEvent::NetworkChange | OnlineEvent::Detached => { pd.pending_failures.clear(); + // Degradation evidence belongs to the network we just left + pd.windowed_failures.clear(); + pd.last_degraded_evidence_ts = None; + pd.degraded_reentries = 0; + pd.degraded_recovered_ts = None; Some(OnlineState::Offline) } } diff --git a/veilid-core/src/network_manager/send_data.rs b/veilid-core/src/network_manager/send_data.rs index 299abc4f..f30434ca 100644 --- a/veilid-core/src/network_manager/send_data.rs +++ b/veilid-core/src/network_manager/send_data.rs @@ -29,12 +29,29 @@ impl NetworkManager { ); if let Some(unique_flow) = sdr.unique_flow() { - self.set_last_flow(node_ref, unique_flow.flow, Timestamp::now()); + self.set_last_flow_from_send(node_ref, unique_flow.flow, Timestamp::now()); } Ok(sdr) } + /// Decision-point suppression gate for existing-flow shortcuts; a UDP "existing + /// flow" is only a remembered address, so reusing one would bypass suppression. + /// Spends the canary when due. + fn transport_deprioritized( + &self, + node_ref: &FilteredNodeRef, + transport: TransportType, + ) -> bool { + node_ref + .best_routing_domain() + .map(|rd| { + self.transport_stats() + .deprioritized_consuming_canary(rd, transport) + }) + .unwrap_or(false) + } + fn report_contact_method_result( &self, node_ref: FilteredNodeRef, @@ -87,6 +104,18 @@ impl NetworkManager { )) .await?; + // Outbound health only; says nothing about peers reaching us + if let (Some(routing_domain), Some(transport)) = + (node_ref.best_routing_domain(), sdr.opt_transport_type()) + { + self.transport_stats().record_send( + routing_domain, + transport, + node_ref.best_node_id(), + TransportSendOutcome::from(&sdr), + ); + } + let (ncm_back, nres) = sdr.destructure(); match &nres { @@ -184,7 +213,8 @@ impl NetworkManager { // how we can reply to inbound requests for that node. NodeContactMethodResult::NoRoutingDomain | NodeContactMethodResult::NoPeerInfo - | NodeContactMethodResult::NoContactMethod => { + | NodeContactMethodResult::NoContactMethod + | NodeContactMethodResult::TransportSuppressed => { pin_future_closure!(self.send_data_unreachable(destination_node_ref, data)).await? } NodeContactMethodResult::Resolved(ContactMethod::Existing) => { @@ -258,8 +288,11 @@ impl NetworkManager { target_node_ref: FilteredNodeRef, data: Bytes, ) -> EyreResult> { - // First try to send data to the last connection we've seen this peer on - let Some(flow) = target_node_ref.last_flow() else { + // Try the last flow seen for this peer, unless its transport is suppressed + let Some(flow) = target_node_ref + .last_flow() + .filter(|flow| !self.transport_deprioritized(&target_node_ref, flow.transport_type())) + else { return Ok(NetworkResult::no_connection_other(format!( "node was unreachable: {}", target_node_ref @@ -277,8 +310,11 @@ impl NetworkManager { )) .await? { - SendDataToExistingFlowResult::Sent(unique_flow) => unique_flow, - SendDataToExistingFlowResult::NotSent(_) => { + NetworkAttempt::Performed(NetworkResult::Value(unique_flow)) => unique_flow, + NetworkAttempt::Performed(NetworkResult::Timeout) => { + return Ok(NetworkResult::timeout()); + } + _ => { return Ok(NetworkResult::no_connection_other( "failed to send to existing flow", )); @@ -286,7 +322,7 @@ impl NetworkManager { }; // Update timestamp for this last connection since we just sent to it - self.set_last_flow( + self.set_last_flow_from_send( target_node_ref.unfiltered(), unique_flow.flow, Timestamp::now(), @@ -329,8 +365,11 @@ impl NetworkManager { )) .await? { - SendDataToExistingFlowResult::Sent(unique_flow) => unique_flow, - SendDataToExistingFlowResult::NotSent(_) => { + NetworkAttempt::Performed(NetworkResult::Value(unique_flow)) => unique_flow, + NetworkAttempt::Performed(NetworkResult::Timeout) => { + return Ok(NetworkResult::timeout()); + } + _ => { return Ok(NetworkResult::no_connection_other( "failed to send to existing flow", )); @@ -338,7 +377,7 @@ impl NetworkManager { }; // Update timestamp for this last connection since we just sent to it - self.set_last_flow(target_node_ref, unique_flow.flow, Timestamp::now()); + self.set_last_flow_from_send(target_node_ref, unique_flow.flow, Timestamp::now()); Ok(NetworkResult::value(unique_flow)) } @@ -359,24 +398,29 @@ impl NetworkManager { // as a signalled node gets to choose its own dial info for the reverse connection. let seq_target_node_ref = target_node_ref.sequencing_only(); - let data = if let Some(flow) = seq_target_node_ref.last_flow() { + let opt_last_flow = seq_target_node_ref.last_flow().filter(|flow| { + !self.transport_deprioritized(&seq_target_node_ref, flow.transport_type()) + }); + let data = if let Some(flow) = opt_last_flow { veilid_log!(self debug target:"contact_method", "ExistingConnection: {:?} for {:?}", flow, seq_target_node_ref ); let net = self.net(); - match pin_future!(net.send_data_to_existing_flow(flow, data).measure_debug( - TimestampDuration::new_secs(1), - veilid_log_dbg!( - self, - "NetworkManager::send_data_cm_signal_reverse send_data_to_existing_flow" - ) - )) + match pin_future!(net + .send_data_to_existing_flow(flow, data.clone()) + .measure_debug( + TimestampDuration::new_secs(1), + veilid_log_dbg!( + self, + "NetworkManager::send_data_cm_signal_reverse send_data_to_existing_flow" + ) + )) .await? { - SendDataToExistingFlowResult::Sent(unique_flow) => { + NetworkAttempt::Performed(NetworkResult::Value(unique_flow)) => { // Update timestamp for this last connection since we just sent to it - self.set_last_flow( + self.set_last_flow_from_send( target_node_ref.unfiltered(), unique_flow.flow, Timestamp::now(), @@ -384,11 +428,10 @@ impl NetworkManager { return Ok(NetworkResult::value(unique_flow)); } - SendDataToExistingFlowResult::NotSent(data) => { - // Couldn't send data to existing connection - // so pass the data back out - data + NetworkAttempt::Performed(NetworkResult::Timeout) => { + return Ok(NetworkResult::timeout()); } + _ => data, } } else { // No last connection @@ -435,24 +478,29 @@ impl NetworkManager { target_node_ref: FilteredNodeRef, data: Bytes, ) -> EyreResult> { - let data = if let Some(flow) = target_node_ref.last_flow() { + let opt_last_flow = target_node_ref + .last_flow() + .filter(|flow| !self.transport_deprioritized(&target_node_ref, flow.transport_type())); + let data = if let Some(flow) = opt_last_flow { veilid_log!(self debug target:"contact_method", "ExistingConnection: {:?} for {:?}", flow, target_node_ref ); let net = self.net(); - match pin_future!(net.send_data_to_existing_flow(flow, data).measure_debug( - TimestampDuration::new_secs(1), - veilid_log_dbg!( - self, - "NetworkManager::send_data_cm_signal_hole_punch send_data_to_existing_flow" - ) - )) + match pin_future!(net + .send_data_to_existing_flow(flow, data.clone()) + .measure_debug( + TimestampDuration::new_secs(1), + veilid_log_dbg!( + self, + "NetworkManager::send_data_cm_signal_hole_punch send_data_to_existing_flow" + ) + )) .await? { - SendDataToExistingFlowResult::Sent(unique_flow) => { + NetworkAttempt::Performed(NetworkResult::Value(unique_flow)) => { // Update timestamp for this last connection since we just sent to it - self.set_last_flow( + self.set_last_flow_from_send( target_node_ref.unfiltered(), unique_flow.flow, Timestamp::now(), @@ -460,11 +508,10 @@ impl NetworkManager { return Ok(NetworkResult::value(unique_flow)); } - SendDataToExistingFlowResult::NotSent(data) => { - // Couldn't send data to existing connection - // so pass the data back out - data + NetworkAttempt::Performed(NetworkResult::Timeout) => { + return Ok(NetworkResult::timeout()); } + _ => data, } } else { // No last connection @@ -517,7 +564,9 @@ impl NetworkManager { let filtered_node_ref = node_ref.custom_filtered(NodeRefFilter::from(dial_info.make_filter())); - let opt_last_flow = filtered_node_ref.last_flow(); + let opt_last_flow = filtered_node_ref.last_flow().filter(|flow| { + !self.transport_deprioritized(&filtered_node_ref, flow.transport_type()) + }); let data = if let Some(flow) = opt_last_flow { veilid_log!(self debug target:"contact_method", @@ -526,22 +575,28 @@ impl NetworkManager { ); let net = self.net(); - match pin_future!(net.send_data_to_existing_flow(flow, data).measure_debug( - TimestampDuration::new_secs(1), - veilid_log_dbg!( - self, - "NetworkManager::send_data_cm_direct send_data_to_existing_flow" - ) - )) + match pin_future!(net + .send_data_to_existing_flow(flow, data.clone()) + .measure_debug( + TimestampDuration::new_secs(1), + veilid_log_dbg!( + self, + "NetworkManager::send_data_cm_direct send_data_to_existing_flow" + ) + )) .await? { - SendDataToExistingFlowResult::Sent(unique_flow) => { - self.set_last_flow(node_ref, unique_flow.flow, Timestamp::now()); + NetworkAttempt::Performed(NetworkResult::Value(unique_flow)) => { + self.set_last_flow_from_send(node_ref, unique_flow.flow, Timestamp::now()); return Ok(NetworkResult::value(unique_flow)); } - SendDataToExistingFlowResult::NotSent(d) => { + NetworkAttempt::Performed(NetworkResult::Timeout) => { node_ref.clear_last_flow(flow); - d + return Ok(NetworkResult::timeout()); + } + _ => { + node_ref.clear_last_flow(flow); + data } } } else { @@ -561,7 +616,7 @@ impl NetworkManager { pin_future!(net.send_data_to_dial_info(dial_info.clone(), data)).await? ); - self.set_last_flow(node_ref, unique_flow.flow, Timestamp::now()); + self.set_last_flow_from_send(node_ref, unique_flow.flow, Timestamp::now()); Ok(NetworkResult::value(unique_flow)) } @@ -619,9 +674,7 @@ impl NetworkManager { target_node_ref_sequencing: target_node_ref.sequencing(), }; if let Some(contact_methods) = self.inner.lock().node_contact_method_cache.get(&ncm_key) { - return Ok(NodeContactMethodResult::Resolved( - self.pick_contact_method(target_node_ref.clone(), contact_methods)?, - )); + return self.pick_contact_method(target_node_ref.clone(), contact_methods); } // Determine if the published peer info is the same as our current peer info @@ -653,39 +706,86 @@ impl NetworkManager { .node_contact_method_cache .insert(ncm_key.clone(), contact_methods.clone()); - NodeContactMethodResult::Resolved( - self.pick_contact_method(target_node_ref.clone(), contact_methods)?, - ) + self.pick_contact_method(target_node_ref.clone(), contact_methods)? }; Ok(ncm_result) } - /// Pick a contact method from a list of contact methods + /// Pick a contact method from a list of contact methods, scored on its first + /// wire hop (a relay on a suppressed transport is as doomed as a direct target + /// on one) and sorted by, in order: + /// 1. our own transport suppressed (aggregated across all peers) + /// 2. this peer's transport failing (per-edge evidence) + /// 3. address type, IPv4 first + /// 4. last failure time, stalest first fn pick_contact_method( &self, target_node_ref: FilteredNodeRef, contact_methods: Vec, - ) -> EyreResult { - let mut contact_method_map = BTreeMap::>::new(); + ) -> EyreResult { + let opt_routing_domain = target_node_ref.best_routing_domain(); + let mut contact_method_map = BTreeMap::<(bool, bool, u8, u64), Vec>::new(); for cm in contact_methods { + let (suppressed, failing, address_pref) = match cm.opt_transport_type() { + Some(transport) => { + let suppressed = opt_routing_domain + .map(|rd| { + self.transport_stats() + .verdict(rd, transport) + .is_deprioritized() + }) + .unwrap_or(false); + let failing = target_node_ref.transport_health(transport).is_failing(); + // Prefer IPv4; IPv6 fails more often in practice + let address_pref = match transport.address_type() { + AddressType::IPV4 => 0u8, + AddressType::IPV6 => 1u8, + }; + (suppressed, failing, address_pref) + } + None => (false, false, 0u8), + }; let failure_ts = target_node_ref .get_contact_method_failure_ts(&cm) .map(|ts| ts.as_u64()) .unwrap_or(0); - contact_method_map.entry(failure_ts).or_default().push(cm); + contact_method_map + .entry((suppressed, failing, address_pref, failure_ts)) + .or_default() + .push(cm); } // Pick the first contact method in sorted order - // The least-recently failed contact method will be first - let Some(first_contact_method) = contact_method_map.into_values().flatten().next() else { + let Some(((suppressed, _, _, _), first_contact_method)) = contact_method_map + .into_iter() + .flat_map(|(k, cms)| cms.into_iter().map(move |cm| (k, cm))) + .next() + else { bail!( "no contact method found for target node ref {:?}", target_node_ref ); }; - Ok(first_contact_method) + let opt_rd_transport = opt_routing_domain.zip(first_contact_method.opt_transport_type()); + + // Every method rides a suppressed transport; the caller falls back or gives up + if suppressed { + if let Some((rd, transport)) = opt_rd_transport { + self.transport_stats().record_skip(rd, transport); + } + return Ok(NodeContactMethodResult::TransportSuppressed); + } + + // Spend the canary at decision time when the winner's transport is due one + if let Some((rd, transport)) = opt_rd_transport { + let _ = self + .transport_stats() + .deprioritized_consuming_canary(rd, transport); + } + + Ok(NodeContactMethodResult::Resolved(first_contact_method)) } /// Figure out how to reach a node from our own node over the best routing domain and reference the nodes we want to access @@ -829,11 +929,12 @@ impl NetworkManager { if let Some(flow) = inbound_nr.last_flow() { let net = self.net(); match pin_future!(net.send_data_to_existing_flow(flow, data)).await? { - SendDataToExistingFlowResult::Sent(unique_flow) => { - self.set_last_flow(target_nr, unique_flow.flow, Timestamp::now()); + NetworkAttempt::Performed(NetworkResult::Value(unique_flow)) => { + self.set_last_flow_from_send(target_nr, unique_flow.flow, Timestamp::now()); Ok(NetworkResult::value(unique_flow)) } - SendDataToExistingFlowResult::NotSent(_) => Ok(NetworkResult::no_connection_other( + NetworkAttempt::Performed(NetworkResult::Timeout) => Ok(NetworkResult::timeout()), + _ => Ok(NetworkResult::no_connection_other( "unable to send over reverse connection", )), } @@ -965,11 +1066,12 @@ impl NetworkManager { // And now use the existing connection to send over if let Some(flow) = inbound_nr.last_flow() { match self.net().send_data_to_existing_flow(flow, data).await? { - SendDataToExistingFlowResult::Sent(unique_flow) => { - self.set_last_flow(target_nr, unique_flow.flow, Timestamp::now()); + NetworkAttempt::Performed(NetworkResult::Value(unique_flow)) => { + self.set_last_flow_from_send(target_nr, unique_flow.flow, Timestamp::now()); Ok(NetworkResult::value(unique_flow)) } - SendDataToExistingFlowResult::NotSent(_) => Ok(NetworkResult::no_connection_other( + NetworkAttempt::Performed(NetworkResult::Timeout) => Ok(NetworkResult::timeout()), + _ => Ok(NetworkResult::no_connection_other( "unable to send over hole punch", )), } diff --git a/veilid-core/src/network_manager/stats.rs b/veilid-core/src/network_manager/stats.rs index 6d7e5cb5..3009282a 100644 --- a/veilid-core/src/network_manager/stats.rs +++ b/veilid-core/src/network_manager/stats.rs @@ -64,8 +64,10 @@ pub(super) struct NetworkManagerStats { #[derive(Debug, Serialize)] struct OnlineDomainStats { online: bool, + degraded: bool, transitions: u64, time_offline: TimestampDuration, + time_degraded: TimestampDuration, } impl NetworkManager { @@ -121,11 +123,14 @@ impl NetworkManager { .unwrap_or_default(); let now = Timestamp::now(); let domain_stats = |rd| { - let (online, transitions, time_offline) = self.online_detector().online_stats(rd, now); + let (online, degraded, transitions, time_offline, time_degraded) = + self.online_detector().online_stats(rd, now); OnlineDomainStats { online, + degraded, transitions, time_offline, + time_degraded, } }; Some(Box::new(NetworkManagerStats { diff --git a/veilid-core/src/network_manager/tasks/mod.rs b/veilid-core/src/network_manager/tasks/mod.rs index 214a05f9..b6e80e67 100644 --- a/veilid-core/src/network_manager/tasks/mod.rs +++ b/veilid-core/src/network_manager/tasks/mod.rs @@ -11,6 +11,19 @@ impl NetworkManager { rolling_transfers_task_routine ); + // Set transport stats health log task + { + let registry = self.registry(); + self.transport_stats_log_task + .set_routine(move |_s, _l, _t| { + let registry = registry.clone(); + Box::pin(async move { + registry.network_manager().transport_stats().log_health(); + Ok(()) + }) + }); + } + // Set address filter task { let registry = self.registry(); @@ -51,6 +64,9 @@ impl NetworkManager { // Run the address filter task self.address_filter_task.tick().await?; + // Log our own per-transport send health + self.transport_stats_log_task.tick().await?; + // Run the low level network tick net.tick().await?; @@ -76,6 +92,11 @@ impl NetworkManager { veilid_log!(self warn "rolling_transfers_task not stopped: {}", e); } + veilid_log!(self debug "stopping transport stats log task"); + if let Err(e) = self.transport_stats_log_task.stop().await { + veilid_log!(self warn "transport_stats_log_task not stopped: {}", e); + } + veilid_log!(self debug "stopping address filter task"); if let Err(e) = self.address_filter_task.stop().await { veilid_log!(self warn "address_filter_task not stopped: {}", e); diff --git a/veilid-core/src/network_manager/tests_network_manager/mod.rs b/veilid-core/src/network_manager/tests_network_manager/mod.rs index 45fd1411..e2041e6a 100644 --- a/veilid-core/src/network_manager/tests_network_manager/mod.rs +++ b/veilid-core/src/network_manager/tests_network_manager/mod.rs @@ -1,6 +1,9 @@ mod mocks; +mod test_address_family_confirm; mod test_bootstrap; mod test_connection_table; +mod test_online_detector; +mod test_transport_stats; use super::*; pub use mocks::*; @@ -11,4 +14,7 @@ pub async fn test_all() { test_connection_table::test_add_get_remove().await; test_connection_table::test_eviction().await; test_connection_table::test_dead_connection_skip().await; + test_online_detector::test_all(); + test_transport_stats::test_all(); + test_address_family_confirm::test_all(); } diff --git a/veilid-core/src/network_manager/tests_network_manager/test_address_family_confirm.rs b/veilid-core/src/network_manager/tests_network_manager/test_address_family_confirm.rs new file mode 100644 index 00000000..8500cece --- /dev/null +++ b/veilid-core/src/network_manager/tests_network_manager/test_address_family_confirm.rs @@ -0,0 +1,148 @@ +use super::*; + +// Candidate handles are irrelevant to the decision logic, so tests use () +fn candidates_with( + configured: AddressTypeSet, + confirmed: AddressTypeSet, + candidate_counts: &[(AddressType, usize)], +) -> AddressFamilyConfirmCandidates<()> { + let mut candidates: BTreeMap> = BTreeMap::new(); + for (at, count) in candidate_counts { + candidates.insert(*at, vec![(); *count]); + } + AddressFamilyConfirmCandidates { + configured_address_types: configured, + confirmed_address_types: confirmed, + candidates, + } +} + +pub fn test_state_classification() { + info!("--- test_state_classification ---"); + + // Nothing configured is its own state, not a failure + assert_eq!( + candidates_with(AddressTypeSet::empty(), AddressTypeSet::empty(), &[]).state(), + AddressFamilyConfirmState::NoConfiguredAddressTypes + ); + + // Everything already proven by live traffic + assert_eq!( + candidates_with(AddressTypeSet::all(), AddressTypeSet::all(), &[]).state(), + AddressFamilyConfirmState::AllConfiguredConfirmed + ); + + // A missing family with too few candidates blocks probing entirely + assert_eq!( + candidates_with( + AddressTypeSet::all(), + AddressTypeSet::only(AddressType::IPV4), + &[(AddressType::IPV6, ADDR_FAMILY_MIN_NODES_TO_CONFIRM - 1)], + ) + .state(), + AddressFamilyConfirmState::NotEnoughCandidates + ); + + // A missing family with no candidate entry at all is the same + assert_eq!( + candidates_with( + AddressTypeSet::all(), + AddressTypeSet::only(AddressType::IPV4), + &[], + ) + .state(), + AddressFamilyConfirmState::NotEnoughCandidates + ); + + // Enough candidates for every missing family: ready to probe + assert_eq!( + candidates_with( + AddressTypeSet::all(), + AddressTypeSet::empty(), + &[ + (AddressType::IPV4, ADDR_FAMILY_MIN_NODES_TO_CONFIRM), + (AddressType::IPV6, ADDR_FAMILY_MIN_NODES_TO_CONFIRM), + ], + ) + .state(), + AddressFamilyConfirmState::ConfirmCandidates + ); +} + +pub fn test_resolve_confirms_probe_survivors() { + info!("--- test_resolve_confirms_probe_survivors ---"); + + // A supported/condemned mix confirms the survivors + assert_eq!( + resolve_probe_dispositions( + AddressTypeSet::empty(), + &[ + (AddressType::IPV4, FamilyProbeDisposition::Supported), + (AddressType::IPV6, FamilyProbeDisposition::Unsupported), + ], + ), + ConfirmationPassOutcome::Confirm(AddressTypeSet::only(AddressType::IPV4)) + ); + + // A family proven by real traffic survives even when every probe fails + assert_eq!( + resolve_probe_dispositions( + AddressTypeSet::only(AddressType::IPV4), + &[(AddressType::IPV6, FamilyProbeDisposition::Unsupported)], + ), + ConfirmationPassOutcome::Confirm(AddressTypeSet::only(AddressType::IPV4)) + ); + + // Inconclusive is not proof of support + assert_eq!( + resolve_probe_dispositions( + AddressTypeSet::empty(), + &[ + (AddressType::IPV4, FamilyProbeDisposition::Supported), + (AddressType::IPV6, FamilyProbeDisposition::Inconclusive), + ], + ), + ConfirmationPassOutcome::Confirm(AddressTypeSet::only(AddressType::IPV4)) + ); +} + +pub fn test_resolve_cancelled_defers() { + info!("--- test_resolve_cancelled_defers ---"); + + // Shutdown mid-probe must not publish a partial verdict + assert_eq!( + resolve_probe_dispositions( + AddressTypeSet::empty(), + &[ + (AddressType::IPV4, FamilyProbeDisposition::Supported), + (AddressType::IPV6, FamilyProbeDisposition::Cancelled), + ], + ), + ConfirmationPassOutcome::Deferred + ); +} + +pub fn test_resolve_all_condemned_defers() { + info!("--- test_resolve_all_condemned_defers ---"); + + // Confirming the empty set would latch the node dark until restart: + // - an empty confirmed set stops all traffic + // - no traffic means no network-change event to re-arm confirmation + assert_eq!( + resolve_probe_dispositions( + AddressTypeSet::empty(), + &[ + (AddressType::IPV4, FamilyProbeDisposition::Unsupported), + (AddressType::IPV6, FamilyProbeDisposition::Unsupported), + ], + ), + ConfirmationPassOutcome::Deferred + ); +} + +pub fn test_all() { + test_state_classification(); + test_resolve_confirms_probe_survivors(); + test_resolve_cancelled_defers(); + test_resolve_all_condemned_defers(); +} diff --git a/veilid-core/src/network_manager/tests_network_manager/test_connection_table.rs b/veilid-core/src/network_manager/tests_network_manager/test_connection_table.rs index f486205c..b0d15db8 100644 --- a/veilid-core/src/network_manager/tests_network_manager/test_connection_table.rs +++ b/veilid-core/src/network_manager/tests_network_manager/test_connection_table.rs @@ -152,7 +152,7 @@ pub async fn test_dead_connection_skip() { c2.close(); assert!(matches!( c2h.send_async(Bytes::new()).await, - ConnectionHandleSendResult::NotSent(_) + ConnectionHandleSendResult::NotSent )); table.add_connection(c1).unwrap(); @@ -170,7 +170,7 @@ pub async fn test_dead_connection_skip() { assert_eq!(table.get_best_connection_by_remote(None, remote), None); assert!(matches!( c1h.send_async(Bytes::new()).await, - ConnectionHandleSendResult::NotSent(_) + ConnectionHandleSendResult::NotSent )); mock_registry::terminate(registry).await; diff --git a/veilid-core/src/network_manager/tests_network_manager/test_online_detector.rs b/veilid-core/src/network_manager/tests_network_manager/test_online_detector.rs new file mode 100644 index 00000000..19dcdb11 --- /dev/null +++ b/veilid-core/src/network_manager/tests_network_manager/test_online_detector.rs @@ -0,0 +1,167 @@ +use super::*; + +// Pure timing rules for the Degraded overlay: fast entry, slow exit + +fn ts(secs: u64) -> Timestamp { + Timestamp::new(secs * 1_000_000) +} + +fn base_clean() -> u32 { + degraded_exit_clean_secs(0) +} + +pub fn test_no_evidence_is_online() { + assert_eq!( + effective_online_state(OnlineState::Online, None, None, base_clean(), ts(100)), + OnlineState::Online + ); + assert_eq!( + effective_online_state( + OnlineState::Online, + None, + Some(ts(99)), + base_clean(), + ts(100) + ), + OnlineState::Online + ); +} + +pub fn test_offline_dominates() { + // Whatever the evidence says, the stored machine's Offline wins + assert_eq!( + effective_online_state( + OnlineState::Offline, + Some(ts(100)), + Some(ts(101)), + base_clean(), + ts(100) + ), + OnlineState::Offline + ); +} + +pub fn test_fresh_evidence_degrades() { + let ev = ts(100); + // Inside the clean window nothing else matters, success included + assert_eq!( + effective_online_state( + OnlineState::Online, + Some(ev), + Some(ts(102)), + base_clean(), + ts(103) + ), + OnlineState::Degraded + ); +} + +pub fn test_success_after_evidence_recovers() { + let ev = ts(100); + let after_clean = ts(100 + DEGRADED_EXIT_CLEAN_SECS as u64); + assert_eq!( + effective_online_state( + OnlineState::Online, + Some(ev), + Some(ts(101)), + base_clean(), + after_clean + ), + OnlineState::Online + ); +} + +pub fn test_stale_success_does_not_recover() { + // A success predating the evidence proves nothing about recovery + let ev = ts(100); + let after_clean = ts(100 + DEGRADED_EXIT_CLEAN_SECS as u64); + assert_eq!( + effective_online_state( + OnlineState::Online, + Some(ev), + Some(ts(99)), + base_clean(), + after_clean + ), + OnlineState::Degraded + ); + // Success exactly AT the evidence is still not after it + assert_eq!( + effective_online_state( + OnlineState::Online, + Some(ev), + Some(ev), + base_clean(), + after_clean + ), + OnlineState::Degraded + ); +} + +pub fn test_no_success_holds_until_decay() { + let ev = ts(100); + let before_decay = ts(100 + DEGRADED_MAX_HOLD_SECS as u64 - 1); + let at_decay = ts(100 + DEGRADED_MAX_HOLD_SECS as u64); + assert_eq!( + effective_online_state( + OnlineState::Online, + Some(ev), + None, + base_clean(), + before_decay + ), + OnlineState::Degraded + ); + // An idle node with no traffic to prove recovery decays back on its own + assert_eq!( + effective_online_state(OnlineState::Online, Some(ev), None, base_clean(), at_decay), + OnlineState::Online + ); +} + +pub fn test_clean_window_escalates_and_caps() { + assert_eq!(degraded_exit_clean_secs(0), DEGRADED_EXIT_CLEAN_SECS); + assert_eq!(degraded_exit_clean_secs(1), DEGRADED_EXIT_CLEAN_SECS * 2); + assert_eq!(degraded_exit_clean_secs(20), DEGRADED_EXIT_CLEAN_MAX_SECS); + // The escalated window must stay under the idle decay or it could never expire + const { assert!(DEGRADED_EXIT_CLEAN_MAX_SECS < DEGRADED_MAX_HOLD_SECS) }; +} + +pub fn test_escalated_window_closes_the_gap() { + // A gap after the success lets the base window open an Online hole before the + // next burst of evidence lands + let ev = ts(100); + let success = ts(101); + let in_gap = ts(100 + DEGRADED_EXIT_CLEAN_SECS as u64 + 1); + assert_eq!( + effective_online_state( + OnlineState::Online, + Some(ev), + Some(success), + base_clean(), + in_gap + ), + OnlineState::Online + ); + assert_eq!( + effective_online_state( + OnlineState::Online, + Some(ev), + Some(success), + degraded_exit_clean_secs(1), + in_gap + ), + OnlineState::Degraded + ); +} + +pub fn test_all() { + test_no_evidence_is_online(); + test_offline_dominates(); + test_fresh_evidence_degrades(); + test_success_after_evidence_recovers(); + test_stale_success_does_not_recover(); + test_no_success_holds_until_decay(); + test_clean_window_escalates_and_caps(); + test_escalated_window_closes_the_gap(); +} diff --git a/veilid-core/src/network_manager/tests_network_manager/test_transport_stats.rs b/veilid-core/src/network_manager/tests_network_manager/test_transport_stats.rs new file mode 100644 index 00000000..e76ad712 --- /dev/null +++ b/veilid-core/src/network_manager/tests_network_manager/test_transport_stats.rs @@ -0,0 +1,216 @@ +use super::*; + +fn now() -> Timestamp { + // Far enough from zero to subtract windows without underflow + Timestamp::new(1_000_000_000u64) +} + +fn secs_ago(secs: u64) -> Timestamp { + Timestamp::new(now().as_u64() - secs * 1_000_000u64) +} + +fn peer(n: u8) -> TransportPeerKey { + let mut b = [0u8; 32]; + b[0] = n; + TransportPeerKey::Node(NodeId::new(CRYPTO_KIND_VLD0, BareNodeId::new(&b))) +} + +// Drive the window directly so the tests don't depend on wall-clock timing. +// Spreads outcomes across enough distinct peers to clear the peer-diversity bar. +fn window_with(outcomes: &[(Timestamp, TransportSendOutcome)]) -> TransportWindow { + let mut w = TransportWindow::default(); + for (i, (ts, o)) in outcomes.iter().enumerate() { + w.record(*ts, *o, peer(i as u8)); + } + w +} + +pub fn test_verdict_needs_min_samples() { + info!("--- test_verdict_needs_min_samples ---"); + + // Below MIN_SAMPLES there is no verdict, however bad the evidence looks + let outcomes: Vec<_> = (0..TRANSPORT_STATS_MIN_SAMPLES - 1) + .map(|_| (now(), TransportSendOutcome::Timeout)) + .collect(); + let w = window_with(&outcomes); + assert_eq!(w.verdict(now()), TransportVerdict::Unknown); + + // One more timeout crosses the threshold with no successes: suppressed + let outcomes: Vec<_> = (0..TRANSPORT_STATS_MIN_SAMPLES) + .map(|_| (now(), TransportSendOutcome::Timeout)) + .collect(); + let w = window_with(&outcomes); + assert_eq!( + w.verdict(now()), + TransportVerdict::Suppressed { + due_for_canary: false + } + ); +} + +pub fn test_verdict_any_success_is_healthy() { + info!("--- test_verdict_any_success_is_healthy ---"); + + // Strict suppression: a single success in the window keeps the transport usable, + // so a merely lossy transport is never suppressed + let mut outcomes: Vec<_> = (0..TRANSPORT_STATS_MIN_SAMPLES) + .map(|_| (now(), TransportSendOutcome::Timeout)) + .collect(); + outcomes.push((now(), TransportSendOutcome::Sent)); + let w = window_with(&outcomes); + assert_eq!(w.verdict(now()), TransportVerdict::Healthy); +} + +pub fn test_window_keeps_min_samples_when_quiet() { + info!("--- test_window_keeps_min_samples_when_quiet ---"); + + // All outcomes are older than the window, but eviction stops at MIN_SAMPLES so a + // quiet steady state still has a verdict instead of decaying to Unknown + let outcomes: Vec<_> = (0..TRANSPORT_STATS_MIN_SAMPLES + 4) + .map(|_| (secs_ago(60), TransportSendOutcome::Timeout)) + .collect(); + let mut w = window_with(&outcomes); + w.evict_aged(now()); + assert_eq!(w.sample_count(), TRANSPORT_STATS_MIN_SAMPLES); + assert!(matches!( + w.verdict(now()), + TransportVerdict::Suppressed { .. } + )); +} + +pub fn test_canary_becomes_due() { + info!("--- test_canary_becomes_due ---"); + + // Freshly suppressed: not yet due, so selection keeps avoiding this transport + let outcomes: Vec<_> = (0..TRANSPORT_STATS_MIN_SAMPLES) + .map(|_| (now(), TransportSendOutcome::Timeout)) + .collect(); + let w = window_with(&outcomes); + assert!(w.verdict(now()).is_deprioritized()); + + // Nothing attempted for longer than the canary interval: let one send through, + // otherwise suppression would starve its own window and never recover + let stale: Vec<_> = (0..TRANSPORT_STATS_MIN_SAMPLES) + .map(|_| (secs_ago(120), TransportSendOutcome::Timeout)) + .collect(); + let w = window_with(&stale); + assert_eq!( + w.verdict(now()), + TransportVerdict::Suppressed { + due_for_canary: true + } + ); + assert!(!w.verdict(now()).is_deprioritized()); +} + +pub fn test_concentrated_failures_are_not_a_dead_path() { + info!("--- test_concentrated_failures_are_not_a_dead_path ---"); + + // A pile of failures against ONE peer means that peer's dial info is stale, + // not that our path is broken, so there must be no verdict + let mut w = TransportWindow::default(); + for _ in 0..TRANSPORT_STATS_MIN_SAMPLES * 2 { + w.record(now(), TransportSendOutcome::Timeout, peer(1)); + } + assert_eq!(w.verdict(now()), TransportVerdict::Unknown); + + // The same failures spread across enough distinct peers do implicate the path + let mut w = TransportWindow::default(); + for i in 0..TRANSPORT_STATS_MIN_SAMPLES { + w.record(now(), TransportSendOutcome::Timeout, peer(i as u8)); + } + assert!(matches!( + w.verdict(now()), + TransportVerdict::Suppressed { .. } + )); +} + +pub fn test_skipping_does_not_defer_the_canary() { + info!("--- test_skipping_does_not_defer_the_canary ---"); + + // Skips must not push last_attempt_ts, or a busy suppressed transport would + // starve its own canary + let outcomes: Vec<_> = (0..TRANSPORT_STATS_MIN_SAMPLES) + .map(|_| (secs_ago(120), TransportSendOutcome::Timeout)) + .collect(); + let w = window_with(&outcomes); + let before = w.sample_count(); + assert_eq!( + w.verdict(now()), + TransportVerdict::Suppressed { + due_for_canary: true + } + ); + assert_eq!(w.sample_count(), before); +} + +pub fn test_suppression_survives_losing_peer_diversity() { + info!("--- test_suppression_survives_losing_peer_diversity ---"); + + // Window decay bunches survivors onto few peers; re-testing the entry bar + // there would un-suppress on thin evidence and oscillate + let mut w = TransportWindow::default(); + for i in 0..TRANSPORT_STATS_MIN_SAMPLES { + w.record(secs_ago(60), TransportSendOutcome::Timeout, peer(i as u8)); + } + assert!(matches!( + w.verdict(secs_ago(60)), + TransportVerdict::Suppressed { .. } + )); + + // The canary trickles retries at one peer while the spread evidence ages out, + // leaving only the concentrated survivors + for _ in 0..TRANSPORT_STATS_MIN_SAMPLES { + w.record(now(), TransportSendOutcome::Timeout, peer(1)); + } + w.evict_aged(now()); + assert!(w.distinct_peers() < TRANSPORT_STATS_MIN_DISTINCT_PEERS); + assert!(matches!( + w.verdict(now()), + TransportVerdict::Suppressed { .. } + )); + + // Only a send that works releases it + w.record(now(), TransportSendOutcome::Sent, peer(1)); + assert_eq!(w.verdict(now()), TransportVerdict::Healthy); +} + +pub fn test_unbound_sends_count_toward_diversity() { + info!("--- test_unbound_sends_count_toward_diversity ---"); + + // Bootstrap and dial info discovery have no node behind them, but they are real + // evidence about our own path, so distinct addresses must satisfy the same bar + fn addr(n: u8) -> TransportPeerKey { + TransportPeerKey::Address(SocketAddress::new( + Address::IPV4(std::net::Ipv4Addr::new(10, 0, 0, n)), + 5150, + )) + } + + let mut w = TransportWindow::default(); + for i in 0..TRANSPORT_STATS_MIN_SAMPLES { + w.record(now(), TransportSendOutcome::Timeout, addr(i as u8)); + } + assert!(matches!( + w.verdict(now()), + TransportVerdict::Suppressed { .. } + )); + + // Repeats against one address are one endpoint, exactly as for one node + let mut w = TransportWindow::default(); + for _ in 0..TRANSPORT_STATS_MIN_SAMPLES * 2 { + w.record(now(), TransportSendOutcome::Timeout, addr(1)); + } + assert_eq!(w.verdict(now()), TransportVerdict::Unknown); +} + +pub fn test_all() { + test_verdict_needs_min_samples(); + test_verdict_any_success_is_healthy(); + test_window_keeps_min_samples_when_quiet(); + test_canary_becomes_due(); + test_concentrated_failures_are_not_a_dead_path(); + test_skipping_does_not_defer_the_canary(); + test_suppression_survives_losing_peer_diversity(); + test_unbound_sends_count_toward_diversity(); +} diff --git a/veilid-core/src/network_manager/transport_stats.rs b/veilid-core/src/network_manager/transport_stats.rs new file mode 100644 index 00000000..ff382114 --- /dev/null +++ b/veilid-core/src/network_manager/transport_stats.rs @@ -0,0 +1,553 @@ +//! Outbound send health per (RoutingDomain, LowLevelTransportType). +//! +//! Windows record only our own sends; they say nothing about inbound reachability +//! and never change what we publish. Design invariants: +//! - Suppression requires a full window of failures across enough distinct peers, +//! so a few peers with stale dial info never indict the path +//! - Suppression is sticky: entered on the evidence bar, released only by a send +//! that works, so window decay cannot un-suppress on thin evidence +//! - While suppressed, one canary send per CANARY_INTERVAL_SECS probes for +//! recovery; it is spent at decision time so concurrent selections cannot all +//! escape with it +//! - Debug `transport disable` fails low-level sends with NoConnection, driving +//! this machinery with real evidence instead of overriding verdicts + +use super::*; + +impl_veilid_log_facility!("net"); + +/// Age at which an outcome leaves the window, down to TRANSPORT_STATS_MIN_SAMPLES +const WINDOW_SECS: u32 = 10; +/// Outcomes retained regardless of age, so a quiet period keeps a verdict +pub(crate) const TRANSPORT_STATS_MIN_SAMPLES: usize = 8; +/// Distinct failed peers required before the path is blamed rather than the peers +pub(crate) const TRANSPORT_STATS_MIN_DISTINCT_PEERS: usize = 4; +/// Hard bound on retained outcomes per transport +const MAX_SAMPLES: usize = 64; +/// How long a suppressed transport goes unattempted before one send is let through +const CANARY_INTERVAL_SECS: u32 = 30; +/// How often the per-transport health summary is logged +pub const TRANSPORT_STATS_LOG_INTERVAL_SECS: u32 = 10; + +/// Peer-diversity key; unbound sends (bootstrap, discovery) have an address but no node +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) enum TransportPeerKey { + /// A peer we know by node id + Node(NodeId), + /// A target known only by address + Address(SocketAddress), +} + +impl From for TransportPeerKey { + fn from(node_id: NodeId) -> Self { + TransportPeerKey::Node(node_id) + } +} + +impl From for TransportPeerKey { + fn from(addr: SocketAddress) -> Self { + TransportPeerKey::Address(addr) + } +} + +/// What one send attempt says about a transport, from our side only +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum TransportSendOutcome { + /// The send completed + Sent, + /// The send returned an error + FailedToSend, + /// The send did not complete in time + Timeout, +} + +impl TransportSendOutcome { + fn is_sent(&self) -> bool { + matches!(self, TransportSendOutcome::Sent) + } + + pub(crate) fn from_network_result(nr: &NetworkResult) -> Self { + match nr { + NetworkResult::Value(_) => TransportSendOutcome::Sent, + NetworkResult::Timeout => TransportSendOutcome::Timeout, + _ => TransportSendOutcome::FailedToSend, + } + } +} + +impl From<&SendDataResult> for TransportSendOutcome { + fn from(sdr: &SendDataResult) -> Self { + Self::from_network_result(sdr.network_result()) + } +} + +/// Outbound send verdict for one transport, aggregated across all peers +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum TransportVerdict { + /// Not enough evidence yet; usable + Unknown, + /// Sending works + Healthy, + /// Nothing has gone out over this transport for a whole window + Suppressed { + /// The canary interval has elapsed since the last attempt + due_for_canary: bool, + }, +} + +impl TransportVerdict { + /// Whether selection should push this transport to the back of the line + pub fn is_deprioritized(&self) -> bool { + matches!( + self, + TransportVerdict::Suppressed { + due_for_canary: false + } + ) + } + + /// Whether we cannot send on this transport. Blame writers void per-transport + /// evidence gathered on it: the aggregate verdict makes the fault ours, not the peer's. + pub fn is_send_suppressed(&self) -> bool { + matches!(self, TransportVerdict::Suppressed { .. }) + } +} + +/// Rolling send outcomes for one transport, plus its suppression latch +#[derive(Debug, Default)] +pub(crate) struct TransportWindow { + outcomes: VecDeque<(Timestamp, TransportSendOutcome, TransportPeerKey)>, + last_attempt_ts: Option, + /// Sends skipped while suppressed since the last health log + skipped: usize, + /// Sticky suppression latch (see module doc) + suppressed: bool, +} + +impl TransportWindow { + /// Outcomes currently retained + pub(crate) fn sample_count(&self) -> usize { + self.outcomes.len() + } + + /// Distinct peers represented in the window + pub(crate) fn distinct_peers(&self) -> usize { + self.outcomes + .iter() + .map(|(_, _, id)| id) + .collect::>() + .len() + } + + /// Add one send outcome, then update the suppression latch + pub(crate) fn record( + &mut self, + cur_ts: Timestamp, + outcome: TransportSendOutcome, + peer: TransportPeerKey, + ) { + self.last_attempt_ts = Some(cur_ts); + self.outcomes.push_back((cur_ts, outcome, peer)); + while self.outcomes.len() > MAX_SAMPLES { + self.outcomes.pop_front(); + } + self.evict_aged(cur_ts); + + if outcome.is_sent() { + self.suppressed = false; + } else if !self.suppressed { + self.suppressed = self.meets_entry_bar(); + } + } + + /// Full window of failures across enough distinct peers + fn meets_entry_bar(&self) -> bool { + self.outcomes.len() >= TRANSPORT_STATS_MIN_SAMPLES + && !self.outcomes.iter().any(|(_, o, _)| o.is_sent()) + && self.distinct_peers() >= TRANSPORT_STATS_MIN_DISTINCT_PEERS + } + + /// Drop aged outcomes, but never below TRANSPORT_STATS_MIN_SAMPLES + pub(crate) fn evict_aged(&mut self, cur_ts: Timestamp) { + let window = TimestampDuration::new_secs(WINDOW_SECS); + while self.outcomes.len() > TRANSPORT_STATS_MIN_SAMPLES { + let Some((ts, _, _)) = self.outcomes.front() else { + break; + }; + if cur_ts.duration_since(*ts) <= window { + break; + } + self.outcomes.pop_front(); + } + } + + fn counts(&self) -> (usize, usize, usize) { + let mut sent = 0; + let mut failed = 0; + let mut timeout = 0; + for (_, o, _) in &self.outcomes { + match o { + TransportSendOutcome::Sent => sent += 1, + TransportSendOutcome::FailedToSend => failed += 1, + TransportSendOutcome::Timeout => timeout += 1, + } + } + (sent, failed, timeout) + } + + /// Verdict from the outcomes currently retained + pub(crate) fn verdict(&self, cur_ts: Timestamp) -> TransportVerdict { + if !self.suppressed { + if self.outcomes.len() < TRANSPORT_STATS_MIN_SAMPLES { + return TransportVerdict::Unknown; + } + return if self.outcomes.iter().any(|(_, o, _)| o.is_sent()) { + TransportVerdict::Healthy + } else { + // Failures below the peer-diversity bar indict the peers, not the path + TransportVerdict::Unknown + }; + } + let due_for_canary = self + .last_attempt_ts + .map(|ts| cur_ts.duration_since(ts) > TimestampDuration::new_secs(CANARY_INTERVAL_SECS)) + .unwrap_or(true); + TransportVerdict::Suppressed { due_for_canary } + } +} + +/// Send-health windows keyed by network path (TCP/WS/WSS pool as TCP); per-protocol +/// trouble is the per-node tables' job +#[derive(Debug)] +pub(crate) struct TransportStats { + /// Registry accessor + registry: VeilidComponentRegistry, + /// One send-outcome window per routing domain and transport + windows: Mutex>, + /// Debug availability overrides; value is the change generation so a timed + /// toggle only reverts state nothing else has touched since + disabled: Mutex>, + /// Bumped by every availability change + #[cfg(feature = "debug-api")] + change_generation: Mutex, +} + +impl_veilid_component_accessors!(TransportStats); + +impl TransportStats { + /// Create an empty send-health tracker + pub fn new(registry: VeilidComponentRegistry) -> Self { + Self { + registry, + windows: Mutex::new(BTreeMap::new()), + disabled: Mutex::new(BTreeMap::new()), + #[cfg(feature = "debug-api")] + change_generation: Mutex::new(0), + } + } + + /// Whether this transport is administratively disabled (debug `transport disable`) + pub fn is_transport_disabled( + &self, + low_level_protocol_type: LowLevelProtocolType, + address_type: AddressType, + ) -> bool { + self.disabled.lock().contains_key(&LowLevelTransportType { + low_level_protocol_type, + address_type, + }) + } + + /// Set availability; returns the change generation for timed reverts + #[cfg(feature = "debug-api")] + pub fn set_transports_disabled( + &self, + transports: &[LowLevelTransportType], + disabled: bool, + ) -> u64 { + let mut generation = self.change_generation.lock(); + *generation += 1; + let gen = *generation; + let mut map = self.disabled.lock(); + for tt in transports { + if disabled { + map.insert(*tt, gen); + } else { + map.remove(tt); + } + } + veilid_log!(self debug "transports {}: {}", + if disabled { "disabled" } else { "enabled" }, + transports.iter().map(|t| t.to_string()).collect::>().join(",")); + gen + } + + /// Revert a timed toggle for transports unchanged since `expect_generation` + #[cfg(feature = "debug-api")] + pub fn set_transports_disabled_if_generation( + &self, + transports: &[LowLevelTransportType], + disabled: bool, + expect_generation: u64, + ) { + let untouched: Vec = { + let map = self.disabled.lock(); + transports + .iter() + .filter(|tt| match map.get(tt) { + Some(gen) => *gen == expect_generation, + // Enabled entries carry no generation; only revert them toward disabled + None => disabled, + }) + .copied() + .collect() + }; + if !untouched.is_empty() { + self.set_transports_disabled(&untouched, disabled); + } + } + + /// Currently disabled transports + #[cfg(feature = "debug-api")] + pub fn disabled_transports(&self) -> Vec { + self.disabled.lock().keys().copied().collect() + } + + /// Purge all windows (debug `transport clear`) + #[cfg(feature = "debug-api")] + pub fn clear_all_windows(&self) { + self.windows.lock().clear(); + veilid_log!(self debug "transport stats cleared (debug)"); + } + + /// Record one send attempt against a peer's routing domain and transport + pub fn record_send( + &self, + routing_domain: RoutingDomain, + transport: TransportType, + peer: impl Into, + outcome: TransportSendOutcome, + ) { + let transport = LowLevelTransportType::from(transport); + let cur_ts = Timestamp::now(); + self.windows + .lock() + .entry((routing_domain, transport)) + .or_default() + .record(cur_ts, outcome, peer.into()); + } + + /// Record a nodeless send (bootstrap, discovery); the address still names the + /// routing domain and transport + pub fn record_unbound_send(&self, dial_info: &DialInfo, outcome: TransportSendOutcome) { + let socket_address = dial_info.socket_address(); + let Some(routing_domain) = self + .routing_table() + .routing_domain_for_address(socket_address.address()) + else { + return; + }; + self.record_send( + routing_domain, + TransportType::from(dial_info), + socket_address, + outcome, + ); + } + + /// Decision point for nodeless sends, whose targets come from configuration + /// rather than contact method selection; spends the canary when due + pub fn unbound_send_deprioritized(&self, dial_info: &DialInfo) -> bool { + let Some(routing_domain) = self + .routing_table() + .routing_domain_for_address(dial_info.socket_address().address()) + else { + return false; + }; + self.deprioritized_consuming_canary(routing_domain, TransportType::from(dial_info)) + } + + /// Whether any non-suppressed transport carries this ordering. Only suppression + /// counts as degraded (Unknown is usable); WS-only WASM has no Unordered carrier + pub fn ordering_is_viable( + &self, + routing_domain: RoutingDomain, + ordering: SequenceOrdering, + ) -> bool { + let routing_table = self.routing_table(); + let peer_info = routing_table.get_current_peer_info(routing_domain); + let node_info = peer_info.node_info(); + let outbound = node_info.outbound_protocols(); + let address_types = node_info.address_types(); + + for protocol in outbound.iter() { + // XXX: WebTransport will offer both orderings over UDP; derive ordering + // support from dialinfo rather than protocol when it lands + if protocol.sequence_ordering() != ordering { + continue; + } + for address_type in address_types.iter() { + let transport = TransportType::new(protocol, address_type); + if !self.verdict(routing_domain, transport).is_deprioritized() { + return true; + } + } + } + false + } + + /// Decision-point verdict check that spends the canary at selection time, so + /// sends picked during the in-flight window cannot all escape with it. + /// Never use for read-only verdict queries. + pub fn deprioritized_consuming_canary( + &self, + routing_domain: RoutingDomain, + transport: TransportType, + ) -> bool { + let transport = LowLevelTransportType::from(transport); + let cur_ts = Timestamp::now(); + let mut windows = self.windows.lock(); + let Some(window) = windows.get_mut(&(routing_domain, transport)) else { + return false; + }; + window.evict_aged(cur_ts); + match window.verdict(cur_ts) { + TransportVerdict::Suppressed { + due_for_canary: false, + } => true, + TransportVerdict::Suppressed { + due_for_canary: true, + } => { + window.last_attempt_ts = Some(cur_ts); + false + } + _ => false, + } + } + + /// Count a send skipped because the transport is suppressed + pub fn record_skip(&self, routing_domain: RoutingDomain, transport: TransportType) { + let transport = LowLevelTransportType::from(transport); + if let Some(window) = self.windows.lock().get_mut(&(routing_domain, transport)) { + window.skipped += 1; + } + } + + /// Verdict for a transport, leaving the canary unspent + pub fn verdict( + &self, + routing_domain: RoutingDomain, + transport: TransportType, + ) -> TransportVerdict { + let transport = LowLevelTransportType::from(transport); + let cur_ts = Timestamp::now(); + let mut windows = self.windows.lock(); + let Some(window) = windows.get_mut(&(routing_domain, transport)) else { + return TransportVerdict::Unknown; + }; + window.evict_aged(cur_ts); + window.verdict(cur_ts) + } + + /// Drop all evidence for a routing domain. Our network position changed, so + /// what we learned from the old one no longer applies. + pub fn reset_routing_domain(&self, routing_domain: RoutingDomain, reason: &str) { + let mut windows = self.windows.lock(); + let before = windows.len(); + windows.retain(|(rd, _), _| *rd != routing_domain); + if before != windows.len() { + veilid_log!(self debug "transport stats reset for {:?}: {}", routing_domain, reason); + } + } + + /// Per-domain health lines. `take_skipped` resets the skip counters, which only + /// the periodic log may do; the debug dump must not steal them from it. + fn health_report(&self, take_skipped: bool) -> Vec<(RoutingDomain, String)> { + let cur_ts = Timestamp::now(); + let mut per_domain = BTreeMap::>::new(); + { + let mut windows = self.windows.lock(); + for ((rd, tt), window) in windows.iter_mut() { + window.evict_aged(cur_ts); + let (sent, failed, timeout) = window.counts(); + let peers = window.distinct_peers(); + let verdict = match window.verdict(cur_ts) { + TransportVerdict::Unknown => "unknown".to_owned(), + TransportVerdict::Healthy => "healthy".to_owned(), + TransportVerdict::Suppressed { due_for_canary } => { + if due_for_canary { + "suppressed(canary due)".to_owned() + } else { + "suppressed".to_owned() + } + } + }; + let since_attempt = window + .last_attempt_ts + .map(|ts| format!("{:#}", cur_ts.duration_since(ts))) + .unwrap_or_else(|| "never".to_owned()); + let skipped = if take_skipped { + std::mem::take(&mut window.skipped) + } else { + window.skipped + }; + per_domain.entry(*rd).or_default().push(format!( + "{}={} n:{} peers:{} sent:{} failed:{} timeout:{} skipped:{} last:{}", + tt, + verdict, + window.sample_count(), + peers, + sent, + failed, + timeout, + skipped, + since_attempt + )); + } + } + per_domain + .into_iter() + .map(|(rd, mut entries)| { + entries.sort(); + (rd, entries.join(" | ")) + }) + .collect() + } + + /// Log one line per routing domain with a transport + pub fn log_health(&self) { + for (rd, line) in self.health_report(true) { + veilid_log!(self debug target: "rtab::stats", "transport health [{:?}]: {}", rd, line); + } + } + + /// Full dump for the debug `transport stats` command + #[cfg(feature = "debug-api")] + pub fn debug_stats(&self) -> String { + let mut out = String::new(); + let report = self.health_report(false); + if report.is_empty() { + out += "no transport windows yet\n"; + } + for (rd, line) in report { + out += &format!("[{:?}]\n", rd); + for entry in line.split(" | ") { + out += &format!(" {}\n", entry); + } + } + let disabled = self.disabled_transports(); + if disabled.is_empty() { + out += "disabled: (none)\n"; + } else { + out += &format!( + "disabled: {}\n", + disabled + .iter() + .map(|t| t.to_string()) + .collect::>() + .join(", ") + ); + } + out + } +} diff --git a/veilid-core/src/network_manager/types/flow.rs b/veilid-core/src/network_manager/types/flow.rs index adf7f115..8f1a8130 100644 --- a/veilid-core/src/network_manager/types/flow.rs +++ b/veilid-core/src/network_manager/types/flow.rs @@ -86,6 +86,9 @@ impl MatchesDialInfoFilter for Flow { pub(crate) struct UniqueFlow { pub flow: Flow, pub connection_id: Option, + /// Whether this flow is outbound-dial evidence (we used the peer's dial info, or an + /// outbound connection) rather than a flow the peer opened to us; UDP always counts + pub dialed: bool, } impl fmt::Display for UniqueFlow { diff --git a/veilid-core/src/network_manager/types/node_contact_method_result.rs b/veilid-core/src/network_manager/types/node_contact_method_result.rs index 25169d07..b3896726 100644 --- a/veilid-core/src/network_manager/types/node_contact_method_result.rs +++ b/veilid-core/src/network_manager/types/node_contact_method_result.rs @@ -12,6 +12,9 @@ pub enum NodeContactMethodResult { NoPeerInfo, /// No compatible contact method could be derived from peer info. NoContactMethod, + /// Every contact method for this node rides a transport our own send history says is + /// dead, and the recovery canary isn't due, so attempting anyway only buys a timeout. + TransportSuppressed, /// Target node id is administratively punished. Punished, } @@ -39,6 +42,7 @@ impl fmt::Display for NodeContactMethodResult { NodeContactMethodResult::NoRoutingDomain => write!(f, "NoRoutingDomain"), NodeContactMethodResult::NoPeerInfo => write!(f, "NoPeerInfo"), NodeContactMethodResult::NoContactMethod => write!(f, "NoContactMethod"), + NodeContactMethodResult::TransportSuppressed => write!(f, "TransportSuppressed"), NodeContactMethodResult::Punished => write!(f, "Punished"), } } diff --git a/veilid-core/src/network_manager/types/send_data_result.rs b/veilid-core/src/network_manager/types/send_data_result.rs index acb6ec69..0f679b88 100644 --- a/veilid-core/src/network_manager/types/send_data_result.rs +++ b/veilid-core/src/network_manager/types/send_data_result.rs @@ -59,6 +59,18 @@ impl SendDataResult { self.unique_flow().map(|uf| uf.flow.transport_type()) } + /// Whether this send is outbound-dial evidence: the flow's own judgment on success, + /// else derived from the attempted contact method, where only Existing is not evidence. + pub fn dialed_evidence(&self) -> bool { + if let NetworkResult::Value(uf) = &self.network_result { + return uf.dialed; + } + !matches!( + self.node_contact_method_result, + NodeContactMethodResult::Resolved(ContactMethod::Existing) + ) + } + pub fn sequence_ordering(&self) -> Option { self.unique_flow() .map(|uf| uf.flow.protocol_type().sequence_ordering()) diff --git a/veilid-core/src/network_manager/types/transport_type.rs b/veilid-core/src/network_manager/types/transport_type.rs index 19ffdc74..94ccd657 100644 --- a/veilid-core/src/network_manager/types/transport_type.rs +++ b/veilid-core/src/network_manager/types/transport_type.rs @@ -38,6 +38,32 @@ impl TransportType { } } +/// A low-level protocol paired with an address type, naming the network path itself +/// (e.g. TCP/IPv4) independent of which high-level protocol rides on it. +/// +/// TCP and WS share one path (as WebTransport's datagram and stream variants will share +/// UDP's), so this is the granularity for asking whether our packets get out at all. +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct LowLevelTransportType { + pub low_level_protocol_type: LowLevelProtocolType, + pub address_type: AddressType, +} + +impl From for LowLevelTransportType { + fn from(tt: TransportType) -> Self { + Self { + low_level_protocol_type: tt.protocol_type.low_level_protocol_type(), + address_type: tt.address_type, + } + } +} + +impl fmt::Display for LowLevelTransportType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.low_level_protocol_type, self.address_type) + } +} + impl From<&DialInfo> for TransportType { fn from(di: &DialInfo) -> Self { Self::new(di.protocol_type(), di.address_type()) diff --git a/veilid-core/src/routing_table/bucket_entry/mod.rs b/veilid-core/src/routing_table/bucket_entry/mod.rs index 6968cbaa..ef6e07b1 100644 --- a/veilid-core/src/routing_table/bucket_entry/mod.rs +++ b/veilid-core/src/routing_table/bucket_entry/mod.rs @@ -174,6 +174,9 @@ pub(crate) struct BucketEntryInner { /// Per-transport reliability stats used to calculate state #[serde(skip)] per_transport_stats: BTreeMap, + /// Cached per-ordering contact-method viability, recomputed by the update_cm_viability task + #[serde(skip)] + cm_viability: Option, /// Per-local-route stats keyed by local-route pubkey (our outbound SR for outbound, /// our inbound reply-PR for inbound answers). Only populated for routed exchanges. #[serde(skip, default = "default_per_route_stats")] @@ -378,6 +381,7 @@ impl<'de> serde::de::Visitor<'de> for BucketEntryInnerVisitor { rpc_stats, per_sequence_ordering_stats: BTreeMap::new(), per_transport_stats: BTreeMap::new(), + cm_viability: None, per_route_stats: default_per_route_stats(), connection_stats: ConnectionStats::default(), punishment: None, @@ -702,7 +706,7 @@ impl BucketEntryInner { // over so that connection is still valid. if node_info_changed { self.clear_last_flows_except_latest(); - self.contact_method_failures.clear(); + // The update_cm_viability task prunes dial evidence to the new contact method list } node_info_changed @@ -1134,10 +1138,13 @@ impl BucketEntryInner { bytes: ByteCount, expects_answer: bool, transport: TransportType, + dialed: bool, ) { - // Update transport stats - let ts_stats = self.transport_stats_mut(transport); - ts_stats.question_sent(ts, expects_answer); + // Only dial evidence goes in the per-transport bucket + if dialed { + let ts_stats = self.transport_stats_mut(transport); + ts_stats.question_sent(ts, expects_answer); + } // Update sequence ordering stats let sequence_ordering = transport.sequence_ordering(); @@ -1151,16 +1158,13 @@ impl BucketEntryInner { self.transfer_stats_accounting.add_up(bytes); } + /// An inbound question is flow evidence, not dial evidence, so no per-transport update pub(super) fn question_rcvd( &mut self, ts: Timestamp, bytes: ByteCount, transport: TransportType, ) { - // Update transport stats - let ts_stats = self.transport_stats_mut(transport); - ts_stats.question_rcvd(ts); - // Update sequence ordering stats let sequence_ordering = transport.sequence_ordering(); let so_stats = self.sequence_ordering_stats_mut(sequence_ordering); @@ -1173,10 +1177,12 @@ impl BucketEntryInner { self.transfer_stats_accounting.add_down(bytes); } - pub(super) fn answer_sent(&mut self, bytes: ByteCount, transport: TransportType) { - // Update transport stats - let ts_stats = self.transport_stats_mut(transport); - ts_stats.answer_sent(); + pub(super) fn answer_sent(&mut self, bytes: ByteCount, transport: TransportType, dialed: bool) { + // Only dial evidence goes in the per-transport bucket + if dialed { + let ts_stats = self.transport_stats_mut(transport); + ts_stats.answer_sent(); + } // Update sequence ordering stats let sequence_ordering = transport.sequence_ordering(); @@ -1196,6 +1202,7 @@ impl BucketEntryInner { recv_ts: Timestamp, bytes: ByteCount, transport: TransportType, + dialed: bool, ) { // Update transfer accounting self.transfer_stats_accounting.add_down(bytes); @@ -1206,9 +1213,11 @@ impl BucketEntryInner { .record_latency(recv_ts.duration_since(send_ts)), ); - // Update transport stats - let ts_stats = self.transport_stats_mut(transport); - ts_stats.answer_rcvd(recv_ts); + // Only dial evidence goes in the per-transport bucket + if dialed { + let ts_stats = self.transport_stats_mut(transport); + ts_stats.answer_rcvd(recv_ts); + } // Update sequence ordering stats let sequence_ordering = transport.sequence_ordering(); @@ -1219,12 +1228,14 @@ impl BucketEntryInner { self.rpc_stats.answer_rcvd(recv_ts); } - pub(super) fn lost_question(&mut self, transport: TransportType) { + pub(super) fn lost_question(&mut self, transport: TransportType, dialed: bool) { let lost_ts = Timestamp::now(); - // Update transport stats - let ts_stats = self.transport_stats_mut(transport); - ts_stats.lost_question(lost_ts); + // Only dial evidence goes in the per-transport bucket + if dialed { + let ts_stats = self.transport_stats_mut(transport); + ts_stats.lost_question(lost_ts); + } // Update sequence ordering stats let sequence_ordering = transport.sequence_ordering(); @@ -1240,10 +1251,13 @@ impl BucketEntryInner { fail_ts: Timestamp, expects_answer: bool, transport: TransportType, + dialed: bool, ) { - // Update transport stats - let ts_stats = self.transport_stats_mut(transport); - ts_stats.failed_to_send(fail_ts, expects_answer); + // Only dial evidence goes in the per-transport bucket + if dialed { + let ts_stats = self.transport_stats_mut(transport); + ts_stats.failed_to_send(fail_ts, expects_answer); + } // Update sequence ordering stats let sequence_ordering = transport.sequence_ordering(); @@ -1276,6 +1290,46 @@ impl BucketEntryInner { } } + /// Health judgment for one transport; no recorded stats means Untried + pub fn transport_health(&self, transport: TransportType) -> TransportHealth { + self.per_transport_stats + .get(&transport) + .map(transport_health) + .unwrap_or(TransportHealth::Untried) + } + + pub fn cm_viability(&self) -> Option<&CMViability> { + self.cm_viability.as_ref() + } + + pub fn set_cm_viability(&mut self, viability: Option) { + self.cm_viability = viability; + } + + /// Our own network changed: all learned dial evidence is suspect because the + /// failures may have been ours, so the nodes get a fresh chance + pub fn clear_transport_evidence(&mut self) { + self.per_transport_stats.clear(); + self.contact_method_failures.clear(); + self.cm_viability = None; + } + + /// Peerinfo changed on either side: keep evidence only for contact methods + /// still in the current list + pub fn prune_evidence_to_contact_methods(&mut self, contact_methods: &[ContactMethod]) { + let transports: BTreeSet = contact_methods + .iter() + .filter_map(|cm| match cm { + ContactMethod::Direct { target_di } => Some(TransportType::from(target_di)), + _ => None, + }) + .collect(); + self.per_transport_stats + .retain(|t, _| transports.contains(t)); + self.contact_method_failures + .retain(|cm, _| contact_methods.contains(cm)); + } + pub fn report_contact_method_result(&mut self, cm: &ContactMethod, success: bool) { if success { self.contact_method_failures.remove(cm); @@ -1359,6 +1413,7 @@ impl BucketEntry { connection_stats: ConnectionStats::default(), per_sequence_ordering_stats: BTreeMap::new(), per_transport_stats: BTreeMap::new(), + cm_viability: None, per_route_stats: LruCache::new(PER_ROUTE_STATS_LRU_SIZE), punishment: None, contact_method_failures: HashMap::new(), diff --git a/veilid-core/src/routing_table/bucket_entry/state_reason.rs b/veilid-core/src/routing_table/bucket_entry/state_reason.rs index 24b4f53e..65ad43d7 100644 --- a/veilid-core/src/routing_table/bucket_entry/state_reason.rs +++ b/veilid-core/src/routing_table/bucket_entry/state_reason.rs @@ -2,6 +2,65 @@ use super::*; impl_veilid_log_facility!("rtab"); +////////////////////////////////////////////////////////////////////////////////////////// +// Per-transport health + +/// Consecutive send failures on one transport before it is judged Failing +pub(crate) const TRANSPORT_FAILING_FAILED_TO_SEND: u32 = 3; +/// Consecutive lost questions on one transport before it is judged Failing +pub(crate) const TRANSPORT_FAILING_LOST_QUESTIONS: u32 = 3; + +/// Evidence judgment for one node over one transport. +/// +/// Untried is viable: it can neither disqualify a contact method nor contribute +/// to node death, so alternates must be tried before a node dies. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum TransportHealth { + /// No attempts recorded + Untried, + /// No recent evidence of failure + Healthy, + /// Recent consecutive failures + Failing, +} + +impl TransportHealth { + pub fn is_failing(&self) -> bool { + matches!(self, TransportHealth::Failing) + } +} + +/// Which credit a time span gets when we could not judge the node +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum TimeCredit { + /// Offline: no test was possible, so the span neither matures nor condemns + Untestable, + /// Offline or degraded: the online detector suppressed blame, so the span cannot condemn + Unblamed, +} + +/// Judge one transport's health for contact method selection. Send failures count +/// here: whatever caused them, this edge is not working. +pub(crate) fn transport_health(stats: &RPCStats) -> TransportHealth { + if stats.failed_to_send >= TRANSPORT_FAILING_FAILED_TO_SEND + || stats.recent_lost_questions >= TRANSPORT_FAILING_LOST_QUESTIONS + { + TransportHealth::Failing + } else { + TransportHealth::Healthy + } +} + +/// Judge one transport's health for node liveness. Only unanswered questions count: +/// a failure to send never left our machine and says nothing about the peer. +pub(crate) fn transport_liveness_health(stats: &RPCStats) -> TransportHealth { + if stats.recent_lost_questions >= TRANSPORT_FAILING_LOST_QUESTIONS { + TransportHealth::Failing + } else { + TransportHealth::Healthy + } +} + ////////////////////////////////////////////////////////////////////////////////////////// // Reliable state thresholds @@ -34,20 +93,9 @@ pub(crate) const UNRELIABLE_PING_INTERVAL: TimestampDuration = TimestampDuration ////////////////////////////////////////////////////////////////////////////////////////// // Missing state thresholds -/// - Number of consecutive lost questions on an unordered transport -/// at which time we call something missing -pub(crate) const MISSING_LOST_QUESTIONS_UNORDERED: u32 = 3; -/// - Number of consecutive lost questions on an ordered transport -/// at which time we call something missing -pub(crate) const MISSING_LOST_QUESTIONS_ORDERED: u32 = 1; -/// Default missing lost-answers threshold per SequenceOrdering -/// Will become adaptive later (per-network baseline). -pub(crate) fn missing_lost_questions_count(sequence_ordering: SequenceOrdering) -> u32 { - match sequence_ordering { - SequenceOrdering::Unordered => MISSING_LOST_QUESTIONS_UNORDERED, - SequenceOrdering::Ordered => MISSING_LOST_QUESTIONS_ORDERED, - } -} +/// Node-level consecutive lost questions (any transport, reset by any answer) +/// at which time we call something missing +pub(crate) const MISSING_LOST_QUESTIONS: u32 = 3; ////////////////////////////////////////////////////////////////////////////////////////// // Dead state thresholds @@ -56,28 +104,74 @@ pub(crate) fn missing_lost_questions_count(sequence_ordering: SequenceOrdering) pub(crate) const DEAD_LOST_QUESTION_SPAN: TimestampDuration = TimestampDuration::new_secs(60); /// Failed node-level unreachable attempts (no usable transport) before we call it dead pub(crate) const DEAD_UNREACHABLE_COUNT: u32 = 1; +/// Node-level lost questions before we call a node dead if we have never seen any traffic from it +pub(crate) const DEAD_NEVER_SEEN_LOST_QUESTIONS: u32 = 3; -/// Number of failed-to-send RPCs to a node over an unordered transport before we call it dead -pub(crate) const DEAD_FAILED_TO_SEND_UNORDERED: u32 = 9; -/// Number of failed-to-send RPCs to a node over an ordered transport before we call it dead -pub(crate) const DEAD_FAILED_TO_SEND_ORDERED: u32 = 3; -/// Default failed-to-send count per sequence ordering. -pub(crate) fn dead_failed_to_send_count(sequence_ordering: SequenceOrdering) -> u32 { - match sequence_ordering { - SequenceOrdering::Unordered => DEAD_FAILED_TO_SEND_UNORDERED, - SequenceOrdering::Ordered => DEAD_FAILED_TO_SEND_ORDERED, - } +////////////////////////////////////////////////////////////////////////////////////////// +// Contact-method viability + +/// Reachability inputs for one SequenceOrdering, derived from the entry's current +/// contact method list by the update_cm_viability task. Direct transports are judged +/// against live per-transport stats at state computation; indirect contact methods +/// (existing flow, signal, relay) veto send-failure death, since their failure modes +/// are covered by unreachable [D1] and steady-lost-question [D4] rules. +#[derive(Clone, Debug, Default)] +pub(crate) struct OrderingReachability { + /// Transports governing Direct contact methods for this ordering + pub direct_transports: BTreeSet, + /// Whether any non-direct contact methods exist + pub has_indirect: bool, } -/// Number of lost questions to a node over an unordered transport before we call it dead if we haven't seen any traffic from it yet -pub(crate) const DEAD_NEVER_SEEN_LOST_QUESTIONS_UNORDERED: u32 = 3; -/// Number of lost questions to a node over an ordered transport before we call it dead if we haven't seen any traffic from it yet -pub(crate) const DEAD_NEVER_SEEN_LOST_QUESTIONS_ORDERED: u32 = 1; -/// Default lost questions count for never-seen nodes per sequence ordering. -pub(crate) fn dead_never_seen_lost_questions_count(sequence_ordering: SequenceOrdering) -> u32 { - match sequence_ordering { - SequenceOrdering::Unordered => DEAD_NEVER_SEEN_LOST_QUESTIONS_UNORDERED, - SequenceOrdering::Ordered => DEAD_NEVER_SEEN_LOST_QUESTIONS_ORDERED, +/// Cached per-ordering contact-method viability summary for one entry +#[derive(Clone, Debug, Default)] +pub(crate) struct CMViability { + /// Reachability inputs keyed by ordering. Orderings with no contact methods are + /// absent (vacuous: a WS-only environment lives or dies entirely on Ordered). + pub per_ordering: BTreeMap, + /// Peer info timestamps this was computed against, per routing domain (ours, theirs) + pub basis: BTreeMap, +} + +impl CMViability { + /// An ordering is fully failing when its only avenues are Direct transports + /// and every one of them is currently judged Failing + fn ordering_all_failing( + reachability: &OrderingReachability, + per_transport_stats: &BTreeMap, + ) -> bool { + if reachability.has_indirect { + return false; + } + !reachability.direct_transports.is_empty() + && reachability.direct_transports.iter().all(|t| { + per_transport_stats + .get(t) + .map(transport_liveness_health) + .is_some_and(|h| h.is_failing()) + }) + } + + /// True if every ordering that has contact methods is fully failing + pub fn all_orderings_failing( + &self, + per_transport_stats: &BTreeMap, + ) -> bool { + !self.per_ordering.is_empty() + && self + .per_ordering + .values() + .all(|r| Self::ordering_all_failing(r, per_transport_stats)) + } + + /// True if at least one ordering that has contact methods is fully failing + pub fn any_ordering_failing( + &self, + per_transport_stats: &BTreeMap, + ) -> bool { + self.per_ordering + .values() + .any(|r| Self::ordering_all_failing(r, per_transport_stats)) } } @@ -86,21 +180,26 @@ impl BucketEntryInner { /// /// Cold start is Unreliable(Unseen). Pure state-reason computation with no side effects. /// + /// Send-failure death is judged from contact-method viability: one viable ordering + /// vetoes death, and Untried transports are viable, so alternates must accumulate + /// failure evidence before a node can die. + /// /// * Punished: /// - [P1] the node id is punished by the address filter /// * Dead: (any of the following are true) /// - [D1] the node has per-entry unreachable >= DEAD_UNREACHABLE_COUNT - /// - [D2] the node has any per-sequence-ordering failed_to_send >= DEAD_FAILED_TO_SEND - /// - [D3] if no transports have been seen (per-transport), and: - /// - the node has any per-sequence-ordering recent_lost_questions >= DEAD_NEVER_SEEN_LOST_QUESTIONS - /// - [D4] if any transport has been seen (per-transport), and: - /// - for any sequence-ordering, all of: - /// - first_steady_lost_question_ts is Some - /// - time_since(first_steady_lost_question_ts) >= DEAD_LOST_QUESTION_SPAN + /// - [D2] cm_viability is computed and every ordering with contact methods is + /// fully failing (all Direct transports Failing, no indirect avenue) + /// - [D3] if no transports have been seen, and node-level + /// recent_lost_questions >= DEAD_NEVER_SEEN_LOST_QUESTIONS + /// - [D4] if any transport has been seen, and node-level + /// time_since(first_steady_lost_question_ts) >= DEAD_LOST_QUESTION_SPAN + /// (any answer on any transport resets it) /// * Missing (any of the following are true): /// - [M1] the node has per-entry unreachable > 0 (only possible if DEAD_UNREACHABLE_COUNT > 1) - /// - [M2] the node has any per-sequence-ordering failed_to_send >= 0 (only possible if DEAD_FAILED_TO_SEND > 1) - /// - [M3] the node has any per-sequence-ordering recent_lost_questions > MISSING_LOST_QUESTIONS + /// - [M2] cm_viability is computed and at least one ordering with contact methods + /// is fully failing + /// - [M3] node-level recent_lost_questions >= MISSING_LOST_QUESTIONS /// * Initial: /// - [I1] for all sequence-orderings: /// - first_steady_answer_ts is None (cold start) @@ -110,18 +209,21 @@ impl BucketEntryInner { /// * Reliable: /// - [R1] Any other condition is reliable pub(super) fn compute_state_reason(&self, cur_ts: Timestamp) -> BucketEntryStateReason { - // Credit reliability windows for time PublicInternet was offline (couldn't - // ping). Zero when there's no network manager (early init / tests). + // Zero when there's no network manager (early init / tests) let opt_nm = self.registry.lookup::(); - let offline_overlap_since = |from: Timestamp| { + let credit_since = |from: Timestamp, credit: TimeCredit| { opt_nm .as_ref() .map(|nm| { - nm.online_detector().offline_overlap( - RoutingDomain::PublicInternet, - from, - cur_ts, - ) + let od = nm.online_detector(); + match credit { + TimeCredit::Untestable => { + od.offline_overlap(RoutingDomain::PublicInternet, from, cur_ts) + } + TimeCredit::Unblamed => { + od.blame_suppressed_overlap(RoutingDomain::PublicInternet, from, cur_ts) + } + } }) .unwrap_or_else(|| TimestampDuration::new(0)) }; @@ -131,7 +233,8 @@ impl BucketEntryInner { &self.rpc_stats, &self.per_sequence_ordering_stats, &self.per_transport_stats, - offline_overlap_since, + self.cm_viability.as_ref(), + credit_since, ) } @@ -143,7 +246,8 @@ impl BucketEntryInner { rpc_stats: &RPCStats, per_sequence_ordering_stats: &BTreeMap, per_transport_stats: &BTreeMap, - offline_overlap_since: impl Fn(Timestamp) -> TimestampDuration, + cm_viability: Option<&CMViability>, + credit_since: impl Fn(Timestamp, TimeCredit) -> TimestampDuration, ) -> BucketEntryStateReason { // ---===/ Punished /===--- @@ -162,42 +266,36 @@ impl BucketEntryInner { return BucketEntryStateReason::Dead(BucketEntryStateDeadReason::ExcessiveUnreachable); } - // [D2] Node is dead if it has any per-sequence-ordering failed_to_send >= DEAD_FAILED_TO_SEND - if per_sequence_ordering_stats - .iter() - .any(|(so, stats)| stats.failed_to_send >= dead_failed_to_send_count(*so)) - { - return BucketEntryStateReason::Dead(BucketEntryStateDeadReason::ExcessiveSendFailures); + // [D2] every ordering with contact methods is fully failing + if let Some(v) = cm_viability { + if v.all_orderings_failing(per_transport_stats) { + return BucketEntryStateReason::Dead( + BucketEntryStateDeadReason::ExcessiveSendFailures, + ); + } } - // [D3] if no transports have been seen (per-transport), and: - // - the node has any per-sequence-ordering recent_lost_questions >= DEAD_NEVER_SEEN_LOST_QUESTIONS + // [D3] never seen any transport, and node-level lost questions at threshold if per_transport_stats.is_empty() - && per_sequence_ordering_stats.iter().any(|(so, stats)| { - stats.recent_lost_questions >= dead_never_seen_lost_questions_count(*so) - }) + && rpc_stats.recent_lost_questions >= DEAD_NEVER_SEEN_LOST_QUESTIONS { return BucketEntryStateReason::Dead( BucketEntryStateDeadReason::NeverSeenLostQuestions, ); } - // [D4] if any transport has been seen (per-transport), and: - // - for any sequence-ordering, all of: - // - cur_ts.since(first_steady_lost_question_ts), minus offline time, - // >= DEAD_LOST_QUESTION_SPAN + // [D4] seen a transport, and node-level steady lost questions (reset by any + // answer on any transport) span at least DEAD_LOST_QUESTION_SPAN, minus unjudgeable time if !per_transport_stats.is_empty() - && per_sequence_ordering_stats.values().any(|stats| { - stats - .first_steady_lost_question_ts - .map(|ts| { - cur_ts - .duration_since(ts) - .saturating_sub(offline_overlap_since(ts)) - >= DEAD_LOST_QUESTION_SPAN - }) - .unwrap_or(false) - }) + && rpc_stats + .first_steady_lost_question_ts + .map(|ts| { + cur_ts + .duration_since(ts) + .saturating_sub(credit_since(ts, TimeCredit::Unblamed)) + >= DEAD_LOST_QUESTION_SPAN + }) + .unwrap_or(false) { return BucketEntryStateReason::Dead(BucketEntryStateDeadReason::SteadyLostQuestions); } @@ -209,19 +307,17 @@ impl BucketEntryInner { return BucketEntryStateReason::Missing(BucketEntryStateMissingReason::Unreachable); } - // [M2] the node has any per-sequence-ordering failed_to_send >= 0 (only possible if DEAD_FAILED_TO_SEND > 1) - if per_sequence_ordering_stats - .values() - .any(|stats| stats.failed_to_send > 0) - { - return BucketEntryStateReason::Missing(BucketEntryStateMissingReason::FailedToSend); + // [M2] at least one ordering with contact methods is fully failing + if let Some(v) = cm_viability { + if v.any_ordering_failing(per_transport_stats) { + return BucketEntryStateReason::Missing( + BucketEntryStateMissingReason::FailedToSend, + ); + } } - // [M3] the node has any per-sequence-ordering recent_lost_questions > UNRELIABLE_LOST_QUESTIONS - if per_sequence_ordering_stats - .iter() - .any(|(so, stats)| stats.recent_lost_questions >= missing_lost_questions_count(*so)) - { + // [M3] node-level recent lost questions at threshold + if rpc_stats.recent_lost_questions >= MISSING_LOST_QUESTIONS { return BucketEntryStateReason::Missing(BucketEntryStateMissingReason::LostQuestions); } @@ -246,7 +342,8 @@ impl BucketEntryInner { // Within span by wall-clock → unreliable regardless; otherwise // subtract offline time so we don't grant untested reliability. raw < UNRELIABLE_ANSWER_SPAN - || raw.saturating_sub(offline_overlap_since(ts)) < UNRELIABLE_ANSWER_SPAN + || raw.saturating_sub(credit_since(ts, TimeCredit::Untestable)) + < UNRELIABLE_ANSWER_SPAN } else { false } @@ -289,6 +386,9 @@ impl BucketEntryInner { /// If a node is dead or missing, reset it to the 'initial' state to give it another chance. /// Called when we get a new peer info for a node so we can try it out. + /// + /// Per-transport stats and contact method failures survive: they are learned dial + /// evidence, pruned only by the update_cm_viability task when the dialinfo changes. pub(super) fn revive(&mut self, cur_ts: Timestamp) { // Keep the stats lock over this operation so we don't have a race condition let mut state_stats_accounting = self.state_stats_accounting.lock(); @@ -304,23 +404,26 @@ impl BucketEntryInner { self.rpc_stats.last_seen_ts = None; self.rpc_stats.first_steady_answer_ts = None; self.rpc_stats.first_steady_lost_question_ts = None; + self.rpc_stats.recent_lost_questions = 0; self.rpc_stats.failed_to_send = 0; self.rpc_stats.unreachable = 0; // Drop per-sequence-ordering stats so each sequence ordering re-validates. self.per_sequence_ordering_stats.clear(); - // Drop per-transport stats so each transport re-validates. - self.per_transport_stats.clear(); - - // Drop contact method failures so each contact method re-validates. - self.contact_method_failures.clear(); - - // Should be at the initial state now, or something is very wrong + // Initial now, except [M2]/[D2], which derive from retained transport evidence let alive_reason = self.compute_and_log_state_reason_inner(cur_ts, &mut state_stats_accounting); - if !matches!(alive_reason, BucketEntryStateReason::Initial) { - veilid_log!(self error "node was ({:?}) but is ({:?}) after reviving: {}", dead_or_missing_reason, alive_reason, self.best_node_id()); + match alive_reason { + BucketEntryStateReason::Initial => {} + BucketEntryStateReason::Missing(BucketEntryStateMissingReason::FailedToSend) + | BucketEntryStateReason::Dead(BucketEntryStateDeadReason::ExcessiveSendFailures) => + { + veilid_log!(self debug "node revived with retained transport evidence ({:?}): {}", alive_reason, self.best_node_id()); + } + _ => { + veilid_log!(self error "node was ({:?}) but is ({:?}) after reviving: {}", dead_or_missing_reason, alive_reason, self.best_node_id()); + } } veilid_log!(self debug target: "rtab::state::node", "Node revived: {:#}", self.best_node_id()); diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 377f5c97..30bfe405 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -57,6 +57,10 @@ const BUCKET_COUNT: usize = HASH_COORDINATE_LENGTH * 8; /// How frequently we flush the routing table and route spec store to storage const ROUTING_TABLE_FLUSH_INTERVAL_SECS: u32 = 30; +/// Contact-method viability recompute is event-driven (cm_viability_dirty); this +/// only rate-limits how often the drain can fire +const UPDATE_CM_VIABILITY_INTERVAL_SECS: u32 = 1; + // Table store keys const ALL_ENTRY_BYTES: &[u8] = b"all_entry_bytes"; const ROUTING_TABLE: &str = "routing_table"; @@ -141,6 +145,11 @@ pub(crate) struct RoutingTable { route_spec_store: RouteSpecStore, /// Buckets to kick on our next kick task kick_queue: Mutex>, + /// Set when peerinfo changed on either side; the cm viability task fires only when set + cm_viability_dirty: core::sync::atomic::AtomicBool, + /// Last route-test enqueue time per route, so a struggling route is not + /// re-tested continuously on lossy networks + route_test_cooldown: Mutex>, /// Interim accounting mechanism for this node's RPC latency to any other node self_latency_stats_accounting: Mutex<(LatencyStatsAccounting, LatencyStats)>, /// Interim accounting mechanism for the total bandwidth to/from this node @@ -162,6 +171,8 @@ pub(crate) struct RoutingTable { rolling_answers_task: TickTask, /// Background process to purge dead routing table entries when necessary kick_buckets_task: TickTask, + /// Background process to recompute per-entry contact-method viability + update_cm_viability_task: TickTask, /// Tick subscription tick_subscription: Mutex>, @@ -200,6 +211,8 @@ impl RoutingTable { public_keys: Default::default(), secret_keys: Default::default(), kick_queue: Default::default(), + cm_viability_dirty: core::sync::atomic::AtomicBool::new(true), + route_test_cooldown: Default::default(), ping_validation_sender: Mutex::new(ping_validation_sender), ping_validation_receiver: Mutex::new(Some(ping_validation_receiver)), ping_validation_processor_jh: Mutex::new(None), @@ -224,6 +237,10 @@ impl RoutingTable { ROLLING_ANSWER_INTERVAL_SECS, ), kick_buckets_task: TickTask::new("kick_buckets_task", 1), + update_cm_viability_task: TickTask::new( + "update_cm_viability_task", + UPDATE_CM_VIABILITY_INTERVAL_SECS, + ), tick_subscription: Default::default(), }; @@ -945,7 +962,7 @@ impl RoutingTable { // Option::<()>::None // }); - // Per-routing-domain state — failures are non-fatal + // Per-routing-domain state; failures are non-fatal for rdc in self.get_routing_domain_controllers(RoutingDomainSet::all()) { if let Err(e) = rdc.load(&db).await { veilid_log!(self warn "failed to load persisted state for {}: {}", rdc.routing_domain(), e); @@ -1437,7 +1454,7 @@ impl RoutingTable { } // Reject if any of the node's own direct dial info exactly matches any of our own - // direct dial info (relay dial info is intentionally excluded — sharing a relay is fine) + // direct dial info (relay dial info excluded; sharing a relay is fine) let our_addrs = self.own_direct_dial_info_addresses(routing_domain); if !our_addrs.is_empty() && node_info @@ -1505,6 +1522,30 @@ impl RoutingTable { Ok(nr) } + /// Drop learned per-peer transport evidence table-wide. + /// + /// Call this only for OUR OWN address changes (a local routable interface address, + /// or a detected external address change): failures we blamed on peers' transports + /// may have been ours. Never on peer info changes: our peer info carries our relay + /// set, a relay swap says nothing about other peers, and clearing would wipe the + /// evidence relay candidacy needs. + pub(crate) fn clear_all_transport_evidence(&self) { + self.inner.write().clear_all_transport_evidence(); + } + + /// Signal that some peerinfo changed (ours or theirs) so contact-method + /// viability gets recomputed; the task's basis check skips unchanged entries + pub(crate) fn mark_cm_viability_dirty(&self) { + self.cm_viability_dirty + .store(true, core::sync::atomic::Ordering::Release); + } + + /// Take the viability dirty flag; changes that land during a recompute re-set it + pub(crate) fn take_cm_viability_dirty(&self) -> bool { + self.cm_viability_dirty + .swap(false, core::sync::atomic::Ordering::AcqRel) + } + /// Called whenever a routing table entry is: /// 1. created or updated with new peer information /// 2. has a node id added or removed (per CryptoKind) @@ -1520,6 +1561,7 @@ impl RoutingTable { old_peer_info: Option>, new_peer_info: Option>, ) { + self.mark_cm_viability_dirty(); let (routing_domain, node_ids) = match (old_peer_info.as_ref(), new_peer_info.as_ref()) { (None, None) => { return; diff --git a/veilid-core/src/routing_table/node_ref/traits.rs b/veilid-core/src/routing_table/node_ref/traits.rs index ff935bb7..3326b1b6 100644 --- a/veilid-core/src/routing_table/node_ref/traits.rs +++ b/veilid-core/src/routing_table/node_ref/traits.rs @@ -294,11 +294,13 @@ pub(crate) trait NodeRefCommonTrait: } fn record_protected_connection_dead(&self, transport: TransportType) { - self.stats_failed_to_send(Timestamp::now_non_decreasing(), false, transport); + // Protected connections are our outbound relay connections: dial evidence + self.stats_failed_to_send(Timestamp::now_non_decreasing(), false, transport, true); } fn report_failed_route_test(&self, transport: TransportType) { - self.stats_failed_to_send(Timestamp::now_non_decreasing(), false, transport); + // Route tests ride our outbound first-hop sends: dial evidence + self.stats_failed_to_send(Timestamp::now_non_decreasing(), false, transport, true); } fn report_contact_method_result(&self, cm: &ContactMethod, success: bool) { @@ -311,16 +313,21 @@ pub(crate) trait NodeRefCommonTrait: self.operate(|e| e.get_contact_method_failure_ts(cm)) } + fn transport_health(&self, transport: TransportType) -> TransportHealth { + self.operate(|e| e.transport_health(transport)) + } + fn stats_question_sent( &self, ts: Timestamp, bytes: ByteCount, expects_answer: bool, transport: TransportType, + dialed: bool, ) { self.operate_mut(|e| { self.routing_table().record_sent_bytes(bytes); - e.question_sent(ts, bytes, expects_answer, transport); + e.question_sent(ts, bytes, expects_answer, transport, dialed); }) } fn stats_question_rcvd(&self, ts: Timestamp, bytes: ByteCount, transport: TransportType) { @@ -329,10 +336,10 @@ pub(crate) trait NodeRefCommonTrait: e.question_rcvd(ts, bytes, transport); }) } - fn stats_answer_sent(&self, bytes: ByteCount, transport: TransportType) { + fn stats_answer_sent(&self, bytes: ByteCount, transport: TransportType, dialed: bool) { self.routing_table().record_sent_bytes(bytes); self.operate_mut(|e| { - e.answer_sent(bytes, transport); + e.answer_sent(bytes, transport, dialed); }) } fn stats_answer_rcvd( @@ -341,17 +348,18 @@ pub(crate) trait NodeRefCommonTrait: recv_ts: Timestamp, bytes: ByteCount, transport: TransportType, + dialed: bool, ) { self.operate_mut(|e| { self.routing_table().record_received_bytes(bytes); self.routing_table() .record_latency(recv_ts.duration_since(send_ts)); - e.answer_rcvd(send_ts, recv_ts, bytes, transport); + e.answer_rcvd(send_ts, recv_ts, bytes, transport, dialed); }) } - fn stats_lost_question(&self, transport: TransportType) { + fn stats_lost_question(&self, transport: TransportType, dialed: bool) { self.operate_mut(|e| { - e.lost_question(transport); + e.lost_question(transport, dialed); }) } fn stats_routed_up(&self, key: PublicKey, bytes: ByteCount) { @@ -366,9 +374,15 @@ pub(crate) trait NodeRefCommonTrait: ) { self.operate_mut(|e| e.record_routed_round_trip(key, send_ts, recv_ts, bytes)) } - fn stats_failed_to_send(&self, ts: Timestamp, expects_answer: bool, transport: TransportType) { + fn stats_failed_to_send( + &self, + ts: Timestamp, + expects_answer: bool, + transport: TransportType, + dialed: bool, + ) { self.operate_mut(|e| { - e.failed_to_send(ts, expects_answer, transport); + e.failed_to_send(ts, expects_answer, transport, dialed); }) } fn stats_unreachable(&self) { diff --git a/veilid-core/src/routing_table/route_spec_store/route_test.rs b/veilid-core/src/routing_table/route_spec_store/route_test.rs index 527773ad..78bdcd26 100644 --- a/veilid-core/src/routing_table/route_spec_store/route_test.rs +++ b/veilid-core/src/routing_table/route_spec_store/route_test.rs @@ -304,9 +304,8 @@ impl RouteSpecStore { let mut probe_transport: Option = None; while let Some((is_loopback, res)) = probes.next().await { match res? { - StatusResult::Answer { - send_data_result, .. - } => { + StatusResult::Sent(status_answer) => { + let send_data_result = status_answer.send_data_result; if probe_transport.is_none() { probe_transport = send_data_result.opt_transport_type(); } @@ -375,7 +374,7 @@ impl RouteSpecStore { // Test with double-round trip ping to self match Box::pin(self.rpc_processor().rpc_call_status(dest)).await? { - StatusResult::Answer { .. } => Ok(Some(true)), + StatusResult::Sent(_) => Ok(Some(true)), StatusResult::Failed(_) | StatusResult::NotSent(_) => Ok(Some(false)), } } diff --git a/veilid-core/src/routing_table/routing_domains/local_network/editor.rs b/veilid-core/src/routing_table/routing_domains/local_network/editor.rs index 06fce295..dedfc5b7 100644 --- a/veilid-core/src/routing_table/routing_domains/local_network/editor.rs +++ b/veilid-core/src/routing_table/routing_domains/local_network/editor.rs @@ -216,6 +216,9 @@ impl<'a> RoutingDomainEditor for RoutingDomainEditorLocalNetwork<'a> { .write() .reset_all_updated_since_last_network_change(); + // Our side of every edge changed; recompute contact-method viability + routing_table.mark_cm_viability_dirty(); + if let Err(e) = self.event_bus().post(RoutingDomainCommitEvent { routing_domain: RoutingDomain::LocalNetwork, peer_info_changed: changed.peer_info_changed, diff --git a/veilid-core/src/routing_table/routing_domains/public_internet/controller/relay_management.rs b/veilid-core/src/routing_table/routing_domains/public_internet/controller/relay_management.rs index ff565ef6..2d5a4f21 100644 --- a/veilid-core/src/routing_table/routing_domains/public_internet/controller/relay_management.rs +++ b/veilid-core/src/routing_table/routing_domains/public_internet/controller/relay_management.rs @@ -4,20 +4,44 @@ impl_veilid_log_facility!("rtab"); /// How frequently we optimize relays const RELAY_OPTIMIZATION_INTERVAL: TimestampDuration = TimestampDuration::new_secs(10); -/// What percentile to keep our relays optimized to -const RELAY_OPTIMIZATION_PERCENTILE: f32 = 66.0; -/// What percentile to choose our relays from (must be greater than RELAY_OPTIMIZATION_PERCENTILE) -const RELAY_SELECTION_PERCENTILE: f32 = 85.0; +/// Drop a relay for being slow only once it falls below this percentile and has degraded +/// against its own baseline. The gap to RELAY_SELECTION_PERCENTILE is the hysteresis dead +/// band: a freshly chosen relay has that much room to degrade before it re-qualifies for +/// replacement. +const RELAY_OPTIMIZATION_PERCENTILE: f32 = 50.0; +/// Below this the relay is dropped on the percentile alone: the incumbent is the +/// only candidate carrying our traffic, but down here no amount of load explains it. +const RELAY_OPTIMIZATION_FLOOR_PERCENTILE: f32 = 10.0; +/// How much worse than its own best tm90 a relay must get before the percentile test +/// may drop it. Judging a relay against itself keeps a good relay in a fast fleet, +/// and re-selection is not free: the flow resettles and our dial info republishes. +const RELAY_DEGRADATION_MARGIN: u64 = 2; +/// What percentile to choose our relays from (must be greater than RELAY_OPTIMIZATION_PERCENTILE). +/// Also the anti-herding knob: selection picks at random from this tier, so a wider tier +/// spreads load across more nodes. +const RELAY_SELECTION_PERCENTILE: f32 = 75.0; +/// How long a newly selected relay is left alone before it can be optimized away. +/// Optimization only: a disqualified, dead, punished or erroring relay is dropped +/// immediately however long it has served. Latency right after selection is the least +/// trustworthy: the flow has not settled and our own traffic load inflates the +/// incumbent's measurements relative to resting candidates. +const RELAY_MIN_DWELL_BEFORE_OPTIMIZATION: TimestampDuration = TimestampDuration::new_secs(300); impl PublicInternetRoutingDomainController { - /// Check is a relay is still valid is this routing domain + /// Check if a relay is still valid in this routing domain /// Does not require access to the routing domain controller itself + /// + /// Past the dead/punished check, every judgment here reads evidence our own degradation + /// can manufacture, so while blame is suppressed the incumbent keeps its job + #[expect(clippy::too_many_arguments)] fn check_relay_valid( routing_table: &RoutingTable, cur_ts: Timestamp, relay: &RoutingDomainRelay, mut relay_state: RoutingDomainRelayState, relay_node_filter: &impl Fn(&BucketEntrySnapshot) -> bool, + relay_retention_filter: &impl Fn(&BucketEntrySnapshot) -> bool, + relay_retention_filter_reason: &impl Fn(&BucketEntrySnapshot) -> Option<&'static str>, snapshot: &RoutingTableSnapshot, ) -> Option { let relay_node = relay.relay_node.clone(); @@ -35,17 +59,24 @@ impl PublicInternetRoutingDomainController { return None; } + let blame_suppressed = routing_table + .network_manager() + .online_detector() + .blame_suppressed(RoutingDomain::PublicInternet); + // Relay node no longer can relay. Tolerate transient unresponsiveness (Missing from a // single lost question): truly dead relays are dropped by the state check above, and // dropping on every blip reselects relays until the flap detector trips. let relay_snap = relay_node .entry() .snapshot(routing_table.registry(), cur_ts); - let relay_filter_passes = relay_node_filter(&relay_snap); + // Retention, not candidacy: an incumbent keeps its job unless something is wrong with it + let relay_filter_passes = relay_retention_filter(&relay_snap); if !relay_filter_passes { - if relay_snap.state.is_responsive() { + if relay_snap.state.is_responsive() && !blame_suppressed { veilid_log!(routing_table debug - "Relay node can no longer relay, dropping relay {}", + "Relay node can no longer relay ({}), dropping relay {}", + relay_retention_filter_reason(&relay_snap).unwrap_or("unknown"), relay.relay_node ); return None; @@ -57,15 +88,17 @@ impl PublicInternetRoutingDomainController { } // Pathological: protected connections die too fast on this relay. - if let Some(drop_stats) = relay_node.connection_stats().protected_drop_span { - if drop_stats.tm90 < RELAY_MIN_PROTECTED_CONNECTION_TM90 { - veilid_log!(routing_table debug - "Relay protected connection lifetime tm90={} < {}, dropping relay {}", - drop_stats.tm90, - RELAY_MIN_PROTECTED_CONNECTION_TM90, - relay_node - ); - return None; + if !blame_suppressed { + if let Some(drop_stats) = relay_node.connection_stats().protected_drop_span { + if drop_stats.tm90 < RELAY_MIN_PROTECTED_CONNECTION_TM90 { + veilid_log!(routing_table debug + "Relay protected connection lifetime tm90={} < {}, dropping relay {}", + drop_stats.tm90, + RELAY_MIN_PROTECTED_CONNECTION_TM90, + relay_node + ); + return None; + } } } @@ -74,8 +107,16 @@ impl PublicInternetRoutingDomainController { // relative-performance set by the same filter, so optimizing it here would drop it as // "unmeasurable" and undo the retention above every interval; let it recover or die // via the Dead state check instead. + // Our own traffic concentrates on the incumbent, so congestion sinks its percentile + // while resting candidates look fast: above the floor it must also have degraded + // against its own settled baseline before we drop it. let last_optimized_duration = cur_ts.duration_since(relay_state.last_optimized); - if relay_filter_passes && last_optimized_duration > RELAY_OPTIMIZATION_INTERVAL { + let dwell_duration = cur_ts.duration_since(relay_state.selected_ts); + if relay_filter_passes + && !blame_suppressed + && last_optimized_duration > RELAY_OPTIMIZATION_INTERVAL + && dwell_duration >= RELAY_MIN_DWELL_BEFORE_OPTIMIZATION + { // See what our relay's current percentile is if let Some(relay_relative_performance) = routing_table.get_node_relative_performance( relay_node_id, @@ -84,38 +125,61 @@ impl PublicInternetRoutingDomainController { |ls| ls.tm90, ) { // Get latency numbers - let latency_stats = if let Some(latency) = relay_node.peer_stats().latency { - latency.to_string() - } else { - "[no stats]".to_owned() + let opt_latency = relay_node.peer_stats().latency; + let opt_tm90 = opt_latency.as_ref().map(|l| l.tm90); + let latency_stats = match &opt_latency { + Some(latency) => latency.to_string(), + None => "[no stats]".to_owned(), }; + // The baseline the margin below is measured against + if let Some(tm90) = opt_tm90 { + relay_state.best_tm90 = Some(match relay_state.best_tm90 { + Some(best) => best.min(tm90), + None => tm90, + }); + } + // Get current relay reliability let state_reason = relay_node.state_reason(cur_ts); - if relay_relative_performance.percentile < RELAY_OPTIMIZATION_PERCENTILE { + // With no latency stats there is nothing to judge against itself + let degraded_from_baseline = match (opt_tm90, relay_state.best_tm90) { + (Some(cur), Some(best)) => cur > best.saturating_mul(RELAY_DEGRADATION_MARGIN), + _ => true, + }; + let below_floor = + relay_relative_performance.percentile < RELAY_OPTIMIZATION_FLOOR_PERCENTILE; + let below_bar = + relay_relative_performance.percentile < RELAY_OPTIMIZATION_PERCENTILE; + + if below_floor || (below_bar && degraded_from_baseline) { // Drop the current relay so we can get the best new one veilid_log!(routing_table debug - "Relay tm90 is ({:.2}% < {:.2}%) ({} out of {}) (latency {}, {:?}) optimizing relay {}", + "Relay tm90 is ({:.2}% < {:.2}%) ({} out of {}) (latency {}, best {}, {:?}) optimizing relay {} after serving {:#} ({})", relay_relative_performance.percentile, - RELAY_OPTIMIZATION_PERCENTILE, + if below_floor { RELAY_OPTIMIZATION_FLOOR_PERCENTILE } else { RELAY_OPTIMIZATION_PERCENTILE }, relay_relative_performance.node_index, relay_relative_performance.node_count, latency_stats, + relay_state.best_tm90.map(|t| t.to_string()).unwrap_or_else(|| "none".to_owned()), state_reason, - relay_node + relay_node, + dwell_duration, + if below_floor { "below floor" } else { "degraded against its own baseline" } ); return None; } else { // Note that we tried to optimize the relay but it was good #[cfg(feature = "verbose-tracing")] veilid_log!(routing_table debug - "Relay tm90 is ({:.2}% >= {:.2}%) ({} out of {}) (latency {}, {:?}) keeping {}", + "Relay tm90 is ({:.2}% vs {:.2}%) ({} out of {}) (latency {}, best {}, {:?}) keeping {}", relay_relative_performance.percentile, RELAY_OPTIMIZATION_PERCENTILE, relay_relative_performance.node_index, relay_relative_performance.node_count, latency_stats, + relay_state.best_tm90.map(|t| t.to_string()).unwrap_or_else(|| "none".to_owned()), state_reason, relay_node ); @@ -162,10 +226,18 @@ impl PublicInternetRoutingDomainController { // Pull everything we need from the routing-domain detail under a brief read lock, // then drop it before any routing-table or bucket-entry work to avoid a // lock-order inversion with last_flows → routing_domain_for_flow. - let (opt_last_relay_compilation, relay_node_filter, prefetched_relay_states) = { + let ( + opt_last_relay_compilation, + relay_node_filter, + relay_retention_filter, + relay_retention_filter_reason, + prefetched_relay_states, + ) = { let detail = self.read(); let opt_last_relay_compilation = detail.relay_compilation(); let relay_node_filter = detail.make_relay_node_filter(); + let relay_retention_filter = detail.make_relay_retention_filter(); + let relay_retention_filter_reason = detail.make_relay_retention_filter_reason(); let prefetched_relay_states: BTreeMap = opt_last_relay_compilation .as_ref() @@ -183,6 +255,8 @@ impl PublicInternetRoutingDomainController { ( opt_last_relay_compilation, relay_node_filter, + relay_retention_filter, + relay_retention_filter_reason, prefetched_relay_states, ) }; @@ -205,6 +279,8 @@ impl PublicInternetRoutingDomainController { relay, relay_state, &relay_node_filter, + &relay_retention_filter, + &relay_retention_filter_reason, &relay_snapshot, ) { if updated_relay_state != relay_state { diff --git a/veilid-core/src/routing_table/routing_domains/public_internet/detail.rs b/veilid-core/src/routing_table/routing_domains/public_internet/detail.rs index e1af1e10..49530fa8 100644 --- a/veilid-core/src/routing_table/routing_domains/public_internet/detail.rs +++ b/veilid-core/src/routing_table/routing_domains/public_internet/detail.rs @@ -93,7 +93,36 @@ impl PublicInternetRoutingDomainDetail { true } + /// Candidate filter: a node must have proven it can carry our traffic before we + /// pick it as a relay. pub fn make_relay_node_filter(&self) -> impl Fn(&BucketEntrySnapshot) -> bool { + let inner = self.make_relay_node_filter_inner(true); + move |snap| inner(snap).is_none() + } + + /// Retention filter for a relay we already chose. Same disqualifiers, except that + /// missing evidence is not one: we only ever dial a relay over the flow we already + /// have, so per-transport dial evidence may never appear. Only positive failure + /// evidence drops an incumbent. + pub fn make_relay_retention_filter(&self) -> impl Fn(&BucketEntrySnapshot) -> bool { + let inner = self.make_relay_node_filter_inner(false); + move |snap| inner(snap).is_none() + } + + /// Why a relay we already chose should be dropped, or None to keep it + pub fn make_relay_retention_filter_reason( + &self, + ) -> impl Fn(&BucketEntrySnapshot) -> Option<&'static str> { + self.make_relay_node_filter_inner(false) + } + + /// Shared body of the relay filters: the disqualifying reason, or None to accept. + /// Transport health keys off last_seen_ts because it is monotone; a streak measure + /// would reset on a single lost question and churn relays under congestion. + fn make_relay_node_filter_inner( + &self, + require_evidence: bool, + ) -> impl Fn(&BucketEntrySnapshot) -> Option<&'static str> { let ip6_prefix_size = self .config() .internal() @@ -109,7 +138,7 @@ impl PublicInternetRoutingDomainDetail { move |snap: &BucketEntrySnapshot| { // Defensively exclude nodes that aren't responsive if !snap.state.is_responsive() { - return false; + return Some("not responsive"); } // Ensure this node is not on the local network @@ -117,24 +146,24 @@ impl PublicInternetRoutingDomainDetail { .routing_domain_set() .contains(RoutingDomain::LocalNetwork) { - return false; + return Some("local network"); } // Exclude any nodes that don't have a 'best node id' for our enabled cryptosystems if snap.best_node_id().is_none() { - return false; + return Some("no best node id"); } // Exclude nodes whose past protected connections died too fast. if let Some(drop_stats) = &snap.connection_stats.protected_drop_span { if drop_stats.tm90 < RELAY_MIN_PROTECTED_CONNECTION_TM90 { - return false; + return Some("protected connections die too fast"); } } // Get the public internet peer info so we can validate it let Some(peer_info) = snap.get_peer_info(RoutingDomain::PublicInternet) else { - return false; + return Some("no public internet peer info"); }; // Exclude any nodes that are relaying directly through us @@ -142,7 +171,7 @@ impl PublicInternetRoutingDomainDetail { .node_ids() .contains_any_from_iter(peer_info.node_info().relay_ids().iter()) { - return false; + return Some("relays through us"); } // Disqualify nodes that don't have relay capability @@ -150,7 +179,7 @@ impl PublicInternetRoutingDomainDetail { .node_info() .has_capability(VEILID_CAPABILITY_RELAY) { - return false; + return Some("no relay capability"); } // Disqualify any nodes that don't speak all of the envelope versions we do @@ -162,11 +191,16 @@ impl PublicInternetRoutingDomainDetail { .copied() .any(|x| !peer_envelope_support.contains(&x)) { - return false; + return Some("envelope version mismatch"); } // Ensure a transport we can reach this relay through is currently healthy. + // Counters record why each candidate transport was rejected, so churn is attributable. let mut has_reachable_transport = false; + let mut considered = 0usize; + let mut no_evidence = 0usize; + let mut unreachable = 0usize; + let mut failing = 0usize; for did in peer_info.node_info().dial_info_detail_list() { if did.class.requires_signal() { continue; @@ -174,13 +208,26 @@ impl PublicInternetRoutingDomainDetail { if !did.dial_info.matches_filter(&outbound_dif) { continue; } + considered += 1; let t: TransportType = (&did.dial_info).into(); - let healthy = snap.per_transport.get(&t).is_some_and(|stats| { - stats.first_steady_answer_ts.is_some() - && stats.unreachable == 0 - && stats.failed_to_send == 0 - && stats.recent_lost_questions == 0 - }); + let opt_stats = snap.per_transport.get(&t); + match opt_stats { + None => no_evidence += 1, + Some(stats) if stats.last_seen_ts.is_none() => no_evidence += 1, + Some(stats) if stats.unreachable != 0 => unreachable += 1, + Some(stats) if transport_health(stats).is_failing() => failing += 1, + _ => {} + } + // Answered a dialed question here at some point and is not now Failing + let healthy = match opt_stats { + // Absent evidence disqualifies a candidate but never an incumbent + None => !require_evidence, + Some(stats) => { + stats.unreachable == 0 + && !transport_health(stats).is_failing() + && (!require_evidence || stats.last_seen_ts.is_some()) + } + }; if healthy { has_reachable_transport = true; break; @@ -188,7 +235,17 @@ impl PublicInternetRoutingDomainDetail { } if !has_reachable_transport { - return false; + return Some(if considered == 0 { + "no dial info matching our outbound capabilities" + } else if require_evidence && no_evidence == considered { + "never dialed on any usable transport" + } else if failing > 0 { + "usable transports are failing" + } else if unreachable > 0 { + "usable transports unreachable" + } else { + "no healthy reachable transport" + }); } // Exclude any nodes that have our same network block @@ -196,10 +253,10 @@ impl PublicInternetRoutingDomainDetail { .node_info() .is_on_same_ipblock(peer_info.node_info(), ip6_prefix_size) { - return false; + return Some("same ip block as us"); } - true + None } } } diff --git a/veilid-core/src/routing_table/routing_domains/public_internet/editor.rs b/veilid-core/src/routing_table/routing_domains/public_internet/editor.rs index ada1dcce..50ea87ee 100644 --- a/veilid-core/src/routing_table/routing_domains/public_internet/editor.rs +++ b/veilid-core/src/routing_table/routing_domains/public_internet/editor.rs @@ -226,6 +226,9 @@ impl<'a> RoutingDomainEditor for RoutingDomainEditorPublicInternet<'a> { .write() .reset_all_updated_since_last_network_change(); + // Our side of every edge changed; recompute contact-method viability + routing_table.mark_cm_viability_dirty(); + if let Err(e) = self.event_bus().post(RoutingDomainCommitEvent { routing_domain: RoutingDomain::PublicInternet, peer_info_changed: changed.peer_info_changed, diff --git a/veilid-core/src/routing_table/routing_domains/routing_domain_detail_common.rs b/veilid-core/src/routing_table/routing_domains/routing_domain_detail_common.rs index e981933d..7bc21b23 100644 --- a/veilid-core/src/routing_table/routing_domains/routing_domain_detail_common.rs +++ b/veilid-core/src/routing_table/routing_domains/routing_domain_detail_common.rs @@ -309,6 +309,8 @@ impl RoutingDomainDetailCommon { .or_insert_with(|| RoutingDomainRelayState { last_keepalive: cur_ts, last_optimized: cur_ts, + selected_ts: cur_ts, + best_tm90: None, }); } } diff --git a/veilid-core/src/routing_table/routing_domains/routing_domain_relay.rs b/veilid-core/src/routing_table/routing_domains/routing_domain_relay.rs index 6bd3ccee..66a6fd24 100644 --- a/veilid-core/src/routing_table/routing_domains/routing_domain_relay.rs +++ b/veilid-core/src/routing_table/routing_domains/routing_domain_relay.rs @@ -83,21 +83,31 @@ impl fmt::Display for RelayPing { } /// The routing domain's state tracking for a single relay -/// Keeps track of when the last keepalive was sent to the relay -/// and when the last attempt to optimize the relay was made #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub struct RoutingDomainRelayState { + /// When we last sent this relay a keepalive pub last_keepalive: Timestamp, + /// When we last attempted to optimize this relay pub last_optimized: Timestamp, + /// When this relay was selected. Never updated while it serves, so + /// `now - selected_ts` is how long it has held the job. + pub selected_ts: Timestamp, + /// Best tm90 seen since this relay settled in, the baseline it is judged against. + /// None until the first measurement past the dwell. + pub best_tm90: Option, } impl fmt::Display for RoutingDomainRelayState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "Keepalive:{}\nOptimized:{}", + "Keepalive:{}\nOptimized:{}\nSelected:{}\nBestTm90:{}", f.to_string(self.last_keepalive), - f.to_string(self.last_optimized) + f.to_string(self.last_optimized), + f.to_string(self.selected_ts), + self.best_tm90 + .map(|t| t.to_string()) + .unwrap_or_else(|| "none".to_owned()) ) } } diff --git a/veilid-core/src/routing_table/routing_table_inner.rs b/veilid-core/src/routing_table/routing_table_inner.rs index b11dcebd..f131f83e 100644 --- a/veilid-core/src/routing_table/routing_table_inner.rs +++ b/veilid-core/src/routing_table/routing_table_inner.rs @@ -51,6 +51,18 @@ impl RoutingTableInner { }); } + /// Drop learned dial evidence for every entry. Callers must obey the address-change + /// restriction on `RoutingTable::clear_all_transport_evidence`. + pub fn clear_all_transport_evidence(&mut self) { + let cur_ts = Timestamp::now(); + self.with_entries(cur_ts, BucketEntryState::Dead, |v| { + v.with_mut(|e| { + e.clear_transport_evidence(); + }); + Option::<()>::None + }); + } + pub(in crate::routing_table) fn bucket_depth(pos: usize) -> usize { match pos { 0 => 256, @@ -75,7 +87,7 @@ impl RoutingTableInner { /// Total count of entries across all buckets (any crypto kind) that exceed /// their bucket's max depth and haven't been kicked yet. This is "over-attached" - /// in the bucket-fill sense — capacity is overrun, lazy kicking will trim later. + /// in the bucket-fill sense (capacity is overrun, lazy kicking will trim later) pub(in crate::routing_table) fn excess_kickable_count(&self) -> usize { let mut total = 0usize; for buckets in self.buckets.values() { diff --git a/veilid-core/src/routing_table/tasks/mod.rs b/veilid-core/src/routing_table/tasks/mod.rs index a96e2d8d..d1e74b5c 100644 --- a/veilid-core/src/routing_table/tasks/mod.rs +++ b/veilid-core/src/routing_table/tasks/mod.rs @@ -1,6 +1,7 @@ pub mod flush; pub mod kick_buckets; pub mod ping_validator; +pub mod update_cm_viability; pub mod update_statistics; use super::*; @@ -39,6 +40,14 @@ impl RoutingTable { // Set kick buckets tick task impl_setup_task!(self, Self, kick_buckets_task, kick_buckets_task_routine); + // Set contact-method viability tick task + impl_setup_task!( + self, + Self, + update_cm_viability_task, + update_cm_viability_task_routine + ); + // Note: ping validation processor is spawned in startup(), not as a TickTask } @@ -72,6 +81,14 @@ impl RoutingTable { // Do rolling answers every ROLLING_ANSWER_INTERVAL_SECS secs self.rolling_answers_task.tick().await?; + // Recompute contact-method viability when peerinfo changed on either side + if self + .cm_viability_dirty + .load(core::sync::atomic::Ordering::Acquire) + { + self.update_cm_viability_task.tick().await?; + } + // Kick buckets task let kick_bucket_queue_count = self.kick_queue.lock().len(); if kick_bucket_queue_count > 0 { @@ -112,6 +129,10 @@ impl RoutingTable { if let Err(e) = self.kick_buckets_task.stop().await { veilid_log!(self warn "kick_buckets_task not stopped: {}", e); } + veilid_log!(self debug "stopping update cm viability task"); + if let Err(e) = self.update_cm_viability_task.stop().await { + veilid_log!(self warn "update_cm_viability_task not stopped: {}", e); + } veilid_log!(self debug "stopping routing domain controller task"); for rdc in self.get_routing_domain_controllers(RoutingDomainSet::all()) { rdc.cancel_tasks().await; diff --git a/veilid-core/src/routing_table/tasks/ping_validator.rs b/veilid-core/src/routing_table/tasks/ping_validator.rs index a1480257..b27b3727 100644 --- a/veilid-core/src/routing_table/tasks/ping_validator.rs +++ b/veilid-core/src/routing_table/tasks/ping_validator.rs @@ -6,6 +6,9 @@ use stop_token::future::FutureExt as _; impl_veilid_log_facility!("rtab"); +/// Minimum seconds between tests of the same route +const ROUTE_TEST_MIN_INTERVAL_SECS: u32 = 15; + ///////////////////////////////////// // Priority Groups @@ -407,14 +410,33 @@ impl RoutingTable { /// Enqueue route test pings for a set of routes. /// Allocated routes: one loopback per SequenceOrdering the route supports. /// Remote routes: a single ping. + /// Returns the number of routes accepted, which excludes those still in test cooldown. pub async fn enqueue_route_tests( &self, cur_ts: Timestamp, route_ids: Vec, priority: usize, - ) { + ) -> usize { let rss = self.route_spec_store(); + // Continuous re-testing of a struggling route is churn, not information + let route_ids: Vec = { + let cooldown = TimestampDuration::new_secs(ROUTE_TEST_MIN_INTERVAL_SECS); + let mut recent = self.route_test_cooldown.lock(); + recent.retain(|_, ts| cur_ts.duration_since(*ts) < cooldown); + route_ids + .into_iter() + .filter(|route_id| { + if recent.contains_key(route_id) { + return false; + } + recent.insert(route_id.clone(), cur_ts); + true + }) + .collect() + }; + let accepted = route_ids.len(); + let mut entries = Vec::new(); for route_id in route_ids { let is_remote = rss.is_route_id_remote(&route_id); @@ -447,10 +469,11 @@ impl RoutingTable { if !entries.is_empty() { self.enqueue_ping_validation_entries("RouteTest".to_owned(), entries); } + accepted } /// Enqueue per-ordering loopback pings for allocated routes. Same shape as a test, - /// driven by the keepalive rate limiter — the entry it produces is identical. + /// driven by the keepalive rate limiter; the entry it produces is identical. pub async fn enqueue_route_loopback_keepalives( &self, cur_ts: Timestamp, @@ -532,8 +555,19 @@ impl RoutingTable { return Vec::new(); }; + // An ordering with no working carrier can only fail, blaming the route for our dead path + let network_manager = rss.network_manager(); + let transport_stats = network_manager.transport_stats(); + let orderings: Vec = req + .orderings + .iter() + .filter(|ordering| { + transport_stats.ordering_is_viable(RoutingDomain::PublicInternet, *ordering) + }) + .collect(); + let mut entries = Vec::new(); - for (idx, ordering) in req.orderings.iter().enumerate() { + for (idx, ordering) in orderings.iter().copied().enumerate() { let dest = Self::build_loopback_destination( &req.route_id, private_route.clone(), @@ -561,8 +595,8 @@ impl RoutingTable { } rss.update_allocated_route_stats(cur_ts, &key, |s| { - for ordering in req.orderings.iter() { - s.record_loopback_keepalive(cur_ts, ordering); + for ordering in orderings.iter() { + s.record_loopback_keepalive(cur_ts, *ordering); } Ok(()) }); @@ -606,7 +640,7 @@ impl RoutingTable { result: Result| -> Vec { match result { - Ok(StatusResult::Answer { .. }) => { + Ok(StatusResult::Sent(_)) => { #[cfg(feature = "verbose-tracing")] veilid_log!(registry trace "Remote route test PASSED: {:#}", route_id_clone); } @@ -615,11 +649,10 @@ impl RoutingTable { let nm = routing_table.network_manager(); if nm .online_detector() - .online_state(RoutingDomain::PublicInternet) - == OnlineState::Offline + .blame_suppressed(RoutingDomain::PublicInternet) { #[cfg(feature = "verbose-tracing")] - veilid_log!(registry debug "Deferring remote route release (offline detected): {:#}", route_id_clone); + veilid_log!(registry debug "Deferring remote route release (offline/degraded): {:#}", route_id_clone); } else { veilid_log!(registry debug "Remote route test failed (no response): {:#}", route_id_clone); routing_table @@ -640,7 +673,7 @@ impl RoutingTable { }) } - /// Completion callback for a per-ordering route loopback ping. On Answer we record a + /// Completion callback for a per-ordering route loopback ping. On Sent we record a /// per-ordering pass on the route (drives last_known_valid_ts / clears failures). On Failed /// we record a per-ordering failure on the route (drives the ordering, and eventually the /// route, to dead) and attribute failure to the transport actually used at the first hop. @@ -658,12 +691,10 @@ impl RoutingTable { let routing_table = registry.routing_table(); let rss = routing_table.route_spec_store(); let sdr = match result { - Ok(StatusResult::Answer { - answer: _answer, .. - }) => { + Ok(StatusResult::Sent(_status_answer)) => { // End-to-end route round-trip (includes inter-hop legs), not a per-node RTT. #[cfg(feature = "verbose-tracing")] - veilid_log!(registry debug "Route loopback rtt: route={:#} ord={:#} rtt={:#} hops={}", route_key, ordering, _answer.answer_context.latency, hops.len()); + veilid_log!(registry debug "Route loopback rtt: route={:#} ord={:#} rtt={:#} hops={}", route_key, ordering, _status_answer.answer.answer_context.latency, hops.len()); let cur_ts = Timestamp::now(); rss.update_allocated_route_stats(cur_ts, &route_key, |s| { @@ -686,14 +717,27 @@ impl RoutingTable { if routing_table .network_manager() .online_detector() - .online_state(RoutingDomain::PublicInternet) - == OnlineState::Offline + .blame_suppressed(RoutingDomain::PublicInternet) { #[cfg(feature = "verbose-tracing")] - veilid_log!(registry debug "Deferring route loopback failure (offline): {:#} ({:#})", route_id, ordering); + veilid_log!(registry debug "Deferring route loopback failure (offline/degraded): {:#} ({:#})", route_id, ordering); return Vec::new(); } + // Neither route nor hops can be blamed for a transport we could not send on + let opt_transport = sdr.opt_transport_type(); + if let Some(transport) = opt_transport { + if routing_table + .network_manager() + .transport_stats() + .verdict(RoutingDomain::PublicInternet, transport) + .is_send_suppressed() + { + veilid_log!(registry debug "Deferring route loopback failure (transport suppressed): {:#} ({:#} via {:#})", route_id, ordering, transport); + return Vec::new(); + } + } + // Record a per-ordering failure on the route so repeated loopback failures // drive this ordering (and eventually the route) to dead and release. let cur_ts = Timestamp::now(); @@ -705,7 +749,7 @@ impl RoutingTable { // now (sticky) so it gets released once its refcount drains. rss.mark_allocated_route_for_release_if_dead(&route_key); - let Some(transport) = sdr.opt_transport_type() else { + let Some(transport) = opt_transport else { veilid_log!(registry debug "Route loopback FAILED with no transport recorded: {:#} ({:#})", route_id, ordering); return Vec::new(); }; diff --git a/veilid-core/src/routing_table/tasks/update_cm_viability.rs b/veilid-core/src/routing_table/tasks/update_cm_viability.rs new file mode 100644 index 00000000..293b15be --- /dev/null +++ b/veilid-core/src/routing_table/tasks/update_cm_viability.rs @@ -0,0 +1,141 @@ +use super::*; + +impl_veilid_log_facility!("rtab"); + +/// Protocols that carry a given sequence ordering +// XXX: WebTransport will offer both orderings over UDP; derive ordering support from +// dialinfo rather than protocol when it lands +fn protocols_for_ordering(ordering: SequenceOrdering) -> ProtocolTypeSet { + let mut protocols = ProtocolTypeSet::new(); + for p in ProtocolTypeSet::all().iter() { + if p.sequence_ordering() == ordering { + protocols.insert(p); + } + } + protocols +} + +impl RoutingTable { + /// Recompute entries' contact-method viability when a peerinfo changes on + /// either side; the per-entry basis check skips unchanged edges. Applies + /// evidence retention pruning on the same edge. + #[cfg_attr(feature = "instrument", instrument(level = "trace", skip(self), err, fields(__VEILID_LOG_KEY = self.log_key())))] + pub fn update_cm_viability_task_routine( + &self, + _stop_token: StopToken, + _last_ts: Timestamp, + _cur_ts: Timestamp, + ) -> EyreResult<()> { + // Consume the flag first; changes landing during this pass re-set it + self.take_cm_viability_dirty(); + + struct DomainContext { + routing_domain: RoutingDomain, + peer_a: Arc, + peer_a_published: bool, + } + let domain_contexts: Vec = + [RoutingDomain::PublicInternet, RoutingDomain::LocalNetwork] + .into_iter() + .map(|routing_domain| { + let peer_a = self.get_current_peer_info(routing_domain); + let peer_a_published = self + .get_published_peer_info(routing_domain) + .map(|p| p.node_info().timestamp()) + == Some(peer_a.node_info().timestamp()); + DomainContext { + routing_domain, + peer_a, + peer_a_published, + } + }) + .collect(); + + let all_entries: Vec> = self.inner.read().all_entries.iter().collect(); + for entry in all_entries { + // Snapshot target peer infos and the basis the current cache was computed against + let (peer_bs, old_basis) = entry.with(|e| { + let peer_bs: Vec<(&DomainContext, Arc)> = domain_contexts + .iter() + .filter_map(|dc| e.get_peer_info(dc.routing_domain).map(|pi| (dc, pi))) + .collect(); + let old_basis = e.cm_viability().map(|v| v.basis.clone()); + (peer_bs, old_basis) + }); + + if peer_bs.is_empty() { + entry.with_mut(|e| e.set_cm_viability(None)); + continue; + } + + let new_basis: BTreeMap = peer_bs + .iter() + .map(|(dc, peer_b)| { + ( + dc.routing_domain, + ( + dc.peer_a.node_info().timestamp(), + peer_b.node_info().timestamp(), + ), + ) + }) + .collect(); + if old_basis.as_ref() == Some(&new_basis) { + continue; + } + + // Compute contact methods per ordering, merged across routing domains + let mut per_ordering = BTreeMap::::new(); + let mut all_cms = Vec::::new(); + for (dc, peer_b) in &peer_bs { + for ordering in [SequenceOrdering::Ordered, SequenceOrdering::Unordered] { + let protocols = protocols_for_ordering(ordering); + let dial_info_filter = DialInfoFilter::all() + .with_protocol_type_set( + protocols & dc.peer_a.node_info().outbound_protocols(), + ) + .with_address_type_set(dc.peer_a.node_info().address_types()); + let cms = self.get_contact_methods( + dc.routing_domain, + ContactMethodRequest { + peer_a: dc.peer_a.clone(), + peer_a_published: dc.peer_a_published, + peer_b: peer_b.clone(), + dial_info_filter, + sequencing: Sequencing::PreferUnordered, + }, + ); + if cms.is_empty() { + continue; + } + let reach = per_ordering.entry(ordering).or_default(); + for cm in &cms { + match cm { + ContactMethod::Direct { target_di } => { + reach + .direct_transports + .insert(TransportType::from(target_di)); + } + _ => { + reach.has_indirect = true; + } + } + } + all_cms.extend(cms); + } + } + + entry.with_mut(|e| { + // The edge changed; keep evidence only for contact methods still offered + if old_basis.is_some() { + e.prune_evidence_to_contact_methods(&all_cms); + } + e.set_cm_viability(Some(CMViability { + per_ordering, + basis: new_basis, + })); + }); + } + Ok(()) + } +} diff --git a/veilid-core/src/routing_table/tests_routing_table/mod.rs b/veilid-core/src/routing_table/tests_routing_table/mod.rs index edd558b3..5daac6cb 100644 --- a/veilid-core/src/routing_table/tests_routing_table/mod.rs +++ b/veilid-core/src/routing_table/tests_routing_table/mod.rs @@ -39,10 +39,13 @@ pub async fn test_all() { test_bucket_entry_state::test_state_punished(); test_bucket_entry_state::test_state_dead_excessive_unreachable(); test_bucket_entry_state::test_state_dead_excessive_send_failures(); + test_bucket_entry_state::test_state_viable_ordering_vetoes_death(); + test_bucket_entry_state::test_state_send_failures_never_kill(); test_bucket_entry_state::test_state_dead_never_seen_lost_questions(); test_bucket_entry_state::test_state_dead_steady_lost_questions(); test_bucket_entry_state::test_state_missing_failed_to_send(); test_bucket_entry_state::test_state_missing_lost_questions(); + test_bucket_entry_state::test_state_revive_reaches_initial(); test_bucket_entry_state::test_state_initial(); test_bucket_entry_state::test_state_unreliable(); test_bucket_entry_state::test_state_reliable(); diff --git a/veilid-core/src/routing_table/tests_routing_table/test_bucket_entry_state.rs b/veilid-core/src/routing_table/tests_routing_table/test_bucket_entry_state.rs index 715e4716..d2180ac0 100644 --- a/veilid-core/src/routing_table/tests_routing_table/test_bucket_entry_state.rs +++ b/veilid-core/src/routing_table/tests_routing_table/test_bucket_entry_state.rs @@ -17,6 +17,7 @@ fn compute( rpc_stats: &RPCStats, per_so: &BTreeMap, per_transport: &BTreeMap, + cm_viability: Option<&CMViability>, ) -> BucketEntryStateReason { BucketEntryInner::compute_state_reason_from_stats( now(), @@ -24,15 +25,17 @@ fn compute( rpc_stats, per_so, per_transport, - |_| TimestampDuration::new(0), + cm_viability, + |_, _| TimestampDuration::new(0), ) } -// As `compute`, but credits `offline_secs` of offline time to every window. +// As `compute`, but credits `offline_secs` of untestable time to every window. fn compute_with_offline( punishment: Option, rpc_stats: &RPCStats, per_so: &BTreeMap, per_transport: &BTreeMap, + cm_viability: Option<&CMViability>, offline_secs: u32, ) -> BucketEntryStateReason { BucketEntryInner::compute_state_reason_from_stats( @@ -41,20 +44,59 @@ fn compute_with_offline( rpc_stats, per_so, per_transport, - |_| TimestampDuration::new_secs(offline_secs), + cm_viability, + |_, _| TimestampDuration::new_secs(offline_secs), ) } fn so_map(entries: &[(SequenceOrdering, RPCStats)]) -> BTreeMap { entries.iter().cloned().collect() } +fn udp4() -> TransportType { + TransportType::new(ProtocolType::UDP, AddressType::IPV4) +} +fn udp6() -> TransportType { + TransportType::new(ProtocolType::UDP, AddressType::IPV6) +} +fn tcp4() -> TransportType { + TransportType::new(ProtocolType::TCP, AddressType::IPV4) +} +// Per-transport stats that judge as Failing for liveness (unanswered questions) +fn failing_stats() -> RPCStats { + RPCStats { + recent_lost_questions: TRANSPORT_FAILING_LOST_QUESTIONS, + ..Default::default() + } +} +// Send failures alone: Failing for selection, never for liveness +fn send_failing_stats() -> RPCStats { + RPCStats { + failed_to_send: TRANSPORT_FAILING_FAILED_TO_SEND, + ..Default::default() + } +} +fn transport_map(entries: &[(TransportType, RPCStats)]) -> BTreeMap { + entries.iter().cloned().collect() +} // A non-empty per-transport map; only its emptiness gates [D3]/[D4], contents are irrelevant. fn seen_transports() -> BTreeMap { - let mut m = BTreeMap::new(); - m.insert( - TransportType::new(ProtocolType::UDP, AddressType::IPV4), - RPCStats::default(), - ); - m + transport_map(&[(udp4(), RPCStats::default())]) +} +fn viability(entries: &[(SequenceOrdering, &[TransportType], bool)]) -> CMViability { + CMViability { + per_ordering: entries + .iter() + .map(|(so, transports, has_indirect)| { + ( + *so, + OrderingReachability { + direct_transports: transports.iter().cloned().collect(), + has_indirect: *has_indirect, + }, + ) + }) + .collect(), + basis: BTreeMap::new(), + } } pub fn test_state_punished() { @@ -73,7 +115,8 @@ pub fn test_state_punished() { Some(PunishmentReason::Manual), &RPCStats::default(), &reliable, - &seen_transports() + &seen_transports(), + None ), BucketEntryStateReason::Punished(PunishmentReason::Manual) ); @@ -89,7 +132,7 @@ pub fn test_state_dead_excessive_unreachable() { ..Default::default() }; assert_eq!( - compute(None, &rpc, &BTreeMap::new(), &BTreeMap::new()), + compute(None, &rpc, &BTreeMap::new(), &BTreeMap::new(), None), BucketEntryStateReason::Dead(BucketEntryStateDeadReason::ExcessiveUnreachable) ); } @@ -97,74 +140,150 @@ pub fn test_state_dead_excessive_unreachable() { pub fn test_state_dead_excessive_send_failures() { info!("--- test_state_dead_excessive_send_failures ---"); - // [D2] failed_to_send at/above threshold on any sequence ordering is dead - for so in [SequenceOrdering::Unordered, SequenceOrdering::Ordered] { - let per_so = so_map(&[( - so, - RPCStats { - failed_to_send: dead_failed_to_send_count(so), - ..Default::default() - }, - )]); - assert_eq!( - compute(None, &RPCStats::default(), &per_so, &BTreeMap::new()), - BucketEntryStateReason::Dead(BucketEntryStateDeadReason::ExcessiveSendFailures), - "{so:?}" - ); - } + // [D2] every ordering's only avenue failing is dead + let v = viability(&[ + (SequenceOrdering::Unordered, &[udp6()], false), + (SequenceOrdering::Ordered, &[tcp4()], false), + ]); + let per_transport = transport_map(&[(udp6(), failing_stats()), (tcp4(), failing_stats())]); + assert_eq!( + compute( + None, + &RPCStats::default(), + &BTreeMap::new(), + &per_transport, + Some(&v) + ), + BucketEntryStateReason::Dead(BucketEntryStateDeadReason::ExcessiveSendFailures) + ); + + // a single-ordering environment (WS-only wasm peer) dies on that ordering alone + let v = viability(&[(SequenceOrdering::Ordered, &[tcp4()], false)]); + let per_transport = transport_map(&[(tcp4(), failing_stats())]); + assert_eq!( + compute( + None, + &RPCStats::default(), + &BTreeMap::new(), + &per_transport, + Some(&v) + ), + BucketEntryStateReason::Dead(BucketEntryStateDeadReason::ExcessiveSendFailures) + ); +} + +pub fn test_state_viable_ordering_vetoes_death() { + info!("--- test_state_viable_ordering_vetoes_death ---"); + + // one healthy ordering keeps the node alive despite the other fully failing; + // the failing ordering makes it Missing, not Dead + let v = viability(&[ + (SequenceOrdering::Unordered, &[udp6()], false), + (SequenceOrdering::Ordered, &[tcp4()], false), + ]); + let per_transport = transport_map(&[(udp6(), failing_stats()), (tcp4(), RPCStats::default())]); + assert_eq!( + compute( + None, + &RPCStats::default(), + &BTreeMap::new(), + &per_transport, + Some(&v) + ), + BucketEntryStateReason::Missing(BucketEntryStateMissingReason::FailedToSend) + ); + + // an Untried transport in the same ordering vetoes that ordering's failure: + // alternates must accumulate evidence before the node can die + let v = viability(&[(SequenceOrdering::Unordered, &[udp6(), udp4()], false)]); + let per_transport = transport_map(&[(udp6(), failing_stats())]); + assert_eq!( + compute( + None, + &RPCStats::default(), + &BTreeMap::new(), + &per_transport, + Some(&v) + ), + BucketEntryStateReason::Initial + ); + + // an indirect contact method (relay/signal/existing flow) vetoes send-failure + // death for its ordering; relay collapse is covered by [D1] unreachable + let v = viability(&[(SequenceOrdering::Unordered, &[udp6()], true)]); + let per_transport = transport_map(&[(udp6(), failing_stats())]); + assert_eq!( + compute( + None, + &RPCStats::default(), + &BTreeMap::new(), + &per_transport, + Some(&v) + ), + BucketEntryStateReason::Initial + ); +} + +pub fn test_state_send_failures_never_kill() { + info!("--- test_state_send_failures_never_kill ---"); + + // Failures to send are our own fault: they steer selection but must never + // make a node Missing or Dead, even as its only transport + let v = viability(&[(SequenceOrdering::Unordered, &[udp6()], false)]); + let per_transport = transport_map(&[(udp6(), send_failing_stats())]); + assert_eq!( + compute( + None, + &RPCStats::default(), + &BTreeMap::new(), + &per_transport, + Some(&v) + ), + BucketEntryStateReason::Initial + ); + // Selection still judges that transport unusable + assert!(transport_health(&send_failing_stats()).is_failing()); } pub fn test_state_dead_never_seen_lost_questions() { info!("--- test_state_dead_never_seen_lost_questions ---"); - // [D3] never-seen (no per-transport stats) with enough lost questions is dead - for so in [SequenceOrdering::Unordered, SequenceOrdering::Ordered] { - let per_so = so_map(&[( - so, - RPCStats { - recent_lost_questions: dead_never_seen_lost_questions_count(so), - ..Default::default() - }, - )]); - // per_transport empty == never seen - assert_eq!( - compute(None, &RPCStats::default(), &per_so, &BTreeMap::new()), - BucketEntryStateReason::Dead(BucketEntryStateDeadReason::NeverSeenLostQuestions), - "{so:?}" - ); - } + // [D3] never-seen (no per-transport stats) with enough node-level lost questions is dead + let rpc = RPCStats { + recent_lost_questions: DEAD_NEVER_SEEN_LOST_QUESTIONS, + ..Default::default() + }; + assert_eq!( + compute(None, &rpc, &BTreeMap::new(), &BTreeMap::new(), None), + BucketEntryStateReason::Dead(BucketEntryStateDeadReason::NeverSeenLostQuestions) + ); } pub fn test_state_dead_steady_lost_questions() { info!("--- test_state_dead_steady_lost_questions ---"); - // [D4] seen before, steadily losing questions past the span, is dead - let per_so = so_map(&[( - SequenceOrdering::Unordered, - RPCStats { - first_steady_lost_question_ts: Some(secs_ago(span_secs(DEAD_LOST_QUESTION_SPAN) + 1)), - ..Default::default() - }, - )]); + // [D4] seen before, node-level steadily losing questions past the span, is dead + let rpc = RPCStats { + first_steady_lost_question_ts: Some(secs_ago(span_secs(DEAD_LOST_QUESTION_SPAN) + 1)), + ..Default::default() + }; assert_eq!( - compute(None, &RPCStats::default(), &per_so, &seen_transports()), + compute(None, &rpc, &BTreeMap::new(), &seen_transports(), None), BucketEntryStateReason::Dead(BucketEntryStateDeadReason::SteadyLostQuestions) ); // boundary: a steady lost question newer than the span is not yet dead - let per_so = so_map(&[( - SequenceOrdering::Unordered, - RPCStats { - first_steady_lost_question_ts: Some(secs_ago(span_secs(DEAD_LOST_QUESTION_SPAN) - 1)), - ..Default::default() - }, - )]); + let rpc = RPCStats { + first_steady_lost_question_ts: Some(secs_ago(span_secs(DEAD_LOST_QUESTION_SPAN) - 1)), + ..Default::default() + }; assert_ne!( BucketEntryState::from(compute( None, - &RPCStats::default(), - &per_so, - &seen_transports() + &rpc, + &BTreeMap::new(), + &seen_transports(), + None )), BucketEntryState::Dead, "should not be dead under the span" @@ -174,41 +293,81 @@ pub fn test_state_dead_steady_lost_questions() { pub fn test_state_missing_failed_to_send() { info!("--- test_state_missing_failed_to_send ---"); - // [M2] some send failures, but below the dead threshold, is missing - assert!(1 < dead_failed_to_send_count(SequenceOrdering::Unordered)); - let per_so = so_map(&[( - SequenceOrdering::Unordered, + // [M2] one fully-failing ordering with another still viable is missing + let v = viability(&[ + (SequenceOrdering::Unordered, &[udp6()], false), + (SequenceOrdering::Ordered, &[tcp4()], false), + ]); + let per_transport = transport_map(&[(udp6(), failing_stats()), (tcp4(), RPCStats::default())]); + assert_eq!( + compute( + None, + &RPCStats::default(), + &BTreeMap::new(), + &per_transport, + Some(&v) + ), + BucketEntryStateReason::Missing(BucketEntryStateMissingReason::FailedToSend) + ); + + // below the transport-failing threshold no ordering is failing + let per_transport = transport_map(&[( + udp6(), RPCStats { - failed_to_send: 1, + recent_lost_questions: TRANSPORT_FAILING_LOST_QUESTIONS - 1, ..Default::default() }, )]); assert_eq!( - compute(None, &RPCStats::default(), &per_so, &BTreeMap::new()), - BucketEntryStateReason::Missing(BucketEntryStateMissingReason::FailedToSend) + compute( + None, + &RPCStats::default(), + &BTreeMap::new(), + &per_transport, + Some(&v) + ), + BucketEntryStateReason::Initial ); } pub fn test_state_missing_lost_questions() { info!("--- test_state_missing_lost_questions ---"); - // [M3] seen before, lost questions at/above the missing threshold but no - // steady span yet, is missing. per_transport is non-empty so [D3] does not apply. - for so in [SequenceOrdering::Unordered, SequenceOrdering::Ordered] { - let per_so = so_map(&[( - so, - RPCStats { - recent_lost_questions: missing_lost_questions_count(so), - first_steady_lost_question_ts: None, - ..Default::default() - }, - )]); - assert_eq!( - compute(None, &RPCStats::default(), &per_so, &seen_transports()), - BucketEntryStateReason::Missing(BucketEntryStateMissingReason::LostQuestions), - "{so:?}" - ); - } + // [M3] seen before, node-level lost questions at the missing threshold but no + // steady span yet, is missing; per_transport is non-empty so [D3] does not apply + let rpc = RPCStats { + recent_lost_questions: MISSING_LOST_QUESTIONS, + first_steady_lost_question_ts: None, + ..Default::default() + }; + assert_eq!( + compute(None, &rpc, &BTreeMap::new(), &seen_transports(), None), + BucketEntryStateReason::Missing(BucketEntryStateMissingReason::LostQuestions) + ); +} + +pub fn test_state_revive_reaches_initial() { + info!("--- test_state_revive_reaches_initial ---"); + + // revive() clears node-level and per-ordering stats but keeps transport + // evidence; that field state must compute Initial or revival is a no-op + let rpc = RPCStats { + recent_lost_questions: MISSING_LOST_QUESTIONS, + first_steady_lost_question_ts: None, + ..Default::default() + }; + assert_eq!( + compute(None, &rpc, &BTreeMap::new(), &seen_transports(), None), + BucketEntryStateReason::Missing(BucketEntryStateMissingReason::LostQuestions) + ); + let revived = RPCStats { + recent_lost_questions: 0, + ..rpc + }; + assert_eq!( + compute(None, &revived, &BTreeMap::new(), &seen_transports(), None), + BucketEntryStateReason::Initial + ); } pub fn test_state_initial() { @@ -220,7 +379,8 @@ pub fn test_state_initial() { None, &RPCStats::default(), &BTreeMap::new(), - &BTreeMap::new() + &BTreeMap::new(), + None ), BucketEntryStateReason::Initial ); @@ -228,7 +388,13 @@ pub fn test_state_initial() { // a sequence ordering tracked but never answered is still initial let per_so = so_map(&[(SequenceOrdering::Unordered, RPCStats::default())]); assert_eq!( - compute(None, &RPCStats::default(), &per_so, &seen_transports()), + compute( + None, + &RPCStats::default(), + &per_so, + &seen_transports(), + None + ), BucketEntryStateReason::Initial ); } @@ -245,7 +411,13 @@ pub fn test_state_unreliable() { }, )]); assert_eq!( - compute(None, &RPCStats::default(), &per_so, &seen_transports()), + compute( + None, + &RPCStats::default(), + &per_so, + &seen_transports(), + None + ), BucketEntryStateReason::Unreliable ); } @@ -262,7 +434,13 @@ pub fn test_state_reliable() { }, )]); assert_eq!( - compute(None, &RPCStats::default(), &per_so, &seen_transports()), + compute( + None, + &RPCStats::default(), + &per_so, + &seen_transports(), + None + ), BucketEntryStateReason::Reliable ); } @@ -270,25 +448,24 @@ pub fn test_state_reliable() { pub fn test_state_precedence_dead_beats_reliable() { info!("--- test_state_precedence_dead_beats_reliable ---"); - // dead on one ordering kills the node even if another ordering is reliable - let per_so = so_map(&[ - ( - SequenceOrdering::Unordered, - RPCStats { - first_steady_answer_ts: Some(secs_ago(120)), - ..Default::default() - }, - ), - ( - SequenceOrdering::Ordered, - RPCStats { - failed_to_send: dead_failed_to_send_count(SequenceOrdering::Ordered), - ..Default::default() - }, - ), - ]); + // all orderings failing kills the node even if its answer history is reliable + let per_so = so_map(&[( + SequenceOrdering::Unordered, + RPCStats { + first_steady_answer_ts: Some(secs_ago(120)), + ..Default::default() + }, + )]); + let v = viability(&[(SequenceOrdering::Unordered, &[udp6()], false)]); + let per_transport = transport_map(&[(udp6(), failing_stats())]); assert_eq!( - compute(None, &RPCStats::default(), &per_so, &seen_transports()), + compute( + None, + &RPCStats::default(), + &per_so, + &per_transport, + Some(&v) + ), BucketEntryStateReason::Dead(BucketEntryStateDeadReason::ExcessiveSendFailures) ); } @@ -301,12 +478,15 @@ pub fn test_state_transition_reliable_degrades_to_missing() { SequenceOrdering::Unordered, RPCStats { first_steady_answer_ts: Some(secs_ago(120)), - recent_lost_questions: missing_lost_questions_count(SequenceOrdering::Unordered), ..Default::default() }, )]); + let rpc = RPCStats { + recent_lost_questions: MISSING_LOST_QUESTIONS, + ..Default::default() + }; assert_eq!( - compute(None, &RPCStats::default(), &per_so, &seen_transports()), + compute(None, &rpc, &per_so, &seen_transports(), None), BucketEntryStateReason::Missing(BucketEntryStateMissingReason::LostQuestions) ); } @@ -315,38 +495,33 @@ pub fn test_state_offline_credit_steady_lost_questions() { info!("--- test_state_offline_credit_steady_lost_questions ---"); // [D4] a node losing questions past the dead span is Dead by raw time. - let per_so = so_map(&[( - SequenceOrdering::Ordered, - RPCStats { - first_steady_lost_question_ts: Some(secs_ago(span_secs(DEAD_LOST_QUESTION_SPAN) + 10)), - recent_lost_questions: missing_lost_questions_count(SequenceOrdering::Ordered), - ..Default::default() - }, - )]); + let rpc = RPCStats { + first_steady_lost_question_ts: Some(secs_ago(span_secs(DEAD_LOST_QUESTION_SPAN) + 10)), + recent_lost_questions: MISSING_LOST_QUESTIONS, + ..Default::default() + }; assert_eq!( - compute(None, &RPCStats::default(), &per_so, &seen_transports()), + compute(None, &rpc, &BTreeMap::new(), &seen_transports(), None), BucketEntryStateReason::Dead(BucketEntryStateDeadReason::SteadyLostQuestions) ); // Crediting offline time pulls the effective span under the threshold: not Dead. assert_eq!( - compute_with_offline(None, &RPCStats::default(), &per_so, &seen_transports(), 20), + compute_with_offline(None, &rpc, &BTreeMap::new(), &seen_transports(), None, 20), BucketEntryStateReason::Missing(BucketEntryStateMissingReason::LostQuestions) ); // A genuinely long lost span stays Dead despite the same offline credit. - let long_lost = so_map(&[( - SequenceOrdering::Ordered, - RPCStats { - first_steady_lost_question_ts: Some(secs_ago(span_secs(DEAD_LOST_QUESTION_SPAN) + 200)), - recent_lost_questions: missing_lost_questions_count(SequenceOrdering::Ordered), - ..Default::default() - }, - )]); + let long_lost = RPCStats { + first_steady_lost_question_ts: Some(secs_ago(span_secs(DEAD_LOST_QUESTION_SPAN) + 200)), + recent_lost_questions: MISSING_LOST_QUESTIONS, + ..Default::default() + }; assert_eq!( compute_with_offline( None, - &RPCStats::default(), &long_lost, + &BTreeMap::new(), &seen_transports(), + None, 20 ), BucketEntryStateReason::Dead(BucketEntryStateDeadReason::SteadyLostQuestions) @@ -365,13 +540,26 @@ pub fn test_state_offline_credit_unreliable_answer() { }, )]); assert_eq!( - compute(None, &RPCStats::default(), &per_so, &seen_transports()), + compute( + None, + &RPCStats::default(), + &per_so, + &seen_transports(), + None + ), BucketEntryStateReason::Reliable ); // Crediting offline time pulls the effective answer span under the threshold: // still Unreliable (we don't grant reliability for time we couldn't test). assert_eq!( - compute_with_offline(None, &RPCStats::default(), &per_so, &seen_transports(), 20), + compute_with_offline( + None, + &RPCStats::default(), + &per_so, + &seen_transports(), + None, + 20 + ), BucketEntryStateReason::Unreliable ); } diff --git a/veilid-core/src/rpc_processor/fanout/fanout_call.rs b/veilid-core/src/rpc_processor/fanout/fanout_call.rs index 95b1801a..90fb549f 100644 --- a/veilid-core/src/rpc_processor/fanout/fanout_call.rs +++ b/veilid-core/src/rpc_processor/fanout/fanout_call.rs @@ -173,7 +173,7 @@ pub type FanoutCheckDone = Arc FanoutDoneDisposition) /// Per-node call routine invoked once per fanout target pub type FanoutCallRoutine = Arc PinBoxFutureStatic) + Send + Sync>; -/// Post-consensus timeout, computed from the result when consensus is first reached +/// Post-consensus rolling window, recomputed from the result on every further accept pub type FanoutPostConsensusTimeoutCallback = Arc TimestampDuration) + Send + Sync>; @@ -217,6 +217,10 @@ pub(crate) struct FanoutCall<'a> { call_routine: FanoutCallRoutine, check_done: FanoutCheckDone, post_consensus_timeout_callback: Option, + /// Accepts this run needs, when it differs from `consensus_count` (see `set_run_target`) + opt_run_target_count: Option, + /// Seed from never-contacted nodes too (see `set_seed_untried`) + seed_untried: bool, } impl VeilidComponentRegistryAccessor for FanoutCall<'_> { @@ -264,11 +268,14 @@ impl<'a> FanoutCall<'a> { call_routine, check_done, post_consensus_timeout_callback: None, + opt_run_target_count: None, + seed_untried: false, } } - /// Once consensus is first reached, the callback computes a tighter deadline - /// from the (consensus-state) result; the fanout exits when that elapses. + /// After consensus the callback sizes a window that re-arms on every further + /// accept. The fanout exits when one elapses with nothing new arriving, or when + /// the pre-consensus deadline is reached, whichever comes first. pub fn with_post_consensus_timeout_callback( mut self, callback: FanoutPostConsensusTimeoutCallback, @@ -277,6 +284,41 @@ impl<'a> FanoutCall<'a> { self } + /// Also seed from nodes we have never contacted. + /// + /// Off by default: `Initial` entries are unproven and a cold table is full of + /// them, so a first pass spends its timeout on nodes that may not answer. A + /// caller that has exhausted every node it HAS reached, without consensus, has + /// nothing cheaper left to try. Unproven is not failed, so this still excludes + /// Missing, Dead and Punished. + pub fn set_seed_untried(&mut self, seed_untried: bool) { + self.seed_untried = seed_untried; + } + + /// Narrow which peers this call will consider, for a second run over the same + /// call routine. A node already holding state from the first run must be + /// filtered out here rather than skipped inside the routine, or it is asked + /// again and answers again. + pub fn set_peer_info_filter(&mut self, peer_info_filter: FanoutPeerInfoFilter) { + self.peer_info_filter = peer_info_filter; + } + + /// Retarget a second run at the accepts still missing rather than the original + /// count, moving the termination check with it. + /// + /// Only the result kind moves. `consensus_count` still drives how many nodes the + /// queue works at once and how many consecutive rejections end the fanout, and + /// neither got easier because we already hold some accepts. + pub fn set_run_target(&mut self, target_count: usize, check_done: FanoutCheckDone) { + self.opt_run_target_count = Some(target_count); + self.check_done = check_done; + } + + /// Accepts this run must collect to call it consensus + fn run_target_count(&self) -> usize { + self.opt_run_target_count.unwrap_or(self.consensus_count) + } + #[cfg_attr( feature = "instrument", instrument(level = "trace", target = "fanout", skip_all, fields(__VEILID_LOG_KEY = self.log_key())) @@ -289,19 +331,33 @@ impl<'a> FanoutCall<'a> { return ctx.done; } - // Calculate fanout result so far. Consensus = `consensus_count` accepts - // anywhere within the `consensus_width` closest nodes, regardless of - // closeness ordering inside the window — a single slow closer node - // doesn't block faster peers within the window from reaching consensus. - // The post-consensus deadline still applies to in-progress stragglers. + // Consensus = `consensus_count` accepts anywhere within the `consensus_width` + // closest viable nodes: + // - closeness ordering inside the window doesn't matter, so one slow closer + // node can't block faster peers from reaching consensus + // - Timeout and Disqualified take no window slot, so an unreachable closest + // node widens the window to the next candidate instead of capping consensus + // below its count + // - a rejection holds its slot, it is a live close node's explicit answer + // - `disqualify` windows the same way let fanout_result = ctx.fanout_queue.with_nodes(|nodes, sorted_nodes| { let mut consensus_nodes: Vec = vec![]; let mut value_nodes: Vec = vec![]; let mut pending_in_window = false; - for (idx, sn) in sorted_nodes.iter().enumerate() { + let mut window_slots = 0usize; + for sn in sorted_nodes.iter() { let node = nodes.get(sn).unwrap_or_log(); - let in_window = idx < self.consensus_width; - match node.status.stage() { + let stage = node.status.stage(); + let in_window = if matches!( + stage, + FanoutNodeStage::Timeout | FanoutNodeStage::Disqualified + ) { + false + } else { + window_slots += 1; + window_slots <= self.consensus_width + }; + match stage { FanoutNodeStage::Queued | FanoutNodeStage::InProgress => { if in_window { pending_in_window = true; @@ -322,7 +378,7 @@ impl<'a> FanoutCall<'a> { } } - let kind = if consensus_nodes.len() >= self.consensus_count { + let kind = if consensus_nodes.len() >= self.run_target_count() { FanoutResultKind::Consensus } else if pending_in_window { FanoutResultKind::Incomplete @@ -496,6 +552,13 @@ impl<'a> FanoutCall<'a> { // Get the 'node_count' closest nodes to the key out of our routing table let closest_nodes = { let peer_info_filter = self.peer_info_filter.clone(); + // Never-contacted nodes are often dead in a cold table, so only a re-plan + // seeds from them + let seed_floor = if self.seed_untried { + BucketEntryState::Initial + } else { + BucketEntryState::Unreliable + }; let filter = Box::new( move |opt_snap: &Option, _cur_ts: Timestamp| { // Exclude our own node @@ -503,8 +566,7 @@ impl<'a> FanoutCall<'a> { return false; }; - // Seed only from nodes we've actually reached; 'initial' (never-contacted) nodes are often dead in a cold table. - if snap.state < BucketEntryState::Unreliable { + if snap.state < seed_floor { return false; } @@ -639,22 +701,35 @@ impl<'a> FanoutCall<'a> { unord.push(make_tick_future()); } - // Effective deadline. Before consensus it tracks a full timeout past the last - // accepting disposition, so the fanout keeps converging on closer nodes while it's - // still making progress (accepted nodes are kept alive server-side by a background - // keepalive). After consensus it only shrinks (post_consensus_timeout_callback). + // Effective deadline: + // - before consensus, a full timeout past the last accepting disposition, so the + // fanout keeps converging on closer nodes while it makes progress (accepted + // nodes are kept alive server-side by a background keepalive) + // - after consensus, a rolling window (post_consensus_timeout_callback) re-armed + // per accept and capped by what the pre-consensus deadline had left + // - with no callback, the pre-consensus deadline stands let mut effective_deadline = Timestamp::now_non_decreasing().later(self.timeout); + let mut post_consensus_cap: Option = None; let inner_result: FanoutLoopResult = loop { let now = Timestamp::now_non_decreasing(); - // Pre-consensus: keep the deadline a full timeout past the last accept. - { - let ctx = context.lock(); - if ctx.consensus_reached_ts.is_none() { - let progress_deadline = ctx.last_accepted_ts.later(self.timeout); + let opt_consensus_ts = context.lock().consensus_reached_ts; + match opt_consensus_ts { + None => { + let progress_deadline = context.lock().last_accepted_ts.later(self.timeout); if progress_deadline > effective_deadline { effective_deadline = progress_deadline; } } + Some(consensus_ts) => { + if let Some(callback) = &self.post_consensus_timeout_callback { + let (anchor, result_snapshot) = { + let ctx = context.lock(); + (ctx.last_accepted_ts.max(consensus_ts), ctx.result.clone()) + }; + let cap = *post_consensus_cap.get_or_insert(effective_deadline); + effective_deadline = cap.min(anchor.later((callback)(&result_snapshot))); + } + } } if now >= effective_deadline { break FanoutLoopResult::DeadlineReached; @@ -686,20 +761,8 @@ impl<'a> FanoutCall<'a> { Some(Err(e)) => break FanoutLoopResult::LaneFailed(e), // Stop all lanes immediately Some(Ok(FanoutProcessorReturn::DoneEarly)) => break FanoutLoopResult::Done, - // Lane finished; arm post-consensus deadline if consensus was just reached - Some(Ok(FanoutProcessorReturn::Done)) => { - if let Some(callback) = &self.post_consensus_timeout_callback { - let context_locked = context.lock(); - if let Some(ts) = context_locked.consensus_reached_ts { - let result_snapshot = context_locked.result.clone(); - drop(context_locked); - let candidate = ts.later((callback)(&result_snapshot)); - if candidate < effective_deadline { - effective_deadline = candidate; - } - } - } - } + // Lane finished; the deadline is recomputed at the top of the loop + Some(Ok(FanoutProcessorReturn::Done)) => {} // Timer tick: push more work and re-arm the tick Some(Ok(FanoutProcessorReturn::Tick)) => { let context_locked = &mut *context.lock(); diff --git a/veilid-core/src/rpc_processor/fanout/fanout_queue.rs b/veilid-core/src/rpc_processor/fanout/fanout_queue.rs index 6f147c28..1e095c5b 100644 --- a/veilid-core/src/rpc_processor/fanout/fanout_queue.rs +++ b/veilid-core/src/rpc_processor/fanout/fanout_queue.rs @@ -519,10 +519,14 @@ impl<'a> FanoutQueue<'a> { } /// Transition InProgress | Queued -> Disqualified if they are too far away from the record key + /// + /// Width counts only nodes that can still answer, mirroring `evaluate_done`: an + /// unreachable closest node must not cull the candidates that could replace it. fn disqualify(&mut self, cur_ts: Timestamp) { let mut consecutive_rejections = 0usize; let mut rejected_consensus = false; - for (index, node_id) in self.sorted_nodes.iter().enumerate() { + let mut window_slots = 0usize; + for node_id in self.sorted_nodes.iter() { let node = self.nodes.get_mut(node_id).unwrap_or_log(); // If the node is finished already we don't have to consider it @@ -533,23 +537,35 @@ impl<'a> FanoutQueue<'a> { if consecutive_rejections >= self.consensus_count { rejected_consensus = true; } + // A rejection holds its slot, it is a live close node's explicit answer + window_slots += 1; continue; } else { consecutive_rejections = 0; } + if !matches!( + node.status.stage, + FanoutNodeStage::Timeout | FanoutNodeStage::Disqualified + ) { + window_slots += 1; + } continue; } // If the node is too far away from the key or we have had enough consecutive rejections, // cancel it immediately and disqualify it - if (self.consensus_width > 0 && index >= self.consensus_width) || rejected_consensus { + if (self.consensus_width > 0 && window_slots >= self.consensus_width) + || rejected_consensus + { node.status .transition(FanoutNodeStage::Disqualified, cur_ts); if let Some(css) = node.lane_stop_source.take() { // Drop the stop source to force the lane to stop processing this node drop(css); } + } else { + window_slots += 1; } } } diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 8e8071ab..c1697c46 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -64,6 +64,12 @@ pub(crate) use destination::*; pub(crate) use error::*; pub(crate) use fanout::*; pub(crate) use rpc_status::StatusResult; +// Only the native discovery context consumes this +#[cfg_attr( + all(target_arch = "wasm32", target_os = "unknown"), + expect(unused_imports) +)] +pub(crate) use rpc_validate_dial_info::DialInfoValidationOutcome; pub(crate) use sender_info::*; pub(crate) use waitable_reply::WaitableReplyContext; @@ -150,6 +156,8 @@ pub(crate) struct RPCProcessor { inner: Mutex, /// The timeout for a direct RPC sent to a node with no extra hops timeout: TimestampDuration, + /// Queue age past which an inbound message is shed: the worst-case routed waiter timeout + worker_shed_timeout: TimestampDuration, /// The queue length for the RPC send/recv channels queue_size: u32, /// The number of RPC workers to spin up @@ -185,7 +193,7 @@ impl RPCProcessor { startup_context: RPCProcessorStartupContext, ) -> Self { // make local copy of node id for easy access - let (concurrency, queue_size, timeout_us) = { + let (concurrency, queue_size, timeout_us, max_route_hop_count) = { let config = registry.config(); // set up channel @@ -193,6 +201,7 @@ impl RPCProcessor { let queue_size = config.internal().network.rpc.queue_size; let timeout_us = TimestampDuration::new(ms_to_us(config.internal().network.rpc.timeout_ms)); + let max_route_hop_count = config.internal().network.rpc.max_route_hop_count; if concurrency == 0 { concurrency = get_concurrency(); if concurrency == 0 { @@ -202,9 +211,15 @@ impl RPCProcessor { // Default RPC concurrency is the number of CPUs * 16 rpc workers per core, as a single worker takes about 1% CPU when relaying and 16% is reasonable for baseline plus relay concurrency *= RPC_WORKERS_PER_CORE; } - (concurrency, queue_size, timeout_us) + (concurrency, queue_size, timeout_us, max_route_hop_count) }; + // Worst-case waiter: get_safety_selection_timeout at the maximum hop count + let worker_shed_timeout = timeout_us.saturating_add( + TimestampDuration::new_ms(500) + .saturating_mul(((max_route_hop_count as u64) * 2).saturating_sub(4)), + ); + Self { waiting_rpc_table: OperationWaiter::new(registry.clone()), waiting_app_call_table: OperationWaiter::new(registry.clone()), @@ -212,6 +227,7 @@ impl RPCProcessor { registry, inner: Mutex::new(Self::new_inner()), timeout: timeout_us, + worker_shed_timeout, queue_size, concurrency, startup_context, @@ -605,7 +621,7 @@ impl RPCProcessor { } // Spawn the actual wait into the reply pool so a dropped caller - // doesn't cancel the answer wait — the question's outbound RPC + // doesn't cancel the answer wait; the question's outbound RPC // always completes, even if no one is left listening. let waiter = self.waiting_rpc_table.clone(); let timeout = waitable_reply.context.timeout; @@ -1178,17 +1194,41 @@ impl RPCProcessor { node_ids } + /// Whether our own spread verdict condemns this transport, voiding any blame + /// evidence carried over it + fn transport_send_suppressed( + &self, + opt_rd: Option, + transport: TransportType, + ) -> bool { + opt_rd.is_some_and(|rd| { + self.network_manager() + .transport_stats() + .verdict(rd, transport) + .is_send_suppressed() + }) + } + /// Record failure to send to node or route + /// + /// Node blame is suppressed while our own network is degraded or offline, or our + /// spread verdict condemns the transport: those failures say nothing about the peer. + /// Route stats are exempt from the degraded case, a route is our own resource + /// selection and releasing a bad one is recovery, not peer blame. Suppressing it + /// would wedge a broken route in place for as long as the losses it causes keep the + /// node degraded. A condemned transport still voids them. #[cfg_attr( feature = "instrument", instrument(level = "trace", target = "rpc", skip_all, fields(__VEILID_LOG_KEY = self.log_key())) )] + #[expect(clippy::too_many_arguments)] fn record_send_failure( &self, rpc_kind: RPCKind, send_ts: Timestamp, node_ref: FilteredNodeRef, opt_transport: Option, + dialed: bool, safety_route: Option, remote_private_route: Option, ) { @@ -1223,16 +1263,18 @@ impl RPCProcessor { }; online_detector.record_failure(rd, node_ids, send_ts, transport); } - let suppress_degradation = opt_rd - .map(|rd| online_detector.online_state(rd) == OnlineState::Offline) - .unwrap_or(false); + let suppress_degradation = opt_rd.is_some_and(|rd| online_detector.blame_suppressed(rd)); + let transport_suppressed = + |transport: TransportType| self.transport_send_suppressed(opt_rd, transport); // Record for node if this was not sent via a route if !is_routed { if !suppress_degradation { if let Some(transport) = opt_transport { - // A specific transport failed to send, record it on the transport - node_ref.stats_failed_to_send(send_ts, wants_answer, transport); + if !transport_suppressed(transport) { + // A specific transport failed to send, record it on the transport + node_ref.stats_failed_to_send(send_ts, wants_answer, transport, dialed); + } } else { // Pre-send abort with no transport attempted: // * NoRoutingDomain / NoPeerInfo / NoContactMethod / Punished @@ -1246,14 +1288,14 @@ impl RPCProcessor { return; } - if suppress_degradation { - return; - } - // Only attribute a route send failure when we know which transport (ordering) was attempted. - let Some(ordering) = opt_transport.map(|t| t.sequence_ordering()) else { + let Some(transport) = opt_transport else { return; }; + if transport_suppressed(transport) { + return; + } + let ordering = transport.sequence_ordering(); if let Some(sr) = &safety_route { // If safety route was in use, record failure to send there sr.with_stats_mut(|s| s.record_send_failed(ordering)); @@ -1264,6 +1306,10 @@ impl RPCProcessor { } /// Record question lost to node or route + /// + /// Suppression follows `record_send_failure`: route stats are exempt from the + /// degraded case but not from a condemned transport, and the out-of-band route test + /// runs regardless. #[cfg_attr( feature = "instrument", instrument(level = "trace", target = "rpc", skip_all, fields(__VEILID_LOG_KEY = self.log_key())) @@ -1294,9 +1340,9 @@ impl RPCProcessor { online_detector.record_failure(rd, node_ids, context.send_ts, transport); } } - let suppress_degradation = opt_rd - .map(|rd| online_detector.online_state(rd) == OnlineState::Offline) - .unwrap_or(false); + let suppress_degradation = opt_rd.is_some_and(|rd| online_detector.blame_suppressed(rd)); + let transport_suppressed = + |transport: TransportType| self.transport_send_suppressed(opt_rd, transport); // Record for node if this was not sent via a route if !context.is_routed() { @@ -1319,7 +1365,12 @@ impl RPCProcessor { .unwrap_or(false); if !suppress_degradation && !protect_after_reattach { if let Some(transport) = context.send_data_result.transport_type() { - context.node_ref.stats_lost_question(transport); + if !transport_suppressed(transport) { + context.node_ref.stats_lost_question( + transport, + context.send_data_result.dialed_evidence(), + ); + } } } // Always clear last_flows: the underlying flow is gone or broken regardless. @@ -1327,12 +1378,12 @@ impl RPCProcessor { return; } - if suppress_degradation { - return; - } - // Ordering the send went out on; attribute the lost question to that ordering on the route. - let opt_ordering = context.send_data_result.sequence_ordering(); + let opt_ordering = context + .send_data_result + .transport_type() + .filter(|transport| !transport_suppressed(*transport)) + .map(|transport| transport.sequence_ordering()); // If safety route was used, record question lost there if let Some(sr) = &context.opt_safety_route_ref { @@ -1344,11 +1395,14 @@ impl RPCProcessor { // get released without waiting for the next regular cycle. The safety // route ref is always allocated, so use its set id directly. let route_id: RouteId = sr.route_set_id().clone().into(); - veilid_log!(self debug "scheduling out-of-band route test for {} after %SR timeout", route_id); let cur_ts = Timestamp::now(); - self.routing_table() - .enqueue_route_tests(cur_ts, vec![route_id], 0) + let enqueued = self + .routing_table() + .enqueue_route_tests(cur_ts, vec![route_id.clone()], 0) .await; + if enqueued > 0 { + veilid_log!(self debug "scheduling out-of-band route test for {} after %SR timeout", route_id); + } } // If remote private route was used, record question lost there if let Some(pr) = &context.opt_remote_private_route_ref { @@ -1384,6 +1438,7 @@ impl RPCProcessor { safety_route: Option, remote_private_route: Option, transport: TransportType, + dialed: bool, ) { let is_routed = safety_route.is_some() || remote_private_route.is_some(); @@ -1393,9 +1448,9 @@ impl RPCProcessor { let is_answer = matches!(rpc_kind, RPCKind::Answer); if is_answer { - node_ref.stats_answer_sent(bytes, transport); + node_ref.stats_answer_sent(bytes, transport, dialed); } else { - node_ref.stats_question_sent(send_ts, bytes, wants_answer, transport); + node_ref.stats_question_sent(send_ts, bytes, wants_answer, transport, dialed); } return; } @@ -1452,9 +1507,13 @@ impl RPCProcessor { && context.opt_reply_private_route_ref.is_none() { if let Some(transport) = context.send_data_result.transport_type() { - context - .node_ref - .stats_answer_rcvd(context.send_ts, recv_ts, bytes, transport); + context.node_ref.stats_answer_rcvd( + context.send_ts, + recv_ts, + bytes, + transport, + context.send_data_result.dialed_evidence(), + ); } return; } @@ -1812,6 +1871,7 @@ impl RPCProcessor { send_ts, node_ref.clone(), None, + false, opt_safety_route_ref.clone(), opt_remote_private_route_ref.clone(), ); @@ -1829,6 +1889,7 @@ impl RPCProcessor { send_ts, node_ref.clone(), send_data_result.opt_transport_type(), + send_data_result.dialed_evidence(), opt_safety_route_ref, opt_remote_private_route_ref, ); @@ -1846,6 +1907,7 @@ impl RPCProcessor { opt_safety_route_ref.clone(), opt_remote_private_route_ref.clone(), transport, + send_data_result.dialed_evidence(), ); } @@ -1963,6 +2025,7 @@ impl RPCProcessor { send_ts, node_ref.clone(), None, + false, safety_route.clone(), remote_private_route.clone(), ); @@ -1980,6 +2043,7 @@ impl RPCProcessor { send_ts, node_ref.clone(), send_data_result.opt_transport_type(), + send_data_result.dialed_evidence(), safety_route.clone(), remote_private_route.clone(), ); @@ -1997,6 +2061,7 @@ impl RPCProcessor { safety_route, remote_private_route, transport, + send_data_result.dialed_evidence(), ); } @@ -2092,6 +2157,7 @@ impl RPCProcessor { send_ts, node_ref.clone(), None, + false, safety_route.clone(), remote_private_route.clone(), ); @@ -2109,6 +2175,7 @@ impl RPCProcessor { send_ts, node_ref.clone(), send_data_result.opt_transport_type(), + send_data_result.dialed_evidence(), safety_route.clone(), remote_private_route.clone(), ); @@ -2126,6 +2193,7 @@ impl RPCProcessor { safety_route, remote_private_route, transport, + send_data_result.dialed_evidence(), ); } @@ -2397,6 +2465,34 @@ impl RPCProcessor { } }; + // Shed instead of process when no waiter can still be listening: + // - direct question: waiter armed self.timeout + // - routed question: waiter armed up to the max-hop routed timeout + // - Route statement: mid-route relay, worst case less the hop that reached us + // - other statements and answers: nobody waits on us, deliver whenever we can + let opt_shed_timeout = match msg.operation.kind() { + RPCOperationKind::Question(_) => Some(match &msg.header.detail { + RPCMessageHeaderDetail::Direct(_) => self.timeout, + RPCMessageHeaderDetail::SafetyRouted(_) + | RPCMessageHeaderDetail::PrivateRouted(_) => self.worker_shed_timeout, + }), + RPCOperationKind::Statement(s) => match s.detail() { + RPCStatementDetail::Route(_) => Some( + self.worker_shed_timeout + .saturating_sub(TimestampDuration::new_ms(500)), + ), + _ => None, + }, + RPCOperationKind::Answer(_) => None, + }; + if let Some(shed_timeout) = opt_shed_timeout { + let queue_age = start_ts.duration_since(msg.header.timestamp); + if queue_age > shed_timeout { + veilid_log!(self debug "shedding stale rpc message: {} age {} > {}", desc, queue_age, shed_timeout); + return Err(RPCError::ignore("shed stale rpc message")); + } + } + // Process specific message kind let out = match msg.operation.kind() { RPCOperationKind::Question(q) => match q.detail() { diff --git a/veilid-core/src/rpc_processor/rpc_status.rs b/veilid-core/src/rpc_processor/rpc_status.rs index 3b2021ca..6fde0350 100644 --- a/veilid-core/src/rpc_processor/rpc_status.rs +++ b/veilid-core/src/rpc_processor/rpc_status.rs @@ -8,21 +8,26 @@ pub struct StatusSenderInfo { pub opt_previous_sender_info: Option, } -/// Outcome of `rpc_call_status`. Callers inspect `send_data_result` (when present) -/// for the transport that was actually attempted at the first hop. +/// Answer to a status RPC plus the transport attempt that carried it at the first hop #[derive(Debug)] -pub enum StatusResult { - /// Question was sent and an answer was received (boxed: much larger than the other variants). - Answer { - answer: Box>, - send_data_result: SendDataResult, - }, - /// Send was attempted but produced no usable answer (send failure, timeout, bad answer). - Failed(SendDataResult), - /// Never reached send_envelope (render failure, no routing domain, etc.). - NotSent(NetworkResult<()>), +pub struct StatusAnswer { + /// The peer's status answer + #[cfg_attr( + all( + target_arch = "wasm32", + target_os = "unknown", + not(feature = "debug-api") + ), + expect(dead_code) + )] + pub answer: Answer, + /// The first-hop send attempt that carried the question + pub send_data_result: SendDataResult, } +/// Outcome of `rpc_call_status` +pub type StatusResult = RpcOutcome; + impl RPCProcessor { // Send StatusQ RPC request, receive StatusA answer // Can be sent via relays or routes, but will have less information via routes @@ -140,7 +145,30 @@ impl RPCProcessor { } => { if matches!(safety_selection, SafetySelection::Unsafe(_)) { if let Some(sender_info) = sender_info { - if send_direct_dial_info.is_some() { + // A Direct send can ride an existing flow to a relay, whose address + // the peer then honestly reports as ours + let sent_to_contact_method_target = + send_direct_dial_info.as_ref().is_some_and(|di| { + di.socket_address() == *send_unique_flow.flow.remote_address() + }); + #[cfg(feature = "verbose-tracing")] + if send_direct_dial_info.is_some() && !sent_to_contact_method_target { + veilid_log!(self debug + "ignoring sender info from {}: Direct to {} but sent via {}", + target, + send_direct_dial_info.as_ref().map(|di| di.to_string()).unwrap_or_default(), + send_unique_flow.flow.remote_address() + ); + } + if sent_to_contact_method_target { + // Log accepts too, rejections alone can't be told apart from starvation + #[cfg(feature = "verbose-tracing")] + veilid_log!(self debug + "accepting sender info from {}: direct to {}, reports {}", + target, + send_unique_flow.flow.remote_address(), + sender_info.socket_address + ); // Directly requested status that actually gets sent directly and not over a relay will tell us what our IP address appears as // If this changes, we'd want to know about that to reset the networking stack opt_previous_sender_info = target.report_sender_info( @@ -175,16 +203,16 @@ impl RPCProcessor { // sender info is irrelevant over relays and routes } }; - Ok(StatusResult::Answer { - answer: Box::new(Answer::new( + Ok(StatusResult::Sent(Box::new(StatusAnswer { + answer: Answer::new( answer_context, StatusSenderInfo { opt_sender_info, opt_previous_sender_info, }, - )), + ), send_data_result, - }) + }))) } //////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs index 342b761b..44bf1f26 100644 --- a/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs +++ b/veilid-core/src/rpc_processor/rpc_validate_dial_info.rs @@ -2,6 +2,20 @@ use super::*; impl_veilid_log_facility!("rpc"); +/// Outcome of asking another node to validate one of our dial infos. +/// +/// Only a statement that actually went out says anything about whether peers can +/// reach us; a failure on our own side is no verdict at all. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum DialInfoValidationOutcome { + /// The receipt came back out-of-band: peers can reach this dial info + Reachable, + /// The request went out but no receipt arrived: peers cannot reach this dial info + Unreachable, + /// The request never got out; this says nothing about reachability + Inconclusive, +} + impl RPCProcessor { // Can only be sent directly, not via relays or routes #[cfg_attr(feature = "instrument", instrument(level = "trace", target = "rpc", skip(self), ret, err(level=Level::DEBUG)))] @@ -11,7 +25,7 @@ impl RPCProcessor { peer: NodeRef, dial_info: DialInfo, redirect: bool, - ) -> Result { + ) -> Result { let _guard = self .startup_context .startup_lock @@ -56,7 +70,7 @@ impl RPCProcessor { peer.custom_filtered(node_ref_filter).with_sequencing(Sequencing::PreferOrdered), None), statement, None, None) .await? => [ format!(": peer={} statement={:?}", peer, statement) ] { - return Ok(false); + return Ok(DialInfoValidationOutcome::Inconclusive); } ); @@ -73,16 +87,17 @@ impl RPCProcessor { ReceiptEvent::ReturnedPrivate { private_route: _ } | ReceiptEvent::ReturnedInBand { inbound_noderef: _ } | ReceiptEvent::ReturnedSafety => { + // Wrong channel, a protocol anomaly rather than evidence either way veilid_log!(self debug "validate_dial_info receipt should be returned out-of-band"); - Ok(false) + Ok(DialInfoValidationOutcome::Inconclusive) } ReceiptEvent::ReturnedOutOfBand => { veilid_log!(self debug "validate_dial_info receipt returned"); - Ok(true) + Ok(DialInfoValidationOutcome::Reachable) } ReceiptEvent::Expired => { veilid_log!(self debug "validate_dial_info receipt expired"); - Ok(false) + Ok(DialInfoValidationOutcome::Unreachable) } ReceiptEvent::Cancelled => { Err(RPCError::internal("receipt was dropped before expiration")) diff --git a/veilid-core/src/rpc_processor/waitable_reply.rs b/veilid-core/src/rpc_processor/waitable_reply.rs index c0104b10..441c0794 100644 --- a/veilid-core/src/rpc_processor/waitable_reply.rs +++ b/veilid-core/src/rpc_processor/waitable_reply.rs @@ -94,25 +94,28 @@ impl WaitableReply { } } -/// Outcome of `RPCProcessor::question` +/// Outcome of an RPC send #[derive(Debug)] #[must_use] -pub(super) enum QuestionOutcome { - /// Envelope sent; caller awaits the WaitableReply for the answer (boxed: much larger than the other variants) - Sent(Box), +pub(crate) enum RpcOutcome { + /// Sent; carries what the caller waits on or received (boxed: much larger than the other variants) + Sent(Box), /// Send was attempted but didn't produce a Value; SendDataResult exposes what was tried Failed(SendDataResult), /// Never made it to send_envelope (render failure, no routing domain, etc.) NotSent(NetworkResult<()>), } -impl QuestionOutcome { - /// Collapse into NetworkResult for callers that don't need the SendDataResult - pub(super) fn into_network_result(self) -> NetworkResult { +impl RpcOutcome { + /// Collapse for callers that don't need the SendDataResult + pub(crate) fn into_network_result(self) -> NetworkResult { match self { - QuestionOutcome::Sent(wr) => NetworkResult::Value(*wr), - QuestionOutcome::Failed(sdr) => sdr.into_network_result().map(|_| unreachable!()), - QuestionOutcome::NotSent(nr) => nr.map(|_| unreachable!()), + RpcOutcome::Sent(v) => NetworkResult::Value(*v), + RpcOutcome::Failed(sdr) => sdr.into_network_result().map(|_| unreachable!()), + RpcOutcome::NotSent(nr) => nr.map(|_| unreachable!()), } } } + +/// Outcome of `RPCProcessor::question` +pub(super) type QuestionOutcome = RpcOutcome; diff --git a/veilid-core/src/storage_manager/constants.rs b/veilid-core/src/storage_manager/constants.rs index c2b53536..7213678b 100644 --- a/veilid-core/src/storage_manager/constants.rs +++ b/veilid-core/src/storage_manager/constants.rs @@ -29,6 +29,13 @@ pub const CHECK_OUTBOUND_TRANSACTIONS_INTERVAL_SECS: u32 = 1; pub const TRANSACTION_TIMEOUT_RPC_MULTIPLIER: u64 = 2; /// Multiplier applied to a node's RTT to derive a post-consensus deadline (capped at `rpc.timeout_ms`) pub const POST_CONSENSUS_RTT_FACTOR: u64 = 3; +/// Hard cap on the post-consensus collection window, which is for the answer wave trailing +/// consensus; under congested RTTs the RTT factor alone degenerates toward the rpc timeout +/// and waits on dead stragglers +pub const POST_CONSENSUS_GRACE_MS: u64 = 500; +/// How long a Background transaction begin queues on the record lock before +/// yielding TryAgain to the caller's retry loop +pub const BACKGROUND_BEGIN_LOCK_TIMEOUT_MS: u64 = 2000; /// Headroom beyond `consensus_count` (= `set_value_count`) that Begin aims for, as a percentage rounded up pub const CONSENSUS_TRIM_PERCENT: usize = 50; /// Frequency to check for expired server-side transactions diff --git a/veilid-core/src/storage_manager/get_value.rs b/veilid-core/src/storage_manager/get_value.rs index 91bfdad8..3129f7fe 100644 --- a/veilid-core/src/storage_manager/get_value.rs +++ b/veilid-core/src/storage_manager/get_value.rs @@ -112,6 +112,9 @@ impl StorageManager { self.record_gated_fanout_outcome(GatedOperationKind::Get, &result.fanout_result); + // The read consumes any watch hint; a later change is news again + self.consume_watch_read_subkeys(&opaque_record_key, &ValueSubkeyRangeSet::single(subkey)); + // Process the returned result let out_encrypted = self .process_outbound_get_value_result_locked(&subkey_lock, last_seq, result) diff --git a/veilid-core/src/storage_manager/inspect_record.rs b/veilid-core/src/storage_manager/inspect_record.rs index 668b53e4..046ad96d 100644 --- a/veilid-core/src/storage_manager/inspect_record.rs +++ b/veilid-core/src/storage_manager/inspect_record.rs @@ -41,6 +41,10 @@ struct OutboundInspectValueContext { pub seqcounts: Vec, /// The descriptor if we got a fresh one or empty if no descriptor was needed pub opt_descriptor_info: Option, + /// Bumped whenever a subkey's seq or consensus set advances + pub convergence: usize, + /// The convergence counter as of the previous done check + pub checked_convergence: usize, } /// The result of the outbound_inspect_value operation @@ -209,9 +213,7 @@ impl StorageManager { // If we're simulating a set, increase the previous sequence number we have by 1 if matches!(scope, DHTReportScope::UpdateSet) { for seq in local_inspect_result.seqs_mut() { - if let Ok(next) = seq.next() { - *seq = next; - } + *seq = seq.saturating_next(); } } @@ -346,6 +348,8 @@ impl StorageManager { }) .collect(), opt_descriptor_info, + convergence: 0, + checked_convergence: 0, })); // Routine to call to generate fanout @@ -482,11 +486,17 @@ impl StorageManager { .iter() .map(|s| SubkeySeqCount { seq: *s, - // One node has shown us the newest sequence numbers so far - consensus_nodes: vec![next_node.clone()], + // Only a returned seq is evidence; a subkey this node + // had nothing for must not count toward its consensus + consensus_nodes: if s.is_some() { + vec![next_node.clone()] + } else { + vec![] + }, value_nodes: vec![next_node.clone()], }) .collect(); + ctx.convergence += 1; } else { if ctx.seqcounts.len() != answer.seqs.len() { veilid_log!(registry debug target:"network_result", "seqs list length should always be equal by now: {} (wanted {})", @@ -497,7 +507,10 @@ impl StorageManager { disposition: FanoutCallDisposition::Invalid, }); } - for pair in ctx.seqcounts.iter_mut().zip(answer.seqs.iter()) { + let mut convergence = 0usize; + for (subkey_index, pair) in + ctx.seqcounts.iter_mut().zip(answer.seqs.iter()).enumerate() + { let ctx_seqcnt = pair.0; let answer_seq = *pair.1; @@ -505,6 +518,18 @@ impl StorageManager { // While we may find a better sequence number if we keep looking, this does not mimic the behavior // of get and set unless we stop here if ctx_seqcnt.consensus_nodes.len() >= consensus_count { + // Unlike get, inspect has no AcceptedNewer restart, so a newer + // seq dropped here is a change the watch fallback never sees + if answer_seq > ctx_seqcnt.seq { + veilid_log!(registry debug + "InspectValue discarded a newer seq after consensus: {} subkey_index={} consensus_seq={} discarded_seq={} node={}", + opaque_record_key, + subkey_index, + ctx_seqcnt.seq, + answer_seq, + next_node + ); + } continue; } @@ -517,13 +542,16 @@ impl StorageManager { // One node has shown us the latest sequence numbers so far ctx_seqcnt.seq = answer_seq; ctx_seqcnt.consensus_nodes = vec![next_node.clone()]; + convergence += 1; } else if answer_seq == ctx_seqcnt.seq { // Keep the nodes that showed us the latest values ctx_seqcnt.consensus_nodes.push(next_node.clone()); + convergence += 1; } } ctx_seqcnt.value_nodes.push(next_node.clone()); } + ctx.convergence += convergence; } // Return peers if we have some @@ -552,17 +580,33 @@ impl StorageManager { let check_done = { let context = context.clone(); Arc::new(move |_: &FanoutResult| { - // If we have reached sufficient consensus on all subkeys, return done - let ctx = context.lock(); - let mut has_consensus = true; + let mut ctx = context.lock(); + + // An unanswered subkey may still live on nodes further out than we have reached, + // so the nodes in hand cannot prove absence: + // - stop only once some subkey has consensus, proving these nodes carry the record + // - every subkey anyone answered for must have reached full consensus + // - the last round must have added nothing + let mut any_consensus = false; + let mut any_converging = false; for cs in ctx.seqcounts.iter() { - if cs.consensus_nodes.len() < consensus_count { - has_consensus = false; - break; + let count = cs.consensus_nodes.len(); + if count >= consensus_count { + any_consensus = true; + } else if count > 0 { + any_converging = true; } } - if !ctx.seqcounts.is_empty() && ctx.opt_descriptor_info.is_some() && has_consensus { + let stalled = ctx.convergence == ctx.checked_convergence; + ctx.checked_convergence = ctx.convergence; + + if !ctx.seqcounts.is_empty() + && ctx.opt_descriptor_info.is_some() + && any_consensus + && !any_converging + && stalled + { FanoutDoneDisposition::DoneEarly } else { FanoutDoneDisposition::NotDone @@ -605,12 +649,18 @@ impl StorageManager { let mut subkey_fanout_results = vec![]; for cs in &ctx.seqcounts { let has_consensus = cs.consensus_nodes.len() >= consensus_count; + let kind = if has_consensus { + FanoutResultKind::Consensus + } else if cs.consensus_nodes.is_empty() + && matches!(fanout_result.kind, FanoutResultKind::Consensus) + { + // No node answered for this subkey, so its absence is unconfirmed + FanoutResultKind::Exhausted + } else { + fanout_result.kind + }; let subkey_fanout_result = FanoutResult { - kind: if has_consensus { - FanoutResultKind::Consensus - } else { - fanout_result.kind - }, + kind, consensus_nodes: cs.consensus_nodes.clone(), value_nodes: cs.value_nodes.clone(), }; diff --git a/veilid-core/src/storage_manager/mod.rs b/veilid-core/src/storage_manager/mod.rs index 7e57c237..f2897fa2 100644 --- a/veilid-core/src/storage_manager/mod.rs +++ b/veilid-core/src/storage_manager/mod.rs @@ -66,6 +66,7 @@ pub(crate) use get_value::InboundGetValueResult; pub(crate) use inspect_record::InboundInspectValueResult; pub(crate) use outbound_transaction_manager::OutboundTransactionHandle; pub(crate) use set_value::InboundSetValueResult; +use set_value::{SeqAssignment, SeqReuse}; pub(crate) use transaction_begin::{InboundTransactBeginResult, TransactBeginSuccess}; pub(crate) use transaction_command::{InboundTransactCommandResult, TransactCommandSuccess}; pub(crate) use watch_value::{InboundWatchParameters, InboundWatchValueResult}; @@ -411,6 +412,19 @@ impl StorageManager { // Start up outbound transaction manager inner.outbound_transaction_manager.init(); + + // Terminal delivery only queues: drop_transaction spawns it while holding + // the inner lock + let registry = self.registry(); + inner + .outbound_transaction_manager + .set_terminal_delivery_spawner(Box::new(move |delivery| { + let registry2 = registry.clone(); + registry + .storage_manager() + .background_operation_processor + .add_future(delivery.run(registry2)); + })); } self.load_metadata().await?; diff --git a/veilid-core/src/storage_manager/outbound_transaction_manager/mod.rs b/veilid-core/src/storage_manager/outbound_transaction_manager/mod.rs index 3c1cc63e..cc58aa44 100644 --- a/veilid-core/src/storage_manager/outbound_transaction_manager/mod.rs +++ b/veilid-core/src/storage_manager/outbound_transaction_manager/mod.rs @@ -8,6 +8,7 @@ mod outbound_transaction_stage; mod outbound_transaction_state; mod remote_node_transaction_id; mod subkey_consensus; +mod terminal_delivery; mod transaction_cleanup; use super::*; @@ -61,7 +62,6 @@ struct LocalNodeTransactionMapping { opaque_record_key: OpaqueRecordKey, } -#[derive(Debug)] pub(in crate::storage_manager) struct OutboundTransactionManager { /// Registry used for logging registry: VeilidComponentRegistry, @@ -79,6 +79,23 @@ pub(in crate::storage_manager) struct OutboundTransactionManager { /// Keepalive processor opt_keepalive_processor: Option>, + /// Runs a `TerminalDelivery` in the background + opt_terminal_delivery_spawner: Option>, +} + +impl fmt::Debug for OutboundTransactionManager { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OutboundTransactionManager") + .field("handles_by_key", &self.handles_by_key) + .field("transactions", &self.transactions) + .field("next_txid", &self.next_txid) + .field( + "local_node_transaction_mappings", + &self.local_node_transaction_mappings, + ) + .field("next_lnxid", &self.next_lnxid) + .finish_non_exhaustive() + } } impl_veilid_component_accessors!(OutboundTransactionManager); @@ -118,6 +135,7 @@ impl OutboundTransactionManager { next_txid: 0u64, next_lnxid: 1_000_000_000u64, opt_keepalive_processor: None, + opt_terminal_delivery_spawner: None, } } @@ -137,6 +155,16 @@ impl OutboundTransactionManager { self.opt_keepalive_processor = Some(keepalive_processor); } + /// Install the closure that runs a terminal delivery in the background. + /// The closure must not take the StorageManager inner lock: drop_transaction + /// invokes it while holding that lock. + pub fn set_terminal_delivery_spawner( + &mut self, + spawner: Box, + ) { + self.opt_terminal_delivery_spawner = Some(spawner); + } + pub fn terminate(&mut self) -> OutboundTransactionManagerTerminate { let opt_keepalive_processor = self.opt_keepalive_processor.take(); Box::pin(async move { @@ -528,6 +556,7 @@ impl OutboundTransactionManager { veilid_log!(self debug target: "network_result", "Dropping transaction: {:?}", transaction_handle); let mut unregister_keepalive_lnxids = LocalNodeTransactionIdSet::new(); + let cur_ts = Timestamp::now(); for record_state in outbound_transaction_state.get_record_states() { let lnxids = record_state.get_node_transaction_ids(); @@ -540,6 +569,54 @@ impl OutboundTransactionManager { unregister_keepalive_lnxids.extend(lnxids); + // Hand unanswered node transactions to terminal delivery: Commit for an + // ended straggler whose record reached commit consensus, Rollback otherwise + if let (Some(spawner), Some(descriptor)) = ( + self.opt_terminal_delivery_spawner.as_ref(), + record_state.descriptor(), + ) { + let record_committed = record_state + .stage_consensus() + .map(|sc| sc.stage == OutboundTransactionStage::Commit) + .unwrap_or(false); + for (_lnxid, node_transaction) in record_state.get_node_transactions() { + match node_transaction.stage() { + OutboundTransactionStage::Commit | OutboundTransactionStage::Rollback => { + continue; + } + OutboundTransactionStage::Failed + | OutboundTransactionStage::Begin + | OutboundTransactionStage::End => {} + } + let Some(expiration_ts) = node_transaction.opt_expiration() else { + continue; + }; + if expiration_ts <= cur_ts { + continue; + } + let command = if record_committed + && node_transaction.stage() == OutboundTransactionStage::End + { + TransactCommand::Commit + } else { + TransactCommand::Rollback + }; + veilid_log!(self debug target: "network_result", + "terminal delivery scheduled: {:?} xid={} node={} record={} stage={:?}", + command, node_transaction.rnxid(), node_transaction.node_ref(), + record_state.record_key().opaque(), node_transaction.stage()); + spawner(terminal_delivery::TerminalDelivery { + opaque_record_key: record_state.record_key().opaque(), + rnxid: node_transaction.rnxid().clone(), + node_ref: node_transaction.node_ref(), + safety_selection: record_state.safety_selection().clone(), + descriptor: descriptor.clone(), + command, + expiration_ts, + }); + } + } + self.handles_by_key .remove(&record_state.record_key().opaque()); } @@ -1624,7 +1701,7 @@ impl OutboundTransactionManager { } DHTReportScope::UpdateSet => { let network_seq = record_state.begin_network_seq(subkey)?; - local_seq = local_seq.next()?; + local_seq = local_seq.saturating_next(); local_seqs.push(local_seq); network_seqs.push(ValueSeqNum::max(local_seq, network_seq)); } diff --git a/veilid-core/src/storage_manager/outbound_transaction_manager/terminal_delivery.rs b/veilid-core/src/storage_manager/outbound_transaction_manager/terminal_delivery.rs new file mode 100644 index 00000000..a2434a26 --- /dev/null +++ b/veilid-core/src/storage_manager/outbound_transaction_manager/terminal_delivery.rs @@ -0,0 +1,83 @@ +use super::*; + +impl_veilid_log_facility!("stor"); + +/// Retries the terminal command for a node transaction that was dropped without an +/// answer, until the remote answers, the server-side expiration passes (the remote +/// drops it then anyway), or shutdown. Any well-formed answer completes delivery: +/// the remote executed the command, or it no longer knows the transaction. +/// +/// A lost terminal RPC leaves the remote holding the transaction until its TTL, and a +/// schema member gets one concurrent write transaction per writer key, so that stale +/// holder blocks the writer's own next begin for the whole TTL. +#[derive(Debug)] +pub(in crate::storage_manager) struct TerminalDelivery { + /// The record the node transaction is over + pub opaque_record_key: OpaqueRecordKey, + /// Server-side transaction id + pub rnxid: RemoteNodeTransactionId, + /// The node holding the transaction + pub node_ref: NodeRef, + /// Safety selection, applied fresh per attempt so retries survive route rotation + pub safety_selection: SafetySelection, + /// Descriptor for the record + pub descriptor: Arc, + /// Commit for an ended straggler in a committed transaction, else Rollback + pub command: TransactCommand, + /// Server-side expiration, past which the remote has dropped it anyway + pub expiration_ts: Timestamp, +} + +/// First retry delay; doubles per attempt, clamped to the remaining server-side lifetime +const TERMINAL_DELIVERY_BACKOFF_START_MS: u64 = 2_000; + +impl TerminalDelivery { + /// Retry until answered, expired, or shutdown; runs detached in the background + /// operation processor. + pub async fn run(self, registry: VeilidComponentRegistry) { + let mut backoff_ms = TERMINAL_DELIVERY_BACKOFF_START_MS; + let mut attempts = 0u32; + loop { + if Timestamp::now() >= self.expiration_ts { + veilid_log!(registry debug target: "network_result", + "terminal delivery expired after {} attempts: {:?} xid={} node={} record={}", + attempts, self.command, self.rnxid, self.node_ref, self.opaque_record_key); + return; + } + + attempts += 1; + let dest = Destination::direct( + self.node_ref + .routing_domain_filtered(RoutingDomain::PublicInternet), + Some(self.safety_selection.clone()), + ); + let rpc_processor = registry.rpc_processor(); + let res = rpc_processor + .rpc_call_transact_command( + dest, + self.opaque_record_key.clone(), + self.descriptor.clone(), + self.rnxid.xid(), + self.command, + None, + None, + None, + ) + .await; + + // Anything short of a well-formed answer retries; shutdown cancels this + // future via the background processor's stop token + if let Ok(NetworkResult::Value(_answer)) = res { + veilid_log!(registry debug target: "network_result", + "terminal delivery complete (attempt {}): {:?} xid={} node={} record={}", + attempts, self.command, self.rnxid, self.node_ref, self.opaque_record_key); + return; + } + + // Never sleep past expiration; the top-of-loop check then exits promptly + let remaining_ms = self.expiration_ts.duration_since(Timestamp::now()).as_u64() / 1_000; + sleep(backoff_ms.min(remaining_ms) as u32).await; + backoff_ms = backoff_ms.saturating_mul(2); + } + } +} diff --git a/veilid-core/src/storage_manager/outbound_watch_manager/mod.rs b/veilid-core/src/storage_manager/outbound_watch_manager/mod.rs index 3c55a5cc..e6cc8291 100644 --- a/veilid-core/src/storage_manager/outbound_watch_manager/mod.rs +++ b/veilid-core/src/storage_manager/outbound_watch_manager/mod.rs @@ -29,6 +29,9 @@ pub(in crate::storage_manager) struct OutboundWatchManager { /// with the earliest time each may run #[serde(skip)] pub needs_change_inspection: HashMap, + /// Subkeys owed a repeat hint per record, drained by the outbound watch check tick + #[serde(skip)] + pub pending_rehints: HashMap, } impl fmt::Display for OutboundWatchManager { @@ -97,6 +100,7 @@ impl OutboundWatchManager { outbound_watches: HashMap::new(), per_node_states: HashMap::new(), needs_change_inspection: HashMap::new(), + pending_rehints: HashMap::new(), } } diff --git a/veilid-core/src/storage_manager/outbound_watch_manager/outbound_watch_state.rs b/veilid-core/src/storage_manager/outbound_watch_manager/outbound_watch_state.rs index 5e178282..3ca78d1d 100644 --- a/veilid-core/src/storage_manager/outbound_watch_manager/outbound_watch_state.rs +++ b/veilid-core/src/storage_manager/outbound_watch_manager/outbound_watch_state.rs @@ -35,6 +35,11 @@ pub(in crate::storage_manager) struct OutboundWatchState { /// Gates hints to be monotonically increasing so replays and duplicate per-node reports are dropped. #[serde(default)] last_reported_subkey_seqs: BTreeMap, + /// Subkeys reported to the app and not yet read back, with a count of range-only copies + /// suppressed while unread. Every serving node sends its own ValueChanged per change, and a + /// seq-less range report has only consumption to dedup by. + #[serde(default, skip)] + unread_reported_subkeys: BTreeMap, } impl fmt::Display for OutboundWatchState { @@ -140,6 +145,7 @@ impl OutboundWatchState { last_fallback_inspect_ts: None, needs_forced_reconcile: false, last_reported_subkey_seqs: BTreeMap::new(), + unread_reported_subkeys: BTreeMap::new(), } } @@ -165,6 +171,46 @@ impl OutboundWatchState { self.last_reported_subkey_seqs.get(&subkey).copied() } + /// Mark `subkeys` hinted to the app and return the unmarked news; an already-marked subkey + /// counts the copy instead. Touches no calculated field, so it takes no editor. + pub fn mark_hinted_subkeys(&mut self, subkeys: &ValueSubkeyRangeSet) -> ValueSubkeyRangeSet { + let mut news = ValueSubkeyRangeSet::new(); + for subkey in subkeys.iter() { + match self.unread_reported_subkeys.entry(subkey) { + std::collections::btree_map::Entry::Vacant(v) => { + v.insert(0); + news.insert(subkey); + } + std::collections::btree_map::Entry::Occupied(mut o) => { + *o.get_mut() = o.get().saturating_add(1); + } + } + } + news + } + + /// Mark `subkeys` reported with data; fresh data satisfies owed copies, so counts reset. + pub fn mark_value_reported_subkeys(&mut self, subkeys: &ValueSubkeyRangeSet) { + for subkey in subkeys.iter() { + self.unread_reported_subkeys.insert(subkey, 0); + } + } + + /// Consume the report marks for subkeys the app read from the network, returning those that + /// had copies suppressed while unread: a copy racing the read may be a newer change, so the + /// watch owes the app one re-report. + pub fn consume_read_subkeys(&mut self, subkeys: &ValueSubkeyRangeSet) -> ValueSubkeyRangeSet { + let mut owed = ValueSubkeyRangeSet::new(); + for subkey in subkeys.iter() { + if let Some(copies) = self.unread_reported_subkeys.remove(&subkey) { + if copies > 0 { + owed.insert(subkey); + } + } + } + owed + } + /// Get the parameters we use if we're updating this state's per node watches pub fn get_per_node_params( &self, diff --git a/veilid-core/src/storage_manager/record_encryption.rs b/veilid-core/src/storage_manager/record_encryption.rs index 848a3061..adfd56f8 100644 --- a/veilid-core/src/storage_manager/record_encryption.rs +++ b/veilid-core/src/storage_manager/record_encryption.rs @@ -10,6 +10,14 @@ impl StorageManager { opaque_record_key: &OpaqueRecordKey, ) -> VeilidAPIResult { let inner = self.inner.lock(); + Self::get_record_key_for_opaque_record_key_inner(&inner, opaque_record_key) + } + + /// [Self::get_record_key_for_opaque_record_key] for a caller that already holds the inner lock + pub(super) fn get_record_key_for_opaque_record_key_inner( + inner: &StorageManagerInner, + opaque_record_key: &OpaqueRecordKey, + ) -> VeilidAPIResult { let Some(opened_record) = inner.opened_records.get(opaque_record_key) else { apibail_generic!("record must be open to resolve opaque record key"); }; diff --git a/veilid-core/src/storage_manager/record_store/record_store_inner/inbound_transactions_interface.rs b/veilid-core/src/storage_manager/record_store/record_store_inner/inbound_transactions_interface.rs index dbba0997..65b7606b 100644 --- a/veilid-core/src/storage_manager/record_store/record_store_inner/inbound_transactions_interface.rs +++ b/veilid-core/src/storage_manager/record_store/record_store_inner/inbound_transactions_interface.rs @@ -524,6 +524,15 @@ where )) } + /// Stage a transactional set of a subkey against the transaction's begin-time snapshot. + /// + /// One of the two places a storage node enforces sequence numbers; the other is + /// `StorageManager::inbound_set_value`, for non-transactional sets. This node stages the + /// proposed value only if its sequence number is strictly greater than the snapshot's, + /// otherwise it leaves the subkey alone and returns the existing value to the writer. + /// Nothing exceeds [ValueSeqNum::MAX], so a subkey stored there refuses every successor + /// for as long as this node holds the value, which is what a `min_seqnum` of `MAX` rests + /// on. pub fn inbound_transaction_set( &mut self, opaque_record_key: &OpaqueRecordKey, diff --git a/veilid-core/src/storage_manager/set_value.rs b/veilid-core/src/storage_manager/set_value.rs index f506fb65..ef2d4f4c 100644 --- a/veilid-core/src/storage_manager/set_value.rs +++ b/veilid-core/src/storage_manager/set_value.rs @@ -19,6 +19,24 @@ pub(super) struct OutboundSetValueResult { pub signed_value_data: Arc, } +/// Whether a set that has run out of sequence numbers may reuse the one already stored +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum SeqReuse { + /// A plain set can resend the same bytes until they reach consensus + Allowed, + /// A transaction only lands with consensus, so a write that cannot raise the seq never commits + Denied, +} + +/// How a set chooses the sequence number it writes +#[derive(Copy, Clone, Debug)] +pub(super) struct SeqAssignment { + /// Raise the sequence number to at least this, after the automatic increment + pub min_seqnum: Option, + /// Whether the stored sequence number may be reused when none are left + pub reuse: SeqReuse, +} + /// The result of the inbound_set_value operation #[derive(Clone, Debug)] pub(crate) enum InboundSetValueResult { @@ -83,6 +101,7 @@ impl StorageManager { .as_ref() .and_then(|o| o.writer.clone()) .or(opt_writer); + let min_seqnum = options.as_ref().and_then(|o| o.min_seqnum); let allow_offline = options .unwrap_or_default() .allow_offline @@ -102,9 +121,23 @@ impl StorageManager { .await?; let (signed_value_data, value_data, descriptor) = self - .prepare_set_value_data(&record_key, subkey, data, &writer, last_get_result) + .prepare_set_value_data( + &record_key, + subkey, + data, + &writer, + SeqAssignment { + min_seqnum, + reuse: SeqReuse::Allowed, + }, + last_get_result, + ) .await?; + // A max-seq write leaves the subkey nothing to write next, so the caller must hear that + // consensus was missed rather than assume it can write again + let closes_subkey = signed_value_data.value_data().seq().is_max(); + // Check if we are offline (this is a race, but an optimization to avoid fanout if it is likely to fail) if !self.dht_is_online() { self.handle_offline_set_single_local_value_with_subkey_lock( @@ -166,7 +199,7 @@ impl StorageManager { ) .await; - if matches!(out, Err(VeilidAPIError::TryAgain { message: _ })) { + if !closes_subkey && matches!(out, Err(VeilidAPIError::TryAgain { message: _ })) { return Ok(None); } @@ -181,8 +214,11 @@ impl StorageManager { subkey: ValueSubkey, data: Vec, writer: &KeyPair, + seq_assignment: SeqAssignment, last_get_result: GetResult, ) -> VeilidAPIResult<(Arc, ValueData, Arc)> { + let SeqAssignment { min_seqnum, reuse } = seq_assignment; + // Get cryptosystem let crypto = self.crypto(); let Some(vcrypto) = crypto.get_async(record_key.kind()) else { @@ -217,10 +253,27 @@ impl StorageManager { seq = last_signed_value_data.value_data().seq(); } else { // New value is different, increment sequence number - seq = last_signed_value_data.value_data().seq().next()?; + let last_seq = last_signed_value_data.value_data().seq(); + seq = match last_seq.next() { + Ok(next) => next, + // A caller whose minimum this seq already satisfies is retrying its own write + Err(_) + if reuse == SeqReuse::Allowed + && min_seqnum.is_some_and(|min_seqnum| last_seq >= min_seqnum) => + { + last_seq + } + Err(_) => { + apibail_generic!("sequence number at max: {}:{}", record_key, subkey); + } + }; } }; + if let Some(min_seqnum) = min_seqnum { + seq = seq.max(min_seqnum); + } + // Make new subkey data let value_data = ValueData::new_with_seq(seq, data, writer.key())?; @@ -678,6 +731,13 @@ impl StorageManager { /// Handle a received 'Set Value' query /// Returns a None if the value passed in was set /// Returns a Some(current value) if the value was older and the current value was kept + /// + /// One of the two places a storage node enforces sequence numbers; the other is + /// `RecordStoreInner::inbound_transaction_set`, for transactional sets. An inbound value is + /// accepted only if its sequence number is greater than the stored one, or equal with + /// byte-identical value data. A conflict at an equal sequence number keeps what arrived first. + /// Nothing exceeds [ValueSeqNum::MAX], so this node refuses every successor to a subkey stored + /// there for as long as it holds the value. A `min_seqnum` of `MAX` rests on that. #[cfg_attr( feature = "instrument", instrument(level = "trace", target = "dht", skip_all) diff --git a/veilid-core/src/storage_manager/tasks/check_outbound_watches.rs b/veilid-core/src/storage_manager/tasks/check_outbound_watches.rs index ff6597a7..56ef5c0c 100644 --- a/veilid-core/src/storage_manager/tasks/check_outbound_watches.rs +++ b/veilid-core/src/storage_manager/tasks/check_outbound_watches.rs @@ -70,6 +70,14 @@ impl StorageManager { // Check for watches that need fallback inspection Self::check_fallback_inspections_inner(self.registry(), &mut inner, cur_ts); + // Send any repeat hints owed from reads that raced suppressed reports + if !inner.outbound_watch_manager.pending_rehints.is_empty() { + let registry = self.registry(); + self.background_operation_processor.add_future(async move { + registry.storage_manager().send_pending_rehints().await; + }); + } + // Iterate all outbound watches and determine what work needs doing if any for outbound_watch in inner.outbound_watch_manager.outbound_watches.values_mut() { // Operate on this watch only if it isn't already being operated on. diff --git a/veilid-core/src/storage_manager/tests_storage_manager/mod.rs b/veilid-core/src/storage_manager/tests_storage_manager/mod.rs index 3381a653..bf6ddf58 100644 --- a/veilid-core/src/storage_manager/tests_storage_manager/mod.rs +++ b/veilid-core/src/storage_manager/tests_storage_manager/mod.rs @@ -4,11 +4,13 @@ mod test_encrypted_value_data; #[cfg(feature = "aimd-dht")] mod test_gated_operation; mod test_inbound_set_value_signature; +mod test_min_seqnum; pub async fn test_all() { test_encrypted_value_data::test_all().await; #[cfg(feature = "aimd-dht")] test_gated_operation::test_all(); test_inbound_set_value_signature::test_all().await; + test_min_seqnum::test_all().await; record_store::tests::test_record_store().await; } diff --git a/veilid-core/src/storage_manager/tests_storage_manager/test_min_seqnum.rs b/veilid-core/src/storage_manager/tests_storage_manager/test_min_seqnum.rs new file mode 100644 index 00000000..e58407e1 --- /dev/null +++ b/veilid-core/src/storage_manager/tests_storage_manager/test_min_seqnum.rs @@ -0,0 +1,134 @@ +//! Sequence-number assignment on an unattached node, so every set takes the offline branch. +//! That still covers the assignment: `prepare_set_value_data` signs before the online check, +//! and the offline path stores what it signed and queues it for replay. + +use super::*; +use crate::tests::*; + +async fn set_value( + rc: &RoutingContext, + record_key: &RecordKey, + subkey: ValueSubkey, + data: &[u8], + min_seqnum: Option, +) -> VeilidAPIResult> { + rc.set_dht_value( + record_key.clone(), + subkey, + data.to_vec(), + Some(SetDHTValueOptions { + writer: None, + allow_offline: None, + min_seqnum, + }), + ) + .await +} + +async fn local_seq( + rc: &RoutingContext, + record_key: &RecordKey, + subkey: ValueSubkey, +) -> ValueSeqNum { + let report = rc + .inspect_dht_record( + record_key.clone(), + Some(ValueSubkeyRangeSet::single(subkey)), + DHTReportScope::Local, + ) + .await + .expect("inspect should succeed"); + report.local_seqs()[0] +} + +pub async fn test_min_seqnum() { + let (update_callback, config) = fixture_veilid_core(); + let api = api_startup(update_callback, config) + .await + .expect("startup failed"); + let rc = api.routing_context().unwrap(); + + let rec = rc + .create_dht_record(CRYPTO_KIND_VLD0, DHTSchema::dflt(8).unwrap(), None) + .await + .unwrap(); + let key = rec.key().clone(); + + // No minimum: the ordinary automatic increment + set_value(&rc, &key, 0, b"a", None).await.unwrap(); + assert_eq!(local_seq(&rc, &key, 0).await, ValueSeqNum::ZERO); + set_value(&rc, &key, 0, b"b", None).await.unwrap(); + assert_eq!(local_seq(&rc, &key, 0).await, ValueSeqNum::from(1)); + + // A minimum at or below the automatic seq changes nothing + set_value(&rc, &key, 0, b"c", Some(ValueSeqNum::ZERO)) + .await + .unwrap(); + assert_eq!(local_seq(&rc, &key, 0).await, ValueSeqNum::from(2)); + + // A minimum above it wins + set_value(&rc, &key, 1, b"a", Some(ValueSeqNum::from(100))) + .await + .unwrap(); + assert_eq!(local_seq(&rc, &key, 1).await, ValueSeqNum::from(100)); + + // Identical data and writer already at the minimum keeps the seq + set_value(&rc, &key, 1, b"a", Some(ValueSeqNum::from(100))) + .await + .unwrap(); + assert_eq!(local_seq(&rc, &key, 1).await, ValueSeqNum::from(100)); + + // Identical data below a raised minimum is still raised + set_value(&rc, &key, 1, b"a", Some(ValueSeqNum::from(200))) + .await + .unwrap(); + assert_eq!(local_seq(&rc, &key, 1).await, ValueSeqNum::from(200)); + + // A minimum the seq already satisfies must not freeze it while increments remain + set_value(&rc, &key, 1, b"b", Some(ValueSeqNum::from(100))) + .await + .unwrap(); + assert_eq!(local_seq(&rc, &key, 1).await, ValueSeqNum::from(201)); + + // A write at the max is deferred like any other offline write, for the replay queue to push + set_value(&rc, &key, 2, b"a", Some(ValueSeqNum::MAX)) + .await + .unwrap(); + assert_eq!(local_seq(&rc, &key, 2).await, ValueSeqNum::MAX); + + // With no minimum there is nothing left to write + let err = set_value(&rc, &key, 2, b"b", None) + .await + .expect_err("a subkey at max has no sequence numbers left"); + assert!(err.to_string().contains("sequence number at max")); + + // A minimum the seq already satisfies reads as a retry, so it reuses the seq: + // - core does not check that the data matches + // - a node only accepts the retry if it does + set_value(&rc, &key, 2, b"b", Some(ValueSeqNum::MAX)) + .await + .unwrap(); + assert_eq!(local_seq(&rc, &key, 2).await, ValueSeqNum::MAX); + + // An identical rewrite at max needs no new seq, so it is accepted with no minimum at all + set_value(&rc, &key, 2, b"b", None).await.unwrap(); + assert_eq!(local_seq(&rc, &key, 2).await, ValueSeqNum::MAX); + + // The retry rule keys on the seq satisfying the minimum, not on the minimum being the max + set_value(&rc, &key, 3, b"a", Some(ValueSeqNum::MAX)) + .await + .unwrap(); + set_value(&rc, &key, 3, b"b", Some(ValueSeqNum::from(100))) + .await + .unwrap(); + assert_eq!(local_seq(&rc, &key, 3).await, ValueSeqNum::MAX); + + rc.close_dht_record(key.clone()).await.unwrap(); + rc.delete_dht_record(key).await.unwrap(); + + api.shutdown().await; +} + +pub async fn test_all() { + test_min_seqnum().await; +} diff --git a/veilid-core/src/storage_manager/transaction.rs b/veilid-core/src/storage_manager/transaction.rs index 167bdd7f..b0de7a72 100644 --- a/veilid-core/src/storage_manager/transaction.rs +++ b/veilid-core/src/storage_manager/transaction.rs @@ -97,19 +97,26 @@ impl StorageManager { let records_lock = match kind { OutboundTransactionKind::Background => { - // Non-blocking acquire: any contention on the Transaction-mode lock - // for any of these records returns TryAgain rather than waiting. - // Prevents cross-record deadlocks that could otherwise form if the - // caller racily issues overlapping begin_transaction calls. - let Some(records_lock) = self.record_lock_table.try_lock_records( - opaque_record_keys.clone(), - StorageManagerRecordLockPurpose::TransactBegin, - ) else { - veilid_log!(self debug target:"network_result", "transaction begin contended on records: [{}] (source=background)", - opaque_record_keys.iter().map(|x| x.to_string()).collect::>().join(", ")); - apibail_try_again!("transaction begin contended"); - }; - records_lock + // Bounded wait instead of storming instant TryAgain retries + // - lock_records sorts, so queueing here is deadlock-safe + // - the timeout breaks any residual cycle and keeps the TryAgain + // contract for genuinely stuck holders + match self + .record_lock_table + .lock_records( + opaque_record_keys.clone(), + StorageManagerRecordLockPurpose::TransactBegin, + ) + .timeout_duration(TimestampDuration::new_ms(BACKGROUND_BEGIN_LOCK_TIMEOUT_MS)) + .await + { + Ok(records_lock) => records_lock, + Err(_) => { + veilid_log!(self debug target:"network_result", "transaction begin contended on records: [{}] (source=background)", + opaque_record_keys.iter().map(|x| x.to_string()).collect::>().join(", ")); + apibail_try_again!("transaction begin contended"); + } + } } OutboundTransactionKind::Foreground => { // Blocking acquire is deadlock-safe (lock_records sorts) and bounded @@ -1305,6 +1312,10 @@ impl StorageManager { // Done with network access, release the semaphore drop(gate_permit); + // A transactional get is the app reading from the network, so it re-arms watch + // reporting like any other read + self.consume_watch_read_subkeys(&record_key.opaque(), &ValueSubkeyRangeSet::single(subkey)); + let get_signed_value_data = { let mut inner = self.inner.lock(); let otm = &mut inner.outbound_transaction_manager; @@ -1386,6 +1397,7 @@ impl StorageManager { } // Build the signed value data for this set from the raw data and the record's writer + let min_seqnum = options.as_ref().and_then(|o| o.min_seqnum); let signed_value_data = { let (writer, last_get_result) = { let inner = &*self.inner.lock(); @@ -1427,7 +1439,17 @@ impl StorageManager { // Make signed value data (encrypted) and value data (unencrypted) and get descriptor for this value let (signed_value_data, _, _) = self - .prepare_set_value_data(&record_key, subkey, data, &writer, last_get_result) + .prepare_set_value_data( + &record_key, + subkey, + data, + &writer, + SeqAssignment { + min_seqnum, + reuse: SeqReuse::Denied, + }, + last_get_result, + ) .await?; signed_value_data }; @@ -1792,7 +1814,7 @@ impl StorageManager { /// Failure-case counterpart to rollback_guard_locked_success. /// /// Drops local transaction state synchronously, then runs the Rollback RPCs in the background - /// holding the records_lock until they finish — so a retry can't re-begin on these records + /// holding the records_lock until they finish, so a retry can't re-begin on these records /// while their nodes still hold this transaction's node-transactions (which starves begin /// consensus). fn rollback_guard_locked_failure( diff --git a/veilid-core/src/storage_manager/transaction_begin.rs b/veilid-core/src/storage_manager/transaction_begin.rs index d565a532..6c522029 100644 --- a/veilid-core/src/storage_manager/transaction_begin.rs +++ b/veilid-core/src/storage_manager/transaction_begin.rs @@ -14,6 +14,8 @@ struct OutboundTransactBeginContext { pub route_refs: Vec, /// Observed RTT per accepted node, used to bound the post-consensus deadline pub accepted_rtts: HashMap, + /// Nodes that answered without giving us a transaction, cancelling the re-plan + pub refused_nodes: HashSet, } /// parameters required to begin a transaction @@ -69,6 +71,15 @@ impl StorageManager { /// Perform a transact begin query on the network for a single record /// This routine uses fanout to find nodes and begin their server-side node transactions + /// + /// Two rules shape the set this builds: + /// - A node that accepts without a transaction id is out of write-transaction slots + /// (`member_transaction_limit`, or `public_transaction_limit` when this node signs + /// anonymously; only sets that take a value count against either). Where a node's + /// limit is 1, the holder is usually another instance of us on the same writer key. + /// - The closest nodes own the record. Placing a transaction further out splits its + /// consensus in two, so a busy answer counts as a rejection and cancels the + /// re-plan; the second pass is only for unreachable nodes. #[cfg_attr( feature = "instrument", instrument(level = "trace", target = "dht", skip_all, err) @@ -125,6 +136,7 @@ impl StorageManager { seqs: vec![], route_refs: Vec::new(), accepted_rtts: HashMap::new(), + refused_nodes: HashSet::new(), })); let descriptor_cache = self.descriptor_cache.clone(); @@ -258,7 +270,11 @@ impl StorageManager { // Get the transaction id let Some(xid) = answer.transaction_id else { - veilid_log!(registry debug target:"network_result", "TransactBegin accepted but returned no transaction id, try again later"); + veilid_log!(registry debug target:"network_result", "TransactBegin busy on {}: no transaction id (out of slots, or this record is already transacted with our writer key)", next_node); + // Busy, not unwilling; nothing here for a second pass either + if let Some(node_id) = next_node.node_ids().get(crypto_kind) { + context.lock().refused_nodes.insert(node_id); + } return Ok(FanoutCallOutput { peer_info_list: answer.peers, disposition: FanoutCallDisposition::Rejected, @@ -383,7 +399,8 @@ impl StorageManager { // post-consensus deadline keeps gathering accepts up to this cap. let safety_count = required_strict_consensus_count + (required_strict_consensus_count * CONSENSUS_TRIM_PERCENT).div_ceil(100); - let check_done = { + // Built from a target count so a re-plan can rebuild it around the deficit + let make_check_done = |safety_count: usize| -> FanoutCheckDone { Arc::new( move |fanout_result: &FanoutResult| match fanout_result.kind { FanoutResultKind::Incomplete => FanoutDoneDisposition::NotDone, @@ -400,13 +417,14 @@ impl StorageManager { }, ) }; + let check_done = make_check_done(safety_count); - // Bound the post-consensus wait by `POST_CONSENSUS_RTT_FACTOR × max(fastest - // consensus_count RTTs)` (capped at `rpc.timeout_ms`). Using only the - // fastest consensus_count keeps a single slow accepter from inflating - // the deadline up to the rpc-timeout cap. + // Rolling post-consensus window: POST_CONSENSUS_RTT_FACTOR * max(fastest + // consensus_count RTTs), capped by rpc.timeout_ms and POST_CONSENSUS_GRACE_MS. + // Only accepts re-arm it, so an arriving wave keeps collecting up to safety_count. let rpc_timeout = TimestampDuration::new_ms(config.internal().network.rpc.timeout_ms.into()); + let post_consensus_grace = TimestampDuration::new_ms(POST_CONSENSUS_GRACE_MS); let post_consensus_timeout_callback: FanoutPostConsensusTimeoutCallback = { let context = context.clone(); Arc::new(move |result: &FanoutResult| -> TimestampDuration { @@ -428,18 +446,14 @@ impl StorageManager { .max() .unwrap_or_else(|| TimestampDuration::new_ms(0)); let bound = bound_rtt.saturating_mul(POST_CONSENSUS_RTT_FACTOR); - if bound < rpc_timeout { - bound - } else { - rpc_timeout - } + bound.min(rpc_timeout).min(post_consensus_grace) }) }; // Call the fanout let routing_table = self.routing_table(); let hash_coordinate = opaque_record_key.to_hash_coordinate(); - let fanout_call = FanoutCall::new( + let mut fanout_call = FanoutCall::new( &routing_table, FanoutCallParams { name: format!( @@ -465,7 +479,7 @@ impl StorageManager { .stop_token() .ok_or_else(VeilidAPIError::not_initialized)?; - let fanout_result = fanout_call + let mut fanout_result = fanout_call .run( init_fanout_queue, FanoutQueueMode::ThrottleAtConsensus, @@ -473,6 +487,83 @@ impl StorageManager { ) .await?; + // Exhausting the fanout short of consensus means this record's planned set is + // unreachable from here, not that the record cannot be written, so re-plan once + // over nodes the first pass never tried: + // - gated on at least one accept, the only cheap proof our own path out works + // (no accepts is exactly what a dead uplink looks like, and a second pass + // would burn another timeout on nodes that were never at fault) + // - asks only for the deficit, since the second pass adds to the first's set + // - any refusal cancels it; see this function's header + let (held_node_transactions, refused_count) = { + let ctx = context.lock(); + (ctx.accepted_rtts.len(), ctx.refused_nodes.len()) + }; + let short_set = matches!(fanout_result.kind, FanoutResultKind::Exhausted) + && held_node_transactions > 0 + && held_node_transactions < required_strict_consensus_count; + // Busy answers are common; log what the slot gate costs + if short_set && refused_count > 0 { + veilid_log!(self debug target: "network_result", + "TransactBegin[{}] short at {}/{} but {} node(s) busy, not re-planning", + opaque_record_key, + held_node_transactions, + required_strict_consensus_count, + refused_count); + } + if short_set && refused_count == 0 { + let deficit = required_strict_consensus_count - held_node_transactions; + let deficit_safety = safety_count + .saturating_sub(held_node_transactions) + .max(deficit); + + veilid_log!(self debug target: "network_result", + "TransactBegin[{}] exhausted at {}/{}, re-planning over untried nodes for the remaining {}", + opaque_record_key, + held_node_transactions, + required_strict_consensus_count, + deficit); + + let shutdown_stop_source = self + .startup_lock + .stop_token() + .ok_or_else(VeilidAPIError::not_initialized)?; + + // Exclude first-pass accepters: a second transaction on the same node would + // count twice toward a consensus meant to be over distinct nodes + let skip_nodes: HashSet = + context.lock().accepted_rtts.keys().cloned().collect(); + let capability_filter = capability_fanout_peer_info_filter(vec![VEILID_CAPABILITY_DHT]); + fanout_call.set_peer_info_filter(Arc::new(move |pi: Arc| { + if pi + .node_ids() + .iter() + .any(|node_id| skip_nodes.contains(node_id)) + { + return false; + } + capability_filter(pi) + })); + fanout_call.set_seed_untried(true); + fanout_call.set_run_target(deficit, make_check_done(deficit_safety)); + // No seed list: the value nodes are what the first pass already exhausted + fanout_result = fanout_call + .run( + Vec::new(), + FanoutQueueMode::ThrottleAtConsensus, + shutdown_stop_source, + ) + .await?; + + // The re-plan result counts only the second pass; log the combined set too + veilid_log!(self debug target: "network_result", + "TransactBegin[{}] re-plan result: {:#}, node transactions now {}/{}", + opaque_record_key, + fanout_result, + context.lock().accepted_rtts.len(), + required_strict_consensus_count); + } + let mut ctx = context.lock(); veilid_log!(self debug target: "network_result", diff --git a/veilid-core/src/storage_manager/transaction_command.rs b/veilid-core/src/storage_manager/transaction_command.rs index 4fb8002f..bd7040cf 100644 --- a/veilid-core/src/storage_manager/transaction_command.rs +++ b/veilid-core/src/storage_manager/transaction_command.rs @@ -426,10 +426,10 @@ impl StorageManager { attempt_responded_xids.insert(pnr.lnxid); per_node_results.push(pnr); - // Arm post-consensus deadline once when consensus is first reached. - // Bound by max of the fastest required_strict_consensus_count RTTs - // among Valid responses we've collected — this keeps a single slow - // responder from inflating the deadline to the rpc-timeout cap. + // Arm the post-consensus deadline once, at first consensus + // - bound: max of the fastest required_strict_consensus_count Valid RTTs + // - capped by POST_CONSENSUS_GRACE_MS, for the answer wave trailing + // consensus rather than dead stragglers if !consensus_reached && valid_response_count >= params.required_strict_consensus_count { @@ -449,7 +449,8 @@ impl StorageManager { .max() .unwrap_or(0) .saturating_mul(POST_CONSENSUS_RTT_FACTOR) - .min(rpc_timeout_us); + .min(rpc_timeout_us) + .min(ms_to_us(POST_CONSENSUS_GRACE_MS as u32)); let now = Timestamp::now_non_decreasing(); post_consensus_deadline = Some(Timestamp::new(now.as_u64().saturating_add(max_bound_us))); diff --git a/veilid-core/src/storage_manager/watch_value.rs b/veilid-core/src/storage_manager/watch_value.rs index b7fb5aae..e04cf87d 100644 --- a/veilid-core/src/storage_manager/watch_value.rs +++ b/veilid-core/src/storage_manager/watch_value.rs @@ -1286,6 +1286,13 @@ impl StorageManager { .await?; let mut newer_online_subkeys = report.newer_online_subkeys(); + // Telemetry to decide whether the periodic inspection earns its cost + if newer_online_subkeys.is_empty() { + veilid_log!(this debug "change inspect: nothing newer online for {}", record_key); + } else { + veilid_log!(this debug "change inspect: PRODUCTIVE: missed value change on {} subkeys {}", record_key, newer_online_subkeys); + } + // Get changed first changed subkey until we find one to report let mut n = 0; while !newer_online_subkeys.is_empty() { @@ -1429,8 +1436,10 @@ impl StorageManager { ) .await?; if report.newer_online_subkeys().is_empty() { + veilid_log!(this debug "change inspect: nothing newer online for {}", record_key); return VeilidAPIResult::Ok(()); } + veilid_log!(this info "change inspect productive: missed value change on {} subkeys {}", record_key, report.newer_online_subkeys()); this.with_background_transaction( record_key.clone(), @@ -1686,7 +1695,7 @@ impl StorageManager { per_node_state.count = count; } - // Record that we received a valid ValueChanged — resets fallback inspection timer + // A valid ValueChanged resets the fallback inspection timer // NOTE: It is possible this should be a per-node state, but for now let's try it on the whole watch // Done after per_node_state usage to avoid borrow conflicts with owm.per_node_states state.edit(&owm.per_node_states, |editor| { @@ -1713,32 +1722,14 @@ impl StorageManager { ) .await?; - // Update the watch state if we're going to report + // Mark the report and take the count under one guard let remaining_count = { let inner = &mut *self.inner.lock(); - - // Get the outbound watch - if let Some(outbound_watch) = inner - .outbound_watch_manager - .outbound_watches - .get_mut(&opaque_record_key) - { - if let Some(state) = outbound_watch.state_mut() { - // If we're going to report, update the remaining change count - let remaining_count = state.remaining_count().saturating_sub(1); - state.edit(&inner.outbound_watch_manager.per_node_states, |editor| { - editor.set_remaining_count(remaining_count); - }); - - state.remaining_count() - } else { - // No outbound watch current state means watch died (no remaining count) - 0 - } - } else { - // No outbound watch means watch died (no remaining count) - 0 + if let Some(state) = Self::watch_state_mut_inner(inner, &opaque_record_key) { + // Data in hand: later range-only copies of this change must not re-notify + state.mark_value_reported_subkeys(&reportable_subkeys); } + Self::note_watch_report_inner(inner, &opaque_record_key, None) }; // Announce ValueChanged VeilidUpdate @@ -1760,8 +1751,12 @@ impl StorageManager { } /// Report a value-change to the app as a non-committing hint (no local-store write) and - /// decrement the watch count. When a value is present its decrypted form is reported and - /// its sequence number recorded so replays/duplicate reports are gated out. + /// decrement the watch count. When a value is present its decrypted form is reported and its + /// sequence number recorded so replays and duplicate reports are gated out. + /// + /// One guard covers the whole decision: re-trim to the currently watched range, mark the + /// reported subkeys (news-filtering range-only reports), and take the count. Sends nothing + /// when no watched subkey remains, the range-only news is empty, or the watch has died. async fn report_value_change_hint( &self, record_lock: StorageManagerRecordLockGuard, @@ -1770,35 +1765,35 @@ impl StorageManager { opt_value: Option>, ) -> VeilidAPIResult<()> { let opaque_record_key = record_lock.record(); - let opt_reported = opt_value.as_ref().map(|v| { - ( - reportable_subkeys.first().unwrap_or_log(), - v.value_data().seq(), - ) - }); - let remaining_count = { + let (subkeys, remaining_count) = { let inner = &mut *self.inner.lock(); - if let Some(outbound_watch) = inner - .outbound_watch_manager - .outbound_watches - .get_mut(&opaque_record_key) - { - if let Some(state) = outbound_watch.state_mut() { - let rc = state.remaining_count().saturating_sub(1); - state.edit(&inner.outbound_watch_manager.per_node_states, |editor| { - editor.set_remaining_count(rc); - if let Some((subkey, seq)) = opt_reported { - editor.set_last_reported_subkey_seq(subkey, seq); - } - }); - rc - } else { - 0 - } - } else { - 0 + let Some(watched) = self.get_watched_subkeys_inner(inner, &record_key)? else { + return Ok(()); + }; + let reportable = reportable_subkeys.intersect(&watched); + if reportable.is_empty() { + return Ok(()); } + let Some(state) = Self::watch_state_mut_inner(inner, &opaque_record_key) else { + return Ok(()); + }; + let subkeys = if opt_value.is_some() { + state.mark_value_reported_subkeys(&reportable); + reportable + } else { + state.mark_hinted_subkeys(&reportable) + }; + if subkeys.is_empty() { + return Ok(()); + } + let opt_reported = opt_value + .as_ref() + .map(|v| (subkeys.first().unwrap_or_log(), v.value_data().seq())); + ( + subkeys, + Self::note_watch_report_inner(inner, &opaque_record_key, opt_reported), + ) }; let opt_value_data = if let Some(value) = &opt_value { @@ -1813,7 +1808,7 @@ impl StorageManager { self.update_callback_value_change( record_lock, record_key, - reportable_subkeys, + subkeys, remaining_count, opt_value_data, ); @@ -1821,6 +1816,33 @@ impl StorageManager { Ok(()) } + /// Decrement and return the watch's remaining change count for a report, + /// recording the reported seq that gates transactional hints. + fn note_watch_report_inner( + inner: &mut StorageManagerInner, + opaque_record_key: &OpaqueRecordKey, + opt_reported: Option<(ValueSubkey, ValueSeqNum)>, + ) -> u32 { + if let Some(outbound_watch) = inner + .outbound_watch_manager + .outbound_watches + .get_mut(opaque_record_key) + { + if let Some(state) = outbound_watch.state_mut() { + let rc = state.remaining_count().saturating_sub(1); + state.edit(&inner.outbound_watch_manager.per_node_states, |editor| { + editor.set_remaining_count(rc); + if let Some((subkey, seq)) = opt_reported { + editor.set_last_reported_subkey_seq(subkey, seq); + } + }); + return rc; + } + } + // No outbound watch or state means the watch died (no remaining count) + 0 + } + /// Handle a received 'Value Changed' statement #[cfg_attr( feature = "instrument", @@ -2000,8 +2022,7 @@ impl StorageManager { .await?; } } else if !reportable_subkeys.is_empty() { - // No single reportable value (e.g. multiple subkeys changed): report the changed - // range immediately so the app fetches it, rather than deferring a change inspection + // No single reportable value (e.g. multiple subkeys changed): report the range so the app fetches it self.report_value_change_hint(record_lock, record_key, reportable_subkeys, None) .await?; } @@ -2009,6 +2030,72 @@ impl StorageManager { Ok(NetworkResult::value(())) } + /// This record's outbound watch state, if the watch and its state exist. + fn watch_state_mut_inner<'a>( + inner: &'a mut StorageManagerInner, + opaque_record_key: &OpaqueRecordKey, + ) -> Option<&'a mut OutboundWatchState> { + inner + .outbound_watch_manager + .outbound_watches + .get_mut(opaque_record_key)? + .state_mut() + } + + /// Consume the report marks for subkeys the app read from the network, so a later change is + /// news again. A subkey whose copies were suppressed mid-read queues one repeat hint (the + /// racing copy may be a newer change), which the outbound watch check tick sends. + pub(super) fn consume_watch_read_subkeys( + &self, + opaque_record_key: &OpaqueRecordKey, + subkeys: &ValueSubkeyRangeSet, + ) { + let inner = &mut *self.inner.lock(); + let Ok(record_key) = + Self::get_record_key_for_opaque_record_key_inner(inner, opaque_record_key) + else { + return; + }; + let Some(owed) = Self::watch_state_mut_inner(inner, opaque_record_key) + .map(|state| state.consume_read_subkeys(subkeys)) + else { + return; + }; + if !owed.is_empty() { + inner + .outbound_watch_manager + .pending_rehints + .entry(record_key) + .and_modify(|s| *s = s.union(&owed)) + .or_insert(owed); + } + } + + /// Send any owed repeat hints. Runs from the outbound watch check tick. + pub(super) async fn send_pending_rehints(&self) { + let rehints: Vec<(RecordKey, ValueSubkeyRangeSet)> = { + let mut inner = self.inner.lock(); + inner + .outbound_watch_manager + .pending_rehints + .drain() + .collect() + }; + for (record_key, subkeys) in rehints { + // Record lock before inner, as everywhere + let record_lock = self + .record_lock_table + .lock_record(record_key.opaque(), StorageManagerRecordLockPurpose::Watch) + .await; + if let Err(e) = self + .report_value_change_hint(record_lock, record_key, subkeys, None) + .await + { + veilid_log!(self debug "failed to send repeat hint: {}", e); + } + } + } + /// Check all watches for changes /// Used when we come back online from being offline and may have /// missed some ValueChanged notifications diff --git a/veilid-core/src/tests/common/test_dht.rs b/veilid-core/src/tests/common/test_dht.rs index ff067fc9..ef2a0762 100644 --- a/veilid-core/src/tests/common/test_dht.rs +++ b/veilid-core/src/tests/common/test_dht.rs @@ -357,6 +357,7 @@ async fn test_open_writer_dht_value(api: VeilidAPI) { Some(SetDHTValueOptions { writer: Some(keypair), allow_offline: None, + min_seqnum: None, }), ) .await; @@ -393,6 +394,7 @@ pub async fn test_set_dht_value_allow_offline(api: VeilidAPI) { Some(SetDHTValueOptions { writer: None, allow_offline: None, + min_seqnum: None, }), ) .await; @@ -407,6 +409,7 @@ pub async fn test_set_dht_value_allow_offline(api: VeilidAPI) { Some(SetDHTValueOptions { writer: None, allow_offline: Some(AllowOffline(true)), + min_seqnum: None, }), ) .await; @@ -421,6 +424,7 @@ pub async fn test_set_dht_value_allow_offline(api: VeilidAPI) { Some(SetDHTValueOptions { writer: None, allow_offline: Some(AllowOffline(false)), + min_seqnum: None, }), ) .await; @@ -447,6 +451,7 @@ async fn test_flush_dht_record_offline(api: VeilidAPI) { Some(SetDHTValueOptions { writer: None, allow_offline: Some(AllowOffline(true)), + min_seqnum: None, }), ) .await diff --git a/veilid-core/src/tests/mocks/mod.rs b/veilid-core/src/tests/mocks/mod.rs index 5996407a..270c634e 100644 --- a/veilid-core/src/tests/mocks/mod.rs +++ b/veilid-core/src/tests/mocks/mod.rs @@ -104,6 +104,7 @@ pub fn fake_veilid_config() -> VeilidConfig { internal: Some(VeilidConfigInternal { network: VeilidConfigInternalNetwork { connection_initial_timeout_ms: 1000, + single_send_timeout_ms: 2000, connection_inactivity_timeout_ms: 2000, max_connections_per_ip4: 3000, max_connections_per_ip6_prefix: 4000, diff --git a/veilid-core/src/veilid_api/api.rs b/veilid-core/src/veilid_api/api.rs index 9e8e9097..26eca32a 100644 --- a/veilid-core/src/veilid_api/api.rs +++ b/veilid-core/src/veilid_api/api.rs @@ -147,6 +147,15 @@ impl VeilidAPI { /// Save a user secret to the protected store. Returns true if a value already existed. /// + /// Uses hardware-backed storage and system keychains (iOS Keychain, Android KeyStore, Windows + /// Credential Manager) where the platform has them. On devices that have none, + /// [VeilidConfigProtectedStore::allow_insecure_fallback] permits less secure storage such as + /// owner-readable disk files. + /// + /// ProtectedStore secrets are size-limited per platform and slow to access. Unless the secret + /// must survive uninstalling and reinstalling the app, prefer the `TableStore`, which is + /// encrypted with the *device encryption key* that already lives in the `ProtectedStore`. + /// /// Errors with [VeilidAPIError::InvalidArgument] for keys veilid reserves for itself. pub async fn save_user_secret(&self, key: String, value: Vec) -> VeilidAPIResult { self.protected_store()? diff --git a/veilid-core/src/veilid_api/debug.rs b/veilid-core/src/veilid_api/debug.rs index 6ab1a412..48a1543a 100644 --- a/veilid-core/src/veilid_api/debug.rs +++ b/veilid-core/src/veilid_api/debug.rs @@ -99,6 +99,25 @@ enum DebugCommand { #[arg(value_name = "relay_token")] args: Vec, }, + /// inspect and control low-level transport availability + Transport { + /// `stats`: dump per-transport send-health windows and disabled set + /// + /// `clear`: purge all transport stats windows + /// + /// `enable|disable ...`: set availability. A disabled + /// transport's low-level sends and connects fail with NoConnection, + /// exercising the normal evidence/suppression machinery. + /// + /// `toggle ... [duration]`: flip availability, reverting + /// after `duration` if given. + /// + /// `` is `udp|tcp` (both families), `ipv4|ipv6` (both + /// protocols), or `/` e.g. `udp/ipv6`. TCP covers + /// WS/WSS (low-level transport granularity). + #[arg(value_name = "transport_token")] + args: Vec, + }, /// send a status RPC question to a destination Ping { /// Destination grammar: @@ -413,9 +432,10 @@ enum DebugRecordSubcommand { #[command(help_template = DEBUG_HELP_TEMPLATE)] Set { /// Positional form: - /// - `[key] [writer] [offline|online|true|false]` + /// - `[key] [writer] [offline|online|true|false] [min_seqnum]` /// - if `key` is omitted, the most recently opened record is used /// - `` may be plain text, quoted text, or `#`-prefixed hex + /// - `min_seqnum` is `max`, `max-` leaving `n` writes, or a sequence number #[arg(allow_hyphen_values = true, value_name = "set_arg")] params: Vec, }, @@ -634,6 +654,51 @@ fn get_dht_report_scope(text: &str) -> Option { parse_dht_report_scope(text).ok() } +fn parse_allow_offline(text: &str) -> Result { + match text.to_ascii_lowercase().trim() { + "online" | "false" => Ok(AllowOffline(false)), + "offline" | "true" => Ok(AllowOffline(true)), + _ => Err(format!("invalid allow_offline: {text}")), + } +} + +fn get_allow_offline(text: &str) -> Option { + parse_allow_offline(text).ok() +} + +/// Accepts `max`, `max-` (leaving `n` writes), or a bare sequence number. +pub(super) fn parse_value_seq_num(text: &str) -> Result { + let text = text.to_ascii_lowercase(); + let text = text.trim(); + let Some(remaining) = text.strip_prefix("max") else { + return text + .parse::() + .map(ValueSeqNum::from) + .map_err(|e| format!("invalid sequence number '{text}': {e}")); + }; + if remaining.is_empty() { + return Ok(ValueSeqNum::MAX); + } + let Some(count) = remaining.strip_prefix('-') else { + return Err(format!("invalid sequence number: {text}")); + }; + let count = count + .parse::() + .map_err(|e| format!("invalid remaining write count '{count}': {e}"))?; + let max = u32::from(ValueSeqNum::MAX); + max.checked_sub(count) + .map(ValueSeqNum::from) + .ok_or_else(|| format!("more remaining writes than sequence numbers: {text}")) +} + +fn transport_list(transports: &[LowLevelTransportType]) -> String { + transports + .iter() + .map(|t| t.to_string()) + .collect::>() + .join(", ") +} + enum PublishedState { Published, Current, @@ -1264,6 +1329,156 @@ impl VeilidAPI { Ok(registry.routing_table().debug_info_entry(node_ref)) } + /// Expand a transport selector token into low-level transports. + /// `udp`/`tcp` = both families; `ipv4`/`ipv6` = both protocols; `udp/ipv6` = one. + fn parse_transport_selector(token: &str) -> Option> { + fn proto(s: &str) -> Option { + match s { + "udp" => Some(LowLevelProtocolType::UDP), + "tcp" => Some(LowLevelProtocolType::TCP), + _ => None, + } + } + fn family(s: &str) -> Option { + match s { + "ipv4" | "v4" => Some(AddressType::IPV4), + "ipv6" | "v6" => Some(AddressType::IPV6), + _ => None, + } + } + let token = token.to_ascii_lowercase(); + let (protos, families) = if let Some((p, f)) = token.split_once('/') { + (vec![proto(p)?], vec![family(f)?]) + } else if let Some(p) = proto(&token) { + (vec![p], vec![AddressType::IPV4, AddressType::IPV6]) + } else if let Some(f) = family(&token) { + ( + vec![LowLevelProtocolType::UDP, LowLevelProtocolType::TCP], + vec![f], + ) + } else { + return None; + }; + let mut out = Vec::new(); + for p in &protos { + for f in &families { + out.push(LowLevelTransportType { + low_level_protocol_type: *p, + address_type: *f, + }); + } + } + Some(out) + } + + fn debug_transport(&self, args: Vec) -> VeilidAPIResult { + let registry = self.core_context()?.registry(); + let network_manager = registry.network_manager(); + let transport_stats = network_manager.transport_stats(); + + let Some(subcommand) = args.first() else { + apibail_missing_argument!("debug_transport", "subcommand"); + }; + match subcommand.as_str() { + "stats" => Ok(transport_stats.debug_stats()), + "clear" => { + transport_stats.clear_all_windows(); + Ok("transport stats cleared".to_owned()) + } + "enable" | "disable" | "toggle" => { + let mut selector_args: Vec<&String> = args[1..].iter().collect(); + let mut opt_duration_us: Option = None; + if *subcommand == "toggle" { + if let Some(last) = selector_args.last() { + if let Some(dur) = parse_duration(last) { + // A token that parses as both stays a selector + if Self::parse_transport_selector(last).is_none() { + opt_duration_us = Some(dur); + selector_args.pop(); + } + } + } + } + if selector_args.is_empty() { + apibail_missing_argument!("debug_transport", "selector"); + } + let mut transports: Vec = Vec::new(); + for token in &selector_args { + let Some(mut tts) = Self::parse_transport_selector(token) else { + apibail_invalid_argument!("debug_transport", "selector", token.to_string()); + }; + transports.append(&mut tts); + } + transports.sort(); + transports.dedup(); + + let mut out = String::new(); + match subcommand.as_str() { + "enable" | "disable" => { + let disable = subcommand.as_str() == "disable"; + transport_stats.set_transports_disabled(&transports, disable); + out += &format!( + "{}: {}\n", + if disable { "disabled" } else { "enabled" }, + transport_list(&transports) + ); + } + "toggle" => { + let currently_disabled = transport_stats.disabled_transports(); + let to_enable: Vec = transports + .iter() + .filter(|t| currently_disabled.contains(t)) + .copied() + .collect(); + let to_disable: Vec = transports + .iter() + .filter(|t| !currently_disabled.contains(t)) + .copied() + .collect(); + let mut gen = 0u64; + if !to_enable.is_empty() { + gen = transport_stats.set_transports_disabled(&to_enable, false); + } + if !to_disable.is_empty() { + gen = transport_stats.set_transports_disabled(&to_disable, true); + } + out += &format!( + "toggled: enabled [{}] disabled [{}]\n", + transport_list(&to_enable), + transport_list(&to_disable) + ); + // The generation check makes the revert skip anything changed since + if let Some(dur_us) = opt_duration_us { + let registry = registry.clone(); + spawn_detached("transport toggle revert", async move { + sleep((dur_us / 1000) as u32).await; + let network_manager = registry.network_manager(); + let ts = network_manager.transport_stats(); + ts.set_transports_disabled_if_generation(&to_enable, true, gen); + ts.set_transports_disabled_if_generation(&to_disable, false, gen); + }); + out += &format!("reverting in {:#}\n", TimestampDuration::new(dur_us)); + } + } + _ => unreachable!(), + } + let now_disabled = transport_stats.disabled_transports(); + out += &format!( + "now disabled: {}\n", + if now_disabled.is_empty() { + "(none)".to_owned() + } else { + transport_list(&now_disabled) + } + ); + Ok(out) + } + _ => { + apibail_invalid_argument!("debug_transport", "subcommand", subcommand.to_string()) + } + } + } + async fn debug_relay(&self, args: Vec) -> VeilidAPIResult { let registry = self.core_context()?.registry(); @@ -1610,7 +1825,7 @@ impl VeilidAPI { .await .map_err(VeilidAPIError::internal)?; match result { - StatusResult::Answer { answer, .. } => Ok(format!("{:#?}", answer)), + StatusResult::Sent(status_answer) => Ok(format!("{:#?}", status_answer.answer)), StatusResult::Failed(sdr) => Ok(format!("send failed: {}", sdr)), StatusResult::NotSent(nr) => Ok(format!("not sent: {:?}", nr)), } @@ -2264,29 +2479,31 @@ impl VeilidAPI { } Err(_) => None, }; - let allow_offline = if args.len() > 2 + opt_arg_add { - get_debug_argument_at( - &args, - 2 + opt_arg_add, - "debug_record_set", - "allow_offline", - get_string, - ) - .ok() - } else { - None - }; - - let allow_offline = if let Some(allow_offline) = allow_offline { - if &allow_offline == "online" || &allow_offline == "false" { - Some(AllowOffline(false)) - } else if &allow_offline == "offline" || &allow_offline == "true" { - Some(AllowOffline(true)) - } else { - return Ok(format!("Unknown allow_offline: {}", allow_offline)); + let allow_offline = match get_debug_argument_at( + &args, + 2 + opt_arg_add, + "debug_record_set", + "allow_offline", + get_allow_offline, + ) { + Ok(v) => { + opt_arg_add += 1; + Some(v) } - } else { - None + Err(_) => None, + }; + let min_seqnum = match get_debug_argument_at( + &args, + 2 + opt_arg_add, + "debug_record_set", + "min_seqnum", + get_string, + ) { + Ok(v) => match parse_value_seq_num(&v) { + Ok(seq) => Some(seq), + Err(e) => return Ok(e), + }, + Err(_) => None, }; // Do a record set @@ -2298,6 +2515,7 @@ impl VeilidAPI { Some(SetDHTValueOptions { writer, allow_offline, + min_seqnum, }), ) .await @@ -3164,6 +3382,7 @@ impl VeilidAPI { DebugCommand::Punish { command } => self.debug_punish(command), DebugCommand::Txtrecord { keypairs } => self.debug_txtrecord(keypairs).await, DebugCommand::Relay { args } => self.debug_relay(args).await, + DebugCommand::Transport { args } => self.debug_transport(args), DebugCommand::Ping { destination } => self.debug_ping(destination).await, DebugCommand::Appmessage { destination, data } => { self.debug_app_message(destination, data).await diff --git a/veilid-core/src/veilid_api/tests_veilid_api/mod.rs b/veilid-core/src/veilid_api/tests_veilid_api/mod.rs index e1e0f9c1..c07848ab 100644 --- a/veilid-core/src/veilid_api/tests_veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/tests_veilid_api/mod.rs @@ -2,6 +2,7 @@ pub mod mocks; pub mod test_config; pub mod test_serialize_json; pub mod test_value_data; +pub mod test_value_seq_num; pub use mocks::*; @@ -12,5 +13,10 @@ pub async fn test_all() { test_serialize_json::test_serialize_json(); test_value_data::value_data_ok(); test_value_data::value_data_too_long(); + test_value_seq_num::value_seq_num_next(); + test_value_seq_num::value_seq_num_saturating_next(); + test_value_seq_num::value_seq_num_ordering(); + #[cfg(feature = "debug-api")] + test_value_seq_num::value_seq_num_parse(); test_config::test_internal_footgun_gate(); } diff --git a/veilid-core/src/veilid_api/tests_veilid_api/test_value_seq_num.rs b/veilid-core/src/veilid_api/tests_veilid_api/test_value_seq_num.rs new file mode 100644 index 00000000..6457bb60 --- /dev/null +++ b/veilid-core/src/veilid_api/tests_veilid_api/test_value_seq_num.rs @@ -0,0 +1,44 @@ +use super::*; + +pub fn value_seq_num_next() { + assert_eq!(ValueSeqNum::NONE.next().unwrap(), ValueSeqNum::ZERO); + assert_eq!(ValueSeqNum::ZERO.next().unwrap(), ValueSeqNum::from(1)); + assert!(ValueSeqNum::MAX.next().is_err()); +} + +pub fn value_seq_num_saturating_next() { + assert_eq!(ValueSeqNum::NONE.saturating_next(), ValueSeqNum::ZERO); + assert_eq!( + ValueSeqNum::from(u32::MAX - 2).saturating_next(), + ValueSeqNum::MAX + ); + assert_eq!(ValueSeqNum::MAX.saturating_next(), ValueSeqNum::MAX); +} + +pub fn value_seq_num_ordering() { + // "never written" sorts below the first write, so max() picks a real seq over NONE + assert!(ValueSeqNum::NONE < ValueSeqNum::ZERO); + assert!(ValueSeqNum::ZERO < ValueSeqNum::MAX); + assert_eq!(ValueSeqNum::ZERO.max(ValueSeqNum::NONE), ValueSeqNum::ZERO); +} + +#[cfg(feature = "debug-api")] +pub fn value_seq_num_parse() { + use crate::veilid_api::debug::parse_value_seq_num as parse; + + assert_eq!(parse("max").unwrap(), ValueSeqNum::MAX); + assert_eq!(parse("MAX").unwrap(), ValueSeqNum::MAX); + assert_eq!(parse("max-0").unwrap(), ValueSeqNum::MAX); + assert_eq!( + parse("max-1").unwrap(), + ValueSeqNum::from(u32::MAX - 2), + "max-1 must leave exactly one write" + ); + assert_eq!(parse("0").unwrap(), ValueSeqNum::ZERO); + assert_eq!(parse("42").unwrap(), ValueSeqNum::from(42)); + + assert!(parse("max-4294967295").is_err()); + assert!(parse("max+1").is_err()); + assert!(parse("maximum").is_err()); + assert!(parse("").is_err()); +} diff --git a/veilid-core/src/veilid_api/types/dht/dht_transaction_set_value_options.rs b/veilid-core/src/veilid_api/types/dht/dht_transaction_set_value_options.rs index f312a6d4..951176e9 100644 --- a/veilid-core/src/veilid_api/types/dht/dht_transaction_set_value_options.rs +++ b/veilid-core/src/veilid_api/types/dht/dht_transaction_set_value_options.rs @@ -7,4 +7,11 @@ pub struct DHTTransactionSetValueOptions { /// Override writer key pair for this operation #[cfg_attr(feature = "schemars", schemars(with = "Option"))] pub writer: Option, + /// Raise the sequence number to at least this, after the automatic increment. + /// [ValueSeqNum::MAX] minus `n` leaves `n` writes after this one. + /// + /// If the subkey already meets the minimum, the write reuses that sequence number instead of + /// failing, so a write at [ValueSeqNum::MAX] can be retried. The caller must send the same data + /// and writer on every retry; nothing here enforces that. + pub min_seqnum: Option, } diff --git a/veilid-core/src/veilid_api/types/dht/set_dht_value_options.rs b/veilid-core/src/veilid_api/types/dht/set_dht_value_options.rs index b4c4bee9..291578da 100644 --- a/veilid-core/src/veilid_api/types/dht/set_dht_value_options.rs +++ b/veilid-core/src/veilid_api/types/dht/set_dht_value_options.rs @@ -9,6 +9,13 @@ pub struct SetDHTValueOptions { /// Defaults to true. If false, the value will not be written if the node is offline, /// and a TryAgain error will be returned. pub allow_offline: Option, + /// Raise the sequence number to at least this, after the automatic increment. + /// [ValueSeqNum::MAX] minus `n` leaves `n` writes after this one. + /// + /// If the subkey already meets the minimum, the write reuses that sequence number instead of + /// failing, so a write at [ValueSeqNum::MAX] can be retried until it reaches consensus. The + /// caller must send the same data and writer on every retry; nothing here enforces that. + pub min_seqnum: Option, } impl Default for SetDHTValueOptions { @@ -16,6 +23,7 @@ impl Default for SetDHTValueOptions { Self { writer: None, allow_offline: Some(AllowOffline(true)), + min_seqnum: None, } } } diff --git a/veilid-core/src/veilid_api/types/dht/value_seq_num.rs b/veilid-core/src/veilid_api/types/dht/value_seq_num.rs index 72a0405a..a45323d3 100644 --- a/veilid-core/src/veilid_api/types/dht/value_seq_num.rs +++ b/veilid-core/src/veilid_api/types/dht/value_seq_num.rs @@ -12,7 +12,10 @@ use super::*; pub struct ValueSeqNum(Option); impl ValueSeqNum { - /// The largest representable sequence number (`u32::MAX` is reserved to mean [ValueSeqNum::NONE]) + /// The largest representable sequence number (`u32::MAX` is reserved to mean [ValueSeqNum::NONE]). + /// + /// A storage node accepts an inbound value only if its sequence number exceeds the one it + /// holds, so a subkey written here has no successor a conforming node will take. pub const MAX: Self = ValueSeqNum(Some(u32::MAX - 1)); /// No sequence number, for a subkey that has never been written pub const NONE: Self = ValueSeqNum(None); @@ -33,6 +36,12 @@ impl ValueSeqNum { } } + /// The next sequence number, saturating at [ValueSeqNum::MAX] instead of failing. + #[must_use] + pub fn saturating_next(&self) -> Self { + self.next().unwrap_or(ValueSeqNum::MAX) + } + /// True if this is [ValueSeqNum::MAX] #[must_use] pub const fn is_max(&self) -> bool { diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index 2f9b5885..76a89f70 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -957,6 +957,9 @@ impl Default for VeilidConfigInternalDHT { pub struct VeilidConfigInternalNetwork { /// Timeout to establish a new connection, in milliseconds. pub connection_initial_timeout_ms: u32, + /// Cap on any single low-level send, in milliseconds, so a send on a wedged + /// transport cannot consume the caller's whole RPC timeout budget. + pub single_send_timeout_ms: u32, /// Idle time before an inactive connection is dropped, in milliseconds. pub connection_inactivity_timeout_ms: u32, /// Maximum simultaneous connections from a single IPv4 address. @@ -987,6 +990,7 @@ impl Default for VeilidConfigInternalNetwork { fn default() -> Self { Self { connection_initial_timeout_ms: 5000, + single_send_timeout_ms: 2000, connection_inactivity_timeout_ms: 60000, max_connections_per_ip4: 32, max_connections_per_ip6_prefix: 32, diff --git a/veilid-flutter/example/integration_test/app_test.dart b/veilid-flutter/example/integration_test/app_test.dart index aad10cda..4bc7c559 100644 --- a/veilid-flutter/example/integration_test/app_test.dart +++ b/veilid-flutter/example/integration_test/app_test.dart @@ -18,205 +18,217 @@ import 'test_table_db.dart'; import 'test_veilid_config.dart'; void main() => runWithIntegrationLog(() { - IntegrationTestWidgetsFlutterBinding.ensureInitialized(); - runApp(const IntegrationLogApp()); + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + runApp(const IntegrationLogApp()); - final fixture = DefaultVeilidFixture( - programName: 'veilid_flutter integration test', - logFixture: LogFixture.instance); - final updateProcessorFixture = - UpdateProcessorFixture(veilidFixture: fixture); + final fixture = DefaultVeilidFixture( + programName: 'veilid_flutter integration test', + logFixture: LogFixture.instance); + final updateProcessorFixture = + UpdateProcessorFixture(veilidFixture: fixture); - // setUp(() { - // final liveTest = Invoker.current?.liveTest; - // print('setUp ${liveTest?.test.name}'); - // }); - // tearDown(() { - // final liveTest = Invoker.current?.liveTest; - // print('tearDown ${liveTest?.test.name}'); - // }); + // setUp(() { + // final liveTest = Invoker.current?.liveTest; + // print('setUp ${liveTest?.test.name}'); + // }); + // tearDown(() { + // final liveTest = Invoker.current?.liveTest; + // print('tearDown ${liveTest?.test.name}'); + // }); - group('Uninitialized', () { - test('veilid config defaults', testVeilidConfigDefaults, tags: ['config']); - }); - - group('Initialized', () { - setUpAll(fixture.setUp); - tearDownAll(fixture.tearDown); - setUpAll(updateProcessorFixture.setUp); - tearDownAll(updateProcessorFixture.tearDown); - // Fail the run if any subsystem (online/relay/route/connection/readiness) flapped - tearDownAll(LogFixture.instance.assertNoFlapping); - - group('Crypto', () { - test('list cryptosystems', testListCryptoSystems, tags: ['crypto']); - test('get cryptosystem', testGetCryptoSystems, tags: ['crypto']); - test('get cryptosystem invalid', testGetCryptoSystemInvalid, - tags: ['crypto']); - test('hash and verify password', testHashAndVerifyPassword, - tags: ['crypto']); - test('sign and verify signature', testSignAndVerifySignature); - test('sign and verify signatures', testSignAndVerifySignatures, - tags: ['crypto']); - test('generate shared secret', testGenerateSharedSecret, - tags: ['crypto']); - test('hpke seal and open', testHpkeSealOpen, tags: ['crypto']); - test('kem bridge from signing keys', testKemBridgeFromSigningKeys, - tags: ['crypto']); - }); - - group('Table DB', () { - test('delete table db nonexistent', testDeleteTableDbNonExistent, - tags: ['table_db']); - test('open delete table db', testOpenDeleteTableDb, tags: ['table_db']); - test('open twice table db', testOpenTwiceTableDb, tags: ['table_db']); - test('open twice table db store load', testOpenTwiceTableDbStoreLoad, - tags: ['table_db']); - test('open twice table db store delete load', - testOpenTwiceTableDbStoreDeleteLoad, - tags: ['table_db']); - test('resize table db', testResizeTableDb, tags: ['table_db']); - }); - - group('Protected Store', () { - test('save load remove user secret', testSaveLoadRemoveUserSecret, - tags: ['protected_store']); - test('binary user secret roundtrip', testBinaryUserSecretRoundtrip, - tags: ['protected_store']); - test('missing user secret', testMissingUserSecret, - tags: ['protected_store']); - test('list user secrets', testListUserSecrets, tags: ['protected_store']); - test('list never returns reserved keys', testListNeverReturnsReservedKeys, - tags: ['protected_store']); - test('reserved keys are rejected', testReservedKeysAreRejected, - tags: ['protected_store']); - test('delete all user secrets', testDeleteAllUserSecrets, - tags: ['protected_store']); - }); - - group('Attached', () { - setUpAll(fixture.attach); - tearDownAll(fixture.detach); - - group('Routing Contexts', () { - test('routing contexts', testRoutingContexts); - test('app message loopback', - () => testAppMessageLoopback(fixture.updateStream), - tags: ['routing_context', 'app_message']); - test('app call loopback', - () => testAppCallLoopback(fixture.updateStream), - tags: ['routing_context', 'app_call']); - test('app message loopback big packets', - () => testAppMessageLoopbackBigPackets(fixture.updateStream), - tags: ['routing_context', 'app_message_big']); - test('app call loopback big packets', - () => testAppCallLoopbackBigPackets(fixture.updateStream), - tags: ['routing_context', 'app_call_big']); + group('Uninitialized', () { + test('veilid config defaults', testVeilidConfigDefaults, + tags: ['config']); }); - for (final cryptoKind in Veilid.instance.validCryptoKinds()) { - group('DHT $cryptoKind', () { - final testDHT = TestDHT(cryptoKind); - setUpAll(testDHT.setUpAll); - tearDownAll(testDHT.tearDownAll); + group('Initialized', () { + setUpAll(fixture.setUp); + tearDownAll(fixture.tearDown); + setUpAll(updateProcessorFixture.setUp); + tearDownAll(updateProcessorFixture.tearDown); + // Fail the run if any subsystem (online/relay/route/connection/readiness) flapped + tearDownAll(LogFixture.instance.assertNoFlapping); - test('get dht value unopened', testDHT.testGetDHTValueUnopened, - tags: ['dht']); - test('open dht record nonexistent no writer', - testDHT.testOpenDHTRecordNonexistentNoWriter, - tags: ['dht']); - test('close dht record nonexistent', - testDHT.testCloseDHTRecordNonexistent, - tags: ['dht']); - test('delete dht record nonexistent', - testDHT.testDeleteDHTRecordNonexistent, - tags: ['dht']); - test('create delete dht record simple', - testDHT.testCreateDeleteDHTRecordSimple, - tags: ['dht']); - test('create delete dht record no close', - testDHT.testCreateDeleteDHTRecordNoClose, - tags: ['dht']); - test('create delete dht record with deterministic key', - testDHT.testCreateDHTRecordWithDeterministicKey, - tags: ['dht']); - test('get dht value nonexistent', testDHT.testGetDHTValueNonexistent); - test('set get dht value', testDHT.testSetGetDHTValue, tags: ['dht']); - test('set get dht value with owner', - testDHT.testSetGetDHTValueWithOwner, - tags: ['dht']); - test('open writer dht value', testDHT.testOpenWriterDHTValue, - tags: ['dht']); - test('inspect dht record', testDHT.testInspectDHTRecord, - tags: ['dht']); - test('flush dht record', testDHT.testFlushDHTRecord, tags: ['dht']); + group('Crypto', () { + test('list cryptosystems', testListCryptoSystems, tags: ['crypto']); + test('get cryptosystem', testGetCryptoSystems, tags: ['crypto']); + test('get cryptosystem invalid', testGetCryptoSystemInvalid, + tags: ['crypto']); + test('hash and verify password', testHashAndVerifyPassword, + tags: ['crypto']); + test('sign and verify signature', testSignAndVerifySignature); + test('sign and verify signatures', testSignAndVerifySignatures, + tags: ['crypto']); + test('generate shared secret', testGenerateSharedSecret, + tags: ['crypto']); + test('hpke seal and open', testHpkeSealOpen, tags: ['crypto']); + test('kem bridge from signing keys', testKemBridgeFromSigningKeys, + tags: ['crypto']); }); - group('DHT Transactions $cryptoKind', () { - final testDHTTransactions = - TestDHTTransactions(cryptoKind, updateProcessorFixture); - setUpAll(testDHTTransactions.setUpAll); - tearDownAll(testDHTTransactions.tearDownAll); - - setUp(testDHTTransactions.setUp); - tearDown(testDHTTransactions.tearDown); - - test('should create empty transaction and drop it explicitly', - testDHTTransactions.testEmptyTxAndDrop, - tags: ['dht_transactions']); - test('should create empty transaction and rollback', - testDHTTransactions.testEmptyTxAndRollback, - tags: ['dht_transactions']); - test('should create empty transaction and commit', - testDHTTransactions.testEmptyTxAndCommit, - tags: ['dht_transactions']); - test('should create transaction, add sets, and rollback', - testDHTTransactions.testTxAddSetsAndRollback, - tags: ['dht_transactions']); - test('should create transaction, add sets, and commit', - testDHTTransactions.testTxAddSetsAndCommit, - tags: ['dht_transactions']); - test('should extend transaction with new record and commit', - testDHTTransactions.testTxExtendAndCommit, - tags: ['dht_transactions']); - test('should extend transaction with new record and rollback', - testDHTTransactions.testTxExtendAndRollback, - tags: ['dht_transactions']); - test('should create transaction, add sets, gets, and commit', - testDHTTransactions.testTxAddSetsGetsAndCommit, - tags: ['dht_transactions']); - test( - 'should create empty transaction, fail non-transactional sets and' - ' then rollback', - testDHTTransactions.testTxFailNonTxSetsAndRollback, - tags: ['dht_transactions']); - test( - 'should create transaction, inspect, add sets to subkey 1, ' - ' inspect and commit. Then a new transaction inspect, and gets, ' - 'and commit', - testDHTTransactions.testTxInspectAddInspectCommit, - tags: ['dht_transactions']); + group('Table DB', () { + test('delete table db nonexistent', testDeleteTableDbNonExistent, + tags: ['table_db']); + test('open delete table db', testOpenDeleteTableDb, + tags: ['table_db']); + test('open twice table db', testOpenTwiceTableDb, tags: ['table_db']); + test('open twice table db store load', testOpenTwiceTableDbStoreLoad, + tags: ['table_db']); + test('open twice table db store delete load', + testOpenTwiceTableDbStoreDeleteLoad, + tags: ['table_db']); + test('resize table db', testResizeTableDb, tags: ['table_db']); }); - group('Veilid DHT Transactions Full $cryptoKind', () { - final testDHTTransactionsFull = TestDHTTransactionsFull(cryptoKind); - setUpAll(testDHTTransactionsFull.setUpAll); - tearDownAll(testDHTTransactionsFull.tearDownAll); - - setUp(testDHTTransactionsFull.setUp); - tearDown(testDHTTransactionsFull.tearDown); - - test('should create empty transaction, inspect, add gets and commit', - testDHTTransactionsFull.testEmptyTxInspectGetsAndCommit, - timeout: const Timeout(Duration(seconds: 120)), tags: ['stress']); - test( - 'should create transaction, fill all records, commit, and then' - ' get all records and rollback', - testDHTTransactionsFull.testTxFillCommitTxGetRollback, - timeout: const Timeout(Duration(seconds: 300)), - tags: ['stress']); + group('Protected Store', () { + test('save load remove user secret', testSaveLoadRemoveUserSecret, + tags: ['protected_store']); + test('binary user secret roundtrip', testBinaryUserSecretRoundtrip, + tags: ['protected_store']); + test('missing user secret', testMissingUserSecret, + tags: ['protected_store']); + test('list user secrets', testListUserSecrets, + tags: ['protected_store']); + test('list never returns reserved keys', + testListNeverReturnsReservedKeys, + tags: ['protected_store']); + test('reserved keys are rejected', testReservedKeysAreRejected, + tags: ['protected_store']); + test('delete all user secrets', testDeleteAllUserSecrets, + tags: ['protected_store']); }); - } + + group('Attached', () { + setUpAll(fixture.attach); + tearDownAll(fixture.detach); + + group('Routing Contexts', () { + test('routing contexts', testRoutingContexts); + test('app message loopback', + () => testAppMessageLoopback(fixture.updateStream), + tags: ['routing_context', 'app_message']); + test('app call loopback', + () => testAppCallLoopback(fixture.updateStream), + tags: ['routing_context', 'app_call']); + test('app message loopback big packets', + () => testAppMessageLoopbackBigPackets(fixture.updateStream), + tags: ['routing_context', 'app_message_big']); + test('app call loopback big packets', + () => testAppCallLoopbackBigPackets(fixture.updateStream), + tags: ['routing_context', 'app_call_big']); + }); + + for (final cryptoKind in Veilid.instance.validCryptoKinds()) { + group('DHT $cryptoKind', () { + final testDHT = TestDHT(cryptoKind); + setUpAll(testDHT.setUpAll); + tearDownAll(testDHT.tearDownAll); + + test('get dht value unopened', testDHT.testGetDHTValueUnopened, + tags: ['dht']); + test('open dht record nonexistent no writer', + testDHT.testOpenDHTRecordNonexistentNoWriter, + tags: ['dht']); + test('close dht record nonexistent', + testDHT.testCloseDHTRecordNonexistent, + tags: ['dht']); + test('delete dht record nonexistent', + testDHT.testDeleteDHTRecordNonexistent, + tags: ['dht']); + test('create delete dht record simple', + testDHT.testCreateDeleteDHTRecordSimple, + tags: ['dht']); + test('create delete dht record no close', + testDHT.testCreateDeleteDHTRecordNoClose, + tags: ['dht']); + test('create delete dht record with deterministic key', + testDHT.testCreateDHTRecordWithDeterministicKey, + tags: ['dht']); + test('get dht value nonexistent', + testDHT.testGetDHTValueNonexistent); + test('set get dht value', testDHT.testSetGetDHTValue, + tags: ['dht']); + test('set get dht value with owner', + testDHT.testSetGetDHTValueWithOwner, + tags: ['dht']); + test('open writer dht value', testDHT.testOpenWriterDHTValue, + tags: ['dht']); + test('inspect dht record', testDHT.testInspectDHTRecord, + tags: ['dht']); + test('min seqnum dht record', testDHT.testMinSeqnumDHTRecord, + tags: ['dht']); + test('flush dht record', testDHT.testFlushDHTRecord, + tags: ['dht']); + }); + + group('DHT Transactions $cryptoKind', () { + final testDHTTransactions = + TestDHTTransactions(cryptoKind, updateProcessorFixture); + setUpAll(testDHTTransactions.setUpAll); + tearDownAll(testDHTTransactions.tearDownAll); + + setUp(testDHTTransactions.setUp); + tearDown(testDHTTransactions.tearDown); + + test('should create empty transaction and drop it explicitly', + testDHTTransactions.testEmptyTxAndDrop, + tags: ['dht_transactions']); + test('should create empty transaction and rollback', + testDHTTransactions.testEmptyTxAndRollback, + tags: ['dht_transactions']); + test('should create empty transaction and commit', + testDHTTransactions.testEmptyTxAndCommit, + tags: ['dht_transactions']); + test('should create transaction, add sets, and rollback', + testDHTTransactions.testTxAddSetsAndRollback, + tags: ['dht_transactions']); + test('should create transaction, add sets, and commit', + testDHTTransactions.testTxAddSetsAndCommit, + tags: ['dht_transactions']); + test('should extend transaction with new record and commit', + testDHTTransactions.testTxExtendAndCommit, + tags: ['dht_transactions']); + test('should extend transaction with new record and rollback', + testDHTTransactions.testTxExtendAndRollback, + tags: ['dht_transactions']); + test('should create transaction, add sets, gets, and commit', + testDHTTransactions.testTxAddSetsGetsAndCommit, + tags: ['dht_transactions']); + test( + 'should create empty transaction, fail non-transactional sets and' + ' then rollback', + testDHTTransactions.testTxFailNonTxSetsAndRollback, + tags: ['dht_transactions']); + test( + 'should create transaction, inspect, add sets to subkey 1, ' + ' inspect and commit. Then a new transaction inspect, and gets, ' + 'and commit', + testDHTTransactions.testTxInspectAddInspectCommit, + tags: ['dht_transactions']); + }); + + group('Veilid DHT Transactions Full $cryptoKind', () { + final testDHTTransactionsFull = + TestDHTTransactionsFull(cryptoKind); + setUpAll(testDHTTransactionsFull.setUpAll); + tearDownAll(testDHTTransactionsFull.tearDownAll); + + setUp(testDHTTransactionsFull.setUp); + tearDown(testDHTTransactionsFull.tearDown); + + test( + 'should create empty transaction, inspect, add gets and commit', + testDHTTransactionsFull.testEmptyTxInspectGetsAndCommit, + timeout: const Timeout(Duration(seconds: 120)), + tags: ['stress']); + test( + 'should create transaction, fill all records, commit, and then' + ' get all records and rollback', + testDHTTransactionsFull.testTxFillCommitTxGetRollback, + timeout: const Timeout(Duration(seconds: 300)), + tags: ['stress']); + }); + } + }); + }); }); - }); -}); diff --git a/veilid-flutter/example/integration_test/test_crypto.dart b/veilid-flutter/example/integration_test/test_crypto.dart index 677f995e..9f064eab 100644 --- a/veilid-flutter/example/integration_test/test_crypto.dart +++ b/veilid-flutter/example/integration_test/test_crypto.dart @@ -122,8 +122,7 @@ Future testHpkeSealOpen() async { expect(await cs.hpkeOpen(kkp1.secret, aad, sealed), equals(plaintext)); // empty aad and empty plaintext round-trip - final sealedEmpty = - await cs.hpkeSeal(kkp1.key, Uint8List(0), Uint8List(0)); + final sealedEmpty = await cs.hpkeSeal(kkp1.key, Uint8List(0), Uint8List(0)); expect(await cs.hpkeOpen(kkp1.secret, Uint8List(0), sealedEmpty), equals(Uint8List(0))); diff --git a/veilid-flutter/example/integration_test/test_dht.dart b/veilid-flutter/example/integration_test/test_dht.dart index 968c47bf..bc1bc306 100644 --- a/veilid-flutter/example/integration_test/test_dht.dart +++ b/veilid-flutter/example/integration_test/test_dht.dart @@ -12,6 +12,17 @@ class TestDHT { TestDHT(this.cryptoKind); + // A closing write has no next write to recover with, so it reports TryAgain + static final _closingWriteRetry = RetryStrategy( + timeout: const Duration(seconds: 30), + rules: [ + RetryRule.delay( + matches: (e) => e is VeilidAPIExceptionTryAgain, + delay: const Duration(milliseconds: 250), + ), + ], + ); + Future setUpAll() async { routingContext = await Veilid.instance.routingContext(); } @@ -297,6 +308,50 @@ class TestDHT { await routingContext.deleteDHTRecord(rec.key); } + Future testMinSeqnumDHTRecord() async { + final rec = await routingContext.createDHTRecord( + cryptoKind, const DHTSchema.dflt(oCnt: 1)); + + const noOffline = SetDHTValueOptions(allowOffline: false); + final va = utf8.encode('Qwertyuiop Asdfghjkl Zxcvbnm'); + final vb = utf8.encode('1234567890'); + + // maxSeqnum - 1 leaves exactly one write + expect( + await routingContext.setDHTValue(rec.key, 0, va, + options: noOffline.copyWith(minSeqnum: ValueData.maxSeqnum - 1)), + isNull); + + var rr = await routingContext.inspectDHTRecord(rec.key); + expect(rr.localSeqs, equals([ValueData.maxSeqnum - 1])); + + expect( + await _closingWriteRetry.retry(() => + routingContext.setDHTValue(rec.key, 0, vb, options: noOffline)), + isNull); + + rr = await routingContext.inspectDHTRecord(rec.key); + expect(rr.localSeqs, equals([ValueData.maxSeqnum])); + + // Nothing writes after maxSeqnum + await expectLater( + routingContext.setDHTValue(rec.key, 0, va, options: noOffline), + throwsA(isA())); + + // A resend at the same minimum reuses the seqnum instead of failing + expect( + await _closingWriteRetry.retry(() => routingContext.setDHTValue( + rec.key, 0, vb, + options: noOffline.copyWith(minSeqnum: ValueData.maxSeqnum))), + isNull); + + rr = await routingContext.inspectDHTRecord(rec.key); + expect(rr.localSeqs, equals([ValueData.maxSeqnum])); + + await routingContext.closeDHTRecord(rec.key); + await routingContext.deleteDHTRecord(rec.key); + } + Future testFlushDHTRecord() async { final rec = await routingContext.createDHTRecord( cryptoKind, const DHTSchema.dflt(oCnt: 1)); diff --git a/veilid-flutter/example/integration_test/test_dht_transactions_full.dart b/veilid-flutter/example/integration_test/test_dht_transactions_full.dart index 978abffb..db2cdb95 100644 --- a/veilid-flutter/example/integration_test/test_dht_transactions_full.dart +++ b/veilid-flutter/example/integration_test/test_dht_transactions_full.dart @@ -112,13 +112,15 @@ class TestDHTTransactionsFull { // Commit final startCommit = Veilid.instance.now(); await tx.commit(); - debugPrint('commit transaction: ${Veilid.instance.now().diff(startCommit)}'); + debugPrint( + 'commit transaction: ${Veilid.instance.now().diff(startCommit)}'); // Transaction 2 final startCommit2 = Veilid.instance.now(); final tx2 = await Veilid.instance.transactDHTRecords(dhtRecords); expect(tx2, isInstanceOf()); - debugPrint('begin transaction 2: ${Veilid.instance.now().diff(startCommit2)}'); + debugPrint( + 'begin transaction 2: ${Veilid.instance.now().diff(startCommit2)}'); // Gets for (var subkey = 0; subkey < dhtSubkeyCount; subkey++) { diff --git a/veilid-flutter/example/integration_test/test_protected_store.dart b/veilid-flutter/example/integration_test/test_protected_store.dart index 71b5d4c0..09c988c1 100644 --- a/veilid-flutter/example/integration_test/test_protected_store.dart +++ b/veilid-flutter/example/integration_test/test_protected_store.dart @@ -19,8 +19,7 @@ Future testSaveLoadRemoveUserSecret() async { await _cleanup(); // Saving a new secret reports that nothing was there before. - expect( - await Veilid.instance.saveUserSecret(testKey, utf8.encode('hello')), + expect(await Veilid.instance.saveUserSecret(testKey, utf8.encode('hello')), isFalse); expect(utf8.decode((await Veilid.instance.loadUserSecret(testKey))!), equals('hello')); @@ -54,7 +53,8 @@ Future testBinaryUserSecretRoundtrip() async { Future testMissingUserSecret() async { expect(await Veilid.instance.loadUserSecret('__dart_never_stored'), isNull); - expect(await Veilid.instance.removeUserSecret('__dart_never_stored'), isFalse); + expect( + await Veilid.instance.removeUserSecret('__dart_never_stored'), isFalse); } Future testListUserSecrets() async { @@ -118,8 +118,8 @@ Future testDeleteAllUserSecrets() async { } // The store is still usable afterwards. - expect(await Veilid.instance.saveUserSecret(testKey, utf8.encode('c')), - isFalse); + expect( + await Veilid.instance.saveUserSecret(testKey, utf8.encode('c')), isFalse); keys = await Veilid.instance.listUserSecrets(); expect(keys, contains(testKey)); diff --git a/veilid-flutter/example/test/widget_test.dart b/veilid-flutter/example/test/widget_test.dart index 4daf4773..e8c6d8af 100644 --- a/veilid-flutter/example/test/widget_test.dart +++ b/veilid-flutter/example/test/widget_test.dart @@ -18,8 +18,7 @@ void main() { // Verify that platform version is retrieved. expect( find.byWidgetPredicate( - (widget) => - widget is Text && widget.data!.startsWith('Running on:'), + (widget) => widget is Text && widget.data!.startsWith('Running on:'), ), findsOneWidget, ); diff --git a/veilid-flutter/hook/build.dart b/veilid-flutter/hook/build.dart index 6cc60984..821c8428 100644 --- a/veilid-flutter/hook/build.dart +++ b/veilid-flutter/hook/build.dart @@ -1,5 +1,6 @@ // Builds veilid-flutter/rust as the code asset lib/veilid_ffi.dart resolves against. // `assetName` below and that file's @DefaultAsset must move together. + import 'package:code_assets/code_assets.dart'; import 'package:hooks/hooks.dart'; import 'package:native_toolchain_rust/native_toolchain_rust.dart'; diff --git a/veilid-flutter/lib/default_config.dart b/veilid-flutter/lib/default_config.dart index a9a3ccc3..13b6eed6 100644 --- a/veilid-flutter/lib/default_config.dart +++ b/veilid-flutter/lib/default_config.dart @@ -24,41 +24,46 @@ Future getDefaultVeilidConfig({ final defaultConfigStr = Veilid.instance.defaultVeilidConfig(); final defaultConfig = VeilidConfig.fromJson(jsonDecode(defaultConfigStr)); return defaultConfig.copyWith( - programName: programName, - namespace: namespace, - tableStore: defaultConfig.tableStore.copyWith( - directory: isWeb - ? '' - : p.join((await getApplicationSupportDirectory()).absolute.path, - 'table_store'), + programName: programName, + namespace: namespace, + tableStore: defaultConfig.tableStore.copyWith( + directory: isWeb + ? '' + : p.join( + (await getApplicationSupportDirectory()).absolute.path, + 'table_store', + ), + ), + blockStore: defaultConfig.blockStore.copyWith( + directory: isWeb + ? '' + : p.join( + (await getApplicationSupportDirectory()).absolute.path, + 'block_store', + ), + ), + protectedStore: defaultConfig.protectedStore.copyWith( + directory: isWeb + ? '' + : p.join( + (await getApplicationSupportDirectory()).absolute.path, + 'protected_store', + ), + deviceEncryptionKeyPassword: deviceEncryptionKeyPassword, + newDeviceEncryptionKeyPassword: newDeviceEncryptionKeyPassword, + ), + network: defaultConfig.network.copyWith( + networkKeyPassword: networkKeyPassword, + routingTable: defaultConfig.network.routingTable.copyWith( + bootstrap: bootstrap.isNotEmpty + ? bootstrap.split(',') + : defaultConfig.network.routingTable.bootstrap, + bootstrapKeys: bootstrapKeys != null + ? bootstrapKeys.isNotEmpty + ? bootstrapKeys.split(',').map(PublicKey.fromString).toList() + : [] + : defaultConfig.network.routingTable.bootstrapKeys, ), - blockStore: defaultConfig.blockStore.copyWith( - directory: isWeb - ? '' - : p.join((await getApplicationSupportDirectory()).absolute.path, - 'block_store'), - ), - protectedStore: defaultConfig.protectedStore.copyWith( - directory: isWeb - ? '' - : p.join((await getApplicationSupportDirectory()).absolute.path, - 'protected_store'), - deviceEncryptionKeyPassword: deviceEncryptionKeyPassword, - newDeviceEncryptionKeyPassword: newDeviceEncryptionKeyPassword, - ), - network: defaultConfig.network.copyWith( - networkKeyPassword: networkKeyPassword, - routingTable: defaultConfig.network.routingTable.copyWith( - bootstrap: bootstrap.isNotEmpty - ? bootstrap.split(',') - : defaultConfig.network.routingTable.bootstrap, - bootstrapKeys: bootstrapKeys != null - ? bootstrapKeys.isNotEmpty - ? bootstrapKeys - .split(',') - .map(PublicKey.fromString) - .toList() - : [] - : defaultConfig.network.routingTable.bootstrapKeys), - )); + ), + ); } diff --git a/veilid-flutter/lib/processor_interface.dart b/veilid-flutter/lib/processor_interface.dart index 23005fee..e52c10d0 100644 --- a/veilid-flutter/lib/processor_interface.dart +++ b/veilid-flutter/lib/processor_interface.dart @@ -27,8 +27,9 @@ extension VeilidProcessorRetry on VeilidProcessorInterface { if (processorConnectionState.isPublicInternetReady) { return; } - var stream = streamProcessorConnectionState() - .where((s) => s.isPublicInternetReady); + var stream = streamProcessorConnectionState().where( + (s) => s.isPublicInternetReady, + ); if (timeout != null) { stream = stream.timeout(timeout); } @@ -50,10 +51,7 @@ extension VeilidProcessorRetry on VeilidProcessorInterface { /// Run [closure], retrying VeilidAPI transients per [strategy] (default /// [VeilidAPIRetryStrategy], waiting on network readiness before a tryAgain /// retry). - Future retry( - Future Function() closure, { - RetryStrategy? strategy, - }) => + Future retry(Future Function() closure, {RetryStrategy? strategy}) => (strategy ?? VeilidAPIRetryStrategy( tryAgainRetry: (attemptNumber) async { diff --git a/veilid-flutter/lib/retry.dart b/veilid-flutter/lib/retry.dart index 708aefb4..ca45a1b7 100644 --- a/veilid-flutter/lib/retry.dart +++ b/veilid-flutter/lib/retry.dart @@ -21,19 +21,16 @@ class RetryRule { /// Retry up to [maxAttempts] times (null = unbounded), waiting [delay] (or /// not, if null) between tries. - RetryRule.delay({ - required this.matches, - int? maxAttempts, - Duration? delay, - }) : beforeRetry = ((n) async { - if (maxAttempts != null && n >= maxAttempts) { - return false; - } - if (delay != null) { - await Future.delayed(delay); - } - return true; - }); + RetryRule.delay({required this.matches, int? maxAttempts, Duration? delay}) + : beforeRetry = ((n) async { + if (maxAttempts != null && n >= maxAttempts) { + return false; + } + if (delay != null) { + await Future.delayed(delay); + } + return true; + }); /// Retry up to [maxAttempts] times (null = unbounded), awaiting [wait] (e.g. /// network readiness) between tries. @@ -42,20 +39,19 @@ class RetryRule { int? maxAttempts, required Future Function() wait, }) : beforeRetry = ((n) async { - if (maxAttempts != null && n >= maxAttempts) { - return false; - } - await wait(); - return true; - }); + if (maxAttempts != null && n >= maxAttempts) { + return false; + } + await wait(); + return true; + }); /// A copy whose action is [chain], which receives the attempt number and this /// rule's current action so it can do its own work and defer to it, e.g. /// `rule.withBeforeRetry((n, prev) async { record(); return prev(n); })`. RetryRule withBeforeRetry( Future Function(int attemptNumber, RetryAction previous) chain, - ) => - RetryRule(matches: matches, beforeRetry: (n) => chain(n, beforeRetry)); + ) => RetryRule(matches: matches, beforeRetry: (n) => chain(n, beforeRetry)); /// A copy that runs [onRetry] (e.g. telemetry) before this rule's action, or /// this rule unchanged if [onRetry] is null. @@ -127,8 +123,8 @@ class RetryStrategy { } /// VeilidAPI-level policy: retry the transients that clear once online. Each -/// rule's action defaults to a bounded fixed delay; override either to change -/// the wait (e.g. block on network readiness) or add telemetry. +/// rule's action defaults to a bounded doubling delay; override either to +/// change the wait (e.g. block on network readiness) or add telemetry. class VeilidAPIRetryStrategy extends RetryStrategy { /// [tryAgainRetry] handles [VeilidAPIExceptionTryAgain]; [transientRetry] /// handles timeout and no-connection errors. Either defaults to @@ -139,27 +135,30 @@ class VeilidAPIRetryStrategy extends RetryStrategy { RetryAction? transientRetry, Duration? timeout, }) : super( - rules: [ - RetryRule( - matches: (e) => e is VeilidAPIExceptionTryAgain, - beforeRetry: tryAgainRetry ?? defaultRetry, - ), - RetryRule( - matches: (e) => - e is VeilidAPIExceptionTimeout || - e is VeilidAPIExceptionNoConnection, - beforeRetry: transientRetry ?? defaultRetry, - ), - ], - timeout: timeout, - ); + rules: [ + RetryRule( + matches: (e) => e is VeilidAPIExceptionTryAgain, + beforeRetry: tryAgainRetry ?? defaultRetry, + ), + RetryRule( + matches: (e) => + e is VeilidAPIExceptionTimeout || + e is VeilidAPIExceptionNoConnection, + beforeRetry: transientRetry ?? defaultRetry, + ), + ], + timeout: timeout, + ); - /// Up to 3 tries, 250ms apart. + /// Up to 3 tries, waiting 500ms then 1s. The delay doubles because a flat one + /// re-offers the whole load to a congested network at once. static Future defaultRetry(int attemptNumber) async { if (attemptNumber >= 3) { return false; } - await Future.delayed(const Duration(milliseconds: 250)); + await Future.delayed( + Duration(milliseconds: 250 * (1 << attemptNumber)), + ); return true; } } diff --git a/veilid-flutter/lib/veilid_api_exception.dart b/veilid-flutter/lib/veilid_api_exception.dart index 04de335a..a8db9aa0 100644 --- a/veilid-flutter/lib/veilid_api_exception.dart +++ b/veilid-flutter/lib/veilid_api_exception.dart @@ -52,22 +52,30 @@ abstract class VeilidAPIException implements Exception { case 'Unimplemented': { return VeilidAPIExceptionUnimplemented( - json['unimplemented']! as String); + json['unimplemented']! as String, + ); } case 'ParseError': { return VeilidAPIExceptionParseError( - json['message']! as String, json['value']! as String); + json['message']! as String, + json['value']! as String, + ); } case 'InvalidArgument': { - return VeilidAPIExceptionInvalidArgument(json['context']! as String, - json['argument']! as String, json['value']! as String); + return VeilidAPIExceptionInvalidArgument( + json['context']! as String, + json['argument']! as String, + json['value']! as String, + ); } case 'MissingArgument': { return VeilidAPIExceptionMissingArgument( - json['context']! as String, json['argument']! as String); + json['context']! as String, + json['argument']! as String, + ); } case 'Generic': { @@ -76,12 +84,14 @@ abstract class VeilidAPIException implements Exception { case 'TransactionNotFound': { return VeilidAPIExceptionTransactionNotFound( - json['message']! as String); + json['message']! as String, + ); } default: { throw VeilidAPIExceptionInternal( - "Invalid VeilidAPIException type: ${json['kind']! as String}"); + "Invalid VeilidAPIException type: ${json['kind']! as String}", + ); } } } @@ -261,10 +271,14 @@ class VeilidAPIExceptionInvalidArgument implements VeilidAPIException { /// Construct with the calling context, argument name, and rejected value. const VeilidAPIExceptionInvalidArgument( - this.context, this.argument, this.value); + this.context, + this.argument, + this.value, + ); @override - String toString() => 'VeilidAPIException: InvalidArgument' + String toString() => + 'VeilidAPIException: InvalidArgument' ' ($context:$argument)\n value: $value'; @override diff --git a/veilid-flutter/lib/veilid_config.dart b/veilid-flutter/lib/veilid_config.dart index 158b438f..eb1a3304 100644 --- a/veilid-flutter/lib/veilid_config.dart +++ b/veilid-flutter/lib/veilid_config.dart @@ -105,7 +105,9 @@ sealed class VeilidFFIConfigLogging with _$VeilidFFIConfigLogging { VeilidFFIConfigLoggingOtlp otlp, @Default(VeilidFFIConfigLoggingFlame(enabled: false, path: '')) VeilidFFIConfigLoggingFlame flame, - @Default(VeilidFFIConfigLoggingStats(enabled: false, intervalSecs: 10, path: '')) + @Default( + VeilidFFIConfigLoggingStats(enabled: false, intervalSecs: 10, path: ''), + ) VeilidFFIConfigLoggingStats stats, }) = _VeilidFFIConfigLogging; @@ -385,9 +387,8 @@ sealed class VeilidConfigDHT with _$VeilidConfigDHT { /// RPC configuration. @freezed sealed class VeilidConfigRPC with _$VeilidConfigRPC { - const factory VeilidConfigRPC({ - required int defaultRouteHopCount, - }) = _VeilidConfigRPC; + const factory VeilidConfigRPC({required int defaultRouteHopCount}) = + _VeilidConfigRPC; factory VeilidConfigRPC.fromJson(dynamic json) => _$VeilidConfigRPCFromJson(json as Map); @@ -424,17 +425,16 @@ enum VeilidConfigAddressType { /// Parse from its JSON name (`IPV4`/`IPV6`). factory VeilidConfigAddressType.fromJson(dynamic j) => switch (j as String) { - 'IPV4' => VeilidConfigAddressType.ipv4, - 'IPV6' => VeilidConfigAddressType.ipv6, - _ => - throw ArgumentError.value(j, 'j', 'invalid VeilidConfigAddressType'), - }; + 'IPV4' => VeilidConfigAddressType.ipv4, + 'IPV6' => VeilidConfigAddressType.ipv6, + _ => throw ArgumentError.value(j, 'j', 'invalid VeilidConfigAddressType'), + }; /// The JSON name (`IPV4`/`IPV6`). String toJson() => switch (this) { - VeilidConfigAddressType.ipv4 => 'IPV4', - VeilidConfigAddressType.ipv6 => 'IPV6', - }; + VeilidConfigAddressType.ipv4 => 'IPV4', + VeilidConfigAddressType.ipv6 => 'IPV6', + }; } //////////// @@ -514,8 +514,8 @@ enum VeilidConfigProtectedStoreBackend { /// WebExtension storage. extensionStorage; -// /// Delegate to a registered secret-resolver callback. -// callback; + // /// Delegate to a registered secret-resolver callback. + // callback; /// Parse from its JSON (PascalCase) name. factory VeilidConfigProtectedStoreBackend.fromJson(dynamic j) => @@ -566,9 +566,8 @@ sealed class VeilidConfigCapabilities with _$VeilidConfigCapabilities { /// Internal "footgun" UDP tuning. See [VeilidConfigInternal]. @freezed sealed class VeilidConfigInternalUDP with _$VeilidConfigInternalUDP { - const factory VeilidConfigInternalUDP({ - required int socketPoolSize, - }) = _VeilidConfigInternalUDP; + const factory VeilidConfigInternalUDP({required int socketPoolSize}) = + _VeilidConfigInternalUDP; factory VeilidConfigInternalUDP.fromJson(dynamic json) => _$VeilidConfigInternalUDPFromJson(json as Map); @@ -640,6 +639,7 @@ sealed class VeilidConfigInternalDHT with _$VeilidConfigInternalDHT { sealed class VeilidConfigInternalNetwork with _$VeilidConfigInternalNetwork { const factory VeilidConfigInternalNetwork({ required int connectionInitialTimeoutMs, + required int singleSendTimeoutMs, required int connectionInactivityTimeoutMs, required int maxConnectionsPerIp4, required int maxConnectionsPerIp6Prefix, diff --git a/veilid-flutter/lib/veilid_config.freezed.dart b/veilid-flutter/lib/veilid_config.freezed.dart index 0bb49c08..5cfed1cb 100644 --- a/veilid-flutter/lib/veilid_config.freezed.dart +++ b/veilid-flutter/lib/veilid_config.freezed.dart @@ -9119,7 +9119,7 @@ as int, /// @nodoc mixin _$VeilidConfigInternalNetwork implements DiagnosticableTreeMixin { - int get connectionInitialTimeoutMs; int get connectionInactivityTimeoutMs; int get maxConnectionsPerIp4; int get maxConnectionsPerIp6Prefix; int get maxConnectionsPerIp6PrefixSize; int get maxConnectionFrequencyPerMin; int get clientAllowlistTimeoutMs; int get reverseConnectionReceiptTimeMs; int get holePunchReceiptTimeMs; int get restrictedNatRetries; VeilidConfigInternalRPC get rpc; VeilidConfigInternalDHT get dht; VeilidConfigInternalProtocol get protocol; + int get connectionInitialTimeoutMs; int get singleSendTimeoutMs; int get connectionInactivityTimeoutMs; int get maxConnectionsPerIp4; int get maxConnectionsPerIp6Prefix; int get maxConnectionsPerIp6PrefixSize; int get maxConnectionFrequencyPerMin; int get clientAllowlistTimeoutMs; int get reverseConnectionReceiptTimeMs; int get holePunchReceiptTimeMs; int get restrictedNatRetries; VeilidConfigInternalRPC get rpc; VeilidConfigInternalDHT get dht; VeilidConfigInternalProtocol get protocol; /// Create a copy of VeilidConfigInternalNetwork /// with the given fields replaced by the non-null parameter values. @JsonKey(includeFromJson: false, includeToJson: false) @@ -9133,21 +9133,21 @@ $VeilidConfigInternalNetworkCopyWith get copyWith = void debugFillProperties(DiagnosticPropertiesBuilder properties) { properties ..add(DiagnosticsProperty('type', 'VeilidConfigInternalNetwork')) - ..add(DiagnosticsProperty('connectionInitialTimeoutMs', connectionInitialTimeoutMs))..add(DiagnosticsProperty('connectionInactivityTimeoutMs', connectionInactivityTimeoutMs))..add(DiagnosticsProperty('maxConnectionsPerIp4', maxConnectionsPerIp4))..add(DiagnosticsProperty('maxConnectionsPerIp6Prefix', maxConnectionsPerIp6Prefix))..add(DiagnosticsProperty('maxConnectionsPerIp6PrefixSize', maxConnectionsPerIp6PrefixSize))..add(DiagnosticsProperty('maxConnectionFrequencyPerMin', maxConnectionFrequencyPerMin))..add(DiagnosticsProperty('clientAllowlistTimeoutMs', clientAllowlistTimeoutMs))..add(DiagnosticsProperty('reverseConnectionReceiptTimeMs', reverseConnectionReceiptTimeMs))..add(DiagnosticsProperty('holePunchReceiptTimeMs', holePunchReceiptTimeMs))..add(DiagnosticsProperty('restrictedNatRetries', restrictedNatRetries))..add(DiagnosticsProperty('rpc', rpc))..add(DiagnosticsProperty('dht', dht))..add(DiagnosticsProperty('protocol', protocol)); + ..add(DiagnosticsProperty('connectionInitialTimeoutMs', connectionInitialTimeoutMs))..add(DiagnosticsProperty('singleSendTimeoutMs', singleSendTimeoutMs))..add(DiagnosticsProperty('connectionInactivityTimeoutMs', connectionInactivityTimeoutMs))..add(DiagnosticsProperty('maxConnectionsPerIp4', maxConnectionsPerIp4))..add(DiagnosticsProperty('maxConnectionsPerIp6Prefix', maxConnectionsPerIp6Prefix))..add(DiagnosticsProperty('maxConnectionsPerIp6PrefixSize', maxConnectionsPerIp6PrefixSize))..add(DiagnosticsProperty('maxConnectionFrequencyPerMin', maxConnectionFrequencyPerMin))..add(DiagnosticsProperty('clientAllowlistTimeoutMs', clientAllowlistTimeoutMs))..add(DiagnosticsProperty('reverseConnectionReceiptTimeMs', reverseConnectionReceiptTimeMs))..add(DiagnosticsProperty('holePunchReceiptTimeMs', holePunchReceiptTimeMs))..add(DiagnosticsProperty('restrictedNatRetries', restrictedNatRetries))..add(DiagnosticsProperty('rpc', rpc))..add(DiagnosticsProperty('dht', dht))..add(DiagnosticsProperty('protocol', protocol)); } @override bool operator ==(Object other) { - return identical(this, other) || (other.runtimeType == runtimeType&&other is VeilidConfigInternalNetwork&&(identical(other.connectionInitialTimeoutMs, connectionInitialTimeoutMs) || other.connectionInitialTimeoutMs == connectionInitialTimeoutMs)&&(identical(other.connectionInactivityTimeoutMs, connectionInactivityTimeoutMs) || other.connectionInactivityTimeoutMs == connectionInactivityTimeoutMs)&&(identical(other.maxConnectionsPerIp4, maxConnectionsPerIp4) || other.maxConnectionsPerIp4 == maxConnectionsPerIp4)&&(identical(other.maxConnectionsPerIp6Prefix, maxConnectionsPerIp6Prefix) || other.maxConnectionsPerIp6Prefix == maxConnectionsPerIp6Prefix)&&(identical(other.maxConnectionsPerIp6PrefixSize, maxConnectionsPerIp6PrefixSize) || other.maxConnectionsPerIp6PrefixSize == maxConnectionsPerIp6PrefixSize)&&(identical(other.maxConnectionFrequencyPerMin, maxConnectionFrequencyPerMin) || other.maxConnectionFrequencyPerMin == maxConnectionFrequencyPerMin)&&(identical(other.clientAllowlistTimeoutMs, clientAllowlistTimeoutMs) || other.clientAllowlistTimeoutMs == clientAllowlistTimeoutMs)&&(identical(other.reverseConnectionReceiptTimeMs, reverseConnectionReceiptTimeMs) || other.reverseConnectionReceiptTimeMs == reverseConnectionReceiptTimeMs)&&(identical(other.holePunchReceiptTimeMs, holePunchReceiptTimeMs) || other.holePunchReceiptTimeMs == holePunchReceiptTimeMs)&&(identical(other.restrictedNatRetries, restrictedNatRetries) || other.restrictedNatRetries == restrictedNatRetries)&&(identical(other.rpc, rpc) || other.rpc == rpc)&&(identical(other.dht, dht) || other.dht == dht)&&(identical(other.protocol, protocol) || other.protocol == protocol)); + return identical(this, other) || (other.runtimeType == runtimeType&&other is VeilidConfigInternalNetwork&&(identical(other.connectionInitialTimeoutMs, connectionInitialTimeoutMs) || other.connectionInitialTimeoutMs == connectionInitialTimeoutMs)&&(identical(other.singleSendTimeoutMs, singleSendTimeoutMs) || other.singleSendTimeoutMs == singleSendTimeoutMs)&&(identical(other.connectionInactivityTimeoutMs, connectionInactivityTimeoutMs) || other.connectionInactivityTimeoutMs == connectionInactivityTimeoutMs)&&(identical(other.maxConnectionsPerIp4, maxConnectionsPerIp4) || other.maxConnectionsPerIp4 == maxConnectionsPerIp4)&&(identical(other.maxConnectionsPerIp6Prefix, maxConnectionsPerIp6Prefix) || other.maxConnectionsPerIp6Prefix == maxConnectionsPerIp6Prefix)&&(identical(other.maxConnectionsPerIp6PrefixSize, maxConnectionsPerIp6PrefixSize) || other.maxConnectionsPerIp6PrefixSize == maxConnectionsPerIp6PrefixSize)&&(identical(other.maxConnectionFrequencyPerMin, maxConnectionFrequencyPerMin) || other.maxConnectionFrequencyPerMin == maxConnectionFrequencyPerMin)&&(identical(other.clientAllowlistTimeoutMs, clientAllowlistTimeoutMs) || other.clientAllowlistTimeoutMs == clientAllowlistTimeoutMs)&&(identical(other.reverseConnectionReceiptTimeMs, reverseConnectionReceiptTimeMs) || other.reverseConnectionReceiptTimeMs == reverseConnectionReceiptTimeMs)&&(identical(other.holePunchReceiptTimeMs, holePunchReceiptTimeMs) || other.holePunchReceiptTimeMs == holePunchReceiptTimeMs)&&(identical(other.restrictedNatRetries, restrictedNatRetries) || other.restrictedNatRetries == restrictedNatRetries)&&(identical(other.rpc, rpc) || other.rpc == rpc)&&(identical(other.dht, dht) || other.dht == dht)&&(identical(other.protocol, protocol) || other.protocol == protocol)); } @JsonKey(includeFromJson: false, includeToJson: false) @override -int get hashCode => Object.hash(runtimeType,connectionInitialTimeoutMs,connectionInactivityTimeoutMs,maxConnectionsPerIp4,maxConnectionsPerIp6Prefix,maxConnectionsPerIp6PrefixSize,maxConnectionFrequencyPerMin,clientAllowlistTimeoutMs,reverseConnectionReceiptTimeMs,holePunchReceiptTimeMs,restrictedNatRetries,rpc,dht,protocol); +int get hashCode => Object.hash(runtimeType,connectionInitialTimeoutMs,singleSendTimeoutMs,connectionInactivityTimeoutMs,maxConnectionsPerIp4,maxConnectionsPerIp6Prefix,maxConnectionsPerIp6PrefixSize,maxConnectionFrequencyPerMin,clientAllowlistTimeoutMs,reverseConnectionReceiptTimeMs,holePunchReceiptTimeMs,restrictedNatRetries,rpc,dht,protocol); @override String toString({ DiagnosticLevel minLevel = DiagnosticLevel.info }) { - return 'VeilidConfigInternalNetwork(connectionInitialTimeoutMs: $connectionInitialTimeoutMs, connectionInactivityTimeoutMs: $connectionInactivityTimeoutMs, maxConnectionsPerIp4: $maxConnectionsPerIp4, maxConnectionsPerIp6Prefix: $maxConnectionsPerIp6Prefix, maxConnectionsPerIp6PrefixSize: $maxConnectionsPerIp6PrefixSize, maxConnectionFrequencyPerMin: $maxConnectionFrequencyPerMin, clientAllowlistTimeoutMs: $clientAllowlistTimeoutMs, reverseConnectionReceiptTimeMs: $reverseConnectionReceiptTimeMs, holePunchReceiptTimeMs: $holePunchReceiptTimeMs, restrictedNatRetries: $restrictedNatRetries, rpc: $rpc, dht: $dht, protocol: $protocol)'; + return 'VeilidConfigInternalNetwork(connectionInitialTimeoutMs: $connectionInitialTimeoutMs, singleSendTimeoutMs: $singleSendTimeoutMs, connectionInactivityTimeoutMs: $connectionInactivityTimeoutMs, maxConnectionsPerIp4: $maxConnectionsPerIp4, maxConnectionsPerIp6Prefix: $maxConnectionsPerIp6Prefix, maxConnectionsPerIp6PrefixSize: $maxConnectionsPerIp6PrefixSize, maxConnectionFrequencyPerMin: $maxConnectionFrequencyPerMin, clientAllowlistTimeoutMs: $clientAllowlistTimeoutMs, reverseConnectionReceiptTimeMs: $reverseConnectionReceiptTimeMs, holePunchReceiptTimeMs: $holePunchReceiptTimeMs, restrictedNatRetries: $restrictedNatRetries, rpc: $rpc, dht: $dht, protocol: $protocol)'; } @@ -9158,7 +9158,7 @@ abstract mixin class $VeilidConfigInternalNetworkCopyWith<$Res> { factory $VeilidConfigInternalNetworkCopyWith(VeilidConfigInternalNetwork value, $Res Function(VeilidConfigInternalNetwork) _then) = _$VeilidConfigInternalNetworkCopyWithImpl; @useResult $Res call({ - int connectionInitialTimeoutMs, int connectionInactivityTimeoutMs, int maxConnectionsPerIp4, int maxConnectionsPerIp6Prefix, int maxConnectionsPerIp6PrefixSize, int maxConnectionFrequencyPerMin, int clientAllowlistTimeoutMs, int reverseConnectionReceiptTimeMs, int holePunchReceiptTimeMs, int restrictedNatRetries, VeilidConfigInternalRPC rpc, VeilidConfigInternalDHT dht, VeilidConfigInternalProtocol protocol + int connectionInitialTimeoutMs, int singleSendTimeoutMs, int connectionInactivityTimeoutMs, int maxConnectionsPerIp4, int maxConnectionsPerIp6Prefix, int maxConnectionsPerIp6PrefixSize, int maxConnectionFrequencyPerMin, int clientAllowlistTimeoutMs, int reverseConnectionReceiptTimeMs, int holePunchReceiptTimeMs, int restrictedNatRetries, VeilidConfigInternalRPC rpc, VeilidConfigInternalDHT dht, VeilidConfigInternalProtocol protocol }); @@ -9175,9 +9175,10 @@ class _$VeilidConfigInternalNetworkCopyWithImpl<$Res> /// Create a copy of VeilidConfigInternalNetwork /// with the given fields replaced by the non-null parameter values. -@pragma('vm:prefer-inline') @override $Res call({Object? connectionInitialTimeoutMs = null,Object? connectionInactivityTimeoutMs = null,Object? maxConnectionsPerIp4 = null,Object? maxConnectionsPerIp6Prefix = null,Object? maxConnectionsPerIp6PrefixSize = null,Object? maxConnectionFrequencyPerMin = null,Object? clientAllowlistTimeoutMs = null,Object? reverseConnectionReceiptTimeMs = null,Object? holePunchReceiptTimeMs = null,Object? restrictedNatRetries = null,Object? rpc = null,Object? dht = null,Object? protocol = null,}) { +@pragma('vm:prefer-inline') @override $Res call({Object? connectionInitialTimeoutMs = null,Object? singleSendTimeoutMs = null,Object? connectionInactivityTimeoutMs = null,Object? maxConnectionsPerIp4 = null,Object? maxConnectionsPerIp6Prefix = null,Object? maxConnectionsPerIp6PrefixSize = null,Object? maxConnectionFrequencyPerMin = null,Object? clientAllowlistTimeoutMs = null,Object? reverseConnectionReceiptTimeMs = null,Object? holePunchReceiptTimeMs = null,Object? restrictedNatRetries = null,Object? rpc = null,Object? dht = null,Object? protocol = null,}) { return _then(_self.copyWith( connectionInitialTimeoutMs: null == connectionInitialTimeoutMs ? _self.connectionInitialTimeoutMs : connectionInitialTimeoutMs // ignore: cast_nullable_to_non_nullable +as int,singleSendTimeoutMs: null == singleSendTimeoutMs ? _self.singleSendTimeoutMs : singleSendTimeoutMs // ignore: cast_nullable_to_non_nullable as int,connectionInactivityTimeoutMs: null == connectionInactivityTimeoutMs ? _self.connectionInactivityTimeoutMs : connectionInactivityTimeoutMs // ignore: cast_nullable_to_non_nullable as int,maxConnectionsPerIp4: null == maxConnectionsPerIp4 ? _self.maxConnectionsPerIp4 : maxConnectionsPerIp4 // ignore: cast_nullable_to_non_nullable as int,maxConnectionsPerIp6Prefix: null == maxConnectionsPerIp6Prefix ? _self.maxConnectionsPerIp6Prefix : maxConnectionsPerIp6Prefix // ignore: cast_nullable_to_non_nullable @@ -9299,10 +9300,10 @@ return $default(_that);case _: /// } /// ``` -@optionalTypeArgs TResult maybeWhen(TResult Function( int connectionInitialTimeoutMs, int connectionInactivityTimeoutMs, int maxConnectionsPerIp4, int maxConnectionsPerIp6Prefix, int maxConnectionsPerIp6PrefixSize, int maxConnectionFrequencyPerMin, int clientAllowlistTimeoutMs, int reverseConnectionReceiptTimeMs, int holePunchReceiptTimeMs, int restrictedNatRetries, VeilidConfigInternalRPC rpc, VeilidConfigInternalDHT dht, VeilidConfigInternalProtocol protocol)? $default,{required TResult orElse(),}) {final _that = this; +@optionalTypeArgs TResult maybeWhen(TResult Function( int connectionInitialTimeoutMs, int singleSendTimeoutMs, int connectionInactivityTimeoutMs, int maxConnectionsPerIp4, int maxConnectionsPerIp6Prefix, int maxConnectionsPerIp6PrefixSize, int maxConnectionFrequencyPerMin, int clientAllowlistTimeoutMs, int reverseConnectionReceiptTimeMs, int holePunchReceiptTimeMs, int restrictedNatRetries, VeilidConfigInternalRPC rpc, VeilidConfigInternalDHT dht, VeilidConfigInternalProtocol protocol)? $default,{required TResult orElse(),}) {final _that = this; switch (_that) { case _VeilidConfigInternalNetwork() when $default != null: -return $default(_that.connectionInitialTimeoutMs,_that.connectionInactivityTimeoutMs,_that.maxConnectionsPerIp4,_that.maxConnectionsPerIp6Prefix,_that.maxConnectionsPerIp6PrefixSize,_that.maxConnectionFrequencyPerMin,_that.clientAllowlistTimeoutMs,_that.reverseConnectionReceiptTimeMs,_that.holePunchReceiptTimeMs,_that.restrictedNatRetries,_that.rpc,_that.dht,_that.protocol);case _: +return $default(_that.connectionInitialTimeoutMs,_that.singleSendTimeoutMs,_that.connectionInactivityTimeoutMs,_that.maxConnectionsPerIp4,_that.maxConnectionsPerIp6Prefix,_that.maxConnectionsPerIp6PrefixSize,_that.maxConnectionFrequencyPerMin,_that.clientAllowlistTimeoutMs,_that.reverseConnectionReceiptTimeMs,_that.holePunchReceiptTimeMs,_that.restrictedNatRetries,_that.rpc,_that.dht,_that.protocol);case _: return orElse(); } @@ -9320,10 +9321,10 @@ return $default(_that.connectionInitialTimeoutMs,_that.connectionInactivityTimeo /// } /// ``` -@optionalTypeArgs TResult when(TResult Function( int connectionInitialTimeoutMs, int connectionInactivityTimeoutMs, int maxConnectionsPerIp4, int maxConnectionsPerIp6Prefix, int maxConnectionsPerIp6PrefixSize, int maxConnectionFrequencyPerMin, int clientAllowlistTimeoutMs, int reverseConnectionReceiptTimeMs, int holePunchReceiptTimeMs, int restrictedNatRetries, VeilidConfigInternalRPC rpc, VeilidConfigInternalDHT dht, VeilidConfigInternalProtocol protocol) $default,) {final _that = this; +@optionalTypeArgs TResult when(TResult Function( int connectionInitialTimeoutMs, int singleSendTimeoutMs, int connectionInactivityTimeoutMs, int maxConnectionsPerIp4, int maxConnectionsPerIp6Prefix, int maxConnectionsPerIp6PrefixSize, int maxConnectionFrequencyPerMin, int clientAllowlistTimeoutMs, int reverseConnectionReceiptTimeMs, int holePunchReceiptTimeMs, int restrictedNatRetries, VeilidConfigInternalRPC rpc, VeilidConfigInternalDHT dht, VeilidConfigInternalProtocol protocol) $default,) {final _that = this; switch (_that) { case _VeilidConfigInternalNetwork(): -return $default(_that.connectionInitialTimeoutMs,_that.connectionInactivityTimeoutMs,_that.maxConnectionsPerIp4,_that.maxConnectionsPerIp6Prefix,_that.maxConnectionsPerIp6PrefixSize,_that.maxConnectionFrequencyPerMin,_that.clientAllowlistTimeoutMs,_that.reverseConnectionReceiptTimeMs,_that.holePunchReceiptTimeMs,_that.restrictedNatRetries,_that.rpc,_that.dht,_that.protocol);} +return $default(_that.connectionInitialTimeoutMs,_that.singleSendTimeoutMs,_that.connectionInactivityTimeoutMs,_that.maxConnectionsPerIp4,_that.maxConnectionsPerIp6Prefix,_that.maxConnectionsPerIp6PrefixSize,_that.maxConnectionFrequencyPerMin,_that.clientAllowlistTimeoutMs,_that.reverseConnectionReceiptTimeMs,_that.holePunchReceiptTimeMs,_that.restrictedNatRetries,_that.rpc,_that.dht,_that.protocol);} } /// A variant of `when` that fallback to returning `null` /// @@ -9337,10 +9338,10 @@ return $default(_that.connectionInitialTimeoutMs,_that.connectionInactivityTimeo /// } /// ``` -@optionalTypeArgs TResult? whenOrNull(TResult? Function( int connectionInitialTimeoutMs, int connectionInactivityTimeoutMs, int maxConnectionsPerIp4, int maxConnectionsPerIp6Prefix, int maxConnectionsPerIp6PrefixSize, int maxConnectionFrequencyPerMin, int clientAllowlistTimeoutMs, int reverseConnectionReceiptTimeMs, int holePunchReceiptTimeMs, int restrictedNatRetries, VeilidConfigInternalRPC rpc, VeilidConfigInternalDHT dht, VeilidConfigInternalProtocol protocol)? $default,) {final _that = this; +@optionalTypeArgs TResult? whenOrNull(TResult? Function( int connectionInitialTimeoutMs, int singleSendTimeoutMs, int connectionInactivityTimeoutMs, int maxConnectionsPerIp4, int maxConnectionsPerIp6Prefix, int maxConnectionsPerIp6PrefixSize, int maxConnectionFrequencyPerMin, int clientAllowlistTimeoutMs, int reverseConnectionReceiptTimeMs, int holePunchReceiptTimeMs, int restrictedNatRetries, VeilidConfigInternalRPC rpc, VeilidConfigInternalDHT dht, VeilidConfigInternalProtocol protocol)? $default,) {final _that = this; switch (_that) { case _VeilidConfigInternalNetwork() when $default != null: -return $default(_that.connectionInitialTimeoutMs,_that.connectionInactivityTimeoutMs,_that.maxConnectionsPerIp4,_that.maxConnectionsPerIp6Prefix,_that.maxConnectionsPerIp6PrefixSize,_that.maxConnectionFrequencyPerMin,_that.clientAllowlistTimeoutMs,_that.reverseConnectionReceiptTimeMs,_that.holePunchReceiptTimeMs,_that.restrictedNatRetries,_that.rpc,_that.dht,_that.protocol);case _: +return $default(_that.connectionInitialTimeoutMs,_that.singleSendTimeoutMs,_that.connectionInactivityTimeoutMs,_that.maxConnectionsPerIp4,_that.maxConnectionsPerIp6Prefix,_that.maxConnectionsPerIp6PrefixSize,_that.maxConnectionFrequencyPerMin,_that.clientAllowlistTimeoutMs,_that.reverseConnectionReceiptTimeMs,_that.holePunchReceiptTimeMs,_that.restrictedNatRetries,_that.rpc,_that.dht,_that.protocol);case _: return null; } @@ -9352,10 +9353,11 @@ return $default(_that.connectionInitialTimeoutMs,_that.connectionInactivityTimeo @JsonSerializable() class _VeilidConfigInternalNetwork with DiagnosticableTreeMixin implements VeilidConfigInternalNetwork { - const _VeilidConfigInternalNetwork({required this.connectionInitialTimeoutMs, required this.connectionInactivityTimeoutMs, required this.maxConnectionsPerIp4, required this.maxConnectionsPerIp6Prefix, required this.maxConnectionsPerIp6PrefixSize, required this.maxConnectionFrequencyPerMin, required this.clientAllowlistTimeoutMs, required this.reverseConnectionReceiptTimeMs, required this.holePunchReceiptTimeMs, required this.restrictedNatRetries, required this.rpc, required this.dht, required this.protocol}); + const _VeilidConfigInternalNetwork({required this.connectionInitialTimeoutMs, required this.singleSendTimeoutMs, required this.connectionInactivityTimeoutMs, required this.maxConnectionsPerIp4, required this.maxConnectionsPerIp6Prefix, required this.maxConnectionsPerIp6PrefixSize, required this.maxConnectionFrequencyPerMin, required this.clientAllowlistTimeoutMs, required this.reverseConnectionReceiptTimeMs, required this.holePunchReceiptTimeMs, required this.restrictedNatRetries, required this.rpc, required this.dht, required this.protocol}); factory _VeilidConfigInternalNetwork.fromJson(Map json) => _$VeilidConfigInternalNetworkFromJson(json); @override final int connectionInitialTimeoutMs; +@override final int singleSendTimeoutMs; @override final int connectionInactivityTimeoutMs; @override final int maxConnectionsPerIp4; @override final int maxConnectionsPerIp6Prefix; @@ -9383,21 +9385,21 @@ Map toJson() { void debugFillProperties(DiagnosticPropertiesBuilder properties) { properties ..add(DiagnosticsProperty('type', 'VeilidConfigInternalNetwork')) - ..add(DiagnosticsProperty('connectionInitialTimeoutMs', connectionInitialTimeoutMs))..add(DiagnosticsProperty('connectionInactivityTimeoutMs', connectionInactivityTimeoutMs))..add(DiagnosticsProperty('maxConnectionsPerIp4', maxConnectionsPerIp4))..add(DiagnosticsProperty('maxConnectionsPerIp6Prefix', maxConnectionsPerIp6Prefix))..add(DiagnosticsProperty('maxConnectionsPerIp6PrefixSize', maxConnectionsPerIp6PrefixSize))..add(DiagnosticsProperty('maxConnectionFrequencyPerMin', maxConnectionFrequencyPerMin))..add(DiagnosticsProperty('clientAllowlistTimeoutMs', clientAllowlistTimeoutMs))..add(DiagnosticsProperty('reverseConnectionReceiptTimeMs', reverseConnectionReceiptTimeMs))..add(DiagnosticsProperty('holePunchReceiptTimeMs', holePunchReceiptTimeMs))..add(DiagnosticsProperty('restrictedNatRetries', restrictedNatRetries))..add(DiagnosticsProperty('rpc', rpc))..add(DiagnosticsProperty('dht', dht))..add(DiagnosticsProperty('protocol', protocol)); + ..add(DiagnosticsProperty('connectionInitialTimeoutMs', connectionInitialTimeoutMs))..add(DiagnosticsProperty('singleSendTimeoutMs', singleSendTimeoutMs))..add(DiagnosticsProperty('connectionInactivityTimeoutMs', connectionInactivityTimeoutMs))..add(DiagnosticsProperty('maxConnectionsPerIp4', maxConnectionsPerIp4))..add(DiagnosticsProperty('maxConnectionsPerIp6Prefix', maxConnectionsPerIp6Prefix))..add(DiagnosticsProperty('maxConnectionsPerIp6PrefixSize', maxConnectionsPerIp6PrefixSize))..add(DiagnosticsProperty('maxConnectionFrequencyPerMin', maxConnectionFrequencyPerMin))..add(DiagnosticsProperty('clientAllowlistTimeoutMs', clientAllowlistTimeoutMs))..add(DiagnosticsProperty('reverseConnectionReceiptTimeMs', reverseConnectionReceiptTimeMs))..add(DiagnosticsProperty('holePunchReceiptTimeMs', holePunchReceiptTimeMs))..add(DiagnosticsProperty('restrictedNatRetries', restrictedNatRetries))..add(DiagnosticsProperty('rpc', rpc))..add(DiagnosticsProperty('dht', dht))..add(DiagnosticsProperty('protocol', protocol)); } @override bool operator ==(Object other) { - return identical(this, other) || (other.runtimeType == runtimeType&&other is _VeilidConfigInternalNetwork&&(identical(other.connectionInitialTimeoutMs, connectionInitialTimeoutMs) || other.connectionInitialTimeoutMs == connectionInitialTimeoutMs)&&(identical(other.connectionInactivityTimeoutMs, connectionInactivityTimeoutMs) || other.connectionInactivityTimeoutMs == connectionInactivityTimeoutMs)&&(identical(other.maxConnectionsPerIp4, maxConnectionsPerIp4) || other.maxConnectionsPerIp4 == maxConnectionsPerIp4)&&(identical(other.maxConnectionsPerIp6Prefix, maxConnectionsPerIp6Prefix) || other.maxConnectionsPerIp6Prefix == maxConnectionsPerIp6Prefix)&&(identical(other.maxConnectionsPerIp6PrefixSize, maxConnectionsPerIp6PrefixSize) || other.maxConnectionsPerIp6PrefixSize == maxConnectionsPerIp6PrefixSize)&&(identical(other.maxConnectionFrequencyPerMin, maxConnectionFrequencyPerMin) || other.maxConnectionFrequencyPerMin == maxConnectionFrequencyPerMin)&&(identical(other.clientAllowlistTimeoutMs, clientAllowlistTimeoutMs) || other.clientAllowlistTimeoutMs == clientAllowlistTimeoutMs)&&(identical(other.reverseConnectionReceiptTimeMs, reverseConnectionReceiptTimeMs) || other.reverseConnectionReceiptTimeMs == reverseConnectionReceiptTimeMs)&&(identical(other.holePunchReceiptTimeMs, holePunchReceiptTimeMs) || other.holePunchReceiptTimeMs == holePunchReceiptTimeMs)&&(identical(other.restrictedNatRetries, restrictedNatRetries) || other.restrictedNatRetries == restrictedNatRetries)&&(identical(other.rpc, rpc) || other.rpc == rpc)&&(identical(other.dht, dht) || other.dht == dht)&&(identical(other.protocol, protocol) || other.protocol == protocol)); + return identical(this, other) || (other.runtimeType == runtimeType&&other is _VeilidConfigInternalNetwork&&(identical(other.connectionInitialTimeoutMs, connectionInitialTimeoutMs) || other.connectionInitialTimeoutMs == connectionInitialTimeoutMs)&&(identical(other.singleSendTimeoutMs, singleSendTimeoutMs) || other.singleSendTimeoutMs == singleSendTimeoutMs)&&(identical(other.connectionInactivityTimeoutMs, connectionInactivityTimeoutMs) || other.connectionInactivityTimeoutMs == connectionInactivityTimeoutMs)&&(identical(other.maxConnectionsPerIp4, maxConnectionsPerIp4) || other.maxConnectionsPerIp4 == maxConnectionsPerIp4)&&(identical(other.maxConnectionsPerIp6Prefix, maxConnectionsPerIp6Prefix) || other.maxConnectionsPerIp6Prefix == maxConnectionsPerIp6Prefix)&&(identical(other.maxConnectionsPerIp6PrefixSize, maxConnectionsPerIp6PrefixSize) || other.maxConnectionsPerIp6PrefixSize == maxConnectionsPerIp6PrefixSize)&&(identical(other.maxConnectionFrequencyPerMin, maxConnectionFrequencyPerMin) || other.maxConnectionFrequencyPerMin == maxConnectionFrequencyPerMin)&&(identical(other.clientAllowlistTimeoutMs, clientAllowlistTimeoutMs) || other.clientAllowlistTimeoutMs == clientAllowlistTimeoutMs)&&(identical(other.reverseConnectionReceiptTimeMs, reverseConnectionReceiptTimeMs) || other.reverseConnectionReceiptTimeMs == reverseConnectionReceiptTimeMs)&&(identical(other.holePunchReceiptTimeMs, holePunchReceiptTimeMs) || other.holePunchReceiptTimeMs == holePunchReceiptTimeMs)&&(identical(other.restrictedNatRetries, restrictedNatRetries) || other.restrictedNatRetries == restrictedNatRetries)&&(identical(other.rpc, rpc) || other.rpc == rpc)&&(identical(other.dht, dht) || other.dht == dht)&&(identical(other.protocol, protocol) || other.protocol == protocol)); } @JsonKey(includeFromJson: false, includeToJson: false) @override -int get hashCode => Object.hash(runtimeType,connectionInitialTimeoutMs,connectionInactivityTimeoutMs,maxConnectionsPerIp4,maxConnectionsPerIp6Prefix,maxConnectionsPerIp6PrefixSize,maxConnectionFrequencyPerMin,clientAllowlistTimeoutMs,reverseConnectionReceiptTimeMs,holePunchReceiptTimeMs,restrictedNatRetries,rpc,dht,protocol); +int get hashCode => Object.hash(runtimeType,connectionInitialTimeoutMs,singleSendTimeoutMs,connectionInactivityTimeoutMs,maxConnectionsPerIp4,maxConnectionsPerIp6Prefix,maxConnectionsPerIp6PrefixSize,maxConnectionFrequencyPerMin,clientAllowlistTimeoutMs,reverseConnectionReceiptTimeMs,holePunchReceiptTimeMs,restrictedNatRetries,rpc,dht,protocol); @override String toString({ DiagnosticLevel minLevel = DiagnosticLevel.info }) { - return 'VeilidConfigInternalNetwork(connectionInitialTimeoutMs: $connectionInitialTimeoutMs, connectionInactivityTimeoutMs: $connectionInactivityTimeoutMs, maxConnectionsPerIp4: $maxConnectionsPerIp4, maxConnectionsPerIp6Prefix: $maxConnectionsPerIp6Prefix, maxConnectionsPerIp6PrefixSize: $maxConnectionsPerIp6PrefixSize, maxConnectionFrequencyPerMin: $maxConnectionFrequencyPerMin, clientAllowlistTimeoutMs: $clientAllowlistTimeoutMs, reverseConnectionReceiptTimeMs: $reverseConnectionReceiptTimeMs, holePunchReceiptTimeMs: $holePunchReceiptTimeMs, restrictedNatRetries: $restrictedNatRetries, rpc: $rpc, dht: $dht, protocol: $protocol)'; + return 'VeilidConfigInternalNetwork(connectionInitialTimeoutMs: $connectionInitialTimeoutMs, singleSendTimeoutMs: $singleSendTimeoutMs, connectionInactivityTimeoutMs: $connectionInactivityTimeoutMs, maxConnectionsPerIp4: $maxConnectionsPerIp4, maxConnectionsPerIp6Prefix: $maxConnectionsPerIp6Prefix, maxConnectionsPerIp6PrefixSize: $maxConnectionsPerIp6PrefixSize, maxConnectionFrequencyPerMin: $maxConnectionFrequencyPerMin, clientAllowlistTimeoutMs: $clientAllowlistTimeoutMs, reverseConnectionReceiptTimeMs: $reverseConnectionReceiptTimeMs, holePunchReceiptTimeMs: $holePunchReceiptTimeMs, restrictedNatRetries: $restrictedNatRetries, rpc: $rpc, dht: $dht, protocol: $protocol)'; } @@ -9408,7 +9410,7 @@ abstract mixin class _$VeilidConfigInternalNetworkCopyWith<$Res> implements $Vei factory _$VeilidConfigInternalNetworkCopyWith(_VeilidConfigInternalNetwork value, $Res Function(_VeilidConfigInternalNetwork) _then) = __$VeilidConfigInternalNetworkCopyWithImpl; @override @useResult $Res call({ - int connectionInitialTimeoutMs, int connectionInactivityTimeoutMs, int maxConnectionsPerIp4, int maxConnectionsPerIp6Prefix, int maxConnectionsPerIp6PrefixSize, int maxConnectionFrequencyPerMin, int clientAllowlistTimeoutMs, int reverseConnectionReceiptTimeMs, int holePunchReceiptTimeMs, int restrictedNatRetries, VeilidConfigInternalRPC rpc, VeilidConfigInternalDHT dht, VeilidConfigInternalProtocol protocol + int connectionInitialTimeoutMs, int singleSendTimeoutMs, int connectionInactivityTimeoutMs, int maxConnectionsPerIp4, int maxConnectionsPerIp6Prefix, int maxConnectionsPerIp6PrefixSize, int maxConnectionFrequencyPerMin, int clientAllowlistTimeoutMs, int reverseConnectionReceiptTimeMs, int holePunchReceiptTimeMs, int restrictedNatRetries, VeilidConfigInternalRPC rpc, VeilidConfigInternalDHT dht, VeilidConfigInternalProtocol protocol }); @@ -9425,9 +9427,10 @@ class __$VeilidConfigInternalNetworkCopyWithImpl<$Res> /// Create a copy of VeilidConfigInternalNetwork /// with the given fields replaced by the non-null parameter values. -@override @pragma('vm:prefer-inline') $Res call({Object? connectionInitialTimeoutMs = null,Object? connectionInactivityTimeoutMs = null,Object? maxConnectionsPerIp4 = null,Object? maxConnectionsPerIp6Prefix = null,Object? maxConnectionsPerIp6PrefixSize = null,Object? maxConnectionFrequencyPerMin = null,Object? clientAllowlistTimeoutMs = null,Object? reverseConnectionReceiptTimeMs = null,Object? holePunchReceiptTimeMs = null,Object? restrictedNatRetries = null,Object? rpc = null,Object? dht = null,Object? protocol = null,}) { +@override @pragma('vm:prefer-inline') $Res call({Object? connectionInitialTimeoutMs = null,Object? singleSendTimeoutMs = null,Object? connectionInactivityTimeoutMs = null,Object? maxConnectionsPerIp4 = null,Object? maxConnectionsPerIp6Prefix = null,Object? maxConnectionsPerIp6PrefixSize = null,Object? maxConnectionFrequencyPerMin = null,Object? clientAllowlistTimeoutMs = null,Object? reverseConnectionReceiptTimeMs = null,Object? holePunchReceiptTimeMs = null,Object? restrictedNatRetries = null,Object? rpc = null,Object? dht = null,Object? protocol = null,}) { return _then(_VeilidConfigInternalNetwork( connectionInitialTimeoutMs: null == connectionInitialTimeoutMs ? _self.connectionInitialTimeoutMs : connectionInitialTimeoutMs // ignore: cast_nullable_to_non_nullable +as int,singleSendTimeoutMs: null == singleSendTimeoutMs ? _self.singleSendTimeoutMs : singleSendTimeoutMs // ignore: cast_nullable_to_non_nullable as int,connectionInactivityTimeoutMs: null == connectionInactivityTimeoutMs ? _self.connectionInactivityTimeoutMs : connectionInactivityTimeoutMs // ignore: cast_nullable_to_non_nullable as int,maxConnectionsPerIp4: null == maxConnectionsPerIp4 ? _self.maxConnectionsPerIp4 : maxConnectionsPerIp4 // ignore: cast_nullable_to_non_nullable as int,maxConnectionsPerIp6Prefix: null == maxConnectionsPerIp6Prefix ? _self.maxConnectionsPerIp6Prefix : maxConnectionsPerIp6Prefix // ignore: cast_nullable_to_non_nullable diff --git a/veilid-flutter/lib/veilid_config.g.dart b/veilid-flutter/lib/veilid_config.g.dart index 5cfbe3b5..a69c171c 100644 --- a/veilid-flutter/lib/veilid_config.g.dart +++ b/veilid-flutter/lib/veilid_config.g.dart @@ -637,6 +637,7 @@ _VeilidConfigInternalNetwork _$VeilidConfigInternalNetworkFromJson( ) => _VeilidConfigInternalNetwork( connectionInitialTimeoutMs: (json['connectionInitialTimeoutMs'] as num) .toInt(), + singleSendTimeoutMs: (json['singleSendTimeoutMs'] as num).toInt(), connectionInactivityTimeoutMs: (json['connectionInactivityTimeoutMs'] as num) .toInt(), maxConnectionsPerIp4: (json['maxConnectionsPerIp4'] as num).toInt(), @@ -660,6 +661,7 @@ Map _$VeilidConfigInternalNetworkToJson( _VeilidConfigInternalNetwork instance, ) => { 'connectionInitialTimeoutMs': instance.connectionInitialTimeoutMs, + 'singleSendTimeoutMs': instance.singleSendTimeoutMs, 'connectionInactivityTimeoutMs': instance.connectionInactivityTimeoutMs, 'maxConnectionsPerIp4': instance.maxConnectionsPerIp4, 'maxConnectionsPerIp6Prefix': instance.maxConnectionsPerIp6Prefix, diff --git a/veilid-flutter/lib/veilid_dht_transaction.dart b/veilid-flutter/lib/veilid_dht_transaction.dart index 3c176ff5..b14b27eb 100644 --- a/veilid-flutter/lib/veilid_dht_transaction.dart +++ b/veilid-flutter/lib/veilid_dht_transaction.dart @@ -16,15 +16,17 @@ part 'veilid_dht_transaction.g.dart'; sealed class DHTTransactionSetValueOptions with _$DHTTransactionSetValueOptions { /// Builds set options, optionally overriding the record's default [writer]. - const factory DHTTransactionSetValueOptions({KeyPair? writer}) = - _DHTTransactionSetValueOptions; + /// `minSeqnum` raises the sequence number to at least that value after the + /// automatic increment; [ValueData.maxSeqnum] minus n leaves n writes after + /// this one. + const factory DHTTransactionSetValueOptions({ + KeyPair? writer, + int? minSeqnum, + }) = _DHTTransactionSetValueOptions; /// Decodes options from JSON. factory DHTTransactionSetValueOptions.fromJson(dynamic json) => _$DHTTransactionSetValueOptionsFromJson(json as Map); - - @override - Map toJson() => {'writer': writer}; } ////////////////////////////////////// diff --git a/veilid-flutter/lib/veilid_dht_transaction.freezed.dart b/veilid-flutter/lib/veilid_dht_transaction.freezed.dart index da661983..3a73d520 100644 --- a/veilid-flutter/lib/veilid_dht_transaction.freezed.dart +++ b/veilid-flutter/lib/veilid_dht_transaction.freezed.dart @@ -15,7 +15,7 @@ T _$identity(T value) => value; /// @nodoc mixin _$DHTTransactionSetValueOptions { - KeyPair? get writer; + KeyPair? get writer; int? get minSeqnum; /// Create a copy of DHTTransactionSetValueOptions /// with the given fields replaced by the non-null parameter values. @JsonKey(includeFromJson: false, includeToJson: false) @@ -28,16 +28,16 @@ $DHTTransactionSetValueOptionsCopyWith get copyWi @override bool operator ==(Object other) { - return identical(this, other) || (other.runtimeType == runtimeType&&other is DHTTransactionSetValueOptions&&(identical(other.writer, writer) || other.writer == writer)); + return identical(this, other) || (other.runtimeType == runtimeType&&other is DHTTransactionSetValueOptions&&(identical(other.writer, writer) || other.writer == writer)&&(identical(other.minSeqnum, minSeqnum) || other.minSeqnum == minSeqnum)); } @JsonKey(includeFromJson: false, includeToJson: false) @override -int get hashCode => Object.hash(runtimeType,writer); +int get hashCode => Object.hash(runtimeType,writer,minSeqnum); @override String toString() { - return 'DHTTransactionSetValueOptions(writer: $writer)'; + return 'DHTTransactionSetValueOptions(writer: $writer, minSeqnum: $minSeqnum)'; } @@ -48,7 +48,7 @@ abstract mixin class $DHTTransactionSetValueOptionsCopyWith<$Res> { factory $DHTTransactionSetValueOptionsCopyWith(DHTTransactionSetValueOptions value, $Res Function(DHTTransactionSetValueOptions) _then) = _$DHTTransactionSetValueOptionsCopyWithImpl; @useResult $Res call({ - KeyPair? writer + KeyPair? writer, int? minSeqnum }); @@ -65,10 +65,11 @@ class _$DHTTransactionSetValueOptionsCopyWithImpl<$Res> /// Create a copy of DHTTransactionSetValueOptions /// with the given fields replaced by the non-null parameter values. -@pragma('vm:prefer-inline') @override $Res call({Object? writer = freezed,}) { +@pragma('vm:prefer-inline') @override $Res call({Object? writer = freezed,Object? minSeqnum = freezed,}) { return _then(_self.copyWith( writer: freezed == writer ? _self.writer : writer // ignore: cast_nullable_to_non_nullable -as KeyPair?, +as KeyPair?,minSeqnum: freezed == minSeqnum ? _self.minSeqnum : minSeqnum // ignore: cast_nullable_to_non_nullable +as int?, )); } @@ -150,10 +151,10 @@ return $default(_that);case _: /// } /// ``` -@optionalTypeArgs TResult maybeWhen(TResult Function( KeyPair? writer)? $default,{required TResult orElse(),}) {final _that = this; +@optionalTypeArgs TResult maybeWhen(TResult Function( KeyPair? writer, int? minSeqnum)? $default,{required TResult orElse(),}) {final _that = this; switch (_that) { case _DHTTransactionSetValueOptions() when $default != null: -return $default(_that.writer);case _: +return $default(_that.writer,_that.minSeqnum);case _: return orElse(); } @@ -171,10 +172,10 @@ return $default(_that.writer);case _: /// } /// ``` -@optionalTypeArgs TResult when(TResult Function( KeyPair? writer) $default,) {final _that = this; +@optionalTypeArgs TResult when(TResult Function( KeyPair? writer, int? minSeqnum) $default,) {final _that = this; switch (_that) { case _DHTTransactionSetValueOptions(): -return $default(_that.writer);} +return $default(_that.writer,_that.minSeqnum);} } /// A variant of `when` that fallback to returning `null` /// @@ -188,10 +189,10 @@ return $default(_that.writer);} /// } /// ``` -@optionalTypeArgs TResult? whenOrNull(TResult? Function( KeyPair? writer)? $default,) {final _that = this; +@optionalTypeArgs TResult? whenOrNull(TResult? Function( KeyPair? writer, int? minSeqnum)? $default,) {final _that = this; switch (_that) { case _DHTTransactionSetValueOptions() when $default != null: -return $default(_that.writer);case _: +return $default(_that.writer,_that.minSeqnum);case _: return null; } @@ -203,10 +204,11 @@ return $default(_that.writer);case _: @JsonSerializable() class _DHTTransactionSetValueOptions implements DHTTransactionSetValueOptions { - const _DHTTransactionSetValueOptions({this.writer}); + const _DHTTransactionSetValueOptions({this.writer, this.minSeqnum}); factory _DHTTransactionSetValueOptions.fromJson(Map json) => _$DHTTransactionSetValueOptionsFromJson(json); @override final KeyPair? writer; +@override final int? minSeqnum; /// Create a copy of DHTTransactionSetValueOptions /// with the given fields replaced by the non-null parameter values. @@ -221,16 +223,16 @@ Map toJson() { @override bool operator ==(Object other) { - return identical(this, other) || (other.runtimeType == runtimeType&&other is _DHTTransactionSetValueOptions&&(identical(other.writer, writer) || other.writer == writer)); + return identical(this, other) || (other.runtimeType == runtimeType&&other is _DHTTransactionSetValueOptions&&(identical(other.writer, writer) || other.writer == writer)&&(identical(other.minSeqnum, minSeqnum) || other.minSeqnum == minSeqnum)); } @JsonKey(includeFromJson: false, includeToJson: false) @override -int get hashCode => Object.hash(runtimeType,writer); +int get hashCode => Object.hash(runtimeType,writer,minSeqnum); @override String toString() { - return 'DHTTransactionSetValueOptions(writer: $writer)'; + return 'DHTTransactionSetValueOptions(writer: $writer, minSeqnum: $minSeqnum)'; } @@ -241,7 +243,7 @@ abstract mixin class _$DHTTransactionSetValueOptionsCopyWith<$Res> implements $D factory _$DHTTransactionSetValueOptionsCopyWith(_DHTTransactionSetValueOptions value, $Res Function(_DHTTransactionSetValueOptions) _then) = __$DHTTransactionSetValueOptionsCopyWithImpl; @override @useResult $Res call({ - KeyPair? writer + KeyPair? writer, int? minSeqnum }); @@ -258,10 +260,11 @@ class __$DHTTransactionSetValueOptionsCopyWithImpl<$Res> /// Create a copy of DHTTransactionSetValueOptions /// with the given fields replaced by the non-null parameter values. -@override @pragma('vm:prefer-inline') $Res call({Object? writer = freezed,}) { +@override @pragma('vm:prefer-inline') $Res call({Object? writer = freezed,Object? minSeqnum = freezed,}) { return _then(_DHTTransactionSetValueOptions( writer: freezed == writer ? _self.writer : writer // ignore: cast_nullable_to_non_nullable -as KeyPair?, +as KeyPair?,minSeqnum: freezed == minSeqnum ? _self.minSeqnum : minSeqnum // ignore: cast_nullable_to_non_nullable +as int?, )); } diff --git a/veilid-flutter/lib/veilid_dht_transaction.g.dart b/veilid-flutter/lib/veilid_dht_transaction.g.dart index ba771ce1..f3a43b13 100644 --- a/veilid-flutter/lib/veilid_dht_transaction.g.dart +++ b/veilid-flutter/lib/veilid_dht_transaction.g.dart @@ -10,8 +10,12 @@ _DHTTransactionSetValueOptions _$DHTTransactionSetValueOptionsFromJson( Map json, ) => _DHTTransactionSetValueOptions( writer: json['writer'] == null ? null : KeyPair.fromJson(json['writer']), + minSeqnum: (json['minSeqnum'] as num?)?.toInt(), ); Map _$DHTTransactionSetValueOptionsToJson( _DHTTransactionSetValueOptions instance, -) => {'writer': instance.writer?.toJson()}; +) => { + 'writer': instance.writer?.toJson(), + 'minSeqnum': instance.minSeqnum, +}; diff --git a/veilid-flutter/lib/veilid_ffi.dart b/veilid-flutter/lib/veilid_ffi.dart index a562491e..7b20b8fc 100644 --- a/veilid-flutter/lib/veilid_ffi.dart +++ b/veilid-flutter/lib/veilid_ffi.dart @@ -1,6 +1,6 @@ // ignore_for_file: public_member_api_docs // Low-level platform binding to veilid-core; not part of the documented public API. -// The asset id below must match `assetName` in hook/build.dart. +// @DefaultAsset below names the asset hook/build.dart declares as `assetName`. @DefaultAsset('package:veilid/veilid_ffi.dart') library; @@ -20,7 +20,7 @@ import 'veilid.dart'; typedef _DartPostCObject = NativeFunction)>; -// fn veilid_version() -> VeilidVersion +// Return layout of veilid_version() final class VeilidVersionFFI extends Struct { @Uint32() external int major; @@ -1591,7 +1591,7 @@ class VeilidFFI extends Veilid { ) external static int _changeLogLevel( Pointer layer, - Pointer logLevel, + Pointer directives, ); @Native, Pointer)>( @@ -1642,7 +1642,7 @@ class VeilidFFI extends Veilid { ) external static int _routingContextWithSafety( int id, - Pointer stability, + Pointer safetySelection, ); @Native)>( @@ -1673,7 +1673,7 @@ class VeilidFFI extends Veilid { int port, int id, Pointer target, - Pointer request, + Pointer message, ); @Native, Pointer)>( @@ -1742,7 +1742,7 @@ class VeilidFFI extends Veilid { Pointer key, int subkey, Pointer data, - Pointer writer, + Pointer options, ); @Native< @@ -1806,8 +1806,8 @@ class VeilidFFI extends Veilid { ) external static void _transactDHTRecords( int port, - Pointer records, - Pointer defaultSigningKey, + Pointer recordKeys, + Pointer options, ); @Native(symbol: 'new_private_route') @@ -1827,14 +1827,14 @@ class VeilidFFI extends Veilid { external static void _importRemotePrivateRoute(int port, Pointer blob); @Native)>(symbol: 'release_private_route') - external static void _releasePrivateRoute(int port, Pointer key); + external static void _releasePrivateRoute(int port, Pointer routeId); @Native, Pointer)>( symbol: 'app_call_reply', ) external static void _appCallReply( int port, - Pointer id, + Pointer callId, Pointer message, ); @@ -2319,7 +2319,7 @@ class VeilidFFI extends Veilid { external static int _nowIncreasing(); @Native)>(symbol: 'debug') - external static void _debug(int port, Pointer logLevel); + external static void _debug(int port, Pointer command); @Native, Pointer)>( symbol: 'save_user_secret', diff --git a/veilid-flutter/lib/veilid_js.dart b/veilid-flutter/lib/veilid_js.dart index 3c71e65d..eed10d4b 100644 --- a/veilid-flutter/lib/veilid_js.dart +++ b/veilid-flutter/lib/veilid_js.dart @@ -451,26 +451,24 @@ class VeilidCryptoSystemJS extends VeilidCryptoSystem { EncapsulationKey recipient, Uint8List aad, Uint8List plaintext, - ) => - _wasmPromiseUint8List('crypto_hpke_seal', [ - _kind.toJS, - jsonEncode(recipient).toJS, - base64UrlNoPadEncode(aad).toJS, - base64UrlNoPadEncode(plaintext).toJS, - ]); + ) => _wasmPromiseUint8List('crypto_hpke_seal', [ + _kind.toJS, + jsonEncode(recipient).toJS, + base64UrlNoPadEncode(aad).toJS, + base64UrlNoPadEncode(plaintext).toJS, + ]); @override Future hpkeOpen( DecapsulationKey secret, Uint8List aad, Uint8List sealed, - ) => - _wasmPromiseUint8List('crypto_hpke_open', [ - _kind.toJS, - jsonEncode(secret).toJS, - base64UrlNoPadEncode(aad).toJS, - base64UrlNoPadEncode(sealed).toJS, - ]); + ) => _wasmPromiseUint8List('crypto_hpke_open', [ + _kind.toJS, + jsonEncode(secret).toJS, + base64UrlNoPadEncode(aad).toJS, + base64UrlNoPadEncode(sealed).toJS, + ]); @override Future randomBytes(int len) => @@ -603,21 +601,20 @@ class VeilidCryptoSystemJS extends VeilidCryptoSystem { @override Future encapsulationKeyFromSigningKey(PublicKey key) => - _wasmPromiseJson('crypto_encapsulation_key_from_signing_key', - EncapsulationKey.fromJson, [ - _kind.toJS, - jsonEncode(key).toJS, - ]); + _wasmPromiseJson( + 'crypto_encapsulation_key_from_signing_key', + EncapsulationKey.fromJson, + [_kind.toJS, jsonEncode(key).toJS], + ); @override Future decapsulationKeyFromSigningSecret( SecretKey secret, - ) => - _wasmPromiseJson('crypto_decapsulation_key_from_signing_secret', - DecapsulationKey.fromJson, [ - _kind.toJS, - jsonEncode(secret).toJS, - ]); + ) => _wasmPromiseJson( + 'crypto_decapsulation_key_from_signing_secret', + DecapsulationKey.fromJson, + [_kind.toJS, jsonEncode(secret).toJS], + ); @override Future generateHash(Uint8List data) => _wasmPromiseJson( @@ -1253,4 +1250,4 @@ class VeilidJS extends Veilid { @override String defaultVeilidConfig() => _wasmCallString('default_veilid_config', []); -} \ No newline at end of file +} diff --git a/veilid-flutter/lib/veilid_plugin_stub_web.dart b/veilid-flutter/lib/veilid_plugin_stub_web.dart index bff6592a..f94d0286 100644 --- a/veilid-flutter/lib/veilid_plugin_stub_web.dart +++ b/veilid-flutter/lib/veilid_plugin_stub_web.dart @@ -17,4 +17,4 @@ class VeilidPluginStubWeb { details: "Veilid for Web doesn't implement '${call.method}'", ); } -} \ No newline at end of file +} diff --git a/veilid-flutter/lib/veilid_stub.dart b/veilid-flutter/lib/veilid_stub.dart index 7a0d6815..6122f45d 100644 --- a/veilid-flutter/lib/veilid_stub.dart +++ b/veilid-flutter/lib/veilid_stub.dart @@ -9,4 +9,4 @@ Veilid getVeilid() => throw UnsupportedError('Cannot create Veilid object'); Uint8List convertUint8ListFromJson(dynamic json) => throw UnsupportedError('Cannot convertUint8ListFromJson'); dynamic convertUint8ListToJson(Uint8List data) => - throw UnsupportedError('Cannot convertUint8ListToJson'); \ No newline at end of file + throw UnsupportedError('Cannot convertUint8ListToJson'); diff --git a/veilid-flutter/lib/veilid_types.dart b/veilid-flutter/lib/veilid_types.dart index b5216d59..37ee2de2 100644 --- a/veilid-flutter/lib/veilid_types.dart +++ b/veilid-flutter/lib/veilid_types.dart @@ -226,6 +226,9 @@ sealed class ValueData with _$ValueData { /// Maximum length in bytes of the data a single subkey may hold. static const maxLen = 32768; + /// The largest sequence number. Subtract n for a `minSeqnum` leaving n writes. + static const maxSeqnum = 0xFFFFFFFE; + /// `seq` time-orders changes to the subkey; `writer` is the public identity /// key of the writer. In debug builds, throws [AssertionError] if `seq` is /// outside `0..=4294967295` or `data` exceeds [maxLen] bytes. @@ -517,19 +520,18 @@ enum DHTReportScope { sealed class SetDHTValueOptions with _$SetDHTValueOptions { /// `writer` overrides the writer key pair for the operation. `allowOffline` /// defaults to true; when false, an offline node returns a TryAgain error - /// instead of writing. - const factory SetDHTValueOptions({KeyPair? writer, bool? allowOffline}) = - _SetDHTValueOptions; + /// instead of writing. `minSeqnum` raises the sequence number to at least + /// that value after the automatic increment; [ValueData.maxSeqnum] minus n + /// leaves n writes after this one. + const factory SetDHTValueOptions({ + KeyPair? writer, + bool? allowOffline, + int? minSeqnum, + }) = _SetDHTValueOptions; /// Decode a [SetDHTValueOptions] from its JSON representation. factory SetDHTValueOptions.fromJson(dynamic json) => _$SetDHTValueOptionsFromJson(json as Map); - - @override - Map toJson() => { - 'writer': writer, - 'allow_offline': allowOffline, - }; } ////////////////////////////////////// diff --git a/veilid-flutter/lib/veilid_types.freezed.dart b/veilid-flutter/lib/veilid_types.freezed.dart index 4979a6ed..6bbfa1db 100644 --- a/veilid-flutter/lib/veilid_types.freezed.dart +++ b/veilid-flutter/lib/veilid_types.freezed.dart @@ -2281,7 +2281,7 @@ as List, /// @nodoc mixin _$SetDHTValueOptions { - KeyPair? get writer; bool? get allowOffline; + KeyPair? get writer; bool? get allowOffline; int? get minSeqnum; /// Create a copy of SetDHTValueOptions /// with the given fields replaced by the non-null parameter values. @JsonKey(includeFromJson: false, includeToJson: false) @@ -2294,16 +2294,16 @@ $SetDHTValueOptionsCopyWith get copyWith => _$SetDHTValueOpt @override bool operator ==(Object other) { - return identical(this, other) || (other.runtimeType == runtimeType&&other is SetDHTValueOptions&&(identical(other.writer, writer) || other.writer == writer)&&(identical(other.allowOffline, allowOffline) || other.allowOffline == allowOffline)); + return identical(this, other) || (other.runtimeType == runtimeType&&other is SetDHTValueOptions&&(identical(other.writer, writer) || other.writer == writer)&&(identical(other.allowOffline, allowOffline) || other.allowOffline == allowOffline)&&(identical(other.minSeqnum, minSeqnum) || other.minSeqnum == minSeqnum)); } @JsonKey(includeFromJson: false, includeToJson: false) @override -int get hashCode => Object.hash(runtimeType,writer,allowOffline); +int get hashCode => Object.hash(runtimeType,writer,allowOffline,minSeqnum); @override String toString() { - return 'SetDHTValueOptions(writer: $writer, allowOffline: $allowOffline)'; + return 'SetDHTValueOptions(writer: $writer, allowOffline: $allowOffline, minSeqnum: $minSeqnum)'; } @@ -2314,7 +2314,7 @@ abstract mixin class $SetDHTValueOptionsCopyWith<$Res> { factory $SetDHTValueOptionsCopyWith(SetDHTValueOptions value, $Res Function(SetDHTValueOptions) _then) = _$SetDHTValueOptionsCopyWithImpl; @useResult $Res call({ - KeyPair? writer, bool? allowOffline + KeyPair? writer, bool? allowOffline, int? minSeqnum }); @@ -2331,11 +2331,12 @@ class _$SetDHTValueOptionsCopyWithImpl<$Res> /// Create a copy of SetDHTValueOptions /// with the given fields replaced by the non-null parameter values. -@pragma('vm:prefer-inline') @override $Res call({Object? writer = freezed,Object? allowOffline = freezed,}) { +@pragma('vm:prefer-inline') @override $Res call({Object? writer = freezed,Object? allowOffline = freezed,Object? minSeqnum = freezed,}) { return _then(_self.copyWith( writer: freezed == writer ? _self.writer : writer // ignore: cast_nullable_to_non_nullable as KeyPair?,allowOffline: freezed == allowOffline ? _self.allowOffline : allowOffline // ignore: cast_nullable_to_non_nullable -as bool?, +as bool?,minSeqnum: freezed == minSeqnum ? _self.minSeqnum : minSeqnum // ignore: cast_nullable_to_non_nullable +as int?, )); } @@ -2417,10 +2418,10 @@ return $default(_that);case _: /// } /// ``` -@optionalTypeArgs TResult maybeWhen(TResult Function( KeyPair? writer, bool? allowOffline)? $default,{required TResult orElse(),}) {final _that = this; +@optionalTypeArgs TResult maybeWhen(TResult Function( KeyPair? writer, bool? allowOffline, int? minSeqnum)? $default,{required TResult orElse(),}) {final _that = this; switch (_that) { case _SetDHTValueOptions() when $default != null: -return $default(_that.writer,_that.allowOffline);case _: +return $default(_that.writer,_that.allowOffline,_that.minSeqnum);case _: return orElse(); } @@ -2438,10 +2439,10 @@ return $default(_that.writer,_that.allowOffline);case _: /// } /// ``` -@optionalTypeArgs TResult when(TResult Function( KeyPair? writer, bool? allowOffline) $default,) {final _that = this; +@optionalTypeArgs TResult when(TResult Function( KeyPair? writer, bool? allowOffline, int? minSeqnum) $default,) {final _that = this; switch (_that) { case _SetDHTValueOptions(): -return $default(_that.writer,_that.allowOffline);} +return $default(_that.writer,_that.allowOffline,_that.minSeqnum);} } /// A variant of `when` that fallback to returning `null` /// @@ -2455,10 +2456,10 @@ return $default(_that.writer,_that.allowOffline);} /// } /// ``` -@optionalTypeArgs TResult? whenOrNull(TResult? Function( KeyPair? writer, bool? allowOffline)? $default,) {final _that = this; +@optionalTypeArgs TResult? whenOrNull(TResult? Function( KeyPair? writer, bool? allowOffline, int? minSeqnum)? $default,) {final _that = this; switch (_that) { case _SetDHTValueOptions() when $default != null: -return $default(_that.writer,_that.allowOffline);case _: +return $default(_that.writer,_that.allowOffline,_that.minSeqnum);case _: return null; } @@ -2470,11 +2471,12 @@ return $default(_that.writer,_that.allowOffline);case _: @JsonSerializable() class _SetDHTValueOptions implements SetDHTValueOptions { - const _SetDHTValueOptions({this.writer, this.allowOffline}); + const _SetDHTValueOptions({this.writer, this.allowOffline, this.minSeqnum}); factory _SetDHTValueOptions.fromJson(Map json) => _$SetDHTValueOptionsFromJson(json); @override final KeyPair? writer; @override final bool? allowOffline; +@override final int? minSeqnum; /// Create a copy of SetDHTValueOptions /// with the given fields replaced by the non-null parameter values. @@ -2489,16 +2491,16 @@ Map toJson() { @override bool operator ==(Object other) { - return identical(this, other) || (other.runtimeType == runtimeType&&other is _SetDHTValueOptions&&(identical(other.writer, writer) || other.writer == writer)&&(identical(other.allowOffline, allowOffline) || other.allowOffline == allowOffline)); + return identical(this, other) || (other.runtimeType == runtimeType&&other is _SetDHTValueOptions&&(identical(other.writer, writer) || other.writer == writer)&&(identical(other.allowOffline, allowOffline) || other.allowOffline == allowOffline)&&(identical(other.minSeqnum, minSeqnum) || other.minSeqnum == minSeqnum)); } @JsonKey(includeFromJson: false, includeToJson: false) @override -int get hashCode => Object.hash(runtimeType,writer,allowOffline); +int get hashCode => Object.hash(runtimeType,writer,allowOffline,minSeqnum); @override String toString() { - return 'SetDHTValueOptions(writer: $writer, allowOffline: $allowOffline)'; + return 'SetDHTValueOptions(writer: $writer, allowOffline: $allowOffline, minSeqnum: $minSeqnum)'; } @@ -2509,7 +2511,7 @@ abstract mixin class _$SetDHTValueOptionsCopyWith<$Res> implements $SetDHTValueO factory _$SetDHTValueOptionsCopyWith(_SetDHTValueOptions value, $Res Function(_SetDHTValueOptions) _then) = __$SetDHTValueOptionsCopyWithImpl; @override @useResult $Res call({ - KeyPair? writer, bool? allowOffline + KeyPair? writer, bool? allowOffline, int? minSeqnum }); @@ -2526,11 +2528,12 @@ class __$SetDHTValueOptionsCopyWithImpl<$Res> /// Create a copy of SetDHTValueOptions /// with the given fields replaced by the non-null parameter values. -@override @pragma('vm:prefer-inline') $Res call({Object? writer = freezed,Object? allowOffline = freezed,}) { +@override @pragma('vm:prefer-inline') $Res call({Object? writer = freezed,Object? allowOffline = freezed,Object? minSeqnum = freezed,}) { return _then(_SetDHTValueOptions( writer: freezed == writer ? _self.writer : writer // ignore: cast_nullable_to_non_nullable as KeyPair?,allowOffline: freezed == allowOffline ? _self.allowOffline : allowOffline // ignore: cast_nullable_to_non_nullable -as bool?, +as bool?,minSeqnum: freezed == minSeqnum ? _self.minSeqnum : minSeqnum // ignore: cast_nullable_to_non_nullable +as int?, )); } diff --git a/veilid-flutter/lib/veilid_types.g.dart b/veilid-flutter/lib/veilid_types.g.dart index 1aba3b82..42ad218a 100644 --- a/veilid-flutter/lib/veilid_types.g.dart +++ b/veilid-flutter/lib/veilid_types.g.dart @@ -153,12 +153,14 @@ _SetDHTValueOptions _$SetDHTValueOptionsFromJson(Map json) => _SetDHTValueOptions( writer: json['writer'] == null ? null : KeyPair.fromJson(json['writer']), allowOffline: json['allowOffline'] as bool?, + minSeqnum: (json['minSeqnum'] as num?)?.toInt(), ); Map _$SetDHTValueOptionsToJson(_SetDHTValueOptions instance) => { 'writer': instance.writer?.toJson(), 'allowOffline': instance.allowOffline, + 'minSeqnum': instance.minSeqnum, }; _TransactDHTRecordsOptions _$TransactDHTRecordsOptionsFromJson( diff --git a/veilid-flutter/packages/veilid_integration_test/lib/src/integration_log.dart b/veilid-flutter/packages/veilid_integration_test/lib/src/integration_log.dart index 45ce7c8c..b4fb537b 100644 --- a/veilid-flutter/packages/veilid_integration_test/lib/src/integration_log.dart +++ b/veilid-flutter/packages/veilid_integration_test/lib/src/integration_log.dart @@ -162,8 +162,9 @@ void runWithIntegrationLog(void Function() body) { } /// Two-pane debug app for integration tests: scrollable xterm log on top, -/// test-status pane on the bottom. Call `runApp(const IntegrationLogApp())` -/// right after `IntegrationTestWidgetsFlutterBinding.ensureInitialized()`. +/// test-status pane on the bottom, separated by a drag-resizable divider. +/// Call `runApp(const IntegrationLogApp())` right after +/// `IntegrationTestWidgetsFlutterBinding.ensureInitialized()`. class IntegrationLogApp extends StatefulWidget { const IntegrationLogApp({super.key}); @@ -172,8 +173,13 @@ class IntegrationLogApp extends StatefulWidget { } class _IntegrationLogAppState extends State { + static const _statusPaneMinHeight = 48.0; + static const _logPaneMinHeight = 64.0; + static const _dividerHeight = 5.0; + final _terminalController = TerminalController(); final _scrollController = ScrollController(); + double _statusPaneHeight = 180; @override void initState() { @@ -212,37 +218,84 @@ class _IntegrationLogAppState extends State { await Clipboard.setData(ClipboardData(text: text)); } + Widget _divider(double maxStatusHeight) => MouseRegion( + cursor: SystemMouseCursors.resizeRow, + child: GestureDetector( + behavior: HitTestBehavior.opaque, + onVerticalDragUpdate: (details) => setState(() { + _statusPaneHeight = (_statusPaneHeight - details.delta.dy).clamp( + _statusPaneMinHeight, + maxStatusHeight, + ); + }), + child: Container( + height: _dividerHeight, + color: Colors.grey.shade600, + alignment: Alignment.center, + child: Container(width: 64, height: 2, color: Colors.grey.shade900), + ), + ), + ); + @override Widget build(BuildContext context) => MaterialApp( debugShowCheckedModeBanner: false, theme: ThemeData.dark(), home: Scaffold( - body: Column( - children: [ - Expanded( - child: Scrollbar( - controller: _scrollController, - thumbVisibility: true, - child: TerminalView( - IntegrationLogSink.instance.terminal, - controller: _terminalController, - scrollController: _scrollController, - autofocus: true, - textStyle: const TerminalStyle(fontSize: 10), + body: LayoutBuilder( + builder: (context, constraints) { + final maxStatusHeight = + (constraints.maxHeight - _dividerHeight - _logPaneMinHeight) + .clamp(_statusPaneMinHeight, double.infinity); + return Column( + children: [ + Expanded( + child: Scrollbar( + controller: _scrollController, + thumbVisibility: true, + child: TerminalView( + IntegrationLogSink.instance.terminal, + controller: _terminalController, + scrollController: _scrollController, + autofocus: true, + textStyle: const TerminalStyle(fontSize: 10), + ), + ), ), - ), - ), - Container(height: 1, color: Colors.grey.shade800), - const SizedBox(height: 180, child: IntegrationTestStatusPane()), - ], + _divider(maxStatusHeight), + SizedBox( + // clamp at render so a resize keeps the dragged height + height: _statusPaneHeight.clamp( + _statusPaneMinHeight, + maxStatusHeight, + ), + child: const IntegrationTestStatusPane(), + ), + ], + ); + }, ), ), ); } -class IntegrationTestStatusPane extends StatelessWidget { +class IntegrationTestStatusPane extends StatefulWidget { const IntegrationTestStatusPane({super.key}); + @override + State createState() => + _IntegrationTestStatusPaneState(); +} + +class _IntegrationTestStatusPaneState extends State { + final _scrollController = ScrollController(); + + @override + void dispose() { + _scrollController.dispose(); + super.dispose(); + } + @override Widget build(BuildContext context) => AnimatedBuilder( animation: IntegrationTestStatusSink.instance, @@ -294,30 +347,36 @@ class IntegrationTestStatusPane extends StatelessWidget { ), ), Expanded( - child: ListView.builder( - reverse: true, - itemCount: sink.results.length, - itemBuilder: (context, i) { - final r = sink.results[sink.results.length - 1 - i]; - final icon = r.status == TestStatus.passed ? '✓' : '✗'; - final color = r.status == TestStatus.passed - ? Colors.greenAccent - : Colors.redAccent; - return Padding( - padding: const EdgeInsets.symmetric( - horizontal: 8, - vertical: 1, - ), - child: Text( - '$icon ${r.name} (${r.elapsedMs}ms)', - style: TextStyle( - color: color, - fontSize: 11, - fontFamily: 'monospace', + child: Scrollbar( + controller: _scrollController, + thumbVisibility: true, + child: ListView.builder( + controller: _scrollController, + reverse: true, + primary: false, + itemCount: sink.results.length, + itemBuilder: (context, i) { + final r = sink.results[sink.results.length - 1 - i]; + final icon = r.status == TestStatus.passed ? '✓' : '✗'; + final color = r.status == TestStatus.passed + ? Colors.greenAccent + : Colors.redAccent; + return Padding( + padding: const EdgeInsets.symmetric( + horizontal: 8, + vertical: 1, ), - ), - ); - }, + child: Text( + '$icon ${r.name} (${r.elapsedMs}ms)', + style: TextStyle( + color: color, + fontSize: 11, + fontFamily: 'monospace', + ), + ), + ); + }, + ), ), ), ], diff --git a/veilid-flutter/packages/veilid_test/lib/src/log_fixture.dart b/veilid-flutter/packages/veilid_test/lib/src/log_fixture.dart index 921a47b6..8f2cc3da 100644 --- a/veilid-flutter/packages/veilid_test/lib/src/log_fixture.dart +++ b/veilid-flutter/packages/veilid_test/lib/src/log_fixture.dart @@ -33,8 +33,7 @@ class LogFixture { } void _log(VeilidConfigLogLevel level, String message) { - final line = - '${_timestamp()} ${level.name.toUpperCase()} [test] $message'; + final line = '${_timestamp()} ${level.name.toUpperCase()} [test] $message'; // ignore: avoid_print print(line); for (final listener in _listeners) { diff --git a/veilid-flutter/packages/veilid_test/lib/src/periodic_ticker_web.dart b/veilid-flutter/packages/veilid_test/lib/src/periodic_ticker_web.dart index ed2ccd8f..ec0d6511 100644 --- a/veilid-flutter/packages/veilid_test/lib/src/periodic_ticker_web.dart +++ b/veilid-flutter/packages/veilid_test/lib/src/periodic_ticker_web.dart @@ -12,8 +12,10 @@ class PeriodicTicker { int? _intervalId; void start(Duration interval, void Function() onTick) { - _intervalId = web.window - .setInterval((() => onTick()).toJS, interval.inMilliseconds.toJS); + _intervalId = web.window.setInterval( + (() => onTick()).toJS, + interval.inMilliseconds.toJS, + ); } void stop() { diff --git a/veilid-flutter/packages/veilid_test/lib/src/unit_fixture.dart b/veilid-flutter/packages/veilid_test/lib/src/unit_fixture.dart new file mode 100644 index 00000000..15716250 --- /dev/null +++ b/veilid-flutter/packages/veilid_test/lib/src/unit_fixture.dart @@ -0,0 +1,83 @@ +// Running veilid-core inside a plain `flutter test` process. +// +// A `flutter test` process has no plugins, and veilid-core startup needs +// exactly one platform channel: `path_provider`, for its storage directory. +// Mocking that channel starts core normally, which gives unit tests crypto +// (keypairs, member ids, signing, nonces) and TableDB. +// +// Core does not attach here: no network, no DHT records, no peers. Tests +// needing those belong in the integration harness. + +import 'dart:io'; + +import 'package:flutter/services.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'veilid_fixture.dart'; + +/// Point `path_provider` at a fresh temporary directory and return it. +/// +/// Every method on that channel answers with a path, so one handler covers all +/// of them. The directory is new on every call, so concurrent test processes +/// never share a storage root. +Directory mockVeilidStorageDirectories({String prefix = 'veilid_unit'}) { + TestWidgetsFlutterBinding.ensureInitialized(); + final dir = Directory.systemTemp.createTempSync(prefix); + for (final channel in const [ + 'plugins.flutter.io/path_provider', + 'plugins.flutter.io/path_provider_macos', + 'plugins.flutter.io/path_provider_linux', + 'plugins.flutter.io/path_provider_windows', + 'plugins.flutter.io/path_provider_android', + ]) { + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler( + MethodChannel(channel), + (call) async => dir.path, + ); + } + return dir; +} + +/// A program name no concurrently-running test process can share. +/// +/// [DefaultVeilidFixture] joins the program name into its table, protected and +/// block store directories, so it is the isolation key: two processes sharing +/// one share their stores. `flutter test` runs test files in separate +/// processes, so the pid separates them. A Dart program holds at most one +/// veilid instance, so no instance counter is needed on top. +String veilidUnitProgramName([String base = 'VeilidUnitTest']) => '$base-$pid'; + +/// Start veilid-core for this test file and shut it down afterwards. +/// +/// Call once at the top of `main()`. veilid-core is a process singleton, so +/// this registers `setUpAll`/`tearDownAll`, not per-test hooks. +/// +/// Startup is [DefaultVeilidFixture], the same one the integration tests use, +/// so logging and config honour the same environment variables; this adds the +/// mocked storage directory a pluginless process lacks and a per-process +/// program name (see [veilidUnitProgramName]). +/// +/// [base] only labels the run; the pid supplies uniqueness, so two files may +/// share a base. +void useVeilidUnitCore({String base = 'VeilidUnitTest'}) { + TestWidgetsFlutterBinding.ensureInitialized(); + + late Directory storageDir; + final programName = veilidUnitProgramName(base); + final fixture = DefaultVeilidFixture(programName: programName); + + setUpAll(() async { + storageDir = mockVeilidStorageDirectories(prefix: programName); + await fixture.setUp(); + }); + + tearDownAll(() async { + await fixture.tearDown(); + try { + storageDir.deleteSync(recursive: true); + } on FileSystemException { + // A leftover temp directory is not worth failing a suite over + } + }); +} diff --git a/veilid-flutter/packages/veilid_test/lib/src/update_processor_fixture.dart b/veilid-flutter/packages/veilid_test/lib/src/update_processor_fixture.dart index 645ffa88..e2988926 100644 --- a/veilid-flutter/packages/veilid_test/lib/src/update_processor_fixture.dart +++ b/veilid-flutter/packages/veilid_test/lib/src/update_processor_fixture.dart @@ -11,8 +11,7 @@ class UpdateProcessorFixture implements VeilidProcessorInterface { VeilidFixture veilidFixture; @override - ProcessorConnectionState processorConnectionState = - ProcessorConnectionState( + ProcessorConnectionState processorConnectionState = ProcessorConnectionState( attachment: VeilidStateAttachment( state: AttachmentState.detached, publicInternetReady: false, diff --git a/veilid-flutter/packages/veilid_test/lib/veilid_test.dart b/veilid-flutter/packages/veilid_test/lib/veilid_test.dart index bfaf382a..776e8f25 100644 --- a/veilid-flutter/packages/veilid_test/lib/veilid_test.dart +++ b/veilid-flutter/packages/veilid_test/lib/veilid_test.dart @@ -3,5 +3,6 @@ library; export 'src/log_fixture.dart'; export 'src/ticker_fixture.dart'; +export 'src/unit_fixture.dart'; export 'src/update_processor_fixture.dart'; export 'src/veilid_fixture.dart'; diff --git a/veilid-flutter/packages/veilid_test/pubspec.yaml b/veilid-flutter/packages/veilid_test/pubspec.yaml index d8ffc00d..04571c4d 100644 --- a/veilid-flutter/packages/veilid_test/pubspec.yaml +++ b/veilid-flutter/packages/veilid_test/pubspec.yaml @@ -14,6 +14,13 @@ dependencies: web: ^1.1.1 veilid: path: ../.. + # For the unit fixture, which mocks a platform channel and registers test + # hooks. This package already depends on veilid, a Flutter plugin, so it was + # never usable outside Flutter. + flutter: + sdk: flutter + flutter_test: + sdk: flutter dev_dependencies: flutter_lints: ^6.0.0 diff --git a/veilid-flutter/run_integration_tests.sh b/veilid-flutter/run_integration_tests.sh index df2bd50e..bcfb4d51 100755 --- a/veilid-flutter/run_integration_tests.sh +++ b/veilid-flutter/run_integration_tests.sh @@ -1,8 +1,8 @@ #!/bin/bash pushd example 2>/dev/null -# Cargo features come from the `hooks: user_defines:` block in example/pubspec.yaml; -# hooks run under a fixed environment allowlist, so nothing can be passed from here. +# Set cargo features in example/pubspec.yaml under `hooks: user_defines:`; hooks run +# under a fixed environment allowlist, so this script cannot pass any through # Default to the #common=debug log tag group unless the caller overrides LOG_DIRECTIVES case "$*" in diff --git a/veilid-flutter/test/test_encoding.dart b/veilid-flutter/test/test_encoding.dart index bef8c57e..5fbbd40d 100644 --- a/veilid-flutter/test/test_encoding.dart +++ b/veilid-flutter/test/test_encoding.dart @@ -12,7 +12,7 @@ const knownVectors = [ ['foo', 'Zm9v'], ['foob', 'Zm9vYg'], ['fooba', 'Zm9vYmE'], - ['foobar', 'Zm9vYmFy'] + ['foobar', 'Zm9vYmFy'], ]; Future _testEncodingKnownVector(Uint8List k, String v) async { diff --git a/veilid-flutter/test/test_native_asset.dart b/veilid-flutter/test/test_native_asset.dart index 9a6cbbd1..7e0f64af 100644 --- a/veilid-flutter/test/test_native_asset.dart +++ b/veilid-flutter/test/test_native_asset.dart @@ -1,8 +1,8 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:veilid/veilid.dart'; -// Renaming either side of the @DefaultAsset/assetName pair still compiles, and only -// fails when the first FFI call runs. +// Renaming either side of the @DefaultAsset/assetName pair still compiles and +// fails only at the first FFI call. void testNativeAssetResolves() { expect(Veilid.instance.veilidVersionString(), isNotEmpty); expect(Veilid.instance.veilidFeatures(), isNotEmpty); diff --git a/veilid-flutter/test/test_value_subkey_range.dart b/veilid-flutter/test/test_value_subkey_range.dart index 84c6aaf5..5a2e9d96 100644 --- a/veilid-flutter/test/test_value_subkey_range.dart +++ b/veilid-flutter/test/test_value_subkey_range.dart @@ -13,12 +13,14 @@ ValueSubkeyRange? vsrIntersect(ValueSubkeyRange a, ValueSubkeyRange b) => a.intersect(b); List lvsrUnion( - List a, List b) => - a.unionSubkeys(b); + List a, + List b, +) => a.unionSubkeys(b); List lvsrIntersect( - List a, List b) => - a.intersectSubkeys(b); + List a, + List b, +) => a.intersectSubkeys(b); Future testValueSubkeyRange() async { final a = ValueSubkeyRange.single(1); @@ -45,7 +47,11 @@ Future testValueSubkeyRangeList() async { expectCommute(lvsrUnion, a, a, a); expectCommute(lvsrUnion, a, b, [const ValueSubkeyRange(low: 1, high: 3)]); expectCommute>( - lvsrUnion, a, c, ListValueSubkeyRangeExt.fromIntPairs([(1, 1), (3, 4)])); + lvsrUnion, + a, + c, + ListValueSubkeyRangeExt.fromIntPairs([(1, 1), (3, 4)]), + ); expectCommute(lvsrUnion, b, c, [const ValueSubkeyRange(low: 2, high: 4)]); const d = [ValueSubkeyRange(low: 0, high: 3)]; @@ -60,8 +66,12 @@ Future testValueSubkeyRangeList() async { expectCommute(lvsrUnion, e, f, g); - final h = ListValueSubkeyRangeExt.fromIntPairs( - [(2, 5), (10, 10), (12, 14), (16, 18)]); + final h = ListValueSubkeyRangeExt.fromIntPairs([ + (2, 5), + (10, 10), + (12, 14), + (16, 18), + ]); expectCommute(lvsrIntersect, e, f, h); expectCommute>(lvsrUnion, [], [], []); diff --git a/veilid-python/tests/test_dht.py b/veilid-python/tests/test_dht.py index e63602b5..8f9a5aee 100644 --- a/veilid-python/tests/test_dht.py +++ b/veilid-python/tests/test_dht.py @@ -539,32 +539,20 @@ async def test_watch_dht_values(): assert vd is None await sync(rc1, [rec1]) - # Server 0: Wait for the update - upd = await asyncio.wait_for(value_change_queue.get(), timeout=10) - - # Server 0: Verify the update came back with the first changed subkey's data - assert isinstance(upd.detail, veilid.VeilidValueChange) - assert upd.detail.key == rec0.key - assert upd.detail.count == 0xFFFFFFFE - assert upd.detail.subkeys == [(3, 3)] - assert upd.detail.value is not None - assert upd.detail.value.data == b"BLAH BLAH" + # Server 0: The change arrives in value or hint form + await expect_watched_change( + value_change_queue, rc0, rec0.key, ValueSubkey(3), b"BLAH BLAH" + ) # Server 1: Now set subkey and trigger an update vd = await rc1.set_dht_value(rec1.key, ValueSubkey(4), b"BZORT") assert vd is None await sync(rc1, [rec1]) - # Server 0: Wait for the update - upd = await asyncio.wait_for(value_change_queue.get(), timeout=10) - - # Server 0: Verify the update came back with the first changed subkey's data - assert isinstance(upd.detail, veilid.VeilidValueChange) - assert upd.detail.key == rec0.key - assert upd.detail.count == 0xFFFFFFFD - assert upd.detail.subkeys == [(4, 4)] - assert upd.detail.value is not None - assert upd.detail.value.data == b"BZORT" + # Server 0: The change arrives in value or hint form + await expect_watched_change( + value_change_queue, rc0, rec0.key, ValueSubkey(4), b"BZORT" + ) # Server 0: Cancel some subkeys we don't care about active = await rc0.cancel_dht_watch(rec0.key, [(ValueSubkey(0), ValueSubkey(3))]) @@ -575,26 +563,26 @@ async def test_watch_dht_values(): assert vd == [None, None] await sync(rc1, [rec1]) - # Server 0: Wait for the update - upd = await asyncio.wait_for(value_change_queue.get(), timeout=10) + # Server 0: subkey 3 is in the cancelled range, so only subkey 4 may surface + await expect_watched_change( + value_change_queue, rc0, rec0.key, ValueSubkey(4), b"BZORT BZORT" + ) - # Server 0: Verify only one update came back - assert isinstance(upd.detail, veilid.VeilidValueChange) - assert upd.detail.key == rec0.key - assert upd.detail.count == 0xFFFFFFFC - assert upd.detail.subkeys == [(4, 4)] - assert upd.detail.value is not None - assert upd.detail.value.data == b"BZORT BZORT" - - # Server 0: Now we should NOT get any other update - upd = None - try: - upd = await asyncio.wait_for(value_change_queue.get(), timeout=10) - except asyncio.TimeoutError: - pass - if upd is not None: - print(f"bad update: {VeilidJSONEncoder.dumps(upd)}") - assert upd is None + # Server 0: a copy that raced the verifying read re-reports once, so + # require every straggler to be well-formed and to go quiet promptly + for _ in range(6): + try: + upd = await asyncio.wait_for(value_change_queue.get(), timeout=10) + except asyncio.TimeoutError: + break + print(f"settling update: {VeilidJSONEncoder.dumps(upd)}") + assert isinstance(upd.detail, veilid.VeilidValueChange) + assert upd.detail.key == rec0.key + assert upd.detail.subkeys == [(4, 4)] + if upd.detail.value is not None: + assert upd.detail.value.data == b"BZORT BZORT" + else: + assert False, "value change updates did not go quiet" # Now cancel the update active = await rc0.cancel_dht_watch(rec0.key, [(ValueSubkey(3), ValueSubkey(9))]) @@ -749,6 +737,109 @@ async def test_watch_many_dht_values(): await rc0.close_dht_record(record.key) await rc0.delete_dht_record(record.key) + +@pytest.mark.asyncio +async def test_min_seqnum_dht_value(api_connection: veilid.VeilidAPI): + rc = await api_connection.new_routing_context() + async with rc: + for kind in await api_connection.valid_crypto_kinds(): + rec = await rc.create_dht_record(kind, veilid.DHTSchema.dflt(3)) + record_key = rec.key + + def no_offline(min_seqnum: Optional[ValueSeqNum] = None) -> veilid.SetDHTValueOptions: + return veilid.SetDHTValueOptions(None, False, min_seqnum) + + va = b"Qwertyuiop Asdfghjkl Zxcvbnm" + vb = b"1234567890" + + # A minimum above the automatic sequence number wins + vdtemp = await rc.set_dht_value(record_key, ValueSubkey(0), va, no_offline(ValueSeqNum(100))) + assert vdtemp is None + + rr = await rc.inspect_dht_record(record_key, [(ValueSubkey(0), ValueSubkey(0))], veilid.DHTReportScope.LOCAL) + assert rr.local_seqs == [100] + + # Identical data already satisfying the minimum keeps the sequence number + vdtemp = await rc.set_dht_value(record_key, ValueSubkey(0), va, no_offline(ValueSeqNum(100))) + assert vdtemp is None + + rr = await rc.inspect_dht_record(record_key, [(ValueSubkey(0), ValueSubkey(0))], veilid.DHTReportScope.LOCAL) + assert rr.local_seqs == [100] + + # Identical data below a raised minimum still gets raised + vdtemp = await rc.set_dht_value(record_key, ValueSubkey(0), va, no_offline(ValueSeqNum(200))) + assert vdtemp is None + + rr = await rc.inspect_dht_record(record_key, [(ValueSubkey(0), ValueSubkey(0))], veilid.DHTReportScope.LOCAL) + assert rr.local_seqs == [200] + + # MAX minus one leaves exactly one write + vdtemp = await rc.set_dht_value(record_key, ValueSubkey(1), va, no_offline(ValueSeqNum(ValueSeqNum.MAX - 1))) + assert vdtemp is None + + vdtemp = await dht_retry( + api_connection, + lambda: rc.set_dht_value(record_key, ValueSubkey(1), vb, no_offline()), + "last remaining write", + ) + assert vdtemp is None + + rr = await rc.inspect_dht_record(record_key, [(ValueSubkey(1), ValueSubkey(1))], veilid.DHTReportScope.LOCAL) + assert rr.local_seqs == [ValueSeqNum.MAX] + + # And nothing after it + with pytest.raises(veilid.VeilidAPIError): + await rc.set_dht_value(record_key, ValueSubkey(1), va, no_offline()) + + # An identical rewrite at max is still accepted, since it needs no new sequence number + vdtemp = await dht_retry( + api_connection, + lambda: rc.set_dht_value(record_key, ValueSubkey(1), vb, no_offline()), + "identical rewrite at max", + ) + assert vdtemp is None + + # A write straight to max leaves nothing after it + vdtemp = await dht_retry( + api_connection, + lambda: rc.set_dht_value(record_key, ValueSubkey(2), va, no_offline(ValueSeqNum(ValueSeqNum.MAX))), + "write straight to max", + ) + assert vdtemp is None + + rr = await rc.inspect_dht_record(record_key, [(ValueSubkey(2), ValueSubkey(2))], veilid.DHTReportScope.LOCAL) + assert rr.local_seqs == [ValueSeqNum.MAX] + + with pytest.raises(veilid.VeilidAPIError): + await rc.set_dht_value(record_key, ValueSubkey(2), vb, no_offline()) + + # Re-sending with the same minimum is a retry: it reuses the sequence number instead of failing + vdtemp = await dht_retry( + api_connection, + lambda: rc.set_dht_value(record_key, ValueSubkey(2), va, no_offline(ValueSeqNum(ValueSeqNum.MAX))), + "min_seqnum retry", + ) + assert vdtemp is None + + # Core does not check that a retry carries the same data, but the network does: + # the value that got there first stands, and the writer is told what it lost to + vdtemp = await dht_retry( + api_connection, + lambda: rc.set_dht_value(record_key, ValueSubkey(2), vb, no_offline(ValueSeqNum(ValueSeqNum.MAX))), + "min_seqnum retry with different data", + ) + assert vdtemp is not None + assert vdtemp.data == va + assert vdtemp.seq == ValueSeqNum.MAX + + # A subkey at max must not break the sequence numbers a set would produce + rr = await rc.inspect_dht_record(record_key, [(ValueSubkey(0), ValueSubkey(2))], veilid.DHTReportScope.UPDATE_SET) + assert rr.local_seqs == [201, ValueSeqNum.MAX, ValueSeqNum.MAX] + + await rc.close_dht_record(record_key) + await rc.delete_dht_record(record_key) + + @pytest.mark.asyncio async def test_inspect_dht_record(api_connection: veilid.VeilidAPI): rc = await api_connection.new_routing_context() @@ -1278,6 +1369,50 @@ async def test_close_reopen_subkey_update_race(api_connection: veilid.VeilidAPI) ) +async def expect_watched_change( + queue: asyncio.Queue, + rc: veilid.RoutingContext, + key, + subkey: ValueSubkey, + data: bytes, +): + """Wait out one watched change, then read the subkey to verify and re-arm it. + + Several nodes serve a watch and each reports independently, so a change + arrives in value form (data inline), range-only hint form (fetch to see it), + or both, in either order, and a node that coalesced adjacent changes may + report a wider range than the subkey that changed. Only delivery is + promised, not form. + + A value-form update may also carry an EARLIER write, a slower node catching + up. Skip that straggler and keep waiting for one covering this change. + """ + saw = False + while True: + try: + upd = await asyncio.wait_for(queue.get(), timeout=10) + except asyncio.TimeoutError: + break + assert isinstance(upd.detail, veilid.VeilidValueChange) + assert upd.detail.key == key + assert len(upd.detail.subkeys) > 0 + covers = any(a <= subkey <= b for (a, b) in upd.detail.subkeys) + if upd.detail.value is not None: + assert covers, f"value update for unexpected subkeys: {upd.detail.subkeys}" + if upd.detail.value.data != data: + continue + saw = True + break + if covers: + saw = True + break + assert saw, "no update covered the changed subkey" + # The read consumes the report, re-arming change reporting for the next change + vd = await rc.get_dht_value(key, subkey, True) + assert vd is not None + assert vd.data == data + + async def sync(rc: veilid.RoutingContext, records: list[veilid.DHTRecordDescriptor]): syncrecords = records.copy() if len(syncrecords) == 0: diff --git a/veilid-python/tests/test_dht_groups.py b/veilid-python/tests/test_dht_groups.py new file mode 100644 index 00000000..a5754e51 --- /dev/null +++ b/veilid-python/tests/test_dht_groups.py @@ -0,0 +1,835 @@ +# DHT group-roster record tests +# +# Structural, semantic and performance validation for the VeilidChat basic-groups +# design: SMPL records carrying one write-once "ballot" subkey per member (roster) +# and one multi-write "status" subkey per member (activity), written +# non-transactionally with allow_offline=False and watched by every member. +# +# Opt-in gates: PERFORMANCE=1 for timing runs, INTEGRATION=1 for multi-server runs. + +import asyncio +import os +import statistics +import time +from typing import Optional + +import pytest + +import veilid +from veilid import * +from veilid.types import * + +from .conftest import dht_retry + +################################################################## + +# storage_manager/types/mod.rs +MAX_SUBKEY_SIZE = 32768 +MAX_RECORD_DATA_SIZE = 1_048_576 + +# Roster layout: owner subkeys carry the epoch header, then one slot per member +OWNER_SUBKEY_COUNT = 2 + + +def max_subkey_len(subkey_count: int) -> int: + """Per-subkey value ceiling the schema enforces for a record of this shape. + + See `check_subkey_value_data_smpl` in storage_manager/schema.rs: a record with + more than 32 subkeys gets a proportionally smaller per-subkey budget, so the + flat 32 KB cap does not apply to wide member records. + """ + return min(MAX_SUBKEY_SIZE, MAX_RECORD_DATA_SIZE // subkey_count) + + +def member_subkey( + member_index: int, + slot: int = 0, + m_cnt: int = 1, + o_cnt: int = OWNER_SUBKEY_COUNT, +) -> ValueSubkey: + """Subkey owned by a given member, per the SMPL owner-block-then-member-blocks layout.""" + return ValueSubkey(o_cnt + member_index * m_cnt + slot) + + +async def make_roster_record( + api: veilid.VeilidAPI, + rc: veilid.RoutingContext, + cs: veilid.CryptoSystem, + member_count: int, + m_cnt: int = 1, + o_cnt: int = OWNER_SUBKEY_COUNT, +) -> tuple[veilid.DHTRecordDescriptor, list[veilid.KeyPair]]: + """Create an SMPL record shaped like a group roster or activity record. + + Returns the descriptor plus the per-member writer keypairs in schema member + order, so `member_subkey(i)` addresses `keypairs[i]`'s slot. + """ + members = [] + keypairs = [] + for _ in range(member_count): + kp = await cs.generate_key_pair() + member_id = await api.generate_member_id(kp.key()) + members.append(veilid.DHTSchemaSMPLMember(member_id.value(), m_cnt)) + keypairs.append(kp) + + schema = veilid.DHTSchema.smpl(o_cnt, members) + desc = await rc.create_dht_record(await cs.kind(), schema) + return (desc, keypairs) + + +def report(name: str, **fields): + """Emit a greppable one-line result record for the spike report.""" + kv = " ".join(f"{k}={v}" for k, v in fields.items()) + print(f"GROUPSPIKE {name} {kv}") + + +################################################################## +# Structural and semantic tests (single server) + + +@pytest.mark.asyncio +async def test_group_roster_layout(api_connection: veilid.VeilidAPI): + """Members write their own slots; the layout formula addresses them correctly.""" + rc = await api_connection.new_routing_context() + async with rc: + for kind in await api_connection.valid_crypto_kinds(): + cs = await api_connection.get_crypto_system(kind) + async with cs: + MEMBERS = 4 + desc, keypairs = await make_roster_record( + api_connection, rc, cs, MEMBERS + ) + + assert isinstance(desc.schema, veilid.DHTSchemaSMPL) + assert len(desc.schema.members) == MEMBERS + + await rc.open_dht_record(desc.key, keypairs[0]) + + for i, kp in enumerate(keypairs): + subkey = member_subkey(i) + data = f"member {i} ballot".encode() + vd = await dht_retry( + api_connection, + lambda k=desc.key, s=subkey, d=data, w=kp: rc.set_dht_value( + k, s, d, veilid.SetDHTValueOptions(w, False) + ), + "set member slot", + ) + assert vd is None, "own-slot write should not conflict" + + for i in range(MEMBERS): + vd = await rc.get_dht_value(desc.key, member_subkey(i), True) + assert vd is not None + assert vd.data == f"member {i} ballot".encode() + assert vd.seq == 0, "first write of a subkey is seq 0" + assert vd.writer == keypairs[i].key() + + await rc.close_dht_record(desc.key) + await rc.delete_dht_record(desc.key) + + +@pytest.mark.asyncio +async def test_group_member_write_authorization(api_connection: veilid.VeilidAPI): + """Slot ownership is enforced: no member may write another's slot, and the + record owner may not write member slots at all. + + The ballot design rests on this: a transition author owns the record but + still cannot forge another member's ballot. + """ + rc = await api_connection.new_routing_context() + async with rc: + for kind in await api_connection.valid_crypto_kinds(): + cs = await api_connection.get_crypto_system(kind) + async with cs: + desc, keypairs = await make_roster_record(api_connection, rc, cs, 3) + owner_keypair = desc.owner_key_pair() + assert owner_keypair is not None + + await rc.open_dht_record(desc.key, keypairs[0]) + + # Own slot: allowed + vd = await dht_retry( + api_connection, + lambda: rc.set_dht_value( + desc.key, + member_subkey(0), + b"mine", + veilid.SetDHTValueOptions(keypairs[0], False), + ), + "own slot", + ) + assert vd is None + + # Another member's slot: rejected + with pytest.raises(Exception): + await rc.set_dht_value( + desc.key, + member_subkey(1), + b"forged", + veilid.SetDHTValueOptions(keypairs[0], False), + ) + + # Owner writing a member slot: rejected + with pytest.raises(Exception): + await rc.set_dht_value( + desc.key, + member_subkey(0), + b"owner override", + veilid.SetDHTValueOptions(owner_keypair, False), + ) + + # Member writing an owner slot: rejected + with pytest.raises(Exception): + await rc.set_dht_value( + desc.key, + ValueSubkey(0), + b"member override", + veilid.SetDHTValueOptions(keypairs[0], False), + ) + + # Owner writing its own header slot: allowed + vd = await dht_retry( + api_connection, + lambda: rc.set_dht_value( + desc.key, + ValueSubkey(0), + b"epoch header", + veilid.SetDHTValueOptions(owner_keypair, False), + ), + "owner header", + ) + assert vd is None + + await rc.close_dht_record(desc.key) + await rc.delete_dht_record(desc.key) + + +@pytest.mark.asyncio +async def test_group_ballot_write_once_seq(api_connection: veilid.VeilidAPI): + """Sequence numbers make a rewritten ballot slot detectable. + + Write-once is enforced app-side by treating any slot past its first write as + poisoned, which requires: first write is seq 0, an identical rewrite does not + advance (so retries are safe), and a differing rewrite does advance. + """ + rc = await api_connection.new_routing_context() + async with rc: + for kind in await api_connection.valid_crypto_kinds(): + cs = await api_connection.get_crypto_system(kind) + async with cs: + desc, keypairs = await make_roster_record(api_connection, rc, cs, 2) + await rc.open_dht_record(desc.key, keypairs[0]) + subkey = member_subkey(0) + opts = veilid.SetDHTValueOptions(keypairs[0], False) + + ballot = b"ballot: propose successor A" + + await dht_retry( + api_connection, + lambda: rc.set_dht_value(desc.key, subkey, ballot, opts), + "first write", + ) + vd = await rc.get_dht_value(desc.key, subkey, True) + assert vd is not None and vd.seq == 0 + + # Idempotent rewrite: same bytes, same writer, no advance + await rc.set_dht_value(desc.key, subkey, ballot, opts) + vd = await rc.get_dht_value(desc.key, subkey, True) + assert vd is not None + assert vd.seq == 0, "identical rewrite must not advance seq" + + # Retraction attempt: differing bytes advance the seq, marking the slot poisoned + await dht_retry( + api_connection, + lambda: rc.set_dht_value( + desc.key, subkey, b"ballot: retracted", opts + ), + "rewrite", + ) + vd = await rc.get_dht_value(desc.key, subkey, True) + assert vd is not None + assert vd.seq == 1, "differing rewrite must advance seq" + + # A reader detects it from the report without fetching values + rr = await rc.inspect_dht_record( + desc.key, [(subkey, subkey)], veilid.DHTReportScope.SYNC_GET + ) + assert rr.network_seqs[0] is not None + assert int(rr.network_seqs[0]) >= 1, "poisoned slot visible via inspect" + + await rc.close_dht_record(desc.key) + await rc.delete_dht_record(desc.key) + + +@pytest.mark.asyncio +async def test_group_subkey_budget(api_connection: veilid.VeilidAPI): + """The per-subkey ceiling shrinks as the member count grows. + + Pins the budget a VLD1-sized ballot must fit into: at N members the ceiling is + 1 MB / (N + owner subkeys), not a flat 32 KB. + """ + rc = await api_connection.new_routing_context() + async with rc: + for kind in await api_connection.valid_crypto_kinds(): + cs = await api_connection.get_crypto_system(kind) + async with cs: + MEMBERS = int(os.getenv("GROUP_MEMBERS", "62")) + desc, keypairs = await make_roster_record( + api_connection, rc, cs, MEMBERS + ) + subkey_count = OWNER_SUBKEY_COUNT + MEMBERS + limit = max_subkey_len(subkey_count) + # A VLD1 adopt ballot embeds an ML-DSA-signed transition statement + vld1_ballot = 3686 + report( + "subkey_budget", + members=MEMBERS, + subkey_count=subkey_count, + limit_bytes=limit, + vld1_ballot_bytes=vld1_ballot, + vld1_headroom_bytes=limit - vld1_ballot, + ) + + await rc.open_dht_record(desc.key, keypairs[0]) + subkey = member_subkey(0) + opts = veilid.SetDHTValueOptions(keypairs[0], False) + + await dht_retry( + api_connection, + lambda: rc.set_dht_value(desc.key, subkey, b"A" * limit, opts), + "at limit", + ) + vd = await rc.get_dht_value(desc.key, subkey, True) + assert vd is not None and len(vd.data) == limit + + with pytest.raises(Exception): + await rc.set_dht_value( + desc.key, subkey, b"A" * (limit + 1), opts + ) + + await rc.close_dht_record(desc.key) + await rc.delete_dht_record(desc.key) + + +@pytest.mark.skipif( + os.getenv("PERFORMANCE") != "1", reason="performance test takes a long time" +) +@pytest.mark.asyncio +async def test_group_roster_scale(api_connection: veilid.VeilidAPI): + """Create rosters at realistic group sizes and confirm the schema holds. + + Reports creation latency and the resulting per-subkey ceiling per size, which + is what sets the advertised group-size maximum. + """ + sizes = [int(s) for s in os.getenv("GROUP_SIZES", "16,64,128,255").split(",")] + rc = await api_connection.new_routing_context() + async with rc: + kind = (await api_connection.valid_crypto_kinds())[0] + cs = await api_connection.get_crypto_system(kind) + async with cs: + for members in sizes: + start = time.monotonic() + desc, keypairs = await make_roster_record( + api_connection, rc, cs, members + ) + create_s = time.monotonic() - start + + subkey_count = OWNER_SUBKEY_COUNT + members + report( + "roster_scale", + members=members, + subkey_count=subkey_count, + limit_bytes=max_subkey_len(subkey_count), + create_s=f"{create_s:.3f}", + ) + + await rc.open_dht_record(desc.key, keypairs[0]) + await rc.close_dht_record(desc.key) + await rc.delete_dht_record(desc.key) + + +@pytest.mark.skipif( + os.getenv("PERFORMANCE") != "1", reason="performance test takes a long time" +) +@pytest.mark.asyncio +async def test_group_status_write_latency(api_connection: veilid.VeilidAPI): + """Per-write cost of a status-slot update, which sets the send-path debounce. + + Each member writes only its own subkey and nothing coordinates them, so this + measures the standalone cost of one non-transactional confirmed write. + """ + members = int(os.getenv("GROUP_MEMBERS", "16")) + rounds = int(os.getenv("GROUP_ROUNDS", "3")) + + rc = await api_connection.new_routing_context() + async with rc: + kind = (await api_connection.valid_crypto_kinds())[0] + cs = await api_connection.get_crypto_system(kind) + async with cs: + desc, keypairs = await make_roster_record(api_connection, rc, cs, members) + await rc.open_dht_record(desc.key, keypairs[0]) + + async def write(i: int, r: int) -> Optional[float]: + payload = f"status {i} round {r} {time.monotonic()}".encode() + opts = veilid.SetDHTValueOptions(keypairs[i], False) + start = time.monotonic() + try: + await rc.set_dht_value(desc.key, member_subkey(i), payload, opts) + except Exception as e: + print(f" write member={i} round={r} failed: {e}") + return None + return time.monotonic() - start + + for label, concurrent in (("sequential", False), ("concurrent", True)): + latencies: list[float] = [] + failures = 0 + for r in range(rounds): + if concurrent: + results = await asyncio.gather( + *[write(i, r) for i in range(members)] + ) + else: + results = [await write(i, r) for i in range(members)] + for x in results: + if x is None: + failures += 1 + else: + latencies.append(x) + + if latencies: + latencies.sort() + report( + "status_write_latency", + mode=label, + members=members, + rounds=rounds, + n=len(latencies), + failures=failures, + p50=f"{statistics.median(latencies):.3f}", + p95=f"{latencies[int(len(latencies) * 0.95) - 1]:.3f}", + max=f"{latencies[-1]:.3f}", + ) + + await rc.close_dht_record(desc.key) + await rc.delete_dht_record(desc.key) + + +@pytest.mark.skipif( + os.getenv("PERFORMANCE") != "1", reason="performance test takes a long time" +) +@pytest.mark.asyncio +async def test_group_inspect_cost(api_connection: veilid.VeilidAPI): + """Cost of scanning every ballot slot, paid once per epoch transition to find + poisoned slots and tally ballots.""" + members = int(os.getenv("GROUP_MEMBERS", "64")) + + rc = await api_connection.new_routing_context() + async with rc: + kind = (await api_connection.valid_crypto_kinds())[0] + cs = await api_connection.get_crypto_system(kind) + async with cs: + desc, keypairs = await make_roster_record(api_connection, rc, cs, members) + await rc.open_dht_record(desc.key, keypairs[0]) + + # Cast a few ballots so the scan has something to find + for i in range(min(members, 8)): + await dht_retry( + api_connection, + lambda i=i: rc.set_dht_value( + desc.key, + member_subkey(i), + f"ballot {i}".encode(), + veilid.SetDHTValueOptions(keypairs[i], False), + ), + "seed ballot", + ) + + full_range = [ + (member_subkey(0), member_subkey(members - 1)) + ] + # First network inspect of a scope pays the locating fanout, later ones hit the cache + repeats = int(os.getenv("GROUP_REPEATS", "5")) + for scope in ( + veilid.DHTReportScope.LOCAL, + veilid.DHTReportScope.SYNC_GET, + ): + for attempt in range(repeats): + start = time.monotonic() + rr = await rc.inspect_dht_record(desc.key, full_range, scope) + elapsed = time.monotonic() - start + seqs = ( + rr.local_seqs + if scope == veilid.DHTReportScope.LOCAL + else rr.network_seqs + ) + written = sum(1 for s in seqs if s is not None) + report( + "inspect_cost", + members=members, + scope=str(scope), + attempt=attempt, + elapsed_s=f"{elapsed:.3f}", + written_slots=written, + ) + + await rc.close_dht_record(desc.key) + await rc.delete_dht_record(desc.key) + + +@pytest.mark.skipif( + os.getenv("PERFORMANCE") != "1", reason="performance test takes a long time" +) +@pytest.mark.asyncio +async def test_group_inspect_fill_sensitivity(api_connection: veilid.VeilidAPI): + """Network inspect cost as a function of how many slots are written. + + A subkey nobody has written returns no seq from every node, so it can never + accumulate consensus and the fanout runs to exhaustion. A roster is mostly + unwritten slots in the common case, so this is the shape that matters. + """ + members = int(os.getenv("GROUP_MEMBERS", "8")) + + rc = await api_connection.new_routing_context() + async with rc: + kind = (await api_connection.valid_crypto_kinds())[0] + cs = await api_connection.get_crypto_system(kind) + async with cs: + for filled in (members, members // 2, 1): + desc, keypairs = await make_roster_record( + api_connection, rc, cs, members + ) + await rc.open_dht_record(desc.key, keypairs[0]) + for i in range(filled): + await dht_retry( + api_connection, + lambda i=i: rc.set_dht_value( + desc.key, + member_subkey(i), + f"slot {i}".encode(), + veilid.SetDHTValueOptions(keypairs[i], False), + ), + "fill", + ) + + rng = [(member_subkey(0), member_subkey(members - 1))] + start = time.monotonic() + await rc.inspect_dht_record( + desc.key, rng, veilid.DHTReportScope.SYNC_GET + ) + elapsed = time.monotonic() - start + report( + "inspect_fill", + members=members, + filled=filled, + unwritten=members - filled, + elapsed_s=f"{elapsed:.3f}", + ) + + await rc.close_dht_record(desc.key) + await rc.delete_dht_record(desc.key) + + +@pytest.mark.skipif( + os.getenv("PERFORMANCE") != "1", reason="performance test takes a long time" +) +@pytest.mark.asyncio +async def test_group_get_value_cost(api_connection: veilid.VeilidAPI): + """Cost of network-verifying a single slot, repeated. + + Separates the per-subkey get path from the range-inspect path: reading one + ballot back from the network is what arbitration actually needs, and the + first get pays the fanout that locates the value nodes. + """ + repeats = int(os.getenv("GROUP_REPEATS", "6")) + + rc = await api_connection.new_routing_context() + async with rc: + kind = (await api_connection.valid_crypto_kinds())[0] + cs = await api_connection.get_crypto_system(kind) + async with cs: + desc, keypairs = await make_roster_record(api_connection, rc, cs, 16) + await rc.open_dht_record(desc.key, keypairs[0]) + subkey = member_subkey(0) + await dht_retry( + api_connection, + lambda: rc.set_dht_value( + desc.key, + subkey, + b"ballot", + veilid.SetDHTValueOptions(keypairs[0], False), + ), + "seed", + ) + + for force in (False, True): + for attempt in range(repeats): + start = time.monotonic() + vd = await rc.get_dht_value(desc.key, subkey, force) + elapsed = time.monotonic() - start + report( + "get_value_cost", + force_refresh=force, + attempt=attempt, + elapsed_s=f"{elapsed:.3f}", + got=vd is not None, + ) + + await rc.close_dht_record(desc.key) + await rc.delete_dht_record(desc.key) + + +################################################################## +# Multi-server tests: real cross-node watch behavior +# +# A node is never notified of its own writes, so watch delivery can only be +# measured with a separate writer and watcher node. + + +@pytest.mark.skipif( + os.getenv("INTEGRATION") != "1", + reason="integration test requires two servers running", +) +@pytest.mark.asyncio +async def test_group_watch_notification_latency(): + """Write-to-notification latency and delivery rate for status-slot updates. + + This is the group message-notification path: one watch on the shared record + stands in for watching every member's log. + """ + value_change_queue: asyncio.Queue[veilid.VeilidUpdate] = asyncio.Queue() + + async def watcher_callback(update: veilid.VeilidUpdate): + if update.kind == veilid.VeilidUpdateKind.VALUE_CHANGE: + await value_change_queue.put((time.monotonic(), update)) + + async def null_callback(update: veilid.VeilidUpdate): + pass + + try: + api_watcher = await veilid.api_connector(watcher_callback, 0) + except veilid.VeilidConnectionError: + pytest.skip("Unable to connect to veilid-server 0.") + return + try: + api_writer = await veilid.api_connector(null_callback, 1) + except veilid.VeilidConnectionError: + pytest.skip("Unable to connect to veilid-server 1.") + return + + async with api_watcher, api_writer: + for api in (api_watcher, api_writer): + await api.debug("record purge local") + await api.debug("record purge remote") + + while True: + try: + await asyncio.wait_for(value_change_queue.get(), timeout=3) + except asyncio.TimeoutError: + break + + rc_watcher = await api_watcher.new_routing_context() + rc_writer = await api_writer.new_routing_context() + async with rc_watcher, rc_writer: + kind = (await api_writer.valid_crypto_kinds())[0] + cs = await api_writer.get_crypto_system(kind) + async with cs: + members = int(os.getenv("GROUP_MEMBERS", "8")) + writes = int(os.getenv("GROUP_WRITES", "10")) + + desc, keypairs = await make_roster_record( + api_writer, rc_writer, cs, members + ) + await rc_writer.open_dht_record(desc.key, keypairs[0]) + + # Publish the record so the watcher node can resolve it + owner_keypair = desc.owner_key_pair() + await dht_retry( + api_writer, + lambda: rc_writer.set_dht_value( + desc.key, + ValueSubkey(0), + b"epoch header", + veilid.SetDHTValueOptions(owner_keypair, False), + ), + "publish record", + ) + + await rc_watcher.open_dht_record(desc.key, None) + active = await rc_watcher.watch_dht_values(desc.key) + assert active, "watch must establish before measuring" + + latencies: list[float] = [] + missed = 0 + for n in range(writes): + i = n % members + sent = time.monotonic() + await dht_retry( + api_writer, + lambda i=i, n=n: rc_writer.set_dht_value( + desc.key, + member_subkey(i), + f"status {i} #{n}".encode(), + veilid.SetDHTValueOptions(keypairs[i], False), + ), + "status write", + ) + try: + recv, upd = await asyncio.wait_for( + value_change_queue.get(), timeout=30 + ) + assert upd.detail.key == desc.key + latencies.append(recv - sent) + except asyncio.TimeoutError: + missed += 1 + print(f" no notification for write #{n}") + + if latencies: + latencies.sort() + report( + "watch_latency", + members=members, + writes=writes, + delivered=len(latencies), + missed=missed, + p50=f"{statistics.median(latencies):.3f}", + p95=f"{latencies[int(len(latencies) * 0.95) - 1]:.3f}", + max=f"{latencies[-1]:.3f}", + ) + else: + report("watch_latency", members=members, writes=writes, delivered=0, missed=missed) + + await rc_watcher.cancel_dht_watch(desc.key) + await rc_watcher.close_dht_record(desc.key) + await rc_writer.close_dht_record(desc.key) + await rc_writer.delete_dht_record(desc.key) + + +@pytest.mark.skipif( + os.getenv("INTEGRATION") != "1", + reason="integration test requires multiple servers running", +) +@pytest.mark.skipif( + os.getenv("PERFORMANCE") != "1", reason="performance test takes a long time" +) +@pytest.mark.asyncio +async def test_group_watch_fanout(): + """Notification cost as the watcher count grows. + + Every status write notifies every member, so fan-out rather than record + capacity is the likely bound on group size. Needs one veilid-server subnode + per watcher: set GROUP_WATCHERS to the number running beyond subnode 0. + """ + watcher_count = int(os.getenv("GROUP_WATCHERS", "2")) + + queues: list[asyncio.Queue] = [] + apis = [] + + def make_callback(q: asyncio.Queue): + async def cb(update: veilid.VeilidUpdate): + if update.kind == veilid.VeilidUpdateKind.VALUE_CHANGE: + await q.put((time.monotonic(), update)) + + return cb + + async def null_callback(update: veilid.VeilidUpdate): + pass + + try: + api_writer = await veilid.api_connector(null_callback, 0) + except veilid.VeilidConnectionError: + pytest.skip("Unable to connect to veilid-server 0.") + return + + for idx in range(1, watcher_count + 1): + try: + q: asyncio.Queue = asyncio.Queue() + apis.append(await veilid.api_connector(make_callback(q), idx)) + queues.append(q) + except veilid.VeilidConnectionError: + pytest.skip(f"Unable to connect to veilid-server {idx}.") + return + + try: + rc_writer = await api_writer.new_routing_context() + async with rc_writer: + kind = (await api_writer.valid_crypto_kinds())[0] + cs = await api_writer.get_crypto_system(kind) + async with cs: + desc, keypairs = await make_roster_record(api_writer, rc_writer, cs, 4) + await rc_writer.open_dht_record(desc.key, keypairs[0]) + + # Publish the record so watcher nodes can resolve it + await dht_retry( + api_writer, + lambda: rc_writer.set_dht_value( + desc.key, + ValueSubkey(0), + b"epoch header", + veilid.SetDHTValueOptions(desc.owner_key_pair(), False), + ), + "publish record", + ) + + watcher_rcs = [] + for api in apis: + rc = await api.new_routing_context() + watcher_rcs.append(rc) + await rc.open_dht_record(desc.key, None) + assert await rc.watch_dht_values(desc.key) + + # Discard watch-establishment notifications: pairing one with a + # later write reads as negative latency + for q in queues: + while True: + try: + await asyncio.wait_for(q.get(), timeout=5) + except asyncio.TimeoutError: + break + + writes = int(os.getenv("GROUP_WRITES", "5")) + per_watcher: list[list[float]] = [[] for _ in queues] + missed = 0 + for n in range(writes): + sent = time.monotonic() + await dht_retry( + api_writer, + lambda n=n: rc_writer.set_dht_value( + desc.key, + member_subkey(n % 4), + f"fanout #{n}".encode(), + veilid.SetDHTValueOptions(keypairs[n % 4], False), + ), + "fanout write", + ) + for w, q in enumerate(queues): + try: + while True: + recv, _ = await asyncio.wait_for(q.get(), timeout=30) + if recv >= sent: + break + per_watcher[w].append(recv - sent) + except asyncio.TimeoutError: + missed += 1 + + for w, lat in enumerate(per_watcher): + if lat: + lat.sort() + report( + "watch_fanout", + watchers=watcher_count, + watcher=w, + delivered=len(lat), + p50=f"{statistics.median(lat):.3f}", + max=f"{lat[-1]:.3f}", + ) + report("watch_fanout_missed", watchers=watcher_count, missed=missed) + + for rc in watcher_rcs: + await rc.cancel_dht_watch(desc.key) + await rc.close_dht_record(desc.key) + await rc_writer.close_dht_record(desc.key) + await rc_writer.delete_dht_record(desc.key) + finally: + for api in apis: + await api.release() + await api_writer.release() diff --git a/veilid-python/tests/test_dht_transactions.py b/veilid-python/tests/test_dht_transactions.py index 11ed7d85..26b3027f 100644 --- a/veilid-python/tests/test_dht_transactions.py +++ b/veilid-python/tests/test_dht_transactions.py @@ -433,6 +433,83 @@ async def test_transact_dht_records_set_commit_get_commit(api_connection: Veilid await rc.delete_dht_record(rec.key) +@pytest.mark.asyncio +async def test_transact_dht_records_min_seqnum(api_connection: VeilidAPI): + rc = await api_connection.new_routing_context() + async with rc: + for kind in await api_connection.valid_crypto_kinds(): + rec = await rc.create_dht_record(kind, DHTSchema.dflt(2)) + + # MAX minus one leaves exactly one write + rec_tx = await api_connection.transact_dht_records([rec.key], None) + async with rec_tx: + vd = await rec_tx.set( + rec.key, + ValueSubkey(0), + TEST_MESSAGE_1, + DHTTransactionSetValueOptions(None, ValueSeqNum(ValueSeqNum.MAX - 1)), + ) + assert vd is None + await rec_tx.commit() + + report = await rc.inspect_dht_record(rec.key, [(ValueSubkey(0), ValueSubkey(0))], DHTReportScope.LOCAL) + assert report.local_seqs == [ValueSeqNum.MAX - 1] + + rec_tx = await api_connection.transact_dht_records([rec.key], None) + async with rec_tx: + vd = await rec_tx.set(rec.key, ValueSubkey(0), TEST_MESSAGE_2) + assert vd is None + await rec_tx.commit() + + report = await rc.inspect_dht_record(rec.key, [(ValueSubkey(0), ValueSubkey(0))], DHTReportScope.LOCAL) + assert report.local_seqs == [ValueSeqNum.MAX] + + rec_tx = await api_connection.transact_dht_records([rec.key], None) + async with rec_tx: + # A snapshot subkey at max must not break the sequence numbers a set would produce + report = await rec_tx.inspect(rec.key, [(ValueSubkey(0), ValueSubkey(1))], DHTReportScope.UPDATE_SET) + assert report.local_seqs == [ValueSeqNum.MAX, 0] + + # And nothing can be written after it. A transaction knows every online sequence + # number at begin and only lands with consensus, so unlike a plain set it does not + # let a minimum the sequence number already satisfies reuse it. + with pytest.raises(VeilidAPIError): + await rec_tx.set(rec.key, ValueSubkey(0), TEST_MESSAGE_1) + with pytest.raises(VeilidAPIError): + await rec_tx.set( + rec.key, + ValueSubkey(0), + TEST_MESSAGE_1, + DHTTransactionSetValueOptions(None, ValueSeqNum(ValueSeqNum.MAX)), + ) + await rec_tx.rollback() + + # PROBE + for label, opts, payload in ( + ("same value, no minimum", None, TEST_MESSAGE_2), + ("same value, same minimum", DHTTransactionSetValueOptions(None, ValueSeqNum(ValueSeqNum.MAX)), TEST_MESSAGE_2), + ("different value, same minimum", DHTTransactionSetValueOptions(None, ValueSeqNum(ValueSeqNum.MAX)), TEST_MESSAGE_1), + ): + rec_tx = await api_connection.transact_dht_records([rec.key], None) + async with rec_tx: + try: + vd = await rec_tx.set(rec.key, ValueSubkey(0), payload, opts) + print(f"PROBE {label}: set -> {vd!r}") + await rec_tx.commit() + print(f"PROBE {label}: commit ok") + except Exception as e: + print(f"PROBE {label}: RAISED {type(e).__name__} {e}") + try: + await rec_tx.rollback() + except Exception: + pass + rr = await rc.inspect_dht_record(rec.key, [(ValueSubkey(0), ValueSubkey(0))], DHTReportScope.LOCAL) + print(f"PROBE {label}: local_seqs={rr.local_seqs}") + + await rc.close_dht_record(rec.key) + await rc.delete_dht_record(rec.key) + + @pytest.mark.asyncio async def test_transact_dht_records_set_commit_delete_get_rollback(api_connection: VeilidAPI): rc = await api_connection.new_routing_context() diff --git a/veilid-python/veilid/api.py b/veilid-python/veilid/api.py index 0328b588..c522c53c 100644 --- a/veilid-python/veilid/api.py +++ b/veilid-python/veilid/api.py @@ -206,16 +206,18 @@ class RoutingContext(ABC): ) -> Optional[types.ValueData]: """Push a changed subkey value to the network. The record must first be opened or created. - The writer in options, if given, overrides the default writer from open. Returns None on - success, or the network's ValueData if the value set was older than what is on the network. + The writer in options, if given, overrides the default writer from open. min_seqnum, if + given, raises the sequence number to at least that after the automatic increment; + ValueSeqNum.MAX minus n leaves n writes after this one. Returns None on success, or the + network's ValueData if the value set was older than what is on the network. Raises VeilidAPIErrorInvalidArgument if the record is not open. Raises VeilidAPIErrorGeneric - if key is an unsupported kind, the record is not writable, or the value fails schema - validation (subkey out of range, data over the per-subkey limit, or wrong writer for the - subkey). Raises VeilidAPIErrorTryAgain (retryable) if the record is currently in a - transaction, VeilidAPIErrorKeyNotFound if the record no longer exists, and - VeilidAPIErrorNotInitialized if the node is shut down. A failed network fanout does not - raise; the write is deferred and None is returned. + if key is an unsupported kind, the record is not writable, the subkey has no sequence numbers + left, or the value fails schema validation (subkey out of range, data over the per-subkey + limit, or wrong writer for the subkey). Raises VeilidAPIErrorTryAgain (retryable) if the + record is currently in a transaction, VeilidAPIErrorKeyNotFound if the record no longer + exists, and VeilidAPIErrorNotInitialized if the node is shut down. A failed network fanout + does not raise; the write is deferred and None is returned. """ pass @@ -931,15 +933,17 @@ class DHTTransaction(ABC): """Add a set operation to the transaction. Fails offline and fails if offline writes exist for the record. The writer in options, if - given, overrides the default writer from open. Returns None on success, or the network's - ValueData if the value set was older than what is on the network. + given, overrides the default writer from open. min_seqnum, if given, raises the sequence + number to at least that after the automatic increment; ValueSeqNum.MAX minus n leaves n + writes after this one. Returns None on success, or the network's ValueData if the value set + was older than what is on the network. Raises VeilidAPIErrorTransactionNotFound if the transaction handle is already completed or no longer in the Begin stage, VeilidAPIErrorInvalidArgument if key is not open in the transaction or subkey is outside the schema range, VeilidAPIErrorGeneric if key is an unsupported kind or - malformed or the subkey has no writer, and VeilidAPIErrorTryAgain (retryable) if the node is - offline or write consensus was not reached this round. A non-responding node is retried rather - than surfaced as a timeout. + malformed or the subkey has no writer or no sequence numbers left, and VeilidAPIErrorTryAgain + (retryable) if the node is offline or write consensus was not reached this round. A + non-responding node is retried rather than surfaced as a timeout. """ pass diff --git a/veilid-python/veilid/config.py b/veilid-python/veilid/config.py index 500f0363..69026843 100644 --- a/veilid-python/veilid/config.py +++ b/veilid-python/veilid/config.py @@ -286,6 +286,7 @@ class VeilidConfigInternalNetwork(ConfigBase): """Internal "footgun" network tuning.""" connection_initial_timeout_ms: int + single_send_timeout_ms: int connection_inactivity_timeout_ms: int max_connections_per_ip4: int max_connections_per_ip6_prefix: int diff --git a/veilid-python/veilid/schema/RecvMessage.json b/veilid-python/veilid/schema/RecvMessage.json index dc4606c3..0072dbd7 100644 --- a/veilid-python/veilid/schema/RecvMessage.json +++ b/veilid-python/veilid/schema/RecvMessage.json @@ -5356,10 +5356,17 @@ "rpc": { "description": "Internal RPC tuning.", "$ref": "#/$defs/VeilidConfigInternalRPC" + }, + "single_send_timeout_ms": { + "description": "Cap on any single low-level send, in milliseconds, so a send on a wedged\ntransport cannot consume the caller's whole RPC timeout budget.", + "type": "integer", + "format": "uint32", + "minimum": 0 } }, "required": [ "connection_initial_timeout_ms", + "single_send_timeout_ms", "connection_inactivity_timeout_ms", "max_connections_per_ip4", "max_connections_per_ip6_prefix", diff --git a/veilid-python/veilid/schema/Request.json b/veilid-python/veilid/schema/Request.json index 0f4d91f9..b1aae545 100644 --- a/veilid-python/veilid/schema/Request.json +++ b/veilid-python/veilid/schema/Request.json @@ -1634,6 +1634,17 @@ "description": "Options that override defaults for DHTTransaction::set", "type": "object", "properties": { + "min_seqnum": { + "description": "Raise the sequence number to at least this, after the automatic increment.\n[ValueSeqNum::MAX] minus `n` leaves `n` writes after this one.\n\nA write whose minimum the subkey's sequence number already satisfies reuses that sequence\nnumber instead of failing, so a write at [ValueSeqNum::MAX] can be retried. Retries must\ncarry the same data and writer to be accepted; that is the caller's responsibility.", + "anyOf": [ + { + "$ref": "#/$defs/ValueSeqNum" + }, + { + "type": "null" + } + ] + }, "writer": { "description": "Override writer key pair for this operation", "type": [ @@ -2398,6 +2409,17 @@ } ] }, + "min_seqnum": { + "description": "Raise the sequence number to at least this, after the automatic increment.\n[ValueSeqNum::MAX] minus `n` leaves `n` writes after this one.\n\nA write whose minimum the subkey's sequence number already satisfies reuses that sequence\nnumber instead of failing, so a write at [ValueSeqNum::MAX] can be retried until it reaches\nconsensus. Retries must carry the same data and writer to be accepted; that is the caller's\nresponsibility, not something this checks.", + "anyOf": [ + { + "$ref": "#/$defs/ValueSeqNum" + }, + { + "type": "null" + } + ] + }, "writer": { "description": "Override writer key pair for this operation", "type": [ @@ -2724,6 +2746,15 @@ } } }, + "ValueSeqNum": { + "description": "Value sequence number", + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0 + }, "ValueSubkeyRangeSet": { "description": "A set of DHT subkeys stored as inclusive ranges of [ValueSubkey]", "type": "array", diff --git a/veilid-python/veilid/types.py b/veilid-python/veilid/types.py index d135dd2c..3d66687e 100644 --- a/veilid-python/veilid/types.py +++ b/veilid-python/veilid/types.py @@ -519,7 +519,8 @@ class ValueSubkey(int): class ValueSeqNum(int): """Increasing sequence number ordering changes to a DHT subkey.""" - pass + MAX = 0xFFFFFFFE + """The largest sequence number. Subtract n for a min_seqnum that leaves n writes.""" #################################################################### @@ -800,17 +801,24 @@ class DHTRecordReport: class SetDHTValueOptions: - """Options that override defaults for set_dht_value: an override writer and the offline-write policy.""" + """Options that override defaults for set_dht_value: an override writer, the offline-write policy, and a minimum sequence number.""" writer: Optional[KeyPair] allow_offline: Optional[bool] + min_seqnum: Optional[ValueSeqNum] - def __init__(self, writer: Optional[KeyPair] = None, allow_offline: Optional[bool] = None): + def __init__( + self, + writer: Optional[KeyPair] = None, + allow_offline: Optional[bool] = None, + min_seqnum: Optional[ValueSeqNum] = None, + ): self.writer = writer self.allow_offline = allow_offline + self.min_seqnum = min_seqnum def __repr__(self) -> str: - return f"<{self.__class__.__name__}(writer={self.writer!r}, allow_offline={self.allow_offline!r})>" + return f"<{self.__class__.__name__}(writer={self.writer!r}, allow_offline={self.allow_offline!r}, min_seqnum={self.min_seqnum!r})>" @classmethod def from_json(cls, j: dict) -> Self: @@ -818,6 +826,7 @@ class SetDHTValueOptions: return cls( KeyPair(j["writer"]) if "writer" in j else None, j["allow_offline"] if "allow_offline" in j else None, + ValueSeqNum(j["min_seqnum"]) if j.get("min_seqnum") is not None else None, ) def to_json(self) -> dict: @@ -826,21 +835,28 @@ class SetDHTValueOptions: class DHTTransactionSetValueOptions: - """Options that override defaults for a transaction set: an override writer key pair.""" + """Options that override defaults for a transaction set: an override writer key pair and a minimum sequence number.""" writer: Optional[KeyPair] + min_seqnum: Optional[ValueSeqNum] - def __init__(self, writer: Optional[KeyPair] = None): + def __init__( + self, + writer: Optional[KeyPair] = None, + min_seqnum: Optional[ValueSeqNum] = None, + ): self.writer = writer + self.min_seqnum = min_seqnum def __repr__(self) -> str: - return f"<{self.__class__.__name__}(writer={self.writer!r})>" + return f"<{self.__class__.__name__}(writer={self.writer!r}, min_seqnum={self.min_seqnum!r})>" @classmethod def from_json(cls, j: dict) -> Self: """Build from its JSON representation.""" return cls( KeyPair(j["writer"]) if "writer" in j else None, + ValueSeqNum(j["min_seqnum"]) if j.get("min_seqnum") is not None else None, ) def to_json(self) -> dict: diff --git a/veilid-remote-api/smoketest/Cargo.lock b/veilid-remote-api/smoketest/Cargo.lock index fbc0928e..5b3ee2a9 100644 --- a/veilid-remote-api/smoketest/Cargo.lock +++ b/veilid-remote-api/smoketest/Cargo.lock @@ -41,9 +41,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -312,9 +312,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", @@ -444,9 +444,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" dependencies = [ "arrayref", "arrayvec", @@ -548,9 +548,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" [[package]] name = "capnp" @@ -581,9 +581,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", "shlex", @@ -676,9 +676,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -686,9 +686,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -938,9 +938,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "dbus" @@ -1198,10 +1198,11 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.12" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" dependencies = [ + "autocfg", "indenter", "once_cell", ] @@ -1235,9 +1236,9 @@ checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "flume" @@ -1295,9 +1296,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -1310,9 +1311,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -1320,15 +1321,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -1337,9 +1338,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-lite" @@ -1356,13 +1357,13 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -1378,21 +1379,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -1592,7 +1593,7 @@ dependencies = [ "once_cell", "rand 0.9.5", "ring", - "thiserror 2.0.19", + "thiserror 2.0.20", "tinyvec", "tokio", "tracing", @@ -1615,7 +1616,7 @@ dependencies = [ "rand 0.9.5", "resolv-conf", "smallvec", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", ] @@ -1700,9 +1701,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -1714,9 +1715,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -1727,9 +1728,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1741,16 +1742,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -1761,15 +1763,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" dependencies = [ "displaydoc", "icu_locale_core", @@ -1859,9 +1861,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "is_terminal_polyfill" @@ -1896,7 +1898,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.19", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -1935,9 +1937,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -2043,9 +2045,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.18" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" dependencies = [ "libc", ] @@ -2069,9 +2071,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -2171,9 +2173,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.15" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -2188,9 +2190,9 @@ dependencies = [ [[package]] name = "netlink-packet-core" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3463cbb78394cb0141e2c926b93fc2197e473394b761986eca3b9da2c63ae0f4" +checksum = "b897d7bd4f0af82e68d40d0344cf37e97f9c97ddf74a098de3e4da05e96ca395" dependencies = [ "paste", ] @@ -2209,9 +2211,9 @@ dependencies = [ [[package]] name = "netlink-proto" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6f7398dddf5f152d2a91a2921a134c6097056e292c0d4b9906007855e7cece6" +checksum = "93af8261786086024cd5e96e0a991dd65ced07bbf7c233a487bbc96b971d5539" dependencies = [ "bytes", "futures-channel", @@ -2219,7 +2221,7 @@ dependencies = [ "log", "netlink-packet-core", "netlink-sys", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -2310,9 +2312,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -2575,9 +2577,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polling" @@ -2605,15 +2607,15 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -2790,7 +2792,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -2827,9 +2829,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -2869,7 +2871,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" dependencies = [ "hashbrown", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -2960,9 +2962,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" dependencies = [ "ring", "rustls-pki-types", @@ -3360,11 +3362,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -3380,9 +3382,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", @@ -3400,9 +3402,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.54" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", "num-conv", @@ -3430,9 +3432,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -3604,7 +3606,7 @@ dependencies = [ "log", "rand 0.9.5", "sha1", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -3667,9 +3669,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -3750,7 +3752,7 @@ dependencies = [ "serde_ignored", "serde_json", "sha2 0.11.0", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", "tracing-subscriber", @@ -3826,7 +3828,7 @@ dependencies = [ "serde", "shell-words", "socket2", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -3880,9 +3882,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -3893,9 +3895,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ "js-sys", "wasm-bindgen", @@ -3903,9 +3905,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3913,9 +3915,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -3926,9 +3928,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] @@ -3941,9 +3943,9 @@ checksum = "323f4da9523e9a669e1eaf9c6e763892769b1d38c623913647bfdc1532fe4549" [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -4197,9 +4199,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "ws_stream_wasm" @@ -4214,7 +4216,7 @@ dependencies = [ "pharos", "rustc_version", "send_wrapper", - "thiserror 2.0.19", + "thiserror 2.0.20", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -4256,18 +4258,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", @@ -4317,9 +4319,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -4328,9 +4330,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" dependencies = [ "yoke", "zerofrom", @@ -4339,13 +4341,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "47402523226a02bfe5230160dc3ccc089aa6f6f19e7fcbb4e6f824bbb1b4aa62" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] diff --git a/veilid-tools/Cargo.toml b/veilid-tools/Cargo.toml index 7940a2fb..fdd20425 100644 --- a/veilid-tools/Cargo.toml +++ b/veilid-tools/Cargo.toml @@ -202,6 +202,8 @@ winapi = { version = "0.3.9", features = [ "sysinfoapi", "winerror", "winsock2", + "ws2def", + "ws2ipdef", ] } windows = { version = "0.62.2", features = [ "Win32_NetworkManagement_Dns", diff --git a/veilid-tools/public-api.txt b/veilid-tools/public-api.txt index b980b1c6..7bc9ccb9 100644 --- a/veilid-tools/public-api.txt +++ b/veilid-tools/public-api.txt @@ -654,6 +654,16 @@ pub fn veilid_tools::MustJoinSingleFuture::new() -> Self pub async fn veilid_tools::MustJoinSingleFuture::single_spawn_local(&self, &str, impl core::future::future::Future + 'static) -> core::result::Result<(core::option::Option, bool), ()> impl core::default::Default for veilid_tools::MustJoinSingleFuture where T: 'static pub fn veilid_tools::MustJoinSingleFuture::default() -> Self +pub mod veilid_tools::network_attempt +pub enum veilid_tools::network_attempt::NetworkAttempt +pub veilid_tools::network_attempt::NetworkAttempt::Performed(veilid_tools::NetworkResult) +pub veilid_tools::network_attempt::NetworkAttempt::Skipped +impl veilid_tools::NetworkAttempt +pub fn veilid_tools::NetworkAttempt::value(T) -> Self +impl core::fmt::Debug for veilid_tools::NetworkAttempt +pub fn veilid_tools::NetworkAttempt::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for veilid_tools::NetworkAttempt +pub fn veilid_tools::NetworkAttempt::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub mod veilid_tools::network_interfaces pub enum veilid_tools::network_interfaces::IfAddr pub veilid_tools::network_interfaces::IfAddr::V4(veilid_tools::Ifv4Addr) @@ -736,6 +746,9 @@ pub fn veilid_tools::NetworkInterface::fmt(&self, &mut core::fmt::Formatter<'_>) pub struct veilid_tools::network_interfaces::NetworkInterfaceAddressState pub veilid_tools::network_interfaces::NetworkInterfaceAddressState::gateway_addresses: alloc::vec::Vec pub veilid_tools::network_interfaces::NetworkInterfaceAddressState::interface_addresses: alloc::vec::Vec +pub veilid_tools::network_interfaces::NetworkInterfaceAddressState::own_addresses: alloc::vec::Vec +impl veilid_tools::NetworkInterfaceAddressState +pub fn veilid_tools::NetworkInterfaceAddressState::is_own_address(&self, core::net::ip_addr::IpAddr) -> bool pub struct veilid_tools::network_interfaces::NetworkInterfaces impl veilid_tools::NetworkInterfaces pub fn veilid_tools::NetworkInterfaces::clear(&self) @@ -861,6 +874,7 @@ pub fn veilid_tools::socket_tools::domain_for_address(core::net::socket_addr::So pub fn veilid_tools::socket_tools::new_default_socket2_tcp(core::ffi::primitives::c_int) -> core::io::error::Result pub fn veilid_tools::socket_tools::primary_source_ipv4() -> core::option::Option pub fn veilid_tools::socket_tools::primary_source_ipv6() -> core::option::Option +pub fn veilid_tools::socket_tools::send_udp_datagram_with_ttl(&S, &[u8], core::net::socket_addr::SocketAddr, u32) -> core::io::error::Result pub fn veilid_tools::socket_tools::set_tcp_stream_linger(&tokio::net::tcp::stream::TcpStream, core::option::Option) -> core::io::error::Result<()> pub fn veilid_tools::socket_tools::socket2_operation R, R>(&S, F) -> R pub fn veilid_tools::socket_tools::split_async_tcp_stream(tokio::net::tcp::stream::TcpStream) -> (veilid_tools::ReadHalf, veilid_tools::WriteHalf) @@ -1159,6 +1173,15 @@ pub veilid_tools::Ipv6MulticastScope::LinkLocal pub veilid_tools::Ipv6MulticastScope::OrganizationLocal pub veilid_tools::Ipv6MulticastScope::RealmLocal pub veilid_tools::Ipv6MulticastScope::SiteLocal +pub enum veilid_tools::NetworkAttempt +pub veilid_tools::NetworkAttempt::Performed(veilid_tools::NetworkResult) +pub veilid_tools::NetworkAttempt::Skipped +impl veilid_tools::NetworkAttempt +pub fn veilid_tools::NetworkAttempt::value(T) -> Self +impl core::fmt::Debug for veilid_tools::NetworkAttempt +pub fn veilid_tools::NetworkAttempt::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for veilid_tools::NetworkAttempt +pub fn veilid_tools::NetworkAttempt::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub enum veilid_tools::NetworkResult pub veilid_tools::NetworkResult::AlreadyExists(core::io::error::Error) pub veilid_tools::NetworkResult::InvalidMessage(alloc::string::String) @@ -1661,6 +1684,9 @@ pub fn veilid_tools::NetworkInterface::fmt(&self, &mut core::fmt::Formatter<'_>) pub struct veilid_tools::NetworkInterfaceAddressState pub veilid_tools::NetworkInterfaceAddressState::gateway_addresses: alloc::vec::Vec pub veilid_tools::NetworkInterfaceAddressState::interface_addresses: alloc::vec::Vec +pub veilid_tools::NetworkInterfaceAddressState::own_addresses: alloc::vec::Vec +impl veilid_tools::NetworkInterfaceAddressState +pub fn veilid_tools::NetworkInterfaceAddressState::is_own_address(&self, core::net::ip_addr::IpAddr) -> bool pub struct veilid_tools::NetworkInterfaces impl veilid_tools::NetworkInterfaces pub fn veilid_tools::NetworkInterfaces::clear(&self) @@ -2031,6 +2057,7 @@ pub fn veilid_tools::random_bytes(&mut [u8]) pub fn veilid_tools::retry_falloff_log(u64, u64, u64, u64, f64) -> bool pub async fn veilid_tools::scaled_yielding R + core::marker::Send + 'static>(N, N, N, T) -> R pub fn veilid_tools::secs_to_timestamp_duration(f64) -> u64 +pub fn veilid_tools::send_udp_datagram_with_ttl(&S, &[u8], core::net::socket_addr::SocketAddr, u32) -> core::io::error::Result pub fn veilid_tools::set_tcp_stream_linger(&tokio::net::tcp::stream::TcpStream, core::option::Option) -> core::io::error::Result<()> pub async fn veilid_tools::sleep(u32) pub fn veilid_tools::socket2_operation R, R>(&S, F) -> R diff --git a/veilid-tools/smoketest/Cargo.lock b/veilid-tools/smoketest/Cargo.lock index 11a7f81d..914ea111 100644 --- a/veilid-tools/smoketest/Cargo.lock +++ b/veilid-tools/smoketest/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -180,9 +180,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", @@ -292,9 +292,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" dependencies = [ "find-msvc-tools", "shlex", @@ -374,9 +374,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "dispatch2" @@ -482,9 +482,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" [[package]] name = "flume" @@ -521,9 +521,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -536,9 +536,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -546,15 +546,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -563,9 +563,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-lite" @@ -582,26 +582,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -615,9 +615,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -743,7 +743,7 @@ dependencies = [ "once_cell", "rand 0.9.5", "ring", - "thiserror 2.0.19", + "thiserror 2.0.20", "tinyvec", "tokio", "tracing", @@ -766,16 +766,16 @@ dependencies = [ "rand 0.9.5", "resolv-conf", "smallvec", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", ] [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -787,9 +787,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -800,9 +800,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -814,16 +814,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -834,15 +835,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" dependencies = [ "displaydoc", "icu_locale_core", @@ -895,9 +896,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "itertools" @@ -926,7 +927,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.19", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -1004,9 +1005,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -1035,9 +1036,9 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "minicov" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" +checksum = "c3aa3aa12b448ac225b3102217d1ac5cc717908f02722926524b0599c933c7a0" dependencies = [ "cc", "walkdir", @@ -1062,9 +1063,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.15" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -1079,9 +1080,9 @@ dependencies = [ [[package]] name = "netlink-packet-core" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3463cbb78394cb0141e2c926b93fc2197e473394b761986eca3b9da2c63ae0f4" +checksum = "b897d7bd4f0af82e68d40d0344cf37e97f9c97ddf74a098de3e4da05e96ca395" dependencies = [ "paste", ] @@ -1100,9 +1101,9 @@ dependencies = [ [[package]] name = "netlink-proto" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6f7398dddf5f152d2a91a2921a134c6097056e292c0d4b9906007855e7cece6" +checksum = "93af8261786086024cd5e96e0a991dd65ced07bbf7c233a487bbc96b971d5539" dependencies = [ "bytes", "futures-channel", @@ -1110,7 +1111,7 @@ dependencies = [ "log", "netlink-packet-core", "netlink-sys", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -1162,9 +1163,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -1320,15 +1321,15 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -1472,9 +1473,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1752,11 +1753,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -1772,9 +1773,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", @@ -1783,9 +1784,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -1910,9 +1911,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.24.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -1971,7 +1972,7 @@ dependencies = [ "serde", "shell-words", "socket2", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -2371,9 +2372,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "ws_stream_wasm" @@ -2388,7 +2389,7 @@ dependencies = [ "pharos", "rustc_version", "send_wrapper", - "thiserror 2.0.19", + "thiserror 2.0.20", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -2419,18 +2420,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", @@ -2460,9 +2461,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -2471,9 +2472,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" dependencies = [ "yoke", "zerofrom", @@ -2482,13 +2483,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "47402523226a02bfe5230160dc3ccc089aa6f6f19e7fcbb4e6f824bbb1b4aa62" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] diff --git a/veilid-tools/src/lib.rs b/veilid-tools/src/lib.rs index 61df8e52..7428bc43 100644 --- a/veilid-tools/src/lib.rs +++ b/veilid-tools/src/lib.rs @@ -83,6 +83,8 @@ pub mod log_unwrap; pub mod must_join_handle; /// Single-slot future runner that joins the prior run before starting the next. pub mod must_join_single_future; +/// Wrapper distinguishing an operation we performed from one we chose not to. +pub mod network_attempt; /// Enumeration of host network interfaces and their addresses. pub mod network_interfaces; /// Result type distinguishing network-level outcomes from hard errors. @@ -276,6 +278,8 @@ pub use must_join_handle::*; #[doc(inline)] pub use must_join_single_future::*; #[doc(inline)] +pub use network_attempt::*; +#[doc(inline)] pub use network_interfaces::*; #[doc(inline)] pub use network_result::*; diff --git a/veilid-tools/src/network_attempt.rs b/veilid-tools/src/network_attempt.rs new file mode 100644 index 00000000..4a2f554d --- /dev/null +++ b/veilid-tools/src/network_attempt.rs @@ -0,0 +1,43 @@ +use super::*; + +use core::fmt::{Debug, Display}; + +/// Whether a network operation was performed at all. +/// +/// Sibling to [`NetworkResult`], one layer out: every `NetworkResult` variant +/// asserts something about the remote, so declining to act needs a variant of its +/// own. `Result, E>` covers all three cases: `Err` for our own +/// fault, `Skipped` for a deliberate choice not to act, `Performed` for anything +/// the remote had a part in. +pub enum NetworkAttempt { + /// The operation happened; the result describes how it went + Performed(NetworkResult), + /// Deliberately not performed. Carries no information about the remote: it must not + /// count against their contact methods, their liveness, or any route or relay built on them. + Skipped, +} + +impl NetworkAttempt { + /// A performed attempt that succeeded + pub fn value(v: T) -> Self { + Self::Performed(NetworkResult::Value(v)) + } +} + +impl Debug for NetworkAttempt { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Performed(nr) => f.debug_tuple("Performed").field(nr).finish(), + Self::Skipped => write!(f, "Skipped"), + } + } +} + +impl Display for NetworkAttempt { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Performed(nr) => write!(f, "Performed({})", nr), + Self::Skipped => write!(f, "Skipped"), + } + } +} diff --git a/veilid-tools/src/network_interfaces/mod.rs b/veilid-tools/src/network_interfaces/mod.rs index 320e3224..932653b8 100644 --- a/veilid-tools/src/network_interfaces/mod.rs +++ b/veilid-tools/src/network_interfaces/mod.rs @@ -456,7 +456,7 @@ impl InterfaceAddress { self.flags.is_preferred } - /// CLAT46 synthesized IPv6 (RFC 6877) — local-only on macOS/iOS + /// CLAT46 synthesized IPv6 (RFC 6877), local-only on macOS/iOS #[must_use] pub fn is_clat(&self) -> bool { self.flags.is_clat @@ -474,7 +474,7 @@ impl InterfaceAddress { } } - /// IPv4 link-local / APIPA (169.254.0.0/16) — DHCP-failed fallback, transient + /// IPv4 link-local / APIPA (169.254.0.0/16), DHCP-failed fallback, transient /// and not reachable by peers; always false for IPv6 (link-local IPv6 is /// handled by `ipv6addr_is_global`) #[must_use] @@ -615,13 +615,26 @@ impl NetworkInterface { } } -/// The best routable gateway and interface addresses distilled from all interfaces. +/// The gateway, publication and identity addresses distilled from all interfaces. #[derive(Clone, Default, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] pub struct NetworkInterfaceAddressState { /// Gateway addresses across all interfaces, sorted and deduplicated. pub gateway_addresses: Vec, - /// Best routable interface addresses, sorted by preference and deduplicated. + /// PUBLICATION set: best routable interface addresses, sorted by preference and + /// deduplicated. Privacy, CLAT and CGNAT addresses are excluded from peer info. pub interface_addresses: Vec, + /// IDENTITY set: every address on a running non-loopback interface, sorted and + /// deduplicated. Includes the privacy addresses we send from but never publish. + pub own_addresses: Vec, +} + +impl NetworkInterfaceAddressState { + /// True when the address belongs to one of our interfaces. Identity, not + /// publishability: matches privacy (RFC 4941) and CLAT addresses that publication excludes. + #[must_use] + pub fn is_own_address(&self, addr: IpAddr) -> bool { + self.own_addresses.binary_search(&addr).is_ok() + } } /// Which address families are present across the running non-loopback interfaces. @@ -793,7 +806,7 @@ impl NetworkInterfaces { // For each interface, keep only addresses that are either non-global (private/ // link-local/ULA, needed by LocalNetwork) or match the kernel's primary outbound // source for that family. A global address on a non-primary interface is - // unreachable from peers — publishing it as our dial info just causes timeouts. + // unreachable from peers; publishing it as our dial info causes timeouts. // Interfaces that end up with no addresses are dropped entirely. #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] @@ -818,6 +831,7 @@ impl NetworkInterfaces { // Reduce interfaces to their best routable ip addresses let mut intf_addrs = Vec::new(); let mut gateway_addresses = Vec::new(); + let mut own_addresses = Vec::new(); for intf in inner.interfaces.values() { // Skip down, loopback, and CLAT virtual interfaces (Android: clat4*/v4-*) if !intf.is_running() || intf.is_loopback() || intf.is_clat() { @@ -825,6 +839,10 @@ impl NetworkInterfaces { } for addr in intf.addresses() { + // Identity set keeps what publication filters out below; peers report those + // back to us as our external address + own_addresses.push(addr.if_addr().ip()); + // Skip RFC 4941 privacy addresses; they rotate and churn published peer info if addr.is_temporary() { continue; @@ -862,10 +880,14 @@ impl NetworkInterfaces { gateway_addresses.sort(); gateway_addresses.dedup(); + own_addresses.sort(); + own_addresses.dedup(); + // Now export just the addresses inner.interface_address_state_cache = Arc::new(NetworkInterfaceAddressState { gateway_addresses, interface_addresses, + own_addresses, }); } } diff --git a/veilid-tools/src/resolver/non_windows/mod.rs b/veilid-tools/src/resolver/non_windows/mod.rs index 2169f243..2ae897e3 100644 --- a/veilid-tools/src/resolver/non_windows/mod.rs +++ b/veilid-tools/src/resolver/non_windows/mod.rs @@ -1,5 +1,7 @@ use super::*; +use futures_util::future::{select, Either}; + cfg_if! { if #[cfg(feature="rt-async-std")] { mod async_std_providers; @@ -28,6 +30,9 @@ cfg_if! { } } +/// Head start for the system resolver so a responsive split-horizon view wins the race +const SYSTEM_RESOLVER_HEAD_START_MS: u32 = 500; + struct Resolvers { system: Option>, default: Arc, @@ -49,7 +54,11 @@ async fn with_resolvers) -> PinBoxFutureStatic>(c options.try_tcp_on_error = true; let default = Arc::new(resolver(config, options)); - let system = if let Ok((config, options)) = read_system_conf() { + let system = if let Ok((config, mut options)) = read_system_conf() { + // UDP-hostile networks need the TCP fallback on the system resolver most of all + options.try_tcp_on_error = true; + // The race with the default resolver provides the retry redundancy + options.attempts = 1; Some(Arc::new(resolver(config, options))) } else { None @@ -59,8 +68,51 @@ async fn with_resolvers) -> PinBoxFutureStatic>(c closure(resolvers).await } -/// Cross-platform DNS resolver. This backend uses hickory-resolver, trying the -/// system configuration first and falling back to a default resolver. +/// Race the system resolver against the default resolver, first success wins. +/// The system resolver gets a head start, and either side failing waits out the +/// other, so a wedged system resolver costs the head start plus the default +/// resolver's latency instead of its own full retry ladder. +async fn race_lookups( + opt_system_fut: Option, + default_fut: DF, + what: &str, +) -> Result +where + E1: fmt::Display, + E2: fmt::Display, + SF: Future>, + DF: Future>, +{ + let default_err = + |e: E2| ResolverError::Generic(format!("default resolver {} error: {}", what, e)); + + let Some(system_fut) = opt_system_fut else { + return default_fut.await.map_err(default_err); + }; + + let delayed_default_fut = Box::pin(async move { + sleep(SYSTEM_RESOLVER_HEAD_START_MS).await; + default_fut.await + }); + + match select(Box::pin(system_fut), delayed_default_fut).await { + Either::Left((Ok(v), _)) => Ok(v), + Either::Left((Err(e), default_rest)) => { + debug!("system resolver {} error: {}", what, e); + default_rest.await.map_err(default_err) + } + Either::Right((Ok(v), _)) => Ok(v), + Either::Right((Err(e), system_rest)) => { + debug!("default resolver {} error: {}", what, e); + system_rest.await.map_err(|e| { + ResolverError::Generic(format!("system resolver {} error: {}", what, e)) + }) + } + } +} + +/// Cross-platform DNS resolver. This backend uses hickory-resolver, racing the +/// system configuration (with a head start) against a default resolver. pub struct Resolver {} impl Resolver { @@ -68,30 +120,18 @@ impl Resolver { /// /// Blocks on a DNS query; the first call also builds and caches the process-wide resolvers. /// - /// Errors with [`ResolverError::Generic`] when the default resolver's lookup fails (no such name, network/DNS failure); a system-resolver failure is logged and falls through to the default. + /// Errors with [`ResolverError::Generic`] when every resolver's lookup fails (no such name, network/DNS failure). pub async fn txt_lookup>(host: S) -> Result, ResolverError> { let host = host.as_ref().to_string(); let txt_result = with_resolvers(|resolvers| { Box::pin(async move { - // Try the system resolver config - if let Some(system_resolver) = &resolvers.system { - match system_resolver.txt_lookup(&host).await { - Ok(v) => { - return Ok(v); - } - Err(e) => { - debug!("system resolver txt_lookup error: {}", e); - } - } - } - - match resolvers.default.txt_lookup(&host).await { - Ok(v) => Ok(v), - Err(e) => Err(ResolverError::Generic(format!( - "default resolver txt_lookup error: {}", - e - ))), - } + let system_fut = resolvers.system.clone().map(|r| { + let host = host.clone(); + async move { r.txt_lookup(&host).await } + }); + let default_resolver = resolvers.default.clone(); + let default_fut = async move { default_resolver.txt_lookup(&host).await }; + race_lookups(system_fut, default_fut, "txt_lookup").await }) }) .await?; @@ -113,29 +153,17 @@ impl Resolver { /// /// Blocks on a DNS query; the first call also builds and caches the process-wide resolvers. /// - /// Errors with [`ResolverError::Generic`] when the default resolver's lookup fails or returns no records; a system-resolver failure is logged and falls through to the default. + /// Errors with [`ResolverError::Generic`] when every resolver's lookup fails or returns no records. pub async fn ptr_lookup(ip_addr: IpAddr) -> Result { let ptr_result = with_resolvers(|resolvers| { Box::pin(async move { - // Try the system resolver config - if let Some(system_resolver) = &resolvers.system { - match system_resolver.reverse_lookup(ip_addr).await { - Ok(v) => { - return Ok(v); - } - Err(e) => { - debug!("system resolver ptr_lookup error: {}", e); - } - } - } - - match resolvers.default.reverse_lookup(ip_addr).await { - Ok(v) => Ok(v), - Err(e) => Err(ResolverError::Generic(format!( - "default resolver ptr_lookup error: {}", - e - ))), - } + let system_fut = resolvers + .system + .clone() + .map(|r| async move { r.reverse_lookup(ip_addr).await }); + let default_resolver = resolvers.default.clone(); + let default_fut = async move { default_resolver.reverse_lookup(ip_addr).await }; + race_lookups(system_fut, default_fut, "ptr_lookup").await }) }) .await?; diff --git a/veilid-tools/src/socket_tools.rs b/veilid-tools/src/socket_tools.rs index 86667fc4..bf15784a 100644 --- a/veilid-tools/src/socket_tools.rs +++ b/veilid-tools/src/socket_tools.rs @@ -407,6 +407,156 @@ pub fn primary_source_ipv6() -> Option { } } +// Send a single UDP datagram with a per-packet TTL override +cfg_if! { + if #[cfg(unix)] { + /// Send one UDP datagram with a best-effort per-packet TTL (IPv4) or hop + /// limit (IPv6) override, never touching the socket's TTL option: + /// - IPv6: `IPV6_HOPLIMIT` cmsg (RFC 3542) + /// - IPv4 on Linux/Android: `IP_TTL` cmsg + /// - IPv4 elsewhere: sent at the socket's default TTL; no per-packet TTL + /// cmsg exists (xnu honors only `IP_PKTINFO`) and a setsockopt swap would + /// race other senders on the shared socket + /// + /// Single nonblocking attempt; a full send buffer surfaces as `WouldBlock`. + pub fn send_udp_datagram_with_ttl( + socket: &S, + data: &[u8], + remote_addr: SocketAddr, + ttl: u32, + ) -> io::Result { + cfg_if! { + if #[cfg(any(target_os = "linux", target_os = "android"))] { + let (cmsg_level, cmsg_type) = match remote_addr { + SocketAddr::V4(_) => (libc::IPPROTO_IP, libc::IP_TTL), + SocketAddr::V6(_) => (libc::IPPROTO_IPV6, libc::IPV6_HOPLIMIT), + }; + sendmsg_with_ttl_cmsg(socket.as_raw_fd(), data, remote_addr, cmsg_level, cmsg_type, ttl) + } else { + match remote_addr { + SocketAddr::V4(_) => socket2_operation(socket, |s| s.send_to(data, &remote_addr.into())), + SocketAddr::V6(_) => sendmsg_with_ttl_cmsg(socket.as_raw_fd(), data, remote_addr, libc::IPPROTO_IPV6, libc::IPV6_HOPLIMIT, ttl), + } + } + } + } + + fn sendmsg_with_ttl_cmsg( + fd: std::os::fd::RawFd, + data: &[u8], + remote_addr: SocketAddr, + cmsg_level: libc::c_int, + cmsg_type: libc::c_int, + ttl: u32, + ) -> io::Result { + let sockaddr = SockAddr::from(remote_addr); + let mut iov = libc::iovec { + iov_base: data.as_ptr() as *mut libc::c_void, + iov_len: data.len(), + }; + // u64 array for cmsg alignment; larger than CMSG_SPACE(sizeof(int)) everywhere + let mut cmsg_buf = [0u64; 8]; + unsafe { + let mut msg: libc::msghdr = std::mem::zeroed(); + msg.msg_name = sockaddr.as_ptr() as *mut libc::c_void; + msg.msg_namelen = sockaddr.len(); + msg.msg_iov = &mut iov; + msg.msg_iovlen = 1; + msg.msg_control = cmsg_buf.as_mut_ptr() as *mut libc::c_void; + msg.msg_controllen = libc::CMSG_SPACE(std::mem::size_of::() as libc::c_uint) as _; + let cmsg = libc::CMSG_FIRSTHDR(&msg); + (*cmsg).cmsg_level = cmsg_level; + (*cmsg).cmsg_type = cmsg_type; + (*cmsg).cmsg_len = libc::CMSG_LEN(std::mem::size_of::() as libc::c_uint) as _; + std::ptr::write_unaligned(libc::CMSG_DATA(cmsg).cast::(), ttl as libc::c_int); + let res = libc::sendmsg(fd, &msg, 0); + if res < 0 { + return Err(io::Error::last_os_error()); + } + Ok(res as usize) + } + } + } else if #[cfg(windows)] { + /// Send one UDP datagram with a best-effort per-packet TTL (IPv4) or hop + /// limit (IPv6) override, never touching the socket's TTL option: + /// - IPv6: `IPV6_HOPLIMIT` cmsg via `WSASendMsg` + /// - IPv4: sent at the socket's default TTL; `WSASendMsg` has no TTL cmsg + /// and a setsockopt swap would race other senders on the shared socket + /// + /// Single nonblocking attempt; a full send buffer surfaces as `WouldBlock`. + pub fn send_udp_datagram_with_ttl( + socket: &S, + data: &[u8], + remote_addr: SocketAddr, + ttl: u32, + ) -> io::Result { + match remote_addr { + SocketAddr::V4(_) => socket2_operation(socket, |s| s.send_to(data, &remote_addr.into())), + SocketAddr::V6(_) => wsasendmsg_with_hoplimit(socket, data, remote_addr, ttl), + } + } + + fn wsasendmsg_with_hoplimit( + socket: &S, + data: &[u8], + remote_addr: SocketAddr, + ttl: u32, + ) -> io::Result { + use winapi::shared::minwindef::DWORD; + use winapi::shared::ws2def::{IPPROTO_IPV6, WSABUF, WSACMSGHDR, WSAMSG}; + use winapi::shared::ws2ipdef::IPV6_HOPLIMIT; + use winapi::um::winsock2::{WSASendMsg, SOCKET, SOCKET_ERROR}; + + // WSA_CMSG_SPACE(sizeof(int)) layout by hand; winapi has no cmsg macros + #[repr(C)] + struct CmsgHopLimit { + hdr: WSACMSGHDR, + value: i32, + } + + let sockaddr = SockAddr::from(remote_addr); + let mut wsabuf = WSABUF { + len: data.len() as u32, + buf: data.as_ptr() as *mut i8, + }; + let mut cmsg = CmsgHopLimit { + hdr: WSACMSGHDR { + cmsg_len: std::mem::size_of::() + std::mem::size_of::(), + cmsg_level: IPPROTO_IPV6 as i32, + cmsg_type: IPV6_HOPLIMIT, + }, + value: ttl as i32, + }; + let mut msg = WSAMSG { + name: sockaddr.as_ptr() as *mut _, + namelen: sockaddr.len(), + lpBuffers: &mut wsabuf, + dwBufferCount: 1, + Control: WSABUF { + len: std::mem::size_of::() as u32, + buf: (&mut cmsg as *mut CmsgHopLimit) as *mut i8, + }, + dwFlags: 0, + }; + let mut sent: DWORD = 0; + let res = unsafe { + WSASendMsg( + socket.as_raw_socket() as SOCKET, + &mut msg, + 0, + &mut sent, + std::ptr::null_mut(), + None, + ) + }; + if res == SOCKET_ERROR { + return Err(io::Error::last_os_error()); + } + Ok(sent as usize) + } + } +} + // Run operations on underlying socket cfg_if! { if #[cfg(unix)] { diff --git a/veilid-tools/src/tests/native/mod.rs b/veilid-tools/src/tests/native/mod.rs index 1d9b0f3e..f687faea 100644 --- a/veilid-tools/src/tests/native/mod.rs +++ b/veilid-tools/src/tests/native/mod.rs @@ -5,6 +5,7 @@ mod test_assembly_buffer; mod test_async_peek_stream; mod test_igd; mod test_network_interfaces; +mod test_udp_ttl_send; use super::*; @@ -78,6 +79,8 @@ pub async fn run_all_tests() { } info!("TEST: exec_test_assembly_buffer"); test_assembly_buffer::test_all().await; + info!("TEST: exec_test_udp_ttl_send"); + test_udp_ttl_send::test_all().await; info!("TEST: exec_test_tracked_mutex"); test_tracked_mutex::test_all(); @@ -274,6 +277,14 @@ cfg_if! { }); } + #[test] + fn run_test_udp_ttl_send() { + setup(); + block_on(async { + test_udp_ttl_send::test_all().await; + }); + } + #[test] fn run_test_tracked_mutex() { setup(); diff --git a/veilid-tools/src/tests/native/test_udp_ttl_send.rs b/veilid-tools/src/tests/native/test_udp_ttl_send.rs new file mode 100644 index 00000000..97dab6af --- /dev/null +++ b/veilid-tools/src/tests/native/test_udp_ttl_send.rs @@ -0,0 +1,63 @@ +use crate::*; + +async fn test_send_with_ttl(local: SocketAddr) { + info!("-- test_send_with_ttl {}", local); + + let receiver = bind_async_udp_socket(local) + .expect("receiver bind should not error") + .expect("receiver bind should succeed"); + let recv_addr = receiver.local_addr().expect("should have local addr"); + let sender = bind_async_udp_socket(local) + .expect("sender bind should not error") + .expect("sender bind should succeed"); + + let read_ttl = || { + socket2_operation(&sender, |s| { + if recv_addr.is_ipv6() { + s.unicast_hops_v6() + } else { + s.ttl_v4() + } + }) + .expect("should get ttl") + }; + + let ttl_before = read_ttl(); + + let sent = + send_udp_datagram_with_ttl(&sender, &[], recv_addr, 3).expect("zero-length send with ttl"); + assert_eq!(sent, 0); + let sent = + send_udp_datagram_with_ttl(&sender, b"punch", recv_addr, 3).expect("payload send with ttl"); + assert_eq!(sent, 5); + + // Receive both datagrams; delivery order is not guaranteed + let mut lens = Vec::new(); + let mut buf = [0u8; 64]; + for _ in 0..2 { + let (n, _) = timeout(5000, receiver.recv_from(&mut buf)) + .await + .expect("should not time out") + .expect("should receive"); + if n == 5 { + assert_eq!(&buf[..5], b"punch"); + } + lens.push(n); + } + lens.sort(); + assert_eq!(lens, vec![0, 5]); + + let ttl_after = read_ttl(); + assert_eq!(ttl_before, ttl_after); +} + +pub async fn test_all() { + test_send_with_ttl(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0))).await; + test_send_with_ttl(SocketAddr::V6(SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ))) + .await; +} diff --git a/veilid-wasm/src/veilid_client_js.rs b/veilid-wasm/src/veilid_client_js.rs index a6410b78..3ad7f216 100644 --- a/veilid-wasm/src/veilid_client_js.rs +++ b/veilid-wasm/src/veilid_client_js.rs @@ -119,6 +119,13 @@ impl VeilidClient { .into() } + /// The largest DHT value sequence number. Subtract `n` for a `minSeqnum` that leaves `n` writes + #[wasm_bindgen(getter, unchecked_return_type = "ValueSeqNum")] + #[must_use] + pub fn VALUE_SEQ_NUM_MAX() -> JsValue { + JsValue::from(u32::from(ValueSeqNum::MAX)) + } + /////////////////////////////////////////////////////////////////////////////////////////// /// Initialize the WASM platform: panic hook and tracing layers from the platform config. diff --git a/veilid-wasm/src/wasm_helpers.rs b/veilid-wasm/src/wasm_helpers.rs index 5a9d9e8c..e13f4e96 100644 --- a/veilid-wasm/src/wasm_helpers.rs +++ b/veilid-wasm/src/wasm_helpers.rs @@ -111,12 +111,16 @@ pub mod ts { pub struct SetDHTValueOptions { /// Override writer key pair for this operation #[tsify(type = "KeyPair", optional)] - #[serde(with = "serde_wasm_bindgen::preserve")] + #[serde(default, with = "serde_wasm_bindgen::preserve")] pub writer: JsValue, /// Defaults to true. If false, the value will not be written if the node is offline, /// and a TryAgain error will be returned. #[tsify(optional)] pub allow_offline: Option, + /// Raise the sequence number to at least this, after the automatic increment. + /// `veilidClient.VALUE_SEQ_NUM_MAX` minus `n` leaves `n` writes after this one. + #[tsify(optional)] + pub min_seqnum: Option, } /// Options that override defaults for transact_dht_records @@ -133,11 +137,16 @@ pub mod ts { /// Options that override defaults for DHTTransaction::set #[derive(Serialize, Deserialize, Clone, Tsify)] #[tsify(from_wasm_abi, into_wasm_abi)] + #[serde(rename_all = "camelCase")] pub struct DHTTransactionSetValueOptions { /// Override writer key pair for this operation #[tsify(type = "KeyPair", optional)] - #[serde(with = "serde_wasm_bindgen::preserve")] + #[serde(default, with = "serde_wasm_bindgen::preserve")] pub writer: JsValue, + /// Raise the sequence number to at least this, after the automatic increment. + /// `veilidClient.VALUE_SEQ_NUM_MAX` minus `n` leaves `n` writes after this one. + #[tsify(optional)] + pub min_seqnum: Option, } /// Simple DHT Schema (SMPL) Member @@ -315,6 +324,7 @@ impl TryFrom for SetDHTValueOptions { Ok(SetDHTValueOptions { writer, allow_offline, + min_seqnum: value.min_seqnum, }) } } @@ -338,7 +348,10 @@ impl TryFrom for DHTTransactionSetValueOption fn try_from(value: ts::DHTTransactionSetValueOptions) -> Result { let writer = ts::try_from_js_option::(value.writer.clone()) .map_err(VeilidAPIError::generic)?; - Ok(DHTTransactionSetValueOptions { writer }) + Ok(DHTTransactionSetValueOptions { + writer, + min_seqnum: value.min_seqnum, + }) } } diff --git a/veilid-wasm/tests/src/suites/dht.ts b/veilid-wasm/tests/src/suites/dht.ts index 19220fe1..821e4d7a 100644 --- a/veilid-wasm/tests/src/suites/dht.ts +++ b/veilid-wasm/tests/src/suites/dht.ts @@ -4,6 +4,7 @@ import { KeyPair, RecordKey, VeilidRoutingContext, + veilidClient, veilidCrypto, } from 'veilid-wasm'; import { textEncoder, textDecoder } from '../utils/marshalling-utils.js'; @@ -231,6 +232,39 @@ export const dhtSuite: Suite = { }), }); + cases.push({ + name: `should raise the sequence number to minSeqnum for ${cryptoKind}`, + run: withRecord(cryptoKind, async ({ expect }, rc, rec) => { + const max = veilidClient.VALUE_SEQ_NUM_MAX; + + // max - 1 leaves exactly one write + await dhtRetry( + () => rc.setDHTValue(rec.key, 0, textEncoder.encode(data), { minSeqnum: max - 1 }), + { label: 'setDHTValue with minSeqnum' }, + ); + + let inspectRes = await rc.inspectDHTRecord(rec.key, [[0, 0]], 'Local'); + expect(inspectRes.localSeqs).toEqual([max - 1]); + + await dhtRetry(() => rc.setDHTValue(rec.key, 0, textEncoder.encode(`${data}👋`)), { + label: 'setDHTValue last remaining write', + }); + + inspectRes = await rc.inspectDHTRecord(rec.key, [[0, 0]], 'Local'); + expect(inspectRes.localSeqs).toEqual([max]); + + // No retry: we're asserting the out-of-sequence-numbers rejection. + let error: { kind?: string; message?: string } | undefined; + try { + await rc.setDHTValue(rec.key, 0, textEncoder.encode(`${data}🙅`)); + } catch (e) { + error = e as { kind?: string; message?: string }; + } + expect(error?.kind).toEqual('Generic'); + expect(error?.message?.startsWith('sequence number at max:')).toBe(true); + }), + }); + cases.push({ name: `should flush a record and confirm no offline subkeys remain for ${cryptoKind}`, run: withRecord(cryptoKind, async ({ expect }, rc, rec) => {