IntenseWebs/veilid
3
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2024-11-21 00:24:14 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'geolocation-2' into 'main'

Browse Source 
IP geolocation, extend BucketEntry

See merge request veilid/veilid!331
This commit is contained in:
Christien Rioux 2024-11-12 17:11:51 +00:00
commit 995f078bd6
11 changed files with 240 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
veilid-core/src/logging/facilities.rs
View File

@ -1,4 +1,4 @@
pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: [&str; 29] = [
pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: &[&str] = &[
"mio",
"h2",
"hyper",
@ -28,9 +28,11 @@ pub static DEFAULT_LOG_FACILITIES_IGNORE_LIST: [&str; 29] = [
"fanout",
"receipt",
"rpc_message",
#[cfg(feature = "geolocation")]
"maxminddb",
];
pub static FLAME_LOG_FACILITIES_IGNORE_LIST: [&str; 22] = [
pub static FLAME_LOG_FACILITIES_IGNORE_LIST: &[&str] = &[
"mio",
"h2",
"hyper",
@ -53,9 +55,11 @@ pub static FLAME_LOG_FACILITIES_IGNORE_LIST: [&str; 22] = [
"hickory_proto",
"attohttpc",
"ws_stream_wasm",
#[cfg(feature = "geolocation")]
"maxminddb",
];
pub static DEFAULT_LOG_FACILITIES_ENABLED_LIST: [&str; 8] = [
pub static DEFAULT_LOG_FACILITIES_ENABLED_LIST: &[&str] = &[
"net",
"rpc",
"rtab",
@ -66,7 +70,7 @@ pub static DEFAULT_LOG_FACILITIES_ENABLED_LIST: [&str; 8] = [
"crypto",
];
pub static DURATION_LOG_FACILITIES: [&str; 1] = ["veilid_api"];
pub static DURATION_LOG_FACILITIES: &[&str] = &["veilid_api"];
#[macro_export]
macro_rules! fn_string {

10
veilid-core/src/logging/veilid_layer_filter.rs
View File

@ -19,8 +19,9 @@ impl VeilidLayerFilter {
ignore_change_list: &[String],
) -> VeilidLayerFilter {
let mut ignore_list = DEFAULT_LOG_FACILITIES_IGNORE_LIST
.map(|x| x.to_owned())
.to_vec();
.iter()
.map(|&x| x.to_owned())
.collect::<Vec<_>>();
Self::apply_ignore_change_list(&mut ignore_list, ignore_change_list);
Self {
inner: Arc::new(RwLock::new(VeilidLayerFilterInner {
@ -64,8 +65,9 @@ impl VeilidLayerFilter {
let mut inner = self.inner.write();
inner.ignore_list = ignore_list.unwrap_or_else(|| {
DEFAULT_LOG_FACILITIES_IGNORE_LIST
.map(|x| x.to_owned())
.to_vec()
.iter()
.map(|&x| x.to_owned())
.collect::<Vec<_>>()
});
}
callsite::rebuild_interest_cache();

24
veilid-core/src/routing_table/bucket_entry.rs
View File

@ -174,6 +174,10 @@ pub(crate) struct BucketEntryInner {
last_sender_info: HashMap<LastSenderInfoKey, SenderInfo>,
/// The node info for this entry on the publicinternet routing domain
public_internet: BucketEntryPublicInternet,
/// Node location
#[cfg(feature = "geolocation")]
#[serde(skip)]
geolocation_info: GeolocationInfo,
/// The node info for this entry on the localnetwork routing domain
local_network: BucketEntryLocalNetwork,
/// Statistics gathered for the peer
@ -461,6 +465,12 @@ impl BucketEntryInner {
self.updated_since_last_network_change = true;
self.make_not_dead(Timestamp::now());
// Update geolocation info
#[cfg(feature = "geolocation")]
{
self.geolocation_info = signed_node_info.get_geolocation_info(routing_domain);
}
// If we're updating an entry's node info, purge all
// but the last connection in our last connections list
// because the dial info could have changed and it's safer to just reconnect.
@ -473,6 +483,13 @@ impl BucketEntryInner {
node_info_changed
}
#[cfg(feature = "geolocation")]
pub(super) fn update_geolocation_info(&mut self) {
if let Some(ref sni) = self.public_internet.signed_node_info {
self.geolocation_info = sni.get_geolocation_info(RoutingDomain::PublicInternet);
}
}
pub fn has_node_info(&self, routing_domain_set: RoutingDomainSet) -> bool {
for routing_domain in routing_domain_set {
// Get the correct signed_node_info for the chosen routing domain
@ -726,6 +743,11 @@ impl BucketEntryInner {
self.state_reason(cur_ts).into()
}
#[cfg(feature = "geolocation")]
pub fn geolocation_info(&self) -> &GeolocationInfo {
&self.geolocation_info
}
pub fn set_punished(&mut self, punished: Option<PunishmentReason>) {
self.punishment = punished;
if punished.is_some() {
@ -1099,6 +1121,8 @@ impl BucketEntry {
signed_node_info: None,
node_status: None,
},
#[cfg(feature = "geolocation")]
geolocation_info: Default::default(),
peer_stats: PeerStats {
time_added: now,
rpc_stats: RPCStats::default(),

101
veilid-core/src/routing_table/debug.rs
View File

@ -144,45 +144,88 @@ impl RoutingTable {
e: &BucketEntryInner,
relay_tag: &str,
) -> String {
format!(
let state_reason = Self::format_state_reason(e.state_reason(cur_ts));
let average_latency = e
.peer_stats()
.latency
.as_ref()
.map(|l| l.to_string())
.unwrap_or_else(|| "???".to_string());
let capabilities = if let Some(ni) = e.node_info(RoutingDomain::PublicInternet) {
ni.capabilities()
.iter()
.map(|x| x.to_string())
.collect::<Vec<String>>()
.join(",")
} else {
"???".to_owned()
};
let since_last_question = e
.peer_stats()
.rpc_stats
.last_question_ts
.as_ref()
.map(|l| cur_ts.saturating_sub(*l).to_string())
.unwrap_or_else(|| "???".to_string());
let since_last_seen = e
.peer_stats()
.rpc_stats
.last_seen_ts
.as_ref()
.map(|l| cur_ts.saturating_sub(*l).to_string())
.unwrap_or_else(|| "???".to_string());
#[allow(unused_mut)]
let mut result = format!(
" {} [{}][{}] {} [{}] lastq@{} seen@{}",
// node id
node,
// state reason
Self::format_state_reason(e.state_reason(cur_ts)),
state_reason,
// Relay tag
relay_tag,
// average latency
e.peer_stats()
.latency
.as_ref()
.map(|l| l.to_string())
.unwrap_or_else(|| "???".to_string()),
average_latency,
// capabilities
if let Some(ni) = e.node_info(RoutingDomain::PublicInternet) {
ni.capabilities()
.iter()
.map(|x| x.to_string())
.collect::<Vec<String>>()
.join(",")
} else {
"???".to_owned()
},
capabilities,
// duration since last question
e.peer_stats()
.rpc_stats
.last_question_ts
.as_ref()
.map(|l| cur_ts.saturating_sub(*l).to_string())
.unwrap_or_else(|| "???".to_string()),
since_last_question,
// duration since last seen
e.peer_stats()
.rpc_stats
.last_seen_ts
.as_ref()
.map(|l| cur_ts.saturating_sub(*l).to_string())
.unwrap_or_else(|| "???".to_string()),
)
since_last_seen,
);
#[cfg(feature = "geolocation")]
{
let geolocation_info = e.geolocation_info();
if let Some(cc) = geolocation_info.country_code() {
result += &format!(" {cc}");
} else {
result += " ??";
}
if !geolocation_info.relay_country_codes().is_empty() {
result += "/";
}
for (i, cc) in geolocation_info.relay_country_codes().iter().enumerate() {
if i > 0 {
result += ",";
}
if let Some(cc) = cc {
result += &format!("{cc}");
} else {
result += "??";
}
}
}
result
}
pub fn debug_info_entries(

29
veilid-core/src/routing_table/geolocation.rs
View File

@ -1,5 +1,3 @@
#![allow(unused)]
use crate::CountryCode;
use maxminddb::MaxMindDBError;
use once_cell::sync::Lazy;
@ -63,6 +61,7 @@ pub fn query_country_code(addr: IpAddr) -> Option<CountryCode> {
mod tests {
use crate::CountryCode;
use core::str::FromStr;
use maxminddb::WithinItem;
#[test]
fn test_query_country_code() {
@ -91,4 +90,30 @@ mod tests {
assert!(super::query_country_code("10.0.0.1".parse().unwrap()).is_none());
assert!(super::query_country_code("::1".parse().unwrap()).is_none());
}
#[test]
fn test_iter_over_ipv4_mmdb() {
let db = super::IPV4.as_ref().unwrap();
let count = db
.within("0.0.0.0/0".parse().unwrap())
.unwrap()
.map(|item: Result<WithinItem<super::Country>, _>| item.unwrap())
.count();
assert!(count > 100, "Expecting some IPv4 subnets in IPv4 MMDB");
}
#[test]
fn test_iter_over_ipv6_mmdb() {
let db = super::IPV6.as_ref().unwrap();
let count = db
.within("::/0".parse().unwrap())
.unwrap()
.map(|item: Result<WithinItem<super::Country>, _>| item.unwrap())
.count();
assert!(count > 100, "Expecting some IPv6 subnets in IPv6 MMDB");
}
}

9
veilid-core/src/routing_table/mod.rs
View File

@ -493,8 +493,15 @@ impl RoutingTable {
) -> EyreResult<()> {
let mut all_entries: Vec<Arc<BucketEntry>> = Vec::with_capacity(all_entry_bytes.len());
for entry_bytes in all_entry_bytes {
let entryinner = deserialize_json_bytes(&entry_bytes)
#[allow(unused_mut)]
let mut entryinner: BucketEntryInner = deserialize_json_bytes(&entry_bytes)
.wrap_err("failed to deserialize bucket entry")?;
#[cfg(feature = "geolocation")]
{
entryinner.update_geolocation_info();
}
let entry = Arc::new(BucketEntry::new_with_inner(entryinner));
// Keep strong reference in table

31
veilid-core/src/routing_table/types/geolocation_info.rs Normal file
View File

@ -0,0 +1,31 @@
use super::*;
#[derive(Debug, Default)]
pub struct GeolocationInfo {
country_code: Option<CountryCode>,
relay_country_codes: Vec<Option<CountryCode>>,
}
impl GeolocationInfo {
pub fn new(
country_code: Option<CountryCode>,
relay_country_codes: Vec<Option<CountryCode>>,
) -> Self {
GeolocationInfo {
country_code,
relay_country_codes,
}
}
/// Get node country code. Might be `None` if unable to determine.
pub fn country_code(&self) -> Option<CountryCode> {
self.country_code
}
/// Get country codes of relays used by the node.
/// There will be exactly one entry for each relay.
/// Empty if no relays are used.
pub fn relay_country_codes(&self) -> &[Option<CountryCode>] {
&self.relay_country_codes
}
}

4
veilid-core/src/routing_table/types/mod.rs
View File

@ -1,6 +1,8 @@
mod contact_method;
mod dial_info_detail;
mod direction;
#[cfg(feature = "geolocation")]
mod geolocation_info;
mod node_info;
mod node_status;
mod peer_info;
@ -14,6 +16,8 @@ use super::*;
pub use contact_method::*;
pub use dial_info_detail::*;
pub use direction::*;
#[cfg(feature = "geolocation")]
pub use geolocation_info::*;
pub use node_info::*;
pub use node_status::*;
pub use peer_info::*;

48
veilid-core/src/routing_table/types/signed_node_info.rs
View File

@ -115,6 +115,54 @@ impl SignedNodeInfo {
.unwrap_or_default();
}
#[cfg(feature = "geolocation")]
/// Get geolocation info of node and its relays.
pub fn get_geolocation_info(&self, routing_domain: RoutingDomain) -> GeolocationInfo {
if routing_domain != RoutingDomain::PublicInternet {
// Country code is irrelevant for local network
return GeolocationInfo::new(None, vec![]);
}
let get_node_country_code = |node_info: &NodeInfo| {
let country_codes = node_info
.dial_info_detail_list()
.iter()
.map(|did| match &did.dial_info {
DialInfo::UDP(di) => di.socket_address.ip_addr(),
DialInfo::TCP(di) => di.socket_address.ip_addr(),
DialInfo::WS(di) => di.socket_address.ip_addr(),
DialInfo::WSS(di) => di.socket_address.ip_addr(),
})
.map(geolocation::query_country_code)
.collect::<Vec<_>>();
if country_codes.is_empty() {
return None;
}
// Indexing cannot panic, guarded by a check above
let cc0 = country_codes[0];
if !country_codes.iter().all(|cc| cc.is_some() && *cc == cc0) {
// Lookup failed for some address or results are different
return None;
}
cc0
};
match self {
SignedNodeInfo::Direct(sni) => {
GeolocationInfo::new(get_node_country_code(sni.node_info()), vec![])
}
SignedNodeInfo::Relayed(sni) => {
let relay_cc = get_node_country_code(sni.relay_info().node_info());
GeolocationInfo::new(get_node_country_code(sni.node_info()), vec![relay_cc])
}
}
}
/// Compare this SignedNodeInfo to another one
/// Exclude the signature and timestamp and any other fields that are not
/// semantically valuable

5
veilid-flutter/rust/src/dart_ffi.rs
View File

@ -321,7 +321,10 @@ pub extern "C" fn initialize_veilid_core(platform_config: FfiStr) {
if platform_config.logging.flame.enabled {
let filter = veilid_core::VeilidLayerFilter::new_no_default(
veilid_core::VeilidConfigLogLevel::Trace,
&veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST.map(|x| x.to_string()),
&veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST
.iter()
.map(|&x| x.to_string())
.collect::<Vec<_>>(),
);
let (flame_layer, guard) =
FlameLayer::with_file(&platform_config.logging.flame.path).unwrap();

10
veilid-server/src/veilid_logs.rs
View File

@ -95,7 +95,10 @@ impl VeilidLogs {
if settingsr.logging.flame.enabled {
let filter = veilid_core::VeilidLayerFilter::new_no_default(
veilid_core::VeilidConfigLogLevel::Trace,
&veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST.map(|x| x.to_string()),
&veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST
.iter()
.map(|&x| x.to_string())
.collect::<Vec<_>>(),
);
let (flame_layer, guard) = FlameLayer::with_file(&settingsr.logging.flame.path)?;
flame_guard = Some(guard);
@ -115,7 +118,10 @@ impl VeilidLogs {
if settingsr.logging.perfetto.enabled {
let filter = veilid_core::VeilidLayerFilter::new_no_default(
veilid_core::VeilidConfigLogLevel::Trace,
&veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST.map(|x| x.to_string()),
&veilid_core::FLAME_LOG_FACILITIES_IGNORE_LIST
.iter()
.map(|&x| x.to_string())
.collect::<Vec<_>>(),
);
let perfetto_layer = PerfettoLayer::new(std::sync::Mutex::new(std::fs::File::create(
&settingsr.logging.perfetto.path,