diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 8197b3cd..365108b7 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -1054,6 +1054,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.0" @@ -2577,6 +2583,18 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", + "serde", + "serde_derive", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -4023,6 +4041,7 @@ dependencies = [ "ip_network_table", "lqos_bus", "lqos_config", + "petgraph", "serde", "serde_cbor", "serde_json", diff --git a/src/rust/lqos_queue_tracker/src/queue_structure/queue_node.rs b/src/rust/lqos_queue_tracker/src/queue_structure/queue_node.rs index bea56490..61e14a04 100644 --- a/src/rust/lqos_queue_tracker/src/queue_structure/queue_node.rs +++ b/src/rust/lqos_queue_tracker/src/queue_structure/queue_node.rs @@ -3,7 +3,7 @@ use lqos_bus::TcHandle; use lqos_utils::hash_to_i64; use lqos_utils::hex_string::read_hex_string; use serde_json::Value; -use tracing::{error, warn}; +use tracing::{error, info, warn}; #[derive(Default, Clone, Debug)] pub struct QueueNode { @@ -238,7 +238,7 @@ impl QueueNode { "idForCircuitsWithoutParentNodes" | "type" => { // Ignore } - _ => error!("I don't know how to parse key: [{key}]"), + _ => info!("I don't know how to parse key: [{key}]"), } } } else { diff --git a/src/rust/uisp/src/lib.rs b/src/rust/uisp/src/lib.rs index 8b1b0cf7..d9e993b5 100644 --- a/src/rust/uisp/src/lib.rs +++ b/src/rust/uisp/src/lib.rs @@ -42,7 +42,7 @@ pub async fn load_all_sites(config: Arc) -> Result> { /// Load all devices from UISP that are authorized, and include their full interface definitions pub async fn load_all_devices_with_interfaces(config: Arc) -> Result> { Ok(nms_request_get_vec( - "devices?withInterfaces=true&authorized=true", + "devices?withInterfaces=true", &config.uisp_integration.token, &config.uisp_integration.url, ) diff --git a/src/rust/uisp_integration/Cargo.toml b/src/rust/uisp_integration/Cargo.toml index f9c0400b..a9cdd794 100644 --- a/src/rust/uisp_integration/Cargo.toml +++ b/src/rust/uisp_integration/Cargo.toml @@ -19,4 +19,5 @@ serde_json = { workspace = true } ip_network_table = { workspace = true } ip_network = { workspace = true } serde_cbor = { workspace = true } -lqos_bus = { path = "../lqos_bus" } \ No newline at end of file +lqos_bus = { path = "../lqos_bus" } +petgraph = { version = "0.7.1", features = ["serde-1"] } diff --git a/src/rust/uisp_integration/src/blackboard.rs b/src/rust/uisp_integration/src/blackboard.rs new file mode 100644 index 00000000..7f15b3b4 --- /dev/null +++ b/src/rust/uisp_integration/src/blackboard.rs @@ -0,0 +1,47 @@ +//! Collects the blackboard functionality together + +use lqos_bus::BlackboardSystem; +use serde::Serialize; +use tracing::info; + +pub async fn blackboard(subsystem: BlackboardSystem, key: &str, value: &str) { + let Ok(config) = lqos_config::load_config() else { + return; + }; + if !config.long_term_stats.use_insight.unwrap_or(false) { + return; + } + let req = vec![lqos_bus::BusRequest::BlackboardData { + subsystem, + key: key.to_string(), + value: value.to_string(), + }]; + let _ = lqos_bus::bus_request(req).await; +} + +pub async fn blackboard_blob(key: &str, value: T) -> anyhow::Result<()> { + let config = lqos_config::load_config()?; + if !config.long_term_stats.use_insight.unwrap_or(false) { + return Ok(()); + } + let blob = serde_cbor::to_vec(&value)?; + let chunks = blob.chunks(1024 * 16); + info!( + "Blob {key} is {} bytes long, split into {} chunks", + blob.len(), + chunks.len() + ); + for (i, chunk) in chunks.enumerate() { + let req = vec![lqos_bus::BusRequest::BlackboardBlob { + tag: key.to_string(), + part: i, + blob: chunk.to_vec(), + }]; + let err = lqos_bus::bus_request(req).await; + #[cfg(debug_assertions)] + if let Err(e) = err { + tracing::error!("Error writing to blackboard (only an error if lqosd is running): {e:?}"); + } + } + Ok(()) +} diff --git a/src/rust/uisp_integration/src/main.rs b/src/rust/uisp_integration/src/main.rs index 173c76dc..419f706e 100644 --- a/src/rust/uisp_integration/src/main.rs +++ b/src/rust/uisp_integration/src/main.rs @@ -2,6 +2,7 @@ //! be ported back to Python, with Rust support structures - but I'll iterate //! faster in Rust. +mod blackboard; #[warn(missing_docs)] mod errors; pub mod ip_ranges; @@ -10,9 +11,8 @@ pub mod uisp_types; use crate::errors::UispIntegrationError; use crate::ip_ranges::IpRanges; -use lqos_bus::BlackboardSystem; +pub use blackboard::*; use lqos_config::Config; -use serde::Serialize; use tokio::time::Instant; use tracing::{error, info}; @@ -35,44 +35,6 @@ fn check_enabled_status(config: &Config) -> Result<(), UispIntegrationError> { } } -pub async fn blackboard(subsystem: BlackboardSystem, key: &str, value: &str) { - let Ok(config) = lqos_config::load_config() else { - return; - }; - if !config.long_term_stats.use_insight.unwrap_or(false) { - return; - } - let req = vec![lqos_bus::BusRequest::BlackboardData { - subsystem, - key: key.to_string(), - value: value.to_string(), - }]; - let _ = lqos_bus::bus_request(req).await; -} - -pub async fn blackboard_blob(key: &str, value: T) -> anyhow::Result<()> { - let config = lqos_config::load_config()?; - if !config.long_term_stats.use_insight.unwrap_or(false) { - return Ok(()); - } - let blob = serde_cbor::to_vec(&value)?; - let chunks = blob.chunks(1024 * 16); - info!( - "Blob {key} is {} bytes long, split into {} chunks", - blob.len(), - chunks.len() - ); - for (i, chunk) in chunks.enumerate() { - let req = vec![lqos_bus::BusRequest::BlackboardBlob { - tag: key.to_string(), - part: i, - blob: chunk.to_vec(), - }]; - let _ = lqos_bus::bus_request(req).await; - } - Ok(()) -} - #[tokio::main] async fn main() -> Result<(), UispIntegrationError> { let now = Instant::now(); diff --git a/src/rust/uisp_integration/src/strategies/ap_only.rs b/src/rust/uisp_integration/src/strategies/ap_only.rs new file mode 100644 index 00000000..4ca1761f --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/ap_only.rs @@ -0,0 +1,118 @@ +use crate::errors::UispIntegrationError; +use crate::ip_ranges::IpRanges; +use crate::strategies::common::UispData; +use crate::strategies::full::shaped_devices_writer::ShapedDevice; +use lqos_config::Config; +use std::fs::write; +use std::path::Path; +use std::sync::Arc; +use tracing::{error, info, warn}; + +/// Creates a network with only APs detected +/// from clients. +pub async fn build_ap_only_network( + config: Arc, + ip_ranges: IpRanges, +) -> Result<(), UispIntegrationError> { + let uisp_data = UispData::fetch_uisp_data(config.clone(), ip_ranges).await?; + + // Find the clients + let mappings = uisp_data.map_clients_to_aps(); + + // Write network.json + let network_path = Path::new(&config.lqos_directory).join("network.json"); + if network_path.exists() && !config.integration_common.always_overwrite_network_json { + warn!( + "Network.json exists, and always overwrite network json is not true - not writing network.json" + ); + return Ok(()); + } + let mut root = serde_json::Map::new(); + for ap in mappings.keys() { + if let Some(ap_device) = uisp_data.devices.iter().find(|d| d.name == *ap) { + let mut ap_object = serde_json::Map::new(); + // Empy children + ap_object.insert("children".to_string(), serde_json::Map::new().into()); + + // Limits + ap_object.insert( + "downloadBandwidthMbps".to_string(), + serde_json::Value::Number(ap_device.download.into()), + ); + ap_object.insert( + "uploadBandwidthMbps".to_string(), + serde_json::Value::Number(ap_device.upload.into()), + ); + + // Metadata + ap_object.insert("type".to_string(), "AP".to_string().into()); + ap_object.insert("uisp_device".to_string(), ap_device.id.clone().into()); + + // Save the entry + root.insert(ap.to_string(), ap_object.into()); + } + } + let json = serde_json::to_string_pretty(&root).unwrap(); + write(network_path, json).map_err(|e| { + error!("Unable to write network.json"); + error!("{e:?}"); + UispIntegrationError::WriteNetJson + })?; + info!("Written network.json"); + + // Write ShapedDevices.csv + let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); + let mut shaped_devices = Vec::new(); + for (parent, client_ids) in mappings.iter() { + for client_id in client_ids { + let site = uisp_data.sites.iter().find(|s| *client_id == s.id).unwrap(); + let devices = uisp_data + .devices + .iter() + .filter(|d| d.site_id == *client_id) + .collect::>(); + for device in devices.iter().filter(|d| d.has_address()) { + let sd = ShapedDevice { + circuit_id: site.id.clone(), + circuit_name: site.name.clone(), + device_id: device.id.clone(), + device_name: device.name.clone(), + parent_node: parent.clone(), + mac: device.mac.clone(), + ipv4: device.ipv4_list(), + ipv6: device.ipv6_list(), + download_min: (site.max_down_mbps as f32 + * config.uisp_integration.commit_bandwidth_multiplier) + as u64, + upload_min: (site.max_up_mbps as f32 + * config.uisp_integration.commit_bandwidth_multiplier) + as u64, + download_max: (site.max_down_mbps as f32 + * config.uisp_integration.bandwidth_overhead_factor) + as u64, + upload_max: (site.max_up_mbps as f32 + * config.uisp_integration.bandwidth_overhead_factor) + as u64, + comment: "".to_string(), + }; + shaped_devices.push(sd); + } + } + } + let mut writer = csv::WriterBuilder::new() + .has_headers(true) + .from_path(file_path) + .unwrap(); + + for d in shaped_devices.iter() { + writer.serialize(d).unwrap(); + } + writer.flush().map_err(|e| { + error!("Unable to flush CSV file"); + error!("{e:?}"); + UispIntegrationError::CsvError + })?; + info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); + + Ok(()) +} diff --git a/src/rust/uisp_integration/src/strategies/ap_site.rs b/src/rust/uisp_integration/src/strategies/ap_site.rs new file mode 100644 index 00000000..3533337c --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/ap_site.rs @@ -0,0 +1,357 @@ +use crate::blackboard_blob; +use crate::errors::UispIntegrationError; +use crate::ip_ranges::IpRanges; +use crate::strategies::common::UispData; +use crate::strategies::full::shaped_devices_writer::ShapedDevice; +use lqos_config::Config; +use std::collections::{HashMap, HashSet}; +use std::fs::write; +use std::path::Path; +use std::sync::Arc; +use tracing::{error, info, warn}; + +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone)] +pub enum GraphMapping { + Root, + SiteByName(String), + //GeneratedSiteByName(String), + AccessPointByName(String), + ClientById(String), +} + +/// Creates a network with APs detected from clients, +/// and then a single site above them (shared if the site +/// matches). +pub async fn build_ap_site_network( + config: Arc, + ip_ranges: IpRanges, +) -> Result<(), UispIntegrationError> { + let uisp_data = UispData::fetch_uisp_data(config.clone(), ip_ranges).await?; + + // Find trouble-spots! + let _trouble = find_troublesome_sites(&uisp_data).await.map_err(|e| { + error!("Error finding troublesome sites"); + error!("{e:?}"); + UispIntegrationError::UnknownSiteType + })?; + + // Find the clients + let ap_mappings = uisp_data.map_clients_to_aps(); + + // Make AP Layer entries + let access_points = get_ap_layer(&ap_mappings); + + // Site mappings + let sites = map_sites_above_aps(&uisp_data, ap_mappings, access_points); + + // Insert the root + let root = Layer { + id: GraphMapping::Root, + children: sites.values().cloned().collect(), + }; + //println!("{:#?}", root); + + let mut shaped_devices = Vec::new(); + let net_json = root.walk_children(None, &uisp_data, &mut shaped_devices, &config); + + let network_path = Path::new(&config.lqos_directory).join("network.json"); + if network_path.exists() && !config.integration_common.always_overwrite_network_json { + warn!( + "Network.json exists, and always overwrite network json is not true - not writing network.json" + ); + return Ok(()); + } + let json = serde_json::to_string_pretty(&net_json).unwrap(); + write(network_path, json).map_err(|e| { + error!("Unable to write network.json"); + error!("{e:?}"); + UispIntegrationError::WriteNetJson + })?; + info!("Written network.json"); + + let _ = write_shaped_devices(&config, &mut shaped_devices); + info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); + + Ok(()) +} + +pub(crate) fn write_shaped_devices( + config: &Arc, + shaped_devices: &mut Vec, +) -> Result<(), UispIntegrationError> { + let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); + let mut writer = csv::WriterBuilder::new() + .has_headers(true) + .from_path(file_path) + .unwrap(); + + for d in shaped_devices.iter() { + writer.serialize(d).unwrap(); + } + writer.flush().map_err(|e| { + error!("Unable to flush CSV file"); + error!("{e:?}"); + UispIntegrationError::CsvError + })?; + info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); + Ok(()) +} + +pub(crate) fn map_sites_above_aps( + uisp_data: &UispData, + ap_mappings: HashMap>, + access_points: HashMap, +) -> HashMap { + let mut sites = HashMap::new(); + for (ap_name, client_ids) in ap_mappings.iter() { + if let Some(device) = uisp_data.find_device_by_name(ap_name) { + if let Some(device_site_id) = device.get_site_id() { + if let Some(device_site) = uisp_data.sites.iter().find(|s| s.id == device_site_id) { + let site_entry = + sites + .entry(device_site.name.clone()) + .or_insert_with(|| Layer { + id: GraphMapping::SiteByName(device_site.name.clone()), + children: Vec::new(), + }); + let ap_map = access_points.get(ap_name).unwrap().clone(); + site_entry.children.push(ap_map); + } + } + } else { + let mut detached = Layer { + id: GraphMapping::SiteByName(ap_name.clone()), + children: vec![], + }; + for client_id in client_ids.iter() { + detached.children.push(Layer { + id: GraphMapping::ClientById(client_id.clone()), + children: vec![], + }); + } + sites.insert(ap_name.clone(), detached); + } + } + sites +} + +pub(crate) fn get_ap_layer(ap_mappings: &HashMap>) -> HashMap { + let mut access_points = HashMap::new(); + for (ap_name, client_ids) in ap_mappings.iter() { + let mut ap_layer = Layer { + id: GraphMapping::AccessPointByName(ap_name.clone()), + children: Vec::new(), + }; + for client_id in client_ids.iter() { + ap_layer.children.push(Layer { + id: GraphMapping::ClientById(client_id.clone()), + children: Vec::new(), + }); + } + access_points.insert(ap_name.clone(), ap_layer); + } + access_points +} + +#[derive(Debug, Clone)] +pub(crate) struct Layer { + pub(crate) id: GraphMapping, + pub(crate) children: Vec, +} + +impl Layer { + pub(crate) fn walk_children( + &self, + parent: Option<&str>, + uisp_data: &UispData, + shaped_devices: &mut Vec, + config: &Config, + ) -> serde_json::Map { + let mut children = serde_json::Map::new(); + let parent_name = match &self.id { + GraphMapping::SiteByName(name) | GraphMapping::AccessPointByName(name) => { + name.to_owned() + } + _ => "".to_owned(), + }; + for child in self.children.iter() { + match &child.id { + GraphMapping::SiteByName(name) | GraphMapping::AccessPointByName(name) => { + children.insert( + name.clone(), + child + .walk_children(Some(&parent_name), uisp_data, shaped_devices, config) + .into(), + ); + } + GraphMapping::ClientById(_client_id) => { + let _ = + child.walk_children(Some(&parent_name), uisp_data, shaped_devices, config); + } + _ => {} + } + } + let mut root = serde_json::Map::new(); + if let Some(parent) = parent { + match &self.id { + GraphMapping::SiteByName(name) => { + root.insert("type".to_string(), "Site".into()); + root.insert("name".to_string(), name.clone().into()); + if let Some(site) = uisp_data.sites.iter().find(|s| s.name == *name) { + root.insert( + "downloadBandwidthMbps".to_owned(), + site.max_down_mbps.into(), + ); + root.insert("uploadBandwidthMbps".to_owned(), site.max_up_mbps.into()); + root.insert("uisp_site".to_string(), site.id.clone().into()); + root.insert("parent_site".to_string(), name.to_string().into()); + } + } + GraphMapping::AccessPointByName(name) => { + root.insert("type".to_string(), "AP".into()); + root.insert("name".to_string(), name.clone().into()); + root.insert("parent_site".to_string(), parent.to_string().into()); + if let Some(device) = uisp_data.devices.iter().find(|d| d.name == *name) { + root.insert("downloadBandwidthMbps".to_owned(), device.download.into()); + root.insert("uploadBandwidthMbps".to_owned(), device.upload.into()); + root.insert("uisp_device".to_string(), device.id.clone().into()); + } + } + GraphMapping::ClientById(client_id) => { + if let Some(site) = uisp_data.sites.iter().find(|c| c.id == *client_id) { + let devices = uisp_data + .devices + .iter() + .filter(|d| d.site_id == *client_id) + .collect::>(); + for device in devices.iter().filter(|d| d.has_address()) { + let sd = ShapedDevice { + circuit_id: site.id.clone(), + circuit_name: site.name.clone(), + device_id: device.id.clone(), + device_name: device.name.clone(), + parent_node: parent.to_owned(), + mac: device.mac.clone(), + ipv4: device.ipv4_list(), + ipv6: device.ipv6_list(), + download_min: (site.max_down_mbps as f32 + * config.uisp_integration.commit_bandwidth_multiplier) + as u64, + upload_min: (site.max_up_mbps as f32 + * config.uisp_integration.commit_bandwidth_multiplier) + as u64, + download_max: (site.max_down_mbps as f32 + * config.uisp_integration.bandwidth_overhead_factor) + as u64, + upload_max: (site.max_up_mbps as f32 + * config.uisp_integration.bandwidth_overhead_factor) + as u64, + comment: "".to_string(), + }; + shaped_devices.push(sd); + } + } else { + warn!("Client not found: {}", client_id); + } + } + _ => {} + } + } + if parent.is_some() { + root.insert( + "children".to_string(), + serde_json::to_value(children).unwrap(), + ); + } else { + for child in children { + root.insert(child.0, child.1); + } + } + root + } +} + +pub struct TroublesomeClients { + #[allow(dead_code)] // Used in serialization + pub multi_entry_points: HashSet, + #[allow(dead_code)] // Used in serialization + pub client_of_clients: HashSet, +} + +pub(crate) async fn find_troublesome_sites(data: &UispData) -> anyhow::Result { + let multi_entry_points = find_clients_with_multiple_entry_points(data)?; + let client_of_clients = find_clients_linked_from_other_clients(data)?; + + let _ = blackboard_blob("uisp-trouble-multi-entry", vec![multi_entry_points.clone()]).await; + let _ = blackboard_blob("uisp-trouble-client-of-client", vec![client_of_clients.clone()]).await; + + Ok(TroublesomeClients { + multi_entry_points, + client_of_clients, + }) +} + +fn find_clients_with_multiple_entry_points(data: &UispData) -> anyhow::Result> { + let mut result = HashSet::new(); + for client in data.find_client_sites() { + let mut links_to_client = HashSet::new(); + for link in data.data_links_raw.iter() { + if let (Some(from_site), Some(to_site)) = (&link.from.site, &link.to.site) { + if from_site.identification.id == client.id + && to_site.identification.id != client.id + { + links_to_client.insert(to_site.identification.id.clone()); + } else if from_site.identification.id != client.id + && to_site.identification.id == client.id + { + links_to_client.insert(from_site.identification.id.clone()); + } + } + } + if links_to_client.len() > 1 { + warn!( + "Client {} has multiple entry points: {:?}", + client.name, links_to_client + ); + result.insert(client.id.clone()); + } + } + + Ok(result) +} + +fn find_clients_linked_from_other_clients(data: &UispData) -> anyhow::Result> { + let all_clients = data.find_client_sites(); + let mut result = HashSet::new(); + for client in &all_clients { + for link in data.data_links_raw.iter() { + if let (Some(from_site), Some(to_site)) = (&link.from.site, &link.to.site) { + if from_site.identification.id == client.id + && to_site.identification.id != client.id + && all_clients + .iter() + .any(|c| c.id == to_site.identification.id) + { + warn!( + "Client {} is linked from another client: {}", + client.name, to_site.identification.id + ); + result.insert(client.id.clone()); + } + if from_site.identification.id != client.id + && to_site.identification.id == client.id + && all_clients + .iter() + .any(|c| c.id == from_site.identification.id) + { + warn!( + "Client {} is linked to another client: {}", + client.name, from_site.identification.id + ); + result.insert(client.id.clone()); + } + } + } + } + Ok(result) +} diff --git a/src/rust/uisp_integration/src/strategies/common.rs b/src/rust/uisp_integration/src/strategies/common.rs new file mode 100644 index 00000000..20020266 --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/common.rs @@ -0,0 +1,215 @@ +use crate::blackboard_blob; +use crate::errors::UispIntegrationError; +use crate::ip_ranges::IpRanges; +use crate::strategies::full::parse::parse_uisp_datasets; +use crate::strategies::full::uisp_fetch::load_uisp_data; +use crate::uisp_types::{UispDevice, UispSite, UispSiteType}; +use lqos_config::Config; +use std::collections::HashMap; +use std::sync::Arc; +use tracing::warn; +use uisp::{DataLink, Device, Site}; + +pub(crate) struct UispData { + pub sites_raw: Vec, + pub devices_raw: Vec, + pub data_links_raw: Vec, + pub sites: Vec, + pub devices: Vec, + //pub data_links: Vec, +} + +impl UispData { + pub(crate) async fn fetch_uisp_data( + config: Arc, + ip_ranges: IpRanges, + ) -> std::result::Result { + // Obtain the UISP data and transform it into easier to work with types + let (sites_raw, devices_raw, data_links_raw) = load_uisp_data(config.clone()).await?; + + if let Err(e) = blackboard_blob("uisp_sites", &sites_raw).await { + warn!("Unable to write sites to blackboard: {e:?}"); + } + if let Err(e) = blackboard_blob("uisp_devices", &devices_raw).await { + warn!("Unable to write devices to blackboard: {e:?}"); + } + if let Err(e) = blackboard_blob("uisp_data_links", &data_links_raw).await { + warn!("Unable to write data links to blackboard: {e:?}"); + } + + // If Mikrotik is enabled, we need to fetch the Mikrotik data + let ipv4_to_v6 = crate::strategies::full::mikrotik::mikrotik_data(&config) + .await + .unwrap_or_else(|_| Vec::new()); + + // Parse the UISP data into a more usable format + let (sites, _data_links, devices) = parse_uisp_datasets( + &sites_raw, + &data_links_raw, + &devices_raw, + &config, + &ip_ranges, + ipv4_to_v6, + ); + + Ok(UispData { + sites_raw, + devices_raw, + data_links_raw, + sites, + devices, + //data_links: _data_links, + }) + } + + pub fn find_client_sites(&self) -> Vec<&UispSite> { + self.sites + .iter() + .filter(|s| s.site_type == UispSiteType::Client) + .collect() + } + + pub fn find_devices_in_site(&self, site_id: &str) -> Vec<&Device> { + self.devices_raw + .iter() + .filter(|d| d.get_site_id().unwrap_or_default() == site_id) + .collect() + } + + pub fn find_device_by_id(&self, device_id: &str) -> Option<&Device> { + self.devices_raw + .iter() + .find(|d| d.identification.id == device_id) + } + + pub fn find_device_by_name(&self, device_name: &str) -> Option<&Device> { + self.devices_raw + .iter() + .find(|d| d.get_name().unwrap_or_default() == device_name) + } + + pub fn map_clients_to_aps(&self) -> HashMap> { + let mut mappings = HashMap::new(); + for client in self.find_client_sites() { + let mut found = false; + let mut parent = None; + for device in self.find_devices_in_site(&client.id) { + //println!("Client {} has a device {:?}", client.name, device.get_name()); + // Look for Parent AP attributes + if let Some(attr) = &device.attributes { + if let Some(ap) = &attr.apDevice { + if let Some(ap_id) = &ap.id { + //println!("AP ID: {}", ap_id); + if let Some(apdev) = self.find_device_by_id(ap_id) { + //println!("AP Device: {:?}", apdev.get_name()); + if apdev.get_site_id().unwrap_or_default() != client.id { + parent = Some(("AP", apdev.get_name().unwrap_or_default())); + found = true; + } + } + } + } + } + + // Look in Site-DeviceSite + if !found { + if let Some(device_site) = &device.identification.site { + if let Some(apdev) = self.find_device_by_id(&device_site.id) { + if apdev.get_site_id().unwrap_or_default() != client.id { + parent = Some(("AP", apdev.get_name().unwrap_or_default())); + found = true; + } + } + } + } + + // Look for data links with this device + if !found { + for link in self.data_links_raw.iter() { + // Check the FROM side + if let Some(from_device) = &link.from.device { + if from_device.identification.id == device.identification.id { + if let Some(to_device) = &link.to.device { + if let Some(apdev) = + self.find_device_by_id(&to_device.identification.id) + { + if apdev.get_site_id().unwrap_or_default() != client.id { + parent = + Some(("AP", apdev.get_name().unwrap_or_default())); + found = true; + } + } + } + } + } + // Check the TO side + if let Some(to_device) = &link.to.device { + if to_device.identification.id == device.identification.id { + if let Some(from_device) = &link.from.device { + if let Some(apdev) = + self.find_device_by_id(&from_device.identification.id) + { + if apdev.get_site_id().unwrap_or_default() != client.id { + parent = + Some(("AP", apdev.get_name().unwrap_or_default())); + found = true; + } + } + } + } + } + } + } + } + + // If we still haven't found anything, let's try data links to the client site as a whole + if !found { + for link in self.data_links_raw.iter() { + if let Some(from_site) = &link.from.site { + if from_site.identification.id == client.id { + if let Some(to_device) = &link.to.device { + if let Some(apdev) = + self.find_device_by_id(&to_device.identification.id) + { + if apdev.get_site_id().unwrap_or_default() != client.id { + parent = Some(("AP", apdev.get_name().unwrap_or_default())); + found = true; + } + } + } + } + } + if let Some(to_site) = &link.to.site { + if to_site.identification.id == client.id { + if let Some(from_device) = &link.from.device { + if let Some(apdev) = + self.find_device_by_id(&from_device.identification.id) + { + if apdev.get_site_id().unwrap_or_default() != client.id { + parent = Some(("AP", apdev.get_name().unwrap_or_default())); + found = true; + } + } + } + } + } + } + } + + if !found { + //println!("Client {} has no obvious parent AP", client.name); + let entry = mappings + .entry("Orphans".to_string()) + .or_insert_with(Vec::new); + entry.push(client.id.clone()); + } else { + //info!("Client {} is connected to {:?}", client.name, parent); + if let Some((_, parent)) = &parent { + let entry = mappings.entry(parent.to_string()).or_insert_with(Vec::new); + entry.push(client.id.clone()); + } + } + } + mappings + } +} diff --git a/src/rust/uisp_integration/src/strategies/full/bandwidth_overrides.rs b/src/rust/uisp_integration/src/strategies/full/bandwidth_overrides.rs index 0908aeec..d448cfd7 100644 --- a/src/rust/uisp_integration/src/strategies/full/bandwidth_overrides.rs +++ b/src/rust/uisp_integration/src/strategies/full/bandwidth_overrides.rs @@ -56,7 +56,7 @@ pub fn get_site_bandwidth_overrides( let parent_node = result[0].to_string(); if let Some(d) = numeric_string_to_f32(&result[1]) { if let Some(u) = numeric_string_to_f32(&result[2]) { - info!("Using bandiwdth override: {}, {}/{}", parent_node, d, u); + info!("Loaded bandiwdth override: {}, {}/{}", parent_node, d, u); overrides.insert(parent_node, (d, u)); } else { error!("Cannot parse {} as float on line {line}", &result[2]); diff --git a/src/rust/uisp_integration/src/strategies/full/mod.rs b/src/rust/uisp_integration/src/strategies/full/mod.rs index 1b526e35..f1552261 100644 --- a/src/rust/uisp_integration/src/strategies/full/mod.rs +++ b/src/rust/uisp_integration/src/strategies/full/mod.rs @@ -1,15 +1,16 @@ +#![allow(dead_code)] mod ap_promotion; -mod bandwidth_overrides; +pub(crate) mod bandwidth_overrides; mod client_site_promotion; -mod mikrotik; +pub mod mikrotik; mod network_json; -mod parse; +pub(crate) mod parse; mod root_site; -mod routes_override; -mod shaped_devices_writer; +pub(crate) mod routes_override; +pub mod shaped_devices_writer; mod squash_single_entry_aps; mod tree_walk; -mod uisp_fetch; +pub(crate) mod uisp_fetch; mod utils; mod zero_capacity_sites; diff --git a/src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs b/src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs index cc411bb5..d248bcd7 100644 --- a/src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs +++ b/src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs @@ -7,7 +7,7 @@ use tracing::{error, info}; /// Represents a shaped device in the ShapedDevices.csv file. #[derive(Serialize, Debug)] -struct ShapedDevice { +pub struct ShapedDevice { pub circuit_id: String, pub circuit_name: String, pub device_id: String, diff --git a/src/rust/uisp_integration/src/strategies/full2.rs b/src/rust/uisp_integration/src/strategies/full2.rs new file mode 100644 index 00000000..2f2aa3f1 --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/full2.rs @@ -0,0 +1,632 @@ +mod dot; +mod graph_mapping; +mod link_mapping; +mod net_json_parent; + +use crate::blackboard_blob; +use crate::errors::UispIntegrationError; +use crate::ip_ranges::IpRanges; +use crate::strategies::common::UispData; +use crate::strategies::full::routes_override::RouteOverride; +use crate::strategies::full::shaped_devices_writer::ShapedDevice; +use crate::strategies::full2::dot::save_dot_file; +use crate::strategies::full2::graph_mapping::GraphMapping; +use crate::strategies::full2::link_mapping::LinkMapping; +use crate::strategies::full2::net_json_parent::{NetJsonParent, walk_parents}; +use crate::uisp_types::UispDevice; +use lqos_config::Config; +use petgraph::graph::NodeIndex; +use petgraph::visit::{EdgeRef, NodeRef}; +use petgraph::Directed; +use std::collections::{HashMap, HashSet}; +use std::fs::write; +use std::path::Path; +use std::sync::Arc; +use tracing::{error, info, warn}; + +type GraphType = petgraph::Graph; + +pub async fn build_full_network_v2( + config: Arc, + ip_ranges: IpRanges, +) -> Result<(), UispIntegrationError> { + // Fetch the data + let uisp_data = UispData::fetch_uisp_data(config.clone(), ip_ranges).await?; + + if let Err(e) = blackboard_blob("uisp_sites", &uisp_data.sites_raw).await { + warn!("Unable to write sites to blackboard: {e:?}"); + } + if let Err(e) = blackboard_blob("uisp_devices", &uisp_data.devices_raw).await { + warn!("Unable to write devices to blackboard: {e:?}"); + } + if let Err(e) = blackboard_blob("uisp_data_links", &uisp_data.data_links_raw).await { + warn!("Unable to write data links to blackboard: {e:?}"); + } + + // Report on obvious UISP errors that should be fixed + let _trouble = crate::strategies::ap_site::find_troublesome_sites(&uisp_data) + .await + .map_err(|e| { + error!("Error finding troublesome sites"); + error!("{e:?}"); + UispIntegrationError::UnknownSiteType + })?; + + // Load overrides + let bandwidth_overrides = + crate::strategies::full::bandwidth_overrides::get_site_bandwidth_overrides(&config)?; + let routing_overrides = crate::strategies::full::routes_override::get_route_overrides(&config)?; + + // Create a new graph + let mut graph = GraphType::new(); + + // Find the root + let root_site_name = config.uisp_integration.site.clone(); + + // Add all sites to the graph + let mut site_map = HashMap::new(); + let mut root_idx = None; + add_all_sites_to_graph( + &uisp_data, + &mut graph, + &root_site_name, + &mut site_map, + &mut root_idx, + ); + let root_idx = root_idx.expect("Root site not found"); + + // Iterate all UISP devices and if their parent site is in the graph, add them + let mut device_map = HashMap::new(); + add_devices_to_graph(&uisp_data, &mut graph, &mut site_map, &mut device_map, &config, &bandwidth_overrides); + + // Now we iterate all the data links looking for DEVICE linkage + add_device_links_to_graph(&uisp_data, &mut graph, &mut device_map, &config); + + // Now we iterate only sites, looking for connectivity to the root + let orphans = graph.add_node(GraphMapping::GeneratedSite { + name: "Orphans".to_string(), + }); + graph.add_edge(root_idx, orphans, LinkMapping::ethernet(config.queues.generated_pn_download_mbps)); + graph.add_edge(orphans, root_idx, LinkMapping::ethernet(config.queues.generated_pn_upload_mbps)); + for (_, site_ref) in site_map.iter() { + if *site_ref == root_idx { + continue; + } + let a_star_run = + petgraph::algo::astar(&graph, *site_ref, |n| n.id() == root_idx.id(), |_| 0, |_| 0); + + if a_star_run.is_none() { + warn!( + "No path is detected from {:?} to {}", + graph[*site_ref], root_site_name + ); + graph.add_edge(*site_ref, orphans, LinkMapping::ethernet(config.queues.generated_pn_download_mbps)); + graph.add_edge(orphans, *site_ref, LinkMapping::ethernet(config.queues.generated_pn_upload_mbps)); + } + } + + // Client mapping + let client_mappings = uisp_data.map_clients_to_aps(); + + // Find the APs that have clients + let mut aps_with_clients = HashSet::new(); + for (ap_name, _client_ids) in client_mappings.iter() { + let Some(ap_device) = uisp_data + .devices_raw + .iter() + .find(|d| d.get_name().unwrap_or_default() == *ap_name) + else { + // Orphaning is already handled + continue; + }; + aps_with_clients.insert(ap_device.identification.id.clone()); + } + + // Count linkages to APs + let mut ap_link_count = HashMap::::new(); + for edge_ref in graph.edge_references() { + if let GraphMapping::AccessPoint { ref id, .. } = graph[edge_ref.source()] { + let entry = ap_link_count.entry(id.clone()).or_insert(0); + *entry += 1; + }; + if let GraphMapping::AccessPoint { ref id, .. } = graph[edge_ref.target()] { + let entry = ap_link_count.entry(id.clone()).or_insert(0); + *entry += 1; + }; + } + + // Cull the APs that have no clients or only one link + let mut to_remove = Vec::new(); + for (ap_id, ap_ref) in device_map.iter() { + if aps_with_clients.contains(ap_id) { + continue; + } + if let Some(link_count) = ap_link_count.get(ap_id) { + if *link_count > 1 { + continue; + } + } + to_remove.push(*ap_ref); + } + info!( + "Removing {} APs with no clients or only one link", + to_remove.len() + ); + for ap_ref in to_remove.iter() { + graph.remove_node(*ap_ref); + } + + // Visualizer + save_dot_file(&graph)?; + let _ = blackboard_blob("uisp-graph", vec![graph.clone()]).await; + + // Figure out the network.json layers + let mut parents = HashMap::::new(); + for node in graph.node_indices() { + if node == root_idx { + continue; + } + match &graph[node] { + GraphMapping::GeneratedSite { name } + | GraphMapping::Site { name, .. } + | GraphMapping::AccessPoint { name, .. } => { + // Generate two routes - one per direction + let route_from_root_to_node = petgraph::algo::astar( + &graph, + root_idx, + |n| n == node, + |e| { + (10_000u64).saturating_sub(link_capacity_mbps_for_routing( + &e.weight(), + &uisp_data.devices, + &routing_overrides, + )) + }, + |_| 0, + ) + .unwrap_or((0, vec![])); + + let route_from_node_to_root = petgraph::algo::astar( + &graph, + node, + |n| n == root_idx, + |e| { + (10_000u64).saturating_sub(link_capacity_mbps_for_routing( + &e.weight(), + &uisp_data.devices, + &routing_overrides, + )) + }, + |_| 0, + ) + .unwrap_or((0, vec![])); + + // println!("From node to root:"); + // println!("{:?}", route_from_node_to_root); + // println!("{:?}", edges_from_node_path(&graph, &route_from_node_to_root.1)); + // + // println!("From root to node:"); + // println!("{:?}", route_from_root_to_node); + // println!("{:?}", edges_from_node_path(&graph, &route_from_root_to_node.1)); + + if route_from_root_to_node.1.is_empty() { + //println!("No path detected from {:?} to {}", graph[node], root_site_name); + parents.insert( + name.to_owned(), + NetJsonParent { + parent_name: "Orphans".to_string(), + mapping: &graph[node], + download: config.queues.generated_pn_download_mbps, + upload: config.queues.generated_pn_upload_mbps, + }, + ); + } else { + // Obtain capacities from route traversal + let mut download_capacity = min_capacity_along_route(&graph, &route_from_root_to_node.1); + let mut upload_capacity = min_capacity_along_route(&graph, &route_from_node_to_root.1); + + // Apply AP capacities + if let GraphMapping::AccessPoint { download_mbps, upload_mbps, .. } = &graph[node] { + //println!("AP device capacity: {download_mbps}/{upload_mbps} (prev {download_capacity}/{upload_capacity})"); + download_capacity = u64::min(download_capacity, *download_mbps); + upload_capacity = u64::min(upload_capacity, *upload_mbps); + //println!("Now: {download_capacity}/{upload_capacity}"); + } + + // 0 isn't possible + if download_capacity < 1 { + download_capacity = config.queues.generated_pn_download_mbps; + } + if upload_capacity < 1 { + upload_capacity = config.queues.generated_pn_upload_mbps; + } + + // Ignore option + if config.uisp_integration.ignore_calculated_capacity { + download_capacity = config.queues.generated_pn_download_mbps; + upload_capacity = config.queues.generated_pn_upload_mbps; + } + + // Overrides + if !bandwidth_overrides.is_empty() { + if let Some(bw_override) = bandwidth_overrides.get(name) { + info!("Applying bandwidth override for {}", name); + info!("Capacity was: {} / {}", download_capacity, upload_capacity); + download_capacity = bw_override.0 as u64; + upload_capacity = bw_override.1 as u64; + info!("Capacity is now: {} / {}", download_capacity, upload_capacity); + } + } + + let parent_node = route_from_root_to_node.1[route_from_root_to_node.1.len() - 2]; + // We need the weight from node to parent_node in the graph edges + if let Some(_edge) = graph.find_edge(parent_node, node) { + let parent = graph[route_from_root_to_node.1[route_from_root_to_node.1.len() - 2]].name(); + parents.insert( + name.to_owned(), + NetJsonParent { + parent_name: parent, + mapping: &graph[node], + download: download_capacity, + upload: upload_capacity, + }, + ); + } else { + panic!("DID NOT FIND THE EDGE"); + } + } + } + _ => {} + } + } + + // Write the network.json file + let mut network_json = serde_json::Map::new(); + let mut visited = HashSet::new(); + for (name, node_info) in parents + .iter() + .filter(|(_name, parent)| parent.parent_name == root_site_name) + { + network_json.insert( + name.into(), + walk_parents(&parents, name, &node_info, &config, &graph, &mut visited).into(), + ); + } + let network_path = Path::new(&config.lqos_directory).join("network.json"); + if network_path.exists() && !config.integration_common.always_overwrite_network_json { + warn!( + "Network.json exists, and always overwrite network json is not true - not writing network.json" + ); + return Ok(()); + } + let json = serde_json::to_string_pretty(&network_json).unwrap(); + write(network_path, json).map_err(|e| { + error!("Unable to write network.json"); + error!("{e:?}"); + UispIntegrationError::WriteNetJson + })?; + info!("Written network.json"); + + // Shaped Devices + let mut shaped_devices = Vec::new(); + + for (ap_id, client_sites) in client_mappings.iter() { + for site_id in client_sites.iter() { + let Some(ap_device) = uisp_data.devices.iter().find(|d| d.name == *ap_id) else { + continue; + }; + let Some(site) = uisp_data.sites.iter().find(|s| s.id == *site_id) else { + continue; + }; + for device in uisp_data.devices.iter().filter(|d| d.site_id == *site_id) { + if !device.has_address() { + continue; + } + + let download = + (site.max_down_mbps as f32) * config.uisp_integration.bandwidth_overhead_factor; + let upload = + (site.max_up_mbps as f32) * config.uisp_integration.bandwidth_overhead_factor; + let download_min = + (download * config.uisp_integration.commit_bandwidth_multiplier) as u64; + let upload_min = + (upload * config.uisp_integration.commit_bandwidth_multiplier) as u64; + let download_max = download as u64; + let upload_max = upload as u64; + + let download_min = u64::max(1, download_min); + let upload_min = u64::max(1, upload_min); + let download_max = u64::max(4, download_max); + let upload_max = u64::max(4, upload_max); + + let parent_node = { + if parents.get(&ap_device.name).is_some() { + ap_device.name.clone() + } else { + "Orphans".to_string() + } + }; + + let shaped_device = ShapedDevice { + circuit_id: site.id.to_owned(), + circuit_name: site.name.to_owned(), + device_id: device.id.to_owned(), + device_name: device.name.to_owned(), + parent_node, + mac: device.mac.to_owned(), + ipv4: device.ipv4_list(), + ipv6: device.ipv6_list(), + download_min, + upload_min, + download_max, + upload_max, + comment: "".to_string(), + }; + shaped_devices.push(shaped_device); + } + } + } + let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); + let mut writer = csv::WriterBuilder::new() + .has_headers(true) + .from_path(file_path) + .unwrap(); + + for d in shaped_devices.iter() { + writer.serialize(d).unwrap(); + } + writer.flush().map_err(|e| { + error!("Unable to flush CSV file"); + error!("{e:?}"); + UispIntegrationError::CsvError + })?; + info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); + + Ok(()) +} + +fn add_device_links_to_graph( + uisp_data: &UispData, + graph: &mut GraphType, + device_map: &mut HashMap, + config: &Arc, +) { + for link in uisp_data.data_links_raw.iter() { + let Some(from_device) = &link.from.device else { + continue; + }; + let Some(to_device) = &link.to.device else { + continue; + }; + let Some(a_ref) = device_map.get(&from_device.identification.id) else { + continue; + }; + let Some(b_ref) = device_map.get(&to_device.identification.id) else { + continue; + }; + if a_ref == b_ref { + // If the devices are the same, we don't need to add an edge + continue; + } + if let Some(dev_a) = uisp_data + .devices_raw + .iter() + .find(|d| d.get_id() == from_device.identification.id) + { + if let Some(dev_b) = uisp_data + .devices_raw + .iter() + .find(|d| d.get_id() == to_device.identification.id) + { + if dev_a.get_site_id().unwrap_or_default() + == dev_b.get_site_id().unwrap_or_default() + { + // If the devices are in the same site, we don't need to add an edge + continue; + } + } + } + if graph.contains_edge(*a_ref, *b_ref) { + // If the edge already exists, we don't need to add it + continue; + } + if graph.contains_edge(*b_ref, *a_ref) { + // If the edge already exists, we don't need to add it + continue; + } + + let (down, up) = get_capacity_from_datalink_device( + &from_device.identification.id, + &uisp_data.devices, + &config, + ); + graph.add_edge( + *a_ref, + *b_ref, + LinkMapping::DevicePair { + device_a: from_device.identification.id.clone(), + device_b: to_device.identification.id.clone(), + speed_mbps: down, + }, + ); + graph.add_edge( + *b_ref, + *a_ref, + LinkMapping::DevicePair { + device_b: from_device.identification.id.clone(), + device_a: to_device.identification.id.clone(), + speed_mbps: up, + }, + ); + } +} + +fn get_capacity_from_datalink_device( + device_id: &str, + devices: &[UispDevice], + config: &Arc, +) -> (u64, u64) { + if let Some(device) = devices.iter().find(|d| d.id == device_id) { + return (device.download, device.upload); + } + + (config.queues.generated_pn_download_mbps, config.queues.generated_pn_upload_mbps) +} + +fn add_devices_to_graph( + uisp_data: &UispData, + graph: &mut GraphType, + site_map: &mut HashMap, + device_map: &mut HashMap, + config: &Arc, + bandwidth_overrides: &BandwidthOverrides, +) { + for device in uisp_data.devices_raw.iter() { + let Some(site_id) = &device.identification.site else { + continue; + }; + let site_id = &site_id.id; + let Some(device_details) = uisp_data.devices.iter().find(|d| d.id == device.identification.id) else { + continue; + }; + let Some(site_ref) = site_map.get(site_id) else { + continue; + }; + let mut download_mbps = device_details.download; + let mut upload_mbps = device_details.upload; + + if download_mbps < 1 { + download_mbps = config.queues.generated_pn_download_mbps; + } + if upload_mbps < 1 { + upload_mbps = config.queues.generated_pn_upload_mbps; + } + + if config.uisp_integration.ignore_calculated_capacity { + download_mbps = config.queues.generated_pn_download_mbps; + upload_mbps = config.queues.generated_pn_upload_mbps; + } + + if let Some(bw_override) = bandwidth_overrides.get(&device.get_name().unwrap_or_default()) { + download_mbps = bw_override.0 as u64; + upload_mbps = bw_override.1 as u64; + } + + let device_entry = GraphMapping::AccessPoint { + name: device.get_name().unwrap_or_default(), + id: device.identification.id.clone(), + site_name: graph[*site_ref].name(), + download_mbps, + upload_mbps, + }; + let device_ref = graph.add_node(device_entry); + device_map.insert(device.identification.id.clone(), device_ref); + let _ = graph.add_edge(device_ref, *site_ref, LinkMapping::ethernet(config.queues.generated_pn_download_mbps)); + let _ = graph.add_edge(*site_ref, device_ref, LinkMapping::ethernet(config.queues.generated_pn_upload_mbps)); + } +} + +pub fn add_all_sites_to_graph( + uisp_data: &UispData, + graph: &mut GraphType, + root_site_name: &String, + site_map: &mut HashMap, + root_idx: &mut Option, +) { + for site in uisp_data.sites_raw.iter().filter(|s| !s.is_client_site()) { + let site_name = site.name_or_blank(); + let id = site.id.clone(); + if site_name == *root_site_name { + // Add the root site + let root_entry = GraphMapping::Root { + name: site_name, + id, + }; + let root_ref = graph.add_node(root_entry); + *root_idx = Some(root_ref); + site_map.insert(site.id.clone(), root_ref); + continue; + } + let site_entry = GraphMapping::Site { + name: site_name, + id, + }; + let site_ref = graph.add_node(site_entry); + site_map.insert(site.id.clone(), site_ref); + } +} + +fn link_capacity_mbps_for_routing( + link_mapping: &LinkMapping, + devices: &[UispDevice], + route_overrides: &[RouteOverride], +) -> u64 { + match link_mapping { + LinkMapping::Ethernet { speed_mbps } => *speed_mbps, + LinkMapping::DevicePair { speed_mbps, device_a, device_b } => { + // Check for overrides: make sure we have both devices (we need names, are storing IDs) + let Some(device_a) = devices.iter().find(|d| d.id == *device_a) else { + return *speed_mbps; + }; + let Some(device_b) = devices.iter().find(|d| d.id == *device_b) else { + return *speed_mbps; + }; + + // Check for the direct direction + if let Some(route_override) = route_overrides + .iter() + .find(|route_override| + (route_override.from_site == device_a.name || route_override.to_site == device_a.name) + && (route_override.from_site == device_b.name || route_override.to_site == device_b.name) + ) { + return route_override.cost as u64; + } + + // Check for the reverse direction + if let Some(route_override) = route_overrides + .iter() + .find(|route_override| + (route_override.from_site == device_b.name || route_override.to_site == device_b.name) + && (route_override.from_site == device_a.name || route_override.to_site == device_a.name) + ) { + return route_override.cost as u64; + } + + *speed_mbps + } + } +} + +use petgraph::prelude::*; +use crate::strategies::full::bandwidth_overrides::BandwidthOverrides; + +fn edges_from_node_path( + graph: &GraphType, + path: &[NodeIndex], +) -> Vec +{ + let mut edge_ids = Vec::new(); + for window in path.windows(2) { + let source = window[0]; + let target = window[1]; + // This assumes a directed graph; if undirected, you might need to check both directions + if let Some(edge) = graph.find_edge(source, target) { + edge_ids.push(edge); + } else { + // Handle case where no edge is found (could be error or special case) + panic!("No edge found between {:?} and {:?}", source, target); + } + } + edge_ids +} + +fn min_capacity_along_route( + graph: &GraphType, + path: &[NodeIndex], +) -> u64 { + edges_from_node_path(graph, path) + .iter() + .map(|edge| graph[*edge].capacity_mbps()) + .min() + .unwrap_or(0) +} \ No newline at end of file diff --git a/src/rust/uisp_integration/src/strategies/full2/dot.rs b/src/rust/uisp_integration/src/strategies/full2/dot.rs new file mode 100644 index 00000000..f0521548 --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/full2/dot.rs @@ -0,0 +1,19 @@ +use crate::errors::UispIntegrationError; +use crate::strategies::full2::GraphType; +use std::fs::write; + +pub fn save_dot_file(graph: &GraphType) -> Result<(), UispIntegrationError> { + // Save the dot file + let dot_data = format!( + "{:?}", + petgraph::dot::Dot::with_config(graph, &[petgraph::dot::Config::EdgeNoLabel]) + ); + let _ = write("graph.dot", dot_data.as_bytes()); + let _ = std::process::Command::new("dot") + .arg("-Tpng") + .arg("graph.dot") + .arg("-o") + .arg("graph.png") + .output(); + Ok(()) +} diff --git a/src/rust/uisp_integration/src/strategies/full2/graph_mapping.rs b/src/rust/uisp_integration/src/strategies/full2/graph_mapping.rs new file mode 100644 index 00000000..b5698dbd --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/full2/graph_mapping.rs @@ -0,0 +1,34 @@ +use serde::Serialize; + +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, Serialize)] +pub enum GraphMapping { + Root { + name: String, + id: String, + }, + Site { + name: String, + id: String, + }, + GeneratedSite { + name: String, + }, + AccessPoint { + name: String, + id: String, + site_name: String, + download_mbps: u64, + upload_mbps: u64, + }, +} + +impl GraphMapping { + pub fn name(&self) -> String { + match self { + GraphMapping::Root { name, .. } => name.clone(), + GraphMapping::Site { name, .. } => name.clone(), + GraphMapping::GeneratedSite { name } => name.clone(), + GraphMapping::AccessPoint { name, .. } => name.clone(), + } + } +} diff --git a/src/rust/uisp_integration/src/strategies/full2/link_mapping.rs b/src/rust/uisp_integration/src/strategies/full2/link_mapping.rs new file mode 100644 index 00000000..9bc71b3e --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/full2/link_mapping.rs @@ -0,0 +1,20 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Serialize)] +pub enum LinkMapping { + Ethernet { speed_mbps: u64 }, + DevicePair { device_a: String, device_b: String, speed_mbps: u64, } +} + +impl LinkMapping { + pub fn ethernet(speed_mbps: u64) -> Self { + LinkMapping::Ethernet { speed_mbps } + } + + pub fn capacity_mbps(&self) -> u64 { + match self { + LinkMapping::Ethernet { speed_mbps } => *speed_mbps, + LinkMapping::DevicePair { speed_mbps, .. } => *speed_mbps, + } + } +} \ No newline at end of file diff --git a/src/rust/uisp_integration/src/strategies/full2/net_json_parent.rs b/src/rust/uisp_integration/src/strategies/full2/net_json_parent.rs new file mode 100644 index 00000000..97e7eadf --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/full2/net_json_parent.rs @@ -0,0 +1,65 @@ +use crate::strategies::full2::GraphType; +use crate::strategies::full2::graph_mapping::GraphMapping; +use lqos_config::Config; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +#[derive(Debug)] +pub struct NetJsonParent<'a> { + pub parent_name: String, + pub mapping: &'a GraphMapping, + pub download: u64, + pub upload: u64, +} + +pub fn walk_parents( + parents: &HashMap, + name: &String, + node_info: &NetJsonParent, + config: &Arc, + graph: &GraphType, + visited: &mut HashSet, +) -> serde_json::Map { + let mut map = serde_json::Map::new(); + // Entries are name, type, uisp_device or site, downloadBandwidthMbps, uploadBandwidthMbps, children + map.insert("name".into(), name.clone().into()); + map.insert("downloadBandwidthMbps".into(), node_info.download.into()); + map.insert("uploadBandwidthMbps".into(), node_info.upload.into()); + match node_info.mapping { + GraphMapping::Root { id, .. } | GraphMapping::Site { id, .. } => { + map.insert("type".into(), "Site".into()); + map.insert("uisp_site".into(), id.clone().into()); + map.insert("parent_site".into(), name.clone().into()); + } + GraphMapping::GeneratedSite { .. } => { + map.insert("type".into(), "Site".into()); + } + GraphMapping::AccessPoint { + name: _, + id, + site_name, + .. + } => { + map.insert("type".into(), "AP".into()); + map.insert("parent_site".into(), site_name.clone().into()); + map.insert("uisp_device".into(), id.clone().into()); + } + } + + let mut children = serde_json::Map::new(); + for (name, node_info) in parents + .iter() + .filter(|(_node_name, node_info)| node_info.parent_name == *name) + { + if visited.contains(name) { + continue; + } + visited.insert(name.to_string()); + let child = walk_parents(parents, name, &node_info, config, graph, visited); + children.insert(name.into(), child.into()); + } + + map.insert("children".into(), children.into()); + + map +} diff --git a/src/rust/uisp_integration/src/strategies/mod.rs b/src/rust/uisp_integration/src/strategies/mod.rs index 3cb5875e..38c0d240 100644 --- a/src/rust/uisp_integration/src/strategies/mod.rs +++ b/src/rust/uisp_integration/src/strategies/mod.rs @@ -1,5 +1,9 @@ +mod ap_only; +mod ap_site; +mod common; mod flat; mod full; +mod full2; use crate::blackboard; use crate::errors::UispIntegrationError; @@ -27,10 +31,25 @@ pub async fn build_with_strategy( flat::build_flat_network(config, ip_ranges).await?; Ok(()) } - "full" => { + /*"full" => { info!("Strategy selected: full"); full::build_full_network(config, ip_ranges).await?; Ok(()) + }*/ + "ap_only" => { + info!("Strategy selected: ap_only"); + ap_only::build_ap_only_network(config, ip_ranges).await?; + Ok(()) + } + "ap_site" => { + info!("Strategy selected: ap_site"); + ap_site::build_ap_site_network(config, ip_ranges).await?; + Ok(()) + } + "full" | "full2" => { + info!("Strategy selected: full2"); + full2::build_full_network_v2(config, ip_ranges).await?; + Ok(()) } _ => { error!( diff --git a/src/rust/uisp_integration/src/uisp_types/uisp_device.rs b/src/rust/uisp_integration/src/uisp_types/uisp_device.rs index 06211719..2ee189fd 100644 --- a/src/rust/uisp_integration/src/uisp_types/uisp_device.rs +++ b/src/rust/uisp_integration/src/uisp_types/uisp_device.rs @@ -52,6 +52,20 @@ impl UispDevice { if let Some(ul) = overview.uplinkCapacity { upload = ul as u64 / 1000000; } + if device.get_model().unwrap_or_default().contains("5AC") { + download = ((download as f64) * config.uisp_integration.airmax_capacity as f64) as u64; + upload = ((upload as f64) * config.uisp_integration.airmax_capacity as f64) as u64; + } + if device.get_model().unwrap_or_default().contains("LTU") { + download = ((download as f64) * config.uisp_integration.ltu_capacity as f64) as u64; + upload = ((upload as f64) * config.uisp_integration.ltu_capacity as f64) as u64; + } + } + if download == 0 { + download = config.queues.generated_pn_download_mbps; + } + if upload == 0 { + upload = config.queues.generated_pn_upload_mbps; } // Accumulate IP address listings