Merge pull request #688 from LibreQoE/uisp_integration_bottom_up

Uisp integration - build from the bottom up

src/rust/Cargo.lock

@@ -1054,6 +1054,12 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "fixedbitset"
+version = "0.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
+
 [[package]]
 name = "flate2"
 version = "1.1.0"
@@ -2577,6 +2583,18 @@ version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
+[[package]]
+name = "petgraph"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+ "serde",
+ "serde_derive",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.1.10"
@@ -4023,6 +4041,7 @@ dependencies = [
  "ip_network_table",
  "lqos_bus",
  "lqos_config",
+ "petgraph",
  "serde",
  "serde_cbor",
  "serde_json",

src/rust/lqos_queue_tracker/src/queue_structure/queue_node.rs

@@ -3,7 +3,7 @@ use lqos_bus::TcHandle;
 use lqos_utils::hash_to_i64;
 use lqos_utils::hex_string::read_hex_string;
 use serde_json::Value;
-use tracing::{error, warn};
+use tracing::{error, info, warn};
 
 #[derive(Default, Clone, Debug)]
 pub struct QueueNode {
@@ -238,7 +238,7 @@ impl QueueNode {
                     "idForCircuitsWithoutParentNodes" | "type" => {
                         // Ignore
                     }
-                    _ => error!("I don't know how to parse key: [{key}]"),
+                    _ => info!("I don't know how to parse key: [{key}]"),
                 }
             }
         } else {

src/rust/uisp/src/lib.rs

@@ -42,7 +42,7 @@ pub async fn load_all_sites(config: Arc<Config>) -> Result<Vec<Site>> {
 /// Load all devices from UISP that are authorized, and include their full interface definitions
 pub async fn load_all_devices_with_interfaces(config: Arc<Config>) -> Result<Vec<Device>> {
     Ok(nms_request_get_vec(
-        "devices?withInterfaces=true&authorized=true",
+        "devices?withInterfaces=true",
         &config.uisp_integration.token,
         &config.uisp_integration.url,
     )

src/rust/uisp_integration/Cargo.toml

@@ -19,4 +19,5 @@ serde_json = { workspace = true }
 ip_network_table = {  workspace = true }
 ip_network = { workspace = true }
 serde_cbor = { workspace = true }
-lqos_bus = { path = "../lqos_bus" }
+lqos_bus = { path = "../lqos_bus" }
+petgraph = {  version = "0.7.1", features = ["serde-1"] }

src/rust/uisp_integration/src/blackboard.rs

@@ -0,0 +1,47 @@
+//! Collects the blackboard functionality together
+
+use lqos_bus::BlackboardSystem;
+use serde::Serialize;
+use tracing::info;
+
+pub async fn blackboard(subsystem: BlackboardSystem, key: &str, value: &str) {
+    let Ok(config) = lqos_config::load_config() else {
+        return;
+    };
+    if !config.long_term_stats.use_insight.unwrap_or(false) {
+        return;
+    }
+    let req = vec![lqos_bus::BusRequest::BlackboardData {
+        subsystem,
+        key: key.to_string(),
+        value: value.to_string(),
+    }];
+    let _ = lqos_bus::bus_request(req).await;
+}
+
+pub async fn blackboard_blob<T: Serialize>(key: &str, value: T) -> anyhow::Result<()> {
+    let config = lqos_config::load_config()?;
+    if !config.long_term_stats.use_insight.unwrap_or(false) {
+        return Ok(());
+    }
+    let blob = serde_cbor::to_vec(&value)?;
+    let chunks = blob.chunks(1024 * 16);
+    info!(
+        "Blob {key} is {} bytes long, split into {} chunks",
+        blob.len(),
+        chunks.len()
+    );
+    for (i, chunk) in chunks.enumerate() {
+        let req = vec![lqos_bus::BusRequest::BlackboardBlob {
+            tag: key.to_string(),
+            part: i,
+            blob: chunk.to_vec(),
+        }];
+        let err = lqos_bus::bus_request(req).await;
+        #[cfg(debug_assertions)]
+        if let Err(e) = err {
+            tracing::error!("Error writing to blackboard (only an error if lqosd is running): {e:?}");
+        }
+    }
+    Ok(())
+}

src/rust/uisp_integration/src/main.rs

@@ -2,6 +2,7 @@
 //! be ported back to Python, with Rust support structures - but I'll iterate
 //! faster in Rust.
 
+mod blackboard;
 #[warn(missing_docs)]
 mod errors;
 pub mod ip_ranges;
@@ -10,9 +11,8 @@ pub mod uisp_types;
 
 use crate::errors::UispIntegrationError;
 use crate::ip_ranges::IpRanges;
-use lqos_bus::BlackboardSystem;
+pub use blackboard::*;
 use lqos_config::Config;
-use serde::Serialize;
 use tokio::time::Instant;
 use tracing::{error, info};
 
@@ -35,44 +35,6 @@ fn check_enabled_status(config: &Config) -> Result<(), UispIntegrationError> {
     }
 }
 
-pub async fn blackboard(subsystem: BlackboardSystem, key: &str, value: &str) {
-    let Ok(config) = lqos_config::load_config() else {
-        return;
-    };
-    if !config.long_term_stats.use_insight.unwrap_or(false) {
-        return;
-    }
-    let req = vec![lqos_bus::BusRequest::BlackboardData {
-        subsystem,
-        key: key.to_string(),
-        value: value.to_string(),
-    }];
-    let _ = lqos_bus::bus_request(req).await;
-}
-
-pub async fn blackboard_blob<T: Serialize>(key: &str, value: T) -> anyhow::Result<()> {
-    let config = lqos_config::load_config()?;
-    if !config.long_term_stats.use_insight.unwrap_or(false) {
-        return Ok(());
-    }
-    let blob = serde_cbor::to_vec(&value)?;
-    let chunks = blob.chunks(1024 * 16);
-    info!(
-        "Blob {key} is {} bytes long, split into {} chunks",
-        blob.len(),
-        chunks.len()
-    );
-    for (i, chunk) in chunks.enumerate() {
-        let req = vec![lqos_bus::BusRequest::BlackboardBlob {
-            tag: key.to_string(),
-            part: i,
-            blob: chunk.to_vec(),
-        }];
-        let _ = lqos_bus::bus_request(req).await;
-    }
-    Ok(())
-}
-
 #[tokio::main]
 async fn main() -> Result<(), UispIntegrationError> {
     let now = Instant::now();

src/rust/uisp_integration/src/strategies/ap_only.rs

@@ -0,0 +1,118 @@
+use crate::errors::UispIntegrationError;
+use crate::ip_ranges::IpRanges;
+use crate::strategies::common::UispData;
+use crate::strategies::full::shaped_devices_writer::ShapedDevice;
+use lqos_config::Config;
+use std::fs::write;
+use std::path::Path;
+use std::sync::Arc;
+use tracing::{error, info, warn};
+
+/// Creates a network with only APs detected
+/// from clients.
+pub async fn build_ap_only_network(
+    config: Arc<Config>,
+    ip_ranges: IpRanges,
+) -> Result<(), UispIntegrationError> {
+    let uisp_data = UispData::fetch_uisp_data(config.clone(), ip_ranges).await?;
+
+    // Find the clients
+    let mappings = uisp_data.map_clients_to_aps();
+
+    // Write network.json
+    let network_path = Path::new(&config.lqos_directory).join("network.json");
+    if network_path.exists() && !config.integration_common.always_overwrite_network_json {
+        warn!(
+            "Network.json exists, and always overwrite network json is not true - not writing network.json"
+        );
+        return Ok(());
+    }
+    let mut root = serde_json::Map::new();
+    for ap in mappings.keys() {
+        if let Some(ap_device) = uisp_data.devices.iter().find(|d| d.name == *ap) {
+            let mut ap_object = serde_json::Map::new();
+            // Empy children
+            ap_object.insert("children".to_string(), serde_json::Map::new().into());
+
+            // Limits
+            ap_object.insert(
+                "downloadBandwidthMbps".to_string(),
+                serde_json::Value::Number(ap_device.download.into()),
+            );
+            ap_object.insert(
+                "uploadBandwidthMbps".to_string(),
+                serde_json::Value::Number(ap_device.upload.into()),
+            );
+
+            // Metadata
+            ap_object.insert("type".to_string(), "AP".to_string().into());
+            ap_object.insert("uisp_device".to_string(), ap_device.id.clone().into());
+
+            // Save the entry
+            root.insert(ap.to_string(), ap_object.into());
+        }
+    }
+    let json = serde_json::to_string_pretty(&root).unwrap();
+    write(network_path, json).map_err(|e| {
+        error!("Unable to write network.json");
+        error!("{e:?}");
+        UispIntegrationError::WriteNetJson
+    })?;
+    info!("Written network.json");
+
+    // Write ShapedDevices.csv
+    let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv");
+    let mut shaped_devices = Vec::new();
+    for (parent, client_ids) in mappings.iter() {
+        for client_id in client_ids {
+            let site = uisp_data.sites.iter().find(|s| *client_id == s.id).unwrap();
+            let devices = uisp_data
+                .devices
+                .iter()
+                .filter(|d| d.site_id == *client_id)
+                .collect::<Vec<_>>();
+            for device in devices.iter().filter(|d| d.has_address()) {
+                let sd = ShapedDevice {
+                    circuit_id: site.id.clone(),
+                    circuit_name: site.name.clone(),
+                    device_id: device.id.clone(),
+                    device_name: device.name.clone(),
+                    parent_node: parent.clone(),
+                    mac: device.mac.clone(),
+                    ipv4: device.ipv4_list(),
+                    ipv6: device.ipv6_list(),
+                    download_min: (site.max_down_mbps as f32
+                        * config.uisp_integration.commit_bandwidth_multiplier)
+                        as u64,
+                    upload_min: (site.max_up_mbps as f32
+                        * config.uisp_integration.commit_bandwidth_multiplier)
+                        as u64,
+                    download_max: (site.max_down_mbps as f32
+                        * config.uisp_integration.bandwidth_overhead_factor)
+                        as u64,
+                    upload_max: (site.max_up_mbps as f32
+                        * config.uisp_integration.bandwidth_overhead_factor)
+                        as u64,
+                    comment: "".to_string(),
+                };
+                shaped_devices.push(sd);
+            }
+        }
+    }
+    let mut writer = csv::WriterBuilder::new()
+        .has_headers(true)
+        .from_path(file_path)
+        .unwrap();
+
+    for d in shaped_devices.iter() {
+        writer.serialize(d).unwrap();
+    }
+    writer.flush().map_err(|e| {
+        error!("Unable to flush CSV file");
+        error!("{e:?}");
+        UispIntegrationError::CsvError
+    })?;
+    info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len());
+
+    Ok(())
+}

src/rust/uisp_integration/src/strategies/ap_site.rs

@@ -0,0 +1,357 @@
+use crate::blackboard_blob;
+use crate::errors::UispIntegrationError;
+use crate::ip_ranges::IpRanges;
+use crate::strategies::common::UispData;
+use crate::strategies::full::shaped_devices_writer::ShapedDevice;
+use lqos_config::Config;
+use std::collections::{HashMap, HashSet};
+use std::fs::write;
+use std::path::Path;
+use std::sync::Arc;
+use tracing::{error, info, warn};
+
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone)]
+pub enum GraphMapping {
+    Root,
+    SiteByName(String),
+    //GeneratedSiteByName(String),
+    AccessPointByName(String),
+    ClientById(String),
+}
+
+/// Creates a network with APs detected from clients,
+/// and then a single site above them (shared if the site
+/// matches).
+pub async fn build_ap_site_network(
+    config: Arc<Config>,
+    ip_ranges: IpRanges,
+) -> Result<(), UispIntegrationError> {
+    let uisp_data = UispData::fetch_uisp_data(config.clone(), ip_ranges).await?;
+
+    // Find trouble-spots!
+    let _trouble = find_troublesome_sites(&uisp_data).await.map_err(|e| {
+        error!("Error finding troublesome sites");
+        error!("{e:?}");
+        UispIntegrationError::UnknownSiteType
+    })?;
+
+    // Find the clients
+    let ap_mappings = uisp_data.map_clients_to_aps();
+
+    // Make AP Layer entries
+    let access_points = get_ap_layer(&ap_mappings);
+
+    // Site mappings
+    let sites = map_sites_above_aps(&uisp_data, ap_mappings, access_points);
+
+    // Insert the root
+    let root = Layer {
+        id: GraphMapping::Root,
+        children: sites.values().cloned().collect(),
+    };
+    //println!("{:#?}", root);
+
+    let mut shaped_devices = Vec::new();
+    let net_json = root.walk_children(None, &uisp_data, &mut shaped_devices, &config);
+
+    let network_path = Path::new(&config.lqos_directory).join("network.json");
+    if network_path.exists() && !config.integration_common.always_overwrite_network_json {
+        warn!(
+            "Network.json exists, and always overwrite network json is not true - not writing network.json"
+        );
+        return Ok(());
+    }
+    let json = serde_json::to_string_pretty(&net_json).unwrap();
+    write(network_path, json).map_err(|e| {
+        error!("Unable to write network.json");
+        error!("{e:?}");
+        UispIntegrationError::WriteNetJson
+    })?;
+    info!("Written network.json");
+
+    let _ = write_shaped_devices(&config, &mut shaped_devices);
+    info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len());
+
+    Ok(())
+}
+
+pub(crate) fn write_shaped_devices(
+    config: &Arc<Config>,
+    shaped_devices: &mut Vec<ShapedDevice>,
+) -> Result<(), UispIntegrationError> {
+    let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv");
+    let mut writer = csv::WriterBuilder::new()
+        .has_headers(true)
+        .from_path(file_path)
+        .unwrap();
+
+    for d in shaped_devices.iter() {
+        writer.serialize(d).unwrap();
+    }
+    writer.flush().map_err(|e| {
+        error!("Unable to flush CSV file");
+        error!("{e:?}");
+        UispIntegrationError::CsvError
+    })?;
+    info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len());
+    Ok(())
+}
+
+pub(crate) fn map_sites_above_aps(
+    uisp_data: &UispData,
+    ap_mappings: HashMap<String, Vec<String>>,
+    access_points: HashMap<String, Layer>,
+) -> HashMap<String, Layer> {
+    let mut sites = HashMap::new();
+    for (ap_name, client_ids) in ap_mappings.iter() {
+        if let Some(device) = uisp_data.find_device_by_name(ap_name) {
+            if let Some(device_site_id) = device.get_site_id() {
+                if let Some(device_site) = uisp_data.sites.iter().find(|s| s.id == device_site_id) {
+                    let site_entry =
+                        sites
+                            .entry(device_site.name.clone())
+                            .or_insert_with(|| Layer {
+                                id: GraphMapping::SiteByName(device_site.name.clone()),
+                                children: Vec::new(),
+                            });
+                    let ap_map = access_points.get(ap_name).unwrap().clone();
+                    site_entry.children.push(ap_map);
+                }
+            }
+        } else {
+            let mut detached = Layer {
+                id: GraphMapping::SiteByName(ap_name.clone()),
+                children: vec![],
+            };
+            for client_id in client_ids.iter() {
+                detached.children.push(Layer {
+                    id: GraphMapping::ClientById(client_id.clone()),
+                    children: vec![],
+                });
+            }
+            sites.insert(ap_name.clone(), detached);
+        }
+    }
+    sites
+}
+
+pub(crate) fn get_ap_layer(ap_mappings: &HashMap<String, Vec<String>>) -> HashMap<String, Layer> {
+    let mut access_points = HashMap::new();
+    for (ap_name, client_ids) in ap_mappings.iter() {
+        let mut ap_layer = Layer {
+            id: GraphMapping::AccessPointByName(ap_name.clone()),
+            children: Vec::new(),
+        };
+        for client_id in client_ids.iter() {
+            ap_layer.children.push(Layer {
+                id: GraphMapping::ClientById(client_id.clone()),
+                children: Vec::new(),
+            });
+        }
+        access_points.insert(ap_name.clone(), ap_layer);
+    }
+    access_points
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct Layer {
+    pub(crate) id: GraphMapping,
+    pub(crate) children: Vec<Layer>,
+}
+
+impl Layer {
+    pub(crate) fn walk_children(
+        &self,
+        parent: Option<&str>,
+        uisp_data: &UispData,
+        shaped_devices: &mut Vec<ShapedDevice>,
+        config: &Config,
+    ) -> serde_json::Map<String, serde_json::Value> {
+        let mut children = serde_json::Map::new();
+        let parent_name = match &self.id {
+            GraphMapping::SiteByName(name) | GraphMapping::AccessPointByName(name) => {
+                name.to_owned()
+            }
+            _ => "".to_owned(),
+        };
+        for child in self.children.iter() {
+            match &child.id {
+                GraphMapping::SiteByName(name) | GraphMapping::AccessPointByName(name) => {
+                    children.insert(
+                        name.clone(),
+                        child
+                            .walk_children(Some(&parent_name), uisp_data, shaped_devices, config)
+                            .into(),
+                    );
+                }
+                GraphMapping::ClientById(_client_id) => {
+                    let _ =
+                        child.walk_children(Some(&parent_name), uisp_data, shaped_devices, config);
+                }
+                _ => {}
+            }
+        }
+        let mut root = serde_json::Map::new();
+        if let Some(parent) = parent {
+            match &self.id {
+                GraphMapping::SiteByName(name) => {
+                    root.insert("type".to_string(), "Site".into());
+                    root.insert("name".to_string(), name.clone().into());
+                    if let Some(site) = uisp_data.sites.iter().find(|s| s.name == *name) {
+                        root.insert(
+                            "downloadBandwidthMbps".to_owned(),
+                            site.max_down_mbps.into(),
+                        );
+                        root.insert("uploadBandwidthMbps".to_owned(), site.max_up_mbps.into());
+                        root.insert("uisp_site".to_string(), site.id.clone().into());
+                        root.insert("parent_site".to_string(), name.to_string().into());
+                    }
+                }
+                GraphMapping::AccessPointByName(name) => {
+                    root.insert("type".to_string(), "AP".into());
+                    root.insert("name".to_string(), name.clone().into());
+                    root.insert("parent_site".to_string(), parent.to_string().into());
+                    if let Some(device) = uisp_data.devices.iter().find(|d| d.name == *name) {
+                        root.insert("downloadBandwidthMbps".to_owned(), device.download.into());
+                        root.insert("uploadBandwidthMbps".to_owned(), device.upload.into());
+                        root.insert("uisp_device".to_string(), device.id.clone().into());
+                    }
+                }
+                GraphMapping::ClientById(client_id) => {
+                    if let Some(site) = uisp_data.sites.iter().find(|c| c.id == *client_id) {
+                        let devices = uisp_data
+                            .devices
+                            .iter()
+                            .filter(|d| d.site_id == *client_id)
+                            .collect::<Vec<_>>();
+                        for device in devices.iter().filter(|d| d.has_address()) {
+                            let sd = ShapedDevice {
+                                circuit_id: site.id.clone(),
+                                circuit_name: site.name.clone(),
+                                device_id: device.id.clone(),
+                                device_name: device.name.clone(),
+                                parent_node: parent.to_owned(),
+                                mac: device.mac.clone(),
+                                ipv4: device.ipv4_list(),
+                                ipv6: device.ipv6_list(),
+                                download_min: (site.max_down_mbps as f32
+                                    * config.uisp_integration.commit_bandwidth_multiplier)
+                                    as u64,
+                                upload_min: (site.max_up_mbps as f32
+                                    * config.uisp_integration.commit_bandwidth_multiplier)
+                                    as u64,
+                                download_max: (site.max_down_mbps as f32
+                                    * config.uisp_integration.bandwidth_overhead_factor)
+                                    as u64,
+                                upload_max: (site.max_up_mbps as f32
+                                    * config.uisp_integration.bandwidth_overhead_factor)
+                                    as u64,
+                                comment: "".to_string(),
+                            };
+                            shaped_devices.push(sd);
+                        }
+                    } else {
+                        warn!("Client not found: {}", client_id);
+                    }
+                }
+                _ => {}
+            }
+        }
+        if parent.is_some() {
+            root.insert(
+                "children".to_string(),
+                serde_json::to_value(children).unwrap(),
+            );
+        } else {
+            for child in children {
+                root.insert(child.0, child.1);
+            }
+        }
+        root
+    }
+}
+
+pub struct TroublesomeClients {
+    #[allow(dead_code)] // Used in serialization
+    pub multi_entry_points: HashSet<String>,
+    #[allow(dead_code)] // Used in serialization
+    pub client_of_clients: HashSet<String>,
+}
+
+pub(crate) async fn find_troublesome_sites(data: &UispData) -> anyhow::Result<TroublesomeClients> {
+    let multi_entry_points = find_clients_with_multiple_entry_points(data)?;
+    let client_of_clients = find_clients_linked_from_other_clients(data)?;
+
+    let _ = blackboard_blob("uisp-trouble-multi-entry", vec![multi_entry_points.clone()]).await;
+    let _ = blackboard_blob("uisp-trouble-client-of-client", vec![client_of_clients.clone()]).await;
+
+    Ok(TroublesomeClients {
+        multi_entry_points,
+        client_of_clients,
+    })
+}
+
+fn find_clients_with_multiple_entry_points(data: &UispData) -> anyhow::Result<HashSet<String>> {
+    let mut result = HashSet::new();
+    for client in data.find_client_sites() {
+        let mut links_to_client = HashSet::new();
+        for link in data.data_links_raw.iter() {
+            if let (Some(from_site), Some(to_site)) = (&link.from.site, &link.to.site) {
+                if from_site.identification.id == client.id
+                    && to_site.identification.id != client.id
+                {
+                    links_to_client.insert(to_site.identification.id.clone());
+                } else if from_site.identification.id != client.id
+                    && to_site.identification.id == client.id
+                {
+                    links_to_client.insert(from_site.identification.id.clone());
+                }
+            }
+        }
+        if links_to_client.len() > 1 {
+            warn!(
+                "Client {} has multiple entry points: {:?}",
+                client.name, links_to_client
+            );
+            result.insert(client.id.clone());
+        }
+    }
+
+    Ok(result)
+}
+
+fn find_clients_linked_from_other_clients(data: &UispData) -> anyhow::Result<HashSet<String>> {
+    let all_clients = data.find_client_sites();
+    let mut result = HashSet::new();
+    for client in &all_clients {
+        for link in data.data_links_raw.iter() {
+            if let (Some(from_site), Some(to_site)) = (&link.from.site, &link.to.site) {
+                if from_site.identification.id == client.id
+                    && to_site.identification.id != client.id
+                    && all_clients
+                        .iter()
+                        .any(|c| c.id == to_site.identification.id)
+                {
+                    warn!(
+                        "Client {} is linked from another client: {}",
+                        client.name, to_site.identification.id
+                    );
+                    result.insert(client.id.clone());
+                }
+                if from_site.identification.id != client.id
+                    && to_site.identification.id == client.id
+                    && all_clients
+                        .iter()
+                        .any(|c| c.id == from_site.identification.id)
+                {
+                    warn!(
+                        "Client {} is linked to another client: {}",
+                        client.name, from_site.identification.id
+                    );
+                    result.insert(client.id.clone());
+                }
+            }
+        }
+    }
+    Ok(result)
+}

src/rust/uisp_integration/src/strategies/common.rs

@@ -0,0 +1,215 @@
+use crate::blackboard_blob;
+use crate::errors::UispIntegrationError;
+use crate::ip_ranges::IpRanges;
+use crate::strategies::full::parse::parse_uisp_datasets;
+use crate::strategies::full::uisp_fetch::load_uisp_data;
+use crate::uisp_types::{UispDevice, UispSite, UispSiteType};
+use lqos_config::Config;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tracing::warn;
+use uisp::{DataLink, Device, Site};
+
+pub(crate) struct UispData {
+    pub sites_raw: Vec<Site>,
+    pub devices_raw: Vec<Device>,
+    pub data_links_raw: Vec<DataLink>,
+    pub sites: Vec<UispSite>,
+    pub devices: Vec<UispDevice>,
+    //pub data_links: Vec<UispDataLink>,
+}
+
+impl UispData {
+    pub(crate) async fn fetch_uisp_data(
+        config: Arc<Config>,
+        ip_ranges: IpRanges,
+    ) -> std::result::Result<Self, UispIntegrationError> {
+        // Obtain the UISP data and transform it into easier to work with types
+        let (sites_raw, devices_raw, data_links_raw) = load_uisp_data(config.clone()).await?;
+
+        if let Err(e) = blackboard_blob("uisp_sites", &sites_raw).await {
+            warn!("Unable to write sites to blackboard: {e:?}");
+        }
+        if let Err(e) = blackboard_blob("uisp_devices", &devices_raw).await {
+            warn!("Unable to write devices to blackboard: {e:?}");
+        }
+        if let Err(e) = blackboard_blob("uisp_data_links", &data_links_raw).await {
+            warn!("Unable to write data links to blackboard: {e:?}");
+        }
+
+        // If Mikrotik is enabled, we need to fetch the Mikrotik data
+        let ipv4_to_v6 = crate::strategies::full::mikrotik::mikrotik_data(&config)
+            .await
+            .unwrap_or_else(|_| Vec::new());
+
+        // Parse the UISP data into a more usable format
+        let (sites, _data_links, devices) = parse_uisp_datasets(
+            &sites_raw,
+            &data_links_raw,
+            &devices_raw,
+            &config,
+            &ip_ranges,
+            ipv4_to_v6,
+        );
+
+        Ok(UispData {
+            sites_raw,
+            devices_raw,
+            data_links_raw,
+            sites,
+            devices,
+            //data_links: _data_links,
+        })
+    }
+
+    pub fn find_client_sites(&self) -> Vec<&UispSite> {
+        self.sites
+            .iter()
+            .filter(|s| s.site_type == UispSiteType::Client)
+            .collect()
+    }
+
+    pub fn find_devices_in_site(&self, site_id: &str) -> Vec<&Device> {
+        self.devices_raw
+            .iter()
+            .filter(|d| d.get_site_id().unwrap_or_default() == site_id)
+            .collect()
+    }
+
+    pub fn find_device_by_id(&self, device_id: &str) -> Option<&Device> {
+        self.devices_raw
+            .iter()
+            .find(|d| d.identification.id == device_id)
+    }
+
+    pub fn find_device_by_name(&self, device_name: &str) -> Option<&Device> {
+        self.devices_raw
+            .iter()
+            .find(|d| d.get_name().unwrap_or_default() == device_name)
+    }
+
+    pub fn map_clients_to_aps(&self) -> HashMap<String, Vec<String>> {
+        let mut mappings = HashMap::new();
+        for client in self.find_client_sites() {
+            let mut found = false;
+            let mut parent = None;
+            for device in self.find_devices_in_site(&client.id) {
+                //println!("Client {} has a device {:?}", client.name, device.get_name());
+                // Look for Parent AP attributes
+                if let Some(attr) = &device.attributes {
+                    if let Some(ap) = &attr.apDevice {
+                        if let Some(ap_id) = &ap.id {
+                            //println!("AP ID: {}", ap_id);
+                            if let Some(apdev) = self.find_device_by_id(ap_id) {
+                                //println!("AP Device: {:?}", apdev.get_name());
+                                if apdev.get_site_id().unwrap_or_default() != client.id {
+                                    parent = Some(("AP", apdev.get_name().unwrap_or_default()));
+                                    found = true;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                // Look in Site-DeviceSite
+                if !found {
+                    if let Some(device_site) = &device.identification.site {
+                        if let Some(apdev) = self.find_device_by_id(&device_site.id) {
+                            if apdev.get_site_id().unwrap_or_default() != client.id {
+                                parent = Some(("AP", apdev.get_name().unwrap_or_default()));
+                                found = true;
+                            }
+                        }
+                    }
+                }
+
+                // Look for data links with this device
+                if !found {
+                    for link in self.data_links_raw.iter() {
+                        // Check the FROM side
+                        if let Some(from_device) = &link.from.device {
+                            if from_device.identification.id == device.identification.id {
+                                if let Some(to_device) = &link.to.device {
+                                    if let Some(apdev) =
+                                        self.find_device_by_id(&to_device.identification.id)
+                                    {
+                                        if apdev.get_site_id().unwrap_or_default() != client.id {
+                                            parent =
+                                                Some(("AP", apdev.get_name().unwrap_or_default()));
+                                            found = true;
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        // Check the TO side
+                        if let Some(to_device) = &link.to.device {
+                            if to_device.identification.id == device.identification.id {
+                                if let Some(from_device) = &link.from.device {
+                                    if let Some(apdev) =
+                                        self.find_device_by_id(&from_device.identification.id)
+                                    {
+                                        if apdev.get_site_id().unwrap_or_default() != client.id {
+                                            parent =
+                                                Some(("AP", apdev.get_name().unwrap_or_default()));
+                                            found = true;
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            // If we still haven't found anything, let's try data links to the client site as a whole
+            if !found {
+                for link in self.data_links_raw.iter() {
+                    if let Some(from_site) = &link.from.site {
+                        if from_site.identification.id == client.id {
+                            if let Some(to_device) = &link.to.device {
+                                if let Some(apdev) =
+                                    self.find_device_by_id(&to_device.identification.id)
+                                {
+                                    if apdev.get_site_id().unwrap_or_default() != client.id {
+                                        parent = Some(("AP", apdev.get_name().unwrap_or_default()));
+                                        found = true;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    if let Some(to_site) = &link.to.site {
+                        if to_site.identification.id == client.id {
+                            if let Some(from_device) = &link.from.device {
+                                if let Some(apdev) =
+                                    self.find_device_by_id(&from_device.identification.id)
+                                {
+                                    if apdev.get_site_id().unwrap_or_default() != client.id {
+                                        parent = Some(("AP", apdev.get_name().unwrap_or_default()));
+                                        found = true;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            if !found {
+                //println!("Client {} has no obvious parent AP", client.name);
+                let entry = mappings
+                    .entry("Orphans".to_string())
+                    .or_insert_with(Vec::new);
+                entry.push(client.id.clone());
+            } else {
+                //info!("Client {} is connected to {:?}", client.name, parent);
+                if let Some((_, parent)) = &parent {
+                    let entry = mappings.entry(parent.to_string()).or_insert_with(Vec::new);
+                    entry.push(client.id.clone());
+                }
+            }
+        }
+        mappings
+    }
+}

src/rust/uisp_integration/src/strategies/full/bandwidth_overrides.rs

@@ -56,7 +56,7 @@ pub fn get_site_bandwidth_overrides(
                 let parent_node = result[0].to_string();
                 if let Some(d) = numeric_string_to_f32(&result[1]) {
                     if let Some(u) = numeric_string_to_f32(&result[2]) {
-                        info!("Using bandiwdth override: {}, {}/{}", parent_node, d, u);
+                        info!("Loaded bandiwdth override: {}, {}/{}", parent_node, d, u);
                         overrides.insert(parent_node, (d, u));
                     } else {
                         error!("Cannot parse {} as float on line {line}", &result[2]);

src/rust/uisp_integration/src/strategies/full/mod.rs

@@ -1,15 +1,16 @@
+#![allow(dead_code)]
 mod ap_promotion;
-mod bandwidth_overrides;
+pub(crate) mod bandwidth_overrides;
 mod client_site_promotion;
-mod mikrotik;
+pub mod mikrotik;
 mod network_json;
-mod parse;
+pub(crate) mod parse;
 mod root_site;
-mod routes_override;
-mod shaped_devices_writer;
+pub(crate) mod routes_override;
+pub mod shaped_devices_writer;
 mod squash_single_entry_aps;
 mod tree_walk;
-mod uisp_fetch;
+pub(crate) mod uisp_fetch;
 mod utils;
 mod zero_capacity_sites;
 

src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs

@@ -7,7 +7,7 @@ use tracing::{error, info};
 
 /// Represents a shaped device in the ShapedDevices.csv file.
 #[derive(Serialize, Debug)]
-struct ShapedDevice {
+pub struct ShapedDevice {
     pub circuit_id: String,
     pub circuit_name: String,
     pub device_id: String,

src/rust/uisp_integration/src/strategies/full2.rs

@@ -0,0 +1,632 @@
+mod dot;
+mod graph_mapping;
+mod link_mapping;
+mod net_json_parent;
+
+use crate::blackboard_blob;
+use crate::errors::UispIntegrationError;
+use crate::ip_ranges::IpRanges;
+use crate::strategies::common::UispData;
+use crate::strategies::full::routes_override::RouteOverride;
+use crate::strategies::full::shaped_devices_writer::ShapedDevice;
+use crate::strategies::full2::dot::save_dot_file;
+use crate::strategies::full2::graph_mapping::GraphMapping;
+use crate::strategies::full2::link_mapping::LinkMapping;
+use crate::strategies::full2::net_json_parent::{NetJsonParent, walk_parents};
+use crate::uisp_types::UispDevice;
+use lqos_config::Config;
+use petgraph::graph::NodeIndex;
+use petgraph::visit::{EdgeRef, NodeRef};
+use petgraph::Directed;
+use std::collections::{HashMap, HashSet};
+use std::fs::write;
+use std::path::Path;
+use std::sync::Arc;
+use tracing::{error, info, warn};
+
+type GraphType = petgraph::Graph<GraphMapping, LinkMapping, Directed>;
+
+pub async fn build_full_network_v2(
+    config: Arc<Config>,
+    ip_ranges: IpRanges,
+) -> Result<(), UispIntegrationError> {
+    // Fetch the data
+    let uisp_data = UispData::fetch_uisp_data(config.clone(), ip_ranges).await?;
+
+    if let Err(e) = blackboard_blob("uisp_sites", &uisp_data.sites_raw).await {
+        warn!("Unable to write sites to blackboard: {e:?}");
+    }
+    if let Err(e) = blackboard_blob("uisp_devices", &uisp_data.devices_raw).await {
+        warn!("Unable to write devices to blackboard: {e:?}");
+    }
+    if let Err(e) = blackboard_blob("uisp_data_links", &uisp_data.data_links_raw).await {
+        warn!("Unable to write data links to blackboard: {e:?}");
+    }
+
+    // Report on obvious UISP errors that should be fixed
+    let _trouble = crate::strategies::ap_site::find_troublesome_sites(&uisp_data)
+        .await
+        .map_err(|e| {
+            error!("Error finding troublesome sites");
+            error!("{e:?}");
+            UispIntegrationError::UnknownSiteType
+        })?;
+
+    // Load overrides
+    let bandwidth_overrides =
+        crate::strategies::full::bandwidth_overrides::get_site_bandwidth_overrides(&config)?;
+    let routing_overrides = crate::strategies::full::routes_override::get_route_overrides(&config)?;
+
+    // Create a new graph
+    let mut graph = GraphType::new();
+
+    // Find the root
+    let root_site_name = config.uisp_integration.site.clone();
+
+    // Add all sites to the graph
+    let mut site_map = HashMap::new();
+    let mut root_idx = None;
+    add_all_sites_to_graph(
+        &uisp_data,
+        &mut graph,
+        &root_site_name,
+        &mut site_map,
+        &mut root_idx,
+    );
+    let root_idx = root_idx.expect("Root site not found");
+
+    // Iterate all UISP devices and if their parent site is in the graph, add them
+    let mut device_map = HashMap::new();
+    add_devices_to_graph(&uisp_data, &mut graph, &mut site_map, &mut device_map, &config, &bandwidth_overrides);
+
+    // Now we iterate all the data links looking for DEVICE linkage
+    add_device_links_to_graph(&uisp_data, &mut graph, &mut device_map, &config);
+
+    // Now we iterate only sites, looking for connectivity to the root
+    let orphans = graph.add_node(GraphMapping::GeneratedSite {
+        name: "Orphans".to_string(),
+    });
+    graph.add_edge(root_idx, orphans, LinkMapping::ethernet(config.queues.generated_pn_download_mbps));
+    graph.add_edge(orphans, root_idx, LinkMapping::ethernet(config.queues.generated_pn_upload_mbps));
+    for (_, site_ref) in site_map.iter() {
+        if *site_ref == root_idx {
+            continue;
+        }
+        let a_star_run =
+            petgraph::algo::astar(&graph, *site_ref, |n| n.id() == root_idx.id(), |_| 0, |_| 0);
+
+        if a_star_run.is_none() {
+            warn!(
+                "No path is detected from {:?} to {}",
+                graph[*site_ref], root_site_name
+            );
+            graph.add_edge(*site_ref, orphans, LinkMapping::ethernet(config.queues.generated_pn_download_mbps));
+            graph.add_edge(orphans, *site_ref, LinkMapping::ethernet(config.queues.generated_pn_upload_mbps));
+        }
+    }
+
+    // Client mapping
+    let client_mappings = uisp_data.map_clients_to_aps();
+
+    // Find the APs that have clients
+    let mut aps_with_clients = HashSet::new();
+    for (ap_name, _client_ids) in client_mappings.iter() {
+        let Some(ap_device) = uisp_data
+            .devices_raw
+            .iter()
+            .find(|d| d.get_name().unwrap_or_default() == *ap_name)
+        else {
+            // Orphaning is already handled
+            continue;
+        };
+        aps_with_clients.insert(ap_device.identification.id.clone());
+    }
+
+    // Count linkages to APs
+    let mut ap_link_count = HashMap::<String, usize>::new();
+    for edge_ref in graph.edge_references() {
+        if let GraphMapping::AccessPoint { ref id, .. } = graph[edge_ref.source()] {
+            let entry = ap_link_count.entry(id.clone()).or_insert(0);
+            *entry += 1;
+        };
+        if let GraphMapping::AccessPoint { ref id, .. } = graph[edge_ref.target()] {
+            let entry = ap_link_count.entry(id.clone()).or_insert(0);
+            *entry += 1;
+        };
+    }
+
+    // Cull the APs that have no clients or only one link
+    let mut to_remove = Vec::new();
+    for (ap_id, ap_ref) in device_map.iter() {
+        if aps_with_clients.contains(ap_id) {
+            continue;
+        }
+        if let Some(link_count) = ap_link_count.get(ap_id) {
+            if *link_count > 1 {
+                continue;
+            }
+        }
+        to_remove.push(*ap_ref);
+    }
+    info!(
+        "Removing {} APs with no clients or only one link",
+        to_remove.len()
+    );
+    for ap_ref in to_remove.iter() {
+        graph.remove_node(*ap_ref);
+    }
+
+    // Visualizer
+    save_dot_file(&graph)?;
+    let _ = blackboard_blob("uisp-graph", vec![graph.clone()]).await;
+
+    // Figure out the network.json layers
+    let mut parents = HashMap::<String, NetJsonParent>::new();
+    for node in graph.node_indices() {
+        if node == root_idx {
+            continue;
+        }
+        match &graph[node] {
+            GraphMapping::GeneratedSite { name }
+            | GraphMapping::Site { name, .. }
+            | GraphMapping::AccessPoint { name, .. } => {
+                // Generate two routes - one per direction
+                let route_from_root_to_node = petgraph::algo::astar(
+                    &graph,
+                    root_idx,
+                    |n| n == node,
+                    |e| {
+                        (10_000u64).saturating_sub(link_capacity_mbps_for_routing(
+                            &e.weight(),
+                            &uisp_data.devices,
+                            &routing_overrides,
+                        ))
+                    },
+                    |_| 0,
+                )
+                .unwrap_or((0, vec![]));
+
+                let route_from_node_to_root = petgraph::algo::astar(
+                    &graph,
+                    node,
+                    |n| n == root_idx,
+                    |e| {
+                        (10_000u64).saturating_sub(link_capacity_mbps_for_routing(
+                            &e.weight(),
+                            &uisp_data.devices,
+                            &routing_overrides,
+                        ))
+                    },
+                    |_| 0,
+                )
+                    .unwrap_or((0, vec![]));
+
+                // println!("From node to root:");
+                // println!("{:?}", route_from_node_to_root);
+                // println!("{:?}", edges_from_node_path(&graph, &route_from_node_to_root.1));
+                //
+                // println!("From root to node:");
+                // println!("{:?}", route_from_root_to_node);
+                // println!("{:?}", edges_from_node_path(&graph, &route_from_root_to_node.1));
+
+                if route_from_root_to_node.1.is_empty() {
+                    //println!("No path detected from {:?} to {}", graph[node], root_site_name);
+                    parents.insert(
+                        name.to_owned(),
+                        NetJsonParent {
+                            parent_name: "Orphans".to_string(),
+                            mapping: &graph[node],
+                            download: config.queues.generated_pn_download_mbps,
+                            upload: config.queues.generated_pn_upload_mbps,
+                        },
+                    );
+                } else {
+                    // Obtain capacities from route traversal
+                    let mut download_capacity = min_capacity_along_route(&graph, &route_from_root_to_node.1);
+                    let mut upload_capacity = min_capacity_along_route(&graph, &route_from_node_to_root.1);
+
+                    // Apply AP capacities
+                    if let GraphMapping::AccessPoint { download_mbps, upload_mbps, .. } = &graph[node] {
+                        //println!("AP device capacity: {download_mbps}/{upload_mbps} (prev {download_capacity}/{upload_capacity})");
+                        download_capacity = u64::min(download_capacity, *download_mbps);
+                        upload_capacity = u64::min(upload_capacity, *upload_mbps);
+                        //println!("Now: {download_capacity}/{upload_capacity}");
+                    }
+
+                    // 0 isn't possible
+                    if download_capacity < 1 {
+                        download_capacity = config.queues.generated_pn_download_mbps;
+                    }
+                    if upload_capacity < 1 {
+                        upload_capacity = config.queues.generated_pn_upload_mbps;
+                    }
+
+                    // Ignore option
+                    if config.uisp_integration.ignore_calculated_capacity {
+                        download_capacity = config.queues.generated_pn_download_mbps;
+                        upload_capacity = config.queues.generated_pn_upload_mbps;
+                    }
+
+                    // Overrides
+                    if !bandwidth_overrides.is_empty() {
+                        if let Some(bw_override) = bandwidth_overrides.get(name) {
+                            info!("Applying bandwidth override for {}", name);
+                            info!("Capacity was: {} / {}", download_capacity, upload_capacity);
+                            download_capacity = bw_override.0 as u64;
+                            upload_capacity = bw_override.1 as u64;
+                            info!("Capacity is now: {} / {}", download_capacity, upload_capacity);
+                        }
+                    }
+
+                    let parent_node = route_from_root_to_node.1[route_from_root_to_node.1.len() - 2];
+                    // We need the weight from node to parent_node in the graph edges
+                    if let Some(_edge) = graph.find_edge(parent_node, node) {
+                        let parent = graph[route_from_root_to_node.1[route_from_root_to_node.1.len() - 2]].name();
+                        parents.insert(
+                            name.to_owned(),
+                            NetJsonParent {
+                                parent_name: parent,
+                                mapping: &graph[node],
+                                download: download_capacity,
+                                upload: upload_capacity,
+                            },
+                        );
+                    } else {
+                        panic!("DID NOT FIND THE EDGE");
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+
+    // Write the network.json file
+    let mut network_json = serde_json::Map::new();
+    let mut visited = HashSet::new();
+    for (name, node_info) in parents
+        .iter()
+        .filter(|(_name, parent)| parent.parent_name == root_site_name)
+    {
+        network_json.insert(
+            name.into(),
+            walk_parents(&parents, name, &node_info, &config, &graph, &mut visited).into(),
+        );
+    }
+    let network_path = Path::new(&config.lqos_directory).join("network.json");
+    if network_path.exists() && !config.integration_common.always_overwrite_network_json {
+        warn!(
+            "Network.json exists, and always overwrite network json is not true - not writing network.json"
+        );
+        return Ok(());
+    }
+    let json = serde_json::to_string_pretty(&network_json).unwrap();
+    write(network_path, json).map_err(|e| {
+        error!("Unable to write network.json");
+        error!("{e:?}");
+        UispIntegrationError::WriteNetJson
+    })?;
+    info!("Written network.json");
+
+    // Shaped Devices
+    let mut shaped_devices = Vec::new();
+
+    for (ap_id, client_sites) in client_mappings.iter() {
+        for site_id in client_sites.iter() {
+            let Some(ap_device) = uisp_data.devices.iter().find(|d| d.name == *ap_id) else {
+                continue;
+            };
+            let Some(site) = uisp_data.sites.iter().find(|s| s.id == *site_id) else {
+                continue;
+            };
+            for device in uisp_data.devices.iter().filter(|d| d.site_id == *site_id) {
+                if !device.has_address() {
+                    continue;
+                }
+
+                let download =
+                    (site.max_down_mbps as f32) * config.uisp_integration.bandwidth_overhead_factor;
+                let upload =
+                    (site.max_up_mbps as f32) * config.uisp_integration.bandwidth_overhead_factor;
+                let download_min =
+                    (download * config.uisp_integration.commit_bandwidth_multiplier) as u64;
+                let upload_min =
+                    (upload * config.uisp_integration.commit_bandwidth_multiplier) as u64;
+                let download_max = download as u64;
+                let upload_max = upload as u64;
+
+                let download_min = u64::max(1, download_min);
+                let upload_min = u64::max(1, upload_min);
+                let download_max = u64::max(4, download_max);
+                let upload_max = u64::max(4, upload_max);
+
+                let parent_node = {
+                    if parents.get(&ap_device.name).is_some() {
+                        ap_device.name.clone()
+                    } else {
+                        "Orphans".to_string()
+                    }
+                };
+
+                let shaped_device = ShapedDevice {
+                    circuit_id: site.id.to_owned(),
+                    circuit_name: site.name.to_owned(),
+                    device_id: device.id.to_owned(),
+                    device_name: device.name.to_owned(),
+                    parent_node,
+                    mac: device.mac.to_owned(),
+                    ipv4: device.ipv4_list(),
+                    ipv6: device.ipv6_list(),
+                    download_min,
+                    upload_min,
+                    download_max,
+                    upload_max,
+                    comment: "".to_string(),
+                };
+                shaped_devices.push(shaped_device);
+            }
+        }
+    }
+    let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv");
+    let mut writer = csv::WriterBuilder::new()
+        .has_headers(true)
+        .from_path(file_path)
+        .unwrap();
+
+    for d in shaped_devices.iter() {
+        writer.serialize(d).unwrap();
+    }
+    writer.flush().map_err(|e| {
+        error!("Unable to flush CSV file");
+        error!("{e:?}");
+        UispIntegrationError::CsvError
+    })?;
+    info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len());
+
+    Ok(())
+}
+
+fn add_device_links_to_graph(
+    uisp_data: &UispData,
+    graph: &mut GraphType,
+    device_map: &mut HashMap<String, NodeIndex>,
+    config: &Arc<Config>,
+) {
+    for link in uisp_data.data_links_raw.iter() {
+        let Some(from_device) = &link.from.device else {
+            continue;
+        };
+        let Some(to_device) = &link.to.device else {
+            continue;
+        };
+        let Some(a_ref) = device_map.get(&from_device.identification.id) else {
+            continue;
+        };
+        let Some(b_ref) = device_map.get(&to_device.identification.id) else {
+            continue;
+        };
+        if a_ref == b_ref {
+            // If the devices are the same, we don't need to add an edge
+            continue;
+        }
+        if let Some(dev_a) = uisp_data
+            .devices_raw
+            .iter()
+            .find(|d| d.get_id() == from_device.identification.id)
+        {
+            if let Some(dev_b) = uisp_data
+                .devices_raw
+                .iter()
+                .find(|d| d.get_id() == to_device.identification.id)
+            {
+                if dev_a.get_site_id().unwrap_or_default()
+                    == dev_b.get_site_id().unwrap_or_default()
+                {
+                    // If the devices are in the same site, we don't need to add an edge
+                    continue;
+                }
+            }
+        }
+        if graph.contains_edge(*a_ref, *b_ref) {
+            // If the edge already exists, we don't need to add it
+            continue;
+        }
+        if graph.contains_edge(*b_ref, *a_ref) {
+            // If the edge already exists, we don't need to add it
+            continue;
+        }
+
+        let (down, up) = get_capacity_from_datalink_device(
+            &from_device.identification.id,
+            &uisp_data.devices,
+            &config,
+        );
+        graph.add_edge(
+            *a_ref,
+            *b_ref,
+            LinkMapping::DevicePair {
+                device_a: from_device.identification.id.clone(),
+                device_b: to_device.identification.id.clone(),
+                speed_mbps: down,
+            },
+        );
+        graph.add_edge(
+            *b_ref,
+            *a_ref,
+            LinkMapping::DevicePair {
+                device_b: from_device.identification.id.clone(),
+                device_a: to_device.identification.id.clone(),
+                speed_mbps: up,
+            },
+        );
+    }
+}
+
+fn get_capacity_from_datalink_device(
+    device_id: &str,
+    devices: &[UispDevice],
+    config: &Arc<Config>,
+) -> (u64, u64) {
+    if let Some(device) = devices.iter().find(|d| d.id == device_id) {
+        return (device.download, device.upload);
+    }
+
+    (config.queues.generated_pn_download_mbps, config.queues.generated_pn_upload_mbps)
+}
+
+fn add_devices_to_graph(
+    uisp_data: &UispData,
+    graph: &mut GraphType,
+    site_map: &mut HashMap<String, NodeIndex>,
+    device_map: &mut HashMap<String, NodeIndex>,
+    config: &Arc<Config>,
+    bandwidth_overrides: &BandwidthOverrides,
+) {
+    for device in uisp_data.devices_raw.iter() {
+        let Some(site_id) = &device.identification.site else {
+            continue;
+        };
+        let site_id = &site_id.id;
+        let Some(device_details) = uisp_data.devices.iter().find(|d| d.id == device.identification.id) else {
+            continue;
+        };
+        let Some(site_ref) = site_map.get(site_id) else {
+            continue;
+        };
+        let mut download_mbps = device_details.download;
+        let mut upload_mbps = device_details.upload;
+
+        if download_mbps < 1 {
+            download_mbps = config.queues.generated_pn_download_mbps;
+        }
+        if upload_mbps < 1 {
+            upload_mbps = config.queues.generated_pn_upload_mbps;
+        }
+
+        if config.uisp_integration.ignore_calculated_capacity {
+            download_mbps = config.queues.generated_pn_download_mbps;
+            upload_mbps = config.queues.generated_pn_upload_mbps;
+        }
+
+        if let Some(bw_override) = bandwidth_overrides.get(&device.get_name().unwrap_or_default()) {
+            download_mbps = bw_override.0 as u64;
+            upload_mbps = bw_override.1 as u64;
+        }
+
+        let device_entry = GraphMapping::AccessPoint {
+            name: device.get_name().unwrap_or_default(),
+            id: device.identification.id.clone(),
+            site_name: graph[*site_ref].name(),
+            download_mbps,
+            upload_mbps,
+        };
+        let device_ref = graph.add_node(device_entry);
+        device_map.insert(device.identification.id.clone(), device_ref);
+        let _ = graph.add_edge(device_ref, *site_ref, LinkMapping::ethernet(config.queues.generated_pn_download_mbps));
+        let _ = graph.add_edge(*site_ref, device_ref, LinkMapping::ethernet(config.queues.generated_pn_upload_mbps));
+    }
+}
+
+pub fn add_all_sites_to_graph(
+    uisp_data: &UispData,
+    graph: &mut GraphType,
+    root_site_name: &String,
+    site_map: &mut HashMap<String, NodeIndex>,
+    root_idx: &mut Option<NodeIndex>,
+) {
+    for site in uisp_data.sites_raw.iter().filter(|s| !s.is_client_site()) {
+        let site_name = site.name_or_blank();
+        let id = site.id.clone();
+        if site_name == *root_site_name {
+            // Add the root site
+            let root_entry = GraphMapping::Root {
+                name: site_name,
+                id,
+            };
+            let root_ref = graph.add_node(root_entry);
+            *root_idx = Some(root_ref);
+            site_map.insert(site.id.clone(), root_ref);
+            continue;
+        }
+        let site_entry = GraphMapping::Site {
+            name: site_name,
+            id,
+        };
+        let site_ref = graph.add_node(site_entry);
+        site_map.insert(site.id.clone(), site_ref);
+    }
+}
+
+fn link_capacity_mbps_for_routing(
+    link_mapping: &LinkMapping,
+    devices: &[UispDevice],
+    route_overrides: &[RouteOverride],
+) -> u64 {
+    match link_mapping {
+        LinkMapping::Ethernet { speed_mbps } => *speed_mbps,
+        LinkMapping::DevicePair { speed_mbps, device_a, device_b } => {
+            // Check for overrides: make sure we have both devices (we need names, are storing IDs)
+            let Some(device_a) = devices.iter().find(|d| d.id == *device_a) else {
+                return *speed_mbps;
+            };
+            let Some(device_b) = devices.iter().find(|d| d.id == *device_b) else {
+                return *speed_mbps;
+            };
+
+            // Check for the direct direction
+            if let Some(route_override) = route_overrides
+                .iter()
+                .find(|route_override|
+                    (route_override.from_site == device_a.name || route_override.to_site == device_a.name)
+                        && (route_override.from_site == device_b.name || route_override.to_site == device_b.name)
+                ) {
+                return route_override.cost as u64;
+            }
+
+            // Check for the reverse direction
+            if let Some(route_override) = route_overrides
+                .iter()
+                .find(|route_override|
+                    (route_override.from_site == device_b.name || route_override.to_site == device_b.name)
+                        && (route_override.from_site == device_a.name || route_override.to_site == device_a.name)
+                ) {
+                return route_override.cost as u64;
+            }
+
+            *speed_mbps
+        }
+    }
+}
+
+use petgraph::prelude::*;
+use crate::strategies::full::bandwidth_overrides::BandwidthOverrides;
+
+fn edges_from_node_path(
+    graph: &GraphType,
+    path: &[NodeIndex],
+) -> Vec<EdgeIndex>
+{
+    let mut edge_ids = Vec::new();
+    for window in path.windows(2) {
+        let source = window[0];
+        let target = window[1];
+        // This assumes a directed graph; if undirected, you might need to check both directions
+        if let Some(edge) = graph.find_edge(source, target) {
+            edge_ids.push(edge);
+        } else {
+            // Handle case where no edge is found (could be error or special case)
+            panic!("No edge found between {:?} and {:?}", source, target);
+        }
+    }
+    edge_ids
+}
+
+fn min_capacity_along_route(
+    graph: &GraphType,
+    path: &[NodeIndex],
+) -> u64 {
+    edges_from_node_path(graph, path)
+        .iter()
+        .map(|edge| graph[*edge].capacity_mbps())
+        .min()
+        .unwrap_or(0)
+}

src/rust/uisp_integration/src/strategies/full2/dot.rs

@@ -0,0 +1,19 @@
+use crate::errors::UispIntegrationError;
+use crate::strategies::full2::GraphType;
+use std::fs::write;
+
+pub fn save_dot_file(graph: &GraphType) -> Result<(), UispIntegrationError> {
+    // Save the dot file
+    let dot_data = format!(
+        "{:?}",
+        petgraph::dot::Dot::with_config(graph, &[petgraph::dot::Config::EdgeNoLabel])
+    );
+    let _ = write("graph.dot", dot_data.as_bytes());
+    let _ = std::process::Command::new("dot")
+        .arg("-Tpng")
+        .arg("graph.dot")
+        .arg("-o")
+        .arg("graph.png")
+        .output();
+    Ok(())
+}

src/rust/uisp_integration/src/strategies/full2/graph_mapping.rs

@@ -0,0 +1,34 @@
+use serde::Serialize;
+
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, Serialize)]
+pub enum GraphMapping {
+    Root {
+        name: String,
+        id: String,
+    },
+    Site {
+        name: String,
+        id: String,
+    },
+    GeneratedSite {
+        name: String,
+    },
+    AccessPoint {
+        name: String,
+        id: String,
+        site_name: String,
+        download_mbps: u64,
+        upload_mbps: u64,
+    },
+}
+
+impl GraphMapping {
+    pub fn name(&self) -> String {
+        match self {
+            GraphMapping::Root { name, .. } => name.clone(),
+            GraphMapping::Site { name, .. } => name.clone(),
+            GraphMapping::GeneratedSite { name } => name.clone(),
+            GraphMapping::AccessPoint { name, .. } => name.clone(),
+        }
+    }
+}

src/rust/uisp_integration/src/strategies/full2/link_mapping.rs

@@ -0,0 +1,20 @@
+use serde::Serialize;
+
+#[derive(Debug, Clone, Serialize)]
+pub enum LinkMapping {
+    Ethernet { speed_mbps: u64 },
+    DevicePair { device_a: String, device_b: String, speed_mbps: u64, }
+}
+
+impl LinkMapping {
+    pub fn ethernet(speed_mbps: u64) -> Self {
+        LinkMapping::Ethernet { speed_mbps }
+    }
+
+    pub fn capacity_mbps(&self) -> u64 {
+        match self {
+            LinkMapping::Ethernet { speed_mbps } => *speed_mbps,
+            LinkMapping::DevicePair { speed_mbps, .. } => *speed_mbps,
+        }
+    }
+}

src/rust/uisp_integration/src/strategies/full2/net_json_parent.rs

@@ -0,0 +1,65 @@
+use crate::strategies::full2::GraphType;
+use crate::strategies::full2::graph_mapping::GraphMapping;
+use lqos_config::Config;
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+
+#[derive(Debug)]
+pub struct NetJsonParent<'a> {
+    pub parent_name: String,
+    pub mapping: &'a GraphMapping,
+    pub download: u64,
+    pub upload: u64,
+}
+
+pub fn walk_parents(
+    parents: &HashMap<String, NetJsonParent>,
+    name: &String,
+    node_info: &NetJsonParent,
+    config: &Arc<Config>,
+    graph: &GraphType,
+    visited: &mut HashSet<String>,
+) -> serde_json::Map<String, serde_json::Value> {
+    let mut map = serde_json::Map::new();
+    // Entries are name, type, uisp_device or site, downloadBandwidthMbps, uploadBandwidthMbps, children
+    map.insert("name".into(), name.clone().into());
+    map.insert("downloadBandwidthMbps".into(), node_info.download.into());
+    map.insert("uploadBandwidthMbps".into(), node_info.upload.into());
+    match node_info.mapping {
+        GraphMapping::Root { id, .. } | GraphMapping::Site { id, .. } => {
+            map.insert("type".into(), "Site".into());
+            map.insert("uisp_site".into(), id.clone().into());
+            map.insert("parent_site".into(), name.clone().into());
+        }
+        GraphMapping::GeneratedSite { .. } => {
+            map.insert("type".into(), "Site".into());
+        }
+        GraphMapping::AccessPoint {
+            name: _,
+            id,
+            site_name,
+            ..
+        } => {
+            map.insert("type".into(), "AP".into());
+            map.insert("parent_site".into(), site_name.clone().into());
+            map.insert("uisp_device".into(), id.clone().into());
+        }
+    }
+
+    let mut children = serde_json::Map::new();
+    for (name, node_info) in parents
+        .iter()
+        .filter(|(_node_name, node_info)| node_info.parent_name == *name)
+    {
+        if visited.contains(name) {
+            continue;
+        }
+        visited.insert(name.to_string());
+        let child = walk_parents(parents, name, &node_info, config, graph, visited);
+        children.insert(name.into(), child.into());
+    }
+
+    map.insert("children".into(), children.into());
+
+    map
+}

src/rust/uisp_integration/src/strategies/mod.rs

@@ -1,5 +1,9 @@
+mod ap_only;
+mod ap_site;
+mod common;
 mod flat;
 mod full;
+mod full2;
 
 use crate::blackboard;
 use crate::errors::UispIntegrationError;
@@ -27,10 +31,25 @@ pub async fn build_with_strategy(
             flat::build_flat_network(config, ip_ranges).await?;
             Ok(())
         }
-        "full" => {
+        /*"full" => {
             info!("Strategy selected: full");
             full::build_full_network(config, ip_ranges).await?;
             Ok(())
+        }*/
+        "ap_only" => {
+            info!("Strategy selected: ap_only");
+            ap_only::build_ap_only_network(config, ip_ranges).await?;
+            Ok(())
+        }
+        "ap_site" => {
+            info!("Strategy selected: ap_site");
+            ap_site::build_ap_site_network(config, ip_ranges).await?;
+            Ok(())
+        }
+        "full" | "full2" => {
+            info!("Strategy selected: full2");
+            full2::build_full_network_v2(config, ip_ranges).await?;
+            Ok(())
         }
         _ => {
             error!(

src/rust/uisp_integration/src/uisp_types/uisp_device.rs

@@ -52,6 +52,20 @@ impl UispDevice {
             if let Some(ul) = overview.uplinkCapacity {
                 upload = ul as u64 / 1000000;
             }
+            if device.get_model().unwrap_or_default().contains("5AC") {
+                download = ((download as f64) * config.uisp_integration.airmax_capacity as f64) as u64;
+                upload = ((upload as f64) * config.uisp_integration.airmax_capacity as f64) as u64;
+            }
+            if device.get_model().unwrap_or_default().contains("LTU") {
+                download = ((download as f64) * config.uisp_integration.ltu_capacity as f64) as u64;
+                upload = ((upload as f64) * config.uisp_integration.ltu_capacity as f64) as u64;
+            }
+        }
+        if download == 0 {
+            download = config.queues.generated_pn_download_mbps;
+        }
+        if upload == 0 {
+            upload = config.queues.generated_pn_upload_mbps;
         }
 
         // Accumulate IP address listings