Add ShapedDevices.csv creation.

src/rust/uisp_integration/Cargo.toml

@@ -17,3 +17,5 @@ thiserror = "1.0.58"
 serde = { version = "1.0.198", features = ["derive"] }
 csv = "1.3.0"
 serde_json = "1.0.116"
+ip_network_table = "0"
+ip_network = "0"

src/rust/uisp_integration/src/errors.rs

@@ -18,4 +18,6 @@ pub enum UispIntegrationError {
     CsvError,
     #[error("Unable to write network.json")]
     WriteNetJson,
+    #[error("Bad IP")]
+    BadIp,
 }

src/rust/uisp_integration/src/ip_ranges.rs

@@ -0,0 +1,64 @@
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
+use ip_network::IpNetwork;
+use ip_network_table::IpNetworkTable;
+use tracing::info;
+use lqos_config::Config;
+use crate::errors::UispIntegrationError;
+
+pub struct IpRanges {
+    allowed: IpNetworkTable<bool>,
+    ignored: IpNetworkTable<bool>,
+}
+
+impl IpRanges {
+    pub fn new(config: &Config) -> Result<Self, UispIntegrationError> {
+        info!("Building allowed/excluded IP range lookups from configuration file");
+
+        let mut allowed = IpNetworkTable::new();
+        let mut ignored = IpNetworkTable::new();
+
+        for allowed_ip in config.ip_ranges.allow_subnets.iter() {
+            let split: Vec<_> = allowed_ip.split('/').collect();
+            if (split[0].contains(':')) {
+                // It's IPv6
+                let ip_network: Ipv6Addr = split[0].parse().unwrap();
+                let ip = IpNetwork::new(ip_network, split[1].parse().unwrap()).unwrap();
+                allowed.insert(ip, true);
+            } else {
+                // It's IPv4
+                let ip_network: Ipv4Addr = split[0].parse().unwrap();
+                let ip = IpNetwork::new(ip_network, split[1].parse().unwrap()).unwrap();
+                allowed.insert(ip, true);
+            }
+        }
+        for excluded_ip in config.ip_ranges.ignore_subnets.iter() {
+            let split: Vec<_> = excluded_ip.split('/').collect();
+            if (split[0].contains(':')) {
+                // It's IPv6
+                let ip_network: Ipv6Addr = split[0].parse().unwrap();
+                let ip = IpNetwork::new(ip_network, split[1].parse().unwrap()).unwrap();
+                ignored.insert(ip, true);
+            } else {
+                // It's IPv4
+                let ip_network: Ipv4Addr = split[0].parse().unwrap();
+                let ip = IpNetwork::new(ip_network, split[1].parse().unwrap()).unwrap();
+                ignored.insert(ip, true);
+            }
+        }
+        info!("{} allowed IP ranges, {} ignored IP ranges", allowed.len().0, ignored.len().0);
+
+        Ok(Self {
+            allowed, ignored
+        })
+    }
+
+    pub fn is_permitted(&self, ip: IpAddr, subnet: u8) -> bool {
+        if let Some(allow) = self.allowed.longest_match(ip) {
+            if let Some(deny) = self.ignored.longest_match(ip) {
+                return false;
+            }
+            return true
+        }
+        false
+    }
+}

src/rust/uisp_integration/src/main.rs

@@ -5,11 +5,13 @@
 mod errors;
 mod strategies;
 pub mod uisp_types;
+pub mod ip_ranges;
 
 use crate::errors::UispIntegrationError;
 use lqos_config::Config;
 use tokio::time::Instant;
 use tracing::{error, info};
+use crate::ip_ranges::IpRanges;
 
 /// Start the tracing/logging system
 fn init_tracing() {
@@ -47,8 +49,11 @@ async fn main() -> Result<(), UispIntegrationError> {
     // Check that we're allowed to run
     check_enabled_status(&config)?;
 
+    // Build our allowed/excluded IP ranges
+    let ip_ranges = IpRanges::new(&config)?;
+
     // Select a strategy and go from there
-    strategies::build_with_strategy(config).await?;
+    strategies::build_with_strategy(config, ip_ranges).await?;
 
     // Print timings
     let elapsed = now.elapsed();

src/rust/uisp_integration/src/strategies/flat.rs

@@ -1,8 +1,9 @@
 use crate::errors::UispIntegrationError;
 use lqos_config::Config;
 use tracing::error;
+use crate::ip_ranges::IpRanges;
 
-pub async fn build_flat_network(_config: Config) -> Result<(), UispIntegrationError> {
+pub async fn build_flat_network(_config: Config, _ip_ranges: IpRanges) -> Result<(), UispIntegrationError> {
     error!("Not implemented yet");
     Ok(())
 }

src/rust/uisp_integration/src/strategies/full/mod.rs

@@ -9,6 +9,7 @@ mod utils;
 mod bandwidth_overrides;
 mod routes_override;
 mod network_json;
+mod shaped_devices_writer;
 
 use crate::errors::UispIntegrationError;
 use crate::strategies::full::ap_promotion::promote_access_points;
@@ -23,11 +24,13 @@ use crate::uisp_types::UispSite;
 use lqos_config::Config;
 use crate::strategies::full::bandwidth_overrides::get_site_bandwidth_overrides;
 pub use bandwidth_overrides::BandwidthOverrides;
+use crate::ip_ranges::IpRanges;
 use crate::strategies::full::network_json::write_network_file;
 use crate::strategies::full::routes_override::get_route_overrides;
+use crate::strategies::full::shaped_devices_writer::write_shaped_devices;
 
 /// Attempt to construct a full hierarchy topology for the UISP network.
-pub async fn build_full_network(config: Config) -> Result<(), UispIntegrationError> {
+pub async fn build_full_network(config: Config, ip_ranges: IpRanges) -> Result<(), UispIntegrationError> {
     // Load any bandwidth overrides
     let bandwidth_overrides = get_site_bandwidth_overrides(&config)?;
 
@@ -37,7 +40,7 @@ pub async fn build_full_network(config: Config) -> Result<(), UispIntegrationErr
     // Obtain the UISP data and transform it into easier to work with types
     let (sites_raw, devices_raw, data_links_raw) = load_uisp_data(config.clone()).await?;
     let (mut sites, data_links, devices) =
-        parse_uisp_datasets(&sites_raw, &data_links_raw, &devices_raw, &config, &bandwidth_overrides);
+        parse_uisp_datasets(&sites_raw, &data_links_raw, &devices_raw, &config, &bandwidth_overrides, &ip_ranges);
 
     // Check root sites
     let root_site = find_root_site(&config, &mut sites, &data_links)?;
@@ -73,6 +76,9 @@ pub async fn build_full_network(config: Config) -> Result<(), UispIntegrationErr
 
         // Output a network.json
         write_network_file(&config, &sites, root_idx)?;
+
+        // Write ShapedDevices.csv
+        write_shaped_devices(&config, &sites, root_idx, &devices)?;
     }
 
 

src/rust/uisp_integration/src/strategies/full/parse.rs

@@ -2,6 +2,7 @@ use crate::uisp_types::{UispDataLink, UispDevice, UispSite};
 use lqos_config::Config;
 use tracing::info;
 use uisp::{DataLink, Device, Site};
+use crate::ip_ranges::IpRanges;
 use crate::strategies::full::bandwidth_overrides::BandwidthOverrides;
 
 pub fn parse_uisp_datasets(
@@ -10,11 +11,12 @@ pub fn parse_uisp_datasets(
     devices_raw: &[Device],
     config: &Config,
     bandwidth_overrides: &BandwidthOverrides,
+    ip_ranges: &IpRanges,
 ) -> (Vec<UispSite>, Vec<UispDataLink>, Vec<UispDevice>) {
     let (mut sites, data_links, devices) = (
         parse_sites(sites_raw, config, bandwidth_overrides),
         parse_data_links(data_links_raw, devices_raw),
-        parse_devices(devices_raw, config),
+        parse_devices(devices_raw, config, ip_ranges),
     );
 
     // Assign devices to sites
@@ -53,10 +55,10 @@ fn parse_data_links(data_links_raw: &[DataLink], devices_raw: &[Device]) -> Vec<
     data_links
 }
 
-fn parse_devices(devices_raw: &[Device], config: &Config) -> Vec<UispDevice> {
+fn parse_devices(devices_raw: &[Device], config: &Config, ip_ranges: &IpRanges) -> Vec<UispDevice> {
     let mut devices: Vec<UispDevice> = devices_raw
         .iter()
-        .map(|d| UispDevice::from_uisp(d, config))
+        .map(|d| UispDevice::from_uisp(d, config, ip_ranges))
         .collect();
     info!("{} devices have been sucessfully parsed", devices.len());
     devices

src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs

@@ -0,0 +1,120 @@
+use std::path::Path;
+use serde::Serialize;
+use tracing::{error, info};
+use lqos_config::Config;
+use crate::errors::UispIntegrationError;
+use crate::uisp_types::{UispDevice, UispSite, UispSiteType};
+
+#[derive(Serialize, Debug)]
+struct ShapedDevice {
+    pub circuit_id: String,
+    pub circuit_name: String,
+    pub device_id: String,
+    pub device_name: String,
+    pub parent_node: String,
+    pub mac: String,
+    pub ipv4: String,
+    pub ipv6: String,
+    pub download_min: u64,
+    pub download_max: u64,
+    pub upload_min: u64,
+    pub upload_max: u64,
+    pub comment: String,
+}
+
+pub fn write_shaped_devices(config: &Config, sites: &[UispSite], root_idx: usize, devices: &[UispDevice]) -> Result<(), UispIntegrationError> {
+    let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv");
+    let mut shaped_devices = Vec::new();
+
+    // Traverse
+    traverse(sites, root_idx, 0, devices, &mut shaped_devices, config);
+
+    // Write the CSV
+    let mut writer = csv::WriterBuilder::new()
+        .has_headers(true)
+        .from_path(file_path)
+        .unwrap();
+
+    for d in shaped_devices.iter() {
+        writer.serialize(d).unwrap();
+    }
+    writer.flush().map_err(|e| {
+        error!("Unable to flush CSV file");
+        error!("{e:?}");
+        UispIntegrationError::CsvError
+    })?;
+    info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len());
+
+    Ok(())
+}
+
+fn traverse(sites: &[UispSite], idx: usize, depth: u32, devices: &[UispDevice], shaped_devices: &mut Vec<ShapedDevice>, config: &Config) {
+    if !sites[idx].device_indices.is_empty() {
+        // We have devices!
+        if sites[idx].site_type == UispSiteType::Client {
+            // Add as normal clients
+            for device in sites[idx].device_indices.iter() {
+                let device = &devices[*device];
+                if device.has_address() {
+                    let download_max = (sites[idx].max_down_mbps as f32 * config.uisp_integration.bandwidth_overhead_factor) as u64;
+                    let upload_max = (sites[idx].max_up_mbps as f32 * config.uisp_integration.bandwidth_overhead_factor) as u64;
+                    let download_min = (download_max as f32 * config.uisp_integration.commit_bandwidth_multiplier) as u64;
+                    let upload_min = (upload_max as f32 * config.uisp_integration.commit_bandwidth_multiplier) as u64;
+                    let sd = ShapedDevice {
+                        circuit_id: sites[idx].id.clone(),
+                        circuit_name: sites[idx].name.clone(),
+                        device_id: device.id.clone(),
+                        device_name: device.name.clone(),
+                        parent_node: sites[idx].name.clone(),
+                        mac: device.mac.clone(),
+                        ipv4: device.ipv4_list(),
+                        ipv6: device.ipv6_list(),
+                        download_min,
+                        download_max,
+                        upload_min,
+                        upload_max,
+                        comment: "".to_string(),
+                    };
+                    shaped_devices.push(sd);
+                }
+            }
+        } else {
+            // It's an infrastructure node
+            for device in sites[idx].device_indices.iter() {
+                let device = &devices[*device];
+                if device.has_address() {
+                    let download_max = (sites[idx].max_down_mbps as f32 * config.uisp_integration.bandwidth_overhead_factor) as u64;
+                    let upload_max = (sites[idx].max_up_mbps as f32 * config.uisp_integration.bandwidth_overhead_factor) as u64;
+                    let download_min = (download_max as f32 * config.uisp_integration.commit_bandwidth_multiplier) as u64;
+                    let upload_min = (upload_max as f32 * config.uisp_integration.commit_bandwidth_multiplier) as u64;
+                    let sd = ShapedDevice {
+                        circuit_id: format!("{}-inf", sites[idx].id),
+                        circuit_name: format!("{} Infrastructure", sites[idx].name),
+                        device_id: device.id.clone(),
+                        device_name: device.name.clone(),
+                        parent_node: sites[idx].name.clone(),
+                        mac: device.mac.clone(),
+                        ipv4: device.ipv4_list(),
+                        ipv6: device.ipv6_list(),
+                        download_min,
+                        download_max,
+                        upload_min,
+                        upload_max,
+                        comment: "Infrastructure Entry".to_string(),
+                    };
+                    shaped_devices.push(sd);
+                }
+            }
+        }
+    }
+
+    if depth < 8 {
+        for (child_idx, child) in sites.iter().enumerate() {
+            if let Some(parent_idx) = child.selected_parent {
+                if parent_idx == idx {
+                    traverse(sites, child_idx, depth+1, devices, shaped_devices, config);
+                }
+            }
+        }
+    }
+}

src/rust/uisp_integration/src/strategies/mod.rs

@@ -5,18 +5,19 @@ use crate::errors::UispIntegrationError;
 use lqos_config::Config;
 use tracing::{error, info};
 pub use full::BandwidthOverrides;
+use crate::ip_ranges::IpRanges;
 
-pub async fn build_with_strategy(config: Config) -> Result<(), UispIntegrationError> {
+pub async fn build_with_strategy(config: Config, ip_ranges: IpRanges) -> Result<(), UispIntegrationError> {
     // Select a Strategy
     match config.uisp_integration.strategy.to_lowercase().as_str() {
         "flat" => {
             info!("Strategy selected: flat");
-            flat::build_flat_network(config).await?;
+            flat::build_flat_network(config, ip_ranges).await?;
             Ok(())
         }
         "full" => {
             info!("Strategy selected: full");
-            full::build_full_network(config).await?;
+            full::build_full_network(config, ip_ranges).await?;
             Ok(())
         }
         _ => {

src/rust/uisp_integration/src/uisp_types/uisp_device.rs

@@ -1,17 +1,25 @@
+use std::collections::HashSet;
+use std::net::{IpAddr, Ipv4Addr};
 use lqos_config::Config;
 use uisp::Device;
+use crate::ip_ranges::IpRanges;
 
 /// Trimmed UISP device for easy use
 pub struct UispDevice {
     pub id: String,
+    pub name: String,
     pub mac: String,
     pub site_id: String,
     pub download: u32,
     pub upload: u32,
+    pub ipv4: HashSet<String>,
+    pub ipv6: HashSet<String>,
 }
 
 impl UispDevice {
-    pub fn from_uisp(device: &Device, config: &Config) -> Self {
+    pub fn from_uisp(device: &Device, config: &Config, ip_ranges: &IpRanges) -> Self {
+        let mut ipv4 = HashSet::new();
+        let mut ipv6 = HashSet::new();
         let mac = if let Some(id) = &device.identification.mac {
             id.clone()
         } else {
@@ -29,12 +37,102 @@ impl UispDevice {
             }
         }
 
+        // Accumulate IP address listings
+        if let Some(interfaces) = &device.interfaces {
+            for interface in interfaces.iter() {
+                if let Some(addr) = &interface.addresses {
+                    for address in addr.iter() {
+                        if let Some(address) = &address.cidr {
+                            if address.contains(':') {
+                                // It's IPv6
+                                ipv6.insert(address.clone());
+                            } else {
+                                // It's IPv4
+                                // We can't trust UISP to provide the correct suffix, so change that to /32
+                                if address.contains('/') {
+                                    let splits: Vec<_> = address.split('/').collect();
+                                    ipv4.insert(format!("{}/32", splits[0]));
+                                } else {
+                                    ipv4.insert(format!("{address}/32"));
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        // Remove IP addresses that are disallowed
+        ipv4.retain(|ip| {
+            let split: Vec<_> = ip.split('/').collect();
+            let subnet: u8 = split[1].parse().unwrap();
+            let addr: IpAddr = split[0].parse().unwrap();
+            ip_ranges.is_permitted(addr, subnet)
+        });
+        ipv6.retain(|ip| {
+            let split: Vec<_> = ip.split('/').collect();
+            let subnet: u8 = split[1].parse().unwrap();
+            let addr: IpAddr = split[0].parse().unwrap();
+            ip_ranges.is_permitted(addr, subnet)
+        });
+
         Self {
             id: device.get_id(),
+            name: device.get_name().unwrap(),
             mac,
             site_id: device.get_site_id().unwrap_or("".to_string()),
             upload,
             download,
+            ipv4,
+            ipv6,
         }
     }
+
+    pub fn has_address(&self) -> bool {
+        if self.ipv4.is_empty() && self.ipv6.is_empty() {
+            false
+        } else {
+            true
+        }
+    }
+
+    pub fn ipv4_list(&self) -> String {
+        if self.ipv4.is_empty() {
+            return "".to_string();
+        }
+        if self.ipv4.len() == 1 {
+            let mut result = "".to_string();
+            for ip in self.ipv4.iter() {
+                result = ip.clone();
+            }
+            return result;
+        }
+        let mut result = "".to_string();
+        for ip in self.ipv4.iter() {
+            result += &format!("{}, ", &ip);
+        }
+        result.truncate(result.len()-2);
+        let result = format!("[{result}]");
+        result
+    }
+
+    pub fn ipv6_list(&self) -> String {
+        if self.ipv6.is_empty() {
+            return "".to_string();
+        }
+        if self.ipv6.len() == 1 {
+            let mut result = "".to_string();
+            for ip in self.ipv6.iter() {
+                result = ip.clone();
+            }
+            return result;
+        }
+        let mut result = "".to_string();
+        for ip in self.ipv6.iter() {
+            result += &format!("{}, ", &ip);
+        }
+        result.truncate(result.len()-2);
+        let result = format!("[{result}]");
+        result
+    }
 }