IntenseWebs/LibreQoS
3
0
Fork 0
mirror of https://github.com/LibreQoE/LibreQoS.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

Format run

Browse Source
This commit is contained in:
Herbert Wolverson 2024-04-22 14:45:37 -05:00
parent 250c091eee
commit 14d466898c
13 changed files with 163 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/rust/uisp_integration/src/ip_ranges.rs
View File

@ -1,9 +1,9 @@
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use crate::errors::UispIntegrationError;
use ip_network::IpNetwork;
use ip_network_table::IpNetworkTable;
use tracing::info;
use lqos_config::Config;
use crate::errors::UispIntegrationError;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use tracing::info;
pub struct IpRanges {
allowed: IpNetworkTable<bool>,
@ -45,11 +45,13 @@ impl IpRanges {
ignored.insert(ip, true);
}
}
info!("{} allowed IP ranges, {} ignored IP ranges", allowed.len().0, ignored.len().0);
info!(
"{} allowed IP ranges, {} ignored IP ranges",
allowed.len().0,
ignored.len().0
);
Ok(Self {
allowed, ignored
})
Ok(Self { allowed, ignored })
}
pub fn is_permitted(&self, ip: IpAddr, subnet: u8) -> bool {
@ -57,8 +59,8 @@ impl IpRanges {
if let Some(deny) = self.ignored.longest_match(ip) {
return false;
}
return true
return true;
}
false
}
}
}

4
src/rust/uisp_integration/src/main.rs
View File

@ -3,15 +3,15 @@
//! faster in Rust.
mod errors;
pub mod ip_ranges;
mod strategies;
pub mod uisp_types;
pub mod ip_ranges;
use crate::errors::UispIntegrationError;
use crate::ip_ranges::IpRanges;
use lqos_config::Config;
use tokio::time::Instant;
use tracing::{error, info};
use crate::ip_ranges::IpRanges;
/// Start the tracing/logging system
fn init_tracing() {

7
src/rust/uisp_integration/src/strategies/flat.rs
View File

@ -1,9 +1,12 @@
use crate::errors::UispIntegrationError;
use crate::ip_ranges::IpRanges;
use lqos_config::Config;
use tracing::error;
use crate::ip_ranges::IpRanges;
pub async fn build_flat_network(_config: Config, _ip_ranges: IpRanges) -> Result<(), UispIntegrationError> {
pub async fn build_flat_network(
_config: Config,
_ip_ranges: IpRanges,
) -> Result<(), UispIntegrationError> {
error!("Not implemented yet");
Ok(())
}

18
src/rust/uisp_integration/src/strategies/full/bandwidth_overrides.rs
View File

@ -1,18 +1,20 @@
use crate::errors::UispIntegrationError;
use csv::ReaderBuilder;
use lqos_config::Config;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::Path;
use csv::ReaderBuilder;
use tracing::{error, info};
use serde::{Deserialize, Serialize};
use lqos_config::Config;
use crate::errors::UispIntegrationError;
pub type BandwidthOverrides = HashMap<String, (f32, f32)>;
/// Attempts to load integrationUISPbandwidths.csv to use for
/// bandwidth overrides. Returns an empty set if not found.
pub fn get_site_bandwidth_overrides(config: &Config) -> Result<BandwidthOverrides, UispIntegrationError> {
pub fn get_site_bandwidth_overrides(
config: &Config,
) -> Result<BandwidthOverrides, UispIntegrationError> {
info!("Looking for integrationUISPbandwidths.csv");
let file_path = Path::new(&config.lqos_directory).join("integrationUISPbandwidths.csv");
let file_path = Path::new(&config.lqos_directory).join("integrationUISPbandwidths.csv");
if file_path.exists() {
let reader = ReaderBuilder::new()
.comment(Some(b'#'))
@ -29,7 +31,7 @@ pub fn get_site_bandwidth_overrides(config: &Config) -> Result<BandwidthOverride
if let Ok(result) = result {
overrides.insert(
result.parent_node,
(result.download_mbps, result.upload_mbps)
(result.download_mbps, result.upload_mbps),
);
}
}
@ -49,4 +51,4 @@ struct IntegrationBandwidthRow {
pub download_mbps: f32,
#[serde(rename = "Upload Mbps")]
pub upload_mbps: f32,
}
}

36
src/rust/uisp_integration/src/strategies/full/mod.rs
View File

@ -1,36 +1,39 @@
mod ap_promotion;
mod bandwidth_overrides;
mod client_site_promotion;
mod network_json;
mod parse;
mod root_site;
mod routes_override;
mod shaped_devices_writer;
mod squash_single_entry_aps;
mod tree_walk;
mod uisp_fetch;
mod utils;
mod bandwidth_overrides;
mod routes_override;
mod network_json;
mod shaped_devices_writer;
use crate::errors::UispIntegrationError;
use crate::ip_ranges::IpRanges;
use crate::strategies::full::ap_promotion::promote_access_points;
use crate::strategies::full::bandwidth_overrides::get_site_bandwidth_overrides;
use crate::strategies::full::client_site_promotion::promote_clients_with_children;
use crate::strategies::full::network_json::write_network_file;
use crate::strategies::full::parse::parse_uisp_datasets;
use crate::strategies::full::root_site::{find_root_site, set_root_site};
use crate::strategies::full::routes_override::get_route_overrides;
use crate::strategies::full::shaped_devices_writer::write_shaped_devices;
use crate::strategies::full::squash_single_entry_aps::squash_single_aps;
use crate::strategies::full::tree_walk::walk_tree_for_routing;
use crate::strategies::full::uisp_fetch::load_uisp_data;
use crate::strategies::full::utils::{print_sites, warn_of_no_parents};
use crate::uisp_types::UispSite;
use lqos_config::Config;
use crate::strategies::full::bandwidth_overrides::get_site_bandwidth_overrides;
pub use bandwidth_overrides::BandwidthOverrides;
use crate::ip_ranges::IpRanges;
use crate::strategies::full::network_json::write_network_file;
use crate::strategies::full::routes_override::get_route_overrides;
use crate::strategies::full::shaped_devices_writer::write_shaped_devices;
use lqos_config::Config;
/// Attempt to construct a full hierarchy topology for the UISP network.
pub async fn build_full_network(config: Config, ip_ranges: IpRanges) -> Result<(), UispIntegrationError> {
pub async fn build_full_network(
config: Config,
ip_ranges: IpRanges,
) -> Result<(), UispIntegrationError> {
// Load any bandwidth overrides
let bandwidth_overrides = get_site_bandwidth_overrides(&config)?;
@ -39,8 +42,14 @@ pub async fn build_full_network(config: Config, ip_ranges: IpRanges) -> Result<(
// Obtain the UISP data and transform it into easier to work with types
let (sites_raw, devices_raw, data_links_raw) = load_uisp_data(config.clone()).await?;
let (mut sites, data_links, devices) =
parse_uisp_datasets(&sites_raw, &data_links_raw, &devices_raw, &config, &bandwidth_overrides, &ip_ranges);
let (mut sites, data_links, devices) = parse_uisp_datasets(
&sites_raw,
&data_links_raw,
&devices_raw,
&config,
&bandwidth_overrides,
&ip_ranges,
);
// Check root sites
let root_site = find_root_site(&config, &mut sites, &data_links)?;
@ -81,6 +90,5 @@ pub async fn build_full_network(config: Config, ip_ranges: IpRanges) -> Result<(
write_shaped_devices(&config, &sites, root_idx, &devices)?;
}
Ok(())
}

46
src/rust/uisp_integration/src/strategies/full/network_json.rs
View File

@ -1,11 +1,15 @@
use crate::errors::UispIntegrationError;
use crate::uisp_types::{UispSite, UispSiteType};
use lqos_config::Config;
use std::fs::write;
use std::path::Path;
use tracing::{error, info};
use lqos_config::Config;
use crate::errors::UispIntegrationError;
use crate::uisp_types::{UispSite, UispSiteType};
pub fn write_network_file(config: &Config, sites: &[UispSite], root_idx: usize) -> Result<(), UispIntegrationError> {
pub fn write_network_file(
config: &Config,
sites: &[UispSite],
root_idx: usize,
) -> Result<(), UispIntegrationError> {
let network_path = Path::new(&config.lqos_directory).join("network.json");
if network_path.exists() && !config.integration_common.always_overwrite_network_json {
tracing::warn!("Network.json exists, and always overwrite network json is not true - not writing network.json");
@ -15,21 +19,30 @@ pub fn write_network_file(config: &Config, sites: &[UispSite], root_idx: usize)
// Write the network JSON file
let root = traverse_sites(sites, root_idx, 0)?;
let json = serde_json::to_string_pretty(&root).unwrap();
write(network_path, &json)
.map_err(|e| {
error!("Unable to write network.json");
error!("{e:?}");
UispIntegrationError::WriteNetJson
})?;
write(network_path, &json).map_err(|e| {
error!("Unable to write network.json");
error!("{e:?}");
UispIntegrationError::WriteNetJson
})?;
info!("Written network.json");
Ok(())
}
fn traverse_sites(sites: &[UispSite], idx: usize, depth: u32) -> Result<serde_json::Map<String, serde_json::Value>, UispIntegrationError> {
fn traverse_sites(
sites: &[UispSite],
idx: usize,
depth: u32,
) -> Result<serde_json::Map<String, serde_json::Value>, UispIntegrationError> {
let mut entry = serde_json::Map::new();
entry.insert("downloadBandwidthMbps".to_string(), serde_json::Value::Number(sites[idx].max_down_mbps.into()));
entry.insert("uploadBandwidthMbps".to_string(), serde_json::Value::Number(sites[idx].max_down_mbps.into()));
entry.insert(
"downloadBandwidthMbps".to_string(),
serde_json::Value::Number(sites[idx].max_down_mbps.into()),
);
entry.insert(
"uploadBandwidthMbps".to_string(),
serde_json::Value::Number(sites[idx].max_down_mbps.into()),
);
let t = match sites[idx].site_type {
UispSiteType::AccessPoint => "ap",
_ => "site",
@ -40,7 +53,10 @@ fn traverse_sites(sites: &[UispSite], idx: usize, depth: u32) -> Result<serde_js
for (child_id, child) in sites.iter().enumerate() {
if let Some(parent) = child.selected_parent {
if parent == idx && should_traverse(&sites[child_id].site_type) {
children.insert(child.name.clone(), serde_json::Value::Object(traverse_sites(sites, child_id, depth + 1)?));
children.insert(
child.name.clone(),
serde_json::Value::Object(traverse_sites(sites, child_id, depth + 1)?),
);
}
}
}
@ -57,4 +73,4 @@ fn should_traverse(t: &UispSiteType) -> bool {
UispSiteType::Client => false,
_ => true,
}
}
}

10
src/rust/uisp_integration/src/strategies/full/parse.rs
View File

@ -1,9 +1,9 @@
use crate::ip_ranges::IpRanges;
use crate::strategies::full::bandwidth_overrides::BandwidthOverrides;
use crate::uisp_types::{UispDataLink, UispDevice, UispSite};
use lqos_config::Config;
use tracing::info;
use uisp::{DataLink, Device, Site};
use crate::ip_ranges::IpRanges;
use crate::strategies::full::bandwidth_overrides::BandwidthOverrides;
pub fn parse_uisp_datasets(
sites_raw: &[Site],
@ -33,7 +33,11 @@ pub fn parse_uisp_datasets(
(sites, data_links, devices)
}
fn parse_sites(sites_raw: &[Site], config: &Config, bandwidth_overrides: &BandwidthOverrides) -> Vec<UispSite> {
fn parse_sites(
sites_raw: &[Site],
config: &Config,
bandwidth_overrides: &BandwidthOverrides,
) -> Vec<UispSite> {
let mut sites: Vec<UispSite> = sites_raw
.iter()
.map(|s| UispSite::from_uisp(s, &config, bandwidth_overrides))

12
src/rust/uisp_integration/src/strategies/full/routes_override.rs
View File

@ -1,9 +1,9 @@
use std::path::Path;
use csv::ReaderBuilder;
use serde::Deserialize;
use tracing::{error, info};
use lqos_config::Config;
use crate::errors::UispIntegrationError;
use csv::ReaderBuilder;
use lqos_config::Config;
use serde::Deserialize;
use std::path::Path;
use tracing::{error, info};
#[derive(Deserialize, Debug)]
pub struct RouteOverride {
@ -37,4 +37,4 @@ pub fn get_route_overrides(config: &Config) -> Result<Vec<RouteOverride>, UispIn
info!("No integrationUISProutes.csv found - no route overrides loaded.");
Ok(Vec::new())
}
}
}

60
src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs
View File

@ -1,9 +1,9 @@
use std::path::Path;
use serde::Serialize;
use tracing::{error, info};
use lqos_config::Config;
use crate::errors::UispIntegrationError;
use crate::uisp_types::{UispDevice, UispSite, UispSiteType};
use lqos_config::Config;
use serde::Serialize;
use std::path::Path;
use tracing::{error, info};
#[derive(Serialize, Debug)]
struct ShapedDevice {
@ -22,7 +22,12 @@ struct ShapedDevice {
pub comment: String,
}
pub fn write_shaped_devices(config: &Config, sites: &[UispSite], root_idx: usize, devices: &[UispDevice]) -> Result<(), UispIntegrationError> {
pub fn write_shaped_devices(
config: &Config,
sites: &[UispSite],
root_idx: usize,
devices: &[UispDevice],
) -> Result<(), UispIntegrationError> {
let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv");
let mut shaped_devices = Vec::new();
@ -48,7 +53,14 @@ pub fn write_shaped_devices(config: &Config, sites: &[UispSite], root_idx: usize
Ok(())
}
fn traverse(sites: &[UispSite], idx: usize, depth: u32, devices: &[UispDevice], shaped_devices: &mut Vec<ShapedDevice>, config: &Config) {
fn traverse(
sites: &[UispSite],
idx: usize,
depth: u32,
devices: &[UispDevice],
shaped_devices: &mut Vec<ShapedDevice>,
config: &Config,
) {
if !sites[idx].device_indices.is_empty() {
// We have devices!
if sites[idx].site_type == UispSiteType::Client {
@ -56,10 +68,18 @@ fn traverse(sites: &[UispSite], idx: usize, depth: u32, devices: &[UispDevice],
for device in sites[idx].device_indices.iter() {
let device = &devices[*device];
if device.has_address() {
let download_max = (sites[idx].max_down_mbps as f32 * config.uisp_integration.bandwidth_overhead_factor) as u64;
let upload_max = (sites[idx].max_up_mbps as f32 * config.uisp_integration.bandwidth_overhead_factor) as u64;
let download_min = (download_max as f32 * config.uisp_integration.commit_bandwidth_multiplier) as u64;
let upload_min = (upload_max as f32 * config.uisp_integration.commit_bandwidth_multiplier) as u64;
let download_max = (sites[idx].max_down_mbps as f32
* config.uisp_integration.bandwidth_overhead_factor)
as u64;
let upload_max = (sites[idx].max_up_mbps as f32
* config.uisp_integration.bandwidth_overhead_factor)
as u64;
let download_min = (download_max as f32
* config.uisp_integration.commit_bandwidth_multiplier)
as u64;
let upload_min = (upload_max as f32
* config.uisp_integration.commit_bandwidth_multiplier)
as u64;
let sd = ShapedDevice {
circuit_id: sites[idx].id.clone(),
circuit_name: sites[idx].name.clone(),
@ -83,10 +103,18 @@ fn traverse(sites: &[UispSite], idx: usize, depth: u32, devices: &[UispDevice],
for device in sites[idx].device_indices.iter() {
let device = &devices[*device];
if device.has_address() {
let download_max = (sites[idx].max_down_mbps as f32 * config.uisp_integration.bandwidth_overhead_factor) as u64;
let upload_max = (sites[idx].max_up_mbps as f32 * config.uisp_integration.bandwidth_overhead_factor) as u64;
let download_min = (download_max as f32 * config.uisp_integration.commit_bandwidth_multiplier) as u64;
let upload_min = (upload_max as f32 * config.uisp_integration.commit_bandwidth_multiplier) as u64;
let download_max = (sites[idx].max_down_mbps as f32
* config.uisp_integration.bandwidth_overhead_factor)
as u64;
let upload_max = (sites[idx].max_up_mbps as f32
* config.uisp_integration.bandwidth_overhead_factor)
as u64;
let download_min = (download_max as f32
* config.uisp_integration.commit_bandwidth_multiplier)
as u64;
let upload_min = (upload_max as f32
* config.uisp_integration.commit_bandwidth_multiplier)
as u64;
let sd = ShapedDevice {
circuit_id: format!("{}-inf", sites[idx].id),
circuit_name: format!("{} Infrastructure", sites[idx].name),
@ -112,9 +140,9 @@ fn traverse(sites: &[UispSite], idx: usize, depth: u32, devices: &[UispDevice],
for (child_idx, child) in sites.iter().enumerate() {
if let Some(parent_idx) = child.selected_parent {
if parent_idx == idx {
traverse(sites, child_idx, depth+1, devices, shaped_devices, config);
traverse(sites, child_idx, depth + 1, devices, shaped_devices, config);
}
}
}
}
}
}

7
src/rust/uisp_integration/src/strategies/full/tree_walk.rs
View File

@ -43,7 +43,10 @@ fn walk_node(
if sites[i].parent_indices.contains(&idx) {
let from = sites[i].name.clone();
let to = sites[idx].name.clone();
if let Some(route_override) = overrides.iter().find(|o| o.from_site == from && o.to_site == to) {
if let Some(route_override) = overrides
.iter()
.find(|o| o.from_site == from && o.to_site == to)
{
sites[i].route_weights.push((idx, route_override.cost));
} else {
sites[i].route_weights.push((idx, weight));
@ -51,4 +54,4 @@ fn walk_node(
walk_node(i, weight + 10, sites, visited, overrides);
}
}
}
}

9
src/rust/uisp_integration/src/strategies/mod.rs
View File

@ -2,12 +2,15 @@ mod flat;
mod full;
use crate::errors::UispIntegrationError;
use crate::ip_ranges::IpRanges;
pub use full::BandwidthOverrides;
use lqos_config::Config;
use tracing::{error, info};
pub use full::BandwidthOverrides;
use crate::ip_ranges::IpRanges;
pub async fn build_with_strategy(config: Config, ip_ranges: IpRanges) -> Result<(), UispIntegrationError> {
pub async fn build_with_strategy(
config: Config,
ip_ranges: IpRanges,
) -> Result<(), UispIntegrationError> {
// Select a Strategy
match config.uisp_integration.strategy.to_lowercase().as_str() {
"flat" => {

8
src/rust/uisp_integration/src/uisp_types/uisp_device.rs
View File

@ -1,8 +1,8 @@
use crate::ip_ranges::IpRanges;
use lqos_config::Config;
use std::collections::HashSet;
use std::net::{IpAddr, Ipv4Addr};
use lqos_config::Config;
use uisp::Device;
use crate::ip_ranges::IpRanges;
/// Trimmed UISP device for easy use
pub struct UispDevice {
@ -111,7 +111,7 @@ impl UispDevice {
for ip in self.ipv4.iter() {
result += &format!("{}, ", &ip);
}
result.truncate(result.len()-2);
result.truncate(result.len() - 2);
let result = format!("[{result}]");
result
}
@ -131,7 +131,7 @@ impl UispDevice {
for ip in self.ipv6.iter() {
result += &format!("{}, ", &ip);
}
result.truncate(result.len()-2);
result.truncate(result.len() - 2);
let result = format!("[{result}]");
result
}

15
src/rust/uisp_integration/src/uisp_types/uisp_site.rs
View File

@ -1,10 +1,10 @@
use crate::strategies::BandwidthOverrides;
use crate::uisp_types::uisp_site_type::UispSiteType;
use crate::uisp_types::DetectedAccessPoint;
use lqos_config::Config;
use std::collections::HashSet;
use tracing::warn;
use uisp::{DataLink, Device, Site};
use crate::strategies::BandwidthOverrides;
/// Shortened/flattened version of the UISP Site type.
#[derive(Debug)]
@ -41,7 +41,11 @@ impl Default for UispSite {
}
impl UispSite {
pub fn from_uisp(value: &Site, config: &Config, bandwidth_overrides: &BandwidthOverrides) -> Self {
pub fn from_uisp(
value: &Site,
config: &Config,
bandwidth_overrides: &BandwidthOverrides,
) -> Self {
let mut uisp_parent_id = None;
if let Some(id) = &value.identification {
@ -80,7 +84,7 @@ impl UispSite {
}
}
if let Some((up,down)) = bandwidth_overrides.get(&value.name_or_blank()) {
if let Some((up, down)) = bandwidth_overrides.get(&value.name_or_blank()) {
// Apply the overrides
max_down_mbps = *down as u32;
max_up_mbps = *up as u32;
@ -145,8 +149,9 @@ impl UispSite {
if from_site.identification.id != self.id {
// We have a data link from this device that goes to
// another site.
if let Some(remote_site) =
sites.iter().find(|s| s.id == from_site.identification.id)
if let Some(remote_site) = sites
.iter()
.find(|s| s.id == from_site.identification.id)
{
potential_ap.child_sites.push(remote_site.id.clone());
}