From 5eec0605c96271cd6cc2e326b42acd41627f361f Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Tue, 21 Nov 2023 15:21:16 -0600 Subject: [PATCH 01/41] First commit for a unified configuration system. * Adds PyO3 as a dependency to the config crate. * Uses PyO3 to load an existing configuration as a Python object. * Adds some generic conversion code for reading Python types to Rust equivalents. * Adds a preliminary "read the whole existing config into Rust" code. Not in a shape to use yet, but a good start. --- src/rust/Cargo.lock | 89 ++++----- src/rust/lqos_config/Cargo.toml | 1 + src/rust/lqos_config/src/lib.rs | 1 + src/rust/lqos_config/src/python_migration.rs | 190 +++++++++++++++++++ 4 files changed, 232 insertions(+), 49 deletions(-) create mode 100644 src/rust/lqos_config/src/python_migration.rs diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index eef6ef64..1ffe02c2 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -134,7 +134,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -145,7 +145,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -264,7 +264,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.29", + "syn", "which", ] @@ -459,7 +459,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -708,7 +708,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -777,7 +777,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1023,7 +1023,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1261,9 +1261,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "inlinable_string" @@ -1530,6 +1530,7 @@ dependencies = [ "ip_network", "ip_network_table", "log", + "pyo3", "serde", "serde_json", "sha2", @@ -2032,7 +2033,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -2108,7 +2109,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -2140,7 +2141,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -2208,7 +2209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", - "syn 2.0.29", + "syn", ] [[package]] @@ -2228,16 +2229,16 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", "version_check", "yansi 1.0.0-rc.1", ] [[package]] name = "pyo3" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" +checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" dependencies = [ "cfg-if", "indoc", @@ -2252,9 +2253,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" +checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" dependencies = [ "once_cell", "target-lexicon", @@ -2262,9 +2263,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" +checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" dependencies = [ "libc", "pyo3-build-config", @@ -2272,25 +2273,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" +checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "pyo3-macros-backend" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" +checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" dependencies = [ + "heck", "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] @@ -2380,7 +2382,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -2552,7 +2554,7 @@ dependencies = [ "proc-macro2", "quote", "rocket_http", - "syn 2.0.29", + "syn", "unicode-xid", ] @@ -2725,7 +2727,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -2927,17 +2929,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.29" @@ -3036,7 +3027,7 @@ checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -3129,7 +3120,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -3250,7 +3241,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -3387,9 +3378,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "url" @@ -3483,7 +3474,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.29", + "syn", "wasm-bindgen-shared", ] @@ -3517,7 +3508,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3790,7 +3781,7 @@ checksum = "8f7f3a471f98d0a61c34322fbbfd10c384b07687f680d4119813713f72308d91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -3810,5 +3801,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] diff --git a/src/rust/lqos_config/Cargo.toml b/src/rust/lqos_config/Cargo.toml index 0b17e88b..215a4eec 100644 --- a/src/rust/lqos_config/Cargo.toml +++ b/src/rust/lqos_config/Cargo.toml @@ -16,3 +16,4 @@ sha2 = "0" uuid = { version = "1", features = ["v4", "fast-rng" ] } log = "0" dashmap = "5" +pyo3 = "0.20" diff --git a/src/rust/lqos_config/src/lib.rs b/src/rust/lqos_config/src/lib.rs index 6f05cecb..4bb2838c 100644 --- a/src/rust/lqos_config/src/lib.rs +++ b/src/rust/lqos_config/src/lib.rs @@ -12,6 +12,7 @@ mod libre_qos_config; mod network_json; mod program_control; mod shaped_devices; +mod python_migration; pub use authentication::{UserRole, WebUsers}; pub use etc::{BridgeConfig, BridgeInterface, BridgeVlan, EtcLqos, Tunables, enable_long_term_stats}; diff --git a/src/rust/lqos_config/src/python_migration.rs b/src/rust/lqos_config/src/python_migration.rs new file mode 100644 index 00000000..17467d73 --- /dev/null +++ b/src/rust/lqos_config/src/python_migration.rs @@ -0,0 +1,190 @@ +//! This module utilizes PyO3 to read an existing ispConfig.py file, and +//! provide conversion services for the new, unified configuration target +//! for version 1.5. + +use crate::EtcLqos; +use pyo3::{prepare_freethreaded_python, Python}; +use std::{ + fs::read_to_string, + path::{Path, PathBuf}, collections::HashMap, +}; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum PythonMigrationError { + #[error("The ispConfig.py file does not exist.")] + ConfigFileNotFound, + #[error("Unable to parse variable")] + ParseError, + #[error("Variable not found")] + VariableNotFound(String), +} + +fn isp_config_py_path(cfg: &EtcLqos) -> PathBuf { + let base_path = Path::new(&cfg.lqos_directory); + let final_path = base_path.join("ispConfig.py"); + final_path +} + +/// Does thie ispConfig.py file exist? +fn config_exists(cfg: &EtcLqos) -> bool { + if let Ok(cfg) = crate::etc::EtcLqos::load() { + isp_config_py_path(&cfg).exists() + } else { + false + } +} + +fn from_python<'a, T>(py: &'a Python, variable_name: &str) -> Result +where + T: pyo3::FromPyObject<'a>, +{ + let result = py + .eval(variable_name, None, None) + .map_err(|_| PythonMigrationError::VariableNotFound(variable_name.to_string()))? + .extract::() + .map_err(|_| PythonMigrationError::ParseError)?; + + Ok(result) +} + +#[derive(Default, Debug)] +pub struct PythonMigration { + pub sqm: String, + pub monitor_only_mode: bool, + pub upstream_bandwidth_capacity_download_mbps: u32, + pub upstream_bandwidth_capacity_upload_mbps: u32, + pub generated_pn_download_mbps: u32, + pub generated_pn_upload_mbps: u32, + pub interface_a: String, + pub interface_b: String, + pub queue_refresh_interval_mins: u32, + pub on_a_stick: bool, + pub stick_vlan_a: u32, + pub stick_vlan_b: u32, + pub enable_actual_shell_commands: bool, + pub run_shell_commands_as_sudo: bool, + pub queues_available_override: u32, + pub use_bin_packing_to_balance_cpu: bool, + pub influx_db_enabled: bool, + pub influx_db_url: String, + pub infux_db_bucket: String, + pub influx_db_org: String, + pub influx_db_token: String, + pub circuit_name_use_address: bool, + pub overwrite_network_json_always: bool, + pub ignore_subnets: Vec, + pub allowed_subnets: Vec, + pub automatic_import_splynx: bool, + pub splynx_api_key: String, + pub spylnx_api_secret: String, + pub spylnx_api_url: String, + pub automatic_import_uisp: bool, + pub uisp_auth_token: String, + pub uisp_base_url: String, + pub uisp_site: String, + pub uisp_strategy: String, + pub uisp_suspended_strategy: String, + pub airmax_capacity: f32, + pub ltu_capacity: f32, + pub exclude_sites: Vec, + pub find_ipv6_using_mikrotik: bool, + pub bandwidth_overhead_factor: f32, + pub committed_bandwidth_multiplier: f32, + pub exception_cpes: HashMap, + pub api_username: String, + pub api_password: String, + pub api_host_ip: String, + pub api_host_port: u32, + + // TODO: httpRestIntegrationConfig +} + +impl PythonMigration { + fn parse(cfg: &mut Self, py: &Python) -> Result<(), PythonMigrationError> { + cfg.sqm = from_python(&py, "sqm")?; + cfg.monitor_only_mode = from_python(&py, "monitorOnlyMode")?; + cfg.upstream_bandwidth_capacity_download_mbps = + from_python(&py, "upstreamBandwidthCapacityDownloadMbps")?; + cfg.upstream_bandwidth_capacity_upload_mbps = + from_python(&py, "upstreamBandwidthCapacityUploadMbps")?; + cfg.generated_pn_download_mbps = from_python(&py, "generatedPNDownloadMbps")?; + cfg.generated_pn_upload_mbps = from_python(&py, "generatedPNUploadMbps")?; + cfg.interface_a = from_python(&py, "interfaceA")?; + cfg.interface_b = from_python(&py, "interfaceB")?; + cfg.queue_refresh_interval_mins = from_python(&py, "queueRefreshIntervalMins")?; + cfg.on_a_stick = from_python(&py, "OnAStick")?; + cfg.stick_vlan_a = from_python(&py, "StickVlanA")?; + cfg.stick_vlan_b = from_python(&py, "StickVlanB")?; + cfg.enable_actual_shell_commands = from_python(&py, "enableActualShellCommands")?; + cfg.run_shell_commands_as_sudo = from_python(&py, "runShellCommandsAsSudo")?; + cfg.queues_available_override = from_python(&py, "queuesAvailableOverride")?; + cfg.use_bin_packing_to_balance_cpu = from_python(&py, "useBinPackingToBalanceCPU")?; + cfg.influx_db_enabled = from_python(&py, "influxDBEnabled")?; + cfg.influx_db_url = from_python(&py, "influxDBurl")?; + cfg.infux_db_bucket = from_python(&py, "influxDBBucket")?; + cfg.influx_db_org = from_python(&py, "influxDBOrg")?; + cfg.influx_db_token = from_python(&py, "influxDBtoken")?; + cfg.circuit_name_use_address = from_python(&py, "circuitNameUseAddress")?; + cfg.overwrite_network_json_always = from_python(&py, "overwriteNetworkJSONalways")?; + cfg.ignore_subnets = from_python(&py, "ignoreSubnets")?; + cfg.allowed_subnets = from_python(&py, "allowedSubnets")?; + cfg.automatic_import_splynx = from_python(&py, "automaticImportSplynx")?; + cfg.splynx_api_key = from_python(&py, "splynx_api_key")?; + cfg.spylnx_api_secret = from_python(&py, "splynx_api_secret")?; + cfg.spylnx_api_url = from_python(&py, "splynx_api_url")?; + cfg.automatic_import_uisp = from_python(&py, "automaticImportUISP")?; + cfg.uisp_auth_token = from_python(&py, "uispAuthToken")?; + cfg.uisp_base_url = from_python(&py, "UISPbaseURL")?; + cfg.uisp_site = from_python(&py, "uispSite")?; + cfg.uisp_strategy = from_python(&py, "uispStrategy")?; + cfg.uisp_suspended_strategy = from_python(&py, "uispSuspendedStrategy")?; + cfg.airmax_capacity = from_python(&py, "airMax_capacity")?; + cfg.ltu_capacity = from_python(&py, "ltu_capacity")?; + cfg.exclude_sites = from_python(&py, "excludeSites")?; + cfg.find_ipv6_using_mikrotik = from_python(&py, "findIPv6usingMikrotik")?; + cfg.bandwidth_overhead_factor = from_python(&py, "bandwidthOverheadFactor")?; + cfg.committed_bandwidth_multiplier = from_python(&py, "committedBandwidthMultiplier")?; + cfg.exception_cpes = from_python(&py, "exceptionCPEs")?; + cfg.api_username = from_python(&py, "apiUsername")?; + cfg.api_password = from_python(&py, "apiPassword")?; + cfg.api_host_ip = from_python(&py, "apiHostIP")?; + cfg.api_host_port = from_python(&py, "apiHostPost")?; + + Ok(()) + } + + pub fn load() -> Result { + let mut old_config = Self::default(); + if let Ok(cfg) = crate::etc::EtcLqos::load() { + if !config_exists(&cfg) { + return Err(PythonMigrationError::ConfigFileNotFound); + } + let code = read_to_string(isp_config_py_path(&cfg)).unwrap(); + + prepare_freethreaded_python(); + Python::with_gil(|py| { + py.run(&code, None, None).unwrap(); + let result = Self::parse(&mut old_config, &py); + if result.is_err() { + println!("Error parsing Python config: {:?}", result); + } + }); + } else { + return Err(PythonMigrationError::ConfigFileNotFound); + } + + Ok(old_config) + } +} + +#[cfg(test)] +mod test { + use super::PythonMigration; + + #[test] + fn load_it() { + let cfg = PythonMigration::load().unwrap(); + println!("{:#?}", cfg); + } +} From fa692d1d3bdc03119dc55c06be55d457e34a93ce Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Tue, 21 Nov 2023 16:17:50 -0600 Subject: [PATCH 02/41] A much better unit test for parsing the default example ispConfig.example.py --- src/rust/lqos_config/src/python_migration.rs | 22 +++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/rust/lqos_config/src/python_migration.rs b/src/rust/lqos_config/src/python_migration.rs index 17467d73..dc144394 100644 --- a/src/rust/lqos_config/src/python_migration.rs +++ b/src/rust/lqos_config/src/python_migration.rs @@ -180,11 +180,23 @@ impl PythonMigration { #[cfg(test)] mod test { - use super::PythonMigration; + use super::*; + + const DEFAULT_ISP_CONFIG_PY: &str = include_str!("../../../ispConfig.example.py"); #[test] - fn load_it() { - let cfg = PythonMigration::load().unwrap(); - println!("{:#?}", cfg); + fn test_parsing_the_default() { + let mut cfg = PythonMigration::default(); + prepare_freethreaded_python(); + let mut worked = true; + Python::with_gil(|py| { + py.run(DEFAULT_ISP_CONFIG_PY, None, None).unwrap(); + let result = PythonMigration::parse(&mut cfg, &py); + if result.is_err() { + println!("Error parsing Python config: {:?}", result); + worked = false; + } + }); + assert!(worked) } -} +} \ No newline at end of file From ff04f3718ed05d94b310e7bf5c4a26a6487b95ac Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 1 Dec 2023 15:29:00 -0600 Subject: [PATCH 03/41] Migrate from to directory, to allow for easier separation of code. --- src/rust/lqos_config/src/{etc.rs => etc/mod.rs} | 1 + src/rust/lqos_config/src/{ => etc}/python_migration.rs | 0 src/rust/lqos_config/src/lib.rs | 1 - 3 files changed, 1 insertion(+), 1 deletion(-) rename src/rust/lqos_config/src/{etc.rs => etc/mod.rs} (99%) rename src/rust/lqos_config/src/{ => etc}/python_migration.rs (100%) diff --git a/src/rust/lqos_config/src/etc.rs b/src/rust/lqos_config/src/etc/mod.rs similarity index 99% rename from src/rust/lqos_config/src/etc.rs rename to src/rust/lqos_config/src/etc/mod.rs index 9647033a..30e4e614 100644 --- a/src/rust/lqos_config/src/etc.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use toml_edit::{Document, value}; use std::{fs, path::Path}; use thiserror::Error; +mod python_migration; /// Represents the top-level of the `/etc/lqos.conf` file. Serialization /// structure. diff --git a/src/rust/lqos_config/src/python_migration.rs b/src/rust/lqos_config/src/etc/python_migration.rs similarity index 100% rename from src/rust/lqos_config/src/python_migration.rs rename to src/rust/lqos_config/src/etc/python_migration.rs diff --git a/src/rust/lqos_config/src/lib.rs b/src/rust/lqos_config/src/lib.rs index 4bb2838c..6f05cecb 100644 --- a/src/rust/lqos_config/src/lib.rs +++ b/src/rust/lqos_config/src/lib.rs @@ -12,7 +12,6 @@ mod libre_qos_config; mod network_json; mod program_control; mod shaped_devices; -mod python_migration; pub use authentication::{UserRole, WebUsers}; pub use etc::{BridgeConfig, BridgeInterface, BridgeVlan, EtcLqos, Tunables, enable_long_term_stats}; From a0c18164ff5d23412ca7ce77ce88fb9e7f281dad Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 1 Dec 2023 15:31:54 -0600 Subject: [PATCH 04/41] Move previous configuration file into its own space. --- .../lqos_config/src/etc/etclqos_migration.rs | 317 +++++++++++++++++ src/rust/lqos_config/src/etc/mod.rs | 318 +----------------- 2 files changed, 319 insertions(+), 316 deletions(-) create mode 100644 src/rust/lqos_config/src/etc/etclqos_migration.rs diff --git a/src/rust/lqos_config/src/etc/etclqos_migration.rs b/src/rust/lqos_config/src/etc/etclqos_migration.rs new file mode 100644 index 00000000..88d34d75 --- /dev/null +++ b/src/rust/lqos_config/src/etc/etclqos_migration.rs @@ -0,0 +1,317 @@ +//! Manages the `/etc/lqos.conf` file. +use log::error; +use serde::{Deserialize, Serialize}; +use toml_edit::{Document, value}; +use std::{fs, path::Path}; +use thiserror::Error; + +/// Represents the top-level of the `/etc/lqos.conf` file. Serialization +/// structure. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct EtcLqos { + /// The directory in which LibreQoS is installed. + pub lqos_directory: String, + + /// How frequently should `lqosd` read the `tc show qdisc` data? + /// In ms. + pub queue_check_period_ms: u64, + + /// If present, provides a unique ID for the node. Used for + /// anonymous stats (to identify nodes without providing an actual + /// identity), and long-term stas. + pub node_id: Option, + + /// If present, provide a name for the node. + pub node_name: Option, + + /// If present, defines how the Bifrost XDP bridge operates. + pub bridge: Option, + + /// If present, defines the values for various `sysctl` and `ethtool` + /// tweaks. + pub tuning: Option, + + /// If present, defined anonymous usage stat sending + pub usage_stats: Option, + + /// Defines for how many seconds a libpcap compatible capture should + /// run. Short times are good, there's a real performance penalty to + /// capturing high-throughput streams. Defaults to 10 seconds. + pub packet_capture_time: Option, + + /// Long-term statistics retention settings. + pub long_term_stats: Option, +} + +/// Represents a set of `sysctl` and `ethtool` tweaks that may be +/// applied (in place of the previous version's offload service) +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct Tunables { + /// Should the `irq_balance` system service be stopped? + pub stop_irq_balance: bool, + + /// Set the netdev budget (usecs) + pub netdev_budget_usecs: u32, + + /// Set the netdev budget (packets) + pub netdev_budget_packets: u32, + + /// Set the RX side polling frequency + pub rx_usecs: u32, + + /// Set the TX side polling frequency + pub tx_usecs: u32, + + /// Disable RXVLAN offloading? You generally want to do this. + pub disable_rxvlan: bool, + + /// Disable TXVLAN offloading? You generally want to do this. + pub disable_txvlan: bool, + + /// A list of `ethtool` offloads to be disabled. + /// The default list is: [ "gso", "tso", "lro", "sg", "gro" ] + pub disable_offload: Vec, +} + +/// Defines the BiFrost XDP bridge accelerator parameters +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct BridgeConfig { + /// Should the XDP bridge be enabled? + pub use_xdp_bridge: bool, + + /// A list of interface mappings. + pub interface_mapping: Vec, + + /// A list of VLAN mappings. + pub vlan_mapping: Vec, +} + +/// An interface within the Bifrost XDP bridge. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct BridgeInterface { + /// The interface name. It *must* match an interface name + /// findable by Linux. + pub name: String, + + /// Should Bifrost read VLAN tags and determine redirect + /// policy from there? + pub scan_vlans: bool, + + /// The outbound interface - data that arrives in the interface + /// defined by `name` will be redirected to this interface. + /// + /// If you are using an "on a stick" configuration, this will + /// be the same as `name`. + pub redirect_to: String, +} + +/// If `scan_vlans` is enabled for an interface, then VLANs +/// are examined on the way through the XDP BiFrost bridge. +/// +/// If a VLAN is on the `parent` interface, and matches `tag` - it +/// will be moved to VLAN `redirect_to`. +/// +/// You often need to make reciprocal pairs of these. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct BridgeVlan { + /// The parent interface name on which the VLAN occurs. + pub parent: String, + + /// The VLAN tag number to redirect if matched. + pub tag: u32, + + /// The destination VLAN tag number if matched. + pub redirect_to: u32, +} + +/// Definitions for anonymous usage submission +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct UsageStats { + /// Are we allowed to send stats at all? + pub send_anonymous: bool, + + /// Where do we send them? + pub anonymous_server: String, +} + +/// Long Term Data Retention +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct LongTermStats { + /// Should we store long-term stats at all? + pub gather_stats: bool, + + /// How frequently should stats be accumulated into a long-term + /// min/max/avg format per tick? + pub collation_period_seconds: u32, + + /// The license key for submitting stats to a LibreQoS hosted + /// statistics server + pub license_key: Option, + + /// UISP reporting period (in seconds). UISP queries can be slow, + /// so hitting it every second or 10 seconds is going to cause problems + /// for some people. A good default may be 5 minutes. Not specifying this + /// disabled UISP integration. + pub uisp_reporting_interval_seconds: Option, +} + +impl EtcLqos { + /// Loads `/etc/lqos.conf`. + pub fn load() -> Result { + if !Path::new("/etc/lqos.conf").exists() { + error!("/etc/lqos.conf does not exist!"); + return Err(EtcLqosError::ConfigDoesNotExist); + } + if let Ok(raw) = std::fs::read_to_string("/etc/lqos.conf") { + let document = raw.parse::(); + match document { + Err(e) => { + error!("Unable to parse TOML from /etc/lqos.conf"); + error!("Full error: {:?}", e); + Err(EtcLqosError::CannotParseToml) + } + Ok(mut config_doc) => { + let cfg = toml_edit::de::from_document::(config_doc.clone()); + match cfg { + Ok(mut cfg) => { + check_config(&mut config_doc, &mut cfg); + Ok(cfg) + } + Err(e) => { + error!("Unable to parse TOML from /etc/lqos.conf"); + error!("Full error: {:?}", e); + Err(EtcLqosError::CannotParseToml) + } + } + } + } + } else { + error!("Unable to read contents of /etc/lqos.conf"); + Err(EtcLqosError::CannotReadFile) + } + } + + /// Saves changes made to /etc/lqos.conf + /// Copies current configuration into /etc/lqos.conf.backup first + pub fn save(&self, document: &mut Document) -> Result<(), EtcLqosError> { + let cfg_path = Path::new("/etc/lqos.conf"); + let backup_path = Path::new("/etc/lqos.conf.backup"); + if let Err(e) = std::fs::copy(cfg_path, backup_path) { + log::error!("Unable to backup /etc/lqos.conf"); + log::error!("{e:?}"); + return Err(EtcLqosError::BackupFail); + } + let new_cfg = document.to_string(); + if let Err(e) = fs::write(cfg_path, new_cfg) { + log::error!("Unable to write to /etc/lqos.conf"); + log::error!("{e:?}"); + return Err(EtcLqosError::WriteFail); + } + Ok(()) + } +} + +/// Run this if you've received the OK from the licensing server, and been +/// sent a license key. This appends a [long_term_stats] section to your +/// config file - ONLY if one doesn't already exist. +pub fn enable_long_term_stats(license_key: String) { + if let Ok(raw) = std::fs::read_to_string("/etc/lqos.conf") { + let document = raw.parse::(); + match document { + Err(e) => { + error!("Unable to parse TOML from /etc/lqos.conf"); + error!("Full error: {:?}", e); + return; + } + Ok(mut config_doc) => { + let cfg = toml_edit::de::from_document::(config_doc.clone()); + match cfg { + Ok(cfg) => { + // Now we enable LTS if its not present + if let Ok(isp_config) = crate::LibreQoSConfig::load() { + if cfg.long_term_stats.is_none() { + + let mut new_section = toml_edit::table(); + new_section["gather_stats"] = value(true); + new_section["collation_period_seconds"] = value(60); + new_section["license_key"] = value(license_key); + if isp_config.automatic_import_uisp { + new_section["uisp_reporting_interval_seconds"] = value(300); + } + config_doc["long_term_stats"] = new_section; + + let new_cfg = config_doc.to_string(); + if let Err(e) = fs::write(Path::new("/etc/lqos.conf"), new_cfg) { + log::error!("Unable to write to /etc/lqos.conf"); + log::error!("{e:?}"); + return; + } + } + } + } + Err(e) => { + error!("Unable to parse TOML from /etc/lqos.conf"); + error!("Full error: {:?}", e); + return; + } + } + } + } + } +} + +fn check_config(cfg_doc: &mut Document, cfg: &mut EtcLqos) { + use sha2::digest::Update; + use sha2::Digest; + + if cfg.node_id.is_none() { + if let Ok(machine_id) = std::fs::read_to_string("/etc/machine-id") { + let hash = sha2::Sha256::new().chain(machine_id).finalize(); + cfg.node_id = Some(format!("{:x}", hash)); + cfg_doc["node_id"] = value(format!("{:x}", hash)); + println!("Updating"); + if let Err(e) = cfg.save(cfg_doc) { + log::error!("Unable to save /etc/lqos.conf"); + log::error!("{e:?}"); + } + } + } +} + +#[derive(Error, Debug)] +pub enum EtcLqosError { + #[error( + "/etc/lqos.conf not found. You must setup this file to use LibreQoS." + )] + ConfigDoesNotExist, + #[error("Unable to read contents of /etc/lqos.conf.")] + CannotReadFile, + #[error("Unable to parse TOML in /etc/lqos.conf")] + CannotParseToml, + #[error("Unable to backup /etc/lqos.conf to /etc/lqos.conf.backup")] + BackupFail, + #[error("Unable to serialize new configuration")] + SerializeFail, + #[error("Unable to write to /etc/lqos.conf")] + WriteFail, +} + +#[cfg(test)] +mod test { + const EXAMPLE_LQOS_CONF: &str = include_str!("../../../../lqos.example"); + + #[test] + fn round_trip_toml() { + let doc = EXAMPLE_LQOS_CONF.parse::().unwrap(); + let reserialized = doc.to_string(); + assert_eq!(EXAMPLE_LQOS_CONF, reserialized); + } + + #[test] + fn add_node_id() { + let mut doc = EXAMPLE_LQOS_CONF.parse::().unwrap(); + doc["node_id"] = toml_edit::value("test"); + let reserialized = doc.to_string(); + assert!(reserialized.contains("node_id = \"test\"")); + } +} \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/mod.rs b/src/rust/lqos_config/src/etc/mod.rs index 30e4e614..f9b73931 100644 --- a/src/rust/lqos_config/src/etc/mod.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -1,318 +1,4 @@ //! Manages the `/etc/lqos.conf` file. -use log::error; -use serde::{Deserialize, Serialize}; -use toml_edit::{Document, value}; -use std::{fs, path::Path}; -use thiserror::Error; -mod python_migration; -/// Represents the top-level of the `/etc/lqos.conf` file. Serialization -/// structure. -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct EtcLqos { - /// The directory in which LibreQoS is installed. - pub lqos_directory: String, - - /// How frequently should `lqosd` read the `tc show qdisc` data? - /// In ms. - pub queue_check_period_ms: u64, - - /// If present, provides a unique ID for the node. Used for - /// anonymous stats (to identify nodes without providing an actual - /// identity), and long-term stas. - pub node_id: Option, - - /// If present, provide a name for the node. - pub node_name: Option, - - /// If present, defines how the Bifrost XDP bridge operates. - pub bridge: Option, - - /// If present, defines the values for various `sysctl` and `ethtool` - /// tweaks. - pub tuning: Option, - - /// If present, defined anonymous usage stat sending - pub usage_stats: Option, - - /// Defines for how many seconds a libpcap compatible capture should - /// run. Short times are good, there's a real performance penalty to - /// capturing high-throughput streams. Defaults to 10 seconds. - pub packet_capture_time: Option, - - /// Long-term statistics retention settings. - pub long_term_stats: Option, -} - -/// Represents a set of `sysctl` and `ethtool` tweaks that may be -/// applied (in place of the previous version's offload service) -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] -pub struct Tunables { - /// Should the `irq_balance` system service be stopped? - pub stop_irq_balance: bool, - - /// Set the netdev budget (usecs) - pub netdev_budget_usecs: u32, - - /// Set the netdev budget (packets) - pub netdev_budget_packets: u32, - - /// Set the RX side polling frequency - pub rx_usecs: u32, - - /// Set the TX side polling frequency - pub tx_usecs: u32, - - /// Disable RXVLAN offloading? You generally want to do this. - pub disable_rxvlan: bool, - - /// Disable TXVLAN offloading? You generally want to do this. - pub disable_txvlan: bool, - - /// A list of `ethtool` offloads to be disabled. - /// The default list is: [ "gso", "tso", "lro", "sg", "gro" ] - pub disable_offload: Vec, -} - -/// Defines the BiFrost XDP bridge accelerator parameters -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct BridgeConfig { - /// Should the XDP bridge be enabled? - pub use_xdp_bridge: bool, - - /// A list of interface mappings. - pub interface_mapping: Vec, - - /// A list of VLAN mappings. - pub vlan_mapping: Vec, -} - -/// An interface within the Bifrost XDP bridge. -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct BridgeInterface { - /// The interface name. It *must* match an interface name - /// findable by Linux. - pub name: String, - - /// Should Bifrost read VLAN tags and determine redirect - /// policy from there? - pub scan_vlans: bool, - - /// The outbound interface - data that arrives in the interface - /// defined by `name` will be redirected to this interface. - /// - /// If you are using an "on a stick" configuration, this will - /// be the same as `name`. - pub redirect_to: String, -} - -/// If `scan_vlans` is enabled for an interface, then VLANs -/// are examined on the way through the XDP BiFrost bridge. -/// -/// If a VLAN is on the `parent` interface, and matches `tag` - it -/// will be moved to VLAN `redirect_to`. -/// -/// You often need to make reciprocal pairs of these. -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct BridgeVlan { - /// The parent interface name on which the VLAN occurs. - pub parent: String, - - /// The VLAN tag number to redirect if matched. - pub tag: u32, - - /// The destination VLAN tag number if matched. - pub redirect_to: u32, -} - -/// Definitions for anonymous usage submission -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct UsageStats { - /// Are we allowed to send stats at all? - pub send_anonymous: bool, - - /// Where do we send them? - pub anonymous_server: String, -} - -/// Long Term Data Retention -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct LongTermStats { - /// Should we store long-term stats at all? - pub gather_stats: bool, - - /// How frequently should stats be accumulated into a long-term - /// min/max/avg format per tick? - pub collation_period_seconds: u32, - - /// The license key for submitting stats to a LibreQoS hosted - /// statistics server - pub license_key: Option, - - /// UISP reporting period (in seconds). UISP queries can be slow, - /// so hitting it every second or 10 seconds is going to cause problems - /// for some people. A good default may be 5 minutes. Not specifying this - /// disabled UISP integration. - pub uisp_reporting_interval_seconds: Option, -} - -impl EtcLqos { - /// Loads `/etc/lqos.conf`. - pub fn load() -> Result { - if !Path::new("/etc/lqos.conf").exists() { - error!("/etc/lqos.conf does not exist!"); - return Err(EtcLqosError::ConfigDoesNotExist); - } - if let Ok(raw) = std::fs::read_to_string("/etc/lqos.conf") { - let document = raw.parse::(); - match document { - Err(e) => { - error!("Unable to parse TOML from /etc/lqos.conf"); - error!("Full error: {:?}", e); - Err(EtcLqosError::CannotParseToml) - } - Ok(mut config_doc) => { - let cfg = toml_edit::de::from_document::(config_doc.clone()); - match cfg { - Ok(mut cfg) => { - check_config(&mut config_doc, &mut cfg); - Ok(cfg) - } - Err(e) => { - error!("Unable to parse TOML from /etc/lqos.conf"); - error!("Full error: {:?}", e); - Err(EtcLqosError::CannotParseToml) - } - } - } - } - } else { - error!("Unable to read contents of /etc/lqos.conf"); - Err(EtcLqosError::CannotReadFile) - } - } - - /// Saves changes made to /etc/lqos.conf - /// Copies current configuration into /etc/lqos.conf.backup first - pub fn save(&self, document: &mut Document) -> Result<(), EtcLqosError> { - let cfg_path = Path::new("/etc/lqos.conf"); - let backup_path = Path::new("/etc/lqos.conf.backup"); - if let Err(e) = std::fs::copy(cfg_path, backup_path) { - log::error!("Unable to backup /etc/lqos.conf"); - log::error!("{e:?}"); - return Err(EtcLqosError::BackupFail); - } - let new_cfg = document.to_string(); - if let Err(e) = fs::write(cfg_path, new_cfg) { - log::error!("Unable to write to /etc/lqos.conf"); - log::error!("{e:?}"); - return Err(EtcLqosError::WriteFail); - } - Ok(()) - } -} - -/// Run this if you've received the OK from the licensing server, and been -/// sent a license key. This appends a [long_term_stats] section to your -/// config file - ONLY if one doesn't already exist. -pub fn enable_long_term_stats(license_key: String) { - if let Ok(raw) = std::fs::read_to_string("/etc/lqos.conf") { - let document = raw.parse::(); - match document { - Err(e) => { - error!("Unable to parse TOML from /etc/lqos.conf"); - error!("Full error: {:?}", e); - return; - } - Ok(mut config_doc) => { - let cfg = toml_edit::de::from_document::(config_doc.clone()); - match cfg { - Ok(cfg) => { - // Now we enable LTS if its not present - if let Ok(isp_config) = crate::LibreQoSConfig::load() { - if cfg.long_term_stats.is_none() { - - let mut new_section = toml_edit::table(); - new_section["gather_stats"] = value(true); - new_section["collation_period_seconds"] = value(60); - new_section["license_key"] = value(license_key); - if isp_config.automatic_import_uisp { - new_section["uisp_reporting_interval_seconds"] = value(300); - } - config_doc["long_term_stats"] = new_section; - - let new_cfg = config_doc.to_string(); - if let Err(e) = fs::write(Path::new("/etc/lqos.conf"), new_cfg) { - log::error!("Unable to write to /etc/lqos.conf"); - log::error!("{e:?}"); - return; - } - } - } - } - Err(e) => { - error!("Unable to parse TOML from /etc/lqos.conf"); - error!("Full error: {:?}", e); - return; - } - } - } - } - } -} - -fn check_config(cfg_doc: &mut Document, cfg: &mut EtcLqos) { - use sha2::digest::Update; - use sha2::Digest; - - if cfg.node_id.is_none() { - if let Ok(machine_id) = std::fs::read_to_string("/etc/machine-id") { - let hash = sha2::Sha256::new().chain(machine_id).finalize(); - cfg.node_id = Some(format!("{:x}", hash)); - cfg_doc["node_id"] = value(format!("{:x}", hash)); - println!("Updating"); - if let Err(e) = cfg.save(cfg_doc) { - log::error!("Unable to save /etc/lqos.conf"); - log::error!("{e:?}"); - } - } - } -} - -#[derive(Error, Debug)] -pub enum EtcLqosError { - #[error( - "/etc/lqos.conf not found. You must setup this file to use LibreQoS." - )] - ConfigDoesNotExist, - #[error("Unable to read contents of /etc/lqos.conf.")] - CannotReadFile, - #[error("Unable to parse TOML in /etc/lqos.conf")] - CannotParseToml, - #[error("Unable to backup /etc/lqos.conf to /etc/lqos.conf.backup")] - BackupFail, - #[error("Unable to serialize new configuration")] - SerializeFail, - #[error("Unable to write to /etc/lqos.conf")] - WriteFail, -} - -#[cfg(test)] -mod test { - const EXAMPLE_LQOS_CONF: &str = include_str!("../../../lqos.example"); - - #[test] - fn round_trip_toml() { - let doc = EXAMPLE_LQOS_CONF.parse::().unwrap(); - let reserialized = doc.to_string(); - assert_eq!(EXAMPLE_LQOS_CONF, reserialized); - } - - #[test] - fn add_node_id() { - let mut doc = EXAMPLE_LQOS_CONF.parse::().unwrap(); - doc["node_id"] = toml_edit::value("test"); - let reserialized = doc.to_string(); - assert!(reserialized.contains("node_id = \"test\"")); - } -} \ No newline at end of file +mod etclqos_migration; +pub use etclqos_migration::*; From 258a0aeef5ea6ad2e9224c18772e0feba34cc671 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 1 Dec 2023 16:09:09 -0600 Subject: [PATCH 05/41] First pass at a suggested version 1.5 format cleanup. None of the Python stuff is integrated yet. --- src/rust/lqos_config/src/etc/mod.rs | 1 + .../src/etc/v15/anonymous_stats.rs | 22 +++++ src/rust/lqos_config/src/etc/v15/bridge.rs | 50 ++++++++++++ src/rust/lqos_config/src/etc/v15/example.toml | 35 ++++++++ .../src/etc/v15/long_term_stats.rs | 34 ++++++++ src/rust/lqos_config/src/etc/v15/mod.rs | 8 ++ .../lqos_config/src/etc/v15/top_config.rs | 80 +++++++++++++++++++ src/rust/lqos_config/src/etc/v15/tuning.rs | 54 +++++++++++++ 8 files changed, 284 insertions(+) create mode 100644 src/rust/lqos_config/src/etc/v15/anonymous_stats.rs create mode 100644 src/rust/lqos_config/src/etc/v15/bridge.rs create mode 100644 src/rust/lqos_config/src/etc/v15/example.toml create mode 100644 src/rust/lqos_config/src/etc/v15/long_term_stats.rs create mode 100644 src/rust/lqos_config/src/etc/v15/mod.rs create mode 100644 src/rust/lqos_config/src/etc/v15/top_config.rs create mode 100644 src/rust/lqos_config/src/etc/v15/tuning.rs diff --git a/src/rust/lqos_config/src/etc/mod.rs b/src/rust/lqos_config/src/etc/mod.rs index f9b73931..9f6a5bc2 100644 --- a/src/rust/lqos_config/src/etc/mod.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -2,3 +2,4 @@ mod etclqos_migration; pub use etclqos_migration::*; +mod v15; diff --git a/src/rust/lqos_config/src/etc/v15/anonymous_stats.rs b/src/rust/lqos_config/src/etc/v15/anonymous_stats.rs new file mode 100644 index 00000000..e2e86359 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/anonymous_stats.rs @@ -0,0 +1,22 @@ +//! Anonymous statistics section of the configuration +//! file. + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct UsageStats { + /// Are we allowed to send stats at all? + pub send_anonymous: bool, + + /// Where do we send them? + pub anonymous_server: String, +} + +impl Default for UsageStats { + fn default() -> Self { + Self { + send_anonymous: true, + anonymous_server: "stats.libreqos.io:9125".to_string(), + } + } +} \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/bridge.rs b/src/rust/lqos_config/src/etc/v15/bridge.rs new file mode 100644 index 00000000..2e5e4f48 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/bridge.rs @@ -0,0 +1,50 @@ +//! Defines a two-interface bridge configuration. +//! A config file must contain EITHER this, or a `single_interface` +//! section, but not both. + +use serde::{Deserialize, Serialize}; + +/// Represents a two-interface bridge configuration. +#[derive(Serialize, Deserialize, Debug)] +pub struct BridgeConfig { + /// Use the XDP-accelerated bridge? + pub use_xdp_bridge: bool, + + /// The name of the first interface, facing the Internet + pub to_internet: String, + + /// The name of the second interface, facing the LAN + pub to_network: String, +} + +impl Default for BridgeConfig { + fn default() -> Self { + Self { + use_xdp_bridge: true, + to_internet: "eth0".to_string(), + to_network: "eth1".to_string(), + } + } +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct SingleInterfaceConfig { + /// The name of the interface + pub interface: String, + + /// The VLAN ID facing the Internet + pub internet_vlan: u16, + + /// The VLAN ID facing the LAN + pub network_vlan: u16, +} + +impl Default for SingleInterfaceConfig { + fn default() -> Self { + Self { + interface: "eth0".to_string(), + internet_vlan: 2, + network_vlan: 3, + } + } +} \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml new file mode 100644 index 00000000..cf4225e3 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -0,0 +1,35 @@ +version = "1.5" +lqos_directory = "/opt/libreqos/src" +node_id = "0000-0000-0000" +node_name = "Example Node" + +[usage_stats] +send_anonymous = true +anonymous_server = "stats.libreqos.io:9125" + +[tuning] +stop_irq_balance = true +netdev_budget_usecs = 8000 +netdev_budget_packets = 300 +rx_usecs = 8 +tx_usecs = 8 +disable_rxvlan = true +disable_txvlan = true +disable_offload = [ "gso", "tso", "lro", "sg", "gro" ] + +# EITHER: +[bridge] +use_xdp_bridge = true +to_internet = "eth0" +to_network = "eth1" + +# OR: +[single_interface] +internet_vlan = 2 +network_vlan = 3 + +[long_term_stats] +gather_stats = true +collation_period_seconds = 10 +license_key = "(data)" +uisp_reporting_interval_seconds = 300 diff --git a/src/rust/lqos_config/src/etc/v15/long_term_stats.rs b/src/rust/lqos_config/src/etc/v15/long_term_stats.rs new file mode 100644 index 00000000..30724d09 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/long_term_stats.rs @@ -0,0 +1,34 @@ +//! Defines configuration for the LTS project + +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct LongTermStats { + /// Should we store long-term stats at all? + pub gather_stats: bool, + + /// How frequently should stats be accumulated into a long-term + /// min/max/avg format per tick? + pub collation_period_seconds: u32, + + /// The license key for submitting stats to a LibreQoS hosted + /// statistics server + pub license_key: Option, + + /// UISP reporting period (in seconds). UISP queries can be slow, + /// so hitting it every second or 10 seconds is going to cause problems + /// for some people. A good default may be 5 minutes. Not specifying this + /// disabled UISP integration. + pub uisp_reporting_interval_seconds: Option, +} + +impl Default for LongTermStats { + fn default() -> Self { + Self { + gather_stats: false, + collation_period_seconds: 10, + license_key: None, + uisp_reporting_interval_seconds: None, + } + } +} \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/mod.rs b/src/rust/lqos_config/src/etc/v15/mod.rs new file mode 100644 index 00000000..5f4adda2 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/mod.rs @@ -0,0 +1,8 @@ +//! Handles the 1.5.0 configuration file format. + +mod top_config; +pub use top_config::Config; +mod anonymous_stats; +mod tuning; +mod bridge; +mod long_term_stats; \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs new file mode 100644 index 00000000..e5b61245 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -0,0 +1,80 @@ +//! Top-level configuration file for LibreQoS. + +use super::anonymous_stats::UsageStats; +use super::tuning::Tunables; +use serde::{Deserialize, Serialize}; +use sha2::digest::Update; +use sha2::Digest; +use uuid::Uuid; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Config { + /// Version number for the configuration file. + /// This will be set to "1.5". Versioning will make + /// it easier to handle schema updates moving forward. + pub version: String, + + /// Directory in which LibreQoS is installed + pub lqos_directory: String, + + /// Node ID - uniquely identifies this shaper. + pub node_id: String, + + /// Node name - human-readable name for this shaper. + pub node_name: String, + + /// Anonymous usage statistics + pub usage_stats: UsageStats, + + /// Tuning instructions + pub tuning: Tunables, + + /// Bridge configuration + pub bridge: Option, + + /// Single-interface configuration + pub single_interface: Option, + + /// Long-term stats configuration + pub long_term_stats: super::long_term_stats::LongTermStats, +} + +impl Config { + /// Calculate a node ID based on the machine ID. If Machine ID is unavailable, + /// generate a random UUID. + pub fn calculate_node_id() -> String { + if let Ok(machine_id) = std::fs::read_to_string("/etc/machine-id") { + let hash = sha2::Sha256::new().chain(machine_id).finalize(); + format!("{:x}", hash) + } else { + Uuid::new_v4().to_string() + } + } + + /// Test is a configuration is valid. + pub fn validate(&self) -> Result<(), String> { + if self.bridge.is_some() && self.single_interface.is_some() { + return Err( + "Configuration file may not contain both a bridge and a single-interface section." + .to_string(), + ); + } + Ok(()) + } +} + +impl Default for Config { + fn default() -> Self { + Self { + version: "1.5".to_string(), + lqos_directory: "/opt/libreqos/src".to_string(), + node_id: Self::calculate_node_id(), + node_name: "LibreQoS".to_string(), + usage_stats: UsageStats::default(), + tuning: Tunables::default(), + bridge: Some(super::bridge::BridgeConfig::default()), + single_interface: None, + long_term_stats: super::long_term_stats::LongTermStats::default(), + } + } +} diff --git a/src/rust/lqos_config/src/etc/v15/tuning.rs b/src/rust/lqos_config/src/etc/v15/tuning.rs new file mode 100644 index 00000000..9c0d8020 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/tuning.rs @@ -0,0 +1,54 @@ +//! Interface tuning instructions + +use serde::{Deserialize, Serialize}; + +/// Represents a set of `sysctl` and `ethtool` tweaks that may be +/// applied (in place of the previous version's offload service) +#[derive(Serialize, Deserialize, Debug)] +pub struct Tunables { + /// Should the `irq_balance` system service be stopped? + pub stop_irq_balance: bool, + + /// Set the netdev budget (usecs) + pub netdev_budget_usecs: u32, + + /// Set the netdev budget (packets) + pub netdev_budget_packets: u32, + + /// Set the RX side polling frequency + pub rx_usecs: u32, + + /// Set the TX side polling frequency + pub tx_usecs: u32, + + /// Disable RXVLAN offloading? You generally want to do this. + pub disable_rxvlan: bool, + + /// Disable TXVLAN offloading? You generally want to do this. + pub disable_txvlan: bool, + + /// A list of `ethtool` offloads to be disabled. + /// The default list is: [ "gso", "tso", "lro", "sg", "gro" ] + pub disable_offload: Vec, +} + +impl Default for Tunables { + fn default() -> Self { + Self { + stop_irq_balance: true, + netdev_budget_usecs: 8000, + netdev_budget_packets: 300, + rx_usecs: 8, + tx_usecs: 8, + disable_rxvlan: true, + disable_txvlan: true, + disable_offload: vec![ + "gso".to_string(), + "tso".to_string(), + "lro".to_string(), + "sg".to_string(), + "gro".to_string(), + ], + } + } +} From 93997f4acab635a32b14efbae42a33f80790a468 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 1 Dec 2023 16:44:14 -0600 Subject: [PATCH 06/41] Started to feed the Python ispConfig.py sections into the config. --- src/rust/lqos_config/src/etc/v15/example.toml | 20 +++++++ .../src/etc/v15/integration_common.rs | 21 ++++++++ src/rust/lqos_config/src/etc/v15/ip_ranges.rs | 21 ++++++++ src/rust/lqos_config/src/etc/v15/mod.rs | 5 +- src/rust/lqos_config/src/etc/v15/queues.rs | 54 +++++++++++++++++++ .../lqos_config/src/etc/v15/top_config.rs | 16 +++++- 6 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 src/rust/lqos_config/src/etc/v15/integration_common.rs create mode 100644 src/rust/lqos_config/src/etc/v15/ip_ranges.rs create mode 100644 src/rust/lqos_config/src/etc/v15/queues.rs diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index cf4225e3..15d0ac4a 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -28,8 +28,28 @@ to_network = "eth1" internet_vlan = 2 network_vlan = 3 +[queues] +default_sqm = "cake diffserv4" +monitor_only = false +uplink_bandwidth_mbps = 1000 +downlink_bandwidth_mbps = 1000 +generated_pn_download_mbps = 1000 +generated_pn_upload_mbps = 1000 +dry_run = false +sudo = false +#override_available_queues = 12 # This can be omitted and be 0 for Python +use_binpacking = false + [long_term_stats] gather_stats = true collation_period_seconds = 10 license_key = "(data)" uisp_reporting_interval_seconds = 300 + +[ip_ranges] +ignore_subnets = [] +allow_subnets = [ "172.16.0.0/12", "10.0.0.0/8", "100.64.0.0/16", "192.168.0.0/16" ] + +[integration_common] +circuit_name_as_address = false +always_overwrite_network_json = false \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/integration_common.rs b/src/rust/lqos_config/src/etc/v15/integration_common.rs new file mode 100644 index 00000000..28104c14 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/integration_common.rs @@ -0,0 +1,21 @@ +//! Common integration variables, shared between integrations + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct IntegrationConfig { + /// Replace names with addresses? + pub circuit_name_as_address: bool, + + /// Always overwrite network.json? + pub always_overwrite_network_json: bool, +} + +impl Default for IntegrationConfig { + fn default() -> Self { + Self { + circuit_name_as_address: false, + always_overwrite_network_json: false, + } + } +} \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/ip_ranges.rs b/src/rust/lqos_config/src/etc/v15/ip_ranges.rs new file mode 100644 index 00000000..e5d357e1 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/ip_ranges.rs @@ -0,0 +1,21 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct IpRanges { + pub ignore_subnets: Vec, + pub allow_subnets: Vec, +} + +impl Default for IpRanges { + fn default() -> Self { + Self { + ignore_subnets: vec![], + allow_subnets: vec![ + "172.16.0.0/12".to_string(), + "10.0.0.0/8".to_string(), + "100.64.0.0/10".to_string(), + "192.168.0.0/16".to_string(), + ], + } + } +} \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/mod.rs b/src/rust/lqos_config/src/etc/v15/mod.rs index 5f4adda2..829ca734 100644 --- a/src/rust/lqos_config/src/etc/v15/mod.rs +++ b/src/rust/lqos_config/src/etc/v15/mod.rs @@ -5,4 +5,7 @@ pub use top_config::Config; mod anonymous_stats; mod tuning; mod bridge; -mod long_term_stats; \ No newline at end of file +mod long_term_stats; +mod queues; +mod integration_common; +mod ip_ranges; \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/queues.rs b/src/rust/lqos_config/src/etc/v15/queues.rs new file mode 100644 index 00000000..76aac4d0 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/queues.rs @@ -0,0 +1,54 @@ +//! Queue Generation definitions (originally from ispConfig.py) + +use serde::{Serialize, Deserialize}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct QueueConfig { + /// Which SQM to use by default + pub default_sqm: String, + + /// Should we monitor only, and not shape traffic? + pub monitor_only: bool, + + /// Upstream bandwidth total - download + pub uplink_bandwidth_mbps: u32, + + /// Downstream bandwidth total - upload + pub downlink_bandwidth_mbps: u32, + + /// Upstream bandwidth per interface queue + pub generated_pn_download_mbps: u32, + + /// Downstream bandwidth per interface queue + pub generated_pn_upload_mbps: u32, + + /// Should shell commands actually execute, or just be printed? + pub dry_run: bool, + + /// Should `sudo` be prefixed on commands? + pub sudo: bool, + + /// Should we override the number of available queues? + pub override_available_queues: Option, + + /// Should we invoke the binpacking algorithm to optimize flat + /// networks? + pub use_binpacking: bool, +} + +impl Default for QueueConfig { + fn default() -> Self { + Self { + default_sqm: "cake diffserv4".to_string(), + monitor_only: false, + uplink_bandwidth_mbps: 1_000, + downlink_bandwidth_mbps: 1_000, + generated_pn_download_mbps: 1_000, + generated_pn_upload_mbps: 1_000, + dry_run: false, + sudo: false, + override_available_queues: None, + use_binpacking: false, + } + } +} \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index e5b61245..c7900dd5 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -21,7 +21,7 @@ pub struct Config { pub node_id: String, /// Node name - human-readable name for this shaper. - pub node_name: String, + pub node_name: String, /// Anonymous usage statistics pub usage_stats: UsageStats, @@ -35,8 +35,17 @@ pub struct Config { /// Single-interface configuration pub single_interface: Option, + /// Queue Definition data (originally from ispConfig.py) + pub queues: super::queues::QueueConfig, + /// Long-term stats configuration pub long_term_stats: super::long_term_stats::LongTermStats, + + /// IP Range definitions + pub ip_ranges: super::ip_ranges::IpRanges, + + /// Integration Common Variables + pub integration_common: super::integration_common::IntegrationConfig, } impl Config { @@ -69,12 +78,15 @@ impl Default for Config { version: "1.5".to_string(), lqos_directory: "/opt/libreqos/src".to_string(), node_id: Self::calculate_node_id(), - node_name: "LibreQoS".to_string(), + node_name: "LibreQoS".to_string(), usage_stats: UsageStats::default(), tuning: Tunables::default(), bridge: Some(super::bridge::BridgeConfig::default()), single_interface: None, + queues: super::queues::QueueConfig::default(), long_term_stats: super::long_term_stats::LongTermStats::default(), + ip_ranges: super::ip_ranges::IpRanges::default(), + integration_common: super::integration_common::IntegrationConfig::default(), } } } From 7bd302191e2c06a7f52e39aaf86d1201a391f539 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 7 Dec 2023 08:46:40 -0600 Subject: [PATCH 07/41] Add unit tests for TOML loading. Integrate the Python migration code into the v15 structure. Activate the Python unit test. --- src/rust/Cargo.lock | 40 +++++++++++++++---- src/rust/lqos_config/Cargo.toml | 1 + src/rust/lqos_config/src/etc/mod.rs | 1 + .../lqos_config/src/etc/python_migration.rs | 2 +- src/rust/lqos_config/src/etc/v15/example.toml | 6 +-- .../lqos_config/src/etc/v15/top_config.rs | 17 ++++++++ 6 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 1ffe02c2..e2b57d46 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -910,7 +910,7 @@ dependencies = [ "atomic", "pear", "serde", - "toml", + "toml 0.7.6", "uncased", "version_check", ] @@ -1535,7 +1535,8 @@ dependencies = [ "serde_json", "sha2", "thiserror", - "toml_edit", + "toml 0.8.8", + "toml_edit 0.19.14", "uuid", ] @@ -2753,9 +2754,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" dependencies = [ "serde", ] @@ -3167,14 +3168,26 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.19.14", +] + +[[package]] +name = "toml" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.21.0", ] [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] @@ -3192,6 +3205,19 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_edit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +dependencies = [ + "indexmap 2.0.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" diff --git a/src/rust/lqos_config/Cargo.toml b/src/rust/lqos_config/Cargo.toml index 215a4eec..9f479b1f 100644 --- a/src/rust/lqos_config/Cargo.toml +++ b/src/rust/lqos_config/Cargo.toml @@ -17,3 +17,4 @@ uuid = { version = "1", features = ["v4", "fast-rng" ] } log = "0" dashmap = "5" pyo3 = "0.20" +toml = "0.8.8" diff --git a/src/rust/lqos_config/src/etc/mod.rs b/src/rust/lqos_config/src/etc/mod.rs index 9f6a5bc2..a6315879 100644 --- a/src/rust/lqos_config/src/etc/mod.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -3,3 +3,4 @@ mod etclqos_migration; pub use etclqos_migration::*; mod v15; +mod python_migration; \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/python_migration.rs b/src/rust/lqos_config/src/etc/python_migration.rs index dc144394..81c846d2 100644 --- a/src/rust/lqos_config/src/etc/python_migration.rs +++ b/src/rust/lqos_config/src/etc/python_migration.rs @@ -182,7 +182,7 @@ impl PythonMigration { mod test { use super::*; - const DEFAULT_ISP_CONFIG_PY: &str = include_str!("../../../ispConfig.example.py"); + const DEFAULT_ISP_CONFIG_PY: &str = include_str!("../../../../ispConfig.example.py"); #[test] fn test_parsing_the_default() { diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index 15d0ac4a..dd5500d7 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -24,9 +24,9 @@ to_internet = "eth0" to_network = "eth1" # OR: -[single_interface] -internet_vlan = 2 -network_vlan = 3 +#[single_interface] +#internet_vlan = 2 +#network_vlan = 3 [queues] default_sqm = "cake diffserv4" diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index c7900dd5..24a3863d 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -70,6 +70,12 @@ impl Config { } Ok(()) } + + fn load_from_string(s: &str) -> Result { + let config: Config = toml::from_str(s).map_err(|e| format!("Error parsing config: {}", e))?; + config.validate()?; + Ok(config) + } } impl Default for Config { @@ -90,3 +96,14 @@ impl Default for Config { } } } + +#[cfg(test)] +mod test { + use super::Config; + + #[test] + fn load_example() { + let config = Config::load_from_string(include_str!("example.toml")).unwrap(); + assert_eq!(config.version, "1.5"); + } +} \ No newline at end of file From 737f81d5a5a4ec03b5ae01e17a31f72b3c5f5dad Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 7 Dec 2023 11:03:17 -0600 Subject: [PATCH 08/41] Finish fleshing out the integration fields. Add some more unit tests and the beginnings of a migration system. --- .../lqos_config/src/etc/etclqos_migration.rs | 14 +- src/rust/lqos_config/src/etc/migration.rs | 227 ++++++++++++++++++ src/rust/lqos_config/src/etc/mod.rs | 8 +- .../lqos_config/src/etc/python_migration.rs | 19 +- src/rust/lqos_config/src/etc/test_data.rs | 202 ++++++++++++++++ src/rust/lqos_config/src/etc/v15/example.toml | 23 +- src/rust/lqos_config/src/etc/v15/mod.rs | 6 +- .../src/etc/v15/spylnx_integration.rs | 20 ++ .../lqos_config/src/etc/v15/top_config.rs | 17 ++ .../src/etc/v15/uisp_integration.rs | 44 ++++ 10 files changed, 569 insertions(+), 11 deletions(-) create mode 100644 src/rust/lqos_config/src/etc/migration.rs create mode 100644 src/rust/lqos_config/src/etc/test_data.rs create mode 100644 src/rust/lqos_config/src/etc/v15/spylnx_integration.rs create mode 100644 src/rust/lqos_config/src/etc/v15/uisp_integration.rs diff --git a/src/rust/lqos_config/src/etc/etclqos_migration.rs b/src/rust/lqos_config/src/etc/etclqos_migration.rs index 88d34d75..ea180561 100644 --- a/src/rust/lqos_config/src/etc/etclqos_migration.rs +++ b/src/rust/lqos_config/src/etc/etclqos_migration.rs @@ -163,7 +163,15 @@ impl EtcLqos { return Err(EtcLqosError::ConfigDoesNotExist); } if let Ok(raw) = std::fs::read_to_string("/etc/lqos.conf") { - let document = raw.parse::(); + Self::load_from_string(&raw) + } else { + error!("Unable to read contents of /etc/lqos.conf"); + Err(EtcLqosError::CannotReadFile) + } + } + + pub(crate) fn load_from_string(raw: &str) -> Result { + let document = raw.parse::(); match document { Err(e) => { error!("Unable to parse TOML from /etc/lqos.conf"); @@ -185,10 +193,6 @@ impl EtcLqos { } } } - } else { - error!("Unable to read contents of /etc/lqos.conf"); - Err(EtcLqosError::CannotReadFile) - } } /// Saves changes made to /etc/lqos.conf diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs new file mode 100644 index 00000000..4975eb9d --- /dev/null +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -0,0 +1,227 @@ +use super::{ + python_migration::{PythonMigration, PythonMigrationError}, + v15::{Config, SingleInterfaceConfig, BridgeConfig}, + EtcLqosError, +}; +use crate::EtcLqos; +/// Provides support for migration from older versions of the configuration file. +use thiserror::Error; +use toml_edit::Document; + +#[derive(Debug, Error)] +pub enum MigrationError { + #[error("Failed to read configuration file: {0}")] + ReadError(#[from] std::io::Error), + #[error("Failed to parse configuration file: {0}")] + ParseError(#[from] toml_edit::TomlError), + #[error("Unknown Version: {0}")] + UnknownVersion(String), + #[error("Unable to load old version: {0}")] + LoadError(#[from] EtcLqosError), + #[error("Unable to load python version: {0}")] + PythonLoadError(#[from] PythonMigrationError), +} + +pub fn migrate_if_needed() -> Result<(), MigrationError> { + let raw = + std::fs::read_to_string("/etc/lqos.conf").map_err(|e| MigrationError::ReadError(e))?; + + let doc = raw + .parse::() + .map_err(|e| MigrationError::ParseError(e))?; + if let Some((_key, version)) = doc.get_key_value("version") { + if version.as_str().unwrap() == "1.5.0" { + log::info!("Configuration file is already at version 1.5.0, no migration needed"); + return Ok(()); + } else { + log::error!("Configuration file is at version {}, but this version of lqos only supports version 1.5.0", version.as_str().unwrap()); + return Err(MigrationError::UnknownVersion( + version.as_str().unwrap().to_string(), + )); + } + } else { + log::info!("No version found in configuration file, assuming 1.4x and migration is needed"); + migrate_14_to_15()?; + } + + Ok(()) +} + +fn migrate_14_to_15() -> Result<(), MigrationError> { + // Load the 1.4 config file + let old_config = EtcLqos::load().map_err(|e| MigrationError::LoadError(e))?; + let python_config = PythonMigration::load().map_err(|e| MigrationError::PythonLoadError(e))?; + let new_config = do_migration_14_to_15(&old_config, &python_config)?; + Ok(()) +} + +fn do_migration_14_to_15( + old_config: &EtcLqos, + python_config: &PythonMigration, +) -> Result { + // This is separated out to make unit testing easier + let mut new_config = Config::default(); + + migrate_top_level(old_config, &mut new_config)?; + migrate_usage_stats(old_config, &mut new_config)?; + migrate_tunables(old_config, &mut new_config)?; + migrate_bridge(old_config, &python_config, &mut new_config)?; + migrate_lts(old_config, &mut new_config)?; + migrate_ip_ranges(python_config, &mut new_config)?; + migrate_integration_common(python_config, &mut new_config)?; + migrate_spylnx(python_config, &mut new_config)?; + migrate_uisp(python_config, &mut new_config)?; + + new_config.validate().unwrap(); // Left as an upwrap because this should *never* happen + Ok(new_config) +} + +fn migrate_top_level(old_config: &EtcLqos, new_config: &mut Config) -> Result<(), MigrationError> { + new_config.version = "1.5".to_string(); + new_config.lqos_directory = old_config.lqos_directory.clone(); + if let Some(node_id) = &old_config.node_id { + new_config.node_id = node_id.clone(); + } else { + new_config.node_id = Config::calculate_node_id(); + } + if let Some(node_name) = &old_config.node_name { + new_config.node_name = node_name.clone(); + } else { + new_config.node_name = "Set my name in /etc/lqos.conf".to_string(); + } + Ok(()) +} + +fn migrate_usage_stats( + old_config: &EtcLqos, + new_config: &mut Config, +) -> Result<(), MigrationError> { + if let Some(usage_stats) = &old_config.usage_stats { + new_config.usage_stats.send_anonymous = usage_stats.send_anonymous; + new_config.usage_stats.anonymous_server = usage_stats.anonymous_server.clone(); + } else { + new_config.usage_stats = Default::default(); + } + Ok(()) +} + +fn migrate_tunables(old_config: &EtcLqos, new_config: &mut Config) -> Result<(), MigrationError> { + if let Some(tunables) = &old_config.tuning { + new_config.tuning.stop_irq_balance = tunables.stop_irq_balance; + new_config.tuning.netdev_budget_packets = tunables.netdev_budget_packets; + new_config.tuning.netdev_budget_usecs = tunables.netdev_budget_usecs; + new_config.tuning.rx_usecs = tunables.rx_usecs; + new_config.tuning.tx_usecs = tunables.tx_usecs; + new_config.tuning.disable_txvlan = tunables.disable_txvlan; + new_config.tuning.disable_rxvlan = tunables.disable_rxvlan; + new_config.tuning.disable_offload = tunables.disable_offload.clone(); + } else { + new_config.tuning = Default::default(); + } + Ok(()) +} + +fn migrate_bridge( + old_config: &EtcLqos, + python_config: &PythonMigration, + new_config: &mut Config, +) -> Result<(), MigrationError> { + if python_config.on_a_stick { + new_config.bridge = None; + new_config.single_interface = Some(SingleInterfaceConfig { + interface: python_config.interface_a.clone(), + internet_vlan: python_config.stick_vlan_b as u16, + network_vlan: python_config.stick_vlan_b as u16, + }); + } else { + new_config.single_interface = None; + new_config.bridge = Some(BridgeConfig { + use_xdp_bridge: old_config.bridge.as_ref().unwrap().use_xdp_bridge, + to_internet: python_config.interface_b.clone(), + to_network: python_config.interface_a.clone(), + }); + + } + Ok(()) +} + +fn migrate_queues(old_config: &EtcLqos, python_config: &PythonMigration, new_config: &mut Config) -> Result<(), MigrationError> { + new_config.queues.default_sqm = python_config.sqm.clone(); + new_config.queues.monitor_only = python_config.monitor_only_mode; + new_config.queues.uplink_bandwidth_mbps = python_config.upstream_bandwidth_capacity_upload_mbps; + new_config.queues.downlink_bandwidth_mbps = python_config.upstream_bandwidth_capacity_download_mbps; + new_config.queues.generated_pn_upload_mbps = python_config.generated_pn_upload_mbps; + new_config.queues.generated_pn_download_mbps = python_config.generated_pn_download_mbps; + new_config.queues.dry_run = !python_config.enable_actual_shell_commands; + new_config.queues.sudo = python_config.run_shell_commands_as_sudo; + if python_config.queues_available_override == 0 { + new_config.queues.override_available_queues = None; + } else { + new_config.queues.override_available_queues = Some(python_config.queues_available_override); + } + new_config.queues.use_binpacking = python_config.use_bin_packing_to_balance_cpu; + Ok(()) +} + +fn migrate_lts(old_config: &EtcLqos, new_config: &mut Config) -> Result<(), MigrationError> { + if let Some(lts) = &old_config.long_term_stats { + new_config.long_term_stats.gather_stats = lts.gather_stats; + new_config.long_term_stats.collation_period_seconds = lts.collation_period_seconds; + new_config.long_term_stats.license_key = lts.license_key.clone(); + new_config.long_term_stats.uisp_reporting_interval_seconds = lts.uisp_reporting_interval_seconds; + } else { + new_config.long_term_stats = super::v15::LongTermStats::default(); + } + Ok(()) +} + +fn migrate_ip_ranges(python_config: &PythonMigration, new_config: &mut Config) -> Result<(), MigrationError> { + new_config.ip_ranges.ignore_subnets = python_config.ignore_subnets.clone(); + new_config.ip_ranges.allow_subnets = python_config.allowed_subnets.clone(); + Ok(()) +} + +fn migrate_integration_common(python_config: &PythonMigration, new_config: &mut Config) -> Result<(), MigrationError> { + new_config.integration_common.circuit_name_as_address = python_config.circuit_name_use_address; + new_config.integration_common.always_overwrite_network_json = python_config.overwrite_network_json_always; + Ok(()) +} + +fn migrate_spylnx(python_config: &PythonMigration, new_config: &mut Config) -> Result<(), MigrationError> { + new_config.spylnx_integration.enable_spylnx = python_config.automatic_import_splynx; + new_config.spylnx_integration.api_key = python_config.splynx_api_key.clone(); + new_config.spylnx_integration.api_secret = python_config.spylnx_api_secret.clone(); + new_config.spylnx_integration.url = python_config.spylnx_api_url.clone(); + Ok(()) +} + +fn migrate_uisp(python_config: &PythonMigration, new_config: &mut Config) -> Result<(), MigrationError> { + new_config.uisp_integration.enable_uisp = python_config.automatic_import_uisp; + new_config.uisp_integration.token = python_config.uisp_auth_token.clone(); + new_config.uisp_integration.url = python_config.uisp_base_url.clone(); + new_config.uisp_integration.site = python_config.uisp_site.clone(); + new_config.uisp_integration.strategy = python_config.uisp_strategy.clone(); + new_config.uisp_integration.suspended_strategy = python_config.uisp_suspended_strategy.clone(); + new_config.uisp_integration.airmax_capacity = python_config.airmax_capacity; + new_config.uisp_integration.ltu_capacity = python_config.ltu_capacity; + new_config.uisp_integration.exclude_sites = python_config.exclude_sites.clone(); + new_config.uisp_integration.ipv6_with_mikrotik = python_config.find_ipv6_using_mikrotik; + new_config.uisp_integration.bandwidth_overhead_factor = python_config.bandwidth_overhead_factor; + new_config.uisp_integration.commit_bandwidth_multiplier = python_config.committed_bandwidth_multiplier; + // TODO: ExceptionCPEs is going to require some real work + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::etc::test_data::{OLD_CONFIG, PYTHON_CONFIG}; + + #[test] + fn test_migration() { + let old_config = EtcLqos::load_from_string(OLD_CONFIG).unwrap(); + let python_config = PythonMigration::load_from_string(PYTHON_CONFIG).unwrap(); + let new_config = do_migration_14_to_15(&old_config, &python_config).unwrap(); + assert_eq!(new_config.version, "1.5"); + } +} diff --git a/src/rust/lqos_config/src/etc/mod.rs b/src/rust/lqos_config/src/etc/mod.rs index a6315879..a17fa99f 100644 --- a/src/rust/lqos_config/src/etc/mod.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -3,4 +3,10 @@ mod etclqos_migration; pub use etclqos_migration::*; mod v15; -mod python_migration; \ No newline at end of file +mod python_migration; +mod migration; + +#[cfg(test)] +pub mod test_data; + +pub use migration::migrate_if_needed; \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/python_migration.rs b/src/rust/lqos_config/src/etc/python_migration.rs index 81c846d2..e71fbf4b 100644 --- a/src/rust/lqos_config/src/etc/python_migration.rs +++ b/src/rust/lqos_config/src/etc/python_migration.rs @@ -176,13 +176,26 @@ impl PythonMigration { Ok(old_config) } + + pub(crate) fn load_from_string(s: &str) -> Result { + let mut old_config = Self::default(); + prepare_freethreaded_python(); + Python::with_gil(|py| { + py.run(s, None, None).unwrap(); + let result = Self::parse(&mut old_config, &py); + if result.is_err() { + println!("Error parsing Python config: {:?}", result); + } + }); + + Ok(old_config) + } } #[cfg(test)] mod test { use super::*; - - const DEFAULT_ISP_CONFIG_PY: &str = include_str!("../../../../ispConfig.example.py"); + use super::super::test_data::*; #[test] fn test_parsing_the_default() { @@ -190,7 +203,7 @@ mod test { prepare_freethreaded_python(); let mut worked = true; Python::with_gil(|py| { - py.run(DEFAULT_ISP_CONFIG_PY, None, None).unwrap(); + py.run(PYTHON_CONFIG, None, None).unwrap(); let result = PythonMigration::parse(&mut cfg, &py); if result.is_err() { println!("Error parsing Python config: {:?}", result); diff --git a/src/rust/lqos_config/src/etc/test_data.rs b/src/rust/lqos_config/src/etc/test_data.rs new file mode 100644 index 00000000..163a516c --- /dev/null +++ b/src/rust/lqos_config/src/etc/test_data.rs @@ -0,0 +1,202 @@ +pub const OLD_CONFIG: &str = " +# This file *must* be installed in `/etc/lqos.conf`. +# Change the values to match your setup. + +# Where is LibreQoS installed? +lqos_directory = '/home/herbert/Rust/LibreQoS/libreqos/LibreQoS/src' +queue_check_period_ms = 1000 +packet_capture_time = 10 # Number of seconds to capture packets in an analysis session +node_id = \"aee0eb53606ef621d386ef5fcfa0d72a5ba64fccd36df2997695dfb6d418c64b\" + +[usage_stats] +send_anonymous = true +anonymous_server = \"127.0.0.1:9125\" + +[tuning] +# IRQ balance breaks XDP_Redirect, which we use. Recommended to leave as true. +stop_irq_balance = false +netdev_budget_usecs = 8000 +netdev_budget_packets = 300 +rx_usecs = 8 +tx_usecs = 8 +disable_rxvlan = true +disable_txvlan = true +# What offload types should be disabled on the NIC. The defaults are recommended here. +disable_offload = [ \"gso\", \"tso\", \"lro\", \"sg\", \"gro\" ] + +# For a two interface setup, use the following - and replace +# \"enp1s0f1\" and \"enp1s0f2\" with your network card names (obtained +# from `ip link`): +[bridge] +use_xdp_bridge = false +interface_mapping = [ + { name = \"veth_toexternal\", redirect_to = \"veth_tointernal\", scan_vlans = false }, + { name = \"veth_tointernal\", redirect_to = \"veth_toexternal\", scan_vlans = false } +] +vlan_mapping = [] + +# For \"on a stick\" (single interface mode): +# [bridge] +# use_xdp_bridge = true +# interface_mapping = [ +# { name = \"enp1s0f1\", redirect_to = \"enp1s0f1\", scan_vlans = true } +# ] +# vlan_mapping = [ +# { parent = \"enp1s0f1\", tag = 3, redirect_to = 4 }, +# { parent = \"enp1s0f1\", tag = 4, redirect_to = 3 } +# ] +"; + +pub const PYTHON_CONFIG : &str = " +# 'fq_codel' or 'cake diffserv4' +# 'cake diffserv4' is recommended +# sqm = 'fq_codel' +sqm = 'cake diffserv4' + +# Used to passively monitor the network for before / after comparisons. Leave as False to +# ensure actual shaping. After changing this value, run \"sudo systemctl restart LibreQoS.service\" +monitorOnlyMode = False + +# How many Mbps are available to the edge of this network. +# Any circuits, generated nodes, or network.json nodes, will all be capped at no more than this amount. +upstreamBandwidthCapacityDownloadMbps = 1000 +upstreamBandwidthCapacityUploadMbps = 1000 + +# Consider these values your bandwidth bottleneck per-CPU-core. +# This will depend on your CPU's single-thread passmark score. +# Devices in ShapedDevices.csv without a defined ParentNode (such as if you have a flat {} network) +# will be placed under one of these generated parent node, evenly spread out across CPU cores. +# This defines the bandwidth limit for each of those generated parent nodes. +# If you are not sure what values to use, simply use the same values as upstreamBandwidthCapacityDownloadMbps and upstreamBandwidthCapacityUploadMbps +generatedPNDownloadMbps = 1000 +generatedPNUploadMbps = 1000 + +# Interface connected to core router +interfaceA = 'veth_tointernal' + +# Interface connected to edge router +interfaceB = 'veth_toexternal' + +# Queue refresh scheduler (lqos_scheduler). Minutes between reloads. +queueRefreshIntervalMins = 30 + +# WORK IN PROGRESS. Note that interfaceA determines the \"stick\" interface +# I could only get scanning to work if I issued ethtool -K enp1s0f1 rxvlan off +OnAStick = False +# VLAN facing the core router +StickVlanA = 0 +# VLAN facing the edge router +StickVlanB = 0 + +# Allow shell commands. False causes commands print to console only without being executed. +# MUST BE ENABLED FOR PROGRAM TO FUNCTION +enableActualShellCommands = True + +# Add 'sudo' before execution of any shell commands. May be required depending on distribution and environment. +runShellCommandsAsSudo = False + +# Allows overriding queues / CPU cores used. When set to 0, the max possible queues / CPU cores are utilized. Please +# leave as 0. +queuesAvailableOverride = 0 + +# Devices in in ShapedDevices.csv without defined Parent Nodes are placed in generated parent nodes. +# When set True, this option balances the subscribers across generatedPNs / CPU cores based on the subscriber's bandwidth plan. +# When set False, devices are placed in generatedPNs sequentially with a near equal number of subscribers per core. +# Whether this impacts balance across CPUs will depend on your subscribers' usage patterns, but if you are observing +# unequal CPU load, and have most subscribers without a defined Parent Node, it is recommended to try this option. +# Most subscribers average about the same bandwidth load regardless of speed plan (typically 5Mbps or so). +# Past 25,000 subscribers this option becomes inefficient and is not advised. +useBinPackingToBalanceCPU = False + +# Bandwidth & Latency Graphing +influxDBEnabled = True +influxDBurl = \"http://localhost:8086\" +influxDBBucket = \"libreqos\" +influxDBOrg = \"Your ISP Name Here\" +influxDBtoken = \"\" + +# NMS/CRM Integration + +# Use Customer Name or Address as Circuit Name +circuitNameUseAddress = True + +# Should integrationUISP overwrite network.json on each run? +overwriteNetworkJSONalways = False + +# If a device shows a WAN IP within these subnets, assume they are behind NAT / un-shapable, and ignore them +ignoreSubnets = ['192.168.0.0/16'] +allowedSubnets = ['100.64.0.0/10'] + +# Splynx Integration +automaticImportSplynx = False +splynx_api_key = '' +splynx_api_secret = '' +# Everything before /api/2.0/ on your Splynx instance +splynx_api_url = 'https://YOUR_URL.splynx.app' + +# UISP integration +automaticImportUISP = False +uispAuthToken = '' +# Everything before /nms/ on your UISP instance +UISPbaseURL = 'https://examplesite.com' +# UISP Site - enter the name of the root site in your network tree +# to act as the starting point for the tree mapping +uispSite = '' +# Strategy: +# * \"flat\" - create all client sites directly off the top of the tree, +# provides maximum performance - at the expense of not offering AP, +# or site options. +# * \"full\" - build a complete network map +uispStrategy = \"full\" +# Handling of UISP suspensions: +# * \"none\" - do not handle suspensions +# * \"ignore\" - do not add suspended customers to the network map +# * \"slow\" - limit suspended customers to 1mbps +uispSuspendedStrategy = \"none\" +# Assumed capacity of AirMax and LTU radios vs reported capacity by UISP. For example, 65% would be 0.65. +# For AirMax, this applies to flexible frame only. AirMax fixed frame will have capacity based on ratio. +airMax_capacity = 0.65 +ltu_capacity = 0.90 +# List any sites that should not be included, with each site name surrounded by '' and separated by commas +excludeSites = [] +# If you use IPv6, this can be used to find associated IPv6 prefixes for your clients' IPv4 addresses, and match them +# to those devices +findIPv6usingMikrotik = False +# If you want to provide a safe cushion for speed test results to prevent customer complains, you can set this to +# 1.15 (15% above plan rate). If not, you can leave as 1.0 +bandwidthOverheadFactor = 1.0 +# Number to multiply the maximum/ceiling bandwidth with to determine the minimum bandwidth. +committedBandwidthMultiplier = 0.98 +# For edge cases, set the respective ParentNode for these CPEs +exceptionCPEs = {} +# exceptionCPEs = { +# 'CPE-SomeLocation1': 'AP-SomeLocation1', +# 'CPE-SomeLocation2': 'AP-SomeLocation2', +# } + +# API Auth +apiUsername = \"testUser\" +apiPassword = \"changeme8343486806\" +apiHostIP = \"127.0.0.1\" +apiHostPost = 5000 + + +httpRestIntegrationConfig = { + 'enabled': False, + 'baseURL': 'https://domain', + 'networkURI': '/some/path', + 'shaperURI': '/some/path/etc', + 'requestsConfig': { + 'verify': True, # Good for Dev if your dev env doesnt have cert + 'params': { # params for query string ie uri?some-arg=some-value + 'search': 'hold-my-beer' + }, + #'headers': { + # 'Origin': 'SomeHeaderValue', + #}, + }, + # If you want to store a timestamped copy/backup of both network.json and Shaper.csv each time they are updated, + # provide a path + # 'logChanges': '/var/log/libreqos' +} +"; diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index dd5500d7..70d58fd2 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -52,4 +52,25 @@ allow_subnets = [ "172.16.0.0/12", "10.0.0.0/8", "100.64.0.0/16", "192.168.0.0/1 [integration_common] circuit_name_as_address = false -always_overwrite_network_json = false \ No newline at end of file +always_overwrite_network_json = false + +[spylnx_integration] +enable_spylnx = false +api_key = "" +api_secret = "" +url = "" + +[uisp_integration] +enable_uisp = false +token = "" +url = "" +site = "" +strategy = "" +suspended_strategy = "" +airmax_capacity = 0.65 +ltu_capacity = 0.9 +exclude_sites = [] +ipv6_with_mikrotik = false +bandwidth_overhead_factor = 1.0 +commit_bandwidth_multiplier = 0.98 +exception_cpes = [] diff --git a/src/rust/lqos_config/src/etc/v15/mod.rs b/src/rust/lqos_config/src/etc/v15/mod.rs index 829ca734..737f082d 100644 --- a/src/rust/lqos_config/src/etc/v15/mod.rs +++ b/src/rust/lqos_config/src/etc/v15/mod.rs @@ -8,4 +8,8 @@ mod bridge; mod long_term_stats; mod queues; mod integration_common; -mod ip_ranges; \ No newline at end of file +mod ip_ranges; +mod spylnx_integration; +mod uisp_integration; +pub use bridge::*; +pub use long_term_stats::LongTermStats; \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/spylnx_integration.rs b/src/rust/lqos_config/src/etc/v15/spylnx_integration.rs new file mode 100644 index 00000000..bb1f9048 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/spylnx_integration.rs @@ -0,0 +1,20 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct SplynxIntegration { + pub enable_spylnx: bool, + pub api_key: String, + pub api_secret: String, + pub url: String, +} + +impl Default for SplynxIntegration { + fn default() -> Self { + SplynxIntegration { + enable_spylnx: false, + api_key: "".to_string(), + api_secret: "".to_string(), + url: "".to_string(), + } + } +} \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index 24a3863d..1fbe6974 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -46,6 +46,12 @@ pub struct Config { /// Integration Common Variables pub integration_common: super::integration_common::IntegrationConfig, + + /// Spylnx Integration + pub spylnx_integration: super::spylnx_integration::SplynxIntegration, + + /// UISP Integration + pub uisp_integration: super::uisp_integration::UispIntegration, } impl Config { @@ -68,6 +74,15 @@ impl Config { .to_string(), ); } + if self.version != "1.5" { + return Err(format!( + "Configuration file is at version {}, but this version of lqos only supports version 1.5.0", + self.version + )); + } + if self.node_id.is_empty() { + return Err("Node ID must be set".to_string()); + } Ok(()) } @@ -93,6 +108,8 @@ impl Default for Config { long_term_stats: super::long_term_stats::LongTermStats::default(), ip_ranges: super::ip_ranges::IpRanges::default(), integration_common: super::integration_common::IntegrationConfig::default(), + spylnx_integration: super::spylnx_integration::SplynxIntegration::default(), + uisp_integration: super::uisp_integration::UispIntegration::default(), } } } diff --git a/src/rust/lqos_config/src/etc/v15/uisp_integration.rs b/src/rust/lqos_config/src/etc/v15/uisp_integration.rs new file mode 100644 index 00000000..54647689 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/uisp_integration.rs @@ -0,0 +1,44 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct UispIntegration { + pub enable_uisp: bool, + pub token: String, + pub url: String, + pub site: String, + pub strategy: String, + pub suspended_strategy: String, + pub airmax_capacity: f32, + pub ltu_capacity: f32, + pub exclude_sites: Vec, + pub ipv6_with_mikrotik: bool, + pub bandwidth_overhead_factor: f32, + pub commit_bandwidth_multiplier: f32, + pub exception_cpes: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ExceptionCpe { + pub cpe: String, + pub parent: String, +} + +impl Default for UispIntegration { + fn default() -> Self { + UispIntegration { + enable_uisp: false, + token: "".to_string(), + url: "".to_string(), + site: "".to_string(), + strategy: "".to_string(), + suspended_strategy: "".to_string(), + airmax_capacity: 0.0, + ltu_capacity: 0.0, + exclude_sites: vec![], + ipv6_with_mikrotik: false, + bandwidth_overhead_factor: 1.0, + commit_bandwidth_multiplier: 1.0, + exception_cpes: vec![], + } + } +} \ No newline at end of file From 720c31e2d811d900a7eccd66bd6294056ed3650b Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 7 Dec 2023 11:26:13 -0600 Subject: [PATCH 09/41] Add (untested) logic for renaming old config files and saving the new one. --- src/rust/lqos_config/src/etc/migration.rs | 24 ++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs index 4975eb9d..14cf1544 100644 --- a/src/rust/lqos_config/src/etc/migration.rs +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use super::{ python_migration::{PythonMigration, PythonMigrationError}, v15::{Config, SingleInterfaceConfig, BridgeConfig}, @@ -41,18 +43,34 @@ pub fn migrate_if_needed() -> Result<(), MigrationError> { } } else { log::info!("No version found in configuration file, assuming 1.4x and migration is needed"); - migrate_14_to_15()?; + let new_config = migrate_14_to_15()?; + // Backup the old configuration + std::fs::rename("/etc/lqos.conf", "/etc/lqos.conf.backup14") + .map_err(|e| MigrationError::ReadError(e))?; + + // Rename the old Python configuration + let from = Path::new(new_config.lqos_directory.as_str()) + .join("ispConfig.py"); + let to = Path::new(new_config.lqos_directory.as_str()) + .join("ispConfig.py.backup14"); + + std::fs::rename(from, to) + .map_err(|e| MigrationError::ReadError(e))?; + + // Save the configuration + let raw = toml::to_string_pretty(&new_config).unwrap(); + std::fs::write("/etc/lqos.conf", raw).map_err(|e| MigrationError::ReadError(e))?; } Ok(()) } -fn migrate_14_to_15() -> Result<(), MigrationError> { +fn migrate_14_to_15() -> Result { // Load the 1.4 config file let old_config = EtcLqos::load().map_err(|e| MigrationError::LoadError(e))?; let python_config = PythonMigration::load().map_err(|e| MigrationError::PythonLoadError(e))?; let new_config = do_migration_14_to_15(&old_config, &python_config)?; - Ok(()) + Ok(new_config) } fn do_migration_14_to_15( From ccaa382202ec0900b94b78d5b15ea86c63ffd902 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 8 Dec 2023 10:38:01 -0600 Subject: [PATCH 10/41] It's a mess, but hit the checkpoint of 'it compiles' with lqosd and node_manager attached to the new configuration file format. There is a lot more to do. --- src/rust/Cargo.lock | 7 +- src/rust/lqos_config/Cargo.toml | 1 + src/rust/lqos_config/src/authentication.rs | 2 +- .../lqos_config/src/etc/etclqos_migration.rs | 4 +- src/rust/lqos_config/src/etc/migration.rs | 67 ++- src/rust/lqos_config/src/etc/mod.rs | 87 ++- .../lqos_config/src/etc/python_migration.rs | 10 +- .../src/etc/v15/anonymous_stats.rs | 2 +- src/rust/lqos_config/src/etc/v15/bridge.rs | 8 +- src/rust/lqos_config/src/etc/v15/example.toml | 2 + .../src/etc/v15/integration_common.rs | 2 +- src/rust/lqos_config/src/etc/v15/ip_ranges.rs | 2 +- src/rust/lqos_config/src/etc/v15/mod.rs | 3 +- src/rust/lqos_config/src/etc/v15/queues.rs | 2 +- .../src/etc/v15/spylnx_integration.rs | 2 +- .../lqos_config/src/etc/v15/top_config.rs | 48 +- src/rust/lqos_config/src/etc/v15/tuning.rs | 2 +- .../src/etc/v15/uisp_integration.rs | 4 +- src/rust/lqos_config/src/lib.rs | 4 +- src/rust/lqos_config/src/libre_qos_config.rs | 541 ------------------ src/rust/lqos_heimdall/src/timeline.rs | 5 +- .../lqos_node_manager/src/config_control.rs | 26 +- src/rust/lqos_node_manager/src/main.rs | 4 +- src/rust/lqos_node_manager/src/toasts.rs | 33 +- .../src/queue_structure/queue_network.rs | 3 +- .../lqos_queue_tracker/src/tracking/mod.rs | 15 +- src/rust/lqos_sys/src/bifrost_maps.rs | 81 ++- src/rust/lqos_sys/src/lqos_kernel.rs | 57 +- src/rust/lqosd/src/anonymous_usage/mod.rs | 51 +- src/rust/lqosd/src/main.rs | 25 +- src/rust/lqosd/src/tuning/mod.rs | 33 +- .../src/collector/collection_manager.rs | 70 ++- .../queue_structure/queue_network.rs | 4 +- .../src/collector/quick_drops/stats_diff.rs | 9 +- src/rust/lts_client/src/collector/uisp_ext.rs | 3 +- .../submission_queue/comm_channel/encode.rs | 20 +- .../src/submission_queue/comm_channel/keys.rs | 12 +- .../src/submission_queue/comm_channel/mod.rs | 19 +- .../src/submission_queue/licensing.rs | 29 +- src/rust/uisp/src/lib.rs | 42 +- 40 files changed, 499 insertions(+), 842 deletions(-) delete mode 100644 src/rust/lqos_config/src/libre_qos_config.rs diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index e2b57d46..75506ffa 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -1530,13 +1530,14 @@ dependencies = [ "ip_network", "ip_network_table", "log", + "once_cell", "pyo3", "serde", "serde_json", "sha2", "thiserror", "toml 0.8.8", - "toml_edit 0.19.14", + "toml_edit 0.21.0", "uuid", ] @@ -2001,9 +2002,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" diff --git a/src/rust/lqos_config/Cargo.toml b/src/rust/lqos_config/Cargo.toml index 9f479b1f..a8f552fd 100644 --- a/src/rust/lqos_config/Cargo.toml +++ b/src/rust/lqos_config/Cargo.toml @@ -18,3 +18,4 @@ log = "0" dashmap = "5" pyo3 = "0.20" toml = "0.8.8" +once_cell = "1.19.0" diff --git a/src/rust/lqos_config/src/authentication.rs b/src/rust/lqos_config/src/authentication.rs index ea9d3fb8..b85ccdca 100644 --- a/src/rust/lqos_config/src/authentication.rs +++ b/src/rust/lqos_config/src/authentication.rs @@ -59,7 +59,7 @@ pub struct WebUsers { impl WebUsers { fn path() -> Result { - let base_path = crate::EtcLqos::load() + let base_path = crate::load_config() .map_err(|_| AuthenticationError::UnableToLoadEtcLqos)? .lqos_directory; let filename = Path::new(&base_path).join("lqusers.toml"); diff --git a/src/rust/lqos_config/src/etc/etclqos_migration.rs b/src/rust/lqos_config/src/etc/etclqos_migration.rs index ea180561..eb416a30 100644 --- a/src/rust/lqos_config/src/etc/etclqos_migration.rs +++ b/src/rust/lqos_config/src/etc/etclqos_migration.rs @@ -232,14 +232,14 @@ pub fn enable_long_term_stats(license_key: String) { match cfg { Ok(cfg) => { // Now we enable LTS if its not present - if let Ok(isp_config) = crate::LibreQoSConfig::load() { + if let Ok(isp_config) = crate::load_config() { if cfg.long_term_stats.is_none() { let mut new_section = toml_edit::table(); new_section["gather_stats"] = value(true); new_section["collation_period_seconds"] = value(60); new_section["license_key"] = value(license_key); - if isp_config.automatic_import_uisp { + if isp_config.uisp_integration.enable_uisp { new_section["uisp_reporting_interval_seconds"] = value(300); } config_doc["long_term_stats"] = new_section; diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs index 14cf1544..3631823e 100644 --- a/src/rust/lqos_config/src/etc/migration.rs +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -1,12 +1,10 @@ +/// Provides support for migration from older versions of the configuration file. use std::path::Path; - use super::{ python_migration::{PythonMigration, PythonMigrationError}, - v15::{Config, SingleInterfaceConfig, BridgeConfig}, - EtcLqosError, + v15::{BridgeConfig, Config, SingleInterfaceConfig}, + EtcLqosError, EtcLqos, }; -use crate::EtcLqos; -/// Provides support for migration from older versions of the configuration file. use thiserror::Error; use toml_edit::Document; @@ -49,14 +47,11 @@ pub fn migrate_if_needed() -> Result<(), MigrationError> { .map_err(|e| MigrationError::ReadError(e))?; // Rename the old Python configuration - let from = Path::new(new_config.lqos_directory.as_str()) - .join("ispConfig.py"); - let to = Path::new(new_config.lqos_directory.as_str()) - .join("ispConfig.py.backup14"); - - std::fs::rename(from, to) - .map_err(|e| MigrationError::ReadError(e))?; - + let from = Path::new(new_config.lqos_directory.as_str()).join("ispConfig.py"); + let to = Path::new(new_config.lqos_directory.as_str()).join("ispConfig.py.backup14"); + + std::fs::rename(from, to).map_err(|e| MigrationError::ReadError(e))?; + // Save the configuration let raw = toml::to_string_pretty(&new_config).unwrap(); std::fs::write("/etc/lqos.conf", raw).map_err(|e| MigrationError::ReadError(e))?; @@ -89,6 +84,7 @@ fn do_migration_14_to_15( migrate_integration_common(python_config, &mut new_config)?; migrate_spylnx(python_config, &mut new_config)?; migrate_uisp(python_config, &mut new_config)?; + migrate_queues( python_config, &mut new_config)?; new_config.validate().unwrap(); // Left as an upwrap because this should *never* happen Ok(new_config) @@ -97,6 +93,7 @@ fn do_migration_14_to_15( fn migrate_top_level(old_config: &EtcLqos, new_config: &mut Config) -> Result<(), MigrationError> { new_config.version = "1.5".to_string(); new_config.lqos_directory = old_config.lqos_directory.clone(); + new_config.packet_capture_time = old_config.packet_capture_time.unwrap_or(10); if let Some(node_id) = &old_config.node_id { new_config.node_id = node_id.clone(); } else { @@ -148,8 +145,8 @@ fn migrate_bridge( new_config.bridge = None; new_config.single_interface = Some(SingleInterfaceConfig { interface: python_config.interface_a.clone(), - internet_vlan: python_config.stick_vlan_b as u16, - network_vlan: python_config.stick_vlan_b as u16, + internet_vlan: python_config.stick_vlan_b, + network_vlan: python_config.stick_vlan_b, }); } else { new_config.single_interface = None; @@ -158,16 +155,19 @@ fn migrate_bridge( to_internet: python_config.interface_b.clone(), to_network: python_config.interface_a.clone(), }); - } Ok(()) } -fn migrate_queues(old_config: &EtcLqos, python_config: &PythonMigration, new_config: &mut Config) -> Result<(), MigrationError> { +fn migrate_queues( + python_config: &PythonMigration, + new_config: &mut Config, +) -> Result<(), MigrationError> { new_config.queues.default_sqm = python_config.sqm.clone(); new_config.queues.monitor_only = python_config.monitor_only_mode; new_config.queues.uplink_bandwidth_mbps = python_config.upstream_bandwidth_capacity_upload_mbps; - new_config.queues.downlink_bandwidth_mbps = python_config.upstream_bandwidth_capacity_download_mbps; + new_config.queues.downlink_bandwidth_mbps = + python_config.upstream_bandwidth_capacity_download_mbps; new_config.queues.generated_pn_upload_mbps = python_config.generated_pn_upload_mbps; new_config.queues.generated_pn_download_mbps = python_config.generated_pn_download_mbps; new_config.queues.dry_run = !python_config.enable_actual_shell_commands; @@ -186,26 +186,37 @@ fn migrate_lts(old_config: &EtcLqos, new_config: &mut Config) -> Result<(), Migr new_config.long_term_stats.gather_stats = lts.gather_stats; new_config.long_term_stats.collation_period_seconds = lts.collation_period_seconds; new_config.long_term_stats.license_key = lts.license_key.clone(); - new_config.long_term_stats.uisp_reporting_interval_seconds = lts.uisp_reporting_interval_seconds; + new_config.long_term_stats.uisp_reporting_interval_seconds = + lts.uisp_reporting_interval_seconds; } else { new_config.long_term_stats = super::v15::LongTermStats::default(); } Ok(()) } -fn migrate_ip_ranges(python_config: &PythonMigration, new_config: &mut Config) -> Result<(), MigrationError> { +fn migrate_ip_ranges( + python_config: &PythonMigration, + new_config: &mut Config, +) -> Result<(), MigrationError> { new_config.ip_ranges.ignore_subnets = python_config.ignore_subnets.clone(); - new_config.ip_ranges.allow_subnets = python_config.allowed_subnets.clone(); + new_config.ip_ranges.allow_subnets = python_config.allowed_subnets.clone(); Ok(()) } -fn migrate_integration_common(python_config: &PythonMigration, new_config: &mut Config) -> Result<(), MigrationError> { +fn migrate_integration_common( + python_config: &PythonMigration, + new_config: &mut Config, +) -> Result<(), MigrationError> { new_config.integration_common.circuit_name_as_address = python_config.circuit_name_use_address; - new_config.integration_common.always_overwrite_network_json = python_config.overwrite_network_json_always; + new_config.integration_common.always_overwrite_network_json = + python_config.overwrite_network_json_always; Ok(()) } -fn migrate_spylnx(python_config: &PythonMigration, new_config: &mut Config) -> Result<(), MigrationError> { +fn migrate_spylnx( + python_config: &PythonMigration, + new_config: &mut Config, +) -> Result<(), MigrationError> { new_config.spylnx_integration.enable_spylnx = python_config.automatic_import_splynx; new_config.spylnx_integration.api_key = python_config.splynx_api_key.clone(); new_config.spylnx_integration.api_secret = python_config.spylnx_api_secret.clone(); @@ -213,7 +224,10 @@ fn migrate_spylnx(python_config: &PythonMigration, new_config: &mut Config) -> R Ok(()) } -fn migrate_uisp(python_config: &PythonMigration, new_config: &mut Config) -> Result<(), MigrationError> { +fn migrate_uisp( + python_config: &PythonMigration, + new_config: &mut Config, +) -> Result<(), MigrationError> { new_config.uisp_integration.enable_uisp = python_config.automatic_import_uisp; new_config.uisp_integration.token = python_config.uisp_auth_token.clone(); new_config.uisp_integration.url = python_config.uisp_base_url.clone(); @@ -225,7 +239,8 @@ fn migrate_uisp(python_config: &PythonMigration, new_config: &mut Config) -> Res new_config.uisp_integration.exclude_sites = python_config.exclude_sites.clone(); new_config.uisp_integration.ipv6_with_mikrotik = python_config.find_ipv6_using_mikrotik; new_config.uisp_integration.bandwidth_overhead_factor = python_config.bandwidth_overhead_factor; - new_config.uisp_integration.commit_bandwidth_multiplier = python_config.committed_bandwidth_multiplier; + new_config.uisp_integration.commit_bandwidth_multiplier = + python_config.committed_bandwidth_multiplier; // TODO: ExceptionCPEs is going to require some real work Ok(()) } diff --git a/src/rust/lqos_config/src/etc/mod.rs b/src/rust/lqos_config/src/etc/mod.rs index a17fa99f..7aefa5b4 100644 --- a/src/rust/lqos_config/src/etc/mod.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -1,12 +1,91 @@ //! Manages the `/etc/lqos.conf` file. mod etclqos_migration; +use self::migration::migrate_if_needed; +pub use self::v15::Config; pub use etclqos_migration::*; -mod v15; -mod python_migration; +use std::sync::Mutex; +use thiserror::Error; mod migration; - +mod python_migration; #[cfg(test)] pub mod test_data; +mod v15; +pub use v15::Tunables; -pub use migration::migrate_if_needed; \ No newline at end of file +static CONFIG: Mutex> = Mutex::new(None); + +/// Load the configuration from `/etc/lqos.conf`. +pub fn load_config() -> Result { + log::info!("Loading configuration file /etc/lqos.conf"); + let mut lock = CONFIG.lock().unwrap(); + if lock.is_none() { + migrate_if_needed().map_err(|e| { + log::error!("Unable to migrate configuration: {:?}", e); + LibreQoSConfigError::FileNotFoud + })?; + + let file_result = std::fs::read_to_string("/etc/lqos.conf"); + if file_result.is_err() { + log::error!("Unable to open /etc/lqos.conf"); + return Err(LibreQoSConfigError::FileNotFoud); + } + let raw = file_result.unwrap(); + + let config_result = Config::load_from_string(&raw); + if config_result.is_err() { + log::error!("Unable to parse /etc/lqos.conf"); + log::error!("Error: {:?}", config_result); + return Err(LibreQoSConfigError::ParseError(format!( + "{:?}", + config_result + ))); + } + *lock = Some(config_result.unwrap()); + } + + Ok(lock.as_ref().unwrap().clone()) +} + +pub fn enable_long_term_stats(license_key: String) -> Result<(), LibreQoSConfigError> { + let mut config = load_config()?; + let mut lock = CONFIG.lock().unwrap(); + + config.long_term_stats.gather_stats = true; + config.long_term_stats.collation_period_seconds = 60; + config.long_term_stats.license_key = Some(license_key); + if config.uisp_integration.enable_uisp { + config.long_term_stats.uisp_reporting_interval_seconds = Some(300); + } + + // Write the file + let raw = toml::to_string_pretty(&config).unwrap(); + std::fs::write("/etc/lqos.conf", raw).map_err(|_| LibreQoSConfigError::CannotWrite)?; + + // Write the lock + *lock = Some(config); + + Ok(()) +} + +#[derive(Debug, Error)] +pub enum LibreQoSConfigError { + #[error("Unable to read /etc/lqos.conf. See other errors for details.")] + CannotOpenEtcLqos, + #[error("Unable to locate (path to LibreQoS)/ispConfig.py. Check your path and that you have configured it.")] + FileNotFoud, + #[error("Unable to read the contents of ispConfig.py. Check file permissions.")] + CannotReadFile, + #[error("Unable to parse ispConfig.py")] + ParseError(String), + #[error("Could not backup configuration")] + CannotCopy, + #[error("Could not remove the previous configuration.")] + CannotRemove, + #[error("Could not open ispConfig.py for write")] + CannotOpenForWrite, + #[error("Unable to write to ispConfig.py")] + CannotWrite, + #[error("Unable to read IP")] + CannotReadIP, +} diff --git a/src/rust/lqos_config/src/etc/python_migration.rs b/src/rust/lqos_config/src/etc/python_migration.rs index e71fbf4b..1f3a3568 100644 --- a/src/rust/lqos_config/src/etc/python_migration.rs +++ b/src/rust/lqos_config/src/etc/python_migration.rs @@ -2,11 +2,12 @@ //! provide conversion services for the new, unified configuration target //! for version 1.5. -use crate::EtcLqos; +use super::EtcLqos; use pyo3::{prepare_freethreaded_python, Python}; use std::{ + collections::HashMap, fs::read_to_string, - path::{Path, PathBuf}, collections::HashMap, + path::{Path, PathBuf}, }; use thiserror::Error; @@ -96,7 +97,6 @@ pub struct PythonMigration { pub api_password: String, pub api_host_ip: String, pub api_host_port: u32, - // TODO: httpRestIntegrationConfig } @@ -194,8 +194,8 @@ impl PythonMigration { #[cfg(test)] mod test { - use super::*; use super::super::test_data::*; + use super::*; #[test] fn test_parsing_the_default() { @@ -212,4 +212,4 @@ mod test { }); assert!(worked) } -} \ No newline at end of file +} diff --git a/src/rust/lqos_config/src/etc/v15/anonymous_stats.rs b/src/rust/lqos_config/src/etc/v15/anonymous_stats.rs index e2e86359..2fe73e90 100644 --- a/src/rust/lqos_config/src/etc/v15/anonymous_stats.rs +++ b/src/rust/lqos_config/src/etc/v15/anonymous_stats.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug)] pub struct UsageStats { /// Are we allowed to send stats at all? pub send_anonymous: bool, diff --git a/src/rust/lqos_config/src/etc/v15/bridge.rs b/src/rust/lqos_config/src/etc/v15/bridge.rs index 2e5e4f48..89618728 100644 --- a/src/rust/lqos_config/src/etc/v15/bridge.rs +++ b/src/rust/lqos_config/src/etc/v15/bridge.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; /// Represents a two-interface bridge configuration. -#[derive(Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug)] pub struct BridgeConfig { /// Use the XDP-accelerated bridge? pub use_xdp_bridge: bool, @@ -27,16 +27,16 @@ impl Default for BridgeConfig { } } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug)] pub struct SingleInterfaceConfig { /// The name of the interface pub interface: String, /// The VLAN ID facing the Internet - pub internet_vlan: u16, + pub internet_vlan: u32, /// The VLAN ID facing the LAN - pub network_vlan: u16, + pub network_vlan: u32, } impl Default for SingleInterfaceConfig { diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index 70d58fd2..79e48a5f 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -2,6 +2,8 @@ version = "1.5" lqos_directory = "/opt/libreqos/src" node_id = "0000-0000-0000" node_name = "Example Node" +packet_capture_time = 10 +queue_check_period_ms = 1000 [usage_stats] send_anonymous = true diff --git a/src/rust/lqos_config/src/etc/v15/integration_common.rs b/src/rust/lqos_config/src/etc/v15/integration_common.rs index 28104c14..0e64f881 100644 --- a/src/rust/lqos_config/src/etc/v15/integration_common.rs +++ b/src/rust/lqos_config/src/etc/v15/integration_common.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug)] pub struct IntegrationConfig { /// Replace names with addresses? pub circuit_name_as_address: bool, diff --git a/src/rust/lqos_config/src/etc/v15/ip_ranges.rs b/src/rust/lqos_config/src/etc/v15/ip_ranges.rs index e5d357e1..a8a681c5 100644 --- a/src/rust/lqos_config/src/etc/v15/ip_ranges.rs +++ b/src/rust/lqos_config/src/etc/v15/ip_ranges.rs @@ -1,6 +1,6 @@ use serde::{Serialize, Deserialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug)] pub struct IpRanges { pub ignore_subnets: Vec, pub allow_subnets: Vec, diff --git a/src/rust/lqos_config/src/etc/v15/mod.rs b/src/rust/lqos_config/src/etc/v15/mod.rs index 737f082d..a6e08d90 100644 --- a/src/rust/lqos_config/src/etc/v15/mod.rs +++ b/src/rust/lqos_config/src/etc/v15/mod.rs @@ -12,4 +12,5 @@ mod ip_ranges; mod spylnx_integration; mod uisp_integration; pub use bridge::*; -pub use long_term_stats::LongTermStats; \ No newline at end of file +pub use long_term_stats::LongTermStats; +pub use tuning::Tunables; \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/queues.rs b/src/rust/lqos_config/src/etc/v15/queues.rs index 76aac4d0..e3560453 100644 --- a/src/rust/lqos_config/src/etc/v15/queues.rs +++ b/src/rust/lqos_config/src/etc/v15/queues.rs @@ -2,7 +2,7 @@ use serde::{Serialize, Deserialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug)] pub struct QueueConfig { /// Which SQM to use by default pub default_sqm: String, diff --git a/src/rust/lqos_config/src/etc/v15/spylnx_integration.rs b/src/rust/lqos_config/src/etc/v15/spylnx_integration.rs index bb1f9048..00c14e14 100644 --- a/src/rust/lqos_config/src/etc/v15/spylnx_integration.rs +++ b/src/rust/lqos_config/src/etc/v15/spylnx_integration.rs @@ -1,6 +1,6 @@ use serde::{Serialize, Deserialize}; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct SplynxIntegration { pub enable_spylnx: bool, pub api_key: String, diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index 1fbe6974..120b7ccd 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -7,7 +7,7 @@ use sha2::digest::Update; use sha2::Digest; use uuid::Uuid; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct Config { /// Version number for the configuration file. /// This will be set to "1.5". Versioning will make @@ -21,7 +21,13 @@ pub struct Config { pub node_id: String, /// Node name - human-readable name for this shaper. - pub node_name: String, + pub node_name: String, + + /// Packet capture time + pub packet_capture_time: usize, + + /// Queue refresh interval + pub queue_check_period_ms: u64, /// Anonymous usage statistics pub usage_stats: UsageStats, @@ -86,7 +92,7 @@ impl Config { Ok(()) } - fn load_from_string(s: &str) -> Result { + pub fn load_from_string(s: &str) -> Result { let config: Config = toml::from_str(s).map_err(|e| format!("Error parsing config: {}", e))?; config.validate()?; Ok(config) @@ -110,6 +116,42 @@ impl Default for Config { integration_common: super::integration_common::IntegrationConfig::default(), spylnx_integration: super::spylnx_integration::SplynxIntegration::default(), uisp_integration: super::uisp_integration::UispIntegration::default(), + packet_capture_time: 10, + queue_check_period_ms: 1000, + } + } +} + +impl Config { + pub fn internet_interface(&self) -> String { + if let Some(bridge) = &self.bridge { + bridge.to_internet.clone() + } else if let Some(single_interface) = &self.single_interface { + single_interface.interface.clone() + } else { + panic!("No internet interface configured") + } + } + + pub fn isp_interface(&self) -> String { + if let Some(bridge) = &self.bridge { + bridge.to_network.clone() + } else if let Some(single_interface) = &self.single_interface { + single_interface.interface.clone() + } else { + panic!("No ISP interface configured") + } + } + + pub fn on_a_stick_mode(&self) -> bool { + self.bridge.is_none() + } + + pub fn stick_vlans(&self) -> (u32, u32) { + if let Some(stick) = &self.single_interface { + (stick.network_vlan, stick.internet_vlan) + } else { + (0, 0) } } } diff --git a/src/rust/lqos_config/src/etc/v15/tuning.rs b/src/rust/lqos_config/src/etc/v15/tuning.rs index 9c0d8020..9ca9bb00 100644 --- a/src/rust/lqos_config/src/etc/v15/tuning.rs +++ b/src/rust/lqos_config/src/etc/v15/tuning.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; /// Represents a set of `sysctl` and `ethtool` tweaks that may be /// applied (in place of the previous version's offload service) -#[derive(Serialize, Deserialize, Debug)] +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] pub struct Tunables { /// Should the `irq_balance` system service be stopped? pub stop_irq_balance: bool, diff --git a/src/rust/lqos_config/src/etc/v15/uisp_integration.rs b/src/rust/lqos_config/src/etc/v15/uisp_integration.rs index 54647689..501a427a 100644 --- a/src/rust/lqos_config/src/etc/v15/uisp_integration.rs +++ b/src/rust/lqos_config/src/etc/v15/uisp_integration.rs @@ -1,6 +1,6 @@ use serde::{Serialize, Deserialize}; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct UispIntegration { pub enable_uisp: bool, pub token: String, @@ -17,7 +17,7 @@ pub struct UispIntegration { pub exception_cpes: Vec, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct ExceptionCpe { pub cpe: String, pub parent: String, diff --git a/src/rust/lqos_config/src/lib.rs b/src/rust/lqos_config/src/lib.rs index 6f05cecb..947cc737 100644 --- a/src/rust/lqos_config/src/lib.rs +++ b/src/rust/lqos_config/src/lib.rs @@ -8,14 +8,12 @@ #![warn(missing_docs)] mod authentication; mod etc; -mod libre_qos_config; mod network_json; mod program_control; mod shaped_devices; pub use authentication::{UserRole, WebUsers}; -pub use etc::{BridgeConfig, BridgeInterface, BridgeVlan, EtcLqos, Tunables, enable_long_term_stats}; -pub use libre_qos_config::LibreQoSConfig; +pub use etc::{load_config, Config, enable_long_term_stats, Tunables}; pub use network_json::{NetworkJson, NetworkJsonNode, NetworkJsonTransport}; pub use program_control::load_libreqos; pub use shaped_devices::{ConfigShapedDevices, ShapedDevice}; diff --git a/src/rust/lqos_config/src/libre_qos_config.rs b/src/rust/lqos_config/src/libre_qos_config.rs deleted file mode 100644 index 9e774299..00000000 --- a/src/rust/lqos_config/src/libre_qos_config.rs +++ /dev/null @@ -1,541 +0,0 @@ -//! `ispConfig.py` is part of the Python side of LibreQoS. This module -//! reads, writes and maps values from the Python file. - -use crate::etc; -use ip_network::IpNetwork; -use log::error; -use serde::{Deserialize, Serialize}; -use std::{ - fs::{self, read_to_string, remove_file, OpenOptions}, - io::Write, - net::IpAddr, - path::{Path, PathBuf}, -}; -use thiserror::Error; - -/// Represents the contents of an `ispConfig.py` file. -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct LibreQoSConfig { - /// Interface facing the Internet - pub internet_interface: String, - - /// Interface facing the ISP Core Router - pub isp_interface: String, - - /// Are we in "on a stick" (single interface) mode? - pub on_a_stick_mode: bool, - - /// If we are, which VLAN represents which direction? - /// In (internet, ISP) order. - pub stick_vlans: (u16, u16), - - /// The value of the SQM field from `ispConfig.py` - pub sqm: String, - - /// Are we in monitor-only mode (not shaping)? - pub monitor_mode: bool, - - /// Total available download (in Mbps) - pub total_download_mbps: u32, - - /// Total available upload (in Mbps) - pub total_upload_mbps: u32, - - /// If a node is generated, how much download (Mbps) should it offer? - pub generated_download_mbps: u32, - - /// If a node is generated, how much upload (Mbps) should it offer? - pub generated_upload_mbps: u32, - - /// Should the Python queue builder use the bin packing strategy to - /// try to optimize CPU assignment? - pub use_binpacking: bool, - - /// Should the Python program use actual shell commands (and execute) - /// them? - pub enable_shell_commands: bool, - - /// Should every issued command be prefixed with `sudo`? - pub run_as_sudo: bool, - - /// WARNING: generally don't touch this. - pub override_queue_count: u32, - - /// Is UISP integration enabled? - pub automatic_import_uisp: bool, - - /// UISP Authentication Token - pub uisp_auth_token: String, - - /// UISP Base URL (e.g. billing.myisp.com) - pub uisp_base_url: String, - - /// Root site for UISP tree generation - pub uisp_root_site: String, - - /// Circuit names use address? - pub circuit_name_use_address: bool, - - /// UISP Strategy - pub uisp_strategy: String, - - /// UISP Suspension Strategy - pub uisp_suspended_strategy: String, - - /// Bandwidth Overhead Factor - pub bandwidth_overhead_factor: f32, - - /// Subnets allowed to be included in device lists - pub allowed_subnets: String, - - /// Subnets explicitly ignored from device lists - pub ignored_subnets: String, - - /// Overwrite network.json even if it exists - pub overwrite_network_json_always: bool, -} - -impl LibreQoSConfig { - /// Does the ispConfig.py file exist? - pub fn config_exists() -> bool { - if let Ok(cfg) = etc::EtcLqos::load() { - let base_path = Path::new(&cfg.lqos_directory); - let final_path = base_path.join("ispConfig.py"); - final_path.exists() - } else { - false - } - } - - /// Loads `ispConfig.py` into a management object. - pub fn load() -> Result { - if let Ok(cfg) = etc::EtcLqos::load() { - let base_path = Path::new(&cfg.lqos_directory); - let final_path = base_path.join("ispConfig.py"); - Ok(Self::load_from_path(&final_path)?) - } else { - error!("Unable to read LibreQoS config from /etc/lqos.conf"); - Err(LibreQoSConfigError::CannotOpenEtcLqos) - } - } - - fn load_from_path(path: &PathBuf) -> Result { - let path = Path::new(path); - if !path.exists() { - error!("Unable to find ispConfig.py"); - return Err(LibreQoSConfigError::FileNotFoud); - } - - // Read the config - let mut result = Self { - internet_interface: String::new(), - isp_interface: String::new(), - on_a_stick_mode: false, - stick_vlans: (0, 0), - sqm: String::new(), - monitor_mode: false, - total_download_mbps: 0, - total_upload_mbps: 0, - generated_download_mbps: 0, - generated_upload_mbps: 0, - use_binpacking: false, - enable_shell_commands: true, - run_as_sudo: false, - override_queue_count: 0, - automatic_import_uisp: false, - uisp_auth_token: "".to_string(), - uisp_base_url: "".to_string(), - uisp_root_site: "".to_string(), - circuit_name_use_address: false, - uisp_strategy: "".to_string(), - uisp_suspended_strategy: "".to_string(), - bandwidth_overhead_factor: 1.0, - allowed_subnets: "".to_string(), - ignored_subnets: "".to_string(), - overwrite_network_json_always: false, - }; - result.parse_isp_config(path)?; - Ok(result) - } - - fn parse_isp_config( - &mut self, - path: &Path, - ) -> Result<(), LibreQoSConfigError> { - let read_result = fs::read_to_string(path); - match read_result { - Err(e) => { - error!("Unable to read contents of ispConfig.py. Check permissions."); - error!("{:?}", e); - return Err(LibreQoSConfigError::CannotReadFile); - } - Ok(content) => { - for line in content.split('\n') { - if line.starts_with("interfaceA") { - self.isp_interface = split_at_equals(line); - } - if line.starts_with("interfaceB") { - self.internet_interface = split_at_equals(line); - } - if line.starts_with("OnAStick") { - let mode = split_at_equals(line); - if mode == "True" { - self.on_a_stick_mode = true; - } - } - if line.starts_with("StickVlanA") { - let vlan_string = split_at_equals(line); - if let Ok(vlan) = vlan_string.parse() { - self.stick_vlans.0 = vlan; - } else { - error!( - "Unable to parse contents of StickVlanA from ispConfig.py" - ); - error!("{line}"); - return Err(LibreQoSConfigError::ParseError(line.to_string())); - } - } - if line.starts_with("StickVlanB") { - let vlan_string = split_at_equals(line); - if let Ok(vlan) = vlan_string.parse() { - self.stick_vlans.1 = vlan; - } else { - error!( - "Unable to parse contents of StickVlanB from ispConfig.py" - ); - error!("{line}"); - return Err(LibreQoSConfigError::ParseError(line.to_string())); - } - } - if line.starts_with("sqm") { - self.sqm = split_at_equals(line); - } - if line.starts_with("upstreamBandwidthCapacityDownloadMbps") { - if let Ok(mbps) = split_at_equals(line).parse() { - self.total_download_mbps = mbps; - } else { - error!("Unable to parse contents of upstreamBandwidthCapacityDownloadMbps from ispConfig.py"); - error!("{line}"); - return Err(LibreQoSConfigError::ParseError(line.to_string())); - } - } - if line.starts_with("upstreamBandwidthCapacityUploadMbps") { - if let Ok(mbps) = split_at_equals(line).parse() { - self.total_upload_mbps = mbps; - } else { - error!("Unable to parse contents of upstreamBandwidthCapacityUploadMbps from ispConfig.py"); - error!("{line}"); - return Err(LibreQoSConfigError::ParseError(line.to_string())); - } - } - if line.starts_with("monitorOnlyMode ") { - let mode = split_at_equals(line); - if mode == "True" { - self.monitor_mode = true; - } - } - if line.starts_with("generatedPNDownloadMbps") { - if let Ok(mbps) = split_at_equals(line).parse() { - self.generated_download_mbps = mbps; - } else { - error!("Unable to parse contents of generatedPNDownloadMbps from ispConfig.py"); - error!("{line}"); - return Err(LibreQoSConfigError::ParseError(line.to_string())); - } - } - if line.starts_with("generatedPNUploadMbps") { - if let Ok(mbps) = split_at_equals(line).parse() { - self.generated_upload_mbps = mbps; - } else { - error!("Unable to parse contents of generatedPNUploadMbps from ispConfig.py"); - error!("{line}"); - return Err(LibreQoSConfigError::ParseError(line.to_string())); - } - } - if line.starts_with("useBinPackingToBalanceCPU") { - let mode = split_at_equals(line); - if mode == "True" { - self.use_binpacking = true; - } - } - if line.starts_with("enableActualShellCommands") { - let mode = split_at_equals(line); - if mode == "True" { - self.enable_shell_commands = true; - } - } - if line.starts_with("runShellCommandsAsSudo") { - let mode = split_at_equals(line); - if mode == "True" { - self.run_as_sudo = true; - } - } - if line.starts_with("queuesAvailableOverride") { - self.override_queue_count = - split_at_equals(line).parse().unwrap_or(0); - } - if line.starts_with("automaticImportUISP") { - let mode = split_at_equals(line); - if mode == "True" { - self.automatic_import_uisp = true; - } - } - if line.starts_with("uispAuthToken") { - self.uisp_auth_token = split_at_equals(line); - } - if line.starts_with("UISPbaseURL") { - self.uisp_base_url = split_at_equals(line); - } - if line.starts_with("uispSite") { - self.uisp_root_site = split_at_equals(line); - } - if line.starts_with("circuitNameUseAddress") { - let mode = split_at_equals(line); - if mode == "True" { - self.circuit_name_use_address = true; - } - } - if line.starts_with("uispStrategy") { - self.uisp_strategy = split_at_equals(line); - } - if line.starts_with("uispSuspendedStrategy") { - self.uisp_suspended_strategy = split_at_equals(line); - } - if line.starts_with("bandwidthOverheadFactor") { - self.bandwidth_overhead_factor = - split_at_equals(line).parse().unwrap_or(1.0); - } - if line.starts_with("allowedSubnets") { - self.allowed_subnets = split_at_equals(line); - } - if line.starts_with("ignoreSubnets") { - self.ignored_subnets = split_at_equals(line); - } - if line.starts_with("overwriteNetworkJSONalways") { - let mode = split_at_equals(line); - if mode == "True" { - self.overwrite_network_json_always = true; - } - } - } - } - } - Ok(()) - } - - /// Saves the current values to `ispConfig.py` and store the - /// previous settings in `ispConfig.py.backup`. - /// - pub fn save(&self) -> Result<(), LibreQoSConfigError> { - // Find the config - let cfg = etc::EtcLqos::load().map_err(|_| { - crate::libre_qos_config::LibreQoSConfigError::CannotOpenEtcLqos - })?; - let base_path = Path::new(&cfg.lqos_directory); - let final_path = base_path.join("ispConfig.py"); - let backup_path = base_path.join("ispConfig.py.backup"); - if std::fs::copy(&final_path, &backup_path).is_err() { - error!( - "Unable to copy {} to {}.", - final_path.display(), - backup_path.display() - ); - return Err(LibreQoSConfigError::CannotCopy); - } - - // Load existing file - let original = read_to_string(&final_path); - if original.is_err() { - error!("Unable to read ispConfig.py"); - return Err(LibreQoSConfigError::CannotReadFile); - } - let original = original.unwrap(); - - // Temporary - //let final_path = base_path.join("ispConfig.py.test"); - - // Update config entries line by line - let mut config = String::new(); - for line in original.split('\n') { - let mut line = line.to_string(); - if line.starts_with("interfaceA") { - line = format!("interfaceA = '{}'", self.isp_interface); - } - if line.starts_with("interfaceB") { - line = format!("interfaceB = '{}'", self.internet_interface); - } - if line.starts_with("OnAStick") { - line = format!( - "OnAStick = {}", - if self.on_a_stick_mode { "True" } else { "False" } - ); - } - if line.starts_with("StickVlanA") { - line = format!("StickVlanA = {}", self.stick_vlans.0); - } - if line.starts_with("StickVlanB") { - line = format!("StickVlanB = {}", self.stick_vlans.1); - } - if line.starts_with("sqm") { - line = format!("sqm = '{}'", self.sqm); - } - if line.starts_with("upstreamBandwidthCapacityDownloadMbps") { - line = format!( - "upstreamBandwidthCapacityDownloadMbps = {}", - self.total_download_mbps - ); - } - if line.starts_with("upstreamBandwidthCapacityUploadMbps") { - line = format!( - "upstreamBandwidthCapacityUploadMbps = {}", - self.total_upload_mbps - ); - } - if line.starts_with("monitorOnlyMode") { - line = format!( - "monitorOnlyMode = {}", - if self.monitor_mode { "True" } else { "False" } - ); - } - if line.starts_with("generatedPNDownloadMbps") { - line = format!( - "generatedPNDownloadMbps = {}", - self.generated_download_mbps - ); - } - if line.starts_with("generatedPNUploadMbps") { - line = - format!("generatedPNUploadMbps = {}", self.generated_upload_mbps); - } - if line.starts_with("useBinPackingToBalanceCPU") { - line = format!( - "useBinPackingToBalanceCPU = {}", - if self.use_binpacking { "True" } else { "False" } - ); - } - if line.starts_with("enableActualShellCommands") { - line = format!( - "enableActualShellCommands = {}", - if self.enable_shell_commands { "True" } else { "False" } - ); - } - if line.starts_with("runShellCommandsAsSudo") { - line = format!( - "runShellCommandsAsSudo = {}", - if self.run_as_sudo { "True" } else { "False" } - ); - } - if line.starts_with("queuesAvailableOverride") { - line = - format!("queuesAvailableOverride = {}", self.override_queue_count); - } - config += &format!("{line}\n"); - } - - // Actually save to disk - if final_path.exists() { - remove_file(&final_path) - .map_err(|_| LibreQoSConfigError::CannotRemove)?; - } - if let Ok(mut file) = - OpenOptions::new().write(true).create_new(true).open(&final_path) - { - if file.write_all(config.as_bytes()).is_err() { - error!("Unable to write to ispConfig.py"); - return Err(LibreQoSConfigError::CannotWrite); - } - } else { - error!("Unable to open ispConfig.py for writing."); - return Err(LibreQoSConfigError::CannotOpenForWrite); - } - Ok(()) - } - - /// Convert the Allowed Subnets list into a Trie for fast search - pub fn allowed_subnets_trie(&self) -> ip_network_table::IpNetworkTable { - let ip_list = ip_list_to_ips(&self.allowed_subnets).unwrap(); - //println!("{ip_list:#?}"); - ip_list_to_trie(&ip_list) - } - - /// Convert the Ignored Subnets list into a Trie for fast search - pub fn ignored_subnets_trie(&self) -> ip_network_table::IpNetworkTable { - let ip_list = ip_list_to_ips(&self.ignored_subnets).unwrap(); - //println!("{ip_list:#?}"); - ip_list_to_trie(&ip_list) - } -} - -fn split_at_equals(line: &str) -> String { - line.split('=').nth(1).unwrap_or("").trim().replace(['\"', '\''], "") -} - -#[derive(Debug, Error)] -pub enum LibreQoSConfigError { - #[error("Unable to read /etc/lqos.conf. See other errors for details.")] - CannotOpenEtcLqos, - #[error("Unable to locate (path to LibreQoS)/ispConfig.py. Check your path and that you have configured it.")] - FileNotFoud, - #[error( - "Unable to read the contents of ispConfig.py. Check file permissions." - )] - CannotReadFile, - #[error("Unable to parse ispConfig.py")] - ParseError(String), - #[error("Could not backup configuration")] - CannotCopy, - #[error("Could not remove the previous configuration.")] - CannotRemove, - #[error("Could not open ispConfig.py for write")] - CannotOpenForWrite, - #[error("Unable to write to ispConfig.py")] - CannotWrite, - #[error("Unable to read IP")] - CannotReadIP, -} - -fn ip_list_to_ips( - source: &str, -) -> Result, LibreQoSConfigError> { - // Remove any square brackets, spaces - let source = source.replace(['[', ']', ' '], ""); - - // Split at commas - Ok( - source - .split(',') - .map(|raw| { - let split: Vec<&str> = raw.split('/').collect(); - let cidr = split[1].parse::().unwrap(); - let addr = split[0].parse::().unwrap(); - (addr, cidr) - }) - .collect(), - ) -} - -fn ip_list_to_trie( - source: &[(IpAddr, u8)], -) -> ip_network_table::IpNetworkTable { - let mut table = ip_network_table::IpNetworkTable::new(); - source - .iter() - .map(|(ip, subnet)| { - ( - match ip { - IpAddr::V4(ip) => ip.to_ipv6_mapped(), - IpAddr::V6(ip) => *ip, - }, - match ip { - IpAddr::V4(..) => *subnet + 96, - IpAddr::V6(..) => *subnet - }, - ) - }) - .map(|(ip, cidr)| IpNetwork::new(ip, cidr).unwrap()) - .enumerate() - .for_each(|(id, net)| { - table.insert(net, id); - }); - table -} diff --git a/src/rust/lqos_heimdall/src/timeline.rs b/src/rust/lqos_heimdall/src/timeline.rs index 3bd2d066..33982b29 100644 --- a/src/rust/lqos_heimdall/src/timeline.rs +++ b/src/rust/lqos_heimdall/src/timeline.rs @@ -7,7 +7,6 @@ use crate::{ }; use dashmap::{DashMap, DashSet}; use lqos_bus::{tos_parser, PacketHeader}; -use lqos_config::EtcLqos; use lqos_utils::{unix_time::time_since_boot, XdpIpAddress}; use once_cell::sync::Lazy; use std::{ @@ -110,8 +109,8 @@ pub fn hyperfocus_on_target(ip: XdpIpAddress) -> Option<(usize, usize)> { { // If explicitly set, obtain the capture time. Otherwise, default to // a reasonable 10 seconds. - let capture_time = if let Ok(cfg) = EtcLqos::load() { - cfg.packet_capture_time.unwrap_or(10) + let capture_time = if let Ok(cfg) = lqos_config::load_config() { + cfg.packet_capture_time } else { 10 }; diff --git a/src/rust/lqos_node_manager/src/config_control.rs b/src/rust/lqos_node_manager/src/config_control.rs index 24602a09..43d501de 100644 --- a/src/rust/lqos_node_manager/src/config_control.rs +++ b/src/rust/lqos_node_manager/src/config_control.rs @@ -1,7 +1,7 @@ use crate::{auth_guard::AuthGuard, cache_control::NoCache}; use default_net::get_interfaces; use lqos_bus::{bus_request, BusRequest, BusResponse}; -use lqos_config::{EtcLqos, LibreQoSConfig, Tunables}; +use lqos_config::{Tunables, Config}; use rocket::{fs::NamedFile, serde::{json::Json, Serialize}}; // Note that NoCache can be replaced with a cache option @@ -30,33 +30,15 @@ pub async fn get_nic_list<'a>( NoCache::new(Json(result)) } -#[get("/api/python_config")] -pub async fn get_current_python_config( - _auth: AuthGuard, -) -> NoCache> { - let config = lqos_config::LibreQoSConfig::load().unwrap(); - println!("{config:#?}"); - NoCache::new(Json(config)) -} - -#[get("/api/lqosd_config")] +#[get("/api/config")] pub async fn get_current_lqosd_config( _auth: AuthGuard, -) -> NoCache> { - let config = lqos_config::EtcLqos::load().unwrap(); +) -> NoCache> { + let config = lqos_config::load_config().unwrap(); println!("{config:#?}"); NoCache::new(Json(config)) } -#[post("/api/python_config", data = "")] -pub async fn update_python_config( - _auth: AuthGuard, - config: Json, -) -> Json { - config.save().unwrap(); - Json("OK".to_string()) -} - #[post("/api/lqos_tuning/", data = "")] pub async fn update_lqos_tuning( auth: AuthGuard, diff --git a/src/rust/lqos_node_manager/src/main.rs b/src/rust/lqos_node_manager/src/main.rs index e104cec4..d680eb58 100644 --- a/src/rust/lqos_node_manager/src/main.rs +++ b/src/rust/lqos_node_manager/src/main.rs @@ -80,9 +80,9 @@ fn rocket() -> _ { queue_info::request_analysis, queue_info::dns_query, config_control::get_nic_list, - config_control::get_current_python_config, + //config_control::get_current_python_config, config_control::get_current_lqosd_config, - config_control::update_python_config, + //config_control::update_python_config, config_control::update_lqos_tuning, auth_guard::create_first_user, auth_guard::login, diff --git a/src/rust/lqos_node_manager/src/toasts.rs b/src/rust/lqos_node_manager/src/toasts.rs index 90706d70..36f31bfb 100644 --- a/src/rust/lqos_node_manager/src/toasts.rs +++ b/src/rust/lqos_node_manager/src/toasts.rs @@ -1,4 +1,4 @@ -use lqos_config::EtcLqos; +use lqos_config::load_config; use lqos_utils::unix_time::unix_now; use rocket::serde::json::Json; use rocket::serde::{Deserialize, Serialize}; @@ -22,12 +22,12 @@ pub struct VersionCheckResponse { } async fn send_version_check() -> anyhow::Result { - if let Ok(cfg) = EtcLqos::load() { + if let Ok(cfg) = load_config() { let current_hash = env!("GIT_HASH"); let request = VersionCheckRequest { current_git_hash: current_hash.to_string(), version_string: VERSION_STRING.to_string(), - node_id: cfg.node_id.unwrap_or("(not configured)".to_string()), + node_id: cfg.node_id.to_string(), }; let response = reqwest::Client::new() .post("https://stats.libreqos.io/api/version_check") @@ -87,24 +87,17 @@ pub async fn stats_check() -> Json { node_id: String::new(), }; - if let Ok(cfg) = EtcLqos::load() { - if let Some(lts) = &cfg.long_term_stats { - if !lts.gather_stats { - response = StatsCheckAction { - action: StatsCheckResponse::Disabled, - node_id: cfg.node_id.unwrap_or("(not configured)".to_string()), - }; - } else { - // Stats are enabled - response = StatsCheckAction { - action: StatsCheckResponse::GoodToGo, - node_id: cfg.node_id.unwrap_or("(not configured)".to_string()), - }; - } - } else { + if let Ok(cfg) = load_config() { + if !cfg.long_term_stats.gather_stats { response = StatsCheckAction { - action: StatsCheckResponse::NotSetup, - node_id: cfg.node_id.unwrap_or("(not configured)".to_string()), + action: StatsCheckResponse::Disabled, + node_id: cfg.node_id.to_string(), + }; + } else { + // Stats are enabled + response = StatsCheckAction { + action: StatsCheckResponse::GoodToGo, + node_id: cfg.node_id.to_string(), }; } } diff --git a/src/rust/lqos_queue_tracker/src/queue_structure/queue_network.rs b/src/rust/lqos_queue_tracker/src/queue_structure/queue_network.rs index 6532e48b..d156058a 100644 --- a/src/rust/lqos_queue_tracker/src/queue_structure/queue_network.rs +++ b/src/rust/lqos_queue_tracker/src/queue_structure/queue_network.rs @@ -1,6 +1,5 @@ use super::{queue_node::QueueNode, QueueStructureError}; use log::error; -use lqos_config::EtcLqos; use serde_json::Value; use std::path::{Path, PathBuf}; @@ -10,7 +9,7 @@ pub struct QueueNetwork { impl QueueNetwork { pub fn path() -> Result { - let cfg = EtcLqos::load(); + let cfg = lqos_config::load_config(); if cfg.is_err() { error!("unable to read /etc/lqos.conf"); return Err(QueueStructureError::LqosConf); diff --git a/src/rust/lqos_queue_tracker/src/tracking/mod.rs b/src/rust/lqos_queue_tracker/src/tracking/mod.rs index 952e8165..1bd7b11d 100644 --- a/src/rust/lqos_queue_tracker/src/tracking/mod.rs +++ b/src/rust/lqos_queue_tracker/src/tracking/mod.rs @@ -3,7 +3,6 @@ use crate::{ queue_store::QueueStore, tracking::reader::read_named_queue_from_interface, }; use log::info; -use lqos_config::LibreQoSConfig; use lqos_utils::fdtimer::periodic; mod reader; mod watched_queues; @@ -16,7 +15,7 @@ fn track_queues() { //info!("No queues marked for read."); return; // There's nothing to do - bail out fast } - let config = LibreQoSConfig::load(); + let config = lqos_config::load_config(); if config.is_err() { //warn!("Unable to read LibreQoS config. Skipping queue collection cycle."); return; @@ -25,22 +24,22 @@ fn track_queues() { WATCHED_QUEUES.iter_mut().for_each(|q| { let (circuit_id, download_class, upload_class) = q.get(); - let (download, upload) = if config.on_a_stick_mode { + let (download, upload) = if config.on_a_stick_mode() { ( read_named_queue_from_interface( - &config.internet_interface, + &config.internet_interface(), download_class, ), read_named_queue_from_interface( - &config.internet_interface, + &config.internet_interface(), upload_class, ), ) } else { ( - read_named_queue_from_interface(&config.isp_interface, download_class), + read_named_queue_from_interface(&config.isp_interface(), download_class), read_named_queue_from_interface( - &config.internet_interface, + &config.internet_interface(), download_class, ), ) @@ -83,7 +82,7 @@ pub fn spawn_queue_monitor() { std::thread::spawn(|| { // Setup the queue monitor loop info!("Starting Queue Monitor Thread."); - let interval_ms = if let Ok(config) = lqos_config::EtcLqos::load() { + let interval_ms = if let Ok(config) = lqos_config::load_config() { config.queue_check_period_ms } else { 1000 diff --git a/src/rust/lqos_sys/src/bifrost_maps.rs b/src/rust/lqos_sys/src/bifrost_maps.rs index d1d39b00..68de6ccb 100644 --- a/src/rust/lqos_sys/src/bifrost_maps.rs +++ b/src/rust/lqos_sys/src/bifrost_maps.rs @@ -1,7 +1,6 @@ use crate::{bpf_map::BpfMap, lqos_kernel::interface_name_to_index}; use anyhow::Result; use log::info; -use lqos_config::{BridgeInterface, BridgeVlan}; #[repr(C)] #[derive(Default, Clone, Debug)] @@ -31,10 +30,86 @@ pub(crate) fn clear_bifrost() -> Result<()> { Ok(()) } -pub(crate) fn map_interfaces(mappings: &[BridgeInterface]) -> Result<()> { +pub(crate) fn map_multi_interface_mode( + to_internet: &str, + to_lan: &str, +) -> Result<()> { + info!("Interface maps (multi-interface)"); + let mut interface_map = + BpfMap::::from_path(INTERFACE_PATH)?; + + // Internet + let mut from = interface_name_to_index(to_internet)?; + let redirect_to = interface_name_to_index(to_lan)?; + let mut mapping = BifrostInterface { + redirect_to, + scan_vlans: 0, + }; + interface_map.insert(&mut from, &mut mapping)?; + info!("Mapped bifrost interface {}->{}", from, redirect_to); + + // LAN + let mut from = interface_name_to_index(to_lan)?; + let redirect_to = interface_name_to_index(to_internet)?; + let mut mapping = BifrostInterface { + redirect_to, + scan_vlans: 0, + }; + interface_map.insert(&mut from, &mut mapping)?; + info!("Mapped bifrost interface {}->{}", from, redirect_to); + + Ok(()) +} + +pub(crate) fn map_single_interface_mode( + interface: &str, + internet_vlan: u32, + lan_vlan: u32, +) -> Result<()> { + info!("Interface maps (single interface)"); + let mut interface_map = + BpfMap::::from_path(INTERFACE_PATH)?; + + let mut vlan_map = BpfMap::::from_path(VLAN_PATH)?; + + // Internet + let mut from = interface_name_to_index(interface)?; + let redirect_to = interface_name_to_index(interface)?; + let mut mapping = BifrostInterface { + redirect_to, + scan_vlans: 1, + }; + interface_map.insert(&mut from, &mut mapping)?; + info!("Mapped bifrost interface {}->{}", from, redirect_to); + + // VLANs - Internet + let mut key: u32 = (interface_name_to_index(&interface)? << 16) | internet_vlan; + let mut val = BifrostVlan { redirect_to: mapping.redirect_to }; + vlan_map.insert(&mut key, &mut val)?; + info!( + "Mapped bifrost VLAN: {}:{} => {}", + interface, internet_vlan, lan_vlan + ); + info!("{key}"); + + // VLANs - LAN + let mut key: u32 = (interface_name_to_index(&interface)? << 16) | lan_vlan; + let mut val = BifrostVlan { redirect_to: mapping.redirect_to }; + vlan_map.insert(&mut key, &mut val)?; + info!( + "Mapped bifrost VLAN: {}:{} => {}", + interface, lan_vlan, internet_vlan + ); + info!("{key}"); + + Ok(()) +} + +/*pub(crate) fn map_interfaces(mappings: &[&str]) -> Result<()> { info!("Interface maps"); let mut interface_map = BpfMap::::from_path(INTERFACE_PATH)?; + for mapping in mappings.iter() { // Key is the parent interface let mut from = interface_name_to_index(&mapping.name)?; @@ -67,4 +142,4 @@ pub(crate) fn map_vlans(mappings: &[BridgeVlan]) -> Result<()> { info!("{key}"); } Ok(()) -} +}*/ diff --git a/src/rust/lqos_sys/src/lqos_kernel.rs b/src/rust/lqos_sys/src/lqos_kernel.rs index d3db199a..66746369 100644 --- a/src/rust/lqos_sys/src/lqos_kernel.rs +++ b/src/rust/lqos_sys/src/lqos_kernel.rs @@ -205,29 +205,48 @@ pub fn attach_xdp_and_tc_to_interface( } // Attach to the ingress IF it is configured - if let Ok(etc) = lqos_config::EtcLqos::load() { + if let Ok(etc) = lqos_config::load_config() { if let Some(bridge) = &etc.bridge { - if bridge.use_xdp_bridge { - // Enable "promiscuous" mode on interfaces - for mapping in bridge.interface_mapping.iter() { - info!("Enabling promiscuous mode on {}", &mapping.name); - std::process::Command::new("/bin/ip") - .args(["link", "set", &mapping.name, "promisc", "on"]) + // Enable "promiscuous" mode on interfaces + info!("Enabling promiscuous mode on {}", &bridge.to_internet); + std::process::Command::new("/bin/ip") + .args(["link", "set", &bridge.to_internet, "promisc", "on"]) .output()?; - } + info!("Enabling promiscuous mode on {}", &bridge.to_network); + std::process::Command::new("/bin/ip") + .args(["link", "set", &bridge.to_network, "promisc", "on"]) + .output()?; - // Build the interface and vlan map entries - crate::bifrost_maps::clear_bifrost()?; - crate::bifrost_maps::map_interfaces(&bridge.interface_mapping)?; - crate::bifrost_maps::map_vlans(&bridge.vlan_mapping)?; + // Build the interface and vlan map entries + crate::bifrost_maps::clear_bifrost()?; + crate::bifrost_maps::map_multi_interface_mode(&bridge.to_internet, &bridge.to_network)?; - // Actually attach the TC ingress program - let error = unsafe { - bpf::tc_attach_ingress(interface_index as i32, false, skeleton) - }; - if error != 0 { - return Err(Error::msg("Unable to attach TC Ingress to interface")); - } + // Actually attach the TC ingress program + let error = unsafe { + bpf::tc_attach_ingress(interface_index as i32, false, skeleton) + }; + if error != 0 { + return Err(Error::msg("Unable to attach TC Ingress to interface")); + } + } + + if let Some(stick) = &etc.single_interface { + // Enable "promiscuous" mode on interface + info!("Enabling promiscuous mode on {}", &stick.interface); + std::process::Command::new("/bin/ip") + .args(["link", "set", &stick.interface, "promisc", "on"]) + .output()?; + + // Build the interface and vlan map entries + crate::bifrost_maps::clear_bifrost()?; + crate::bifrost_maps::map_single_interface_mode(&stick.interface, stick.internet_vlan as u32, stick.network_vlan as u32)?; + + // Actually attach the TC ingress program + let error = unsafe { + bpf::tc_attach_ingress(interface_index as i32, false, skeleton) + }; + if error != 0 { + return Err(Error::msg("Unable to attach TC Ingress to interface")); } } } diff --git a/src/rust/lqosd/src/anonymous_usage/mod.rs b/src/rust/lqosd/src/anonymous_usage/mod.rs index c9f7378c..7292cc74 100644 --- a/src/rust/lqosd/src/anonymous_usage/mod.rs +++ b/src/rust/lqosd/src/anonymous_usage/mod.rs @@ -2,7 +2,6 @@ mod lshw; mod version; use std::{time::Duration, net::TcpStream, io::Write}; use lqos_bus::anonymous::{AnonymousUsageV1, build_stats}; -use lqos_config::{EtcLqos, LibreQoSConfig}; use lqos_sys::num_possible_cpus; use sysinfo::{System, SystemExt, CpuExt}; use crate::{shaped_devices_tracker::{SHAPED_DEVICES, NETWORK_JSON}, stats::{HIGH_WATERMARK_DOWN, HIGH_WATERMARK_UP}}; @@ -11,17 +10,15 @@ const SLOW_START_SECS: u64 = 1; const INTERVAL_SECS: u64 = 60 * 60 * 24; pub async fn start_anonymous_usage() { - if let Ok(cfg) = EtcLqos::load() { - if let Some(usage) = cfg.usage_stats { - if usage.send_anonymous { - std::thread::spawn(|| { - std::thread::sleep(Duration::from_secs(SLOW_START_SECS)); - loop { - let _ = anonymous_usage_dump(); - std::thread::sleep(Duration::from_secs(INTERVAL_SECS)); - } - }); - } + if let Ok(cfg) = lqos_config::load_config() { + if cfg.usage_stats.send_anonymous { + std::thread::spawn(|| { + std::thread::sleep(Duration::from_secs(SLOW_START_SECS)); + loop { + let _ = anonymous_usage_dump(); + std::thread::sleep(Duration::from_secs(INTERVAL_SECS)); + } + }); } } } @@ -52,30 +49,24 @@ fn anonymous_usage_dump() -> anyhow::Result<()> { data.distro = pv.trim().to_string(); } - if let Ok(cfg) = LibreQoSConfig::load() { - data.sqm = cfg.sqm; - data.monitor_mode = cfg.monitor_mode; + if let Ok(cfg) = lqos_config::load_config() { + data.sqm = cfg.queues.default_sqm.clone(); + data.monitor_mode = cfg.queues.monitor_only; data.total_capacity = ( - cfg.total_download_mbps, - cfg.total_upload_mbps, + cfg.queues.downlink_bandwidth_mbps, + cfg.queues.uplink_bandwidth_mbps, ); data.generated_pdn_capacity = ( - cfg.generated_download_mbps, - cfg.generated_upload_mbps, + cfg.queues.generated_pn_download_mbps, + cfg.queues.generated_pn_upload_mbps, ); - data.on_a_stick = cfg.on_a_stick_mode; - } + data.on_a_stick = cfg.on_a_stick_mode(); - if let Ok(cfg) = EtcLqos::load() { - if let Some(node_id) = cfg.node_id { - data.node_id = node_id; - if let Some(bridge) = cfg.bridge { - data.using_xdp_bridge = bridge.use_xdp_bridge; - } - } - if let Some(anon) = cfg.usage_stats { - server = anon.anonymous_server; + data.node_id = cfg.node_id.clone(); + if let Some(bridge) = cfg.bridge { + data.using_xdp_bridge = bridge.use_xdp_bridge; } + server = cfg.usage_stats.anonymous_server; } data.git_hash = env!("GIT_HASH").to_string(); diff --git a/src/rust/lqosd/src/main.rs b/src/rust/lqosd/src/main.rs index 73cca66a..d537d65a 100644 --- a/src/rust/lqosd/src/main.rs +++ b/src/rust/lqosd/src/main.rs @@ -17,7 +17,6 @@ use crate::{ use anyhow::Result; use log::{info, warn}; use lqos_bus::{BusRequest, BusResponse, UnixSocketServer, StatsRequest}; -use lqos_config::LibreQoSConfig; use lqos_heimdall::{n_second_packet_dump, perf_interface::heimdall_handle_events, start_heimdall}; use lqos_queue_tracker::{ add_watched_queue, get_raw_circuit_data, spawn_queue_monitor, @@ -57,19 +56,19 @@ async fn main() -> Result<()> { } info!("LibreQoS Daemon Starting"); - let config = LibreQoSConfig::load()?; - tuning::tune_lqosd_from_config_file(&config)?; + let config = lqos_config::load_config()?; + tuning::tune_lqosd_from_config_file()?; // Start the XDP/TC kernels - let kernels = if config.on_a_stick_mode { + let kernels = if config.on_a_stick_mode() { LibreQoSKernels::on_a_stick_mode( - &config.internet_interface, - config.stick_vlans.1, - config.stick_vlans.0, + &config.internet_interface(), + config.stick_vlans().1 as u16, + config.stick_vlans().0 as u16, Some(heimdall_handle_events), )? } else { - LibreQoSKernels::new(&config.internet_interface, &config.isp_interface, Some(heimdall_handle_events))? + LibreQoSKernels::new(&config.internet_interface(), &config.isp_interface(), Some(heimdall_handle_events))? }; // Spawn tracking sub-systems @@ -107,13 +106,9 @@ async fn main() -> Result<()> { } SIGHUP => { warn!("Reloading configuration because of SIGHUP"); - if let Ok(config) = LibreQoSConfig::load() { - let result = tuning::tune_lqosd_from_config_file(&config); - if let Err(err) = result { - warn!("Unable to HUP tunables: {:?}", err) - } - } else { - warn!("Unable to reload configuration"); + let result = tuning::tune_lqosd_from_config_file(); + if let Err(err) = result { + warn!("Unable to HUP tunables: {:?}", err) } } _ => warn!("No handler for signal: {sig}"), diff --git a/src/rust/lqosd/src/tuning/mod.rs b/src/rust/lqosd/src/tuning/mod.rs index 86fc393b..693f347e 100644 --- a/src/rust/lqosd/src/tuning/mod.rs +++ b/src/rust/lqosd/src/tuning/mod.rs @@ -1,26 +1,23 @@ mod offloads; use anyhow::Result; use lqos_bus::{BusRequest, BusResponse}; -use lqos_config::{EtcLqos, LibreQoSConfig}; use lqos_queue_tracker::set_queue_refresh_interval; -pub fn tune_lqosd_from_config_file(config: &LibreQoSConfig) -> Result<()> { - let etc_lqos = EtcLqos::load()?; +pub fn tune_lqosd_from_config_file() -> Result<()> { + let config = lqos_config::load_config()?; // Disable offloading - if let Some(tuning) = &etc_lqos.tuning { - offloads::bpf_sysctls(); - if tuning.stop_irq_balance { - offloads::stop_irq_balance(); - } - offloads::netdev_budget( - tuning.netdev_budget_usecs, - tuning.netdev_budget_packets, - ); - offloads::ethtool_tweaks(&config.internet_interface, tuning); - offloads::ethtool_tweaks(&config.isp_interface, tuning); + offloads::bpf_sysctls(); + if config.tuning.stop_irq_balance { + offloads::stop_irq_balance(); } - let interval = etc_lqos.queue_check_period_ms; + offloads::netdev_budget( + config.tuning.netdev_budget_usecs, + config.tuning.netdev_budget_packets, + ); + offloads::ethtool_tweaks(&config.internet_interface(), &config.tuning); + offloads::ethtool_tweaks(&config.isp_interface(), &config.tuning); + let interval = config.queue_check_period_ms; set_queue_refresh_interval(interval); Ok(()) } @@ -29,7 +26,7 @@ pub fn tune_lqosd_from_bus(request: &BusRequest) -> BusResponse { match request { BusRequest::UpdateLqosDTuning(interval, tuning) => { // Real-time tuning changes. Probably dangerous. - if let Ok(config) = LibreQoSConfig::load() { + if let Ok(config) = lqos_config::load_config() { if tuning.stop_irq_balance { offloads::stop_irq_balance(); } @@ -37,8 +34,8 @@ pub fn tune_lqosd_from_bus(request: &BusRequest) -> BusResponse { tuning.netdev_budget_usecs, tuning.netdev_budget_packets, ); - offloads::ethtool_tweaks(&config.internet_interface, tuning); - offloads::ethtool_tweaks(&config.isp_interface, tuning); + offloads::ethtool_tweaks(&config.internet_interface(), &config.tuning); + offloads::ethtool_tweaks(&config.isp_interface(), &config.tuning); } set_queue_refresh_interval(*interval); lqos_bus::BusResponse::Ack diff --git a/src/rust/lts_client/src/collector/collection_manager.rs b/src/rust/lts_client/src/collector/collection_manager.rs index 59e74c15..c2693377 100644 --- a/src/rust/lts_client/src/collector/collection_manager.rs +++ b/src/rust/lts_client/src/collector/collection_manager.rs @@ -7,12 +7,22 @@ //! lose min/max values. use super::StatsUpdateMessage; -use crate::{collector::{collation::{collate_stats, StatsSession}, SESSION_BUFFER, uisp_ext::gather_uisp_data}, submission_queue::{enqueue_shaped_devices_if_allowed, comm_channel::{SenderChannelMessage, start_communication_channel}}}; -use lqos_config::EtcLqos; +use crate::{ + collector::{ + collation::{collate_stats, StatsSession}, + uisp_ext::gather_uisp_data, + SESSION_BUFFER, + }, + submission_queue::{ + comm_channel::{start_communication_channel, SenderChannelMessage}, + enqueue_shaped_devices_if_allowed, + }, +}; +use dashmap::DashSet; +use lqos_config::load_config; use once_cell::sync::Lazy; use std::{sync::atomic::AtomicU64, time::Duration}; use tokio::sync::mpsc::{self, Receiver, Sender}; -use dashmap::DashSet; static STATS_COUNTER: AtomicU64 = AtomicU64::new(0); pub(crate) static DEVICE_ID_LIST: Lazy> = Lazy::new(DashSet::new); @@ -22,21 +32,21 @@ pub(crate) static DEVICE_ID_LIST: Lazy> = Lazy::new(DashSet::new /// /// Returns a channel that may be used to notify of data availability. pub async fn start_long_term_stats() -> Sender { - let (update_tx, mut update_rx): (Sender, Receiver) = mpsc::channel(10); - let (comm_tx, comm_rx): (Sender, Receiver) = mpsc::channel(10); + let (update_tx, mut update_rx): (Sender, Receiver) = + mpsc::channel(10); + let (comm_tx, comm_rx): (Sender, Receiver) = + mpsc::channel(10); - if let Ok(cfg) = lqos_config::EtcLqos::load() { - if let Some(lts) = cfg.long_term_stats { - if !lts.gather_stats { - // Wire up a null recipient to the channel, so it receives messages - // but doesn't do anything with them. - tokio::spawn(async move { - while let Some(_msg) = update_rx.recv().await { - // Do nothing - } - }); - return update_tx; - } + if let Ok(cfg) = load_config() { + if !cfg.long_term_stats.gather_stats { + // Wire up a null recipient to the channel, so it receives messages + // but doesn't do anything with them. + tokio::spawn(async move { + while let Some(_msg) = update_rx.recv().await { + // Do nothing + } + }); + return update_tx; } } @@ -85,7 +95,10 @@ async fn lts_manager(mut rx: Receiver, comm_tx: Sender { log::info!("Collation time reached"); @@ -108,20 +121,18 @@ async fn lts_manager(mut rx: Receiver, comm_tx: Sender Duration { - if let Ok(cfg) = EtcLqos::load() { - if let Some(lts) = &cfg.long_term_stats { - return Duration::from_secs(lts.collation_period_seconds.into()); - } + if let Ok(cfg) = load_config() { + return Duration::from_secs(cfg.long_term_stats.collation_period_seconds.into()); } Duration::from_secs(60) } fn get_uisp_collation_period() -> Option { - if let Ok(cfg) = EtcLqos::load() { - if let Some(lts) = &cfg.long_term_stats { - return Some(Duration::from_secs(lts.uisp_reporting_interval_seconds.unwrap_or(300))); - } + if let Ok(cfg) = load_config() { + return Some(Duration::from_secs( + cfg.long_term_stats.uisp_reporting_interval_seconds.unwrap_or(300), + )); } None @@ -136,7 +147,10 @@ async fn uisp_collection_manager(control_tx: Sender) { if let Some(period) = get_uisp_collation_period() { log::info!("Starting UISP poller with period {:?}", period); loop { - control_tx.send(StatsUpdateMessage::UispCollationTime).await.unwrap(); + control_tx + .send(StatsUpdateMessage::UispCollationTime) + .await + .unwrap(); tokio::time::sleep(period).await; } } else { @@ -144,4 +158,4 @@ async fn uisp_collection_manager(control_tx: Sender) { tokio::time::sleep(Duration::from_secs(3600)).await; } } -} \ No newline at end of file +} diff --git a/src/rust/lts_client/src/collector/quick_drops/queue_structure/queue_network.rs b/src/rust/lts_client/src/collector/quick_drops/queue_structure/queue_network.rs index 6532e48b..05b3cfa7 100644 --- a/src/rust/lts_client/src/collector/quick_drops/queue_structure/queue_network.rs +++ b/src/rust/lts_client/src/collector/quick_drops/queue_structure/queue_network.rs @@ -1,6 +1,6 @@ use super::{queue_node::QueueNode, QueueStructureError}; use log::error; -use lqos_config::EtcLqos; +use lqos_config::load_config; use serde_json::Value; use std::path::{Path, PathBuf}; @@ -10,7 +10,7 @@ pub struct QueueNetwork { impl QueueNetwork { pub fn path() -> Result { - let cfg = EtcLqos::load(); + let cfg = load_config(); if cfg.is_err() { error!("unable to read /etc/lqos.conf"); return Err(QueueStructureError::LqosConf); diff --git a/src/rust/lts_client/src/collector/quick_drops/stats_diff.rs b/src/rust/lts_client/src/collector/quick_drops/stats_diff.rs index 512aa920..0aaaa736 100644 --- a/src/rust/lts_client/src/collector/quick_drops/stats_diff.rs +++ b/src/rust/lts_client/src/collector/quick_drops/stats_diff.rs @@ -1,3 +1,4 @@ +use lqos_config::load_config; use tokio::sync::Mutex; use once_cell::sync::Lazy; use super::CakeStats; @@ -23,10 +24,10 @@ impl CakeTracker { } pub(crate) async fn update(&mut self) -> Option<(Vec, Vec)> { - if let Ok(cfg) = lqos_config::LibreQoSConfig::load() { - let outbound = &cfg.internet_interface; - let inbound = &cfg.isp_interface; - if cfg.on_a_stick_mode { + if let Ok(cfg) = load_config() { + let outbound = &cfg.internet_interface(); + let inbound = &cfg.isp_interface(); + if cfg.on_a_stick_mode() { let reader = super::AsyncQueueReader::new(outbound); if let Ok((Some(up), Some(down))) = reader.run_on_a_stick().await { return self.read_up_down(up, down); diff --git a/src/rust/lts_client/src/collector/uisp_ext.rs b/src/rust/lts_client/src/collector/uisp_ext.rs index 3a543e82..8e2da167 100644 --- a/src/rust/lts_client/src/collector/uisp_ext.rs +++ b/src/rust/lts_client/src/collector/uisp_ext.rs @@ -1,3 +1,4 @@ +use lqos_config::load_config; use lqos_utils::unix_time::unix_now; use tokio::sync::mpsc::Sender; use crate::{submission_queue::{comm_channel::SenderChannelMessage, new_submission}, transport_data::{StatsSubmission, UispExtDevice}, collector::collection_manager::DEVICE_ID_LIST}; @@ -9,7 +10,7 @@ pub(crate) async fn gather_uisp_data(comm_tx: Sender) { return; // We're not ready } - if let Ok(config) = lqos_config::LibreQoSConfig::load() { + if let Ok(config) = load_config() { if let Ok(devices) = uisp::load_all_devices_with_interfaces(config).await { log::info!("Loaded {} UISP devices", devices.len()); diff --git a/src/rust/lts_client/src/submission_queue/comm_channel/encode.rs b/src/rust/lts_client/src/submission_queue/comm_channel/encode.rs index d9d7b054..2808b307 100644 --- a/src/rust/lts_client/src/submission_queue/comm_channel/encode.rs +++ b/src/rust/lts_client/src/submission_queue/comm_channel/encode.rs @@ -1,5 +1,5 @@ use dryoc::{dryocbox::{Nonce, DryocBox}, types::{NewByteArray, ByteArray}}; -use lqos_config::EtcLqos; +use lqos_config::load_config; use thiserror::Error; use crate::{transport_data::{LtsCommand, NodeIdAndLicense, HelloVersion2}, submission_queue::queue::QueueError}; use super::keys::{SERVER_PUBLIC_KEY, KEYPAIR}; @@ -104,17 +104,13 @@ pub(crate) async fn encode_submission(submission: &LtsCommand) -> Result } fn get_license_key_and_node_id(nonce: &Nonce) -> Result { - let cfg = EtcLqos::load().map_err(|_| QueueError::SendFail)?; - if let Some(node_id) = cfg.node_id { - if let Some(lts) = &cfg.long_term_stats { - if let Some(license_key) = <s.license_key { - return Ok(NodeIdAndLicense { - node_id, - license_key: license_key.clone(), - nonce: *nonce.as_array(), - }); - } - } + let cfg = load_config().map_err(|_| QueueError::SendFail)?; + if let Some(license_key) = &cfg.long_term_stats.license_key { + return Ok(NodeIdAndLicense { + node_id: cfg.node_id.clone(), + license_key: license_key.clone(), + nonce: *nonce.as_array(), + }); } Err(QueueError::SendFail) } diff --git a/src/rust/lts_client/src/submission_queue/comm_channel/keys.rs b/src/rust/lts_client/src/submission_queue/comm_channel/keys.rs index d437dd5a..d157023e 100644 --- a/src/rust/lts_client/src/submission_queue/comm_channel/keys.rs +++ b/src/rust/lts_client/src/submission_queue/comm_channel/keys.rs @@ -1,5 +1,5 @@ use crate::{pki::generate_new_keypair, dryoc::dryocbox::{KeyPair, PublicKey}, transport_data::{exchange_keys_with_license_server, LicenseReply}}; -use lqos_config::EtcLqos; +use lqos_config::load_config; use once_cell::sync::Lazy; use tokio::sync::RwLock; @@ -11,14 +11,14 @@ pub(crate) async fn store_server_public_key(key: &PublicKey) { } pub(crate) async fn key_exchange() -> bool { - let cfg = EtcLqos::load().unwrap(); - let node_id = cfg.node_id.unwrap(); - let node_name = if let Some(node_name) = cfg.node_name { - node_name + let cfg = load_config().unwrap(); + let node_id = cfg.node_id.clone(); + let node_name = if !cfg.node_name.is_empty() { + cfg.node_name } else { node_id.clone() }; - let license_key = cfg.long_term_stats.unwrap().license_key.unwrap(); + let license_key = cfg.long_term_stats.license_key.unwrap(); let keypair = (KEYPAIR.read().await).clone(); match exchange_keys_with_license_server(node_id, node_name, license_key, keypair.public_key.clone()).await { Ok(LicenseReply::MyPublicKey { public_key }) => { diff --git a/src/rust/lts_client/src/submission_queue/comm_channel/mod.rs b/src/rust/lts_client/src/submission_queue/comm_channel/mod.rs index e87c470f..2221313e 100644 --- a/src/rust/lts_client/src/submission_queue/comm_channel/mod.rs +++ b/src/rust/lts_client/src/submission_queue/comm_channel/mod.rs @@ -1,5 +1,5 @@ use std::time::Duration; -use lqos_config::EtcLqos; +use lqos_config::load_config; use tokio::{sync::mpsc::Receiver, time::sleep, net::TcpStream, io::{AsyncWriteExt, AsyncReadExt}}; use crate::submission_queue::comm_channel::keys::store_server_public_key; use self::encode::encode_submission_hello; @@ -49,24 +49,17 @@ pub(crate) async fn start_communication_channel(mut rx: Receiver Result { log::info!("Connecting to stats.libreqos.io"); // Check that we have a local license key and are enabled - let cfg = EtcLqos::load().map_err(|_| { + let cfg = load_config().map_err(|_| { log::error!("Unable to load config file."); QueueError::NoLocalLicenseKey })?; - let node_id = cfg.node_id.ok_or_else(|| { - log::warn!("No node ID configured."); - QueueError::NoLocalLicenseKey - })?; - let node_name = cfg.node_name.unwrap_or(node_id.clone()); - let usage_cfg = cfg.long_term_stats.ok_or_else(|| { - log::warn!("Long-term stats are not configured."); - QueueError::NoLocalLicenseKey - })?; - if !usage_cfg.gather_stats { + let node_id = cfg.node_id.clone(); + let node_name = cfg.node_name.clone(); + if !cfg.long_term_stats.gather_stats { log::warn!("Gathering long-term stats is disabled."); return Err(QueueError::StatsDisabled); } - let license_key = usage_cfg.license_key.ok_or_else(|| { + let license_key = cfg.long_term_stats.license_key.ok_or_else(|| { log::warn!("No license key configured."); QueueError::NoLocalLicenseKey })?; diff --git a/src/rust/lts_client/src/submission_queue/licensing.rs b/src/rust/lts_client/src/submission_queue/licensing.rs index 225dec35..62138f5e 100644 --- a/src/rust/lts_client/src/submission_queue/licensing.rs +++ b/src/rust/lts_client/src/submission_queue/licensing.rs @@ -1,5 +1,5 @@ use crate::transport_data::{ask_license_server, LicenseReply, ask_license_server_for_new_account}; -use lqos_config::EtcLqos; +use lqos_config::load_config; use lqos_utils::unix_time::unix_now; use once_cell::sync::Lazy; use tokio::sync::RwLock; @@ -45,12 +45,12 @@ const MISERLY_NO_KEY: &str = "IDontSupportDevelopersAndShouldFeelBad"; async fn check_license(unix_time: u64) -> LicenseState { log::info!("Checking LTS stats license"); - if let Ok(cfg) = EtcLqos::load() { + if let Ok(cfg) = load_config() { // The config file is good. Is LTS enabled? // If it isn't, we need to try very gently to see if a pending // request has been submitted. - if let Some(cfg) = cfg.long_term_stats { - if let Some(key) = cfg.license_key { + if cfg.long_term_stats.gather_stats { + if let Some(key) = cfg.long_term_stats.license_key { if key == MISERLY_NO_KEY { log::warn!("You are using the self-hosting license key. We'd be happy to sell you a real one."); return LicenseState::Valid { expiry: 0, stats_host: "192.168.100.11:9127".to_string() } @@ -90,20 +90,15 @@ async fn check_license(unix_time: u64) -> LicenseState { // So we need to check if we have a pending request. // If a license key has been assigned, then we'll setup // LTS. If it hasn't, we'll just return Unknown. - if let Some(node_id) = &cfg.node_id { - if let Ok(result) = ask_license_server_for_new_account(node_id.to_string()).await { - if let LicenseReply::NewActivation { license_key } = result { - // We have a new license! - let _ = lqos_config::enable_long_term_stats(license_key); - // Note that we're not doing anything beyond this - the next cycle - // will pick up on there actually being a license - } else { - log::info!("No pending LTS license found"); - } + if let Ok(result) = ask_license_server_for_new_account(cfg.node_id.to_string()).await { + if let LicenseReply::NewActivation { license_key } = result { + // We have a new license! + let _ = lqos_config::enable_long_term_stats(license_key); + // Note that we're not doing anything beyond this - the next cycle + // will pick up on there actually being a license + } else { + log::info!("No pending LTS license found"); } - } else { - // There's no node ID either - we can't talk to this - log::warn!("No NodeID is configured. No online services are possible."); } } } else { diff --git a/src/rust/uisp/src/lib.rs b/src/rust/uisp/src/lib.rs index 6b72d3e8..9345c8ce 100644 --- a/src/rust/uisp/src/lib.rs +++ b/src/rust/uisp/src/lib.rs @@ -1,36 +1,46 @@ +mod data_link; +mod device; // UISP data definition for a device, including interfaces /// UISP Data Structures -/// +/// /// Strong-typed implementation of the UISP API system. Used by long-term /// stats to attach device information, possibly in the future used to /// accelerate the UISP integration. - mod rest; // REST HTTP services mod site; // UISP data definition for a site, pulled from the JSON -mod device; // UISP data definition for a device, including interfaces -mod data_link; // UISP data link definitions -use lqos_config::LibreQoSConfig; -pub use site::Site; -pub use device::Device; -pub use data_link::DataLink; +use lqos_config::Config; +// UISP data link definitions use self::rest::nms_request_get_vec; use anyhow::Result; +pub use data_link::DataLink; +pub use device::Device; +pub use site::Site; /// Loads a complete list of all sites from UISP -pub async fn load_all_sites(config: LibreQoSConfig) -> Result> { - Ok(nms_request_get_vec("sites", &config.uisp_auth_token, &config.uisp_base_url).await?) +pub async fn load_all_sites(config: Config) -> Result> { + Ok(nms_request_get_vec( + "sites", + &config.uisp_integration.token, + &config.uisp_integration.url, + ) + .await?) } /// Load all devices from UISP that are authorized, and include their full interface definitions -pub async fn load_all_devices_with_interfaces(config: LibreQoSConfig) -> Result> { +pub async fn load_all_devices_with_interfaces(config: Config) -> Result> { Ok(nms_request_get_vec( "devices?withInterfaces=true&authorized=true", - &config.uisp_auth_token, - &config.uisp_base_url, + &config.uisp_integration.token, + &config.uisp_integration.url, ) .await?) } /// Loads all data links from UISP (including links in client sites) -pub async fn load_all_data_links(config: LibreQoSConfig) -> Result> { - Ok(nms_request_get_vec("data-links", &config.uisp_auth_token, &config.uisp_base_url).await?) -} \ No newline at end of file +pub async fn load_all_data_links(config: Config) -> Result> { + Ok(nms_request_get_vec( + "data-links", + &config.uisp_integration.token, + &config.uisp_integration.url, + ) + .await?) +} From 861c0b770419521ae8630dca54508680cc660e8e Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 8 Dec 2023 11:30:15 -0600 Subject: [PATCH 11/41] Tweaks to get it running. No python support yet, but the basics now seem to work. --- src/rust/lqos_config/src/etc/etclqos_migration.rs | 2 ++ src/rust/lqos_config/src/etc/migration.rs | 8 +++++--- src/rust/lqos_config/src/etc/mod.rs | 4 +++- src/rust/lqos_config/src/etc/python_migration.rs | 6 +----- src/rust/lqos_config/src/etc/v15/top_config.rs | 4 ++-- src/rust/lqos_config/src/network_json/mod.rs | 3 +-- src/rust/lqos_config/src/program_control.rs | 6 ++---- src/rust/lqos_config/src/shaped_devices/mod.rs | 8 +++++--- 8 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/rust/lqos_config/src/etc/etclqos_migration.rs b/src/rust/lqos_config/src/etc/etclqos_migration.rs index eb416a30..1c2e6d3a 100644 --- a/src/rust/lqos_config/src/etc/etclqos_migration.rs +++ b/src/rust/lqos_config/src/etc/etclqos_migration.rs @@ -171,6 +171,7 @@ impl EtcLqos { } pub(crate) fn load_from_string(raw: &str) -> Result { + log::info!("Trying to load old TOML version from /etc/lqos.conf"); let document = raw.parse::(); match document { Err(e) => { @@ -188,6 +189,7 @@ impl EtcLqos { Err(e) => { error!("Unable to parse TOML from /etc/lqos.conf"); error!("Full error: {:?}", e); + panic!(); Err(EtcLqosError::CannotParseToml) } } diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs index 3631823e..92079c54 100644 --- a/src/rust/lqos_config/src/etc/migration.rs +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -23,6 +23,7 @@ pub enum MigrationError { } pub fn migrate_if_needed() -> Result<(), MigrationError> { + log::info!("Checking config file version"); let raw = std::fs::read_to_string("/etc/lqos.conf").map_err(|e| MigrationError::ReadError(e))?; @@ -30,11 +31,12 @@ pub fn migrate_if_needed() -> Result<(), MigrationError> { .parse::() .map_err(|e| MigrationError::ParseError(e))?; if let Some((_key, version)) = doc.get_key_value("version") { - if version.as_str().unwrap() == "1.5.0" { - log::info!("Configuration file is already at version 1.5.0, no migration needed"); + log::info!("Configuration file is at version {}", version.as_str().unwrap()); + if version.as_str().unwrap().trim() == "1.5" { + log::info!("Configuration file is already at version 1.5, no migration needed"); return Ok(()); } else { - log::error!("Configuration file is at version {}, but this version of lqos only supports version 1.5.0", version.as_str().unwrap()); + log::error!("Configuration file is at version {}, but this version of lqos only supports version 1.5", version.as_str().unwrap()); return Err(MigrationError::UnknownVersion( version.as_str().unwrap().to_string(), )); diff --git a/src/rust/lqos_config/src/etc/mod.rs b/src/rust/lqos_config/src/etc/mod.rs index 7aefa5b4..bb64e2c6 100644 --- a/src/rust/lqos_config/src/etc/mod.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -17,9 +17,9 @@ static CONFIG: Mutex> = Mutex::new(None); /// Load the configuration from `/etc/lqos.conf`. pub fn load_config() -> Result { - log::info!("Loading configuration file /etc/lqos.conf"); let mut lock = CONFIG.lock().unwrap(); if lock.is_none() { + log::info!("Loading configuration file /etc/lqos.conf"); migrate_if_needed().map_err(|e| { log::error!("Unable to migrate configuration: {:?}", e); LibreQoSConfigError::FileNotFoud @@ -41,9 +41,11 @@ pub fn load_config() -> Result { config_result ))); } + log::info!("Set cached version of config file"); *lock = Some(config_result.unwrap()); } + log::info!("Returning cached config"); Ok(lock.as_ref().unwrap().clone()) } diff --git a/src/rust/lqos_config/src/etc/python_migration.rs b/src/rust/lqos_config/src/etc/python_migration.rs index 1f3a3568..4751f52c 100644 --- a/src/rust/lqos_config/src/etc/python_migration.rs +++ b/src/rust/lqos_config/src/etc/python_migration.rs @@ -29,11 +29,7 @@ fn isp_config_py_path(cfg: &EtcLqos) -> PathBuf { /// Does thie ispConfig.py file exist? fn config_exists(cfg: &EtcLqos) -> bool { - if let Ok(cfg) = crate::etc::EtcLqos::load() { - isp_config_py_path(&cfg).exists() - } else { - false - } + isp_config_py_path(&cfg).exists() } fn from_python<'a, T>(py: &'a Python, variable_name: &str) -> Result diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index 120b7ccd..8d8da1bb 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -80,9 +80,9 @@ impl Config { .to_string(), ); } - if self.version != "1.5" { + if self.version.trim() != "1.5" { return Err(format!( - "Configuration file is at version {}, but this version of lqos only supports version 1.5.0", + "Configuration file is at version [{}], but this version of lqos only supports version 1.5.0", self.version )); } diff --git a/src/rust/lqos_config/src/network_json/mod.rs b/src/rust/lqos_config/src/network_json/mod.rs index ff37a527..1414c4b6 100644 --- a/src/rust/lqos_config/src/network_json/mod.rs +++ b/src/rust/lqos_config/src/network_json/mod.rs @@ -1,4 +1,3 @@ -use crate::etc; use dashmap::DashSet; use log::{error, info, warn}; use serde::{Deserialize, Serialize}; @@ -105,7 +104,7 @@ impl NetworkJson { /// file. pub fn path() -> Result { let cfg = - etc::EtcLqos::load().map_err(|_| NetworkJsonError::ConfigLoadError)?; + crate::load_config().map_err(|_| NetworkJsonError::ConfigLoadError)?; let base_path = Path::new(&cfg.lqos_directory); Ok(base_path.join("network.json")) } diff --git a/src/rust/lqos_config/src/program_control.rs b/src/rust/lqos_config/src/program_control.rs index e975834a..00cd2f12 100644 --- a/src/rust/lqos_config/src/program_control.rs +++ b/src/rust/lqos_config/src/program_control.rs @@ -1,7 +1,5 @@ use log::error; use thiserror::Error; - -use crate::etc; use std::{ path::{Path, PathBuf}, process::Command, @@ -11,14 +9,14 @@ const PYTHON_PATH: &str = "/usr/bin/python3"; fn path_to_libreqos() -> Result { let cfg = - etc::EtcLqos::load().map_err(|_| ProgramControlError::ConfigLoadError)?; + crate::load_config().map_err(|_| ProgramControlError::ConfigLoadError)?; let base_path = Path::new(&cfg.lqos_directory); Ok(base_path.join("LibreQoS.py")) } fn working_directory() -> Result { let cfg = - etc::EtcLqos::load().map_err(|_| ProgramControlError::ConfigLoadError)?; + crate::load_config().map_err(|_| ProgramControlError::ConfigLoadError)?; let base_path = Path::new(&cfg.lqos_directory); Ok(base_path.to_path_buf()) } diff --git a/src/rust/lqos_config/src/shaped_devices/mod.rs b/src/rust/lqos_config/src/shaped_devices/mod.rs index dc041046..6b9eea1a 100644 --- a/src/rust/lqos_config/src/shaped_devices/mod.rs +++ b/src/rust/lqos_config/src/shaped_devices/mod.rs @@ -34,9 +34,11 @@ impl ConfigShapedDevices { /// file. pub fn path() -> Result { let cfg = - etc::EtcLqos::load().map_err(|_| ShapedDevicesError::ConfigLoadError)?; + crate::load_config().map_err(|_| ShapedDevicesError::ConfigLoadError)?; let base_path = Path::new(&cfg.lqos_directory); - Ok(base_path.join("ShapedDevices.csv")) + let full_path = base_path.join("ShapedDevices.csv"); + log::info!("ShapedDevices.csv path: {:?}", full_path); + Ok(full_path) } /// Does ShapedDevices.csv exist? @@ -146,7 +148,7 @@ impl ConfigShapedDevices { /// Saves the current shaped devices list to `ShapedDevices.csv` pub fn write_csv(&self, filename: &str) -> Result<(), ShapedDevicesError> { let cfg = - etc::EtcLqos::load().map_err(|_| ShapedDevicesError::ConfigLoadError)?; + crate::load_config().map_err(|_| ShapedDevicesError::ConfigLoadError)?; let base_path = Path::new(&cfg.lqos_directory); let path = base_path.join(filename); let csv = self.to_csv_string()?; From 3cf097c78f754c9a52d788be68e860f72af13f48 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 8 Dec 2023 12:23:54 -0600 Subject: [PATCH 12/41] First pass at a python config API. LibreQoS.py now executes with the new configuration regime. --- src/LibreQoS.py | 128 +++++++++++++++++--------------- src/rust/Cargo.lock | 1 + src/rust/lqos_python/Cargo.toml | 1 + src/rust/lqos_python/src/lib.rs | 102 +++++++++++++++++++++++++ 4 files changed, 171 insertions(+), 61 deletions(-) diff --git a/src/LibreQoS.py b/src/LibreQoS.py index 9a1a2d76..873c4492 100755 --- a/src/LibreQoS.py +++ b/src/LibreQoS.py @@ -20,21 +20,20 @@ import shutil import binpacking from deepdiff import DeepDiff -from ispConfig import sqm, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps, \ - interfaceA, interfaceB, enableActualShellCommands, useBinPackingToBalanceCPU, monitorOnlyMode, \ - runShellCommandsAsSudo, generatedPNDownloadMbps, generatedPNUploadMbps, queuesAvailableOverride, \ - OnAStick - from liblqos_python import is_lqosd_alive, clear_ip_mappings, delete_ip_mapping, validate_shaped_devices, \ - is_libre_already_running, create_lock_file, free_lock_file, add_ip_mapping, BatchedCommands + is_libre_already_running, create_lock_file, free_lock_file, add_ip_mapping, BatchedCommands, \ + check_config, sqm, upstream_bandwidth_capacity_download_mbps, upstream_bandwidth_capacity_upload_mbps, \ + interface_a, interface_b, enable_actual_shell_commands, use_bin_packing_to_balance_cpu, monitor_mode_only, \ + run_shell_commands_as_sudo, generated_pn_download_mbps, generated_pn_upload_mbps, queues_available_override, \ + on_a_stick # Automatically account for TCP overhead of plans. For example a 100Mbps plan needs to be set to 109Mbps for the user to ever see that result on a speed test # Does not apply to nodes of any sort, just endpoint devices tcpOverheadFactor = 1.09 def shell(command): - if enableActualShellCommands: - if runShellCommandsAsSudo: + if enable_actual_shell_commands(): + if run_shell_commands_as_sudo(): command = 'sudo ' + command logging.info(command) commands = command.split(' ') @@ -49,7 +48,7 @@ def shell(command): def shellReturn(command): returnableString = '' - if enableActualShellCommands: + if enable_actual_shell_commands(): commands = command.split(' ') proc = subprocess.Popen(commands, stdout=subprocess.PIPE) for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding @@ -72,11 +71,11 @@ def checkIfFirstRunSinceBoot(): return True def clearPriorSettings(interfaceA, interfaceB): - if enableActualShellCommands: + if enable_actual_shell_commands(): if 'mq' in shellReturn('tc qdisc show dev ' + interfaceA + ' root'): print('MQ detected. Will delete and recreate mq qdisc.') # Clear tc filter - if OnAStick == True: + if on_a_stick() == True: shell('tc qdisc delete dev ' + interfaceA + ' root') else: shell('tc qdisc delete dev ' + interfaceA + ' root') @@ -84,7 +83,7 @@ def clearPriorSettings(interfaceA, interfaceB): def tearDown(interfaceA, interfaceB): # Full teardown of everything for exiting LibreQoS - if enableActualShellCommands: + if enable_actual_shell_commands(): # Clear IP filters and remove xdp program from interfaces #result = os.system('./bin/xdp_iphash_to_cpu_cmdline clear') clear_ip_mappings() # Use the bus @@ -92,8 +91,8 @@ def tearDown(interfaceA, interfaceB): def findQueuesAvailable(interfaceName): # Find queues and CPU cores available. Use min between those two as queuesAvailable - if enableActualShellCommands: - if queuesAvailableOverride == 0: + if enable_actual_shell_commands(): + if queues_available_override() == 0: queuesAvailable = 0 path = '/sys/class/net/' + interfaceName + '/queues/' directory_contents = os.listdir(path) @@ -102,7 +101,7 @@ def findQueuesAvailable(interfaceName): queuesAvailable += 1 print(f"Interface {interfaceName} NIC queues:\t\t\t" + str(queuesAvailable)) else: - queuesAvailable = queuesAvailableOverride + queuesAvailable = queues_available_override() print(f"Interface {interfaceName} NIC queues (Override):\t\t\t" + str(queuesAvailable)) cpuCount = multiprocessing.cpu_count() print("CPU cores:\t\t\t" + str(cpuCount)) @@ -298,7 +297,7 @@ def loadSubscriberCircuits(shapedDevicesFile): for row in commentsRemoved: circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row # If in monitorOnlyMode, override bandwidth rates to where no shaping will actually occur - if monitorOnlyMode == True: + if monitor_mode_only() == True: downloadMin = 10000 uploadMin = 10000 downloadMax = 10000 @@ -333,7 +332,7 @@ def loadSubscriberCircuits(shapedDevicesFile): errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID raise ValueError(errorMessageString) # Check if bandwidth parameters match other cdevices of this same circuit ID, but only check if monitorOnlyMode is Off - if monitorOnlyMode == False: + if monitor_mode_only() == False: if ((circuit['minDownload'] != round(int(downloadMin)*tcpOverheadFactor)) or (circuit['minUpload'] != round(int(uploadMin)*tcpOverheadFactor)) or (circuit['maxDownload'] != round(int(downloadMax)*tcpOverheadFactor)) @@ -427,10 +426,10 @@ def refreshShapers(): ipMapBatch = BatchedCommands() # Warn user if enableActualShellCommands is False, because that would mean no actual commands are executing - if enableActualShellCommands == False: + if enable_actual_shell_commands() == False: warnings.warn("enableActualShellCommands is set to False. None of the commands below will actually be executed. Simulated run.", stacklevel=2) # Warn user if monitorOnlyMode is True, because that would mean no actual shaping is happening - if monitorOnlyMode == True: + if monitor_mode_only() == True: warnings.warn("monitorOnlyMode is set to True. Shaping will not occur.", stacklevel=2) @@ -474,18 +473,18 @@ def refreshShapers(): # Pull rx/tx queues / CPU cores available # Handling the case when the number of queues for interfaces are different - InterfaceAQueuesAvailable = findQueuesAvailable(interfaceA) - InterfaceBQueuesAvailable = findQueuesAvailable(interfaceB) + InterfaceAQueuesAvailable = findQueuesAvailable(interface_a()) + InterfaceBQueuesAvailable = findQueuesAvailable(interface_b()) queuesAvailable = min(InterfaceAQueuesAvailable, InterfaceBQueuesAvailable) stickOffset = 0 - if OnAStick: + if on_a_stick(): print("On-a-stick override dividing queues") # The idea here is that download use queues 0 - n/2, upload uses the other half queuesAvailable = math.floor(queuesAvailable / 2) stickOffset = queuesAvailable # If in monitorOnlyMode, override network.json bandwidth rates to where no shaping will actually occur - if monitorOnlyMode == True: + if monitor_mode_only() == True: def overrideNetworkBandwidths(data): for elem in data: if 'children' in data[elem]: @@ -499,12 +498,12 @@ def refreshShapers(): generatedPNs = [] numberOfGeneratedPNs = queuesAvailable # If in monitorOnlyMode, override bandwidth rates to where no shaping will actually occur - if monitorOnlyMode == True: + if monitor_mode_only() == True: chosenDownloadMbps = 10000 chosenUploadMbps = 10000 else: - chosenDownloadMbps = generatedPNDownloadMbps - chosenUploadMbps = generatedPNDownloadMbps + chosenDownloadMbps = generated_pn_download_mbps() + chosenUploadMbps = generated_pn_upload_mbps() for x in range(numberOfGeneratedPNs): genPNname = "Generated_PN_" + str(x+1) network[genPNname] = { @@ -512,7 +511,7 @@ def refreshShapers(): "uploadBandwidthMbps": chosenUploadMbps } generatedPNs.append(genPNname) - if useBinPackingToBalanceCPU: + if use_bin_packing_to_balance_cpu(): print("Using binpacking module to sort circuits by CPU core") bins = binpacking.to_constant_bin_number(dictForCircuitsWithoutParentNodes, numberOfGeneratedPNs) genPNcounter = 0 @@ -574,7 +573,7 @@ def refreshShapers(): inheritBandwidthMaxes(data[node]['children'], data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps']) #return data # Here is the actual call to the recursive function - inheritBandwidthMaxes(network, parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps) + inheritBandwidthMaxes(network, parentMaxDL=upstream_bandwidth_capacity_download_mbps(), parentMaxUL=upstream_bandwidth_capacity_upload_mbps()) # Compress network.json. HTB only supports 8 levels of HTB depth. Compress to 8 layers if beyond 8. @@ -629,7 +628,7 @@ def refreshShapers(): data[node]['parentClassID'] = parentClassID data[node]['up_parentClassID'] = upParentClassID # If in monitorOnlyMode, override bandwidth rates to where no shaping will actually occur - if monitorOnlyMode == True: + if monitor_mode_only() == True: data[node]['downloadBandwidthMbps'] = 10000 data[node]['uploadBandwidthMbps'] = 10000 # If not in monitorOnlyMode @@ -659,7 +658,7 @@ def refreshShapers(): for circuit in subscriberCircuits: #If a device from ShapedDevices.csv lists this node as its Parent Node, attach it as a leaf to this node HTB if node == circuit['ParentNode']: - if monitorOnlyMode == False: + if monitor_mode_only() == False: if circuit['maxDownload'] > data[node]['downloadBandwidthMbps']: logging.info("downloadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2) if circuit['maxUpload'] > data[node]['uploadBandwidthMbps']: @@ -712,50 +711,50 @@ def refreshShapers(): major += 1 return minorByCPU # Here is the actual call to the recursive traverseNetwork() function. finalMinor is not used. - minorByCPU = traverseNetwork(network, 0, major=1, minorByCPU=minorByCPUpreloaded, queue=1, parentClassID=None, upParentClassID=None, parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps) + minorByCPU = traverseNetwork(network, 0, major=1, minorByCPU=minorByCPUpreloaded, queue=1, parentClassID=None, upParentClassID=None, parentMaxDL=upstream_bandwidth_capacity_download_mbps(), parentMaxUL=upstream_bandwidth_capacity_upload_mbps()) linuxTCcommands = [] devicesShaped = [] # Root HTB Setup # Create MQ qdisc for each CPU core / rx-tx queue. Generate commands to create corresponding HTB and leaf classes. Prepare commands for execution later - thisInterface = interfaceA + thisInterface = interface_a() logging.info("# MQ Setup for " + thisInterface) command = 'qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq' linuxTCcommands.append(command) for queue in range(queuesAvailable): command = 'qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2' linuxTCcommands.append(command) - command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityDownloadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityDownloadMbps) + 'mbit' + command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstream_bandwidth_capacity_download_mbps()) + 'mbit ceil ' + str(upstream_bandwidth_capacity_download_mbps()) + 'mbit' linuxTCcommands.append(command) - command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + sqm + command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + sqm() linuxTCcommands.append(command) # Default class - traffic gets passed through this limiter with lower priority if it enters the top HTB without a specific class. # Technically, that should not even happen. So don't expect much if any traffic in this default class. # Only 1/4 of defaultClassCapacity is guaranteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. - command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(round((upstreamBandwidthCapacityDownloadMbps-1)/4)) + 'mbit ceil ' + str(upstreamBandwidthCapacityDownloadMbps-1) + 'mbit prio 5' + command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(round((upstream_bandwidth_capacity_download_mbps()-1)/4)) + 'mbit ceil ' + str(upstream_bandwidth_capacity_download_mbps()-1) + 'mbit prio 5' linuxTCcommands.append(command) - command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + sqm + command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + sqm() linuxTCcommands.append(command) # Note the use of stickOffset, and not replacing the root queue if we're on a stick - thisInterface = interfaceB + thisInterface = interface_b() logging.info("# MQ Setup for " + thisInterface) - if not OnAStick: + if not on_a_stick(): command = 'qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq' linuxTCcommands.append(command) for queue in range(queuesAvailable): command = 'qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+stickOffset+1) + ' handle ' + hex(queue+stickOffset+1) + ': htb default 2' linuxTCcommands.append(command) - command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ': classid ' + hex(queue+stickOffset+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityUploadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityUploadMbps) + 'mbit' + command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ': classid ' + hex(queue+stickOffset+1) + ':1 htb rate '+ str(upstream_bandwidth_capacity_upload_mbps()) + 'mbit ceil ' + str(upstream_bandwidth_capacity_upload_mbps()) + 'mbit' linuxTCcommands.append(command) - command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ':1 ' + sqm + command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ':1 ' + sqm() linuxTCcommands.append(command) # Default class - traffic gets passed through this limiter with lower priority if it enters the top HTB without a specific class. # Technically, that should not even happen. So don't expect much if any traffic in this default class. # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. - command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ':1 classid ' + hex(queue+stickOffset+1) + ':2 htb rate ' + str(round((upstreamBandwidthCapacityUploadMbps-1)/4)) + 'mbit ceil ' + str(upstreamBandwidthCapacityUploadMbps-1) + 'mbit prio 5' + command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ':1 classid ' + hex(queue+stickOffset+1) + ':2 htb rate ' + str(round((upstream_bandwidth_capacity_upload_mbps()-1)/4)) + 'mbit ceil ' + str(upstream_bandwidth_capacity_upload_mbps()-1) + 'mbit prio 5' linuxTCcommands.append(command) - command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ':2 ' + sqm + command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ':2 ' + sqm() linuxTCcommands.append(command) @@ -767,7 +766,7 @@ def refreshShapers(): def sqmFixupRate(rate:int, sqm:str) -> str: # If we aren't using cake, just return the sqm string if not sqm.startswith("cake") or "rtt" in sqm: - return sqm + return sqm() # If we are using cake, we need to fixup the rate # Based on: 1 MTU is 1500 bytes, or 12,000 bits. # At 1 Mbps, (1,000 bits per ms) transmitting an MTU takes 12ms. Add 3ms for overhead, and we get 15ms. @@ -783,11 +782,11 @@ def refreshShapers(): case _: return sqm for node in data: - command = 'class add dev ' + interfaceA + ' parent ' + data[node]['parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ str(data[node]['downloadBandwidthMbpsMin']) + 'mbit ceil '+ str(data[node]['downloadBandwidthMbps']) + 'mbit prio 3' + command = 'class add dev ' + interface_a() + ' parent ' + data[node]['parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ str(data[node]['downloadBandwidthMbpsMin']) + 'mbit ceil '+ str(data[node]['downloadBandwidthMbps']) + 'mbit prio 3' linuxTCcommands.append(command) logging.info("Up ParentClassID: " + data[node]['up_parentClassID']) logging.info("ClassMinor: " + data[node]['classMinor']) - command = 'class add dev ' + interfaceB + ' parent ' + data[node]['up_parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ str(data[node]['uploadBandwidthMbpsMin']) + 'mbit ceil '+ str(data[node]['uploadBandwidthMbps']) + 'mbit prio 3' + command = 'class add dev ' + interface_b() + ' parent ' + data[node]['up_parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ str(data[node]['uploadBandwidthMbpsMin']) + 'mbit ceil '+ str(data[node]['uploadBandwidthMbps']) + 'mbit prio 3' linuxTCcommands.append(command) if 'circuits' in data[node]: for circuit in data[node]['circuits']: @@ -799,21 +798,21 @@ def refreshShapers(): if 'comment' in circuit['devices'][0]: tcComment = tcComment + '| Comment: ' + circuit['devices'][0]['comment'] tcComment = tcComment.replace("\n", "") - command = 'class add dev ' + interfaceA + ' parent ' + data[node]['classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ str(circuit['minDownload']) + 'mbit ceil '+ str(circuit['maxDownload']) + 'mbit prio 3' + tcComment + command = 'class add dev ' + interface_a() + ' parent ' + data[node]['classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ str(circuit['minDownload']) + 'mbit ceil '+ str(circuit['maxDownload']) + 'mbit prio 3' + tcComment linuxTCcommands.append(command) # Only add CAKE / fq_codel qdisc if monitorOnlyMode is Off - if monitorOnlyMode == False: + if monitor_mode_only() == False: # SQM Fixup for lower rates - useSqm = sqmFixupRate(circuit['maxDownload'], sqm) - command = 'qdisc add dev ' + interfaceA + ' parent ' + circuit['classMajor'] + ':' + circuit['classMinor'] + ' ' + useSqm + useSqm = sqmFixupRate(circuit['maxDownload'], sqm()) + command = 'qdisc add dev ' + interface_a() + ' parent ' + circuit['classMajor'] + ':' + circuit['classMinor'] + ' ' + useSqm linuxTCcommands.append(command) - command = 'class add dev ' + interfaceB + ' parent ' + data[node]['up_classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ str(circuit['minUpload']) + 'mbit ceil '+ str(circuit['maxUpload']) + 'mbit prio 3' + command = 'class add dev ' + interface_b() + ' parent ' + data[node]['up_classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ str(circuit['minUpload']) + 'mbit ceil '+ str(circuit['maxUpload']) + 'mbit prio 3' linuxTCcommands.append(command) # Only add CAKE / fq_codel qdisc if monitorOnlyMode is Off - if monitorOnlyMode == False: + if monitor_mode_only() == False: # SQM Fixup for lower rates - useSqm = sqmFixupRate(circuit['maxUpload'], sqm) - command = 'qdisc add dev ' + interfaceB + ' parent ' + circuit['up_classMajor'] + ':' + circuit['classMinor'] + ' ' + useSqm + useSqm = sqmFixupRate(circuit['maxUpload'], sqm()) + command = 'qdisc add dev ' + interface_b() + ' parent ' + circuit['up_classMajor'] + ':' + circuit['classMinor'] + ' ' + useSqm linuxTCcommands.append(command) pass for device in circuit['devices']: @@ -821,14 +820,14 @@ def refreshShapers(): for ipv4 in device['ipv4s']: ipMapBatch.add_ip_mapping(str(ipv4), circuit['classid'], data[node]['cpuNum'], False) #xdpCPUmapCommands.append('./bin/xdp_iphash_to_cpu_cmdline add --ip ' + str(ipv4) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['classid']) - if OnAStick: + if on_a_stick(): ipMapBatch.add_ip_mapping(str(ipv4), circuit['up_classid'], data[node]['up_cpuNum'], True) #xdpCPUmapCommands.append('./bin/xdp_iphash_to_cpu_cmdline add --ip ' + str(ipv4) + ' --cpu ' + data[node]['up_cpuNum'] + ' --classid ' + circuit['up_classid'] + ' --upload 1') if device['ipv6s']: for ipv6 in device['ipv6s']: ipMapBatch.add_ip_mapping(str(ipv6), circuit['classid'], data[node]['cpuNum'], False) #xdpCPUmapCommands.append('./bin/xdp_iphash_to_cpu_cmdline add --ip ' + str(ipv6) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['classid']) - if OnAStick: + if on_a_stick(): ipMapBatch.add_ip_mapping(str(ipv6), circuit['up_classid'], data[node]['up_cpuNum'], True) #xdpCPUmapCommands.append('./bin/xdp_iphash_to_cpu_cmdline add --ip ' + str(ipv6) + ' --cpu ' + data[node]['up_cpuNum'] + ' --classid ' + circuit['up_classid'] + ' --upload 1') if device['deviceName'] not in devicesShaped: @@ -853,12 +852,12 @@ def refreshShapers(): # Clear Prior Settings - clearPriorSettings(interfaceA, interfaceB) + clearPriorSettings(interface_a(), interface_b()) # Setup XDP and disable XPS regardless of whether it is first run or not (necessary to handle cases where systemctl stop was used) xdpStartTime = datetime.now() - if enableActualShellCommands: + if enable_actual_shell_commands(): # Here we use os.system for the command, because otherwise it sometimes gltiches out with Popen in shell() #result = os.system('./bin/xdp_iphash_to_cpu_cmdline clear') clear_ip_mappings() # Use the bus @@ -894,7 +893,7 @@ def refreshShapers(): xdpFilterStartTime = datetime.now() print("Executing XDP-CPUMAP-TC IP filter commands") numXdpCommands = ipMapBatch.length(); - if enableActualShellCommands: + if enable_actual_shell_commands(): ipMapBatch.submit() #for command in xdpCPUmapCommands: # logging.info(command) @@ -971,7 +970,7 @@ def refreshShapersUpdateOnly(): # Warn user if enableActualShellCommands is False, because that would mean no actual commands are executing - if enableActualShellCommands == False: + if enable_actual_shell_commands() == False: warnings.warn("enableActualShellCommands is set to False. None of the commands below will actually be executed. Simulated run.", stacklevel=2) @@ -1052,6 +1051,13 @@ if __name__ == '__main__': print("ERROR: lqosd is not running. Aborting") os._exit(-1) + # Check that the configuration file is usable + if check_config(): + print("Configuration from /etc/lqos.conf is usable") + else: + print("ERROR: Unable to load configuration from /etc/lqos.conf") + os.exit(-1) + # Check that we aren't running LibreQoS.py more than once at a time if is_libre_already_running(): print("LibreQoS.py is already running in another process. Aborting.") @@ -1093,10 +1099,10 @@ if __name__ == '__main__': if args.validate: status = validateNetworkAndDevices() elif args.clearrules: - tearDown(interfaceA, interfaceB) + tearDown(interface_a(), interface_b()) elif args.updateonly: # Single-interface updates don't work at all right now. - if OnAStick: + if on_a_stick(): print("--updateonly is not supported for single-interface configurations") os.exit(-1) refreshShapersUpdateOnly() diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 75506ffa..83da0774 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -1589,6 +1589,7 @@ version = "0.1.0" dependencies = [ "anyhow", "lqos_bus", + "lqos_config", "lqos_utils", "nix 0.27.1", "pyo3", diff --git a/src/rust/lqos_python/Cargo.toml b/src/rust/lqos_python/Cargo.toml index 732bf9fa..3935e5ac 100644 --- a/src/rust/lqos_python/Cargo.toml +++ b/src/rust/lqos_python/Cargo.toml @@ -12,6 +12,7 @@ crate-type = ["cdylib"] pyo3 = "0" lqos_bus = { path = "../lqos_bus" } lqos_utils = { path = "../lqos_utils" } +lqos_config = { path = "../lqos_config" } tokio = { version = "1", features = [ "full" ] } anyhow = "1" sysinfo = "0" diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index 7fe69365..b7139c78 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -32,6 +32,20 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(is_libre_already_running))?; m.add_wrapped(wrap_pyfunction!(create_lock_file))?; m.add_wrapped(wrap_pyfunction!(free_lock_file))?; + m.add_wrapped(wrap_pyfunction!(check_config))?; + m.add_wrapped(wrap_pyfunction!(sqm))?; + m.add_wrapped(wrap_pyfunction!(upstream_bandwidth_capacity_download_mbps))?; + m.add_wrapped(wrap_pyfunction!(upstream_bandwidth_capacity_upload_mbps))?; + m.add_wrapped(wrap_pyfunction!(interface_a))?; + m.add_wrapped(wrap_pyfunction!(interface_b))?; + m.add_wrapped(wrap_pyfunction!(enable_actual_shell_commands))?; + m.add_wrapped(wrap_pyfunction!(use_bin_packing_to_balance_cpu))?; + m.add_wrapped(wrap_pyfunction!(monitor_mode_only))?; + m.add_wrapped(wrap_pyfunction!(run_shell_commands_as_sudo))?; + m.add_wrapped(wrap_pyfunction!(generated_pn_download_mbps))?; + m.add_wrapped(wrap_pyfunction!(generated_pn_upload_mbps))?; + m.add_wrapped(wrap_pyfunction!(queues_available_override))?; + m.add_wrapped(wrap_pyfunction!(on_a_stick))?; Ok(()) } @@ -254,3 +268,91 @@ fn free_lock_file() -> PyResult<()> { let _ = remove_file(LOCK_FILE); // Ignore result Ok(()) } + +#[pyfunction] +fn check_config() -> PyResult { + let config = lqos_config::load_config(); + if let Err(e) = config { + println!("Error loading config: {e}"); + return Ok(false); + } + Ok(true) +} + +#[pyfunction] +fn sqm() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.queues.default_sqm.clone()) +} + +#[pyfunction] +fn upstream_bandwidth_capacity_download_mbps() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.queues.uplink_bandwidth_mbps) +} + +#[pyfunction] +fn upstream_bandwidth_capacity_upload_mbps() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.queues.uplink_bandwidth_mbps) +} + +#[pyfunction] +fn interface_a() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.isp_interface()) +} + +#[pyfunction] +fn interface_b() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.internet_interface()) +} + +#[pyfunction] +fn enable_actual_shell_commands() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(!config.queues.dry_run) +} + +#[pyfunction] +fn use_bin_packing_to_balance_cpu() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.queues.use_binpacking) +} + +#[pyfunction] +fn monitor_mode_only() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.queues.monitor_only) +} + +#[pyfunction] +fn run_shell_commands_as_sudo() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.queues.sudo) +} + +#[pyfunction] +fn generated_pn_download_mbps() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.queues.generated_pn_download_mbps) +} + +#[pyfunction] +fn generated_pn_upload_mbps() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.queues.generated_pn_upload_mbps) +} + +#[pyfunction] +fn queues_available_override() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.queues.override_available_queues.unwrap_or(0)) +} + +#[pyfunction] +fn on_a_stick() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.on_a_stick_mode()) +} \ No newline at end of file From 342176d6c2b23ccab575e42b43dbcd530563c8e3 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 8 Dec 2023 12:36:03 -0600 Subject: [PATCH 13/41] Update csvToNetworkJSON.py to use the new config --- src/csvToNetworkJSON.py | 5 ++--- src/rust/lqos_python/src/lib.rs | 7 +++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/csvToNetworkJSON.py b/src/csvToNetworkJSON.py index b5e2de76..a7ff5c2e 100644 --- a/src/csvToNetworkJSON.py +++ b/src/csvToNetworkJSON.py @@ -3,8 +3,7 @@ checkPythonVersion() import os import csv import json -from ispConfig import uispSite, uispStrategy, overwriteNetworkJSONalways -from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps +from liblqos_python import overwrite_network_json_always from integrationCommon import NetworkGraph, NetworkNode, NodeType def csvToNetworkJSONfile(): @@ -46,7 +45,7 @@ def csvToNetworkJSONfile(): net.prepareTree() net.plotNetworkGraph(False) if net.doesNetworkJsonExist(): - if overwriteNetworkJSONalways: + if overwrite_network_json_always(): net.createNetworkJson() else: print("network.json already exists and overwriteNetworkJSONalways set to False. Leaving in-place.") diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index b7139c78..a7b88c1f 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -46,6 +46,7 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(generated_pn_upload_mbps))?; m.add_wrapped(wrap_pyfunction!(queues_available_override))?; m.add_wrapped(wrap_pyfunction!(on_a_stick))?; + m.add_wrapped(wrap_pyfunction!(overwrite_network_json_always))?; Ok(()) } @@ -355,4 +356,10 @@ fn queues_available_override() -> PyResult { fn on_a_stick() -> PyResult { let config = lqos_config::load_config().unwrap(); Ok(config.on_a_stick_mode()) +} + +#[pyfunction] +fn overwrite_network_json_always() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.integration_common.always_overwrite_network_json) } \ No newline at end of file From fb638ea43957f17eb6fa93c68a5179e95f13885d Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 8 Dec 2023 13:25:49 -0600 Subject: [PATCH 14/41] Untested - integrationCommon.py ported to the new config setup --- src/integrationCommon.py | 37 ++++++++--------- src/rust/lqos_python/src/lib.rs | 71 +++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 20 deletions(-) diff --git a/src/integrationCommon.py b/src/integrationCommon.py index 0a478f24..f5e04547 100644 --- a/src/integrationCommon.py +++ b/src/integrationCommon.py @@ -2,7 +2,10 @@ # integrations. from typing import List, Any -from ispConfig import allowedSubnets, ignoreSubnets, generatedPNUploadMbps, generatedPNDownloadMbps, circuitNameUseAddress, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps +from liblqos_python import allowed_subnets, ignore_subnets, generated_pn_download_mbps, generated_pn_upload_mbps, \ + circuit_name_use_address, upstream_bandwidth_capacity_download_mbps, upstream_bandwidth_capacity_upload_mbps, \ + find_ipv6_using_mikrotik, exclude_sites, bandwidth_overhead_factor, committed_bandwidth_multiplier, \ + exception_cpes import ipaddress import enum import os @@ -12,7 +15,7 @@ def isInAllowedSubnets(inputIP): isAllowed = False if '/' in inputIP: inputIP = inputIP.split('/')[0] - for subnet in allowedSubnets: + for subnet in allowed_subnets(): if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)): isAllowed = True return isAllowed @@ -23,7 +26,7 @@ def isInIgnoredSubnets(inputIP): isIgnored = False if '/' in inputIP: inputIP = inputIP.split('/')[0] - for subnet in ignoreSubnets: + for subnet in ignore_subnets(): if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)): isIgnored = True return isIgnored @@ -98,7 +101,7 @@ class NetworkNode: address: str mac: str - def __init__(self, id: str, displayName: str = "", parentId: str = "", type: NodeType = NodeType.site, download: int = generatedPNDownloadMbps, upload: int = generatedPNUploadMbps, ipv4: List = [], ipv6: List = [], address: str = "", mac: str = "", customerName: str = "") -> None: + def __init__(self, id: str, displayName: str = "", parentId: str = "", type: NodeType = NodeType.site, download: int = generated_pn_download_mbps(), upload: int = generated_pn_upload_mbps(), ipv4: List = [], ipv6: List = [], address: str = "", mac: str = "", customerName: str = "") -> None: self.id = id self.parentIndex = 0 self.type = type @@ -129,14 +132,13 @@ class NetworkGraph: exceptionCPEs: Any def __init__(self) -> None: - from ispConfig import findIPv6usingMikrotik, excludeSites, exceptionCPEs self.nodes = [ NetworkNode("FakeRoot", type=NodeType.root, parentId="", displayName="Shaper Root") ] - self.excludeSites = excludeSites - self.exceptionCPEs = exceptionCPEs - if findIPv6usingMikrotik: + self.excludeSites = exclude_sites() + self.exceptionCPEs = exception_cpes() + if find_ipv6_using_mikrotik(): from mikrotikFindIPv6 import pullMikrotikIPv6 self.ipv4ToIPv6 = pullMikrotikIPv6() else: @@ -315,7 +317,7 @@ class NetworkGraph: data[node]['uploadBandwidthMbps'] = min(int(data[node]['uploadBandwidthMbps']),int(parentMaxUL)) if 'children' in data[node]: inheritBandwidthMaxes(data[node]['children'], data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps']) - inheritBandwidthMaxes(topLevelNode, parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps) + inheritBandwidthMaxes(topLevelNode, parentMaxDL=upstream_bandwidth_capacity_download_mbps, parentMaxUL=upstream_bandwidth_capacity_upload_mbps) with open('network.json', 'w') as f: json.dump(topLevelNode, f, indent=4) @@ -355,19 +357,14 @@ class NetworkGraph: def createShapedDevices(self): import csv - from ispConfig import bandwidthOverheadFactor - try: - from ispConfig import committedBandwidthMultiplier - except: - committedBandwidthMultiplier = 0.98 - # Builds ShapedDevices.csv from the network tree. + # Builds ShapedDevices.csv from the network tree. circuits = [] for (i, node) in enumerate(self.nodes): if node.type == NodeType.client: parent = self.nodes[node.parentIndex].displayName if parent == "Shaper Root": parent = "" - if circuitNameUseAddress: + if circuit_name_use_address(): displayNameToUse = node.address else: if node.type == NodeType.client: @@ -420,10 +417,10 @@ class NetworkGraph: device["mac"], device["ipv4"], device["ipv6"], - int(float(circuit["download"]) * committedBandwidthMultiplier), - int(float(circuit["upload"]) * committedBandwidthMultiplier), - int(float(circuit["download"]) * bandwidthOverheadFactor), - int(float(circuit["upload"]) * bandwidthOverheadFactor), + int(float(circuit["download"]) * committed_bandwidth_multiplier()), + int(float(circuit["upload"]) * committed_bandwidth_multiplier()), + int(float(circuit["download"]) * bandwidth_overhead_factor()), + int(float(circuit["upload"]) * bandwidth_overhead_factor()), "" ] wr.writerow(row) diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index a7b88c1f..ea8e5c1f 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -23,6 +23,7 @@ const LOCK_FILE: &str = "/run/lqos/libreqos.lock"; fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_wrapped(wrap_pyfunction!(is_lqosd_alive))?; m.add_wrapped(wrap_pyfunction!(list_ip_mappings))?; m.add_wrapped(wrap_pyfunction!(clear_ip_mappings))?; @@ -47,6 +48,15 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(queues_available_override))?; m.add_wrapped(wrap_pyfunction!(on_a_stick))?; m.add_wrapped(wrap_pyfunction!(overwrite_network_json_always))?; + m.add_wrapped(wrap_pyfunction!(allowed_subnets))?; + m.add_wrapped(wrap_pyfunction!(ignore_subnets))?; + m.add_wrapped(wrap_pyfunction!(circuit_name_use_address))?; + m.add_wrapped(wrap_pyfunction!(find_ipv6_using_mikrotik))?; + m.add_wrapped(wrap_pyfunction!(exclude_sites))?; + m.add_wrapped(wrap_pyfunction!(bandwidth_overhead_factor))?; + m.add_wrapped(wrap_pyfunction!(committed_bandwidth_multiplier))?; + m.add_wrapped(wrap_pyfunction!(exception_cpes))?; + Ok(()) } @@ -362,4 +372,65 @@ fn on_a_stick() -> PyResult { fn overwrite_network_json_always() -> PyResult { let config = lqos_config::load_config().unwrap(); Ok(config.integration_common.always_overwrite_network_json) +} + +#[pyfunction] +fn allowed_subnets() -> PyResult> { + let config = lqos_config::load_config().unwrap(); + Ok(config.ip_ranges.allow_subnets.clone()) +} + +#[pyfunction] +fn ignore_subnets() -> PyResult> { + let config = lqos_config::load_config().unwrap(); + Ok(config.ip_ranges.ignore_subnets.clone()) +} + +#[pyfunction] +fn circuit_name_use_address() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.integration_common.circuit_name_as_address) +} + +#[pyfunction] +fn find_ipv6_using_mikrotik() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.ipv6_with_mikrotik) +} + +#[pyfunction] +fn exclude_sites() -> PyResult> { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.exclude_sites.clone()) +} + +#[pyfunction] +fn bandwidth_overhead_factor() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.bandwidth_overhead_factor) +} + +#[pyfunction] +fn committed_bandwidth_multiplier() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.commit_bandwidth_multiplier) +} + +#[pyclass] +pub struct PyExceptionCpe { + pub cpe: String, + pub parent: String, +} + +#[pyfunction] +fn exception_cpes() -> PyResult> { + let config = lqos_config::load_config().unwrap(); + let mut result = Vec::new(); + for cpe in config.uisp_integration.exception_cpes.iter() { + result.push(PyExceptionCpe { + cpe: cpe.cpe.clone(), + parent: cpe.parent.clone(), + }); + } + Ok(result) } \ No newline at end of file From c5cd1fdf6d27f30c1a47557963067c515842058c Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 14 Dec 2023 09:51:17 -0600 Subject: [PATCH 15/41] Not tested - but the integrationUISP.py file has been ported. Testing required. --- src/integrationUISP.py | 64 +++++++------------ src/rust/lqos_config/src/etc/v15/example.toml | 1 + .../src/etc/v15/uisp_integration.rs | 2 + src/rust/lqos_python/src/lib.rs | 57 +++++++++++++++++ 4 files changed, 82 insertions(+), 42 deletions(-) diff --git a/src/integrationUISP.py b/src/integrationUISP.py index af27988a..cad47376 100644 --- a/src/integrationUISP.py +++ b/src/integrationUISP.py @@ -5,35 +5,16 @@ import os import csv from datetime import datetime, timedelta from integrationCommon import isIpv4Permitted, fixSubnet -try: - from ispConfig import uispSite, uispStrategy, overwriteNetworkJSONalways -except: - from ispConfig import uispSite, uispStrategy - overwriteNetworkJSONalways = False -try: - from ispConfig import uispSuspendedStrategy -except: - uispSuspendedStrategy = "none" -try: - from ispConfig import airMax_capacity -except: - airMax_capacity = 0.65 -try: - from ispConfig import ltu_capacity -except: - ltu_capacity = 0.90 -try: - from ispConfig import usePtMPasParent -except: - usePtMPasParent = False +from liblqos_python import uisp_site, uisp_strategy, overwrite_network_json_always, uisp_suspended_strategy, \ + airmax_capacity, ltu_capacity, use_ptmp_as_parent, uisp_base_url, uisp_auth_token, \ + generated_pn_download_mbps, generated_pn_upload_mbps def uispRequest(target): # Sends an HTTP request to UISP and returns the # result in JSON. You only need to specify the # tail end of the URL, e.g. "sites" - from ispConfig import UISPbaseURL, uispAuthToken - url = UISPbaseURL + "/nms/api/v2.1/" + target - headers = {'accept': 'application/json', 'x-auth-token': uispAuthToken} + url = uisp_base_url() + "/nms/api/v2.1/" + target + headers = {'accept': 'application/json', 'x-auth-token': uisp_auth_token()} r = requests.get(url, headers=headers, timeout=60) return r.json() @@ -41,7 +22,6 @@ def buildFlatGraph(): # Builds a high-performance (but lacking in site or AP bandwidth control) # network. from integrationCommon import NetworkGraph, NetworkNode, NodeType - from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps # Load network sites print("Loading Data from UISP") @@ -60,8 +40,8 @@ def buildFlatGraph(): customerName = '' name = site['identification']['name'] type = site['identification']['type'] - download = generatedPNDownloadMbps - upload = generatedPNUploadMbps + download = generated_pn_download_mbps() + upload = generated_pn_upload_mbps() if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']): download = int(round(site['qos']['downloadSpeed']/1000000)) upload = int(round(site['qos']['uploadSpeed']/1000000)) @@ -92,7 +72,7 @@ def buildFlatGraph(): net.prepareTree() net.plotNetworkGraph(False) if net.doesNetworkJsonExist(): - if overwriteNetworkJSONalways: + if overwrite_network_json_always(): net.createNetworkJson() else: print("network.json already exists and overwriteNetworkJSONalways set to False. Leaving in-place.") @@ -156,8 +136,8 @@ def findApCapacities(devices, siteBandwidth): if device['identification']['type'] == 'airMax': download, upload = airMaxCapacityCorrection(device, download, upload) elif device['identification']['model'] == 'LTU-Rocket': - download = download * ltu_capacity - upload = upload * ltu_capacity + download = download * ltu_capacity() + upload = upload * ltu_capacity() if device['identification']['model'] == 'WaveAP': if (download < 500) or (upload < 500): download = 2450 @@ -188,8 +168,8 @@ def airMaxCapacityCorrection(device, download, upload): upload = upload * 0.50 # Flexible frame elif dlRatio == None: - download = download * airMax_capacity - upload = upload * airMax_capacity + download = download * airmax_capacity() + upload = upload * airmax_capacity() return (download, upload) def findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps): @@ -344,7 +324,7 @@ def findNodesBranchedOffPtMP(siteList, dataLinks, sites, rootSite, foundAirFiber 'upload': upload, parent: apID } - if usePtMPasParent: + if use_ptmp_as_parent(): site['parent'] = apID print('Site ' + name + ' will use PtMP AP as parent.') return siteList, nodeOffPtMP @@ -375,7 +355,7 @@ def buildFullGraph(): # Attempts to build a full network graph, incorporating as much of the UISP # hierarchy as possible. from integrationCommon import NetworkGraph, NetworkNode, NodeType - from ispConfig import uispSite, generatedPNUploadMbps, generatedPNDownloadMbps + uispSite = uisp_site() # Load network sites print("Loading Data from UISP") @@ -397,7 +377,7 @@ def buildFullGraph(): siteList = buildSiteList(sites, dataLinks) rootSite = findInSiteList(siteList, uispSite) print("Finding PtP Capacities") - foundAirFibersBySite = findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps) + foundAirFibersBySite = findAirfibers(devices, generated_pn_download_mbps(), generated_pn_upload_mbps()) print('Creating list of route overrides') routeOverrides = loadRoutingOverrides() if rootSite is None: @@ -425,8 +405,8 @@ def buildFullGraph(): id = site['identification']['id'] name = site['identification']['name'] type = site['identification']['type'] - download = generatedPNDownloadMbps - upload = generatedPNUploadMbps + download = generated_pn_download_mbps + upload = generated_pn_upload_mbps() address = "" customerName = "" parent = findInSiteListById(siteList, id)['parent'] @@ -469,10 +449,10 @@ def buildFullGraph(): download = int(round(site['qos']['downloadSpeed']/1000000)) upload = int(round(site['qos']['uploadSpeed']/1000000)) if site['identification'] is not None and site['identification']['suspended'] is not None and site['identification']['suspended'] == True: - if uispSuspendedStrategy == "ignore": + if uisp_suspended_strategy() == "ignore": print("WARNING: Site " + name + " is suspended") continue - if uispSuspendedStrategy == "slow": + if uisp_suspended_strategy() == "slow": print("WARNING: Site " + name + " is suspended") download = 1 upload = 1 @@ -530,13 +510,13 @@ def buildFullGraph(): else: # Add some defaults in case they want to change them siteBandwidth[node.displayName] = { - "download": generatedPNDownloadMbps, "upload": generatedPNUploadMbps} + "download": generated_pn_download_mbps(), "upload": generated_pn_upload_mbps()} net.prepareTree() print('Plotting network graph') net.plotNetworkGraph(False) if net.doesNetworkJsonExist(): - if overwriteNetworkJSONalways: + if overwrite_network_json_always(): net.createNetworkJson() else: print("network.json already exists and overwriteNetworkJSONalways set to False. Leaving in-place.") @@ -558,7 +538,7 @@ def buildFullGraph(): def importFromUISP(): startTime = datetime.now() - match uispStrategy: + match uisp_strategy(): case "full": buildFullGraph() case default: buildFlatGraph() endTime = datetime.now() diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index 79e48a5f..28833bdc 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -76,3 +76,4 @@ ipv6_with_mikrotik = false bandwidth_overhead_factor = 1.0 commit_bandwidth_multiplier = 0.98 exception_cpes = [] +use_ptmp_as_parent = false diff --git a/src/rust/lqos_config/src/etc/v15/uisp_integration.rs b/src/rust/lqos_config/src/etc/v15/uisp_integration.rs index 501a427a..cda3bcde 100644 --- a/src/rust/lqos_config/src/etc/v15/uisp_integration.rs +++ b/src/rust/lqos_config/src/etc/v15/uisp_integration.rs @@ -15,6 +15,7 @@ pub struct UispIntegration { pub bandwidth_overhead_factor: f32, pub commit_bandwidth_multiplier: f32, pub exception_cpes: Vec, + pub use_ptmp_as_parent: bool, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -39,6 +40,7 @@ impl Default for UispIntegration { bandwidth_overhead_factor: 1.0, commit_bandwidth_multiplier: 1.0, exception_cpes: vec![], + use_ptmp_as_parent: false, } } } \ No newline at end of file diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index ea8e5c1f..06313fb8 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -33,6 +33,7 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(is_libre_already_running))?; m.add_wrapped(wrap_pyfunction!(create_lock_file))?; m.add_wrapped(wrap_pyfunction!(free_lock_file))?; + // Unified configuration items m.add_wrapped(wrap_pyfunction!(check_config))?; m.add_wrapped(wrap_pyfunction!(sqm))?; m.add_wrapped(wrap_pyfunction!(upstream_bandwidth_capacity_download_mbps))?; @@ -56,6 +57,14 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(bandwidth_overhead_factor))?; m.add_wrapped(wrap_pyfunction!(committed_bandwidth_multiplier))?; m.add_wrapped(wrap_pyfunction!(exception_cpes))?; + m.add_wrapped(wrap_pyfunction!(uisp_site))?; + m.add_wrapped(wrap_pyfunction!(uisp_strategy))?; + m.add_wrapped(wrap_pyfunction!(uisp_suspended_strategy))?; + m.add_wrapped(wrap_pyfunction!(airmax_capacity))?; + m.add_wrapped(wrap_pyfunction!(ltu_capacity))?; + m.add_wrapped(wrap_pyfunction!(use_ptmp_as_parent))?; + m.add_wrapped(wrap_pyfunction!(uisp_base_url))?; + m.add_wrapped(wrap_pyfunction!(uisp_auth_token))?; Ok(()) } @@ -433,4 +442,52 @@ fn exception_cpes() -> PyResult> { }); } Ok(result) +} + +#[pyfunction] +fn uisp_site() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.site) +} + +#[pyfunction] +fn uisp_strategy() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.strategy) +} + +#[pyfunction] +fn uisp_suspended_strategy() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.suspended_strategy) +} + +#[pyfunction] +fn airmax_capacity() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.airmax_capacity) +} + +#[pyfunction] +fn ltu_capacity() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.ltu_capacity) +} + +#[pyfunction] +fn use_ptmp_as_parent() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.use_ptmp_as_parent) +} + +#[pyfunction] +fn uisp_base_url() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.url) +} + +#[pyfunction] +fn uisp_auth_token() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.token) } \ No newline at end of file From 2e9f96552fcabbaece013860bf1f85b22567f7ca Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 14 Dec 2023 10:08:03 -0600 Subject: [PATCH 16/41] Patch integrationUISP and integrationCommon to run successfully. The exception CPEs system still needs fixing. --- src/integrationCommon.py | 6 ++++-- src/integrationUISP.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/integrationCommon.py b/src/integrationCommon.py index f5e04547..805e1500 100644 --- a/src/integrationCommon.py +++ b/src/integrationCommon.py @@ -149,8 +149,10 @@ class NetworkGraph: # If a site is excluded (via excludedSites in ispConfig) # it won't be added if not node.displayName in self.excludeSites: - if node.displayName in self.exceptionCPEs.keys(): - node.parentId = self.exceptionCPEs[node.displayName] + # TODO: Fixup exceptionCPE handling + #print(self.excludeSites) + #if node.displayName in self.exceptionCPEs.keys(): + # node.parentId = self.exceptionCPEs[node.displayName] self.nodes.append(node) def replaceRootNode(self, node: NetworkNode) -> None: diff --git a/src/integrationUISP.py b/src/integrationUISP.py index cad47376..d15fabac 100644 --- a/src/integrationUISP.py +++ b/src/integrationUISP.py @@ -405,7 +405,7 @@ def buildFullGraph(): id = site['identification']['id'] name = site['identification']['name'] type = site['identification']['type'] - download = generated_pn_download_mbps + download = generated_pn_download_mbps() upload = generated_pn_upload_mbps() address = "" customerName = "" From ba61f30fa49af4d37c03edf062109a1fb33698c2 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 14 Dec 2023 10:14:06 -0600 Subject: [PATCH 17/41] Update the Spylnx integration to use the new config format. --- src/integrationSplynx.py | 9 +++++---- src/rust/lqos_python/src/lib.rs | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/integrationSplynx.py b/src/integrationSplynx.py index 11feae84..53397aeb 100644 --- a/src/integrationSplynx.py +++ b/src/integrationSplynx.py @@ -2,23 +2,24 @@ from pythonCheck import checkPythonVersion checkPythonVersion() import requests import warnings -from ispConfig import excludeSites, findIPv6usingMikrotik, bandwidthOverheadFactor, exceptionCPEs, splynx_api_key, splynx_api_secret, splynx_api_url +from liblqos_python import exclude_sites, find_ipv6_using_mikrotik, bandwidth_overhead_factor, splynx_api_key, \ + splynx_api_secret, splynx_api_url from integrationCommon import isIpv4Permitted import base64 from requests.auth import HTTPBasicAuth -if findIPv6usingMikrotik == True: +if find_ipv6_using_mikrotik() == True: from mikrotikFindIPv6 import pullMikrotikIPv6 from integrationCommon import NetworkGraph, NetworkNode, NodeType def buildHeaders(): - credentials = splynx_api_key + ':' + splynx_api_secret + credentials = splynx_api_key() + ':' + splynx_api_secret() credentials = base64.b64encode(credentials.encode()).decode() return {'Authorization' : "Basic %s" % credentials} def spylnxRequest(target, headers): # Sends a REST GET request to Spylnx and returns the # result in JSON - url = splynx_api_url + "/api/2.0/" + target + url = splynx_api_url() + "/api/2.0/" + target r = requests.get(url, headers=headers, timeout=10) return r.json() diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index 06313fb8..6e35865c 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -65,6 +65,9 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(use_ptmp_as_parent))?; m.add_wrapped(wrap_pyfunction!(uisp_base_url))?; m.add_wrapped(wrap_pyfunction!(uisp_auth_token))?; + m.add_wrapped(wrap_pyfunction!(splynx_api_key))?; + m.add_wrapped(wrap_pyfunction!(splynx_api_secret))?; + m.add_wrapped(wrap_pyfunction!(splynx_api_url))?; Ok(()) } @@ -490,4 +493,22 @@ fn uisp_base_url() -> PyResult { fn uisp_auth_token() -> PyResult { let config = lqos_config::load_config().unwrap(); Ok(config.uisp_integration.token) +} + +#[pyfunction] +fn splynx_api_key() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.spylnx_integration.api_key) +} + +#[pyfunction] +fn splynx_api_secret() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.spylnx_integration.api_secret) +} + +#[pyfunction] +fn splynx_api_url() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.spylnx_integration.url) } \ No newline at end of file From adc0174e64d6afd6a6111acb01d248d1e020215b Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 14 Dec 2023 10:49:11 -0600 Subject: [PATCH 18/41] lqTools.py updated to use unified config --- src/lqTools.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lqTools.py b/src/lqTools.py index 7c3b3aaf..a1032d05 100755 --- a/src/lqTools.py +++ b/src/lqTools.py @@ -10,10 +10,11 @@ import subprocess import warnings import argparse import logging -from ispConfig import interfaceA, interfaceB, enableActualShellCommands, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps, generatedPNDownloadMbps, generatedPNUploadMbps +from liblqos_python import interface_a, interface_b, enable_actual_shell_commands, upstream_bandwidth_capacity_download_mbps, \ + upstream_bandwidth_capacity_upload_mbps, generated_pn_download_mbps, generated_pn_upload_mbps def shell(command): - if enableActualShellCommands: + if enable_actual_shell_commands(): logging.info(command) commands = command.split(' ') proc = subprocess.Popen(commands, stdout=subprocess.PIPE) @@ -24,7 +25,7 @@ def shell(command): def safeShell(command): safelyRan = True - if enableActualShellCommands: + if enable_actual_shell_commands(): commands = command.split(' ') proc = subprocess.Popen(commands, stdout=subprocess.PIPE) for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding @@ -61,7 +62,7 @@ def getQdiscForIPaddress(ipAddress): def printStatsFromIP(ipAddress): qDiscID = getQdiscForIPaddress(ipAddress) if qDiscID != None: - interfaces = [interfaceA, interfaceB] + interfaces = [interface_a(), interface_b()] for interface in interfaces: command = 'tc -s qdisc show dev ' + interface + ' parent ' + qDiscID commands = command.split(' ') @@ -77,7 +78,7 @@ def printCircuitClassInfo(ipAddress): print("IP: " + ipAddress + " | Class ID: " + qDiscID) print() theClassID = '' - interfaces = [interfaceA, interfaceB] + interfaces = [interface_a(), interface_b()] downloadMin = '' downloadMax = '' uploadMin = '' @@ -91,7 +92,7 @@ def printCircuitClassInfo(ipAddress): for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding if "htb" in line: listOfThings = line.split(" ") - if interface == interfaceA: + if interface == interface_a(): downloadMin = line.split(' rate ')[1].split(' ')[0] downloadMax = line.split(' ceil ')[1].split(' ')[0] burst = line.split(' burst ')[1].split(' ')[0] @@ -103,8 +104,8 @@ def printCircuitClassInfo(ipAddress): print("Upload rate/ceil: " + uploadMin + "/" + uploadMax) print("burst/cburst: " + burst + "/" + cburst) else: - download = min(upstreamBandwidthCapacityDownloadMbps, generatedPNDownloadMbps) - upload = min(upstreamBandwidthCapacityUploadMbps, generatedPNUploadMbps) + download = min(upstream_bandwidth_capacity_download_mbps(), generated_pn_download_mbps()) + upload = min(upstream_bandwidth_capacity_upload_mbps(), generated_pn_upload_mbps()) bwString = str(download) + '/' + str(upload) print("Invalid IP address provided (default queue limit is " + bwString + " Mbps)") From b604dba613e56d6f644390f2fcad79b3d3c7caee Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 14 Dec 2023 11:08:23 -0600 Subject: [PATCH 19/41] Update scheduler.py to use the new config, and comment out the influx stuff --- src/rust/lqos_config/src/etc/migration.rs | 2 + src/rust/lqos_config/src/etc/v15/example.toml | 1 + .../src/etc/v15/integration_common.rs | 4 ++ src/rust/lqos_python/src/lib.rs | 21 ++++++++++ src/scheduler.py | 40 +++++++++---------- 5 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs index 92079c54..c947a84c 100644 --- a/src/rust/lqos_config/src/etc/migration.rs +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -212,6 +212,8 @@ fn migrate_integration_common( new_config.integration_common.circuit_name_as_address = python_config.circuit_name_use_address; new_config.integration_common.always_overwrite_network_json = python_config.overwrite_network_json_always; + new_config.integration_common.queue_refresh_interval_mins = + python_config.queue_refresh_interval_mins; Ok(()) } diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index 28833bdc..dba70997 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -55,6 +55,7 @@ allow_subnets = [ "172.16.0.0/12", "10.0.0.0/8", "100.64.0.0/16", "192.168.0.0/1 [integration_common] circuit_name_as_address = false always_overwrite_network_json = false +queue_refresh_interval_mins = 30 [spylnx_integration] enable_spylnx = false diff --git a/src/rust/lqos_config/src/etc/v15/integration_common.rs b/src/rust/lqos_config/src/etc/v15/integration_common.rs index 0e64f881..f0cc9f3d 100644 --- a/src/rust/lqos_config/src/etc/v15/integration_common.rs +++ b/src/rust/lqos_config/src/etc/v15/integration_common.rs @@ -9,6 +9,9 @@ pub struct IntegrationConfig { /// Always overwrite network.json? pub always_overwrite_network_json: bool, + + /// Queue refresh interval in minutes + pub queue_refresh_interval_mins: u32, } impl Default for IntegrationConfig { @@ -16,6 +19,7 @@ impl Default for IntegrationConfig { Self { circuit_name_as_address: false, always_overwrite_network_json: false, + queue_refresh_interval_mins: 30, } } } \ No newline at end of file diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index 6e35865c..bfe56cd2 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -68,6 +68,9 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(splynx_api_key))?; m.add_wrapped(wrap_pyfunction!(splynx_api_secret))?; m.add_wrapped(wrap_pyfunction!(splynx_api_url))?; + m.add_wrapped(wrap_pyfunction!(automatic_import_uisp))?; + m.add_wrapped(wrap_pyfunction!(automatic_import_splynx))?; + m.add_wrapped(wrap_pyfunction!(queue_refresh_interval_mins))?; Ok(()) } @@ -511,4 +514,22 @@ fn splynx_api_secret() -> PyResult { fn splynx_api_url() -> PyResult { let config = lqos_config::load_config().unwrap(); Ok(config.spylnx_integration.url) +} + +#[pyfunction] +fn automatic_import_uisp() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.enable_uisp) +} + +#[pyfunction] +fn automatic_import_splynx() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.uisp_integration.enable_uisp) +} + +#[pyfunction] +fn queue_refresh_interval_mins() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.integration_common.queue_refresh_interval_mins) } \ No newline at end of file diff --git a/src/scheduler.py b/src/scheduler.py index f35bfd10..c99c676f 100644 --- a/src/scheduler.py +++ b/src/scheduler.py @@ -1,15 +1,11 @@ import time import datetime from LibreQoS import refreshShapers, refreshShapersUpdateOnly -from graphInfluxDB import refreshBandwidthGraphs, refreshLatencyGraphs -from ispConfig import influxDBEnabled, automaticImportUISP, automaticImportSplynx -try: - from ispConfig import queueRefreshIntervalMins -except: - queueRefreshIntervalMins = 30 -if automaticImportUISP: +#from graphInfluxDB import refreshBandwidthGraphs, refreshLatencyGraphs +from liblqos_python import automatic_import_uisp, automatic_import_splynx, queue_refresh_interval_mins +if automatic_import_uisp(): from integrationUISP import importFromUISP -if automaticImportSplynx: +if automatic_import_splynx(): from integrationSplynx import importFromSplynx from apscheduler.schedulers.background import BlockingScheduler from apscheduler.executors.pool import ThreadPoolExecutor @@ -17,26 +13,26 @@ from apscheduler.executors.pool import ThreadPoolExecutor ads = BlockingScheduler(executors={'default': ThreadPoolExecutor(1)}) def importFromCRM(): - if automaticImportUISP: + if automatic_import_uisp(): try: importFromUISP() except: print("Failed to import from UISP") - elif automaticImportSplynx: + elif automatic_import_splynx(): try: importFromSplynx() except: print("Failed to import from Splynx") -def graphHandler(): - try: - refreshBandwidthGraphs() - except: - print("Failed to update bandwidth graphs") - try: - refreshLatencyGraphs() - except: - print("Failed to update latency graphs") +#def graphHandler(): +# try: +# refreshBandwidthGraphs() +# except: +# print("Failed to update bandwidth graphs") +# try: +# refreshLatencyGraphs() +# except: +# print("Failed to update latency graphs") def importAndShapeFullReload(): importFromCRM() @@ -49,9 +45,9 @@ def importAndShapePartialReload(): if __name__ == '__main__': importAndShapeFullReload() - ads.add_job(importAndShapePartialReload, 'interval', minutes=queueRefreshIntervalMins, max_instances=1) + ads.add_job(importAndShapePartialReload, 'interval', minutes=queue_refresh_interval_mins(), max_instances=1) - if influxDBEnabled: - ads.add_job(graphHandler, 'interval', seconds=10, max_instances=1) + #if influxDBEnabled: + # ads.add_job(graphHandler, 'interval', seconds=10, max_instances=1) ads.start() From 5a10838dae7f3e91a95a13aaca2d7aaa86bc47ea Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 14 Dec 2023 11:09:55 -0600 Subject: [PATCH 20/41] Deprecate the graphInfluxDb system. --- src/graphInfluxDB.py | 1064 +++++++++++++++++++++--------------------- 1 file changed, 533 insertions(+), 531 deletions(-) diff --git a/src/graphInfluxDB.py b/src/graphInfluxDB.py index cafe7dec..327497b6 100644 --- a/src/graphInfluxDB.py +++ b/src/graphInfluxDB.py @@ -1,619 +1,621 @@ -import subprocess -import json -import subprocess -from datetime import datetime -from pathlib import Path -import statistics -import time -import psutil +print("influxDB Support is Deperecated. Use the Long-Term Stats system instead.") -from influxdb_client import InfluxDBClient, Point -from influxdb_client.client.write_api import SYNCHRONOUS +# import subprocess +# import json +# import subprocess +# from datetime import datetime +# from pathlib import Path +# import statistics +# import time +# import psutil -from ispConfig import interfaceA, interfaceB, influxDBEnabled, influxDBBucket, influxDBOrg, influxDBtoken, influxDBurl, sqm +# from influxdb_client import InfluxDBClient, Point +# from influxdb_client.client.write_api import SYNCHRONOUS + +# from ispConfig import interfaceA, interfaceB, influxDBEnabled, influxDBBucket, influxDBOrg, influxDBtoken, influxDBurl, sqm -def getInterfaceStats(interface): - command = 'tc -j -s qdisc show dev ' + interface - jsonAr = json.loads(subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8')) - jsonDict = {} - for element in filter(lambda e: 'parent' in e, jsonAr): - flowID = ':'.join(map(lambda p: f'0x{p}', element['parent'].split(':')[0:2])) - jsonDict[flowID] = element - del jsonAr - return jsonDict +# def getInterfaceStats(interface): +# command = 'tc -j -s qdisc show dev ' + interface +# jsonAr = json.loads(subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8')) +# jsonDict = {} +# for element in filter(lambda e: 'parent' in e, jsonAr): +# flowID = ':'.join(map(lambda p: f'0x{p}', element['parent'].split(':')[0:2])) +# jsonDict[flowID] = element +# del jsonAr +# return jsonDict -def chunk_list(l, n): - for i in range(0, len(l), n): - yield l[i:i + n] +# def chunk_list(l, n): +# for i in range(0, len(l), n): +# yield l[i:i + n] -def getCircuitBandwidthStats(subscriberCircuits, tinsStats): - interfaces = [interfaceA, interfaceB] - ifaceStats = list(map(getInterfaceStats, interfaces)) +# def getCircuitBandwidthStats(subscriberCircuits, tinsStats): +# interfaces = [interfaceA, interfaceB] +# ifaceStats = list(map(getInterfaceStats, interfaces)) - for circuit in subscriberCircuits: - if 'stats' not in circuit: - circuit['stats'] = {} - if 'currentQuery' in circuit['stats']: - circuit['stats']['priorQuery'] = circuit['stats']['currentQuery'] - circuit['stats']['currentQuery'] = {} - circuit['stats']['sinceLastQuery'] = {} - else: - #circuit['stats']['priorQuery'] = {} - #circuit['stats']['priorQuery']['time'] = datetime.now().isoformat() - circuit['stats']['currentQuery'] = {} - circuit['stats']['sinceLastQuery'] = {} +# for circuit in subscriberCircuits: +# if 'stats' not in circuit: +# circuit['stats'] = {} +# if 'currentQuery' in circuit['stats']: +# circuit['stats']['priorQuery'] = circuit['stats']['currentQuery'] +# circuit['stats']['currentQuery'] = {} +# circuit['stats']['sinceLastQuery'] = {} +# else: +# #circuit['stats']['priorQuery'] = {} +# #circuit['stats']['priorQuery']['time'] = datetime.now().isoformat() +# circuit['stats']['currentQuery'] = {} +# circuit['stats']['sinceLastQuery'] = {} - #for entry in tinsStats: - if 'currentQuery' in tinsStats: - tinsStats['priorQuery'] = tinsStats['currentQuery'] - tinsStats['currentQuery'] = {} - tinsStats['sinceLastQuery'] = {} - else: - tinsStats['currentQuery'] = {} - tinsStats['sinceLastQuery'] = {} +# #for entry in tinsStats: +# if 'currentQuery' in tinsStats: +# tinsStats['priorQuery'] = tinsStats['currentQuery'] +# tinsStats['currentQuery'] = {} +# tinsStats['sinceLastQuery'] = {} +# else: +# tinsStats['currentQuery'] = {} +# tinsStats['sinceLastQuery'] = {} - tinsStats['currentQuery'] = { 'Bulk': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'BestEffort': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'Video': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'Voice': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - } - tinsStats['sinceLastQuery'] = { 'Bulk': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'BestEffort': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'Video': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'Voice': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - } +# tinsStats['currentQuery'] = { 'Bulk': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, +# 'BestEffort': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, +# 'Video': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, +# 'Voice': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, +# } +# tinsStats['sinceLastQuery'] = { 'Bulk': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, +# 'BestEffort': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, +# 'Video': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, +# 'Voice': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, +# } - for circuit in subscriberCircuits: - for (interface, stats, dirSuffix) in zip(interfaces, ifaceStats, ['Download', 'Upload']): +# for circuit in subscriberCircuits: +# for (interface, stats, dirSuffix) in zip(interfaces, ifaceStats, ['Download', 'Upload']): - element = stats[circuit['classid']] if circuit['classid'] in stats else False +# element = stats[circuit['classid']] if circuit['classid'] in stats else False - if element: - bytesSent = float(element['bytes']) - drops = float(element['drops']) - packets = float(element['packets']) - if (element['drops'] > 0) and (element['packets'] > 0): - overloadFactor = float(round(element['drops']/element['packets'],3)) - else: - overloadFactor = 0.0 +# if element: +# bytesSent = float(element['bytes']) +# drops = float(element['drops']) +# packets = float(element['packets']) +# if (element['drops'] > 0) and (element['packets'] > 0): +# overloadFactor = float(round(element['drops']/element['packets'],3)) +# else: +# overloadFactor = 0.0 - if 'cake diffserv4' in sqm: - tinCounter = 1 - for tin in element['tins']: - sent_packets = float(tin['sent_packets']) - ack_drops = float(tin['ack_drops']) - ecn_mark = float(tin['ecn_mark']) - tinDrops = float(tin['drops']) - trueDrops = ecn_mark + tinDrops - ack_drops - if tinCounter == 1: - tinsStats['currentQuery']['Bulk'][dirSuffix]['sent_packets'] += sent_packets - tinsStats['currentQuery']['Bulk'][dirSuffix]['drops'] += trueDrops - elif tinCounter == 2: - tinsStats['currentQuery']['BestEffort'][dirSuffix]['sent_packets'] += sent_packets - tinsStats['currentQuery']['BestEffort'][dirSuffix]['drops'] += trueDrops - elif tinCounter == 3: - tinsStats['currentQuery']['Video'][dirSuffix]['sent_packets'] += sent_packets - tinsStats['currentQuery']['Video'][dirSuffix]['drops'] += trueDrops - elif tinCounter == 4: - tinsStats['currentQuery']['Voice'][dirSuffix]['sent_packets'] += sent_packets - tinsStats['currentQuery']['Voice'][dirSuffix]['drops'] += trueDrops - tinCounter += 1 +# if 'cake diffserv4' in sqm: +# tinCounter = 1 +# for tin in element['tins']: +# sent_packets = float(tin['sent_packets']) +# ack_drops = float(tin['ack_drops']) +# ecn_mark = float(tin['ecn_mark']) +# tinDrops = float(tin['drops']) +# trueDrops = ecn_mark + tinDrops - ack_drops +# if tinCounter == 1: +# tinsStats['currentQuery']['Bulk'][dirSuffix]['sent_packets'] += sent_packets +# tinsStats['currentQuery']['Bulk'][dirSuffix]['drops'] += trueDrops +# elif tinCounter == 2: +# tinsStats['currentQuery']['BestEffort'][dirSuffix]['sent_packets'] += sent_packets +# tinsStats['currentQuery']['BestEffort'][dirSuffix]['drops'] += trueDrops +# elif tinCounter == 3: +# tinsStats['currentQuery']['Video'][dirSuffix]['sent_packets'] += sent_packets +# tinsStats['currentQuery']['Video'][dirSuffix]['drops'] += trueDrops +# elif tinCounter == 4: +# tinsStats['currentQuery']['Voice'][dirSuffix]['sent_packets'] += sent_packets +# tinsStats['currentQuery']['Voice'][dirSuffix]['drops'] += trueDrops +# tinCounter += 1 - circuit['stats']['currentQuery']['bytesSent' + dirSuffix] = bytesSent - circuit['stats']['currentQuery']['packetDrops' + dirSuffix] = drops - circuit['stats']['currentQuery']['packetsSent' + dirSuffix] = packets - circuit['stats']['currentQuery']['overloadFactor' + dirSuffix] = overloadFactor +# circuit['stats']['currentQuery']['bytesSent' + dirSuffix] = bytesSent +# circuit['stats']['currentQuery']['packetDrops' + dirSuffix] = drops +# circuit['stats']['currentQuery']['packetsSent' + dirSuffix] = packets +# circuit['stats']['currentQuery']['overloadFactor' + dirSuffix] = overloadFactor - #if 'cake diffserv4' in sqm: - # circuit['stats']['currentQuery']['tins'] = theseTins +# #if 'cake diffserv4' in sqm: +# # circuit['stats']['currentQuery']['tins'] = theseTins - circuit['stats']['currentQuery']['time'] = datetime.now().isoformat() +# circuit['stats']['currentQuery']['time'] = datetime.now().isoformat() - allPacketsDownload = 0.0 - allPacketsUpload = 0.0 - for circuit in subscriberCircuits: - circuit['stats']['sinceLastQuery']['bitsDownload'] = circuit['stats']['sinceLastQuery']['bitsUpload'] = None - circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None - circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None - circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None +# allPacketsDownload = 0.0 +# allPacketsUpload = 0.0 +# for circuit in subscriberCircuits: +# circuit['stats']['sinceLastQuery']['bitsDownload'] = circuit['stats']['sinceLastQuery']['bitsUpload'] = None +# circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None +# circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None +# circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None - try: - if (circuit['stats']['currentQuery']['bytesSentDownload'] - circuit['stats']['priorQuery']['bytesSentDownload']) >= 0.0: - circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['currentQuery']['bytesSentDownload'] - circuit['stats']['priorQuery']['bytesSentDownload'] - else: - circuit['stats']['sinceLastQuery']['bytesSentDownload'] = None - if (circuit['stats']['currentQuery']['bytesSentUpload'] - circuit['stats']['priorQuery']['bytesSentUpload']) >= 0.0: - circuit['stats']['sinceLastQuery']['bytesSentUpload'] = circuit['stats']['currentQuery']['bytesSentUpload'] - circuit['stats']['priorQuery']['bytesSentUpload'] - else: - circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None - except: - circuit['stats']['sinceLastQuery']['bytesSentDownload'] = None - circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None - try: - if (circuit['stats']['currentQuery']['packetDropsDownload'] - circuit['stats']['priorQuery']['packetDropsDownload']) >= 0.0: - circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['currentQuery']['packetDropsDownload'] - circuit['stats']['priorQuery']['packetDropsDownload'] - else: - circuit['stats']['sinceLastQuery']['packetDropsDownload'] = None - if (circuit['stats']['currentQuery']['packetDropsUpload'] - circuit['stats']['priorQuery']['packetDropsUpload']) >= 0.0: - circuit['stats']['sinceLastQuery']['packetDropsUpload'] = circuit['stats']['currentQuery']['packetDropsUpload'] - circuit['stats']['priorQuery']['packetDropsUpload'] - else: - circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None - except: - circuit['stats']['sinceLastQuery']['packetDropsDownload'] = None - circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None - try: - if (circuit['stats']['currentQuery']['packetsSentDownload'] - circuit['stats']['priorQuery']['packetsSentDownload']) >= 0.0: - circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['currentQuery']['packetsSentDownload'] - circuit['stats']['priorQuery']['packetsSentDownload'] - else: - circuit['stats']['sinceLastQuery']['packetsSentDownload'] = None - if (circuit['stats']['currentQuery']['packetsSentUpload'] - circuit['stats']['priorQuery']['packetsSentUpload']) >= 0.0: - circuit['stats']['sinceLastQuery']['packetsSentUpload'] = circuit['stats']['currentQuery']['packetsSentUpload'] - circuit['stats']['priorQuery']['packetsSentUpload'] - else: - circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None - except: - circuit['stats']['sinceLastQuery']['packetsSentDownload'] = None - circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None +# try: +# if (circuit['stats']['currentQuery']['bytesSentDownload'] - circuit['stats']['priorQuery']['bytesSentDownload']) >= 0.0: +# circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['currentQuery']['bytesSentDownload'] - circuit['stats']['priorQuery']['bytesSentDownload'] +# else: +# circuit['stats']['sinceLastQuery']['bytesSentDownload'] = None +# if (circuit['stats']['currentQuery']['bytesSentUpload'] - circuit['stats']['priorQuery']['bytesSentUpload']) >= 0.0: +# circuit['stats']['sinceLastQuery']['bytesSentUpload'] = circuit['stats']['currentQuery']['bytesSentUpload'] - circuit['stats']['priorQuery']['bytesSentUpload'] +# else: +# circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None +# except: +# circuit['stats']['sinceLastQuery']['bytesSentDownload'] = None +# circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None +# try: +# if (circuit['stats']['currentQuery']['packetDropsDownload'] - circuit['stats']['priorQuery']['packetDropsDownload']) >= 0.0: +# circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['currentQuery']['packetDropsDownload'] - circuit['stats']['priorQuery']['packetDropsDownload'] +# else: +# circuit['stats']['sinceLastQuery']['packetDropsDownload'] = None +# if (circuit['stats']['currentQuery']['packetDropsUpload'] - circuit['stats']['priorQuery']['packetDropsUpload']) >= 0.0: +# circuit['stats']['sinceLastQuery']['packetDropsUpload'] = circuit['stats']['currentQuery']['packetDropsUpload'] - circuit['stats']['priorQuery']['packetDropsUpload'] +# else: +# circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None +# except: +# circuit['stats']['sinceLastQuery']['packetDropsDownload'] = None +# circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None +# try: +# if (circuit['stats']['currentQuery']['packetsSentDownload'] - circuit['stats']['priorQuery']['packetsSentDownload']) >= 0.0: +# circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['currentQuery']['packetsSentDownload'] - circuit['stats']['priorQuery']['packetsSentDownload'] +# else: +# circuit['stats']['sinceLastQuery']['packetsSentDownload'] = None +# if (circuit['stats']['currentQuery']['packetsSentUpload'] - circuit['stats']['priorQuery']['packetsSentUpload']) >= 0.0: +# circuit['stats']['sinceLastQuery']['packetsSentUpload'] = circuit['stats']['currentQuery']['packetsSentUpload'] - circuit['stats']['priorQuery']['packetsSentUpload'] +# else: +# circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None +# except: +# circuit['stats']['sinceLastQuery']['packetsSentDownload'] = None +# circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None - if(circuit['stats']['sinceLastQuery']['packetsSentDownload']): - allPacketsDownload += circuit['stats']['sinceLastQuery']['packetsSentDownload'] - if(circuit['stats']['sinceLastQuery']['packetsSentUpload']): - allPacketsUpload += circuit['stats']['sinceLastQuery']['packetsSentUpload'] +# if(circuit['stats']['sinceLastQuery']['packetsSentDownload']): +# allPacketsDownload += circuit['stats']['sinceLastQuery']['packetsSentDownload'] +# if(circuit['stats']['sinceLastQuery']['packetsSentUpload']): +# allPacketsUpload += circuit['stats']['sinceLastQuery']['packetsSentUpload'] - if 'priorQuery' in circuit['stats']: - if 'time' in circuit['stats']['priorQuery']: - currentQueryTime = datetime.fromisoformat(circuit['stats']['currentQuery']['time']) - priorQueryTime = datetime.fromisoformat(circuit['stats']['priorQuery']['time']) - deltaSeconds = (currentQueryTime - priorQueryTime).total_seconds() - if (circuit['stats']['sinceLastQuery']['bytesSentDownload']): - circuit['stats']['sinceLastQuery']['bitsDownload'] = round((circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 - else: - circuit['stats']['sinceLastQuery']['bitsDownload'] = None - if (circuit['stats']['sinceLastQuery']['bytesSentUpload']): - circuit['stats']['sinceLastQuery']['bitsUpload'] = round((circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 - else: - circuit['stats']['sinceLastQuery']['bitsUpload'] = None +# if 'priorQuery' in circuit['stats']: +# if 'time' in circuit['stats']['priorQuery']: +# currentQueryTime = datetime.fromisoformat(circuit['stats']['currentQuery']['time']) +# priorQueryTime = datetime.fromisoformat(circuit['stats']['priorQuery']['time']) +# deltaSeconds = (currentQueryTime - priorQueryTime).total_seconds() +# if (circuit['stats']['sinceLastQuery']['bytesSentDownload']): +# circuit['stats']['sinceLastQuery']['bitsDownload'] = round((circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 +# else: +# circuit['stats']['sinceLastQuery']['bitsDownload'] = None +# if (circuit['stats']['sinceLastQuery']['bytesSentUpload']): +# circuit['stats']['sinceLastQuery']['bitsUpload'] = round((circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 +# else: +# circuit['stats']['sinceLastQuery']['bitsUpload'] = None - else: - circuit['stats']['sinceLastQuery']['bitsDownload'] = None - if(circuit['stats']['sinceLastQuery']['bytesSentDownload']): - circuit['stats']['sinceLastQuery']['bitsDownload'] = (circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) - circuit['stats']['sinceLastQuery']['bitsUpload'] = None - if(circuit['stats']['sinceLastQuery']['bytesSentUpload']): - circuit['stats']['sinceLastQuery']['bitsUpload'] = (circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) +# else: +# circuit['stats']['sinceLastQuery']['bitsDownload'] = None +# if(circuit['stats']['sinceLastQuery']['bytesSentDownload']): +# circuit['stats']['sinceLastQuery']['bitsDownload'] = (circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) +# circuit['stats']['sinceLastQuery']['bitsUpload'] = None +# if(circuit['stats']['sinceLastQuery']['bytesSentUpload']): +# circuit['stats']['sinceLastQuery']['bitsUpload'] = (circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) - tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = 0.0 - tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = 0.0 - tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = 0.0 - tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = 0.0 +# tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = 0.0 +# tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = 0.0 +# tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = 0.0 +# tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = 0.0 - try: - tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] = tinsStats['currentQuery']['Bulk']['Download']['sent_packets'] - tinsStats['priorQuery']['Bulk']['Download']['sent_packets'] - tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] = tinsStats['currentQuery']['BestEffort']['Download']['sent_packets'] - tinsStats['priorQuery']['BestEffort']['Download']['sent_packets'] - tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] = tinsStats['currentQuery']['Video']['Download']['sent_packets'] - tinsStats['priorQuery']['Video']['Download']['sent_packets'] - tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] = tinsStats['currentQuery']['Voice']['Download']['sent_packets'] - tinsStats['priorQuery']['Voice']['Download']['sent_packets'] - tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] = tinsStats['currentQuery']['Bulk']['Upload']['sent_packets'] - tinsStats['priorQuery']['Bulk']['Upload']['sent_packets'] - tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] = tinsStats['currentQuery']['BestEffort']['Upload']['sent_packets'] - tinsStats['priorQuery']['BestEffort']['Upload']['sent_packets'] - tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] = tinsStats['currentQuery']['Video']['Upload']['sent_packets'] - tinsStats['priorQuery']['Video']['Upload']['sent_packets'] - tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] = tinsStats['currentQuery']['Voice']['Upload']['sent_packets'] - tinsStats['priorQuery']['Voice']['Upload']['sent_packets'] - except: - tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] = tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] = 0.0 - tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] = tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] = 0.0 - tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] = 0.0 - tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] = tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] = 0.0 +# try: +# tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] = tinsStats['currentQuery']['Bulk']['Download']['sent_packets'] - tinsStats['priorQuery']['Bulk']['Download']['sent_packets'] +# tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] = tinsStats['currentQuery']['BestEffort']['Download']['sent_packets'] - tinsStats['priorQuery']['BestEffort']['Download']['sent_packets'] +# tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] = tinsStats['currentQuery']['Video']['Download']['sent_packets'] - tinsStats['priorQuery']['Video']['Download']['sent_packets'] +# tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] = tinsStats['currentQuery']['Voice']['Download']['sent_packets'] - tinsStats['priorQuery']['Voice']['Download']['sent_packets'] +# tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] = tinsStats['currentQuery']['Bulk']['Upload']['sent_packets'] - tinsStats['priorQuery']['Bulk']['Upload']['sent_packets'] +# tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] = tinsStats['currentQuery']['BestEffort']['Upload']['sent_packets'] - tinsStats['priorQuery']['BestEffort']['Upload']['sent_packets'] +# tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] = tinsStats['currentQuery']['Video']['Upload']['sent_packets'] - tinsStats['priorQuery']['Video']['Upload']['sent_packets'] +# tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] = tinsStats['currentQuery']['Voice']['Upload']['sent_packets'] - tinsStats['priorQuery']['Voice']['Upload']['sent_packets'] +# except: +# tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] = tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] = 0.0 +# tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] = tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] = 0.0 +# tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] = 0.0 +# tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] = tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] = 0.0 - try: - tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] = tinsStats['currentQuery']['Bulk']['Download']['drops'] - tinsStats['priorQuery']['Bulk']['Download']['drops'] - tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] = tinsStats['currentQuery']['BestEffort']['Download']['drops'] - tinsStats['priorQuery']['BestEffort']['Download']['drops'] - tinsStats['sinceLastQuery']['Video']['Download']['drops'] = tinsStats['currentQuery']['Video']['Download']['drops'] - tinsStats['priorQuery']['Video']['Download']['drops'] - tinsStats['sinceLastQuery']['Voice']['Download']['drops'] = tinsStats['currentQuery']['Voice']['Download']['drops'] - tinsStats['priorQuery']['Voice']['Download']['drops'] - tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] = tinsStats['currentQuery']['Bulk']['Upload']['drops'] - tinsStats['priorQuery']['Bulk']['Upload']['drops'] - tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] = tinsStats['currentQuery']['BestEffort']['Upload']['drops'] - tinsStats['priorQuery']['BestEffort']['Upload']['drops'] - tinsStats['sinceLastQuery']['Video']['Upload']['drops'] = tinsStats['currentQuery']['Video']['Upload']['drops'] - tinsStats['priorQuery']['Video']['Upload']['drops'] - tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] = tinsStats['currentQuery']['Voice']['Upload']['drops'] - tinsStats['priorQuery']['Voice']['Upload']['drops'] - except: - tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] = tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] = 0.0 - tinsStats['sinceLastQuery']['Video']['Download']['drops'] = tinsStats['sinceLastQuery']['Voice']['Download']['drops'] = 0.0 - tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] = 0.0 - tinsStats['sinceLastQuery']['Video']['Upload']['drops'] = tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] = 0.0 +# try: +# tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] = tinsStats['currentQuery']['Bulk']['Download']['drops'] - tinsStats['priorQuery']['Bulk']['Download']['drops'] +# tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] = tinsStats['currentQuery']['BestEffort']['Download']['drops'] - tinsStats['priorQuery']['BestEffort']['Download']['drops'] +# tinsStats['sinceLastQuery']['Video']['Download']['drops'] = tinsStats['currentQuery']['Video']['Download']['drops'] - tinsStats['priorQuery']['Video']['Download']['drops'] +# tinsStats['sinceLastQuery']['Voice']['Download']['drops'] = tinsStats['currentQuery']['Voice']['Download']['drops'] - tinsStats['priorQuery']['Voice']['Download']['drops'] +# tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] = tinsStats['currentQuery']['Bulk']['Upload']['drops'] - tinsStats['priorQuery']['Bulk']['Upload']['drops'] +# tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] = tinsStats['currentQuery']['BestEffort']['Upload']['drops'] - tinsStats['priorQuery']['BestEffort']['Upload']['drops'] +# tinsStats['sinceLastQuery']['Video']['Upload']['drops'] = tinsStats['currentQuery']['Video']['Upload']['drops'] - tinsStats['priorQuery']['Video']['Upload']['drops'] +# tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] = tinsStats['currentQuery']['Voice']['Upload']['drops'] - tinsStats['priorQuery']['Voice']['Upload']['drops'] +# except: +# tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] = tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] = 0.0 +# tinsStats['sinceLastQuery']['Video']['Download']['drops'] = tinsStats['sinceLastQuery']['Voice']['Download']['drops'] = 0.0 +# tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] = 0.0 +# tinsStats['sinceLastQuery']['Video']['Upload']['drops'] = tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] = 0.0 - try: - dlPerc = tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] / tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] - ulPerc = tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] / tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] - tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) - tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) +# try: +# dlPerc = tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] / tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] +# ulPerc = tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] / tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] +# tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) +# tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) - dlPerc = tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] / tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] - ulPerc = tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] / tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] - tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) - tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) +# dlPerc = tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] / tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] +# ulPerc = tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] / tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] +# tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) +# tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) - dlPerc = tinsStats['sinceLastQuery']['Video']['Download']['drops'] / tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] - ulPerc = tinsStats['sinceLastQuery']['Video']['Upload']['drops'] / tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] - tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) - tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) +# dlPerc = tinsStats['sinceLastQuery']['Video']['Download']['drops'] / tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] +# ulPerc = tinsStats['sinceLastQuery']['Video']['Upload']['drops'] / tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] +# tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) +# tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) - dlPerc = tinsStats['sinceLastQuery']['Voice']['Download']['drops'] / tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] - ulPerc = tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] / tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] - tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) - tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) - except: - tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = 0.0 - tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = 0.0 +# dlPerc = tinsStats['sinceLastQuery']['Voice']['Download']['drops'] / tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] +# ulPerc = tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] / tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] +# tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) +# tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) +# except: +# tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = 0.0 +# tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = 0.0 - try: - tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) - tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) - tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) - tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) - tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Video']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) - tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) - tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) - tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) - except: - # To avoid graphing 0.0 for all categories, which would show unusual graph results upon each queue reload, we just set these to None if the above calculations fail. - tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = None - tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = None - tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = None - tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = None +# try: +# tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) +# tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) +# tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) +# tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) +# tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Video']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) +# tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) +# tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) +# tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) +# except: +# # To avoid graphing 0.0 for all categories, which would show unusual graph results upon each queue reload, we just set these to None if the above calculations fail. +# tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = None +# tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = None +# tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = None +# tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = None - return subscriberCircuits, tinsStats +# return subscriberCircuits, tinsStats -def getParentNodeBandwidthStats(parentNodes, subscriberCircuits): - for parentNode in parentNodes: - thisNodeDropsDownload = 0 - thisNodeDropsUpload = 0 - thisNodeDropsTotal = 0 - thisNodeBitsDownload = 0 - thisNodeBitsUpload = 0 - packetsSentDownloadAggregate = 0.0 - packetsSentUploadAggregate = 0.0 - packetsSentTotalAggregate = 0.0 - circuitsMatched = 0 - thisParentNodeStats = {'sinceLastQuery': {}} - for circuit in subscriberCircuits: - if circuit['ParentNode'] == parentNode['parentNodeName']: - if circuit['stats']['sinceLastQuery']['bitsDownload']: - thisNodeBitsDownload += circuit['stats']['sinceLastQuery']['bitsDownload'] - if circuit['stats']['sinceLastQuery']['bitsUpload']: - thisNodeBitsUpload += circuit['stats']['sinceLastQuery']['bitsUpload'] - #thisNodeDropsDownload += circuit['packetDropsDownloadSinceLastQuery'] - #thisNodeDropsUpload += circuit['packetDropsUploadSinceLastQuery'] - if circuit['stats']['sinceLastQuery']['packetDropsDownload'] and circuit['stats']['sinceLastQuery']['packetDropsUpload']: - thisNodeDropsTotal += (circuit['stats']['sinceLastQuery']['packetDropsDownload'] + circuit['stats']['sinceLastQuery']['packetDropsUpload']) - if circuit['stats']['sinceLastQuery']['packetsSentDownload']: - packetsSentDownloadAggregate += circuit['stats']['sinceLastQuery']['packetsSentDownload'] - if circuit['stats']['sinceLastQuery']['packetsSentUpload']: - packetsSentUploadAggregate += circuit['stats']['sinceLastQuery']['packetsSentUpload'] - if circuit['stats']['sinceLastQuery']['packetsSentDownload'] and circuit['stats']['sinceLastQuery']['packetsSentUpload']: - packetsSentTotalAggregate += (circuit['stats']['sinceLastQuery']['packetsSentDownload'] + circuit['stats']['sinceLastQuery']['packetsSentUpload']) - circuitsMatched += 1 - if (packetsSentDownloadAggregate > 0) and (packetsSentUploadAggregate > 0) and (packetsSentTotalAggregate > 0): - #overloadFactorDownloadSinceLastQuery = float(round((thisNodeDropsDownload/packetsSentDownloadAggregate)*100.0, 3)) - #overloadFactorUploadSinceLastQuery = float(round((thisNodeDropsUpload/packetsSentUploadAggregate)*100.0, 3)) - overloadFactorTotalSinceLastQuery = float(round((thisNodeDropsTotal/packetsSentTotalAggregate)*100.0, 1)) - else: - #overloadFactorDownloadSinceLastQuery = 0.0 - #overloadFactorUploadSinceLastQuery = 0.0 - overloadFactorTotalSinceLastQuery = 0.0 +# def getParentNodeBandwidthStats(parentNodes, subscriberCircuits): +# for parentNode in parentNodes: +# thisNodeDropsDownload = 0 +# thisNodeDropsUpload = 0 +# thisNodeDropsTotal = 0 +# thisNodeBitsDownload = 0 +# thisNodeBitsUpload = 0 +# packetsSentDownloadAggregate = 0.0 +# packetsSentUploadAggregate = 0.0 +# packetsSentTotalAggregate = 0.0 +# circuitsMatched = 0 +# thisParentNodeStats = {'sinceLastQuery': {}} +# for circuit in subscriberCircuits: +# if circuit['ParentNode'] == parentNode['parentNodeName']: +# if circuit['stats']['sinceLastQuery']['bitsDownload']: +# thisNodeBitsDownload += circuit['stats']['sinceLastQuery']['bitsDownload'] +# if circuit['stats']['sinceLastQuery']['bitsUpload']: +# thisNodeBitsUpload += circuit['stats']['sinceLastQuery']['bitsUpload'] +# #thisNodeDropsDownload += circuit['packetDropsDownloadSinceLastQuery'] +# #thisNodeDropsUpload += circuit['packetDropsUploadSinceLastQuery'] +# if circuit['stats']['sinceLastQuery']['packetDropsDownload'] and circuit['stats']['sinceLastQuery']['packetDropsUpload']: +# thisNodeDropsTotal += (circuit['stats']['sinceLastQuery']['packetDropsDownload'] + circuit['stats']['sinceLastQuery']['packetDropsUpload']) +# if circuit['stats']['sinceLastQuery']['packetsSentDownload']: +# packetsSentDownloadAggregate += circuit['stats']['sinceLastQuery']['packetsSentDownload'] +# if circuit['stats']['sinceLastQuery']['packetsSentUpload']: +# packetsSentUploadAggregate += circuit['stats']['sinceLastQuery']['packetsSentUpload'] +# if circuit['stats']['sinceLastQuery']['packetsSentDownload'] and circuit['stats']['sinceLastQuery']['packetsSentUpload']: +# packetsSentTotalAggregate += (circuit['stats']['sinceLastQuery']['packetsSentDownload'] + circuit['stats']['sinceLastQuery']['packetsSentUpload']) +# circuitsMatched += 1 +# if (packetsSentDownloadAggregate > 0) and (packetsSentUploadAggregate > 0) and (packetsSentTotalAggregate > 0): +# #overloadFactorDownloadSinceLastQuery = float(round((thisNodeDropsDownload/packetsSentDownloadAggregate)*100.0, 3)) +# #overloadFactorUploadSinceLastQuery = float(round((thisNodeDropsUpload/packetsSentUploadAggregate)*100.0, 3)) +# overloadFactorTotalSinceLastQuery = float(round((thisNodeDropsTotal/packetsSentTotalAggregate)*100.0, 1)) +# else: +# #overloadFactorDownloadSinceLastQuery = 0.0 +# #overloadFactorUploadSinceLastQuery = 0.0 +# overloadFactorTotalSinceLastQuery = 0.0 - thisParentNodeStats['sinceLastQuery']['bitsDownload'] = thisNodeBitsDownload - thisParentNodeStats['sinceLastQuery']['bitsUpload'] = thisNodeBitsUpload - thisParentNodeStats['sinceLastQuery']['packetDropsTotal'] = thisNodeDropsTotal - thisParentNodeStats['sinceLastQuery']['overloadFactorTotal'] = overloadFactorTotalSinceLastQuery - parentNode['stats'] = thisParentNodeStats +# thisParentNodeStats['sinceLastQuery']['bitsDownload'] = thisNodeBitsDownload +# thisParentNodeStats['sinceLastQuery']['bitsUpload'] = thisNodeBitsUpload +# thisParentNodeStats['sinceLastQuery']['packetDropsTotal'] = thisNodeDropsTotal +# thisParentNodeStats['sinceLastQuery']['overloadFactorTotal'] = overloadFactorTotalSinceLastQuery +# parentNode['stats'] = thisParentNodeStats - return parentNodes +# return parentNodes -def getParentNodeLatencyStats(parentNodes, subscriberCircuits): - for parentNode in parentNodes: - if 'stats' not in parentNode: - parentNode['stats'] = {} - parentNode['stats']['sinceLastQuery'] = {} +# def getParentNodeLatencyStats(parentNodes, subscriberCircuits): +# for parentNode in parentNodes: +# if 'stats' not in parentNode: +# parentNode['stats'] = {} +# parentNode['stats']['sinceLastQuery'] = {} - for parentNode in parentNodes: - thisParentNodeStats = {'sinceLastQuery': {}} - circuitsMatchedLatencies = [] - for circuit in subscriberCircuits: - if circuit['ParentNode'] == parentNode['parentNodeName']: - if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: - circuitsMatchedLatencies.append(circuit['stats']['sinceLastQuery']['tcpLatency']) - if len(circuitsMatchedLatencies) > 0: - thisParentNodeStats['sinceLastQuery']['tcpLatency'] = statistics.median(circuitsMatchedLatencies) - else: - thisParentNodeStats['sinceLastQuery']['tcpLatency'] = None - parentNode['stats'] = thisParentNodeStats - return parentNodes +# for parentNode in parentNodes: +# thisParentNodeStats = {'sinceLastQuery': {}} +# circuitsMatchedLatencies = [] +# for circuit in subscriberCircuits: +# if circuit['ParentNode'] == parentNode['parentNodeName']: +# if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: +# circuitsMatchedLatencies.append(circuit['stats']['sinceLastQuery']['tcpLatency']) +# if len(circuitsMatchedLatencies) > 0: +# thisParentNodeStats['sinceLastQuery']['tcpLatency'] = statistics.median(circuitsMatchedLatencies) +# else: +# thisParentNodeStats['sinceLastQuery']['tcpLatency'] = None +# parentNode['stats'] = thisParentNodeStats +# return parentNodes -def getCircuitLatencyStats(subscriberCircuits): - command = './bin/xdp_pping' - consoleOutput = subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8') - consoleOutput = consoleOutput.replace('\n','').replace('}{', '}, {') - listOfEntries = json.loads(consoleOutput) +# def getCircuitLatencyStats(subscriberCircuits): +# command = './bin/xdp_pping' +# consoleOutput = subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8') +# consoleOutput = consoleOutput.replace('\n','').replace('}{', '}, {') +# listOfEntries = json.loads(consoleOutput) - tcpLatencyForClassID = {} - for entry in listOfEntries: - if 'tc' in entry: - handle = '0x' + entry['tc'].split(':')[0] + ':' + '0x' + entry['tc'].split(':')[1] - # To avoid outliers messing up avg for each circuit - cap at ceiling of 200ms - ceiling = 200.0 - tcpLatencyForClassID[handle] = min(entry['median'], ceiling) - for circuit in subscriberCircuits: - if 'stats' not in circuit: - circuit['stats'] = {} - circuit['stats']['sinceLastQuery'] = {} +# tcpLatencyForClassID = {} +# for entry in listOfEntries: +# if 'tc' in entry: +# handle = '0x' + entry['tc'].split(':')[0] + ':' + '0x' + entry['tc'].split(':')[1] +# # To avoid outliers messing up avg for each circuit - cap at ceiling of 200ms +# ceiling = 200.0 +# tcpLatencyForClassID[handle] = min(entry['median'], ceiling) +# for circuit in subscriberCircuits: +# if 'stats' not in circuit: +# circuit['stats'] = {} +# circuit['stats']['sinceLastQuery'] = {} - for circuit in subscriberCircuits: - classID = circuit['classid'] - if classID in tcpLatencyForClassID: - circuit['stats']['sinceLastQuery']['tcpLatency'] = tcpLatencyForClassID[classID] - else: - # If we can't identify RTT this time around, use most recently recorded RTT - # None by default, change if found in priorQuery - circuit['stats']['sinceLastQuery']['tcpLatency'] = None - if 'priorQuery' in circuit['stats']: - if circuit['stats']['priorQuery'] != None: - if 'priorQuery' in circuit['stats']: - if 'tcpLatency' in circuit['stats']['priorQuery']: - circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency'] +# for circuit in subscriberCircuits: +# classID = circuit['classid'] +# if classID in tcpLatencyForClassID: +# circuit['stats']['sinceLastQuery']['tcpLatency'] = tcpLatencyForClassID[classID] +# else: +# # If we can't identify RTT this time around, use most recently recorded RTT +# # None by default, change if found in priorQuery +# circuit['stats']['sinceLastQuery']['tcpLatency'] = None +# if 'priorQuery' in circuit['stats']: +# if circuit['stats']['priorQuery'] != None: +# if 'priorQuery' in circuit['stats']: +# if 'tcpLatency' in circuit['stats']['priorQuery']: +# circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency'] - return subscriberCircuits +# return subscriberCircuits -def getParentNodeDict(data, depth, parentNodeNameDict): - if parentNodeNameDict == None: - parentNodeNameDict = {} +# def getParentNodeDict(data, depth, parentNodeNameDict): +# if parentNodeNameDict == None: +# parentNodeNameDict = {} - for elem in data: - if 'children' in data[elem]: - for child in data[elem]['children']: - parentNodeNameDict[child] = elem - tempDict = getParentNodeDict(data[elem]['children'], depth + 1, parentNodeNameDict) - parentNodeNameDict = dict(parentNodeNameDict, **tempDict) - return parentNodeNameDict +# for elem in data: +# if 'children' in data[elem]: +# for child in data[elem]['children']: +# parentNodeNameDict[child] = elem +# tempDict = getParentNodeDict(data[elem]['children'], depth + 1, parentNodeNameDict) +# parentNodeNameDict = dict(parentNodeNameDict, **tempDict) +# return parentNodeNameDict -def parentNodeNameDictPull(): - # Load network hierarchy - with open('network.json', 'r') as j: - network = json.loads(j.read()) - parentNodeNameDict = getParentNodeDict(network, 0, None) - return parentNodeNameDict +# def parentNodeNameDictPull(): +# # Load network hierarchy +# with open('network.json', 'r') as j: +# network = json.loads(j.read()) +# parentNodeNameDict = getParentNodeDict(network, 0, None) +# return parentNodeNameDict -def refreshBandwidthGraphs(): - startTime = datetime.now() - with open('statsByParentNode.json', 'r') as j: - parentNodes = json.loads(j.read()) +# def refreshBandwidthGraphs(): +# startTime = datetime.now() +# with open('statsByParentNode.json', 'r') as j: +# parentNodes = json.loads(j.read()) - with open('statsByCircuit.json', 'r') as j: - subscriberCircuits = json.loads(j.read()) +# with open('statsByCircuit.json', 'r') as j: +# subscriberCircuits = json.loads(j.read()) - fileLoc = Path("tinsStats.json") - if fileLoc.is_file(): - with open(fileLoc, 'r') as j: - tinsStats = json.loads(j.read()) - else: - tinsStats = {} +# fileLoc = Path("tinsStats.json") +# if fileLoc.is_file(): +# with open(fileLoc, 'r') as j: +# tinsStats = json.loads(j.read()) +# else: +# tinsStats = {} - fileLoc = Path("longTermStats.json") - if fileLoc.is_file(): - with open(fileLoc, 'r') as j: - longTermStats = json.loads(j.read()) - droppedPacketsAllTime = longTermStats['droppedPacketsTotal'] - else: - longTermStats = {} - longTermStats['droppedPacketsTotal'] = 0.0 - droppedPacketsAllTime = 0.0 +# fileLoc = Path("longTermStats.json") +# if fileLoc.is_file(): +# with open(fileLoc, 'r') as j: +# longTermStats = json.loads(j.read()) +# droppedPacketsAllTime = longTermStats['droppedPacketsTotal'] +# else: +# longTermStats = {} +# longTermStats['droppedPacketsTotal'] = 0.0 +# droppedPacketsAllTime = 0.0 - parentNodeNameDict = parentNodeNameDictPull() +# parentNodeNameDict = parentNodeNameDictPull() - print("Retrieving circuit statistics") - subscriberCircuits, tinsStats = getCircuitBandwidthStats(subscriberCircuits, tinsStats) - print("Computing parent node statistics") - parentNodes = getParentNodeBandwidthStats(parentNodes, subscriberCircuits) - print("Writing data to InfluxDB") - client = InfluxDBClient( - url=influxDBurl, - token=influxDBtoken, - org=influxDBOrg - ) +# print("Retrieving circuit statistics") +# subscriberCircuits, tinsStats = getCircuitBandwidthStats(subscriberCircuits, tinsStats) +# print("Computing parent node statistics") +# parentNodes = getParentNodeBandwidthStats(parentNodes, subscriberCircuits) +# print("Writing data to InfluxDB") +# client = InfluxDBClient( +# url=influxDBurl, +# token=influxDBtoken, +# org=influxDBOrg +# ) - # Record current timestamp, use for all points added - timestamp = time.time_ns() - write_api = client.write_api(write_options=SYNCHRONOUS) +# # Record current timestamp, use for all points added +# timestamp = time.time_ns() +# write_api = client.write_api(write_options=SYNCHRONOUS) - chunkedsubscriberCircuits = list(chunk_list(subscriberCircuits, 200)) +# chunkedsubscriberCircuits = list(chunk_list(subscriberCircuits, 200)) - queriesToSendCount = 0 - for chunk in chunkedsubscriberCircuits: - seenSomethingBesides0s = False - queriesToSend = [] - for circuit in chunk: - bitsDownloadMin = float(circuit['minDownload']) * 1000000 if circuit['minDownload'] else None - bitsDownloadMax = float(circuit['maxDownload']) * 1000000 if circuit['maxDownload'] else None - bitsUploadMin = float(circuit['minUpload']) * 1000000 if circuit['minUpload'] else None - bitsUploadMax = float(circuit['maxUpload']) * 1000000 if circuit['maxUpload'] else None - bitsDownload = float(circuit['stats']['sinceLastQuery']['bitsDownload']) if circuit['stats']['sinceLastQuery']['bitsDownload'] else None - bitsUpload = float(circuit['stats']['sinceLastQuery']['bitsUpload']) if circuit['stats']['sinceLastQuery']['bitsUpload'] else None - bytesSentDownload = float(circuit['stats']['sinceLastQuery']['bytesSentDownload']) if circuit['stats']['sinceLastQuery']['bytesSentDownload'] else None - bytesSentUpload = float(circuit['stats']['sinceLastQuery']['bytesSentUpload']) if circuit['stats']['sinceLastQuery']['bytesSentUpload'] else None - percentUtilizationDownload = round((bitsDownload / round(circuit['maxDownload'] * 1000000))*100.0, 1) if bitsDownload and circuit['maxDownload'] else None - percentUtilizationUpload = round((bitsUpload / round(circuit['maxUpload'] * 1000000))*100.0, 1) if bitsUpload and circuit['maxUpload'] else None - if bitsDownload and bitsUpload: - if (bitsDownload > 0.0) or (bitsUpload > 0.0): - seenSomethingBesides0s = True - p = Point('Bandwidth').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", bitsDownload).field("Upload", bitsUpload).time(timestamp) - queriesToSend.append(p) - p = Point('Utilization').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) - queriesToSend.append(p) +# queriesToSendCount = 0 +# for chunk in chunkedsubscriberCircuits: +# seenSomethingBesides0s = False +# queriesToSend = [] +# for circuit in chunk: +# bitsDownloadMin = float(circuit['minDownload']) * 1000000 if circuit['minDownload'] else None +# bitsDownloadMax = float(circuit['maxDownload']) * 1000000 if circuit['maxDownload'] else None +# bitsUploadMin = float(circuit['minUpload']) * 1000000 if circuit['minUpload'] else None +# bitsUploadMax = float(circuit['maxUpload']) * 1000000 if circuit['maxUpload'] else None +# bitsDownload = float(circuit['stats']['sinceLastQuery']['bitsDownload']) if circuit['stats']['sinceLastQuery']['bitsDownload'] else None +# bitsUpload = float(circuit['stats']['sinceLastQuery']['bitsUpload']) if circuit['stats']['sinceLastQuery']['bitsUpload'] else None +# bytesSentDownload = float(circuit['stats']['sinceLastQuery']['bytesSentDownload']) if circuit['stats']['sinceLastQuery']['bytesSentDownload'] else None +# bytesSentUpload = float(circuit['stats']['sinceLastQuery']['bytesSentUpload']) if circuit['stats']['sinceLastQuery']['bytesSentUpload'] else None +# percentUtilizationDownload = round((bitsDownload / round(circuit['maxDownload'] * 1000000))*100.0, 1) if bitsDownload and circuit['maxDownload'] else None +# percentUtilizationUpload = round((bitsUpload / round(circuit['maxUpload'] * 1000000))*100.0, 1) if bitsUpload and circuit['maxUpload'] else None +# if bitsDownload and bitsUpload: +# if (bitsDownload > 0.0) or (bitsUpload > 0.0): +# seenSomethingBesides0s = True +# p = Point('Bandwidth').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", bitsDownload).field("Upload", bitsUpload).time(timestamp) +# queriesToSend.append(p) +# p = Point('Utilization').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) +# queriesToSend.append(p) - if seenSomethingBesides0s: - write_api.write(bucket=influxDBBucket, record=queriesToSend) - # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") - queriesToSendCount += len(queriesToSend) +# if seenSomethingBesides0s: +# write_api.write(bucket=influxDBBucket, record=queriesToSend) +# # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") +# queriesToSendCount += len(queriesToSend) - queriesToSend = [] - seenSomethingBesides0s = False - for parentNode in parentNodes: - bitsDownload = float(parentNode['stats']['sinceLastQuery']['bitsDownload']) - bitsUpload = float(parentNode['stats']['sinceLastQuery']['bitsUpload']) - dropsTotal = float(parentNode['stats']['sinceLastQuery']['packetDropsTotal']) - overloadFactor = float(parentNode['stats']['sinceLastQuery']['overloadFactorTotal']) - droppedPacketsAllTime += dropsTotal - percentUtilizationDownload = round((bitsDownload / round(parentNode['maxDownload'] * 1000000))*100.0, 1) - percentUtilizationUpload = round((bitsUpload / round(parentNode['maxUpload'] * 1000000))*100.0, 1) - if bitsDownload and bitsUpload: - if (bitsDownload > 0.0) or (bitsUpload > 0.0): - seenSomethingBesides0s = True - p = Point('Bandwidth').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", bitsDownload).field("Upload", bitsUpload).time(timestamp) - queriesToSend.append(p) - p = Point('Utilization').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) - queriesToSend.append(p) - p = Point('Overload').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Overload", overloadFactor).time(timestamp) - queriesToSend.append(p) +# queriesToSend = [] +# seenSomethingBesides0s = False +# for parentNode in parentNodes: +# bitsDownload = float(parentNode['stats']['sinceLastQuery']['bitsDownload']) +# bitsUpload = float(parentNode['stats']['sinceLastQuery']['bitsUpload']) +# dropsTotal = float(parentNode['stats']['sinceLastQuery']['packetDropsTotal']) +# overloadFactor = float(parentNode['stats']['sinceLastQuery']['overloadFactorTotal']) +# droppedPacketsAllTime += dropsTotal +# percentUtilizationDownload = round((bitsDownload / round(parentNode['maxDownload'] * 1000000))*100.0, 1) +# percentUtilizationUpload = round((bitsUpload / round(parentNode['maxUpload'] * 1000000))*100.0, 1) +# if bitsDownload and bitsUpload: +# if (bitsDownload > 0.0) or (bitsUpload > 0.0): +# seenSomethingBesides0s = True +# p = Point('Bandwidth').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", bitsDownload).field("Upload", bitsUpload).time(timestamp) +# queriesToSend.append(p) +# p = Point('Utilization').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) +# queriesToSend.append(p) +# p = Point('Overload').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Overload", overloadFactor).time(timestamp) +# queriesToSend.append(p) - if seenSomethingBesides0s: - write_api.write(bucket=influxDBBucket, record=queriesToSend) - # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") - queriesToSendCount += len(queriesToSend) +# if seenSomethingBesides0s: +# write_api.write(bucket=influxDBBucket, record=queriesToSend) +# # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") +# queriesToSendCount += len(queriesToSend) - if 'cake diffserv4' in sqm: - seenSomethingBesides0s = False - queriesToSend = [] - listOfTins = ['Bulk', 'BestEffort', 'Video', 'Voice'] - for tin in listOfTins: - p = Point('Tin Drop Percentage').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['dropPercentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['dropPercentage']).time(timestamp) - queriesToSend.append(p) - # Check to ensure tin percentage has value (!= None) before graphing. During partial or full reload these will have a value of None. - if (tinsStats['sinceLastQuery'][tin]['Download']['percentage'] != None) and (tinsStats['sinceLastQuery'][tin]['Upload']['percentage'] != None): - if (tinsStats['sinceLastQuery'][tin]['Download']['percentage'] > 0.0) or (tinsStats['sinceLastQuery'][tin]['Upload']['percentage'] > 0.0): - seenSomethingBesides0s = True - p = Point('Tins Assigned').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['percentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['percentage']).time(timestamp) - queriesToSend.append(p) +# if 'cake diffserv4' in sqm: +# seenSomethingBesides0s = False +# queriesToSend = [] +# listOfTins = ['Bulk', 'BestEffort', 'Video', 'Voice'] +# for tin in listOfTins: +# p = Point('Tin Drop Percentage').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['dropPercentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['dropPercentage']).time(timestamp) +# queriesToSend.append(p) +# # Check to ensure tin percentage has value (!= None) before graphing. During partial or full reload these will have a value of None. +# if (tinsStats['sinceLastQuery'][tin]['Download']['percentage'] != None) and (tinsStats['sinceLastQuery'][tin]['Upload']['percentage'] != None): +# if (tinsStats['sinceLastQuery'][tin]['Download']['percentage'] > 0.0) or (tinsStats['sinceLastQuery'][tin]['Upload']['percentage'] > 0.0): +# seenSomethingBesides0s = True +# p = Point('Tins Assigned').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['percentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['percentage']).time(timestamp) +# queriesToSend.append(p) - if seenSomethingBesides0s: - write_api.write(bucket=influxDBBucket, record=queriesToSend) - # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") - queriesToSendCount += len(queriesToSend) +# if seenSomethingBesides0s: +# write_api.write(bucket=influxDBBucket, record=queriesToSend) +# # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") +# queriesToSendCount += len(queriesToSend) - # Graph CPU use - cpuVals = psutil.cpu_percent(percpu=True) - queriesToSend = [] - for index, item in enumerate(cpuVals): - p = Point('CPU').field('CPU_' + str(index), item) - queriesToSend.append(p) - write_api.write(bucket=influxDBBucket, record=queriesToSend) - queriesToSendCount += len(queriesToSend) +# # Graph CPU use +# cpuVals = psutil.cpu_percent(percpu=True) +# queriesToSend = [] +# for index, item in enumerate(cpuVals): +# p = Point('CPU').field('CPU_' + str(index), item) +# queriesToSend.append(p) +# write_api.write(bucket=influxDBBucket, record=queriesToSend) +# queriesToSendCount += len(queriesToSend) - print("Added " + str(queriesToSendCount) + " points to InfluxDB.") +# print("Added " + str(queriesToSendCount) + " points to InfluxDB.") - client.close() +# client.close() - with open('statsByParentNode.json', 'w') as f: - f.write(json.dumps(parentNodes, indent=4)) +# with open('statsByParentNode.json', 'w') as f: +# f.write(json.dumps(parentNodes, indent=4)) - with open('statsByCircuit.json', 'w') as f: - f.write(json.dumps(subscriberCircuits, indent=4)) +# with open('statsByCircuit.json', 'w') as f: +# f.write(json.dumps(subscriberCircuits, indent=4)) - longTermStats['droppedPacketsTotal'] = droppedPacketsAllTime - with open('longTermStats.json', 'w') as f: - f.write(json.dumps(longTermStats, indent=4)) +# longTermStats['droppedPacketsTotal'] = droppedPacketsAllTime +# with open('longTermStats.json', 'w') as f: +# f.write(json.dumps(longTermStats, indent=4)) - with open('tinsStats.json', 'w') as f: - f.write(json.dumps(tinsStats, indent=4)) +# with open('tinsStats.json', 'w') as f: +# f.write(json.dumps(tinsStats, indent=4)) - endTime = datetime.now() - durationSeconds = round((endTime - startTime).total_seconds(), 2) - print("Graphs updated within " + str(durationSeconds) + " seconds.") +# endTime = datetime.now() +# durationSeconds = round((endTime - startTime).total_seconds(), 2) +# print("Graphs updated within " + str(durationSeconds) + " seconds.") -def refreshLatencyGraphs(): - startTime = datetime.now() - with open('statsByParentNode.json', 'r') as j: - parentNodes = json.loads(j.read()) +# def refreshLatencyGraphs(): +# startTime = datetime.now() +# with open('statsByParentNode.json', 'r') as j: +# parentNodes = json.loads(j.read()) - with open('statsByCircuit.json', 'r') as j: - subscriberCircuits = json.loads(j.read()) +# with open('statsByCircuit.json', 'r') as j: +# subscriberCircuits = json.loads(j.read()) - parentNodeNameDict = parentNodeNameDictPull() +# parentNodeNameDict = parentNodeNameDictPull() - print("Retrieving circuit statistics") - subscriberCircuits = getCircuitLatencyStats(subscriberCircuits) - print("Computing parent node statistics") - parentNodes = getParentNodeLatencyStats(parentNodes, subscriberCircuits) - print("Writing data to InfluxDB") - client = InfluxDBClient( - url=influxDBurl, - token=influxDBtoken, - org=influxDBOrg - ) +# print("Retrieving circuit statistics") +# subscriberCircuits = getCircuitLatencyStats(subscriberCircuits) +# print("Computing parent node statistics") +# parentNodes = getParentNodeLatencyStats(parentNodes, subscriberCircuits) +# print("Writing data to InfluxDB") +# client = InfluxDBClient( +# url=influxDBurl, +# token=influxDBtoken, +# org=influxDBOrg +# ) - # Record current timestamp, use for all points added - timestamp = time.time_ns() +# # Record current timestamp, use for all points added +# timestamp = time.time_ns() - write_api = client.write_api(write_options=SYNCHRONOUS) +# write_api = client.write_api(write_options=SYNCHRONOUS) - chunkedsubscriberCircuits = list(chunk_list(subscriberCircuits, 200)) +# chunkedsubscriberCircuits = list(chunk_list(subscriberCircuits, 200)) - queriesToSendCount = 0 - for chunk in chunkedsubscriberCircuits: - queriesToSend = [] - for circuit in chunk: - if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: - tcpLatency = float(circuit['stats']['sinceLastQuery']['tcpLatency']) - p = Point('TCP Latency').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("TCP Latency", tcpLatency).time(timestamp) - queriesToSend.append(p) - write_api.write(bucket=influxDBBucket, record=queriesToSend) - queriesToSendCount += len(queriesToSend) +# queriesToSendCount = 0 +# for chunk in chunkedsubscriberCircuits: +# queriesToSend = [] +# for circuit in chunk: +# if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: +# tcpLatency = float(circuit['stats']['sinceLastQuery']['tcpLatency']) +# p = Point('TCP Latency').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("TCP Latency", tcpLatency).time(timestamp) +# queriesToSend.append(p) +# write_api.write(bucket=influxDBBucket, record=queriesToSend) +# queriesToSendCount += len(queriesToSend) - queriesToSend = [] - for parentNode in parentNodes: - if parentNode['stats']['sinceLastQuery']['tcpLatency'] != None: - tcpLatency = float(parentNode['stats']['sinceLastQuery']['tcpLatency']) - p = Point('TCP Latency').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("TCP Latency", tcpLatency).time(timestamp) - queriesToSend.append(p) +# queriesToSend = [] +# for parentNode in parentNodes: +# if parentNode['stats']['sinceLastQuery']['tcpLatency'] != None: +# tcpLatency = float(parentNode['stats']['sinceLastQuery']['tcpLatency']) +# p = Point('TCP Latency').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("TCP Latency", tcpLatency).time(timestamp) +# queriesToSend.append(p) - write_api.write(bucket=influxDBBucket, record=queriesToSend) - queriesToSendCount += len(queriesToSend) +# write_api.write(bucket=influxDBBucket, record=queriesToSend) +# queriesToSendCount += len(queriesToSend) - listOfAllLatencies = [] - for circuit in subscriberCircuits: - if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: - listOfAllLatencies.append(circuit['stats']['sinceLastQuery']['tcpLatency']) - if len(listOfAllLatencies) > 0: - currentNetworkLatency = statistics.median(listOfAllLatencies) - p = Point('TCP Latency').tag("Type", "Network").field("TCP Latency", currentNetworkLatency).time(timestamp) - write_api.write(bucket=influxDBBucket, record=p) - queriesToSendCount += 1 +# listOfAllLatencies = [] +# for circuit in subscriberCircuits: +# if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: +# listOfAllLatencies.append(circuit['stats']['sinceLastQuery']['tcpLatency']) +# if len(listOfAllLatencies) > 0: +# currentNetworkLatency = statistics.median(listOfAllLatencies) +# p = Point('TCP Latency').tag("Type", "Network").field("TCP Latency", currentNetworkLatency).time(timestamp) +# write_api.write(bucket=influxDBBucket, record=p) +# queriesToSendCount += 1 - print("Added " + str(queriesToSendCount) + " points to InfluxDB.") +# print("Added " + str(queriesToSendCount) + " points to InfluxDB.") - client.close() +# client.close() - with open('statsByParentNode.json', 'w') as f: - f.write(json.dumps(parentNodes, indent=4)) +# with open('statsByParentNode.json', 'w') as f: +# f.write(json.dumps(parentNodes, indent=4)) - with open('statsByCircuit.json', 'w') as f: - f.write(json.dumps(subscriberCircuits, indent=4)) +# with open('statsByCircuit.json', 'w') as f: +# f.write(json.dumps(subscriberCircuits, indent=4)) - endTime = datetime.now() - durationSeconds = round((endTime - startTime).total_seconds(), 2) - print("Graphs updated within " + str(durationSeconds) + " seconds.") +# endTime = datetime.now() +# durationSeconds = round((endTime - startTime).total_seconds(), 2) +# print("Graphs updated within " + str(durationSeconds) + " seconds.") -if __name__ == '__main__': - refreshBandwidthGraphs() - refreshLatencyGraphs() +# if __name__ == '__main__': +# refreshBandwidthGraphs() +# refreshLatencyGraphs() From f9fe8fef02fc8e2a5aaf9a806aaefc543c426fef Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 14 Dec 2023 13:58:24 -0600 Subject: [PATCH 21/41] Update lqos_setup to use the combined config format. --- src/rust/Cargo.lock | 2 + src/rust/lqos_config/src/etc/mod.rs | 2 +- src/rust/lqos_config/src/lib.rs | 2 +- src/rust/lqos_setup/Cargo.toml | 2 + src/rust/lqos_setup/src/main.rs | 116 ++++++---------------------- 5 files changed, 28 insertions(+), 96 deletions(-) diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 83da0774..801d968a 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -1626,6 +1626,8 @@ version = "0.1.0" dependencies = [ "colored", "default-net", + "lqos_config", + "toml 0.8.8", "uuid", ] diff --git a/src/rust/lqos_config/src/etc/mod.rs b/src/rust/lqos_config/src/etc/mod.rs index bb64e2c6..54d49b30 100644 --- a/src/rust/lqos_config/src/etc/mod.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -11,7 +11,7 @@ mod python_migration; #[cfg(test)] pub mod test_data; mod v15; -pub use v15::Tunables; +pub use v15::{Tunables, BridgeConfig}; static CONFIG: Mutex> = Mutex::new(None); diff --git a/src/rust/lqos_config/src/lib.rs b/src/rust/lqos_config/src/lib.rs index 947cc737..eb905e9c 100644 --- a/src/rust/lqos_config/src/lib.rs +++ b/src/rust/lqos_config/src/lib.rs @@ -13,7 +13,7 @@ mod program_control; mod shaped_devices; pub use authentication::{UserRole, WebUsers}; -pub use etc::{load_config, Config, enable_long_term_stats, Tunables}; +pub use etc::{load_config, Config, enable_long_term_stats, Tunables, BridgeConfig}; pub use network_json::{NetworkJson, NetworkJsonNode, NetworkJsonTransport}; pub use program_control::load_libreqos; pub use shaped_devices::{ConfigShapedDevices, ShapedDevice}; diff --git a/src/rust/lqos_setup/Cargo.toml b/src/rust/lqos_setup/Cargo.toml index 629339f1..dc40f38d 100644 --- a/src/rust/lqos_setup/Cargo.toml +++ b/src/rust/lqos_setup/Cargo.toml @@ -8,3 +8,5 @@ license = "GPL-2.0-only" colored = "2" default-net = "0" # For obtaining an easy-to-use NIC list uuid = { version = "1", features = ["v4", "fast-rng" ] } +lqos_config = { path = "../lqos_config" } +toml = "0.8.8" diff --git a/src/rust/lqos_setup/src/main.rs b/src/rust/lqos_setup/src/main.rs index 24504391..8e81c95e 100644 --- a/src/rust/lqos_setup/src/main.rs +++ b/src/rust/lqos_setup/src/main.rs @@ -52,7 +52,6 @@ pub fn read_line_as_number() -> u32 { } const LQOS_CONF: &str = "/etc/lqos.conf"; -const ISP_CONF: &str = "/opt/libreqos/src/ispConfig.py"; const NETWORK_JSON: &str = "/opt/libreqos/src/network.json"; const SHAPED_DEVICES: &str = "/opt/libreqos/src/ShapedDevices.csv"; const LQUSERS: &str = "/opt/libreqos/src/lqusers.toml"; @@ -116,82 +115,6 @@ fn get_bandwidth(up: bool) -> u32 { } } -const ETC_LQOS_CONF: &str = "lqos_directory = '/opt/libreqos/src' -queue_check_period_ms = 1000 -node_id = \"{NODE_ID}\" - -[tuning] -stop_irq_balance = true -netdev_budget_usecs = 8000 -netdev_budget_packets = 300 -rx_usecs = 8 -tx_usecs = 8 -disable_rxvlan = true -disable_txvlan = true -disable_offload = [ \"gso\", \"tso\", \"lro\", \"sg\", \"gro\" ] - -[bridge] -use_xdp_bridge = true -interface_mapping = [ - { name = \"{INTERNET}\", redirect_to = \"{ISP}\", scan_vlans = false }, - { name = \"{ISP}\", redirect_to = \"{INTERNET}\", scan_vlans = false } -] -vlan_mapping = [] - -[usage_stats] -send_anonymous = {ALLOW_ANONYMOUS} -anonymous_server = \"stats.libreqos.io:9125\" -"; - -fn write_etc_lqos_conf(internet: &str, isp: &str, allow_anonymous: bool) { - let new_id = Uuid::new_v4().to_string(); - let output = - ETC_LQOS_CONF.replace("{INTERNET}", internet).replace("{ISP}", isp) - .replace("{NODE_ID}", &new_id) - .replace("{ALLOW_ANONYMOUS}", &allow_anonymous.to_string()); - fs::write(LQOS_CONF, output).expect("Unable to write file"); -} - -pub fn write_isp_config_py( - dir: &str, - download: u32, - upload: u32, - lan: &str, - internet: &str, -) { - // Copy ispConfig.example.py to ispConfig.py - let orig = format!("{dir}ispConfig.example.py"); - let dest = format!("{dir}ispConfig.py"); - std::fs::copy(orig, &dest).unwrap(); - - let config_file = std::fs::read_to_string(&dest).unwrap(); - let mut new_config_file = String::new(); - config_file.split('\n').for_each(|line| { - if line.starts_with('#') { - new_config_file += line; - new_config_file += "\n"; - } else if line.contains("upstreamBandwidthCapacityDownloadMbps") { - new_config_file += - &format!("upstreamBandwidthCapacityDownloadMbps = {download}\n"); - } else if line.contains("upstreamBandwidthCapacityUploadMbps") { - new_config_file += - &format!("upstreamBandwidthCapacityUploadMbps = {upload}\n"); - } else if line.contains("interfaceA") { - new_config_file += &format!("interfaceA = \"{lan}\"\n"); - } else if line.contains("interfaceB") { - new_config_file += &format!("interfaceB = \"{internet}\"\n"); - } else if line.contains("generatedPNDownloadMbps") { - new_config_file += &format!("generatedPNDownloadMbps = {download}\n"); - } else if line.contains("generatedPNUploadMbps") { - new_config_file += &format!("generatedPNUploadMbps = {upload}\n"); - } else { - new_config_file += line; - new_config_file += "\n"; - } - }); - std::fs::write(&dest, new_config_file).unwrap(); -} - fn write_network_json() { let output = "{}\n"; fs::write(NETWORK_JSON, output).expect("Unable to write file"); @@ -223,6 +146,26 @@ fn anonymous() -> bool { } } +fn write_combined_config( + to_internet: &str, + to_network: &str, + download: u32, + upload: u32, + allow_anonymous: bool, +) { + let mut config = lqos_config::Config::default(); + config.node_id = lqos_config::Config::calculate_node_id(); + config.single_interface = None; + config.bridge = Some(lqos_config::BridgeConfig { use_xdp_bridge:true, to_internet: to_internet.to_string(), to_network: to_network.to_string() }); + config.queues.downlink_bandwidth_mbps = download; + config.queues.uplink_bandwidth_mbps = upload; + config.queues.generated_pn_download_mbps = download; + config.queues.generated_pn_upload_mbps = upload; + config.usage_stats.send_anonymous = allow_anonymous; + let raw = toml::to_string_pretty(&config).unwrap(); + std::fs::write("/etc/lqos.conf", raw).unwrap(); +} + fn main() { println!("{:^80}", "LibreQoS 1.4 Setup Assistant".yellow().on_blue()); println!(); @@ -237,26 +180,11 @@ fn main() { ); get_internet_interface(&interfaces, &mut if_internet); get_isp_interface(&interfaces, &mut if_isp); - let allow_anonymous = anonymous(); - if let (Some(internet), Some(isp)) = (&if_internet, &if_isp) { - write_etc_lqos_conf(internet, isp, allow_anonymous); - } - } - - if should_build(ISP_CONF) { - println!("{}{}", ISP_CONF.cyan(), "does not exist, building one.".white()); - get_internet_interface(&interfaces, &mut if_internet); - get_isp_interface(&interfaces, &mut if_isp); let upload = get_bandwidth(true); let download = get_bandwidth(false); + let allow_anonymous = anonymous(); if let (Some(internet), Some(isp)) = (&if_internet, &if_isp) { - write_isp_config_py( - "/opt/libreqos/src/", - download, - upload, - isp, - internet, - ) + write_combined_config(internet, isp, download, upload, allow_anonymous); } } From a0f04840cc36243e5156e1f78942881facaa8a6a Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 8 Jan 2024 10:12:22 -0600 Subject: [PATCH 22/41] Update powercode integration to use the unified config --- src/integrationPowercode.py | 12 +++++------ src/rust/lqos_config/src/etc/v15/mod.rs | 1 + .../lqos_config/src/etc/v15/top_config.rs | 4 ++++ src/rust/lqos_python/src/lib.rs | 21 +++++++++++++++++++ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/integrationPowercode.py b/src/integrationPowercode.py index a1ed97df..756fcba2 100644 --- a/src/integrationPowercode.py +++ b/src/integrationPowercode.py @@ -2,20 +2,20 @@ from pythonCheck import checkPythonVersion checkPythonVersion() import requests import warnings -from ispConfig import excludeSites, findIPv6usingMikrotik, bandwidthOverheadFactor, exceptionCPEs, powercode_api_key, powercode_api_url +from liblqos_python import find_ipv6_using_mikrotik, powercode_api_key, powercode_api_url from integrationCommon import isIpv4Permitted import base64 from requests.auth import HTTPBasicAuth -if findIPv6usingMikrotik == True: +if find_ipv6_using_mikrotik() == True: from mikrotikFindIPv6 import pullMikrotikIPv6 from integrationCommon import NetworkGraph, NetworkNode, NodeType from urllib3.exceptions import InsecureRequestWarning def getCustomerInfo(): headers= {'Content-Type': 'application/x-www-form-urlencoded'} - url = powercode_api_url + ":444/api/preseem/index.php" + url = powercode_api_url() + ":444/api/preseem/index.php" data = {} - data['apiKey'] = powercode_api_key + data['apiKey'] = powercode_api_key() data['action'] = 'list_customers' r = requests.post(url, data=data, headers=headers, verify=False, timeout=10) @@ -23,9 +23,9 @@ def getCustomerInfo(): def getListServices(): headers= {'Content-Type': 'application/x-www-form-urlencoded'} - url = powercode_api_url + ":444/api/preseem/index.php" + url = powercode_api_url() + ":444/api/preseem/index.php" data = {} - data['apiKey'] = powercode_api_key + data['apiKey'] = powercode_api_key() data['action'] = 'list_services' r = requests.post(url, data=data, headers=headers, verify=False, timeout=10) diff --git a/src/rust/lqos_config/src/etc/v15/mod.rs b/src/rust/lqos_config/src/etc/v15/mod.rs index a6e08d90..83420406 100644 --- a/src/rust/lqos_config/src/etc/v15/mod.rs +++ b/src/rust/lqos_config/src/etc/v15/mod.rs @@ -11,6 +11,7 @@ mod integration_common; mod ip_ranges; mod spylnx_integration; mod uisp_integration; +mod powercode_integration; pub use bridge::*; pub use long_term_stats::LongTermStats; pub use tuning::Tunables; \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index 8d8da1bb..a855b24e 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -58,6 +58,9 @@ pub struct Config { /// UISP Integration pub uisp_integration: super::uisp_integration::UispIntegration, + + /// Powercode Integration + pub powercode_integration: super::powercode_integration::PowercodeIntegration, } impl Config { @@ -116,6 +119,7 @@ impl Default for Config { integration_common: super::integration_common::IntegrationConfig::default(), spylnx_integration: super::spylnx_integration::SplynxIntegration::default(), uisp_integration: super::uisp_integration::UispIntegration::default(), + powercode_integration: super::powercode_integration::PowercodeIntegration::default(), packet_capture_time: 10, queue_check_period_ms: 1000, } diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index bfe56cd2..b4e9ccb3 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -71,6 +71,9 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(automatic_import_uisp))?; m.add_wrapped(wrap_pyfunction!(automatic_import_splynx))?; m.add_wrapped(wrap_pyfunction!(queue_refresh_interval_mins))?; + m.add_wrapped(wrap_pyfunction!(automatic_import_powercode))?; + m.add_wrapped(wrap_pyfunction!(powercode_api_key))?; + m.add_wrapped(wrap_pyfunction!(powercode_api_url))?; Ok(()) } @@ -532,4 +535,22 @@ fn automatic_import_splynx() -> PyResult { fn queue_refresh_interval_mins() -> PyResult { let config = lqos_config::load_config().unwrap(); Ok(config.integration_common.queue_refresh_interval_mins) +} + +#[pyfunction] +fn automatic_import_powercode() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.powercode_integration.enable_powercode) +} + +#[pyfunction] +fn powercode_api_key() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.powercode_integration.powercode_api_key) +} + +#[pyfunction] +fn powercode_api_url() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.powercode_integration.powercode_api_url) } \ No newline at end of file From 69baca106149e96807600fde8fd0d11339d32186 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 11 Jan 2024 08:43:01 -0600 Subject: [PATCH 23/41] File missed from previous commit --- .../src/etc/v15/powercode_integration.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/rust/lqos_config/src/etc/v15/powercode_integration.rs diff --git a/src/rust/lqos_config/src/etc/v15/powercode_integration.rs b/src/rust/lqos_config/src/etc/v15/powercode_integration.rs new file mode 100644 index 00000000..9f032cc4 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/powercode_integration.rs @@ -0,0 +1,18 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PowercodeIntegration { + pub enable_powercode: bool, + pub powercode_api_key: String, + pub powercode_api_url: String, +} + +impl Default for PowercodeIntegration { + fn default() -> Self { + PowercodeIntegration { + enable_powercode: false, + powercode_api_key: "".to_string(), + powercode_api_url: "".to_string(), + } + } +} \ No newline at end of file From d9acd551f01f5bdb5cb2ac28028f98191d0ebd8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Thu, 18 Jan 2024 09:58:09 -0700 Subject: [PATCH 24/41] Update quickstart-prereq.md --- docs/Quickstart/quickstart-prereq.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/Quickstart/quickstart-prereq.md b/docs/Quickstart/quickstart-prereq.md index 43576b62..0be54e39 100644 --- a/docs/Quickstart/quickstart-prereq.md +++ b/docs/Quickstart/quickstart-prereq.md @@ -27,7 +27,8 @@ There are two options for the bridge to pass data through your two interfaces: - Bifrost XDP-Accelerated Bridge - Regular Linux Bridge -The Bifrost Bridge is faster and generally recommended, but may not work perfectly in a VM setup using virtualized NICs. +The Bifrost Bridge is recommended for Intel NICs with XDP support, such as the X520 and X710. +The regular Linux bridge is recommended for Nvidea/Mellanox NICs such as the ConnectX-5 series (which have superior bridge performance), and VM setups using virtualized NICs. To use the Bifrost bridge, skip the regular Linux bridge section below, and be sure to enable Bifrost/XDP in lqos.conf a few sections below. ### Adding a regular Linux bridge (if not using Bifrost XDP bridge) From d39322eaaf5f05b44e58a05190b1674b5c1ce2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Thu, 18 Jan 2024 10:06:05 -0700 Subject: [PATCH 25/41] Update Compute.md --- docs/SystemRequirements/Compute.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/SystemRequirements/Compute.md b/docs/SystemRequirements/Compute.md index 2e27dcb1..b775c7be 100644 --- a/docs/SystemRequirements/Compute.md +++ b/docs/SystemRequirements/Compute.md @@ -14,8 +14,8 @@ Single-thread CPU performance will determine the max throughput of a single HTB | 250 Mbps | 1250 | | 500 Mbps | 1500 | | 1 Gbps | 2000 | -| 2 Gbps | 3000 | -| 4 Gbps | 4000 | +| 3 Gbps | 3000 | +| 10 Gbps | 4000 | Below is a table of approximate aggregate throughput capacity, assuming a a CPU with a [single thread](https://www.cpubenchmark.net/singleThread.html#server-thread) performance of 2700 or greater: @@ -26,8 +26,8 @@ Below is a table of approximate aggregate throughput capacity, assuming a a CPU | 5 Gbps | 6 | | 10 Gbps | 8 | | 20 Gbps | 16 | -| 50 Gbps* | 32 | - +| 50 Gbps | 32 | +| 100 Gbps * | 64 | (* Estimated) So for example, an ISP delivering 1Gbps service plans with 10Gbps aggregate throughput would choose a CPU with a 2500+ single-thread score and 8 cores, such as the Intel Xeon E-2388G @ 3.20GHz. From 232942a21659f64c0867624c34a32fc4fa55836d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Thu, 18 Jan 2024 10:14:32 -0700 Subject: [PATCH 26/41] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 619b3acb..5727b7ff 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ LibreQoS is a Quality of Experience (QoE) Smart Queue Management (SQM) system designed for Internet Service Providers to optimize the flow of their network traffic and thus reduce bufferbloat, keep the network responsive, and improve the end-user experience. -Servers running LibreQoS can shape traffic for many thousands of customers. +Servers running LibreQoS can shape traffic for thousands of customers. On higher-end servers, LibreQoS is capable of shaping 50-80 Gbps of traffic. Learn more at [LibreQoS.io](https://libreqos.io/)! From bb3700202e9b6da65e6b593ec2d5d4ca76415ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Tue, 23 Jan 2024 11:03:34 -0700 Subject: [PATCH 27/41] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5727b7ff..f43ab720 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ Please support the continued development of LibreQoS by sponsoring us via [GitHu [ReadTheDocs](https://libreqos.readthedocs.io/en/latest/) -## Matrix Chat +## LibreQoS Chat -Our Matrix chat channel is available at [https://matrix.to/#/#libreqos:matrix.org](https://matrix.to/#/#libreqos:matrix.org). +Our Zulip chat server is available at [https://chat.libreqos.io/join/fvu3cerayyaumo377xwvpev6/](https://chat.libreqos.io/join/fvu3cerayyaumo377xwvpev6/). ## Long-Term Stats (LTS) From 8cd8b7d4045fa6adb7a6b8359ee394f18f5e36e5 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jan 2024 11:49:24 -0600 Subject: [PATCH 28/41] Update example.toml to include the powercode required fields --- src/rust/lqos_config/src/etc/v15/example.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index dba70997..34303165 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -78,3 +78,8 @@ bandwidth_overhead_factor = 1.0 commit_bandwidth_multiplier = 0.98 exception_cpes = [] use_ptmp_as_parent = false + +[powercode_integration] +enable_powercode = false +powercode_api_key = "" +powercode_api_url = "" From b1b01b89a63bc53a48d3f24d9c48f00d56ea6fac Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jan 2024 12:50:30 -0600 Subject: [PATCH 29/41] Update cargo packages to latest releases, update rocket and sysinfo to newer syntaxes. --- src/rust/Cargo.lock | 1054 +++++++++-------- src/rust/lqos_node_manager/src/auth_guard.rs | 4 +- .../src/tracker/cache_manager.rs | 2 - src/rust/lqos_python/src/lib.rs | 2 +- src/rust/lqosd/src/anonymous_usage/mod.rs | 4 +- src/rust/lqosd/src/file_lock.rs | 2 +- .../src/collector/collation/system_stats.rs | 3 +- 7 files changed, 553 insertions(+), 518 deletions(-) diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 801d968a..205f2544 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -49,9 +49,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -63,43 +63,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "async-compression" @@ -139,9 +139,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", @@ -154,6 +154,15 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -226,9 +235,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.3" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "binascii" @@ -247,11 +256,11 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.66.1" +version = "0.69.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" +checksum = "a4c69fae65a523209d34240b60abe0c42d33d1045d445c0839d8a4894a736e2d" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "cexpr", "clang-sys", "lazy_static", @@ -276,9 +285,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" dependencies = [ "serde", ] @@ -294,9 +303,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.3.4" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -305,9 +314,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.4" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -315,21 +324,27 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "bytemuck" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2490600f404f2b94c167e31d3ed1d5f3c225a0f3b80230053b3e0b7b962bd9" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cassowary" @@ -380,9 +395,9 @@ dependencies = [ [[package]] name = "ciborium" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", @@ -391,18 +406,18 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", - "half", + "half 2.3.1", ] [[package]] @@ -418,9 +433,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" dependencies = [ "glob", "libc", @@ -429,20 +444,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.1" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.4.1" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -452,9 +466,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.0" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", @@ -464,9 +478,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "colorchoice" @@ -476,20 +490,19 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "colored" -version = "2.0.4" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" dependencies = [ - "is-terminal", "lazy_static", "windows-sys 0.48.0", ] [[package]] name = "cookie" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7efb37c3e1ccb1ff97164ad95ac1606e8ccd35b3fa0a7d99a304c7f4a428cc24" +checksum = "3cd91cf61412820176e137621345ee43b3f4423e589e7ae4e50d601d93e35ef8" dependencies = [ "percent-encoding", "time", @@ -498,9 +511,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -508,15 +521,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -568,48 +581,30 @@ dependencies = [ "itertools", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crossterm" @@ -633,7 +628,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "crossterm_winapi", "libc", "mio", @@ -653,6 +648,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-common" version = "0.1.6" @@ -665,9 +666,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ "csv-core", "itoa", @@ -677,18 +678,18 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ "memchr", ] [[package]] name = "curve25519-dalek" -version = "4.0.0" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2" +checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c" dependencies = [ "cfg-if", "cpufeatures", @@ -702,9 +703,9 @@ dependencies = [ [[package]] name = "curve25519-dalek-derive" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", @@ -713,12 +714,12 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.5.2" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b101bb8960ab42ada6ae98eb82afcea4452294294c45b681295af26610d6d28" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown", "lock_api", "once_cell", "parking_lot_core", @@ -726,9 +727,9 @@ dependencies = [ [[package]] name = "default-net" -version = "0.17.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d1b93f7700c45c26856522aed3982b685edb0532e4dbdd3a4ec84bb4185526" +checksum = "85dc7576d8346d3c86ad64dc64d26d0f6c970ba4795b850f15ee94467d8e53eb" dependencies = [ "dlopen2", "libc", @@ -738,14 +739,17 @@ dependencies = [ "netlink-sys", "once_cell", "system-configuration", - "windows", + "windows 0.48.0", ] [[package]] name = "deranged" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] [[package]] name = "devise" @@ -773,7 +777,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35b50dba0afdca80b187392b24f2499a88c336d5a8493e4b4ccfb608708be56a" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "proc-macro2", "proc-macro2-diagnostics", "quote", @@ -809,17 +813,17 @@ checksum = "53ecafc952c4528d9b51a458d1a8904b81783feff9fde08ab6ed2545ff396872" dependencies = [ "cfg-if", "libc", - "socket2 0.4.9", + "socket2 0.4.10", "winapi", ] [[package]] name = "dryoc" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65f5013c48133363c5a8db6bc74511b8b254680929c7757d9d833dea18c12f13" +checksum = "d446216f1822c429bbc9f96b4227248fc09a4213897439504a672a1b052d0804" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "chacha20", "curve25519-dalek", "generic-array", @@ -850,16 +854,26 @@ dependencies = [ ] [[package]] -name = "env_logger" -version = "0.10.0" +name = "env_filter" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" dependencies = [ - "humantime", - "is-terminal", "log", "regex", - "termcolor", +] + +[[package]] +name = "env_logger" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eeb342678d785662fd2514be38c459bb925f02b68dd2a3e0f21d7ef82d979dd" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", ] [[package]] @@ -870,68 +884,57 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.3" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fiat-crypto" -version = "0.1.20" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" +checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" [[package]] name = "figment" -version = "0.10.10" +version = "0.10.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4547e226f4c9ab860571e070a9034192b3175580ecea38da34fcdb53a018c9a5" +checksum = "2b6e5bc7bd59d60d0d45a6ccab6cf0f4ce28698fb4e81e750ddf229c9b824026" dependencies = [ - "atomic", + "atomic 0.6.0", "pear", "serde", - "toml 0.7.6", + "toml", "uncased", "version_check", ] [[package]] name = "filetime" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -960,18 +963,18 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -984,9 +987,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -994,15 +997,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1011,15 +1014,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -1028,21 +1031,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1066,7 +1069,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows", + "windows 0.48.0", ] [[package]] @@ -1081,9 +1084,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -1092,9 +1095,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "glob" @@ -1104,9 +1107,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -1114,7 +1117,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap", "slab", "tokio", "tokio-util", @@ -1128,16 +1131,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" [[package]] -name = "hashbrown" -version = "0.12.3" +name = "half" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +dependencies = [ + "cfg-if", + "crunchy", +] [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "heck" @@ -1147,15 +1154,24 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -1164,9 +1180,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -1193,9 +1209,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -1208,7 +1224,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -1230,9 +1246,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1240,23 +1256,13 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.3" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - -[[package]] -name = "indexmap" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown", + "serde", ] [[package]] @@ -1324,19 +1330,19 @@ checksum = "8e537132deb99c0eb4b752f0346b6a836200eaaa3516dd7e5514b63930a09e5d" [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1350,9 +1356,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jemalloc-sys" @@ -1376,9 +1382,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -1417,43 +1423,43 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libbpf-sys" -version = "1.2.1+v1.2.0" +version = "1.3.0+v1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75adb4021282a72ca63ebbc0e4247750ad74ede68ff062d247691072d709ad8b" +checksum = "b6e68987fe8f2dd7b76930f800a4e5b958c766171fc3a7c33dd67c06a0f1e801" dependencies = [ "cc", - "nix 0.26.4", + "nix", "num_cpus", "pkg-config", ] [[package]] name = "libc" -version = "0.2.147" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" dependencies = [ "cfg-if", - "winapi", + "windows-sys 0.48.0", ] [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -1467,9 +1473,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "log-once" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae8737764ac12e9cfc5ab42a9ec2bdb7e2b376ce9694a4dffc92b7adee09cb5" +checksum = "6d8a05e3879b317b1b6dbf353e5bba7062bedcc59815267bb23eaa0c576cebf0" dependencies = [ "log", ] @@ -1514,7 +1520,7 @@ dependencies = [ "lqos_config", "lqos_utils", "lts_client", - "nix 0.27.1", + "nix", "serde", "serde_cbor", "thiserror", @@ -1536,8 +1542,8 @@ dependencies = [ "serde_json", "sha2", "thiserror", - "toml 0.8.8", - "toml_edit 0.21.0", + "toml", + "toml_edit", "uuid", ] @@ -1575,7 +1581,7 @@ dependencies = [ "lqos_bus", "lqos_config", "lqos_utils", - "nix 0.27.1", + "nix", "once_cell", "reqwest", "rocket", @@ -1591,7 +1597,7 @@ dependencies = [ "lqos_bus", "lqos_config", "lqos_utils", - "nix 0.27.1", + "nix", "pyo3", "sysinfo", "tokio", @@ -1627,7 +1633,7 @@ dependencies = [ "colored", "default-net", "lqos_config", - "toml 0.8.8", + "toml", "uuid", ] @@ -1644,7 +1650,7 @@ dependencies = [ "lqos_bus", "lqos_config", "lqos_utils", - "nix 0.27.1", + "nix", "once_cell", "thiserror", "zerocopy", @@ -1656,7 +1662,7 @@ version = "0.1.0" dependencies = [ "byteorder", "log", - "nix 0.27.1", + "nix", "notify", "serde", "thiserror", @@ -1679,7 +1685,7 @@ dependencies = [ "lqos_sys", "lqos_utils", "lts_client", - "nix 0.27.1", + "nix", "num-traits", "once_cell", "serde", @@ -1753,9 +1759,9 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memalloc" @@ -1765,18 +1771,9 @@ checksum = "df39d232f5c40b0891c10216992c2f250c054105cb1e56f0fc9032db6203ecc1" [[package]] name = "memchr" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76fc44e2588d5b436dbc3c6cf62aef290f90dab6235744a93dfe1cc18f451e2c" - -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -1810,9 +1807,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "log", @@ -1871,9 +1868,9 @@ dependencies = [ [[package]] name = "netlink-packet-route" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6de2fe935f44cbdfcab77dce2150d68eda75be715cd42d4d6f52b0bd4dcc5b1" +checksum = "053998cea5a306971f88580d0829e90f270f940befd7cf928da179d4187a5a66" dependencies = [ "anyhow", "bitflags 1.3.2", @@ -1906,26 +1903,13 @@ dependencies = [ "log", ] -[[package]] -name = "nix" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", - "memoffset 0.7.1", - "pin-utils", -] - [[package]] name = "nix" version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "cfg-if", "libc", ] @@ -1977,9 +1961,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", ] @@ -1996,9 +1980,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.0" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -2017,11 +2001,11 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "cfg-if", "foreign-types", "libc", @@ -2049,9 +2033,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.92" +version = "0.9.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", @@ -2077,9 +2061,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", @@ -2096,20 +2080,20 @@ checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "pear" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a386cd715229d399604b50d1361683fe687066f42d56f54be995bc6868f71c" +checksum = "4ccca0f6c17acc81df8e242ed473ec144cbf5c98037e69aa6d144780aad103c8" dependencies = [ "inlinable_string", "pear_codegen", - "yansi 1.0.0-rc.1", + "yansi", ] [[package]] name = "pear_codegen" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f0f13dac8069c139e8300a6510e3f4143ecf5259c60b116a9b271b4ca0d54" +checksum = "2e22670e8eb757cff11d6c199ca7b987f352f0346e0be4dd23869ec72cb53c77" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", @@ -2125,24 +2109,24 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", @@ -2163,15 +2147,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "platforms" -version = "3.1.2" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8" +checksum = "626dec3cac7cc0e1577a2ec3fc496277ec2baa084bebad95bb6fdbfae235f84c" [[package]] name = "plotters" @@ -2201,6 +2185,12 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2209,9 +2199,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.12" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", "syn", @@ -2219,9 +2209,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -2236,19 +2226,19 @@ dependencies = [ "quote", "syn", "version_check", - "yansi 1.0.0-rc.1", + "yansi", ] [[package]] name = "pyo3" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" +checksum = "9a89dc7a5850d0e983be1ec2a463a171d20990487c3cfcd68b5363f1ee3d6fe0" dependencies = [ "cfg-if", "indoc", "libc", - "memoffset 0.9.0", + "memoffset", "parking_lot", "pyo3-build-config", "pyo3-ffi", @@ -2258,9 +2248,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" +checksum = "07426f0d8fe5a601f26293f300afd1a7b1ed5e78b2a705870c5f30893c5163be" dependencies = [ "once_cell", "target-lexicon", @@ -2268,9 +2258,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" +checksum = "dbb7dec17e17766b46bca4f1a4215a85006b4c2ecde122076c562dd058da6cf1" dependencies = [ "libc", "pyo3-build-config", @@ -2278,9 +2268,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" +checksum = "05f738b4e40d50b5711957f142878cfa0f28e054aa0ebdfc3fd137a843f74ed3" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2290,9 +2280,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" +checksum = "0fc910d4851847827daf9d6cdd4a823fbdaab5b8818325c5e97a86da79e8881f" dependencies = [ "heck", "proc-macro2", @@ -2302,9 +2292,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2341,9 +2331,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" dependencies = [ "either", "rayon-core", @@ -2351,39 +2341,37 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "ref-cast" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acde58d073e9c79da00f2b5b84eed919c8326832648a5b109b3fce1bb1175280" +checksum = "c4846d4c50d1721b1a3bef8af76924eef20d5e723647333798c1b519b3a9473f" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" +checksum = "5fddb4f8d99b0a2ebafc65a87a69a7b9875e4b1ae1f00db265d300ef7f28bccc" dependencies = [ "proc-macro2", "quote", @@ -2392,14 +2380,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.4" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.7", - "regex-syntax 0.7.5", + "regex-automata 0.4.5", + "regex-syntax 0.8.2", ] [[package]] @@ -2413,13 +2401,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.7" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", ] [[package]] @@ -2430,15 +2418,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "base64", "bytes", @@ -2461,6 +2449,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tower-service", @@ -2495,20 +2484,19 @@ dependencies = [ [[package]] name = "rocket" -version = "0.5.0-rc.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58734f7401ae5cfd129685b48f61182331745b357b96f2367f01aebaf1cc9cc9" +checksum = "9e7bb57ccb26670d73b6a47396c83139447b9e7878cab627fdfe9ea8da489150" dependencies = [ "async-stream", "async-trait", - "atomic", + "atomic 0.5.3", "binascii", "bytes", "either", "figment", "futures", - "indexmap 1.9.3", - "is-terminal", + "indexmap", "log", "memchr", "multer", @@ -2531,7 +2519,7 @@ dependencies = [ "ubyte", "uuid", "version_check", - "yansi 0.5.1", + "yansi", ] [[package]] @@ -2549,32 +2537,33 @@ dependencies = [ [[package]] name = "rocket_codegen" -version = "0.5.0-rc.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7093353f14228c744982e409259fb54878ba9563d08214f2d880d59ff2fc508b" +checksum = "a2238066abf75f21be6cd7dc1a09d5414a671f4246e384e49fe3f8a4936bd04c" dependencies = [ "devise", "glob", - "indexmap 1.9.3", + "indexmap", "proc-macro2", "quote", "rocket_http", "syn", "unicode-xid", + "version_check", ] [[package]] name = "rocket_http" -version = "0.5.0-rc.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936012c99162a03a67f37f9836d5f938f662e26f2717809761a9ac46432090f4" +checksum = "37a1663694d059fe5f943ea5481363e48050acedd241d46deb2e27f71110389e" dependencies = [ "cookie", "either", "futures", "http", "hyper", - "indexmap 1.9.3", + "indexmap", "log", "memchr", "pear", @@ -2614,15 +2603,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.10" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6248e1caa625eb708e266e06159f135e8c26f2bb7ceb72dc4b2766d0340964" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2633,9 +2622,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "salsa20" @@ -2657,11 +2646,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2701,15 +2690,15 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] @@ -2720,15 +2709,15 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" dependencies = [ - "half", + "half 1.8.2", "serde", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", @@ -2737,9 +2726,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -2748,9 +2737,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" dependencies = [ "itoa", "serde", @@ -2758,9 +2747,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ "serde", ] @@ -2779,9 +2768,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -2790,18 +2779,18 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] [[package]] name = "shlex" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook" @@ -2844,15 +2833,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -2860,9 +2849,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2915,9 +2904,9 @@ dependencies = [ [[package]] name = "state" -version = "0.5.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe866e1e51e8260c9eed836a042a5e7f6726bb2b411dffeaa712e19c388f23b" +checksum = "2b8c4a4445d81357df8b1a650d0d0d6fbbbfe99d064aa5e02f3e4022061476d8" dependencies = [ "loom", ] @@ -2936,9 +2925,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.29" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -2953,9 +2942,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sysinfo" -version = "0.29.9" +version = "0.30.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d0e9cc2273cc8d31377bdd638d72e3ac3e5607b18621062b169d02787f1bab" +checksum = "1fb4f3438c8f6389c864e61221cbc97e9bca98b4daf39a5beb7bea660f528bb2" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2963,7 +2952,7 @@ dependencies = [ "ntapi", "once_cell", "rayon", - "winapi", + "windows 0.52.0", ] [[package]] @@ -2989,46 +2978,37 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.11" +version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" +checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys 0.48.0", -] - -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", + "windows-sys 0.52.0", ] [[package]] name = "thiserror" -version = "1.0.47" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.47" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", @@ -3047,12 +3027,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "itoa", + "powerfmt", "serde", "time-core", "time-macros", @@ -3060,15 +3041,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] @@ -3100,9 +3081,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", @@ -3112,16 +3093,16 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.3", + "socket2 0.5.5", "tokio-macros", "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", @@ -3151,9 +3132,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -3163,18 +3144,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "toml" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.19.14", -] - [[package]] name = "toml" version = "0.8.8" @@ -3184,7 +3153,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.21.0", + "toml_edit", ] [[package]] @@ -3196,26 +3165,13 @@ dependencies = [ "serde", ] -[[package]] -name = "toml_edit" -version = "0.19.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" -dependencies = [ - "indexmap 2.0.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "toml_edit" version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap 2.0.0", + "indexmap", "serde", "serde_spanned", "toml_datetime", @@ -3252,11 +3208,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -3265,9 +3220,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", @@ -3276,9 +3231,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -3286,20 +3241,20 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "matchers", "nu-ansi-term", @@ -3315,9 +3270,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tui" @@ -3334,15 +3289,15 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ubyte" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c81f0dae7d286ad0d9366d7679a77934cfc3cf3a8d67e82669794412b2368fe6" +checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea" dependencies = [ "serde", ] @@ -3359,9 +3314,9 @@ dependencies = [ [[package]] name = "uncased" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b9bc53168a4be7402ab86c3aad243a84dd7381d09be0eddc81280c1da95ca68" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" dependencies = [ "serde", "version_check", @@ -3369,15 +3324,15 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -3396,9 +3351,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unicode-xid" @@ -3414,9 +3369,9 @@ checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -3431,9 +3386,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", "rand", @@ -3460,9 +3415,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -3485,9 +3440,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3495,9 +3450,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", @@ -3510,9 +3465,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -3522,9 +3477,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3532,9 +3487,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", @@ -3545,15 +3500,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -3561,13 +3516,14 @@ dependencies = [ [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix", ] [[package]] @@ -3588,9 +3544,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -3610,6 +3566,25 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core", + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -3628,6 +3603,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -3658,6 +3642,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -3670,6 +3669,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -3682,6 +3687,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -3694,6 +3705,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -3706,6 +3723,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -3718,6 +3741,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -3730,6 +3759,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -3743,10 +3778,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] -name = "winnow" -version = "0.5.15" +name = "windows_x86_64_msvc" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "winnow" +version = "0.5.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1931d78a9c73861da0134f453bb1f790ce49b2e30eba8410b4b79bac72b46a2d" dependencies = [ "memchr", ] @@ -3781,23 +3822,20 @@ dependencies = [ "tokio", ] -[[package]] -name = "yansi" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" - [[package]] name = "yansi" version = "1.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1367295b8f788d371ce2dbc842c7b709c73ee1364d30351dd300ec2203b12377" +dependencies = [ + "is-terminal", +] [[package]] name = "zerocopy" -version = "0.6.3" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3b9c234616391070b0b173963ebc65a9195068e7ed3731c6edac2ec45ebe106" +checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" dependencies = [ "byteorder", "zerocopy-derive", @@ -3805,9 +3843,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.6.3" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f7f3a471f98d0a61c34322fbbfd10c384b07687f680d4119813713f72308d91" +checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" dependencies = [ "proc-macro2", "quote", @@ -3816,9 +3854,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" dependencies = [ "zeroize_derive", ] diff --git a/src/rust/lqos_node_manager/src/auth_guard.rs b/src/rust/lqos_node_manager/src/auth_guard.rs index e714f342..00ed1484 100644 --- a/src/rust/lqos_node_manager/src/auth_guard.rs +++ b/src/rust/lqos_node_manager/src/auth_guard.rs @@ -46,7 +46,7 @@ impl<'r> FromRequest<'r> for AuthGuard { return Outcome::Success(AuthGuard::ReadOnly) } _ => { - return Outcome::Failure(( + return Outcome::Error(( Status::Unauthorized, Error::msg("Invalid token"), )) @@ -60,7 +60,7 @@ impl<'r> FromRequest<'r> for AuthGuard { } } - Outcome::Failure((Status::Unauthorized, Error::msg("Access Denied"))) + Outcome::Error((Status::Unauthorized, Error::msg("Access Denied"))) } } diff --git a/src/rust/lqos_node_manager/src/tracker/cache_manager.rs b/src/rust/lqos_node_manager/src/tracker/cache_manager.rs index cdc0cb72..75083303 100644 --- a/src/rust/lqos_node_manager/src/tracker/cache_manager.rs +++ b/src/rust/lqos_node_manager/src/tracker/cache_manager.rs @@ -18,9 +18,7 @@ use std::{sync::atomic::AtomicBool, time::Duration}; /// it runs as part of start-up - and keeps running. /// Designed to never return or fail on error. pub async fn update_tracking() { - use sysinfo::CpuExt; use sysinfo::System; - use sysinfo::SystemExt; let mut sys = System::new_all(); spawn_blocking(|| { diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index b4e9ccb3..61f27621 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -13,7 +13,7 @@ use std::{ mod blocking; use anyhow::{Error, Result}; use blocking::run_query; -use sysinfo::{ProcessExt, System, SystemExt}; +use sysinfo::System; const LOCK_FILE: &str = "/run/lqos/libreqos.lock"; diff --git a/src/rust/lqosd/src/anonymous_usage/mod.rs b/src/rust/lqosd/src/anonymous_usage/mod.rs index 7292cc74..24339e16 100644 --- a/src/rust/lqosd/src/anonymous_usage/mod.rs +++ b/src/rust/lqosd/src/anonymous_usage/mod.rs @@ -3,7 +3,7 @@ mod version; use std::{time::Duration, net::TcpStream, io::Write}; use lqos_bus::anonymous::{AnonymousUsageV1, build_stats}; use lqos_sys::num_possible_cpus; -use sysinfo::{System, SystemExt, CpuExt}; +use sysinfo::System; use crate::{shaped_devices_tracker::{SHAPED_DEVICES, NETWORK_JSON}, stats::{HIGH_WATERMARK_DOWN, HIGH_WATERMARK_UP}}; const SLOW_START_SECS: u64 = 1; @@ -30,7 +30,7 @@ fn anonymous_usage_dump() -> anyhow::Result<()> { sys.refresh_all(); data.total_memory = sys.total_memory(); data.available_memory = sys.available_memory(); - if let Some(kernel) = sys.kernel_version() { + if let Some(kernel) = sysinfo::System::kernel_version() { data.kernel_version = kernel; } data.usable_cores = num_possible_cpus().unwrap_or(0); diff --git a/src/rust/lqosd/src/file_lock.rs b/src/rust/lqosd/src/file_lock.rs index d8d6699d..1193ca3d 100644 --- a/src/rust/lqosd/src/file_lock.rs +++ b/src/rust/lqosd/src/file_lock.rs @@ -6,7 +6,7 @@ use std::{ io::{Read, Write}, path::Path, }; -use sysinfo::{ProcessExt, System, SystemExt}; +use sysinfo::System; const LOCK_PATH: &str = "/run/lqos/lqosd.lock"; const LOCK_DIR: &str = "/run/lqos"; diff --git a/src/rust/lts_client/src/collector/collation/system_stats.rs b/src/rust/lts_client/src/collector/collation/system_stats.rs index 10132e00..fed5f83e 100644 --- a/src/rust/lts_client/src/collector/collation/system_stats.rs +++ b/src/rust/lts_client/src/collector/collation/system_stats.rs @@ -1,11 +1,10 @@ use once_cell::sync::Lazy; -use sysinfo::{System, SystemExt}; +use sysinfo::System; use tokio::sync::Mutex; static SYS: Lazy> = Lazy::new(|| Mutex::new(System::new_all())); pub(crate) async fn get_cpu_ram() -> (Vec, u32) { - use sysinfo::CpuExt; let mut lock = SYS.lock().await; lock.refresh_cpu(); lock.refresh_memory(); From 84f7562d5470cbb40e02193e05f76bf9da8f2e4f Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jan 2024 16:08:29 -0600 Subject: [PATCH 30/41] Add line to example file to complete on-a-stick outline. --- src/rust/lqos_config/src/etc/v15/example.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index 34303165..cf73b6ec 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -27,6 +27,7 @@ to_network = "eth1" # OR: #[single_interface] +#interface = "eth0" #internet_vlan = 2 #network_vlan = 3 From 50bc071db15b09395dc516f8fc85c6c6c59f8da5 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jan 2024 16:33:26 -0600 Subject: [PATCH 31/41] Fix issue with queueRefreshInterval being optional but not treated as such, and breaking things. --- src/rust/lqos_config/src/etc/python_migration.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rust/lqos_config/src/etc/python_migration.rs b/src/rust/lqos_config/src/etc/python_migration.rs index 4751f52c..c1077f55 100644 --- a/src/rust/lqos_config/src/etc/python_migration.rs +++ b/src/rust/lqos_config/src/etc/python_migration.rs @@ -108,7 +108,7 @@ impl PythonMigration { cfg.generated_pn_upload_mbps = from_python(&py, "generatedPNUploadMbps")?; cfg.interface_a = from_python(&py, "interfaceA")?; cfg.interface_b = from_python(&py, "interfaceB")?; - cfg.queue_refresh_interval_mins = from_python(&py, "queueRefreshIntervalMins")?; + cfg.queue_refresh_interval_mins = from_python(&py, "queueRefreshIntervalMins").unwrap_or(15); cfg.on_a_stick = from_python(&py, "OnAStick")?; cfg.stick_vlan_a = from_python(&py, "StickVlanA")?; cfg.stick_vlan_b = from_python(&py, "StickVlanB")?; @@ -173,6 +173,7 @@ impl PythonMigration { Ok(old_config) } + #[allow(dead_code)] pub(crate) fn load_from_string(s: &str) -> Result { let mut old_config = Self::default(); prepare_freethreaded_python(); From d851161670f42dfa14324464fff21c56ba0bd3c7 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jan 2024 16:33:35 -0600 Subject: [PATCH 32/41] General warning and documentation pass. --- src/rust/lqos_config/src/etc/etclqos_migration.rs | 5 ++--- src/rust/lqos_config/src/etc/mod.rs | 1 + src/rust/lqos_config/src/etc/v15/top_config.rs | 7 +++++++ src/rust/lqos_config/src/shaped_devices/mod.rs | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/rust/lqos_config/src/etc/etclqos_migration.rs b/src/rust/lqos_config/src/etc/etclqos_migration.rs index 1c2e6d3a..78ccab96 100644 --- a/src/rust/lqos_config/src/etc/etclqos_migration.rs +++ b/src/rust/lqos_config/src/etc/etclqos_migration.rs @@ -190,7 +190,7 @@ impl EtcLqos { error!("Unable to parse TOML from /etc/lqos.conf"); error!("Full error: {:?}", e); panic!(); - Err(EtcLqosError::CannotParseToml) + //Err(EtcLqosError::CannotParseToml) } } } @@ -220,6 +220,7 @@ impl EtcLqos { /// Run this if you've received the OK from the licensing server, and been /// sent a license key. This appends a [long_term_stats] section to your /// config file - ONLY if one doesn't already exist. +#[allow(dead_code)] pub fn enable_long_term_stats(license_key: String) { if let Ok(raw) = std::fs::read_to_string("/etc/lqos.conf") { let document = raw.parse::(); @@ -296,8 +297,6 @@ pub enum EtcLqosError { CannotParseToml, #[error("Unable to backup /etc/lqos.conf to /etc/lqos.conf.backup")] BackupFail, - #[error("Unable to serialize new configuration")] - SerializeFail, #[error("Unable to write to /etc/lqos.conf")] WriteFail, } diff --git a/src/rust/lqos_config/src/etc/mod.rs b/src/rust/lqos_config/src/etc/mod.rs index 54d49b30..58a1df95 100644 --- a/src/rust/lqos_config/src/etc/mod.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -49,6 +49,7 @@ pub fn load_config() -> Result { Ok(lock.as_ref().unwrap().clone()) } +/// Enables LTS reporting in the configuration file. pub fn enable_long_term_stats(license_key: String) -> Result<(), LibreQoSConfigError> { let mut config = load_config()?; let mut lock = CONFIG.lock().unwrap(); diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index a855b24e..39900748 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -7,6 +7,7 @@ use sha2::digest::Update; use sha2::Digest; use uuid::Uuid; +/// Top-level configuration file for LibreQoS. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Config { /// Version number for the configuration file. @@ -95,6 +96,8 @@ impl Config { Ok(()) } + /// Loads a config file from a string (used for testing only) + #[allow(dead_code)] pub fn load_from_string(s: &str) -> Result { let config: Config = toml::from_str(s).map_err(|e| format!("Error parsing config: {}", e))?; config.validate()?; @@ -127,6 +130,7 @@ impl Default for Config { } impl Config { + /// Calculate the unterface facing the Internet pub fn internet_interface(&self) -> String { if let Some(bridge) = &self.bridge { bridge.to_internet.clone() @@ -137,6 +141,7 @@ impl Config { } } + /// Calculate the interface facing the ISP pub fn isp_interface(&self) -> String { if let Some(bridge) = &self.bridge { bridge.to_network.clone() @@ -147,10 +152,12 @@ impl Config { } } + /// Are we in single-interface mode? pub fn on_a_stick_mode(&self) -> bool { self.bridge.is_none() } + /// Get the VLANs for the stick interface pub fn stick_vlans(&self) -> (u32, u32) { if let Some(stick) = &self.single_interface { (stick.network_vlan, stick.internet_vlan) diff --git a/src/rust/lqos_config/src/shaped_devices/mod.rs b/src/rust/lqos_config/src/shaped_devices/mod.rs index 6b9eea1a..be05e8fa 100644 --- a/src/rust/lqos_config/src/shaped_devices/mod.rs +++ b/src/rust/lqos_config/src/shaped_devices/mod.rs @@ -1,6 +1,6 @@ mod serializable; mod shaped_device; -use crate::{etc, SUPPORTED_CUSTOMERS}; +use crate::SUPPORTED_CUSTOMERS; use csv::{QuoteStyle, ReaderBuilder, WriterBuilder}; use log::error; use serializable::SerializableShapedDevice; From a24c51065f14875e254355f067b7e2b0a52dc5a9 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jan 2024 16:36:36 -0600 Subject: [PATCH 33/41] Fix on-a-stick typo messing up vlans --- src/rust/lqos_config/src/etc/migration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs index c947a84c..e0e4481e 100644 --- a/src/rust/lqos_config/src/etc/migration.rs +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -147,7 +147,7 @@ fn migrate_bridge( new_config.bridge = None; new_config.single_interface = Some(SingleInterfaceConfig { interface: python_config.interface_a.clone(), - internet_vlan: python_config.stick_vlan_b, + internet_vlan: python_config.stick_vlan_a, network_vlan: python_config.stick_vlan_b, }); } else { From ed5cbbc29cff0c3604c36c1fc60fa12035211c95 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jan 2024 09:30:54 -0600 Subject: [PATCH 34/41] Add the beginnings of the Sonar integration into the config system. Add defaults for every Python field to reduce the chances of getting a partial config. --- src/rust/lqos_config/src/etc/migration.rs | 23 ++++ .../lqos_config/src/etc/python_migration.rs | 121 +++++++++++------- src/rust/lqos_config/src/etc/v15/example.toml | 6 + src/rust/lqos_config/src/etc/v15/mod.rs | 1 + .../src/etc/v15/sonar_integration.rs | 22 ++++ .../lqos_config/src/etc/v15/top_config.rs | 4 + 6 files changed, 132 insertions(+), 45 deletions(-) create mode 100644 src/rust/lqos_config/src/etc/v15/sonar_integration.rs diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs index e0e4481e..3d09ac03 100644 --- a/src/rust/lqos_config/src/etc/migration.rs +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -86,6 +86,8 @@ fn do_migration_14_to_15( migrate_integration_common(python_config, &mut new_config)?; migrate_spylnx(python_config, &mut new_config)?; migrate_uisp(python_config, &mut new_config)?; + migrate_powercode(python_config, &mut new_config)?; + migrate_sonar(python_config, &mut new_config)?; migrate_queues( python_config, &mut new_config)?; new_config.validate().unwrap(); // Left as an upwrap because this should *never* happen @@ -228,6 +230,27 @@ fn migrate_spylnx( Ok(()) } +fn migrate_powercode( + python_config: &PythonMigration, + new_config: &mut Config, +) -> Result<(), MigrationError> { + new_config.powercode_integration.enable_powercode = python_config.automatic_import_powercode; + new_config.powercode_integration.powercode_api_url = python_config.powercode_api_url.clone(); + new_config.powercode_integration.powercode_api_key = python_config.powercode_api_key.clone(); + Ok(()) +} + +fn migrate_sonar( + python_config: &PythonMigration, + new_config: &mut Config, +) -> Result<(), MigrationError> { + new_config.sonar_integration.enable_sonar = python_config.automatic_import_sonar; + new_config.sonar_integration.sonar_api_url = python_config.sonar_api_url.clone(); + new_config.sonar_integration.sonar_api_key = python_config.sonar_api_key.clone(); + new_config.sonar_integration.snmp_community = python_config.snmp_community.clone(); + Ok(()) +} + fn migrate_uisp( python_config: &PythonMigration, new_config: &mut Config, diff --git a/src/rust/lqos_config/src/etc/python_migration.rs b/src/rust/lqos_config/src/etc/python_migration.rs index c1077f55..797bbde7 100644 --- a/src/rust/lqos_config/src/etc/python_migration.rs +++ b/src/rust/lqos_config/src/etc/python_migration.rs @@ -93,59 +93,90 @@ pub struct PythonMigration { pub api_password: String, pub api_host_ip: String, pub api_host_port: u32, + pub automatic_import_powercode: bool, + pub powercode_api_key: String, + pub powercode_api_url: String, + pub automatic_import_sonar: bool, + pub sonar_api_url: String, + pub sonar_api_key: String, + pub snmp_community: String, + // TODO: It isn't clear what types `sonar_api_key,sonar_airmax_ap_model_ids,sonar_active_status_ids,sonar_ltu_ap_model_ids` + // are supposed to be. + // TODO: httpRestIntegrationConfig } impl PythonMigration { fn parse(cfg: &mut Self, py: &Python) -> Result<(), PythonMigrationError> { - cfg.sqm = from_python(&py, "sqm")?; - cfg.monitor_only_mode = from_python(&py, "monitorOnlyMode")?; + cfg.sqm = from_python(&py, "sqm").unwrap_or("cake diffserv4".to_string()); + cfg.monitor_only_mode = from_python(&py, "monitorOnlyMode").unwrap_or(false); cfg.upstream_bandwidth_capacity_download_mbps = - from_python(&py, "upstreamBandwidthCapacityDownloadMbps")?; + from_python(&py, "upstreamBandwidthCapacityDownloadMbps").unwrap_or(1000); cfg.upstream_bandwidth_capacity_upload_mbps = - from_python(&py, "upstreamBandwidthCapacityUploadMbps")?; - cfg.generated_pn_download_mbps = from_python(&py, "generatedPNDownloadMbps")?; - cfg.generated_pn_upload_mbps = from_python(&py, "generatedPNUploadMbps")?; - cfg.interface_a = from_python(&py, "interfaceA")?; - cfg.interface_b = from_python(&py, "interfaceB")?; + from_python(&py, "upstreamBandwidthCapacityUploadMbps").unwrap_or(1000); + cfg.generated_pn_download_mbps = from_python(&py, "generatedPNDownloadMbps").unwrap_or(1000); + cfg.generated_pn_upload_mbps = from_python(&py, "generatedPNUploadMbps").unwrap_or(1000); + cfg.interface_a = from_python(&py, "interfaceA").unwrap_or("eth1".to_string()); + cfg.interface_b = from_python(&py, "interfaceB").unwrap_or("eth2".to_string()); cfg.queue_refresh_interval_mins = from_python(&py, "queueRefreshIntervalMins").unwrap_or(15); - cfg.on_a_stick = from_python(&py, "OnAStick")?; - cfg.stick_vlan_a = from_python(&py, "StickVlanA")?; - cfg.stick_vlan_b = from_python(&py, "StickVlanB")?; - cfg.enable_actual_shell_commands = from_python(&py, "enableActualShellCommands")?; - cfg.run_shell_commands_as_sudo = from_python(&py, "runShellCommandsAsSudo")?; - cfg.queues_available_override = from_python(&py, "queuesAvailableOverride")?; - cfg.use_bin_packing_to_balance_cpu = from_python(&py, "useBinPackingToBalanceCPU")?; - cfg.influx_db_enabled = from_python(&py, "influxDBEnabled")?; - cfg.influx_db_url = from_python(&py, "influxDBurl")?; - cfg.infux_db_bucket = from_python(&py, "influxDBBucket")?; - cfg.influx_db_org = from_python(&py, "influxDBOrg")?; - cfg.influx_db_token = from_python(&py, "influxDBtoken")?; - cfg.circuit_name_use_address = from_python(&py, "circuitNameUseAddress")?; - cfg.overwrite_network_json_always = from_python(&py, "overwriteNetworkJSONalways")?; - cfg.ignore_subnets = from_python(&py, "ignoreSubnets")?; - cfg.allowed_subnets = from_python(&py, "allowedSubnets")?; - cfg.automatic_import_splynx = from_python(&py, "automaticImportSplynx")?; - cfg.splynx_api_key = from_python(&py, "splynx_api_key")?; - cfg.spylnx_api_secret = from_python(&py, "splynx_api_secret")?; - cfg.spylnx_api_url = from_python(&py, "splynx_api_url")?; - cfg.automatic_import_uisp = from_python(&py, "automaticImportUISP")?; - cfg.uisp_auth_token = from_python(&py, "uispAuthToken")?; - cfg.uisp_base_url = from_python(&py, "UISPbaseURL")?; - cfg.uisp_site = from_python(&py, "uispSite")?; - cfg.uisp_strategy = from_python(&py, "uispStrategy")?; - cfg.uisp_suspended_strategy = from_python(&py, "uispSuspendedStrategy")?; - cfg.airmax_capacity = from_python(&py, "airMax_capacity")?; - cfg.ltu_capacity = from_python(&py, "ltu_capacity")?; - cfg.exclude_sites = from_python(&py, "excludeSites")?; - cfg.find_ipv6_using_mikrotik = from_python(&py, "findIPv6usingMikrotik")?; - cfg.bandwidth_overhead_factor = from_python(&py, "bandwidthOverheadFactor")?; - cfg.committed_bandwidth_multiplier = from_python(&py, "committedBandwidthMultiplier")?; - cfg.exception_cpes = from_python(&py, "exceptionCPEs")?; - cfg.api_username = from_python(&py, "apiUsername")?; - cfg.api_password = from_python(&py, "apiPassword")?; - cfg.api_host_ip = from_python(&py, "apiHostIP")?; - cfg.api_host_port = from_python(&py, "apiHostPost")?; + cfg.on_a_stick = from_python(&py, "OnAStick").unwrap_or(false); + cfg.stick_vlan_a = from_python(&py, "StickVlanA").unwrap_or(0); + cfg.stick_vlan_b = from_python(&py, "StickVlanB").unwrap_or(0); + cfg.enable_actual_shell_commands = from_python(&py, "enableActualShellCommands").unwrap_or(true); + cfg.run_shell_commands_as_sudo = from_python(&py, "runShellCommandsAsSudo").unwrap_or(false); + cfg.queues_available_override = from_python(&py, "queuesAvailableOverride").unwrap_or(0); + cfg.use_bin_packing_to_balance_cpu = from_python(&py, "useBinPackingToBalanceCPU").unwrap_or(false); + + // Influx + cfg.influx_db_enabled = from_python(&py, "influxDBEnabled").unwrap_or(false); + cfg.influx_db_url = from_python(&py, "influxDBurl").unwrap_or("http://localhost:8086".to_string()); + cfg.infux_db_bucket = from_python(&py, "influxDBBucket").unwrap_or("libreqos".to_string()); + cfg.influx_db_org = from_python(&py, "influxDBOrg").unwrap_or("Your ISP Name Here".to_string()); + cfg.influx_db_token = from_python(&py, "influxDBtoken").unwrap_or("".to_string()); + + // Common + cfg.circuit_name_use_address = from_python(&py, "circuitNameUseAddress").unwrap_or(true); + cfg.overwrite_network_json_always = from_python(&py, "overwriteNetworkJSONalways").unwrap_or(false); + cfg.ignore_subnets = from_python(&py, "ignoreSubnets").unwrap_or(vec!["192.168.0.0/16".to_string()]); + cfg.allowed_subnets = from_python(&py, "allowedSubnets").unwrap_or(vec!["100.64.0.0/10".to_string()]); + cfg.exclude_sites = from_python(&py, "excludeSites").unwrap_or(vec![]); + cfg.find_ipv6_using_mikrotik = from_python(&py, "findIPv6usingMikrotik").unwrap_or(false); + + // Spylnx + cfg.automatic_import_splynx = from_python(&py, "automaticImportSplynx").unwrap_or(false); + cfg.splynx_api_key = from_python(&py, "splynx_api_key").unwrap_or("Your API Key Here".to_string()); + cfg.spylnx_api_secret = from_python(&py, "splynx_api_secret").unwrap_or("Your API Secret Here".to_string()); + cfg.spylnx_api_url = from_python(&py, "splynx_api_url").unwrap_or("https://your.splynx.url/api/v1".to_string()); + + // UISP + cfg.automatic_import_uisp = from_python(&py, "automaticImportUISP").unwrap_or(false); + cfg.uisp_auth_token = from_python(&py, "uispAuthToken").unwrap_or("Your API Token Here".to_string()); + cfg.uisp_base_url = from_python(&py, "UISPbaseURL").unwrap_or("https://your.uisp.url".to_string()); + cfg.uisp_site = from_python(&py, "uispSite").unwrap_or("Your parent site name here".to_string()); + cfg.uisp_strategy = from_python(&py, "uispStrategy").unwrap_or("full".to_string()); + cfg.uisp_suspended_strategy = from_python(&py, "uispSuspendedStrategy").unwrap_or("none".to_string()); + cfg.airmax_capacity = from_python(&py, "airMax_capacity").unwrap_or(0.65); + cfg.ltu_capacity = from_python(&py, "ltu_capacity").unwrap_or(0.9); + cfg.bandwidth_overhead_factor = from_python(&py, "bandwidthOverheadFactor").unwrap_or(1.0); + cfg.committed_bandwidth_multiplier = from_python(&py, "committedBandwidthMultiplier").unwrap_or(0.98); + cfg.exception_cpes = from_python(&py, "exceptionCPEs").unwrap_or(HashMap::new()); + + // API + cfg.api_username = from_python(&py, "apiUsername").unwrap_or("testUser".to_string()); + cfg.api_password = from_python(&py, "apiPassword").unwrap_or("testPassword".to_string()); + cfg.api_host_ip = from_python(&py, "apiHostIP").unwrap_or("127.0.0.1".to_string()); + cfg.api_host_port = from_python(&py, "apiHostPost").unwrap_or(5000); + + // Powercode + cfg.automatic_import_powercode = from_python(&py, "automaticImportPowercode").unwrap_or(false); + cfg.powercode_api_key = from_python(&py,"powercode_api_key").unwrap_or("".to_string()); + cfg.powercode_api_url = from_python(&py,"powercode_api_url").unwrap_or("".to_string()); + + // Sonar + cfg.automatic_import_sonar = from_python(&py, "automaticImportSonar").unwrap_or(false); + cfg.sonar_api_key = from_python(&py, "sonar_api_key").unwrap_or("".to_string()); + cfg.sonar_api_url = from_python(&py, "sonar_api_url").unwrap_or("".to_string()); + cfg.snmp_community = from_python(&py, "snmp_community").unwrap_or("public".to_string()); Ok(()) } diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index cf73b6ec..0fbdfe3a 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -84,3 +84,9 @@ use_ptmp_as_parent = false enable_powercode = false powercode_api_key = "" powercode_api_url = "" + +[sonar_integration] +enable_sonar = false +sonar_api_key = "" +sonar_api_url = "" +snmp_community = "public" diff --git a/src/rust/lqos_config/src/etc/v15/mod.rs b/src/rust/lqos_config/src/etc/v15/mod.rs index 83420406..fc95839a 100644 --- a/src/rust/lqos_config/src/etc/v15/mod.rs +++ b/src/rust/lqos_config/src/etc/v15/mod.rs @@ -12,6 +12,7 @@ mod ip_ranges; mod spylnx_integration; mod uisp_integration; mod powercode_integration; +mod sonar_integration; pub use bridge::*; pub use long_term_stats::LongTermStats; pub use tuning::Tunables; \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/sonar_integration.rs b/src/rust/lqos_config/src/etc/v15/sonar_integration.rs new file mode 100644 index 00000000..94cc16e7 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/sonar_integration.rs @@ -0,0 +1,22 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SonarIntegration { + pub enable_sonar: bool, + pub sonar_api_url: String, + pub sonar_api_key: String, + pub snmp_community: String, + // TODO: It isn't clear what types `sonar_api_key,sonar_airmax_ap_model_ids,sonar_active_status_ids,sonar_ltu_ap_model_ids` + // are supposed to be. +} + +impl Default for SonarIntegration { + fn default() -> Self { + SonarIntegration { + enable_sonar: false, + sonar_api_url: "".to_string(), + sonar_api_key: "".to_string(), + snmp_community: "public".to_string(), + } + } +} \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index 39900748..f322d7da 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -62,6 +62,9 @@ pub struct Config { /// Powercode Integration pub powercode_integration: super::powercode_integration::PowercodeIntegration, + + /// Sonar Integration + pub sonar_integration: super::sonar_integration::SonarIntegration, } impl Config { @@ -123,6 +126,7 @@ impl Default for Config { spylnx_integration: super::spylnx_integration::SplynxIntegration::default(), uisp_integration: super::uisp_integration::UispIntegration::default(), powercode_integration: super::powercode_integration::PowercodeIntegration::default(), + sonar_integration: super::sonar_integration::SonarIntegration::default(), packet_capture_time: 10, queue_check_period_ms: 1000, } From 9c2747de3703c9a23948d809411315bfc0533bfd Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jan 2024 09:42:11 -0600 Subject: [PATCH 35/41] Update the scheduler to use the latest config system --- src/rust/lqos_python/src/lib.rs | 7 +++++++ src/scheduler.py | 19 ++++++------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index 61f27621..35cf75e7 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -74,6 +74,7 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(automatic_import_powercode))?; m.add_wrapped(wrap_pyfunction!(powercode_api_key))?; m.add_wrapped(wrap_pyfunction!(powercode_api_url))?; + m.add_wrapped(wrap_pyfunction!(automatic_import_sonar))?; Ok(()) } @@ -553,4 +554,10 @@ fn powercode_api_key() -> PyResult { fn powercode_api_url() -> PyResult { let config = lqos_config::load_config().unwrap(); Ok(config.powercode_integration.powercode_api_url) +} + +#[pyfunction] +fn automatic_import_sonar() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.sonar_integration.enable_sonar) } \ No newline at end of file diff --git a/src/scheduler.py b/src/scheduler.py index 7794a757..25b16b4a 100644 --- a/src/scheduler.py +++ b/src/scheduler.py @@ -2,22 +2,15 @@ import time import datetime from LibreQoS import refreshShapers, refreshShapersUpdateOnly #from graphInfluxDB import refreshBandwidthGraphs, refreshLatencyGraphs -from liblqos_python import automatic_import_uisp, automatic_import_splynx, queue_refresh_interval_mins +from liblqos_python import automatic_import_uisp, automatic_import_splynx, queue_refresh_interval_mins, \ + automatic_import_powercode, automatic_import_sonar if automatic_import_uisp(): from integrationUISP import importFromUISP if automatic_import_splynx(): from integrationSplynx import importFromSplynx -try: - from ispConfig import automaticImportPowercode -except: - automaticImportPowercode = False -if automaticImportPowercode: +if automatic_import_powercode(): from integrationPowercode import importFromPowercode -try: - from ispConfig import automaticImportSonar -except: - automaticImportSonar = False -if automaticImportSonar: +if automatic_import_sonar(): from integrationSonar import importFromSonar from apscheduler.schedulers.background import BlockingScheduler from apscheduler.executors.pool import ThreadPoolExecutor @@ -35,12 +28,12 @@ def importFromCRM(): importFromSplynx() except: print("Failed to import from Splynx") - elif automaticImportPowercode: + elif automatic_import_powercode(): try: importFromPowercode() except: print("Failed to import from Powercode") - elif automaticImportSonar: + elif automatic_import_sonar(): try: importFromSonar() except: From 8a812fe1901cfec41e21a73499222f7a34a43fdb Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jan 2024 09:46:46 -0600 Subject: [PATCH 36/41] Update the parts of the Sonar configuration that are ported. Still have to figure out some types. --- src/integrationSonar.py | 11 ++++++----- src/rust/lqos_python/src/lib.rs | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/integrationSonar.py b/src/integrationSonar.py index 18fc9399..8de14288 100644 --- a/src/integrationSonar.py +++ b/src/integrationSonar.py @@ -2,7 +2,8 @@ from pythonCheck import checkPythonVersion checkPythonVersion() import requests import subprocess -from ispConfig import sonar_api_url,sonar_api_key,sonar_airmax_ap_model_ids,sonar_active_status_ids,sonar_ltu_ap_model_ids,snmp_community +from liblqos_python import sonar_api_key, sonar_api_url, snmp_community +from ispConfig import sonar_airmax_ap_model_ids,sonar_active_status_ids,sonar_ltu_ap_model_ids all_models = sonar_airmax_ap_model_ids + sonar_ltu_ap_model_ids from integrationCommon import NetworkGraph, NetworkNode, NodeType from multiprocessing.pool import ThreadPool @@ -26,7 +27,7 @@ from multiprocessing.pool import ThreadPool def sonarRequest(query,variables={}): - r = requests.post(sonar_api_url, json={'query': query, 'variables': variables}, headers={'Authorization': 'Bearer ' + sonar_api_key}, timeout=10) + r = requests.post(sonar_api_url(), json={'query': query, 'variables': variables}, headers={'Authorization': 'Bearer ' + sonar_api_key()}, timeout=10) r_json = r.json() # Sonar responses look like this: {"data": {"accounts": {"entities": [{"id": '1'},{"id": 2}]}}} @@ -247,11 +248,11 @@ def mapApCpeMacs(ap): macs = [] macs_output = None if ap['model'] in sonar_airmax_ap_model_ids: #Tested with Prism Gen2AC and Rocket M5. - macs_output = subprocess.run(['snmpwalk', '-Os', '-v', '1', '-c', snmp_community, ap['ip'], '.1.3.6.1.4.1.41112.1.4.7.1.1.1'], capture_output=True).stdout.decode('utf8') + macs_output = subprocess.run(['snmpwalk', '-Os', '-v', '1', '-c', snmp_community(), ap['ip'], '.1.3.6.1.4.1.41112.1.4.7.1.1.1'], capture_output=True).stdout.decode('utf8') if ap['model'] in sonar_ltu_ap_model_ids: #Tested with LTU Rocket - macs_output = subprocess.run(['snmpwalk', '-Os', '-v', '1', '-c', snmp_community, ap['ip'], '.1.3.6.1.4.1.41112.1.10.1.4.1.11'], capture_output=True).stdout.decode('utf8') + macs_output = subprocess.run(['snmpwalk', '-Os', '-v', '1', '-c', snmp_community(), ap['ip'], '.1.3.6.1.4.1.41112.1.10.1.4.1.11'], capture_output=True).stdout.decode('utf8') if macs_output: - name_output = subprocess.run(['snmpwalk', '-Os', '-v', '1', '-c', snmp_community, ap['ip'], '.1.3.6.1.2.1.1.5.0'], capture_output=True).stdout.decode('utf8') + name_output = subprocess.run(['snmpwalk', '-Os', '-v', '1', '-c', snmp_community(), ap['ip'], '.1.3.6.1.2.1.1.5.0'], capture_output=True).stdout.decode('utf8') ap['name'] = name_output[name_output.find('"')+1:name_output.rfind('"')] for mac_line in macs_output.splitlines(): mac = mac_line[mac_line.find(':')+1:] diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index 35cf75e7..d0340d67 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -75,6 +75,9 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(powercode_api_key))?; m.add_wrapped(wrap_pyfunction!(powercode_api_url))?; m.add_wrapped(wrap_pyfunction!(automatic_import_sonar))?; + m.add_wrapped(wrap_pyfunction!(sonar_api_url))?; + m.add_wrapped(wrap_pyfunction!(sonar_api_key))?; + m.add_wrapped(wrap_pyfunction!(snmp_community))?; Ok(()) } @@ -560,4 +563,22 @@ fn powercode_api_url() -> PyResult { fn automatic_import_sonar() -> PyResult { let config = lqos_config::load_config().unwrap(); Ok(config.sonar_integration.enable_sonar) +} + +#[pyfunction] +fn sonar_api_url() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.sonar_integration.sonar_api_url) +} + +#[pyfunction] +fn sonar_api_key() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.sonar_integration.sonar_api_key) +} + +#[pyfunction] +fn snmp_community() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.sonar_integration.snmp_community) } \ No newline at end of file From e0e81ed7154d7de3f105fea3184465f8ca33acab Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jan 2024 10:06:46 -0600 Subject: [PATCH 37/41] Finish porting the Sonar integration over --- src/integrationSonar.py | 17 ++++++++------- .../lqos_config/src/etc/python_migration.rs | 8 +++++-- src/rust/lqos_config/src/etc/v15/example.toml | 3 +++ .../src/etc/v15/sonar_integration.rs | 8 +++++-- src/rust/lqos_python/src/lib.rs | 21 +++++++++++++++++++ 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/integrationSonar.py b/src/integrationSonar.py index 8de14288..9044d221 100644 --- a/src/integrationSonar.py +++ b/src/integrationSonar.py @@ -2,9 +2,10 @@ from pythonCheck import checkPythonVersion checkPythonVersion() import requests import subprocess -from liblqos_python import sonar_api_key, sonar_api_url, snmp_community -from ispConfig import sonar_airmax_ap_model_ids,sonar_active_status_ids,sonar_ltu_ap_model_ids -all_models = sonar_airmax_ap_model_ids + sonar_ltu_ap_model_ids +from liblqos_python import sonar_api_key, sonar_api_url, snmp_community, sonar_airmax_ap_model_ids, \ + sonar_ltu_ap_model_ids, sonar_active_status_ids +#from ispConfig import sonar_airmax_ap_model_ids,sonar_active_status_ids,sonar_ltu_ap_model_ids +all_models = sonar_airmax_ap_model_ids() + sonar_ltu_ap_model_ids() from integrationCommon import NetworkGraph, NetworkNode, NodeType from multiprocessing.pool import ThreadPool @@ -37,7 +38,7 @@ def sonarRequest(query,variables={}): return sonar_list def getActiveStatuses(): - if not sonar_active_status_ids: + if not sonar_active_status_ids(): query = """query getActiveStatuses { account_statuses (activates_account: true) { entities { @@ -53,7 +54,7 @@ def getActiveStatuses(): status_ids.append(status['id']) return status_ids else: - return sonar_active_status_ids + return sonar_active_status_ids() # Sometimes the IP will be under the field data for an item and sometimes it will be assigned to the inventory item itself. def findIPs(inventory_item): @@ -185,7 +186,7 @@ def getAccounts(sonar_active_status_ids): }""" active_status_ids = [] - for status_id in sonar_active_status_ids: + for status_id in sonar_active_status_ids(): active_status_ids.append({ "attribute": "account_status_id", "operator": "EQ", @@ -247,9 +248,9 @@ def getAccounts(sonar_active_status_ids): def mapApCpeMacs(ap): macs = [] macs_output = None - if ap['model'] in sonar_airmax_ap_model_ids: #Tested with Prism Gen2AC and Rocket M5. + if ap['model'] in sonar_airmax_ap_model_ids(): #Tested with Prism Gen2AC and Rocket M5. macs_output = subprocess.run(['snmpwalk', '-Os', '-v', '1', '-c', snmp_community(), ap['ip'], '.1.3.6.1.4.1.41112.1.4.7.1.1.1'], capture_output=True).stdout.decode('utf8') - if ap['model'] in sonar_ltu_ap_model_ids: #Tested with LTU Rocket + if ap['model'] in sonar_ltu_ap_model_ids(): #Tested with LTU Rocket macs_output = subprocess.run(['snmpwalk', '-Os', '-v', '1', '-c', snmp_community(), ap['ip'], '.1.3.6.1.4.1.41112.1.10.1.4.1.11'], capture_output=True).stdout.decode('utf8') if macs_output: name_output = subprocess.run(['snmpwalk', '-Os', '-v', '1', '-c', snmp_community(), ap['ip'], '.1.3.6.1.2.1.1.5.0'], capture_output=True).stdout.decode('utf8') diff --git a/src/rust/lqos_config/src/etc/python_migration.rs b/src/rust/lqos_config/src/etc/python_migration.rs index 797bbde7..92454f3f 100644 --- a/src/rust/lqos_config/src/etc/python_migration.rs +++ b/src/rust/lqos_config/src/etc/python_migration.rs @@ -100,8 +100,9 @@ pub struct PythonMigration { pub sonar_api_url: String, pub sonar_api_key: String, pub snmp_community: String, - // TODO: It isn't clear what types `sonar_api_key,sonar_airmax_ap_model_ids,sonar_active_status_ids,sonar_ltu_ap_model_ids` - // are supposed to be. + pub sonar_airmax_ap_model_ids: Vec, + pub sonar_ltu_ap_model_ids: Vec, + pub sonar_active_status_ids: Vec, // TODO: httpRestIntegrationConfig } @@ -177,6 +178,9 @@ impl PythonMigration { cfg.sonar_api_key = from_python(&py, "sonar_api_key").unwrap_or("".to_string()); cfg.sonar_api_url = from_python(&py, "sonar_api_url").unwrap_or("".to_string()); cfg.snmp_community = from_python(&py, "snmp_community").unwrap_or("public".to_string()); + cfg.sonar_active_status_ids = from_python(&py, "sonar_active_status_ids").unwrap_or(vec![]); + cfg.sonar_airmax_ap_model_ids = from_python(&py, "sonar_airmax_ap_model_ids").unwrap_or(vec![]); + cfg.sonar_ltu_ap_model_ids = from_python(&py, "sonar_ltu_ap_model_ids").unwrap_or(vec![]); Ok(()) } diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index 0fbdfe3a..184ae217 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -90,3 +90,6 @@ enable_sonar = false sonar_api_key = "" sonar_api_url = "" snmp_community = "public" +airmax_model_ids = [ "" ] +ltu_model_ids = [ "" ] +active_status_ids = [ "" ] diff --git a/src/rust/lqos_config/src/etc/v15/sonar_integration.rs b/src/rust/lqos_config/src/etc/v15/sonar_integration.rs index 94cc16e7..1dfb5ca4 100644 --- a/src/rust/lqos_config/src/etc/v15/sonar_integration.rs +++ b/src/rust/lqos_config/src/etc/v15/sonar_integration.rs @@ -6,8 +6,9 @@ pub struct SonarIntegration { pub sonar_api_url: String, pub sonar_api_key: String, pub snmp_community: String, - // TODO: It isn't clear what types `sonar_api_key,sonar_airmax_ap_model_ids,sonar_active_status_ids,sonar_ltu_ap_model_ids` - // are supposed to be. + pub airmax_model_ids: Vec, + pub ltu_model_ids: Vec, + pub active_status_ids: Vec, } impl Default for SonarIntegration { @@ -17,6 +18,9 @@ impl Default for SonarIntegration { sonar_api_url: "".to_string(), sonar_api_key: "".to_string(), snmp_community: "public".to_string(), + airmax_model_ids: vec![], + ltu_model_ids: vec![], + active_status_ids: vec![], } } } \ No newline at end of file diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index d0340d67..a7afa1a6 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -78,6 +78,9 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(sonar_api_url))?; m.add_wrapped(wrap_pyfunction!(sonar_api_key))?; m.add_wrapped(wrap_pyfunction!(snmp_community))?; + m.add_wrapped(wrap_pyfunction!(sonar_airmax_ap_model_ids))?; + m.add_wrapped(wrap_pyfunction!(sonar_ltu_ap_model_ids))?; + m.add_wrapped(wrap_pyfunction!(sonar_active_status_ids))?; Ok(()) } @@ -581,4 +584,22 @@ fn sonar_api_key() -> PyResult { fn snmp_community() -> PyResult { let config = lqos_config::load_config().unwrap(); Ok(config.sonar_integration.snmp_community) +} + +#[pyfunction] +fn sonar_airmax_ap_model_ids() -> PyResult> { + let config = lqos_config::load_config().unwrap(); + Ok(config.sonar_integration.airmax_model_ids) +} + +#[pyfunction] +fn sonar_ltu_ap_model_ids() -> PyResult> { + let config = lqos_config::load_config().unwrap(); + Ok(config.sonar_integration.ltu_model_ids) +} + +#[pyfunction] +fn sonar_active_status_ids() -> PyResult> { + let config = lqos_config::load_config().unwrap(); + Ok(config.sonar_integration.active_status_ids) } \ No newline at end of file From 699e265850b9f07cd75b8626d95dbfd707f7dc34 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jan 2024 10:12:57 -0600 Subject: [PATCH 38/41] Add some small documentation for adding config items. --- src/rust/lqos_config/README.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/rust/lqos_config/README.md b/src/rust/lqos_config/README.md index e117b979..dda8461f 100644 --- a/src/rust/lqos_config/README.md +++ b/src/rust/lqos_config/README.md @@ -1,15 +1,23 @@ # LQosConfig -`lqos_config` is designed to manage configuration of LibreQoS. +`lqos_config` is designed to manage configuration of LibreQoS. Starting in 1.5, all configuration is +centralized into `/etc/lqos.conf`. -Since all of the parts of the system need to know where to find LibreQoS, it first looks for a file named `/etc/lqos.conf` and uses that to locate the LibreQoS installation. +The `lqos_python` module contains functions that mirror each of these, using their original Python +names for integration purposes. -`/etc/lqos.conf` looks like this: +You can find the full definitions of each configuration entry in `src/etc/v15`. -```toml -lqos_directory = '/opt/libreqos' -``` +## Adding Configuration Items -The entries are: +There are two ways to add a configuration: -* `lqos_directory`: where LibreQoS is installed (e.g. `/opt/libreqos`) +1. Declare a Major Version Break. This is a whole new setup that will require a new configuration and migration. We should avoid doing this very often. + 1. You need to create a new folder, e.g. `src/etc/v16`. + 2. You need to port as much of the old config as you are creating. + 3. You need to update `src/etc/migration.rs` to include code to read a "v15" file and create a "v16" configuration. + 4. *This is a lot of work and should be a planned effort!* +2. Declare an optional new version item. This is how you handle "oh, I needed to snoo the foo" - and add an *optional* configuration item - so nothing will snarl up because it isn't there. + 1. Find the section you want to include it in in `src/etc/v15`. If there isn't one, create it using one of the others as a template and be sure to include the defaults. Add it into `top_config` as the type `Option`. + 2. Update `example.toml` to include what *should* go there. + 3. Go into `lqos_python` and in `lib.rs` add a Python "getter" for the field. Remember to use `if let` to read the `Option` and return a default if it isn't present. From 1ee3543eb198a5005852bc551407971111faa25d Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sun, 4 Feb 2024 20:55:31 -0600 Subject: [PATCH 39/41] Add documentation updates for the unified config --- docs/ChangeNotes/v1.5.md | 5 + docs/Quickstart/configuration.md | 26 ++-- docs/TechnicalDocs/integrations.md | 16 +- src/ispConfig.example.py | 180 ----------------------- src/lqos.conf.new.discuss | 226 ----------------------------- src/lqos.example | 104 +++++++++---- src/rust/lqosd/README.md | 2 +- 7 files changed, 102 insertions(+), 457 deletions(-) create mode 100644 docs/ChangeNotes/v1.5.md delete mode 100644 src/ispConfig.example.py delete mode 100644 src/lqos.conf.new.discuss diff --git a/docs/ChangeNotes/v1.5.md b/docs/ChangeNotes/v1.5.md new file mode 100644 index 00000000..94c0f252 --- /dev/null +++ b/docs/ChangeNotes/v1.5.md @@ -0,0 +1,5 @@ +# LibreQoS v1.4 to v1.5 Change Summary + +## Unified Configuration + +All configuration has been moved into `/etc/lqos.conf`. \ No newline at end of file diff --git a/docs/Quickstart/configuration.md b/docs/Quickstart/configuration.md index 9c4ccec6..430bc0b1 100644 --- a/docs/Quickstart/configuration.md +++ b/docs/Quickstart/configuration.md @@ -15,27 +15,23 @@ Now edit the file to match your setup with sudo nano /etc/lqos.conf ``` -Change `enp1s0f1` and `enp1s0f2` to match your network interfaces. It doesn't matter which one is which. Notice, it's paring the interfaces, so when you first enter enps0f**1** in the first line, the `redirect_to` parameter is enp1s0f**2** (replacing with your actual interface names). - -- First Line: `name = "enp1s0f1", redirect_to = "enp1s0f2"` -- Second Line: `name = "enp1s0f2", redirect_to = "enp1s0f1"` +Change `eth0` and `eth1` to match your network interfaces. The interface facing the Internet should be specified in `to_internet`. The interfacing facing your ISP network should be in `to_network`. Then, if using Bifrost/XDP set `use_xdp_bridge = true` under that same `[bridge]` section. -## Configure ispConfig.py +For example: -Copy ispConfig.example.py to ispConfig.py and edit as needed - -```shell -cd /opt/libreqos/src/ -cp ispConfig.example.py ispConfig.py -nano ispConfig.py +```toml +[bridge] +use_xdp_bridge = true +to_internet = "eth0" +to_network = "eth1" ``` -- Set upstreamBandwidthCapacityDownloadMbps and upstreamBandwidthCapacityUploadMbps to match the bandwidth in Mbps of your network's upstream / WAN internet connection. The same can be done for generatedPNDownloadMbps and generatedPNUploadMbps. -- Set interfaceA to the interface facing your core router (or bridged internal network if your network is bridged) -- Set interfaceB to the interface facing your edge router -- Set ```enableActualShellCommands = True``` to allow the program to actually run the commands. +## Configure Your Network Settings + +- Set `uplink_bandwidth_mbps` and `downlink_bandwidth_mbps` to match the bandwidth in Mbps of your network's upstream / WAN internet connection. The same can be done for `generated_pn_download_mbps` and `generated_pn_upload_mbps`. +- Set ```dry_run = false``` to allow the program to actually run the commands. ## Network.json diff --git a/docs/TechnicalDocs/integrations.md b/docs/TechnicalDocs/integrations.md index 9d46eae6..c34d884a 100644 --- a/docs/TechnicalDocs/integrations.md +++ b/docs/TechnicalDocs/integrations.md @@ -2,7 +2,7 @@ ## UISP Integration -First, set the relevant parameters for UISP (uispAuthToken, UISPbaseURL, etc.) in ispConfig.py. +First, set the relevant parameters for UISP (uispAuthToken, UISPbaseURL, etc.) in `/etc/lqos.conf`. To test the UISP Integration, use @@ -14,11 +14,11 @@ On the first successful run, it will create a network.json and ShapedDevices.csv If a network.json file exists, it will not be overwritten. You can modify the network.json file to more accurately reflect bandwidth limits. ShapedDevices.csv will be overwritten every time the UISP integration is run. -You have the option to run integrationUISP.py automatically on boot and every 10 minutes, which is recommended. This can be enabled by setting ```automaticImportUISP = True``` in ispConfig.py +You have the option to run integrationUISP.py automatically on boot and every 10 minutes, which is recommended. This can be enabled by setting ```enable_uisp = true``` in `/etc/lqos.conf` ## Powercode Integration -First, set the relevant parameters for Sonar (powercode_api_key, powercode_api_url, etc.) in ispConfig.py. +First, set the relevant parameters for Powercode (powercode_api_key, powercode_api_url, etc.) in `/etc/lqos.conf`. To test the Powercode Integration, use @@ -29,11 +29,11 @@ python3 integrationPowercode.py On the first successful run, it will create a ShapedDevices.csv file. You can modify the network.json file manually to reflect Site/AP bandwidth limits. ShapedDevices.csv will be overwritten every time the Powercode integration is run. -You have the option to run integrationPowercode.py automatically on boot and every 10 minutes, which is recommended. This can be enabled by setting ```automaticImportPowercode = True``` in ispConfig.py +You have the option to run integrationPowercode.py automatically on boot and every 10 minutes, which is recommended. This can be enabled by setting ```enable_powercode = true``` in `/etc/lqos.conf` ## Sonar Integration -First, set the relevant parameters for Sonar (sonar_api_key, sonar_api_url, etc.) in ispConfig.py. +First, set the relevant parameters for Sonar (sonar_api_key, sonar_api_url, etc.) in `/etc/lqos.conf`. To test the Sonar Integration, use @@ -45,11 +45,11 @@ On the first successful run, it will create a ShapedDevices.csv file. If a network.json file exists, it will not be overwritten. You can modify the network.json file to more accurately reflect bandwidth limits. ShapedDevices.csv will be overwritten every time the Sonar integration is run. -You have the option to run integrationSonar.py automatically on boot and every 10 minutes, which is recommended. This can be enabled by setting ```automaticImportSonar = True``` in ispConfig.py +You have the option to run integrationSonar.py automatically on boot and every 10 minutes, which is recommended. This can be enabled by setting ```enable_sonar = true``` in `/etc/lqos.conf` ## Splynx Integration -First, set the relevant parameters for Splynx (splynx_api_key, splynx_api_secret, etc.) in ispConfig.py. +First, set the relevant parameters for Splynx (splynx_api_key, splynx_api_secret, etc.) in `/etc/lqos.conf`. The Splynx Integration uses Basic authentication. For using this type of authentication, please make sure you enable [Unsecure access](https://splynx.docs.apiary.io/#introduction/authentication) in your Splynx API key settings. Also the Splynx API key should be granted access to the necessary permissions. @@ -62,4 +62,4 @@ python3 integrationSplynx.py On the first successful run, it will create a ShapedDevices.csv file. You can manually create your network.json file to more accurately reflect bandwidth limits. ShapedDevices.csv will be overwritten every time the Splynx integration is run. -You have the option to run integrationSplynx.py automatically on boot and every 10 minutes, which is recommended. This can be enabled by setting ```automaticImportSplynx = True``` in ispConfig.py +You have the option to run integrationSplynx.py automatically on boot and every 10 minutes, which is recommended. This can be enabled by setting ```enable_spylnx = true``` in `/etc/lqos.conf`. diff --git a/src/ispConfig.example.py b/src/ispConfig.example.py deleted file mode 100644 index 53220bd2..00000000 --- a/src/ispConfig.example.py +++ /dev/null @@ -1,180 +0,0 @@ -# 'fq_codel' or 'cake diffserv4' -# 'cake diffserv4' is recommended -# sqm = 'fq_codel' -sqm = 'cake diffserv4' - -# Used to passively monitor the network for before / after comparisons. Leave as False to -# ensure actual shaping. After changing this value, run "sudo systemctl restart LibreQoS.service" -monitorOnlyMode = False - -# How many Mbps are available to the edge of this network. -# Any circuits, generated nodes, or network.json nodes, will all be capped at no more than this amount. -upstreamBandwidthCapacityDownloadMbps = 1000 -upstreamBandwidthCapacityUploadMbps = 1000 - -# Consider these values your bandwidth bottleneck per-CPU-core. -# This will depend on your CPU's single-thread passmark score. -# Devices in ShapedDevices.csv without a defined ParentNode (such as if you have a flat {} network) -# will be placed under one of these generated parent node, evenly spread out across CPU cores. -# This defines the bandwidth limit for each of those generated parent nodes. -# If you are not sure what values to use, simply use the same values as upstreamBandwidthCapacityDownloadMbps and upstreamBandwidthCapacityUploadMbps -generatedPNDownloadMbps = 1000 -generatedPNUploadMbps = 1000 - -# Interface connected to core router -interfaceA = 'eth1' - -# Interface connected to edge router -interfaceB = 'eth2' - -# Queue refresh scheduler (lqos_scheduler). Minutes between reloads. -queueRefreshIntervalMins = 30 - -# WORK IN PROGRESS. Note that interfaceA determines the "stick" interface -# I could only get scanning to work if I issued ethtool -K enp1s0f1 rxvlan off -OnAStick = False -# VLAN facing the core router -StickVlanA = 0 -# VLAN facing the edge router -StickVlanB = 0 - -# Allow shell commands. False causes commands print to console only without being executed. -# MUST BE ENABLED FOR PROGRAM TO FUNCTION -enableActualShellCommands = True - -# Add 'sudo' before execution of any shell commands. May be required depending on distribution and environment. -runShellCommandsAsSudo = False - -# Allows overriding queues / CPU cores used. When set to 0, the max possible queues / CPU cores are utilized. Please -# leave as 0. -queuesAvailableOverride = 0 - -# Devices in in ShapedDevices.csv without defined Parent Nodes are placed in generated parent nodes. -# When set True, this option balances the subscribers across generatedPNs / CPU cores based on the subscriber's bandwidth plan. -# When set False, devices are placed in generatedPNs sequentially with a near equal number of subscribers per core. -# Whether this impacts balance across CPUs will depend on your subscribers' usage patterns, but if you are observing -# unequal CPU load, and have most subscribers without a defined Parent Node, it is recommended to try this option. -# Most subscribers average about the same bandwidth load regardless of speed plan (typically 5Mbps or so). -# Past 25,000 subscribers this option becomes inefficient and is not advised. -useBinPackingToBalanceCPU = False - -# Bandwidth & Latency Graphing -influxDBEnabled = True -influxDBurl = "http://localhost:8086" -influxDBBucket = "libreqos" -influxDBOrg = "Your ISP Name Here" -influxDBtoken = "" - -# NMS/CRM Integration - -# Use Customer Name or Address as Circuit Name -circuitNameUseAddress = True - -# Should integrationUISP overwrite network.json on each run? -overwriteNetworkJSONalways = False - -# If a device shows a WAN IP within these subnets, assume they are behind NAT / un-shapable, and ignore them -ignoreSubnets = ['192.168.0.0/16'] -allowedSubnets = ['100.64.0.0/10'] - -# Powercode Integration -automaticImportPowercode = False -powercode_api_key = '' -# Everything before :444/api/ in your Powercode instance URL -powercode_api_url = '' - -# Sonar Integration -automaticImportSonar = False -sonar_api_key = '' -sonar_api_url = '' # ex 'https://company.sonar.software/api/graphql' -# If there are radios in these lists, we will try to get the clients using snmp. This requires snmpwalk to be install on the server. You can use "sudo apt-get install snmp" for that. You will also need to fill in the snmp_community. -sonar_airmax_ap_model_ids = [] # ex ['29','43'] -sonar_ltu_ap_model_ids = [] # ex ['4'] -snmp_community = '' -# This is for all account statuses where we should be applying QoS. If you leave it blank, we'll use any status in account marked with "Activates Account" in Sonar. -sonar_active_status_ids = [] - -# Splynx Integration -automaticImportSplynx = False -splynx_api_key = '' -splynx_api_secret = '' -# Everything before /api/2.0/ on your Splynx instance -splynx_api_url = 'https://YOUR_URL.splynx.app' - -#Sonar Integration -automaticImportSonar = False -sonar_api_key = '' -sonar_api_url = '' # ex 'https://company.sonar.software/api/graphql' -# If there are radios in these lists, we will try to get the clients using snmp. This requires snmpwalk to be install on the server. You can use "sudo apt-get install snmp" for that. You will also need to fill in the snmp_community. -sonar_airmax_ap_model_ids = [] # ex ['29','43'] -sonar_ltu_ap_model_ids = [] # ex ['4'] -snmp_community = '' -# This is for all account statuses where we should be applying QoS. If you leave it blank, we'll use any status in account marked with "Activates Account" in Sonar. -sonar_active_status_ids = [] - - -# UISP integration -automaticImportUISP = False -uispAuthToken = '' -# Everything before /nms/ on your UISP instance -UISPbaseURL = 'https://examplesite.com' -# UISP Site - enter the name of the root site in your network tree -# to act as the starting point for the tree mapping -uispSite = '' -# Strategy: -# * "flat" - create all client sites directly off the top of the tree, -# provides maximum performance - at the expense of not offering AP, -# or site options. -# * "full" - build a complete network map -uispStrategy = "full" -# Handling of UISP suspensions: -# * "none" - do not handle suspensions -# * "ignore" - do not add suspended customers to the network map -# * "slow" - limit suspended customers to 1mbps -uispSuspendedStrategy = "none" -# Assumed capacity of AirMax and LTU radios vs reported capacity by UISP. For example, 65% would be 0.65. -# For AirMax, this applies to flexible frame only. AirMax fixed frame will have capacity based on ratio. -airMax_capacity = 0.65 -ltu_capacity = 0.90 -# List any sites that should not be included, with each site name surrounded by '' and separated by commas -excludeSites = [] -# If you use IPv6, this can be used to find associated IPv6 prefixes for your clients' IPv4 addresses, and match them -# to those devices -findIPv6usingMikrotik = False -# If you want to provide a safe cushion for speed test results to prevent customer complains, you can set this to -# 1.15 (15% above plan rate). If not, you can leave as 1.0 -bandwidthOverheadFactor = 1.0 -# Number to multiply the maximum/ceiling bandwidth with to determine the minimum bandwidth. -committedBandwidthMultiplier = 0.98 -# For edge cases, set the respective ParentNode for these CPEs -exceptionCPEs = {} -# exceptionCPEs = { -# 'CPE-SomeLocation1': 'AP-SomeLocation1', -# 'CPE-SomeLocation2': 'AP-SomeLocation2', -# } - -# API Auth -apiUsername = "testUser" -apiPassword = "changeme8343486806" -apiHostIP = "127.0.0.1" -apiHostPost = 5000 - - -httpRestIntegrationConfig = { - 'enabled': False, - 'baseURL': 'https://domain', - 'networkURI': '/some/path', - 'shaperURI': '/some/path/etc', - 'requestsConfig': { - 'verify': True, # Good for Dev if your dev env doesnt have cert - 'params': { # params for query string ie uri?some-arg=some-value - 'search': 'hold-my-beer' - }, - #'headers': { - # 'Origin': 'SomeHeaderValue', - #}, - }, - # If you want to store a timestamped copy/backup of both network.json and Shaper.csv each time they are updated, - # provide a path - # 'logChanges': '/var/log/libreqos' -} diff --git a/src/lqos.conf.new.discuss b/src/lqos.conf.new.discuss deleted file mode 100644 index 0b586ade..00000000 --- a/src/lqos.conf.new.discuss +++ /dev/null @@ -1,226 +0,0 @@ -[main] - -lqos_directory = '/etc/lqos/' # /etc/lqos seems saner -lqos_bus = '/run/lqos' - -[perms] - -max_users = 0 # limiting connects is sane -group = 'lqos' -umask = 0770 # Restrict access to the bus to lqos group and root - -[stats] - -queue_check_period_us = 1000000 # 1/2 rx_usecs would be nice - -[tuning] -stop_irq_balance = true -netdev_budget_usecs = 8000 -netdev_budget_packets = 300 -rx_usecs = 8 -tx_usecs = 8 -disable_rxvlan = true -disable_txvlan = true -disable_offload = [ "gso", "tso", "lro", "sg", "gro" ] - -# For a two interface setup, use the following - and replace -# "enp1s0f1" and "enp1s0f2" with your network card names (obtained -# from `ip link`): - -[bridge] -use_xdp_bridge = true -interface_mapping = [ - { name = "enp1s0f1", redirect_to = "enp1s0f2", scan_vlans = false }, - { name = "enp1s0f2", redirect_to = "enp1s0f1", scan_vlans = false } -] -vlan_mapping = [] -# For "on a stick" (single interface mode): -# [bridge] -# use_xdp_bridge = true -# interface_mapping = [ -# { name = "enp1s0f1", redirect_to = "enp1s0f1", scan_vlans = true } -# ] -# vlan_mapping = [ -# { parent = "enp1s0f1", tag = 3, redirect_to = 4 }, -# { parent = "enp1s0f1", tag = 4, redirect_to = 3 } -# ] - -# Does the linux bridge still work? How do you set it up? It seems -# as hot as we are on all this new stuff the lowest barrier to entry -# is a default of the linux bridge. - -# How does one setup a Proxmox VM? Everyone except the testbed is on a vm. - -# NMS/CRM Integration - -[NMS] - -# If a device shows a WAN IP within these subnets... -# assume they are behind NAT / un-shapable, and ignore them - -ignoreSubnets = ['192.168.0.0/16'] -allowedSubnets = ['100.64.0.0/10'] - -# Stuff appearing on the bridge not on these networks is bad -# Spoofed traffic, non BCP38 issues from customers, etc also bad -# I am also not big on caseING variable names - -mySubnets = ['x.y.z.x/22'] -myTunnels = ['192.168.0.0/16'] # Say we use a subset of 10/8 or ... - -[IspConfig] - -# 'fq_codel' or 'cake diffserv4' -# 'cake diffserv4' is recommended -# sqm = 'fq_codel' - -sqm = 'cake diffserv4' -sqm_in = 'why do we think in and out should be the same?' -sqm_out = 'why do we think in and out should be the same?' - -# Used to passively monitor the network for before / after comparisons. Leave as False to -# ensure actual shaping. After changing this value, run "sudo systemctl restart LibreQoS.service" - -monitorOnlyMode = False - -# How many Mbps are available to the edge of this network - -# Does this mean we are ALSO applying this as a shaped rate in or out of the network? - -upstreamBandwidthCapacityDownloadMbps = 1000 -upstreamBandwidthCapacityUploadMbps = 1000 - -# Devices in ShapedDevices.csv without a defined ParentNode will be placed under a generated -# parent node, evenly spread out across CPU cores. Here, define the bandwidth limit for each -# of those generated parent nodes. - -# and if that is the case, why does this make sense? - -generatedPNDownloadMbps = 1000 -generatedPNUploadMbps = 1000 - -# These seem to be duplicate and incomplete from the other stuff above -# How does one (assuming we keep this file) use on a stick here? -# There should be one way only to configure on a stick mode - -# We should retire these and just attach to the bridge per the rust -# Interface connected to core router -interfaceA = 'eth1' - -# Interface connected to edge router -interfaceB = 'eth2' - -# WORK IN PROGRESS. Note that interfaceA determines the "stick" interface -# I could only get scanning to work if I issued ethtool -K enp1s0f1 rxvlan off - -OnAStick = False -# VLAN facing the core router - -StickVlanA = 0 -# VLAN facing the edge router - -StickVlanB = 0 - -# Allow shell commands. False causes commands print to console only without being executed. -# MUST BE ENABLED FOR PROGRAM TO FUNCTION - -enableActualShellCommands = True - -# Add 'sudo' before execution of any shell commands. May be required depending on distribution and environment. -# what happens when run from systemd, vs the command line? -runShellCommandsAsSudo = False - -# Allows overriding queues / CPU cores used. When set to 0, the max possible queues / CPU cores are utilized. Please leave as 0. Why? - -queuesAvailableOverride = 0 - -# Some networks are flat - where there are no Parent Nodes defined in ShapedDevices.csv -# For such flat networks, just define network.json as {} and enable this setting -# By default, it balances the subscribers across CPU cores, factoring in their max bandwidth rates -# Past 25,000 subsribers this algorithm becomes inefficient and is not advised - -useBinPackingToBalanceCPU = True - -[InfluxDB] - -# Bandwidth & Latency Graphing -influxDBEnabled = True -influxDBurl = "http://localhost:8086" -influxDBBucket = "libreqos" -influxDBOrg = "Your ISP Name Here" -influxDBtoken = "" - -[Splynx] - -# Splynx Integration -automaticImportSplynx = False -splynx_api_key = '' -splynx_api_secret = '' -# Everything before /api/2.0/ on your Splynx instance -splynx_api_url = 'https://YOUR_URL.splynx.app' - -# UISP integration -[UISP] -automaticImportUISP = False -uispAuthToken = '' -# Everything before /nms/ on your UISP instance -UISPbaseURL = 'https://examplesite.com' -# UISP Site - enter the name of the root site in your network tree -# to act as the starting point for the tree mapping -uispSite = '' - -# Strategy: -# * "flat" - create all client sites directly off the top of the tree, -# provides maximum performance - at the expense of not offering AP, -# or site options. -# * "full" - build a complete network map -uispStrategy = "full" - -# List any sites that should not be included, with each site name surrounded by '' -# and separated by commas - -excludeSites = [] - -# If you use IPv6, this can be used to find associated IPv6 prefixes -# for your clients' IPv4 addresses, and match them -# to those devices - -findIPv6usingMikrotik = False - -# If you want to provide a safe cushion for speed test results to prevent customer complaints, -# you can set this to 1.15 (15% above plan rate). If not, you can leave as 1.0 - -bandwidthOverheadFactor = 1.0 - -# For edge cases, set the respective ParentNode for these CPEs -exceptionCPEs = {} - -# exceptionCPEs = { -# 'CPE-SomeLocation1': 'AP-SomeLocation1', -# 'CPE-SomeLocation2': 'AP-SomeLocation2', -# } - -# API Auth -apiUsername = "testUser" -apiPassword = "changeme8343486806" -apiHostIP = "127.0.0.1" -apiHostPost = 5000 - -httpRestIntegrationConfig = { - 'enabled': False, - 'baseURL': 'https://domain', - 'networkURI': '/some/path', - 'shaperURI': '/some/path/etc', - 'requestsConfig': { - 'verify': True, # Good for Dev if your dev env doesnt have cert - 'params': { # params for query string ie uri?some-arg=some-value - 'search': 'hold-my-beer' - }, - #'headers': { - # 'Origin': 'SomeHeaderValue', - #}, - }, - # If you want to store a timestamped copy/backup of both network.json and Shaper.csv each time they are updated, - # provide a path - # 'logChanges': '/var/log/libreqos' -} diff --git a/src/lqos.example b/src/lqos.example index 4fb5806c..184ae217 100644 --- a/src/lqos.example +++ b/src/lqos.example @@ -1,17 +1,15 @@ -# This file *must* be installed in `/etc/lqos.conf`. -# Change the values to match your setup. - -# Where is LibreQoS installed? -lqos_directory = '/opt/libreqos/src' +version = "1.5" +lqos_directory = "/opt/libreqos/src" +node_id = "0000-0000-0000" +node_name = "Example Node" +packet_capture_time = 10 queue_check_period_ms = 1000 -packet_capture_time = 10 # Number of seconds to capture packets in an analysis session [usage_stats] send_anonymous = true -anonymous_server = "127.0.0.1:9125" +anonymous_server = "stats.libreqos.io:9125" [tuning] -# IRQ balance breaks XDP_Redirect, which we use. Recommended to leave as true. stop_irq_balance = true netdev_budget_usecs = 8000 netdev_budget_packets = 300 @@ -19,27 +17,79 @@ rx_usecs = 8 tx_usecs = 8 disable_rxvlan = true disable_txvlan = true -# What offload types should be disabled on the NIC. The defaults are recommended here. disable_offload = [ "gso", "tso", "lro", "sg", "gro" ] -# For a two interface setup, use the following - and replace -# "enp1s0f1" and "enp1s0f2" with your network card names (obtained -# from `ip link`): +# EITHER: [bridge] use_xdp_bridge = true -interface_mapping = [ - { name = "enp1s0f1", redirect_to = "enp1s0f2", scan_vlans = false }, - { name = "enp1s0f2", redirect_to = "enp1s0f1", scan_vlans = false } -] -vlan_mapping = [] +to_internet = "eth0" +to_network = "eth1" -# For "on a stick" (single interface mode): -# [bridge] -# use_xdp_bridge = true -# interface_mapping = [ -# { name = "enp1s0f1", redirect_to = "enp1s0f1", scan_vlans = true } -# ] -# vlan_mapping = [ -# { parent = "enp1s0f1", tag = 3, redirect_to = 4 }, -# { parent = "enp1s0f1", tag = 4, redirect_to = 3 } -# ] +# OR: +#[single_interface] +#interface = "eth0" +#internet_vlan = 2 +#network_vlan = 3 + +[queues] +default_sqm = "cake diffserv4" +monitor_only = false +uplink_bandwidth_mbps = 1000 +downlink_bandwidth_mbps = 1000 +generated_pn_download_mbps = 1000 +generated_pn_upload_mbps = 1000 +dry_run = false +sudo = false +#override_available_queues = 12 # This can be omitted and be 0 for Python +use_binpacking = false + +[long_term_stats] +gather_stats = true +collation_period_seconds = 10 +license_key = "(data)" +uisp_reporting_interval_seconds = 300 + +[ip_ranges] +ignore_subnets = [] +allow_subnets = [ "172.16.0.0/12", "10.0.0.0/8", "100.64.0.0/16", "192.168.0.0/16" ] + +[integration_common] +circuit_name_as_address = false +always_overwrite_network_json = false +queue_refresh_interval_mins = 30 + +[spylnx_integration] +enable_spylnx = false +api_key = "" +api_secret = "" +url = "" + +[uisp_integration] +enable_uisp = false +token = "" +url = "" +site = "" +strategy = "" +suspended_strategy = "" +airmax_capacity = 0.65 +ltu_capacity = 0.9 +exclude_sites = [] +ipv6_with_mikrotik = false +bandwidth_overhead_factor = 1.0 +commit_bandwidth_multiplier = 0.98 +exception_cpes = [] +use_ptmp_as_parent = false + +[powercode_integration] +enable_powercode = false +powercode_api_key = "" +powercode_api_url = "" + +[sonar_integration] +enable_sonar = false +sonar_api_key = "" +sonar_api_url = "" +snmp_community = "public" +airmax_model_ids = [ "" ] +ltu_model_ids = [ "" ] +active_status_ids = [ "" ] diff --git a/src/rust/lqosd/README.md b/src/rust/lqosd/README.md index 1e68e29e..92f6eb50 100644 --- a/src/rust/lqosd/README.md +++ b/src/rust/lqosd/README.md @@ -3,7 +3,7 @@ **The LibreQoS Daemon** is designed to run as a `systemd` service at all times. It provides: * Load/Unload the XDP/TC programs (they unload when the program exits) -* Configure XDP/TC, based on the content of `ispConfig.py`. +* Configure XDP/TC, based on the content of `/etc/lqos.conf`. * Includes support for "on a stick" mode, using `OnAStick = True, StickVlanA = 1, StickVlanB = 2`. * Hosts a lightweight server offering "bus" queries for clients (such as `lqtop` and `xdp_iphash_to_cpu_cmdline`). * See the `lqos_bus` sub-project for bus details. From 3ab165a591154b0d39d32f0c9b86344fcfb4fa4b Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sun, 4 Feb 2024 21:29:02 -0600 Subject: [PATCH 40/41] Re-add InfluxDB Support to the unified configuration. --- src/graphInfluxDB.py | 1065 ++++++++--------- src/integrationCommon.py | 2 +- src/integrationRestHttp.py | 114 +- src/integrationSonar.py | 3 +- src/lqos.example | 7 + .../lqos_config/src/etc/etclqos_migration.rs | 2 +- src/rust/lqos_config/src/etc/migration.rs | 13 + .../lqos_config/src/etc/python_migration.rs | 7 + src/rust/lqos_config/src/etc/v15/example.toml | 7 + src/rust/lqos_config/src/etc/v15/influxdb.rs | 22 + src/rust/lqos_config/src/etc/v15/mod.rs | 1 + .../lqos_config/src/etc/v15/top_config.rs | 4 + src/rust/lqos_python/src/lib.rs | 35 + 13 files changed, 688 insertions(+), 594 deletions(-) create mode 100644 src/rust/lqos_config/src/etc/v15/influxdb.rs diff --git a/src/graphInfluxDB.py b/src/graphInfluxDB.py index 327497b6..4e04abcf 100644 --- a/src/graphInfluxDB.py +++ b/src/graphInfluxDB.py @@ -1,621 +1,618 @@ -print("influxDB Support is Deperecated. Use the Long-Term Stats system instead.") +import subprocess +import json +import subprocess +from datetime import datetime +from pathlib import Path +import statistics +import time +import psutil -# import subprocess -# import json -# import subprocess -# from datetime import datetime -# from pathlib import Path -# import statistics -# import time -# import psutil +from influxdb_client import InfluxDBClient, Point +from influxdb_client.client.write_api import SYNCHRONOUS -# from influxdb_client import InfluxDBClient, Point -# from influxdb_client.client.write_api import SYNCHRONOUS +from liblqos_python import interface_a, interface_b, influx_db_enabled, influx_db_bucket, influx_db_org, influx_db_token, influx_db_url, sqm -# from ispConfig import interfaceA, interfaceB, influxDBEnabled, influxDBBucket, influxDBOrg, influxDBtoken, influxDBurl, sqm +def getInterfaceStats(interface): + command = 'tc -j -s qdisc show dev ' + interface + jsonAr = json.loads(subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8')) + jsonDict = {} + for element in filter(lambda e: 'parent' in e, jsonAr): + flowID = ':'.join(map(lambda p: f'0x{p}', element['parent'].split(':')[0:2])) + jsonDict[flowID] = element + del jsonAr + return jsonDict -# def getInterfaceStats(interface): -# command = 'tc -j -s qdisc show dev ' + interface -# jsonAr = json.loads(subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8')) -# jsonDict = {} -# for element in filter(lambda e: 'parent' in e, jsonAr): -# flowID = ':'.join(map(lambda p: f'0x{p}', element['parent'].split(':')[0:2])) -# jsonDict[flowID] = element -# del jsonAr -# return jsonDict +def chunk_list(l, n): + for i in range(0, len(l), n): + yield l[i:i + n] - -# def chunk_list(l, n): -# for i in range(0, len(l), n): -# yield l[i:i + n] - -# def getCircuitBandwidthStats(subscriberCircuits, tinsStats): -# interfaces = [interfaceA, interfaceB] -# ifaceStats = list(map(getInterfaceStats, interfaces)) +def getCircuitBandwidthStats(subscriberCircuits, tinsStats): + interfaces = [interface_a(), interface_b()] + ifaceStats = list(map(getInterfaceStats, interfaces)) -# for circuit in subscriberCircuits: -# if 'stats' not in circuit: -# circuit['stats'] = {} -# if 'currentQuery' in circuit['stats']: -# circuit['stats']['priorQuery'] = circuit['stats']['currentQuery'] -# circuit['stats']['currentQuery'] = {} -# circuit['stats']['sinceLastQuery'] = {} -# else: -# #circuit['stats']['priorQuery'] = {} -# #circuit['stats']['priorQuery']['time'] = datetime.now().isoformat() -# circuit['stats']['currentQuery'] = {} -# circuit['stats']['sinceLastQuery'] = {} + for circuit in subscriberCircuits: + if 'stats' not in circuit: + circuit['stats'] = {} + if 'currentQuery' in circuit['stats']: + circuit['stats']['priorQuery'] = circuit['stats']['currentQuery'] + circuit['stats']['currentQuery'] = {} + circuit['stats']['sinceLastQuery'] = {} + else: + #circuit['stats']['priorQuery'] = {} + #circuit['stats']['priorQuery']['time'] = datetime.now().isoformat() + circuit['stats']['currentQuery'] = {} + circuit['stats']['sinceLastQuery'] = {} -# #for entry in tinsStats: -# if 'currentQuery' in tinsStats: -# tinsStats['priorQuery'] = tinsStats['currentQuery'] -# tinsStats['currentQuery'] = {} -# tinsStats['sinceLastQuery'] = {} -# else: -# tinsStats['currentQuery'] = {} -# tinsStats['sinceLastQuery'] = {} + #for entry in tinsStats: + if 'currentQuery' in tinsStats: + tinsStats['priorQuery'] = tinsStats['currentQuery'] + tinsStats['currentQuery'] = {} + tinsStats['sinceLastQuery'] = {} + else: + tinsStats['currentQuery'] = {} + tinsStats['sinceLastQuery'] = {} -# tinsStats['currentQuery'] = { 'Bulk': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, -# 'BestEffort': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, -# 'Video': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, -# 'Voice': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, -# } -# tinsStats['sinceLastQuery'] = { 'Bulk': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, -# 'BestEffort': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, -# 'Video': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, -# 'Voice': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, -# } + tinsStats['currentQuery'] = { 'Bulk': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, + 'BestEffort': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, + 'Video': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, + 'Voice': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, + } + tinsStats['sinceLastQuery'] = { 'Bulk': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, + 'BestEffort': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, + 'Video': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, + 'Voice': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, + } -# for circuit in subscriberCircuits: -# for (interface, stats, dirSuffix) in zip(interfaces, ifaceStats, ['Download', 'Upload']): + for circuit in subscriberCircuits: + for (interface, stats, dirSuffix) in zip(interfaces, ifaceStats, ['Download', 'Upload']): -# element = stats[circuit['classid']] if circuit['classid'] in stats else False + element = stats[circuit['classid']] if circuit['classid'] in stats else False -# if element: -# bytesSent = float(element['bytes']) -# drops = float(element['drops']) -# packets = float(element['packets']) -# if (element['drops'] > 0) and (element['packets'] > 0): -# overloadFactor = float(round(element['drops']/element['packets'],3)) -# else: -# overloadFactor = 0.0 + if element: + bytesSent = float(element['bytes']) + drops = float(element['drops']) + packets = float(element['packets']) + if (element['drops'] > 0) and (element['packets'] > 0): + overloadFactor = float(round(element['drops']/element['packets'],3)) + else: + overloadFactor = 0.0 -# if 'cake diffserv4' in sqm: -# tinCounter = 1 -# for tin in element['tins']: -# sent_packets = float(tin['sent_packets']) -# ack_drops = float(tin['ack_drops']) -# ecn_mark = float(tin['ecn_mark']) -# tinDrops = float(tin['drops']) -# trueDrops = ecn_mark + tinDrops - ack_drops -# if tinCounter == 1: -# tinsStats['currentQuery']['Bulk'][dirSuffix]['sent_packets'] += sent_packets -# tinsStats['currentQuery']['Bulk'][dirSuffix]['drops'] += trueDrops -# elif tinCounter == 2: -# tinsStats['currentQuery']['BestEffort'][dirSuffix]['sent_packets'] += sent_packets -# tinsStats['currentQuery']['BestEffort'][dirSuffix]['drops'] += trueDrops -# elif tinCounter == 3: -# tinsStats['currentQuery']['Video'][dirSuffix]['sent_packets'] += sent_packets -# tinsStats['currentQuery']['Video'][dirSuffix]['drops'] += trueDrops -# elif tinCounter == 4: -# tinsStats['currentQuery']['Voice'][dirSuffix]['sent_packets'] += sent_packets -# tinsStats['currentQuery']['Voice'][dirSuffix]['drops'] += trueDrops -# tinCounter += 1 + if 'cake diffserv4' in sqm(): + tinCounter = 1 + for tin in element['tins']: + sent_packets = float(tin['sent_packets']) + ack_drops = float(tin['ack_drops']) + ecn_mark = float(tin['ecn_mark']) + tinDrops = float(tin['drops']) + trueDrops = ecn_mark + tinDrops - ack_drops + if tinCounter == 1: + tinsStats['currentQuery']['Bulk'][dirSuffix]['sent_packets'] += sent_packets + tinsStats['currentQuery']['Bulk'][dirSuffix]['drops'] += trueDrops + elif tinCounter == 2: + tinsStats['currentQuery']['BestEffort'][dirSuffix]['sent_packets'] += sent_packets + tinsStats['currentQuery']['BestEffort'][dirSuffix]['drops'] += trueDrops + elif tinCounter == 3: + tinsStats['currentQuery']['Video'][dirSuffix]['sent_packets'] += sent_packets + tinsStats['currentQuery']['Video'][dirSuffix]['drops'] += trueDrops + elif tinCounter == 4: + tinsStats['currentQuery']['Voice'][dirSuffix]['sent_packets'] += sent_packets + tinsStats['currentQuery']['Voice'][dirSuffix]['drops'] += trueDrops + tinCounter += 1 -# circuit['stats']['currentQuery']['bytesSent' + dirSuffix] = bytesSent -# circuit['stats']['currentQuery']['packetDrops' + dirSuffix] = drops -# circuit['stats']['currentQuery']['packetsSent' + dirSuffix] = packets -# circuit['stats']['currentQuery']['overloadFactor' + dirSuffix] = overloadFactor + circuit['stats']['currentQuery']['bytesSent' + dirSuffix] = bytesSent + circuit['stats']['currentQuery']['packetDrops' + dirSuffix] = drops + circuit['stats']['currentQuery']['packetsSent' + dirSuffix] = packets + circuit['stats']['currentQuery']['overloadFactor' + dirSuffix] = overloadFactor -# #if 'cake diffserv4' in sqm: -# # circuit['stats']['currentQuery']['tins'] = theseTins + #if 'cake diffserv4' in sqm(): + # circuit['stats']['currentQuery']['tins'] = theseTins -# circuit['stats']['currentQuery']['time'] = datetime.now().isoformat() + circuit['stats']['currentQuery']['time'] = datetime.now().isoformat() -# allPacketsDownload = 0.0 -# allPacketsUpload = 0.0 -# for circuit in subscriberCircuits: -# circuit['stats']['sinceLastQuery']['bitsDownload'] = circuit['stats']['sinceLastQuery']['bitsUpload'] = None -# circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None -# circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None -# circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None + allPacketsDownload = 0.0 + allPacketsUpload = 0.0 + for circuit in subscriberCircuits: + circuit['stats']['sinceLastQuery']['bitsDownload'] = circuit['stats']['sinceLastQuery']['bitsUpload'] = None + circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None + circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None + circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None -# try: -# if (circuit['stats']['currentQuery']['bytesSentDownload'] - circuit['stats']['priorQuery']['bytesSentDownload']) >= 0.0: -# circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['currentQuery']['bytesSentDownload'] - circuit['stats']['priorQuery']['bytesSentDownload'] -# else: -# circuit['stats']['sinceLastQuery']['bytesSentDownload'] = None -# if (circuit['stats']['currentQuery']['bytesSentUpload'] - circuit['stats']['priorQuery']['bytesSentUpload']) >= 0.0: -# circuit['stats']['sinceLastQuery']['bytesSentUpload'] = circuit['stats']['currentQuery']['bytesSentUpload'] - circuit['stats']['priorQuery']['bytesSentUpload'] -# else: -# circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None -# except: -# circuit['stats']['sinceLastQuery']['bytesSentDownload'] = None -# circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None -# try: -# if (circuit['stats']['currentQuery']['packetDropsDownload'] - circuit['stats']['priorQuery']['packetDropsDownload']) >= 0.0: -# circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['currentQuery']['packetDropsDownload'] - circuit['stats']['priorQuery']['packetDropsDownload'] -# else: -# circuit['stats']['sinceLastQuery']['packetDropsDownload'] = None -# if (circuit['stats']['currentQuery']['packetDropsUpload'] - circuit['stats']['priorQuery']['packetDropsUpload']) >= 0.0: -# circuit['stats']['sinceLastQuery']['packetDropsUpload'] = circuit['stats']['currentQuery']['packetDropsUpload'] - circuit['stats']['priorQuery']['packetDropsUpload'] -# else: -# circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None -# except: -# circuit['stats']['sinceLastQuery']['packetDropsDownload'] = None -# circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None -# try: -# if (circuit['stats']['currentQuery']['packetsSentDownload'] - circuit['stats']['priorQuery']['packetsSentDownload']) >= 0.0: -# circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['currentQuery']['packetsSentDownload'] - circuit['stats']['priorQuery']['packetsSentDownload'] -# else: -# circuit['stats']['sinceLastQuery']['packetsSentDownload'] = None -# if (circuit['stats']['currentQuery']['packetsSentUpload'] - circuit['stats']['priorQuery']['packetsSentUpload']) >= 0.0: -# circuit['stats']['sinceLastQuery']['packetsSentUpload'] = circuit['stats']['currentQuery']['packetsSentUpload'] - circuit['stats']['priorQuery']['packetsSentUpload'] -# else: -# circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None -# except: -# circuit['stats']['sinceLastQuery']['packetsSentDownload'] = None -# circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None + try: + if (circuit['stats']['currentQuery']['bytesSentDownload'] - circuit['stats']['priorQuery']['bytesSentDownload']) >= 0.0: + circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['currentQuery']['bytesSentDownload'] - circuit['stats']['priorQuery']['bytesSentDownload'] + else: + circuit['stats']['sinceLastQuery']['bytesSentDownload'] = None + if (circuit['stats']['currentQuery']['bytesSentUpload'] - circuit['stats']['priorQuery']['bytesSentUpload']) >= 0.0: + circuit['stats']['sinceLastQuery']['bytesSentUpload'] = circuit['stats']['currentQuery']['bytesSentUpload'] - circuit['stats']['priorQuery']['bytesSentUpload'] + else: + circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None + except: + circuit['stats']['sinceLastQuery']['bytesSentDownload'] = None + circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None + try: + if (circuit['stats']['currentQuery']['packetDropsDownload'] - circuit['stats']['priorQuery']['packetDropsDownload']) >= 0.0: + circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['currentQuery']['packetDropsDownload'] - circuit['stats']['priorQuery']['packetDropsDownload'] + else: + circuit['stats']['sinceLastQuery']['packetDropsDownload'] = None + if (circuit['stats']['currentQuery']['packetDropsUpload'] - circuit['stats']['priorQuery']['packetDropsUpload']) >= 0.0: + circuit['stats']['sinceLastQuery']['packetDropsUpload'] = circuit['stats']['currentQuery']['packetDropsUpload'] - circuit['stats']['priorQuery']['packetDropsUpload'] + else: + circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None + except: + circuit['stats']['sinceLastQuery']['packetDropsDownload'] = None + circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None + try: + if (circuit['stats']['currentQuery']['packetsSentDownload'] - circuit['stats']['priorQuery']['packetsSentDownload']) >= 0.0: + circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['currentQuery']['packetsSentDownload'] - circuit['stats']['priorQuery']['packetsSentDownload'] + else: + circuit['stats']['sinceLastQuery']['packetsSentDownload'] = None + if (circuit['stats']['currentQuery']['packetsSentUpload'] - circuit['stats']['priorQuery']['packetsSentUpload']) >= 0.0: + circuit['stats']['sinceLastQuery']['packetsSentUpload'] = circuit['stats']['currentQuery']['packetsSentUpload'] - circuit['stats']['priorQuery']['packetsSentUpload'] + else: + circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None + except: + circuit['stats']['sinceLastQuery']['packetsSentDownload'] = None + circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None -# if(circuit['stats']['sinceLastQuery']['packetsSentDownload']): -# allPacketsDownload += circuit['stats']['sinceLastQuery']['packetsSentDownload'] -# if(circuit['stats']['sinceLastQuery']['packetsSentUpload']): -# allPacketsUpload += circuit['stats']['sinceLastQuery']['packetsSentUpload'] + if(circuit['stats']['sinceLastQuery']['packetsSentDownload']): + allPacketsDownload += circuit['stats']['sinceLastQuery']['packetsSentDownload'] + if(circuit['stats']['sinceLastQuery']['packetsSentUpload']): + allPacketsUpload += circuit['stats']['sinceLastQuery']['packetsSentUpload'] -# if 'priorQuery' in circuit['stats']: -# if 'time' in circuit['stats']['priorQuery']: -# currentQueryTime = datetime.fromisoformat(circuit['stats']['currentQuery']['time']) -# priorQueryTime = datetime.fromisoformat(circuit['stats']['priorQuery']['time']) -# deltaSeconds = (currentQueryTime - priorQueryTime).total_seconds() -# if (circuit['stats']['sinceLastQuery']['bytesSentDownload']): -# circuit['stats']['sinceLastQuery']['bitsDownload'] = round((circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 -# else: -# circuit['stats']['sinceLastQuery']['bitsDownload'] = None -# if (circuit['stats']['sinceLastQuery']['bytesSentUpload']): -# circuit['stats']['sinceLastQuery']['bitsUpload'] = round((circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 -# else: -# circuit['stats']['sinceLastQuery']['bitsUpload'] = None + if 'priorQuery' in circuit['stats']: + if 'time' in circuit['stats']['priorQuery']: + currentQueryTime = datetime.fromisoformat(circuit['stats']['currentQuery']['time']) + priorQueryTime = datetime.fromisoformat(circuit['stats']['priorQuery']['time']) + deltaSeconds = (currentQueryTime - priorQueryTime).total_seconds() + if (circuit['stats']['sinceLastQuery']['bytesSentDownload']): + circuit['stats']['sinceLastQuery']['bitsDownload'] = round((circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 + else: + circuit['stats']['sinceLastQuery']['bitsDownload'] = None + if (circuit['stats']['sinceLastQuery']['bytesSentUpload']): + circuit['stats']['sinceLastQuery']['bitsUpload'] = round((circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 + else: + circuit['stats']['sinceLastQuery']['bitsUpload'] = None -# else: -# circuit['stats']['sinceLastQuery']['bitsDownload'] = None -# if(circuit['stats']['sinceLastQuery']['bytesSentDownload']): -# circuit['stats']['sinceLastQuery']['bitsDownload'] = (circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) -# circuit['stats']['sinceLastQuery']['bitsUpload'] = None -# if(circuit['stats']['sinceLastQuery']['bytesSentUpload']): -# circuit['stats']['sinceLastQuery']['bitsUpload'] = (circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) + else: + circuit['stats']['sinceLastQuery']['bitsDownload'] = None + if(circuit['stats']['sinceLastQuery']['bytesSentDownload']): + circuit['stats']['sinceLastQuery']['bitsDownload'] = (circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) + circuit['stats']['sinceLastQuery']['bitsUpload'] = None + if(circuit['stats']['sinceLastQuery']['bytesSentUpload']): + circuit['stats']['sinceLastQuery']['bitsUpload'] = (circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) -# tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = 0.0 -# tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = 0.0 -# tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = 0.0 -# tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = 0.0 + tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = 0.0 + tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = 0.0 + tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = 0.0 -# try: -# tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] = tinsStats['currentQuery']['Bulk']['Download']['sent_packets'] - tinsStats['priorQuery']['Bulk']['Download']['sent_packets'] -# tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] = tinsStats['currentQuery']['BestEffort']['Download']['sent_packets'] - tinsStats['priorQuery']['BestEffort']['Download']['sent_packets'] -# tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] = tinsStats['currentQuery']['Video']['Download']['sent_packets'] - tinsStats['priorQuery']['Video']['Download']['sent_packets'] -# tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] = tinsStats['currentQuery']['Voice']['Download']['sent_packets'] - tinsStats['priorQuery']['Voice']['Download']['sent_packets'] -# tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] = tinsStats['currentQuery']['Bulk']['Upload']['sent_packets'] - tinsStats['priorQuery']['Bulk']['Upload']['sent_packets'] -# tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] = tinsStats['currentQuery']['BestEffort']['Upload']['sent_packets'] - tinsStats['priorQuery']['BestEffort']['Upload']['sent_packets'] -# tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] = tinsStats['currentQuery']['Video']['Upload']['sent_packets'] - tinsStats['priorQuery']['Video']['Upload']['sent_packets'] -# tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] = tinsStats['currentQuery']['Voice']['Upload']['sent_packets'] - tinsStats['priorQuery']['Voice']['Upload']['sent_packets'] -# except: -# tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] = tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] = 0.0 -# tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] = tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] = 0.0 -# tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] = 0.0 -# tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] = tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] = 0.0 + try: + tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] = tinsStats['currentQuery']['Bulk']['Download']['sent_packets'] - tinsStats['priorQuery']['Bulk']['Download']['sent_packets'] + tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] = tinsStats['currentQuery']['BestEffort']['Download']['sent_packets'] - tinsStats['priorQuery']['BestEffort']['Download']['sent_packets'] + tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] = tinsStats['currentQuery']['Video']['Download']['sent_packets'] - tinsStats['priorQuery']['Video']['Download']['sent_packets'] + tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] = tinsStats['currentQuery']['Voice']['Download']['sent_packets'] - tinsStats['priorQuery']['Voice']['Download']['sent_packets'] + tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] = tinsStats['currentQuery']['Bulk']['Upload']['sent_packets'] - tinsStats['priorQuery']['Bulk']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] = tinsStats['currentQuery']['BestEffort']['Upload']['sent_packets'] - tinsStats['priorQuery']['BestEffort']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] = tinsStats['currentQuery']['Video']['Upload']['sent_packets'] - tinsStats['priorQuery']['Video']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] = tinsStats['currentQuery']['Voice']['Upload']['sent_packets'] - tinsStats['priorQuery']['Voice']['Upload']['sent_packets'] + except: + tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] = tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] = tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] = 0.0 + tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] = tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] = 0.0 -# try: -# tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] = tinsStats['currentQuery']['Bulk']['Download']['drops'] - tinsStats['priorQuery']['Bulk']['Download']['drops'] -# tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] = tinsStats['currentQuery']['BestEffort']['Download']['drops'] - tinsStats['priorQuery']['BestEffort']['Download']['drops'] -# tinsStats['sinceLastQuery']['Video']['Download']['drops'] = tinsStats['currentQuery']['Video']['Download']['drops'] - tinsStats['priorQuery']['Video']['Download']['drops'] -# tinsStats['sinceLastQuery']['Voice']['Download']['drops'] = tinsStats['currentQuery']['Voice']['Download']['drops'] - tinsStats['priorQuery']['Voice']['Download']['drops'] -# tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] = tinsStats['currentQuery']['Bulk']['Upload']['drops'] - tinsStats['priorQuery']['Bulk']['Upload']['drops'] -# tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] = tinsStats['currentQuery']['BestEffort']['Upload']['drops'] - tinsStats['priorQuery']['BestEffort']['Upload']['drops'] -# tinsStats['sinceLastQuery']['Video']['Upload']['drops'] = tinsStats['currentQuery']['Video']['Upload']['drops'] - tinsStats['priorQuery']['Video']['Upload']['drops'] -# tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] = tinsStats['currentQuery']['Voice']['Upload']['drops'] - tinsStats['priorQuery']['Voice']['Upload']['drops'] -# except: -# tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] = tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] = 0.0 -# tinsStats['sinceLastQuery']['Video']['Download']['drops'] = tinsStats['sinceLastQuery']['Voice']['Download']['drops'] = 0.0 -# tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] = 0.0 -# tinsStats['sinceLastQuery']['Video']['Upload']['drops'] = tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] = 0.0 + try: + tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] = tinsStats['currentQuery']['Bulk']['Download']['drops'] - tinsStats['priorQuery']['Bulk']['Download']['drops'] + tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] = tinsStats['currentQuery']['BestEffort']['Download']['drops'] - tinsStats['priorQuery']['BestEffort']['Download']['drops'] + tinsStats['sinceLastQuery']['Video']['Download']['drops'] = tinsStats['currentQuery']['Video']['Download']['drops'] - tinsStats['priorQuery']['Video']['Download']['drops'] + tinsStats['sinceLastQuery']['Voice']['Download']['drops'] = tinsStats['currentQuery']['Voice']['Download']['drops'] - tinsStats['priorQuery']['Voice']['Download']['drops'] + tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] = tinsStats['currentQuery']['Bulk']['Upload']['drops'] - tinsStats['priorQuery']['Bulk']['Upload']['drops'] + tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] = tinsStats['currentQuery']['BestEffort']['Upload']['drops'] - tinsStats['priorQuery']['BestEffort']['Upload']['drops'] + tinsStats['sinceLastQuery']['Video']['Upload']['drops'] = tinsStats['currentQuery']['Video']['Upload']['drops'] - tinsStats['priorQuery']['Video']['Upload']['drops'] + tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] = tinsStats['currentQuery']['Voice']['Upload']['drops'] - tinsStats['priorQuery']['Voice']['Upload']['drops'] + except: + tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] = tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Download']['drops'] = tinsStats['sinceLastQuery']['Voice']['Download']['drops'] = 0.0 + tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Upload']['drops'] = tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] = 0.0 -# try: -# dlPerc = tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] / tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] -# ulPerc = tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] / tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] -# tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) -# tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) + try: + dlPerc = tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] / tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] + ulPerc = tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] / tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) + tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) -# dlPerc = tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] / tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] -# ulPerc = tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] / tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] -# tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) -# tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) + dlPerc = tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] / tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] + ulPerc = tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] / tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) + tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) -# dlPerc = tinsStats['sinceLastQuery']['Video']['Download']['drops'] / tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] -# ulPerc = tinsStats['sinceLastQuery']['Video']['Upload']['drops'] / tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] -# tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) -# tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) + dlPerc = tinsStats['sinceLastQuery']['Video']['Download']['drops'] / tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] + ulPerc = tinsStats['sinceLastQuery']['Video']['Upload']['drops'] / tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) + tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) -# dlPerc = tinsStats['sinceLastQuery']['Voice']['Download']['drops'] / tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] -# ulPerc = tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] / tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] -# tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) -# tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) -# except: -# tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = 0.0 -# tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = 0.0 + dlPerc = tinsStats['sinceLastQuery']['Voice']['Download']['drops'] / tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] + ulPerc = tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] / tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) + tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) + except: + tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = 0.0 -# try: -# tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) -# tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) -# tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) -# tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) -# tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Video']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) -# tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) -# tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) -# tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) -# except: -# # To avoid graphing 0.0 for all categories, which would show unusual graph results upon each queue reload, we just set these to None if the above calculations fail. -# tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = None -# tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = None -# tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = None -# tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = None + try: + tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Video']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) + except: + # To avoid graphing 0.0 for all categories, which would show unusual graph results upon each queue reload, we just set these to None if the above calculations fail. + tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = None + tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = None + tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = None + tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = None -# return subscriberCircuits, tinsStats + return subscriberCircuits, tinsStats -# def getParentNodeBandwidthStats(parentNodes, subscriberCircuits): -# for parentNode in parentNodes: -# thisNodeDropsDownload = 0 -# thisNodeDropsUpload = 0 -# thisNodeDropsTotal = 0 -# thisNodeBitsDownload = 0 -# thisNodeBitsUpload = 0 -# packetsSentDownloadAggregate = 0.0 -# packetsSentUploadAggregate = 0.0 -# packetsSentTotalAggregate = 0.0 -# circuitsMatched = 0 -# thisParentNodeStats = {'sinceLastQuery': {}} -# for circuit in subscriberCircuits: -# if circuit['ParentNode'] == parentNode['parentNodeName']: -# if circuit['stats']['sinceLastQuery']['bitsDownload']: -# thisNodeBitsDownload += circuit['stats']['sinceLastQuery']['bitsDownload'] -# if circuit['stats']['sinceLastQuery']['bitsUpload']: -# thisNodeBitsUpload += circuit['stats']['sinceLastQuery']['bitsUpload'] -# #thisNodeDropsDownload += circuit['packetDropsDownloadSinceLastQuery'] -# #thisNodeDropsUpload += circuit['packetDropsUploadSinceLastQuery'] -# if circuit['stats']['sinceLastQuery']['packetDropsDownload'] and circuit['stats']['sinceLastQuery']['packetDropsUpload']: -# thisNodeDropsTotal += (circuit['stats']['sinceLastQuery']['packetDropsDownload'] + circuit['stats']['sinceLastQuery']['packetDropsUpload']) -# if circuit['stats']['sinceLastQuery']['packetsSentDownload']: -# packetsSentDownloadAggregate += circuit['stats']['sinceLastQuery']['packetsSentDownload'] -# if circuit['stats']['sinceLastQuery']['packetsSentUpload']: -# packetsSentUploadAggregate += circuit['stats']['sinceLastQuery']['packetsSentUpload'] -# if circuit['stats']['sinceLastQuery']['packetsSentDownload'] and circuit['stats']['sinceLastQuery']['packetsSentUpload']: -# packetsSentTotalAggregate += (circuit['stats']['sinceLastQuery']['packetsSentDownload'] + circuit['stats']['sinceLastQuery']['packetsSentUpload']) -# circuitsMatched += 1 -# if (packetsSentDownloadAggregate > 0) and (packetsSentUploadAggregate > 0) and (packetsSentTotalAggregate > 0): -# #overloadFactorDownloadSinceLastQuery = float(round((thisNodeDropsDownload/packetsSentDownloadAggregate)*100.0, 3)) -# #overloadFactorUploadSinceLastQuery = float(round((thisNodeDropsUpload/packetsSentUploadAggregate)*100.0, 3)) -# overloadFactorTotalSinceLastQuery = float(round((thisNodeDropsTotal/packetsSentTotalAggregate)*100.0, 1)) -# else: -# #overloadFactorDownloadSinceLastQuery = 0.0 -# #overloadFactorUploadSinceLastQuery = 0.0 -# overloadFactorTotalSinceLastQuery = 0.0 +def getParentNodeBandwidthStats(parentNodes, subscriberCircuits): + for parentNode in parentNodes: + thisNodeDropsDownload = 0 + thisNodeDropsUpload = 0 + thisNodeDropsTotal = 0 + thisNodeBitsDownload = 0 + thisNodeBitsUpload = 0 + packetsSentDownloadAggregate = 0.0 + packetsSentUploadAggregate = 0.0 + packetsSentTotalAggregate = 0.0 + circuitsMatched = 0 + thisParentNodeStats = {'sinceLastQuery': {}} + for circuit in subscriberCircuits: + if circuit['ParentNode'] == parentNode['parentNodeName']: + if circuit['stats']['sinceLastQuery']['bitsDownload']: + thisNodeBitsDownload += circuit['stats']['sinceLastQuery']['bitsDownload'] + if circuit['stats']['sinceLastQuery']['bitsUpload']: + thisNodeBitsUpload += circuit['stats']['sinceLastQuery']['bitsUpload'] + #thisNodeDropsDownload += circuit['packetDropsDownloadSinceLastQuery'] + #thisNodeDropsUpload += circuit['packetDropsUploadSinceLastQuery'] + if circuit['stats']['sinceLastQuery']['packetDropsDownload'] and circuit['stats']['sinceLastQuery']['packetDropsUpload']: + thisNodeDropsTotal += (circuit['stats']['sinceLastQuery']['packetDropsDownload'] + circuit['stats']['sinceLastQuery']['packetDropsUpload']) + if circuit['stats']['sinceLastQuery']['packetsSentDownload']: + packetsSentDownloadAggregate += circuit['stats']['sinceLastQuery']['packetsSentDownload'] + if circuit['stats']['sinceLastQuery']['packetsSentUpload']: + packetsSentUploadAggregate += circuit['stats']['sinceLastQuery']['packetsSentUpload'] + if circuit['stats']['sinceLastQuery']['packetsSentDownload'] and circuit['stats']['sinceLastQuery']['packetsSentUpload']: + packetsSentTotalAggregate += (circuit['stats']['sinceLastQuery']['packetsSentDownload'] + circuit['stats']['sinceLastQuery']['packetsSentUpload']) + circuitsMatched += 1 + if (packetsSentDownloadAggregate > 0) and (packetsSentUploadAggregate > 0) and (packetsSentTotalAggregate > 0): + #overloadFactorDownloadSinceLastQuery = float(round((thisNodeDropsDownload/packetsSentDownloadAggregate)*100.0, 3)) + #overloadFactorUploadSinceLastQuery = float(round((thisNodeDropsUpload/packetsSentUploadAggregate)*100.0, 3)) + overloadFactorTotalSinceLastQuery = float(round((thisNodeDropsTotal/packetsSentTotalAggregate)*100.0, 1)) + else: + #overloadFactorDownloadSinceLastQuery = 0.0 + #overloadFactorUploadSinceLastQuery = 0.0 + overloadFactorTotalSinceLastQuery = 0.0 -# thisParentNodeStats['sinceLastQuery']['bitsDownload'] = thisNodeBitsDownload -# thisParentNodeStats['sinceLastQuery']['bitsUpload'] = thisNodeBitsUpload -# thisParentNodeStats['sinceLastQuery']['packetDropsTotal'] = thisNodeDropsTotal -# thisParentNodeStats['sinceLastQuery']['overloadFactorTotal'] = overloadFactorTotalSinceLastQuery -# parentNode['stats'] = thisParentNodeStats + thisParentNodeStats['sinceLastQuery']['bitsDownload'] = thisNodeBitsDownload + thisParentNodeStats['sinceLastQuery']['bitsUpload'] = thisNodeBitsUpload + thisParentNodeStats['sinceLastQuery']['packetDropsTotal'] = thisNodeDropsTotal + thisParentNodeStats['sinceLastQuery']['overloadFactorTotal'] = overloadFactorTotalSinceLastQuery + parentNode['stats'] = thisParentNodeStats -# return parentNodes + return parentNodes -# def getParentNodeLatencyStats(parentNodes, subscriberCircuits): -# for parentNode in parentNodes: -# if 'stats' not in parentNode: -# parentNode['stats'] = {} -# parentNode['stats']['sinceLastQuery'] = {} +def getParentNodeLatencyStats(parentNodes, subscriberCircuits): + for parentNode in parentNodes: + if 'stats' not in parentNode: + parentNode['stats'] = {} + parentNode['stats']['sinceLastQuery'] = {} -# for parentNode in parentNodes: -# thisParentNodeStats = {'sinceLastQuery': {}} -# circuitsMatchedLatencies = [] -# for circuit in subscriberCircuits: -# if circuit['ParentNode'] == parentNode['parentNodeName']: -# if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: -# circuitsMatchedLatencies.append(circuit['stats']['sinceLastQuery']['tcpLatency']) -# if len(circuitsMatchedLatencies) > 0: -# thisParentNodeStats['sinceLastQuery']['tcpLatency'] = statistics.median(circuitsMatchedLatencies) -# else: -# thisParentNodeStats['sinceLastQuery']['tcpLatency'] = None -# parentNode['stats'] = thisParentNodeStats -# return parentNodes + for parentNode in parentNodes: + thisParentNodeStats = {'sinceLastQuery': {}} + circuitsMatchedLatencies = [] + for circuit in subscriberCircuits: + if circuit['ParentNode'] == parentNode['parentNodeName']: + if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: + circuitsMatchedLatencies.append(circuit['stats']['sinceLastQuery']['tcpLatency']) + if len(circuitsMatchedLatencies) > 0: + thisParentNodeStats['sinceLastQuery']['tcpLatency'] = statistics.median(circuitsMatchedLatencies) + else: + thisParentNodeStats['sinceLastQuery']['tcpLatency'] = None + parentNode['stats'] = thisParentNodeStats + return parentNodes -# def getCircuitLatencyStats(subscriberCircuits): -# command = './bin/xdp_pping' -# consoleOutput = subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8') -# consoleOutput = consoleOutput.replace('\n','').replace('}{', '}, {') -# listOfEntries = json.loads(consoleOutput) +def getCircuitLatencyStats(subscriberCircuits): + command = './bin/xdp_pping' + consoleOutput = subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8') + consoleOutput = consoleOutput.replace('\n','').replace('}{', '}, {') + listOfEntries = json.loads(consoleOutput) -# tcpLatencyForClassID = {} -# for entry in listOfEntries: -# if 'tc' in entry: -# handle = '0x' + entry['tc'].split(':')[0] + ':' + '0x' + entry['tc'].split(':')[1] -# # To avoid outliers messing up avg for each circuit - cap at ceiling of 200ms -# ceiling = 200.0 -# tcpLatencyForClassID[handle] = min(entry['median'], ceiling) -# for circuit in subscriberCircuits: -# if 'stats' not in circuit: -# circuit['stats'] = {} -# circuit['stats']['sinceLastQuery'] = {} + tcpLatencyForClassID = {} + for entry in listOfEntries: + if 'tc' in entry: + handle = '0x' + entry['tc'].split(':')[0] + ':' + '0x' + entry['tc'].split(':')[1] + # To avoid outliers messing up avg for each circuit - cap at ceiling of 200ms + ceiling = 200.0 + tcpLatencyForClassID[handle] = min(entry['median'], ceiling) + for circuit in subscriberCircuits: + if 'stats' not in circuit: + circuit['stats'] = {} + circuit['stats']['sinceLastQuery'] = {} -# for circuit in subscriberCircuits: -# classID = circuit['classid'] -# if classID in tcpLatencyForClassID: -# circuit['stats']['sinceLastQuery']['tcpLatency'] = tcpLatencyForClassID[classID] -# else: -# # If we can't identify RTT this time around, use most recently recorded RTT -# # None by default, change if found in priorQuery -# circuit['stats']['sinceLastQuery']['tcpLatency'] = None -# if 'priorQuery' in circuit['stats']: -# if circuit['stats']['priorQuery'] != None: -# if 'priorQuery' in circuit['stats']: -# if 'tcpLatency' in circuit['stats']['priorQuery']: -# circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency'] + for circuit in subscriberCircuits: + classID = circuit['classid'] + if classID in tcpLatencyForClassID: + circuit['stats']['sinceLastQuery']['tcpLatency'] = tcpLatencyForClassID[classID] + else: + # If we can't identify RTT this time around, use most recently recorded RTT + # None by default, change if found in priorQuery + circuit['stats']['sinceLastQuery']['tcpLatency'] = None + if 'priorQuery' in circuit['stats']: + if circuit['stats']['priorQuery'] != None: + if 'priorQuery' in circuit['stats']: + if 'tcpLatency' in circuit['stats']['priorQuery']: + circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency'] -# return subscriberCircuits + return subscriberCircuits -# def getParentNodeDict(data, depth, parentNodeNameDict): -# if parentNodeNameDict == None: -# parentNodeNameDict = {} +def getParentNodeDict(data, depth, parentNodeNameDict): + if parentNodeNameDict == None: + parentNodeNameDict = {} -# for elem in data: -# if 'children' in data[elem]: -# for child in data[elem]['children']: -# parentNodeNameDict[child] = elem -# tempDict = getParentNodeDict(data[elem]['children'], depth + 1, parentNodeNameDict) -# parentNodeNameDict = dict(parentNodeNameDict, **tempDict) -# return parentNodeNameDict + for elem in data: + if 'children' in data[elem]: + for child in data[elem]['children']: + parentNodeNameDict[child] = elem + tempDict = getParentNodeDict(data[elem]['children'], depth + 1, parentNodeNameDict) + parentNodeNameDict = dict(parentNodeNameDict, **tempDict) + return parentNodeNameDict -# def parentNodeNameDictPull(): -# # Load network hierarchy -# with open('network.json', 'r') as j: -# network = json.loads(j.read()) -# parentNodeNameDict = getParentNodeDict(network, 0, None) -# return parentNodeNameDict +def parentNodeNameDictPull(): + # Load network hierarchy + with open('network.json', 'r') as j: + network = json.loads(j.read()) + parentNodeNameDict = getParentNodeDict(network, 0, None) + return parentNodeNameDict -# def refreshBandwidthGraphs(): -# startTime = datetime.now() -# with open('statsByParentNode.json', 'r') as j: -# parentNodes = json.loads(j.read()) +def refreshBandwidthGraphs(): + startTime = datetime.now() + with open('statsByParentNode.json', 'r') as j: + parentNodes = json.loads(j.read()) -# with open('statsByCircuit.json', 'r') as j: -# subscriberCircuits = json.loads(j.read()) + with open('statsByCircuit.json', 'r') as j: + subscriberCircuits = json.loads(j.read()) -# fileLoc = Path("tinsStats.json") -# if fileLoc.is_file(): -# with open(fileLoc, 'r') as j: -# tinsStats = json.loads(j.read()) -# else: -# tinsStats = {} + fileLoc = Path("tinsStats.json") + if fileLoc.is_file(): + with open(fileLoc, 'r') as j: + tinsStats = json.loads(j.read()) + else: + tinsStats = {} -# fileLoc = Path("longTermStats.json") -# if fileLoc.is_file(): -# with open(fileLoc, 'r') as j: -# longTermStats = json.loads(j.read()) -# droppedPacketsAllTime = longTermStats['droppedPacketsTotal'] -# else: -# longTermStats = {} -# longTermStats['droppedPacketsTotal'] = 0.0 -# droppedPacketsAllTime = 0.0 + fileLoc = Path("longTermStats.json") + if fileLoc.is_file(): + with open(fileLoc, 'r') as j: + longTermStats = json.loads(j.read()) + droppedPacketsAllTime = longTermStats['droppedPacketsTotal'] + else: + longTermStats = {} + longTermStats['droppedPacketsTotal'] = 0.0 + droppedPacketsAllTime = 0.0 -# parentNodeNameDict = parentNodeNameDictPull() + parentNodeNameDict = parentNodeNameDictPull() -# print("Retrieving circuit statistics") -# subscriberCircuits, tinsStats = getCircuitBandwidthStats(subscriberCircuits, tinsStats) -# print("Computing parent node statistics") -# parentNodes = getParentNodeBandwidthStats(parentNodes, subscriberCircuits) -# print("Writing data to InfluxDB") -# client = InfluxDBClient( -# url=influxDBurl, -# token=influxDBtoken, -# org=influxDBOrg -# ) + print("Retrieving circuit statistics") + subscriberCircuits, tinsStats = getCircuitBandwidthStats(subscriberCircuits, tinsStats) + print("Computing parent node statistics") + parentNodes = getParentNodeBandwidthStats(parentNodes, subscriberCircuits) + print("Writing data to InfluxDB") + client = InfluxDBClient( + url=influx_db_url(), + token=influx_db_token(), + org=influx_db_org() + ) -# # Record current timestamp, use for all points added -# timestamp = time.time_ns() -# write_api = client.write_api(write_options=SYNCHRONOUS) + # Record current timestamp, use for all points added + timestamp = time.time_ns() + write_api = client.write_api(write_options=SYNCHRONOUS) -# chunkedsubscriberCircuits = list(chunk_list(subscriberCircuits, 200)) + chunkedsubscriberCircuits = list(chunk_list(subscriberCircuits, 200)) -# queriesToSendCount = 0 -# for chunk in chunkedsubscriberCircuits: -# seenSomethingBesides0s = False -# queriesToSend = [] -# for circuit in chunk: -# bitsDownloadMin = float(circuit['minDownload']) * 1000000 if circuit['minDownload'] else None -# bitsDownloadMax = float(circuit['maxDownload']) * 1000000 if circuit['maxDownload'] else None -# bitsUploadMin = float(circuit['minUpload']) * 1000000 if circuit['minUpload'] else None -# bitsUploadMax = float(circuit['maxUpload']) * 1000000 if circuit['maxUpload'] else None -# bitsDownload = float(circuit['stats']['sinceLastQuery']['bitsDownload']) if circuit['stats']['sinceLastQuery']['bitsDownload'] else None -# bitsUpload = float(circuit['stats']['sinceLastQuery']['bitsUpload']) if circuit['stats']['sinceLastQuery']['bitsUpload'] else None -# bytesSentDownload = float(circuit['stats']['sinceLastQuery']['bytesSentDownload']) if circuit['stats']['sinceLastQuery']['bytesSentDownload'] else None -# bytesSentUpload = float(circuit['stats']['sinceLastQuery']['bytesSentUpload']) if circuit['stats']['sinceLastQuery']['bytesSentUpload'] else None -# percentUtilizationDownload = round((bitsDownload / round(circuit['maxDownload'] * 1000000))*100.0, 1) if bitsDownload and circuit['maxDownload'] else None -# percentUtilizationUpload = round((bitsUpload / round(circuit['maxUpload'] * 1000000))*100.0, 1) if bitsUpload and circuit['maxUpload'] else None -# if bitsDownload and bitsUpload: -# if (bitsDownload > 0.0) or (bitsUpload > 0.0): -# seenSomethingBesides0s = True -# p = Point('Bandwidth').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", bitsDownload).field("Upload", bitsUpload).time(timestamp) -# queriesToSend.append(p) -# p = Point('Utilization').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) -# queriesToSend.append(p) + queriesToSendCount = 0 + for chunk in chunkedsubscriberCircuits: + seenSomethingBesides0s = False + queriesToSend = [] + for circuit in chunk: + bitsDownloadMin = float(circuit['minDownload']) * 1000000 if circuit['minDownload'] else None + bitsDownloadMax = float(circuit['maxDownload']) * 1000000 if circuit['maxDownload'] else None + bitsUploadMin = float(circuit['minUpload']) * 1000000 if circuit['minUpload'] else None + bitsUploadMax = float(circuit['maxUpload']) * 1000000 if circuit['maxUpload'] else None + bitsDownload = float(circuit['stats']['sinceLastQuery']['bitsDownload']) if circuit['stats']['sinceLastQuery']['bitsDownload'] else None + bitsUpload = float(circuit['stats']['sinceLastQuery']['bitsUpload']) if circuit['stats']['sinceLastQuery']['bitsUpload'] else None + bytesSentDownload = float(circuit['stats']['sinceLastQuery']['bytesSentDownload']) if circuit['stats']['sinceLastQuery']['bytesSentDownload'] else None + bytesSentUpload = float(circuit['stats']['sinceLastQuery']['bytesSentUpload']) if circuit['stats']['sinceLastQuery']['bytesSentUpload'] else None + percentUtilizationDownload = round((bitsDownload / round(circuit['maxDownload'] * 1000000))*100.0, 1) if bitsDownload and circuit['maxDownload'] else None + percentUtilizationUpload = round((bitsUpload / round(circuit['maxUpload'] * 1000000))*100.0, 1) if bitsUpload and circuit['maxUpload'] else None + if bitsDownload and bitsUpload: + if (bitsDownload > 0.0) or (bitsUpload > 0.0): + seenSomethingBesides0s = True + p = Point('Bandwidth').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", bitsDownload).field("Upload", bitsUpload).time(timestamp) + queriesToSend.append(p) + p = Point('Utilization').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) + queriesToSend.append(p) -# if seenSomethingBesides0s: -# write_api.write(bucket=influxDBBucket, record=queriesToSend) -# # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") -# queriesToSendCount += len(queriesToSend) + if seenSomethingBesides0s: + write_api.write(bucket=influx_db_bucket(), record=queriesToSend) + # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") + queriesToSendCount += len(queriesToSend) -# queriesToSend = [] -# seenSomethingBesides0s = False -# for parentNode in parentNodes: -# bitsDownload = float(parentNode['stats']['sinceLastQuery']['bitsDownload']) -# bitsUpload = float(parentNode['stats']['sinceLastQuery']['bitsUpload']) -# dropsTotal = float(parentNode['stats']['sinceLastQuery']['packetDropsTotal']) -# overloadFactor = float(parentNode['stats']['sinceLastQuery']['overloadFactorTotal']) -# droppedPacketsAllTime += dropsTotal -# percentUtilizationDownload = round((bitsDownload / round(parentNode['maxDownload'] * 1000000))*100.0, 1) -# percentUtilizationUpload = round((bitsUpload / round(parentNode['maxUpload'] * 1000000))*100.0, 1) -# if bitsDownload and bitsUpload: -# if (bitsDownload > 0.0) or (bitsUpload > 0.0): -# seenSomethingBesides0s = True -# p = Point('Bandwidth').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", bitsDownload).field("Upload", bitsUpload).time(timestamp) -# queriesToSend.append(p) -# p = Point('Utilization').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) -# queriesToSend.append(p) -# p = Point('Overload').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Overload", overloadFactor).time(timestamp) -# queriesToSend.append(p) + queriesToSend = [] + seenSomethingBesides0s = False + for parentNode in parentNodes: + bitsDownload = float(parentNode['stats']['sinceLastQuery']['bitsDownload']) + bitsUpload = float(parentNode['stats']['sinceLastQuery']['bitsUpload']) + dropsTotal = float(parentNode['stats']['sinceLastQuery']['packetDropsTotal']) + overloadFactor = float(parentNode['stats']['sinceLastQuery']['overloadFactorTotal']) + droppedPacketsAllTime += dropsTotal + percentUtilizationDownload = round((bitsDownload / round(parentNode['maxDownload'] * 1000000))*100.0, 1) + percentUtilizationUpload = round((bitsUpload / round(parentNode['maxUpload'] * 1000000))*100.0, 1) + if bitsDownload and bitsUpload: + if (bitsDownload > 0.0) or (bitsUpload > 0.0): + seenSomethingBesides0s = True + p = Point('Bandwidth').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", bitsDownload).field("Upload", bitsUpload).time(timestamp) + queriesToSend.append(p) + p = Point('Utilization').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) + queriesToSend.append(p) + p = Point('Overload').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Overload", overloadFactor).time(timestamp) + queriesToSend.append(p) -# if seenSomethingBesides0s: -# write_api.write(bucket=influxDBBucket, record=queriesToSend) -# # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") -# queriesToSendCount += len(queriesToSend) + if seenSomethingBesides0s: + write_api.write(bucket=influx_db_bucket(), record=queriesToSend) + # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") + queriesToSendCount += len(queriesToSend) -# if 'cake diffserv4' in sqm: -# seenSomethingBesides0s = False -# queriesToSend = [] -# listOfTins = ['Bulk', 'BestEffort', 'Video', 'Voice'] -# for tin in listOfTins: -# p = Point('Tin Drop Percentage').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['dropPercentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['dropPercentage']).time(timestamp) -# queriesToSend.append(p) -# # Check to ensure tin percentage has value (!= None) before graphing. During partial or full reload these will have a value of None. -# if (tinsStats['sinceLastQuery'][tin]['Download']['percentage'] != None) and (tinsStats['sinceLastQuery'][tin]['Upload']['percentage'] != None): -# if (tinsStats['sinceLastQuery'][tin]['Download']['percentage'] > 0.0) or (tinsStats['sinceLastQuery'][tin]['Upload']['percentage'] > 0.0): -# seenSomethingBesides0s = True -# p = Point('Tins Assigned').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['percentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['percentage']).time(timestamp) -# queriesToSend.append(p) + if 'cake diffserv4' in sqm(): + seenSomethingBesides0s = False + queriesToSend = [] + listOfTins = ['Bulk', 'BestEffort', 'Video', 'Voice'] + for tin in listOfTins: + p = Point('Tin Drop Percentage').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['dropPercentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['dropPercentage']).time(timestamp) + queriesToSend.append(p) + # Check to ensure tin percentage has value (!= None) before graphing. During partial or full reload these will have a value of None. + if (tinsStats['sinceLastQuery'][tin]['Download']['percentage'] != None) and (tinsStats['sinceLastQuery'][tin]['Upload']['percentage'] != None): + if (tinsStats['sinceLastQuery'][tin]['Download']['percentage'] > 0.0) or (tinsStats['sinceLastQuery'][tin]['Upload']['percentage'] > 0.0): + seenSomethingBesides0s = True + p = Point('Tins Assigned').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['percentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['percentage']).time(timestamp) + queriesToSend.append(p) -# if seenSomethingBesides0s: -# write_api.write(bucket=influxDBBucket, record=queriesToSend) -# # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") -# queriesToSendCount += len(queriesToSend) + if seenSomethingBesides0s: + write_api.write(bucket=influx_db_bucket(), record=queriesToSend) + # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") + queriesToSendCount += len(queriesToSend) -# # Graph CPU use -# cpuVals = psutil.cpu_percent(percpu=True) -# queriesToSend = [] -# for index, item in enumerate(cpuVals): -# p = Point('CPU').field('CPU_' + str(index), item) -# queriesToSend.append(p) -# write_api.write(bucket=influxDBBucket, record=queriesToSend) -# queriesToSendCount += len(queriesToSend) + # Graph CPU use + cpuVals = psutil.cpu_percent(percpu=True) + queriesToSend = [] + for index, item in enumerate(cpuVals): + p = Point('CPU').field('CPU_' + str(index), item) + queriesToSend.append(p) + write_api.write(bucket=influx_db_bucket(), record=queriesToSend) + queriesToSendCount += len(queriesToSend) -# print("Added " + str(queriesToSendCount) + " points to InfluxDB.") + print("Added " + str(queriesToSendCount) + " points to InfluxDB.") -# client.close() + client.close() -# with open('statsByParentNode.json', 'w') as f: -# f.write(json.dumps(parentNodes, indent=4)) + with open('statsByParentNode.json', 'w') as f: + f.write(json.dumps(parentNodes, indent=4)) -# with open('statsByCircuit.json', 'w') as f: -# f.write(json.dumps(subscriberCircuits, indent=4)) + with open('statsByCircuit.json', 'w') as f: + f.write(json.dumps(subscriberCircuits, indent=4)) -# longTermStats['droppedPacketsTotal'] = droppedPacketsAllTime -# with open('longTermStats.json', 'w') as f: -# f.write(json.dumps(longTermStats, indent=4)) + longTermStats['droppedPacketsTotal'] = droppedPacketsAllTime + with open('longTermStats.json', 'w') as f: + f.write(json.dumps(longTermStats, indent=4)) -# with open('tinsStats.json', 'w') as f: -# f.write(json.dumps(tinsStats, indent=4)) + with open('tinsStats.json', 'w') as f: + f.write(json.dumps(tinsStats, indent=4)) -# endTime = datetime.now() -# durationSeconds = round((endTime - startTime).total_seconds(), 2) -# print("Graphs updated within " + str(durationSeconds) + " seconds.") + endTime = datetime.now() + durationSeconds = round((endTime - startTime).total_seconds(), 2) + print("Graphs updated within " + str(durationSeconds) + " seconds.") -# def refreshLatencyGraphs(): -# startTime = datetime.now() -# with open('statsByParentNode.json', 'r') as j: -# parentNodes = json.loads(j.read()) +def refreshLatencyGraphs(): + startTime = datetime.now() + with open('statsByParentNode.json', 'r') as j: + parentNodes = json.loads(j.read()) -# with open('statsByCircuit.json', 'r') as j: -# subscriberCircuits = json.loads(j.read()) + with open('statsByCircuit.json', 'r') as j: + subscriberCircuits = json.loads(j.read()) -# parentNodeNameDict = parentNodeNameDictPull() + parentNodeNameDict = parentNodeNameDictPull() -# print("Retrieving circuit statistics") -# subscriberCircuits = getCircuitLatencyStats(subscriberCircuits) -# print("Computing parent node statistics") -# parentNodes = getParentNodeLatencyStats(parentNodes, subscriberCircuits) -# print("Writing data to InfluxDB") -# client = InfluxDBClient( -# url=influxDBurl, -# token=influxDBtoken, -# org=influxDBOrg -# ) + print("Retrieving circuit statistics") + subscriberCircuits = getCircuitLatencyStats(subscriberCircuits) + print("Computing parent node statistics") + parentNodes = getParentNodeLatencyStats(parentNodes, subscriberCircuits) + print("Writing data to InfluxDB") + client = InfluxDBClient( + url=influx_db_url(), + token=influx_db_token(), + org=influx_db_org() + ) -# # Record current timestamp, use for all points added -# timestamp = time.time_ns() + # Record current timestamp, use for all points added + timestamp = time.time_ns() -# write_api = client.write_api(write_options=SYNCHRONOUS) + write_api = client.write_api(write_options=SYNCHRONOUS) -# chunkedsubscriberCircuits = list(chunk_list(subscriberCircuits, 200)) + chunkedsubscriberCircuits = list(chunk_list(subscriberCircuits, 200)) -# queriesToSendCount = 0 -# for chunk in chunkedsubscriberCircuits: -# queriesToSend = [] -# for circuit in chunk: -# if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: -# tcpLatency = float(circuit['stats']['sinceLastQuery']['tcpLatency']) -# p = Point('TCP Latency').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("TCP Latency", tcpLatency).time(timestamp) -# queriesToSend.append(p) -# write_api.write(bucket=influxDBBucket, record=queriesToSend) -# queriesToSendCount += len(queriesToSend) + queriesToSendCount = 0 + for chunk in chunkedsubscriberCircuits: + queriesToSend = [] + for circuit in chunk: + if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: + tcpLatency = float(circuit['stats']['sinceLastQuery']['tcpLatency']) + p = Point('TCP Latency').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("TCP Latency", tcpLatency).time(timestamp) + queriesToSend.append(p) + write_api.write(bucket=influx_db_bucket(), record=queriesToSend) + queriesToSendCount += len(queriesToSend) -# queriesToSend = [] -# for parentNode in parentNodes: -# if parentNode['stats']['sinceLastQuery']['tcpLatency'] != None: -# tcpLatency = float(parentNode['stats']['sinceLastQuery']['tcpLatency']) -# p = Point('TCP Latency').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("TCP Latency", tcpLatency).time(timestamp) -# queriesToSend.append(p) + queriesToSend = [] + for parentNode in parentNodes: + if parentNode['stats']['sinceLastQuery']['tcpLatency'] != None: + tcpLatency = float(parentNode['stats']['sinceLastQuery']['tcpLatency']) + p = Point('TCP Latency').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("TCP Latency", tcpLatency).time(timestamp) + queriesToSend.append(p) -# write_api.write(bucket=influxDBBucket, record=queriesToSend) -# queriesToSendCount += len(queriesToSend) + write_api.write(bucket=influx_db_bucket(), record=queriesToSend) + queriesToSendCount += len(queriesToSend) -# listOfAllLatencies = [] -# for circuit in subscriberCircuits: -# if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: -# listOfAllLatencies.append(circuit['stats']['sinceLastQuery']['tcpLatency']) -# if len(listOfAllLatencies) > 0: -# currentNetworkLatency = statistics.median(listOfAllLatencies) -# p = Point('TCP Latency').tag("Type", "Network").field("TCP Latency", currentNetworkLatency).time(timestamp) -# write_api.write(bucket=influxDBBucket, record=p) -# queriesToSendCount += 1 + listOfAllLatencies = [] + for circuit in subscriberCircuits: + if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: + listOfAllLatencies.append(circuit['stats']['sinceLastQuery']['tcpLatency']) + if len(listOfAllLatencies) > 0: + currentNetworkLatency = statistics.median(listOfAllLatencies) + p = Point('TCP Latency').tag("Type", "Network").field("TCP Latency", currentNetworkLatency).time(timestamp) + write_api.write(bucket=influx_db_bucket(), record=p) + queriesToSendCount += 1 -# print("Added " + str(queriesToSendCount) + " points to InfluxDB.") + print("Added " + str(queriesToSendCount) + " points to InfluxDB.") -# client.close() + client.close() -# with open('statsByParentNode.json', 'w') as f: -# f.write(json.dumps(parentNodes, indent=4)) + with open('statsByParentNode.json', 'w') as f: + f.write(json.dumps(parentNodes, indent=4)) -# with open('statsByCircuit.json', 'w') as f: -# f.write(json.dumps(subscriberCircuits, indent=4)) + with open('statsByCircuit.json', 'w') as f: + f.write(json.dumps(subscriberCircuits, indent=4)) -# endTime = datetime.now() -# durationSeconds = round((endTime - startTime).total_seconds(), 2) -# print("Graphs updated within " + str(durationSeconds) + " seconds.") + endTime = datetime.now() + durationSeconds = round((endTime - startTime).total_seconds(), 2) + print("Graphs updated within " + str(durationSeconds) + " seconds.") -# if __name__ == '__main__': -# refreshBandwidthGraphs() -# refreshLatencyGraphs() +if __name__ == '__main__': + refreshBandwidthGraphs() + refreshLatencyGraphs() diff --git a/src/integrationCommon.py b/src/integrationCommon.py index 805e1500..a020e7bf 100644 --- a/src/integrationCommon.py +++ b/src/integrationCommon.py @@ -146,7 +146,7 @@ class NetworkGraph: def addRawNode(self, node: NetworkNode) -> None: # Adds a NetworkNode to the graph, unchanged. - # If a site is excluded (via excludedSites in ispConfig) + # If a site is excluded (via excludedSites in lqos.conf) # it won't be added if not node.displayName in self.excludeSites: # TODO: Fixup exceptionCPE handling diff --git a/src/integrationRestHttp.py b/src/integrationRestHttp.py index a31b1549..9465a271 100644 --- a/src/integrationRestHttp.py +++ b/src/integrationRestHttp.py @@ -1,79 +1,81 @@ -import csv -import os -import shutil -from datetime import datetime +print("Deprecated for now.") -from requests import get +# import csv +# import os +# import shutil +# from datetime import datetime -from ispConfig import automaticImportRestHttp as restconf -from pydash import objects +# from requests import get -requestsBaseConfig = { - 'verify': True, - 'headers': { - 'accept': 'application/json' - } -} +# from ispConfig import automaticImportRestHttp as restconf +# from pydash import objects + +# requestsBaseConfig = { +# 'verify': True, +# 'headers': { +# 'accept': 'application/json' +# } +# } -def createShaper(): +# def createShaper(): - # shutil.copy('Shaper.csv', 'Shaper.csv.bak') - ts = datetime.now().strftime('%Y-%m-%d.%H-%M-%S') +# # shutil.copy('Shaper.csv', 'Shaper.csv.bak') +# ts = datetime.now().strftime('%Y-%m-%d.%H-%M-%S') - devicesURL = restconf.get('baseURL') + '/' + restconf.get('devicesURI').strip('/') +# devicesURL = restconf.get('baseURL') + '/' + restconf.get('devicesURI').strip('/') - requestConfig = objects.defaults_deep({'params': {}}, restconf.get('requestsConfig'), requestsBaseConfig) +# requestConfig = objects.defaults_deep({'params': {}}, restconf.get('requestsConfig'), requestsBaseConfig) - raw = get(devicesURL, **requestConfig, timeout=10) +# raw = get(devicesURL, **requestConfig, timeout=10) - if raw.status_code != 200: - print('Failed to request ' + devicesURL + ', got ' + str(raw.status_code)) - return False +# if raw.status_code != 200: +# print('Failed to request ' + devicesURL + ', got ' + str(raw.status_code)) +# return False - devicesCsvFP = os.path.dirname(os.path.realpath(__file__)) + '/ShapedDevices.csv' +# devicesCsvFP = os.path.dirname(os.path.realpath(__file__)) + '/ShapedDevices.csv' - with open(devicesCsvFP, 'w') as csvfile: - wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL) - wr.writerow( - ['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC', 'IPv4', 'IPv6', - 'Download Min Mbps', 'Upload Min Mbps', 'Download Max Mbps', 'Upload Max Mbps', 'Comment']) - for row in raw.json(): - wr.writerow(row.values()) +# with open(devicesCsvFP, 'w') as csvfile: +# wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL) +# wr.writerow( +# ['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC', 'IPv4', 'IPv6', +# 'Download Min Mbps', 'Upload Min Mbps', 'Download Max Mbps', 'Upload Max Mbps', 'Comment']) +# for row in raw.json(): +# wr.writerow(row.values()) - if restconf['logChanges']: - devicesBakFilePath = restconf['logChanges'].rstrip('/') + '/ShapedDevices.' + ts + '.csv' - try: - shutil.copy(devicesCsvFP, devicesBakFilePath) - except: - os.makedirs(restconf['logChanges'], exist_ok=True) - shutil.copy(devicesCsvFP, devicesBakFilePath) +# if restconf['logChanges']: +# devicesBakFilePath = restconf['logChanges'].rstrip('/') + '/ShapedDevices.' + ts + '.csv' +# try: +# shutil.copy(devicesCsvFP, devicesBakFilePath) +# except: +# os.makedirs(restconf['logChanges'], exist_ok=True) +# shutil.copy(devicesCsvFP, devicesBakFilePath) - networkURL = restconf['baseURL'] + '/' + restconf['networkURI'].strip('/') +# networkURL = restconf['baseURL'] + '/' + restconf['networkURI'].strip('/') - raw = get(networkURL, **requestConfig, timeout=10) +# raw = get(networkURL, **requestConfig, timeout=10) - if raw.status_code != 200: - print('Failed to request ' + networkURL + ', got ' + str(raw.status_code)) - return False +# if raw.status_code != 200: +# print('Failed to request ' + networkURL + ', got ' + str(raw.status_code)) +# return False - networkJsonFP = os.path.dirname(os.path.realpath(__file__)) + '/network.json' +# networkJsonFP = os.path.dirname(os.path.realpath(__file__)) + '/network.json' - with open(networkJsonFP, 'w') as handler: - handler.write(raw.text) +# with open(networkJsonFP, 'w') as handler: +# handler.write(raw.text) - if restconf['logChanges']: - networkBakFilePath = restconf['logChanges'].rstrip('/') + '/network.' + ts + '.json' - try: - shutil.copy(networkJsonFP, networkBakFilePath) - except: - os.makedirs(restconf['logChanges'], exist_ok=True) - shutil.copy(networkJsonFP, networkBakFilePath) +# if restconf['logChanges']: +# networkBakFilePath = restconf['logChanges'].rstrip('/') + '/network.' + ts + '.json' +# try: +# shutil.copy(networkJsonFP, networkBakFilePath) +# except: +# os.makedirs(restconf['logChanges'], exist_ok=True) +# shutil.copy(networkJsonFP, networkBakFilePath) -def importFromRestHttp(): - createShaper() +# def importFromRestHttp(): +# createShaper() -if __name__ == '__main__': - importFromRestHttp() +# if __name__ == '__main__': +# importFromRestHttp() diff --git a/src/integrationSonar.py b/src/integrationSonar.py index 9044d221..3cbffca4 100644 --- a/src/integrationSonar.py +++ b/src/integrationSonar.py @@ -4,7 +4,6 @@ import requests import subprocess from liblqos_python import sonar_api_key, sonar_api_url, snmp_community, sonar_airmax_ap_model_ids, \ sonar_ltu_ap_model_ids, sonar_active_status_ids -#from ispConfig import sonar_airmax_ap_model_ids,sonar_active_status_ids,sonar_ltu_ap_model_ids all_models = sonar_airmax_ap_model_ids() + sonar_ltu_ap_model_ids() from integrationCommon import NetworkGraph, NetworkNode, NodeType from multiprocessing.pool import ThreadPool @@ -120,7 +119,7 @@ def getSitesAndAps(): } sites_and_aps = sonarRequest(query,variables) - # This should only return sites that have equipment on them that is in the list sonar_ubiquiti_ap_model_ids in ispConfig.py + # This should only return sites that have equipment on them that is in the list sonar_ubiquiti_ap_model_ids in lqos.conf sites = [] aps = [] for site in sites_and_aps: diff --git a/src/lqos.example b/src/lqos.example index 184ae217..deb197de 100644 --- a/src/lqos.example +++ b/src/lqos.example @@ -93,3 +93,10 @@ snmp_community = "public" airmax_model_ids = [ "" ] ltu_model_ids = [ "" ] active_status_ids = [ "" ] + +[influxdb] +enable_influxdb = false +url = "http://localhost:8086" +org = "libreqos" +bucket = "Your ISP Name Here" +token = "" \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/etclqos_migration.rs b/src/rust/lqos_config/src/etc/etclqos_migration.rs index 78ccab96..17ed2190 100644 --- a/src/rust/lqos_config/src/etc/etclqos_migration.rs +++ b/src/rust/lqos_config/src/etc/etclqos_migration.rs @@ -309,7 +309,7 @@ mod test { fn round_trip_toml() { let doc = EXAMPLE_LQOS_CONF.parse::().unwrap(); let reserialized = doc.to_string(); - assert_eq!(EXAMPLE_LQOS_CONF, reserialized); + assert_eq!(EXAMPLE_LQOS_CONF.trim(), reserialized.trim()); } #[test] diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs index 3d09ac03..98dcf53d 100644 --- a/src/rust/lqos_config/src/etc/migration.rs +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -89,6 +89,7 @@ fn do_migration_14_to_15( migrate_powercode(python_config, &mut new_config)?; migrate_sonar(python_config, &mut new_config)?; migrate_queues( python_config, &mut new_config)?; + migrate_influx(python_config, &mut new_config)?; new_config.validate().unwrap(); // Left as an upwrap because this should *never* happen Ok(new_config) @@ -272,6 +273,18 @@ fn migrate_uisp( Ok(()) } +fn migrate_influx( + python_config: &PythonMigration, + new_config: &mut Config, +) -> Result<(), MigrationError> { + new_config.influxdb.enable_influxdb = python_config.influx_db_enabled; + new_config.influxdb.url = python_config.influx_db_url.clone(); + new_config.influxdb.bucket = python_config.infux_db_bucket.clone(); + new_config.influxdb.org = python_config.influx_db_org.clone(); + new_config.influxdb.token = python_config.influx_db_token.clone(); + Ok(()) +} + #[cfg(test)] mod test { use super::*; diff --git a/src/rust/lqos_config/src/etc/python_migration.rs b/src/rust/lqos_config/src/etc/python_migration.rs index 92454f3f..6f5d6b6f 100644 --- a/src/rust/lqos_config/src/etc/python_migration.rs +++ b/src/rust/lqos_config/src/etc/python_migration.rs @@ -182,6 +182,13 @@ impl PythonMigration { cfg.sonar_airmax_ap_model_ids = from_python(&py, "sonar_airmax_ap_model_ids").unwrap_or(vec![]); cfg.sonar_ltu_ap_model_ids = from_python(&py, "sonar_ltu_ap_model_ids").unwrap_or(vec![]); + // InfluxDB + cfg.influx_db_enabled = from_python(&py, "influxDBEnabled").unwrap_or(false); + cfg.influx_db_url = from_python(&py, "influxDBurl").unwrap_or("http://localhost:8086".to_string()); + cfg.infux_db_bucket = from_python(&py, "influxDBBucket").unwrap_or("libreqos".to_string()); + cfg.influx_db_org = from_python(&py, "influxDBOrg").unwrap_or("Your ISP Name Here".to_string()); + cfg.influx_db_token = from_python(&py, "influxDBtoken").unwrap_or("".to_string()); + Ok(()) } diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index 184ae217..deb197de 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -93,3 +93,10 @@ snmp_community = "public" airmax_model_ids = [ "" ] ltu_model_ids = [ "" ] active_status_ids = [ "" ] + +[influxdb] +enable_influxdb = false +url = "http://localhost:8086" +org = "libreqos" +bucket = "Your ISP Name Here" +token = "" \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/influxdb.rs b/src/rust/lqos_config/src/etc/v15/influxdb.rs new file mode 100644 index 00000000..eb7788e3 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/influxdb.rs @@ -0,0 +1,22 @@ +use serde::{Serialize, Deserialize}; + +#[derive(Clone, Serialize, Deserialize, Debug)] +pub struct InfluxDbConfig { + pub enable_influxdb: bool, + pub url: String, + pub bucket: String, + pub org: String, + pub token: String, +} + +impl Default for InfluxDbConfig { + fn default() -> Self { + Self { + enable_influxdb: false, + url: "http://localhost:8086".to_string(), + bucket: "libreqos".to_string(), + org: "Your ISP Name".to_string(), + token: "".to_string() + } + } +} \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/mod.rs b/src/rust/lqos_config/src/etc/v15/mod.rs index fc95839a..44567b47 100644 --- a/src/rust/lqos_config/src/etc/v15/mod.rs +++ b/src/rust/lqos_config/src/etc/v15/mod.rs @@ -13,6 +13,7 @@ mod spylnx_integration; mod uisp_integration; mod powercode_integration; mod sonar_integration; +mod influxdb; pub use bridge::*; pub use long_term_stats::LongTermStats; pub use tuning::Tunables; \ No newline at end of file diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index f322d7da..5d7175d1 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -65,6 +65,9 @@ pub struct Config { /// Sonar Integration pub sonar_integration: super::sonar_integration::SonarIntegration, + + /// InfluxDB Configuration + pub influxdb: super::influxdb::InfluxDbConfig, } impl Config { @@ -127,6 +130,7 @@ impl Default for Config { uisp_integration: super::uisp_integration::UispIntegration::default(), powercode_integration: super::powercode_integration::PowercodeIntegration::default(), sonar_integration: super::sonar_integration::SonarIntegration::default(), + influxdb: super::influxdb::InfluxDbConfig::default(), packet_capture_time: 10, queue_check_period_ms: 1000, } diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index a7afa1a6..32adfd29 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -81,6 +81,11 @@ fn liblqos_python(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(sonar_airmax_ap_model_ids))?; m.add_wrapped(wrap_pyfunction!(sonar_ltu_ap_model_ids))?; m.add_wrapped(wrap_pyfunction!(sonar_active_status_ids))?; + m.add_wrapped(wrap_pyfunction!(influx_db_enabled))?; + m.add_wrapped(wrap_pyfunction!(influx_db_bucket))?; + m.add_wrapped(wrap_pyfunction!(influx_db_org))?; + m.add_wrapped(wrap_pyfunction!(influx_db_token))?; + m.add_wrapped(wrap_pyfunction!(influx_db_url))?; Ok(()) } @@ -602,4 +607,34 @@ fn sonar_ltu_ap_model_ids() -> PyResult> { fn sonar_active_status_ids() -> PyResult> { let config = lqos_config::load_config().unwrap(); Ok(config.sonar_integration.active_status_ids) +} + +#[pyfunction] +fn influx_db_enabled() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.influxdb.enable_influxdb) +} + +#[pyfunction] +fn influx_db_bucket() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.influxdb.bucket) +} + +#[pyfunction] +fn influx_db_org() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.influxdb.org) +} + +#[pyfunction] +fn influx_db_token() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.influxdb.token) +} + +#[pyfunction] +fn influx_db_url() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config.influxdb.url) } \ No newline at end of file From 7180dd7950c5f11a91971222ca90c3a409b30f92 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 5 Feb 2024 11:33:32 -0600 Subject: [PATCH 41/41] BUGFIX: We weren't correctly honouring requests to use a Linux bridge. This patch fixes it. --- src/rust/lqos_sys/src/lqos_kernel.rs | 38 +++++++++++++++------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/rust/lqos_sys/src/lqos_kernel.rs b/src/rust/lqos_sys/src/lqos_kernel.rs index 66746369..405d865b 100644 --- a/src/rust/lqos_sys/src/lqos_kernel.rs +++ b/src/rust/lqos_sys/src/lqos_kernel.rs @@ -207,26 +207,28 @@ pub fn attach_xdp_and_tc_to_interface( // Attach to the ingress IF it is configured if let Ok(etc) = lqos_config::load_config() { if let Some(bridge) = &etc.bridge { - // Enable "promiscuous" mode on interfaces - info!("Enabling promiscuous mode on {}", &bridge.to_internet); - std::process::Command::new("/bin/ip") - .args(["link", "set", &bridge.to_internet, "promisc", "on"]) - .output()?; - info!("Enabling promiscuous mode on {}", &bridge.to_network); - std::process::Command::new("/bin/ip") - .args(["link", "set", &bridge.to_network, "promisc", "on"]) - .output()?; + if bridge.use_xdp_bridge { + // Enable "promiscuous" mode on interfaces + info!("Enabling promiscuous mode on {}", &bridge.to_internet); + std::process::Command::new("/bin/ip") + .args(["link", "set", &bridge.to_internet, "promisc", "on"]) + .output()?; + info!("Enabling promiscuous mode on {}", &bridge.to_network); + std::process::Command::new("/bin/ip") + .args(["link", "set", &bridge.to_network, "promisc", "on"]) + .output()?; - // Build the interface and vlan map entries - crate::bifrost_maps::clear_bifrost()?; - crate::bifrost_maps::map_multi_interface_mode(&bridge.to_internet, &bridge.to_network)?; + // Build the interface and vlan map entries + crate::bifrost_maps::clear_bifrost()?; + crate::bifrost_maps::map_multi_interface_mode(&bridge.to_internet, &bridge.to_network)?; - // Actually attach the TC ingress program - let error = unsafe { - bpf::tc_attach_ingress(interface_index as i32, false, skeleton) - }; - if error != 0 { - return Err(Error::msg("Unable to attach TC Ingress to interface")); + // Actually attach the TC ingress program + let error = unsafe { + bpf::tc_attach_ingress(interface_index as i32, false, skeleton) + }; + if error != 0 { + return Err(Error::msg("Unable to attach TC Ingress to interface")); + } } }