First commit for a unified configuration system.

* Adds PyO3 as a dependency to the config crate.
* Uses PyO3 to load an existing configuration as a Python object.
* Adds some generic conversion code for reading Python types to Rust
  equivalents.
* Adds a preliminary "read the whole existing config into Rust" code.

Not in a shape to use yet, but a good start.

src/rust/Cargo.lock

@@ -134,7 +134,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -145,7 +145,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -264,7 +264,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.29",
+ "syn",
  "which",
 ]
 
@@ -459,7 +459,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -708,7 +708,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -777,7 +777,7 @@ dependencies = [
  "proc-macro2",
  "proc-macro2-diagnostics",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -1023,7 +1023,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -1261,9 +1261,9 @@ dependencies = [
 
 [[package]]
 name = "indoc"
-version = "1.0.9"
+version = "2.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306"
+checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8"
 
 [[package]]
 name = "inlinable_string"
@@ -1530,6 +1530,7 @@ dependencies = [
  "ip_network",
  "ip_network_table",
  "log",
+ "pyo3",
  "serde",
  "serde_json",
  "sha2",
@@ -2032,7 +2033,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -2108,7 +2109,7 @@ dependencies = [
  "proc-macro2",
  "proc-macro2-diagnostics",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -2140,7 +2141,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -2208,7 +2209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62"
 dependencies = [
  "proc-macro2",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -2228,16 +2229,16 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
  "version_check",
  "yansi 1.0.0-rc.1",
 ]
 
 [[package]]
 name = "pyo3"
-version = "0.19.2"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38"
+checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b"
 dependencies = [
  "cfg-if",
  "indoc",
@@ -2252,9 +2253,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.19.2"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5"
+checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5"
 dependencies = [
  "once_cell",
  "target-lexicon",
@@ -2262,9 +2263,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.19.2"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9"
+checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -2272,25 +2273,26 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.19.2"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1"
+checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
  "quote",
- "syn 1.0.109",
+ "syn",
 ]
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.19.2"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536"
+checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424"
 dependencies = [
+ "heck",
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn",
 ]
 
 [[package]]
@@ -2380,7 +2382,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -2552,7 +2554,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rocket_http",
- "syn 2.0.29",
+ "syn",
  "unicode-xid",
 ]
 
@@ -2725,7 +2727,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -2927,17 +2929,6 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
 
-[[package]]
-name = "syn"
-version = "1.0.109"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
 [[package]]
 name = "syn"
 version = "2.0.29"
@@ -3036,7 +3027,7 @@ checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -3129,7 +3120,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -3250,7 +3241,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -3387,9 +3378,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
 
 [[package]]
 name = "unindent"
-version = "0.1.11"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c"
+checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
 
 [[package]]
 name = "url"
@@ -3483,7 +3474,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
  "wasm-bindgen-shared",
 ]
 
@@ -3517,7 +3508,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -3790,7 +3781,7 @@ checksum = "8f7f3a471f98d0a61c34322fbbfd10c384b07687f680d4119813713f72308d91"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]
 
 [[package]]
@@ -3810,5 +3801,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn",
 ]

src/rust/lqos_config/Cargo.toml

@@ -16,3 +16,4 @@ sha2 = "0"
 uuid = { version = "1", features = ["v4", "fast-rng" ] }
 log = "0"
 dashmap = "5"
+pyo3 = "0.20"

src/rust/lqos_config/src/lib.rs

@@ -12,6 +12,7 @@ mod libre_qos_config;
 mod network_json;
 mod program_control;
 mod shaped_devices;
+mod python_migration;
 
 pub use authentication::{UserRole, WebUsers};
 pub use etc::{BridgeConfig, BridgeInterface, BridgeVlan, EtcLqos, Tunables, enable_long_term_stats};

src/rust/lqos_config/src/python_migration.rs

@@ -0,0 +1,190 @@
+//! This module utilizes PyO3 to read an existing ispConfig.py file, and
+//! provide conversion services for the new, unified configuration target
+//! for version 1.5.
+
+use crate::EtcLqos;
+use pyo3::{prepare_freethreaded_python, Python};
+use std::{
+    fs::read_to_string,
+    path::{Path, PathBuf}, collections::HashMap,
+};
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum PythonMigrationError {
+    #[error("The ispConfig.py file does not exist.")]
+    ConfigFileNotFound,
+    #[error("Unable to parse variable")]
+    ParseError,
+    #[error("Variable not found")]
+    VariableNotFound(String),
+}
+
+fn isp_config_py_path(cfg: &EtcLqos) -> PathBuf {
+    let base_path = Path::new(&cfg.lqos_directory);
+    let final_path = base_path.join("ispConfig.py");
+    final_path
+}
+
+/// Does thie ispConfig.py file exist?
+fn config_exists(cfg: &EtcLqos) -> bool {
+    if let Ok(cfg) = crate::etc::EtcLqos::load() {
+        isp_config_py_path(&cfg).exists()
+    } else {
+        false
+    }
+}
+
+fn from_python<'a, T>(py: &'a Python, variable_name: &str) -> Result<T, PythonMigrationError>
+where
+    T: pyo3::FromPyObject<'a>,
+{
+    let result = py
+        .eval(variable_name, None, None)
+        .map_err(|_| PythonMigrationError::VariableNotFound(variable_name.to_string()))?
+        .extract::<T>()
+        .map_err(|_| PythonMigrationError::ParseError)?;
+
+    Ok(result)
+}
+
+#[derive(Default, Debug)]
+pub struct PythonMigration {
+    pub sqm: String,
+    pub monitor_only_mode: bool,
+    pub upstream_bandwidth_capacity_download_mbps: u32,
+    pub upstream_bandwidth_capacity_upload_mbps: u32,
+    pub generated_pn_download_mbps: u32,
+    pub generated_pn_upload_mbps: u32,
+    pub interface_a: String,
+    pub interface_b: String,
+    pub queue_refresh_interval_mins: u32,
+    pub on_a_stick: bool,
+    pub stick_vlan_a: u32,
+    pub stick_vlan_b: u32,
+    pub enable_actual_shell_commands: bool,
+    pub run_shell_commands_as_sudo: bool,
+    pub queues_available_override: u32,
+    pub use_bin_packing_to_balance_cpu: bool,
+    pub influx_db_enabled: bool,
+    pub influx_db_url: String,
+    pub infux_db_bucket: String,
+    pub influx_db_org: String,
+    pub influx_db_token: String,
+    pub circuit_name_use_address: bool,
+    pub overwrite_network_json_always: bool,
+    pub ignore_subnets: Vec<String>,
+    pub allowed_subnets: Vec<String>,
+    pub automatic_import_splynx: bool,
+    pub splynx_api_key: String,
+    pub spylnx_api_secret: String,
+    pub spylnx_api_url: String,
+    pub automatic_import_uisp: bool,
+    pub uisp_auth_token: String,
+    pub uisp_base_url: String,
+    pub uisp_site: String,
+    pub uisp_strategy: String,
+    pub uisp_suspended_strategy: String,
+    pub airmax_capacity: f32,
+    pub ltu_capacity: f32,
+    pub exclude_sites: Vec<String>,
+    pub find_ipv6_using_mikrotik: bool,
+    pub bandwidth_overhead_factor: f32,
+    pub committed_bandwidth_multiplier: f32,
+    pub exception_cpes: HashMap<String, String>,
+    pub api_username: String,
+    pub api_password: String,
+    pub api_host_ip: String,
+    pub api_host_port: u32,
+
+    // TODO: httpRestIntegrationConfig
+}
+
+impl PythonMigration {
+    fn parse(cfg: &mut Self, py: &Python) -> Result<(), PythonMigrationError> {
+        cfg.sqm = from_python(&py, "sqm")?;
+        cfg.monitor_only_mode = from_python(&py, "monitorOnlyMode")?;
+        cfg.upstream_bandwidth_capacity_download_mbps =
+            from_python(&py, "upstreamBandwidthCapacityDownloadMbps")?;
+        cfg.upstream_bandwidth_capacity_upload_mbps =
+            from_python(&py, "upstreamBandwidthCapacityUploadMbps")?;
+        cfg.generated_pn_download_mbps = from_python(&py, "generatedPNDownloadMbps")?;
+        cfg.generated_pn_upload_mbps = from_python(&py, "generatedPNUploadMbps")?;
+        cfg.interface_a = from_python(&py, "interfaceA")?;
+        cfg.interface_b = from_python(&py, "interfaceB")?;
+        cfg.queue_refresh_interval_mins = from_python(&py, "queueRefreshIntervalMins")?;
+        cfg.on_a_stick = from_python(&py, "OnAStick")?;
+        cfg.stick_vlan_a = from_python(&py, "StickVlanA")?;
+        cfg.stick_vlan_b = from_python(&py, "StickVlanB")?;
+        cfg.enable_actual_shell_commands = from_python(&py, "enableActualShellCommands")?;
+        cfg.run_shell_commands_as_sudo = from_python(&py, "runShellCommandsAsSudo")?;
+        cfg.queues_available_override = from_python(&py, "queuesAvailableOverride")?;
+        cfg.use_bin_packing_to_balance_cpu = from_python(&py, "useBinPackingToBalanceCPU")?;
+        cfg.influx_db_enabled = from_python(&py, "influxDBEnabled")?;
+        cfg.influx_db_url = from_python(&py, "influxDBurl")?;
+        cfg.infux_db_bucket = from_python(&py, "influxDBBucket")?;
+        cfg.influx_db_org = from_python(&py, "influxDBOrg")?;
+        cfg.influx_db_token = from_python(&py, "influxDBtoken")?;
+        cfg.circuit_name_use_address = from_python(&py, "circuitNameUseAddress")?;
+        cfg.overwrite_network_json_always = from_python(&py, "overwriteNetworkJSONalways")?;
+        cfg.ignore_subnets = from_python(&py, "ignoreSubnets")?;
+        cfg.allowed_subnets = from_python(&py, "allowedSubnets")?;
+        cfg.automatic_import_splynx = from_python(&py, "automaticImportSplynx")?;
+        cfg.splynx_api_key = from_python(&py, "splynx_api_key")?;
+        cfg.spylnx_api_secret = from_python(&py, "splynx_api_secret")?;
+        cfg.spylnx_api_url = from_python(&py, "splynx_api_url")?;
+        cfg.automatic_import_uisp = from_python(&py, "automaticImportUISP")?;
+        cfg.uisp_auth_token = from_python(&py, "uispAuthToken")?;
+        cfg.uisp_base_url = from_python(&py, "UISPbaseURL")?;
+        cfg.uisp_site = from_python(&py, "uispSite")?;
+        cfg.uisp_strategy = from_python(&py, "uispStrategy")?;
+        cfg.uisp_suspended_strategy = from_python(&py, "uispSuspendedStrategy")?;
+        cfg.airmax_capacity = from_python(&py, "airMax_capacity")?;
+        cfg.ltu_capacity = from_python(&py, "ltu_capacity")?;
+        cfg.exclude_sites = from_python(&py, "excludeSites")?;
+        cfg.find_ipv6_using_mikrotik = from_python(&py, "findIPv6usingMikrotik")?;
+        cfg.bandwidth_overhead_factor = from_python(&py, "bandwidthOverheadFactor")?;
+        cfg.committed_bandwidth_multiplier = from_python(&py, "committedBandwidthMultiplier")?;
+        cfg.exception_cpes = from_python(&py, "exceptionCPEs")?;
+        cfg.api_username = from_python(&py, "apiUsername")?;
+        cfg.api_password = from_python(&py, "apiPassword")?;
+        cfg.api_host_ip = from_python(&py, "apiHostIP")?;
+        cfg.api_host_port = from_python(&py, "apiHostPost")?;
+
+        Ok(())
+    }
+
+    pub fn load() -> Result<Self, PythonMigrationError> {
+        let mut old_config = Self::default();
+        if let Ok(cfg) = crate::etc::EtcLqos::load() {
+            if !config_exists(&cfg) {
+                return Err(PythonMigrationError::ConfigFileNotFound);
+            }
+            let code = read_to_string(isp_config_py_path(&cfg)).unwrap();
+
+            prepare_freethreaded_python();
+            Python::with_gil(|py| {
+                py.run(&code, None, None).unwrap();
+                let result = Self::parse(&mut old_config, &py);
+                if result.is_err() {
+                    println!("Error parsing Python config: {:?}", result);
+                }
+            });
+        } else {
+            return Err(PythonMigrationError::ConfigFileNotFound);
+        }
+
+        Ok(old_config)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::PythonMigration;
+
+    #[test]
+    fn load_it() {
+        let cfg = PythonMigration::load().unwrap();
+        println!("{:#?}", cfg);
+    }
+}