WIP: Start adding json_overrides.json and the library/app to manage it.

.gitignore

@@ -141,3 +141,5 @@ src/bin/lqos_api
 /src/linux_tc_rust.txt
 /src/linux_tc_rust_live.txt
 /src/linux_tc_rust_prune.txt
+/src/lqos_overrides.json
+

src/rust/Cargo.lock

@@ -1962,6 +1962,18 @@ dependencies = [
  "lqos_sys",
 ]
 
+[[package]]
+name = "lqos_overrides"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "lqos_config",
+ "nix",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "lqos_python"
 version = "0.1.0"

src/rust/Cargo.toml

@@ -33,6 +33,7 @@ members = [
     "lqos_support_tool", # A Helper tool to make it easier to request/receive support
     "lqos_stormguard", # An implementation of CAKE AutoRotate using dynamic bus information. EXPERIMENTAL.
     "lqos_bakery", # The bakery makes CAKEs - controls queue creation.
+    "lqos_overrides", # A CLI tool and library for unifying the override system and allowing API support for changing network.json and ShapedDevices.csv
 ]
 
 [dependencies]
@@ -79,4 +80,4 @@ mimalloc = "0.1.43"
 
 # For memory debugging
 allocative = {  version = "0.3.3", features = [ "dashmap" ] }
-allocative_derive = "0.3.3"
+allocative_derive = "0.3.3"

src/rust/lqos_overrides/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "lqos_overrides"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+serde.workspace = true
+serde_json.workspace = true
+anyhow.workspace = true
+nix.workspace = true
+lqos_config = { path = "../lqos_config" }
+clap = { workspace = true, features = ["derive"] }

src/rust/lqos_overrides/src/lib.rs

@@ -0,0 +1,2 @@
+mod overrides_file;
+pub use overrides_file::OverrideFile;

src/rust/lqos_overrides/src/main.rs

@@ -0,0 +1,166 @@
+use std::net::{Ipv4Addr, Ipv6Addr};
+
+use anyhow::{anyhow, Result};
+use clap::{Args, Parser, Subcommand};
+
+use lqos_config::ShapedDevice;
+use lqos_overrides::OverrideFile;
+
+#[derive(Parser, Debug)]
+#[command(name = "lqos_overrides")]
+#[command(about = "Manage LibreQoS overrides", version, author)]
+struct Cli {
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand, Debug)]
+enum Commands {
+    /// Manage appended shaped devices
+    AppendDevices {
+        #[command(subcommand)]
+        command: AppendDevicesCommand,
+    },
+}
+
+#[derive(Subcommand, Debug)]
+enum AppendDevicesCommand {
+    /// Add a shaped device to append list
+    Add(AddArgs),
+    /// Delete all appended devices with a circuit ID
+    DeleteCircuitId { #[arg(long)] circuit_id: String },
+    /// Delete appended device(s) by device ID
+    DeleteDeviceId { #[arg(long)] device_id: String },
+}
+
+#[derive(Args, Debug)]
+struct AddArgs {
+    #[arg(long)]
+    circuit_id: String,
+    #[arg(long)]
+    circuit_name: String,
+    #[arg(long)]
+    device_id: String,
+    #[arg(long)]
+    device_name: String,
+    #[arg(long)]
+    parent_node: String,
+    #[arg(long)]
+    mac: String,
+    /// Repeat for multiple IPv4 entries (CIDR supported, default /32)
+    #[arg(long = "ipv4")]
+    ipv4_list: Vec<String>,
+    /// Repeat for multiple IPv6 entries (CIDR supported, default /128)
+    #[arg(long = "ipv6")]
+    ipv6_list: Vec<String>,
+    #[arg(long)]
+    download_min_mbps: f32,
+    #[arg(long)]
+    upload_min_mbps: f32,
+    #[arg(long)]
+    download_max_mbps: f32,
+    #[arg(long)]
+    upload_max_mbps: f32,
+    #[arg(long, default_value = "")]
+    comment: String,
+}
+
+fn parse_ipv4(s: &str) -> Result<(Ipv4Addr, u32)> {
+    if let Some((ip, prefix)) = s.split_once('/') {
+        Ok((ip.parse()?, prefix.parse()?))
+    } else {
+        Ok((s.parse()?, 32))
+    }
+}
+
+fn parse_ipv6(s: &str) -> Result<(Ipv6Addr, u32)> {
+    if let Some((ip, prefix)) = s.split_once('/') {
+        Ok((ip.parse()?, prefix.parse()?))
+    } else {
+        Ok((s.parse()?, 128))
+    }
+}
+
+impl AddArgs {
+    fn into_device(self) -> Result<ShapedDevice> {
+        let ipv4 = self
+            .ipv4_list
+            .iter()
+            .map(|s| parse_ipv4(s))
+            .collect::<Result<Vec<_>>>()?;
+        let ipv6 = self
+            .ipv6_list
+            .iter()
+            .map(|s| parse_ipv6(s))
+            .collect::<Result<Vec<_>>>()?;
+
+        if self.download_min_mbps <= 0.0
+            || self.upload_min_mbps <= 0.0
+            || self.download_max_mbps <= 0.0
+            || self.upload_max_mbps <= 0.0
+        {
+            return Err(anyhow!("Bandwidth values must be positive"));
+        }
+
+        Ok(ShapedDevice {
+            circuit_id: self.circuit_id,
+            circuit_name: self.circuit_name,
+            device_id: self.device_id,
+            device_name: self.device_name,
+            parent_node: self.parent_node,
+            mac: self.mac,
+            ipv4,
+            ipv6,
+            download_min_mbps: self.download_min_mbps,
+            upload_min_mbps: self.upload_min_mbps,
+            download_max_mbps: self.download_max_mbps,
+            upload_max_mbps: self.upload_max_mbps,
+            comment: self.comment,
+            circuit_hash: 0,
+            device_hash: 0,
+            parent_hash: 0,
+        })
+    }
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+
+    // Load overrides file at start
+    let mut overrides = OverrideFile::load()?;
+
+    match cli.command {
+        Commands::AppendDevices { command: cmd } => match cmd {
+            AppendDevicesCommand::Add(args) => {
+                let device = args.into_device()?;
+                let changed = overrides.add_append_shaped_device_return_changed(device);
+                if changed {
+                    overrides.save()?;
+                    println!("Added device; overrides saved.");
+                } else {
+                    println!("No changes (device already present).");
+                }
+            }
+            AppendDevicesCommand::DeleteCircuitId { circuit_id } => {
+                let removed = overrides.remove_append_shaped_device_by_circuit_count(&circuit_id);
+                if removed > 0 {
+                    overrides.save()?;
+                    println!("Removed {removed} device(s) by circuit_id; overrides saved.");
+                } else {
+                    println!("No devices matched circuit_id {circuit_id}.");
+                }
+            }
+            AppendDevicesCommand::DeleteDeviceId { device_id } => {
+                let removed = overrides.remove_append_shaped_device_by_device_count(&device_id);
+                if removed > 0 {
+                    overrides.save()?;
+                    println!("Removed {removed} device(s) by device_id; overrides saved.");
+                } else {
+                    println!("No devices matched device_id {device_id}.");
+                }
+            }
+        },
+    }
+
+    Ok(())
+}

src/rust/lqos_overrides/src/overrides_file.rs

@@ -0,0 +1,79 @@
+use std::{fs::read_to_string, path::Path};
+
+use serde::{Deserialize, Serialize};
+use anyhow::Result;
+use lqos_config::ShapedDevice;
+
+use crate::overrides_file::file_lock::FileLock;
+
+mod file_lock;
+
+#[derive(Serialize, Deserialize, Default)]
+pub struct OverrideFile {
+    /// Devices that will be appended to ShapedDevices.csv by LibreQoS.py when rebuilding. Useful
+    /// for adding persistent "catch all", or API-controlled new devices that are somehow detached
+    /// from the scheduler integration.
+    devices_to_append: Vec<ShapedDevice>,
+}
+
+impl OverrideFile {
+    pub fn load() -> Result<Self> {
+        let lock = FileLock::new()?;
+        let config = lqos_config::load_config()?;
+        let path = Path::new(&config.lqos_directory).join("lqos_overrides.json");
+        if !path.exists() {
+            // Create a default empty file
+            let new_override_file = OverrideFile::default();
+            let as_json = serde_json::to_string(&new_override_file)?;
+            std::fs::write(&path, as_json.as_bytes())?;
+        }
+        let raw = read_to_string(path)?;
+        let as_json = serde_json::from_str(&raw)?;
+        drop(lock); // Explicitly drop for clarity. RAII does it anyway.
+        Ok(as_json)
+    }
+
+    pub fn save(&self) -> Result<()> {
+        let lock = FileLock::new()?;
+        let config = lqos_config::load_config()?;
+        let path = Path::new(&config.lqos_directory).join("lqos_overrides.json");
+        let as_json = serde_json::to_string(self)?;
+        std::fs::write(&path, as_json.as_bytes())?;
+        drop(lock); // Explicitly drop for clarity. RAII does it anyway.
+        Ok(())
+    }
+
+    /// Add or replace a shaped device by `device_id`. Returns true if changed.
+    pub fn add_append_shaped_device_return_changed(&mut self, device: ShapedDevice) -> bool {
+        if let Some(existing) = self
+            .devices_to_append
+            .iter()
+            .find(|d| d.device_id == device.device_id)
+        {
+            if existing == &device {
+                // No change needed
+                return false;
+            }
+        }
+        self.devices_to_append
+            .retain(|d| d.device_id != device.device_id);
+        self.devices_to_append.push(device);
+        true
+    }
+
+    /// Remove all devices matching `circuit_id`. Returns number removed.
+    pub fn remove_append_shaped_device_by_circuit_count(&mut self, circuit_id: &str) -> usize {
+        let before = self.devices_to_append.len();
+        self.devices_to_append
+            .retain(|d| d.circuit_id != circuit_id);
+        before.saturating_sub(self.devices_to_append.len())
+    }
+
+    /// Remove all devices matching `device_id`. Returns number removed.
+    pub fn remove_append_shaped_device_by_device_count(&mut self, device_id: &str) -> usize {
+        let before = self.devices_to_append.len();
+        self.devices_to_append
+            .retain(|d| d.device_id != device_id);
+        before.saturating_sub(self.devices_to_append.len())
+    }
+}

src/rust/lqos_overrides/src/overrides_file/file_lock.rs

@@ -0,0 +1,90 @@
+use std::{ffi::CString, fs::{remove_file, File}, io::{Read, Write}, path::Path};
+use anyhow::{Error, Result};
+use nix::{errno::Errno, libc::{getpid, mode_t}};
+
+const LOCK_PATH: &str = "/run/lqos/lqos_overrides.lock";
+const LOCK_DIR: &str = "/run/lqos";
+const LOCK_DIR_PERMS: &str = "/run/lqos";
+
+pub struct FileLock {
+
+}
+
+impl FileLock {
+    pub fn new() -> Result<Self> {
+        Self::check_directory()?;
+        let lock_path = Path::new(LOCK_PATH);
+        if lock_path.exists() {
+            if Self::is_lock_valid()? {
+                return Err(Error::msg("The lqos_overrides file is locked by another process."));
+            }
+
+            // It's a stale pid, so we need to replace it
+            Self::create_lock()?;
+
+            Ok(Self {})
+        } else {
+            Self::create_lock()?;
+            Ok(Self {})
+        }
+    }
+
+    fn is_lock_valid() -> Result<bool> {
+        let mut f = File::open(LOCK_PATH)?;
+        let mut contents = String::new();
+        f.read_to_string(&mut contents)?;
+        let pid: i32 = contents.trim().parse()?;
+
+        // Check if the process exists by sending signal 0.
+        // Returns 0 if the process exists; -1 with ESRCH if it doesn't;
+        // other errors (e.g., EPERM) mean the process exists but we lack permission.
+        let ret = unsafe { nix::libc::kill(pid, 0) };
+        if ret == 0 {
+            return Ok(true);
+        }
+        let err = Errno::last();
+        if err == Errno::ESRCH {
+            Ok(false)
+        } else {
+            Ok(true)
+        }
+    }
+
+    fn create_lock() -> Result<()> {
+        let pid = unsafe { getpid() };
+        let pid_format = format!("{pid}");
+        {
+            let mut f = File::create(LOCK_PATH)?;
+            f.write_all(pid_format.as_bytes())?;
+        }
+        let unix_path = CString::new(LOCK_PATH)?;
+        unsafe {
+            nix::libc::chmod(unix_path.as_ptr(), mode_t::from_le(666));
+        }
+        Ok(())
+    }
+
+    fn check_directory() -> Result<()> {
+        let dir_path = std::path::Path::new(LOCK_DIR);
+        if dir_path.exists() && dir_path.is_dir() {
+            Ok(())
+        } else {
+            std::fs::create_dir(dir_path)?;
+            let unix_path = CString::new(LOCK_DIR_PERMS)?;
+            unsafe {
+                nix::libc::chmod(unix_path.as_ptr(), 777);
+            }
+            Ok(())
+        }
+    }
+
+    pub fn remove_lock() {
+        let _ = remove_file(LOCK_PATH); // Ignore result
+    }
+}
+
+impl Drop for FileLock {
+    fn drop(&mut self) {
+        Self::remove_lock();
+    }
+}