Add Rust source tree to the main repo.

Replaces the previously separate repo, merging the Rust
system into the mainline ready for 1.4.

The Rust system currently provides:

* lqos_bus: type definitions for localhost control bus.
* lqos_config: handler for configuration files.
* lqos_node_manager: local web-based monitor and manager.
* lqos_sys: eBPF program that handles all of the tracking,
  CPU assignment, VLAN redirect, TC queue assignment,
  and RTT tracking. Wrapped in a Rust core, providing
  consistent in-Rust building and mostly-safe wrappers
  to C code.
* lqosd: a daemon designed to run continually while
  LibreQoS is operating. On start, it parses the configuration
  files and sets up interface mapping (the Python code is still
  required to actually build queues). It then assigns the various
  eBPF programs to appropriate interfaces. The eBPF systems are
  removed when the daemon terminates.

  lqosd also provides a "bus", listening to requests for changes
  or information on localhost, providing a control plane for
  the rest of the project.
* lqtop: An example program demonstrating how to use the bus,
  acts like "top", showing current network traffic and mappings.
* xdp_iphash_to_cpu_cmdline: a Rust wrapper providing the same
  services as the cpumap originated tool of the same name. This is
  a "shim" - it will go away once the native Python library is
  ready.
* xdp_pping: also a shim, providing equivalent services to the
  cpumap service of the same name.

A helper shell script "remove_pinned_maps.sh" can be used to
remove all pinned eBPF maps from the system, allowing for eBPF
program upgrades that change persistent map structures without
a reboot.

Signed-off-by: Herbert Wolverson <herberticus@gmail.com>

src/rust/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "lqos_rs"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+
+[profile.release]
+strip = "debuginfo"
+lto = "thin"
+
+[workspace]
+members = [
+    "lqos_sys", # System support for handling the XDP component
+    "lqos_config", # Configuration support
+    "lqosd", # LibreQoS Daemon
+    "lqos_bus", # Bus data types
+    "lqtop", # A command line utility to show current activity
+    "xdp_iphash_to_cpu_cmdline", # Rust port of the C xdp_iphash_to_cpu_cmdline tool, for compatibility
+    "xdp_pping", # Rust port of cpumap's `xdp_pping` tool, for compatibility
+    "lqos_node_manager", # A lightweight web interface for management and local monitoring
+]

src/rust/README.md

@@ -0,0 +1,32 @@
+# Rust Management System for LibreQoS
+
+> Very much a work in progress. Details will be filled out as it stabilizes.
+
+## Sub Projects
+
+This project contains a number of projects arranged in a workspace. The projects are:
+
+* `lqos_sys` - a library that builds, installs, removes and manages the LibreQoS XDP and TC programs.
+* `lqos_bus` - definitions and helper functions for passing data across the local management bus.
+* `lqos_config` - a crate that handles pulling configuration from the Python manager.
+* `lqosd` - the management daemon that should eventually be run as a `systemd` service.
+    * When started, the daemon sets up XDP/TC eBPF programs for the interfaces specified in the LibreQoS configuration.
+    * When exiting, all eBPF programs are unloaded.
+    * Listens for bus commands and applies them.
+* `lqtop` - A CLI tool that outputs the top X downloaders and mostly verifies that the bus and daemons work.
+* `xdp_iphash_to_cpu_cmdline` - An almost-compatible command that acts like the tool of the same name from the previous verion.
+* `xdp_pping` - Port of the previous release's `xdp_pping` tool, for compatibility. Will eventually not be needed.
+
+## Required Ubuntu packages
+
+* `clang`
+* `linux-tools-common` (for `bpftool`)
+* `libbpf-dev`
+* `gcc-multilib`
+* `llvm`
+* `pkg-config`
+* `linux-tools-5.15.0-56-generic` (the common version doesn't work?)
+
+## Helper Scripts
+
+* `remove_pinned_maps.sh` deletes all of the BPF shared maps. Useful during development.

src/rust/lqos_bus/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "lqos_bus"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+default = ["equinix_tests"]
+equinix_tests = []
+
+[dependencies]
+serde = { version = "1.0", features = ["derive"] }
+bincode = "1"
+anyhow = "1"
+
+[build-dependencies]
+cc = "1.0"

src/rust/lqos_bus/build.rs

@@ -0,0 +1,5 @@
+fn main() {
+    cc::Build::new()
+        .file("src/tc_handle_parser.c")
+        .compile("tc_handle_parse.o");
+}

src/rust/lqos_bus/src/ip_stats.rs

@@ -0,0 +1,29 @@
+use serde::{Deserialize, Serialize};
+use crate::TcHandle;
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct IpStats {
+    pub ip_address: String,
+    pub bits_per_second: (u64, u64),
+    pub packets_per_second: (u64, u64),
+    pub median_tcp_rtt: f32,
+    pub tc_handle: TcHandle,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct IpMapping {
+    pub ip_address: String,
+    pub prefix_length: u32,
+    pub tc_handle: TcHandle,
+    pub cpu: u32,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct XdpPpingResult {
+    pub tc: String,
+    pub avg: f32,
+    pub min: f32,
+    pub max: f32,
+    pub median: f32,
+    pub samples: u32,
+}

src/rust/lqos_bus/src/lib.rs

@@ -0,0 +1,88 @@
+mod ip_stats;
+use anyhow::Result;
+pub use ip_stats::{IpMapping, IpStats, XdpPpingResult};
+use serde::{Deserialize, Serialize};
+mod tc_handle;
+pub use tc_handle::TcHandle;
+
+pub const BUS_BIND_ADDRESS: &str = "127.0.0.1:9999";
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct BusSession {
+    pub auth_cookie: u32,
+    pub requests: Vec<BusRequest>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub enum BusRequest {
+    Ping, // A generic "is it alive" test
+    GetCurrentThroughput,
+    GetTopNDownloaders(u32),
+    GetWorstRtt(u32),
+    MapIpToFlow {
+        ip_address: String,
+        tc_handle: TcHandle,
+        cpu: u32,
+        upload: bool,
+    },
+    DelIpFlow {
+        ip_address: String,
+        upload: bool,
+    },
+    ClearIpFlow,
+    ListIpFlow,
+    XdpPping,
+    RttHistogram,
+    HostCounts,
+    AllUnknownIps,
+    ReloadLibreQoS,
+    GetRawQueueData(String), // The string is the circuit ID
+    #[cfg(feature = "equinix_tests")]
+    RequestLqosEquinixTest, // TODO: Feature flag this so it doesn't go into production
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct BusReply {
+    pub auth_cookie: u32,
+    pub responses: Vec<BusResponse>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub enum BusResponse {
+    Ack,          // Yes, we're alive
+    Fail(String), // The operation failed
+    CurrentThroughput {
+        bits_per_second: (u64, u64),
+        packets_per_second: (u64, u64),
+        shaped_bits_per_second: (u64, u64),
+    },
+    TopDownloaders(Vec<IpStats>),
+    WorstRtt(Vec<IpStats>),
+    MappedIps(Vec<IpMapping>),
+    XdpPping(Vec<XdpPpingResult>),
+    RttHistogram(Vec<u32>),
+    HostCounts((u32, u32)),
+    AllUnknownIps(Vec<IpStats>),
+    ReloadLibreQoS(String),
+    RawQueueData(String),
+}
+
+pub fn encode_request(request: &BusSession) -> Result<Vec<u8>> {
+    Ok(bincode::serialize(request)?)
+}
+
+pub fn decode_request(bytes: &[u8]) -> Result<BusSession> {
+    Ok(bincode::deserialize(&bytes)?)
+}
+
+pub fn encode_response(request: &BusReply) -> Result<Vec<u8>> {
+    Ok(bincode::serialize(request)?)
+}
+
+pub fn decode_response(bytes: &[u8]) -> Result<BusReply> {
+    Ok(bincode::deserialize(&bytes)?)
+}
+
+pub fn cookie_value() -> u32 {
+    1234
+}

src/rust/lqos_bus/src/tc_handle.rs

@@ -0,0 +1,131 @@
+use std::ffi::CString;
+use anyhow::{Result, Error};
+use serde::{Serialize, Deserialize};
+
+/// Provides consistent handling of TC handle types.
+#[derive(Copy, Clone, Serialize, Deserialize, Debug, Default)]
+pub struct TcHandle(u32);
+
+#[allow(non_camel_case_types)]
+type __u32 = ::std::os::raw::c_uint;
+#[allow(dead_code)]
+const TC_H_ROOT: u32 = 4294967295;
+#[allow(dead_code)]
+const TC_H_UNSPEC: u32 = 0;
+
+extern "C" {
+    pub fn get_tc_classid(
+        h: *mut __u32,
+        str_: *const ::std::os::raw::c_char,
+    ) -> ::std::os::raw::c_int;
+}
+
+impl TcHandle {
+    #[inline(always)]
+    pub fn get_major_minor(&self) -> (u16, u16) {
+        // According to xdp_pping.c handles are minor:major u16s inside
+        // a u32.
+        (
+            (self.0 >> 16) as u16,
+            (self.0 & 0xFFFF) as u16,
+        )
+    }
+
+    pub fn from_string<S:ToString>(handle: S) -> Result<Self> {
+        let mut tc_handle : __u32 = 0;
+        let str = CString::new(handle.to_string())?;
+        let handle_pointer : *mut __u32 = &mut tc_handle;
+        let result = unsafe {
+            get_tc_classid(handle_pointer, str.as_ptr())
+        };
+        if result != 0 {
+            Err(Error::msg("Unable to parse TC handle string"))
+        } else {
+            Ok(Self(tc_handle))
+        }
+    }
+
+    pub fn from_u32(tc: u32) -> Self {
+        Self(tc)
+    }
+
+    pub fn as_u32(&self) -> u32 {
+        self.0
+    }
+
+    pub fn zero() -> Self {
+        Self(0)
+    }
+}
+
+impl ToString for TcHandle {
+    fn to_string(&self) -> String {
+        let (major, minor) = self.get_major_minor();
+        format!("{major:x}:{minor:x}")
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn make_root() {
+        let tc = TcHandle::from_string("root").unwrap();
+        assert_eq!(tc.0, TC_H_ROOT);
+    }
+
+    #[test]
+    fn make_unspecified() {
+        let tc = TcHandle::from_string("none").unwrap();
+        assert_eq!(tc.0, TC_H_UNSPEC);
+    }
+
+    #[test]
+    fn test_invalid() {
+        let tc = TcHandle::from_string("not_a_number");
+        assert!(tc.is_err());
+    }
+
+    #[test]
+    fn oversize_major() {
+        let tc = TcHandle::from_string("65540:0");
+        assert!(tc.is_err());
+    }
+
+    #[test]
+    fn oversize_minor() {
+        let tc = TcHandle::from_string("0:65540");
+        assert!(tc.is_err());
+    }
+
+    #[test]
+    fn zero() {
+        let tc = TcHandle::from_string("0:0").unwrap();
+        assert_eq!(tc.0, 0);
+    }
+
+    #[test]
+    fn roundtrip() {
+        let tc = TcHandle::from_string("1:2").unwrap();
+        assert_eq!(tc.to_string(), "1:2");
+    }
+
+    #[test]
+    fn hex() {
+        let tc = TcHandle::from_string("7FFF:2").unwrap();
+        assert_eq!(tc.to_string().to_uppercase(), "7FFF:2");
+    }
+
+    #[test]
+    fn roundtrip_extreme() {
+        for major in 0 .. 2000 {
+            for minor in 0..2000 {
+                let handle = format!("{major:x}:{minor:x}");
+                let tc = TcHandle::from_string(&handle).unwrap();
+                assert_eq!(tc.to_string(), handle);
+            }
+        }
+        
+    }
+}

src/rust/lqos_bus/src/tc_handle_parser.c

@@ -0,0 +1,46 @@
+// Imported from https://github.com/thebracket/cpumap-pping/blob/master/src/xdp_iphash_to_cpu_cmdline.c
+// Because it uses strtoul and is based on the TC source, including it directly
+// seemed like the path of least resistance.
+
+#include <stdlib.h>
+#include <stdbool.h>
+#include <string.h>
+#include <linux/types.h>
+#include <linux/pkt_sched.h> /* TC macros */
+
+/* Handle classid parsing based on iproute source */
+int get_tc_classid(__u32 *h, const char *str)
+{
+	__u32 major, minor;
+	char *p;
+
+	major = TC_H_ROOT;
+	if (strcmp(str, "root") == 0)
+		goto ok;
+	major = TC_H_UNSPEC;
+	if (strcmp(str, "none") == 0)
+		goto ok;
+	major = strtoul(str, &p, 16);
+	if (p == str) {
+		major = 0;
+		if (*p != ':')
+			return -1;
+	}
+	if (*p == ':') {
+		if (major >= (1<<16))
+			return -1;
+		major <<= 16;
+		str = p+1;
+		minor = strtoul(str, &p, 16);
+		if (*p != 0)
+			return -1;
+		if (minor >= (1<<16))
+			return -1;
+		major |= minor;
+	} else if (*p != 0)
+		return -1;
+
+ok:
+	*h = major;
+	return 0;
+}

src/rust/lqos_config/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "lqos_config"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow = "1"
+toml = "0.5"
+serde = { version = "1.0", features = [ "derive" ] }
+csv = "1"
+ip_network_table = "0"
+ip_network = "0"

src/rust/lqos_config/README.md

@@ -0,0 +1,15 @@
+# LQosConfig
+
+`lqos_config` is designed to manage configuration of LibreQoS.
+
+Since all of the parts of the system need to know where to find LibreQoS, it first looks for a file named `/etc/lqos` and uses that to locate the LibreQoS installation.
+
+`/etc/lqos` looks like this:
+
+```toml
+lqos_directory = '/opt/libreqos'
+```
+
+The entries are:
+
+* `lqos_directory`: where LibreQoS is installed (e.g. `/opt/libreqos`)

src/rust/lqos_config/src/etc.rs

@@ -0,0 +1,55 @@
+use std::path::Path;
+use serde::Deserialize;
+use anyhow::{Result, Error};
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct EtcLqos {
+    pub lqos_directory: String,
+    pub bridge: Option<BridgeConfig>,
+    pub tuning: Option<Tunables>,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct Tunables {
+    pub stop_irq_balance: bool,
+    pub netdev_budget_usecs: u32,
+    pub netdev_budget_packets: u32,
+    pub rx_usecs: u32,
+    pub tx_usecs: u32,
+    pub disable_rxvlan: bool,
+    pub disable_txvlan: bool,
+    pub disable_offload: Vec<String>,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct BridgeConfig {
+    pub use_kernel_bridge: bool,
+    pub interface_mapping: Vec<BridgeInterface>,
+    pub vlan_mapping: Vec<BridgeVlan>,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct BridgeInterface {
+    pub name: String,
+    pub scan_vlans: bool,
+    pub redirect_to: String,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct BridgeVlan {
+    pub parent: String,
+    pub tag: u32,
+    pub redirect_to: u32,
+}
+
+impl EtcLqos {
+    pub fn load() -> Result<Self> {
+        if !Path::new("/etc/lqos").exists() {
+            return Err(Error::msg("You must setup /etc/lqos"));
+        }
+        let raw = std::fs::read_to_string("/etc/lqos")?;
+        let config: Self = toml::from_str(&raw)?;
+        //println!("{:?}", config);
+        Ok(config)
+    }
+}

src/rust/lqos_config/src/lib.rs

@@ -0,0 +1,9 @@
+mod etc;
+mod libre_qos_config;
+mod shaped_devices;
+mod program_control;
+
+pub use libre_qos_config::LibreQoSConfig;
+pub use shaped_devices::{ConfigShapedDevices, ShapedDevice};
+pub use program_control::load_libreqos;
+pub use etc::{EtcLqos, BridgeConfig, Tunables, BridgeInterface, BridgeVlan};

src/rust/lqos_config/src/libre_qos_config.rs

@@ -0,0 +1,74 @@
+use anyhow::{Error, Result};
+use std::{fs, path::{Path, PathBuf}};
+use crate::etc;
+
+pub struct LibreQoSConfig {
+    pub internet_interface: String,
+    pub isp_interface: String,
+    pub on_a_stick_mode: bool,
+    pub stick_vlans: (u16, u16),
+}
+
+impl LibreQoSConfig {
+    pub fn load() -> Result<Self> {
+        let cfg = etc::EtcLqos::load()?;
+        let base_path = Path::new(&cfg.lqos_directory);
+        let final_path = base_path.join("ispConfig.py");
+        Ok(Self::load_from_path(&final_path)?)
+    }
+
+    fn load_from_path(path: &PathBuf) -> Result<Self> {
+        let path = Path::new(path);
+        if !path.exists() {
+            return Err(Error::msg("Unable to find ispConfig.py"));
+        }
+
+        // Read the config
+        let mut result = Self {
+            internet_interface: String::new(),
+            isp_interface: String::new(),
+            on_a_stick_mode: false,
+            stick_vlans: (0,0),
+        };
+        result.parse_isp_config(path)?;
+        Ok(result)
+    }
+
+    fn parse_isp_config(&mut self, path: &Path) -> Result<()> {
+        let content = fs::read_to_string(path)?;
+        for line in content.split("\n") {
+            if line.starts_with("interfaceA") {
+                self.isp_interface = split_at_equals(line);
+            }
+            if line.starts_with("interfaceB") {
+                self.internet_interface = split_at_equals(line);
+            }
+            if line.starts_with("OnAStick") {
+                let mode = split_at_equals(line);
+                if mode == "True" {
+                    self.on_a_stick_mode = true;
+                }
+            }
+            if line.starts_with("StickVlanA") {
+                let vlan_string = split_at_equals(line);
+                let vlan : u16 = vlan_string.parse()?;
+                self.stick_vlans.0 = vlan;
+            }
+            if line.starts_with("StickVlanB") {
+                let vlan_string = split_at_equals(line);
+                let vlan : u16 = vlan_string.parse()?;
+                self.stick_vlans.1 = vlan;
+            }
+        }
+        Ok(())
+    }
+}
+
+fn split_at_equals(line: &str) -> String {
+    line.split('=')
+        .nth(1)
+        .unwrap_or("")
+        .trim()
+        .replace("\"", "")
+        .replace("'", "")
+}

src/rust/lqos_config/src/program_control.rs

@@ -0,0 +1,35 @@
+use std::{path::{PathBuf, Path}, process::Command};
+use anyhow::{Result, Error};
+use crate::etc;
+
+const PYTHON_PATH: &str = "/usr/bin/python3";
+
+fn path_to_libreqos() -> Result<PathBuf> {
+    let cfg = etc::EtcLqos::load()?;
+    let base_path = Path::new(&cfg.lqos_directory);
+    Ok(base_path.join("LibreQoS.py"))
+}
+
+fn working_directory() -> Result<PathBuf> {
+    let cfg = etc::EtcLqos::load()?;
+    let base_path = Path::new(&cfg.lqos_directory);
+    Ok(base_path.to_path_buf())
+}
+
+pub fn load_libreqos() -> Result<String> {
+    let path = path_to_libreqos()?;
+    if !path.exists() {
+        return Err(Error::msg("LibreQoS.py not found"));
+    }
+    if !Path::new(PYTHON_PATH).exists() {
+        return Err(Error::msg("Python not found"));
+    }
+
+    let result = Command::new(PYTHON_PATH)
+        .current_dir(working_directory()?)
+        .arg("LibreQoS.py")
+        .output()?;
+    let stdout = String::from_utf8(result.stdout)?;
+    let stderr = String::from_utf8(result.stderr)?;
+    Ok(stdout + &stderr)
+}

src/rust/lqos_config/src/shaped_devices/mod.rs

@@ -0,0 +1,200 @@
+mod shaped_device;
+mod serializable;
+use csv::{WriterBuilder, QuoteStyle};
+pub use shaped_device::ShapedDevice;
+use std::{path::{Path, PathBuf}};
+use anyhow::Result;
+use crate::etc;
+use serializable::SerializableShapedDevice;
+
+pub struct ConfigShapedDevices {
+    pub devices: Vec<ShapedDevice>,
+    pub trie: ip_network_table::IpNetworkTable<usize>,
+}
+
+impl ConfigShapedDevices {
+    pub fn path() -> Result<PathBuf> {
+        let cfg = etc::EtcLqos::load()?;
+        let base_path = Path::new(&cfg.lqos_directory);
+        Ok(base_path.join("ShapedDevices.csv"))
+    }
+
+    pub fn load() -> Result<Self> {
+        let final_path = ConfigShapedDevices::path()?;
+        let mut reader = csv::Reader::from_path(final_path)?;
+        
+        // Example: StringRecord(["1", "968 Circle St., Gurnee, IL 60031", "1", "Device 1", "", "", "192.168.101.2", "", "25", "5", "10000", "10000", ""])
+        let mut devices = Vec::new();
+        for result in reader.records() {
+            if let Ok(result) = result {
+                if let Ok(device) = ShapedDevice::from_csv(&result) {
+                    devices.push(device);
+                }
+            }
+        }
+        let trie = ConfigShapedDevices::make_trie(&devices);
+        Ok(Self{ devices, trie })
+    }
+
+    fn make_trie(devices: &[ShapedDevice]) -> ip_network_table::IpNetworkTable<usize> {
+        use ip_network::IpNetwork;
+        let mut table = ip_network_table::IpNetworkTable::new();
+        devices
+            .iter()
+            .enumerate()
+            .map(|(i,d)| { (i, d.to_ipv6_list()) })
+            .for_each(|(id, ips)| {
+                ips.iter().for_each(|(ip, cidr)| {
+                    if let Ok(net) = IpNetwork::new(*ip, (*cidr) as u8) {
+                        table.insert(net, id);
+                    }
+                });
+            });
+        table
+    }
+
+    fn to_csv_string(&self) -> Result<String> {
+        let mut writer = WriterBuilder::new().quote_style(QuoteStyle::NonNumeric).from_writer(vec![]);
+        for d in self.devices
+            .iter()
+            .map(|d| SerializableShapedDevice::from(d))
+            {
+                writer.serialize(d)?;
+            };
+
+        let data = String::from_utf8(writer.into_inner()?)?;
+        Ok(data)
+    }
+
+    pub fn write_csv(&self, filename: &str) -> Result<()> {
+        let cfg = etc::EtcLqos::load()?;
+        let base_path = Path::new(&cfg.lqos_directory);
+        let path = base_path.join(filename);
+        let csv = self.to_csv_string()?;
+        std::fs::write(path, csv)?;
+        Ok(())
+    }
+}
+
+
+#[cfg(test)]
+mod test {
+    use std::net::{Ipv4Addr, Ipv6Addr};
+
+    use super::*;
+
+    #[test]
+    fn test_simple_ipv4_parse() {
+        let (ip, cidr) = ShapedDevice::parse_cidr_v4("1.2.3.4").unwrap();
+        assert_eq!(cidr, 32);
+        assert_eq!("1.2.3.4".parse::<Ipv4Addr>().unwrap(), ip);
+    }
+
+    #[test]
+    fn test_cidr_ipv4_parse() {
+        let (ip, cidr) = ShapedDevice::parse_cidr_v4("1.2.3.4/24").unwrap();
+        assert_eq!(cidr, 24);
+        assert_eq!("1.2.3.4".parse::<Ipv4Addr>().unwrap(), ip);
+    }
+
+    #[test]
+    fn test_bad_ipv4_parse() {
+        let r = ShapedDevice::parse_cidr_v4("bad wolf");
+        assert!(r.is_err());
+    }
+
+    #[test]
+    fn test_nearly_ok_ipv4_parse() {
+        let r = ShapedDevice::parse_cidr_v4("192.168.1.256/32");
+        assert!(r.is_err());
+    }
+
+    #[test]
+    fn test_single_ipv4() {
+        let r = ShapedDevice::parse_ipv4("1.2.3.4");
+        assert_eq!(r.len(), 1);
+        assert_eq!(r[0].0, "1.2.3.4".parse::<Ipv4Addr>().unwrap());
+        assert_eq!(r[0].1, 32);
+    }
+
+    #[test]
+    fn test_two_ipv4() {
+        let r = ShapedDevice::parse_ipv4("1.2.3.4, 1.2.3.4/24");
+        assert_eq!(r.len(), 2);
+        assert_eq!(r[0].0, "1.2.3.4".parse::<Ipv4Addr>().unwrap());
+        assert_eq!(r[0].1, 32);
+        assert_eq!(r[1].0, "1.2.3.4".parse::<Ipv4Addr>().unwrap());
+        assert_eq!(r[1].1, 24);
+    }
+
+    #[test]
+    fn test_simple_ipv6_parse() {
+        let (ip, cidr) = ShapedDevice::parse_cidr_v6("fd77::1:5").unwrap();
+        assert_eq!(cidr, 128);
+        assert_eq!("fd77::1:5".parse::<Ipv6Addr>().unwrap(), ip);
+    }
+
+    #[test]
+    fn test_cidr_ipv6_parse() {
+        let (ip, cidr) = ShapedDevice::parse_cidr_v6("fd77::1:5/64").unwrap();
+        assert_eq!(cidr, 64);
+        assert_eq!("fd77::1:5".parse::<Ipv6Addr>().unwrap(), ip);
+    }
+
+    #[test]
+    fn test_bad_ipv6_parse() {
+        let r = ShapedDevice::parse_cidr_v6("bad wolf");
+        assert!(r.is_err());
+    }
+
+    #[test]
+    fn test_nearly_ok_ipv6_parse() {
+        let r = ShapedDevice::parse_cidr_v6("fd77::1::5");
+        assert!(r.is_err());
+    }
+
+    #[test]
+    fn test_single_ipv6() {
+        let r = ShapedDevice::parse_ipv6("fd77::1:5");
+        assert_eq!(r.len(), 1);
+        assert_eq!(r[0].0, "fd77::1:5".parse::<Ipv6Addr>().unwrap());
+        assert_eq!(r[0].1, 128);
+    }
+
+    #[test]
+    fn test_two_ipv6() {
+        let r = ShapedDevice::parse_ipv6("fd77::1:5, fd77::1:5/64");
+        assert_eq!(r.len(), 2);
+        assert_eq!(r[0].0, "fd77::1:5".parse::<Ipv6Addr>().unwrap());
+        assert_eq!(r[0].1, 128);
+        assert_eq!(r[1].0, "fd77::1:5".parse::<Ipv6Addr>().unwrap());
+        assert_eq!(r[1].1, 64);
+    }
+
+    #[test]
+    fn build_and_test_simple_trie() {
+        let devices = vec![
+            ShapedDevice{
+                circuit_id: "One".to_string(),
+                ipv4: ShapedDevice::parse_ipv4("192.168.1.0/24"),
+                ..Default::default()
+            },
+            ShapedDevice{
+                circuit_id: "One".to_string(),
+                ipv4: ShapedDevice::parse_ipv4("1.2.3.4"),
+                ..Default::default()
+            },
+        ];
+        let trie = ConfigShapedDevices::make_trie(&devices);
+        assert_eq!(trie.len(), (0, 2));
+        assert!(trie.longest_match(ShapedDevice::parse_cidr_v4("192.168.2.2").unwrap().0).is_none());
+        
+        let addr: Ipv4Addr = "192.168.1.2".parse().unwrap();
+        let v6 = addr.to_ipv6_mapped();
+        assert!(trie.longest_match(v6).is_some());
+
+        let addr: Ipv4Addr = "1.2.3.4".parse().unwrap();
+        let v6 = addr.to_ipv6_mapped();
+        assert!(trie.longest_match(v6).is_some());
+    }
+}

src/rust/lqos_config/src/shaped_devices/serializable.rs

@@ -0,0 +1,88 @@
+
+use std::net::{Ipv4Addr, Ipv6Addr};
+use serde::Serialize;
+use crate::ShapedDevice;
+
+// Example: StringRecord(["1", "968 Circle St., Gurnee, IL 60031", "1", "Device 1", "", "", "192.168.101.2", "", "25", "5", "10000", "10000", ""])
+#[derive(Serialize)]
+pub(crate) struct SerializableShapedDevice {
+    pub circuit_id: String,
+    pub circuit_name: String,
+    pub device_id: String,
+    pub device_name: String,
+    pub parent_node: String,
+    pub mac: String,
+    pub ipv4: String,
+    pub ipv6: String,
+    pub download_min_mbps: u32,
+    pub upload_min_mbps: u32,
+    pub download_max_mbps: u32,
+    pub upload_max_mbps: u32,
+    pub comment: String,
+}
+
+impl From<&ShapedDevice> for SerializableShapedDevice {
+    fn from(d: &ShapedDevice) -> Self {
+        Self {
+            circuit_id: d.circuit_id.clone(),
+            circuit_name: d.circuit_name.clone(),
+            device_id: d.device_id.clone(),
+            device_name: d.device_name.clone(),
+            parent_node: d.parent_node.clone(),
+            mac: d.mac.clone(),
+            ipv4: ipv4_list_to_string(&d.ipv4),
+            ipv6: ipv6_list_to_string(&d.ipv6),
+            download_min_mbps: d.download_min_mbps,
+            upload_min_mbps: d.upload_min_mbps,
+            download_max_mbps: d.download_max_mbps,
+            upload_max_mbps: d.upload_max_mbps,
+            comment: d.comment.clone()
+        }
+    }
+}
+
+fn ipv4_to_string(ip: &(Ipv4Addr, u32)) -> String {
+    if ip.1 == 32 {
+        format!("{}", ip.0)
+    } else {
+        format!{"{}/{}", ip.0, ip.1}
+    }
+}
+
+fn ipv4_list_to_string(ips: &[(Ipv4Addr, u32)]) -> String {
+    if ips.len() == 0 {
+        return String::new();
+    }
+    if ips.len() == 1 {
+        return ipv4_to_string(&ips[0]);
+    }
+    let mut buffer = String::new();
+    for i in 0..ips.len()-1 {
+        buffer += &format!("{}, ", ipv4_to_string(&ips[i]));
+    }
+    buffer += &ipv4_to_string(&ips[ips.len()-1]);
+    String::new()
+}
+
+fn ipv6_to_string(ip: &(Ipv6Addr, u32)) -> String {
+    if ip.1 == 32 {
+        format!("{}", ip.0)
+    } else {
+        format!{"{}/{}", ip.0, ip.1}
+    }
+}
+
+fn ipv6_list_to_string(ips: &[(Ipv6Addr, u32)]) -> String {
+    if ips.len() == 0 {
+        return String::new();
+    }
+    if ips.len() == 1 {
+        return ipv6_to_string(&ips[0]);
+    }
+    let mut buffer = String::new();
+    for i in 0..ips.len()-1 {
+        buffer += &format!("{}, ", ipv6_to_string(&ips[i]));
+    }
+    buffer += &ipv6_to_string(&ips[ips.len()-1]);
+    String::new()
+}

src/rust/lqos_config/src/shaped_devices/shaped_device.rs

@@ -0,0 +1,151 @@
+use std::net::{Ipv4Addr, Ipv6Addr};
+use anyhow::{Result, Error};
+use csv::StringRecord;
+use serde::{Serialize, Deserialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct ShapedDevice {
+    // Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment
+    pub circuit_id: String,
+    pub circuit_name: String,
+    pub device_id: String,
+    pub device_name: String,
+    pub parent_node: String,
+    pub mac: String,
+    pub ipv4: Vec<(Ipv4Addr, u32)>,
+    pub ipv6: Vec<(Ipv6Addr, u32)>,
+    pub download_min_mbps: u32,
+    pub upload_min_mbps: u32,
+    pub download_max_mbps: u32,
+    pub upload_max_mbps: u32,
+    pub comment: String,
+}
+
+impl Default for ShapedDevice {
+    fn default() -> Self {
+        Self {
+            circuit_id: String::new(),
+            circuit_name: String::new(),
+            device_id: String::new(),
+            device_name: String::new(),
+            parent_node: String::new(),
+            mac: String::new(),
+            ipv4: Vec::new(),
+            ipv6: Vec::new(),
+            download_min_mbps: 0,
+            download_max_mbps: 0,
+            upload_min_mbps: 0,
+            upload_max_mbps: 0,
+            comment: String::new(),
+        }
+    }
+}
+
+impl ShapedDevice {
+    pub(crate) fn from_csv(record: &StringRecord) -> Result<Self> {
+        Ok(Self {
+            circuit_id: record[0].to_string(),
+            circuit_name: record[1].to_string(),
+            device_id: record[2].to_string(),
+            device_name: record[3].to_string(),
+            parent_node: record[4].to_string(),
+            mac: record[5].to_string(),
+            ipv4: ShapedDevice::parse_ipv4(&record[6]),
+            ipv6: ShapedDevice::parse_ipv6(&record[7]),
+            download_min_mbps: record[8].parse()?,
+            upload_min_mbps: record[9].parse()?,
+            download_max_mbps: record[10].parse()?,
+            upload_max_mbps: record[11].parse()?,
+            comment: record[12].to_string(),
+        })
+    }
+
+    pub(crate) fn parse_cidr_v4(address: &str) -> Result<(Ipv4Addr, u32)> {
+        if address.contains("/") {
+            let split : Vec<&str> = address.split("/").collect();
+            if split.len() != 2 {
+                return Err(Error::msg("Unable to parse IPv4"));
+            }
+            return Ok((
+                split[0].parse()?,
+                split[1].parse()?
+            ))
+        } else {
+            return Ok((
+                address.parse()?,
+                32
+            ));
+        }
+    }
+
+    pub(crate) fn parse_ipv4(str: &str) -> Vec<(Ipv4Addr, u32)> {
+        let mut result = Vec::new();
+        if str.contains(",") {
+            for ip in str.split(",") {
+                let ip = ip.trim();
+                if let Ok((ipv4, subnet)) = ShapedDevice::parse_cidr_v4(ip) {
+                    result.push((ipv4, subnet));
+                }
+            }
+        } else {
+            // No Commas
+            if let Ok((ipv4, subnet)) = ShapedDevice::parse_cidr_v4(str) {
+                result.push((ipv4, subnet));
+            }
+        }
+
+        result
+    }
+
+    pub(crate) fn parse_cidr_v6(address: &str) -> Result<(Ipv6Addr, u32)> {
+        if address.contains("/") {
+            let split : Vec<&str> = address.split("/").collect();
+            if split.len() != 2 {
+                return Err(Error::msg("Unable to parse IPv6"));
+            }
+            return Ok((
+                split[0].parse()?,
+                split[1].parse()?
+            ))
+        } else {
+            return Ok((
+                address.parse()?,
+                128
+            ));
+        }
+    }
+
+    pub(crate) fn parse_ipv6(str: &str) -> Vec<(Ipv6Addr, u32)> {
+        let mut result = Vec::new();
+        if str.contains(",") {
+            for ip in str.split(",") {
+                let ip = ip.trim();
+                if let Ok((ipv6, subnet)) = ShapedDevice::parse_cidr_v6(ip) {
+                    result.push((ipv6, subnet));
+                }
+            }
+        } else {
+            // No Commas
+            if let Ok((ipv6, subnet)) = ShapedDevice::parse_cidr_v6(str) {
+                result.push((ipv6, subnet));
+            }
+        }
+
+        result
+    }
+
+    pub(crate) fn to_ipv6_list(&self) -> Vec<(Ipv6Addr, u32)> {
+        let mut result = Vec::new();
+
+        for (ipv4, cidr) in &self.ipv4 {
+            result.push((
+                ipv4.to_ipv6_mapped(),
+                cidr + 96
+            ));
+        }
+        result.extend_from_slice(&self.ipv6);
+
+        result
+    }
+}
+

src/rust/lqos_node_manager/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "lqos_node_manager"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+default = ["equinix_tests"]
+equinix_tests = []
+
+[dependencies]
+rocket = { version = "0.5.0-rc.2", features = [ "json", "msgpack", "uuid" ] }
+rocket_async_compression = "0.2.0"
+lazy_static = "1.4"
+parking_lot = "0.12"
+lqos_bus = { path = "../lqos_bus" }
+lqos_config = { path = "../lqos_config" }
+anyhow = "1"
+sysinfo = "0"
+notify = { version = "5.0.0", default-features = false, feature=["macos_kqueue"] } # Not using crossbeam because of Tokio

src/rust/lqos_node_manager/Rocket.toml

@@ -0,0 +1,3 @@
+[default]
+port = 9123
+address = "::"

src/rust/lqos_node_manager/src/cache_control.rs

@@ -0,0 +1,50 @@
+use rocket::http::Header;
+use rocket::response::Responder;
+
+/// Use to wrap a responder when you want to tell the user's
+/// browser to try and cache a response.
+/// 
+/// For example:
+/// 
+/// ```
+/// pub async fn bootsrap_css<'a>() -> LongCache<Option<NamedFile>> {
+///     LongCache::new(NamedFile::open("static/vendor/bootstrap.min.css").await.ok())
+/// }
+/// ```
+#[derive(Responder)]
+pub struct LongCache<T> {
+    inner: T,
+    my_header: Header<'static>,
+}
+impl<'r, 'o: 'r, T: Responder<'r, 'o>> LongCache<T> {
+    pub fn new(inner: T) -> Self {
+        Self {
+            inner,
+            my_header: Header::new("cache-control", "max-age=604800, public"),
+        }
+    }
+}
+
+/// Use to wrap a responder when you want to tell the user's
+/// browser to keep data private and never cahce it.
+/// 
+/// For example:
+/// 
+/// ```
+/// pub async fn bootsrap_css<'a>() -> LongCache<Option<NamedFile>> {
+///     LongCache::new(NamedFile::open("static/vendor/bootstrap.min.css").await.ok())
+/// }
+/// ```
+#[derive(Responder)]
+pub struct NoCache<T> {
+    inner: T,
+    my_header: Header<'static>,
+}
+impl<'r, 'o: 'r, T: Responder<'r, 'o>> NoCache<T> {
+    pub fn new(inner: T) -> Self {
+        Self {
+            inner,
+            my_header: Header::new("cache-control", "no-cache, private"),
+        }
+    }
+}

src/rust/lqos_node_manager/src/main.rs

@@ -0,0 +1,71 @@
+#[macro_use] extern crate rocket;
+use rocket::fairing::AdHoc;
+mod static_pages;
+mod tracker;
+mod shaped_devices;
+mod unknown_devices;
+mod cache_control;
+use rocket_async_compression::Compression;
+mod queue_info;
+
+#[launch]
+fn rocket() -> _ {
+    //tracker::SHAPED_DEVICES.read().write_csv("ShapedDeviceWriteTest.csv").unwrap();
+    let server = rocket::build()
+        .attach(AdHoc::on_liftoff("Poll lqosd", |_| {
+            Box::pin(async move {
+                rocket::tokio::spawn(tracker::update_tracking());
+            })
+        }))
+        .mount("/", routes![
+            static_pages::index,
+            static_pages::shaped_devices_csv_page,
+            static_pages::shaped_devices_add_page,
+            static_pages::unknown_devices_page,
+            static_pages::circuit_queue,
+
+            // Our JS library
+            static_pages::lqos_js,
+
+            // API calls
+            tracker::current_throughput,
+            tracker::throughput_ring,
+            tracker::cpu_usage,
+            tracker::ram_usage,
+            tracker::top_10_downloaders,
+            tracker::worst_10_rtt,
+            tracker::rtt_histogram,
+            tracker::host_counts,
+            shaped_devices::all_shaped_devices,
+            shaped_devices::shaped_devices_count,
+            shaped_devices::shaped_devices_range,
+            shaped_devices::shaped_devices_search,
+            shaped_devices::reload_required,
+            shaped_devices::reload_libreqos,
+            unknown_devices::all_unknown_devices,
+            unknown_devices::unknown_devices_count,
+            unknown_devices::unknown_devices_range,
+            queue_info::raw_queue_by_circuit,
+            queue_info::run_btest,
+
+            // Supporting files
+            static_pages::bootsrap_css,
+            static_pages::plotly_js,
+            static_pages::jquery_js,
+            static_pages::bootsrap_js,
+            static_pages::tinylogo,
+            static_pages::favicon,
+            static_pages::fontawesome_solid,
+            static_pages::fontawesome_webfont,
+            static_pages::fontawesome_woff,
+        ]
+    );
+    
+    // Compression is slow in debug builds,
+    // so only enable it on release builds.
+    if cfg!(debug_assertions) {
+        server
+    } else {
+        server.attach(Compression::fairing())
+    }
+}

src/rust/lqos_node_manager/src/queue_info.rs

@@ -0,0 +1,59 @@
+use lqos_bus::{BusResponse, BUS_BIND_ADDRESS, BusSession, BusRequest, encode_request, decode_response};
+use rocket::response::content::RawJson;
+use rocket::tokio::io::{AsyncWriteExt, AsyncReadExt};
+use rocket::tokio::net::TcpStream;
+use crate::cache_control::NoCache;
+
+#[get("/api/raw_queue_by_circuit/<circuit_id>")]
+pub async fn raw_queue_by_circuit(circuit_id: String) -> NoCache<RawJson<String>> {
+    let mut stream = TcpStream::connect(BUS_BIND_ADDRESS).await.unwrap();
+    let test = BusSession {
+        auth_cookie: 1234,
+        requests: vec![
+            BusRequest::GetRawQueueData(circuit_id),
+        ],
+    };
+    let msg = encode_request(&test).unwrap();
+    stream.write(&msg).await.unwrap();
+
+    // Receive reply
+    let mut buf = Vec::new();
+    let _ = stream.read_to_end(&mut buf).await.unwrap();
+    let reply = decode_response(&buf).unwrap();
+
+    let result = match &reply.responses[0] {
+        BusResponse::RawQueueData(msg) => msg.clone(),
+        _ => "Unable to request queue".to_string(),
+    };
+    NoCache::new(RawJson(result))
+}
+
+#[cfg(feature = "equinix_tests")]
+#[get("/api/run_btest")]
+pub async fn run_btest() -> NoCache<RawJson<String>> {
+    let mut stream = TcpStream::connect(BUS_BIND_ADDRESS).await.unwrap();
+    let test = BusSession {
+        auth_cookie: 1234,
+        requests: vec![
+            BusRequest::RequestLqosEquinixTest,
+        ],
+    };
+    let msg = encode_request(&test).unwrap();
+    stream.write(&msg).await.unwrap();
+
+    // Receive reply
+    let mut buf = Vec::new();
+    let _ = stream.read_to_end(&mut buf).await.unwrap();
+    let reply = decode_response(&buf).unwrap();
+
+    let result = match &reply.responses[0] {
+        BusResponse::Ack => String::new(),
+        _ => "Unable to request test".to_string(),
+    };
+    NoCache::new(RawJson(result))
+}
+
+#[cfg(not(feature = "equinix_tests"))]
+pub async fn run_btest() -> NoCache<RawJson<String>> {
+    NoCache::new(RawJson("No!"))
+}

src/rust/lqos_node_manager/src/shaped_devices.rs

@@ -0,0 +1,78 @@
+use lqos_bus::{BusResponse, BUS_BIND_ADDRESS, BusSession, BusRequest, encode_request, decode_response};
+use lqos_config::ShapedDevice;
+use rocket::serde::json::Json;
+use rocket::tokio::io::{AsyncWriteExt, AsyncReadExt};
+use rocket::tokio::net::TcpStream;
+use crate::cache_control::NoCache;
+use crate::tracker::SHAPED_DEVICES;
+use lazy_static::*;
+use parking_lot::RwLock;
+
+lazy_static! {
+    static ref RELOAD_REQUIRED : RwLock<bool> = RwLock::new(false);
+}
+
+#[get("/api/all_shaped_devices")]
+pub fn all_shaped_devices() -> NoCache<Json<Vec<ShapedDevice>>> {
+    NoCache::new(Json(SHAPED_DEVICES.read().devices.clone()))
+}
+
+#[get("/api/shaped_devices_count")]
+pub fn shaped_devices_count() -> NoCache<Json<usize>> {
+    NoCache::new(Json(SHAPED_DEVICES.read().devices.len()))
+}
+
+#[get("/api/shaped_devices_range/<start>/<end>")]
+pub fn shaped_devices_range(start: usize, end: usize) -> NoCache<Json<Vec<ShapedDevice>>> {
+    let reader = SHAPED_DEVICES.read();
+    let result: Vec<ShapedDevice> = reader.devices.iter().skip(start).take(end).cloned().collect();
+    NoCache::new(Json(result))
+}
+
+#[get("/api/shaped_devices_search/<term>")]
+pub fn shaped_devices_search(term: String) -> NoCache<Json<Vec<ShapedDevice>>> {
+    let term = term.trim().to_lowercase();
+    let reader = SHAPED_DEVICES.read();
+    let result: Vec<ShapedDevice> = reader
+        .devices
+        .iter()
+        .filter(|s| 
+            s.circuit_name.trim().to_lowercase().contains(&term) ||
+            s.device_name.trim().to_lowercase().contains(&term)
+        )
+        .cloned()
+        .collect();
+    NoCache::new(Json(result))
+}
+
+#[get("/api/reload_required")]
+pub fn reload_required() -> NoCache<Json<bool>> {
+    NoCache::new(Json(*RELOAD_REQUIRED.read()))
+}
+
+#[get("/api/reload_libreqos")]
+pub async fn reload_libreqos() -> NoCache<Json<String>> {
+    // Send request to lqosd
+    let mut stream = TcpStream::connect(BUS_BIND_ADDRESS).await.unwrap();
+    let test = BusSession {
+        auth_cookie: 1234,
+        requests: vec![
+            BusRequest::ReloadLibreQoS,
+        ],
+    };
+    let msg = encode_request(&test).unwrap();
+    stream.write(&msg).await.unwrap();
+
+    // Receive reply
+    let mut buf = Vec::new();
+    let _ = stream.read_to_end(&mut buf).await.unwrap();
+    let reply = decode_response(&buf).unwrap();
+
+    let result = match &reply.responses[0] {
+        BusResponse::ReloadLibreQoS(msg) => msg.clone(),
+        _ => "Unable to reload LibreQoS".to_string(),
+    };
+
+    *RELOAD_REQUIRED.write() = false;
+    NoCache::new(Json(result))
+}

src/rust/lqos_node_manager/src/static_pages.rs

@@ -0,0 +1,90 @@
+use rocket::fs::NamedFile;
+use crate::cache_control::{LongCache, NoCache};
+
+// Note that NoCache can be replaced with a cache option
+// once the design work is complete.
+#[get("/")]
+pub async fn index<'a>() -> NoCache<Option<NamedFile>> {
+    NoCache::new(NamedFile::open("static/main.html").await.ok())
+}
+
+// Note that NoCache can be replaced with a cache option
+// once the design work is complete.
+#[get("/shaped")]
+pub async fn shaped_devices_csv_page<'a>() -> NoCache<Option<NamedFile>> {
+    NoCache::new(NamedFile::open("static/shaped.html").await.ok())
+}
+
+// Note that NoCache can be replaced with a cache option
+// once the design work is complete.
+#[get("/circuit_queue")]
+pub async fn circuit_queue<'a>() -> NoCache<Option<NamedFile>> {
+    NoCache::new(NamedFile::open("static/circuit_queue.html").await.ok())
+}
+
+// Note that NoCache can be replaced with a cache option
+// once the design work is complete.
+#[get("/unknown")]
+pub async fn unknown_devices_page<'a>() -> NoCache<Option<NamedFile>> {
+    NoCache::new(NamedFile::open("static/unknown-ip.html").await.ok())
+}
+
+// Note that NoCache can be replaced with a cache option
+// once the design work is complete.
+#[get("/shaped-add")]
+pub async fn shaped_devices_add_page<'a>() -> NoCache<Option<NamedFile>> {
+    NoCache::new(NamedFile::open("static/shaped-add.html").await.ok())
+}
+
+#[get("/vendor/bootstrap.min.css")]
+pub async fn bootsrap_css<'a>() -> LongCache<Option<NamedFile>> {
+    LongCache::new(NamedFile::open("static/vendor/bootstrap.min.css").await.ok())
+}
+
+// Note that NoCache can be replaced with a cache option
+// once the design work is complete.
+#[get("/lqos.js")]
+pub async fn lqos_js<'a>() -> NoCache<Option<NamedFile>> {
+    NoCache::new(NamedFile::open("static/lqos.js").await.ok())
+}
+
+#[get("/vendor/plotly-2.16.1.min.js")]
+pub async fn plotly_js<'a>() -> LongCache<Option<NamedFile>> {
+    LongCache::new(NamedFile::open("static/vendor/plotly-2.16.1.min.js").await.ok())
+}
+
+#[get("/vendor/jquery.min.js")]
+pub async fn jquery_js<'a>() -> LongCache<Option<NamedFile>> {
+    LongCache::new(NamedFile::open("static/vendor/jquery.min.js").await.ok())
+}
+
+#[get("/vendor/bootstrap.bundle.min.js")]
+pub async fn bootsrap_js<'a>() -> LongCache<Option<NamedFile>> {
+    LongCache::new(NamedFile::open("static/vendor/bootstrap.bundle.min.js").await.ok())
+}
+
+#[get("/vendor/tinylogo.svg")]
+pub async fn tinylogo<'a>() -> LongCache<Option<NamedFile>> {
+    LongCache::new(NamedFile::open("static/tinylogo.svg").await.ok())
+}
+
+#[get("/favicon.ico")]
+pub async fn favicon<'a>() -> LongCache<Option<NamedFile>> {
+    LongCache::new(NamedFile::open("static/favicon.ico").await.ok())
+}
+
+/// FontAwesome icons
+#[get("/vendor/solid.min.css")]
+pub async fn fontawesome_solid<'a>() -> LongCache<Option<NamedFile>> {
+    LongCache::new(NamedFile::open("static/vendor/solid.min.css").await.ok())
+}
+
+#[get("/fonts/fontawesome-webfont.ttf")]
+pub async fn fontawesome_webfont<'a>() -> LongCache<Option<NamedFile>> {
+    LongCache::new(NamedFile::open("static/vendor/fa-webfont.ttf").await.ok())
+}
+
+#[get("/fonts/fontawesome-webfont.woff2")]
+pub async fn fontawesome_woff<'a>() -> LongCache<Option<NamedFile>> {
+    LongCache::new(NamedFile::open("static/vendor/fa-webfont.ttf").await.ok())
+}

src/rust/lqos_node_manager/src/tracker/cache/cpu_ram.rs

@@ -0,0 +1,12 @@
+use lazy_static::*;
+use parking_lot::RwLock;
+
+lazy_static! {
+    /// Global storage of current CPU usage
+    pub static ref CPU_USAGE : RwLock<Vec<f32>> = RwLock::new(Vec::new());
+}
+
+lazy_static! {
+    /// Global storage of current RAM usage
+    pub static ref MEMORY_USAGE : RwLock<Vec<u64>> = RwLock::new(vec![0, 0]);
+}

src/rust/lqos_node_manager/src/tracker/cache/lqosd_stats.rs

@@ -0,0 +1,19 @@
+use lazy_static::*;
+use lqos_bus::IpStats;
+use parking_lot::RwLock;
+
+lazy_static! {
+    pub static ref TOP_10_DOWNLOADERS : RwLock<Vec<IpStats>> = RwLock::new(Vec::new());
+}
+
+lazy_static! {
+    pub static ref WORST_10_RTT : RwLock<Vec<IpStats>> = RwLock::new(Vec::new());
+}
+
+lazy_static! {
+    pub static ref RTT_HISTOGRAM : RwLock<Vec<u32>> = RwLock::new(Vec::new());
+}
+
+lazy_static! {
+    pub static ref HOST_COUNTS : RwLock<(u32, u32)> = RwLock::new((0, 0));
+}

src/rust/lqos_node_manager/src/tracker/cache/mod.rs

@@ -0,0 +1,13 @@
+//! The cache module stores cached data, periodically
+//! obtained from the `lqosd` server and other parts
+//! of the system.
+
+mod throughput;
+mod cpu_ram;
+mod lqosd_stats;
+mod shaped_devices;
+
+pub use throughput::*;
+pub use cpu_ram::*;
+pub use lqosd_stats::*;
+pub use shaped_devices::*;

src/rust/lqos_node_manager/src/tracker/cache/shaped_devices.rs

@@ -0,0 +1,18 @@
+use lazy_static::*;
+use lqos_bus::IpStats;
+use lqos_config::ConfigShapedDevices;
+use parking_lot::RwLock;
+
+lazy_static! {
+    /// Global storage of the shaped devices csv data.
+    /// Updated by the file system watcher whenever
+    /// the underlying file changes.
+    pub static ref SHAPED_DEVICES : RwLock<ConfigShapedDevices> = RwLock::new(ConfigShapedDevices::load().unwrap());
+}
+
+lazy_static! {
+    /// Global storage of the shaped devices csv data.
+    /// Updated by the file system watcher whenever
+    /// the underlying file changes.
+    pub static ref UNKNOWN_DEVICES : RwLock<Vec<IpStats>> = RwLock::new(Vec::new());
+}

src/rust/lqos_node_manager/src/tracker/cache/throughput.rs

@@ -0,0 +1,73 @@
+use lazy_static::*;
+use parking_lot::RwLock;
+use rocket::serde::Serialize;
+
+lazy_static! {
+    /// Global storage of the current throughput counter.
+    pub static ref CURRENT_THROUGHPUT : RwLock<ThroughputPerSecond> = RwLock::new(ThroughputPerSecond::default());
+}
+
+lazy_static! {
+    /// Global storage of the last N seconds throughput buffer.
+    pub static ref THROUGHPUT_BUFFER : RwLock<ThroughputRingbuffer> = RwLock::new(ThroughputRingbuffer::new());
+}
+
+/// Stores total system throughput per second.
+#[derive(Debug, Clone, Copy, Serialize)]
+#[serde(crate = "rocket::serde")]
+pub struct ThroughputPerSecond {
+    pub bits_per_second: (u64, u64),
+    pub packets_per_second: (u64, u64),
+    pub shaped_bits_per_second: (u64, u64),
+}
+
+impl Default for ThroughputPerSecond {
+    fn default() -> Self {
+        Self {
+            bits_per_second: (0,0),
+            packets_per_second: (0,0),
+            shaped_bits_per_second: (0, 0),
+        }
+    }
+}
+
+/// How many entries (at one per second) should we keep in the
+/// throughput ringbuffer?
+const RINGBUFFER_SAMPLES: usize = 300;
+
+/// Stores Throughput samples in a ringbuffer, continually
+/// updating. There are always RINGBUFFER_SAMPLES available,
+/// allowing for non-allocating/non-growing storage of
+/// throughput for the dashboard summaries.
+pub struct ThroughputRingbuffer {
+    readings: Vec<ThroughputPerSecond>,
+    next: usize,
+}
+
+impl ThroughputRingbuffer {
+    fn new() -> Self {
+        Self {
+            readings: vec![ThroughputPerSecond::default(); RINGBUFFER_SAMPLES],
+            next: 0,
+        }
+    }
+
+    pub fn store(&mut self, reading: ThroughputPerSecond) {
+        self.readings[self.next] = reading;
+        self.next += 1;
+        self.next %= RINGBUFFER_SAMPLES;
+    }
+
+    pub fn get_result(&self) -> Vec<ThroughputPerSecond> {
+        let mut result = Vec::new();
+
+        for i in self.next .. RINGBUFFER_SAMPLES {
+            result.push(self.readings[i]);
+        }
+        for i in 0..self.next {
+            result.push(self.readings[i]);
+        }
+
+        result
+    }
+}

src/rust/lqos_node_manager/src/tracker/cache_manager.rs

@@ -0,0 +1,142 @@
+//! The Cache mod stores data that is periodically updated
+//! on the server-side, to avoid re-requesting repeatedly
+//! when there are multiple clients.
+use std::{time::Duration, net::IpAddr};
+use anyhow::Result;
+use lqos_bus::{BUS_BIND_ADDRESS, BusSession, BusRequest, encode_request, BusResponse, decode_response, IpStats};
+use lqos_config::ConfigShapedDevices;
+use rocket::tokio::{task::spawn_blocking, net::TcpStream, io::{AsyncWriteExt, AsyncReadExt}};
+use super::cache::*;
+
+/// Once per second, update CPU and RAM usage and ask
+/// `lqosd` for updated system statistics.
+/// Called from the main program as a "fairing", meaning
+/// it runs as part of start-up - and keeps running.
+/// Designed to never return or fail on error.
+pub async fn update_tracking() {
+    use sysinfo::System;
+    use sysinfo::CpuExt;
+    use sysinfo::SystemExt;
+    let mut sys = System::new_all();
+
+    spawn_blocking(|| {
+        let _ = watch_for_shaped_devices_changing();
+    });
+
+    loop {
+        //println!("Updating tracking data");
+        sys.refresh_cpu();
+        sys.refresh_memory();
+        let cpu_usage = sys
+            .cpus()
+            .iter()
+            .map(|cpu| cpu.cpu_usage())
+            .collect::<Vec<f32>>();
+        *CPU_USAGE.write() = cpu_usage;
+        {
+            let mut mem_use = MEMORY_USAGE.write();
+            mem_use[0] = sys.used_memory();
+            mem_use[1] = sys.total_memory();
+        }
+        let _ = get_data_from_server().await; // Ignoring errors to keep running
+        rocket::tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+}
+
+
+/// Fires up a Linux file system watcher than notifies
+/// when `ShapedDevices.csv` changes, and triggers a reload.
+fn watch_for_shaped_devices_changing() -> Result<()> {
+    use notify::{Watcher, RecursiveMode, Config};
+
+    let (tx, rx) = std::sync::mpsc::channel();
+    let mut watcher = notify::RecommendedWatcher::new(tx, Config::default())?;
+
+    watcher.watch(&ConfigShapedDevices::path()?, RecursiveMode::NonRecursive)?;
+    loop {
+        let _ = rx.recv();
+        if let Ok(new_file) = ConfigShapedDevices::load() {
+            println!("ShapedDevices.csv changed");
+            *SHAPED_DEVICES.write() = new_file;
+        }
+    }
+}
+
+/// Requests data from `lqosd` and stores it in local
+/// caches.
+async fn get_data_from_server() -> Result<()> {
+    // Send request to lqosd
+    let mut stream = TcpStream::connect(BUS_BIND_ADDRESS).await?;
+    let test = BusSession {
+        auth_cookie: 1234,
+        requests: vec![
+            BusRequest::GetCurrentThroughput,
+            BusRequest::GetTopNDownloaders(10),
+            BusRequest::GetWorstRtt(10),
+            BusRequest::RttHistogram,
+            BusRequest::AllUnknownIps,
+        ],
+    };
+    let msg = encode_request(&test)?;
+    stream.write(&msg).await?;
+
+    // Receive reply
+    let mut buf = Vec::new();
+    let _ = stream.read_to_end(&mut buf).await.unwrap();
+    let reply = decode_response(&buf)?;
+
+    // Process the reply
+    for r in reply.responses.iter() {
+        match r {
+            BusResponse::CurrentThroughput {
+                bits_per_second,
+                packets_per_second,
+                shaped_bits_per_second
+            } => {
+                {
+                    let mut lock = CURRENT_THROUGHPUT.write();
+                    lock.bits_per_second = *bits_per_second;
+                    lock.packets_per_second = *packets_per_second;
+                } // Lock scope
+                {
+                    let mut lock = THROUGHPUT_BUFFER.write();
+                    lock.store(ThroughputPerSecond {
+                        packets_per_second: *packets_per_second,
+                        bits_per_second: *bits_per_second,
+                        shaped_bits_per_second: *shaped_bits_per_second,
+                    });
+                }
+            }
+            BusResponse::TopDownloaders(stats) => {
+                *TOP_10_DOWNLOADERS.write() = stats.clone();
+            }
+            BusResponse::WorstRtt(stats) => {
+                *WORST_10_RTT.write() = stats.clone();
+            }
+            BusResponse::RttHistogram(stats) => {
+                *RTT_HISTOGRAM.write() = stats.clone();
+            }
+            BusResponse::AllUnknownIps(unknowns) => {
+                *HOST_COUNTS.write() = (unknowns.len() as u32, 0);
+                let cfg = SHAPED_DEVICES.read();
+                let really_unknown: Vec<IpStats> = unknowns.iter().filter(|ip| {
+                    if let Ok(ip) = ip.ip_address.parse::<IpAddr>() {
+                        let lookup = match ip {
+                            IpAddr::V4(ip) => ip.to_ipv6_mapped(),
+                            IpAddr::V6(ip) => ip,
+                        };
+                        cfg.trie.longest_match(lookup).is_none()
+                    } else {
+                        false
+                    }
+                }).cloned().collect();
+                *HOST_COUNTS.write() = (really_unknown.len() as u32, 0);
+                *UNKNOWN_DEVICES.write() = really_unknown;
+            }
+            // Default
+            _ => {}
+        }
+    }
+
+    Ok(())
+}

src/rust/lqos_node_manager/src/tracker/mod.rs

@@ -0,0 +1,101 @@
+mod cache_manager;
+mod cache;
+pub use cache::{SHAPED_DEVICES, UNKNOWN_DEVICES};
+pub use cache_manager::update_tracking;
+use std::net::IpAddr;
+use lqos_bus::{IpStats, TcHandle};
+use rocket::serde::{json::Json, Serialize, Deserialize};
+use crate::tracker::cache::ThroughputPerSecond;
+use self::cache::{CURRENT_THROUGHPUT, THROUGHPUT_BUFFER, CPU_USAGE, MEMORY_USAGE, TOP_10_DOWNLOADERS, WORST_10_RTT, RTT_HISTOGRAM, HOST_COUNTS};
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(crate = "rocket::serde")]
+pub struct IpStatsWithPlan {
+    pub ip_address: String,
+    pub bits_per_second: (u64, u64),
+    pub packets_per_second: (u64, u64),
+    pub median_tcp_rtt: f32,
+    pub tc_handle: TcHandle,
+    pub circuit_id: String,
+    pub plan: (u32, u32),
+}
+
+impl From<&IpStats> for IpStatsWithPlan {
+    fn from(i: &IpStats) -> Self {
+        let mut result = Self {
+            ip_address: i.ip_address.clone(),
+            bits_per_second: i.bits_per_second,
+            packets_per_second: i.packets_per_second,
+            median_tcp_rtt: i.median_tcp_rtt,
+            tc_handle: i.tc_handle,
+            circuit_id: String::new(),
+            plan: (0, 0),
+        };
+        if let Ok(ip) = result.ip_address.parse::<IpAddr>() {
+            let lookup = match ip {
+                IpAddr::V4(ip) => ip.to_ipv6_mapped(),
+                IpAddr::V6(ip) => ip,
+            };
+            let cfg = SHAPED_DEVICES.read();
+            if let Some((_, id)) = cfg.trie.longest_match(lookup) {
+                result.ip_address = format!("{} ({})", cfg.devices[*id].circuit_name, result.ip_address);
+                result.plan.0 = cfg.devices[*id].download_max_mbps;
+                result.plan.1 = cfg.devices[*id].upload_max_mbps;
+                result.circuit_id = cfg.devices[*id].circuit_id.clone();
+            }
+        }
+        result
+    }
+}
+
+#[get("/api/current_throughput")]
+pub fn current_throughput() -> Json<ThroughputPerSecond> {
+    let result = CURRENT_THROUGHPUT.read().clone();
+    Json(result)
+}
+
+#[get("/api/throughput_ring")]
+pub fn throughput_ring() -> Json<Vec<ThroughputPerSecond>> {
+    let result = THROUGHPUT_BUFFER.read().get_result();
+    Json(result)
+}
+
+#[get("/api/cpu")]
+pub fn cpu_usage() -> Json<Vec<f32>> {
+    let cpu_usage = CPU_USAGE.read().clone();
+
+    Json(cpu_usage)
+}
+
+#[get("/api/ram")]
+pub fn ram_usage() -> Json<Vec<u64>> {
+    let ram_usage = MEMORY_USAGE.read().clone();
+    Json(ram_usage)
+}
+
+#[get("/api/top_10_downloaders")]
+pub fn top_10_downloaders() -> Json<Vec<IpStatsWithPlan>> {
+    let tt : Vec<IpStatsWithPlan> = TOP_10_DOWNLOADERS.read().iter().map(|tt| tt.into()).collect();
+    Json(tt)
+}
+
+#[get("/api/worst_10_rtt")]
+pub fn worst_10_rtt() -> Json<Vec<IpStatsWithPlan>> {
+    let tt : Vec<IpStatsWithPlan> = WORST_10_RTT.read().iter().map(|tt| tt.into()).collect();
+    Json(tt)
+}
+
+
+#[get("/api/rtt_histogram")]
+pub fn rtt_histogram() -> Json<Vec<u32>> {
+    Json(RTT_HISTOGRAM.read().clone())
+}
+
+#[get("/api/host_counts")]
+pub fn host_counts() -> Json<(u32, u32)> {
+    let shaped_reader = SHAPED_DEVICES.read();
+    let n_devices = shaped_reader.devices.len();
+    let host_counts = HOST_COUNTS.read();
+    let unknown = host_counts.0 - host_counts.1;
+    Json((n_devices as u32, unknown))
+}

src/rust/lqos_node_manager/src/unknown_devices.rs

@@ -0,0 +1,20 @@
+use lqos_bus::IpStats;
+use rocket::serde::json::Json;
+use crate::{cache_control::NoCache, tracker::UNKNOWN_DEVICES};
+
+#[get("/api/all_unknown_devices")]
+pub fn all_unknown_devices() -> NoCache<Json<Vec<IpStats>>> {
+    NoCache::new(Json(UNKNOWN_DEVICES.read().clone()))
+}
+
+#[get("/api/unknown_devices_count")]
+pub fn unknown_devices_count() -> NoCache<Json<usize>> {
+    NoCache::new(Json(UNKNOWN_DEVICES.read().len()))
+}
+
+#[get("/api/unknown_devices_range/<start>/<end>")]
+pub fn unknown_devices_range(start: usize, end: usize) -> NoCache<Json<Vec<IpStats>>> {
+    let reader = UNKNOWN_DEVICES.read();
+    let result: Vec<IpStats> = reader.iter().skip(start).take(end).cloned().collect();
+    NoCache::new(Json(result))
+}

src/rust/lqos_node_manager/static/circuit_queue.html

@@ -0,0 +1,84 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="/vendor/bootstrap.min.css" rel="stylesheet">
+    <link rel="stylesheet" href="/vendor/solid.min.css">
+    <title>LibreQoS - Local Node Manager</title>
+    <script src="/lqos.js"></script>
+    <script src="/vendor/plotly-2.16.1.min.js"></script>
+    <script src="/vendor/jquery.min.js"></script>
+</head>
+<body class="bg-secondary">
+    <!-- Navigation -->
+    <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
+        <div class="container-fluid">
+            <a class="navbar-brand" href="/"><img src="/vendor/tinylogo.svg" alt="LibreQoS SVG Logo" width="25" height="25" />&nbsp;LibreQoS</a>
+            <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+                <span class="navbar-toggler-icon"></span>
+            </button>
+            <div class="collapse navbar-collapse" id="navbarSupportedContent">
+                <ul class="navbar-nav me-auto mb-2 mb-lg-0">
+                    <li class="nav-item">
+                        <a class="nav-link" href="/"><i class="fa fa-home"></i> Dashboard</a>
+                    </li>
+                    
+                    <li class="nav-item">
+                        <a class="nav-link" href="#"><i class="fa fa-globe"></i> Network Layout</a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link active" aria-current="page" href="/shaped"><i class="fa fa-users"></i> Shaped Devices <span id="shapedCount" style="background-color: green;" class="badge badge-pill badge-success">?</span></a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="/unknown"><i class="fa fa-address-card"></i> Unknown IPs <span id="unshapedCount" style="background-color:darkgoldenrod" class="badge badge-warning">?</span></a>
+                    </li>
+                </ul>
+            </div>
+
+            <ul class="navbar-nav ms-auto">
+                <li class="nav-item ms-auto">
+                    <a class="nav-link" href="#"><i class="fa fa-gear"></i> Configuration</a>
+                </li>
+                <li>
+                    <a class="nav-link btn btn-small" style="color: black" href="#" id="btnReload"><i class="fa fa-refresh"></i> Reload LibreQoS</a>
+                </li>
+            </ul>
+        </div>
+    </nav>
+
+    <div id="container" style="padding: 4px;">
+
+        <div class="row">
+            <div class="col-sm-12">
+                <div class="card bg-light">
+                    <div class="card-body">
+                        <h5 class="card-title"><i class="fa fa-users"></i> Queue Info Placeholder</h5>
+                        <div id="raw"></div>
+                </div>
+            </div>
+        </div>
+
+    </div>
+
+    <footer>Copyright (c) 2022, LibreQoE LLC</footer>
+
+    <script>
+        function start() {
+            colorReloadButton();
+            updateHostCounts();
+            const params = new Proxy(new URLSearchParams(window.location.search), {
+                get: (searchParams, prop) => searchParams.get(prop),
+            });
+            if (params.id != null) {
+                $("#raw").html("<a class='btn btn-info' href='/api/raw_queue_by_circuit/" + encodeURI(params.id) + "'><i class='fa fa-search'></i> Raw Data</a>");
+            }
+        }
+
+        $(document).ready(start);
+    </script>
+
+    <!-- Leave to last -->
+    <script src="vendor/bootstrap.bundle.min.js"></script>
+</body>
+</html>

src/rust/lqos_node_manager/static/lqos.js

@@ -0,0 +1,104 @@
+var colorPreference = 0;
+
+function metaverse_color_ramp(n) {
+    if (n <= 9) {
+        return "#32b08c";
+    } else if (n <= 20) {
+        return "#ffb94a";
+    } else if (n <=50) {
+        return "#f95f53";
+    } else if (n <=70) {
+        return "#bf3d5e";
+    } else {
+        return "#dc4e58";
+    }
+}
+
+function regular_color_ramp(n) {
+    if (n <= 100) {
+        return "#aaffaa";
+    } else if (n <= 150) {
+        return "goldenrod";
+    } else {
+        return "#ffaaaa";
+    }
+}
+
+function color_ramp(n) {
+    if (colorPreference == 0) {
+        return regular_color_ramp(n);
+    } else {
+        return metaverse_color_ramp(n);
+    }
+}
+
+function bindColorToggle() {
+    $("#toggleColors").on('click', () => {
+        if (colorPreference == 0) {
+            colorPreference = 1;
+            $("#toggleColors").text("(metaverse colors)");
+        } else {
+            colorPreference = 0;
+            $("#toggleColors").text("(regular colors)");
+        }
+    });
+}
+
+function updateHostCounts() {
+    $.get("/api/host_counts", (hc) => {
+        $("#shapedCount").text(hc[0]);
+        $("#unshapedCount").text(hc[1]);
+        setTimeout(updateHostCounts, 5000);
+    });
+}
+
+function colorReloadButton() {
+    $("body").append(reloadModal);
+    $("#btnReload").on('click', () => {
+        $.get("/api/reload_libreqos", (result) => {
+            const myModal = new bootstrap.Modal(document.getElementById('reloadModal'), {focus: true});
+            $("#reloadLibreResult").text(result);
+            myModal.show();    
+        });
+    });
+    $.get("/api/reload_required", (req) => {
+        if (req) {
+            $("#btnReload").addClass('btn-warning');
+            $("#btnReload").css('color', 'darkred');
+        } else {
+            $("#btnReload").addClass('btn-secondary');
+        }
+    })
+}
+
+function scaleNumber(n) {
+    if (n > 1000000000000) {
+        return (n/1000000000000).toFixed(2) + "T";
+    } else if (n > 1000000000) {
+        return (n/1000000000).toFixed(2) + "G";
+    } else if (n > 1000000) {
+        return (n/1000000).toFixed(2) + "M";
+    } else if (n > 1000) {
+        return (n/1000).toFixed(2) + "K";
+    }
+    return n;
+}
+
+const reloadModal = `
+<div class='modal fade' id='reloadModal' tabindex='-1' aria-labelledby='reloadModalLabel' aria-hidden='true'>
+    <div class='modal-dialog modal-fullscreen'>
+      <div class='modal-content'>
+        <div class='modal-header'>
+          <h1 class='modal-title fs-5' id='reloadModalLabel'>LibreQoS Reload Result</h1>
+          <button type='button' class='btn-close' data-bs-dismiss='modal' aria-label='Close'></button>
+        </div>
+        <div class='modal-body'>
+          <pre id='reloadLibreResult' style='overflow: vertical; height: 100%; width: 100%;'>
+          </pre>
+        </div>
+        <div class='modal-footer'>
+          <button type='button' class='btn btn-secondary' data-bs-dismiss='modal'>Close</button>
+        </div>
+      </div>
+    </div>
+  </div>`;

src/rust/lqos_node_manager/static/main.html

@@ -0,0 +1,293 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="/vendor/bootstrap.min.css" rel="stylesheet">
+    <link rel="stylesheet" href="/vendor/solid.min.css">
+    <title>LibreQoS - Local Node Manager</title>
+    <script src="/lqos.js"></script>
+    <script src="/vendor/plotly-2.16.1.min.js"></script>
+    <script src="/vendor/jquery.min.js"></script>
+</head>
+<body class="bg-secondary">
+    <!-- Navigation -->
+    <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
+        <div class="container-fluid">
+            <a class="navbar-brand" href="/"><img src="/vendor/tinylogo.svg" alt="LibreQoS SVG Logo" width="25" height="25" />&nbsp;LibreQoS</a>
+            <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+                <span class="navbar-toggler-icon"></span>
+            </button>
+            <div class="collapse navbar-collapse" id="navbarSupportedContent">
+                <ul class="navbar-nav me-auto mb-2 mb-lg-0">
+                    <li class="nav-item">
+                        <a class="nav-link active" aria-current="page" href="/"><i class="fa fa-home"></i> Dashboard</a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="#"><i class="fa fa-globe"></i> Network Layout</a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="/shaped"><i class="fa fa-users"></i> Shaped Devices <span id="shapedCount" style="background-color: green;" class="badge badge-pill badge-success">?</span></a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="/unknown"><i class="fa fa-address-card"></i> Unknown IPs <span id="unshapedCount" style="background-color:darkgoldenrod" class="badge badge-warning">?</span></a>
+                    </li>
+                </ul>
+            </div>
+
+            <ul class="navbar-nav ms-auto">
+                <li class="nav-item ms-auto">
+                    <a class="nav-link" href="#"><i class="fa fa-gear"></i> Configuration</a>
+                </li>
+                <li class="nav-item">
+                    <a class="nav-link" href="#" id="startTest"><i class="fa fa-flag-checkered"></i> Run Bandwidth Test</a>
+                </li>
+                <li>
+                    <a class="nav-link btn btn-small" href="#" id="btnReload"><i class="fa fa-refresh"></i> Reload LibreQoS</a>
+                </li>
+            </ul>
+        </div>
+    </nav>
+
+    <div id="container" style="padding: 4px;">
+
+    <!-- Dashboard Row 1 -->
+    <div class="row" style="margin-bottom: 8px;">
+        <!-- THROUGHPUT -->
+        <div class="col-sm-4">
+            <div class="card bg-light">
+                <div class="card-body">
+                    <h5 class="card-title"><i class="fa fa-bolt"></i> Current Throughput</h5>
+                    <table class="table">
+                        <tr>
+                            <td style="font-weight: bold">Packets/Second</td>
+                            <td id="ppsDown"></td>
+                            <td id="ppsUp"></td>
+                        </tr>
+                        <tr>
+                            <td style="font-weight: bold">Bits/Second</td>
+                            <td id="bpsDown"></td>
+                            <td id="bpsUp"></td>
+                        </tr>
+                    </table>
+                </div>
+            </div>
+        </div>
+
+        <!-- RAM INFO -->
+        <div class="col-sm-2">
+            <div class="card bg-light">
+                <div class="card-body">
+                    <h5 class="card-title"><i class="fa fa-database"></i> Memory Status</h5>
+                    <div id="ram" style="height: 97px"></div>
+                </div>
+            </div>
+        </div>
+
+        <!-- CPU INFO -->
+        <div class="col-sm-6">
+            <div class="card bg-light">
+                <div class="card-body">
+                    <h5 class="card-title"><i class="fa fa-microchip"></i> CPU Status</h5>
+                    <div id="cpu" style="height: 97px"></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- Dashboard Row 2 -->
+    <div class="row" style="height: 200px; margin-bottom: 20px;">
+        <!-- 5 minutes of throughput -->
+        <div class="col-sm-6">
+            <div class="card bg-light">
+                <div class="card-body">
+                    <h5 class="card-title"><i class="fa fa-hourglass"></i> Last 5 Minutes</h5>
+                    <div id="tpGraph" style="height: 150px"></div>
+                </div>
+            </div>
+        </div>
+
+        <!-- RTT Histogram -->
+        <div class="col-sm-6">
+            <div class="card bg-light">
+                <div class="card-body">
+                    <h5 class="card-title"><i class="fa fa-bar-chart"></i> TCP Round-Trip Time Histogram</h5>
+                    <div id="rttHistogram" style="height: 150px"></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- Dashboard Row 3 -->
+    <div class="row">
+        <!-- Top 10 downloaders -->
+        <div class="col-sm-6">
+            <div class="card bg-light">
+                <div class="card-body">
+                    <h5 class="card-title"><i class='fa fa-arrow-down'></i> Top 10 Downloaders <a href="#" id="toggleColors" style="font-size: 8pt;">(Regular colors)</a></h5>
+                    <div id="top10dl"></div>
+                </div>
+            </div>
+        </div>
+
+        <!-- Worst 10 RTT -->
+        <div class="col-sm-6">
+            <div class="card bg-light">
+                <div class="card-body">
+                    <h5 class="card-title"><i class='fa fa-exclamation'></i> Worst 10 RTT</h5>
+                    <div id="worstRtt"></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    </div>
+
+    <footer>Copyright (c) 2022, LibreQoE LLC</footer>
+
+    <script>
+        function updateCurrentThroughput() {
+            $.get("/api/current_throughput", (tp) => {
+                $("#ppsDown").text(scaleNumber(tp.packets_per_second[0]));
+                $("#ppsUp").text(scaleNumber(tp.packets_per_second[1]));
+                $("#bpsDown").text(scaleNumber(tp.bits_per_second[0]));
+                $("#bpsUp").text(scaleNumber(tp.bits_per_second[1]));
+                setTimeout(updateCurrentThroughput, 1000);
+            });
+        }
+
+        function updateThroughputGraph() {
+            $.get("/api/throughput_ring", (tp) => {
+                let graph = document.getElementById("tpGraph");
+                let x = [];
+                let y = []; // Down
+                let y2 = []; // Up
+                let y3 = []; // Shaped Down
+                let y4 = []; // Shaped Up
+                for (i=0; i<300; i++) {
+                    x.push(i);
+                    y.push(tp[i].bits_per_second[0]);
+                    y2.push(0.0 - tp[i].bits_per_second[1]);
+                    y3.push(tp[i].shaped_bits_per_second[0]);
+                    y4.push(0.0 - tp[i].shaped_bits_per_second[1]);
+                }
+                let data = [
+                    {x: x, y:y, name: 'Download', type: 'scatter', fill: 'tozeroy'},
+                    {x: x, y:y2, name: 'Upload', type: 'scatter', fill: 'tozeroy'},
+                    {x: x, y:y3, name: 'Shaped Download', type: 'scatter', fill: 'tozeroy'},
+                    {x: x, y:y4, name: 'Shaped Upload', type: 'scatter', fill: 'tozeroy'},
+                ];
+                Plotly.newPlot(graph, data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true }, xaxis: {automargin: true} });
+                //console.log(tp);
+                setTimeout(updateThroughputGraph, 1000);
+            });
+        }
+
+        function updateCpu() {
+            $.get("/api/cpu", (cpu) => {
+                let graph = document.getElementById("cpu");
+                let x = [];
+                let y = [];
+                for (i=0; i<cpu.length; i++) {
+                    x.push(i);
+                    y.push(cpu[i]);
+                }
+                let data = [ {x: x, y:y, type: 'bar' } ];
+                Plotly.newPlot(graph, data, { margin: { l:0,r:0,b:15,t:0 }, yaxis: { automargin: true, autorange: false, range: [0.0, 100.0 ] } });
+                setTimeout(updateCpu, 2000);
+            });
+        }
+
+        function updateRam() {
+            $.get("/api/ram", (ram) => {
+                let graph = document.getElementById("ram");
+                let data = [ {
+                    values: [ram[0], ram[1]-ram[0]],
+                    labels: ['Used', 'Available'], 
+                    type: 'pie' 
+                } ];
+                Plotly.newPlot(graph, data, { margin: { l:4,r:0,b:0,t:4 } });
+                setTimeout(updateRam, 30000);
+            });
+        }
+
+        function updateNTable(target, tt) {
+            let html = "<table class='table'>";
+            html += "<thead><th>IP Address</th><th>DL ⬇️</th><th>UL ⬆️</th><th>RTT (ms)</th><th>Shaped</th></thead>";
+            for (let i=0; i<tt.length; i++) {
+                let color = color_ramp(tt[i].median_tcp_rtt);
+                html += "<tr style='background-color: " + color + "'>";
+                if (tt[i].circuit_id != "") {
+                    html += "<td><a href='/circuit_queue?id=" + encodeURI(tt[i].circuit_id)  + "'>" + tt[i].ip_address + "</td>";
+                } else {
+                    html += "<td>" + tt[i].ip_address + "</td>";
+                }
+                html += "<td>" + scaleNumber(tt[i].bits_per_second[0]) + "</td>";
+                html += "<td>" + scaleNumber(tt[i].bits_per_second[1]) + "</td>";
+                html += "<td>" + tt[i].median_tcp_rtt.toFixed(2) + "</td>";
+                if (tt[i].tc_handle !=0) {
+                    html += "<td><i class='fa fa-check-circle'></i> (" + tt[i].plan[0] + "/" + tt[i].plan[1] + ")</td>";
+                } else {
+                    html += "<td><a class='btn btn-small btn-success' href='/shaped-add?ip=" + tt[i].ip_address + "'>Add Shaper</a></td>";
+                }
+                html += "</tr>";
+            }
+            html += "</table>";
+            $(target).html(html);
+        }
+
+        function updateTop10() {
+            $.get("/api/top_10_downloaders", (tt) => {
+                updateNTable('#top10dl', tt);
+                setTimeout(updateTop10, 5000);
+            });
+        }
+
+        function updateWorst10() {
+            $.get("/api/worst_10_rtt", (tt) => {
+                updateNTable('#worstRtt', tt);
+                setTimeout(updateWorst10, 5000);
+            });
+        }
+
+        function updateHistogram() {
+            $.get("/api/rtt_histogram", (rtt) => {
+                let graph = document.getElementById("rttHistogram");
+                let x = [];
+                let y = [];
+                for (let i=0; i<rtt.length; i++) {
+                    x.push(i*10.0);
+                    y.push(rtt[i]);
+                }
+                let data = [
+                    {x:x, y:y, type: 'bar'}
+                ]
+                Plotly.newPlot(graph, data, { margin: { l:0,r:0,b:15,t:0 }});
+                setTimeout(updateHistogram, 5000);
+            });
+        }
+
+        function start() {
+            colorReloadButton();
+            updateCurrentThroughput();
+            updateThroughputGraph();
+            updateCpu();
+            updateRam();
+            updateTop10();
+            updateWorst10();
+            updateHistogram();
+            updateHostCounts();
+            bindColorToggle();
+
+            $("#startTest").on('click', () => {
+                $.get("/api/run_btest", () => {});
+            });
+        }
+
+        $(document).ready(start);
+    </script>
+
+    <!-- Leave to last -->
+    <script src="vendor/bootstrap.bundle.min.js"></script>
+</body>
+</html>

src/rust/lqos_node_manager/static/shaped-add.html

@@ -0,0 +1,169 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="/vendor/bootstrap.min.css" rel="stylesheet">
+    <link rel="stylesheet" href="/vendor/solid.min.css">
+    <title>LibreQoS - Local Node Manager</title>
+    <script src="/lqos.js"></script>
+    <script src="/vendor/plotly-2.16.1.min.js"></script>
+    <script src="/vendor/jquery.min.js"></script>
+</head>
+<body class="bg-secondary">
+    <!-- Navigation -->
+    <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
+        <div class="container-fluid">
+            <a class="navbar-brand" href="/"><img src="/vendor/tinylogo.svg" alt="LibreQoS SVG Logo" width="25" height="25" />&nbsp;LibreQoS</a>
+            <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+                <span class="navbar-toggler-icon"></span>
+            </button>
+            <div class="collapse navbar-collapse" id="navbarSupportedContent">
+                <ul class="navbar-nav me-auto mb-2 mb-lg-0">
+                    <li class="nav-item">
+                        <a class="nav-link" href="/"><i class="fa fa-home"></i> Dashboard</a>
+                    </li>
+                    
+                    <li class="nav-item">
+                        <a class="nav-link" href="#"><i class="fa fa-globe"></i> Network Layout</a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link active" aria-current="page" href="/shaped"><i class="fa fa-users"></i> Shaped Devices <span id="shapedCount" style="background-color: green;" class="badge badge-pill badge-success">?</span></a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="/unknown"><i class="fa fa-address-card"></i> Unknown IPs <span id="unshapedCount" style="background-color:darkgoldenrod" class="badge badge-warning">?</span></a>
+                    </li>
+                </ul>
+            </div>
+
+            <ul class="navbar-nav ms-auto">
+                <li class="nav-item ms-auto">
+                    <a class="nav-link" href="#"><i class="fa fa-gear"></i> Configuration</a>
+                </li>
+                <li>
+                    <a class="nav-link btn btn-small" style="color: black" href="#" id="btnReload"><i class="fa fa-refresh"></i> Reload LibreQoS</a>
+                </li>
+            </ul>
+        </div>
+    </nav>
+
+    <div id="container" style="padding: 4px;">
+
+        <div class="row">
+            <div class="col-sm-12">
+                <div class="card bg-light">
+                    <div class="card-body">
+                        <h5 class="card-title"><i class="fa fa-users"></i> Add Shaped Circuit</h5>
+
+                        <div class="row">
+                            <div class="col">
+                                <label for="circuitId" class="form-label">Circuit ID</label>
+                                <input type="text" id="circuitId" class="form-control" />
+                            </div>
+                            <div class="col">
+                                <label for="circuitName" class="form-label">Circuit Name</label>
+                                <input type="text" id="circuitName" class="form-control" />
+                            </div>
+                        </div>
+                        <div class="row">
+                            <div class="col">
+                                <label for="deviceId" class="form-label">Device ID</label>
+                                <input type="text" id="deviceId" class="form-control" />
+                            </div>
+                            <div class="col">
+                                <label for="circuitName" class="form-label">Device Name</label>
+                                <input type="text" id="deviceName" class="form-control" />
+                            </div>
+                            <div class="col">
+                                <label for="parent" class="form-label">Parent</label>
+                                <input type="text" id="parent" class="form-control" />
+                            </div>
+                            <div class="col">
+                                <label for="mac" class="form-label">MAC Address</label>
+                                <input type="text" id="mac" class="form-control" />
+                            </div>
+                        </div>
+                        <div class="row">
+                            <div class="col">
+                                <label for="dlMin" class="form-label">Download Minimum (Mbps)</label>
+                                <input type="number" id="dlMin" class="form-control" />
+                            </div>
+                            <div class="col">
+                                <label for="ulMin" class="form-label">Upload Minimum (Mbps)</label>
+                                <input type="number" id="ulMin" class="form-control" />
+                            </div>
+                            <div class="col">
+                                <label for="dlMax" class="form-label">Download Maximum (Mbps)</label>
+                                <input type="number" id="dlMax" class="form-control" />
+                            </div>
+                            <div class="col">
+                                <label for="ulMax" class="form-label">Upload Maximum (Mbps)</label>
+                                <input type="number" id="ulMax" class="form-control" />
+                            </div>
+                        </div>
+                        <div class="row" style="margin-bottom: 8px;">
+                            <div class="col">
+                                <label for="comment" class="form-label">Comment</label>
+                                <input type="text" id="comment" class="form-control" />
+                            </div>
+                        </div>
+                        <div class="row" style="margin-bottom: 8px;">
+                            <div class="col">
+                                <strong>IPv4 Addresses</strong> (You can use 1.2.3.4/X to match a CIDR subnet)<br />
+                                <label for="ipv4_1" class="form-label">Address 1</label>
+                                <input type="text" id="ipv4_1" class="form-control" />
+                                <label for="ipv4_2" class="form-label">Address 2</label>
+                                <input type="text" id="ipv4_2" class="form-control" />
+                                <label for="ipv4_3" class="form-label">Address 3</label>
+                                <input type="text" id="ipv4_3" class="form-control" />
+                            </div>
+                            <div class="col">
+                                <strong>IPv6 Addresses</strong> (You can use /X to match a subnet)<br />
+                                <label for="ipv6_1" class="form-label">Address 1</label>
+                                <input type="text" id="ipv6_1" class="form-control" />
+                                <label for="ipv6_2" class="form-label">Address 2</label>
+                                <input type="text" id="ipv6_2" class="form-control" />
+                                <label for="ipv6_3" class="form-label">Address 3</label>
+                                <input type="text" id="ip64_3" class="form-control" />
+                            </div>
+                        </div>
+
+                        <div class="row">
+                            <div class="col" align="center">
+                                <a href="#" class="btn btn-success"><i class='fa fa-plus'></i> Add Record</a>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+    </div>
+
+    <footer>Copyright (c) 2022, LibreQoE LLC</footer>
+
+    <script>
+        function start() {
+            colorReloadButton();
+            updateHostCounts();
+
+            // Get the ? search params
+            const params = new Proxy(new URLSearchParams(window.location.search), {
+                get: (searchParams, prop) => searchParams.get(prop),
+            });
+            if (params.ip != null) {
+                if (params.ip.includes(":")) {
+                    $("#ipv6_1").val(params.ip + "/128");
+                } else {
+                    $("#ipv4_1").val(params.ip + "/32");
+                }
+            }
+        }
+
+        $(document).ready(start);
+    </script>
+
+    <!-- Leave to last -->
+    <script src="vendor/bootstrap.bundle.min.js"></script>
+</body>
+</html>

src/rust/lqos_node_manager/static/shaped.html

@@ -0,0 +1,165 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="/vendor/bootstrap.min.css" rel="stylesheet">
+    <link rel="stylesheet" href="/vendor/solid.min.css">
+    <title>LibreQoS - Local Node Manager</title>
+    <script src="/lqos.js"></script>
+    <script src="/vendor/plotly-2.16.1.min.js"></script>
+    <script src="/vendor/jquery.min.js"></script>
+</head>
+<body class="bg-secondary">
+    <!-- Navigation -->
+    <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
+        <div class="container-fluid">
+            <a class="navbar-brand" href="/"><img src="/vendor/tinylogo.svg" alt="LibreQoS SVG Logo" width="25" height="25" />&nbsp;LibreQoS</a>
+            <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+                <span class="navbar-toggler-icon"></span>
+            </button>
+            <div class="collapse navbar-collapse" id="navbarSupportedContent">
+                <ul class="navbar-nav me-auto mb-2 mb-lg-0">
+                    <li class="nav-item">
+                        <a class="nav-link" href="/"><i class="fa fa-home"></i> Dashboard</a>
+                    </li>
+                    
+                    <li class="nav-item">
+                        <a class="nav-link" href="#"><i class="fa fa-globe"></i> Network Layout</a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link active" aria-current="page" href="/shaped"><i class="fa fa-users"></i> Shaped Devices <span id="shapedCount" style="background-color: green;" class="badge badge-pill badge-success">?</span></a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="/unknown"><i class="fa fa-address-card"></i> Unknown IPs <span id="unshapedCount" style="background-color:darkgoldenrod" class="badge badge-warning">?</span></a>
+                    </li>
+                </ul>
+            </div>
+
+            <ul class="navbar-nav ms-auto">
+                <li class="nav-item ms-auto">
+                    <a class="nav-link" href="#"><i class="fa fa-gear"></i> Configuration</a>
+                </li>
+                <li>
+                    <a class="nav-link btn btn-small" style="color: black" href="#" id="btnReload"><i class="fa fa-refresh"></i> Reload LibreQoS</a>
+                </li>
+            </ul>
+        </div>
+    </nav>
+
+    <div id="container" style="padding: 4px;">
+
+        <div class="row">
+            <div class="col-sm-12">
+                <div class="card bg-light">
+                    <div class="card-body">
+                        <h5 class="card-title"><i class="fa fa-users"></i> Shaped Devices</h5>
+
+                        <div class="row">
+                            <div class="col">
+                                <input id="search" class="form-control" placeholder="Search" style="min-width: 150px">
+                            </div>
+                            <div class="col">
+                                <a href="#" class="btn btn-primary" id="btnSearch"><i class='fa fa-search'></i></a>
+                            </div>
+                            <div class="col">
+                                <a href="/shaped-add" class="btn btn-success"><i class='fa fa-plus'></i> Add</a>
+                            </div>
+                        </div>
+
+                        <table class="table table-striped">
+                            <thead>
+                                <th>Circuit</th>
+                                <th>Device</th>
+                                <th>Plan</th>
+                                <th>IPs</th>
+                                <th><i class="fa fa-gear"></i></th>
+                            </thead>
+                            <tbody id="shapedList"></tbody>
+                        </table>
+
+                        <p>
+                            Go to page: <span id="shapedPaginator"></span><br />
+                            Total Shaped Devices: <span id="shapedTotal"></span>
+                        </p>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+    </div>
+
+    <footer>Copyright (c) 2022, LibreQoE LLC</footer>
+
+    <script>
+        function fillDeviceTable(devices) {
+            let html = "";
+            for (let i=0; i<devices.length; i++) {
+                html += "<tr>";
+                html += "<td><a href='/circuit_queue?id=" + encodeURI(devices[i].circuit_id) + "'>" + devices[i].circuit_id + ": " +devices[i].circuit_name + "</a></td>";
+                html += "<td>" + devices[i].device_id + ": " + devices[i].device_name + "</td>";
+                html += "<td>" + devices[i].download_min_mbps + "/" + devices[i].upload_max_mbps + "</td>";
+                html += "<td style='font-size: 8pt'>";
+                for (let j=0; j<devices[i].ipv4.length; j++) {
+                    html += devices[i].ipv4[j][0] + "/" + devices[i].ipv4[j][1] + "<br />";
+                }
+                for (let j=0; j<devices[i].ipv6.length; j++) {
+                    html += devices[i].ipv6[j][0] + "/" + devices[i].ipv6[j][1] + "<br />";
+                }
+                html += "</td>";
+                html += "<td><a class='btn btn-primary btn-sm' href='#'><i class='fa fa-pencil'></i></a>";
+                html +=" <a href='#' class='btn btn-danger btn-sm'><i class='fa fa-trash'></i></a></td>";
+                html += "</tr>";
+            }
+            $("#shapedList").html(html);
+        }
+
+        function paginator(page) {
+            $.get("/api/shaped_devices_range/" + page * 25 + "/" + (page+1)*25, (devices) => {
+                fillDeviceTable(devices);
+            });
+        }
+
+        function doSearch() {
+            let term = $("#search").val();
+            if (term == "") {
+                paginator(0);
+            } else {
+                // /api/shaped_devices_search/<term>
+                let safe_term = encodeURIComponent(term);
+                $.get("/api/shaped_devices_search/" + safe_term, (devices) => {
+                    fillDeviceTable(devices);
+                })
+            }
+        }
+
+        function start() {
+            colorReloadButton();
+            updateHostCounts();
+            $.get("/api/shaped_devices_count", (count) => {
+                let n_pages = count / 25;
+                $("#shapedTotal").text(count);
+                let paginator = "";
+                for (let i=0; i<n_pages; i++) {
+                    paginator += "<a href='#' onclick='paginator(" + i + ")'>" + (i+1) + "</a> ";
+                }
+                $("#shapedPaginator").html(paginator);
+            });
+            $.get("/api/shaped_devices_range/0/25", (devices) => {
+                fillDeviceTable(devices);
+            });
+            $("#btnSearch").on('click', () => {
+                doSearch();
+            });
+            $("#search").on('keyup', (k) => {
+                if (k.originalEvent.keyCode == 13) doSearch();
+            });
+        }
+
+        $(document).ready(start);
+    </script>
+
+    <!-- Leave to last -->
+    <script src="vendor/bootstrap.bundle.min.js"></script>
+</body>
+</html>

src/rust/lqos_node_manager/static/tinylogo.svg

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg id="Layer_2" data-name="Layer 2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 283.08 323.96">
+  <defs>
+    <style>
+      .cls-1 {
+        fill: #fff;
+      }
+    </style>
+  </defs>
+  <g id="Layer_1-2" data-name="Layer 1">
+    <path class="cls-1" d="M137.98,0c2.33,0,4.67,0,7,0,.98,.83,1.87,1.82,2.97,2.45,42.65,24.58,85.27,49.2,128.04,73.58,5.18,2.95,7.14,6.46,7.1,12.35-.3,48.79-.51,97.59-.3,146.38,.03,6.9-2.51,10.56-8.15,13.78-33.8,19.28-67.47,38.78-101.14,58.3-9.59,5.56-19.01,11.4-28.51,17.12h-7c-7.85-3.9-15.98-7.33-23.51-11.79-32.3-19.11-64.46-38.46-96.57-57.89C11.71,250.54,5.96,246.09,0,241.97,0,188.31,0,134.65,0,80.99c1.99-1.42,3.87-3.03,5.97-4.25C42.03,55.88,78.12,35.09,114.17,14.22,122.17,9.59,130.05,4.75,137.98,0Zm129.74,98.28c-13.51,16.02-26.06,31.09-38.87,45.93-2.62,3.04-3.49,6.04-3.46,9.91,.16,15.99,.18,31.98-.02,47.97-.05,3.64,1.09,5.72,4.25,7.48,11.77,6.56,23.4,13.38,35.1,20.06,.89,.51,1.94,.75,3,1.15V98.28Zm-9.62-13.78c.04-.43,.08-.86,.11-1.29-37.74-21.74-75.47-43.49-113.95-65.66,0,16.25-.09,31.22,.13,46.19,.02,1.54,1.51,3.65,2.91,4.47,15.04,8.86,30.17,17.56,45.38,26.14,1.51,.85,3.73,1.06,5.49,.78,12.09-1.93,24.16-4.03,36.21-6.18,7.92-1.41,15.81-2.97,23.71-4.46ZM15.06,98.41V231.35c14-8.08,27.4-15.71,40.64-23.6,1.15-.69,1.85-3.02,1.86-4.6,.13-16.98,.12-33.96,.02-50.94-.01-1.69-.44-3.76-1.47-5-13.36-16.05-26.87-31.99-41.06-48.81Zm2.82,138.31c38.75,22.38,76.47,44.16,115.44,66.67-7.56-21.03-14.52-40.52-21.64-59.96-.5-1.37-1.96-2.63-3.29-3.41-15.07-8.8-30.18-17.54-45.35-26.16-1.22-.69-3.37-1.12-4.44-.52-13.31,7.48-26.51,15.17-40.72,23.38Zm246.92,0c-13.5-7.81-26.09-14.91-38.45-22.38-3.11-1.88-5.31-1.48-8.13,.17-14.48,8.49-29.07,16.79-43.53,25.32-1.69,1-3.36,2.82-4.03,4.63-6.61,17.9-13.01,35.87-19.44,53.83-.45,1.27-.7,2.62-1.25,4.77,38.52-22.25,76.18-44.01,114.82-66.33ZM23.57,83.77c2.22,.69,3.26,1.15,4.35,1.34,18.89,3.38,37.79,6.78,56.7,10.02,1.74,.3,3.96,.09,5.46-.76,14.93-8.4,29.73-17.02,44.62-25.49,2.4-1.37,3.53-2.92,3.5-5.88-.17-13.82-.06-27.65-.1-41.47,0-1.03-.34-2.05-.59-3.44C99.6,39.94,62.17,61.52,23.57,83.77Zm225.87,9.21c-.1-.19-.2-.38-.3-.57-11.23,1.94-22.48,3.76-33.67,5.87-7.3,1.37-15.23,1.75-21.63,5.02-15.98,8.15-31.27,17.67-46.78,26.73-1.14,.67-2.6,2.05-2.64,3.14-.25,6.42-.12,12.86-.12,20.38,35.63-20.52,70.39-40.54,105.15-60.57Zm-99.23,70.53c5.74,3.36,10.66,6.12,15.44,9.09,2.57,1.6,4.64,1.48,7.31-.1,14.18-8.39,28.54-16.48,42.72-24.86,2.95-1.74,5.73-4.02,8.02-6.56,5.56-6.17,10.81-12.63,16.16-19,4.92-5.87,9.8-11.78,14.7-17.67-.25-.26-.5-.52-.75-.77-34.23,19.79-68.46,39.58-103.59,59.88Zm-2.63,124.5l.86,.16c5.78-16.02,11.56-32.04,17.3-48.08,.27-.75,.21-1.65,.21-2.47,.01-18.31,.04-36.62-.04-54.93,0-1.31-.53-3.23-1.47-3.82-5.28-3.31-10.76-6.31-16.86-9.81v118.95Zm-13.09,.76c.24-.1,.47-.21,.71-.31v-119.41c-5.37,3.03-9.89,5.78-14.59,8.16-2.89,1.46-3.96,3.31-3.91,6.62,.21,13.15,.46,26.32-.03,39.45-.41,11,1.26,21.36,5.41,31.57,4.53,11.15,8.31,22.59,12.42,33.91Zm-2.08-125.22c-35.12-20.32-69.59-40.27-104.05-60.21,7.86,10.87,16.28,20.91,24.85,30.84,3.56,4.12,6.79,9,11.26,11.79,15.08,9.39,30.64,18.04,46.08,26.85,1.27,.72,3.42,1.27,4.52,.69,5.7-2.99,11.19-6.37,17.34-9.96Zm5.7-10.18c0-6.55-.25-12.16,.09-17.73,.21-3.4-1.23-5.08-3.94-6.61-12.6-7.11-25.23-14.18-37.62-21.65-6.07-3.66-12.27-6.24-19.33-7.34-14.48-2.25-28.88-5.03-43.31-7.59-.09,.27-.17,.53-.26,.8,34.51,19.88,69.02,39.76,104.37,60.12Zm36.27,25.14c15.23,8.81,29.68,17.18,44.81,25.93v-51.81c-14.97,8.64-29.42,16.99-44.81,25.88Zm-64.05,3.73c-15.27,8.83-29.76,17.21-44.85,25.93,15.34,8.87,29.77,17.21,44.85,25.93v-51.86Zm61.97-.05v52.04c15.04-8.77,29.51-17.21,44.81-26.13-15.45-8.94-29.8-17.23-44.81-25.91Zm-108.52,22.08c14.91-8.63,29.44-17.04,44.51-25.77-15.03-8.69-29.36-16.97-44.51-25.72v51.49ZM138.74,73.37c-14.99,8.66-29.36,16.97-44.57,25.76,15.31,8.81,29.72,17.09,44.57,25.63v-51.38Zm49.81,25.73c-15.29-8.81-29.7-17.11-44.55-25.67v51.29c14.96-8.6,29.31-16.86,44.55-25.62Z"/>
+  </g>
+</svg>

src/rust/lqos_node_manager/static/unknown-ip.html

@@ -0,0 +1,125 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="/vendor/bootstrap.min.css" rel="stylesheet">
+    <link rel="stylesheet" href="/vendor/solid.min.css">
+    <title>LibreQoS - Local Node Manager</title>
+    <script src="/lqos.js"></script>
+    <script src="/vendor/plotly-2.16.1.min.js"></script>
+    <script src="/vendor/jquery.min.js"></script>
+</head>
+<body class="bg-secondary">
+    <!-- Navigation -->
+    <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
+        <div class="container-fluid">
+            <a class="navbar-brand" href="/"><img src="/vendor/tinylogo.svg" alt="LibreQoS SVG Logo" width="25" height="25" />&nbsp;LibreQoS</a>
+            <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+                <span class="navbar-toggler-icon"></span>
+            </button>
+            <div class="collapse navbar-collapse" id="navbarSupportedContent">
+                <ul class="navbar-nav me-auto mb-2 mb-lg-0">
+                    <li class="nav-item">
+                        <a class="nav-link" href="/"><i class="fa fa-home"></i> Dashboard</a>
+                    </li>
+                    
+                    <li class="nav-item">
+                        <a class="nav-link" href="#"><i class="fa fa-globe"></i> Network Layout</a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link" href="/shaped"><i class="fa fa-users"></i> Shaped Devices <span id="shapedCount" style="background-color: green;" class="badge badge-pill badge-success">?</span></a>
+                    </li>
+                    <li class="nav-item">
+                        <a class="nav-link active" aria-current="page" href="/unknown"><i class="fa fa-address-card"></i> Unknown IPs <span id="unshapedCount" style="background-color:darkgoldenrod" class="badge badge-warning">?</span></a>
+                    </li>
+                </ul>
+            </div>
+
+            <ul class="navbar-nav ms-auto">
+                <li class="nav-item ms-auto">
+                    <a class="nav-link" href="#"><i class="fa fa-gear"></i> Configuration</a>
+                </li>
+                <li>
+                    <a class="nav-link btn btn-small" style="color: black" href="#" id="btnReload"><i class="fa fa-refresh"></i> Reload LibreQoS</a>
+                </li>
+            </ul>
+        </div>
+    </nav>
+
+    <div id="container" style="padding: 4px;">
+
+        <div class="row">
+            <div class="col-sm-12">
+                <div class="card bg-light">
+                    <div class="card-body">
+                        <h5 class="card-title"><i class="fa fa-address-card"></i> Unmapped IP Addresses (Most recently seen first)</h5>
+
+                        <table class="table table-striped">
+                            <thead>
+                                <th>IP</th>
+                                <th>Total Bandwidth</th>
+                                <th>Total Packets</th>
+                                <th><i class='fa fa-gear'></i></th>
+                            </thead>
+                            <tbody id="unknownList"></tbody>
+                        </table>
+
+                        <p>
+                            Go to page: <span id="unknownPaginator"></span><br />
+                            Total Shaped Devices: <span id="unknownTotal"></span>
+                        </p>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+    </div>
+
+    <footer>Copyright (c) 2022, LibreQoE LLC</footer>
+
+    <script>
+        function fillDeviceTable(devices) {
+            let html = "";
+            for (let i=0; i<devices.length; i++) {
+                html += "<tr>";
+                html += "<td>" + devices[i].ip_address + "</td>";
+                html += "<td>" + scaleNumber(devices[i].bits_per_second[0]) + " / " + scaleNumber(devices[i].bits_per_second[1]) + "</td>";
+                html += "<td>" + scaleNumber(devices[i].packets_per_second[0]) + " / " + scaleNumber(devices[i].packets_per_second[1]) + "</td>";
+                html += "<td><a class='btn btn-small btn-success' href='/shaped-add?ip=" + devices[i].ip_address + "'><i class='fa fa-plus'></i></a></td>";
+                html += "</tr>";
+            }
+            $("#unknownList").html(html);
+        }
+
+        function paginator(page) {
+            $.get("/api/unknown_devices_range/" + page * 25 + "/" + (page+1)*25, (devices) => {
+                fillDeviceTable(devices);
+            });
+        }
+
+        function start() {
+            colorReloadButton();
+            updateHostCounts();
+            $.get("/api/unknown_devices_count", (count) => {
+                let n_pages = count / 25;
+                $("#unknownTotal").text(count);
+                let paginator = "";
+                for (let i=0; i<n_pages; i++) {
+                    paginator += "<a href='#' onclick='paginator(" + i + ")'>" + (i+1) + "</a> ";
+                }
+                $("#unknownPaginator").html(paginator);
+            });
+            $.get("/api/unknown_devices_range/0/25", (devices) => {
+                console.log(devices);
+                fillDeviceTable(devices);
+            });
+        }
+
+        $(document).ready(start);
+    </script>
+
+    <!-- Leave to last -->
+    <script src="vendor/bootstrap.bundle.min.js"></script>
+</body>
+</html>

src/rust/lqos_sys/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "lqos_sys"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+nix = "0.25"
+libbpf-sys = "1"
+anyhow = "1"
+byteorder = "1.4"
+lqos_bus = { path = "../lqos_bus" }
+lqos_config = { path = "../lqos_config" }
+
+[build-dependencies]
+bindgen = "0.53.1"

src/rust/lqos_sys/README.md

@@ -0,0 +1,7 @@
+## lqos_sys
+
+This crate wraps the XDP component in externally callable Rust. This is
+used by other systems to manage the XDP/TC eBPF system.
+
+The `src/bpf` directory contains the C for the eBPF program, as well as
+some wrapper helpers to bring it into Rust-space.

src/rust/lqos_sys/build.rs

@@ -0,0 +1,143 @@
+use std::env;
+use std::path::PathBuf;
+use std::process::{Command, Output};
+
+fn command_warnings(section: &str, command_result: &std::io::Result<Output>) {
+    if command_result.is_err() {
+        println!("cargo:warning=[{section}]{:?}", command_result);
+    }
+
+    let r = command_result.as_ref().unwrap().stdout.clone();
+    if !r.is_empty() {
+        println!("cargo:warning=[{section}]{}", String::from_utf8(r).unwrap());
+    }
+
+    let r = command_result.as_ref().unwrap().stderr.clone();
+    if !r.is_empty() {
+        panic!("{}", String::from_utf8(r).unwrap());
+    }
+}
+
+fn command_warnings_errors_only(section: &str, command_result: &std::io::Result<Output>) {
+    if command_result.is_err() {
+        println!("cargo:warning=[{section}]{:?}", command_result);
+    }
+
+    let r = command_result.as_ref().unwrap().stderr.clone();
+    if !r.is_empty() {
+        println!("cargo:warning=[{section}] {}", String::from_utf8(r).unwrap());
+    }
+}
+
+fn main() {
+    let out_dir = env::var_os("OUT_DIR").unwrap();
+
+    // 1: Shell out to build the lqos_bpf.ll XDP/TC combined program.
+    // Command line to wrap:
+    // clang -S -target bpf -Wall -Wno-unused-value -Wno-pointer-sign -Wno-compare-distinct-pointer-types -Werror -emit-llvm -c -g -I../headers/ -O2 -o "../bin"/libre_xdp_kern.ll libre_xdp_kern.c
+
+    let build_target = format!("{}/lqos_kern.ll", out_dir.to_str().unwrap());
+    let compile_result = Command::new("clang")
+        .current_dir("src/bpf")
+        .args([
+            "-S",
+            "-target",
+            "bpf",
+            "-Wall",
+            "-Wno-unused-value",
+            "-Wno-pointer-sign",
+            "-Wno-compare-distinct-pointer-types",
+            "-Werror",
+            "-emit-llvm",
+            "-c",
+            "-g",
+            "-O2",
+            "-o",
+            &build_target,
+            "lqos_kern.c",
+        ])
+        .output();
+    command_warnings("clang", &compile_result);
+
+    // 2: Link the .ll file into a .o file
+    // Command line:
+    // llc -march=bpf -filetype=obj -o "../bin"/libre_xdp_kern.o "../bin"/libre_xdp_kern.ll
+    let link_target = format!("{}/lqos_kern.o", out_dir.to_str().unwrap());
+    let link_result = Command::new("llc")
+        .args([
+            "-march=bpf",
+            "-filetype=obj",
+            "-o",
+            &link_target,
+            &build_target,
+        ])
+        .output();
+    command_warnings("llc", &link_result);
+
+    // 3: Use bpftool to build the skeleton file
+    // Command line:
+    // bpftool gen skeleton ../bin/libre_xdp_kern.o > libre_xdp_skel.h
+    let skel_target = format!("{}/lqos_kern_skel.h", out_dir.to_str().unwrap());
+    let skel_result = Command::new("bpftool")
+        .args(["gen", "skeleton", &link_target])
+        .output();
+    command_warnings_errors_only("bpf skel", &skel_result);
+    let header_file = String::from_utf8(skel_result.unwrap().stdout).unwrap();
+    std::fs::write(&skel_target, header_file).unwrap();
+
+    // 4: Copy the wrapper to our out dir
+    let wrapper_target = format!("{}/wrapper.h", out_dir.to_str().unwrap());
+    let wrapper_target_c = format!("{}/wrapper.c", out_dir.to_str().unwrap());
+    let shrinkwrap_lib = format!("{}/libshrinkwrap.o", out_dir.to_str().unwrap());
+    let shrinkwrap_a = format!("{}/libshrinkwrap.a", out_dir.to_str().unwrap());
+    std::fs::copy("src/bpf/wrapper.h", &wrapper_target).unwrap();
+    std::fs::copy("src/bpf/wrapper.c", &wrapper_target_c).unwrap();
+
+    // 5: Build the intermediary library
+    let build_result = Command::new("clang")
+        .current_dir("src/bpf")
+        .args([
+            "-c",
+            "wrapper.c",
+            &format!("-I{}", out_dir.to_str().unwrap()),
+            "-o",
+            &shrinkwrap_lib,
+        ])
+        .output();
+    command_warnings("clang - wrapper", &build_result);
+
+    let _build_result = Command::new("ar")
+        .args([
+            "r",
+            &shrinkwrap_a,
+            &shrinkwrap_lib,
+            //"/usr/lib/x86_64-linux-gnu/libbpf.a",
+        ])
+        .output();
+    //command_warnings(&build_result);
+
+    println!(
+        "cargo:rustc-link-search=native={}",
+        out_dir.to_str().unwrap()
+    );
+    println!("cargo:rustc-link-lib=static=shrinkwrap");
+
+    // 6: Use bindgen to generate a Rust wrapper
+    let bindings = bindgen::Builder::default()
+        // The input header we would like to generate
+        // bindings for.
+        .header(&wrapper_target)
+        // Tell cargo to invalidate the built crate whenever any of the
+        // included header files changed.
+        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
+        // Finish the builder and generate the bindings.
+        .generate()
+        // Unwrap the Result and panic on failure.
+        .expect("Unable to generate bindings");
+
+    // Write the bindings to the $OUT_DIR/bindings.rs file.
+    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    bindings
+        .write_to_file(out_path.join("bindings.rs"))
+        .expect("Couldn't write bindings!");
+}

src/rust/lqos_sys/src/bifrost_maps.rs

@@ -0,0 +1,67 @@
+use anyhow::Result;
+use lqos_config::{BridgeInterface, BridgeVlan};
+
+use crate::{bpf_map::BpfMap, lqos_kernel::interface_name_to_index};
+
+#[repr(C)]
+#[derive(Default, Clone, Debug)]
+struct BifrostInterface {
+    redirect_to: u32,
+    scan_vlans: u32,
+}
+
+#[repr(C)]
+#[derive(Default, Clone, Debug)]
+struct BifrostVlan {
+    redirect_to: u32,
+}
+
+const INTERFACE_PATH: &str = "/sys/fs/bpf/bifrost_interface_map";
+const VLAN_PATH: &str = "/sys/fs/bpf/bifrost_vlan_map";
+
+pub(crate) fn clear_bifrost() -> Result<()> {
+    println!("Clearing bifrost maps");
+    let mut interface_map = BpfMap::<u32, BifrostInterface>::from_path(INTERFACE_PATH)?;
+    let mut vlan_map = BpfMap::<u32, BifrostVlan>::from_path(VLAN_PATH)?;
+    println!("Clearing VLANs");
+    vlan_map.clear_no_repeat()?;
+    println!("Clearing Interfaces");
+    interface_map.clear_no_repeat()?;
+    println!("Done");
+    Ok(())
+}
+
+pub(crate) fn map_interfaces(mappings: &[BridgeInterface]) -> Result<()> {
+    println!("Interface maps");
+    let mut interface_map = BpfMap::<u32, BifrostInterface>::from_path(INTERFACE_PATH)?;
+    for mapping in mappings.iter() {
+        // Key is the parent interface
+        let mut from = interface_name_to_index(&mapping.name)?;
+        let redirect_to = interface_name_to_index(&mapping.redirect_to)?;
+        let mut mapping = BifrostInterface {
+            redirect_to,
+            scan_vlans: match mapping.scan_vlans {
+                true => 1,
+                false => 0,
+            },
+        };
+        interface_map.insert(&mut from, &mut mapping)?;
+        println!("Mapped bifrost interface {}->{}", from, redirect_to);
+    }
+    Ok(())
+}
+
+pub(crate) fn map_vlans(mappings: &[BridgeVlan]) -> Result<()> {
+    println!("VLAN maps");
+    let mut vlan_map = BpfMap::<u32, BifrostVlan>::from_path(VLAN_PATH)?;
+    for mapping in mappings.iter() {
+        let mut key: u32 = (interface_name_to_index(&mapping.parent)? << 16) | mapping.tag;
+        let mut val = BifrostVlan {
+            redirect_to: mapping.redirect_to,
+        };
+        vlan_map.insert(&mut key, &mut val)?;
+        println!("Mapped bifrost VLAN: {}:{} => {}", mapping.parent, mapping.tag, mapping.redirect_to);
+        println!("{key}");
+    }
+    Ok(())
+}

src/rust/lqos_sys/src/bpf/common/bifrost.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+#include <linux/if_ether.h>
+#include <stdbool.h>
+#include "maximums.h"
+#include "debug.h"
+
+// Defines a bridge-free redirect interface.
+struct bifrost_interface {
+	// The interface index to which this interface (from the key)
+	// should redirect.
+    __u32 redirect_to;
+	// Should VLANs be scanned (for VLAN redirection)?
+	// > 0 = true. 32-bit for padding reasons.
+    __u32 scan_vlans;
+};
+
+// Hash map defining up to 64 interface redirects.
+// Keyed on the source interface index, value is a bifrost_interface
+// structure.
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 64);
+	__type(key, __u32);
+	__type(value, struct bifrost_interface);
+	__uint(pinning, LIBBPF_PIN_BY_NAME);
+} bifrost_interface_map SEC(".maps");
+
+// TODO: This could be a u32 if we don't need any additional info.
+// Which VLAN should the keyed VLAN be redirected to?
+struct bifrost_vlan {
+    __u32 redirect_to;
+};
+
+// Hash map of VLANs that should be redirected.
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 64);
+	__type(key, __u32);
+	__type(value, struct bifrost_vlan);
+	__uint(pinning, LIBBPF_PIN_BY_NAME);
+} bifrost_vlan_map SEC(".maps");

src/rust/lqos_sys/src/bpf/common/cpu_map.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+#include <linux/if_ether.h>
+#include <stdbool.h>
+#include "maximums.h"
+#include "debug.h"
+
+// Data structure used for map_txq_config.
+// This is used to apply the queue_mapping in the TC part.
+struct txq_config {
+	/* lookup key: __u32 cpu; */
+	__u16 queue_mapping;
+	__u16 htb_major;
+};
+
+/* Special map type that can XDP_REDIRECT frames to another CPU */
+struct {
+	__uint(type, BPF_MAP_TYPE_CPUMAP);
+	__uint(max_entries, MAX_CPUS);
+	__type(key, __u32);
+	__type(value, __u32);
+	__uint(pinning, LIBBPF_PIN_BY_NAME);
+} cpu_map SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, MAX_CPUS);
+	__type(key, __u32);
+	__type(value, __u32);
+	__uint(pinning, LIBBPF_PIN_BY_NAME);
+} cpus_available SEC(".maps");
+
+// Map used to store queue mappings
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, MAX_CPUS);
+	__type(key, __u32);
+	__type(value, struct txq_config);
+	__uint(pinning, LIBBPF_PIN_BY_NAME);
+} map_txq_config SEC(".maps");

src/rust/lqos_sys/src/bpf/common/debug.h

@@ -0,0 +1,13 @@
+#pragma once
+
+// Define VERBOSE if you want to fill
+// `/sys/kernel/debug/tracing/trace_pipe` with per-packet debug
+// info. You usually don't want this.
+//#define VERBOSE 1
+
+#define bpf_debug(fmt, ...)                        \
+	({                                             \
+		char ____fmt[] = " " fmt;             \
+		bpf_trace_printk(____fmt, sizeof(____fmt), \
+						 ##__VA_ARGS__);           \
+	})

src/rust/lqos_sys/src/bpf/common/dissector.h

@@ -0,0 +1,247 @@
+#pragma once
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+#include <linux/if_ether.h>
+#include <stdbool.h>
+#include "../common/skb_safety.h"
+#include "../common/debug.h"
+#include "../common/ip_hash.h"
+#include "../common/bifrost.h"
+
+// Packet dissector for XDP. We don't have any help from Linux at this
+// point.
+struct dissector_t
+{
+    // Pointer to the XDP context.
+    struct xdp_md *ctx;
+    // Start of data
+    void *start;
+    // End of data
+    void *end;
+    // Total length (end - start)
+    __u32 skb_len;
+    // Ethernet header once found (NULL until then)
+    struct ethhdr *ethernet_header;
+    // Ethernet packet type once found (0 until then)
+    __u16 eth_type;
+    // Layer-3 offset if found (0 until then)
+    __u32 l3offset;
+    // IPv4/6 header once found
+    union iph_ptr ip_header;
+    // Source IP address, encoded by `ip_hash.h`
+    struct in6_addr src_ip;
+    // Destination IP address, encoded by `ip_hash.h`
+    struct in6_addr dst_ip;
+    // Current VLAN tag. If there are multiple tags, it will be
+    // the INNER tag.
+    __be16 current_vlan;
+};
+
+// Representation of the VLAN header type.
+struct vlan_hdr
+{
+    // Tagged VLAN number
+    __be16 h_vlan_TCI;
+    // Protocol for the next section
+    __be16 h_vlan_encapsulated_proto;
+};
+
+// Representation of the PPPoE protocol header.
+struct pppoe_proto
+{
+    __u8 pppoe_version_type;
+    __u8 ppoe_code;
+    __be16 session_id;
+    __be16 pppoe_length;
+    __be16 proto;
+};
+
+#define PPPOE_SES_HLEN 8
+#define PPP_IP 0x21
+#define PPP_IPV6 0x57
+
+// Constructor for a dissector
+// Connects XDP/TC SKB structure to a dissector structure.
+// Arguments:
+// * ctx - an xdp_md structure, passed from the entry-point
+// * dissector - pointer to a local dissector object to be initialized
+//
+// Returns TRUE if all is good, FALSE if the process cannot be completed
+static __always_inline bool dissector_new(
+    struct xdp_md *ctx, 
+    struct dissector_t *dissector
+) {
+    dissector->ctx = ctx;
+    dissector->start = (void *)(long)ctx->data;
+    dissector->end = (void *)(long)ctx->data_end;
+    dissector->ethernet_header = (struct ethhdr *)NULL;
+    dissector->l3offset = 0;
+    dissector->skb_len = dissector->end - dissector->start;
+    dissector->current_vlan = 0;
+
+    // Check that there's room for an ethernet header
+    if SKB_OVERFLOW (dissector->start, dissector->end, ethhdr)
+    {
+        return false;
+    }
+    dissector->ethernet_header = (struct ethhdr *)dissector->start;
+
+    return true;
+}
+
+// Helper function - is an eth_type an IPv4 or v6 type?
+static __always_inline bool is_ip(__u16 eth_type)
+{
+    return eth_type == ETH_P_IP || eth_type == ETH_P_IPV6;
+}
+
+// Locates the layer-3 offset, if present. Fast returns for various
+// common non-IP types. Will perform VLAN redirection if requested.
+static __always_inline bool dissector_find_l3_offset(
+    struct dissector_t *dissector, 
+    bool vlan_redirect
+) {
+    if (dissector->ethernet_header == NULL)
+    {
+        bpf_debug("Ethernet header is NULL, still called offset check.");
+        return false;
+    }
+    __u32 offset = sizeof(struct ethhdr);
+    __u16 eth_type = bpf_ntohs(dissector->ethernet_header->h_proto);
+
+    // Fast return for unwrapped IP
+    if (eth_type == ETH_P_IP || eth_type == ETH_P_IPV6)
+    {
+        dissector->eth_type = eth_type;
+        dissector->l3offset = offset;
+        return true;
+    }
+
+    // Fast return for ARP or non-802.3 ether types
+    if (eth_type == ETH_P_ARP || eth_type < ETH_P_802_3_MIN)
+    {
+        return false;
+    }
+
+    // Walk the headers until we find IP
+    __u8 i = 0;
+    while (i < 10 && !is_ip(eth_type))
+    {
+        switch (eth_type)
+        {
+        // Read inside VLAN headers
+        case ETH_P_8021AD:
+        case ETH_P_8021Q:
+        {
+            if SKB_OVERFLOW_OFFSET (dissector->start, dissector->end, 
+                offset, vlan_hdr)
+            {
+                return false;
+            }
+            struct vlan_hdr *vlan = (struct vlan_hdr *)
+                (dissector->start + offset);
+            dissector->current_vlan = vlan->h_vlan_TCI;
+            eth_type = bpf_ntohs(vlan->h_vlan_encapsulated_proto);
+            offset += sizeof(struct vlan_hdr);
+            // VLAN Redirection is requested, so lookup a detination and
+            // switch the VLAN tag if required
+            if (vlan_redirect) {
+                #ifdef VERBOSE
+                bpf_debug("Searching for redirect %u:%u", 
+                    dissector->ctx->ingress_ifindex, 
+                    bpf_ntohs(dissector->current_vlan)
+                );
+                #endif
+                __u32 key = (dissector->ctx->ingress_ifindex << 16) | 
+                    bpf_ntohs(dissector->current_vlan);
+                struct bifrost_vlan * vlan_info = NULL;
+                vlan_info = bpf_map_lookup_elem(&bifrost_vlan_map, &key);
+                if (vlan_info) {
+                    #ifdef VERBOSE
+                    bpf_debug("Redirect to VLAN %u", 
+                        bpf_htons(vlan_info->redirect_to)
+                    );
+                    #endif
+                    vlan->h_vlan_TCI = bpf_htons(vlan_info->redirect_to);
+                }
+            }
+        }
+        break;
+
+        // Handle PPPoE
+        case ETH_P_PPP_SES:
+        {
+            if SKB_OVERFLOW_OFFSET (dissector->start, dissector->end, 
+                offset, pppoe_proto)
+            {
+                return false;
+            }
+            struct pppoe_proto *pppoe = (struct pppoe_proto *)
+                (dissector->start + offset);
+            __u16 proto = bpf_ntohs(pppoe->proto);
+            switch (proto)
+            {
+            case PPP_IP:
+                eth_type = ETH_P_IP;
+                break;
+            case PPP_IPV6:
+                eth_type = ETH_P_IPV6;
+                break;
+            default:
+                return false;
+            }
+            offset += PPPOE_SES_HLEN;
+        }
+        break;
+        // We found something we don't know how to handle - bail out
+        default:
+            return false;
+        }
+        ++i;
+    }
+
+    dissector->l3offset = offset;
+    dissector->eth_type = eth_type;
+    return true;
+}
+
+// Searches for an IP header.
+static __always_inline bool dissector_find_ip_header(
+    struct dissector_t *dissector
+) {
+    switch (dissector->eth_type)
+    {
+    case ETH_P_IP:
+    {
+        if (dissector->start + dissector->l3offset + sizeof(struct iphdr) > 
+            dissector->end) {
+                return false;
+        }
+        dissector->ip_header.iph = dissector->start + dissector->l3offset;
+        if (dissector->ip_header.iph + 1 > dissector->end)
+            return false;
+        encode_ipv4(dissector->ip_header.iph->saddr, &dissector->src_ip);
+        encode_ipv4(dissector->ip_header.iph->daddr, &dissector->dst_ip);
+        return true;
+    }
+    break;
+    case ETH_P_IPV6:
+    {
+        if (dissector->start + dissector->l3offset + 
+            sizeof(struct ipv6hdr) > dissector->end) {
+                return false;
+        }
+        dissector->ip_header.ip6h = dissector->start + dissector->l3offset;
+        if (dissector->ip_header.iph + 1 > dissector->end)
+            return false;
+        encode_ipv6(&dissector->ip_header.ip6h->saddr, &dissector->src_ip);
+        encode_ipv6(&dissector->ip_header.ip6h->daddr, &dissector->dst_ip);
+        return true;
+    }
+    break;
+    default:
+        return false;
+    }
+}

src/rust/lqos_sys/src/bpf/common/dissector_tc.h

@@ -0,0 +1,194 @@
+#pragma once
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+#include <linux/if_ether.h>
+#include <stdbool.h>
+#include "../common/skb_safety.h"
+#include "../common/debug.h"
+#include "../common/ip_hash.h"
+#include "dissector.h"
+
+// Structure holding packet dissection information (obtained at the TC level)
+struct tc_dissector_t
+{
+    // Pointer to the SKB context.
+    struct __sk_buff *ctx;
+    // Pointer to the data start
+    void *start;
+    // Pointer to the data end
+    void *end;
+    // Pointer to the Ethernet header once obtained (NULL until then)
+    struct ethhdr *ethernet_header;
+    // Ethernet packet type, once obtained
+    __u16 eth_type;
+    // Start of layer-3 data, once obtained
+    __u32 l3offset;
+    // IP header (either v4 or v6), once obtained.
+    union iph_ptr ip_header;
+    // Source IP, encoded by `ip_hash.h` functions.
+    struct in6_addr src_ip;
+    // Destination IP, encoded by `ip_hash.h` functions.
+    struct in6_addr dst_ip;
+    // Current VLAN detected.
+    // TODO: This can probably be removed since the packet dissector
+    // now finds this.
+    __be16 current_vlan;
+};
+
+// Constructor for a dissector
+// Connects XDP/TC SKB structure to a dissector structure.
+// Arguments:
+// * ctx - an xdp_md structure, passed from the entry-point
+// * dissector - pointer to a local dissector object to be initialized
+//
+// Returns TRUE if all is good, FALSE if the process cannot be completed
+static __always_inline bool tc_dissector_new(
+    struct __sk_buff *ctx, 
+    struct tc_dissector_t *dissector
+) {
+    dissector->ctx = ctx;
+    dissector->start = (void *)(long)ctx->data;
+    dissector->end = (void *)(long)ctx->data_end;
+    dissector->ethernet_header = (struct ethhdr *)NULL;
+    dissector->l3offset = 0;
+    dissector->current_vlan = bpf_htons(ctx->vlan_tci);
+
+    // Check that there's room for an ethernet header
+    if SKB_OVERFLOW (dissector->start, dissector->end, ethhdr)
+    {
+        return false;
+    }
+    dissector->ethernet_header = (struct ethhdr *)dissector->start;
+
+    return true;
+}
+
+// Search a context to find the layer-3 offset.
+static __always_inline bool tc_dissector_find_l3_offset(
+    struct tc_dissector_t *dissector
+) {
+    if (dissector->ethernet_header == NULL)
+    {
+        bpf_debug("Ethernet header is NULL, still called offset check.");
+        return false;
+    }
+    __u32 offset = sizeof(struct ethhdr);
+    __u16 eth_type = bpf_ntohs(dissector->ethernet_header->h_proto);
+
+    // Fast return for unwrapped IP
+    if (eth_type == ETH_P_IP || eth_type == ETH_P_IPV6)
+    {
+        dissector->eth_type = eth_type;
+        dissector->l3offset = offset;
+        return true;
+    }
+
+    // Fast return for ARP or non-802.3 ether types
+    if (eth_type == ETH_P_ARP || eth_type < ETH_P_802_3_MIN)
+    {
+        return false;
+    }
+
+    // Walk the headers until we find IP
+    __u8 i = 0;
+    while (i < 10 && !is_ip(eth_type))
+    {
+        switch (eth_type)
+        {
+        // Read inside VLAN headers
+        case ETH_P_8021AD:
+        case ETH_P_8021Q:
+        {
+            if SKB_OVERFLOW_OFFSET (dissector->start, dissector->end, 
+                offset, vlan_hdr)
+            {
+                return false;
+            }
+            //bpf_debug("TC Found VLAN");
+            struct vlan_hdr *vlan = (struct vlan_hdr *)
+                (dissector->start + offset);
+            // Calculated from the SKB
+            //dissector->current_vlan = vlan->h_vlan_TCI;
+            eth_type = bpf_ntohs(vlan->h_vlan_encapsulated_proto);
+            offset += sizeof(struct vlan_hdr);
+        }
+        break;
+
+        // Handle PPPoE
+        case ETH_P_PPP_SES:
+        {
+            if SKB_OVERFLOW_OFFSET (dissector->start, dissector->end, 
+                offset, pppoe_proto)
+            {
+                return false;
+            }
+            struct pppoe_proto *pppoe = (struct pppoe_proto *)
+                (dissector->start + offset);
+            __u16 proto = bpf_ntohs(pppoe->proto);
+            switch (proto)
+            {
+            case PPP_IP:
+                eth_type = ETH_P_IP;
+                break;
+            case PPP_IPV6:
+                eth_type = ETH_P_IPV6;
+                break;
+            default:
+                return false;
+            }
+            offset += PPPOE_SES_HLEN;
+        }
+        break;
+        // We found something we don't know how to handle - bail out
+        default:
+            return false;
+        }
+        ++i;
+    }
+
+    dissector->l3offset = offset;
+    dissector->eth_type = eth_type;
+    return true;
+}
+
+// Locate the IP header if present
+static __always_inline bool tc_dissector_find_ip_header(
+    struct tc_dissector_t *dissector
+) {
+    switch (dissector->eth_type)
+    {
+    case ETH_P_IP:
+    {
+        if (dissector->start + dissector->l3offset + 
+            sizeof(struct iphdr) > dissector->end) {
+                return false;
+        }
+        dissector->ip_header.iph = dissector->start + dissector->l3offset;
+        if (dissector->ip_header.iph + 1 > dissector->end) {
+            return false;
+        }
+        encode_ipv4(dissector->ip_header.iph->saddr, &dissector->src_ip);
+        encode_ipv4(dissector->ip_header.iph->daddr, &dissector->dst_ip);
+        return true;
+    }
+    break;
+    case ETH_P_IPV6:
+    {
+        if (dissector->start + dissector->l3offset + 
+            sizeof(struct ipv6hdr) > dissector->end) {
+                return false;
+        }
+        dissector->ip_header.ip6h = dissector->start + dissector->l3offset;
+        if (dissector->ip_header.iph + 1 > dissector->end)
+            return false;
+        encode_ipv6(&dissector->ip_header.ip6h->saddr, &dissector->src_ip);
+        encode_ipv6(&dissector->ip_header.ip6h->daddr, &dissector->dst_ip);
+        return true;
+    }
+    break;
+    default:
+        return false;
+    }
+}

src/rust/lqos_sys/src/bpf/common/ip_hash.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <linux/in6.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+
+// Provides hashing services for merging IPv4 and IPv6 addresses into
+// the same memory format.
+
+// Union that contains either a pointer to an IPv4 header or an IPv6
+// header. NULL if not present.
+// Note that you also need to keep track of the header type, since
+// accessing it directly without checking is undefined behavior.
+union iph_ptr
+{
+    // IPv4 Header
+    struct iphdr *iph;
+    // IPv6 Header
+    struct ipv6hdr *ip6h;
+};
+
+// Encodes an IPv4 address into an IPv6 address. All 0xFF except for the
+// last 32-bits.
+static __always_inline void encode_ipv4(
+    __be32 addr, 
+    struct in6_addr * out_address
+) {
+    __builtin_memset(&out_address->in6_u.u6_addr8, 0xFF, 16);
+    out_address->in6_u.u6_addr32[3] = addr;
+}
+
+// Encodes an IPv6 address into an IPv6 address. Unsurprisingly, that's
+// just a memcpy operation.
+static __always_inline void encode_ipv6(
+    struct in6_addr * ipv6_address, 
+    struct in6_addr * out_address
+) {
+    __builtin_memcpy(
+        &out_address->in6_u.u6_addr8, 
+        &ipv6_address->in6_u.u6_addr8, 
+        16
+    );
+}

src/rust/lqos_sys/src/bpf/common/lpm.h

@@ -0,0 +1,170 @@
+#pragma once
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+#include <linux/if_ether.h>
+#include <stdbool.h>
+#include <linux/in6.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include "maximums.h"
+#include "debug.h"
+#include "dissector.h"
+#include "dissector_tc.h"
+
+// Data structure used for map_ip_hash
+struct ip_hash_info {
+	__u32 cpu;
+	__u32 tc_handle; // TC handle MAJOR:MINOR combined in __u32
+};
+
+// Key type used for map_ip_hash trie
+struct ip_hash_key {
+	__u32 prefixlen; // Length of the prefix to match
+	struct in6_addr address; // An IPv6 address. IPv4 uses the last 32 bits.
+};
+
+// Map describing IP to CPU/TC mappings
+struct {
+	__uint(type, BPF_MAP_TYPE_LPM_TRIE);
+	__uint(max_entries, IP_HASH_ENTRIES_MAX);
+	__type(key, struct ip_hash_key);
+	__type(value, struct ip_hash_info);
+	__uint(pinning, LIBBPF_PIN_BY_NAME);
+	__uint(map_flags, BPF_F_NO_PREALLOC);
+} map_ip_to_cpu_and_tc SEC(".maps");
+
+// RECIPROCAL Map describing IP to CPU/TC mappings
+// If in "on a stick" mode, this is used to
+// fetch the UPLOAD mapping.
+struct {
+	__uint(type, BPF_MAP_TYPE_LPM_TRIE);
+	__uint(max_entries, IP_HASH_ENTRIES_MAX);
+	__type(key, struct ip_hash_key);
+	__type(value, struct ip_hash_info);
+	__uint(pinning, LIBBPF_PIN_BY_NAME);
+	__uint(map_flags, BPF_F_NO_PREALLOC);
+} map_ip_to_cpu_and_tc_recip SEC(".maps");
+
+// Performs an LPM lookup for an `ip_hash.h` encoded address, taking
+// into account redirection and "on a stick" setup.
+static __always_inline struct ip_hash_info * setup_lookup_key_and_tc_cpu(
+    // The "direction" constant from the main program. 1 = Internet,
+    // 2 = LAN, 3 = Figure it out from VLAN tags
+    int direction,
+    // Pointer to the "lookup key", which should contain the IP address
+    // to search for. Prefix length will be set for you.
+    struct ip_hash_key * lookup_key,
+    // Pointer to the traffic dissector.
+    struct dissector_t * dissector,
+    // Which VLAN represents the Internet, in redirection scenarios? (i.e.
+    // when direction == 3)
+    __be16 internet_vlan,
+    // Out variable setting the real "direction" of traffic when it has to
+    // be calculated.
+    int * out_effective_direction
+) 
+{
+    lookup_key->prefixlen = 128;
+    // Normal preset 2-interface setup, no need to calculate any direction
+    // related VLANs.
+    if (direction < 3) {
+        lookup_key->address = (direction == 1) ? dissector->dst_ip : 
+            dissector->src_ip;
+        *out_effective_direction = direction;
+        struct ip_hash_info * ip_info = bpf_map_lookup_elem(
+            &map_ip_to_cpu_and_tc, 
+            lookup_key
+        );
+        return ip_info;
+    } else {
+        if (dissector->current_vlan == internet_vlan) {
+            // Packet is coming IN from the Internet.
+            // Therefore it is download.
+            lookup_key->address = dissector->dst_ip;
+            *out_effective_direction = 1;
+            struct ip_hash_info * ip_info = bpf_map_lookup_elem(
+                &map_ip_to_cpu_and_tc, 
+                lookup_key
+            );
+            return ip_info;
+        } else {
+            // Packet is coming IN from the ISP.
+            // Therefore it is UPLOAD.
+            lookup_key->address = dissector->src_ip;
+            *out_effective_direction = 2;
+            struct ip_hash_info * ip_info = bpf_map_lookup_elem(
+                &map_ip_to_cpu_and_tc_recip, 
+                lookup_key
+            );
+            return ip_info;
+        }
+    }
+}
+
+// For the TC side, the dissector is different. Operates similarly to
+// `setup_lookup_key_and_tc_cpu`. Performs an LPM lookup for an `ip_hash.h` 
+// encoded address, taking into account redirection and "on a stick" setup.
+static __always_inline struct ip_hash_info * tc_setup_lookup_key_and_tc_cpu(
+    // The "direction" constant from the main program. 1 = Internet,
+    // 2 = LAN, 3 = Figure it out from VLAN tags
+    int direction,
+    // Pointer to the "lookup key", which should contain the IP address
+    // to search for. Prefix length will be set for you.
+    struct ip_hash_key * lookup_key, 
+    // Pointer to the traffic dissector.
+    struct tc_dissector_t * dissector,
+    // Which VLAN represents the Internet, in redirection scenarios? (i.e.
+    // when direction == 3)
+    __be16 internet_vlan,
+    // Out variable setting the real "direction" of traffic when it has to
+    // be calculated.
+    int * out_effective_direction
+) 
+{
+    lookup_key->prefixlen = 128;
+	// Direction is reversed because we are operating on egress
+    if (direction < 3) {
+        lookup_key->address = (direction == 1) ? dissector->src_ip : 
+            dissector->dst_ip;
+        *out_effective_direction = direction;
+        struct ip_hash_info * ip_info = bpf_map_lookup_elem(
+            &map_ip_to_cpu_and_tc, 
+            lookup_key
+        );
+        return ip_info;
+    } else {
+        //bpf_debug("Current VLAN (TC): %d", dissector->current_vlan);
+        //bpf_debug("Source: %x", dissector->src_ip.in6_u.u6_addr32[3]);
+        //bpf_debug("Dest: %x", dissector->dst_ip.in6_u.u6_addr32[3]);
+        if (dissector->current_vlan == internet_vlan) {
+            // Packet is going OUT to the Internet.
+            // Therefore, it is UPLOAD.
+            lookup_key->address = dissector->src_ip;
+            *out_effective_direction = 2;
+            //bpf_debug("Reciprocal lookup");
+            struct ip_hash_info * ip_info = bpf_map_lookup_elem(
+                &map_ip_to_cpu_and_tc_recip, 
+                lookup_key
+            );
+            return ip_info;
+        } else {
+            // Packet is going OUT to the LAN.
+            // Therefore, it is DOWNLOAD.
+            lookup_key->address = dissector->dst_ip;
+            *out_effective_direction = 1;
+            //bpf_debug("Forward lookup");
+            struct ip_hash_info * ip_info = bpf_map_lookup_elem(
+                &map_ip_to_cpu_and_tc, 
+                lookup_key
+            );
+            return ip_info;
+        }
+    }
+    struct ip_hash_info * ip_info = bpf_map_lookup_elem(
+        &map_ip_to_cpu_and_tc, 
+        lookup_key
+    );
+    return ip_info;
+}

src/rust/lqos_sys/src/bpf/common/maximums.h

@@ -0,0 +1,16 @@
+#pragma once
+
+// Maximum number of client IPs we are tracking
+#define MAX_TRACKED_IPS 128000
+
+// Maximum number of TC class mappings to support
+#define IP_HASH_ENTRIES_MAX	128000
+
+// Maximum number of supported CPUs
+#define MAX_CPUS 1024
+
+// Maximum number of TCP flows to track at once
+#define MAX_FLOWS IP_HASH_ENTRIES_MAX*2
+
+// Maximum number of packet pairs to track per flow.
+#define MAX_PACKETS MAX_FLOWS

src/rust/lqos_sys/src/bpf/common/skb_safety.h

@@ -0,0 +1,4 @@
+#pragma once
+
+#define SKB_OVERFLOW(start, end, T) ((void *)start + sizeof(struct T) > end)
+#define SKB_OVERFLOW_OFFSET(start, end, offset, T) (start + offset + sizeof(struct T) > end)

src/rust/lqos_sys/src/bpf/common/tc_classify_kern_pping_common.h

@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+Based on the GPLv2 xdp-pping project 
+(https://github.com/xdp-project/bpf-examples/tree/master/pping)
+
+xdp_pping is based on the ideas in Dr. Kathleen Nichols' pping
+utility: https://github.com/pollere/pping
+   and the papers around "Listening to Networks":
+http://www.pollere.net/Pdfdocs/ListeningGoog.pdf
+
+My modifications are Copyright 2022, Herbert Wolverson
+(Bracket Productions)
+*/
+/* Shared structures between userspace and kernel space
+ */
+#ifndef __TC_CLASSIFY_KERN_PPING_COMMON_H
+#define __TC_CLASSIFY_KERN_PPING_COMMON_H
+
+/* 30 second rotating performance buffer, per-TC handle */
+#define MAX_PERF_SECONDS 60
+#define NS_PER_MS            1000000UL
+#define NS_PER_MS_TIMES_100    10000UL
+#define NS_PER_SECOND NS_PER_MS 1000000000UL
+#define RECYCLE_RTT_INTERVAL 10000000000UL
+
+/* Quick way to access a TC handle as either two 16-bit numbers or a single u32 */
+union tc_handle_type
+{
+    __u32 handle;
+    __u16 majmin[2];
+};
+
+/*
+ * Struct that can hold the source or destination address for a flow (l3+l4).
+ * Works for both IPv4 and IPv6, as IPv4 addresses can be mapped to IPv6 ones
+ * based on RFC 4291 Section 2.5.5.2.
+ */
+struct flow_address
+{
+    struct in6_addr ip;
+    __u16 port;
+    __u16 reserved;
+};
+
+/*
+ * Struct to hold a full network tuple
+ * The ipv member is technically not necessary, but makes it easier to
+ * determine if saddr/daddr are IPv4 or IPv6 address (don't need to look at the
+ * first 12 bytes of address). The proto memeber is not currently used, but
+ * could be useful once pping is extended to work for other protocols than TCP.
+ *
+ * Note that I've removed proto, ipv and reserved.
+ */
+struct network_tuple
+{
+    struct flow_address saddr;
+    struct flow_address daddr;
+    __u16 proto; // IPPROTO_TCP, IPPROTO_ICMP, QUIC etc
+    __u8 ipv;    // AF_INET or AF_INET6
+    __u8 reserved;
+};
+
+/* Packet identifier */
+struct packet_id
+{
+    struct network_tuple flow;
+    __u32 identifier;
+};
+
+/* Ring-buffer of performance readings for each TC handle */
+struct rotating_performance
+{
+    __u32 rtt[MAX_PERF_SECONDS];
+    __u32 next_entry;
+    __u64 recycle_time;
+    __u32 has_fresh_data;
+};
+
+#endif /* __TC_CLASSIFY_KERN_PPING_COMMON_H */

src/rust/lqos_sys/src/bpf/common/tcp_rtt.h

@@ -0,0 +1,777 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+Based on the GPLv2 xdp-pping project 
+(https://github.com/xdp-project/bpf-examples/tree/master/pping)
+
+xdp_pping is based on the ideas in Dr. Kathleen Nichols' pping
+utility: https://github.com/pollere/pping
+   and the papers around "Listening to Networks":
+http://www.pollere.net/Pdfdocs/ListeningGoog.pdf
+
+My modifications are Copyright 2022, Herbert Wolverson
+(Bracket Productions)
+*/
+/* Shared structures between userspace and kernel space
+ */
+ 
+/* Implementation of pping inside the kernel
+ * classifier
+ */
+#ifndef __TC_CLASSIFY_KERN_PPING_H
+#define __TC_CLASSIFY_KERN_PPING_H
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <linux/pkt_cls.h>
+#include <linux/in.h>
+#include <linux/in6.h>
+#include <linux/if_ether.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/tcp.h>
+#include <bpf/bpf_endian.h>
+#include <stdbool.h>
+#include "tc_classify_kern_pping_common.h"
+#include "maximums.h"
+#include "debug.h"
+#include "ip_hash.h"
+#include "dissector_tc.h"
+
+#define MAX_MEMCMP_SIZE 128
+
+struct parsing_context
+{
+    struct tcphdr *tcp;
+    __u64 now;
+    struct tc_dissector_t * dissector;
+    struct in6_addr * active_host;
+};
+
+/* Event type recorded for a packet flow */
+enum __attribute__((__packed__)) flow_event_type
+{
+    FLOW_EVENT_NONE,
+    FLOW_EVENT_OPENING,
+    FLOW_EVENT_CLOSING,
+    FLOW_EVENT_CLOSING_BOTH
+};
+
+enum __attribute__((__packed__)) connection_state
+{
+    CONNECTION_STATE_EMPTY,
+    CONNECTION_STATE_WAITOPEN,
+    CONNECTION_STATE_OPEN,
+    CONNECTION_STATE_CLOSED
+};
+
+struct flow_state
+{
+    __u64 last_timestamp;
+    __u32 last_id;
+    __u32 outstanding_timestamps;
+    enum connection_state conn_state;
+    __u8 reserved[2];
+};
+
+/*
+ * Stores flowstate for both direction (src -> dst and dst -> src) of a flow
+ *
+ * Uses two named members instead of array of size 2 to avoid hassels with
+ * convincing verifier that member access is not out of bounds
+ */
+struct dual_flow_state
+{
+    struct flow_state dir1;
+    struct flow_state dir2;
+};
+
+/*
+ * Struct filled in by parse_packet_id.
+ *
+ * Note: As long as parse_packet_id is successful, the flow-parts of pid
+ * and reply_pid should be valid, regardless of value for pid_valid and
+ * reply_pid valid. The *pid_valid members are there to indicate that the
+ * identifier part of *pid are valid and can be used for timestamping/lookup.
+ * The reason for not keeping the flow parts as an entirely separate members
+ * is to save some performance by avoid doing a copy for lookup/insertion
+ * in the packet_ts map.
+ */
+struct packet_info
+{
+    __u64 time; // Arrival time of packet
+    //__u32 payload;              // Size of packet data (excluding headers)
+    struct packet_id pid;       // flow + identifier to timestamp (ex. TSval)
+    struct packet_id reply_pid; // rev. flow + identifier to match against (ex. TSecr)
+    //__u32 ingress_ifindex;      // Interface packet arrived on (if is_ingress, otherwise not valid)    
+    bool pid_flow_is_dfkey;              // Used to determine which member of dualflow state to use for forward direction
+    bool pid_valid;                      // identifier can be used to timestamp packet
+    bool reply_pid_valid;                // reply_identifier can be used to match packet
+    enum flow_event_type event_type;     // flow event triggered by packet
+};
+
+/*
+ * Struct filled in by protocol id parsers (ex. parse_tcp_identifier)
+ */
+struct protocol_info
+{
+    __u32 pid;
+    __u32 reply_pid;
+    bool pid_valid;
+    bool reply_pid_valid;
+    enum flow_event_type event_type;
+};
+
+
+
+/* Map Definitions */
+struct
+{
+    __uint(type, BPF_MAP_TYPE_LRU_HASH);
+    __type(key, struct packet_id);
+    __type(value, __u64);
+    __uint(max_entries, MAX_PACKETS);
+    __uint(pinning, LIBBPF_PIN_BY_NAME);
+//    __uint(map_flags, BPF_F_NO_PREALLOC);
+} packet_ts SEC(".maps");
+
+struct
+{
+    __uint(type, BPF_MAP_TYPE_LRU_HASH);
+    __type(key, struct network_tuple);
+    __type(value, struct dual_flow_state);
+    __uint(max_entries, MAX_FLOWS);
+    __uint(pinning, LIBBPF_PIN_BY_NAME);
+//    __uint(map_flags, BPF_F_NO_PREALLOC);
+} flow_state SEC(".maps");
+
+struct
+{
+    __uint(type, BPF_MAP_TYPE_LRU_HASH);
+    __type(key, struct in6_addr); // Keyed to the IP address
+    __type(value, struct rotating_performance);
+    __uint(max_entries, IP_HASH_ENTRIES_MAX);
+    __uint(pinning, LIBBPF_PIN_BY_NAME);
+//    __uint(map_flags, BPF_F_NO_PREALLOC);
+
+} rtt_tracker SEC(".maps");
+
+// Mask for IPv6 flowlabel + traffic class -  used in fib lookup
+#define IPV6_FLOWINFO_MASK __cpu_to_be32(0x0FFFFFFF)
+
+#ifndef AF_INET
+#define AF_INET 2
+#endif
+#ifndef AF_INET6
+#define AF_INET6 10
+#endif
+
+#define MAX_TCP_OPTIONS 10
+
+/* Functions */
+
+/*
+ * Convenience function for getting the corresponding reverse flow.
+ * PPing needs to keep track of flow in both directions, and sometimes
+ * also needs to reverse the flow to report the "correct" (consistent
+ * with Kathie's PPing) src and dest address.
+ */
+static __always_inline void reverse_flow(
+    struct network_tuple *dest, 
+    struct network_tuple *src
+) {
+    dest->ipv = src->ipv;
+    dest->proto = src->proto;
+    dest->saddr = src->daddr;
+    dest->daddr = src->saddr;
+    dest->reserved = 0;
+}
+
+/*
+ * Can't seem to get __builtin_memcmp to work, so hacking my own
+ *
+ * Based on https://githubhot.com/repo/iovisor/bcc/issues/3559,
+ * __builtin_memcmp should work constant size but I still get the "failed to
+ * find BTF for extern" error.
+ */
+static __always_inline int my_memcmp(
+    const void *s1_, 
+    const void *s2_, 
+    __u32 size
+) {
+    const __u8 *s1 = (const __u8 *)s1_, *s2 = (const __u8 *)s2_;
+    int i;
+
+    for (i = 0; i < MAX_MEMCMP_SIZE && i < size; i++)
+    {
+        if (s1[i] != s2[i])
+            return s1[i] > s2[i] ? 1 : -1;
+    }
+
+    return 0;
+}
+
+static __always_inline bool is_dualflow_key(struct network_tuple *flow)
+{
+    return my_memcmp(&flow->saddr, &flow->daddr, sizeof(flow->saddr)) <= 0;
+}
+
+static __always_inline struct flow_state *fstate_from_dfkey(
+    struct dual_flow_state *df_state,
+    bool is_dfkey
+) {
+    if (!df_state) {
+        return (struct flow_state *)NULL;
+    }
+
+    return is_dfkey ? &df_state->dir1 : &df_state->dir2;
+}
+
+/*
+ * Parses the TSval and TSecr values from the TCP options field. If sucessful
+ * the TSval and TSecr values will be stored at tsval and tsecr (in network
+ * byte order).
+ * Returns 0 if sucessful and -1 on failure
+ */
+static __always_inline int parse_tcp_ts(
+    struct tcphdr *tcph, 
+    void *data_end, 
+    __u32 *tsval,
+    __u32 *tsecr
+) {
+    int len = tcph->doff << 2;
+    void *opt_end = (void *)tcph + len;
+    __u8 *pos = (__u8 *)(tcph + 1); // Current pos in TCP options
+    __u8 i, opt;
+    volatile __u8
+        opt_size; // Seems to ensure it's always read of from stack as u8
+
+    if (tcph + 1 > data_end || len <= sizeof(struct tcphdr))
+        return -1;
+#pragma unroll // temporary solution until we can identify why the non-unrolled loop gets stuck in an infinite loop
+    for (i = 0; i < MAX_TCP_OPTIONS; i++)
+    {
+        if (pos + 1 > opt_end || pos + 1 > data_end)
+            return -1;
+
+        opt = *pos;
+        if (opt == 0) // Reached end of TCP options
+            return -1;
+
+        if (opt == 1)
+        { // TCP NOP option - advance one byte
+            pos++;
+            continue;
+        }
+
+        // Option > 1, should have option size
+        if (pos + 2 > opt_end || pos + 2 > data_end)
+            return -1;
+        opt_size = *(pos + 1);
+        if (opt_size < 2) // Stop parsing options if opt_size has an invalid value
+            return -1;
+
+        // Option-kind is TCP timestap (yey!)
+        if (opt == 8 && opt_size == 10)
+        {
+            if (pos + 10 > opt_end || pos + 10 > data_end)
+                return -1;
+            *tsval = bpf_ntohl(*(__u32 *)(pos + 2));
+            *tsecr = bpf_ntohl(*(__u32 *)(pos + 6));
+            return 0;
+        }
+
+        // Some other TCP option - advance option-length bytes
+        pos += opt_size;
+    }
+    return -1;
+}
+
+/*
+ * Attempts to fetch an identifier for TCP packets, based on the TCP timestamp
+ * option.
+ *
+ * Will use the TSval as pid and TSecr as reply_pid, and the TCP source and dest
+ * as port numbers.
+ *
+ * If successful, tcph, sport, dport and proto_info will be set
+ * appropriately and 0 will be returned.
+ * On failure -1 will be returned (and arguments will not be set).
+ */
+static __always_inline int parse_tcp_identifier(
+    struct parsing_context *context,
+    __u16 *sport,
+    __u16 *dport, 
+    struct protocol_info *proto_info
+) {
+    if (parse_tcp_ts(context->tcp, context->dissector->end, &proto_info->pid, 
+        &proto_info->reply_pid) < 0) {
+        return -1; // Possible TODO, fall back on seq/ack instead
+    }
+
+    // Do not timestamp pure ACKs (no payload)
+    void *nh_pos = (context->tcp + 1) + (context->tcp->doff << 2);
+    proto_info->pid_valid = nh_pos - context->dissector->start < context->dissector->ctx->len || context->tcp->syn;
+
+    // Do not match on non-ACKs (TSecr not valid)
+    proto_info->reply_pid_valid = context->tcp->ack;
+
+    // Check if connection is opening/closing
+    if (context->tcp->rst)
+    {
+        proto_info->event_type = FLOW_EVENT_CLOSING_BOTH;
+    }
+    else if (context->tcp->fin)
+    {
+        proto_info->event_type = FLOW_EVENT_CLOSING;
+    }
+    else if (context->tcp->syn)
+    {
+        proto_info->event_type = FLOW_EVENT_OPENING;
+    }
+    else
+    {
+        proto_info->event_type = FLOW_EVENT_NONE;
+    }
+
+    *sport = bpf_ntohs(context->tcp->dest);
+    *dport = bpf_ntohs(context->tcp->source);
+
+    return 0;
+}
+
+/* This is a bit of a hackjob from the original */
+static __always_inline int parse_packet_identifier(
+    struct parsing_context *context, 
+    struct packet_info *p_info
+) {
+    p_info->time = context->now;
+    if (context->dissector->eth_type == ETH_P_IP)
+    {
+        p_info->pid.flow.ipv = AF_INET;
+        p_info->pid.flow.saddr.ip = context->dissector->src_ip;
+        p_info->pid.flow.daddr.ip = context->dissector->dst_ip;
+    }
+    else if (context->dissector->eth_type == ETH_P_IPV6)
+    {
+        p_info->pid.flow.ipv = AF_INET6;
+        p_info->pid.flow.saddr.ip = context->dissector->src_ip;
+        p_info->pid.flow.daddr.ip = context->dissector->dst_ip;
+    }
+    else
+    {
+        bpf_debug("Unknown protocol");
+        return -1;
+    }
+    //bpf_debug("IPs: %u %u", p_info->pid.flow.saddr.ip.in6_u.u6_addr32[3], p_info->pid.flow.daddr.ip.in6_u.u6_addr32[3]);
+
+    struct protocol_info proto_info;
+    int err = parse_tcp_identifier(context,
+                                   &p_info->pid.flow.saddr.port,
+                                   &p_info->pid.flow.daddr.port,
+                                   &proto_info);
+    if (err)
+        return -1;
+    //bpf_debug("Ports: %u %u", p_info->pid.flow.saddr.port, p_info->pid.flow.daddr.port);
+
+    // Sucessfully parsed packet identifier - fill in remaining members and return
+    p_info->pid.identifier = proto_info.pid;
+    p_info->pid_valid = proto_info.pid_valid;
+    p_info->reply_pid.identifier = proto_info.reply_pid;
+    p_info->reply_pid_valid = proto_info.reply_pid_valid;
+    p_info->event_type = proto_info.event_type;
+
+    if (p_info->pid.flow.ipv == AF_INET && p_info->pid.flow.ipv == AF_INET6) {
+        bpf_debug("Unknown internal protocol");
+        return -1;
+    }
+
+    p_info->pid_flow_is_dfkey = is_dualflow_key(&p_info->pid.flow);
+
+    reverse_flow(&p_info->reply_pid.flow, &p_info->pid.flow);
+
+    return 0;
+}
+
+static __always_inline struct network_tuple *
+get_dualflow_key_from_packet(struct packet_info *p_info)
+{
+    return p_info->pid_flow_is_dfkey ? &p_info->pid.flow : &p_info->reply_pid.flow;
+}
+
+/*
+ * Initilizes an "empty" flow state based on the forward direction of the
+ * current packet
+ */
+static __always_inline void init_flowstate(struct flow_state *f_state,
+                                           struct packet_info *p_info)
+{
+    f_state->conn_state = CONNECTION_STATE_WAITOPEN;
+    f_state->last_timestamp = p_info->time;
+}
+
+static __always_inline void init_empty_flowstate(struct flow_state *f_state)
+{
+    f_state->conn_state = CONNECTION_STATE_EMPTY;
+}
+
+static __always_inline struct flow_state *
+get_flowstate_from_packet(struct dual_flow_state *df_state,
+                          struct packet_info *p_info)
+{
+    return fstate_from_dfkey(df_state, p_info->pid_flow_is_dfkey);
+}
+
+static __always_inline struct flow_state *
+get_reverse_flowstate_from_packet(struct dual_flow_state *df_state,
+                                  struct packet_info *p_info)
+{
+    return fstate_from_dfkey(df_state, !p_info->pid_flow_is_dfkey);
+}
+
+/*
+ * Initilize a new (assumed 0-initlized) dual flow state based on the current
+ * packet.
+ */
+static __always_inline void init_dualflow_state(
+    struct dual_flow_state *df_state,
+    struct packet_info *p_info
+) {
+    struct flow_state *fw_state =
+        get_flowstate_from_packet(df_state, p_info);
+    struct flow_state *rev_state =
+        get_reverse_flowstate_from_packet(df_state, p_info);
+
+    init_flowstate(fw_state, p_info);
+    init_empty_flowstate(rev_state);
+}
+
+static __always_inline struct dual_flow_state *
+create_dualflow_state(
+    struct parsing_context *ctx, 
+    struct packet_info *p_info, 
+    bool *new_flow
+) {
+    struct network_tuple *key = get_dualflow_key_from_packet(p_info);
+    struct dual_flow_state new_state = {0};
+
+    init_dualflow_state(&new_state, p_info);
+    //new_state.dir1.tc_handle.handle = ctx->tc_handle;
+    //new_state.dir2.tc_handle.handle = ctx->tc_handle;
+
+    if (bpf_map_update_elem(&flow_state, key, &new_state, BPF_NOEXIST) ==
+        0)
+    {
+        if (new_flow)
+            *new_flow = true;
+    }
+    else
+    {
+        return (struct dual_flow_state *)NULL;
+    }
+
+    return (struct dual_flow_state *)bpf_map_lookup_elem(&flow_state, key);
+}
+
+static __always_inline struct dual_flow_state *
+lookup_or_create_dualflow_state(
+    struct parsing_context *ctx, 
+    struct packet_info *p_info, 
+    bool *new_flow
+) {
+    struct dual_flow_state *df_state;
+
+    struct network_tuple *key = get_dualflow_key_from_packet(p_info);
+    df_state = (struct dual_flow_state *)bpf_map_lookup_elem(&flow_state, key);
+
+    if (df_state)
+    {
+        return df_state;
+    }
+
+    // Only try to create new state if we have a valid pid
+    if (!p_info->pid_valid || p_info->event_type == FLOW_EVENT_CLOSING ||
+        p_info->event_type == FLOW_EVENT_CLOSING_BOTH)
+        return (struct dual_flow_state *)NULL;
+
+    return create_dualflow_state(ctx, p_info, new_flow);
+}
+
+static __always_inline bool is_flowstate_active(struct flow_state *f_state)
+{
+    return f_state->conn_state != CONNECTION_STATE_EMPTY &&
+           f_state->conn_state != CONNECTION_STATE_CLOSED;
+}
+
+static __always_inline void update_forward_flowstate(
+    struct packet_info *p_info,
+    struct flow_state *f_state, 
+    bool *new_flow
+) {
+    // "Create" flowstate if it's empty
+    if (f_state->conn_state == CONNECTION_STATE_EMPTY &&
+        p_info->pid_valid)
+    {
+        init_flowstate(f_state, p_info);
+        if (new_flow)
+            *new_flow = true;
+    }
+}
+
+static __always_inline void update_reverse_flowstate(
+    void *ctx, 
+    struct packet_info *p_info,
+    struct flow_state *f_state
+) {
+    if (!is_flowstate_active(f_state))
+        return;
+
+    // First time we see reply for flow?
+    if (f_state->conn_state == CONNECTION_STATE_WAITOPEN &&
+        p_info->event_type != FLOW_EVENT_CLOSING_BOTH)
+    {
+        f_state->conn_state = CONNECTION_STATE_OPEN;
+    }
+}
+
+static __always_inline bool is_new_identifier(
+    struct packet_id *pid, 
+    struct flow_state *f_state
+) {
+    if (pid->flow.proto == IPPROTO_TCP)
+        /* TCP timestamps should be monotonically non-decreasing
+         * Check that pid > last_ts (considering wrap around) by
+         * checking 0 < pid - last_ts < 2^31 as specified by
+         * RFC7323 Section 5.2*/
+        return pid->identifier - f_state->last_id > 0 &&
+               pid->identifier - f_state->last_id < 1UL << 31;
+
+    return pid->identifier != f_state->last_id;
+}
+
+static __always_inline bool is_rate_limited(__u64 now, __u64 last_ts)
+{
+    if (now < last_ts)
+        return true;
+
+    // Static rate limit
+    //return now - last_ts < DELAY_BETWEEN_RTT_REPORTS_MS * NS_PER_MS;
+    return false; // Max firehose drinking speed
+}
+
+/*
+ * Attempt to create a timestamp-entry for packet p_info for flow in f_state
+ */
+static __always_inline void pping_timestamp_packet(
+    struct flow_state *f_state, 
+    void *ctx,
+    struct packet_info *p_info, 
+    bool new_flow
+) {
+    if (!is_flowstate_active(f_state) || !p_info->pid_valid)
+        return;
+
+    // Check if identfier is new
+    if (!new_flow && !is_new_identifier(&p_info->pid, f_state))
+        return;
+    f_state->last_id = p_info->pid.identifier;
+
+    // Check rate-limit
+    if (!new_flow && is_rate_limited(p_info->time, f_state->last_timestamp))
+        return;
+
+    /*
+     * Updates attempt at creating timestamp, even if creation of timestamp
+     * fails (due to map being full). This should make the competition for
+     * the next available map slot somewhat fairer between heavy and sparse
+     * flows.
+     */
+    f_state->last_timestamp = p_info->time;
+
+    if (bpf_map_update_elem(&packet_ts, &p_info->pid, &p_info->time,
+                            BPF_NOEXIST) == 0)
+        __sync_fetch_and_add(&f_state->outstanding_timestamps, 1);
+}
+
+/*
+ * Attempt to match packet in p_info with a timestamp from flow in f_state
+ */
+static __always_inline void pping_match_packet(struct flow_state *f_state,
+                                               struct packet_info *p_info,
+                                               struct in6_addr *active_host)
+{
+    __u64 *p_ts;
+
+    if (!is_flowstate_active(f_state) || !p_info->reply_pid_valid)
+        return;
+
+    if (f_state->outstanding_timestamps == 0)
+        return;
+
+    p_ts = (__u64 *)bpf_map_lookup_elem(&packet_ts, &p_info->reply_pid);
+    if (!p_ts || p_info->time < *p_ts)
+        return;
+
+    __u64 rtt = (p_info->time - *p_ts) / NS_PER_MS_TIMES_100;
+
+    // Delete timestamp entry as soon as RTT is calculated
+    if (bpf_map_delete_elem(&packet_ts, &p_info->reply_pid) == 0)
+    {
+        __sync_fetch_and_add(&f_state->outstanding_timestamps, -1);
+    }
+
+    // Update the most performance map to include this data
+    struct rotating_performance *perf = 
+        (struct rotating_performance *)bpf_map_lookup_elem(
+            &rtt_tracker, active_host);
+    if (perf == NULL) return;
+    __sync_fetch_and_add(&perf->next_entry, 1);
+    __u32 next_entry = perf->next_entry;
+    if (next_entry < MAX_PERF_SECONDS) {
+        __sync_fetch_and_add(&perf->rtt[next_entry], rtt);
+        perf->has_fresh_data = 1;
+    }
+}
+
+static __always_inline void close_and_delete_flows(
+    void *ctx, 
+    struct packet_info *p_info,
+    struct flow_state *fw_flow,
+    struct flow_state *rev_flow
+) {
+    // Forward flow closing
+    if (p_info->event_type == FLOW_EVENT_CLOSING ||
+        p_info->event_type == FLOW_EVENT_CLOSING_BOTH)
+    {
+        fw_flow->conn_state = CONNECTION_STATE_CLOSED;
+    }
+
+    // Reverse flow closing
+    if (p_info->event_type == FLOW_EVENT_CLOSING_BOTH)
+    {
+        rev_flow->conn_state = CONNECTION_STATE_CLOSED;
+    }
+
+    // Delete flowstate entry if neither flow is open anymore
+    if (!is_flowstate_active(fw_flow) && !is_flowstate_active(rev_flow))
+    {
+        bpf_map_delete_elem(&flow_state, get_dualflow_key_from_packet(p_info));
+    }
+}
+
+/*
+ * Contains the actual pping logic that is applied after a packet has been
+ * parsed and deemed to contain some valid identifier.
+ * Looks up and updates flowstate (in both directions), tries to save a
+ * timestamp of the packet, tries to match packet against previous timestamps,
+ * calculates RTTs and pushes messages to userspace as appropriate.
+ */
+static __always_inline void pping_parsed_packet(
+    struct parsing_context *context, 
+    struct packet_info *p_info
+) {
+    struct dual_flow_state *df_state;
+    struct flow_state *fw_flow, *rev_flow;
+    bool new_flow = false;
+
+    df_state = lookup_or_create_dualflow_state(context, p_info, &new_flow);
+    if (!df_state)
+    {
+        // bpf_debug("No flow state - stop");
+        return;
+    }
+
+    fw_flow = get_flowstate_from_packet(df_state, p_info);
+    update_forward_flowstate(p_info, fw_flow, &new_flow);
+    pping_timestamp_packet(fw_flow, context, p_info, new_flow);
+
+    rev_flow = get_reverse_flowstate_from_packet(df_state, p_info);
+    update_reverse_flowstate(context, p_info, rev_flow);
+    pping_match_packet(rev_flow, p_info, context->active_host);
+
+    close_and_delete_flows(context, p_info, fw_flow, rev_flow);
+}
+
+/* Entry poing for running pping in the tc context */
+static __always_inline void tc_pping_start(struct parsing_context *context)
+{
+    // Check to see if we can store perf info. Bail if we've hit the limit.
+    // Copying occurs because otherwise the validator complains.
+    struct rotating_performance *perf = 
+        (struct rotating_performance *)bpf_map_lookup_elem(
+            &rtt_tracker, context->active_host);
+    if (perf) {
+        if (perf->next_entry >= MAX_PERF_SECONDS-1) {
+            //bpf_debug("Flow has max samples. Not sampling further until next reset.");
+            //for (int i=0; i<MAX_PERF_SECONDS; ++i) {
+            //    bpf_debug("%u", perf->rtt[i]);
+            //}
+            if (context->now > perf->recycle_time) {
+                // If the time-to-live for the sample is exceeded, recycle it to be
+                // usable again.
+                //bpf_debug("Recycling flow, %u > %u", context->now, perf->recycle_time);
+                __builtin_memset(perf->rtt, 0, sizeof(__u32) * MAX_PERF_SECONDS);
+                perf->recycle_time = context->now + RECYCLE_RTT_INTERVAL;
+                perf->next_entry = 0;
+                perf->has_fresh_data = 0;
+            }
+            return;
+        }
+    }
+
+    // Populate the TCP Header
+    if (context->dissector->eth_type == ETH_P_IP)
+    {
+        // If its not TCP, stop
+        if (context->dissector->ip_header.iph + 1 > context->dissector->end)
+            return; // Stops the error checking from crashing
+        if (context->dissector->ip_header.iph->protocol != IPPROTO_TCP)
+        {
+            return;
+        }
+        context->tcp = (struct tcphdr *)((char *)context->dissector->ip_header.iph + (context->dissector->ip_header.iph->ihl * 4));
+    }
+    else if (context->dissector->eth_type == ETH_P_IPV6)
+    {
+        // If its not TCP, stop
+        if (context->dissector->ip_header.ip6h + 1 > context->dissector->end)
+            return; // Stops the error checking from crashing
+        if (context->dissector->ip_header.ip6h->nexthdr != IPPROTO_TCP)
+        {
+            return;
+        }
+        context->tcp = (struct tcphdr *)(context->dissector->ip_header.ip6h + 1);
+    }
+    else
+    {
+        bpf_debug("UNKNOWN PROTOCOL TYPE");
+        return;
+    }
+
+    // Bail out if the packet is incomplete
+    if (context->tcp + 1 > context->dissector->end)
+    {
+        return;
+    }
+
+    // If we didn't get a handle, make one
+    if (perf == NULL)
+    {
+        struct rotating_performance new_perf = {0};
+        new_perf.recycle_time = context->now + RECYCLE_RTT_INTERVAL;
+        new_perf.has_fresh_data = 0;
+        if (bpf_map_update_elem(&rtt_tracker, context->active_host, &new_perf, BPF_NOEXIST) != 0) return;
+    }
+
+
+    // Start the parsing process
+    struct packet_info p_info = {0};
+    if (parse_packet_identifier(context, &p_info) < 0)
+    {
+        //bpf_debug("Unable to parse packet identifier");
+        return;
+    }
+
+    pping_parsed_packet(context, &p_info);
+}
+
+#endif /* __TC_CLASSIFY_KERN_PPING_H */

src/rust/lqos_sys/src/bpf/common/throughput.h

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+#include <linux/if_ether.h>
+#include <stdbool.h>
+#include "maximums.h"
+#include "debug.h"
+
+// Counter for each host
+struct host_counter {
+    __u64 download_bytes;
+    __u64 upload_bytes;
+    __u64 download_packets;
+    __u64 upload_packets;
+    __u32 tc_handle;
+};
+
+// Pinned map storing counters per host. its an LRU structure: if it
+// runs out of space, the least recently seen host will be removed.
+struct
+{
+	__uint(type, BPF_MAP_TYPE_LRU_PERCPU_HASH);
+	__type(key, struct in6_addr);
+	__type(value, struct host_counter);
+    __uint(max_entries, MAX_TRACKED_IPS);
+	__uint(pinning, LIBBPF_PIN_BY_NAME);
+} map_traffic SEC(".maps");
+
+static __always_inline void track_traffic(
+    int direction, 
+    struct in6_addr * key, 
+    __u32 size, 
+    __u32 tc_handle
+) {
+    // Count the bits. It's per-CPU, so we can't be interrupted - no sync required
+    struct host_counter * counter = 
+        (struct host_counter *)bpf_map_lookup_elem(&map_traffic, key);
+    if (counter) {
+        if (direction == 1) {
+            // Download
+            counter->download_packets += 1;
+            counter->download_bytes += size;
+            counter->tc_handle = tc_handle;
+        } else {
+            // Upload
+            counter->upload_packets += 1;
+            counter->upload_bytes += size;
+            counter->tc_handle = tc_handle;
+        }
+    } else {
+        struct host_counter new_host = {0};
+        new_host.tc_handle = tc_handle;
+        if (direction == 1) {
+            new_host.download_packets = 1;
+            new_host.download_bytes = size;
+            new_host.upload_bytes = 0;
+            new_host.upload_packets = 0;
+        } else {
+            new_host.upload_packets = 1;
+            new_host.upload_bytes = size;
+            new_host.download_bytes = 0;
+            new_host.download_packets = 0;
+        }
+        if (bpf_map_update_elem(&map_traffic, key, &new_host, BPF_NOEXIST) != 0) {
+            bpf_debug("Failed to insert flow");
+        }
+    }
+}

src/rust/lqos_sys/src/bpf/lqos_kern.c

@@ -0,0 +1,309 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+// Minimal XDP program that passes all packets.
+// Used to verify XDP functionality.
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <linux/in6.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/pkt_cls.h>
+#include <linux/pkt_sched.h> /* TC_H_MAJ + TC_H_MIN */
+#include "common/debug.h"
+#include "common/dissector.h"
+#include "common/dissector_tc.h"
+#include "common/maximums.h"
+#include "common/throughput.h"
+#include "common/lpm.h"
+#include "common/cpu_map.h"
+#include "common/tcp_rtt.h"
+#include "common/bifrost.h"
+
+/* Theory of operation:
+1. (Packet arrives at interface)
+2. XDP (ingress) starts
+  * Check that "direction" is set and any VLAN mappings
+  * Dissect the packet to find VLANs and L3 offset
+      * If VLAN redirection is enabled, change VLAN tags
+      * to swap ingress/egress VLANs.
+  * Perform LPM lookup to determine CPU destination
+  * Track traffic totals
+  * Perform CPU redirection
+3. TC (ingress) starts
+  * If interface redirection is enabled, bypass the bridge
+    and redirect to the outbound interface.
+  * If VLAN redirection has happened, ONLY redirect if
+    there is a VLAN tag to avoid STP loops.
+4. TC (egress) starts on the outbound interface
+  * LPM lookup to find TC handle
+  * If TCP, track RTT via ringbuffer and sampling
+  * Send TC redirect to track at the appropriate handle.
+*/
+
+// Constant passed in during loading to either
+// 1 (facing the Internet)
+// 2 (facing the LAN)
+// 3 (use VLAN mode, we're running on a stick)
+// If it stays at 255, we have a configuration error.
+int direction = 255;
+
+// Also configured during loading. For "on a stick" support,
+// these are mapped to the respective VLAN facing directions.
+__be16 internet_vlan = 0; // Note: turn these into big-endian
+__be16 isp_vlan = 0;
+
+// XDP Entry Point
+SEC("xdp")
+int xdp_prog(struct xdp_md *ctx)
+{
+#ifdef VERBOSE
+    bpf_debug("XDP-RDR");
+#endif
+    if (direction == 255) {
+        bpf_debug("Error: interface direction unspecified, aborting.");
+        return XDP_PASS;
+    }
+
+    // Do we need to perform a VLAN redirect?
+    bool vlan_redirect = false;
+    { // Note: scope for removing temporaries from the stack
+        __u32 my_interface = ctx->ingress_ifindex;
+        struct bifrost_interface * redirect_info = NULL;
+        redirect_info = bpf_map_lookup_elem(
+            &bifrost_interface_map, 
+            &my_interface
+        );
+        if (redirect_info) {
+            // If we have a redirect, mark it - the dissector will
+            // apply it
+            vlan_redirect = true;
+#ifdef VERBOSE
+            bpf_debug("(XDP) VLAN redirection requested for this interface");
+#endif
+        }
+    }
+
+    struct dissector_t dissector = {0};
+#ifdef VERBOSE
+    bpf_debug("(XDP) START XDP");
+    bpf_debug("(XDP) Running mode %u", direction);
+    bpf_debug("(XDP) Scan VLANs: %u %u", internet_vlan, isp_vlan);
+#endif
+    // If the dissector is unable to figure out what's going on, bail
+    // out.
+    if (!dissector_new(ctx, &dissector)) return XDP_PASS;
+    // Note that this step rewrites the VLAN tag if redirection
+    // is requested.
+    if (!dissector_find_l3_offset(&dissector, vlan_redirect)) return XDP_PASS;
+    if (!dissector_find_ip_header(&dissector)) return XDP_PASS;
+#ifdef VERBOSE
+    bpf_debug("(XDP) Spotted VLAN: %u", dissector.current_vlan);
+#endif
+
+    // Determine the lookup key by direction
+    struct ip_hash_key lookup_key;
+    int effective_direction = 0;
+    struct ip_hash_info * ip_info = setup_lookup_key_and_tc_cpu(
+        direction, 
+        &lookup_key, 
+        &dissector, 
+        internet_vlan, 
+        &effective_direction
+    );
+#ifdef VERBOSE
+    bpf_debug("(XDP) Effective direction: %d", effective_direction);
+#endif
+
+    // Find the desired TC handle and CPU target
+    __u32 tc_handle = 0;
+    __u32 cpu = 0;
+    if (ip_info) {
+        tc_handle = ip_info->tc_handle;
+        cpu = ip_info->cpu;
+    }
+    // Update the traffic tracking buffers
+    track_traffic(
+        effective_direction, 
+        &lookup_key.address, 
+        ctx->data_end - ctx->data, // end - data = length
+        tc_handle
+    );
+
+    // Send on its way
+    if (tc_handle != 0) {
+        // Handle CPU redirection if there is one specified
+        __u32 *cpu_lookup;
+        cpu_lookup = bpf_map_lookup_elem(&cpus_available, &cpu);
+        if (!cpu_lookup) {
+            bpf_debug("Error: CPU %u is not mapped", cpu);
+            return XDP_PASS; // No CPU found
+        }
+        __u32 cpu_dest = *cpu_lookup;
+
+        // Redirect based on CPU
+#ifdef VERBOSE
+        bpf_debug("(XDP) Zooming to CPU: %u", cpu_dest);
+        bpf_debug("(XDP) Mapped to handle: %u", tc_handle);
+#endif
+        long redirect_result = bpf_redirect_map(&cpu_map, cpu_dest, 0);
+#ifdef VERBOSE
+        bpf_debug("(XDP) Redirect result: %u", redirect_result);
+#endif
+        return redirect_result;
+    }
+	return XDP_PASS;
+}
+
+// TC-Egress Entry Point
+SEC("tc")
+int tc_iphash_to_cpu(struct __sk_buff *skb)
+{
+#ifdef VERBOSE
+    bpf_debug("TC-MAP");
+#endif
+    if (direction == 255) {
+        bpf_debug("(TC) Error: interface direction unspecified, aborting.");
+        return TC_ACT_OK;
+    }
+#ifdef VERBOSE
+    bpf_debug("(TC) SKB VLAN TCI: %u", skb->vlan_tci);    
+#endif
+
+    __u32 cpu = bpf_get_smp_processor_id();
+
+    // Lookup the queue
+    {
+        struct txq_config *txq_cfg;
+        txq_cfg = bpf_map_lookup_elem(&map_txq_config, &cpu);
+        if (!txq_cfg) return TC_ACT_SHOT;
+        if (txq_cfg->queue_mapping != 0) {
+            skb->queue_mapping = txq_cfg->queue_mapping;
+        } else {
+            bpf_debug("(TC) Misconf: CPU:%u no conf (curr qm:%d)\n", 
+                cpu, skb->queue_mapping);
+        }
+    } // Scope to remove tcq_cfg when done with it
+
+    // Once again parse the packet
+    // Note that we are returning OK on failure, which is a little odd.
+    // The reasoning being that if its a packet we don't know how to handle,
+    // we probably don't want to drop it - to ensure that IS-IS, ARP, STP
+    // and other packet types are still handled by the default queues.
+    struct tc_dissector_t dissector = {0};
+    if (!tc_dissector_new(skb, &dissector)) return TC_ACT_OK;
+    if (!tc_dissector_find_l3_offset(&dissector)) return TC_ACT_OK;
+    if (!tc_dissector_find_ip_header(&dissector)) return TC_ACT_OK;
+
+    // Determine the lookup key by direction
+    struct ip_hash_key lookup_key;
+    int effective_direction = 0;
+    struct ip_hash_info * ip_info = tc_setup_lookup_key_and_tc_cpu(
+        direction, 
+        &lookup_key, 
+        &dissector, 
+        internet_vlan, 
+        &effective_direction
+    );
+#ifdef VERBOSE
+    bpf_debug("(TC) effective direction: %d", effective_direction);
+#endif
+
+    // Call pping to obtain RTT times
+    struct parsing_context context = {0};
+    context.now = bpf_ktime_get_ns();
+    context.tcp = NULL;
+    context.dissector = &dissector;
+    context.active_host = &lookup_key.address;
+    tc_pping_start(&context);
+
+    if (ip_info && ip_info->tc_handle != 0) {
+        // We found a matching mapped TC flow
+#ifdef VERBOSE
+        bpf_debug("(TC) Mapped to TC handle %x", ip_info->tc_handle);
+#endif
+        skb->priority = ip_info->tc_handle;
+        return TC_ACT_OK;
+    } else {
+        // We didn't find anything
+#ifdef VERBOSE
+        bpf_debug("(TC) didn't map anything");
+#endif
+        return TC_ACT_OK;
+    }
+
+    return TC_ACT_OK;
+}
+
+// Helper function to call the bpf_redirect function and note
+// errors from the TC-egress context.
+static __always_inline long do_tc_redirect(__u32 target) {
+    //bpf_debug("Packet would have been redirected to ifindex %u", target);
+    //return TC_ACT_UNSPEC; // Don't actually redirect, we're testing
+    long ret = bpf_redirect(target, 0);
+    if (ret != TC_ACT_REDIRECT) {
+        bpf_debug("(TC-IN) TC Redirect call failed");
+        return TC_ACT_UNSPEC;
+    } else {
+        return ret;
+    }
+}
+
+// TC-Ingress entry-point. eBPF Bridge ("bifrost")
+SEC("tc")
+int bifrost(struct __sk_buff *skb)
+{
+#ifdef VERBOSE
+    bpf_debug("TC-Ingress invoked on interface: %u . %u", 
+        skb->ifindex, skb->vlan_tci);
+#endif
+    // Lookup to see if we have redirection setup
+    struct bifrost_interface * redirect_info = NULL;
+    __u32 my_interface = skb->ifindex;
+    redirect_info = bpf_map_lookup_elem(&bifrost_interface_map, &my_interface);
+    if (redirect_info) {
+#ifdef VERBOSE
+        bpf_debug("(TC-IN) Redirect info: to: %u, scan vlans: %d", 
+            redirect_info->redirect_to, redirect_info->scan_vlans);
+#endif
+
+        if (redirect_info->scan_vlans) {
+            // We are in VLAN redirect mode. If VLAN redirection is required,
+            // it already happened in the XDP stage (rewriting the header).
+            //
+            // We need to ONLY redirect if we have tagged packets, otherwise
+            // we create STP loops and Bad Things (TM) happen.
+            if (skb->vlan_tci > 0) {
+#ifdef VERBOSE
+                bpf_debug("(TC-IN) Redirecting back to same interface, \
+                    VLAN %u", skb->vlan_tci);
+#endif                
+                return do_tc_redirect(redirect_info->redirect_to);
+            } else {
+#ifdef VERBOSE
+                bpf_debug("(TC-IN) Not redirecting: No VLAN tag, bare \
+                    redirect unsupported in VLAN mode.");
+#endif
+                return TC_ACT_UNSPEC;
+            }
+        } else {
+            // We're in regular redirect mode. So if we aren't trying to send
+            // a packet out via the interface it arrived, we can redirect.
+            if (skb->ifindex == redirect_info->redirect_to) {
+#ifdef VERBOSE
+                bpf_debug("(TC-IN) Not redirecting: src and dst are the \
+                same.");
+#endif
+                return TC_ACT_UNSPEC;
+            } else {
+                return do_tc_redirect(redirect_info->redirect_to);
+            }
+        }
+    } else {
+#ifdef VERBOSE
+        bpf_debug("(TC-IN) No matching redirect record for interface %u", 
+        my_interface);
+#endif
+    }
+    return TC_ACT_UNSPEC;
+}
+
+char _license[] SEC("license") = "GPL";

src/rust/lqos_sys/src/bpf/wrapper.c

@@ -0,0 +1,306 @@
+#include "wrapper.h"
+#include "common/maximums.h"
+
+struct lqos_kern * lqos_kern_open() {
+    return lqos_kern__open();
+}
+
+int lqos_kern_load(struct lqos_kern * skel) {
+    return lqos_kern__load(skel);
+}
+
+extern __u64 max_tracker_ips() {
+	return MAX_TRACKED_IPS;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////
+// The following is derived from
+// https://github.com/xdp-project/bpf-examples/blob/master/tc-policy/tc_txq_policy.c
+// It needs converting to Rust, but I wanted to get something
+// working relatively quickly.
+
+#include <linux/bpf.h>
+#include <bpf/libbpf.h>
+#include <bpf/bpf.h>
+
+#define EGRESS_HANDLE		0x1;
+#define EGRESS_PRIORITY 	0xC02F;
+
+int teardown_hook(int ifindex, const char * ifname, bool verbose)
+{
+	DECLARE_LIBBPF_OPTS(bpf_tc_hook, hook,
+			    .attach_point = BPF_TC_EGRESS,
+			    .ifindex = ifindex);
+	int err;
+
+	/* When destroying the hook, any and ALL attached TC-BPF (filter)
+	 * programs are also detached.
+	 */
+	err = bpf_tc_hook_destroy(&hook);
+	if (err)
+		fprintf(stderr, "Couldn't remove clsact qdisc on %s\n", ifname);
+
+	if (verbose)
+		printf("Flushed all TC-BPF egress programs (via destroy hook)\n");
+
+	return err;
+}
+
+int tc_detach_egress(int ifindex, bool verbose, bool flush_hook, const char * ifname)
+{
+	int err;
+	DECLARE_LIBBPF_OPTS(bpf_tc_hook, hook, .ifindex = ifindex,
+			    .attach_point = BPF_TC_EGRESS);
+	DECLARE_LIBBPF_OPTS(bpf_tc_opts, opts_info);
+
+	opts_info.handle   = EGRESS_HANDLE;
+	opts_info.priority = EGRESS_PRIORITY;
+
+	/* Check what program we are removing */
+	err = bpf_tc_query(&hook, &opts_info);
+	if (err) {
+		fprintf(stderr, "No egress program to detach "
+			"for ifindex %d (err:%d)\n", ifindex, err);
+		return err;
+	}
+	if (verbose)
+		printf("Detaching TC-BPF prog id:%d\n", opts_info.prog_id);
+
+	/* Attempt to detach program */
+	opts_info.prog_fd = 0;
+	opts_info.prog_id = 0;
+	opts_info.flags = 0;
+	err = bpf_tc_detach(&hook, &opts_info);
+	if (err) {
+		fprintf(stderr, "Cannot detach TC-BPF program id:%d "
+			"for ifindex %d (err:%d)\n", opts_info.prog_id,
+			ifindex, err);
+	}
+
+	if (flush_hook)
+		return teardown_hook(ifindex, ifname, verbose);
+
+	return err;
+}
+
+int tc_attach_egress(int ifindex, bool verbose, struct lqos_kern *obj)
+{
+	int err = 0;
+	int fd;
+	DECLARE_LIBBPF_OPTS(bpf_tc_hook, hook, .attach_point = BPF_TC_EGRESS);
+	DECLARE_LIBBPF_OPTS(bpf_tc_opts, attach_egress);
+
+	/* Selecting BPF-prog here: */
+	//fd = bpf_program__fd(obj->progs.queue_map_4);
+	fd = bpf_program__fd(obj->progs.tc_iphash_to_cpu);
+	if (fd < 0) {
+		fprintf(stderr, "Couldn't find egress program\n");
+		err = -ENOENT;
+		goto out;
+	}
+	attach_egress.prog_fd = fd;
+
+	hook.ifindex = ifindex;
+
+	err = bpf_tc_hook_create(&hook);
+	if (err && err != -EEXIST) {
+		fprintf(stderr, "Couldn't create TC-BPF hook for "
+			"ifindex %d (err:%d)\n", ifindex, err);
+		goto out;
+	}
+	if (verbose && err == -EEXIST) {
+		printf("Success: TC-BPF hook already existed "
+		       "(Ignore: \"libbpf: Kernel error message\")\n");
+	}
+
+	hook.attach_point = BPF_TC_EGRESS;
+	attach_egress.flags    = BPF_TC_F_REPLACE;
+	attach_egress.handle   = EGRESS_HANDLE;
+	attach_egress.priority = EGRESS_PRIORITY;
+	err = bpf_tc_attach(&hook, &attach_egress);
+	if (err) {
+		fprintf(stderr, "Couldn't attach egress program to "
+			"ifindex %d (err:%d)\n", hook.ifindex, err);
+		goto out;
+	}
+
+	if (verbose) {
+		printf("Attached TC-BPF program id:%d\n",
+		       attach_egress.prog_id);
+	}
+out:
+	return err;
+}
+
+int teardown_hook_ingress(int ifindex, const char * ifname, bool verbose)
+{
+	DECLARE_LIBBPF_OPTS(bpf_tc_hook, hook,
+			    .attach_point = BPF_TC_INGRESS,
+			    .ifindex = ifindex);
+	int err;
+
+	/* When destroying the hook, any and ALL attached TC-BPF (filter)
+	 * programs are also detached.
+	 */
+	err = bpf_tc_hook_destroy(&hook);
+	if (err)
+		fprintf(stderr, "Couldn't remove clsact qdisc on %s\n", ifname);
+
+	if (verbose)
+		printf("Flushed all TC-BPF egress programs (via destroy hook)\n");
+
+	return err;
+}
+
+int tc_detach_ingress(int ifindex, bool verbose, bool flush_hook, const char * ifname)
+{
+	int err;
+	DECLARE_LIBBPF_OPTS(bpf_tc_hook, hook, .ifindex = ifindex,
+			    .attach_point = BPF_TC_INGRESS);
+	DECLARE_LIBBPF_OPTS(bpf_tc_opts, opts_info);
+
+	opts_info.handle   = EGRESS_HANDLE;
+	opts_info.priority = EGRESS_PRIORITY;
+
+	/* Check what program we are removing */
+	err = bpf_tc_query(&hook, &opts_info);
+	if (err) {
+		fprintf(stderr, "No ingress program to detach "
+			"for ifindex %d (err:%d)\n", ifindex, err);
+		return err;
+	}
+	if (verbose)
+		printf("Detaching TC-BPF prog id:%d\n", opts_info.prog_id);
+
+	/* Attempt to detach program */
+	opts_info.prog_fd = 0;
+	opts_info.prog_id = 0;
+	opts_info.flags = 0;
+	err = bpf_tc_detach(&hook, &opts_info);
+	if (err) {
+		fprintf(stderr, "Cannot detach TC-BPF program id:%d "
+			"for ifindex %d (err:%d)\n", opts_info.prog_id,
+			ifindex, err);
+	}
+
+	if (flush_hook)
+		return teardown_hook(ifindex, ifname, verbose);
+
+	return err;
+}
+
+int tc_attach_ingress(int ifindex, bool verbose, struct lqos_kern *obj)
+{
+	int err = 0;
+	int fd;
+	DECLARE_LIBBPF_OPTS(bpf_tc_hook, hook, .attach_point = BPF_TC_INGRESS);
+	DECLARE_LIBBPF_OPTS(bpf_tc_opts, attach_egress);
+
+	/* Selecting BPF-prog here: */
+	//fd = bpf_program__fd(obj->progs.queue_map_4);
+	fd = bpf_program__fd(obj->progs.bifrost);
+	if (fd < 0) {
+		fprintf(stderr, "Couldn't find ingress program\n");
+		err = -ENOENT;
+		goto out;
+	}
+	attach_egress.prog_fd = fd;
+
+	hook.ifindex = ifindex;
+
+	err = bpf_tc_hook_create(&hook);
+	if (err && err != -EEXIST) {
+		fprintf(stderr, "Couldn't create TC-BPF hook for "
+			"ifindex %d (err:%d)\n", ifindex, err);
+		goto out;
+	}
+	if (verbose && err == -EEXIST) {
+		printf("Success: TC-BPF hook already existed "
+		       "(Ignore: \"libbpf: Kernel error message\")\n");
+	}
+
+	hook.attach_point = BPF_TC_INGRESS;
+	attach_egress.flags    = BPF_TC_F_REPLACE;
+	attach_egress.handle   = EGRESS_HANDLE;
+	attach_egress.priority = EGRESS_PRIORITY;
+	err = bpf_tc_attach(&hook, &attach_egress);
+	if (err) {
+		fprintf(stderr, "Couldn't attach egress program to "
+			"ifindex %d (err:%d)\n", hook.ifindex, err);
+		goto out;
+	}
+
+	if (verbose) {
+		printf("Attached TC-BPF program id:%d\n",
+		       attach_egress.prog_id);
+	}
+out:
+	return err;
+}
+
+/*******************************/
+
+static inline unsigned int bpf_num_possible_cpus(void)
+{
+	static const char *fcpu = "/sys/devices/system/cpu/possible";
+	unsigned int start, end, possible_cpus = 0;
+	char buff[128];
+	FILE *fp;
+	int n;
+
+	fp = fopen(fcpu, "r");
+	if (!fp) {
+		printf("Failed to open %s: '%s'!\n", fcpu, strerror(errno));
+		exit(1);
+	}
+
+	while (fgets(buff, sizeof(buff), fp)) {
+		n = sscanf(buff, "%u-%u", &start, &end);
+		if (n == 0) {
+			printf("Failed to retrieve # possible CPUs!\n");
+			exit(1);
+		} else if (n == 1) {
+			end = start;
+		}
+		possible_cpus = start == 0 ? end + 1 : 0;
+		break;
+	}
+	fclose(fp);
+
+	return possible_cpus;
+}
+
+struct txq_config {
+	/* lookup key: __u32 cpu; */
+	__u16 queue_mapping;
+	__u16 htb_major;
+};
+
+bool map_txq_config_base_setup(int map_fd) {
+	unsigned int possible_cpus = bpf_num_possible_cpus();
+	struct txq_config txq_cfg;
+	__u32 cpu;
+	int err;
+
+	if (map_fd < 0) {
+		fprintf(stderr, "ERR: (bad map_fd:%d) "
+			"cannot proceed without access to txq_config map\n",
+			map_fd);
+		return false;
+	}
+
+	for (cpu = 0; cpu < possible_cpus; cpu++) {
+		txq_cfg.queue_mapping = cpu + 1;
+		txq_cfg.htb_major     = cpu + 1;
+
+		err = bpf_map_update_elem(map_fd, &cpu, &txq_cfg, 0);
+		if (err) {
+			fprintf(stderr,
+				"ERR: %s() updating cpu-key:%d err(%d):%s\n",
+				__func__, cpu, errno, strerror(errno));
+			return false;
+		}
+	}
+
+	return true;
+}

src/rust/lqos_sys/src/bpf/wrapper.h

@@ -0,0 +1,11 @@
+#include "lqos_kern_skel.h"
+#include <stdbool.h>
+
+extern struct lqos_kern * lqos_kern_open();
+extern int lqos_kern_load(struct lqos_kern * skel);
+extern int tc_attach_egress(int ifindex, bool verbose, struct lqos_kern *obj);
+extern int tc_detach_egress(int ifindex, bool verbose, bool flush_hook, const char * ifname);
+extern int tc_attach_ingress(int ifindex, bool verbose, struct lqos_kern *obj);
+extern int tc_detach_ingress(int ifindex, bool verbose, bool flush_hook, const char * ifname);
+extern __u64 max_tracker_ips();
+extern bool map_txq_config_base_setup(int map_fd);

src/rust/lqos_sys/src/bpf_map.rs

@@ -0,0 +1,168 @@
+#![allow(dead_code)]
+use anyhow::{Error, Result};
+use libbpf_sys::{
+    bpf_map_delete_elem, bpf_map_get_next_key, bpf_map_lookup_elem, bpf_map_update_elem,
+    bpf_obj_get, BPF_NOEXIST,
+};
+use std::{
+    ffi::{c_void, CString},
+    marker::PhantomData,
+    ptr::null_mut,
+};
+
+/// Represents an underlying BPF map, accessed via the filesystem.
+/// `BpfMap` *only* talks to shared (not PER-CPU) variants of maps.
+/// 
+/// `K` is the *key* type, indexing the map.
+/// `V` is the *value* type, and must exactly match the underlying C data type.
+pub(crate) struct BpfMap<K, V> {
+    fd: i32,
+    _key_phantom: PhantomData<K>,
+    _val_phantom: PhantomData<V>,
+}
+
+impl<K, V> BpfMap<K, V>
+where
+    K: Default + Clone,
+    V: Default + Clone,
+{
+    /// Connect to a BPF map via a filename. Connects the internal
+    /// file descriptor, which is held until the structure is
+    /// dropped.
+    pub(crate) fn from_path(filename: &str) -> Result<Self> {
+        let filename_c = CString::new(filename)?;
+        let fd = unsafe { bpf_obj_get(filename_c.as_ptr()) };
+        if fd < 0 {
+            Err(Error::msg("Unable to open BPF map"))
+        } else {
+            Ok(Self {
+                fd,
+                _key_phantom: PhantomData,
+                _val_phantom: PhantomData,
+            })
+        }
+    }
+
+    /// Iterates the undlering BPF map, and adds the results
+    /// to a vector. Each entry contains a `key, value` tuple.
+    pub(crate) fn dump_vec(&self) -> Vec<(K, V)> {
+        let mut result = Vec::new();
+
+        let mut prev_key: *mut K = null_mut();
+        let mut key: K = K::default();
+        let key_ptr: *mut K = &mut key;
+        let mut value = V::default();
+        let value_ptr: *mut V = &mut value;
+
+        unsafe {
+            while bpf_map_get_next_key(self.fd, prev_key as *mut c_void, key_ptr as *mut c_void)
+                == 0
+            {
+                bpf_map_lookup_elem(self.fd, key_ptr as *mut c_void, value_ptr as *mut c_void);
+                result.push((key.clone(), value.clone()));
+                prev_key = key_ptr;
+            }
+        }
+
+        result
+    }
+
+    /// Inserts an entry into a BPF map.
+    /// Use this sparingly, because it briefly pauses XDP access to the
+    /// underlying map (through internal locking we can't reach from
+    /// userland).
+    /// 
+    /// ## Arguments
+    /// 
+    /// * `key` - the key to insert.
+    /// * `value` - the value to insert.
+    /// 
+    /// Returns Ok if insertion succeeded, a generic error (no details yet)
+    /// if it fails.
+    pub(crate) fn insert(&mut self, key: &mut K, value: &mut V) -> Result<()> {
+        let key_ptr: *mut K = key;
+        let val_ptr: *mut V = value;
+        let err = unsafe {
+            bpf_map_update_elem(
+                self.fd,
+                key_ptr as *mut c_void,
+                val_ptr as *mut c_void,
+                BPF_NOEXIST.into(),
+            )
+        };
+        if err != 0 {
+            Err(Error::msg(format!("Unable to insert into map ({err})")))
+        } else {
+            Ok(())
+        }
+    }
+
+    /// Deletes an entry from the underlying eBPF map.
+    /// Use this sparingly, it locks the underlying map in the
+    /// kernel. This can cause *long* delays under heavy load.
+    /// 
+    /// ## Arguments
+    /// 
+    /// * `key` - the key to delete.
+    /// 
+    /// Return `Ok` if deletion succeeded.
+    pub(crate) fn delete(&mut self, key: &mut K) -> Result<()> {
+        let key_ptr: *mut K = key;
+        let err = unsafe { bpf_map_delete_elem(self.fd, key_ptr as *mut c_void) };
+        if err != 0 {
+            Err(Error::msg("Unable to delete from map"))
+        } else {
+            Ok(())
+        }
+    }
+
+    /// Delete all entries in the underlying eBPF map.
+    /// Use this sparingly, it locks the underlying map. Under
+    /// heavy load, it WILL eventually terminate - but it might
+    /// take a very long time. Only use this for cleaning up
+    /// sparsely allocated map data.
+    pub(crate) fn clear(&mut self) -> Result<()> {
+        loop {
+            let mut key = K::default();
+            let mut prev_key: *mut K = null_mut();
+            unsafe {
+                let key_ptr: *mut K = &mut key;
+                while bpf_map_get_next_key(self.fd, prev_key as *mut c_void, key_ptr as *mut c_void)== 0
+                {
+                    bpf_map_delete_elem(self.fd, key_ptr as *mut c_void);
+                    prev_key = key_ptr;
+                }
+            }
+
+            key = K::default();
+            prev_key = null_mut();
+            unsafe {
+                let key_ptr: *mut K = &mut key;
+                if bpf_map_get_next_key(self.fd, prev_key as *mut c_void, key_ptr as *mut c_void) != 0 {
+                    break;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    pub(crate) fn clear_no_repeat(&mut self) -> Result<()> {
+        let mut key = K::default();
+        let mut prev_key: *mut K = null_mut();
+        unsafe {
+            let key_ptr: *mut K = &mut key;
+            while bpf_map_get_next_key(self.fd, prev_key as *mut c_void, key_ptr as *mut c_void)== 0
+            {
+                bpf_map_delete_elem(self.fd, key_ptr as *mut c_void);
+                prev_key = key_ptr;
+            }
+        }
+        Ok(())
+    }
+}
+
+impl<K, V> Drop for BpfMap<K, V> {
+    fn drop(&mut self) {
+        let _ = nix::unistd::close(self.fd);
+    }
+}

src/rust/lqos_sys/src/bpf_per_cpu_map.rs

@@ -0,0 +1,77 @@
+use anyhow::{Error, Result};
+use libbpf_sys::{
+    bpf_map_get_next_key, bpf_map_lookup_elem, bpf_obj_get, libbpf_num_possible_cpus,
+};
+use std::fmt::Debug;
+use std::{
+    ffi::{c_void, CString},
+    marker::PhantomData,
+    ptr::null_mut,
+};
+
+/// Represents an underlying BPF map, accessed via the filesystem.
+/// `BpfMap` *only* talks to PER-CPU variants of maps.
+/// 
+/// `K` is the *key* type, indexing the map.
+/// `V` is the *value* type, and must exactly match the underlying C data type.
+pub(crate) struct BpfPerCpuMap<K, V> {
+    fd: i32,
+    _key_phantom: PhantomData<K>,
+    _val_phantom: PhantomData<V>,
+}
+
+impl<K, V> BpfPerCpuMap<K, V>
+where
+    K: Default + Clone,
+    V: Default + Clone + Debug,
+{
+    /// Connect to a PER-CPU BPF map via a filename. Connects the internal
+    /// file descriptor, which is held until the structure is
+    /// dropped. The index of the CPU is *not* specified.
+    pub(crate) fn from_path(filename: &str) -> Result<Self> {
+        let filename_c = CString::new(filename)?;
+        let fd = unsafe { bpf_obj_get(filename_c.as_ptr()) };
+        if fd < 0 {
+            Err(Error::msg("Unable to open BPF map"))
+        } else {
+            Ok(Self {
+                fd,
+                _key_phantom: PhantomData,
+                _val_phantom: PhantomData,
+            })
+        }
+    }
+
+    /// Iterates the entire contents of the underlying eBPF per-cpu map.
+    /// Each iteration returns one entry per CPU, even if there isn't a
+    /// CPU-local map entry. Each result is therefore returned as one
+    /// key and a vector of values.
+    pub(crate) fn dump_vec(&self) -> Vec<(K, Vec<V>)> {
+        let mut result = Vec::new();
+        let num_cpus = unsafe { libbpf_num_possible_cpus() };
+
+        let mut prev_key: *mut K = null_mut();
+        let mut key: K = K::default();
+        let key_ptr: *mut K = &mut key;
+        let mut value = vec![V::default(); num_cpus as usize];
+        let value_ptr = value.as_mut_ptr();
+
+        unsafe {
+            while bpf_map_get_next_key(self.fd, prev_key as *mut c_void, key_ptr as *mut c_void)
+                == 0
+            {
+                bpf_map_lookup_elem(self.fd, key_ptr as *mut c_void, value_ptr as *mut c_void);
+                result.push((key.clone(), value.clone()));
+                prev_key = key_ptr;
+            }
+        }
+
+        result
+    }
+}
+
+impl<K, V> Drop for BpfPerCpuMap<K, V> {
+    fn drop(&mut self) {
+        let _ = nix::unistd::close(self.fd);
+    }
+}

src/rust/lqos_sys/src/cpu_map.rs

@@ -0,0 +1,131 @@
+use std::{ffi::CString, os::raw::c_void};
+use anyhow::{Error, Result};
+use libbpf_sys::{bpf_map_update_elem, bpf_obj_get, libbpf_num_possible_cpus};
+
+//* Provides an interface for querying the number of CPUs eBPF can
+//* see, and marking CPUs as available. Currently marks ALL eBPF
+//* usable CPUs as available.
+
+pub(crate) struct CpuMapping {
+    fd_cpu_map: i32,
+    fd_cpu_available: i32,
+    fd_txq_config: i32,
+}
+
+fn get_map_fd(filename: &str) -> Result<i32> {
+    let filename_c = CString::new(filename)?;
+    let fd = unsafe { bpf_obj_get(filename_c.as_ptr()) };
+    if fd < 0 {
+        Err(Error::msg("Unable to open BPF map"))
+    } else {
+        Ok(fd)
+    }
+}
+
+impl CpuMapping {
+    pub(crate) fn new() -> Result<Self> {
+        Ok(Self {
+            fd_cpu_map: get_map_fd("/sys/fs/bpf/cpu_map")?,
+            fd_cpu_available: get_map_fd("/sys/fs/bpf/cpus_available")?,
+            fd_txq_config: get_map_fd("/sys/fs/bpf/map_txq_config")?,
+        })
+    }
+
+    pub(crate) fn mark_cpus_available(&self) -> Result<()> {
+        let cpu_count = unsafe { libbpf_num_possible_cpus() } as u32;
+
+        let queue_size = 2048u32;
+        let val_ptr: *const u32 = &queue_size;
+        for cpu in 0..cpu_count {
+            println!("Mapping core #{cpu}");
+            // Insert into the cpu map
+            let cpu_ptr: *const u32 = &cpu;
+            let error = unsafe {
+                bpf_map_update_elem(
+                    self.fd_cpu_map,
+                    cpu_ptr as *const c_void,
+                    val_ptr as *const c_void,
+                    0,
+                )
+            };
+            if error != 0 {
+                return Err(Error::msg("Unable to map CPU"));
+            }
+
+            // Insert into the available list
+            let error = unsafe {
+                bpf_map_update_elem(
+                    self.fd_cpu_available,
+                    cpu_ptr as *const c_void,
+                    cpu_ptr as *const c_void,
+                    0,
+                )
+            };
+            if error != 0 {
+                return Err(Error::msg("Unable to add to available CPUs list"));
+            }
+        } // CPU loop
+        Ok(())
+    }
+
+    pub(crate) fn setup_base_txq_config(&self) -> Result<()> {
+        use crate::lqos_kernel::bpf::map_txq_config_base_setup;
+        // Should we shell out to the C and do it the easy way?
+        let result = unsafe {
+            map_txq_config_base_setup(self.fd_txq_config)
+        };
+        if !result {
+            Err(Error::msg("Unable to setup TXQ map"))
+        } else {
+            Ok(())
+        }
+    }
+}
+
+impl Drop for CpuMapping {
+    fn drop(&mut self) {
+        let _ = nix::unistd::close(self.fd_cpu_available);
+        let _ = nix::unistd::close(self.fd_cpu_map);
+        let _ = nix::unistd::close(self.fd_txq_config);
+    }
+}
+
+/// Emulates xd_setup from cpumap
+pub(crate) fn xps_setup_default_disable(interface: &str) -> Result<()> {
+    use std::io::Write;
+    println!("xps_setup");
+    let queues = sorted_txq_xps_cpus(interface)?;
+    for (cpu, xps_cpu) in queues.iter().enumerate() {
+        let mask = cpu_to_mask_disabled(cpu);
+        let mut f = std::fs::OpenOptions::new().write(true).open(xps_cpu)?;
+        f.write_all(&mask.to_string().as_bytes())?;
+        f.flush()?;
+        println!("Mapped TX queue for CPU {cpu}");
+    }
+
+    Ok(())
+}
+
+fn sorted_txq_xps_cpus(interface: &str) -> Result<Vec<String>> {
+    let mut result = Vec::new();
+    let paths = std::fs::read_dir(&format!("/sys/class/net/{interface}/queues/"))?;
+    for path in paths {
+        if let Ok(path) = &path {
+            if path.path().is_dir() {
+                if let Some(filename) = path.path().file_name() {
+                    let base_fn = format!("/sys/class/net/{interface}/queues/{}/xps_cpus", filename.to_str().unwrap());
+                    if std::path::Path::new(&base_fn).exists() {
+                        result.push(base_fn);
+                    }
+                }
+            }
+        }
+    }
+    result.sort();
+
+    Ok(result)
+}
+
+fn cpu_to_mask_disabled(_cpu: usize) -> usize {
+    0
+}

src/rust/lqos_sys/src/ip_mapping/ip_hash_data.rs

@@ -0,0 +1,15 @@
+#[repr(C)]
+#[derive(Clone)]
+pub struct IpHashData {
+    pub cpu: u32,
+    pub tc_handle: u32,
+}
+
+impl Default for IpHashData {
+    fn default() -> Self {
+        Self {
+            cpu: 0,
+            tc_handle: 0,
+        }
+    }
+}

src/rust/lqos_sys/src/ip_mapping/ip_hash_key.rs

@@ -0,0 +1,15 @@
+#[repr(C)]
+#[derive(Clone)]
+pub struct IpHashKey {
+    pub prefixlen: u32,
+    pub address: [u8; 16],
+}
+
+impl Default for IpHashKey {
+    fn default() -> Self {
+        Self {
+            prefixlen: 0,
+            address: [0xFF; 16],
+        }
+    }
+}

src/rust/lqos_sys/src/ip_mapping/ip_to_map.rs

@@ -0,0 +1,127 @@
+use std::net::{IpAddr, Ipv6Addr, Ipv4Addr};
+use anyhow::{Result, Error};
+use lqos_bus::TcHandle;
+
+pub(crate) struct IpToMap {
+    pub(crate) subnet: IpAddr,
+    pub(crate) prefix: u32,
+    pub(crate) tc_handle: TcHandle,
+    pub(crate) cpu: u32,
+}
+
+impl IpToMap {
+    pub(crate) fn new(address: &str, tc_handle: TcHandle, cpu: u32) -> Result<Self> {
+        let address_part; // Filled in later
+        let mut subnet_part = 128;
+        if address.contains("/") {
+            let parts: Vec<&str> = address.split('/').collect();
+            address_part = parts[0].to_string();
+            subnet_part = parts[1].replace("/", "").parse()?;
+        } else {
+            address_part = address.to_string();
+        }
+
+        let subnet = if address_part.contains(":") {
+            // It's an IPv6
+            let ipv6 = address_part.parse::<Ipv6Addr>()?;
+            IpAddr::V6(ipv6)
+        } else {
+            // It's an IPv4
+            if subnet_part != 128 {
+                subnet_part += 96;
+            }
+            let ipv4 = address_part.parse::<Ipv4Addr>()?;
+            IpAddr::V4(ipv4)
+        };
+
+        if subnet_part > 128 {
+            return Err(Error::msg("Invalid subnet mask"));
+        }
+
+        Ok(Self {
+            subnet,
+            prefix: subnet_part,
+            tc_handle,
+            cpu,
+        })
+    }
+
+    pub(crate) fn handle(&self) -> u32 {
+        self.tc_handle.as_u32()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn parse_ipv4_single() {
+        let map = IpToMap::new("1.2.3.4", TcHandle::from_string("1:2").unwrap(), 1).unwrap();
+        let rust_ip : IpAddr = "1.2.3.4".parse().unwrap();
+        assert_eq!(rust_ip, map.subnet);
+        assert_eq!(map.prefix, 128);
+        assert_eq!(map.tc_handle.to_string(), "1:2");
+        assert_eq!(map.cpu, 1);
+    }
+
+    #[test]
+    fn parse_ipv4_subnet() {
+        let map = IpToMap::new("1.2.3.0/24", TcHandle::from_string("1:2").unwrap(), 1).unwrap();
+        let rust_ip : IpAddr = "1.2.3.0".parse().unwrap();
+        assert_eq!(rust_ip, map.subnet);
+        assert_eq!(map.prefix, 24+96);
+        assert_eq!(map.tc_handle.to_string(), "1:2");
+        assert_eq!(map.cpu, 1);
+    }
+
+    #[test]
+    fn parse_ipv4_invalid_ip() {
+        let map = IpToMap::new("1.2.3.256/24", TcHandle::from_string("1:2").unwrap(), 1);
+        assert!(map.is_err());
+    }
+
+    #[test]
+    fn parse_ipv4_super_invalid_ip() {
+        let map = IpToMap::new("I like sheep", TcHandle::from_string("1:2").unwrap(), 1);
+        assert!(map.is_err());
+    }
+
+    #[test]
+    fn parse_ipv4_invalid_cidr() {
+        let map = IpToMap::new("1.2.3.256/33", TcHandle::from_string("1:2").unwrap(), 1);
+        assert!(map.is_err());
+    }
+
+    #[test]
+    fn parse_ipv4_negative_cidr() {
+        let map = IpToMap::new("1.2.3.256/-1", TcHandle::from_string("1:2").unwrap(), 1);
+        assert!(map.is_err());
+    }
+
+    #[test]
+    fn parse_ipv6_single() {
+        let map = IpToMap::new("dead::beef", TcHandle::from_string("1:2").unwrap(), 1).unwrap();
+        let rust_ip : IpAddr = "dead::beef".parse().unwrap();
+        assert_eq!(rust_ip, map.subnet);
+        assert_eq!(map.prefix, 128);
+        assert_eq!(map.tc_handle.to_string(), "1:2");
+        assert_eq!(map.cpu, 1);
+    }
+
+    #[test]
+    fn parse_ipv6_subnet() {
+        let map = IpToMap::new("dead:beef::/64", TcHandle::from_string("1:2").unwrap(), 1).unwrap();
+        let rust_ip : IpAddr = "dead:beef::".parse().unwrap();
+        assert_eq!(rust_ip, map.subnet);
+        assert_eq!(map.prefix, 64);
+        assert_eq!(map.tc_handle.to_string(), "1:2");
+        assert_eq!(map.cpu, 1);
+    }
+
+    #[test]
+    fn parse_ipv6_invalid_ip() {
+        let map = IpToMap::new("dead:beef", TcHandle::from_string("1:2").unwrap(), 1);
+        assert!(map.is_err());
+    }
+}

src/rust/lqos_sys/src/ip_mapping/mod.rs

@@ -0,0 +1,89 @@
+use anyhow::Result;
+use lqos_bus::TcHandle;
+use std::net::IpAddr;
+use crate::{bpf_map::BpfMap, XdpIpAddress};
+mod ip_to_map;
+mod ip_hash_data;
+mod ip_hash_key;
+use ip_to_map::IpToMap;
+use ip_hash_data::IpHashData;
+use ip_hash_key::IpHashKey;
+
+/// Adds an IP address to the underlying TC map.
+/// 
+/// ## Arguments
+/// 
+/// * `address` - a string containing an IPv4 or IPv6 address, with or without a prefix-length.
+/// * `tc_handle` - the TC classifier handle to associate with the IP address, in (major,minor) format.
+/// * `cpu` - the CPU index on which the TC class should be handled.
+pub fn add_ip_to_tc(address: &str, tc_handle: TcHandle, cpu: u32, upload: bool) -> Result<()> {
+    let bpf_path = if upload {
+        "/sys/fs/bpf/map_ip_to_cpu_and_tc_recip"
+    } else {
+        "/sys/fs/bpf/map_ip_to_cpu_and_tc"
+    };
+
+    let ip_to_add = IpToMap::new(address, tc_handle, cpu)?;
+    let mut bpf_map =
+        BpfMap::<IpHashKey, IpHashData>::from_path(bpf_path)?;
+    let address = XdpIpAddress::from_ip(ip_to_add.subnet);
+    let mut key = IpHashKey {
+        prefixlen: ip_to_add.prefix,
+        address: address.0,
+    };
+    let mut value = IpHashData {
+        cpu: ip_to_add.cpu,
+        tc_handle: ip_to_add.handle(),
+    };
+    bpf_map.insert(&mut key, &mut value)?;
+    Ok(())
+}
+
+/// Removes an IP address from the underlying TC map.
+/// 
+/// ## Arguments
+/// 
+/// * `address` - the IP address to remove. If no prefix (e.g. `/24`) is provided, the longest prefix to match a single IP address will be assumed.
+pub fn del_ip_from_tc(address: &str, upload: bool) -> Result<()> {
+    let bpf_path = if upload {
+        "/sys/fs/bpf/map_ip_to_cpu_and_tc_recip"
+    } else {
+        "/sys/fs/bpf/map_ip_to_cpu_and_tc"
+    };
+    let ip_to_add = IpToMap::new(address, TcHandle::from_string("0:0")?, 0)?;
+    let mut bpf_map =
+        BpfMap::<IpHashKey, IpHashData>::from_path(bpf_path)?;
+    let ip = address.parse::<IpAddr>()?;
+    let ip = XdpIpAddress::from_ip(ip);
+    let mut key = IpHashKey {
+        prefixlen: ip_to_add.prefix,
+        address: ip.0,
+    };
+    bpf_map.delete(&mut key)?;
+    Ok(())
+}
+
+/// Remove all IP addresses from the underlying TC map.
+pub fn clear_ips_from_tc() -> Result<()> {
+    let mut bpf_map =
+        BpfMap::<IpHashKey, IpHashData>::from_path("/sys/fs/bpf/map_ip_to_cpu_and_tc")?;
+    bpf_map.clear()?;
+
+    let mut bpf_map =
+        BpfMap::<IpHashKey, IpHashData>::from_path("/sys/fs/bpf/map_ip_to_cpu_and_tc_recip")?;
+    bpf_map.clear()?;
+
+    Ok(())
+}
+
+/// Query the underlying IP address to TC map and return the currently active dataset.
+pub fn list_mapped_ips() -> Result<Vec<(IpHashKey, IpHashData)>> {
+    let bpf_map = BpfMap::<IpHashKey, IpHashData>::from_path("/sys/fs/bpf/map_ip_to_cpu_and_tc")?;
+    let mut raw = bpf_map.dump_vec();
+
+    let bpf_map2 = BpfMap::<IpHashKey, IpHashData>::from_path("/sys/fs/bpf/map_ip_to_cpu_and_tc_recip")?;
+    let raw2 = bpf_map2.dump_vec();
+    raw.extend_from_slice(&raw2);
+
+    Ok(raw)
+}

src/rust/lqos_sys/src/kernel_wrapper.rs

@@ -0,0 +1,68 @@
+use crate::lqos_kernel::{
+    attach_xdp_and_tc_to_interface, unload_xdp_from_interface, InterfaceDirection,
+};
+
+/// A wrapper-type that stores the interfaces to which the XDP and TC programs should
+/// be attached. Performs the attachment process, and hooks "drop" to unattach the
+/// programs when the structure falls out of scope.
+pub struct LibreQoSKernels {
+    to_internet: String,
+    to_isp: String,
+    on_a_stick: bool,
+}
+
+impl LibreQoSKernels {
+    /// Create a new `LibreQosKernels` structure, using the specified interfaces.
+    /// Returns Ok(self) if attaching to the XDP/TC interfaces succeeded, otherwise
+    /// returns an error containing a string describing what went wrong.
+    /// 
+    /// Outputs progress to `stdio` during execution, and detailed errors to `stderr`.
+    /// 
+    /// ## Arguments
+    /// 
+    /// * `to_internet` - the name of the Internet-facing interface (e.g. `eth1`).
+    /// * `to_isp` - the name of the ISP-network facing interface (e.g. `eth2`).
+    pub fn new<S: ToString>(to_internet: S, to_isp: S) -> anyhow::Result<Self> {
+        let kernel = Self {
+            to_internet: to_internet.to_string(),
+            to_isp: to_isp.to_string(),
+            on_a_stick: false,
+        };
+        attach_xdp_and_tc_to_interface(&kernel.to_internet, InterfaceDirection::Internet)?;
+        attach_xdp_and_tc_to_interface(&kernel.to_isp, InterfaceDirection::IspNetwork)?;
+        Ok(kernel)
+    }
+
+    /// Creates a new `LibreQosKernels` structure, in "on a stick mode" - only a single interface is
+    /// bound, and internal VLANs are used to map ingress vs. egress. Returns Ok(self) if everything
+    /// loaded correctly, an error otherwise.
+    /// 
+    /// Prints to `stdio` during execution and detailed errors to `stderr`.
+    /// 
+    /// ## Arguments
+    /// 
+    /// * `stick_interfaace` - the name of the VLAN trunked interface.
+    /// * `internet_vlan` - the VLAN ID facing the Internet. Endianness is fixed for you.
+    /// * `isp_vlan` - the VLAN ID facing the ISP core router. Endianness is fixed for you.
+    pub fn on_a_stick_mode<S:ToString>(stick_interface : S, internet_vlan: u16, isp_vlan: u16) -> anyhow::Result<Self> {
+        let kernel = Self {
+            to_internet: stick_interface.to_string(),
+            to_isp: String::new(),
+            on_a_stick: true,
+        };
+        attach_xdp_and_tc_to_interface(&kernel.to_internet, InterfaceDirection::OnAStick(internet_vlan, isp_vlan))?;
+
+        Ok(kernel)
+    }
+}
+
+impl Drop for LibreQoSKernels {
+    fn drop(&mut self) {
+        if !self.on_a_stick {
+            let _ = unload_xdp_from_interface(&self.to_internet);
+            let _ = unload_xdp_from_interface(&self.to_isp);
+        } else {
+            let _ = unload_xdp_from_interface(&self.to_internet);
+        }
+    }
+}

src/rust/lqos_sys/src/lib.rs

@@ -0,0 +1,26 @@
+#![warn(missing_docs)]
+
+//! `lqos_sys` is a system-library that builds LibreQoS's eBPF code
+//! and wraps it in a safe external Rust wrapper.
+//! 
+//! The `build.rs` script compiles the C code found in `src/bpf`
+//! and statically embeds the result in this crate.
+
+mod bpf_map;
+mod bpf_per_cpu_map;
+mod cpu_map;
+mod ip_mapping;
+mod kernel_wrapper;
+mod lqos_kernel;
+mod tcp_rtt;
+mod throughput;
+mod xdp_ip_address;
+mod bifrost_maps;
+
+pub use ip_mapping::{add_ip_to_tc, clear_ips_from_tc, del_ip_from_tc, list_mapped_ips};
+pub use kernel_wrapper::LibreQoSKernels;
+pub use tcp_rtt::{get_tcp_round_trip_times, RttTrackingEntry};
+pub use throughput::{get_throughput_map, HostCounter};
+pub use xdp_ip_address::XdpIpAddress;
+pub use lqos_kernel::max_tracked_ips;
+pub use libbpf_sys::libbpf_num_possible_cpus;

src/rust/lqos_sys/src/lqos_kernel.rs

@@ -0,0 +1,225 @@
+#![allow(non_upper_case_globals)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(dead_code)]
+
+use crate::cpu_map::CpuMapping;
+use anyhow::{Error, Result};
+use libbpf_sys::{
+    bpf_xdp_attach, libbpf_set_strict_mode, LIBBPF_STRICT_ALL, XDP_FLAGS_UPDATE_IF_NOEXIST, XDP_FLAGS_HW_MODE, XDP_FLAGS_DRV_MODE, XDP_FLAGS_SKB_MODE,
+};
+use nix::libc::{geteuid, if_nametoindex};
+use std::{ffi::CString, process::Command};
+
+pub(crate) mod bpf {
+    #![allow(warnings, unused)]
+    include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
+}
+
+/// Returns the value set in the C XDP system's MAX_TRACKED_IPS
+/// constant.
+pub fn max_tracked_ips() -> usize {
+    (unsafe {
+        bpf::max_tracker_ips()
+     }) as usize
+}
+
+pub fn check_root() -> Result<()> {
+    unsafe {
+        if geteuid() == 0 {
+            Ok(())
+        } else {
+            Err(Error::msg("You need to be root to do this."))
+        }
+    }
+}
+
+pub fn interface_name_to_index(interface_name: &str) -> Result<u32> {
+    let if_name = CString::new(interface_name)?;
+    let index = unsafe { if_nametoindex(if_name.as_ptr()) };
+    if index == 0 {
+        Err(Error::msg(format!("Unknown interface: {interface_name}")))
+    } else {
+        Ok(index)
+    }
+}
+
+pub fn unload_xdp_from_interface(interface_name: &str) -> Result<()> {
+    println!("Unloading XDP/TC");
+    check_root()?;
+    let interface_index = interface_name_to_index(interface_name)?.try_into()?;
+    unsafe {
+        let err = bpf_xdp_attach(interface_index, -1, 1 << 0, std::ptr::null());
+        if err != 0 {
+            return Err(Error::msg("Unable to unload from interface."));
+        }
+
+        let interface_c = CString::new(interface_name)?;
+        let _ = bpf::tc_detach_egress(interface_index as i32, true, true, interface_c.as_ptr());
+        let _ = bpf::tc_detach_ingress(interface_index as i32, true, true, interface_c.as_ptr());
+    }
+    Ok(())
+}
+
+fn set_strict_mode() -> Result<()> {
+    let err = unsafe { libbpf_set_strict_mode(LIBBPF_STRICT_ALL) };
+    if err != 0 {
+        Err(Error::msg("Unable to activate BPF Strict Mode"))
+    } else {
+        Ok(())
+    }
+}
+
+unsafe fn open_kernel() -> Result<*mut bpf::lqos_kern> {
+    let result = bpf::lqos_kern_open();
+    if result.is_null() {
+        Err(Error::msg("Unable to open LibreQoS XDP/TC Kernel"))
+    } else {
+        Ok(result)
+    }
+}
+
+unsafe fn load_kernel(skeleton: *mut bpf::lqos_kern) -> Result<()> {
+    let error = bpf::lqos_kern_load(skeleton);
+    if error != 0 {
+        Err(Error::msg("Unable to load the XDP/TC kernel"))
+    } else {
+        Ok(())
+    }
+}
+
+#[derive(PartialEq, Eq)]
+pub enum InterfaceDirection {
+    Internet,
+    IspNetwork,
+    OnAStick(u16, u16),
+}
+
+pub fn attach_xdp_and_tc_to_interface(
+    interface_name: &str,
+    direction: InterfaceDirection,
+) -> Result<()> {
+    check_root()?;
+    // Check the interface is valid
+    let interface_index = interface_name_to_index(interface_name)?;
+    set_strict_mode()?;
+    let skeleton = unsafe {
+        let skeleton = open_kernel()?;
+        (*(*skeleton).data).direction = match direction {
+            InterfaceDirection::Internet => 1,
+            InterfaceDirection::IspNetwork => 2,
+            InterfaceDirection::OnAStick(..) => 3,
+        };
+        if let InterfaceDirection::OnAStick(internet, isp) = direction {
+            (*(*skeleton).bss).internet_vlan = internet.to_be();
+            (*(*skeleton).bss).isp_vlan = isp.to_be();
+        }
+        load_kernel(skeleton)?;
+        let _ = unload_xdp_from_interface(interface_name); // Ignoring error, it's ok if there isn't one
+        let prog_fd = bpf::bpf_program__fd((*skeleton).progs.xdp_prog);
+        attach_xdp_best_available(interface_index, prog_fd)?;
+        skeleton
+    };
+
+    // Configure CPU Maps
+    {
+        let cpu_map = CpuMapping::new()?;
+        crate::cpu_map::xps_setup_default_disable(interface_name)?;
+        cpu_map.mark_cpus_available()?;
+        cpu_map.setup_base_txq_config()?;
+    } // Scope block to ensure the CPU maps are closed
+
+    // Attach the TC program
+    // extern int tc_attach_egress(int ifindex, bool verbose, struct lqos_kern *obj);
+    // extern int tc_detach_egress(int ifindex, bool verbose, bool flush_hook, char * ifname);
+    let interface_c = CString::new(interface_name)?;
+    let _ =
+        unsafe { bpf::tc_detach_egress(interface_index as i32, true, true, interface_c.as_ptr()) }; // Ignoring error, because it's ok to not have something to detach
+
+    // Remove any previous entry
+    let r = Command::new("tc")
+        .args(["qdisc", "del", "dev", interface_name, "clsact"])
+        .output()?;
+    println!("{}", String::from_utf8(r.stderr).unwrap());
+    
+    // Add the classifier
+    let r = Command::new("tc")
+        .args(["filter", "add", "dev", interface_name, "clsact"])
+        .output()?;
+    println!("{}", String::from_utf8(r.stderr).unwrap());
+
+    // Attach to the egress
+    let error = unsafe { bpf::tc_attach_egress(interface_index as i32, true, skeleton) };
+    if error != 0 {
+        return Err(Error::msg("Unable to attach TC to interface"));
+    }
+
+    // Attach to the ingress IF it is configured
+    if let Ok(etc) = lqos_config::EtcLqos::load() {
+        if let Some(bridge) = &etc.bridge {
+            if bridge.use_kernel_bridge {
+                // Enable "promiscuous" mode on interfaces
+                for mapping in bridge.interface_mapping.iter() {
+                    std::process::Command::new("/bin/ip")
+                        .args(["link", "set", &mapping.name, "promisc", "on"])
+                        .output()?;
+                }
+
+                // Build the interface and vlan map entries
+                crate::bifrost_maps::clear_bifrost()?;
+                crate::bifrost_maps::map_interfaces(&bridge.interface_mapping)?;
+                crate::bifrost_maps::map_vlans(&bridge.vlan_mapping)?;
+
+                // Actually attach the TC ingress program
+                let error = unsafe { bpf::tc_attach_ingress(interface_index as i32, true, skeleton) };
+                if error != 0 {
+                    return Err(Error::msg("Unable to attach TC Ingress to interface"));
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+unsafe fn attach_xdp_best_available(interface_index: u32, prog_fd: i32) -> Result<()> {
+    // Try hardware offload first
+    if try_xdp_attach(interface_index, prog_fd, XDP_FLAGS_HW_MODE).is_err() {
+        // Try driver attach
+        if try_xdp_attach(interface_index, prog_fd, XDP_FLAGS_DRV_MODE).is_err() {
+            // Try SKB mode
+            if try_xdp_attach(interface_index, prog_fd, XDP_FLAGS_SKB_MODE).is_err() {
+                // Try no flags
+                let error = bpf_xdp_attach(
+                    interface_index.try_into().unwrap(),
+                    prog_fd,
+                    XDP_FLAGS_UPDATE_IF_NOEXIST,
+                    std::ptr::null(),
+                );
+                if error != 0 {
+                    return Err(Error::msg("Unable to attach to interface"));
+                }
+            } else {
+                println!("Attached in SKB compatibility mode. (Not so fast)");
+            }
+        } else {
+            println!("Attached in driver mode. (Fast)");
+        }
+    } else {
+        println!("Attached in hardware accelerated mode. (Fastest)");
+    }
+    Ok(())
+}
+
+unsafe fn try_xdp_attach(interface_index: u32, prog_fd: i32, connect_mode: u32) -> Result<()> {
+    let error = bpf_xdp_attach(
+        interface_index.try_into().unwrap(),
+        prog_fd,
+        XDP_FLAGS_UPDATE_IF_NOEXIST | connect_mode,
+        std::ptr::null(),
+    );
+    if error != 0 {
+        return Err(Error::msg("Unable to attach to interface"));
+    }
+    Ok(())
+}

src/rust/lqos_sys/src/tcp_rtt.rs

@@ -0,0 +1,47 @@
+use anyhow::Result;
+
+use crate::bpf_map::BpfMap;
+
+/// Entry from the XDP rtt_tracker map.
+#[repr(C)]
+#[derive(Clone, Copy, Debug)]
+pub struct RttTrackingEntry {
+    /// Array containing TCP round-trip times. Convert to an `f32` and divide by `100.0` for actual numbers.
+    pub rtt: [u32; 60],
+    
+    /// Used internally by the XDP program to store the current position in the storage array. Do not modify.
+    next_entry: u32,
+
+    /// Used internally by the XDP program to determine when it is time to recycle and reuse a record. Do not modify.
+    recycle_time: u64,
+
+    /// Flag indicating that an entry has been updated recently (last 30 seconds by default).
+    pub has_fresh_data: u32,
+}
+
+impl Default for RttTrackingEntry {
+    fn default() -> Self {
+        Self {
+            rtt: [0; 60],
+            next_entry: 0,
+            recycle_time: 0,
+            has_fresh_data: 0,
+        }
+    }
+}
+
+/// Queries the active XDP/TC programs for TCP round-trip time tracking
+/// data (from the `rtt_tracker` pinned eBPF map).
+/// 
+/// Only IP addresses facing the ISP Network side are tracked.
+/// 
+/// ## Returns
+/// 
+/// A vector containing:
+/// * `[u8; 16]` - a byte representation of the encoded IP address. See `XdpIpAddress` for details.
+/// * An `RttTrackingEntry` structure containing the current RTT results for the IP address.
+pub fn get_tcp_round_trip_times() -> Result<Vec<([u8; 16], RttTrackingEntry)>> {
+    let rtt_tracker = BpfMap::<[u8; 16], RttTrackingEntry>::from_path("/sys/fs/bpf/rtt_tracker")?;
+    let rtt_data = rtt_tracker.dump_vec();
+    Ok(rtt_data)
+}

src/rust/lqos_sys/src/throughput.rs

@@ -0,0 +1,41 @@
+use crate::{bpf_per_cpu_map::BpfPerCpuMap, XdpIpAddress};
+use anyhow::Result;
+
+/// Representation of the XDP map from map_traffic
+#[repr(C)]
+#[derive(Debug, Clone)]
+pub struct HostCounter {
+    /// Download bytes counter (keeps incrementing)
+    pub download_bytes: u64,
+
+    /// Upload bytes counter (keeps incrementing)
+    pub upload_bytes: u64,
+
+    /// Download packets counter (keeps incrementing)
+    pub download_packets: u64,
+
+    /// Upload packets counter (keeps incrementing)
+    pub upload_packets: u64,
+
+    /// Mapped TC handle, 0 if there isn't one.
+    pub tc_handle: u32,
+}
+
+impl Default for HostCounter {
+    fn default() -> Self {
+        Self {
+            download_bytes: 0,
+            download_packets: 0,
+            upload_bytes: 0,
+            upload_packets: 0,
+            tc_handle: 0,
+        }
+    }
+}
+
+/// Queries the underlying `map_traffic` eBPF pinned map, and returns every entry.
+pub fn get_throughput_map() -> Result<Vec<(XdpIpAddress, Vec<HostCounter>)>> {
+    Ok(BpfPerCpuMap::<XdpIpAddress, HostCounter>::from_path(
+        "/sys/fs/bpf/map_traffic",
+    )?.dump_vec())
+}

src/rust/lqos_sys/src/xdp_ip_address.rs

@@ -0,0 +1,131 @@
+use byteorder::{BigEndian, ByteOrder};
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
+
+/// XdpIpAddress provides helpful conversion between the XDP program's
+/// native storage of IP addresses in `[u8; 16]` blocks of bytes and
+/// Rust `IpAddr` types.
+#[repr(C)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct XdpIpAddress(pub [u8; 16]);
+
+impl Default for XdpIpAddress {
+    fn default() -> Self {
+        Self ([0xFF; 16])
+    }
+}
+
+impl XdpIpAddress {
+    /// Converts a Rust `IpAddr` type into an `XdpIpAddress`.
+    /// 
+    /// # Arguments
+    /// 
+    /// * `ip` - the IP Address to convert.
+    pub fn from_ip(ip: IpAddr) -> Self {
+        let mut result = Self::default();
+        match ip {
+            IpAddr::V4(ip) => {
+                result.0[12] = ip.octets()[0];
+                result.0[13] = ip.octets()[1];
+                result.0[14] = ip.octets()[2];
+                result.0[15] = ip.octets()[3];
+            }
+            IpAddr::V6(ip) => {
+                for i in 0..8 {
+                    let base = i * 2;
+                    result.0[base] = ip.octets()[base];
+                    result.0[base+1] = ip.octets()[base + 1];
+                }
+            }
+        }
+
+        result
+    }
+
+    /// Converts an `XdpIpAddress` type to a Rust `IpAddr` type
+    pub fn as_ip(&self) -> IpAddr {
+        if self.0[0] == 0xFF
+            && self.0[1] == 0xFF
+            && self.0[2] == 0xFF
+            && self.0[3] == 0xFF
+            && self.0[4] == 0xFF
+            && self.0[5] == 0xFF
+            && self.0[6] == 0xFF
+            && self.0[7] == 0xFF
+            && self.0[8] == 0xFF
+            && self.0[9] == 0xFF
+            && self.0[10] == 0xFF
+            && self.0[11] == 0xFF
+        {
+            // It's an IPv4 Address
+            IpAddr::V4(Ipv4Addr::new(
+                self.0[12],
+                self.0[13],
+                self.0[14],
+                self.0[15],
+            ))
+        } else {
+            // It's an IPv6 address
+            IpAddr::V6(Ipv6Addr::new(
+                BigEndian::read_u16(&self.0[0..2]),
+                BigEndian::read_u16(&self.0[2..4]),
+                BigEndian::read_u16(&self.0[4..6]),
+                BigEndian::read_u16(&self.0[6..8]),
+                BigEndian::read_u16(&self.0[8..10]),
+                BigEndian::read_u16(&self.0[10..12]),
+                BigEndian::read_u16(&self.0[12..14]),
+                BigEndian::read_u16(&self.0[14..]),
+            ))
+        }
+    }
+}
+
+impl Into<IpAddr> for XdpIpAddress {
+    fn into(self) -> IpAddr {
+        self.as_ip()
+    }
+}
+
+impl From<IpAddr> for XdpIpAddress {
+    fn from(ip: IpAddr) -> Self {
+        Self::from_ip(ip)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_default_xdp_ip() {
+        let default = XdpIpAddress::default();
+        assert_eq!(default.0, [0xFF; 16]);
+    }
+
+    #[test]
+    fn test_from_ipv4() {
+        let ip = XdpIpAddress::from_ip("1.2.3.4".parse().unwrap());
+        for n in 0..12 {
+            assert_eq!(ip.0[n], 0xFF);
+        }
+        assert_eq!(ip.0[12], 1);
+        assert_eq!(ip.0[13], 2);
+        assert_eq!(ip.0[14], 3);
+        assert_eq!(ip.0[15], 4);
+    }
+
+    #[test]
+    fn test_to_ipv4() {
+        let raw_ip = XdpIpAddress([0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 1, 2, 3, 4]);
+        let ip = raw_ip.as_ip();
+        let intended_ip : IpAddr = "1.2.3.4".parse().unwrap();
+        assert_eq!(ip, intended_ip);
+    }
+
+    #[test]
+    fn test_ipv6_round_trip() {
+        let ipv6 = IpAddr::V6("2001:db8:85a3::8a2e:370:7334".parse().unwrap());
+        let xip = XdpIpAddress::from_ip(ipv6);
+        let test = xip.as_ip();
+        assert_eq!(ipv6, test);
+    }
+}

src/rust/lqosd/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "lqosd"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+default = ["equinix_tests"]
+equinix_tests = []
+
+[dependencies]
+anyhow = "1"
+lqos_config = { path = "../lqos_config" }
+lqos_sys = { path = "../lqos_sys" }
+tokio = { version = "1.22", features = [ "full", "parking_lot" ] }
+lazy_static = "1.4"
+parking_lot = "0.12"
+lqos_bus = { path = "../lqos_bus" }
+signal-hook = "0.3"
+serde_json = "1"
+serde = { version = "1.0", features = ["derive"] }
+notify = { version = "5.0.0", default-features = false, feature=["macos_kqueue"] } # Not using crossbeam because of Tokio
+env_logger = "0"
+log = "0"

src/rust/lqosd/README.md

@@ -0,0 +1,66 @@
+# LQOSD
+
+**The LibreQoS Daemon** is designed to run as a `systemd` service at all times. It provides:
+
+* Load/Unload the XDP/TC programs (they unload when the program exits)
+* Configure XDP/TC, based on the content of `ispConfig.py`.
+   * Includes support for "on a stick" mode, using `OnAStick = True, StickVlanA = 1, StickVlanB = 2`.
+* Hosts a lightweight server offering "bus" queries for clients (such as `lqtop` and `xdp_iphash_to_cpu_cmdline`).
+   * See the `lqos_bus` sub-project for bus details.
+* Periodically gathers statistics for distribution to other systems via the bus.
+
+## Required Configuration
+
+You *must* have a file present called `/etc/lqos`. At a minimum, it must tell `lqosd` where to find the LibreQoS configuration. For example:
+
+```toml
+lqos_directory = '/opt/libreqos/v1.3'
+```
+
+## Offload Tuning
+
+`lqosd` can set kernel tunables for you on start-up. These are specified in `/etc/lqos` also, in the `[tuning]` section:
+
+```toml
+[tuning]
+stop_irq_balance = true
+netdev_budget_usecs = 20
+netdev_budget_packets = 1
+rx_usecs = 0
+tx_usecs = 0
+disable_rxvlan = true
+disable_txvlan = true
+disable_offload = [ "gso", "tso", "lro", "sg", "gro" ]
+```
+
+> If this section is not present, no tuning will be performed.
+
+## Bifrost - eBPF Kernel Bridge
+
+To enable the kernel-side eBPF bridge, edit `/etc/lqos`:
+
+```toml
+[bridge]
+use_kernel_bridge = true
+interface_mapping = [
+	{ name = "eth1", redirect_to = "eth2", scan_vlans = false },
+	{ name = "eth2", redirect_to = "eth1", scan_vlans = false }
+]
+vlan_mapping = []
+```
+
+Each interface must be a *physical* interface, not a VLAN. If you set `scan_vlans` to `true`, you can specify mapping rules.
+
+```toml
+[bridge]
+use_kernel_bridge = true
+interface_mapping = [
+	{ name = "eth1", redirect_to = "eth1", scan_vlans = true },
+]
+vlan_mapping = [
+	{ parent = "eth1", tag = 3, redirect_to = 4 },
+	{ parent = "eth1", tag = 4, redirect_to = 3 },
+]
+```
+
+Reciprocal mappings are created NOT automatically, you have to specify each mapping. When you are using "on a stick" mode, you need to redirect to the same interface.

src/rust/lqosd/src/ip_mapping.rs

@@ -0,0 +1,50 @@
+use anyhow::Result;
+use lqos_bus::{BusResponse, IpMapping, TcHandle};
+use lqos_sys::XdpIpAddress;
+
+fn expect_ack(result: Result<()>) -> BusResponse {
+    if result.is_ok() {
+        BusResponse::Ack
+    } else {
+        BusResponse::Fail(format!("{:?}", result))
+    }
+}
+
+pub(crate) fn map_ip_to_flow(
+    ip_address: &str,
+    tc_handle: &TcHandle,
+    cpu: u32,
+    upload: bool,
+) -> BusResponse {
+    expect_ack(lqos_sys::add_ip_to_tc(
+        &ip_address,
+        *tc_handle,
+        cpu,
+        upload,
+    ))
+}
+
+pub(crate) fn del_ip_flow(ip_address: &str, upload: bool) -> BusResponse {
+    expect_ack(lqos_sys::del_ip_from_tc(ip_address, upload))
+}
+
+pub(crate) fn clear_ip_flows() -> BusResponse {
+    expect_ack(lqos_sys::clear_ips_from_tc())
+}
+
+pub(crate) fn list_mapped_ips() -> BusResponse {
+    if let Ok(raw) = lqos_sys::list_mapped_ips() {
+        let data = raw
+            .iter()
+            .map(|(ip_key, ip_data)| IpMapping {
+                ip_address: XdpIpAddress(ip_key.address).as_ip().to_string(),
+                prefix_length: ip_key.prefixlen,
+                tc_handle: TcHandle::from_u32(ip_data.tc_handle),
+                cpu: ip_data.cpu,
+            })
+            .collect();
+        BusResponse::MappedIps(data)
+    } else {
+        BusResponse::Fail("Unable to get IP map".to_string())
+    }
+}

src/rust/lqosd/src/libreqos_tracker/mod.rs

@@ -0,0 +1,7 @@
+//mod shaped_devices;
+mod queue_structure;
+mod queueing_structure;
+
+//pub(crate) use shaped_devices::spawn_shaped_devices_monitor;
+pub(crate) use queue_structure::spawn_queue_structure_monitor;
+pub(crate) use queue_structure::QUEUE_STRUCTURE;

src/rust/lqosd/src/libreqos_tracker/queue_structure.rs

@@ -0,0 +1,37 @@
+//! Tracks changes to the ShapedDevices.csv file in LibreQoS.
+
+use lazy_static::*;
+use parking_lot::RwLock;
+use anyhow::Result;
+use tokio::task::spawn_blocking;
+use crate::libreqos_tracker::queueing_structure::{QueueNetwork, read_queueing_structure, QueueNode};
+
+lazy_static! {
+    /// Global storage of the shaped devices csv data.
+    /// Updated by the file system watcher whenever
+    /// the underlying file changes.
+    pub(crate) static ref QUEUE_STRUCTURE : RwLock<Result<Vec<QueueNode>>> = RwLock::new(read_queueing_structure());
+}
+
+pub async fn spawn_queue_structure_monitor() {
+    spawn_blocking(|| {
+        let _ = watch_for_shaped_devices_changing();
+    });
+}
+
+/// Fires up a Linux file system watcher than notifies
+/// when `ShapedDevices.csv` changes, and triggers a reload.
+fn watch_for_shaped_devices_changing() -> Result<()> {
+    use notify::{Watcher, RecursiveMode, Config};
+
+    let (tx, rx) = std::sync::mpsc::channel();
+    let mut watcher = notify::RecommendedWatcher::new(tx, Config::default())?;
+
+    watcher.watch(&QueueNetwork::path()?, RecursiveMode::NonRecursive)?;
+    loop {
+        let _ = rx.recv();
+        let new_file = read_queueing_structure();
+        log::info!("queuingStructure.csv changed");
+        *QUEUE_STRUCTURE.write() = new_file;
+    }
+}

src/rust/lqosd/src/libreqos_tracker/queueing_structure/mod.rs

@@ -0,0 +1,9 @@
+mod reader;
+use anyhow::Result;
+
+pub use reader::{QueueNode, QueueNetwork};
+
+pub fn read_queueing_structure() -> Result<Vec<reader::QueueNode>> {
+    let network = reader::QueueNetwork::from_json()?;
+    Ok(network.to_flat())
+}

src/rust/lqosd/src/libreqos_tracker/queueing_structure/reader.rs

@@ -0,0 +1,165 @@
+use std::path::{PathBuf, Path};
+use anyhow::{Result, Error};
+use lqos_bus::TcHandle;
+use serde_json::Value;
+use lqos_config::EtcLqos;
+
+pub struct QueueNetwork {
+    cpu_node: Vec<QueueNode>,
+}
+
+#[derive(Default, Clone, Debug)]
+pub struct QueueNode {
+    pub download_bandwidth_mbps: u64,
+    pub upload_bandwidth_mbps: u64,
+    pub download_bandwidth_mbps_min: u64,
+    pub upload_bandwidth_mbps_min: u64,
+    pub class_id: TcHandle,
+    pub up_class_id: TcHandle,
+    pub parent_class_id: TcHandle,
+    pub up_parent_class_id: TcHandle,
+    pub class_major: u32,
+    pub up_class_major: u32,
+    pub class_minor: u32,
+    pub cpu_num: u32,
+    pub up_cpu_num: u32,
+    pub circuits: Vec<QueueNode>,
+    pub circuit_id: Option<String>,
+    pub circuit_name: Option<String>,
+    pub parent_node: Option<String>,
+    pub devices: Vec<QueueNode>,
+    pub comment: String,
+    pub device_id: Option<String>,
+    pub device_name: Option<String>,
+    pub mac: Option<String>,
+}
+
+fn read_hex_string(s: &str) -> Result<u32> {
+    Ok(u32::from_str_radix(&s.replace("0x", ""), 16)?)
+}
+
+impl QueueNetwork {
+    pub fn path() -> Result<PathBuf> {
+        let cfg = EtcLqos::load()?;
+        let base_path = Path::new(&cfg.lqos_directory);
+        Ok(base_path.join("queuingStructure.json"))
+    }
+
+    fn exists() -> bool {
+        if let Ok(path) = QueueNetwork::path() {
+            path.exists()
+        } else {
+            false
+        }
+    }
+
+    pub(crate) fn from_json() -> Result<Self> {
+        let path = QueueNetwork::path()?;
+        if !QueueNetwork::exists() {
+            return Err(Error::msg("queueStructure.json does not exist yet. Try running LibreQoS?"));
+        }
+        let raw_string = std::fs::read_to_string(path)?;
+        let mut result = Self {
+            cpu_node: Vec::new(),
+        };
+        let json: Value = serde_json::from_str(&raw_string)?;
+        if let Value::Object(map) = &json {
+            if let Some(network) = map.get("Network") {
+                if let Value::Object(map) = network {
+                    for (key, value) in map.iter() {
+                        result.cpu_node.push(QueueNode::from_json(&key, value)?);
+                    }
+                } else {
+                    return Err(Error::msg("Unable to parse network object structure"));
+                }
+            } else {
+                return Err(Error::msg("Network entry not found"));
+            }
+        } else {
+            return Err(Error::msg("Unable to parse queueStructure.json"));
+        }
+
+        Ok(result)
+    }
+
+    pub fn to_flat(&self) -> Vec<QueueNode> {
+        let mut result = Vec::new();
+        for cpu in self.cpu_node.iter() {
+            result.push(cpu.clone());
+            let children = cpu.to_flat();
+            result.extend_from_slice(&children);
+        }
+        for c in result.iter_mut() {
+            c.circuits.clear();
+            c.devices.clear();
+        }
+        result
+    }
+}
+
+impl QueueNode {
+    fn from_json(key: &str, value: &Value) -> Result<Self> {
+        let mut result = Self::default();
+        if let Value::Object(map) = value {
+            for (key, value) in map.iter() {
+                match key.as_str() {
+                    "downloadBandwidthMbps" | "maxDownload" => result.download_bandwidth_mbps = value.as_u64().unwrap(),
+                    "uploadBandwidthMbps" | "maxUpload" => result.upload_bandwidth_mbps = value.as_u64().unwrap(),
+                    "downloadBandwidthMbpsMin"| "minDownload" => result.download_bandwidth_mbps_min = value.as_u64().unwrap(),
+                    "uploadBandwidthMbpsMin" | "minUpload" => result.upload_bandwidth_mbps_min = value.as_u64().unwrap(),
+                    "classid" => result.class_id = TcHandle::from_string(&value.as_str().unwrap().to_string())?,
+                    "up_classid" => result.up_class_id = TcHandle::from_string(value.as_str().unwrap().to_string())?,
+                    "classMajor" => result.class_major = read_hex_string(value.as_str().unwrap())?,
+                    "up_classMajor" => result.up_class_major = read_hex_string(value.as_str().unwrap())?,
+                    "classMinor" => result.class_minor = read_hex_string(value.as_str().unwrap())?,
+                    "cpuNum" => result.cpu_num = read_hex_string(value.as_str().unwrap())?,
+                    "up_cpuNum" => result.up_cpu_num = read_hex_string(value.as_str().unwrap())?,
+                    "parentClassID" => result.parent_class_id = TcHandle::from_string(value.as_str().unwrap().to_string())?,
+                    "up_parentClassID" => result.up_parent_class_id = TcHandle::from_string(value.as_str().unwrap().to_string())?,
+                    "circuitId" | "circuitID" => result.circuit_id = Some(value.as_str().unwrap().to_string()),
+                    "circuitName" => result.circuit_name = Some(value.as_str().unwrap().to_string()),
+                    "parentNode" | "ParentNode" => result.parent_node = Some(value.as_str().unwrap().to_string()),
+                    "comment" => result.comment = value.as_str().unwrap().to_string(),
+                    "deviceId" | "deviceID" => result.device_id = Some(value.as_str().unwrap().to_string()),
+                    "deviceName" => result.device_name = Some(value.as_str().unwrap().to_string()),
+                    "mac" => result.mac = Some(value.as_str().unwrap().to_string()),
+                    "ipv4s" => {}, // Ignore
+                    "ipv6s" => {},
+                    "circuits" => {
+                        if let Value::Array(array) = value {
+                            for c in array.iter() {
+                                result.circuits.push(QueueNode::from_json(key, c)?);
+                            }
+                        }
+                    }
+                    "devices" => {
+                        if let Value::Array(array) = value {
+                            for c in array.iter() {
+                                result.devices.push(QueueNode::from_json(key, c)?);
+                            }
+                        }
+                    }
+                    _ => log::error!("I don't know how to parse key: [{key}]"),
+                }
+            }
+        } else {
+            return Err(Error::msg(format!("Unable to parse node structure for [{key}]")));
+        }
+        Ok(result)
+    }
+
+    fn to_flat(&self) -> Vec<QueueNode> {
+        let mut result = Vec::new();
+        for c in self.circuits.iter() {
+            result.push(c.clone());
+            let children = c.to_flat();
+            result.extend_from_slice(&children);
+        }
+        for c in self.devices.iter() {
+            result.push(c.clone());
+            let children = c.to_flat();
+            result.extend_from_slice(&children);
+        }
+        result
+    }
+}

src/rust/lqosd/src/libreqos_tracker/shaped_devices.rs

@@ -0,0 +1,38 @@
+//! Tracks changes to the ShapedDevices.csv file in LibreQoS.
+
+use lazy_static::*;
+use lqos_config::ConfigShapedDevices;
+use parking_lot::RwLock;
+use anyhow::Result;
+use tokio::task::spawn_blocking;
+
+lazy_static! {
+    /// Global storage of the shaped devices csv data.
+    /// Updated by the file system watcher whenever
+    /// the underlying file changes.
+    pub(crate) static ref SHAPED_DEVICES : RwLock<ConfigShapedDevices> = RwLock::new(ConfigShapedDevices::load().unwrap());
+}
+
+pub async fn spawn_shaped_devices_monitor() {
+    spawn_blocking(|| {
+        let _ = watch_for_shaped_devices_changing();
+    });
+}
+
+/// Fires up a Linux file system watcher than notifies
+/// when `ShapedDevices.csv` changes, and triggers a reload.
+fn watch_for_shaped_devices_changing() -> Result<()> {
+    use notify::{Watcher, RecursiveMode, Config};
+
+    let (tx, rx) = std::sync::mpsc::channel();
+    let mut watcher = notify::RecommendedWatcher::new(tx, Config::default())?;
+
+    watcher.watch(&ConfigShapedDevices::path()?, RecursiveMode::NonRecursive)?;
+    loop {
+        let _ = rx.recv();
+        if let Ok(new_file) = ConfigShapedDevices::load() {
+            println!("ShapedDevices.csv changed");
+            *SHAPED_DEVICES.write() = new_file;
+        }
+    }
+}

src/rust/lqosd/src/lqos_daht_test.rs

@@ -0,0 +1,21 @@
+use std::process::Command;
+use lqos_bus::BusResponse;
+use lazy_static::*;
+use parking_lot::Mutex;
+use tokio::task::spawn_blocking;
+
+lazy_static! {
+    static ref TEST_BUSY : Mutex<bool> = Mutex::new(false);
+}
+
+pub async fn lqos_daht_test() -> BusResponse {
+    spawn_blocking(|| {
+        if let Some(_lock) = TEST_BUSY.try_lock() {
+            Command::new("/bin/ssh")
+                .args(["-t", "lqtest@lqos.taht.net", "\"/home/lqtest/bin/v6vsv4.sh\""])
+                .output()
+                .unwrap();
+        }
+    });
+    BusResponse::Ack
+}

src/rust/lqosd/src/main.rs

@@ -0,0 +1,124 @@
+mod ip_mapping;
+mod throughput_tracker;
+mod program_control;
+mod queue_tracker;
+mod libreqos_tracker;
+#[cfg(feature = "equinix_tests")]
+mod lqos_daht_test;
+mod offloads;
+use crate::ip_mapping::{clear_ip_flows, del_ip_flow, list_mapped_ips, map_ip_to_flow};
+use anyhow::Result;
+use lqos_bus::{
+    cookie_value, decode_request, encode_response, BusReply, BusRequest, BUS_BIND_ADDRESS,
+};
+use lqos_config::{LibreQoSConfig, EtcLqos};
+use lqos_sys::LibreQoSKernels;
+use signal_hook::{consts::SIGINT, iterator::Signals};
+use tokio::{
+    io::{AsyncReadExt, AsyncWriteExt},
+    net::{TcpListener, TcpStream}, join,
+};
+use log::{info, warn};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    env_logger::init(); // Configure log level with RUST_LOG environment variable
+    info!("LibreQoS Daemon Starting");
+    let config = LibreQoSConfig::load()?;
+    let etc_lqos = EtcLqos::load()?;
+
+    // Disable offloading
+    if let Some(tuning) = &etc_lqos.tuning {
+        offloads::bpf_sysctls().await;
+        if tuning.stop_irq_balance {
+            offloads::stop_irq_balance().await;
+        }        
+        offloads::netdev_budget(tuning.netdev_budget_usecs, tuning.netdev_budget_packets).await;
+        offloads::ethtool_tweaks(&config.internet_interface, tuning).await;
+        offloads::ethtool_tweaks(&config.isp_interface, tuning).await;
+    }
+
+    // Start the XDP/TC kernels
+    let kernels = if config.on_a_stick_mode {
+        LibreQoSKernels::on_a_stick_mode(&config.internet_interface, config.stick_vlans.1, config.stick_vlans.0)?
+    } else {
+        LibreQoSKernels::new(&config.internet_interface, &config.isp_interface)?
+    };
+
+    // Spawn tracking sub-systems
+    join!(
+        throughput_tracker::spawn_throughput_monitor(),
+        queue_tracker::spawn_queue_monitor(),
+        //libreqos_tracker::spawn_shaped_devices_monitor(),
+        libreqos_tracker::spawn_queue_structure_monitor(),
+    );
+
+    let mut signals = Signals::new(&[SIGINT])?;
+
+    std::thread::spawn(move || {
+        for sig in signals.forever() {
+            warn!("Received signal {:?}", sig);
+            std::mem::drop(kernels);
+            std::process::exit(0);
+        }
+    });
+
+    // Main bus listen loop
+    let listener = TcpListener::bind(BUS_BIND_ADDRESS).await?;
+    info!("Listening on: {}", BUS_BIND_ADDRESS);
+    loop {
+        let (mut socket, _) = listener.accept().await?;
+        tokio::spawn(async move {
+            let mut buf = vec![0; 1024];
+
+            let _ = socket
+                .read(&mut buf)
+                .await
+                .expect("failed to read data from socket");
+
+            if let Ok(request) = decode_request(&buf) {
+                if request.auth_cookie == cookie_value() {
+                    let mut response = BusReply {
+                        auth_cookie: request.auth_cookie,
+                        responses: Vec::new(),
+                    };
+                    for req in request.requests.iter() {
+                        //println!("Request: {:?}", req);
+                        response.responses.push(match req {
+                            BusRequest::Ping => lqos_bus::BusResponse::Ack,
+                            BusRequest::GetCurrentThroughput => {
+                                throughput_tracker::current_throughput()
+                            }
+                            BusRequest::GetTopNDownloaders(n) => throughput_tracker::top_n(*n),
+                            BusRequest::GetWorstRtt(n) => throughput_tracker::worst_n(*n),
+                            BusRequest::MapIpToFlow {
+                                ip_address,
+                                tc_handle,
+                                cpu,
+                                upload,
+                            } => map_ip_to_flow(ip_address, tc_handle, *cpu, *upload),
+                            BusRequest::DelIpFlow { ip_address, upload } => del_ip_flow(&ip_address, *upload),
+                            BusRequest::ClearIpFlow => clear_ip_flows(),
+                            BusRequest::ListIpFlow => list_mapped_ips(),
+                            BusRequest::XdpPping => throughput_tracker::xdp_pping_compat(),
+                            BusRequest::RttHistogram => throughput_tracker::rtt_histogram(),
+                            BusRequest::HostCounts => throughput_tracker::host_counts(),
+                            BusRequest::AllUnknownIps => throughput_tracker::all_unknown_ips(),
+                            BusRequest::ReloadLibreQoS => program_control::reload_libre_qos(),
+                            BusRequest::GetRawQueueData(circuit_id) => queue_tracker::get_raw_circuit_data(&circuit_id),
+                            #[cfg(feature = "equinix_tests")]
+                            BusRequest::RequestLqosEquinixTest => lqos_daht_test::lqos_daht_test().await,
+                        });
+                    }
+                    //println!("{:?}", response);
+                    let _ = reply(&encode_response(&response).unwrap(), &mut socket).await;
+                }
+            }
+        });
+    }
+}
+
+async fn reply(response: &[u8], socket: &mut TcpStream) -> Result<()> {
+    socket.write_all(&response).await?;
+    Ok(())
+}

src/rust/lqosd/src/offloads.rs

@@ -0,0 +1,57 @@
+use lqos_config::Tunables;
+use tokio::process::Command;
+
+pub async fn bpf_sysctls() {
+    let _ = Command::new("/sbin/sysctl")
+        .arg(format!("net.core.bpf_jit_enable=1"))
+        .output().await;
+}
+
+pub async fn stop_irq_balance() {
+    let _ = Command::new("/bin/systemctl")
+        .args(["stop", "irqbalance"])
+        .output().await;
+}
+
+pub async fn netdev_budget(usecs: u32, packets: u32) {
+    let _ = Command::new("/sbin/sysctl")
+        .arg(format!("net.core.netdev_budget_usecs={usecs}"))
+        .output().await;
+
+    let _ = Command::new("/sbin/sysctl")
+        .arg(format!("net.core.netdev_budget={packets}"))
+        .output().await;
+}
+
+async fn disable_individual_offload(interface: &str, feature: &str) {
+    let _ = Command::new("/sbin/ethtool")
+        .args(["--offload", interface, feature, "off"])
+        .output().await;
+}
+
+pub async fn ethtool_tweaks(interface: &str, tuning: &Tunables) {
+    // Disabling individually to avoid complaints that a card doesn't support a feature anyway
+    for feature in tuning.disable_offload.iter() {
+        disable_individual_offload(interface, feature).await;
+    }
+    
+    let _ = Command::new("/sbin/ethtool")
+        .args(["-C", interface, "rx-usecs", &format!("\"{}\"", tuning.rx_usecs)])
+        .output().await;
+
+    let _ = Command::new("/sbin/ethtool")
+        .args(["-C", interface, "tx-usecs", &format!("\"{}\"", tuning.tx_usecs)])
+        .output().await;
+
+    if tuning.disable_rxvlan {
+        let _ = Command::new("/sbin/ethtool")
+            .args(["-K", interface, "rxvlan", "off"])
+            .output().await;
+    }
+
+    if tuning.disable_txvlan {
+        let _ = Command::new("/sbin/ethtool")
+            .args(["-K", interface, "txvlan", "off"])
+            .output().await;
+    }
+}

src/rust/lqosd/src/program_control.rs

@@ -0,0 +1,9 @@
+use lqos_bus::BusResponse;
+
+pub fn reload_libre_qos() -> BusResponse {
+    let result = lqos_config::load_libreqos();
+    match result {
+        Ok(message) => BusResponse::ReloadLibreQoS(message),
+        Err(..) => BusResponse::Fail("Unable to reload LibreQoS".to_string()),
+    }
+}

src/rust/lqosd/src/queue_tracker/mod.rs

@@ -0,0 +1,135 @@
+use std::{time::{Duration, Instant}, collections::HashMap};
+use lqos_bus::BusResponse;
+use lqos_config::LibreQoSConfig;
+use tokio::{task, time};
+use crate::libreqos_tracker::QUEUE_STRUCTURE;
+use self::queue_reader::QueueType;
+mod queue_reader;
+use lazy_static::*;
+use parking_lot::RwLock;
+
+lazy_static! {
+    pub(crate) static ref CIRCUIT_TO_QUEUE : RwLock<HashMap<String, (QueueType, QueueType)>> = RwLock::new(HashMap::new());
+}
+
+fn track_queues() {
+    let config = LibreQoSConfig::load().unwrap();
+    let queues = if config.on_a_stick_mode {
+        let queues = queue_reader::read_tc_queues(&config.internet_interface).unwrap();
+        vec![queues]
+    } else {
+        vec![
+            queue_reader::read_tc_queues(&config.isp_interface).unwrap(),
+            queue_reader::read_tc_queues(&config.internet_interface).unwrap(),
+        ]
+    };
+
+    // Time to associate queues with circuits
+    let mut mapping: HashMap<String, (QueueType, QueueType)> = HashMap::new();
+    let structure_lock = QUEUE_STRUCTURE.read();
+    // Do a quick check that we have a queue association
+    if let Ok(structure) = &*structure_lock {
+        for circuit in structure.iter().filter(|c| c.circuit_id.is_some()) {
+            if config.on_a_stick_mode {
+                let download = queues[0].iter().find(|q| {
+                    match q {
+                        QueueType::Cake(cake) => {
+                            let (maj,min) = cake.parent.get_major_minor();
+                            let (cmaj,cmin) = circuit.class_id.get_major_minor();
+                            maj==cmaj && min == cmin
+                        }
+                        QueueType::FqCodel(fq) => {
+                            fq.parent.as_u32() == circuit.class_id.as_u32()
+                        }
+                        _ => false,
+                    }
+                });
+                let upload = queues[0].iter().find(|q| {
+                    match q {
+                        QueueType::Cake(cake) => {
+                            let (maj,min) = cake.parent.get_major_minor();
+                            let (cmaj,cmin) = circuit.up_class_id.get_major_minor();
+                            maj==cmaj && min == cmin
+                        }
+                        QueueType::FqCodel(fq) => {
+                            fq.parent.as_u32() == circuit.up_class_id.as_u32()
+                        }
+                        _ => false,
+                    }
+                });
+                mapping.insert(
+                    circuit.circuit_id.as_ref().unwrap().clone(),
+                    (download.unwrap().clone(), upload.unwrap().clone())
+                );
+            } else {
+                let download = queues[0].iter().find(|q| {
+                    match q {
+                        QueueType::Cake(cake) => {
+                            let (maj,min) = cake.parent.get_major_minor();
+                            let (cmaj,cmin) = circuit.class_id.get_major_minor();
+                            maj==cmaj && min == cmin
+                        }
+                        QueueType::FqCodel(fq) => {
+                            fq.parent.as_u32() == circuit.class_id.as_u32()
+                        }
+                        _ => false,
+                    }
+                });
+                let upload = queues[1].iter().find(|q| {
+                    match q {
+                        QueueType::Cake(cake) => {
+                            let (maj,min) = cake.parent.get_major_minor();
+                            let (cmaj,cmin) = circuit.class_id.get_major_minor();
+                            maj==cmaj && min == cmin
+                        }
+                        QueueType::FqCodel(fq) => {
+                            fq.parent.as_u32() == circuit.class_id.as_u32()
+                        }
+                        _ => false,
+                    }
+                });
+                mapping.insert(
+                    circuit.circuit_id.as_ref().unwrap().clone(),
+                    (download.unwrap().clone(), upload.unwrap().clone())
+                );
+            }
+        }
+        *CIRCUIT_TO_QUEUE.write() = mapping;
+    }
+}
+
+pub async fn spawn_queue_monitor() {
+    let _ = task::spawn(async {
+        let mut interval = time::interval(Duration::from_secs(10));
+
+        loop {
+            let now = Instant::now();
+            let _ = task::spawn_blocking(move || {
+                track_queues()
+            })
+            .await;
+            let elapsed = now.elapsed();
+            //println!("TC Reader tick with mapping consumed {:.4} seconds.", elapsed.as_secs_f32());
+            if elapsed.as_secs_f32() < 10.0 {
+                let duration = Duration::from_secs(10) - elapsed;
+                //println!("Sleeping for {:.2} seconds", duration.as_secs_f32());
+                tokio::time::sleep(duration).await;
+            } else {
+                interval.tick().await;
+            }
+        }
+    });
+}
+
+pub fn get_raw_circuit_data(circuit_id: &str) -> BusResponse {
+    let reader = CIRCUIT_TO_QUEUE.read();
+    if let Some(circuit) = reader.get(circuit_id) {
+        if let Ok(json) = serde_json::to_string(circuit) {
+            BusResponse::RawQueueData(json)
+        } else {
+            BusResponse::RawQueueData(String::new())
+        }
+    } else {
+        BusResponse::RawQueueData(String::new())
+    }
+}

src/rust/lqosd/src/queue_tracker/queue_reader/CAKE.md

@@ -0,0 +1,28 @@
+ways_collisions, if you are seeing a lot of these, it's indicative of a ping flood of some kind.
+  thresh          6Mbit
+  target          5.0ms
+  interval      100.0ms
+these are the bandwidth and codel settiings. On a vpn interface, or in a place where the typical rtt is to sydney and back, you can set rtt lower in cake, otherwise, don't touch.
+ pk_delay        440us
+  av_delay         13us
+  sp_delay          2us
+All these are ewmas (I forget the interval). pk_delay would be a good statistic to graph, tho I suspect that the highest peak delays would be on the lowest end plans, so peak delay by plan would pull out folk that were above average here.
+av_delay is pretty useless, like all averages, IMHO. Still I could be wrong.
+sp_delay = sparse delay and is an indicator of overload. Customers with the largest sparse delay would be interesting. Ideally this number never cracks 2ms.
+backlog - a persistent backlog is OK, but a sign that someone is using a torrent-like app, doing a big backup, or underprovisioned in some way. Seeing a persistent backlog in an AP on the other hand says the AP is overloaded.
+pkts
+bytes
+do what they say and are a summary statistic since invokation of cake. 
+way_inds
+way_mis
+way_cols are indicators of how well the FQ is working, but not ver good ones. 
+drops
+marks
+ack_drop
+in general I care about pkts, ack_drops, drops and marks, most, these are best plotted together on an invlog scale. When last I looked robert was combining drops and marks to treat as drops (which they are), but I am caring about the rise of ECN in general. As for the top 10 approach, the largest number of drops or marks relative to packets, would be useful.
+  sp_flows            1
+  bk_flows            1
+  un_flows            0
+bulk_flows indicate large transfers. They have nothing to do with diffserv marks, they are just bulk. 
+max_len - if this is greater than your MTU, GRO and GSO are loose on your network. 
+Quantum varies as a function of bandwidth, it may be 300 is more optimal than an MTU even at higher bandwidths than we first tested cake at.

src/rust/lqosd/src/queue_tracker/queue_reader/mod.rs

@@ -0,0 +1,58 @@
+mod tc_mq;
+mod tc_htb;
+mod tc_fq_codel;
+mod tc_cake;
+use anyhow::{Result, Error};
+use serde::Serialize;
+use serde_json::Value;
+use std::process::Command;
+
+#[derive(Debug, Clone, Serialize)]
+pub(crate) enum QueueType {
+    Mq(tc_mq::TcMultiQueue),
+    Htb(tc_htb::TcHtb),
+    FqCodel(tc_fq_codel::TcFqCodel),
+    Cake(tc_cake::TcCake),
+    ClsAct,
+}
+
+impl QueueType {
+    fn parse(kind: &str, map: &serde_json::Map<std::string::String, Value>) -> Result<QueueType> {
+        match kind {
+            "mq" => Ok(QueueType::Mq(tc_mq::TcMultiQueue::from_json(map)?)),
+            "htb" => Ok(QueueType::Htb(tc_htb::TcHtb::from_json(map)?)),
+            "fq_codel" => Ok(QueueType::FqCodel(tc_fq_codel::TcFqCodel::from_json(map)?)),
+            "cake" => Ok(QueueType::Cake(tc_cake::TcCake::from_json(map)?)),
+            "clsact" => Ok(QueueType::ClsAct),
+            _ => Err(Error::msg(format!("Unknown queue kind: {kind}"))),
+        }   
+    }
+}
+
+pub(crate) fn read_tc_queues(interface: &str) -> Result<Vec<QueueType>> {
+    let mut result = Vec::new();
+    let command_output = Command::new("/sbin/tc")
+        .args(["-s", "-j", "qdisc", "show", "dev", interface])
+        .output()?;
+    let json = String::from_utf8(command_output.stdout)?;
+    let json: Value = serde_json::from_str(&json)?;
+    if let Value::Array(array) = &json {
+        for entry in array.iter() {
+            match entry {
+                Value::Object(map) => {
+                    if let Some(kind) = map.get("kind") {
+                        if let Some(kind) = kind.as_str() {
+                            let qdisc = QueueType::parse(kind, map)?;
+                            result.push(qdisc);
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+    } else {
+        return Err(Error::msg("Unable to parse TC data array"));
+    }
+
+    Ok(result)
+}

src/rust/lqosd/src/queue_tracker/queue_reader/tc_cake.rs

@@ -0,0 +1,309 @@
+/*
+
+{
+    "kind": "cake",
+    "handle": "9cb1:",
+    "parent": "3:205",
+    "options": {
+        "bandwidth": "unlimited",
+        "diffserv": "diffserv4",
+        "flowmode": "triple-isolate",
+        "nat": false,
+        "wash": false,
+        "ingress": false,
+        "ack-filter": "disabled",
+        "split_gso": true,
+        "rtt": 100000,
+        "raw": true,
+        "overhead": 0,
+        "fwmark": "0"
+    },
+    "bytes": 49072087981,
+    "packets": 35792920,
+    "drops": 1162331,
+    "overlimits": 0,
+    "requeues": 0,
+    "backlog": 0,
+    "qlen": 0,
+    "memory_used": 2002176,
+    "memory_limit": 15503360,
+    "capacity_estimate": 0,
+    "min_network_size": 56,
+    "max_network_size": 1514,
+    "min_adj_size": 56,
+    "max_adj_size": 1514,
+    "avg_hdr_offset": 14,
+    "tins": [
+        {
+            "threshold_rate": 0,
+            "sent_bytes": 0,
+            "backlog_bytes": 0,
+            "target_us": 5000,
+            "interval_us": 100000,
+            "peak_delay_us": 0,
+            "avg_delay_us": 0,
+            "base_delay_us": 0,
+            "sent_packets": 0,
+            "way_indirect_hits": 0,
+            "way_misses": 0,
+            "way_collisions": 0,
+            "drops": 0,
+            "ecn_mark": 0,
+            "ack_drops": 0,
+            "sparse_flows": 0,
+            "bulk_flows": 0,
+            "unresponsive_flows": 0,
+            "max_pkt_len": 0,
+            "flow_quantum": 1514
+        },
+        {
+            "threshold_rate": 0,
+            "sent_bytes": 47096460394,
+            "backlog_bytes": 0,
+            "target_us": 5000,
+            "interval_us": 100000,
+            "peak_delay_us": 152,
+            "avg_delay_us": 7,
+            "base_delay_us": 1,
+            "sent_packets": 34376628,
+            "way_indirect_hits": 156580,
+            "way_misses": 89285,
+            "way_collisions": 0,
+            "drops": 984524,
+            "ecn_mark": 10986,
+            "ack_drops": 0,
+            "sparse_flows": 1,
+            "bulk_flows": 0,
+            "unresponsive_flows": 0,
+            "max_pkt_len": 1514,
+            "flow_quantum": 1514
+        },
+        {
+            "threshold_rate": 0,
+            "sent_bytes": 3481013747,
+            "backlog_bytes": 0,
+            "target_us": 5000,
+            "interval_us": 100000,
+            "peak_delay_us": 1080,
+            "avg_delay_us": 141,
+            "base_delay_us": 1,
+            "sent_packets": 2456582,
+            "way_indirect_hits": 282,
+            "way_misses": 3916,
+            "way_collisions": 0,
+            "drops": 177080,
+            "ecn_mark": 25,
+            "ack_drops": 0,
+            "sparse_flows": 0,
+            "bulk_flows": 0,
+            "unresponsive_flows": 0,
+            "max_pkt_len": 1514,
+            "flow_quantum": 1514
+        },
+        {
+            "threshold_rate": 0,
+            "sent_bytes": 145417781,
+            "backlog_bytes": 0,
+            "target_us": 5000,
+            "interval_us": 100000,
+            "peak_delay_us": 566715,
+            "avg_delay_us": 421103,
+            "base_delay_us": 3,
+            "sent_packets": 122041,
+            "way_indirect_hits": 11,
+            "way_misses": 148,
+            "way_collisions": 0,
+            "drops": 727,
+            "ecn_mark": 0,
+            "ack_drops": 0,
+            "sparse_flows": 2,
+            "bulk_flows": 0,
+            "unresponsive_flows": 0,
+            "max_pkt_len": 1242,
+            "flow_quantum": 1514
+        }
+    ]
+},
+
+ */
+
+use anyhow::{Result, Error};
+use lqos_bus::TcHandle;
+use serde::Serialize;
+use serde_json::Value;
+
+#[derive(Default, Clone, Debug, Serialize)]
+pub(crate) struct TcCake {
+    pub(crate) handle: TcHandle,
+    pub(crate) parent: TcHandle,
+    options: TcCakeOptions,
+    bytes: u64,
+    packets: u64,
+    overlimits: u64,
+    requeues: u64,
+    backlog: u64,
+    qlen: u64,
+    memory_used: u64,
+    memory_limit: u64,
+    capacity_estimate: u64,
+    min_network_size: u64,
+    max_network_size: u64,
+    min_adj_size: u64,
+    max_adj_size: u64,
+    avg_hdr_offset: u64,
+    tins: Vec<TcCakeTin>,
+    drops: u64,
+ }
+
+ #[derive(Default, Clone, Debug, Serialize)]
+  struct TcCakeOptions {
+    bandwidth: String,
+    diffserv: String,
+    flowmode: String,
+    nat: bool,
+    wash: bool,
+    ingress: bool,
+    ack_filter: String,
+    split_gso: bool,
+    rtt: u64,
+    raw: bool,
+    overhead: u64,
+    fwmark: String,
+ }
+
+ #[derive(Default, Clone, Debug, Serialize)]
+ struct TcCakeTin {
+    threshold_rate: u64,
+    sent_bytes: u64,
+    backlog_bytes: u64,
+    target_us: u64,
+    interval_us: u64,
+    peak_delay_us: u64,
+    avg_delay_us: u64,
+    base_delay_us: u64,
+    sent_packets: u64,
+    way_indirect_hits: u64,
+    way_misses: u64,
+    way_collisions: u64,
+    drops: u64,
+    ecn_marks: u64,
+    ack_drops: u64,
+    sparse_flows: u64,
+    bulk_flows: u64,
+    unresponsive_flows: u64,
+    max_pkt_len: u64,
+    flow_quantum: u64,
+ }
+
+ impl TcCake {
+    pub(crate) fn from_json(map: &serde_json::Map<std::string::String, Value>) -> Result<Self> {
+        let mut result = Self::default();
+        for (key, value) in map.iter() {
+            match key.as_str() {
+                "handle" => result.handle = TcHandle::from_string(value.as_str().unwrap())?,
+                "parent" => result.parent = TcHandle::from_string(value.as_str().unwrap())?,
+                "bytes" => result.bytes = value.as_u64().unwrap(),
+                "packets" => result.packets = value.as_u64().unwrap(),
+                "overlimits" => result.overlimits = value.as_u64().unwrap(),
+                "requeues" => result.requeues = value.as_u64().unwrap(),
+                "backlog" => result.backlog = value.as_u64().unwrap(),
+                "qlen" => result.qlen = value.as_u64().unwrap(),
+                "memory_used" => result.memory_used = value.as_u64().unwrap(),
+                "memory_limit" => result.memory_limit = value.as_u64().unwrap(),
+                "capacity_estimate" => result.capacity_estimate = value.as_u64().unwrap(),
+                "min_network_size" => result.min_network_size = value.as_u64().unwrap(),
+                "max_network_size" => result.max_network_size = value.as_u64().unwrap(),
+                "min_adj_size" => result.min_adj_size = value.as_u64().unwrap(),
+                "max_adj_size" => result.max_adj_size = value.as_u64().unwrap(),
+                "avg_hdr_offset" => result.avg_hdr_offset = value.as_u64().unwrap(),
+                "drops" => result.drops = value.as_u64().unwrap(),
+                "options" => result.options = TcCakeOptions::from_json(value)?,
+                "tins" => {
+                    match value {
+                        Value::Array(array) => {
+                            for value in array.iter() {
+                                result.tins.push(TcCakeTin::from_json(value)?);
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+                "kind" => {},
+               _ => {
+                    log::error!("Unknown entry in Tc-cake: {key}");
+                }
+            }
+        }
+        Ok(result)
+    }
+}
+
+impl TcCakeOptions {
+    fn from_json(value: &Value) -> Result<Self> {
+        match value {
+            Value::Object(map) => {
+                let mut result = Self::default();
+                for (key, value) in map.iter() {
+                    match key.as_str() {
+                        "bandwidth" => result.bandwidth = value.as_str().unwrap().to_string(),
+                        "diffserv" => result.diffserv = value.as_str().unwrap().to_string(),
+                        "flowmode" => result.flowmode = value.as_str().unwrap().to_string(),
+                        "nat" => result.nat = value.as_bool().unwrap(),
+                        "wash" => result.wash = value.as_bool().unwrap(),
+                        "ingress" => result.ingress = value.as_bool().unwrap(),
+                        "ack-filter" => result.ack_filter = value.as_str().unwrap().to_string(),
+                        "split_gso" => result.split_gso = value.as_bool().unwrap(),
+                        "rtt" => result.rtt = value.as_u64().unwrap(),
+                        "raw" => result.raw = value.as_bool().unwrap(),
+                        "overhead" => result.overhead = value.as_u64().unwrap(),
+                        "fwmark" => result.fwmark = value.as_str().unwrap().to_string(),
+                        _ => {
+                            log::error!("Unknown entry in Tc-cake-options: {key}");
+                        }
+                    }
+                }
+                Ok(result)
+            }
+            _ => Err(Error::msg("Unable to parse cake options")),
+        }        
+    }
+}
+
+impl TcCakeTin {
+    fn from_json(value: &Value) -> Result<Self> {
+        match value {
+            Value::Object(map) => {
+                let mut result = Self::default();
+                for (key, value) in map.iter() {
+                    match key.as_str() {
+                        "threshold_rate" => result.threshold_rate = value.as_u64().unwrap(),
+                        "sent_bytes" => result.sent_bytes = value.as_u64().unwrap(),
+                        "backlog_bytes" => result.backlog_bytes = value.as_u64().unwrap(),
+                        "target_us" => result.target_us = value.as_u64().unwrap(),
+                        "interval_us" => result.interval_us = value.as_u64().unwrap(),
+                        "peak_delay_us" => result.peak_delay_us = value.as_u64().unwrap(),
+                        "avg_delay_us" => result.avg_delay_us = value.as_u64().unwrap(),
+                        "base_delay_us" => result.base_delay_us = value.as_u64().unwrap(),
+                        "sent_packets" => result.sent_packets = value.as_u64().unwrap(),
+                        "way_indirect_hits" => result.way_indirect_hits = value.as_u64().unwrap(),
+                        "way_misses" => result.way_misses = value.as_u64().unwrap(),
+                        "way_collisions" => result.way_collisions = value.as_u64().unwrap(),
+                        "drops" => result.drops = value.as_u64().unwrap(),
+                        "ecn_mark" => result.ecn_marks = value.as_u64().unwrap(),
+                        "ack_drops" => result.ack_drops = value.as_u64().unwrap(),
+                        "sparse_flows" => result.sparse_flows = value.as_u64().unwrap(),
+                        "bulk_flows" => result.bulk_flows = value.as_u64().unwrap(),
+                        "unresponsive_flows" => result.unresponsive_flows = value.as_u64().unwrap(),
+                        "max_pkt_len" => result.max_pkt_len = value.as_u64().unwrap(),
+                        "flow_quantum" => result.flow_quantum = value.as_u64().unwrap(),
+                        _ => {
+                            log::error!("Unknown entry in Tc-cake-tin: {key}");
+                        }
+                    }
+                }
+                Ok(result)
+            }
+            _ => Err(Error::msg("Unable to parse cake tin options")),
+        }        
+    }
+}

src/rust/lqosd/src/queue_tracker/queue_reader/tc_fq_codel.rs

@@ -0,0 +1,101 @@
+/*
+{"kind":"fq_codel","handle":"0:","parent":"7fff:a",
+    "options":{"limit":10240,"flows":1024,"quantum":1514,"target":4999,"interval":99999,"memory_limit":33554432,"ecn":true,"drop_batch":64},
+    "bytes":560,"packets":8,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0,"maxpacket":0,"drop_overlimit":0,"new_flow_count":0,
+    "ecn_mark":0,"new_flows_len":0,"old_flows_len":0},
+*/
+
+use anyhow::{Result, Error};
+use lqos_bus::TcHandle;
+use serde::Serialize;
+use serde_json::Value;
+
+#[derive(Default, Clone, Debug, Serialize)]
+pub(crate) struct TcFqCodel {
+    handle: TcHandle,
+    pub(crate) parent: TcHandle,
+    options: TcFqCodelOptions,
+    bytes: u64,
+    packets: u64,
+    drops: u64,
+    overlimits: u64,
+    requeues: u64,
+    backlog: u64,
+    qlen: u64,
+    maxpacket: u64,
+    drop_overlimit: u64,
+    new_flow_count: u64,
+    ecn_mark: u64,
+    new_flows_len: u64,
+    old_flows_len: u64,
+}
+
+#[derive(Default, Clone, Debug, Serialize)]
+struct TcFqCodelOptions {
+    limit: u64,
+    flows: u64,
+    quantum: u64,
+    target: u64,
+    interval: u64,
+    memory_limit: u64,
+    ecn: bool,
+    drop_batch: u64,
+}
+
+impl TcFqCodel {
+    pub(crate) fn from_json(map: &serde_json::Map<std::string::String, Value>) -> Result<Self> {
+        let mut result = Self::default();
+        for (key, value) in map.iter() {
+            match key.as_str() {
+                "handle" => result.handle = TcHandle::from_string(value.as_str().unwrap())?,
+                "parent" => result.parent = TcHandle::from_string(value.as_str().unwrap())?,
+                "bytes" => result.bytes = value.as_u64().unwrap(),
+                "packets" => result.packets = value.as_u64().unwrap(),
+                "drops" => result.drops = value.as_u64().unwrap(),
+                "overlimits" => result.overlimits = value.as_u64().unwrap(),
+                "requeues" => result.requeues = value.as_u64().unwrap(),
+                "backlog" => result.backlog = value.as_u64().unwrap(),
+                "qlen" => result.qlen = value.as_u64().unwrap(),
+                "maxpacket" => result.maxpacket = value.as_u64().unwrap(),
+                "drop_overlimit" => result.drop_overlimit = value.as_u64().unwrap(),
+                "new_flow_count" => result.new_flow_count = value.as_u64().unwrap(),
+                "ecn_mark" => result.ecn_mark = value.as_u64().unwrap(),
+                "new_flows_len" => result.new_flows_len = value.as_u64().unwrap(),
+                "old_flows_len" => result.old_flows_len = value.as_u64().unwrap(),
+                "options" => result.options = TcFqCodelOptions::from_json(value)?,
+                "kind" => {},
+                _ => {
+                    log::error!("Unknown entry in Tc-codel: {key}");
+                }
+            }
+        }
+        Ok(result)
+    }
+}
+
+impl TcFqCodelOptions {
+    fn from_json(value: &Value) -> Result<Self> {
+        match value {
+            Value::Object(map) => {
+                let mut result = Self::default();
+                for (key, value) in map.iter() {
+                    match key.as_str() {
+                        "limit" => result.limit = value.as_u64().unwrap(),
+                        "flows" => result.flows = value.as_u64().unwrap(),
+                        "quantum" => result.quantum = value.as_u64().unwrap(),
+                        "target" => result.target = value.as_u64().unwrap(),
+                        "interval" => result.interval = value.as_u64().unwrap(),
+                        "memory_limit" => result.memory_limit = value.as_u64().unwrap(),
+                        "ecn" => result.ecn = value.as_bool().unwrap(),
+                        "drop_batch" => result.drop_batch = value.as_u64().unwrap(),
+                        _ => {
+                            log::error!("Unknown entry in Tc-codel-options: {key}");
+                        }
+                    }
+                }
+                Ok(result)
+            }
+            _ => Err(Error::msg("Unable to parse fq_codel options")),
+        }        
+    }
+}

src/rust/lqosd/src/queue_tracker/queue_reader/tc_htb.rs

@@ -0,0 +1,79 @@
+/*
+{"kind":"htb","handle":"2:","parent":"7fff:2","options":{"r2q":10,"default":"0x2","direct_packets_stat":7,"direct_qlen":1000},
+"bytes":1920791512305,"packets":1466145855,"drops":32136937,"overlimits":2627500070,"requeues":1224,"backlog":0,"qlen":0}
+*/
+
+use anyhow::{Result, Error};
+use lqos_bus::TcHandle;
+use serde::Serialize;
+use serde_json::Value;
+
+#[derive(Default, Clone, Debug, Serialize)]
+pub(crate) struct TcHtb {
+    handle: TcHandle,
+    parent: TcHandle,
+    options: TcHtbOptions,
+    bytes: u64,
+    packets: u64,
+    drops: u64,
+    overlimits: u64,
+    requeues: u64,
+    backlog: u64,
+    qlen: u64,
+}
+
+#[derive(Default, Clone, Debug, Serialize)]
+struct TcHtbOptions {
+    r2q: u64,
+    default: TcHandle,
+    direct_packets_stat: u64,
+    direct_qlen: u64,
+}
+
+impl TcHtb {
+    pub(crate) fn from_json(map: &serde_json::Map<std::string::String, Value>) -> Result<Self> {
+        let mut result = Self::default();
+        for (key, value) in map.iter() {
+            match key.as_str() {
+                "handle" => result.handle = TcHandle::from_string(value.as_str().unwrap())?,
+                "parent" => result.parent = TcHandle::from_string(value.as_str().unwrap())?,
+                "bytes" => result.bytes = value.as_u64().unwrap(),
+                "packets" => result.packets = value.as_u64().unwrap(),
+                "drops" => result.drops = value.as_u64().unwrap(),
+                "overlimits" => result.overlimits = value.as_u64().unwrap(),
+                "requeues" => result.requeues = value.as_u64().unwrap(),
+                "backlog" => result.backlog = value.as_u64().unwrap(),
+                "qlen" => result.qlen = value.as_u64().unwrap(),
+                "options" => result.options = TcHtbOptions::from_json(value)?,
+                "kind" => {},
+                _ => {
+                    log::error!("Unknown entry in Tc-HTB: {key}");
+                }
+            }
+        }
+        Ok(result)
+    }
+}
+
+impl TcHtbOptions {
+    fn from_json(value: &Value) -> Result<Self> {
+        match value {
+            Value::Object(map) => {
+                let mut result = Self::default();
+                for (key, value) in map.iter() {
+                    match key.as_str() {
+                        "r2q" => result.r2q = value.as_u64().unwrap(),
+                        "default" => result.default = TcHandle::from_string(value.as_str().unwrap())?,
+                        "direct_packets_stat" => result.direct_packets_stat = value.as_u64().unwrap(),
+                        "direct_qlen" => result.direct_qlen = value.as_u64().unwrap(),
+                        _ => {
+                            log::error!("Unknown entry in Tc-HTB: {key}");
+                        }
+                    }
+                }
+                Ok(result)
+            }
+            _ => Err(Error::msg("Unable to parse HTB options")),
+        }        
+    }
+}

src/rust/lqosd/src/queue_tracker/queue_reader/tc_mq.rs

@@ -0,0 +1,46 @@
+/*
+{"kind":"mq","handle":"7fff:","root":true,"options":{},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0}
+*/
+
+use lqos_bus::TcHandle;
+use serde::Serialize;
+use serde_json::Value;
+use anyhow::Result;
+
+#[derive(Default, Clone, Debug, Serialize)]
+pub(crate) struct TcMultiQueue {
+    handle: TcHandle,
+    root: bool,
+    bytes: u64,
+    packets: u64,
+    drops: u64,
+    overlimits: u64,
+    requeues: u64,
+    backlog: u64,
+    qlen: u64,
+}
+
+impl TcMultiQueue {
+    pub(crate) fn from_json(map: &serde_json::Map<std::string::String, Value>) -> Result<Self> {
+        let mut result = Self::default();
+        for (key, value) in map.iter() {
+            match key.as_str() {
+                "handle" => result.handle = TcHandle::from_string(value.as_str().unwrap())?,
+                "root" => result.root = value.as_bool().unwrap(),
+                "bytes" => result.bytes = value.as_u64().unwrap(),
+                "packets" => result.packets = value.as_u64().unwrap(),
+                "drops" => result.drops = value.as_u64().unwrap(),
+                "overlimits" => result.overlimits = value.as_u64().unwrap(),
+                "requeues" => result.requeues = value.as_u64().unwrap(),
+                "backlog" => result.backlog = value.as_u64().unwrap(),
+                "qlen" => result.qlen = value.as_u64().unwrap(),
+                "kind" => {},
+                "options" => {},
+                _ => {
+                    log::error!("Unknown entry in Tc-MQ: {key}");
+                }
+            }
+        }
+        Ok(result)
+    }
+}

src/rust/lqosd/src/throughput_tracker/mod.rs

@@ -0,0 +1,233 @@
+mod tracking_data;
+mod throughput_entry;
+use lazy_static::*;
+use lqos_bus::{BusResponse, IpStats, XdpPpingResult, TcHandle};
+use lqos_sys::{XdpIpAddress, get_throughput_map};
+use parking_lot::RwLock;
+use std::time::{Duration, Instant};
+use tokio::{task, time};
+use crate::throughput_tracker::tracking_data::ThroughputTracker;
+
+const RETIRE_AFTER_SECONDS: u64 = 30;
+
+lazy_static! {
+    static ref THROUGHPUT_TRACKER: RwLock<ThroughputTracker> =
+        RwLock::new(ThroughputTracker::new());
+}
+
+pub async fn spawn_throughput_monitor() {
+    let _ = task::spawn(async {
+        let mut interval = time::interval(Duration::from_secs(1));
+
+        loop {
+            let now = Instant::now();
+            let _ = task::spawn_blocking(move || {
+                let rtt = lqos_sys::get_tcp_round_trip_times();
+                if let Ok(value_dump) = get_throughput_map() {
+                    let mut thoughput = THROUGHPUT_TRACKER.write();
+                    let _ = thoughput.tick(&value_dump, rtt);
+                }
+            })
+            .await;
+            let elapsed = now.elapsed();
+            //println!("Tick consumed {:.2} seconds.", elapsed.as_secs_f32());
+            if elapsed.as_secs_f32() < 1.0 {
+                let duration = Duration::from_secs(1) - elapsed;
+                //println!("Sleeping for {:.2} seconds", duration.as_secs_f32());
+                tokio::time::sleep(duration).await;
+            } else {
+                interval.tick().await;
+            }
+        }
+    });
+}
+
+pub fn current_throughput() -> BusResponse {
+    let (bits_per_second, packets_per_second, shaped_bits_per_second) = {
+        let tp = THROUGHPUT_TRACKER.read();
+        (tp.bits_per_second(), tp.packets_per_second(), tp.shaped_bits_per_second())
+    };
+    BusResponse::CurrentThroughput {
+        bits_per_second,
+        packets_per_second,
+        shaped_bits_per_second,
+    }
+}
+
+#[inline(always)]
+fn retire_check(cycle: u64, recent_cycle: u64) -> bool {
+    cycle < recent_cycle + RETIRE_AFTER_SECONDS
+}
+
+pub fn top_n(n: u32) -> BusResponse {
+    let mut full_list: Vec<(XdpIpAddress, (u64, u64), (u64, u64), f32, TcHandle)> = {
+        let tp = THROUGHPUT_TRACKER.read();
+        tp.raw_data
+            .iter()
+            .filter(|(ip, _)| !ip.as_ip().is_loopback())
+            .filter(|(_, d)| retire_check(tp.cycle, d.most_recent_cycle))
+            .map(|(ip, te)| {
+                (
+                    *ip,
+                    te.bytes_per_second,
+                    te.packets_per_second,
+                    te.median_latency(),
+                    te.tc_handle,
+                )
+            })
+            .collect()
+    };
+    full_list.sort_by(|a, b| b.1 .0.cmp(&a.1 .0));
+    let result = full_list
+        .iter()
+        .take(n as usize)
+        .map(
+            |(ip, (bytes_dn, bytes_up), (packets_dn, packets_up), median_rtt, tc_handle)| IpStats {
+                ip_address: ip.as_ip().to_string(),
+                bits_per_second: (bytes_dn * 8, bytes_up * 8),
+                packets_per_second: (*packets_dn, *packets_up),
+                median_tcp_rtt: *median_rtt,
+                tc_handle: *tc_handle,
+            },
+        )
+        .collect();
+    BusResponse::TopDownloaders(result)
+}
+
+pub fn worst_n(n: u32) -> BusResponse {
+    let mut full_list: Vec<(XdpIpAddress, (u64, u64), (u64, u64), f32, TcHandle)> = {
+        let tp = THROUGHPUT_TRACKER.read();
+        tp.raw_data
+            .iter()
+            .filter(|(ip, _)| !ip.as_ip().is_loopback())
+            .filter(|(_, d)| retire_check(tp.cycle, d.most_recent_cycle))
+            .map(|(ip, te)| {
+                (
+                    *ip,
+                    te.bytes_per_second,
+                    te.packets_per_second,
+                    te.median_latency(),
+                    te.tc_handle,
+                )
+            })
+            .collect()
+    };
+    full_list.sort_by(|a, b| b.3.partial_cmp(&a.3).unwrap());
+    let result = full_list
+        .iter()
+        .take(n as usize)
+        .map(
+            |(ip, (bytes_dn, bytes_up), (packets_dn, packets_up), median_rtt, tc_handle)| IpStats {
+                ip_address: ip.as_ip().to_string(),
+                bits_per_second: (bytes_dn * 8, bytes_up * 8),
+                packets_per_second: (*packets_dn, *packets_up),
+                median_tcp_rtt: *median_rtt,
+                tc_handle: *tc_handle,
+            },
+        )
+        .collect();
+    BusResponse::WorstRtt(result)
+}
+
+pub fn xdp_pping_compat() -> BusResponse {
+    let raw = THROUGHPUT_TRACKER.read();
+    let result = raw.raw_data
+        .iter()
+        .filter(|(_, d)| retire_check(raw.cycle, d.most_recent_cycle))
+        .filter_map(|(_ip, data)| {
+            if data.tc_handle.as_u32() > 0 {
+                let mut valid_samples : Vec<u32> = data.recent_rtt_data.iter().filter(|d| **d > 0).map(|d| *d).collect();
+                let samples = valid_samples.len() as u32;
+                if samples > 0 {
+                    valid_samples.sort_by(|a,b| (*a).cmp(&b));
+                    let median = valid_samples[valid_samples.len() / 2] as f32 / 100.0;
+                    let max = *(valid_samples.iter().max().unwrap()) as f32 / 100.0;
+                    let min = *(valid_samples.iter().min().unwrap()) as f32 / 100.0;
+                    let sum = valid_samples.iter().sum::<u32>() as f32 / 100.0;
+                    let avg = sum / samples as f32;
+
+                    Some(XdpPpingResult {
+                        tc: format!("{}", data.tc_handle.to_string()),
+                        median,
+                        avg,
+                        max,
+                        min,
+                        samples
+                    })
+                } else {
+                    None
+                }
+            } else {
+                None
+            }
+        })
+        .collect();
+    BusResponse::XdpPping(result)
+}
+
+pub fn rtt_histogram() -> BusResponse {
+    let mut result = vec![0; 20];
+    let reader = THROUGHPUT_TRACKER.read();
+    for (_, data) in reader.raw_data.iter().filter(|(_, d)| retire_check(reader.cycle, d.most_recent_cycle))
+    {
+        let valid_samples : Vec<u32> = data.recent_rtt_data.iter().filter(|d| **d > 0).map(|d| *d).collect();
+        let samples = valid_samples.len() as u32;
+        if samples > 0 {
+            let median = valid_samples[valid_samples.len() / 2] as f32 / 100.0;
+            let median = f32::min(200.0, median);
+            let column = (median / 10.0) as usize;
+            result[usize::min(column, 19)] += 1;
+        }
+    }
+
+    BusResponse::RttHistogram(result)
+}
+
+pub fn host_counts() -> BusResponse {
+    let mut total = 0;
+    let mut shaped = 0;
+    let tp = THROUGHPUT_TRACKER.read();
+    tp.raw_data.iter().filter(|(_, d)| retire_check(tp.cycle, d.most_recent_cycle))
+    .for_each(|(_,d)| {
+        total += 1;
+        if d.tc_handle.as_u32() != 0 {
+            shaped += 1;
+        }
+    });
+    BusResponse::HostCounts((total, shaped))
+}
+
+pub fn all_unknown_ips() -> BusResponse {
+    let mut full_list: Vec<(XdpIpAddress, (u64, u64), (u64, u64), f32, TcHandle, u64)> = {
+        let tp = THROUGHPUT_TRACKER.read();
+        tp.raw_data
+            .iter()
+            .filter(|(ip, _)| !ip.as_ip().is_loopback())
+            .filter(|(_, d)| d.tc_handle.as_u32() == 0)
+            .map(|(ip, te)| {
+                (
+                    *ip,
+                    te.bytes,
+                    te.packets,
+                    te.median_latency(),
+                    te.tc_handle,
+                    te.most_recent_cycle
+                )
+            })
+            .collect()
+    };
+    full_list.sort_by(|a, b| b.5.partial_cmp(&a.5).unwrap());
+    let result = full_list
+        .iter()
+        .map(
+            |(ip, (bytes_dn, bytes_up), (packets_dn, packets_up), median_rtt, tc_handle, _last_seen)| IpStats {
+                ip_address: ip.as_ip().to_string(),
+                bits_per_second: (bytes_dn * 8, bytes_up * 8),
+                packets_per_second: (*packets_dn, *packets_up),
+                median_tcp_rtt: *median_rtt,
+                tc_handle: *tc_handle,
+            },
+        )
+        .collect();
+    BusResponse::AllUnknownIps(result)
+}

src/rust/lqosd/src/throughput_tracker/throughput_entry.rs

@@ -0,0 +1,32 @@
+use lqos_bus::TcHandle;
+
+#[derive(Debug)]
+pub(crate) struct ThroughputEntry {
+    pub(crate) first_cycle: u64,
+    pub(crate) most_recent_cycle: u64,
+    pub(crate) bytes: (u64, u64),
+    pub(crate) packets: (u64, u64),
+    pub(crate) prev_bytes: (u64, u64),
+    pub(crate) prev_packets: (u64, u64),
+    pub(crate) bytes_per_second: (u64, u64),
+    pub(crate) packets_per_second: (u64, u64),
+    pub(crate) tc_handle: TcHandle,
+    pub(crate) recent_rtt_data: [u32; 60],
+    pub(crate) last_fresh_rtt_data_cycle: u64,
+}
+
+impl ThroughputEntry {
+    pub(crate) fn median_latency(&self) -> f32 {
+        let mut shifted: Vec<f32> = self
+            .recent_rtt_data
+            .iter()
+            .filter(|n| **n != 0)
+            .map(|n| *n as f32 / 100.0)
+            .collect();
+        if shifted.is_empty() {
+            return 0.0;
+        }
+        shifted.sort_by(|a, b| a.partial_cmp(&b).unwrap());
+        shifted[shifted.len() / 2]
+    }
+}