From e6cb58bc650d1d3265e3fa466f8da2b3189fe4d9 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 16:20:56 +0000
Subject: [PATCH 01/67] Adds "client throughput" to the queue tree tab of
 circuit_queue

Also moves all the circuit graphs to use the same graphing
system as the other graphs on the plot.

FIXES #315
---
 .../static/circuit_queue.html                 | 124 +++++++++++-------
 1 file changed, 75 insertions(+), 49 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 43815d67..3e7daeb6 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -94,8 +94,9 @@
                         </ul>
                     </div>
                     <div class="col-sm-2">
-                        <a href="#" class="btn btn-small btn-info" id="btnPause"><i class="fa fa-pause"></i> Pause</a> 
-                        <a href="#" class="btn btn-small btn-info" id="btnSlow"><i class="fa fa-hourglass"></i> Slow Mode</a>
+                        <a href="#" class="btn btn-small btn-info" id="btnPause"><i class="fa fa-pause"></i> Pause</a>
+                        <a href="#" class="btn btn-small btn-info" id="btnSlow"><i class="fa fa-hourglass"></i> Slow
+                            Mode</a>
                     </div>
                 </div>
             </div>
@@ -385,16 +386,17 @@
                     { x: this.x_axis, y: this.backlog.tins[2].y, type: 'scattergl', mode: 'markers', name: 'Video', marker: { size: 4 } },
                     { x: this.x_axis, y: this.backlog.tins[3].y, type: 'scattergl', mode: 'markers', name: 'Voice', marker: { size: 4 } },
                 ];
-                
+
                 if (this.backlogPlotted == null) {
                     this.backlogPlotted = true;
                     Plotly.newPlot(
-                        graph, 
-                        graphData, 
-                        { 
-                            margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, 
-                            yaxis: { automargin: true, title: "Bytes" }, 
-                            xaxis: { automargin: true, title: "Time since now" } });
+                        graph,
+                        graphData,
+                        {
+                            margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 },
+                            yaxis: { automargin: true, title: "Bytes" },
+                            xaxis: { automargin: true, title: "Time since now" }
+                        });
                 } else {
                     Plotly.redraw(graph, graphData);
                 }
@@ -411,11 +413,13 @@
 
                 if (this.delaysPlotted == null) {
                     Plotly.newPlot(
-                        graph, 
-                        graphData, 
-                        { margin: { l: 8, r: 0, b: 0, t: 0, pad: 4 }, 
-                        yaxis: { automargin: true, title: "log10(ms)", range: [-1.0, 1.0] }, 
-                        xaxis: { automargin: true, title: "Time since now" } });
+                        graph,
+                        graphData,
+                        {
+                            margin: { l: 8, r: 0, b: 0, t: 0, pad: 4 },
+                            yaxis: { automargin: true, title: "log10(ms)", range: [-1.0, 1.0] },
+                            xaxis: { automargin: true, title: "Time since now" }
+                        });
                     this.delaysPlotted = true;
                 } else {
                     Plotly.redraw(graph, graphData);
@@ -441,7 +445,7 @@
                 let graphData = [
                     { x: this.x_axis, y: this.throughput.tins[tin].y, type: 'scatter', mode: 'markers' }
                 ];
-                if (this.tinsPlotted == null) {   
+                if (this.tinsPlotted == null) {
                     Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Bits" }, xaxis: { automargin: true, title: "Time since now" } });
                 } else {
                     Plotly.redraw(graph, graphData);
@@ -502,7 +506,7 @@
                 this.per_ip = {};
                 this.y = {};
                 this.x_axis = [];
-                for (let i=0; i<capacity; ++i) {
+                for (let i = 0; i < capacity; ++i) {
                     this.x_axis.push(i);
                     this.x_axis.push(i);
                 }
@@ -514,7 +518,7 @@
             }
 
             clearQuantiles() {
-                for (let i=0; i<12; ++i) {
+                for (let i = 0; i < 12; ++i) {
                     this.quantiles[0][i] = 0;
                     this.quantiles[1][i] = 0;
                 }
@@ -526,7 +530,7 @@
                 if (!this.per_ip.hasOwnProperty(ip)) {
                     this.per_ip[ip] = [];
                     this.y[ip] = [];
-                    for (let i=0; i<this.capacity; ++i) {
+                    for (let i = 0; i < this.capacity; ++i) {
                         this.per_ip[ip].push(0);
                         this.per_ip[ip].push(0);
                         this.y[ip].push(0);
@@ -536,7 +540,7 @@
                 this.per_ip[ip][this.head] = down;
                 this.per_ip[ip][this.head + 1] = 0.0 - up;
                 this.head += 2;
-                if (this.head > this.capacity*2) {
+                if (this.head > this.capacity * 2) {
                     this.head = 0;
                 }
             }
@@ -557,30 +561,30 @@
             prepare() {
                 this.clearQuantiles();
                 for (const ip in this.per_ip) {
-                    let counter = this.capacity*2;
-                    for (let i = this.head; i < this.capacity*2; i++) {
-                        this.y[ip][counter] = this.per_ip[ip][i];              
+                    let counter = this.capacity * 2;
+                    for (let i = this.head; i < this.capacity * 2; i++) {
+                        this.y[ip][counter] = this.per_ip[ip][i];
                         counter--;
                     }
                     for (let i = 0; i < this.head; i++) {
                         this.y[ip][counter] = this.per_ip[ip][i];
                         counter--;
                     }
-                    for (let i=2; i<22; i+=2) {
-                        this.addQuantile(this.y[ip][i], this.y[ip][i+1]);
+                    for (let i = 2; i < 22; i += 2) {
+                        this.addQuantile(this.y[ip][i], this.y[ip][i + 1]);
                     }
                 }
             }
 
-            plot() {
-                let graph = document.getElementById("throughputGraph");
+            plot(target) {
+                let graph = document.getElementById(target);
                 let graphData = [];
                 for (const ip in this.per_ip) {
                     graphData.push({ x: this.x_axis, y: this.y[ip], name: ip, mode: 'markers', type: 'scattergl', marker: { size: 3 } });
                 }
-                if (this.plotted == null) {
-                    Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Traffic (bits)" }, xaxis: { automargin: true, title: "Time since now" } });                    
-                    this.plotted = true;
+                if (!this.hasOwnProperty("plotted" + target)) {
+                    Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Traffic (bits)" }, xaxis: { automargin: true, title: "Time since now" } });
+                    this["plotted" + target] = true;
                 } else {
                     Plotly.redraw(graph, graphData);
                 }
@@ -608,7 +612,7 @@
                 msgPackGet("/api/circuit_throughput/" + encodeURI(id), (data) => {
                     if (tpData == null) tpData = new ThroughputMonitor(300);
                     ips = [];
-                    for (let i=0; i < data.length; i++) {
+                    for (let i = 0; i < data.length; i++) {
                         let ip = data[i][0];
                         ips.push(ip);
                         let down = data[i][1];
@@ -616,13 +620,13 @@
                         tpData.ingest(ip, down, up);
                     }
                     tpData.prepare();
-                    tpData.plot();
+                    tpData.plot("throughputGraph");
                     tpData.plotQuantiles();
                 });
             }
         }
 
-        let funnels = new MultiRingBuffer(300);
+        let funnels = new ThroughputMonitor(300);
         let rtts = {};
         let circuitId = "";
         let builtFunnelDivs = false;
@@ -635,8 +639,24 @@
             circuitId = encodeURI(id);
             msgPackGet("/api/funnel_for_queue/" + circuitId, (data) => {
                 let html = "";
+
+                // Add the client on top
+                let row = "<div class='row row220'>";
+
+                row += "<div class='col-sm-12'>";
+                row += "<div class='card bg-light'>";
+                row += "<h5 class='card-title'><i class='fa fa-hourglass'></i> Client Throughput</h5>";
+                row += "<div id='tp_client' class='graph98 graph150'></div>";
+                row += "</div>";
+                row += "</div>";
+
+                row += "</div>";
+                html += row;
+
+                // Funnels
                 for (let i = 0; i < data.length; ++i) {
-                    funnels.push(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
+                    //funnels.push(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
+                    funnels.ingest(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
                     rtts[data[i][0]] = new RttHistogram();
 
                     let row = "<div class='row row220'>";
@@ -660,33 +680,38 @@
                 }
                 $("#pills-funnel").html(html);
                 builtFunnelDivs = true;
-                setTimeout(plotFunnels, 10);
             });
         }
 
         let plottedFunnels = {};
 
         function plotFunnels() {
+            if (tpData != null) tpData.plot("tp_client");
+            funnels.prepare();
             msgPackGet("/api/funnel_for_queue/" + encodeURI(circuitId), (data) => {
                 for (let i = 0; i < data.length; ++i) {
-                    funnels.push(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
-                    for (const [k, v] of Object.entries(funnels.data)) {
-                        let target_div = "tp" + k;                        
-                        let graphData = v.toScatterGraphData();
-                        let graph = document.getElementById(target_div);
-                        if (!plotFunnels.hasOwnProperty(target_div)) {
-                            Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true }, xaxis: { automargin: true, title: "Time since now" } }, { responsive: true });
-                        } else {
-                            Plotly.redraw(graph, graphData);
-                        }
-                    }
-
                     rtts[data[i][0]].clear();
+                    funnels.ingest(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
                     for (let j = 0; j < data[i][1][NetTrans.rtts].length; j++) {
                         rtts[data[i][0]].push(data[i][1][NetTrans.rtts][j]);
                     }
+
                     rtts[data[i][0]].plot("rtt" + data[i][0]);
                 }
+
+                for (const [k, v] of Object.entries(funnels.y)) {
+                    let target_div = "tp" + k;
+                    let graphData = [
+                        { x: funnels.x_axis, y: v, type: 'scatter', mode: 'markers', marker: { size: 3 } }
+                    ];
+                    let graph = document.getElementById(target_div);
+                    if (!plotFunnels.hasOwnProperty(target_div)) {
+                        Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Traffic (bits)" }, xaxis: { automargin: true, title: "Time since now" } });
+                    } else {
+                        Plotly.redraw(graph, graphData);
+                    }
+                }
+
             });
         }
 
@@ -828,14 +853,14 @@
         var slowMode = false;
 
         function showFps() {
-            if(!lastCalledTime) {
+            if (!lastCalledTime) {
                 lastCalledTime = Date.now();
                 fps = 0;
                 return;
             }
-            delta = (Date.now() - lastCalledTime)/1000;
+            delta = (Date.now() - lastCalledTime) / 1000;
             lastCalledTime = Date.now();
-            fps = 1/delta;
+            fps = 1 / delta;
             //$("#fps").text(fps.toFixed(0));
             worstDelta = Math.max(delta, worstDelta);
         }
@@ -846,6 +871,7 @@
                 switch (activeTab) {
                     case "pills-funnel-tab": {
                         getFunnel();
+                        getThroughput();
                     } break;
                     case "pills-flows-tab": {
                         getFlows();

From aa7cf1fc684d1b8e1602bb457ed4f75c739aa958 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 17:10:43 +0000
Subject: [PATCH 02/67] Add DNS resolution to displayed flow IP addresses

---
 src/rust/Cargo.lock                              | 14 ++++++++++++++
 src/rust/lqos_node_manager/Cargo.toml            |  2 ++
 src/rust/lqos_node_manager/src/main.rs           |  1 +
 src/rust/lqos_node_manager/src/queue_info.rs     | 11 ++++++++++-
 .../lqos_node_manager/src/tracker/cache/mod.rs   |  2 ++
 src/rust/lqos_node_manager/src/tracker/mod.rs    |  1 +
 .../lqos_node_manager/static/circuit_queue.html  |  4 ++--
 src/rust/lqos_node_manager/static/lqos.js        | 16 ++++++++++++++++
 8 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
index 5d681b67..299ff8b1 100644
--- a/src/rust/Cargo.lock
+++ b/src/rust/Cargo.lock
@@ -733,6 +733,18 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "dns-lookup"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53ecafc952c4528d9b51a458d1a8904b81783feff9fde08ab6ed2545ff396872"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "socket2",
+ "winapi",
+]
+
 [[package]]
 name = "either"
 version = "1.8.1"
@@ -1448,7 +1460,9 @@ name = "lqos_node_manager"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "dashmap",
  "default-net",
+ "dns-lookup",
  "jemallocator",
  "lqos_bus",
  "lqos_config",
diff --git a/src/rust/lqos_node_manager/Cargo.toml b/src/rust/lqos_node_manager/Cargo.toml
index 69b3c05b..818c8d42 100644
--- a/src/rust/lqos_node_manager/Cargo.toml
+++ b/src/rust/lqos_node_manager/Cargo.toml
@@ -19,6 +19,8 @@ sysinfo = "0"
 default-net = "0"
 nix = "0"
 once_cell = "1"
+dns-lookup = "1"
+dashmap = "5"
 
 # Support JemAlloc on supported platforms
 [target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies]
diff --git a/src/rust/lqos_node_manager/src/main.rs b/src/rust/lqos_node_manager/src/main.rs
index c0e13f32..1fe50750 100644
--- a/src/rust/lqos_node_manager/src/main.rs
+++ b/src/rust/lqos_node_manager/src/main.rs
@@ -77,6 +77,7 @@ fn rocket() -> _ {
         queue_info::packet_dump,
         queue_info::pcap,
         queue_info::request_analysis,
+        queue_info::dns_query,
         config_control::get_nic_list,
         config_control::get_current_python_config,
         config_control::get_current_lqosd_config,
diff --git a/src/rust/lqos_node_manager/src/queue_info.rs b/src/rust/lqos_node_manager/src/queue_info.rs
index fe5e7d46..3a0a34ab 100644
--- a/src/rust/lqos_node_manager/src/queue_info.rs
+++ b/src/rust/lqos_node_manager/src/queue_info.rs
@@ -1,6 +1,6 @@
 use crate::auth_guard::AuthGuard;
 use crate::cache_control::NoCache;
-use crate::tracker::SHAPED_DEVICES;
+use crate::tracker::{SHAPED_DEVICES, lookup_dns};
 use lqos_bus::{bus_request, BusRequest, BusResponse, FlowTransport, PacketHeader, QueueStoreTransit};
 use rocket::fs::NamedFile;
 use rocket::http::Status;
@@ -162,6 +162,15 @@ pub async fn pcap(id: usize, filename: String) -> Result<NoCache<NamedFile>, Sta
   Err(Status::NotFound)
 }
 
+#[get("/api/dns/<ip>")]
+pub async fn dns_query(ip: String) -> NoCache<String> {
+  if let Ok(ip) = ip.parse::<IpAddr>() {
+    NoCache::new(lookup_dns(ip))
+  } else {
+    NoCache::new(ip)
+  }
+}
+
 #[cfg(feature = "equinix_tests")]
 #[get("/api/run_btest")]
 pub async fn run_btest() -> NoCache<RawJson<String>> {
diff --git a/src/rust/lqos_node_manager/src/tracker/cache/mod.rs b/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
index 8829e9f5..2e0a7d89 100644
--- a/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
+++ b/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
@@ -5,7 +5,9 @@
 mod cpu_ram;
 mod shaped_devices;
 mod throughput;
+mod dns_cache;
 
 pub use cpu_ram::*;
 pub use shaped_devices::*;
 pub use throughput::THROUGHPUT_BUFFER;
+pub use dns_cache::lookup_dns;
diff --git a/src/rust/lqos_node_manager/src/tracker/mod.rs b/src/rust/lqos_node_manager/src/tracker/mod.rs
index 9eed911f..67cdf69c 100644
--- a/src/rust/lqos_node_manager/src/tracker/mod.rs
+++ b/src/rust/lqos_node_manager/src/tracker/mod.rs
@@ -10,6 +10,7 @@ pub use cache::SHAPED_DEVICES;
 pub use cache_manager::{update_tracking, update_total_throughput_buffer};
 use lqos_bus::{bus_request, BusRequest, BusResponse, IpStats, TcHandle};
 use rocket::serde::{Deserialize, Serialize, msgpack::MsgPack};
+pub use cache::lookup_dns;
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(crate = "rocket::serde")]
diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 3e7daeb6..718a5327 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -813,13 +813,13 @@
                     }
                     html += "<tr>";
                     html += "<td>" + data[i][0][FlowTrans.proto] + "</td>";
-                    html += "<td>" + data[i][0][FlowTrans.src] + "</td>";
+                    html += "<td>" + ipToHostname(data[i][0][FlowTrans.src]) + "</td>";
                     if (data[i][0].proto == "ICMP") {
                         html += "<td>" + icmpType(data[i][0][FlowTrans.src_port]) + "</td>";
                     } else {
                         html += "<td>" + data[i][0][FlowTrans.src_port] + "</td>";
                     }
-                    html += "<td>" + data[i][0][FlowTrans.dst] + "</td>";
+                    html += "<td>" + ipToHostname(data[i][0][FlowTrans.dst]) + "</td>";
                     if (data[i][0][FlowTrans.proto] == "ICMP") {
                         if (data[i][1] != null) {
                             html += "<td>" + icmpType(data[i][1][FlowTrans.src_port]) + "</td>";
diff --git a/src/rust/lqos_node_manager/static/lqos.js b/src/rust/lqos_node_manager/static/lqos.js
index dd40994a..b1633dd5 100644
--- a/src/rust/lqos_node_manager/static/lqos.js
+++ b/src/rust/lqos_node_manager/static/lqos.js
@@ -447,4 +447,20 @@ function zero_to_null(array) {
     for (let i=0; i<array.length; ++i) {
         if (array[i] == 0) array[i] = null;
     }
+}
+
+var dnsCache = {};
+
+function ipToHostname(ip) {
+    if (dnsCache.hasOwnProperty(ip)) {
+        if (dnsCache[ip] != ip) {
+            return ip + "<br /><span style='font-size: 6pt'>" + dnsCache[ip] + "</span>";
+        } else {
+            return ip;
+        }
+    }
+    $.get("/api/dns/" + encodeURI(ip), (hostname) => {
+        dnsCache[ip] = hostname;
+    })
+    return ip;
 }
\ No newline at end of file

From 28e1b020afd3f400c2c900bb4024fee8755af02e Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 17:12:18 +0000
Subject: [PATCH 03/67] Add file missing from previous commit

---
 .../src/tracker/cache/dns_cache.rs            | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 src/rust/lqos_node_manager/src/tracker/cache/dns_cache.rs

diff --git a/src/rust/lqos_node_manager/src/tracker/cache/dns_cache.rs b/src/rust/lqos_node_manager/src/tracker/cache/dns_cache.rs
new file mode 100644
index 00000000..7165a137
--- /dev/null
+++ b/src/rust/lqos_node_manager/src/tracker/cache/dns_cache.rs
@@ -0,0 +1,38 @@
+//! Implements a lock-free DNS least-recently-used DNS cache.
+
+use std::net::IpAddr;
+use dashmap::DashMap;
+use dns_lookup::lookup_addr;
+use lqos_utils::unix_time::unix_now;
+use once_cell::sync::Lazy;
+
+const CACHE_SIZE: usize = 1000;
+
+struct DnsEntry {
+    hostname: String,
+    last_accessed: u64,
+}
+
+static DNS_CACHE: Lazy<DashMap<IpAddr, DnsEntry>> = Lazy::new(|| DashMap::with_capacity(CACHE_SIZE));
+
+pub fn lookup_dns(ip: IpAddr) -> String {
+    // If the cached value exists, just return it
+    if let Some(mut dns) = DNS_CACHE.get_mut(&ip) {
+        if let Ok(now) = unix_now() {
+            dns.last_accessed = now;
+        }
+        return dns.hostname.clone();
+    }
+
+    // If it doesn't, we'll be adding it.
+    if DNS_CACHE.len() >= CACHE_SIZE {
+        let mut entries : Vec<(IpAddr, u64)> = DNS_CACHE.iter().map(|v| (*v.key(), v.last_accessed)).collect();
+        entries.sort_by(|a,b| b.1.cmp(&a.1));
+        DNS_CACHE.remove(&entries[0].0);
+    }
+    let hostname = lookup_addr(&ip).unwrap_or(ip.to_string());
+    DNS_CACHE.insert(ip, DnsEntry { hostname, last_accessed: unix_now().unwrap_or(0) });
+
+
+    String::new()
+}
\ No newline at end of file

From 7b571213d95962ff33dc9a0e75abed2f27d19be0 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 17:31:59 +0000
Subject: [PATCH 04/67] Fix a missing break - switch fallthrough by mistake -
 in dissector.h. This fixes a problem with misidentifying traffic as TCP when
 it is't.

---
 src/rust/lqos_sys/src/bpf/common/dissector.h | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/rust/lqos_sys/src/bpf/common/dissector.h b/src/rust/lqos_sys/src/bpf/common/dissector.h
index 25653d3f..cd43eb0d 100644
--- a/src/rust/lqos_sys/src/bpf/common/dissector.h
+++ b/src/rust/lqos_sys/src/bpf/common/dissector.h
@@ -345,8 +345,7 @@ static __always_inline void snoop(struct dissector_t *dissector)
 
             parse_tcp_ts(hdr, dissector->end, &dissector->tsval, &dissector->tsecr);
         }
-    }
-    break;
+    } break;
     case IPPROTO_UDP:
     {
         struct udphdr *hdr = get_udp_header(dissector);
@@ -354,12 +353,13 @@ static __always_inline void snoop(struct dissector_t *dissector)
         {
             if (hdr + 1 > dissector->end)
             {
+                bpf_debug("UDP header past end");
                 return;
             }
             dissector->src_port = hdr->source;
             dissector->dst_port = hdr->dest;
         }
-    }
+    } break;
     case IPPROTO_ICMP:
     {
         struct icmphdr *hdr = get_icmp_header(dissector);
@@ -367,14 +367,14 @@ static __always_inline void snoop(struct dissector_t *dissector)
         {
             if ((char *)hdr + sizeof(struct icmphdr) > dissector->end)
             {
+                bpf_debug("ICMP header past end");
                 return;
             }
             dissector->ip_protocol = 1;
             dissector->src_port = bpf_ntohs(hdr->type);
-            dissector->dst_port = bpf_ntohs(hdr->type);
-        }
-    }
-    break;
+            dissector->dst_port = bpf_ntohs(hdr->code);
+        }    
+    } break;
     }
 }
 

From 182dbcf8d9e408f409e45ed6521325d493f39797 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Mon, 27 Mar 2023 16:21:46 +0000
Subject: [PATCH 05/67] IP Dump supports flow filtering

Every packet is assigned a "flow id", and the UI allows you to
filter displayed packet data by flow id.
---
 .../lqos_node_manager/static/ip_dump.html     | 115 +++++++++++++-----
 1 file changed, 82 insertions(+), 33 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/ip_dump.html b/src/rust/lqos_node_manager/static/ip_dump.html
index df56005d..f98759fb 100644
--- a/src/rust/lqos_node_manager/static/ip_dump.html
+++ b/src/rust/lqos_node_manager/static/ip_dump.html
@@ -73,11 +73,26 @@
 
     <script>
         var packets = [];
+        var flows = {};
         var pages = 0;
         var PAGE_SIZE = 1000;
         var target = "";
         var current_page = 0;
         var capacity = [];
+        var activeFilter = null;
+        var activeSet = null;
+
+        function filter(newFilter) {
+            activeFilter = newFilter;
+            if (newFilter == null) {
+                activeSet = packets;
+            } else {
+                activeSet = packets.filter(packet => packet.flow_id == activeFilter);
+            }
+            pages = Math.ceil((activeSet.length / PAGE_SIZE));            
+            paginator(0);
+            viewPage(0);
+        }
 
         function proto(n) {
             switch (n) {
@@ -133,19 +148,39 @@ if (hdr->cwr) flags |= 128;
             paginator += "<a href='#' class='btn btn-info' onClick='zoomOut();'>Zoom Out</a> (ℹ️ Or drag an area of the graph) <br />";
 
             paginator += "<strong>Jump to page</strong>: ";
-                for (let i=0; i<pages; i++) {
-                    if (i == active) {
-                        paginator += " " + i + " ";
-                    } else {
-                        paginator += "<a href='#' onclick='viewPage(" + i + ");'>" + i + "</a> ";
-                    }
+            paginator += "<select>"
+            for (let i=0; i<pages; i++) {
+                if (i == active) {
+                    paginator += "<option selected>" + i + "</option>";
+                } else {
+                    paginator += "<option onclick='viewPage(" + i + ");'>" + i + "</option> ";
                 }
-                $("#pages").html(paginator);
+            }
+            paginator += "</select><br />";
+
+            // Offer flow filtering
+            paginator += "<strong>Filter Flows</strong>: ";
+            paginator += "<select>";
+            if (activeFilter == null) {
+                paginator += "<option selected onclick='filter(null);'>View All</option>";
+            } else {
+                paginator += "<option onclick='filter(null);'>View All</option>";
+            }
+            Object.keys(flows).forEach(key => {
+                if (activeFilter == key) {
+                    paginator += "<option selected onclick='filter(\"" + key + "\");'>" + key + "</option>";
+                } else {
+                    paginator += "<option onclick='filter(\"" + key + "\");'>" + key + "</option>";
+                }
+            });
+            paginator += "</select>";
+
+            $("#pages").html(paginator);
         }
 
         function viewPage(n) {
             let start = n * PAGE_SIZE;
-            let end = Math.min(start + PAGE_SIZE, packets.length);
+            let end = Math.min(start + PAGE_SIZE, activeSet.length);
             if (start > packets.length) {
                 console.log("OOps");
             }
@@ -156,33 +191,33 @@ if (hdr->cwr) flags |= 128;
             let y2_axis = [];
             for (let i=start; i<end; ++i) {                
                 html += "<tr>";
-                html += "<td>" + packets[i].timestamp + "</td>";
-                html += "<td>" + proto(packets[i].ip_protocol) + "</td>";
+                html += "<td>" + activeSet[i].timestamp + "</td>";
+                html += "<td>" + proto(activeSet[i].ip_protocol) + "</td>";
 
-                if (packets[i].ip_protocol == 6) {
-                    html += "<td>" + tcp_flags(packets[i].tcp_flags) + "</td>";
-                    html += "<td>" + packets[i].tcp_tsval + "/" + packets[i].tcp_tsecr + "</td>";
-                    html += "<td>" + packets[i].tcp_window + "</td>";
+                if (activeSet[i].ip_protocol == 6) {
+                    html += "<td>" + tcp_flags(activeSet[i].tcp_flags) + "</td>";
+                    html += "<td>" + activeSet[i].tcp_tsval + "/" + activeSet[i].tcp_tsecr + "</td>";
+                    html += "<td>" + activeSet[i].tcp_window + "</td>";
                 } else {
                     html += "<td></td><td></td><td></td>";
                 }
 
-                if (packets[i].ip_protocol != 1) {
-                    html += "<td>" + packets[i].src + ":" + packets[i].src_port + " -> " + packets[i].dst + ":" + packets[i].dst_port + "</td>";
+                if (activeSet[i].ip_protocol != 1) {
+                    html += "<td>" + activeSet[i].src + ":" + activeSet[i].src_port + " -> " + activeSet[i].dst + ":" + activeSet[i].dst_port + "</td>";
                 } else {
-                    html += "<td>" + packets[i].src + " -> " + packets[i].dst + "</td>";
+                    html += "<td>" + activeSet[i].src + " -> " + activeSet[i].dst + "</td>";
                 }
-                html += "<td>" + packets[i].size + "</td>";
-                html += "<td>" + ecn(packets[i].ecn) + "</td>";
-                html += "<td>0x" + packets[i].dscp.toString(16) + "</td>";
+                html += "<td>" + activeSet[i].size + "</td>";
+                html += "<td>" + ecn(activeSet[i].ecn) + "</td>";
+                html += "<td>0x" + activeSet[i].dscp.toString(16) + "</td>";
                 html += "</tr>";
-                x_axis.push(packets[i].timestamp);
-                if (packets[i].src == target) {
-                    y1_axis.push(packets[i].size);
+                x_axis.push(activeSet[i].timestamp);
+                if (activeSet[i].src == target) {
+                    y1_axis.push(activeSet[i].size);
                     y2_axis.push(0);
                 } else {
                     y1_axis.push(0);
-                    y2_axis.push(0.0 - packets[i].size);
+                    y2_axis.push(0.0 - activeSet[i].size);
                 }
             }
             html += "</table>";
@@ -217,17 +252,31 @@ if (hdr->cwr) flags |= 128;
             target = params.id;
             $.get("/api/packet_dump/" + params.id, (data) => {
                 data.sort((a,b) => a.timestamp - b.timestamp);
-                let min_ts = null;
-                for (let i=0; i<data.length; ++i) {
-                    if (min_ts == null || min_ts > data[i].timestamp) {
-                        min_ts = data[i].timestamp;
+
+                // Find the minimum timestamp
+                let min_ts = data.reduce((prev, curr) => prev.timestamp < curr.timestamp ? prev : curr).timestamp;
+
+                // Set the displayed timestamp to be (ts - min)
+                data.forEach(packet => packet.timestamp -= min_ts);
+
+                // Divide the packets into flows and append the flow_id
+                data.forEach(packet => {
+                    let flow_id = proto(packet.ip_protocol) + " " + packet.src + ":" + packet.src_port + " <-> " + packet.dst + ":" + packet.dst_port;
+                    let reverse_flow_id = proto(packet.ip_protocol) + " " + packet.dst + ":" + packet.dst_port + " <-> " + packet.src + ":" + packet.src_port;
+                    if (flows.hasOwnProperty(flow_id)) {
+                        packet.flow_id = flow_id;
+                    } else if (flows.hasOwnProperty(reverse_flow_id)) {
+                        packet.flow_id = reverse_flow_id;
+                    } else {
+                        flows[flow_id] = {};
+                        packet.flow_id = flow_id;
                     }
-                }
-                for (let i=0; i<data.length; ++i) {
-                    data[i].timestamp -= min_ts;
-                }
+                });
+                console.log(flows);
+
                 packets = data;
-                pages = Math.ceil((packets.length / PAGE_SIZE));
+                activeSet = packets;
+                pages = Math.ceil((activeSet.length / PAGE_SIZE));
                 starting_timestamp = min_ts;
                 paginator(0);                
                 viewPage(0);

From 869f97ff3b442671a52599f7f0245ffd15e6a957 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Mon, 27 Mar 2023 17:42:50 +0000
Subject: [PATCH 06/67] Packet analysis page has a lot more options

See ISSUE #302

* Filter by FlowID
* Pagination is a select/drop-down
* Filtering is a select/drop-down
* Graph type is selectable from:
   * Packet size (the previous view)
   * Piano Roll (flows plotted by time)
   * TCP Window (differntiated by flow)
---
 .../lqos_node_manager/static/ip_dump.html     | 103 ++++++++++++++++--
 1 file changed, 94 insertions(+), 9 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/ip_dump.html b/src/rust/lqos_node_manager/static/ip_dump.html
index f98759fb..7e8d320c 100644
--- a/src/rust/lqos_node_manager/static/ip_dump.html
+++ b/src/rust/lqos_node_manager/static/ip_dump.html
@@ -81,6 +81,8 @@
         var capacity = [];
         var activeFilter = null;
         var activeSet = null;
+        var activeChart = 0;
+        var activePage = 0;
 
         function filter(newFilter) {
             activeFilter = newFilter;
@@ -94,6 +96,12 @@
             viewPage(0);
         }
 
+        function setChart(n) {
+            activeChart = n;
+            paginator(activePage);
+            viewPage(activePage);
+        }
+
         function proto(n) {
             switch (n) {
                 case 6: return "TCP"
@@ -143,10 +151,12 @@ if (hdr->cwr) flags |= 128;
         }
 
         function paginator(active) {
+            activePage = active;
             let paginator = "<a href='/api/pcap/" + target + "/capture-" + circuit_id + "-" + starting_timestamp + ".pcap' class='btn btn-warning'>Download PCAP Dump</a> ";
             paginator += "<a href='#' class='btn btn-info' onClick='zoomIn();'>Zoom In</a> ";
             paginator += "<a href='#' class='btn btn-info' onClick='zoomOut();'>Zoom Out</a> (ℹ️ Or drag an area of the graph) <br />";
 
+            paginator += "<div style='margin: 4px; padding: 6px; background-color: #ddd; border: solid 1px black;'>";
             paginator += "<strong>Jump to page</strong>: ";
             paginator += "<select>"
             for (let i=0; i<pages; i++) {
@@ -156,7 +166,7 @@ if (hdr->cwr) flags |= 128;
                     paginator += "<option onclick='viewPage(" + i + ");'>" + i + "</option> ";
                 }
             }
-            paginator += "</select><br />";
+            paginator += "</select> | ";
 
             // Offer flow filtering
             paginator += "<strong>Filter Flows</strong>: ";
@@ -173,7 +183,28 @@ if (hdr->cwr) flags |= 128;
                     paginator += "<option onclick='filter(\"" + key + "\");'>" + key + "</option>";
                 }
             });
+            paginator += "</select> | ";
+
+            // Offer graph choices
+            paginator += "<strong>Graph</strong>: ";
+            paginator += "<select>";
+            if (activeChart == 0) {
+                paginator += "<option selected>Packet-Size Chart</option>";
+            } else {
+                paginator += "<option onclick='setChart(0);'>Packet-Size Chart</option>";
+            }
+            if (activeChart == 1) {
+                paginator += "<option selected>Piano Roll Flow Chart</option>";
+            } else {
+                paginator += "<option onclick='setChart(1);'>Piano Roll Flow Chart</option>";
+            }
+            if (activeChart == 2) {
+                paginator += "<option selected>TCP Window Chart</option>";
+            } else {
+                paginator += "<option onclick='setChart(2);'>TCP Window Chart</option>";
+            }
             paginator += "</select>";
+            paginator += "</div>";
 
             $("#pages").html(paginator);
         }
@@ -224,13 +255,66 @@ if (hdr->cwr) flags |= 128;
             $("#dump").html(html);
             paginator(n);
 
-            // Make the graph
+            // Make the graph            
             let graph = document.getElementById("graph");
-            let data = [
-                {x: x_axis, y:y1_axis, name: 'Download', type: 'scatter', mode: 'markers', error_x: { type: 'percent', value: capacity[0], symetric: false, valueminus: 0 }},
-                {x: x_axis, y:y2_axis, name: 'Upload', type: 'scatter', mode: 'markers', error_x: { type: 'percent', value: capacity[1], symetric: false, valueminus: 0 }},                
-            ];
-            Plotly.newPlot(graph, data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: 'Bytes' }, xaxis: {automargin: true, title: "Nanoseconds"} }, { responsive: true });
+            if (activeChart == 0) {
+                // Render the timeline chart
+                let data = [
+                    {x: x_axis, y:y1_axis, name: 'Download', type: 'scatter', mode: 'markers', error_x: { type: 'percent', value: capacity[0], symetric: false, valueminus: 0 }},
+                    {x: x_axis, y:y2_axis, name: 'Upload', type: 'scatter', mode: 'markers', error_x: { type: 'percent', value: capacity[1], symetric: false, valueminus: 0 }},                
+                ];
+                Plotly.newPlot(graph, data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: 'Bytes' }, xaxis: {automargin: true, title: "Nanoseconds"} }, { responsive: true });
+            } else if (activeChart == 1) {
+                // Render the piano roll chart
+                let flowGraphY = {};
+                for (var i=start; i<end; ++i) {
+                    let flow_id = activeSet[i].flow_id;
+                    if (flowGraphY.hasOwnProperty(flow_id)) {
+                        flowGraphY[flow_id].x.push(activeSet[i].timestamp);
+                        flowGraphY[flow_id].y.push(flows[flow_id].flowCounter);
+                    } else {
+                        flowGraphY[flow_id] = {
+                            "x": [ activeSet[i].timestamp ],
+                            "y": [ flows[flow_id].flowCounter ],
+                        }
+                    }
+                }
+
+                let data = [];
+                for (flow in flowGraphY) {
+                    //console.log(flowGraphY[flow]);
+                    data.push({
+                        x: flowGraphY[flow].x, y: flowGraphY[flow].y, name: flow, type: 'scatter', mode: 'markers',
+                    });
+                }
+                Plotly.newPlot(graph, data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: 'Flow' }, xaxis: {automargin: true, title: "Nanoseconds"} }, { responsive: true });
+            } else if (activeChart == 2) {
+                // Render the window chart
+                let flowGraphY = {};
+                for (var i=start; i<end; ++i) {
+                    let flow_id = activeSet[i].flow_id;
+                    if (flow_id.includes("TCP")) {
+                        if (flowGraphY.hasOwnProperty(flow_id)) {
+                            flowGraphY[flow_id].x.push(activeSet[i].timestamp);
+                            flowGraphY[flow_id].y.push(activeSet[i].tcp_window);
+                        } else {
+                            flowGraphY[flow_id] = {
+                                "x": [ activeSet[i].timestamp ],
+                                "y": [ activeSet[i].tcp_window ],
+                            }
+                        }
+                    }
+                }
+
+                let data = [];
+                for (flow in flowGraphY) {
+                    //console.log(flowGraphY[flow]);
+                    data.push({
+                        x: flowGraphY[flow].x, y: flowGraphY[flow].y, name: flow, type: 'scatter', mode: 'markers',
+                    });
+                }
+                Plotly.newPlot(graph, data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: 'Window Size' }, xaxis: {automargin: true, title: "Nanoseconds"} }, { responsive: true });
+            }
         }
 
         let circuit_id = null;
@@ -260,6 +344,7 @@ if (hdr->cwr) flags |= 128;
                 data.forEach(packet => packet.timestamp -= min_ts);
 
                 // Divide the packets into flows and append the flow_id
+                let flowCounter = 0;
                 data.forEach(packet => {
                     let flow_id = proto(packet.ip_protocol) + " " + packet.src + ":" + packet.src_port + " <-> " + packet.dst + ":" + packet.dst_port;
                     let reverse_flow_id = proto(packet.ip_protocol) + " " + packet.dst + ":" + packet.dst_port + " <-> " + packet.src + ":" + packet.src_port;
@@ -268,11 +353,11 @@ if (hdr->cwr) flags |= 128;
                     } else if (flows.hasOwnProperty(reverse_flow_id)) {
                         packet.flow_id = reverse_flow_id;
                     } else {
-                        flows[flow_id] = {};
+                        flows[flow_id] = { flowCounter };
                         packet.flow_id = flow_id;
+                        flowCounter++;
                     }
                 });
-                console.log(flows);
 
                 packets = data;
                 activeSet = packets;

From 536d165f9bb455aa559ed108a3b9e0ad392cd242 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 28 Mar 2023 13:11:53 +0000
Subject: [PATCH 07/67] Fix loss of context when clicking zoom in/out buttons.

---
 src/rust/lqos_node_manager/static/ip_dump.html | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/ip_dump.html b/src/rust/lqos_node_manager/static/ip_dump.html
index 7e8d320c..14fed0fe 100644
--- a/src/rust/lqos_node_manager/static/ip_dump.html
+++ b/src/rust/lqos_node_manager/static/ip_dump.html
@@ -77,7 +77,6 @@
         var pages = 0;
         var PAGE_SIZE = 1000;
         var target = "";
-        var current_page = 0;
         var capacity = [];
         var activeFilter = null;
         var activeSet = null;
@@ -138,16 +137,16 @@ if (hdr->cwr) flags |= 128;
 
         function zoomIn() {
             PAGE_SIZE /= 2;
-            current_page /= 2;
+            activePage /= 2;
             pages = packets.length / PAGE_SIZE;
-            viewPage(current_page);
+            viewPage(activePage);
         }
 
         function zoomOut() {
             PAGE_SIZE *= 2;
-            current_page *= 2;
+            activePage *= 2;
             pages = packets.length / PAGE_SIZE;
-            viewPage(current_page);
+            viewPage(activePage);
         }
 
         function paginator(active) {

From 7321c00234d4dbc444b7d6af250bfdb6acad0aee Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 28 Mar 2023 13:15:53 +0000
Subject: [PATCH 08/67] Change units on backlog graph to packets. FIXES #304

---
 src/rust/lqos_node_manager/static/circuit_queue.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 1bf47f69..9086dabe 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -374,7 +374,7 @@
                 
                 if (this.backlogPlotted == null) {
                     this.backlogPlotted = true;
-                    Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Bytes" }, xaxis: { automargin: true, title: "Time since now" } });
+                    Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Packets" }, xaxis: { automargin: true, title: "Time since now" } });
                 } else {
                     Plotly.redraw(graph, graphData);
                 }

From 53acf906003f45cfa857ba7c0e63ff62dd0a6d5a Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 28 Mar 2023 14:13:36 +0000
Subject: [PATCH 09/67] Add pause button and fast/slow modes to circuit_queue
 page

FIXES #303

Adds a pause button to freeze a snapshot of the current circuit
queue page.

Adds a "fast mode" and "slow mode" toggle. Slow mode continues
at a 1s update cadence. Fast mode uses animation frame support
to update at a good speed, at the expense of CPU usage.

Stores the preference in local storage, so it will resume the
same mode next time you visit the page.
---
 .../static/circuit_queue.html                 | 103 +++++++++++++++---
 1 file changed, 85 insertions(+), 18 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 9086dabe..28d0bd4e 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -94,7 +94,8 @@
                         </ul>
                     </div>
                     <div class="col-sm-2">
-                        <div id="raw"></div>
+                        <a href="#" class="btn btn-small btn-info" id="btnPause"><i class="fa fa-pause"></i> Pause</a> 
+                        <a href="#" class="btn btn-small btn-info" id="btnSlow"><i class="fa fa-hourglass"></i> Slow Mode</a>
                     </div>
                 </div>
             </div>
@@ -520,10 +521,10 @@
                 up = 0 - up;
                 let down_slot = Math.floor((down / circuit_info[CircuitInfo.capacity][0]) * 10.0);
                 let up_slot = Math.floor((up / circuit_info[CircuitInfo.capacity][1]) * 10.0);
-                /*if (down_slot < 0) down_slot = 0;
+                if (down_slot < 0) down_slot = 0;
                 if (up_slot < 0) up_slot = 0;
-                if (down_slot > 11) down_slot = 11;
-                if (up_slot > 11) up_slot = 11;*/
+                if (down_slot > 10) down_slot = 10;
+                if (up_slot > 10) up_slot = 10;
                 this.quantiles[0][down_slot] += 1;
                 this.quantiles[1][up_slot] += 1;
                 //console.log(down_slot, up_slot);            
@@ -796,22 +797,50 @@
 
         let id = 0;
         let activeTab = "pills-home-tab";
+        var lastCalledTime;
+        var fps;
+        var worstDelta = 0;
+        var paused = false;
+        var slowMode = false;
 
-        function oneSecondCadence() {
-            //console.log(activeTab);
-            switch (activeTab) {
-                case "pills-funnel-tab": {
-                    getFunnel();
-                } break;
-                case "pills-flows-tab": {
-                    getFlows();
-                } break;
-                default: {
-                    pollQueue();
-                    getThroughput();
+        function showFps() {
+            if(!lastCalledTime) {
+                lastCalledTime = Date.now();
+                fps = 0;
+                return;
+            }
+            delta = (Date.now() - lastCalledTime)/1000;
+            lastCalledTime = Date.now();
+            fps = 1/delta;
+            //$("#fps").text(fps.toFixed(0));
+            worstDelta = Math.max(delta, worstDelta);
+        }
+
+        function updateFrame() {
+            showFps();
+            if (!paused) {
+                switch (activeTab) {
+                    case "pills-funnel-tab": {
+                        getFunnel();
+                    } break;
+                    case "pills-flows-tab": {
+                        getFlows();
+                    } break;
+                    default: {
+                        pollQueue();
+                        getThroughput();
+                    }
                 }
             }
-            setTimeout(oneSecondCadence, 1000);
+            // Doing this to balance out the FPS
+            // It will tend towards the slowest
+            if (slowMode) {
+                setTimeout(updateFrame, 1000);
+            } else {
+                setTimeout(() => {
+                    requestAnimationFrame(updateFrame);
+                }, worstDelta * 200);
+            }
         }
 
         function wireUpTabEvents() {
@@ -822,8 +851,45 @@
             });
         }
 
+        function isSlowMode() {
+            let slow = localStorage.getItem("slowMode");
+            if (slow == null) {
+                localStorage.setItem("slowMode", false);
+                slow = false;
+            }
+            if (slow == "false") {
+                slow = false;
+            } else if (slow == "true") {
+                slow = true;
+            }
+            return slow;
+        }
+
         function start() {
             wireUpTabEvents();
+            $("#btnPause").on('click', () => {
+                paused = !paused;
+                if (paused) {
+                    $("#btnPause").html("<i class='fa fa-play'></i> Resume");
+                } else {
+                    $("#btnPause").html("<i class='fa fa-pause'></i> Pause");
+                }
+            });
+            slowMode = isSlowMode();
+            if (slowMode) {
+                $("#btnSlow").html("<i class='fa fa-fast-forward'></i> Fast Mode");
+            } else {
+                $("#btnSlow").html("<i class='fa fa-hourglass'></i> Slow Mode");
+            }
+            $("#btnSlow").on('click', () => {
+                slowMode = !slowMode;
+                localStorage.setItem("slowMode", slowMode);
+                if (slowMode) {
+                    $("#btnSlow").html("<i class='fa fa-fast-forward'></i> Fast Mode");
+                } else {
+                    $("#btnSlow").html("<i class='fa fa-hourglass'></i> Slow Mode");
+                }
+            });
             colorReloadButton();
             updateHostCounts();
             const params = new Proxy(new URLSearchParams(window.location.search), {
@@ -831,7 +897,8 @@
             });
             id = params.id;
             $.get("/api/watch_circuit/" + params.id, () => {
-                oneSecondCadence();
+                //updateFrame();
+                requestAnimationFrame(updateFrame);
             });
         }
 

From 3f4ee288aeaedef381091ba121d431e2cb896b08 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 28 Mar 2023 16:07:49 +0000
Subject: [PATCH 10/67] Use base delay (which is sparse delay) rather than
 average delay, and use log y values on the delays chart.

---
 src/rust/lqos_bus/src/bus/queue_data.rs       |  2 +-
 .../static/circuit_queue.html                 | 32 ++++++++++++++++---
 src/rust/lqos_queue_tracker/src/queue_diff.rs |  4 +--
 .../lqos_queue_tracker/src/queue_store.rs     |  2 +-
 4 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/src/rust/lqos_bus/src/bus/queue_data.rs b/src/rust/lqos_bus/src/bus/queue_data.rs
index 42b74680..3ea63146 100644
--- a/src/rust/lqos_bus/src/bus/queue_data.rs
+++ b/src/rust/lqos_bus/src/bus/queue_data.rs
@@ -30,7 +30,7 @@ pub struct CakeDiffTinTransit {
   pub backlog_bytes: u32,
   pub drops: u32,
   pub marks: u32,
-  pub avg_delay_us: u32,
+  pub base_delay_us: u32,
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 28d0bd4e..75cf9c12 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -305,8 +305,21 @@
             }
 
             ingestDelays(subData, currentX, tin) {
-                this.delays.store(tin, currentX, 0, subData[0][CDT.tins][tin][CDTT.avg_delay_us] * 0.001);
-                this.delays.store(tin, currentX, 1, 0.0 - subData[1][CDT.tins][tin][CDTT.avg_delay_us] * 0.001);
+                let down = subData[0][CDT.tins][tin][CDTT.avg_delay_us] * 0.001;
+                let up = subData[1][CDT.tins][tin][CDTT.avg_delay_us] * 0.001;
+                if (down == 0.0) {
+                    down = null;
+                } else {
+                    down = Math.log10(down);
+                }
+                if (up == 0.0) {
+                    up = null;
+                } else {
+                    //console.log(up);
+                    up = 0.0 - Math.log10(up);
+                }
+                this.delays.store(tin, currentX, 0, down);
+                this.delays.store(tin, currentX, 1, up);
             }
 
             ingestQueueLen(subData, currentX) {
@@ -375,7 +388,13 @@
                 
                 if (this.backlogPlotted == null) {
                     this.backlogPlotted = true;
-                    Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Packets" }, xaxis: { automargin: true, title: "Time since now" } });
+                    Plotly.newPlot(
+                        graph, 
+                        graphData, 
+                        { 
+                            margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, 
+                            yaxis: { automargin: true, title: "Packets" }, 
+                            xaxis: { automargin: true, title: "Time since now" } });
                 } else {
                     Plotly.redraw(graph, graphData);
                 }
@@ -391,7 +410,12 @@
                 ];
 
                 if (this.delaysPlotted == null) {
-                    Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "ms" }, xaxis: { automargin: true, title: "Time since now" } });
+                    Plotly.newPlot(
+                        graph, 
+                        graphData, 
+                        { margin: { l: 8, r: 0, b: 0, t: 0, pad: 4 }, 
+                        yaxis: { automargin: true, title: "log10(ms)", range: [-1.0, 1.0] }, 
+                        xaxis: { automargin: true, title: "Time since now" } });
                     this.delaysPlotted = true;
                 } else {
                     Plotly.redraw(graph, graphData);
diff --git a/src/rust/lqos_queue_tracker/src/queue_diff.rs b/src/rust/lqos_queue_tracker/src/queue_diff.rs
index 22c4d486..7becc43b 100644
--- a/src/rust/lqos_queue_tracker/src/queue_diff.rs
+++ b/src/rust/lqos_queue_tracker/src/queue_diff.rs
@@ -46,7 +46,7 @@ pub struct CakeDiffTin {
   pub backlog_bytes: u32,
   pub drops: u32,
   pub marks: u32,
-  pub avg_delay_us: u32,
+  pub base_delay_us: u32,
 }
 
 fn cake_diff(
@@ -65,7 +65,7 @@ fn cake_diff(
           backlog_bytes: new.backlog_bytes,
           drops: new.drops.saturating_sub(prev.drops),
           marks: new.ecn_marks.saturating_sub(prev.ecn_marks),
-          avg_delay_us: new.avg_delay_us,
+          base_delay_us: new.base_delay_us,
         })
         .collect();
       return Ok(QueueDiff::Cake(CakeDiff {
diff --git a/src/rust/lqos_queue_tracker/src/queue_store.rs b/src/rust/lqos_queue_tracker/src/queue_store.rs
index 55171057..2047825c 100644
--- a/src/rust/lqos_queue_tracker/src/queue_store.rs
+++ b/src/rust/lqos_queue_tracker/src/queue_store.rs
@@ -101,7 +101,7 @@ impl Into<CakeDiffTinTransit> for CakeDiffTin {
       backlog_bytes: self.backlog_bytes,
       drops: self.drops,
       marks: self.marks,
-      avg_delay_us: self.avg_delay_us,
+      base_delay_us: self.base_delay_us,
     }
   }
 }

From 1b73c3765c2c5c0023713a14c3f7618788abe0d1 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 28 Mar 2023 16:21:14 +0000
Subject: [PATCH 11/67] Backlog is in bytes

---
 src/rust/lqos_node_manager/static/circuit_queue.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 75cf9c12..43815d67 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -393,7 +393,7 @@
                         graphData, 
                         { 
                             margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, 
-                            yaxis: { automargin: true, title: "Packets" }, 
+                            yaxis: { automargin: true, title: "Bytes" }, 
                             xaxis: { automargin: true, title: "Time since now" } });
                 } else {
                     Plotly.redraw(graph, graphData);

From ebc980c782b4bb689a30c1e06b1da44664a899c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Wed, 29 Mar 2023 09:58:12 -0600
Subject: [PATCH 12/67] When network.json is created by integration, node
 bandwidth will be capped by parent nodes

---
 src/integrationCommon.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/integrationCommon.py b/src/integrationCommon.py
index f9c4920b..8dffaac3 100644
--- a/src/integrationCommon.py
+++ b/src/integrationCommon.py
@@ -2,7 +2,7 @@
 # integrations.
 
 from typing import List, Any
-from ispConfig import allowedSubnets, ignoreSubnets, generatedPNUploadMbps, generatedPNDownloadMbps, circuitNameUseAddress
+from ispConfig import allowedSubnets, ignoreSubnets, generatedPNUploadMbps, generatedPNDownloadMbps, circuitNameUseAddress, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps
 import ipaddress
 import enum
 
@@ -290,7 +290,22 @@ class NetworkGraph:
 					child)
 
 		del self.__visited
-
+		
+		# Child nodes inherit bandwidth maximums of parents. We apply this here to avoid bugs when compression is applied with flattenA().
+		def inheritBandwidthMaxes(data, parentMaxDL, parentMaxUL):
+			for node in data:
+				if isinstance(node, str):
+					if (isinstance(data[node], dict)) and (node != 'children'):
+						# Cap based on this node's max bandwidth, or parent node's max bandwidth, whichever is lower
+						data[node]['downloadBandwidthMbps'] = min(int(data[node]['downloadBandwidthMbps']),int(parentMaxDL))
+						data[node]['uploadBandwidthMbps'] = min(int(data[node]['uploadBandwidthMbps']),int(parentMaxUL))
+						# Recursive call this function for children nodes attached to this node
+						if 'children' in data[node]:
+							# We need to keep tabs on the minor counter, because we can't have repeating class IDs. Here, we bring back the minor counter from the recursive function
+							inheritBandwidthMaxes(data[node]['children'], data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps'])
+		# Here is the actual call to the recursive function
+		inheritBandwidthMaxes(topLevelNode, parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps)
+		
 		with open('network.json', 'w') as f:
 			json.dump(topLevelNode, f, indent=4)
 

From ec9a7e71b8e42465ee94ee35bdde7549c0e45b5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Wed, 29 Mar 2023 09:59:01 -0600
Subject: [PATCH 13/67] Clean up excessive comments

---
 src/integrationCommon.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/integrationCommon.py b/src/integrationCommon.py
index 8dffaac3..943ad889 100644
--- a/src/integrationCommon.py
+++ b/src/integrationCommon.py
@@ -291,19 +291,14 @@ class NetworkGraph:
 
 		del self.__visited
 		
-		# Child nodes inherit bandwidth maximums of parents. We apply this here to avoid bugs when compression is applied with flattenA().
 		def inheritBandwidthMaxes(data, parentMaxDL, parentMaxUL):
 			for node in data:
 				if isinstance(node, str):
 					if (isinstance(data[node], dict)) and (node != 'children'):
-						# Cap based on this node's max bandwidth, or parent node's max bandwidth, whichever is lower
 						data[node]['downloadBandwidthMbps'] = min(int(data[node]['downloadBandwidthMbps']),int(parentMaxDL))
 						data[node]['uploadBandwidthMbps'] = min(int(data[node]['uploadBandwidthMbps']),int(parentMaxUL))
-						# Recursive call this function for children nodes attached to this node
 						if 'children' in data[node]:
-							# We need to keep tabs on the minor counter, because we can't have repeating class IDs. Here, we bring back the minor counter from the recursive function
 							inheritBandwidthMaxes(data[node]['children'], data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps'])
-		# Here is the actual call to the recursive function
 		inheritBandwidthMaxes(topLevelNode, parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps)
 		
 		with open('network.json', 'w') as f:

From 52a961181e3b80289af0c8f1e0d53b8b705d36f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Wed, 29 Mar 2023 10:02:01 -0600
Subject: [PATCH 14/67] Use airfiber backhaul capacities to limit site
 bandwidths

---
 src/integrationUISP.py | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 2d437c19..3f239f88 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -99,7 +99,7 @@ def buildFullGraph():
 				download = int(download)
 				upload = int(upload)
 				siteBandwidth[name] = {"download": download, "upload": upload}
-
+	
 	# Find AP capacities from UISP
 	for device in devices:
 		if device['identification']['role'] == "ap":
@@ -110,7 +110,27 @@ def buildFullGraph():
 				upload = int(device['overview']['uplinkCapacity'] / 1000000)
 				siteBandwidth[device['identification']['name']] = {
 					"download": download, "upload": upload}
-
+	
+	# Find Site Capacities by AirFiber capacities
+	foundAirFibersBySite = {}
+	for device in devices:
+		if device['identification']['site']['type'] == 'site':
+			if device['identification']['role'] == "station":
+				if device['identification']['type'] == "airFiber":
+					if device['overview']['status'] == 'active':
+						download = int(device['overview']['downlinkCapacity']/ 1000000)
+						upload = int(device['overview']['uplinkCapacity']/ 1000000)
+						# Make sure to use half of reported bandwidth for AF60-LRs
+						if device['identification']['model'] == "AF60-LR":
+							download = int(download / 2)
+							upload = int(download / 2)
+						if device['identification']['site']['id'] in foundAirFibersBySite:
+							if (download > foundAirFibersBySite['download']) or (upload > foundAirFibersBySite['upload']):
+								foundAirFibersBySite[device['identification']['site']['id']]['download'] = download
+								foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
+						else:
+							foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload}
+	
 	print("Building Topology")
 	net = NetworkGraph()
 	# Add all sites and client sites
@@ -133,6 +153,9 @@ def buildFullGraph():
 					# Use the CSV bandwidth values
 					download = siteBandwidth[name]["download"]
 					upload = siteBandwidth[name]["upload"]
+				elif id in foundAirFibersBySite:
+					download = foundAirFibersBySite[id]['download']
+					upload = foundAirFibersBySite[id]['upload']
 				else:
 					# Add them just in case
 					siteBandwidth[name] = {
@@ -209,7 +232,8 @@ def buildFullGraph():
 
 	# Save integrationUISPbandwidths.csv
 	# (the newLine fixes generating extra blank lines)
-	with open('integrationUISPbandwidths.csv', 'w', newline='') as csvfile:
+	# Saves as .template as to not overwrite
+	with open('integrationUISPbandwidths.template.csv', 'w', newline='') as csvfile:
 		wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
 		wr.writerow(['ParentNode', 'Download Mbps', 'Upload Mbps'])
 		for device in siteBandwidth:

From f38553c72b702a9e3cf3ba87c3c54f832b933df9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Thu, 30 Mar 2023 12:19:23 -0600
Subject: [PATCH 15/67] Fix error when multiple airFibers at site

---
 src/integrationUISP.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 3f239f88..fdddea6e 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -125,7 +125,7 @@ def buildFullGraph():
 							download = int(download / 2)
 							upload = int(download / 2)
 						if device['identification']['site']['id'] in foundAirFibersBySite:
-							if (download > foundAirFibersBySite['download']) or (upload > foundAirFibersBySite['upload']):
+							if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']):
 								foundAirFibersBySite[device['identification']['site']['id']]['download'] = download
 								foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
 						else:

From 5d36828b3521ea49002dd6eff01e4a26aab8dfe5 Mon Sep 17 00:00:00 2001
From: Matthew McTague <matthew.mctague@vetta.nz>
Date: Fri, 31 Mar 2023 13:07:15 +1300
Subject: [PATCH 16/67] replaceRootNote should be replaceRootNode

---
 old/v1.3/integrationCommon.py | 2 +-
 old/v1.3/integrationUISP.py   | 2 +-
 old/v1.3/testGraph.py         | 2 +-
 src/integrationCommon.py      | 2 +-
 src/integrationUISP.py        | 2 +-
 src/testGraph.py              | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/old/v1.3/integrationCommon.py b/old/v1.3/integrationCommon.py
index 710713ef..c70aa085 100644
--- a/old/v1.3/integrationCommon.py
+++ b/old/v1.3/integrationCommon.py
@@ -134,7 +134,7 @@ class NetworkGraph:
                 node.parentId = self.exceptionCPEs[node.displayName]
             self.nodes.append(node)
 
-    def replaceRootNote(self, node: NetworkNode) -> None:
+    def replaceRootNode(self, node: NetworkNode) -> None:
         # Replaces the automatically generated root node
         # with a new node. Useful when you have a top-level
         # node specified (e.g. "uispSite" in the UISP
diff --git a/old/v1.3/integrationUISP.py b/old/v1.3/integrationUISP.py
index b3f67fc4..48ba0b9d 100644
--- a/old/v1.3/integrationUISP.py
+++ b/old/v1.3/integrationUISP.py
@@ -145,7 +145,7 @@ def buildFullGraph():
         # If this is the uispSite node, it becomes the root. Otherwise, add it to the
         # node soup.
         if name == uispSite:
-            net.replaceRootNote(node)
+            net.replaceRootNode(node)
         else:
             net.addRawNode(node)
 
diff --git a/old/v1.3/testGraph.py b/old/v1.3/testGraph.py
index e127ac37..0ce07dc8 100644
--- a/old/v1.3/testGraph.py
+++ b/old/v1.3/testGraph.py
@@ -53,7 +53,7 @@ class TestGraph(unittest.TestCase):
         from integrationCommon import NetworkGraph, NetworkNode, NodeType
         graph = NetworkGraph()
         node = NetworkNode("Test", type = NodeType.site)
-        graph.replaceRootNote(node)
+        graph.replaceRootNode(node)
         self.assertEqual(graph.nodes[0].id, "Test")
 
     def add_child_by_named_parent(self):
diff --git a/src/integrationCommon.py b/src/integrationCommon.py
index 943ad889..33f20e31 100644
--- a/src/integrationCommon.py
+++ b/src/integrationCommon.py
@@ -135,7 +135,7 @@ class NetworkGraph:
 				node.parentId = self.exceptionCPEs[node.displayName]
 			self.nodes.append(node)
 
-	def replaceRootNote(self, node: NetworkNode) -> None:
+	def replaceRootNode(self, node: NetworkNode) -> None:
 		# Replaces the automatically generated root node
 		# with a new node. Useful when you have a top-level
 		# node specified (e.g. "uispSite" in the UISP
diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index fdddea6e..36937fb7 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -176,7 +176,7 @@ def buildFullGraph():
 		# If this is the uispSite node, it becomes the root. Otherwise, add it to the
 		# node soup.
 		if name == uispSite:
-			net.replaceRootNote(node)
+			net.replaceRootNode(node)
 		else:
 			net.addRawNode(node)
 
diff --git a/src/testGraph.py b/src/testGraph.py
index e127ac37..0ce07dc8 100644
--- a/src/testGraph.py
+++ b/src/testGraph.py
@@ -53,7 +53,7 @@ class TestGraph(unittest.TestCase):
         from integrationCommon import NetworkGraph, NetworkNode, NodeType
         graph = NetworkGraph()
         node = NetworkNode("Test", type = NodeType.site)
-        graph.replaceRootNote(node)
+        graph.replaceRootNode(node)
         self.assertEqual(graph.nodes[0].id, "Test")
 
     def add_child_by_named_parent(self):

From 691a1d9e6de376618d8413e3759c1d1ee5c2d594 Mon Sep 17 00:00:00 2001
From: Matthew McTague <matthew.mctague@vetta.nz>
Date: Fri, 31 Mar 2023 13:08:47 +1300
Subject: [PATCH 17/67] Check that priorQuery exists before using it

---
 src/graphInfluxDB.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/graphInfluxDB.py b/src/graphInfluxDB.py
index 13b38de7..b8b59f86 100644
--- a/src/graphInfluxDB.py
+++ b/src/graphInfluxDB.py
@@ -331,7 +331,7 @@ def getCircuitLatencyStats(subscriberCircuits):
 			# If we can't identify RTT this time around, use most recently recorded RTT
 			# None by default, change if found in priorQuery
 			circuit['stats']['sinceLastQuery']['tcpLatency'] = None
-			if circuit['stats']['priorQuery'] != None:
+			if 'priorQuery' in circuit['stats'] and circuit['stats']['priorQuery'] != None:
 				if 'priorQuery' in circuit['stats']:
 					if 'tcpLatency' in circuit['stats']['priorQuery']:
 						circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency']

From 33ece34df6998e8ac281fcefe934624bdcd30957 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Thu, 30 Mar 2023 18:22:38 -0600
Subject: [PATCH 18/67] Fix bug pertaining to
 https://github.com/LibreQoE/LibreQoS/pull/320

---
 src/graphInfluxDB.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/graphInfluxDB.py b/src/graphInfluxDB.py
index b8b59f86..6aa9c314 100644
--- a/src/graphInfluxDB.py
+++ b/src/graphInfluxDB.py
@@ -331,10 +331,11 @@ def getCircuitLatencyStats(subscriberCircuits):
 			# If we can't identify RTT this time around, use most recently recorded RTT
 			# None by default, change if found in priorQuery
 			circuit['stats']['sinceLastQuery']['tcpLatency'] = None
-			if 'priorQuery' in circuit['stats'] and circuit['stats']['priorQuery'] != None:
-				if 'priorQuery' in circuit['stats']:
-					if 'tcpLatency' in circuit['stats']['priorQuery']:
-						circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency']
+			if 'priorQuery' in circuit['stats']:
+				if circuit['stats']['priorQuery'] != None:
+					if 'priorQuery' in circuit['stats']:
+						if 'tcpLatency' in circuit['stats']['priorQuery']:
+							circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency']
 
 	return subscriberCircuits
 

From 3bc2c16328e5b1560c2230b0f538239e0570bd09 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Thu, 30 Mar 2023 14:46:30 +0000
Subject: [PATCH 19/67] Last 5 minutes of throughput data are stored
 server-side on node_manager

A server-side ringbuffer retains just the last 5 minutes of throughput.
When the dashboard first loads, it collects all 5 minutes of data,
so nothing appears skipped. Subsequent updates query the latest
value in the buffer.

FIXES #317
---
 src/rust/lqos_node_manager/src/main.rs        |  7 ++-
 .../src/tracker/cache/mod.rs                  |  2 +
 .../src/tracker/cache/throughput.rs           | 60 +++++++++++++++++++
 .../src/tracker/cache_manager.rs              | 17 +++++-
 src/rust/lqos_node_manager/src/tracker/mod.rs | 30 ++++------
 src/rust/lqos_node_manager/static/main.html   | 27 ++++++++-
 6 files changed, 119 insertions(+), 24 deletions(-)
 create mode 100644 src/rust/lqos_node_manager/src/tracker/cache/throughput.rs

diff --git a/src/rust/lqos_node_manager/src/main.rs b/src/rust/lqos_node_manager/src/main.rs
index 3787e635..c0e13f32 100644
--- a/src/rust/lqos_node_manager/src/main.rs
+++ b/src/rust/lqos_node_manager/src/main.rs
@@ -22,13 +22,17 @@ static GLOBAL: Jemalloc = Jemalloc;
 
 #[launch]
 fn rocket() -> _ {
-  //tracker::SHAPED_DEVICES.read().write_csv("ShapedDeviceWriteTest.csv").unwrap();
   let server = rocket::build()
     .attach(AdHoc::on_liftoff("Poll lqosd", |_| {
       Box::pin(async move {
         rocket::tokio::spawn(tracker::update_tracking());
       })
     }))
+    .attach(AdHoc::on_liftoff("Poll throughput", |_| {
+      Box::pin(async move {
+        rocket::tokio::spawn(tracker::update_total_throughput_buffer());
+      })
+    }))
     .register("/", catchers![static_pages::login])
     .mount(
       "/",
@@ -47,6 +51,7 @@ fn rocket() -> _ {
         static_pages::klingon,
         // API calls
         tracker::current_throughput,
+        tracker::throughput_ring_buffer,
         tracker::cpu_usage,
         tracker::ram_usage,
         tracker::top_10_downloaders,
diff --git a/src/rust/lqos_node_manager/src/tracker/cache/mod.rs b/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
index dcc23457..8829e9f5 100644
--- a/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
+++ b/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
@@ -4,6 +4,8 @@
 
 mod cpu_ram;
 mod shaped_devices;
+mod throughput;
 
 pub use cpu_ram::*;
 pub use shaped_devices::*;
+pub use throughput::THROUGHPUT_BUFFER;
diff --git a/src/rust/lqos_node_manager/src/tracker/cache/throughput.rs b/src/rust/lqos_node_manager/src/tracker/cache/throughput.rs
new file mode 100644
index 00000000..b15e7ef1
--- /dev/null
+++ b/src/rust/lqos_node_manager/src/tracker/cache/throughput.rs
@@ -0,0 +1,60 @@
+use crate::tracker::ThroughputPerSecond;
+use lqos_bus::{bus_request, BusRequest, BusResponse};
+use once_cell::sync::Lazy;
+use rocket::tokio::sync::RwLock;
+
+pub static THROUGHPUT_BUFFER: Lazy<RwLock<TotalThroughput>> =
+  Lazy::new(|| RwLock::new(TotalThroughput::new()));
+
+/// Maintains an in-memory ringbuffer of the last 5 minutes of
+/// throughput data.
+pub struct TotalThroughput {
+  data: Vec<ThroughputPerSecond>,
+  head: usize,
+  prev_head: usize,
+}
+
+impl TotalThroughput {
+  /// Create a new throughput ringbuffer system
+  pub fn new() -> Self {
+    Self {
+      data: vec![ThroughputPerSecond::default(); 300],
+      head: 0,
+      prev_head: 0,
+    }
+  }
+
+  /// Run once per second to update the ringbuffer with current data
+  pub async fn tick(&mut self) {
+    if let Ok(messages) =
+      bus_request(vec![BusRequest::GetCurrentThroughput]).await
+    {
+      for msg in messages {
+        if let BusResponse::CurrentThroughput {
+          bits_per_second,
+          packets_per_second,
+          shaped_bits_per_second,
+        } = msg
+        {
+          self.data[self.head].bits_per_second = bits_per_second;
+          self.data[self.head].packets_per_second = packets_per_second;
+          self.data[self.head].shaped_bits_per_second = shaped_bits_per_second;
+          self.prev_head = self.head;
+          self.head += 1;
+          self.head %= 300;
+        }
+      }
+    }
+  }
+
+  /// Retrieve just the current throughput data (1 tick)
+  pub fn current(&self) -> ThroughputPerSecond {
+    self.data[self.prev_head]
+  }
+
+  /// Retrieve the head (0-299) and the full current throughput
+  /// buffer. Used to populate the dashboard the first time.
+  pub fn copy(&self) -> (usize, Vec<ThroughputPerSecond>) {
+    (self.head, self.data.clone())
+  }
+}
diff --git a/src/rust/lqos_node_manager/src/tracker/cache_manager.rs b/src/rust/lqos_node_manager/src/tracker/cache_manager.rs
index 2de8c8e5..cdc0cb72 100644
--- a/src/rust/lqos_node_manager/src/tracker/cache_manager.rs
+++ b/src/rust/lqos_node_manager/src/tracker/cache_manager.rs
@@ -9,8 +9,8 @@ use nix::sys::{
   time::{TimeSpec, TimeValLike},
   timerfd::{ClockId, Expiration, TimerFd, TimerFlags, TimerSetTimeFlags},
 };
-use rocket::tokio::task::spawn_blocking;
-use std::{sync::atomic::AtomicBool};
+use rocket::tokio::{task::spawn_blocking, time::Instant};
+use std::{sync::atomic::AtomicBool, time::Duration};
 
 /// Once per second, update CPU and RAM usage and ask
 /// `lqosd` for updated system statistics.
@@ -118,3 +118,16 @@ fn watch_for_shaped_devices_changing() -> Result<()> {
     info!("ShapedDevices watcher returned: {result:?}");
   }
 }
+
+/// Fires once per second and updates the global traffic ringbuffer.
+pub async fn update_total_throughput_buffer() {
+  loop {
+    let now = Instant::now();
+    let mut lock = THROUGHPUT_BUFFER.write().await;
+    lock.tick().await;
+    let wait_time = Duration::from_secs(1) - now.elapsed();
+    if wait_time.as_micros() > 0 {
+      rocket::tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/rust/lqos_node_manager/src/tracker/mod.rs b/src/rust/lqos_node_manager/src/tracker/mod.rs
index fc3e4f72..9eed911f 100644
--- a/src/rust/lqos_node_manager/src/tracker/mod.rs
+++ b/src/rust/lqos_node_manager/src/tracker/mod.rs
@@ -3,11 +3,11 @@ mod cache_manager;
 use std::net::IpAddr;
 
 use self::cache::{
-  CPU_USAGE, NUM_CPUS, RAM_USED, TOTAL_RAM,
+  CPU_USAGE, NUM_CPUS, RAM_USED, TOTAL_RAM, THROUGHPUT_BUFFER,
 };
 use crate::{auth_guard::AuthGuard, cache_control::NoCache};
 pub use cache::SHAPED_DEVICES;
-pub use cache_manager::update_tracking;
+pub use cache_manager::{update_tracking, update_total_throughput_buffer};
 use lqos_bus::{bus_request, BusRequest, BusResponse, IpStats, TcHandle};
 use rocket::serde::{Deserialize, Serialize, msgpack::MsgPack};
 
@@ -70,23 +70,15 @@ pub struct ThroughputPerSecond {
 pub async fn current_throughput(
   _auth: AuthGuard,
 ) -> NoCache<MsgPack<ThroughputPerSecond>> {
-  let mut result = ThroughputPerSecond::default();
-  if let Ok(messages) =
-    bus_request(vec![BusRequest::GetCurrentThroughput]).await
-  {
-    for msg in messages {
-      if let BusResponse::CurrentThroughput {
-        bits_per_second,
-        packets_per_second,
-        shaped_bits_per_second,
-      } = msg
-      {
-        result.bits_per_second = bits_per_second;
-        result.packets_per_second = packets_per_second;
-        result.shaped_bits_per_second = shaped_bits_per_second;
-      }
-    }
-  }
+  let result = THROUGHPUT_BUFFER.read().await.current();
+  NoCache::new(MsgPack(result))
+}
+
+#[get("/api/throughput_ring_buffer")]
+pub async fn throughput_ring_buffer(
+  _auth: AuthGuard,
+) -> NoCache<MsgPack<(usize, Vec<ThroughputPerSecond>)>> {
+  let result = THROUGHPUT_BUFFER.read().await.copy();
   NoCache::new(MsgPack(result))
 }
 
diff --git a/src/rust/lqos_node_manager/static/main.html b/src/rust/lqos_node_manager/static/main.html
index f9561028..95be9b97 100644
--- a/src/rust/lqos_node_manager/static/main.html
+++ b/src/rust/lqos_node_manager/static/main.html
@@ -168,6 +168,29 @@
     <script>
         var throughput = new MultiRingBuffer(300);
 
+        // Loads the complete ringbuffer for initial display
+        function fillCurrentThroughput() {
+            msgPackGet("/api/throughput_ring_buffer", (tp) => {
+                console.log(tp);
+                const bits = 0;
+                const packets = 1;
+                const shaped = 2;
+
+                let head = tp[0];
+                for (let i=head; i<300; ++i) {
+                    throughput.push("pps", tp[1][i][packets][0], tp[1][i][packets][1]);
+                    throughput.push("total", tp[1][i][bits][0], tp[1][i][bits][1]);
+                    throughput.push("shaped", tp[1][i][shaped][0], tp[1][i][shaped][1]);
+                }
+                for (let i=0; i<head; ++i) {
+                    throughput.push("pps", tp[1][i][packets][0], tp[1][i][packets][1]);
+                    throughput.push("total", tp[1][i][bits][0], tp[1][i][bits][1]);
+                    throughput.push("shaped", tp[1][i][shaped][0], tp[1][i][shaped][1]);
+                }
+                throughput.plotTotalThroughput("tpGraph");
+            });
+        }
+
         function updateCurrentThroughput() {
             msgPackGet("/api/current_throughput", (tp) => {
                 const bits = 0;
@@ -178,7 +201,7 @@
                 $("#bpsDown").text(scaleNumber(tp[bits][0]));
                 $("#bpsUp").text(scaleNumber(tp[bits][1]));
 
-                throughput.push("pps", tp[1][0], tp[packets][1]);
+                throughput.push("pps", tp[packets][0], tp[packets][1]);
                 throughput.push("total", tp[bits][0], tp[bits][1]);
                 throughput.push("shaped", tp[shaped][0], tp[shaped][1]);
                 throughput.plotTotalThroughput("tpGraph");
@@ -319,7 +342,7 @@
             }
 
             colorReloadButton();
-            updateCurrentThroughput();
+            fillCurrentThroughput();
             updateCpu();
             updateRam();
             updateTop10();

From 9fce5d51def458f88073262664063ac26f4399a4 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 16:20:56 +0000
Subject: [PATCH 20/67] Adds "client throughput" to the queue tree tab of
 circuit_queue

Also moves all the circuit graphs to use the same graphing
system as the other graphs on the plot.

FIXES #315
---
 .../static/circuit_queue.html                 | 124 +++++++++++-------
 1 file changed, 75 insertions(+), 49 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 43815d67..3e7daeb6 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -94,8 +94,9 @@
                         </ul>
                     </div>
                     <div class="col-sm-2">
-                        <a href="#" class="btn btn-small btn-info" id="btnPause"><i class="fa fa-pause"></i> Pause</a> 
-                        <a href="#" class="btn btn-small btn-info" id="btnSlow"><i class="fa fa-hourglass"></i> Slow Mode</a>
+                        <a href="#" class="btn btn-small btn-info" id="btnPause"><i class="fa fa-pause"></i> Pause</a>
+                        <a href="#" class="btn btn-small btn-info" id="btnSlow"><i class="fa fa-hourglass"></i> Slow
+                            Mode</a>
                     </div>
                 </div>
             </div>
@@ -385,16 +386,17 @@
                     { x: this.x_axis, y: this.backlog.tins[2].y, type: 'scattergl', mode: 'markers', name: 'Video', marker: { size: 4 } },
                     { x: this.x_axis, y: this.backlog.tins[3].y, type: 'scattergl', mode: 'markers', name: 'Voice', marker: { size: 4 } },
                 ];
-                
+
                 if (this.backlogPlotted == null) {
                     this.backlogPlotted = true;
                     Plotly.newPlot(
-                        graph, 
-                        graphData, 
-                        { 
-                            margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, 
-                            yaxis: { automargin: true, title: "Bytes" }, 
-                            xaxis: { automargin: true, title: "Time since now" } });
+                        graph,
+                        graphData,
+                        {
+                            margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 },
+                            yaxis: { automargin: true, title: "Bytes" },
+                            xaxis: { automargin: true, title: "Time since now" }
+                        });
                 } else {
                     Plotly.redraw(graph, graphData);
                 }
@@ -411,11 +413,13 @@
 
                 if (this.delaysPlotted == null) {
                     Plotly.newPlot(
-                        graph, 
-                        graphData, 
-                        { margin: { l: 8, r: 0, b: 0, t: 0, pad: 4 }, 
-                        yaxis: { automargin: true, title: "log10(ms)", range: [-1.0, 1.0] }, 
-                        xaxis: { automargin: true, title: "Time since now" } });
+                        graph,
+                        graphData,
+                        {
+                            margin: { l: 8, r: 0, b: 0, t: 0, pad: 4 },
+                            yaxis: { automargin: true, title: "log10(ms)", range: [-1.0, 1.0] },
+                            xaxis: { automargin: true, title: "Time since now" }
+                        });
                     this.delaysPlotted = true;
                 } else {
                     Plotly.redraw(graph, graphData);
@@ -441,7 +445,7 @@
                 let graphData = [
                     { x: this.x_axis, y: this.throughput.tins[tin].y, type: 'scatter', mode: 'markers' }
                 ];
-                if (this.tinsPlotted == null) {   
+                if (this.tinsPlotted == null) {
                     Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Bits" }, xaxis: { automargin: true, title: "Time since now" } });
                 } else {
                     Plotly.redraw(graph, graphData);
@@ -502,7 +506,7 @@
                 this.per_ip = {};
                 this.y = {};
                 this.x_axis = [];
-                for (let i=0; i<capacity; ++i) {
+                for (let i = 0; i < capacity; ++i) {
                     this.x_axis.push(i);
                     this.x_axis.push(i);
                 }
@@ -514,7 +518,7 @@
             }
 
             clearQuantiles() {
-                for (let i=0; i<12; ++i) {
+                for (let i = 0; i < 12; ++i) {
                     this.quantiles[0][i] = 0;
                     this.quantiles[1][i] = 0;
                 }
@@ -526,7 +530,7 @@
                 if (!this.per_ip.hasOwnProperty(ip)) {
                     this.per_ip[ip] = [];
                     this.y[ip] = [];
-                    for (let i=0; i<this.capacity; ++i) {
+                    for (let i = 0; i < this.capacity; ++i) {
                         this.per_ip[ip].push(0);
                         this.per_ip[ip].push(0);
                         this.y[ip].push(0);
@@ -536,7 +540,7 @@
                 this.per_ip[ip][this.head] = down;
                 this.per_ip[ip][this.head + 1] = 0.0 - up;
                 this.head += 2;
-                if (this.head > this.capacity*2) {
+                if (this.head > this.capacity * 2) {
                     this.head = 0;
                 }
             }
@@ -557,30 +561,30 @@
             prepare() {
                 this.clearQuantiles();
                 for (const ip in this.per_ip) {
-                    let counter = this.capacity*2;
-                    for (let i = this.head; i < this.capacity*2; i++) {
-                        this.y[ip][counter] = this.per_ip[ip][i];              
+                    let counter = this.capacity * 2;
+                    for (let i = this.head; i < this.capacity * 2; i++) {
+                        this.y[ip][counter] = this.per_ip[ip][i];
                         counter--;
                     }
                     for (let i = 0; i < this.head; i++) {
                         this.y[ip][counter] = this.per_ip[ip][i];
                         counter--;
                     }
-                    for (let i=2; i<22; i+=2) {
-                        this.addQuantile(this.y[ip][i], this.y[ip][i+1]);
+                    for (let i = 2; i < 22; i += 2) {
+                        this.addQuantile(this.y[ip][i], this.y[ip][i + 1]);
                     }
                 }
             }
 
-            plot() {
-                let graph = document.getElementById("throughputGraph");
+            plot(target) {
+                let graph = document.getElementById(target);
                 let graphData = [];
                 for (const ip in this.per_ip) {
                     graphData.push({ x: this.x_axis, y: this.y[ip], name: ip, mode: 'markers', type: 'scattergl', marker: { size: 3 } });
                 }
-                if (this.plotted == null) {
-                    Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Traffic (bits)" }, xaxis: { automargin: true, title: "Time since now" } });                    
-                    this.plotted = true;
+                if (!this.hasOwnProperty("plotted" + target)) {
+                    Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Traffic (bits)" }, xaxis: { automargin: true, title: "Time since now" } });
+                    this["plotted" + target] = true;
                 } else {
                     Plotly.redraw(graph, graphData);
                 }
@@ -608,7 +612,7 @@
                 msgPackGet("/api/circuit_throughput/" + encodeURI(id), (data) => {
                     if (tpData == null) tpData = new ThroughputMonitor(300);
                     ips = [];
-                    for (let i=0; i < data.length; i++) {
+                    for (let i = 0; i < data.length; i++) {
                         let ip = data[i][0];
                         ips.push(ip);
                         let down = data[i][1];
@@ -616,13 +620,13 @@
                         tpData.ingest(ip, down, up);
                     }
                     tpData.prepare();
-                    tpData.plot();
+                    tpData.plot("throughputGraph");
                     tpData.plotQuantiles();
                 });
             }
         }
 
-        let funnels = new MultiRingBuffer(300);
+        let funnels = new ThroughputMonitor(300);
         let rtts = {};
         let circuitId = "";
         let builtFunnelDivs = false;
@@ -635,8 +639,24 @@
             circuitId = encodeURI(id);
             msgPackGet("/api/funnel_for_queue/" + circuitId, (data) => {
                 let html = "";
+
+                // Add the client on top
+                let row = "<div class='row row220'>";
+
+                row += "<div class='col-sm-12'>";
+                row += "<div class='card bg-light'>";
+                row += "<h5 class='card-title'><i class='fa fa-hourglass'></i> Client Throughput</h5>";
+                row += "<div id='tp_client' class='graph98 graph150'></div>";
+                row += "</div>";
+                row += "</div>";
+
+                row += "</div>";
+                html += row;
+
+                // Funnels
                 for (let i = 0; i < data.length; ++i) {
-                    funnels.push(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
+                    //funnels.push(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
+                    funnels.ingest(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
                     rtts[data[i][0]] = new RttHistogram();
 
                     let row = "<div class='row row220'>";
@@ -660,33 +680,38 @@
                 }
                 $("#pills-funnel").html(html);
                 builtFunnelDivs = true;
-                setTimeout(plotFunnels, 10);
             });
         }
 
         let plottedFunnels = {};
 
         function plotFunnels() {
+            if (tpData != null) tpData.plot("tp_client");
+            funnels.prepare();
             msgPackGet("/api/funnel_for_queue/" + encodeURI(circuitId), (data) => {
                 for (let i = 0; i < data.length; ++i) {
-                    funnels.push(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
-                    for (const [k, v] of Object.entries(funnels.data)) {
-                        let target_div = "tp" + k;                        
-                        let graphData = v.toScatterGraphData();
-                        let graph = document.getElementById(target_div);
-                        if (!plotFunnels.hasOwnProperty(target_div)) {
-                            Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true }, xaxis: { automargin: true, title: "Time since now" } }, { responsive: true });
-                        } else {
-                            Plotly.redraw(graph, graphData);
-                        }
-                    }
-
                     rtts[data[i][0]].clear();
+                    funnels.ingest(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
                     for (let j = 0; j < data[i][1][NetTrans.rtts].length; j++) {
                         rtts[data[i][0]].push(data[i][1][NetTrans.rtts][j]);
                     }
+
                     rtts[data[i][0]].plot("rtt" + data[i][0]);
                 }
+
+                for (const [k, v] of Object.entries(funnels.y)) {
+                    let target_div = "tp" + k;
+                    let graphData = [
+                        { x: funnels.x_axis, y: v, type: 'scatter', mode: 'markers', marker: { size: 3 } }
+                    ];
+                    let graph = document.getElementById(target_div);
+                    if (!plotFunnels.hasOwnProperty(target_div)) {
+                        Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Traffic (bits)" }, xaxis: { automargin: true, title: "Time since now" } });
+                    } else {
+                        Plotly.redraw(graph, graphData);
+                    }
+                }
+
             });
         }
 
@@ -828,14 +853,14 @@
         var slowMode = false;
 
         function showFps() {
-            if(!lastCalledTime) {
+            if (!lastCalledTime) {
                 lastCalledTime = Date.now();
                 fps = 0;
                 return;
             }
-            delta = (Date.now() - lastCalledTime)/1000;
+            delta = (Date.now() - lastCalledTime) / 1000;
             lastCalledTime = Date.now();
-            fps = 1/delta;
+            fps = 1 / delta;
             //$("#fps").text(fps.toFixed(0));
             worstDelta = Math.max(delta, worstDelta);
         }
@@ -846,6 +871,7 @@
                 switch (activeTab) {
                     case "pills-funnel-tab": {
                         getFunnel();
+                        getThroughput();
                     } break;
                     case "pills-flows-tab": {
                         getFlows();

From 16721208e6b5ba91e327194c75ae6f6f1be3526e Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 17:10:43 +0000
Subject: [PATCH 21/67] Add DNS resolution to displayed flow IP addresses

---
 src/rust/Cargo.lock                              | 14 ++++++++++++++
 src/rust/lqos_node_manager/Cargo.toml            |  2 ++
 src/rust/lqos_node_manager/src/main.rs           |  1 +
 src/rust/lqos_node_manager/src/queue_info.rs     | 11 ++++++++++-
 .../lqos_node_manager/src/tracker/cache/mod.rs   |  2 ++
 src/rust/lqos_node_manager/src/tracker/mod.rs    |  1 +
 .../lqos_node_manager/static/circuit_queue.html  |  4 ++--
 src/rust/lqos_node_manager/static/lqos.js        | 16 ++++++++++++++++
 8 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
index 5d681b67..299ff8b1 100644
--- a/src/rust/Cargo.lock
+++ b/src/rust/Cargo.lock
@@ -733,6 +733,18 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "dns-lookup"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53ecafc952c4528d9b51a458d1a8904b81783feff9fde08ab6ed2545ff396872"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "socket2",
+ "winapi",
+]
+
 [[package]]
 name = "either"
 version = "1.8.1"
@@ -1448,7 +1460,9 @@ name = "lqos_node_manager"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "dashmap",
  "default-net",
+ "dns-lookup",
  "jemallocator",
  "lqos_bus",
  "lqos_config",
diff --git a/src/rust/lqos_node_manager/Cargo.toml b/src/rust/lqos_node_manager/Cargo.toml
index 69b3c05b..818c8d42 100644
--- a/src/rust/lqos_node_manager/Cargo.toml
+++ b/src/rust/lqos_node_manager/Cargo.toml
@@ -19,6 +19,8 @@ sysinfo = "0"
 default-net = "0"
 nix = "0"
 once_cell = "1"
+dns-lookup = "1"
+dashmap = "5"
 
 # Support JemAlloc on supported platforms
 [target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies]
diff --git a/src/rust/lqos_node_manager/src/main.rs b/src/rust/lqos_node_manager/src/main.rs
index c0e13f32..1fe50750 100644
--- a/src/rust/lqos_node_manager/src/main.rs
+++ b/src/rust/lqos_node_manager/src/main.rs
@@ -77,6 +77,7 @@ fn rocket() -> _ {
         queue_info::packet_dump,
         queue_info::pcap,
         queue_info::request_analysis,
+        queue_info::dns_query,
         config_control::get_nic_list,
         config_control::get_current_python_config,
         config_control::get_current_lqosd_config,
diff --git a/src/rust/lqos_node_manager/src/queue_info.rs b/src/rust/lqos_node_manager/src/queue_info.rs
index fe5e7d46..3a0a34ab 100644
--- a/src/rust/lqos_node_manager/src/queue_info.rs
+++ b/src/rust/lqos_node_manager/src/queue_info.rs
@@ -1,6 +1,6 @@
 use crate::auth_guard::AuthGuard;
 use crate::cache_control::NoCache;
-use crate::tracker::SHAPED_DEVICES;
+use crate::tracker::{SHAPED_DEVICES, lookup_dns};
 use lqos_bus::{bus_request, BusRequest, BusResponse, FlowTransport, PacketHeader, QueueStoreTransit};
 use rocket::fs::NamedFile;
 use rocket::http::Status;
@@ -162,6 +162,15 @@ pub async fn pcap(id: usize, filename: String) -> Result<NoCache<NamedFile>, Sta
   Err(Status::NotFound)
 }
 
+#[get("/api/dns/<ip>")]
+pub async fn dns_query(ip: String) -> NoCache<String> {
+  if let Ok(ip) = ip.parse::<IpAddr>() {
+    NoCache::new(lookup_dns(ip))
+  } else {
+    NoCache::new(ip)
+  }
+}
+
 #[cfg(feature = "equinix_tests")]
 #[get("/api/run_btest")]
 pub async fn run_btest() -> NoCache<RawJson<String>> {
diff --git a/src/rust/lqos_node_manager/src/tracker/cache/mod.rs b/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
index 8829e9f5..2e0a7d89 100644
--- a/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
+++ b/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
@@ -5,7 +5,9 @@
 mod cpu_ram;
 mod shaped_devices;
 mod throughput;
+mod dns_cache;
 
 pub use cpu_ram::*;
 pub use shaped_devices::*;
 pub use throughput::THROUGHPUT_BUFFER;
+pub use dns_cache::lookup_dns;
diff --git a/src/rust/lqos_node_manager/src/tracker/mod.rs b/src/rust/lqos_node_manager/src/tracker/mod.rs
index 9eed911f..67cdf69c 100644
--- a/src/rust/lqos_node_manager/src/tracker/mod.rs
+++ b/src/rust/lqos_node_manager/src/tracker/mod.rs
@@ -10,6 +10,7 @@ pub use cache::SHAPED_DEVICES;
 pub use cache_manager::{update_tracking, update_total_throughput_buffer};
 use lqos_bus::{bus_request, BusRequest, BusResponse, IpStats, TcHandle};
 use rocket::serde::{Deserialize, Serialize, msgpack::MsgPack};
+pub use cache::lookup_dns;
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(crate = "rocket::serde")]
diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 3e7daeb6..718a5327 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -813,13 +813,13 @@
                     }
                     html += "<tr>";
                     html += "<td>" + data[i][0][FlowTrans.proto] + "</td>";
-                    html += "<td>" + data[i][0][FlowTrans.src] + "</td>";
+                    html += "<td>" + ipToHostname(data[i][0][FlowTrans.src]) + "</td>";
                     if (data[i][0].proto == "ICMP") {
                         html += "<td>" + icmpType(data[i][0][FlowTrans.src_port]) + "</td>";
                     } else {
                         html += "<td>" + data[i][0][FlowTrans.src_port] + "</td>";
                     }
-                    html += "<td>" + data[i][0][FlowTrans.dst] + "</td>";
+                    html += "<td>" + ipToHostname(data[i][0][FlowTrans.dst]) + "</td>";
                     if (data[i][0][FlowTrans.proto] == "ICMP") {
                         if (data[i][1] != null) {
                             html += "<td>" + icmpType(data[i][1][FlowTrans.src_port]) + "</td>";
diff --git a/src/rust/lqos_node_manager/static/lqos.js b/src/rust/lqos_node_manager/static/lqos.js
index dd40994a..b1633dd5 100644
--- a/src/rust/lqos_node_manager/static/lqos.js
+++ b/src/rust/lqos_node_manager/static/lqos.js
@@ -447,4 +447,20 @@ function zero_to_null(array) {
     for (let i=0; i<array.length; ++i) {
         if (array[i] == 0) array[i] = null;
     }
+}
+
+var dnsCache = {};
+
+function ipToHostname(ip) {
+    if (dnsCache.hasOwnProperty(ip)) {
+        if (dnsCache[ip] != ip) {
+            return ip + "<br /><span style='font-size: 6pt'>" + dnsCache[ip] + "</span>";
+        } else {
+            return ip;
+        }
+    }
+    $.get("/api/dns/" + encodeURI(ip), (hostname) => {
+        dnsCache[ip] = hostname;
+    })
+    return ip;
 }
\ No newline at end of file

From 34e3381af4bd45b31f70abcbc209654a76d096c9 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 17:12:18 +0000
Subject: [PATCH 22/67] Add file missing from previous commit

---
 .../src/tracker/cache/dns_cache.rs            | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 src/rust/lqos_node_manager/src/tracker/cache/dns_cache.rs

diff --git a/src/rust/lqos_node_manager/src/tracker/cache/dns_cache.rs b/src/rust/lqos_node_manager/src/tracker/cache/dns_cache.rs
new file mode 100644
index 00000000..7165a137
--- /dev/null
+++ b/src/rust/lqos_node_manager/src/tracker/cache/dns_cache.rs
@@ -0,0 +1,38 @@
+//! Implements a lock-free DNS least-recently-used DNS cache.
+
+use std::net::IpAddr;
+use dashmap::DashMap;
+use dns_lookup::lookup_addr;
+use lqos_utils::unix_time::unix_now;
+use once_cell::sync::Lazy;
+
+const CACHE_SIZE: usize = 1000;
+
+struct DnsEntry {
+    hostname: String,
+    last_accessed: u64,
+}
+
+static DNS_CACHE: Lazy<DashMap<IpAddr, DnsEntry>> = Lazy::new(|| DashMap::with_capacity(CACHE_SIZE));
+
+pub fn lookup_dns(ip: IpAddr) -> String {
+    // If the cached value exists, just return it
+    if let Some(mut dns) = DNS_CACHE.get_mut(&ip) {
+        if let Ok(now) = unix_now() {
+            dns.last_accessed = now;
+        }
+        return dns.hostname.clone();
+    }
+
+    // If it doesn't, we'll be adding it.
+    if DNS_CACHE.len() >= CACHE_SIZE {
+        let mut entries : Vec<(IpAddr, u64)> = DNS_CACHE.iter().map(|v| (*v.key(), v.last_accessed)).collect();
+        entries.sort_by(|a,b| b.1.cmp(&a.1));
+        DNS_CACHE.remove(&entries[0].0);
+    }
+    let hostname = lookup_addr(&ip).unwrap_or(ip.to_string());
+    DNS_CACHE.insert(ip, DnsEntry { hostname, last_accessed: unix_now().unwrap_or(0) });
+
+
+    String::new()
+}
\ No newline at end of file

From ebfcde2618d011d356adef79dea71ea0b6454965 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 17:31:59 +0000
Subject: [PATCH 23/67] Fix a missing break - switch fallthrough by mistake -
 in dissector.h. This fixes a problem with misidentifying traffic as TCP when
 it is't.

---
 src/rust/lqos_sys/src/bpf/common/dissector.h | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/rust/lqos_sys/src/bpf/common/dissector.h b/src/rust/lqos_sys/src/bpf/common/dissector.h
index 25653d3f..cd43eb0d 100644
--- a/src/rust/lqos_sys/src/bpf/common/dissector.h
+++ b/src/rust/lqos_sys/src/bpf/common/dissector.h
@@ -345,8 +345,7 @@ static __always_inline void snoop(struct dissector_t *dissector)
 
             parse_tcp_ts(hdr, dissector->end, &dissector->tsval, &dissector->tsecr);
         }
-    }
-    break;
+    } break;
     case IPPROTO_UDP:
     {
         struct udphdr *hdr = get_udp_header(dissector);
@@ -354,12 +353,13 @@ static __always_inline void snoop(struct dissector_t *dissector)
         {
             if (hdr + 1 > dissector->end)
             {
+                bpf_debug("UDP header past end");
                 return;
             }
             dissector->src_port = hdr->source;
             dissector->dst_port = hdr->dest;
         }
-    }
+    } break;
     case IPPROTO_ICMP:
     {
         struct icmphdr *hdr = get_icmp_header(dissector);
@@ -367,14 +367,14 @@ static __always_inline void snoop(struct dissector_t *dissector)
         {
             if ((char *)hdr + sizeof(struct icmphdr) > dissector->end)
             {
+                bpf_debug("ICMP header past end");
                 return;
             }
             dissector->ip_protocol = 1;
             dissector->src_port = bpf_ntohs(hdr->type);
-            dissector->dst_port = bpf_ntohs(hdr->type);
-        }
-    }
-    break;
+            dissector->dst_port = bpf_ntohs(hdr->code);
+        }    
+    } break;
     }
 }
 

From 01433160acdf489448b76a7be9bf34e8ec674676 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 14:36:43 +0000
Subject: [PATCH 24/67] Fix displayed plans in Top/Worst N tables

A value was transposed in the IPStats to IPStats transit
conversion. The UI now displays (max download/max upload)
correctly.

Fix an accidental inclusion of a console.log statement
logging needlessly.
---
 src/rust/lqos_node_manager/src/tracker/mod.rs | 2 +-
 src/rust/lqos_node_manager/static/main.html   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rust/lqos_node_manager/src/tracker/mod.rs b/src/rust/lqos_node_manager/src/tracker/mod.rs
index 67cdf69c..6795c2de 100644
--- a/src/rust/lqos_node_manager/src/tracker/mod.rs
+++ b/src/rust/lqos_node_manager/src/tracker/mod.rs
@@ -50,7 +50,7 @@ impl From<&IpStats> for IpStatsWithPlan {
           &circuit.circuit_name
         };
         result.ip_address = format!("{} ({})", name, result.ip_address);
-        result.plan = (circuit.download_max_mbps, circuit.download_min_mbps);
+        result.plan = (circuit.download_max_mbps, circuit.upload_max_mbps);
       }
     }
 
diff --git a/src/rust/lqos_node_manager/static/main.html b/src/rust/lqos_node_manager/static/main.html
index 95be9b97..82d8f57f 100644
--- a/src/rust/lqos_node_manager/static/main.html
+++ b/src/rust/lqos_node_manager/static/main.html
@@ -171,7 +171,7 @@
         // Loads the complete ringbuffer for initial display
         function fillCurrentThroughput() {
             msgPackGet("/api/throughput_ring_buffer", (tp) => {
-                console.log(tp);
+                //console.log(tp);
                 const bits = 0;
                 const packets = 1;
                 const shaped = 2;

From eb9b32ef6568dc5c30be86f4d0b5273618350ae1 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 15:00:35 +0000
Subject: [PATCH 25/67] Format-preserving TOML editing

Restore the automatic generation of a node_id if one isn't present,
based on a hash of /etc/machine_id. Switch from the "toml" crate
to the "toml_edit" crate, and use its Document type to preserve
the user's comments and formatting when editing the configuration
file for saving.
---
 src/rust/Cargo.lock                        | 16 +------
 src/rust/lqos_config/Cargo.toml            |  2 +-
 src/rust/lqos_config/src/authentication.rs |  6 +--
 src/rust/lqos_config/src/etc.rs            | 52 +++++++++++++---------
 4 files changed, 36 insertions(+), 40 deletions(-)

diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
index 299ff8b1..2f8f803f 100644
--- a/src/rust/Cargo.lock
+++ b/src/rust/Cargo.lock
@@ -812,7 +812,7 @@ dependencies = [
  "atomic",
  "pear",
  "serde",
- "toml 0.5.11",
+ "toml",
  "uncased",
  "version_check",
 ]
@@ -1436,7 +1436,7 @@ dependencies = [
  "serde_json",
  "sha2",
  "thiserror",
- "toml 0.7.2",
+ "toml_edit",
  "uuid",
 ]
 
@@ -2787,18 +2787,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "toml"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6"
-dependencies = [
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit",
-]
-
 [[package]]
 name = "toml_datetime"
 version = "0.6.1"
diff --git a/src/rust/lqos_config/Cargo.toml b/src/rust/lqos_config/Cargo.toml
index 41e037be..0b17e88b 100644
--- a/src/rust/lqos_config/Cargo.toml
+++ b/src/rust/lqos_config/Cargo.toml
@@ -6,7 +6,7 @@ license = "GPL-2.0-only"
 
 [dependencies]
 thiserror = "1"
-toml = "0"
+toml_edit = { version = "0", features = [ "serde" ] }
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = "1"
 csv = "1"
diff --git a/src/rust/lqos_config/src/authentication.rs b/src/rust/lqos_config/src/authentication.rs
index e6c3effe..a80de0d6 100644
--- a/src/rust/lqos_config/src/authentication.rs
+++ b/src/rust/lqos_config/src/authentication.rs
@@ -68,7 +68,7 @@ impl WebUsers {
 
   fn save_to_disk(&self) -> Result<(), AuthenticationError> {
     let path = Self::path()?;
-    let new_contents = toml::to_string(&self);
+    let new_contents = toml_edit::ser::to_string(&self);
     if let Err(e) = new_contents {
       return Err(AuthenticationError::SerializationError(e));
     }
@@ -109,7 +109,7 @@ impl WebUsers {
     } else {
       // Load from disk
       if let Ok(raw) = read_to_string(path) {
-        let parse_result = toml::from_str(&raw);
+        let parse_result = toml_edit::de::from_str(&raw);
         if let Ok(users) = parse_result {
           Ok(users)
         } else {
@@ -255,7 +255,7 @@ pub enum AuthenticationError {
   #[error("Unable to load /etc/lqos.conf")]
   UnableToLoadEtcLqos,
   #[error("Unable to serialize to TOML")]
-  SerializationError(toml::ser::Error),
+  SerializationError(toml_edit::ser::Error),
   #[error("Unable to remove existing web users file")]
   UnableToDelete,
   #[error("Unable to open lqusers.toml for writing. Check permissions?")]
diff --git a/src/rust/lqos_config/src/etc.rs b/src/rust/lqos_config/src/etc.rs
index 83090d70..ff2069c1 100644
--- a/src/rust/lqos_config/src/etc.rs
+++ b/src/rust/lqos_config/src/etc.rs
@@ -1,6 +1,7 @@
 //! Manages the `/etc/lqos.conf` file.
 use log::error;
 use serde::{Deserialize, Serialize};
+use toml_edit::{Document, value};
 use std::{fs, path::Path};
 use thiserror::Error;
 
@@ -136,17 +137,27 @@ impl EtcLqos {
       return Err(EtcLqosError::ConfigDoesNotExist);
     }
     if let Ok(raw) = std::fs::read_to_string("/etc/lqos.conf") {
-      let config_result: Result<Self, toml::de::Error> = toml::from_str(&raw);
-      match config_result {
-        Ok(mut config) => {
-          check_config(&mut config);
-          Ok(config)
-        }
+      let document = raw.parse::<Document>();
+      match document {
         Err(e) => {
           error!("Unable to parse TOML from /etc/lqos.conf");
           error!("Full error: {:?}", e);
           Err(EtcLqosError::CannotParseToml)
         }
+        Ok(mut config_doc) => {
+          let cfg = toml_edit::de::from_document::<EtcLqos>(config_doc.clone());
+          match cfg {
+            Ok(mut cfg) => {
+              check_config(&mut config_doc, &mut cfg);
+              Ok(cfg)
+            }
+            Err(e) => {
+              error!("Unable to parse TOML from /etc/lqos.conf");
+              error!("Full error: {:?}", e);
+              Err(EtcLqosError::CannotParseToml)
+            }
+          }
+        }
       }
     } else {
       error!("Unable to read contents of /etc/lqos.conf");
@@ -156,7 +167,7 @@ impl EtcLqos {
 
   /// Saves changes made to /etc/lqos.conf
   /// Copies current configuration into /etc/lqos.conf.backup first
-  pub fn save(&self) -> Result<(), EtcLqosError> {
+  pub fn save(&self, document: &mut Document) -> Result<(), EtcLqosError> {
     let cfg_path = Path::new("/etc/lqos.conf");
     let backup_path = Path::new("/etc/lqos.conf.backup");
     if let Err(e) = std::fs::copy(cfg_path, backup_path) {
@@ -164,26 +175,17 @@ impl EtcLqos {
       log::error!("{e:?}");
       return Err(EtcLqosError::BackupFail);
     }
-    let new_cfg = toml::to_string_pretty(&self);
-    match new_cfg {
-      Err(e) => {
-        log::error!("Unable to serialize new /etc/lqos.conf");
-        log::error!("{e:?}");
-        return Err(EtcLqosError::SerializeFail);
-      }
-      Ok(new_cfg) => {
-        if let Err(e) = fs::write(cfg_path, new_cfg) {
-          log::error!("Unable to write to /etc/lqos.conf");
-          log::error!("{e:?}");
-          return Err(EtcLqosError::WriteFail);
-        }
-      }
+    let new_cfg = document.to_string();
+    if let Err(e) = fs::write(cfg_path, new_cfg) {
+      log::error!("Unable to write to /etc/lqos.conf");
+      log::error!("{e:?}");
+      return Err(EtcLqosError::WriteFail);
     }
     Ok(())
   }
 }
 
-fn check_config(cfg: &mut EtcLqos) {
+fn check_config(cfg_doc: &mut Document, cfg: &mut EtcLqos) {
   use sha2::digest::Update;
   use sha2::Digest;
 
@@ -191,6 +193,12 @@ fn check_config(cfg: &mut EtcLqos) {
     if let Ok(machine_id) = std::fs::read_to_string("/etc/machine-id") {
       let hash = sha2::Sha256::new().chain(machine_id).finalize();
       cfg.node_id = Some(format!("{:x}", hash));
+      cfg_doc["node_id"] = value(format!("{:x}", hash));
+      println!("Updating");
+      if let Err(e) = cfg.save(cfg_doc) {
+        log::error!("Unable to save /etc/lqos.conf");
+        log::error!("{e:?}");
+      }
     }
   }
 }

From c9a443f77b5ac828a0b2b1117666a348d6408044 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 16:36:10 +0000
Subject: [PATCH 26/67] Add a couple of unit test.

---
 src/rust/lqos_config/src/etc.rs | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/rust/lqos_config/src/etc.rs b/src/rust/lqos_config/src/etc.rs
index ff2069c1..8d47327b 100644
--- a/src/rust/lqos_config/src/etc.rs
+++ b/src/rust/lqos_config/src/etc.rs
@@ -220,3 +220,23 @@ pub enum EtcLqosError {
   #[error("Unable to write to /etc/lqos.conf")]
   WriteFail,
 }
+
+#[cfg(test)]
+mod test {
+  const EXAMPLE_LQOS_CONF: &str = include_str!("../../../lqos.example");
+
+  #[test]
+  fn round_trip_toml() {
+    let doc = EXAMPLE_LQOS_CONF.parse::<toml_edit::Document>().unwrap();
+    let reserialized = doc.to_string();
+    assert_eq!(EXAMPLE_LQOS_CONF, reserialized);
+  }
+
+  #[test]
+  fn add_node_id() {
+    let mut doc = EXAMPLE_LQOS_CONF.parse::<toml_edit::Document>().unwrap();
+    doc["node_id"] = toml_edit::value("test");
+    let reserialized = doc.to_string();
+    assert!(reserialized.contains("node_id = \"test\""));
+  }
+}
\ No newline at end of file

From bb53be9bfc470cc9a22601b334e8aca0816f7f71 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 16:51:15 +0000
Subject: [PATCH 27/67] Add some unit tests and documentation to the  crate. No
 functional changes.

---
 src/rust/lqos_utils/src/fdtimer.rs           |  7 ++++
 src/rust/lqos_utils/src/file_watcher.rs      |  4 ++
 src/rust/lqos_utils/src/hex_string.rs        |  1 +
 src/rust/lqos_utils/src/lib.rs               | 16 +++++++-
 src/rust/lqos_utils/src/packet_scale.rs      | 42 ++++++++++++++++----
 src/rust/lqos_utils/src/string_table_enum.rs | 13 ++++++
 src/rust/lqos_utils/src/unix_time.rs         |  2 +
 7 files changed, 76 insertions(+), 9 deletions(-)

diff --git a/src/rust/lqos_utils/src/fdtimer.rs b/src/rust/lqos_utils/src/fdtimer.rs
index f9f80a96..baa7c36c 100644
--- a/src/rust/lqos_utils/src/fdtimer.rs
+++ b/src/rust/lqos_utils/src/fdtimer.rs
@@ -5,6 +5,13 @@ use nix::sys::{
 };
 use std::sync::atomic::AtomicBool;
 
+/// `periodic` runs a function at a given interval.
+/// 
+/// ## Parameters
+/// 
+/// * `interval_ms`: the interval in milliseconds.
+/// * `task_name`: the name of the task to run.
+/// * `tick_function`: the function to run at the given interval.
 pub fn periodic(
   interval_ms: u64,
   task_name: &str,
diff --git a/src/rust/lqos_utils/src/file_watcher.rs b/src/rust/lqos_utils/src/file_watcher.rs
index 9b2d4b2e..0eb3d2f1 100644
--- a/src/rust/lqos_utils/src/file_watcher.rs
+++ b/src/rust/lqos_utils/src/file_watcher.rs
@@ -145,10 +145,14 @@ impl FileWatcher {
   }
 }
 
+/// Errors that can occur when watching a file.
 #[derive(Error, Debug)]
 pub enum WatchedFileError {
+  /// Unable to create the file watcher.
   #[error("Unable to create watcher")]
   CreateWatcherError,
+
+  /// Unable to start the file watcher system.
   #[error("Unable to start watcher")]
   StartWatcherError,
 }
diff --git a/src/rust/lqos_utils/src/hex_string.rs b/src/rust/lqos_utils/src/hex_string.rs
index 7e47ceb8..8c11e6a6 100644
--- a/src/rust/lqos_utils/src/hex_string.rs
+++ b/src/rust/lqos_utils/src/hex_string.rs
@@ -37,6 +37,7 @@ pub fn read_hex_string(s: &str) -> Result<u32, HexParseError> {
 /// parsing a string into a `u32` hex number.
 #[derive(Error, Debug)]
 pub enum HexParseError {
+  /// The hex string could not be decoded
   #[error("Unable to decode string into valid hex")]
   ParseError,
 }
diff --git a/src/rust/lqos_utils/src/lib.rs b/src/rust/lqos_utils/src/lib.rs
index b0654c0d..352ac9d1 100644
--- a/src/rust/lqos_utils/src/lib.rs
+++ b/src/rust/lqos_utils/src/lib.rs
@@ -1,10 +1,24 @@
+//! Collection of utility functions for LibreQoS
+
+#![warn(missing_docs)]
 mod commands;
+
+/// Provides a Linux file-descriptor based timing service.
 pub mod fdtimer;
+
+/// Wrapper for watching when a file has changed.
 pub mod file_watcher;
+
+/// Utilities for handling strings in hex format
 pub mod hex_string;
+
+/// Utilities for scaling bits and packets to human-readable format
 pub mod packet_scale;
 mod string_table_enum;
+
+/// Utilities dealing with Unix Timestamps
 pub mod unix_time;
 mod xdp_ip_address;
 
-pub use xdp_ip_address::XdpIpAddress;
\ No newline at end of file
+/// XDP compatible IP Address
+pub use xdp_ip_address::XdpIpAddress;
diff --git a/src/rust/lqos_utils/src/packet_scale.rs b/src/rust/lqos_utils/src/packet_scale.rs
index 6f88504a..56af8c9f 100644
--- a/src/rust/lqos_utils/src/packet_scale.rs
+++ b/src/rust/lqos_utils/src/packet_scale.rs
@@ -1,23 +1,49 @@
+/// Scale a number of packets to a human readable string.
+/// 
+/// ## Parameters
+/// * `n`: the number of packets to scale
 pub fn scale_packets(n: u64) -> String {
-  if n > 1_000_000_000 {
+  if n >= 1_000_000_000 {
     format!("{:.2} gpps", n as f32 / 1_000_000_000.0)
-  } else if n > 1_000_000 {
+  } else if n >= 1_000_000 {
     format!("{:.2} mpps", n as f32 / 1_000_000.0)
-  } else if n > 1_000 {
+  } else if n >= 1_000 {
     format!("{:.2} kpps", n as f32 / 1_000.0)
   } else {
-    format!("{n}  pps")
+    format!("{n} pps")
   }
 }
 
+/// Scale a number of bits to a human readable string.
+/// 
+/// ## Parameters
+/// * `n`: the number of bits to scale
 pub fn scale_bits(n: u64) -> String {
-  if n > 1_000_000_000 {
+  if n >= 1_000_000_000 {
     format!("{:.2} gbit/s", n as f32 / 1_000_000_000.0)
-  } else if n > 1_000_000 {
+  } else if n >= 1_000_000 {
     format!("{:.2} mbit/s", n as f32 / 1_000_000.0)
-  } else if n > 1_000 {
+  } else if n >= 1_000 {
     format!("{:.2} kbit/s", n as f32 / 1_000.0)
   } else {
-    format!("{n}  bit/s")
+    format!("{n} bit/s")
   }
 }
+
+#[cfg(test)]
+mod test {
+  #[test]
+  fn test_scale_packets() {
+    assert_eq!(super::scale_packets(1), "1 pps");
+    assert_eq!(super::scale_packets(1000), "1.00 kpps");
+    assert_eq!(super::scale_packets(1000000), "1.00 mpps");
+    assert_eq!(super::scale_packets(1000000000), "1.00 gpps");
+  }
+
+  #[test]
+  fn test_scale_bits() {
+    assert_eq!(super::scale_bits(1), "1 bit/s");
+    assert_eq!(super::scale_bits(1000), "1.00 kbit/s");
+    assert_eq!(super::scale_bits(1000000), "1.00 mbit/s");
+    assert_eq!(super::scale_bits(1000000000), "1.00 gbit/s");}
+}
\ No newline at end of file
diff --git a/src/rust/lqos_utils/src/string_table_enum.rs b/src/rust/lqos_utils/src/string_table_enum.rs
index aa0cde90..f7849aed 100644
--- a/src/rust/lqos_utils/src/string_table_enum.rs
+++ b/src/rust/lqos_utils/src/string_table_enum.rs
@@ -1,3 +1,9 @@
+/// Helper macro to create an enum that can be serialized to a string
+/// and deserialized from a string.
+/// 
+/// ## Parameters
+/// * `$enum_name`: the name of the enum to create
+/// * `$($option:ident),*`: the options of the enum
 #[macro_export]
 macro_rules! string_table_enum {
     ($enum_name: ident, $($option:ident),*) => {
@@ -36,6 +42,13 @@ macro_rules! string_table_enum {
     };
 }
 
+/// Helper macro to create an enum that can be serialized to a string
+/// and deserialized from a string. Adds explicit support for dashes
+/// in identifiers.
+/// 
+/// ## Parameters
+/// * `$enum_name`: the name of the enum to create
+/// * `$($option:ident),*`: the options of the enum
 #[macro_export]
 macro_rules! dashy_table_enum {
     ($enum_name: ident, $($option:ident),*) => {
diff --git a/src/rust/lqos_utils/src/unix_time.rs b/src/rust/lqos_utils/src/unix_time.rs
index a60e714e..33eee4a4 100644
--- a/src/rust/lqos_utils/src/unix_time.rs
+++ b/src/rust/lqos_utils/src/unix_time.rs
@@ -32,8 +32,10 @@ pub fn time_since_boot() -> Result<TimeSpec, TimeError> {
   }
 }
 
+/// Error type for time functions.
 #[derive(Error, Debug)]
 pub enum TimeError {
+  /// The clock isn't ready yet.
   #[error("Clock not ready")]
   ClockNotReady,
 }

From bcb4a8a53ae3a362177655ec90e11417ed61510a Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 16:58:19 +0000
Subject: [PATCH 28/67] Add documentation headers for Heimdall.

---
 src/rust/lqos_heimdall/src/flows.rs          |  2 ++
 src/rust/lqos_heimdall/src/lib.rs            |  3 ++-
 src/rust/lqos_heimdall/src/perf_interface.rs | 16 ++++++++++++++++
 src/rust/lqos_heimdall/src/timeline.rs       | 15 +++++++++++++++
 4 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/src/rust/lqos_heimdall/src/flows.rs b/src/rust/lqos_heimdall/src/flows.rs
index 41a2b364..c88313e2 100644
--- a/src/rust/lqos_heimdall/src/flows.rs
+++ b/src/rust/lqos_heimdall/src/flows.rs
@@ -133,6 +133,7 @@ pub fn read_flows() {
   });
 }
 
+/// Expire flows that have not been seen in a while.
 pub fn expire_heimdall_flows() {
   if let Ok(now) = time_since_boot() {
     let since_boot = Duration::from(now);
@@ -142,6 +143,7 @@ pub fn expire_heimdall_flows() {
   }
 }
 
+/// Get the flow stats for a given IP address.
 pub fn get_flow_stats(ip: XdpIpAddress) -> BusResponse {
   let mut result = Vec::new();
 
diff --git a/src/rust/lqos_heimdall/src/lib.rs b/src/rust/lqos_heimdall/src/lib.rs
index 267af1f8..5531858b 100644
--- a/src/rust/lqos_heimdall/src/lib.rs
+++ b/src/rust/lqos_heimdall/src/lib.rs
@@ -1,8 +1,9 @@
 //! Provides an interface to the Heimdall packet watching
 //! system. Heimdall watches traffic flows, and is notified
 //! about their contents via the eBPF Perf system.
-
+#![warn(missing_docs)]
 mod config;
+/// Interface to the performance tracking system
 pub mod perf_interface;
 pub mod stats;
 pub use config::{HeimdalConfig, HeimdallMode};
diff --git a/src/rust/lqos_heimdall/src/perf_interface.rs b/src/rust/lqos_heimdall/src/perf_interface.rs
index 49243019..81fb76b3 100644
--- a/src/rust/lqos_heimdall/src/perf_interface.rs
+++ b/src/rust/lqos_heimdall/src/perf_interface.rs
@@ -6,21 +6,37 @@ use crate::timeline::store_on_timeline;
 /// This constant MUST exactly match PACKET_OCTET_STATE in heimdall.h
 pub(crate) const PACKET_OCTET_SIZE: usize = 128;
 
+/// A representation of the eBPF `heimdall_event` type.
+/// This is the type that is sent from the eBPF program to userspace.
+/// It is a representation of the `heimdall_event` type in heimdall.h
 #[derive(FromBytes, Debug, Clone, PartialEq, Eq, Hash)]
 #[repr(C)]
 pub struct HeimdallEvent {
+  /// Timestamp of the event, in nanoseconds since boot time.
   pub timestamp: u64,
+  /// Source IP address
   pub src: XdpIpAddress,
+  /// Destination IP address
   pub dst: XdpIpAddress,
+  /// Source port number, or ICMP type.
   pub src_port : u16,
+  /// Destination port number.
   pub dst_port: u16,
+  /// IP protocol number
   pub ip_protocol: u8,
+  /// IP header TOS value
   pub tos: u8,
+  /// Total size of the packet, in bytes
   pub size: u32,
+  /// TCP flags
   pub tcp_flags: u8,
+  /// TCP window size
   pub tcp_window: u16,
+  /// TCP sequence number
   pub tcp_tsval: u32,
+  /// TCP acknowledgement number
   pub tcp_tsecr: u32,
+  /// Raw packet data
   pub packet_data: [u8; PACKET_OCTET_SIZE],
 }
 
diff --git a/src/rust/lqos_heimdall/src/timeline.rs b/src/rust/lqos_heimdall/src/timeline.rs
index 1be06db6..3bd2d066 100644
--- a/src/rust/lqos_heimdall/src/timeline.rs
+++ b/src/rust/lqos_heimdall/src/timeline.rs
@@ -150,6 +150,13 @@ pub fn hyperfocus_on_target(ip: XdpIpAddress) -> Option<(usize, usize)> {
   }
 }
 
+/// Request a dump of the packet headers collected during a hyperfocus session.
+/// This will return `None` if the session id is invalid or the session has
+/// expired.
+/// ## Returns
+/// * Either `None` or a vector of packet headers.
+/// ## Arguments
+/// * `session_id` - The session id of the hyperfocus session.
 pub fn n_second_packet_dump(session_id: usize) -> Option<Vec<PacketHeader>> {
   if let Some(session) = FOCUS_SESSIONS.get(&session_id) {
     Some(session.data.iter().map(|e| e.as_header()).collect())
@@ -158,6 +165,14 @@ pub fn n_second_packet_dump(session_id: usize) -> Option<Vec<PacketHeader>> {
   }
 }
 
+/// Request a dump of the packet headers collected during a hyperfocus session,
+/// in LibPCAP format. This will return `None` if the session id is invalid or
+/// the session has expired, or the temporary filename used to store the dump
+/// if it is available.
+/// ## Returns
+/// * Either `None` or the filename of the dump.
+/// ## Arguments
+/// * `session_id` - The session id of the hyperfocus session.
 pub fn n_second_pcap(session_id: usize) -> Option<String> {
   if let Some(mut session) = FOCUS_SESSIONS.get_mut(&session_id) {
     let filename = format!("/tmp/cap_sess_{session_id}");

From 415f939715eea6fdd0686ba859881c2312749153 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 19:21:39 +0000
Subject: [PATCH 29/67] Add documentation headers for Queue Tracker.

---
 src/rust/lqos_queue_tracker/src/bus.rs               |  4 ++++
 src/rust/lqos_queue_tracker/src/interval.rs          |  6 ++++++
 src/rust/lqos_queue_tracker/src/lib.rs               |  7 +++++++
 src/rust/lqos_queue_tracker/src/tracking/mod.rs      |  3 +++
 .../src/tracking/watched_queues.rs                   | 12 ++++++++++++
 5 files changed, 32 insertions(+)

diff --git a/src/rust/lqos_queue_tracker/src/bus.rs b/src/rust/lqos_queue_tracker/src/bus.rs
index 8904989f..bbbb07c6 100644
--- a/src/rust/lqos_queue_tracker/src/bus.rs
+++ b/src/rust/lqos_queue_tracker/src/bus.rs
@@ -3,6 +3,10 @@ use crate::{
 };
 use lqos_bus::BusResponse;
 
+/// Retrieves the raw queue data for a given circuit ID.
+/// 
+/// # Arguments
+/// * `circuit_id` - The circuit ID to retrieve data for.
 pub fn get_raw_circuit_data(circuit_id: &str) -> BusResponse {
   still_watching(circuit_id);
   if let Some(circuit) = CIRCUIT_TO_QUEUE.get(circuit_id) {
diff --git a/src/rust/lqos_queue_tracker/src/interval.rs b/src/rust/lqos_queue_tracker/src/interval.rs
index 126d8622..f5327bf8 100644
--- a/src/rust/lqos_queue_tracker/src/interval.rs
+++ b/src/rust/lqos_queue_tracker/src/interval.rs
@@ -2,6 +2,12 @@ use std::sync::atomic::AtomicU64;
 
 pub(crate) static QUEUE_MONITOR_INTERVAL: AtomicU64 = AtomicU64::new(1000);
 
+/// Sets the interval at which the queue monitor thread will poll the
+/// Linux `tc` shaper for queue statistics.
+/// 
+/// # Arguments
+/// * `interval_ms` - The interval, in milliseconds, at which the queue
+///   monitor thread will poll the Linux `tc` shaper for queue statistics.
 pub fn set_queue_refresh_interval(interval_ms: u64) {
   QUEUE_MONITOR_INTERVAL
     .store(interval_ms, std::sync::atomic::Ordering::Relaxed);
diff --git a/src/rust/lqos_queue_tracker/src/lib.rs b/src/rust/lqos_queue_tracker/src/lib.rs
index 31c01050..e5a14f25 100644
--- a/src/rust/lqos_queue_tracker/src/lib.rs
+++ b/src/rust/lqos_queue_tracker/src/lib.rs
@@ -1,3 +1,10 @@
+//! Retrieves queue statistics from the Linux `tc` shaper, and stores
+//! them in a `QueueStore` for later retrieval. The `QueueStore` is
+//! thread-safe, and can be accessed from multiple threads. It is
+//! updated periodically by a separate thread, and accumulates statistics
+//! between polling periods.
+
+#![warn(missing_docs)]
 mod bus;
 mod circuit_to_queue;
 mod interval;
diff --git a/src/rust/lqos_queue_tracker/src/tracking/mod.rs b/src/rust/lqos_queue_tracker/src/tracking/mod.rs
index 93a13fcb..952e8165 100644
--- a/src/rust/lqos_queue_tracker/src/tracking/mod.rs
+++ b/src/rust/lqos_queue_tracker/src/tracking/mod.rs
@@ -76,6 +76,9 @@ fn track_queues() {
   expire_watched_queues();
 }
 
+/// Spawns a thread that periodically reads the queue statistics from
+/// the Linux `tc` shaper, and stores them in a `QueueStore` for later
+/// retrieval.
 pub fn spawn_queue_monitor() {
   std::thread::spawn(|| {
     // Setup the queue monitor loop
diff --git a/src/rust/lqos_queue_tracker/src/tracking/watched_queues.rs b/src/rust/lqos_queue_tracker/src/tracking/watched_queues.rs
index 3c6d49aa..2196c880 100644
--- a/src/rust/lqos_queue_tracker/src/tracking/watched_queues.rs
+++ b/src/rust/lqos_queue_tracker/src/tracking/watched_queues.rs
@@ -31,6 +31,13 @@ pub fn expiration_in_the_future() -> u64 {
   unix_now().unwrap_or(0) + 10
 }
 
+/// Start watching a queue. This will cause the queue to be read
+/// periodically, and its statistics stored in the `QueueStore`.
+/// If the queue is already being watched, this function will
+/// do nothing.
+/// 
+/// # Arguments
+/// * `circuit_id` - The circuit ID to watch
 pub fn add_watched_queue(circuit_id: &str) {
   //info!("Watching queue {circuit_id}");
   let max = num_possible_cpus().unwrap() * 2;
@@ -74,6 +81,11 @@ pub(crate) fn expire_watched_queues() {
   WATCHED_QUEUES.retain(|_,w| w.expires_unix_time > now);
 }
 
+/// Indicates that a watched queue is still being watched. Update the
+/// expiration time for the queue.
+/// 
+/// # Arguments
+/// * `circuit_id` - The circuit ID to watch
 pub fn still_watching(circuit_id: &str) {
   if let Some(mut q) = WATCHED_QUEUES.get_mut(circuit_id) {
     //info!("Still watching circuit: {circuit_id}");

From 5f9655b529a08f4297968e850208630f202c93ae Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 20:16:57 +0000
Subject: [PATCH 30/67] Add a couple of 'is none' checks to remove crashes I
 experienced.

---
 src/integrationCommon.py | 12 ++++++++----
 src/integrationUISP.py   |  8 ++++++--
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/integrationCommon.py b/src/integrationCommon.py
index 33f20e31..122fc3ef 100644
--- a/src/integrationCommon.py
+++ b/src/integrationCommon.py
@@ -383,6 +383,10 @@ class NetworkGraph:
 						#Remove brackets and quotes of list so LibreQoS.py can parse it
 						device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'')
 						device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'')
+						if circuit["upload"] is None: 
+							circuit["upload"] = 0.0
+						if circuit["download"] is None: 
+							circuit["download"] = 0.0
 						row = [
 							circuit["id"],
 							circuit["name"],
@@ -392,10 +396,10 @@ class NetworkGraph:
 							device["mac"],
 							device["ipv4"],
 							device["ipv6"],
-							int(circuit["download"] * 0.98),
-							int(circuit["upload"] * 0.98),
-							int(circuit["download"] * bandwidthOverheadFactor),
-							int(circuit["upload"] * bandwidthOverheadFactor),
+							int(float(circuit["download"]) * 0.98),
+							int(float(circuit["upload"]) * 0.98),
+							int(float(circuit["download"]) * bandwidthOverheadFactor),
+							int(float(circuit["upload"]) * bandwidthOverheadFactor),
 							""
 						]
 						wr.writerow(row)
diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 36937fb7..208594a7 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -118,8 +118,12 @@ def buildFullGraph():
 			if device['identification']['role'] == "station":
 				if device['identification']['type'] == "airFiber":
 					if device['overview']['status'] == 'active':
-						download = int(device['overview']['downlinkCapacity']/ 1000000)
-						upload = int(device['overview']['uplinkCapacity']/ 1000000)
+						if device['overview']['downlinkCapacity'] is not None and device['overview']['uplinkCapacity'] is not None:
+							download = int(device['overview']['downlinkCapacity']/ 1000000)
+							upload = int(device['overview']['uplinkCapacity']/ 1000000)
+						else:
+							download = generatedPNDownloadMbps
+							upload = generatedPNUploadMbps
 						# Make sure to use half of reported bandwidth for AF60-LRs
 						if device['identification']['model'] == "AF60-LR":
 							download = int(download / 2)

From d33cd0d3e65b71a662f50d5ba72cd09b335d1368 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 20:34:32 +0000
Subject: [PATCH 31/67] Use dot renderer for PDF

---
 src/integrationCommon.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/integrationCommon.py b/src/integrationCommon.py
index 122fc3ef..6e89c3c9 100644
--- a/src/integrationCommon.py
+++ b/src/integrationCommon.py
@@ -418,7 +418,7 @@ class NetworkGraph:
 
 		import graphviz
 		dot = graphviz.Digraph(
-			'network', comment="Network Graph", engine="fdp")
+			'network', comment="Network Graph", engine="dot")
 
 		for (i, node) in enumerate(self.nodes):
 			if ((node.type != NodeType.client and node.type != NodeType.device) or showClients):

From ff4df61d543c5972382407ea5f89206a8e9aa100 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Thu, 6 Apr 2023 16:22:01 +0000
Subject: [PATCH 32/67] Build complete spanning tree for data links and use it
 to orient tree

Rather than obeying the "parent" field, which leads to wrong-way up
graphs when building from a non-root site - instead build a complete
cost-based spanning tree from the specified root node. Then use
the generated tree - falling back on parent if nothing is found -
to populate the tree in order.

Tested on Herbert's network, correctly generates topology from several
different locations.

Still to come: adding a mechanism for providing parenting overloads
to allow for cases where a long path is actually optimal, but we
have no way of knowing that.
---
 src/integrationUISP.py | 152 +++++++++++++++++++++++++++++++++++------
 1 file changed, 133 insertions(+), 19 deletions(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 208594a7..bdc31cfc 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -76,19 +76,36 @@ def buildFlatGraph():
 		net.createNetworkJson()
 	net.createShapedDevices()
 
-def buildFullGraph():
-	# Attempts to build a full network graph, incorporating as much of the UISP
-	# hierarchy as possible.
-	from integrationCommon import NetworkGraph, NetworkNode, NodeType
-	from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps
+def linkSiteTarget(link, direction):
+	# Helper function to extract the site ID from a data link. Returns
+	# None if not present.
+	if link[direction]['site'] is not None:
+		return link[direction]['site']['identification']['id']
+	
+	return None
 
-	# Load network sites
-	print("Loading Data from UISP")
-	sites = uispRequest("sites")
-	devices = uispRequest("devices?withInterfaces=true&authorized=true")
-	dataLinks = uispRequest("data-links?siteLinksOnly=true")
+def findSiteLinks(dataLinks, siteId):
+	# Searches the Data Links for any links to/from the specified site.
+	# Returns a list of site IDs that are linked to the specified site.
+	links = []
+	for dl in dataLinks:
+		fromSiteId = linkSiteTarget(dl, "from")
+		if fromSiteId is not None and fromSiteId == siteId:
+			# We have a link originating in this site.
+			target = linkSiteTarget(dl, "to")
+			if target is not None:
+				links.append(target)
 
-	# Do we already have a integrationUISPbandwidths.csv file?
+		toSiteId = linkSiteTarget(dl, "to")
+		if toSiteId is not None and toSiteId == siteId:
+			# We have a link originating in this site.
+			target = linkSiteTarget(dl, "from")
+			if target is not None:
+				links.append(target)
+	return links
+
+def buildSiteBandwidths():
+	# Builds a dictionary of site bandwidths from the integrationUISPbandwidths.csv file.
 	siteBandwidth = {}
 	if os.path.isfile("integrationUISPbandwidths.csv"):
 		with open('integrationUISPbandwidths.csv') as csv_file:
@@ -99,8 +116,10 @@ def buildFullGraph():
 				download = int(download)
 				upload = int(upload)
 				siteBandwidth[name] = {"download": download, "upload": upload}
-	
-	# Find AP capacities from UISP
+	return siteBandwidth
+
+def findApCapacities(devices, siteBandwidth):
+	# Searches the UISP devices for APs and adds their capacities to the siteBandwidth dictionary.
 	for device in devices:
 		if device['identification']['role'] == "ap":
 			name = device['identification']['name']
@@ -110,8 +129,8 @@ def buildFullGraph():
 				upload = int(device['overview']['uplinkCapacity'] / 1000000)
 				siteBandwidth[device['identification']['name']] = {
 					"download": download, "upload": upload}
-	
-	# Find Site Capacities by AirFiber capacities
+
+def findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps):
 	foundAirFibersBySite = {}
 	for device in devices:
 		if device['identification']['site']['type'] == 'site':
@@ -134,7 +153,100 @@ def buildFullGraph():
 								foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
 						else:
 							foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload}
+	return foundAirFibersBySite
+
+def buildSiteList(sites, dataLinks):
+	# Builds a list of sites, including their IDs, names, and connections.
+	# Connections are determined by the dataLinks list.
+	siteList = []
+	for site in sites:
+		newSite = {
+			'id': site['identification']['id'], 
+			'name': site['identification']['name'],
+			'connections': findSiteLinks(dataLinks, site['identification']['id']),
+			'cost': 10000,
+			'parent': "",
+			'type': type,
+		}
+		siteList.append(newSite)
+	return siteList
+
+def findInSiteList(siteList, name):
+	# Searches the siteList for a site with the specified name.
+	for site in siteList:
+		if site['name'] == name:
+			return site
+	return None
+
+def findInSiteListById(siteList, id):
+	# Searches the siteList for a site with the specified name.
+	for site in siteList:
+		if site['id'] == id:
+			return site
+	return None
+
+def debugSpaces(n):
+	# Helper function to print n spaces.
+	spaces = ""
+	for i in range(int(n)):
+		spaces = spaces + " "
+	return spaces
+
+def walkGraphOutwards(siteList, root):
+	def walkGraph(node, parent, cost, backPath):
+		site = findInSiteListById(siteList, node)
+		if cost < site['cost']:
+			# It's cheaper to get here this way, so update the cost and parent.
+			site['cost'] = cost
+			site['parent'] = parent['id']
+			#print(debugSpaces(cost/10) + parent['name'] + "->" + site['name'] + " -> New cost: " + str(cost))
+
+		for connection in site['connections']:
+			if not connection in backPath:
+				#target = findInSiteListById(siteList, connection)
+				#print(debugSpaces((cost+10)/10) + site['name'] + " -> " + target['name'] + " (" + str(target['cost']) + ")")
+				newBackPath = backPath.copy()
+				newBackPath.append(site['id'])
+				walkGraph(connection, site, cost+10, newBackPath)
+
+	for connection in root['connections']:
+		# Force the parent since we're at the top
+		site = findInSiteListById(siteList, connection)
+		site['parent'] = root['id']
+		walkGraph(connection, root, 20, [root['id']])
+
+def buildFullGraph():
+	# Attempts to build a full network graph, incorporating as much of the UISP
+	# hierarchy as possible.
+	from integrationCommon import NetworkGraph, NetworkNode, NodeType
+	from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps
+
+	# Load network sites
+	print("Loading Data from UISP")
+	sites = uispRequest("sites")
+	devices = uispRequest("devices?withInterfaces=true&authorized=true")
+	dataLinks = uispRequest("data-links?siteLinksOnly=true")
+
+	# Build Site Capacities
+	siteBandwidth = buildSiteBandwidths()
+	findApCapacities(devices, siteBandwidth)
+	foundAirFibersBySite = findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps)
 	
+	# Create a list of just network sites
+	siteList = buildSiteList(sites, dataLinks)
+	rootSite = findInSiteList(siteList, uispSite)
+	if rootSite is None:
+		print("ERROR: Unable to find root site in UISP")
+		return
+	walkGraphOutwards(siteList, rootSite)
+	# Debug code: dump the list of site parents
+	# for s in siteList:
+	# 	if s['parent'] == "":
+	# 		p = "None"
+	# 	else:
+	# 		p = findInSiteListById(siteList, s['parent'])['name']
+	# 		print(s['name'] + " (" + str(s['cost']) + ") <-- " + p)
+
 	print("Building Topology")
 	net = NetworkGraph()
 	# Add all sites and client sites
@@ -146,10 +258,12 @@ def buildFullGraph():
 		upload = generatedPNUploadMbps
 		address = ""
 		customerName = ""
-		if site['identification']['parent'] is None:
-			parent = ""
-		else:
-			parent = site['identification']['parent']['id']
+		parent = findInSiteListById(siteList, id)['parent']
+		if parent == "":
+			if site['identification']['parent'] is None:
+				parent = ""
+			else:
+				parent = site['identification']['parent']['id']
 		match type:
 			case "site":
 				nodeType = NodeType.site

From cb63968e81211e5de78810fee98e44d75b9a2fae Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 11 Apr 2023 15:30:47 +0000
Subject: [PATCH 33/67] Allow integrationUISProutes.csv to override default
 costs in UISP route determination.

Requested by D. Denson.

Now that the UISP network tree is built as a spanning tree, with 10
cost per hop we can reliably flip the tree from A->B->C to C->B->A
depending upon root positioning.

This addition allows you to specify additional routes (that MUST
exist!) e.g. A->C and specify a cost to use instead of the default
10. This allows for topologies in which A-B-C is actually faster
than a direct A-C route (for example, becuase of short 60ghz
hops).
---
 src/integrationUISP.py        | 25 ++++++++++++++++++++++---
 src/integrationUISProutes.csv |  5 +++++
 2 files changed, 27 insertions(+), 3 deletions(-)
 create mode 100644 src/integrationUISProutes.csv

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index bdc31cfc..55fb3411 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -192,9 +192,14 @@ def debugSpaces(n):
 		spaces = spaces + " "
 	return spaces
 
-def walkGraphOutwards(siteList, root):
+def walkGraphOutwards(siteList, root, routeOverrides):
 	def walkGraph(node, parent, cost, backPath):
 		site = findInSiteListById(siteList, node)
+		routeName = parent['name'] + "->" + site['name']
+		if routeName in routeOverrides:
+			# We have an overridden cost for this route, so use it instead
+			#print("--> Using overridden cost for " + routeName + ". New cost: " + str(routeOverrides[routeName]) + ".")
+			cost = routeOverrides[routeName]
 		if cost < site['cost']:
 			# It's cheaper to get here this way, so update the cost and parent.
 			site['cost'] = cost
@@ -213,7 +218,20 @@ def walkGraphOutwards(siteList, root):
 		# Force the parent since we're at the top
 		site = findInSiteListById(siteList, connection)
 		site['parent'] = root['id']
-		walkGraph(connection, root, 20, [root['id']])
+		walkGraph(connection, root, 10, [root['id']])
+
+def loadRoutingOverrides():
+	# Loads integrationUISProutes.csv and returns a set of "from", "to", "cost"
+	overrides = {}
+	if os.path.isfile("integrationUISProutes.csv"):
+		with open("integrationUISProutes.csv", "r") as f:
+			reader = csv.reader(f)
+			for row in reader:
+				if not row[0].startswith("#") and len(row) == 3:
+					fromSite, toSite, cost = row
+					overrides[fromSite.strip() + "->" + toSite.strip()] = int(cost)
+	#print(overrides)
+	return overrides
 
 def buildFullGraph():
 	# Attempts to build a full network graph, incorporating as much of the UISP
@@ -235,10 +253,11 @@ def buildFullGraph():
 	# Create a list of just network sites
 	siteList = buildSiteList(sites, dataLinks)
 	rootSite = findInSiteList(siteList, uispSite)
+	routeOverrides = loadRoutingOverrides()
 	if rootSite is None:
 		print("ERROR: Unable to find root site in UISP")
 		return
-	walkGraphOutwards(siteList, rootSite)
+	walkGraphOutwards(siteList, rootSite, routeOverrides)
 	# Debug code: dump the list of site parents
 	# for s in siteList:
 	# 	if s['parent'] == "":
diff --git a/src/integrationUISProutes.csv b/src/integrationUISProutes.csv
new file mode 100644
index 00000000..643c4eb0
--- /dev/null
+++ b/src/integrationUISProutes.csv
@@ -0,0 +1,5 @@
+# Allows you to override route costs in the UISP integration, to better
+# represent your network. Costs by default increment 10 at each hop.
+# So if you want to skip 10 links, put a cost of 100 in.
+# From Site Name, To Site Name, New Cost
+# MYCORE, MYTOWER, 100

From 19928a5e5e605c23a9922e60cc09ef21e8018482 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Tue, 11 Apr 2023 10:38:00 -0600
Subject: [PATCH 34/67] Update ispConfig.example.py

---
 src/ispConfig.example.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/ispConfig.example.py b/src/ispConfig.example.py
index cbca50ff..9da3813c 100644
--- a/src/ispConfig.example.py
+++ b/src/ispConfig.example.py
@@ -63,6 +63,9 @@ influxDBtoken = ""
 # Use Customer Name or Address as Circuit Name
 circuitNameUseAddress = True
 
+# Should integrationUISP overwrite network.json on each run?
+overwriteNetworkJSONalways = False
+
 # If a device shows a WAN IP within these subnets, assume they are behind NAT / un-shapable, and ignore them
 ignoreSubnets = ['192.168.0.0/16']
 allowedSubnets = ['100.64.0.0/10']

From ddc30894cf6397e53c7a2ba06a2d5cfc97e57997 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Tue, 11 Apr 2023 10:39:30 -0600
Subject: [PATCH 35/67] Update integrationUISP.py

---
 src/integrationUISP.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 55fb3411..9c73e745 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -3,8 +3,12 @@ checkPythonVersion()
 import requests
 import os
 import csv
-from ispConfig import uispSite, uispStrategy
 from integrationCommon import isIpv4Permitted, fixSubnet
+try:
+	from ispConfig import uispSite, uispStrategy, overwriteNetworkJSONalways
+except:
+	from ispConfig import uispSite, uispStrategy
+	overwriteNetworkJSONalways = False
 
 def uispRequest(target):
 	# Sends an HTTP request to UISP and returns the
@@ -71,7 +75,10 @@ def buildFlatGraph():
 	net.prepareTree()
 	net.plotNetworkGraph(False)
 	if net.doesNetworkJsonExist():
-		print("network.json already exists. Leaving in-place.")
+		if overwriteNetworkJSONalways:
+			net.createNetworkJson()
+		else:
+			print("network.json already exists and overwriteNetworkJSONalways set to False. Leaving in-place.")
 	else:
 		net.createNetworkJson()
 	net.createShapedDevices()
@@ -362,7 +369,10 @@ def buildFullGraph():
 	net.prepareTree()
 	net.plotNetworkGraph(False)
 	if net.doesNetworkJsonExist():
-		print("network.json already exists. Leaving in-place.")
+		if overwriteNetworkJSONalways:
+			net.createNetworkJson()
+		else:
+			print("network.json already exists and overwriteNetworkJSONalways set to False. Leaving in-place.")
 	else:
 		net.createNetworkJson()
 	net.createShapedDevices()

From b00f99beec20eabff1fd1cff777759b0c1567628 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Tue, 11 Apr 2023 15:30:28 -0600
Subject: [PATCH 36/67] Update integrationUISP.py

---
 src/integrationUISP.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 9c73e745..dce7eda7 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -272,7 +272,21 @@ def buildFullGraph():
 	# 	else:
 	# 		p = findInSiteListById(siteList, s['parent'])['name']
 	# 		print(s['name'] + " (" + str(s['cost']) + ") <-- " + p)
-
+	
+	# Find Nodes Connected By PtMP
+	nodeOffPtMP = {}
+	for site in sites:
+		id = site['identification']['id']
+		name = site['identification']['name']
+		type = site['identification']['type']
+		parent = findInSiteListById(siteList, id)['parent']
+		if type == 'site':
+			for link in dataLinks:
+				if link['from']['site']['identification']['id'] == parent:
+					if link['to']['site']['identification']['id'] == id:
+						if link['from']['device']['overview']['wirelessMode'] == 'ap-ptmp':
+							nodeOffPtMP[id] = link['from']['device']['identification']['id']
+	
 	print("Building Topology")
 	net = NetworkGraph()
 	# Add all sites and client sites
@@ -293,6 +307,8 @@ def buildFullGraph():
 		match type:
 			case "site":
 				nodeType = NodeType.site
+				if id in nodeOffPtMP:
+					parent = nodeOffPtMP[id]
 				if name in siteBandwidth:
 					# Use the CSV bandwidth values
 					download = siteBandwidth[name]["download"]
@@ -365,7 +381,7 @@ def buildFullGraph():
 									# Add some defaults in case they want to change them
 									siteBandwidth[node.displayName] = {
 										"download": generatedPNDownloadMbps, "upload": generatedPNUploadMbps}
-
+	
 	net.prepareTree()
 	net.plotNetworkGraph(False)
 	if net.doesNetworkJsonExist():

From f3bcf614754779e230e375233c83ab5defc1c185 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Tue, 11 Apr 2023 16:13:26 -0600
Subject: [PATCH 37/67] Update integrationUISP.py

---
 src/integrationUISP.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index dce7eda7..543ce734 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -285,8 +285,13 @@ def buildFullGraph():
 				if link['from']['site']['identification']['id'] == parent:
 					if link['to']['site']['identification']['id'] == id:
 						if link['from']['device']['overview']['wirelessMode'] == 'ap-ptmp':
-							nodeOffPtMP[id] = link['from']['device']['identification']['id']
-	
+							# Capacity of the PtMP client radio feeding the PoP will be used as the site bandwidth limit
+							download = int(round(link['to']['device']['overview']['downlinkCapacity']/1000000))
+							upload = int(round(link['to']['device']['overview']['uplinkCapacity']/1000000))
+							nodeOffPtMP[id] = { 'parent': link['from']['device']['identification']['id'],
+												'download': download,
+												'upload': upload
+												}
 	print("Building Topology")
 	net = NetworkGraph()
 	# Add all sites and client sites
@@ -308,11 +313,14 @@ def buildFullGraph():
 			case "site":
 				nodeType = NodeType.site
 				if id in nodeOffPtMP:
-					parent = nodeOffPtMP[id]
+					parent = nodeOffPtMP[id]['parent']
 				if name in siteBandwidth:
 					# Use the CSV bandwidth values
 					download = siteBandwidth[name]["download"]
 					upload = siteBandwidth[name]["upload"]
+				elif id in nodeOffPtMP:
+					download = nodeOffPtMP[id]['download']
+					upload = nodeOffPtMP[id]['upload']
 				elif id in foundAirFibersBySite:
 					download = foundAirFibersBySite[id]['download']
 					upload = foundAirFibersBySite[id]['upload']

From da0489b638bc8f80a4b05a4f889921b344ec74c7 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Mon, 27 Mar 2023 16:21:46 +0000
Subject: [PATCH 38/67] IP Dump supports flow filtering

Every packet is assigned a "flow id", and the UI allows you to
filter displayed packet data by flow id.
---
 .../lqos_node_manager/static/ip_dump.html     | 36 ++-----------------
 1 file changed, 3 insertions(+), 33 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/ip_dump.html b/src/rust/lqos_node_manager/static/ip_dump.html
index 14fed0fe..56e51f37 100644
--- a/src/rust/lqos_node_manager/static/ip_dump.html
+++ b/src/rust/lqos_node_manager/static/ip_dump.html
@@ -80,8 +80,6 @@
         var capacity = [];
         var activeFilter = null;
         var activeSet = null;
-        var activeChart = 0;
-        var activePage = 0;
 
         function filter(newFilter) {
             activeFilter = newFilter;
@@ -95,12 +93,6 @@
             viewPage(0);
         }
 
-        function setChart(n) {
-            activeChart = n;
-            paginator(activePage);
-            viewPage(activePage);
-        }
-
         function proto(n) {
             switch (n) {
                 case 6: return "TCP"
@@ -165,7 +157,7 @@ if (hdr->cwr) flags |= 128;
                     paginator += "<option onclick='viewPage(" + i + ");'>" + i + "</option> ";
                 }
             }
-            paginator += "</select> | ";
+            paginator += "</select><br />";
 
             // Offer flow filtering
             paginator += "<strong>Filter Flows</strong>: ";
@@ -182,28 +174,7 @@ if (hdr->cwr) flags |= 128;
                     paginator += "<option onclick='filter(\"" + key + "\");'>" + key + "</option>";
                 }
             });
-            paginator += "</select> | ";
-
-            // Offer graph choices
-            paginator += "<strong>Graph</strong>: ";
-            paginator += "<select>";
-            if (activeChart == 0) {
-                paginator += "<option selected>Packet-Size Chart</option>";
-            } else {
-                paginator += "<option onclick='setChart(0);'>Packet-Size Chart</option>";
-            }
-            if (activeChart == 1) {
-                paginator += "<option selected>Piano Roll Flow Chart</option>";
-            } else {
-                paginator += "<option onclick='setChart(1);'>Piano Roll Flow Chart</option>";
-            }
-            if (activeChart == 2) {
-                paginator += "<option selected>TCP Window Chart</option>";
-            } else {
-                paginator += "<option onclick='setChart(2);'>TCP Window Chart</option>";
-            }
             paginator += "</select>";
-            paginator += "</div>";
 
             $("#pages").html(paginator);
         }
@@ -343,7 +314,6 @@ if (hdr->cwr) flags |= 128;
                 data.forEach(packet => packet.timestamp -= min_ts);
 
                 // Divide the packets into flows and append the flow_id
-                let flowCounter = 0;
                 data.forEach(packet => {
                     let flow_id = proto(packet.ip_protocol) + " " + packet.src + ":" + packet.src_port + " <-> " + packet.dst + ":" + packet.dst_port;
                     let reverse_flow_id = proto(packet.ip_protocol) + " " + packet.dst + ":" + packet.dst_port + " <-> " + packet.src + ":" + packet.src_port;
@@ -352,11 +322,11 @@ if (hdr->cwr) flags |= 128;
                     } else if (flows.hasOwnProperty(reverse_flow_id)) {
                         packet.flow_id = reverse_flow_id;
                     } else {
-                        flows[flow_id] = { flowCounter };
+                        flows[flow_id] = {};
                         packet.flow_id = flow_id;
-                        flowCounter++;
                     }
                 });
+                console.log(flows);
 
                 packets = data;
                 activeSet = packets;

From 83825e03763acee3f375a01f7948c0f392244697 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Mon, 27 Mar 2023 17:42:50 +0000
Subject: [PATCH 39/67] Packet analysis page has a lot more options

See ISSUE #302

* Filter by FlowID
* Pagination is a select/drop-down
* Filtering is a select/drop-down
* Graph type is selectable from:
   * Packet size (the previous view)
   * Piano Roll (flows plotted by time)
   * TCP Window (differntiated by flow)
---
 .../lqos_node_manager/static/ip_dump.html     | 36 +++++++++++++++++--
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/ip_dump.html b/src/rust/lqos_node_manager/static/ip_dump.html
index 56e51f37..14fed0fe 100644
--- a/src/rust/lqos_node_manager/static/ip_dump.html
+++ b/src/rust/lqos_node_manager/static/ip_dump.html
@@ -80,6 +80,8 @@
         var capacity = [];
         var activeFilter = null;
         var activeSet = null;
+        var activeChart = 0;
+        var activePage = 0;
 
         function filter(newFilter) {
             activeFilter = newFilter;
@@ -93,6 +95,12 @@
             viewPage(0);
         }
 
+        function setChart(n) {
+            activeChart = n;
+            paginator(activePage);
+            viewPage(activePage);
+        }
+
         function proto(n) {
             switch (n) {
                 case 6: return "TCP"
@@ -157,7 +165,7 @@ if (hdr->cwr) flags |= 128;
                     paginator += "<option onclick='viewPage(" + i + ");'>" + i + "</option> ";
                 }
             }
-            paginator += "</select><br />";
+            paginator += "</select> | ";
 
             // Offer flow filtering
             paginator += "<strong>Filter Flows</strong>: ";
@@ -174,7 +182,28 @@ if (hdr->cwr) flags |= 128;
                     paginator += "<option onclick='filter(\"" + key + "\");'>" + key + "</option>";
                 }
             });
+            paginator += "</select> | ";
+
+            // Offer graph choices
+            paginator += "<strong>Graph</strong>: ";
+            paginator += "<select>";
+            if (activeChart == 0) {
+                paginator += "<option selected>Packet-Size Chart</option>";
+            } else {
+                paginator += "<option onclick='setChart(0);'>Packet-Size Chart</option>";
+            }
+            if (activeChart == 1) {
+                paginator += "<option selected>Piano Roll Flow Chart</option>";
+            } else {
+                paginator += "<option onclick='setChart(1);'>Piano Roll Flow Chart</option>";
+            }
+            if (activeChart == 2) {
+                paginator += "<option selected>TCP Window Chart</option>";
+            } else {
+                paginator += "<option onclick='setChart(2);'>TCP Window Chart</option>";
+            }
             paginator += "</select>";
+            paginator += "</div>";
 
             $("#pages").html(paginator);
         }
@@ -314,6 +343,7 @@ if (hdr->cwr) flags |= 128;
                 data.forEach(packet => packet.timestamp -= min_ts);
 
                 // Divide the packets into flows and append the flow_id
+                let flowCounter = 0;
                 data.forEach(packet => {
                     let flow_id = proto(packet.ip_protocol) + " " + packet.src + ":" + packet.src_port + " <-> " + packet.dst + ":" + packet.dst_port;
                     let reverse_flow_id = proto(packet.ip_protocol) + " " + packet.dst + ":" + packet.dst_port + " <-> " + packet.src + ":" + packet.src_port;
@@ -322,11 +352,11 @@ if (hdr->cwr) flags |= 128;
                     } else if (flows.hasOwnProperty(reverse_flow_id)) {
                         packet.flow_id = reverse_flow_id;
                     } else {
-                        flows[flow_id] = {};
+                        flows[flow_id] = { flowCounter };
                         packet.flow_id = flow_id;
+                        flowCounter++;
                     }
                 });
-                console.log(flows);
 
                 packets = data;
                 activeSet = packets;

From ec9028c3b2223b3379de25de7ffa146802a3db9b Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 28 Mar 2023 13:15:53 +0000
Subject: [PATCH 40/67] Change units on backlog graph to packets. FIXES #304

---
 src/rust/lqos_node_manager/static/circuit_queue.html | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 718a5327..6b0e96aa 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -389,14 +389,7 @@
 
                 if (this.backlogPlotted == null) {
                     this.backlogPlotted = true;
-                    Plotly.newPlot(
-                        graph,
-                        graphData,
-                        {
-                            margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 },
-                            yaxis: { automargin: true, title: "Bytes" },
-                            xaxis: { automargin: true, title: "Time since now" }
-                        });
+                    Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Packets" }, xaxis: { automargin: true, title: "Time since now" } });
                 } else {
                     Plotly.redraw(graph, graphData);
                 }

From b8921fb7b391d1da85702839afcc8ff42cd13e10 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 28 Mar 2023 14:13:36 +0000
Subject: [PATCH 41/67] Add pause button and fast/slow modes to circuit_queue
 page

FIXES #303

Adds a pause button to freeze a snapshot of the current circuit
queue page.

Adds a "fast mode" and "slow mode" toggle. Slow mode continues
at a 1s update cadence. Fast mode uses animation frame support
to update at a good speed, at the expense of CPU usage.

Stores the preference in local storage, so it will resume the
same mode next time you visit the page.
---
 src/rust/lqos_node_manager/static/circuit_queue.html | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 6b0e96aa..d678f6c0 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -94,9 +94,8 @@
                         </ul>
                     </div>
                     <div class="col-sm-2">
-                        <a href="#" class="btn btn-small btn-info" id="btnPause"><i class="fa fa-pause"></i> Pause</a>
-                        <a href="#" class="btn btn-small btn-info" id="btnSlow"><i class="fa fa-hourglass"></i> Slow
-                            Mode</a>
+                        <a href="#" class="btn btn-small btn-info" id="btnPause"><i class="fa fa-pause"></i> Pause</a> 
+                        <a href="#" class="btn btn-small btn-info" id="btnSlow"><i class="fa fa-hourglass"></i> Slow Mode</a>
                     </div>
                 </div>
             </div>
@@ -846,14 +845,14 @@
         var slowMode = false;
 
         function showFps() {
-            if (!lastCalledTime) {
+            if(!lastCalledTime) {
                 lastCalledTime = Date.now();
                 fps = 0;
                 return;
             }
-            delta = (Date.now() - lastCalledTime) / 1000;
+            delta = (Date.now() - lastCalledTime)/1000;
             lastCalledTime = Date.now();
-            fps = 1 / delta;
+            fps = 1/delta;
             //$("#fps").text(fps.toFixed(0));
             worstDelta = Math.max(delta, worstDelta);
         }
@@ -864,7 +863,6 @@
                 switch (activeTab) {
                     case "pills-funnel-tab": {
                         getFunnel();
-                        getThroughput();
                     } break;
                     case "pills-flows-tab": {
                         getFlows();

From a34505eca0631f6d34c97f953fe73d9c978c5c48 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 28 Mar 2023 16:07:49 +0000
Subject: [PATCH 42/67] Use base delay (which is sparse delay) rather than
 average delay, and use log y values on the delays chart.

---
 .../static/circuit_queue.html                 | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index d678f6c0..b678d4f7 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -388,7 +388,13 @@
 
                 if (this.backlogPlotted == null) {
                     this.backlogPlotted = true;
-                    Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Packets" }, xaxis: { automargin: true, title: "Time since now" } });
+                    Plotly.newPlot(
+                        graph, 
+                        graphData, 
+                        { 
+                            margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, 
+                            yaxis: { automargin: true, title: "Packets" }, 
+                            xaxis: { automargin: true, title: "Time since now" } });
                 } else {
                     Plotly.redraw(graph, graphData);
                 }
@@ -405,13 +411,11 @@
 
                 if (this.delaysPlotted == null) {
                     Plotly.newPlot(
-                        graph,
-                        graphData,
-                        {
-                            margin: { l: 8, r: 0, b: 0, t: 0, pad: 4 },
-                            yaxis: { automargin: true, title: "log10(ms)", range: [-1.0, 1.0] },
-                            xaxis: { automargin: true, title: "Time since now" }
-                        });
+                        graph, 
+                        graphData, 
+                        { margin: { l: 8, r: 0, b: 0, t: 0, pad: 4 }, 
+                        yaxis: { automargin: true, title: "log10(ms)", range: [-1.0, 1.0] }, 
+                        xaxis: { automargin: true, title: "Time since now" } });
                     this.delaysPlotted = true;
                 } else {
                     Plotly.redraw(graph, graphData);

From da167fb84dd00e817edf6c18a14c30851158fdb2 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 28 Mar 2023 16:21:14 +0000
Subject: [PATCH 43/67] Backlog is in bytes

---
 src/rust/lqos_node_manager/static/circuit_queue.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index b678d4f7..11c32cea 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -393,7 +393,7 @@
                         graphData, 
                         { 
                             margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, 
-                            yaxis: { automargin: true, title: "Packets" }, 
+                            yaxis: { automargin: true, title: "Bytes" }, 
                             xaxis: { automargin: true, title: "Time since now" } });
                 } else {
                     Plotly.redraw(graph, graphData);

From 7797f183c31914c095daf27404dd1beb8aaa7441 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Wed, 29 Mar 2023 09:58:12 -0600
Subject: [PATCH 44/67] When network.json is created by integration, node
 bandwidth will be capped by parent nodes

---
 src/integrationCommon.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/integrationCommon.py b/src/integrationCommon.py
index 33f20e31..52d7b6f2 100644
--- a/src/integrationCommon.py
+++ b/src/integrationCommon.py
@@ -291,14 +291,19 @@ class NetworkGraph:
 
 		del self.__visited
 		
+		# Child nodes inherit bandwidth maximums of parents. We apply this here to avoid bugs when compression is applied with flattenA().
 		def inheritBandwidthMaxes(data, parentMaxDL, parentMaxUL):
 			for node in data:
 				if isinstance(node, str):
 					if (isinstance(data[node], dict)) and (node != 'children'):
+						# Cap based on this node's max bandwidth, or parent node's max bandwidth, whichever is lower
 						data[node]['downloadBandwidthMbps'] = min(int(data[node]['downloadBandwidthMbps']),int(parentMaxDL))
 						data[node]['uploadBandwidthMbps'] = min(int(data[node]['uploadBandwidthMbps']),int(parentMaxUL))
+						# Recursive call this function for children nodes attached to this node
 						if 'children' in data[node]:
+							# We need to keep tabs on the minor counter, because we can't have repeating class IDs. Here, we bring back the minor counter from the recursive function
 							inheritBandwidthMaxes(data[node]['children'], data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps'])
+		# Here is the actual call to the recursive function
 		inheritBandwidthMaxes(topLevelNode, parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps)
 		
 		with open('network.json', 'w') as f:

From 59df418d1b42f632b3dbedf6fdb4eef6824f596c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Wed, 29 Mar 2023 09:59:01 -0600
Subject: [PATCH 45/67] Clean up excessive comments

---
 src/integrationCommon.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/integrationCommon.py b/src/integrationCommon.py
index 52d7b6f2..33f20e31 100644
--- a/src/integrationCommon.py
+++ b/src/integrationCommon.py
@@ -291,19 +291,14 @@ class NetworkGraph:
 
 		del self.__visited
 		
-		# Child nodes inherit bandwidth maximums of parents. We apply this here to avoid bugs when compression is applied with flattenA().
 		def inheritBandwidthMaxes(data, parentMaxDL, parentMaxUL):
 			for node in data:
 				if isinstance(node, str):
 					if (isinstance(data[node], dict)) and (node != 'children'):
-						# Cap based on this node's max bandwidth, or parent node's max bandwidth, whichever is lower
 						data[node]['downloadBandwidthMbps'] = min(int(data[node]['downloadBandwidthMbps']),int(parentMaxDL))
 						data[node]['uploadBandwidthMbps'] = min(int(data[node]['uploadBandwidthMbps']),int(parentMaxUL))
-						# Recursive call this function for children nodes attached to this node
 						if 'children' in data[node]:
-							# We need to keep tabs on the minor counter, because we can't have repeating class IDs. Here, we bring back the minor counter from the recursive function
 							inheritBandwidthMaxes(data[node]['children'], data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps'])
-		# Here is the actual call to the recursive function
 		inheritBandwidthMaxes(topLevelNode, parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps)
 		
 		with open('network.json', 'w') as f:

From c1c0bb23852597a8079c21edc36dd17e857e7c55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Wed, 29 Mar 2023 10:02:01 -0600
Subject: [PATCH 46/67] Use airfiber backhaul capacities to limit site
 bandwidths

---
 src/integrationUISP.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 36937fb7..538c0c32 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -125,7 +125,7 @@ def buildFullGraph():
 							download = int(download / 2)
 							upload = int(download / 2)
 						if device['identification']['site']['id'] in foundAirFibersBySite:
-							if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']):
+							if (download > foundAirFibersBySite['download']) or (upload > foundAirFibersBySite['upload']):
 								foundAirFibersBySite[device['identification']['site']['id']]['download'] = download
 								foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
 						else:

From 3f0ce354547781c24463619491fbb4cd4b75b621 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Thu, 30 Mar 2023 12:19:23 -0600
Subject: [PATCH 47/67] Fix error when multiple airFibers at site

---
 src/integrationUISP.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 538c0c32..36937fb7 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -125,7 +125,7 @@ def buildFullGraph():
 							download = int(download / 2)
 							upload = int(download / 2)
 						if device['identification']['site']['id'] in foundAirFibersBySite:
-							if (download > foundAirFibersBySite['download']) or (upload > foundAirFibersBySite['upload']):
+							if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']):
 								foundAirFibersBySite[device['identification']['site']['id']]['download'] = download
 								foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
 						else:

From 567cde6b7c5f84dcf13d18e0209e033160d121f7 Mon Sep 17 00:00:00 2001
From: Matthew McTague <matthew.mctague@vetta.nz>
Date: Fri, 31 Mar 2023 13:08:47 +1300
Subject: [PATCH 48/67] Check that priorQuery exists before using it

---
 src/graphInfluxDB.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/graphInfluxDB.py b/src/graphInfluxDB.py
index 6aa9c314..b8b59f86 100644
--- a/src/graphInfluxDB.py
+++ b/src/graphInfluxDB.py
@@ -331,11 +331,10 @@ def getCircuitLatencyStats(subscriberCircuits):
 			# If we can't identify RTT this time around, use most recently recorded RTT
 			# None by default, change if found in priorQuery
 			circuit['stats']['sinceLastQuery']['tcpLatency'] = None
-			if 'priorQuery' in circuit['stats']:
-				if circuit['stats']['priorQuery'] != None:
-					if 'priorQuery' in circuit['stats']:
-						if 'tcpLatency' in circuit['stats']['priorQuery']:
-							circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency']
+			if 'priorQuery' in circuit['stats'] and circuit['stats']['priorQuery'] != None:
+				if 'priorQuery' in circuit['stats']:
+					if 'tcpLatency' in circuit['stats']['priorQuery']:
+						circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency']
 
 	return subscriberCircuits
 

From cfaae176d4547b497b95e60bfff5cc45dd60721e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Thu, 30 Mar 2023 18:22:38 -0600
Subject: [PATCH 49/67] Fix bug pertaining to
 https://github.com/LibreQoE/LibreQoS/pull/320

---
 src/graphInfluxDB.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/graphInfluxDB.py b/src/graphInfluxDB.py
index b8b59f86..6aa9c314 100644
--- a/src/graphInfluxDB.py
+++ b/src/graphInfluxDB.py
@@ -331,10 +331,11 @@ def getCircuitLatencyStats(subscriberCircuits):
 			# If we can't identify RTT this time around, use most recently recorded RTT
 			# None by default, change if found in priorQuery
 			circuit['stats']['sinceLastQuery']['tcpLatency'] = None
-			if 'priorQuery' in circuit['stats'] and circuit['stats']['priorQuery'] != None:
-				if 'priorQuery' in circuit['stats']:
-					if 'tcpLatency' in circuit['stats']['priorQuery']:
-						circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency']
+			if 'priorQuery' in circuit['stats']:
+				if circuit['stats']['priorQuery'] != None:
+					if 'priorQuery' in circuit['stats']:
+						if 'tcpLatency' in circuit['stats']['priorQuery']:
+							circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency']
 
 	return subscriberCircuits
 

From 4ed077c6341581a97ccf951fbd0a3573f3f0adae Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Thu, 30 Mar 2023 14:46:30 +0000
Subject: [PATCH 50/67] Last 5 minutes of throughput data are stored
 server-side on node_manager

A server-side ringbuffer retains just the last 5 minutes of throughput.
When the dashboard first loads, it collects all 5 minutes of data,
so nothing appears skipped. Subsequent updates query the latest
value in the buffer.

FIXES #317
---
 src/rust/lqos_node_manager/src/tracker/cache/mod.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/rust/lqos_node_manager/src/tracker/cache/mod.rs b/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
index 2e0a7d89..8829e9f5 100644
--- a/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
+++ b/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
@@ -5,9 +5,7 @@
 mod cpu_ram;
 mod shaped_devices;
 mod throughput;
-mod dns_cache;
 
 pub use cpu_ram::*;
 pub use shaped_devices::*;
 pub use throughput::THROUGHPUT_BUFFER;
-pub use dns_cache::lookup_dns;

From 2c63393356aadd1668024135ff29fca7edee4c83 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 16:20:56 +0000
Subject: [PATCH 51/67] Adds "client throughput" to the queue tree tab of
 circuit_queue

Also moves all the circuit graphs to use the same graphing
system as the other graphs on the plot.

FIXES #315
---
 .../static/circuit_queue.html                 | 37 +++++++++++--------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/src/rust/lqos_node_manager/static/circuit_queue.html b/src/rust/lqos_node_manager/static/circuit_queue.html
index 11c32cea..718a5327 100644
--- a/src/rust/lqos_node_manager/static/circuit_queue.html
+++ b/src/rust/lqos_node_manager/static/circuit_queue.html
@@ -94,8 +94,9 @@
                         </ul>
                     </div>
                     <div class="col-sm-2">
-                        <a href="#" class="btn btn-small btn-info" id="btnPause"><i class="fa fa-pause"></i> Pause</a> 
-                        <a href="#" class="btn btn-small btn-info" id="btnSlow"><i class="fa fa-hourglass"></i> Slow Mode</a>
+                        <a href="#" class="btn btn-small btn-info" id="btnPause"><i class="fa fa-pause"></i> Pause</a>
+                        <a href="#" class="btn btn-small btn-info" id="btnSlow"><i class="fa fa-hourglass"></i> Slow
+                            Mode</a>
                     </div>
                 </div>
             </div>
@@ -389,12 +390,13 @@
                 if (this.backlogPlotted == null) {
                     this.backlogPlotted = true;
                     Plotly.newPlot(
-                        graph, 
-                        graphData, 
-                        { 
-                            margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, 
-                            yaxis: { automargin: true, title: "Bytes" }, 
-                            xaxis: { automargin: true, title: "Time since now" } });
+                        graph,
+                        graphData,
+                        {
+                            margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 },
+                            yaxis: { automargin: true, title: "Bytes" },
+                            xaxis: { automargin: true, title: "Time since now" }
+                        });
                 } else {
                     Plotly.redraw(graph, graphData);
                 }
@@ -411,11 +413,13 @@
 
                 if (this.delaysPlotted == null) {
                     Plotly.newPlot(
-                        graph, 
-                        graphData, 
-                        { margin: { l: 8, r: 0, b: 0, t: 0, pad: 4 }, 
-                        yaxis: { automargin: true, title: "log10(ms)", range: [-1.0, 1.0] }, 
-                        xaxis: { automargin: true, title: "Time since now" } });
+                        graph,
+                        graphData,
+                        {
+                            margin: { l: 8, r: 0, b: 0, t: 0, pad: 4 },
+                            yaxis: { automargin: true, title: "log10(ms)", range: [-1.0, 1.0] },
+                            xaxis: { automargin: true, title: "Time since now" }
+                        });
                     this.delaysPlotted = true;
                 } else {
                     Plotly.redraw(graph, graphData);
@@ -849,14 +853,14 @@
         var slowMode = false;
 
         function showFps() {
-            if(!lastCalledTime) {
+            if (!lastCalledTime) {
                 lastCalledTime = Date.now();
                 fps = 0;
                 return;
             }
-            delta = (Date.now() - lastCalledTime)/1000;
+            delta = (Date.now() - lastCalledTime) / 1000;
             lastCalledTime = Date.now();
-            fps = 1/delta;
+            fps = 1 / delta;
             //$("#fps").text(fps.toFixed(0));
             worstDelta = Math.max(delta, worstDelta);
         }
@@ -867,6 +871,7 @@
                 switch (activeTab) {
                     case "pills-funnel-tab": {
                         getFunnel();
+                        getThroughput();
                     } break;
                     case "pills-flows-tab": {
                         getFlows();

From 04bc7326adad36bed85a3a4f3bc513b3452b5e27 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 17:10:43 +0000
Subject: [PATCH 52/67] Add DNS resolution to displayed flow IP addresses

---
 src/rust/lqos_node_manager/src/tracker/cache/mod.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/rust/lqos_node_manager/src/tracker/cache/mod.rs b/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
index 8829e9f5..2e0a7d89 100644
--- a/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
+++ b/src/rust/lqos_node_manager/src/tracker/cache/mod.rs
@@ -5,7 +5,9 @@
 mod cpu_ram;
 mod shaped_devices;
 mod throughput;
+mod dns_cache;
 
 pub use cpu_ram::*;
 pub use shaped_devices::*;
 pub use throughput::THROUGHPUT_BUFFER;
+pub use dns_cache::lookup_dns;

From fada0a6e92f66d9f0c7fb38561104a79c9dc63f8 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Fri, 31 Mar 2023 14:36:43 +0000
Subject: [PATCH 53/67] Fix displayed plans in Top/Worst N tables

A value was transposed in the IPStats to IPStats transit
conversion. The UI now displays (max download/max upload)
correctly.

Fix an accidental inclusion of a console.log statement
logging needlessly.
---
 src/rust/lqos_node_manager/src/tracker/mod.rs | 2 +-
 src/rust/lqos_node_manager/static/main.html   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rust/lqos_node_manager/src/tracker/mod.rs b/src/rust/lqos_node_manager/src/tracker/mod.rs
index 67cdf69c..6795c2de 100644
--- a/src/rust/lqos_node_manager/src/tracker/mod.rs
+++ b/src/rust/lqos_node_manager/src/tracker/mod.rs
@@ -50,7 +50,7 @@ impl From<&IpStats> for IpStatsWithPlan {
           &circuit.circuit_name
         };
         result.ip_address = format!("{} ({})", name, result.ip_address);
-        result.plan = (circuit.download_max_mbps, circuit.download_min_mbps);
+        result.plan = (circuit.download_max_mbps, circuit.upload_max_mbps);
       }
     }
 
diff --git a/src/rust/lqos_node_manager/static/main.html b/src/rust/lqos_node_manager/static/main.html
index 95be9b97..82d8f57f 100644
--- a/src/rust/lqos_node_manager/static/main.html
+++ b/src/rust/lqos_node_manager/static/main.html
@@ -171,7 +171,7 @@
         // Loads the complete ringbuffer for initial display
         function fillCurrentThroughput() {
             msgPackGet("/api/throughput_ring_buffer", (tp) => {
-                console.log(tp);
+                //console.log(tp);
                 const bits = 0;
                 const packets = 1;
                 const shaped = 2;

From a16bc66d79ba3b4bea0afa021a280bc9289781b8 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 15:00:35 +0000
Subject: [PATCH 54/67] Format-preserving TOML editing

Restore the automatic generation of a node_id if one isn't present,
based on a hash of /etc/machine_id. Switch from the "toml" crate
to the "toml_edit" crate, and use its Document type to preserve
the user's comments and formatting when editing the configuration
file for saving.
---
 src/rust/Cargo.lock                        | 16 +------
 src/rust/lqos_config/Cargo.toml            |  2 +-
 src/rust/lqos_config/src/authentication.rs |  6 +--
 src/rust/lqos_config/src/etc.rs            | 52 +++++++++++++---------
 4 files changed, 36 insertions(+), 40 deletions(-)

diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
index 299ff8b1..2f8f803f 100644
--- a/src/rust/Cargo.lock
+++ b/src/rust/Cargo.lock
@@ -812,7 +812,7 @@ dependencies = [
  "atomic",
  "pear",
  "serde",
- "toml 0.5.11",
+ "toml",
  "uncased",
  "version_check",
 ]
@@ -1436,7 +1436,7 @@ dependencies = [
  "serde_json",
  "sha2",
  "thiserror",
- "toml 0.7.2",
+ "toml_edit",
  "uuid",
 ]
 
@@ -2787,18 +2787,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "toml"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6"
-dependencies = [
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit",
-]
-
 [[package]]
 name = "toml_datetime"
 version = "0.6.1"
diff --git a/src/rust/lqos_config/Cargo.toml b/src/rust/lqos_config/Cargo.toml
index 41e037be..0b17e88b 100644
--- a/src/rust/lqos_config/Cargo.toml
+++ b/src/rust/lqos_config/Cargo.toml
@@ -6,7 +6,7 @@ license = "GPL-2.0-only"
 
 [dependencies]
 thiserror = "1"
-toml = "0"
+toml_edit = { version = "0", features = [ "serde" ] }
 serde = { version = "1.0", features = [ "derive" ] }
 serde_json = "1"
 csv = "1"
diff --git a/src/rust/lqos_config/src/authentication.rs b/src/rust/lqos_config/src/authentication.rs
index e6c3effe..a80de0d6 100644
--- a/src/rust/lqos_config/src/authentication.rs
+++ b/src/rust/lqos_config/src/authentication.rs
@@ -68,7 +68,7 @@ impl WebUsers {
 
   fn save_to_disk(&self) -> Result<(), AuthenticationError> {
     let path = Self::path()?;
-    let new_contents = toml::to_string(&self);
+    let new_contents = toml_edit::ser::to_string(&self);
     if let Err(e) = new_contents {
       return Err(AuthenticationError::SerializationError(e));
     }
@@ -109,7 +109,7 @@ impl WebUsers {
     } else {
       // Load from disk
       if let Ok(raw) = read_to_string(path) {
-        let parse_result = toml::from_str(&raw);
+        let parse_result = toml_edit::de::from_str(&raw);
         if let Ok(users) = parse_result {
           Ok(users)
         } else {
@@ -255,7 +255,7 @@ pub enum AuthenticationError {
   #[error("Unable to load /etc/lqos.conf")]
   UnableToLoadEtcLqos,
   #[error("Unable to serialize to TOML")]
-  SerializationError(toml::ser::Error),
+  SerializationError(toml_edit::ser::Error),
   #[error("Unable to remove existing web users file")]
   UnableToDelete,
   #[error("Unable to open lqusers.toml for writing. Check permissions?")]
diff --git a/src/rust/lqos_config/src/etc.rs b/src/rust/lqos_config/src/etc.rs
index 83090d70..ff2069c1 100644
--- a/src/rust/lqos_config/src/etc.rs
+++ b/src/rust/lqos_config/src/etc.rs
@@ -1,6 +1,7 @@
 //! Manages the `/etc/lqos.conf` file.
 use log::error;
 use serde::{Deserialize, Serialize};
+use toml_edit::{Document, value};
 use std::{fs, path::Path};
 use thiserror::Error;
 
@@ -136,17 +137,27 @@ impl EtcLqos {
       return Err(EtcLqosError::ConfigDoesNotExist);
     }
     if let Ok(raw) = std::fs::read_to_string("/etc/lqos.conf") {
-      let config_result: Result<Self, toml::de::Error> = toml::from_str(&raw);
-      match config_result {
-        Ok(mut config) => {
-          check_config(&mut config);
-          Ok(config)
-        }
+      let document = raw.parse::<Document>();
+      match document {
         Err(e) => {
           error!("Unable to parse TOML from /etc/lqos.conf");
           error!("Full error: {:?}", e);
           Err(EtcLqosError::CannotParseToml)
         }
+        Ok(mut config_doc) => {
+          let cfg = toml_edit::de::from_document::<EtcLqos>(config_doc.clone());
+          match cfg {
+            Ok(mut cfg) => {
+              check_config(&mut config_doc, &mut cfg);
+              Ok(cfg)
+            }
+            Err(e) => {
+              error!("Unable to parse TOML from /etc/lqos.conf");
+              error!("Full error: {:?}", e);
+              Err(EtcLqosError::CannotParseToml)
+            }
+          }
+        }
       }
     } else {
       error!("Unable to read contents of /etc/lqos.conf");
@@ -156,7 +167,7 @@ impl EtcLqos {
 
   /// Saves changes made to /etc/lqos.conf
   /// Copies current configuration into /etc/lqos.conf.backup first
-  pub fn save(&self) -> Result<(), EtcLqosError> {
+  pub fn save(&self, document: &mut Document) -> Result<(), EtcLqosError> {
     let cfg_path = Path::new("/etc/lqos.conf");
     let backup_path = Path::new("/etc/lqos.conf.backup");
     if let Err(e) = std::fs::copy(cfg_path, backup_path) {
@@ -164,26 +175,17 @@ impl EtcLqos {
       log::error!("{e:?}");
       return Err(EtcLqosError::BackupFail);
     }
-    let new_cfg = toml::to_string_pretty(&self);
-    match new_cfg {
-      Err(e) => {
-        log::error!("Unable to serialize new /etc/lqos.conf");
-        log::error!("{e:?}");
-        return Err(EtcLqosError::SerializeFail);
-      }
-      Ok(new_cfg) => {
-        if let Err(e) = fs::write(cfg_path, new_cfg) {
-          log::error!("Unable to write to /etc/lqos.conf");
-          log::error!("{e:?}");
-          return Err(EtcLqosError::WriteFail);
-        }
-      }
+    let new_cfg = document.to_string();
+    if let Err(e) = fs::write(cfg_path, new_cfg) {
+      log::error!("Unable to write to /etc/lqos.conf");
+      log::error!("{e:?}");
+      return Err(EtcLqosError::WriteFail);
     }
     Ok(())
   }
 }
 
-fn check_config(cfg: &mut EtcLqos) {
+fn check_config(cfg_doc: &mut Document, cfg: &mut EtcLqos) {
   use sha2::digest::Update;
   use sha2::Digest;
 
@@ -191,6 +193,12 @@ fn check_config(cfg: &mut EtcLqos) {
     if let Ok(machine_id) = std::fs::read_to_string("/etc/machine-id") {
       let hash = sha2::Sha256::new().chain(machine_id).finalize();
       cfg.node_id = Some(format!("{:x}", hash));
+      cfg_doc["node_id"] = value(format!("{:x}", hash));
+      println!("Updating");
+      if let Err(e) = cfg.save(cfg_doc) {
+        log::error!("Unable to save /etc/lqos.conf");
+        log::error!("{e:?}");
+      }
     }
   }
 }

From c9025f654c7be6f91e3152b87c6201d42f52de04 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 16:36:10 +0000
Subject: [PATCH 55/67] Add a couple of unit test.

---
 src/rust/lqos_config/src/etc.rs | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/rust/lqos_config/src/etc.rs b/src/rust/lqos_config/src/etc.rs
index ff2069c1..8d47327b 100644
--- a/src/rust/lqos_config/src/etc.rs
+++ b/src/rust/lqos_config/src/etc.rs
@@ -220,3 +220,23 @@ pub enum EtcLqosError {
   #[error("Unable to write to /etc/lqos.conf")]
   WriteFail,
 }
+
+#[cfg(test)]
+mod test {
+  const EXAMPLE_LQOS_CONF: &str = include_str!("../../../lqos.example");
+
+  #[test]
+  fn round_trip_toml() {
+    let doc = EXAMPLE_LQOS_CONF.parse::<toml_edit::Document>().unwrap();
+    let reserialized = doc.to_string();
+    assert_eq!(EXAMPLE_LQOS_CONF, reserialized);
+  }
+
+  #[test]
+  fn add_node_id() {
+    let mut doc = EXAMPLE_LQOS_CONF.parse::<toml_edit::Document>().unwrap();
+    doc["node_id"] = toml_edit::value("test");
+    let reserialized = doc.to_string();
+    assert!(reserialized.contains("node_id = \"test\""));
+  }
+}
\ No newline at end of file

From f56581f1d6bc499b8545d6537dbeaeb82c697956 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 16:51:15 +0000
Subject: [PATCH 56/67] Add some unit tests and documentation to the  crate. No
 functional changes.

---
 src/rust/lqos_utils/src/fdtimer.rs           |  7 ++++
 src/rust/lqos_utils/src/file_watcher.rs      |  4 ++
 src/rust/lqos_utils/src/hex_string.rs        |  1 +
 src/rust/lqos_utils/src/lib.rs               | 16 +++++++-
 src/rust/lqos_utils/src/packet_scale.rs      | 42 ++++++++++++++++----
 src/rust/lqos_utils/src/string_table_enum.rs | 13 ++++++
 src/rust/lqos_utils/src/unix_time.rs         |  2 +
 7 files changed, 76 insertions(+), 9 deletions(-)

diff --git a/src/rust/lqos_utils/src/fdtimer.rs b/src/rust/lqos_utils/src/fdtimer.rs
index f9f80a96..baa7c36c 100644
--- a/src/rust/lqos_utils/src/fdtimer.rs
+++ b/src/rust/lqos_utils/src/fdtimer.rs
@@ -5,6 +5,13 @@ use nix::sys::{
 };
 use std::sync::atomic::AtomicBool;
 
+/// `periodic` runs a function at a given interval.
+/// 
+/// ## Parameters
+/// 
+/// * `interval_ms`: the interval in milliseconds.
+/// * `task_name`: the name of the task to run.
+/// * `tick_function`: the function to run at the given interval.
 pub fn periodic(
   interval_ms: u64,
   task_name: &str,
diff --git a/src/rust/lqos_utils/src/file_watcher.rs b/src/rust/lqos_utils/src/file_watcher.rs
index 9b2d4b2e..0eb3d2f1 100644
--- a/src/rust/lqos_utils/src/file_watcher.rs
+++ b/src/rust/lqos_utils/src/file_watcher.rs
@@ -145,10 +145,14 @@ impl FileWatcher {
   }
 }
 
+/// Errors that can occur when watching a file.
 #[derive(Error, Debug)]
 pub enum WatchedFileError {
+  /// Unable to create the file watcher.
   #[error("Unable to create watcher")]
   CreateWatcherError,
+
+  /// Unable to start the file watcher system.
   #[error("Unable to start watcher")]
   StartWatcherError,
 }
diff --git a/src/rust/lqos_utils/src/hex_string.rs b/src/rust/lqos_utils/src/hex_string.rs
index 7e47ceb8..8c11e6a6 100644
--- a/src/rust/lqos_utils/src/hex_string.rs
+++ b/src/rust/lqos_utils/src/hex_string.rs
@@ -37,6 +37,7 @@ pub fn read_hex_string(s: &str) -> Result<u32, HexParseError> {
 /// parsing a string into a `u32` hex number.
 #[derive(Error, Debug)]
 pub enum HexParseError {
+  /// The hex string could not be decoded
   #[error("Unable to decode string into valid hex")]
   ParseError,
 }
diff --git a/src/rust/lqos_utils/src/lib.rs b/src/rust/lqos_utils/src/lib.rs
index b0654c0d..352ac9d1 100644
--- a/src/rust/lqos_utils/src/lib.rs
+++ b/src/rust/lqos_utils/src/lib.rs
@@ -1,10 +1,24 @@
+//! Collection of utility functions for LibreQoS
+
+#![warn(missing_docs)]
 mod commands;
+
+/// Provides a Linux file-descriptor based timing service.
 pub mod fdtimer;
+
+/// Wrapper for watching when a file has changed.
 pub mod file_watcher;
+
+/// Utilities for handling strings in hex format
 pub mod hex_string;
+
+/// Utilities for scaling bits and packets to human-readable format
 pub mod packet_scale;
 mod string_table_enum;
+
+/// Utilities dealing with Unix Timestamps
 pub mod unix_time;
 mod xdp_ip_address;
 
-pub use xdp_ip_address::XdpIpAddress;
\ No newline at end of file
+/// XDP compatible IP Address
+pub use xdp_ip_address::XdpIpAddress;
diff --git a/src/rust/lqos_utils/src/packet_scale.rs b/src/rust/lqos_utils/src/packet_scale.rs
index 6f88504a..56af8c9f 100644
--- a/src/rust/lqos_utils/src/packet_scale.rs
+++ b/src/rust/lqos_utils/src/packet_scale.rs
@@ -1,23 +1,49 @@
+/// Scale a number of packets to a human readable string.
+/// 
+/// ## Parameters
+/// * `n`: the number of packets to scale
 pub fn scale_packets(n: u64) -> String {
-  if n > 1_000_000_000 {
+  if n >= 1_000_000_000 {
     format!("{:.2} gpps", n as f32 / 1_000_000_000.0)
-  } else if n > 1_000_000 {
+  } else if n >= 1_000_000 {
     format!("{:.2} mpps", n as f32 / 1_000_000.0)
-  } else if n > 1_000 {
+  } else if n >= 1_000 {
     format!("{:.2} kpps", n as f32 / 1_000.0)
   } else {
-    format!("{n}  pps")
+    format!("{n} pps")
   }
 }
 
+/// Scale a number of bits to a human readable string.
+/// 
+/// ## Parameters
+/// * `n`: the number of bits to scale
 pub fn scale_bits(n: u64) -> String {
-  if n > 1_000_000_000 {
+  if n >= 1_000_000_000 {
     format!("{:.2} gbit/s", n as f32 / 1_000_000_000.0)
-  } else if n > 1_000_000 {
+  } else if n >= 1_000_000 {
     format!("{:.2} mbit/s", n as f32 / 1_000_000.0)
-  } else if n > 1_000 {
+  } else if n >= 1_000 {
     format!("{:.2} kbit/s", n as f32 / 1_000.0)
   } else {
-    format!("{n}  bit/s")
+    format!("{n} bit/s")
   }
 }
+
+#[cfg(test)]
+mod test {
+  #[test]
+  fn test_scale_packets() {
+    assert_eq!(super::scale_packets(1), "1 pps");
+    assert_eq!(super::scale_packets(1000), "1.00 kpps");
+    assert_eq!(super::scale_packets(1000000), "1.00 mpps");
+    assert_eq!(super::scale_packets(1000000000), "1.00 gpps");
+  }
+
+  #[test]
+  fn test_scale_bits() {
+    assert_eq!(super::scale_bits(1), "1 bit/s");
+    assert_eq!(super::scale_bits(1000), "1.00 kbit/s");
+    assert_eq!(super::scale_bits(1000000), "1.00 mbit/s");
+    assert_eq!(super::scale_bits(1000000000), "1.00 gbit/s");}
+}
\ No newline at end of file
diff --git a/src/rust/lqos_utils/src/string_table_enum.rs b/src/rust/lqos_utils/src/string_table_enum.rs
index aa0cde90..f7849aed 100644
--- a/src/rust/lqos_utils/src/string_table_enum.rs
+++ b/src/rust/lqos_utils/src/string_table_enum.rs
@@ -1,3 +1,9 @@
+/// Helper macro to create an enum that can be serialized to a string
+/// and deserialized from a string.
+/// 
+/// ## Parameters
+/// * `$enum_name`: the name of the enum to create
+/// * `$($option:ident),*`: the options of the enum
 #[macro_export]
 macro_rules! string_table_enum {
     ($enum_name: ident, $($option:ident),*) => {
@@ -36,6 +42,13 @@ macro_rules! string_table_enum {
     };
 }
 
+/// Helper macro to create an enum that can be serialized to a string
+/// and deserialized from a string. Adds explicit support for dashes
+/// in identifiers.
+/// 
+/// ## Parameters
+/// * `$enum_name`: the name of the enum to create
+/// * `$($option:ident),*`: the options of the enum
 #[macro_export]
 macro_rules! dashy_table_enum {
     ($enum_name: ident, $($option:ident),*) => {
diff --git a/src/rust/lqos_utils/src/unix_time.rs b/src/rust/lqos_utils/src/unix_time.rs
index a60e714e..33eee4a4 100644
--- a/src/rust/lqos_utils/src/unix_time.rs
+++ b/src/rust/lqos_utils/src/unix_time.rs
@@ -32,8 +32,10 @@ pub fn time_since_boot() -> Result<TimeSpec, TimeError> {
   }
 }
 
+/// Error type for time functions.
 #[derive(Error, Debug)]
 pub enum TimeError {
+  /// The clock isn't ready yet.
   #[error("Clock not ready")]
   ClockNotReady,
 }

From 6106c26899f58acd709464be85f27e092e37381a Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 16:58:19 +0000
Subject: [PATCH 57/67] Add documentation headers for Heimdall.

---
 src/rust/lqos_heimdall/src/flows.rs          |  2 ++
 src/rust/lqos_heimdall/src/lib.rs            |  3 ++-
 src/rust/lqos_heimdall/src/perf_interface.rs | 16 ++++++++++++++++
 src/rust/lqos_heimdall/src/timeline.rs       | 15 +++++++++++++++
 4 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/src/rust/lqos_heimdall/src/flows.rs b/src/rust/lqos_heimdall/src/flows.rs
index 41a2b364..c88313e2 100644
--- a/src/rust/lqos_heimdall/src/flows.rs
+++ b/src/rust/lqos_heimdall/src/flows.rs
@@ -133,6 +133,7 @@ pub fn read_flows() {
   });
 }
 
+/// Expire flows that have not been seen in a while.
 pub fn expire_heimdall_flows() {
   if let Ok(now) = time_since_boot() {
     let since_boot = Duration::from(now);
@@ -142,6 +143,7 @@ pub fn expire_heimdall_flows() {
   }
 }
 
+/// Get the flow stats for a given IP address.
 pub fn get_flow_stats(ip: XdpIpAddress) -> BusResponse {
   let mut result = Vec::new();
 
diff --git a/src/rust/lqos_heimdall/src/lib.rs b/src/rust/lqos_heimdall/src/lib.rs
index 267af1f8..5531858b 100644
--- a/src/rust/lqos_heimdall/src/lib.rs
+++ b/src/rust/lqos_heimdall/src/lib.rs
@@ -1,8 +1,9 @@
 //! Provides an interface to the Heimdall packet watching
 //! system. Heimdall watches traffic flows, and is notified
 //! about their contents via the eBPF Perf system.
-
+#![warn(missing_docs)]
 mod config;
+/// Interface to the performance tracking system
 pub mod perf_interface;
 pub mod stats;
 pub use config::{HeimdalConfig, HeimdallMode};
diff --git a/src/rust/lqos_heimdall/src/perf_interface.rs b/src/rust/lqos_heimdall/src/perf_interface.rs
index 49243019..81fb76b3 100644
--- a/src/rust/lqos_heimdall/src/perf_interface.rs
+++ b/src/rust/lqos_heimdall/src/perf_interface.rs
@@ -6,21 +6,37 @@ use crate::timeline::store_on_timeline;
 /// This constant MUST exactly match PACKET_OCTET_STATE in heimdall.h
 pub(crate) const PACKET_OCTET_SIZE: usize = 128;
 
+/// A representation of the eBPF `heimdall_event` type.
+/// This is the type that is sent from the eBPF program to userspace.
+/// It is a representation of the `heimdall_event` type in heimdall.h
 #[derive(FromBytes, Debug, Clone, PartialEq, Eq, Hash)]
 #[repr(C)]
 pub struct HeimdallEvent {
+  /// Timestamp of the event, in nanoseconds since boot time.
   pub timestamp: u64,
+  /// Source IP address
   pub src: XdpIpAddress,
+  /// Destination IP address
   pub dst: XdpIpAddress,
+  /// Source port number, or ICMP type.
   pub src_port : u16,
+  /// Destination port number.
   pub dst_port: u16,
+  /// IP protocol number
   pub ip_protocol: u8,
+  /// IP header TOS value
   pub tos: u8,
+  /// Total size of the packet, in bytes
   pub size: u32,
+  /// TCP flags
   pub tcp_flags: u8,
+  /// TCP window size
   pub tcp_window: u16,
+  /// TCP sequence number
   pub tcp_tsval: u32,
+  /// TCP acknowledgement number
   pub tcp_tsecr: u32,
+  /// Raw packet data
   pub packet_data: [u8; PACKET_OCTET_SIZE],
 }
 
diff --git a/src/rust/lqos_heimdall/src/timeline.rs b/src/rust/lqos_heimdall/src/timeline.rs
index 1be06db6..3bd2d066 100644
--- a/src/rust/lqos_heimdall/src/timeline.rs
+++ b/src/rust/lqos_heimdall/src/timeline.rs
@@ -150,6 +150,13 @@ pub fn hyperfocus_on_target(ip: XdpIpAddress) -> Option<(usize, usize)> {
   }
 }
 
+/// Request a dump of the packet headers collected during a hyperfocus session.
+/// This will return `None` if the session id is invalid or the session has
+/// expired.
+/// ## Returns
+/// * Either `None` or a vector of packet headers.
+/// ## Arguments
+/// * `session_id` - The session id of the hyperfocus session.
 pub fn n_second_packet_dump(session_id: usize) -> Option<Vec<PacketHeader>> {
   if let Some(session) = FOCUS_SESSIONS.get(&session_id) {
     Some(session.data.iter().map(|e| e.as_header()).collect())
@@ -158,6 +165,14 @@ pub fn n_second_packet_dump(session_id: usize) -> Option<Vec<PacketHeader>> {
   }
 }
 
+/// Request a dump of the packet headers collected during a hyperfocus session,
+/// in LibPCAP format. This will return `None` if the session id is invalid or
+/// the session has expired, or the temporary filename used to store the dump
+/// if it is available.
+/// ## Returns
+/// * Either `None` or the filename of the dump.
+/// ## Arguments
+/// * `session_id` - The session id of the hyperfocus session.
 pub fn n_second_pcap(session_id: usize) -> Option<String> {
   if let Some(mut session) = FOCUS_SESSIONS.get_mut(&session_id) {
     let filename = format!("/tmp/cap_sess_{session_id}");

From 0cc1a6eaee88f3ff7e9df91d31830fdc2957272b Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 19:21:39 +0000
Subject: [PATCH 58/67] Add documentation headers for Queue Tracker.

---
 src/rust/lqos_queue_tracker/src/bus.rs               |  4 ++++
 src/rust/lqos_queue_tracker/src/interval.rs          |  6 ++++++
 src/rust/lqos_queue_tracker/src/lib.rs               |  7 +++++++
 src/rust/lqos_queue_tracker/src/tracking/mod.rs      |  3 +++
 .../src/tracking/watched_queues.rs                   | 12 ++++++++++++
 5 files changed, 32 insertions(+)

diff --git a/src/rust/lqos_queue_tracker/src/bus.rs b/src/rust/lqos_queue_tracker/src/bus.rs
index 8904989f..bbbb07c6 100644
--- a/src/rust/lqos_queue_tracker/src/bus.rs
+++ b/src/rust/lqos_queue_tracker/src/bus.rs
@@ -3,6 +3,10 @@ use crate::{
 };
 use lqos_bus::BusResponse;
 
+/// Retrieves the raw queue data for a given circuit ID.
+/// 
+/// # Arguments
+/// * `circuit_id` - The circuit ID to retrieve data for.
 pub fn get_raw_circuit_data(circuit_id: &str) -> BusResponse {
   still_watching(circuit_id);
   if let Some(circuit) = CIRCUIT_TO_QUEUE.get(circuit_id) {
diff --git a/src/rust/lqos_queue_tracker/src/interval.rs b/src/rust/lqos_queue_tracker/src/interval.rs
index 126d8622..f5327bf8 100644
--- a/src/rust/lqos_queue_tracker/src/interval.rs
+++ b/src/rust/lqos_queue_tracker/src/interval.rs
@@ -2,6 +2,12 @@ use std::sync::atomic::AtomicU64;
 
 pub(crate) static QUEUE_MONITOR_INTERVAL: AtomicU64 = AtomicU64::new(1000);
 
+/// Sets the interval at which the queue monitor thread will poll the
+/// Linux `tc` shaper for queue statistics.
+/// 
+/// # Arguments
+/// * `interval_ms` - The interval, in milliseconds, at which the queue
+///   monitor thread will poll the Linux `tc` shaper for queue statistics.
 pub fn set_queue_refresh_interval(interval_ms: u64) {
   QUEUE_MONITOR_INTERVAL
     .store(interval_ms, std::sync::atomic::Ordering::Relaxed);
diff --git a/src/rust/lqos_queue_tracker/src/lib.rs b/src/rust/lqos_queue_tracker/src/lib.rs
index 31c01050..e5a14f25 100644
--- a/src/rust/lqos_queue_tracker/src/lib.rs
+++ b/src/rust/lqos_queue_tracker/src/lib.rs
@@ -1,3 +1,10 @@
+//! Retrieves queue statistics from the Linux `tc` shaper, and stores
+//! them in a `QueueStore` for later retrieval. The `QueueStore` is
+//! thread-safe, and can be accessed from multiple threads. It is
+//! updated periodically by a separate thread, and accumulates statistics
+//! between polling periods.
+
+#![warn(missing_docs)]
 mod bus;
 mod circuit_to_queue;
 mod interval;
diff --git a/src/rust/lqos_queue_tracker/src/tracking/mod.rs b/src/rust/lqos_queue_tracker/src/tracking/mod.rs
index 93a13fcb..952e8165 100644
--- a/src/rust/lqos_queue_tracker/src/tracking/mod.rs
+++ b/src/rust/lqos_queue_tracker/src/tracking/mod.rs
@@ -76,6 +76,9 @@ fn track_queues() {
   expire_watched_queues();
 }
 
+/// Spawns a thread that periodically reads the queue statistics from
+/// the Linux `tc` shaper, and stores them in a `QueueStore` for later
+/// retrieval.
 pub fn spawn_queue_monitor() {
   std::thread::spawn(|| {
     // Setup the queue monitor loop
diff --git a/src/rust/lqos_queue_tracker/src/tracking/watched_queues.rs b/src/rust/lqos_queue_tracker/src/tracking/watched_queues.rs
index 3c6d49aa..2196c880 100644
--- a/src/rust/lqos_queue_tracker/src/tracking/watched_queues.rs
+++ b/src/rust/lqos_queue_tracker/src/tracking/watched_queues.rs
@@ -31,6 +31,13 @@ pub fn expiration_in_the_future() -> u64 {
   unix_now().unwrap_or(0) + 10
 }
 
+/// Start watching a queue. This will cause the queue to be read
+/// periodically, and its statistics stored in the `QueueStore`.
+/// If the queue is already being watched, this function will
+/// do nothing.
+/// 
+/// # Arguments
+/// * `circuit_id` - The circuit ID to watch
 pub fn add_watched_queue(circuit_id: &str) {
   //info!("Watching queue {circuit_id}");
   let max = num_possible_cpus().unwrap() * 2;
@@ -74,6 +81,11 @@ pub(crate) fn expire_watched_queues() {
   WATCHED_QUEUES.retain(|_,w| w.expires_unix_time > now);
 }
 
+/// Indicates that a watched queue is still being watched. Update the
+/// expiration time for the queue.
+/// 
+/// # Arguments
+/// * `circuit_id` - The circuit ID to watch
 pub fn still_watching(circuit_id: &str) {
   if let Some(mut q) = WATCHED_QUEUES.get_mut(circuit_id) {
     //info!("Still watching circuit: {circuit_id}");

From 6e15181fabd9d401d908e585d1553a3f7c970bc0 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 20:16:57 +0000
Subject: [PATCH 59/67] Add a couple of 'is none' checks to remove crashes I
 experienced.

---
 src/integrationCommon.py | 12 ++++++++----
 src/integrationUISP.py   |  8 ++++++--
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/integrationCommon.py b/src/integrationCommon.py
index 33f20e31..122fc3ef 100644
--- a/src/integrationCommon.py
+++ b/src/integrationCommon.py
@@ -383,6 +383,10 @@ class NetworkGraph:
 						#Remove brackets and quotes of list so LibreQoS.py can parse it
 						device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'')
 						device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'')
+						if circuit["upload"] is None: 
+							circuit["upload"] = 0.0
+						if circuit["download"] is None: 
+							circuit["download"] = 0.0
 						row = [
 							circuit["id"],
 							circuit["name"],
@@ -392,10 +396,10 @@ class NetworkGraph:
 							device["mac"],
 							device["ipv4"],
 							device["ipv6"],
-							int(circuit["download"] * 0.98),
-							int(circuit["upload"] * 0.98),
-							int(circuit["download"] * bandwidthOverheadFactor),
-							int(circuit["upload"] * bandwidthOverheadFactor),
+							int(float(circuit["download"]) * 0.98),
+							int(float(circuit["upload"]) * 0.98),
+							int(float(circuit["download"]) * bandwidthOverheadFactor),
+							int(float(circuit["upload"]) * bandwidthOverheadFactor),
 							""
 						]
 						wr.writerow(row)
diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 36937fb7..208594a7 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -118,8 +118,12 @@ def buildFullGraph():
 			if device['identification']['role'] == "station":
 				if device['identification']['type'] == "airFiber":
 					if device['overview']['status'] == 'active':
-						download = int(device['overview']['downlinkCapacity']/ 1000000)
-						upload = int(device['overview']['uplinkCapacity']/ 1000000)
+						if device['overview']['downlinkCapacity'] is not None and device['overview']['uplinkCapacity'] is not None:
+							download = int(device['overview']['downlinkCapacity']/ 1000000)
+							upload = int(device['overview']['uplinkCapacity']/ 1000000)
+						else:
+							download = generatedPNDownloadMbps
+							upload = generatedPNUploadMbps
 						# Make sure to use half of reported bandwidth for AF60-LRs
 						if device['identification']['model'] == "AF60-LR":
 							download = int(download / 2)

From d2aa804ace96b86dd1dc897912f9fb592786b327 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 5 Apr 2023 20:34:32 +0000
Subject: [PATCH 60/67] Use dot renderer for PDF

---
 src/integrationCommon.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/integrationCommon.py b/src/integrationCommon.py
index 122fc3ef..6e89c3c9 100644
--- a/src/integrationCommon.py
+++ b/src/integrationCommon.py
@@ -418,7 +418,7 @@ class NetworkGraph:
 
 		import graphviz
 		dot = graphviz.Digraph(
-			'network', comment="Network Graph", engine="fdp")
+			'network', comment="Network Graph", engine="dot")
 
 		for (i, node) in enumerate(self.nodes):
 			if ((node.type != NodeType.client and node.type != NodeType.device) or showClients):

From 2d3874e812adfab1c0436cd9d2c9be7f89590620 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Thu, 6 Apr 2023 16:22:01 +0000
Subject: [PATCH 61/67] Build complete spanning tree for data links and use it
 to orient tree

Rather than obeying the "parent" field, which leads to wrong-way up
graphs when building from a non-root site - instead build a complete
cost-based spanning tree from the specified root node. Then use
the generated tree - falling back on parent if nothing is found -
to populate the tree in order.

Tested on Herbert's network, correctly generates topology from several
different locations.

Still to come: adding a mechanism for providing parenting overloads
to allow for cases where a long path is actually optimal, but we
have no way of knowing that.
---
 src/integrationUISP.py | 152 +++++++++++++++++++++++++++++++++++------
 1 file changed, 133 insertions(+), 19 deletions(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 208594a7..bdc31cfc 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -76,19 +76,36 @@ def buildFlatGraph():
 		net.createNetworkJson()
 	net.createShapedDevices()
 
-def buildFullGraph():
-	# Attempts to build a full network graph, incorporating as much of the UISP
-	# hierarchy as possible.
-	from integrationCommon import NetworkGraph, NetworkNode, NodeType
-	from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps
+def linkSiteTarget(link, direction):
+	# Helper function to extract the site ID from a data link. Returns
+	# None if not present.
+	if link[direction]['site'] is not None:
+		return link[direction]['site']['identification']['id']
+	
+	return None
 
-	# Load network sites
-	print("Loading Data from UISP")
-	sites = uispRequest("sites")
-	devices = uispRequest("devices?withInterfaces=true&authorized=true")
-	dataLinks = uispRequest("data-links?siteLinksOnly=true")
+def findSiteLinks(dataLinks, siteId):
+	# Searches the Data Links for any links to/from the specified site.
+	# Returns a list of site IDs that are linked to the specified site.
+	links = []
+	for dl in dataLinks:
+		fromSiteId = linkSiteTarget(dl, "from")
+		if fromSiteId is not None and fromSiteId == siteId:
+			# We have a link originating in this site.
+			target = linkSiteTarget(dl, "to")
+			if target is not None:
+				links.append(target)
 
-	# Do we already have a integrationUISPbandwidths.csv file?
+		toSiteId = linkSiteTarget(dl, "to")
+		if toSiteId is not None and toSiteId == siteId:
+			# We have a link originating in this site.
+			target = linkSiteTarget(dl, "from")
+			if target is not None:
+				links.append(target)
+	return links
+
+def buildSiteBandwidths():
+	# Builds a dictionary of site bandwidths from the integrationUISPbandwidths.csv file.
 	siteBandwidth = {}
 	if os.path.isfile("integrationUISPbandwidths.csv"):
 		with open('integrationUISPbandwidths.csv') as csv_file:
@@ -99,8 +116,10 @@ def buildFullGraph():
 				download = int(download)
 				upload = int(upload)
 				siteBandwidth[name] = {"download": download, "upload": upload}
-	
-	# Find AP capacities from UISP
+	return siteBandwidth
+
+def findApCapacities(devices, siteBandwidth):
+	# Searches the UISP devices for APs and adds their capacities to the siteBandwidth dictionary.
 	for device in devices:
 		if device['identification']['role'] == "ap":
 			name = device['identification']['name']
@@ -110,8 +129,8 @@ def buildFullGraph():
 				upload = int(device['overview']['uplinkCapacity'] / 1000000)
 				siteBandwidth[device['identification']['name']] = {
 					"download": download, "upload": upload}
-	
-	# Find Site Capacities by AirFiber capacities
+
+def findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps):
 	foundAirFibersBySite = {}
 	for device in devices:
 		if device['identification']['site']['type'] == 'site':
@@ -134,7 +153,100 @@ def buildFullGraph():
 								foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
 						else:
 							foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload}
+	return foundAirFibersBySite
+
+def buildSiteList(sites, dataLinks):
+	# Builds a list of sites, including their IDs, names, and connections.
+	# Connections are determined by the dataLinks list.
+	siteList = []
+	for site in sites:
+		newSite = {
+			'id': site['identification']['id'], 
+			'name': site['identification']['name'],
+			'connections': findSiteLinks(dataLinks, site['identification']['id']),
+			'cost': 10000,
+			'parent': "",
+			'type': type,
+		}
+		siteList.append(newSite)
+	return siteList
+
+def findInSiteList(siteList, name):
+	# Searches the siteList for a site with the specified name.
+	for site in siteList:
+		if site['name'] == name:
+			return site
+	return None
+
+def findInSiteListById(siteList, id):
+	# Searches the siteList for a site with the specified name.
+	for site in siteList:
+		if site['id'] == id:
+			return site
+	return None
+
+def debugSpaces(n):
+	# Helper function to print n spaces.
+	spaces = ""
+	for i in range(int(n)):
+		spaces = spaces + " "
+	return spaces
+
+def walkGraphOutwards(siteList, root):
+	def walkGraph(node, parent, cost, backPath):
+		site = findInSiteListById(siteList, node)
+		if cost < site['cost']:
+			# It's cheaper to get here this way, so update the cost and parent.
+			site['cost'] = cost
+			site['parent'] = parent['id']
+			#print(debugSpaces(cost/10) + parent['name'] + "->" + site['name'] + " -> New cost: " + str(cost))
+
+		for connection in site['connections']:
+			if not connection in backPath:
+				#target = findInSiteListById(siteList, connection)
+				#print(debugSpaces((cost+10)/10) + site['name'] + " -> " + target['name'] + " (" + str(target['cost']) + ")")
+				newBackPath = backPath.copy()
+				newBackPath.append(site['id'])
+				walkGraph(connection, site, cost+10, newBackPath)
+
+	for connection in root['connections']:
+		# Force the parent since we're at the top
+		site = findInSiteListById(siteList, connection)
+		site['parent'] = root['id']
+		walkGraph(connection, root, 20, [root['id']])
+
+def buildFullGraph():
+	# Attempts to build a full network graph, incorporating as much of the UISP
+	# hierarchy as possible.
+	from integrationCommon import NetworkGraph, NetworkNode, NodeType
+	from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps
+
+	# Load network sites
+	print("Loading Data from UISP")
+	sites = uispRequest("sites")
+	devices = uispRequest("devices?withInterfaces=true&authorized=true")
+	dataLinks = uispRequest("data-links?siteLinksOnly=true")
+
+	# Build Site Capacities
+	siteBandwidth = buildSiteBandwidths()
+	findApCapacities(devices, siteBandwidth)
+	foundAirFibersBySite = findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps)
 	
+	# Create a list of just network sites
+	siteList = buildSiteList(sites, dataLinks)
+	rootSite = findInSiteList(siteList, uispSite)
+	if rootSite is None:
+		print("ERROR: Unable to find root site in UISP")
+		return
+	walkGraphOutwards(siteList, rootSite)
+	# Debug code: dump the list of site parents
+	# for s in siteList:
+	# 	if s['parent'] == "":
+	# 		p = "None"
+	# 	else:
+	# 		p = findInSiteListById(siteList, s['parent'])['name']
+	# 		print(s['name'] + " (" + str(s['cost']) + ") <-- " + p)
+
 	print("Building Topology")
 	net = NetworkGraph()
 	# Add all sites and client sites
@@ -146,10 +258,12 @@ def buildFullGraph():
 		upload = generatedPNUploadMbps
 		address = ""
 		customerName = ""
-		if site['identification']['parent'] is None:
-			parent = ""
-		else:
-			parent = site['identification']['parent']['id']
+		parent = findInSiteListById(siteList, id)['parent']
+		if parent == "":
+			if site['identification']['parent'] is None:
+				parent = ""
+			else:
+				parent = site['identification']['parent']['id']
 		match type:
 			case "site":
 				nodeType = NodeType.site

From 6df648e29977f957aded56ffa40647afd8edd5e4 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Tue, 11 Apr 2023 15:30:47 +0000
Subject: [PATCH 62/67] Allow integrationUISProutes.csv to override default
 costs in UISP route determination.

Requested by D. Denson.

Now that the UISP network tree is built as a spanning tree, with 10
cost per hop we can reliably flip the tree from A->B->C to C->B->A
depending upon root positioning.

This addition allows you to specify additional routes (that MUST
exist!) e.g. A->C and specify a cost to use instead of the default
10. This allows for topologies in which A-B-C is actually faster
than a direct A-C route (for example, becuase of short 60ghz
hops).
---
 src/integrationUISP.py        | 25 ++++++++++++++++++++++---
 src/integrationUISProutes.csv |  5 +++++
 2 files changed, 27 insertions(+), 3 deletions(-)
 create mode 100644 src/integrationUISProutes.csv

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index bdc31cfc..55fb3411 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -192,9 +192,14 @@ def debugSpaces(n):
 		spaces = spaces + " "
 	return spaces
 
-def walkGraphOutwards(siteList, root):
+def walkGraphOutwards(siteList, root, routeOverrides):
 	def walkGraph(node, parent, cost, backPath):
 		site = findInSiteListById(siteList, node)
+		routeName = parent['name'] + "->" + site['name']
+		if routeName in routeOverrides:
+			# We have an overridden cost for this route, so use it instead
+			#print("--> Using overridden cost for " + routeName + ". New cost: " + str(routeOverrides[routeName]) + ".")
+			cost = routeOverrides[routeName]
 		if cost < site['cost']:
 			# It's cheaper to get here this way, so update the cost and parent.
 			site['cost'] = cost
@@ -213,7 +218,20 @@ def walkGraphOutwards(siteList, root):
 		# Force the parent since we're at the top
 		site = findInSiteListById(siteList, connection)
 		site['parent'] = root['id']
-		walkGraph(connection, root, 20, [root['id']])
+		walkGraph(connection, root, 10, [root['id']])
+
+def loadRoutingOverrides():
+	# Loads integrationUISProutes.csv and returns a set of "from", "to", "cost"
+	overrides = {}
+	if os.path.isfile("integrationUISProutes.csv"):
+		with open("integrationUISProutes.csv", "r") as f:
+			reader = csv.reader(f)
+			for row in reader:
+				if not row[0].startswith("#") and len(row) == 3:
+					fromSite, toSite, cost = row
+					overrides[fromSite.strip() + "->" + toSite.strip()] = int(cost)
+	#print(overrides)
+	return overrides
 
 def buildFullGraph():
 	# Attempts to build a full network graph, incorporating as much of the UISP
@@ -235,10 +253,11 @@ def buildFullGraph():
 	# Create a list of just network sites
 	siteList = buildSiteList(sites, dataLinks)
 	rootSite = findInSiteList(siteList, uispSite)
+	routeOverrides = loadRoutingOverrides()
 	if rootSite is None:
 		print("ERROR: Unable to find root site in UISP")
 		return
-	walkGraphOutwards(siteList, rootSite)
+	walkGraphOutwards(siteList, rootSite, routeOverrides)
 	# Debug code: dump the list of site parents
 	# for s in siteList:
 	# 	if s['parent'] == "":
diff --git a/src/integrationUISProutes.csv b/src/integrationUISProutes.csv
new file mode 100644
index 00000000..643c4eb0
--- /dev/null
+++ b/src/integrationUISProutes.csv
@@ -0,0 +1,5 @@
+# Allows you to override route costs in the UISP integration, to better
+# represent your network. Costs by default increment 10 at each hop.
+# So if you want to skip 10 links, put a cost of 100 in.
+# From Site Name, To Site Name, New Cost
+# MYCORE, MYTOWER, 100

From 8c6a901ab90f43a65a5c093119ad623391b675c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Tue, 11 Apr 2023 10:38:00 -0600
Subject: [PATCH 63/67] Update ispConfig.example.py

---
 src/ispConfig.example.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/ispConfig.example.py b/src/ispConfig.example.py
index cbca50ff..9da3813c 100644
--- a/src/ispConfig.example.py
+++ b/src/ispConfig.example.py
@@ -63,6 +63,9 @@ influxDBtoken = ""
 # Use Customer Name or Address as Circuit Name
 circuitNameUseAddress = True
 
+# Should integrationUISP overwrite network.json on each run?
+overwriteNetworkJSONalways = False
+
 # If a device shows a WAN IP within these subnets, assume they are behind NAT / un-shapable, and ignore them
 ignoreSubnets = ['192.168.0.0/16']
 allowedSubnets = ['100.64.0.0/10']

From a47c8a065d349b1cd712d95f5e58f66c9b58b962 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Tue, 11 Apr 2023 10:39:30 -0600
Subject: [PATCH 64/67] Update integrationUISP.py

---
 src/integrationUISP.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 55fb3411..9c73e745 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -3,8 +3,12 @@ checkPythonVersion()
 import requests
 import os
 import csv
-from ispConfig import uispSite, uispStrategy
 from integrationCommon import isIpv4Permitted, fixSubnet
+try:
+	from ispConfig import uispSite, uispStrategy, overwriteNetworkJSONalways
+except:
+	from ispConfig import uispSite, uispStrategy
+	overwriteNetworkJSONalways = False
 
 def uispRequest(target):
 	# Sends an HTTP request to UISP and returns the
@@ -71,7 +75,10 @@ def buildFlatGraph():
 	net.prepareTree()
 	net.plotNetworkGraph(False)
 	if net.doesNetworkJsonExist():
-		print("network.json already exists. Leaving in-place.")
+		if overwriteNetworkJSONalways:
+			net.createNetworkJson()
+		else:
+			print("network.json already exists and overwriteNetworkJSONalways set to False. Leaving in-place.")
 	else:
 		net.createNetworkJson()
 	net.createShapedDevices()
@@ -362,7 +369,10 @@ def buildFullGraph():
 	net.prepareTree()
 	net.plotNetworkGraph(False)
 	if net.doesNetworkJsonExist():
-		print("network.json already exists. Leaving in-place.")
+		if overwriteNetworkJSONalways:
+			net.createNetworkJson()
+		else:
+			print("network.json already exists and overwriteNetworkJSONalways set to False. Leaving in-place.")
 	else:
 		net.createNetworkJson()
 	net.createShapedDevices()

From 00839dfe88d00968261d5ba9c8a88d8db18b4e0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Tue, 11 Apr 2023 15:30:28 -0600
Subject: [PATCH 65/67] Update integrationUISP.py

---
 src/integrationUISP.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 9c73e745..dce7eda7 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -272,7 +272,21 @@ def buildFullGraph():
 	# 	else:
 	# 		p = findInSiteListById(siteList, s['parent'])['name']
 	# 		print(s['name'] + " (" + str(s['cost']) + ") <-- " + p)
-
+	
+	# Find Nodes Connected By PtMP
+	nodeOffPtMP = {}
+	for site in sites:
+		id = site['identification']['id']
+		name = site['identification']['name']
+		type = site['identification']['type']
+		parent = findInSiteListById(siteList, id)['parent']
+		if type == 'site':
+			for link in dataLinks:
+				if link['from']['site']['identification']['id'] == parent:
+					if link['to']['site']['identification']['id'] == id:
+						if link['from']['device']['overview']['wirelessMode'] == 'ap-ptmp':
+							nodeOffPtMP[id] = link['from']['device']['identification']['id']
+	
 	print("Building Topology")
 	net = NetworkGraph()
 	# Add all sites and client sites
@@ -293,6 +307,8 @@ def buildFullGraph():
 		match type:
 			case "site":
 				nodeType = NodeType.site
+				if id in nodeOffPtMP:
+					parent = nodeOffPtMP[id]
 				if name in siteBandwidth:
 					# Use the CSV bandwidth values
 					download = siteBandwidth[name]["download"]
@@ -365,7 +381,7 @@ def buildFullGraph():
 									# Add some defaults in case they want to change them
 									siteBandwidth[node.displayName] = {
 										"download": generatedPNDownloadMbps, "upload": generatedPNUploadMbps}
-
+	
 	net.prepareTree()
 	net.plotNetworkGraph(False)
 	if net.doesNetworkJsonExist():

From f3414257adb841f0ebbcb67f6322b87c7dd21d9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Robert=20Chac=C3=B3n?= <robertchacon@protonmail.com>
Date: Tue, 11 Apr 2023 16:13:26 -0600
Subject: [PATCH 66/67] Update integrationUISP.py

---
 src/integrationUISP.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index dce7eda7..543ce734 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -285,8 +285,13 @@ def buildFullGraph():
 				if link['from']['site']['identification']['id'] == parent:
 					if link['to']['site']['identification']['id'] == id:
 						if link['from']['device']['overview']['wirelessMode'] == 'ap-ptmp':
-							nodeOffPtMP[id] = link['from']['device']['identification']['id']
-	
+							# Capacity of the PtMP client radio feeding the PoP will be used as the site bandwidth limit
+							download = int(round(link['to']['device']['overview']['downlinkCapacity']/1000000))
+							upload = int(round(link['to']['device']['overview']['uplinkCapacity']/1000000))
+							nodeOffPtMP[id] = { 'parent': link['from']['device']['identification']['id'],
+												'download': download,
+												'upload': upload
+												}
 	print("Building Topology")
 	net = NetworkGraph()
 	# Add all sites and client sites
@@ -308,11 +313,14 @@ def buildFullGraph():
 			case "site":
 				nodeType = NodeType.site
 				if id in nodeOffPtMP:
-					parent = nodeOffPtMP[id]
+					parent = nodeOffPtMP[id]['parent']
 				if name in siteBandwidth:
 					# Use the CSV bandwidth values
 					download = siteBandwidth[name]["download"]
 					upload = siteBandwidth[name]["upload"]
+				elif id in nodeOffPtMP:
+					download = nodeOffPtMP[id]['download']
+					upload = nodeOffPtMP[id]['upload']
 				elif id in foundAirFibersBySite:
 					download = foundAirFibersBySite[id]['download']
 					upload = foundAirFibersBySite[id]['upload']

From 0fcc5d0bac3e50d4c45e2025363ae04038dd9b21 Mon Sep 17 00:00:00 2001
From: Herbert Wolverson <herberticus@gmail.com>
Date: Wed, 12 Apr 2023 13:46:04 +0000
Subject: [PATCH 67/67] Add a lot of is not None tests to the nodeOffPtmp code,
 to account for offline devices and 'other' devices that don't have all of the
 required data.

---
 src/integrationUISP.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/integrationUISP.py b/src/integrationUISP.py
index 543ce734..079c3b4d 100644
--- a/src/integrationUISP.py
+++ b/src/integrationUISP.py
@@ -282,9 +282,9 @@ def buildFullGraph():
 		parent = findInSiteListById(siteList, id)['parent']
 		if type == 'site':
 			for link in dataLinks:
-				if link['from']['site']['identification']['id'] == parent:
-					if link['to']['site']['identification']['id'] == id:
-						if link['from']['device']['overview']['wirelessMode'] == 'ap-ptmp':
+				if link['from']['site'] is not None and link['from']['site']['identification']['id'] == parent:
+					if link['to']['site'] is not None and link['to']['site']['identification']['id'] == id:
+						if link['from']['device'] is not None and link['to']['device'] is not None and link['to']['device']['overview'] is not None and link['to']['device']['overview']['downlinkCapacity'] is not None and link['from']['device']['overview']['wirelessMode'] == 'ap-ptmp':
 							# Capacity of the PtMP client radio feeding the PoP will be used as the site bandwidth limit
 							download = int(round(link['to']['device']['overview']['downlinkCapacity']/1000000))
 							upload = int(round(link['to']['device']['overview']['uplinkCapacity']/1000000))