From b9ad794dc55bebb2cb343c43da078435f29ab703 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jul 2024 09:53:46 -0500 Subject: [PATCH 01/28] Start tracking bytes/second over time per-flow. This is experimental; in theory it won't cause problems, but it needs to be thoroughly tested. --- .../lqosd/src/throughput_tracker/flow_data/flow_tracker.rs | 3 +++ src/rust/lqosd/src/throughput_tracker/tracking_data.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs index 943b3600..49c52484 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs @@ -41,6 +41,8 @@ pub struct FlowbeeLocalData { pub flags: u8, /// Recent RTT median pub rtt: [RttData; 2], + /// Throughput Buffer + pub throughput_buffer: Vec>, } impl From<&FlowbeeData> for FlowbeeLocalData { @@ -56,6 +58,7 @@ impl From<&FlowbeeData> for FlowbeeLocalData { tos: data.tos, flags: data.flags, rtt: [RttData::from_nanos(0); 2], + throughput_buffer: vec![ data.bytes_sent ], } } } \ No newline at end of file diff --git a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs index 7467269d..a7e60ed9 100644 --- a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs +++ b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs @@ -200,6 +200,7 @@ impl ThroughputTracker { get_flowbee_event_count_and_reset(); let since_boot = Duration::from(now); let expire = (since_boot - Duration::from_secs(timeout_seconds)).as_nanos() as u64; + let zeroed = DownUpOrder::zeroed(); // Tracker for per-circuit RTT data. We're losing some of the smoothness by sampling // every flow; the idea is to combine them into a single entry for the circuit. This @@ -234,6 +235,8 @@ impl ThroughputTracker { this_flow.0.end_status = data.end_status; this_flow.0.tos = data.tos; this_flow.0.flags = data.flags; + let prev_bytes = this_flow.0.throughput_buffer.last().unwrap_or(&zeroed); + this_flow.0.throughput_buffer.push(data.bytes_sent.checked_sub_or_zero(*prev_bytes)); if let Some([up, down]) = rtt_samples.get(&key) { if up.as_nanos() != 0 { this_flow.0.rtt[0] = *up; From 403a0e0dcdbcb9eb60c0a7cfd50c62e9f4055d13 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jul 2024 10:18:05 -0500 Subject: [PATCH 02/28] We should now have a timeline of when retransmits happened. --- .../lqosd/src/throughput_tracker/flow_data/flow_tracker.rs | 6 ++++++ src/rust/lqosd/src/throughput_tracker/tracking_data.rs | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs index 49c52484..eb548562 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs @@ -43,6 +43,10 @@ pub struct FlowbeeLocalData { pub rtt: [RttData; 2], /// Throughput Buffer pub throughput_buffer: Vec>, + /// When did the retries happen? In nanoseconds since kernel boot + pub retry_times_down: Vec, + /// When did the retries happen? In nanoseconds since kernel boot + pub retry_times_up: Vec, } impl From<&FlowbeeData> for FlowbeeLocalData { @@ -59,6 +63,8 @@ impl From<&FlowbeeData> for FlowbeeLocalData { flags: data.flags, rtt: [RttData::from_nanos(0); 2], throughput_buffer: vec![ data.bytes_sent ], + retry_times_down: Vec::new(), + retry_times_up: Vec::new(), } } } \ No newline at end of file diff --git a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs index a7e60ed9..eec3827e 100644 --- a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs +++ b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs @@ -227,6 +227,13 @@ impl ThroughputTracker { } else { // We have a valid flow, so it needs to be tracked if let Some(this_flow) = all_flows_lock.get_mut(&key) { + // If retransmits have changed, add the time to the retry list + if data.tcp_retransmits.down != this_flow.0.tcp_retransmits.down { + this_flow.0.retry_times_down.push(data.last_seen); + } + if data.tcp_retransmits.up != this_flow.0.tcp_retransmits.up { + this_flow.0.retry_times_up.push(data.last_seen); + } this_flow.0.last_seen = data.last_seen; this_flow.0.bytes_sent = data.bytes_sent; this_flow.0.packets_sent = data.packets_sent; From 48a6ca57046b7d9b3a465ac24df5b9e9c47460b3 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jul 2024 10:45:04 -0500 Subject: [PATCH 03/28] Formatting fix --- src/rust/lqosd/src/throughput_tracker/tracking_data.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs index eec3827e..9d37fdc8 100644 --- a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs +++ b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs @@ -229,7 +229,7 @@ impl ThroughputTracker { if let Some(this_flow) = all_flows_lock.get_mut(&key) { // If retransmits have changed, add the time to the retry list if data.tcp_retransmits.down != this_flow.0.tcp_retransmits.down { - this_flow.0.retry_times_down.push(data.last_seen); + this_flow.0.retry_times_down.push(data.last_seen); } if data.tcp_retransmits.up != this_flow.0.tcp_retransmits.up { this_flow.0.retry_times_up.push(data.last_seen); From b250196d07ade8bdf8ba91a3d12336d36836e14e Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jul 2024 14:09:50 -0500 Subject: [PATCH 04/28] First try at a sparkline based ASN explorer! --- .../src/node_manager/js_build/esbuild.sh | 2 +- .../node_manager/js_build/src/asn_explorer.js | 226 ++++++++++++++++++ src/rust/lqosd/src/node_manager/local_api.rs | 3 + .../node_manager/local_api/flow_explorer.rs | 58 +++++ .../node_manager/static2/asn_explorer.html | 11 + .../src/node_manager/static2/template.html | 6 + .../lqosd/src/node_manager/static_pages.rs | 1 + .../flow_data/flow_analysis/asn.rs | 9 + .../flow_data/flow_analysis/finished_flows.rs | 35 ++- .../flow_data/flow_analysis/mod.rs | 10 + .../src/throughput_tracker/flow_data/mod.rs | 3 +- 11 files changed, 360 insertions(+), 4 deletions(-) create mode 100644 src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js create mode 100644 src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs create mode 100644 src/rust/lqosd/src/node_manager/static2/asn_explorer.html diff --git a/src/rust/lqosd/src/node_manager/js_build/esbuild.sh b/src/rust/lqosd/src/node_manager/js_build/esbuild.sh index 3afb860f..f3072966 100755 --- a/src/rust/lqosd/src/node_manager/js_build/esbuild.sh +++ b/src/rust/lqosd/src/node_manager/js_build/esbuild.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -scripts=( index.js template.js login.js first-run.js shaped-devices.js tree.js help.js unknown-ips.js configuration.js circuit.js flow_map.js all_tree_sankey.js ) +scripts=( index.js template.js login.js first-run.js shaped-devices.js tree.js help.js unknown-ips.js configuration.js circuit.js flow_map.js all_tree_sankey.js asn_explorer.js ) for script in "${scripts[@]}" do echo "Building {$script}" diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js new file mode 100644 index 00000000..897832b1 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -0,0 +1,226 @@ +import {clearDiv} from "./helpers/builders"; +import {scaleNanos, scaleNumber} from "./helpers/scaling"; + +let asnList = []; +let asnData = []; +let graphMinTime = Number.MAX_SAFE_INTEGER; +let graphMaxTime = Number.MIN_SAFE_INTEGER; + +function unixTimeToDate(unixTime) { + return new Date(unixTime * 1000); +} + +function asnDropdown() { + $.get("local-api/asnList", (data) => { + asnList = data; + + // Sort data by row.count, descending + data.sort((a, b) => { + return b.count - a.count; + }); + + // Build the dropdown + let parentDiv = document.createElement("div"); + parentDiv.classList.add("dropdown"); + let button = document.createElement("button"); + button.classList.add("btn", "btn-secondary", "dropdown-toggle"); + button.type = "button"; + button.innerHTML = "Select ASN"; + button.setAttribute("data-bs-toggle", "dropdown"); + button.setAttribute("aria-expanded", "false"); + parentDiv.appendChild(button); + let dropdownList = document.createElement("ul"); + dropdownList.classList.add("dropdown-menu"); + + // Add items + data.forEach((row) => { + let li = document.createElement("li"); + li.innerHTML = row.name + " (" + row.count + ")"; + li.classList.add("dropdown-item"); + li.onclick = () => { + selectAsn(row.asn); + }; + dropdownList.appendChild(li); + }); + + parentDiv.appendChild(dropdownList); + let target = document.getElementById("asnList"); + target.appendChild(parentDiv); + + if (data.length > 0) { + selectAsn(data[0].asn); + } + }); +} + +function selectAsn(asn) { + let targetAsn = asnList.find((row) => row.asn === asn); + if (targetAsn === undefined || targetAsn === null) { + console.error("Could not find ASN: " + asn); + return; + } + + let target = document.getElementById("asnDetails"); + + // Build the heading + let heading = document.createElement("h2"); + heading.innerText = "ASN #" + asn.toFixed(0) + " (" + targetAsn.name + ")"; + + // Get the flow data + $.get("local-api/flowTimeline/" + asn, (data) => { + asnData = data; + + // Sort data by row.start, ascending + data.sort((a, b) => { + return a.start - b.start; + }); + + // Build the flows display + let flowsDiv = document.createElement("div"); + let count = 0; + let minTime = Number.MAX_SAFE_INTEGER; + let maxTime = Number.MIN_SAFE_INTEGER; + data.forEach((row) => { + // Update min/max time + if (row.start < minTime) { + minTime = row.start; + } + if (row.end > maxTime) { + maxTime = row.end; + } + + let div = document.createElement("div"); + div.classList.add("row"); + + // Build the heading + let headingCol = document.createElement("div"); + headingCol.classList.add("col-1"); + + let ht = "

" + scaleNumber(row.total_bytes.down, 0) + " / " + scaleNumber(row.total_bytes.up); + + if (row.rtt[0] !== undefined) { + ht += "
RTT: " + scaleNanos(row.rtt[0].nanoseconds, 0); + } else { + ht += "
RTT: -"; + } + if (row.rtt[1] !== undefined) { + ht += " / " + scaleNanos(row.rtt[1].nanoseconds, 0); + } + ht += "

"; + headingCol.innerHTML = ht; + div.appendChild(headingCol); + + // Build a canvas div, we'll decorate this later + let canvasCol = document.createElement("div"); + canvasCol.classList.add("col-11"); + let canvas = document.createElement("canvas"); + canvas.id = "flowCanvas" + count; + canvas.style.width = "100%"; + canvas.style.height = "50px"; + canvasCol.appendChild(canvas); + div.appendChild(canvasCol); + + flowsDiv.appendChild(div); + count++; + }); + + // Store the global time range + graphMinTime = minTime; + graphMaxTime = maxTime; + + // Apply the data to the page + clearDiv(target); + target.appendChild(heading); + target.appendChild(flowsDiv); + + // Wait for the page to render before drawing the graphs + requestAnimationFrame(() => { + setTimeout(() => { + drawTimeline(); + }); + }); + }); +} + +function timeToX(time, width) { + let range = graphMaxTime - graphMinTime; + let offset = time - graphMinTime; + return (offset / range) * width; +} + +function drawTimeline() { + var style = getComputedStyle(document.body) + let regionBg = style.getPropertyValue('--bs-tertiary-bg'); + let lineColor = style.getPropertyValue('--bs-primary'); + + for (let i=0; i { + // Start at y/2, end at y + ctx.beginPath(); + ctx.moveTo(timeToX(time, width), height / 2); + ctx.lineTo(timeToX(time, width), height); + ctx.stroke(); + }); + row.retransmit_times_up.forEach((time) => { + // Start at 0, end at y/2 + ctx.beginPath(); + ctx.moveTo(timeToX(time, width), 0); + ctx.lineTo(timeToX(time, width), height / 2); + ctx.stroke(); + }); + + // Find the max of row.throughput.down and row.throughput.up + let maxThroughputDown = 0; + let maxThroughputUp = 0; + row.throughput.forEach((value) => { + if (value.down > maxThroughputDown) { + maxThroughputDown = value.down; + } + if (value.up > maxThroughputUp) { + maxThroughputUp = value.up; + } + }); + + // Draw a throughput down line. Y from y/2 to height, scaled to maxThroughputDown + ctx.strokeStyle = lineColor; + ctx.beginPath(); + let duration = row.end - row.start; + let numberOfSamples = row.throughput.length; + let startX = timeToX(row.start, width); + let endX = timeToX(row.end, width); + let sampleWidth = (endX - startX) / numberOfSamples; + let x = timeToX(row.start, width); + row.throughput.forEach((value, index) => { + let downPercent = value.down / maxThroughputDown; + let downHeight = downPercent * (height / 2); + let y = height - downHeight; + ctx.moveTo(x, y); + + let upPercent = value.up / maxThroughputUp; + let upHeight = upPercent * (height / 2); + ctx.lineTo(x, upHeight); + + x += sampleWidth; + }); + ctx.stroke(); + + } +} + +asnDropdown(); diff --git a/src/rust/lqosd/src/node_manager/local_api.rs b/src/rust/lqosd/src/node_manager/local_api.rs index 07fc7173..f8b0ad72 100644 --- a/src/rust/lqosd/src/node_manager/local_api.rs +++ b/src/rust/lqosd/src/node_manager/local_api.rs @@ -13,6 +13,7 @@ mod circuit; mod packet_analysis; mod flow_map; mod warnings; +mod flow_explorer; use axum::Router; use axum::routing::{get, post}; @@ -48,5 +49,7 @@ pub fn local_api() -> Router { .route("/pcapDump/:id", get(packet_analysis::pcap_dump)) .route("/flowMap", get(flow_map::flow_lat_lon)) .route("/globalWarnings", get(warnings::get_global_warnings)) + .route("/asnList", get(flow_explorer::asn_list)) + .route("/flowTimeline/:asn_id", get(flow_explorer::flow_timeline)) .route_layer(axum::middleware::from_fn(auth_layer)) } \ No newline at end of file diff --git a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs new file mode 100644 index 00000000..dc8ba177 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs @@ -0,0 +1,58 @@ +use std::time::Duration; +use axum::extract::Path; +use axum::Json; +use serde::Serialize; +use lqos_utils::units::DownUpOrder; +use lqos_utils::unix_time::{time_since_boot, unix_now}; +use crate::throughput_tracker::flow_data::{AsnListEntry, RECENT_FLOWS, RttData}; + +pub async fn asn_list() -> Json> { + Json(RECENT_FLOWS.asn_list()) +} + +#[derive(Serialize)] +pub struct FlowTimeline { + start: u64, + end: u64, + duration_nanos: u64, + throughput: Vec>, + tcp_retransmits: DownUpOrder, + rtt: [RttData; 2], + retransmit_times_down: Vec, + retransmit_times_up: Vec, + total_bytes: DownUpOrder, +} + +pub async fn flow_timeline(Path(asn_id): Path) -> Json> { + let time_since_boot = time_since_boot().unwrap(); + let since_boot = Duration::from(time_since_boot); + let boot_time = unix_now().unwrap() - since_boot.as_secs(); + + let all_flows_for_asn = RECENT_FLOWS.all_flows_for_asn(asn_id); + + let flows = all_flows_for_asn + .iter() + .map(|flow| { + + FlowTimeline { + start: boot_time + Duration::from_nanos(flow.1.start_time).as_secs(), + end: boot_time + Duration::from_nanos(flow.1.last_seen).as_secs(), + duration_nanos: flow.1.last_seen - flow.1.start_time, + tcp_retransmits: flow.1.tcp_retransmits.clone(), + throughput: flow.1.throughput_buffer.clone(), + rtt: flow.1.rtt.clone(), + retransmit_times_down: flow.1.retry_times_down + .iter() + .map(|t| boot_time + *t) + .collect(), + retransmit_times_up: flow.1.retry_times_up + .iter() + .map(|t| boot_time + *t) + .collect(), + total_bytes: flow.1.bytes_sent.clone(), + } + }) + .collect::>(); + + Json(flows) +} \ No newline at end of file diff --git a/src/rust/lqosd/src/node_manager/static2/asn_explorer.html b/src/rust/lqosd/src/node_manager/static2/asn_explorer.html new file mode 100644 index 00000000..835c7785 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/static2/asn_explorer.html @@ -0,0 +1,11 @@ +
+
+ +
+
+
+
+
+
+ + \ No newline at end of file diff --git a/src/rust/lqosd/src/node_manager/static2/template.html b/src/rust/lqosd/src/node_manager/static2/template.html index 50b8f769..7659138c 100644 --- a/src/rust/lqosd/src/node_manager/static2/template.html +++ b/src/rust/lqosd/src/node_manager/static2/template.html @@ -77,6 +77,12 @@ Tree Overview + +
  • + + ASN Explorer + +
  • diff --git a/src/rust/lqosd/src/node_manager/static_pages.rs b/src/rust/lqosd/src/node_manager/static_pages.rs index c4a61d37..4d85b2dd 100644 --- a/src/rust/lqosd/src/node_manager/static_pages.rs +++ b/src/rust/lqosd/src/node_manager/static_pages.rs @@ -32,6 +32,7 @@ pub(super) fn static_routes() -> Result { "index.html", "shaped_devices.html", "tree.html", "help.html", "unknown_ips.html", "configuration.html", "circuit.html", "flow_map.html", "all_tree_sankey.html", + "asn_explorer.html", ]; // Iterate through pages and construct the router diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs index 1141a398..f82bf272 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs @@ -168,6 +168,15 @@ impl GeoTable { (0.0, 0.0) } + + pub fn find_name_by_id(&self, id: u32) -> String { + for (_, entry) in self.asn_trie.iter() { + if entry.asn == id { + return entry.organization.clone(); + } + } + "Unknown".to_string() + } } #[derive(Default)] diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs index eb77b04c..7ed0535f 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs @@ -1,4 +1,4 @@ -use super::{get_asn_lat_lon, get_asn_name_and_country, FlowAnalysis}; +use super::{get_asn_lat_lon, get_asn_name_and_country, FlowAnalysis, get_asn_name_by_id}; use crate::throughput_tracker::flow_data::{FlowbeeLocalData, FlowbeeRecipient}; use fxhash::FxHashMap; use lqos_bus::BusResponse; @@ -14,6 +14,7 @@ pub struct TimeBuffer { buffer: Mutex>, } +#[derive(Clone, Debug)] struct TimeEntry { time: u64, data: (FlowbeeKey, FlowbeeLocalData, FlowAnalysis), @@ -25,6 +26,13 @@ pub struct FlowDurationSummary { duration: u64, } +#[derive(Debug, Serialize)] +pub struct AsnListEntry { + count: usize, + asn: u32, + name: String, +} + impl TimeBuffer { fn new() -> Self { Self { @@ -261,6 +269,31 @@ impl TimeBuffer { .map(|(count, duration)| FlowDurationSummary { count, duration }) .collect() } + + pub fn all_flows_for_asn(&self, id: u32) -> Vec<(FlowbeeKey, FlowbeeLocalData, FlowAnalysis)> { + let buffer = self.buffer.lock().unwrap(); + buffer + .iter() + .filter(|flow| flow.data.2.asn_id.0 == id ) + .map(|flow| flow.data.clone()) + .collect() + } + + /// Builds a list of all ASNs with recent data, and how many flows they have. + pub fn asn_list(&self) -> Vec { + let buffer = self.buffer.lock().unwrap(); + buffer + .iter() + .map(|flow| flow.data.2.asn_id.0) + .sorted() + .dedup_with_count() + .map(|(count, asn)| AsnListEntry { + count, + asn, + name: get_asn_name_by_id(asn), + }) + .collect() + } } pub static RECENT_FLOWS: Lazy = Lazy::new(|| TimeBuffer::new()); diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs index 6e4ee4c4..35cf7a77 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs @@ -14,6 +14,7 @@ mod kernel_ringbuffer; pub use kernel_ringbuffer::*; mod rtt_types; pub use rtt_types::RttData; +pub use finished_flows::AsnListEntry; use crate::throughput_tracker::flow_data::flow_analysis::asn::AsnNameCountryFlag; static ANALYSIS: Lazy = Lazy::new(|| FlowAnalysisSystem::new()); @@ -97,4 +98,13 @@ pub fn get_asn_lat_lon(ip: IpAddr) -> (f64, f64) { } } (0.0, 0.0) +} + +pub fn get_asn_name_by_id(id: u32) -> String { + if let Ok(table_lock) = ANALYSIS.asn_table.lock() { + if let Some(table) = table_lock.as_ref() { + return table.find_name_by_id(id); + } + } + "Unknown".to_string() } \ No newline at end of file diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs index 7c1756fc..bcdcb012 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs @@ -15,10 +15,9 @@ use std::sync::{ }; pub(crate) use flow_analysis::{setup_flow_analysis, get_asn_name_and_country, FlowAnalysis, RECENT_FLOWS, flowbee_handle_events, get_flowbee_event_count_and_reset, - expire_rtt_flows, flowbee_rtt_map, RttData, get_rtt_events_per_second, + expire_rtt_flows, flowbee_rtt_map, RttData, get_rtt_events_per_second, AsnListEntry }; - trait FlowbeeRecipient { fn enqueue(&self, key: FlowbeeKey, data: FlowbeeLocalData, analysis: FlowAnalysis); } From f920e8e1f5fd2be9b74c68f2282a77242e193968 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jul 2024 14:20:16 -0500 Subject: [PATCH 05/28] A few tweaks to get it demo-able. Don't merge! --- .../src/node_manager/js_build/src/asn_explorer.js | 10 +++++++--- .../lqosd/src/node_manager/static2/asn_explorer.html | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 897832b1..82e90208 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -47,9 +47,9 @@ function asnDropdown() { let target = document.getElementById("asnList"); target.appendChild(parentDiv); - if (data.length > 0) { + /*if (data.length > 0) { selectAsn(data[0].asn); - } + }*/ }); } @@ -68,6 +68,10 @@ function selectAsn(asn) { // Get the flow data $.get("local-api/flowTimeline/" + asn, (data) => { + // If data has more than 20 entries, only show the first 20 (temporary) + if (data.length > 20) { + data = data.slice(0, 20); + } asnData = data; // Sort data by row.start, ascending @@ -116,7 +120,7 @@ function selectAsn(asn) { let canvas = document.createElement("canvas"); canvas.id = "flowCanvas" + count; canvas.style.width = "100%"; - canvas.style.height = "50px"; + canvas.style.height = "30px"; canvasCol.appendChild(canvas); div.appendChild(canvasCol); diff --git a/src/rust/lqosd/src/node_manager/static2/asn_explorer.html b/src/rust/lqosd/src/node_manager/static2/asn_explorer.html index 835c7785..d22a5d4c 100644 --- a/src/rust/lqosd/src/node_manager/static2/asn_explorer.html +++ b/src/rust/lqosd/src/node_manager/static2/asn_explorer.html @@ -1,6 +1,6 @@
    - + Loading, Please Wait (this can take a minute)
    From 0c6ecd3e916a6992797287c95e87434fa58bf4bd Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jul 2024 14:39:43 -0500 Subject: [PATCH 06/28] Filter to non-ephemeral flows, and collect flow tracking data a bit better. --- .../src/node_manager/js_build/src/asn_explorer.js | 1 + .../src/node_manager/local_api/flow_explorer.rs | 4 ++++ .../flow_data/flow_analysis/finished_flows.rs | 4 ++++ .../lqosd/src/throughput_tracker/tracking_data.rs | 14 +++++++++++--- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 82e90208..c0433d38 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -159,6 +159,7 @@ function drawTimeline() { for (let i=0; i) -> Json> { let flows = all_flows_for_asn .iter() + .filter(|flow| { + // Total flow time > 2 seconds + flow.1.last_seen - flow.1.start_time > 2_000_000_000 + }) .map(|flow| { FlowTimeline { diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs index 7ed0535f..9f4186fc 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs @@ -284,6 +284,10 @@ impl TimeBuffer { let buffer = self.buffer.lock().unwrap(); buffer .iter() + .filter(|flow| { + // Total flow time > 2 seconds + flow.data.1.last_seen - flow.data.1.start_time > 2_000_000_000 + }) .map(|flow| flow.data.2.asn_id.0) .sorted() .dedup_with_count() diff --git a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs index 9d37fdc8..7f2cdd53 100644 --- a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs +++ b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs @@ -200,7 +200,6 @@ impl ThroughputTracker { get_flowbee_event_count_and_reset(); let since_boot = Duration::from(now); let expire = (since_boot - Duration::from_secs(timeout_seconds)).as_nanos() as u64; - let zeroed = DownUpOrder::zeroed(); // Tracker for per-circuit RTT data. We're losing some of the smoothness by sampling // every flow; the idea is to combine them into a single entry for the circuit. This @@ -234,6 +233,11 @@ impl ThroughputTracker { if data.tcp_retransmits.up != this_flow.0.tcp_retransmits.up { this_flow.0.retry_times_up.push(data.last_seen); } + + let change_since_last_time = data.bytes_sent.checked_sub_or_zero(this_flow.0.bytes_sent); + this_flow.0.throughput_buffer.push(change_since_last_time); + println!("{change_since_last_time:?}"); + this_flow.0.last_seen = data.last_seen; this_flow.0.bytes_sent = data.bytes_sent; this_flow.0.packets_sent = data.packets_sent; @@ -242,8 +246,12 @@ impl ThroughputTracker { this_flow.0.end_status = data.end_status; this_flow.0.tos = data.tos; this_flow.0.flags = data.flags; - let prev_bytes = this_flow.0.throughput_buffer.last().unwrap_or(&zeroed); - this_flow.0.throughput_buffer.push(data.bytes_sent.checked_sub_or_zero(*prev_bytes)); + + + + //let prev_bytes = this_flow.0.throughput_buffer.last().unwrap_or(&zeroed); + //this_flow.0.throughput_buffer.push(data.bytes_sent.checked_sub_or_zero(*prev_bytes)); + if let Some([up, down]) = rtt_samples.get(&key) { if up.as_nanos() != 0 { this_flow.0.rtt[0] = *up; From 680e8ba6710c8b48c317e79066f34a110f81e543 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jul 2024 15:11:07 -0500 Subject: [PATCH 07/28] Trim off the excess empty seconds on the graphs. --- .../node_manager/js_build/src/asn_explorer.js | 14 ++++++----- .../flow_data/flow_analysis/finished_flows.rs | 3 ++- .../flow_data/flow_tracker.rs | 25 +++++++++++++++++++ .../src/throughput_tracker/tracking_data.rs | 7 +----- 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index c0433d38..8fd4b75d 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -1,17 +1,18 @@ import {clearDiv} from "./helpers/builders"; import {scaleNanos, scaleNumber} from "./helpers/scaling"; +//const API_URL = "local-api/"; +const API_URL = "local-api/"; +const LIST_URL = API_URL + "asnList"; +const FLOW_URL = API_URL + "flowTimeline/"; + let asnList = []; let asnData = []; let graphMinTime = Number.MAX_SAFE_INTEGER; let graphMaxTime = Number.MIN_SAFE_INTEGER; -function unixTimeToDate(unixTime) { - return new Date(unixTime * 1000); -} - function asnDropdown() { - $.get("local-api/asnList", (data) => { + $.get(LIST_URL, (data) => { asnList = data; // Sort data by row.count, descending @@ -45,6 +46,7 @@ function asnDropdown() { parentDiv.appendChild(dropdownList); let target = document.getElementById("asnList"); + clearDiv(target); target.appendChild(parentDiv); /*if (data.length > 0) { @@ -67,7 +69,7 @@ function selectAsn(asn) { heading.innerText = "ASN #" + asn.toFixed(0) + " (" + targetAsn.name + ")"; // Get the flow data - $.get("local-api/flowTimeline/" + asn, (data) => { + $.get(FLOW_URL + asn, (data) => { // If data has more than 20 entries, only show the first 20 (temporary) if (data.length > 20) { data = data.slice(0, 20); diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs index 9f4186fc..af56851b 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs @@ -318,8 +318,9 @@ impl FinishedFlowAnalysis { } impl FlowbeeRecipient for FinishedFlowAnalysis { - fn enqueue(&self, key: FlowbeeKey, data: FlowbeeLocalData, analysis: FlowAnalysis) { + fn enqueue(&self, key: FlowbeeKey, mut data: FlowbeeLocalData, analysis: FlowAnalysis) { log::debug!("Finished flow analysis"); + data.trim(); // Remove the trailing 30 seconds of zeroes RECENT_FLOWS.push(TimeEntry { time: std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs index eb548562..5cc6286a 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs @@ -67,4 +67,29 @@ impl From<&FlowbeeData> for FlowbeeLocalData { retry_times_up: Vec::new(), } } +} + +impl FlowbeeLocalData { + pub fn trim(&mut self) { + // Find the point at which the throughput buffer starts being all zeroes + let mut last_start: Option = None; + let mut in_zero_run = false; + + for (i, &value) in self.throughput_buffer.iter().enumerate() { + if value.down == 0 && value.up == 0 { + if !in_zero_run { + in_zero_run = true; + last_start = Some(i); + } + } else { + in_zero_run = false; + } + } + + if let Some(start_index) = last_start { + // There's a run of zeroes terminating the throughput buffer + // That means we need to truncate the buffer + self.throughput_buffer.truncate(start_index); + } + } } \ No newline at end of file diff --git a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs index 7f2cdd53..81c04d36 100644 --- a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs +++ b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs @@ -236,7 +236,7 @@ impl ThroughputTracker { let change_since_last_time = data.bytes_sent.checked_sub_or_zero(this_flow.0.bytes_sent); this_flow.0.throughput_buffer.push(change_since_last_time); - println!("{change_since_last_time:?}"); + //println!("{change_since_last_time:?}"); this_flow.0.last_seen = data.last_seen; this_flow.0.bytes_sent = data.bytes_sent; @@ -247,11 +247,6 @@ impl ThroughputTracker { this_flow.0.tos = data.tos; this_flow.0.flags = data.flags; - - - //let prev_bytes = this_flow.0.throughput_buffer.last().unwrap_or(&zeroed); - //this_flow.0.throughput_buffer.push(data.bytes_sent.checked_sub_or_zero(*prev_bytes)); - if let Some([up, down]) = rtt_samples.get(&key) { if up.as_nanos() != 0 { this_flow.0.rtt[0] = *up; From 7877b8f5aef57edc3e4e8edbeb5b26b8c5afc3f1 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jul 2024 16:15:45 -0500 Subject: [PATCH 08/28] Continued tweaking - better sparklines, should fix the TCP retransmit time, add pagination. --- .../node_manager/js_build/src/asn_explorer.js | 230 +++++++++++------- .../node_manager/local_api/flow_explorer.rs | 4 +- 2 files changed, 145 insertions(+), 89 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 8fd4b75d..fd12b492 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -10,6 +10,11 @@ let asnList = []; let asnData = []; let graphMinTime = Number.MAX_SAFE_INTEGER; let graphMaxTime = Number.MIN_SAFE_INTEGER; +let throughputDownMax = 0; +let throughputUpMax = 0; + +const itemsPerPage = 20; +let page = 0; function asnDropdown() { $.get(LIST_URL, (data) => { @@ -33,6 +38,10 @@ function asnDropdown() { let dropdownList = document.createElement("ul"); dropdownList.classList.add("dropdown-menu"); + if (data.length === 0) { + data.push({asn: 0, name: "No data", count: 0}); + } + // Add items data.forEach((row) => { let li = document.createElement("li"); @@ -56,6 +65,13 @@ function asnDropdown() { } function selectAsn(asn) { + $.get(FLOW_URL + asn, (data) => { + page = 0; + renderAsn(asn, data); + }); +} + +function renderAsn(asn, data) { let targetAsn = asnList.find((row) => row.asn === asn); if (targetAsn === undefined || targetAsn === null) { console.error("Could not find ASN: " + asn); @@ -69,81 +85,117 @@ function selectAsn(asn) { heading.innerText = "ASN #" + asn.toFixed(0) + " (" + targetAsn.name + ")"; // Get the flow data - $.get(FLOW_URL + asn, (data) => { - // If data has more than 20 entries, only show the first 20 (temporary) - if (data.length > 20) { - data = data.slice(0, 20); + asnData = data; + + // Sort data by row.start, ascending + data.sort((a, b) => { + return a.start - b.start; + }); + + // Build the flows display + let flowsDiv = document.createElement("div"); + let minTime = Number.MAX_SAFE_INTEGER; + let maxTime = Number.MIN_SAFE_INTEGER; + for (let i= page * itemsPerPage; i<(page+1) * itemsPerPage; i++) { + if (i >= data.length) break; + let row = data[i]; + + // Update min/max time + if (row.start < minTime) { + minTime = row.start; + } + if (row.end > maxTime) { + maxTime = row.end; } - asnData = data; - // Sort data by row.start, ascending - data.sort((a, b) => { - return a.start - b.start; + let div = document.createElement("div"); + div.classList.add("row"); + + // Build the heading + let headingCol = document.createElement("div"); + headingCol.classList.add("col-1"); + + let ht = "

    " + scaleNumber(row.total_bytes.down, 0) + " / " + scaleNumber(row.total_bytes.up); + + if (row.rtt[0] !== undefined) { + ht += "
    RTT: " + scaleNanos(row.rtt[0].nanoseconds, 0); + } else { + ht += "
    RTT: -"; + } + if (row.rtt[1] !== undefined) { + ht += " / " + scaleNanos(row.rtt[1].nanoseconds, 0); + } + ht += "

    "; + headingCol.innerHTML = ht; + div.appendChild(headingCol); + + // Build a canvas div, we'll decorate this later + let canvasCol = document.createElement("div"); + canvasCol.classList.add("col-11"); + let canvas = document.createElement("canvas"); + canvas.id = "flowCanvas" + i; + canvas.style.width = "100%"; + canvas.style.height = "30px"; + canvasCol.appendChild(canvas); + div.appendChild(canvasCol); + + flowsDiv.appendChild(div); + } + + // Store the global time range + graphMinTime = minTime; + graphMaxTime = maxTime; + + // Calculate the max down and up for every item + let maxDown = 0; + let maxUp = 0; + data.forEach((row) => { + row.throughput.forEach((value) => { + if (value.down > maxDown) { + maxDown = value.down; + } + if (value.up > maxUp) { + maxUp = value.up; + } }); + }); + if (maxDown > throughputDownMax) { + throughputDownMax = maxDown; + } + if (maxUp > throughputUpMax) { + throughputUpMax = maxUp; + } - // Build the flows display - let flowsDiv = document.createElement("div"); - let count = 0; - let minTime = Number.MAX_SAFE_INTEGER; - let maxTime = Number.MIN_SAFE_INTEGER; - data.forEach((row) => { - // Update min/max time - if (row.start < minTime) { - minTime = row.start; - } - if (row.end > maxTime) { - maxTime = row.end; - } + // Apply the data to the page + clearDiv(target); + target.appendChild(heading); - let div = document.createElement("div"); - div.classList.add("row"); + let nextButton = document.createElement("button"); + nextButton.classList.add("btn", "btn-primary"); + nextButton.innerText = "Next"; + nextButton.onclick = () => { + page++; + if (page * itemsPerPage >= data.length) page = Math.floor(data.length / itemsPerPage); + renderAsn(asn, data); + }; + target.appendChild(nextButton); - // Build the heading - let headingCol = document.createElement("div"); - headingCol.classList.add("col-1"); + let prevButton = document.createElement("button"); + prevButton.classList.add("btn", "btn-primary"); + prevButton.innerText = "Previous"; + prevButton.onclick = () => { + page--; + if (page < 0) page = 0; + renderAsn(asn, data); + } + target.appendChild(prevButton); - let ht = "

    " + scaleNumber(row.total_bytes.down, 0) + " / " + scaleNumber(row.total_bytes.up); + target.appendChild(flowsDiv); - if (row.rtt[0] !== undefined) { - ht += "
    RTT: " + scaleNanos(row.rtt[0].nanoseconds, 0); - } else { - ht += "
    RTT: -"; - } - if (row.rtt[1] !== undefined) { - ht += " / " + scaleNanos(row.rtt[1].nanoseconds, 0); - } - ht += "

    "; - headingCol.innerHTML = ht; - div.appendChild(headingCol); - - // Build a canvas div, we'll decorate this later - let canvasCol = document.createElement("div"); - canvasCol.classList.add("col-11"); - let canvas = document.createElement("canvas"); - canvas.id = "flowCanvas" + count; - canvas.style.width = "100%"; - canvas.style.height = "30px"; - canvasCol.appendChild(canvas); - div.appendChild(canvasCol); - - flowsDiv.appendChild(div); - count++; - }); - - // Store the global time range - graphMinTime = minTime; - graphMaxTime = maxTime; - - // Apply the data to the page - clearDiv(target); - target.appendChild(heading); - target.appendChild(flowsDiv); - - // Wait for the page to render before drawing the graphs - requestAnimationFrame(() => { - setTimeout(() => { - drawTimeline(); - }); + // Wait for the page to render before drawing the graphs + requestAnimationFrame(() => { + setTimeout(() => { + drawTimeline(); }); }); } @@ -158,14 +210,16 @@ function drawTimeline() { var style = getComputedStyle(document.body) let regionBg = style.getPropertyValue('--bs-tertiary-bg'); let lineColor = style.getPropertyValue('--bs-primary'); + let axisColor = style.getPropertyValue('--bs-secondary'); - for (let i=0; i { - if (value.down > maxThroughputDown) { - maxThroughputDown = value.down; - } - if (value.up > maxThroughputUp) { - maxThroughputUp = value.up; - } - }); + // Draw a horizontal axis line the length of the canvas area at y/2 + ctx.strokeStyle = axisColor; + ctx.beginPath(); + ctx.moveTo(timeToX(row.start, width), height / 2); + ctx.lineTo(timeToX(row.end, width), height / 2); + ctx.stroke(); // Draw a throughput down line. Y from y/2 to height, scaled to maxThroughputDown ctx.strokeStyle = lineColor; @@ -213,15 +262,22 @@ function drawTimeline() { let endX = timeToX(row.end, width); let sampleWidth = (endX - startX) / numberOfSamples; let x = timeToX(row.start, width); + ctx.moveTo(x, height/2); + let trimmedHeight = height - 10; row.throughput.forEach((value, index) => { - let downPercent = value.down / maxThroughputDown; - let downHeight = downPercent * (height / 2); - let y = height - downHeight; - ctx.moveTo(x, y); + let downPercent = value.down / throughputDownMax; + let y = (height/2) - (downPercent * (trimmedHeight / 2)); + ctx.lineTo(x, y); - let upPercent = value.up / maxThroughputUp; - let upHeight = upPercent * (height / 2); - ctx.lineTo(x, upHeight); + x += sampleWidth; + }); + ctx.stroke(); + + x = timeToX(row.start, width); + row.throughput.forEach((value, index) => { + let upPercent = value.up / throughputUpMax; + let y = (height/2) + (upPercent * (trimmedHeight / 2)); + ctx.lineTo(x, y); x += sampleWidth; }); diff --git a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs index cb5234c6..a9c7fd9d 100644 --- a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs +++ b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs @@ -47,11 +47,11 @@ pub async fn flow_timeline(Path(asn_id): Path) -> Json> { rtt: flow.1.rtt.clone(), retransmit_times_down: flow.1.retry_times_down .iter() - .map(|t| boot_time + *t) + .map(|t| boot_time + Duration::from_nanos(*t).as_secs()) .collect(), retransmit_times_up: flow.1.retry_times_up .iter() - .map(|t| boot_time + *t) + .map(|t| boot_time + Duration::from_nanos(*t).as_secs()) .collect(), total_bytes: flow.1.bytes_sent.clone(), } From f37c95612edf772a7de2cd9f13eb652c7b843c41 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 26 Jul 2024 16:32:23 -0500 Subject: [PATCH 09/28] Temporarily remove the label column. --- .../node_manager/js_build/src/asn_explorer.js | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index fd12b492..3d50d102 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -127,11 +127,11 @@ function renderAsn(asn, data) { } ht += "

    "; headingCol.innerHTML = ht; - div.appendChild(headingCol); + //div.appendChild(headingCol); // Build a canvas div, we'll decorate this later let canvasCol = document.createElement("div"); - canvasCol.classList.add("col-11"); + canvasCol.classList.add("col-12"); let canvas = document.createElement("canvas"); canvas.id = "flowCanvas" + i; canvas.style.width = "100%"; @@ -171,24 +171,28 @@ function renderAsn(asn, data) { target.appendChild(heading); let nextButton = document.createElement("button"); - nextButton.classList.add("btn", "btn-primary"); - nextButton.innerText = "Next"; + nextButton.classList.add("btn", "btn-secondary", "btn-sm", "ms-2"); + nextButton.innerHTML = " Next"; nextButton.onclick = () => { page++; if (page * itemsPerPage >= data.length) page = Math.floor(data.length / itemsPerPage); renderAsn(asn, data); }; - target.appendChild(nextButton); + let prevButton = document.createElement("button"); - prevButton.classList.add("btn", "btn-primary"); - prevButton.innerText = "Previous"; + nextButton.classList.add("btn", "btn-secondary", "btn-sm", "me-2"); + prevButton.innerHTML = " Previous"; prevButton.onclick = () => { page--; if (page < 0) page = 0; renderAsn(asn, data); } - target.appendChild(prevButton); + let controlDiv = document.createElement("div"); + controlDiv.classList.add("mb-2"); + controlDiv.appendChild(prevButton); + controlDiv.appendChild(nextButton); + target.appendChild(controlDiv); target.appendChild(flowsDiv); @@ -274,6 +278,7 @@ function drawTimeline() { ctx.stroke(); x = timeToX(row.start, width); + ctx.moveTo(x, height/2); row.throughput.forEach((value, index) => { let upPercent = value.up / throughputUpMax; let y = (height/2) + (upPercent * (trimmedHeight / 2)); From a73e3af50ad644ea2ad391b25e9047e61c35f755 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jul 2024 08:10:49 -0500 Subject: [PATCH 10/28] Allow CORS to the local-api --- src/rust/lqosd/Cargo.toml | 2 +- src/rust/lqosd/src/node_manager/local_api.rs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rust/lqosd/Cargo.toml b/src/rust/lqosd/Cargo.toml index 47f74f63..cef9594e 100644 --- a/src/rust/lqosd/Cargo.toml +++ b/src/rust/lqosd/Cargo.toml @@ -39,7 +39,7 @@ zerocopy = { workspace = true } fxhash = "0.2.1" axum = { version = "0.7.5", features = ["ws", "http2"] } axum-extra = { version = "0.9.3", features = ["cookie", "cookie-private"] } -tower-http = { version = "0.5.2", features = ["fs"] } +tower-http = { version = "0.5.2", features = ["fs", "cors"] } strum = { version = "0.26.3", features = ["derive"] } default-net = { workspace = true } surge-ping = "0.8.1" diff --git a/src/rust/lqosd/src/node_manager/local_api.rs b/src/rust/lqosd/src/node_manager/local_api.rs index f8b0ad72..7b8013de 100644 --- a/src/rust/lqosd/src/node_manager/local_api.rs +++ b/src/rust/lqosd/src/node_manager/local_api.rs @@ -18,9 +18,11 @@ mod flow_explorer; use axum::Router; use axum::routing::{get, post}; use crate::node_manager::auth::auth_layer; +use tower_http::cors::CorsLayer; pub fn local_api() -> Router { Router::new() + .layer(CorsLayer::very_permissive()) .route("/dashletThemes", get(dashboard_themes::list_themes)) .route("/dashletSave", post(dashboard_themes::save_theme)) .route("/dashletDelete", post(dashboard_themes::delete_theme)) From b4d1d5deff19233d3ff296d374bc9a4f20b8a6f6 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jul 2024 08:20:16 -0500 Subject: [PATCH 11/28] Add a hashmap for looking up ASNs by number. Refactor the ASN-count code, it should be MUCH faster and gentler now. --- .../flow_data/flow_analysis/asn.rs | 15 ++++++------ .../flow_data/flow_analysis/finished_flows.rs | 24 +++++++++++++++---- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs index f82bf272..25f8c6b6 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs @@ -3,6 +3,7 @@ use std::{io::Read, net::IpAddr, path::Path}; +use fxhash::FxHashMap; use serde::Deserialize; #[derive(Deserialize, Clone, Debug)] @@ -44,6 +45,7 @@ struct Geobin { pub struct GeoTable { asn_trie: ip_network_table::IpNetworkTable, geo_trie: ip_network_table::IpNetworkTable, + asn_lookup: FxHashMap, } impl GeoTable { @@ -78,10 +80,13 @@ impl GeoTable { flate2::read::GzDecoder::new(file).read_to_end(&mut buffer)?; let geobin: Geobin = bincode::deserialize(&buffer)?; - // Build the ASN trie + // Build the ASN trie and ASN lookup map + let mut asn_lookup = FxHashMap::default(); + log::info!("Building ASN trie"); let mut asn_trie = ip_network_table::IpNetworkTable::::new(); for entry in geobin.asn { + asn_lookup.insert(entry.asn, entry.organization.clone()); let (ip, prefix) = match entry.network { IpAddr::V4(ip) => (ip.to_ipv6_mapped(), entry.prefix+96 ), IpAddr::V6(ip) => (ip, entry.prefix), @@ -109,6 +114,7 @@ impl GeoTable { Ok(Self { asn_trie, geo_trie, + asn_lookup, }) } @@ -170,12 +176,7 @@ impl GeoTable { } pub fn find_name_by_id(&self, id: u32) -> String { - for (_, entry) in self.asn_trie.iter() { - if entry.asn == id { - return entry.organization.clone(); - } - } - "Unknown".to_string() + self.asn_lookup.get(&id).cloned().unwrap_or_else(|| "Unknown".to_string()) } } diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs index af56851b..14370d1e 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs @@ -281,14 +281,28 @@ impl TimeBuffer { /// Builds a list of all ASNs with recent data, and how many flows they have. pub fn asn_list(&self) -> Vec { - let buffer = self.buffer.lock().unwrap(); - buffer - .iter() + // 1: Clone: large operation, don't keep the buffer locked longer than we have to + let buffer = { + let buffer = self.buffer.lock().unwrap(); + buffer.clone() + }; + + // Filter out short flows and reduce to the ASN ID# only + let mut buffer: Vec<_> = buffer + .into_iter() .filter(|flow| { - // Total flow time > 2 seconds - flow.data.1.last_seen - flow.data.1.start_time > 2_000_000_000 + // Total flow time > 3 seconds + flow.data.1.last_seen - flow.data.1.start_time > 3_000_000_000 }) .map(|flow| flow.data.2.asn_id.0) + .collect(); + + // Sort the buffer + buffer.sort_unstable(); + + // Deduplicate and count, decorate with name + buffer + .into_iter() .sorted() .dedup_with_count() .map(|(count, asn)| AsnListEntry { From 80995b7f95aa89ef165598ccfd66a242b0cec244 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jul 2024 09:16:29 -0500 Subject: [PATCH 12/28] The ASN explorer now displays bytes, protocol, RTT and client links. --- .../node_manager/js_build/src/asn_explorer.js | 174 +++++++++++------- .../node_manager/local_api/flow_explorer.rs | 15 ++ 2 files changed, 125 insertions(+), 64 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 3d50d102..0e5c118d 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -10,12 +10,14 @@ let asnList = []; let asnData = []; let graphMinTime = Number.MAX_SAFE_INTEGER; let graphMaxTime = Number.MIN_SAFE_INTEGER; -let throughputDownMax = 0; -let throughputUpMax = 0; const itemsPerPage = 20; let page = 0; +function unixTimeToDate(unixTime) { + return new Date(unixTime * 1000).toLocaleString(); +} + function asnDropdown() { $.get(LIST_URL, (data) => { asnList = data; @@ -45,7 +47,7 @@ function asnDropdown() { // Add items data.forEach((row) => { let li = document.createElement("li"); - li.innerHTML = row.name + " (" + row.count + ")"; + li.innerHTML = "#" + row.asn + " " + row.name + " (" + row.count + ")"; li.classList.add("dropdown-item"); li.onclick = () => { selectAsn(row.asn); @@ -92,14 +94,14 @@ function renderAsn(asn, data) { return a.start - b.start; }); - // Build the flows display - let flowsDiv = document.createElement("div"); + let div = document.createElement("div"); + div.classList.add("row"); + let minTime = Number.MAX_SAFE_INTEGER; let maxTime = Number.MIN_SAFE_INTEGER; - for (let i= page * itemsPerPage; i<(page+1) * itemsPerPage; i++) { - if (i >= data.length) break; - let row = data[i]; + // Calculate time overall + data.forEach((row) => { // Update min/max time if (row.start < minTime) { minTime = row.start; @@ -107,63 +109,87 @@ function renderAsn(asn, data) { if (row.end > maxTime) { maxTime = row.end; } - - let div = document.createElement("div"); - div.classList.add("row"); - - // Build the heading - let headingCol = document.createElement("div"); - headingCol.classList.add("col-1"); - - let ht = "

    " + scaleNumber(row.total_bytes.down, 0) + " / " + scaleNumber(row.total_bytes.up); - - if (row.rtt[0] !== undefined) { - ht += "
    RTT: " + scaleNanos(row.rtt[0].nanoseconds, 0); - } else { - ht += "
    RTT: -"; - } - if (row.rtt[1] !== undefined) { - ht += " / " + scaleNanos(row.rtt[1].nanoseconds, 0); - } - ht += "

    "; - headingCol.innerHTML = ht; - //div.appendChild(headingCol); - - // Build a canvas div, we'll decorate this later - let canvasCol = document.createElement("div"); - canvasCol.classList.add("col-12"); - let canvas = document.createElement("canvas"); - canvas.id = "flowCanvas" + i; - canvas.style.width = "100%"; - canvas.style.height = "30px"; - canvasCol.appendChild(canvas); - div.appendChild(canvasCol); - - flowsDiv.appendChild(div); - } + }); // Store the global time range graphMinTime = minTime; graphMaxTime = maxTime; - // Calculate the max down and up for every item - let maxDown = 0; - let maxUp = 0; - data.forEach((row) => { - row.throughput.forEach((value) => { - if (value.down > maxDown) { - maxDown = value.down; - } - if (value.up > maxUp) { - maxUp = value.up; - } - }); - }); - if (maxDown > throughputDownMax) { - throughputDownMax = maxDown; - } - if (maxUp > throughputUpMax) { - throughputUpMax = maxUp; + // Header row (explain the columns) + let headerDiv = document.createElement("div"); + headerDiv.classList.add("row"); + let headerBytes = document.createElement("div"); + headerBytes.classList.add("col-1", "text-secondary"); + headerBytes.innerText = "Bytes"; + headerDiv.appendChild(headerBytes); + let headerRtt = document.createElement("div"); + headerRtt.classList.add("col-1", "text-secondary"); + headerRtt.innerText = "RTT"; + headerDiv.appendChild(headerRtt); + let headerClient = document.createElement("div"); + headerClient.classList.add("col-1", "text-secondary"); + headerClient.innerText = "Client"; + headerDiv.appendChild(headerClient); + let headerProtocol = document.createElement("div"); + headerProtocol.classList.add("col-1", "text-secondary"); + headerProtocol.innerText = "Protocol"; + headerDiv.appendChild(headerProtocol); + let headerTime1 = document.createElement("div"); + headerTime1.classList.add("col-4", "text-secondary"); + headerTime1.innerText = unixTimeToDate(minTime); + headerDiv.appendChild(headerTime1); + let headerTime2 = document.createElement("div"); + headerTime2.classList.add("col-4", "text-secondary", "text-end"); + console.log(maxTime); + headerTime2.innerText = unixTimeToDate(maxTime); + headerDiv.appendChild(headerTime2); + + let flowsDiv = document.createElement("div"); + for (let i= page * itemsPerPage; i<(page+1) * itemsPerPage; i++) { + if (i >= data.length) break; + let row = data[i]; + + // Build the headings + let totalCol = document.createElement("div"); + totalCol.classList.add("col-1", "text-secondary", "small"); + totalCol.innerText = scaleNumber(row.total_bytes.down, 0) + " / " + scaleNumber(row.total_bytes.up); + div.appendChild(totalCol); + + let rttCol = document.createElement("div"); + rttCol.classList.add("col-1", "text-secondary", "small"); + let rttDown = row.rtt[0] !== undefined ? scaleNanos(row.rtt[0].nanoseconds, 0) : "-"; + let rttUp = row.rtt[1] !== undefined ? scaleNanos(row.rtt[1].nanoseconds, 0) : "-"; + rttCol.innerText = rttDown + " / " + rttUp; + div.appendChild(rttCol); + + let clientCol = document.createElement("div"); + clientCol.classList.add("col-1", "text-secondary", "small"); + if (row.circuit_id !== "") { + let clientLink = document.createElement("a"); + clientLink.href = "/circuit/" + encodeURI(row.circuit_id); + clientLink.innerText = row.circuit_name; + clientCol.appendChild(clientLink); + } else { + clientCol.innerText = row.circuit_name; + } + div.appendChild(clientCol); + + let protocolCol = document.createElement("div"); + protocolCol.classList.add("col-1", "text-secondary", "small"); + protocolCol.innerText = row.protocol; + div.appendChild(protocolCol); + + // Build a canvas div, we'll decorate this later + let canvasCol = document.createElement("div"); + canvasCol.classList.add("col-8"); + let canvas = document.createElement("canvas"); + canvas.id = "flowCanvas" + i; + canvas.style.width = "100%"; + canvas.style.height = "20px"; + canvasCol.appendChild(canvas); + div.appendChild(canvasCol); + + flowsDiv.appendChild(div); } // Apply the data to the page @@ -182,17 +208,25 @@ function renderAsn(asn, data) { let prevButton = document.createElement("button"); nextButton.classList.add("btn", "btn-secondary", "btn-sm", "me-2"); - prevButton.innerHTML = " Previous"; + prevButton.innerHTML = " Prev"; prevButton.onclick = () => { page--; if (page < 0) page = 0; renderAsn(asn, data); } + + let paginator = document.createElement("span"); + paginator.classList.add("text-secondary", "small", "ms-2", "me-2"); + paginator.innerText = "Page " + (page + 1) + " of " + Math.ceil(data.length / itemsPerPage); + paginator.id = "paginator"; + let controlDiv = document.createElement("div"); controlDiv.classList.add("mb-2"); controlDiv.appendChild(prevButton); + controlDiv.appendChild(paginator); controlDiv.appendChild(nextButton); target.appendChild(controlDiv); + target.appendChild(headerDiv); target.appendChild(flowsDiv); @@ -257,6 +291,18 @@ function drawTimeline() { ctx.lineTo(timeToX(row.end, width), height / 2); ctx.stroke(); + // Calculate maxThroughputUp and maxThroughputDown for this row + let maxThroughputDown = 0; + let maxThroughputUp = 0; + row.throughput.forEach((value) => { + if (value.down > maxThroughputDown) { + maxThroughputDown = value.down; + } + if (value.up > maxThroughputUp) { + maxThroughputUp = value.up; + } + }); + // Draw a throughput down line. Y from y/2 to height, scaled to maxThroughputDown ctx.strokeStyle = lineColor; ctx.beginPath(); @@ -267,9 +313,9 @@ function drawTimeline() { let sampleWidth = (endX - startX) / numberOfSamples; let x = timeToX(row.start, width); ctx.moveTo(x, height/2); - let trimmedHeight = height - 10; + let trimmedHeight = height - 4; row.throughput.forEach((value, index) => { - let downPercent = value.down / throughputDownMax; + let downPercent = value.down / maxThroughputDown; let y = (height/2) - (downPercent * (trimmedHeight / 2)); ctx.lineTo(x, y); @@ -280,7 +326,7 @@ function drawTimeline() { x = timeToX(row.start, width); ctx.moveTo(x, height/2); row.throughput.forEach((value, index) => { - let upPercent = value.up / throughputUpMax; + let upPercent = value.up / maxThroughputUp; let y = (height/2) + (upPercent * (trimmedHeight / 2)); ctx.lineTo(x, y); diff --git a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs index a9c7fd9d..b53a9c66 100644 --- a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs +++ b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs @@ -4,6 +4,7 @@ use axum::Json; use serde::Serialize; use lqos_utils::units::DownUpOrder; use lqos_utils::unix_time::{time_since_boot, unix_now}; +use crate::shaped_devices_tracker::SHAPED_DEVICES; use crate::throughput_tracker::flow_data::{AsnListEntry, RECENT_FLOWS, RttData}; pub async fn asn_list() -> Json> { @@ -21,6 +22,9 @@ pub struct FlowTimeline { retransmit_times_down: Vec, retransmit_times_up: Vec, total_bytes: DownUpOrder, + protocol: String, + circuit_id: String, + circuit_name: String, } pub async fn flow_timeline(Path(asn_id): Path) -> Json> { @@ -38,6 +42,14 @@ pub async fn flow_timeline(Path(asn_id): Path) -> Json> { }) .map(|flow| { + let (circuit_id, mut circuit_name) = { + let sd = SHAPED_DEVICES.read().unwrap(); + sd.get_circuit_id_and_name_from_ip(&flow.0.local_ip).unwrap_or((String::new(), String::new())) + }; + if circuit_name.is_empty() { + circuit_name = flow.0.local_ip.as_ip().to_string(); + } + FlowTimeline { start: boot_time + Duration::from_nanos(flow.1.start_time).as_secs(), end: boot_time + Duration::from_nanos(flow.1.last_seen).as_secs(), @@ -54,6 +66,9 @@ pub async fn flow_timeline(Path(asn_id): Path) -> Json> { .map(|t| boot_time + Duration::from_nanos(*t).as_secs()) .collect(), total_bytes: flow.1.bytes_sent.clone(), + protocol: flow.2.protocol_analysis.to_string(), + circuit_id, + circuit_name, } }) .collect::>(); From ac700677eef104db3cfd0f26fe7f298e4be65e10 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jul 2024 09:31:43 -0500 Subject: [PATCH 13/28] You can sort the timeline entries. --- .../node_manager/js_build/src/asn_explorer.js | 54 +++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 0e5c118d..cf8ecc6f 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -14,6 +14,13 @@ let graphMaxTime = Number.MIN_SAFE_INTEGER; const itemsPerPage = 20; let page = 0; +let sortBy = "start"; +let sortOptionsList = [ + { tag: "start", label: "Start Time" }, + { tag: "duration", label: "Duration" }, + { tag: "bytes", label: "Bytes" }, +]; + function unixTimeToDate(unixTime) { return new Date(unixTime * 1000).toLocaleString(); } @@ -89,10 +96,24 @@ function renderAsn(asn, data) { // Get the flow data asnData = data; - // Sort data by row.start, ascending - data.sort((a, b) => { - return a.start - b.start; - }); + // Sort by the selected sort key + switch (sortBy) { + case "start": { + data.sort((a, b) => { + return a.start - b.start; + }); + } break; + case "duration": { + data.sort((a, b) => { + return b.duration_nanos - a.duration_nanos; + }); + } break; + case "bytes": { + data.sort((a, b) => { + return (b.total_bytes.down + b.total_bytes.up) - (a.total_bytes.down + a.total_bytes.up); + }); + } + } let div = document.createElement("div"); div.classList.add("row"); @@ -140,7 +161,6 @@ function renderAsn(asn, data) { headerDiv.appendChild(headerTime1); let headerTime2 = document.createElement("div"); headerTime2.classList.add("col-4", "text-secondary", "text-end"); - console.log(maxTime); headerTime2.innerText = unixTimeToDate(maxTime); headerDiv.appendChild(headerTime2); @@ -220,11 +240,35 @@ function renderAsn(asn, data) { paginator.innerText = "Page " + (page + 1) + " of " + Math.ceil(data.length / itemsPerPage); paginator.id = "paginator"; + let sortOptions = document.createElement("span"); + sortOptions.classList.add("text-secondary", "small", "ms-2", "me-2"); + sortOptions.innerText = "Sort by: "; + + let sortBox = document.createElement("select"); + sortBox.classList.add("small"); + sortBox.id = "sortBox"; + sortOptionsList.forEach((option) => { + let opt = document.createElement("option"); + opt.value = option.tag; + opt.innerText = option.label; + if (option.tag === sortBy) { + opt.selected = true; + } + sortBox.appendChild(opt); + }); + sortBox.onchange = () => { + let sortBox = document.getElementById("sortBox"); + sortBy = sortBox.value; + renderAsn(asn, data); + } + let controlDiv = document.createElement("div"); controlDiv.classList.add("mb-2"); controlDiv.appendChild(prevButton); controlDiv.appendChild(paginator); controlDiv.appendChild(nextButton); + controlDiv.appendChild(sortOptions); + controlDiv.appendChild(sortBox); target.appendChild(controlDiv); target.appendChild(headerDiv); From 1f8379d0e1a28e39545ffcc768ea41bd2338a7b6 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jul 2024 09:52:51 -0500 Subject: [PATCH 14/28] Redact client names, add elipses if it grows too long. --- src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index cf8ecc6f..1a823398 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -188,6 +188,8 @@ function renderAsn(asn, data) { let clientLink = document.createElement("a"); clientLink.href = "/circuit/" + encodeURI(row.circuit_id); clientLink.innerText = row.circuit_name; + clientLink.classList.add("redactable"); + clientLink.style.textOverflow = "ellipsis"; clientCol.appendChild(clientLink); } else { clientCol.innerText = row.circuit_name; From 7b14dc1276eb04d566fd6362b0e2a3c146b691f4 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jul 2024 10:47:29 -0500 Subject: [PATCH 15/28] Experimental: list ASN timelines by country --- .../node_manager/js_build/src/asn_explorer.js | 83 ++++++++++++++++--- src/rust/lqosd/src/node_manager/local_api.rs | 2 + .../node_manager/local_api/flow_explorer.rs | 28 ++++++- .../node_manager/static2/asn_explorer.html | 5 +- .../flow_data/flow_analysis/asn.rs | 6 +- .../flow_data/flow_analysis/finished_flows.rs | 56 +++++++++++++ .../flow_data/flow_analysis/mod.rs | 2 +- .../src/throughput_tracker/flow_data/mod.rs | 3 +- 8 files changed, 164 insertions(+), 21 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 1a823398..4c15681a 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -7,12 +7,14 @@ const LIST_URL = API_URL + "asnList"; const FLOW_URL = API_URL + "flowTimeline/"; let asnList = []; +let countryList = []; let asnData = []; let graphMinTime = Number.MAX_SAFE_INTEGER; let graphMaxTime = Number.MIN_SAFE_INTEGER; const itemsPerPage = 20; let page = 0; +let renderMode = "asn"; let sortBy = "start"; let sortOptionsList = [ @@ -58,6 +60,7 @@ function asnDropdown() { li.classList.add("dropdown-item"); li.onclick = () => { selectAsn(row.asn); + renderMode = "asn"; }; dropdownList.appendChild(li); }); @@ -66,10 +69,48 @@ function asnDropdown() { let target = document.getElementById("asnList"); clearDiv(target); target.appendChild(parentDiv); + }); +} - /*if (data.length > 0) { - selectAsn(data[0].asn); - }*/ +function countryDropdown() { + $.get(API_URL + "countryList", (data) => { + countryList = data; + + // Sort data by row.count, descending + data.sort((a, b) => { + return b.count - a.count; + }); + //console.log(data); + + // Build the dropdown + let parentDiv = document.createElement("div"); + parentDiv.classList.add("dropdown"); + let button = document.createElement("button"); + button.classList.add("btn", "btn-secondary", "dropdown-toggle"); + button.type = "button"; + button.innerHTML = "Select Country"; + button.setAttribute("data-bs-toggle", "dropdown"); + button.setAttribute("aria-expanded", "false"); + parentDiv.appendChild(button); + let dropdownList = document.createElement("ul"); + dropdownList.classList.add("dropdown-menu"); + + // Add items + data.forEach((row) => { + let li = document.createElement("li"); + li.innerHTML = "" + row.iso_code + "" + row.name + " (" + row.count + ")"; + li.classList.add("dropdown-item"); + li.onclick = () => { + selectCountry(row.iso_code); + renderMode = "country"; + }; + dropdownList.appendChild(li); + }); + + parentDiv.appendChild(dropdownList); + let target = document.getElementById("countryList"); + clearDiv(target); + target.appendChild(parentDiv); }); } @@ -80,19 +121,38 @@ function selectAsn(asn) { }); } +function selectCountry(country) { + let url = API_URL + "countryTimeline/" + country; + $.get(url, (data) => { + page = 0; + renderAsn(country, data); + }); +} + function renderAsn(asn, data) { - let targetAsn = asnList.find((row) => row.asn === asn); - if (targetAsn === undefined || targetAsn === null) { - console.error("Could not find ASN: " + asn); - return; + let heading = document.createElement("h2"); + if (renderMode === "asn") { + let targetAsn = asnList.find((row) => row.asn === asn); + if (targetAsn === undefined || targetAsn === null) { + console.error("Could not find ASN: " + asn); + return; + } + + // Build the heading + heading.innerText = "ASN #" + asn.toFixed(0) + " (" + targetAsn.name + ")"; + } else if (renderMode === "country") { + let targetCountry = countryList.find((row) => row.iso_code === asn); + if (targetCountry === undefined || targetCountry === null) { + console.error("Could not find country: " + asn); + return; + } + + // Build the heading + heading.innerHTML = "" + targetCountry.iso_code + "" + targetCountry.name; } let target = document.getElementById("asnDetails"); - // Build the heading - let heading = document.createElement("h2"); - heading.innerText = "ASN #" + asn.toFixed(0) + " (" + targetAsn.name + ")"; - // Get the flow data asnData = data; @@ -384,3 +444,4 @@ function drawTimeline() { } asnDropdown(); +countryDropdown(); \ No newline at end of file diff --git a/src/rust/lqosd/src/node_manager/local_api.rs b/src/rust/lqosd/src/node_manager/local_api.rs index 7b8013de..3c1b072a 100644 --- a/src/rust/lqosd/src/node_manager/local_api.rs +++ b/src/rust/lqosd/src/node_manager/local_api.rs @@ -52,6 +52,8 @@ pub fn local_api() -> Router { .route("/flowMap", get(flow_map::flow_lat_lon)) .route("/globalWarnings", get(warnings::get_global_warnings)) .route("/asnList", get(flow_explorer::asn_list)) + .route("/countryList", get(flow_explorer::country_list)) .route("/flowTimeline/:asn_id", get(flow_explorer::flow_timeline)) + .route("/countryTimeline/:iso_code", get(flow_explorer::country_timeline)) .route_layer(axum::middleware::from_fn(auth_layer)) } \ No newline at end of file diff --git a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs index b53a9c66..2a82b599 100644 --- a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs +++ b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs @@ -2,15 +2,20 @@ use std::time::Duration; use axum::extract::Path; use axum::Json; use serde::Serialize; +use lqos_sys::flowbee_data::FlowbeeKey; use lqos_utils::units::DownUpOrder; use lqos_utils::unix_time::{time_since_boot, unix_now}; use crate::shaped_devices_tracker::SHAPED_DEVICES; -use crate::throughput_tracker::flow_data::{AsnListEntry, RECENT_FLOWS, RttData}; +use crate::throughput_tracker::flow_data::{AsnListEntry, AsnCountryListEntry, RECENT_FLOWS, RttData, FlowbeeLocalData, FlowAnalysis}; pub async fn asn_list() -> Json> { Json(RECENT_FLOWS.asn_list()) } +pub async fn country_list() -> Json> { + Json(RECENT_FLOWS.country_list()) +} + #[derive(Serialize)] pub struct FlowTimeline { start: u64, @@ -34,14 +39,19 @@ pub async fn flow_timeline(Path(asn_id): Path) -> Json> { let all_flows_for_asn = RECENT_FLOWS.all_flows_for_asn(asn_id); - let flows = all_flows_for_asn + let flows = all_flows_to_transport(boot_time, all_flows_for_asn); + + Json(flows) +} + +fn all_flows_to_transport(boot_time: u64, all_flows_for_asn: Vec<(FlowbeeKey, FlowbeeLocalData, FlowAnalysis)>) -> Vec { + all_flows_for_asn .iter() .filter(|flow| { // Total flow time > 2 seconds flow.1.last_seen - flow.1.start_time > 2_000_000_000 }) .map(|flow| { - let (circuit_id, mut circuit_name) = { let sd = SHAPED_DEVICES.read().unwrap(); sd.get_circuit_id_and_name_from_ip(&flow.0.local_ip).unwrap_or((String::new(), String::new())) @@ -71,7 +81,17 @@ pub async fn flow_timeline(Path(asn_id): Path) -> Json> { circuit_name, } }) - .collect::>(); + .collect::>() +} + +pub async fn country_timeline(Path(iso_code): Path) -> Json> { + let time_since_boot = time_since_boot().unwrap(); + let since_boot = Duration::from(time_since_boot); + let boot_time = unix_now().unwrap() - since_boot.as_secs(); + + let all_flows_for_asn = RECENT_FLOWS.all_flows_for_country(&iso_code); + + let flows = all_flows_to_transport(boot_time, all_flows_for_asn); Json(flows) } \ No newline at end of file diff --git a/src/rust/lqosd/src/node_manager/static2/asn_explorer.html b/src/rust/lqosd/src/node_manager/static2/asn_explorer.html index d22a5d4c..4cd6f2b6 100644 --- a/src/rust/lqosd/src/node_manager/static2/asn_explorer.html +++ b/src/rust/lqosd/src/node_manager/static2/asn_explorer.html @@ -1,7 +1,10 @@
    -
    +
    Loading, Please Wait (this can take a minute)
    +
    + Loading, Please Wait (this can take a minute) +
    diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs index 25f8c6b6..17b3597f 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs @@ -139,9 +139,9 @@ impl GeoTable { IpAddr::V4(ip) => ip.to_ipv6_mapped(), IpAddr::V6(ip) => ip, }; - let mut owners = String::new(); - let mut country = String::new(); - let mut flag = String::new(); + let mut owners = "Unknown".to_string(); + let mut country = "Unknown".to_string(); + let mut flag = "Unknown".to_string(); if let Some(matched) = self.asn_trie.longest_match(ip) { log::debug!("Matched ASN: {:?}", matched.1.asn); diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs index 14370d1e..99f85de1 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs @@ -33,6 +33,13 @@ pub struct AsnListEntry { name: String, } +#[derive(Debug, Serialize)] +pub struct AsnCountryListEntry { + count: usize, + name: String, + iso_code: String, +} + impl TimeBuffer { fn new() -> Self { Self { @@ -279,6 +286,18 @@ impl TimeBuffer { .collect() } + pub fn all_flows_for_country(&self, iso_code: &str) -> Vec<(FlowbeeKey, FlowbeeLocalData, FlowAnalysis)> { + let buffer = self.buffer.lock().unwrap(); + buffer + .iter() + .filter(|flow| { + let country = get_asn_name_and_country(flow.data.0.remote_ip.as_ip()); + country.flag == iso_code + }) + .map(|flow| flow.data.clone()) + .collect() + } + /// Builds a list of all ASNs with recent data, and how many flows they have. pub fn asn_list(&self) -> Vec { // 1: Clone: large operation, don't keep the buffer locked longer than we have to @@ -312,6 +331,43 @@ impl TimeBuffer { }) .collect() } + + /// Builds a list of ASNs by country with recent data, and how many flows they have. + pub fn country_list(&self) -> Vec { + // 1: Clone: large operation, don't keep the buffer locked longer than we have to + let buffer = { + let buffer = self.buffer.lock().unwrap(); + buffer.clone() + }; + + // Filter out the short flows and get the country & flag + let mut buffer: Vec<(String, String)> = buffer + .into_iter() + .filter(|flow| { + // Total flow time > 3 seconds + flow.data.1.last_seen - flow.data.1.start_time > 3_000_000_000 + }) + .map(|flow| { + let country = get_asn_name_and_country(flow.data.0.remote_ip.as_ip()); + (country.country, country.flag) + }) + .collect(); + + // Sort the buffer + buffer.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + + // Deduplicate and count, decorate with name + buffer + .into_iter() + .sorted() + .dedup_with_count() + .map(|(count, asn)| AsnCountryListEntry { + count, + name: asn.0, + iso_code: asn.1, + }) + .collect() + } } pub static RECENT_FLOWS: Lazy = Lazy::new(|| TimeBuffer::new()); diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs index 35cf7a77..e6dcfe34 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs @@ -14,7 +14,7 @@ mod kernel_ringbuffer; pub use kernel_ringbuffer::*; mod rtt_types; pub use rtt_types::RttData; -pub use finished_flows::AsnListEntry; +pub use finished_flows::{AsnListEntry, AsnCountryListEntry}; use crate::throughput_tracker::flow_data::flow_analysis::asn::AsnNameCountryFlag; static ANALYSIS: Lazy = Lazy::new(|| FlowAnalysisSystem::new()); diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs index bcdcb012..785eec82 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs @@ -15,7 +15,8 @@ use std::sync::{ }; pub(crate) use flow_analysis::{setup_flow_analysis, get_asn_name_and_country, FlowAnalysis, RECENT_FLOWS, flowbee_handle_events, get_flowbee_event_count_and_reset, - expire_rtt_flows, flowbee_rtt_map, RttData, get_rtt_events_per_second, AsnListEntry + expire_rtt_flows, flowbee_rtt_map, RttData, get_rtt_events_per_second, AsnListEntry, + AsnCountryListEntry }; trait FlowbeeRecipient { From eebc6372e4aa08eb39eb54255a5979c5470e60ff Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jul 2024 10:50:55 -0500 Subject: [PATCH 16/28] Fix case on flag path --- src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 4c15681a..7eea8b18 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -148,7 +148,7 @@ function renderAsn(asn, data) { } // Build the heading - heading.innerHTML = "" + targetCountry.iso_code + "" + targetCountry.name; + heading.innerHTML = "" + targetCountry.iso_code + "" + targetCountry.name; } let target = document.getElementById("asnDetails"); From ec8707ac6fb3875c7d45f616501e323e244fa417 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 27 Jul 2024 10:51:32 -0500 Subject: [PATCH 17/28] Fix case on flag path --- src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 7eea8b18..dca96d4d 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -98,7 +98,7 @@ function countryDropdown() { // Add items data.forEach((row) => { let li = document.createElement("li"); - li.innerHTML = "" + row.iso_code + "" + row.name + " (" + row.count + ")"; + li.innerHTML = "" + row.iso_code + "" + row.name + " (" + row.count + ")"; li.classList.add("dropdown-item"); li.onclick = () => { selectCountry(row.iso_code); From 1a3c8df787f26506039c2243bb0e1abd164732c8 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 09:27:18 -0500 Subject: [PATCH 18/28] Change country selection from flag to name matching, to ensure that America/Foo matches only that region - and not everything with the stars and stripes. --- src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js | 2 +- src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs | 4 ++-- .../flow_data/flow_analysis/finished_flows.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index dca96d4d..68a49391 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -101,7 +101,7 @@ function countryDropdown() { li.innerHTML = "" + row.iso_code + "" + row.name + " (" + row.count + ")"; li.classList.add("dropdown-item"); li.onclick = () => { - selectCountry(row.iso_code); + selectCountry(row.name); renderMode = "country"; }; dropdownList.appendChild(li); diff --git a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs index 2a82b599..2d52bf15 100644 --- a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs +++ b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs @@ -84,12 +84,12 @@ fn all_flows_to_transport(boot_time: u64, all_flows_for_asn: Vec<(FlowbeeKey, Fl .collect::>() } -pub async fn country_timeline(Path(iso_code): Path) -> Json> { +pub async fn country_timeline(Path(country_name): Path) -> Json> { let time_since_boot = time_since_boot().unwrap(); let since_boot = Duration::from(time_since_boot); let boot_time = unix_now().unwrap() - since_boot.as_secs(); - let all_flows_for_asn = RECENT_FLOWS.all_flows_for_country(&iso_code); + let all_flows_for_asn = RECENT_FLOWS.all_flows_for_country(&country_name); let flows = all_flows_to_transport(boot_time, all_flows_for_asn); diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs index 99f85de1..87f982e1 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs @@ -286,13 +286,13 @@ impl TimeBuffer { .collect() } - pub fn all_flows_for_country(&self, iso_code: &str) -> Vec<(FlowbeeKey, FlowbeeLocalData, FlowAnalysis)> { + pub fn all_flows_for_country(&self, country_name: &str) -> Vec<(FlowbeeKey, FlowbeeLocalData, FlowAnalysis)> { let buffer = self.buffer.lock().unwrap(); buffer .iter() .filter(|flow| { let country = get_asn_name_and_country(flow.data.0.remote_ip.as_ip()); - country.flag == iso_code + country.name == country_name }) .map(|flow| flow.data.clone()) .collect() From bb73aea9c6281d484d6e85cb8795d542e9a4401f Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 09:55:11 -0500 Subject: [PATCH 19/28] Support display by protocol --- .../node_manager/js_build/src/asn_explorer.js | 62 +++++++++++++++++-- src/rust/lqosd/src/node_manager/local_api.rs | 2 + .../node_manager/local_api/flow_explorer.rs | 20 +++++- .../node_manager/static2/asn_explorer.html | 3 + .../flow_data/flow_analysis/finished_flows.rs | 52 ++++++++++++++++ .../flow_data/flow_analysis/mod.rs | 2 +- .../src/throughput_tracker/flow_data/mod.rs | 2 +- 7 files changed, 135 insertions(+), 8 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 68a49391..f7b8bd42 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -8,6 +8,7 @@ const FLOW_URL = API_URL + "flowTimeline/"; let asnList = []; let countryList = []; +let protocolList = []; let asnData = []; let graphMinTime = Number.MAX_SAFE_INTEGER; let graphMaxTime = Number.MIN_SAFE_INTEGER; @@ -40,7 +41,7 @@ function asnDropdown() { let parentDiv = document.createElement("div"); parentDiv.classList.add("dropdown"); let button = document.createElement("button"); - button.classList.add("btn", "btn-secondary", "dropdown-toggle"); + button.classList.add("btn", "btn-secondary", "dropdown-toggle", "btn-sm"); button.type = "button"; button.innerHTML = "Select ASN"; button.setAttribute("data-bs-toggle", "dropdown"); @@ -86,7 +87,7 @@ function countryDropdown() { let parentDiv = document.createElement("div"); parentDiv.classList.add("dropdown"); let button = document.createElement("button"); - button.classList.add("btn", "btn-secondary", "dropdown-toggle"); + button.classList.add("btn", "btn-secondary", "dropdown-toggle", "btn-sm"); button.type = "button"; button.innerHTML = "Select Country"; button.setAttribute("data-bs-toggle", "dropdown"); @@ -114,21 +115,71 @@ function countryDropdown() { }); } +function protocolDropdown() { + $.get(API_URL + "protocolList", (data) => { + protocolList = data; + + // Sort data by row.count, descending + data.sort((a, b) => { + return b.count - a.count; + }); + //console.log(data); + + // Build the dropdown + let parentDiv = document.createElement("div"); + parentDiv.classList.add("dropdown"); + let button = document.createElement("button"); + button.classList.add("btn", "btn-secondary", "dropdown-toggle", "btn-sm"); + button.type = "button"; + button.innerHTML = "Select Protocol"; + button.setAttribute("data-bs-toggle", "dropdown"); + button.setAttribute("aria-expanded", "false"); + parentDiv.appendChild(button); + let dropdownList = document.createElement("ul"); + dropdownList.classList.add("dropdown-menu"); + + // Add items + data.forEach((row) => { + let li = document.createElement("li"); + li.innerHTML = row.protocol + " (" + row.count + ")"; + li.classList.add("dropdown-item"); + li.onclick = () => { + selectProtocol(row.protocol); + renderMode = "protocol"; + }; + dropdownList.appendChild(li); + }); + + parentDiv.appendChild(dropdownList); + let target = document.getElementById("protocolList"); + clearDiv(target); + target.appendChild(parentDiv); + }); +} + function selectAsn(asn) { - $.get(FLOW_URL + asn, (data) => { + $.get(FLOW_URL + encodeURI(asn), (data) => { page = 0; renderAsn(asn, data); }); } function selectCountry(country) { - let url = API_URL + "countryTimeline/" + country; + let url = API_URL + "countryTimeline/" + encodeURI(country); $.get(url, (data) => { page = 0; renderAsn(country, data); }); } +function selectProtocol(protocol) { + let url = API_URL + "protocolTimeline/" + encodeURI(protocol.replace('/', '_')); + $.get(url, (data) => { + page = 0; + renderAsn(protocol, data); + }); +} + function renderAsn(asn, data) { let heading = document.createElement("h2"); if (renderMode === "asn") { @@ -444,4 +495,5 @@ function drawTimeline() { } asnDropdown(); -countryDropdown(); \ No newline at end of file +countryDropdown(); +protocolDropdown(); \ No newline at end of file diff --git a/src/rust/lqosd/src/node_manager/local_api.rs b/src/rust/lqosd/src/node_manager/local_api.rs index 3c1b072a..974c57f8 100644 --- a/src/rust/lqosd/src/node_manager/local_api.rs +++ b/src/rust/lqosd/src/node_manager/local_api.rs @@ -53,7 +53,9 @@ pub fn local_api() -> Router { .route("/globalWarnings", get(warnings::get_global_warnings)) .route("/asnList", get(flow_explorer::asn_list)) .route("/countryList", get(flow_explorer::country_list)) + .route("/protocolList", get(flow_explorer::protocol_list)) .route("/flowTimeline/:asn_id", get(flow_explorer::flow_timeline)) .route("/countryTimeline/:iso_code", get(flow_explorer::country_timeline)) + .route("/protocolTimeline/:protocol", get(flow_explorer::protocol_timeline)) .route_layer(axum::middleware::from_fn(auth_layer)) } \ No newline at end of file diff --git a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs index 2d52bf15..18360b11 100644 --- a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs +++ b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs @@ -6,7 +6,8 @@ use lqos_sys::flowbee_data::FlowbeeKey; use lqos_utils::units::DownUpOrder; use lqos_utils::unix_time::{time_since_boot, unix_now}; use crate::shaped_devices_tracker::SHAPED_DEVICES; -use crate::throughput_tracker::flow_data::{AsnListEntry, AsnCountryListEntry, RECENT_FLOWS, RttData, FlowbeeLocalData, FlowAnalysis}; +use crate::throughput_tracker::flow_data::{AsnListEntry, AsnCountryListEntry, AsnProtocolListEntry, + RECENT_FLOWS, RttData, FlowbeeLocalData, FlowAnalysis}; pub async fn asn_list() -> Json> { Json(RECENT_FLOWS.asn_list()) @@ -16,6 +17,10 @@ pub async fn country_list() -> Json> { Json(RECENT_FLOWS.country_list()) } +pub async fn protocol_list() -> Json> { + Json(RECENT_FLOWS.protocol_list()) +} + #[derive(Serialize)] pub struct FlowTimeline { start: u64, @@ -93,5 +98,18 @@ pub async fn country_timeline(Path(country_name): Path) -> Json) -> Json> { + let protocol_name = protocol_name.replace("_", "/"); + let time_since_boot = time_since_boot().unwrap(); + let since_boot = Duration::from(time_since_boot); + let boot_time = unix_now().unwrap() - since_boot.as_secs(); + + let all_flows_for_asn = RECENT_FLOWS.all_flows_for_protocol(&protocol_name); + + let flows = all_flows_to_transport(boot_time, all_flows_for_asn); + Json(flows) } \ No newline at end of file diff --git a/src/rust/lqosd/src/node_manager/static2/asn_explorer.html b/src/rust/lqosd/src/node_manager/static2/asn_explorer.html index 4cd6f2b6..dfa8f860 100644 --- a/src/rust/lqosd/src/node_manager/static2/asn_explorer.html +++ b/src/rust/lqosd/src/node_manager/static2/asn_explorer.html @@ -5,6 +5,9 @@
    Loading, Please Wait (this can take a minute)
    +
    + Loading, Please Wait (this can take a minute) +
    diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs index 87f982e1..dc5b73ac 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs @@ -40,6 +40,12 @@ pub struct AsnCountryListEntry { iso_code: String, } +#[derive(Debug, Serialize)] +pub struct AsnProtocolListEntry { + count: usize, + protocol: String, +} + impl TimeBuffer { fn new() -> Self { Self { @@ -298,6 +304,17 @@ impl TimeBuffer { .collect() } + pub fn all_flows_for_protocol(&self, protocol_name: &str) -> Vec<(FlowbeeKey, FlowbeeLocalData, FlowAnalysis)> { + let buffer = self.buffer.lock().unwrap(); + buffer + .iter() + .filter(|flow| { + flow.data.2.protocol_analysis.to_string() == protocol_name + }) + .map(|flow| flow.data.clone()) + .collect() + } + /// Builds a list of all ASNs with recent data, and how many flows they have. pub fn asn_list(&self) -> Vec { // 1: Clone: large operation, don't keep the buffer locked longer than we have to @@ -368,6 +385,41 @@ impl TimeBuffer { }) .collect() } + + /// Builds a list of protocols with recent data, and how many flows they have. + pub fn protocol_list(&self) -> Vec { + // 1: Clone: large operation, don't keep the buffer locked longer than we have to + let buffer = { + let buffer = self.buffer.lock().unwrap(); + buffer.clone() + }; + + // Filter out the short flows and get the country & flag + let mut buffer: Vec = buffer + .into_iter() + .filter(|flow| { + // Total flow time > 3 seconds + flow.data.1.last_seen - flow.data.1.start_time > 3_000_000_000 + }) + .map(|flow| { + flow.data.2.protocol_analysis.to_string() + }) + .collect(); + + // Sort the buffer + buffer.sort_unstable_by(|a, b| a.cmp(&b)); + + // Deduplicate and count, decorate with name + buffer + .into_iter() + .sorted() + .dedup_with_count() + .map(|(count, protocol)| AsnProtocolListEntry { + count, + protocol, + }) + .collect() + } } pub static RECENT_FLOWS: Lazy = Lazy::new(|| TimeBuffer::new()); diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs index e6dcfe34..010808b8 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs @@ -14,7 +14,7 @@ mod kernel_ringbuffer; pub use kernel_ringbuffer::*; mod rtt_types; pub use rtt_types::RttData; -pub use finished_flows::{AsnListEntry, AsnCountryListEntry}; +pub use finished_flows::{AsnListEntry, AsnCountryListEntry, AsnProtocolListEntry}; use crate::throughput_tracker::flow_data::flow_analysis::asn::AsnNameCountryFlag; static ANALYSIS: Lazy = Lazy::new(|| FlowAnalysisSystem::new()); diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs index 785eec82..f5b4248e 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs @@ -16,7 +16,7 @@ use std::sync::{ pub(crate) use flow_analysis::{setup_flow_analysis, get_asn_name_and_country, FlowAnalysis, RECENT_FLOWS, flowbee_handle_events, get_flowbee_event_count_and_reset, expire_rtt_flows, flowbee_rtt_map, RttData, get_rtt_events_per_second, AsnListEntry, - AsnCountryListEntry + AsnCountryListEntry, AsnProtocolListEntry, }; trait FlowbeeRecipient { From b4872a4f094c55035a0995c3366fbe1fe763b762 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 09:58:53 -0500 Subject: [PATCH 20/28] Display Remote IP --- .../node_manager/js_build/src/asn_explorer.js | 18 +++++++++++++----- .../node_manager/local_api/flow_explorer.rs | 2 ++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index f7b8bd42..3bf37791 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -262,6 +262,10 @@ function renderAsn(asn, data) { headerClient.classList.add("col-1", "text-secondary"); headerClient.innerText = "Client"; headerDiv.appendChild(headerClient); + let headerRemote = document.createElement("div"); + headerRemote.classList.add("col-1", "text-secondary"); + headerRemote.innerText = "Remote"; + headerDiv.appendChild(headerRemote); let headerProtocol = document.createElement("div"); headerProtocol.classList.add("col-1", "text-secondary"); headerProtocol.innerText = "Protocol"; @@ -271,7 +275,7 @@ function renderAsn(asn, data) { headerTime1.innerText = unixTimeToDate(minTime); headerDiv.appendChild(headerTime1); let headerTime2 = document.createElement("div"); - headerTime2.classList.add("col-4", "text-secondary", "text-end"); + headerTime2.classList.add("col-3", "text-secondary", "text-end"); headerTime2.innerText = unixTimeToDate(maxTime); headerDiv.appendChild(headerTime2); @@ -307,6 +311,11 @@ function renderAsn(asn, data) { } div.appendChild(clientCol); + let remoteCol = document.createElement("div"); + remoteCol.classList.add("col-1", "text-secondary", "small"); + remoteCol.innerText = row.remote_ip; + div.appendChild(remoteCol); + let protocolCol = document.createElement("div"); protocolCol.classList.add("col-1", "text-secondary", "small"); protocolCol.innerText = row.protocol; @@ -314,7 +323,7 @@ function renderAsn(asn, data) { // Build a canvas div, we'll decorate this later let canvasCol = document.createElement("div"); - canvasCol.classList.add("col-8"); + canvasCol.classList.add("col-7"); let canvas = document.createElement("canvas"); canvas.id = "flowCanvas" + i; canvas.style.width = "100%"; @@ -463,7 +472,6 @@ function drawTimeline() { // Draw a throughput down line. Y from y/2 to height, scaled to maxThroughputDown ctx.strokeStyle = lineColor; ctx.beginPath(); - let duration = row.end - row.start; let numberOfSamples = row.throughput.length; let startX = timeToX(row.start, width); let endX = timeToX(row.end, width); @@ -471,7 +479,7 @@ function drawTimeline() { let x = timeToX(row.start, width); ctx.moveTo(x, height/2); let trimmedHeight = height - 4; - row.throughput.forEach((value, index) => { + row.throughput.forEach((value) => { let downPercent = value.down / maxThroughputDown; let y = (height/2) - (downPercent * (trimmedHeight / 2)); ctx.lineTo(x, y); @@ -482,7 +490,7 @@ function drawTimeline() { x = timeToX(row.start, width); ctx.moveTo(x, height/2); - row.throughput.forEach((value, index) => { + row.throughput.forEach((value) => { let upPercent = value.up / maxThroughputUp; let y = (height/2) + (upPercent * (trimmedHeight / 2)); ctx.lineTo(x, y); diff --git a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs index 18360b11..6331ceca 100644 --- a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs +++ b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs @@ -35,6 +35,7 @@ pub struct FlowTimeline { protocol: String, circuit_id: String, circuit_name: String, + remote_ip: String, } pub async fn flow_timeline(Path(asn_id): Path) -> Json> { @@ -84,6 +85,7 @@ fn all_flows_to_transport(boot_time: u64, all_flows_for_asn: Vec<(FlowbeeKey, Fl protocol: flow.2.protocol_analysis.to_string(), circuit_id, circuit_name, + remote_ip: flow.0.remote_ip.as_ip().to_string(), } }) .collect::>() From 59d1116893befb1ccb6b72ab343f7f2bc69432fa Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 10:04:18 -0500 Subject: [PATCH 21/28] Remember to display the protocol in the header! --- src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 3bf37791..6049d48d 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -200,6 +200,9 @@ function renderAsn(asn, data) { // Build the heading heading.innerHTML = "" + targetCountry.iso_code + "" + targetCountry.name; + } else if (renderMode === "protocol") { + // Build the heading + heading.innerText = "Protocol: " + asn; } let target = document.getElementById("asnDetails"); From 5174571cd8145b89dcc7551bbfa52c5d79d05839 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 10:19:58 -0500 Subject: [PATCH 22/28] Fix search by country - using wrong key --- src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 6049d48d..c650e7af 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -192,7 +192,7 @@ function renderAsn(asn, data) { // Build the heading heading.innerText = "ASN #" + asn.toFixed(0) + " (" + targetAsn.name + ")"; } else if (renderMode === "country") { - let targetCountry = countryList.find((row) => row.iso_code === asn); + let targetCountry = countryList.find((row) => row.name === asn); if (targetCountry === undefined || targetCountry === null) { console.error("Could not find country: " + asn); return; From c6f351352bd2de87db44dec04fb289edea1e6036 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 10:28:06 -0500 Subject: [PATCH 23/28] Put CORS at the bottom of Local API --- src/rust/lqosd/src/node_manager/local_api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rust/lqosd/src/node_manager/local_api.rs b/src/rust/lqosd/src/node_manager/local_api.rs index 974c57f8..9550ffcd 100644 --- a/src/rust/lqosd/src/node_manager/local_api.rs +++ b/src/rust/lqosd/src/node_manager/local_api.rs @@ -22,7 +22,6 @@ use tower_http::cors::CorsLayer; pub fn local_api() -> Router { Router::new() - .layer(CorsLayer::very_permissive()) .route("/dashletThemes", get(dashboard_themes::list_themes)) .route("/dashletSave", post(dashboard_themes::save_theme)) .route("/dashletDelete", post(dashboard_themes::delete_theme)) @@ -57,5 +56,6 @@ pub fn local_api() -> Router { .route("/flowTimeline/:asn_id", get(flow_explorer::flow_timeline)) .route("/countryTimeline/:iso_code", get(flow_explorer::country_timeline)) .route("/protocolTimeline/:protocol", get(flow_explorer::protocol_timeline)) + .layer(CorsLayer::very_permissive()) .route_layer(axum::middleware::from_fn(auth_layer)) } \ No newline at end of file From 5d33a7ed6baa3394670be24dbdf3ba9bf21fbb49 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 10:29:20 -0500 Subject: [PATCH 24/28] Revert the country selection to something that functions. --- src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs | 4 ++-- .../flow_data/flow_analysis/finished_flows.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs index 6331ceca..d5b89783 100644 --- a/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs +++ b/src/rust/lqosd/src/node_manager/local_api/flow_explorer.rs @@ -91,12 +91,12 @@ fn all_flows_to_transport(boot_time: u64, all_flows_for_asn: Vec<(FlowbeeKey, Fl .collect::>() } -pub async fn country_timeline(Path(country_name): Path) -> Json> { +pub async fn country_timeline(Path(iso_code): Path) -> Json> { let time_since_boot = time_since_boot().unwrap(); let since_boot = Duration::from(time_since_boot); let boot_time = unix_now().unwrap() - since_boot.as_secs(); - let all_flows_for_asn = RECENT_FLOWS.all_flows_for_country(&country_name); + let all_flows_for_asn = RECENT_FLOWS.all_flows_for_country(&iso_code); let flows = all_flows_to_transport(boot_time, all_flows_for_asn); diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs index dc5b73ac..4e2ce9a6 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs @@ -292,13 +292,13 @@ impl TimeBuffer { .collect() } - pub fn all_flows_for_country(&self, country_name: &str) -> Vec<(FlowbeeKey, FlowbeeLocalData, FlowAnalysis)> { + pub fn all_flows_for_country(&self, iso_code: &str) -> Vec<(FlowbeeKey, FlowbeeLocalData, FlowAnalysis)> { let buffer = self.buffer.lock().unwrap(); buffer .iter() .filter(|flow| { let country = get_asn_name_and_country(flow.data.0.remote_ip.as_ip()); - country.name == country_name + country.flag == iso_code }) .map(|flow| flow.data.clone()) .collect() From 42d6efc8400d789b11a68770d5d31f09c14f9e93 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 10:37:24 -0500 Subject: [PATCH 25/28] Revert the country selection to something that functions. --- src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index c650e7af..668948ff 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -102,7 +102,7 @@ function countryDropdown() { li.innerHTML = "" + row.iso_code + "" + row.name + " (" + row.count + ")"; li.classList.add("dropdown-item"); li.onclick = () => { - selectCountry(row.name); + selectCountry(row.flag); renderMode = "country"; }; dropdownList.appendChild(li); From 32d746894a450b96d05b90bdfdd125e89fdc6b34 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 10:38:11 -0500 Subject: [PATCH 26/28] Revert the country selection to something that functions. --- src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 668948ff..01e8c3e5 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -192,7 +192,7 @@ function renderAsn(asn, data) { // Build the heading heading.innerText = "ASN #" + asn.toFixed(0) + " (" + targetAsn.name + ")"; } else if (renderMode === "country") { - let targetCountry = countryList.find((row) => row.name === asn); + let targetCountry = countryList.find((row) => row.flag === asn); if (targetCountry === undefined || targetCountry === null) { console.error("Could not find country: " + asn); return; From f41e4c9133d0a8ffdd6557dc088261521bd4a7b0 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 10:39:29 -0500 Subject: [PATCH 27/28] Revert the country selection to something that functions. --- src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js index 01e8c3e5..94506c0f 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_explorer.js @@ -102,7 +102,7 @@ function countryDropdown() { li.innerHTML = "" + row.iso_code + "" + row.name + " (" + row.count + ")"; li.classList.add("dropdown-item"); li.onclick = () => { - selectCountry(row.flag); + selectCountry(row.iso_code); renderMode = "country"; }; dropdownList.appendChild(li); @@ -192,7 +192,7 @@ function renderAsn(asn, data) { // Build the heading heading.innerText = "ASN #" + asn.toFixed(0) + " (" + targetAsn.name + ")"; } else if (renderMode === "country") { - let targetCountry = countryList.find((row) => row.flag === asn); + let targetCountry = countryList.find((row) => row.iso_code === asn); if (targetCountry === undefined || targetCountry === null) { console.error("Could not find country: " + asn); return; From cabbcf92c07561976cbe928ce3875264f12ace9e Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Mon, 29 Jul 2024 10:57:49 -0500 Subject: [PATCH 28/28] Improved redaction for Sankey displays --- .../lqosd/src/node_manager/js_build/src/all_tree_sankey.js | 2 +- .../node_manager/js_build/src/dashlets/top_tree_sankey.js | 5 ++--- .../src/node_manager/js_build/src/graphs/top_n_sankey.js | 2 ++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/all_tree_sankey.js b/src/rust/lqosd/src/node_manager/js_build/src/all_tree_sankey.js index 46ed2881..91876805 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/all_tree_sankey.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/all_tree_sankey.js @@ -98,7 +98,7 @@ function start() { fontSize: 6, color: "#999" }; - if (redact) label.fontSize = 0; + if (redact) label.backgroundColor = label.color; nodes.push({ name: data[i][1].name, diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_tree_sankey.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_tree_sankey.js index 45819201..a93f621d 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_tree_sankey.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_tree_sankey.js @@ -1,6 +1,5 @@ import {BaseDashlet} from "./base_dashlet"; -import {clearDiv, simpleRowHtml, theading} from "../helpers/builders"; -import {formatThroughput, formatRetransmit, formatCakeStat, lerpGreenToRedViaOrange} from "../helpers/scaling"; +import {lerpGreenToRedViaOrange} from "../helpers/scaling"; import {DashboardGraph} from "../graphs/dashboard_graph"; import {isRedacted} from "../helpers/redact"; @@ -92,7 +91,7 @@ export class TopTreeSankey extends BaseDashlet { fontSize: 9, color: "#999" }; - if (redact) label.fontSize = 0; + if (redact) label.backgroundColor = label.color; let name = r[1].name; let bytes = r[1].current_throughput[0]; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/top_n_sankey.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/top_n_sankey.js index fddcbbe4..757bd539 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/top_n_sankey.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/top_n_sankey.js @@ -1,5 +1,6 @@ import {DashboardGraph} from "./dashboard_graph"; import {lerpColor, lerpGreenToRedViaOrange, scaleNumber} from "../helpers/scaling"; +import {isRedacted} from "../helpers/redact"; export class TopNSankey extends DashboardGraph { constructor(id) { @@ -59,6 +60,7 @@ export class TopNSankey extends DashboardGraph { fontSize: 9, color: "#999" }; + if (isRedacted()) label.backgroundColor = label.color; let name = r.ip_address+ " (" + scaleNumber(r.bits_per_second.down, 0) + ", " + r.tcp_retransmits.down + "/" + r.tcp_retransmits.up + ")"; let bytes = r.bits_per_second.down / 8;