IntenseWebs/LibreQoS
3
0
Fork 0
mirror of https://github.com/LibreQoE/LibreQoS.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

Heimdall circuit UI speed (#296)

Browse Source 
* Circuit queue - lazy rendering by active tab

Only perform network requests for the active tab on the
circuit_queue.html page. Small to moderate performance
improvement, but it greatly reduces the time spent polling.

* Significantly optimize network and rendering performance of the queues funnel display.

* Substantially improve performance on the flows display by using msgpack with a dictionary.

* Remove some commented code

* Fetch circuit info via efficient msgpack

* Use MsgPack for circuit throughput

* Get rid of the monstrosity that was copying queue data over the bus in a raw JSON string, hideously wasteful. Instead, we now have a 'transit' series of objects in the bus definition for tight encoding. This also cleaned up some node manager code. It's mostly useful for the next stage, which will start to reduce the amount of data we are transmitting.

* A lot of the redundant network transport is culled.

* More unused fields culled.

* Elimate a HUGE amount of garbage collection by allocating and reusing a single object, and cleaning up the JS rendering. Still not good enough.

* Switch to an efficient msgpack transmission format.

* Cleanup handling of 'none' in msgpack

* Fix scale delays to ms

* Commit to send to payne

* Use WebGL for a slight rendering boost.

* Further reduce draw time of circuit page by using redraw commands.

* Finish previous commit

* Use redraw with preallocated/non-GC data for all ringbuffer renders.

* Fix a rare issue with reloading network.json that could cause a stall.

* Optimize RTT graphs with the reload system.
This commit is contained in:
Herbert "TheBracket
2023-03-25 09:37:04 -05:00
committed by GitHub
parent d53200f43d
commit 0cf7d5dd0a
15 changed files with 845 additions and 429 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/rust/lqos_bus/src/bus/mod.rs
View File

@@ -5,6 +5,7 @@ mod request;
mod response;
mod session;
mod unix_socket_server;
mod queue_data;
pub use client::bus_request;
use log::error;
pub use persistent_client::BusClient;
@@ -14,6 +15,7 @@ pub use response::BusResponse;
pub use session::BusSession;
use thiserror::Error;
pub use unix_socket_server::UnixSocketServer;
pub use queue_data::*;
/// The local socket path to which `lqosd` will bind itself,
/// listening for requets.

103
src/rust/lqos_bus/src/bus/queue_data.rs Normal file
View File

@@ -0,0 +1,103 @@
use serde::{Serialize, Deserialize};
/// Type used for *displaying* the queue store data. It deliberately
/// doesn't include data that we aren't going to display in a GUI.
#[allow(missing_docs)]
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
pub struct QueueStoreTransit {
pub history: Vec<(CakeDiffTransit, CakeDiffTransit)>,
pub history_head: usize,
//pub prev_download: Option<CakeTransit>,
//pub prev_upload: Option<CakeTransit>,
pub current_download: CakeTransit,
pub current_upload: CakeTransit,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
#[allow(missing_docs)]
pub struct CakeDiffTransit {
pub bytes: u64,
pub packets: u32,
pub qlen: u32,
pub tins: Vec<CakeDiffTinTransit>,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
#[allow(missing_docs)]
pub struct CakeDiffTinTransit {
pub sent_bytes: u64,
pub backlog_bytes: u32,
pub drops: u32,
pub marks: u32,
pub avg_delay_us: u32,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
#[allow(missing_docs)]
pub struct CakeTransit {
//pub handle: TcHandle,
//pub parent: TcHandle,
//pub bytes: u64,
//pub packets: u32,
//pub overlimits: u32,
//pub requeues: u32,
//pub backlog: u32,
//pub qlen: u32,
pub memory_used: u32,
//pub memory_limit: u32,
//pub capacity_estimate: u32,
//pub min_network_size: u16,
//pub max_network_size: u16,
//pub min_adj_size: u16,
//pub max_adj_size: u16,
//pub avg_hdr_offset: u16,
//pub tins: Vec<CakeTinTransit>,
//pub drops: u32,
}
/*
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
#[allow(missing_docs)]
pub struct CakeOptionsTransit {
pub rtt: u64,
pub bandwidth: u8,
pub diffserv: u8,
pub flowmode: u8,
pub ack_filter: u8,
pub nat: bool,
pub wash: bool,
pub ingress: bool,
pub split_gso: bool,
pub raw: bool,
pub overhead: u16,
pub fwmark: TcHandle,
}
// Commented out data is collected but not used
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
#[allow(missing_docs)]
pub struct CakeTinTransit {
//pub threshold_rate: u64,
//pub sent_bytes: u64,
//pub backlog_bytes: u32,
//pub target_us: u32,
//pub interval_us: u32,
//pub peak_delay_us: u32,
//pub avg_delay_us: u32,
//pub base_delay_us: u32,
//pub sent_packets: u32,
//pub way_indirect_hits: u16,
//pub way_misses: u16,
//pub way_collisions: u16,
//pub drops: u32,
//pub ecn_marks: u32,
//pub ack_drops: u32,
//pub sparse_flows: u16,
//pub bulk_flows: u16,
//pub unresponsive_flows: u16,
//pub max_pkt_len: u16,
//pub flow_quantum: u16,
}
*/

3
src/rust/lqos_bus/src/bus/response.rs
View File

@@ -1,6 +1,7 @@
use crate::{IpMapping, IpStats, XdpPpingResult, FlowTransport, ip_stats::PacketHeader};
use serde::{Deserialize, Serialize};
use std::net::IpAddr;
use super::QueueStoreTransit;
/// A `BusResponse` object represents a single
/// reply generated from a `BusRequest`, and batched
@@ -67,7 +68,7 @@ pub enum BusResponse {
/// A string containing a JSON dump of a queue stats. Analagos to
/// the response from `tc show qdisc`.
RawQueueData(String),
RawQueueData(Option<Box<QueueStoreTransit>>),
/// Results from network map queries
NetworkMap(Vec<(usize, lqos_config::NetworkJsonTransport)>),

6
src/rust/lqos_bus/src/lib.rs
View File

@@ -12,11 +12,15 @@
#![warn(missing_docs)]
mod bus;
mod ip_stats;
pub use ip_stats::{IpMapping, IpStats, XdpPpingResult, FlowProto, FlowTransport, tos_parser, PacketHeader};
pub use ip_stats::{
tos_parser, FlowProto, FlowTransport, IpMapping, IpStats, PacketHeader,
XdpPpingResult,
};
mod tc_handle;
pub use bus::{
bus_request, decode_request, decode_response, encode_request,
encode_response, BusClient, BusReply, BusRequest, BusResponse, BusSession,
CakeDiffTinTransit, CakeDiffTransit, CakeTransit, QueueStoreTransit,
UnixSocketServer, BUS_SOCKET_PATH,
};
pub use tc_handle::TcHandle;

1
src/rust/lqos_config/src/network_json/mod.rs
View File

@@ -179,6 +179,7 @@ impl NetworkJson {
&self,
circuit_id: &str,
) -> Option<Vec<usize>> {
//println!("Looking for parents of {circuit_id}");
self
.nodes
.iter()

4
src/rust/lqos_node_manager/src/network_tree.rs
View File

@@ -106,7 +106,7 @@ pub async fn node_names(
#[get("/api/funnel_for_queue/<circuit_id>")]
pub async fn funnel_for_queue(
circuit_id: String,
) -> NoCache<Json<Vec<(usize, NetworkJsonTransport)>>> {
) -> NoCache<MsgPack<Vec<(usize, NetworkJsonTransport)>>> {
let mut result = Vec::new();
let target = SHAPED_DEVICES
@@ -127,5 +127,5 @@ pub async fn funnel_for_queue(
result.extend_from_slice(map);
}
}
NoCache::new(Json(result))
NoCache::new(MsgPack(result))
}

29
src/rust/lqos_node_manager/src/queue_info.rs
View File

@@ -1,12 +1,13 @@
use crate::auth_guard::AuthGuard;
use crate::cache_control::NoCache;
use crate::tracker::SHAPED_DEVICES;
use lqos_bus::{bus_request, BusRequest, BusResponse, FlowTransport, PacketHeader};
use lqos_bus::{bus_request, BusRequest, BusResponse, FlowTransport, PacketHeader, QueueStoreTransit};
use rocket::fs::NamedFile;
use rocket::http::Status;
use rocket::response::content::RawJson;
use rocket::serde::json::Json;
use rocket::serde::Serialize;
use rocket::serde::msgpack::MsgPack;
use std::net::IpAddr;
#[derive(Serialize, Clone)]
@@ -30,7 +31,7 @@ pub async fn watch_circuit(
pub async fn circuit_info(
circuit_id: String,
_auth: AuthGuard,
) -> NoCache<Json<CircuitInfo>> {
) -> NoCache<MsgPack<CircuitInfo>> {
if let Some(device) = SHAPED_DEVICES
.read()
.unwrap()
@@ -45,13 +46,13 @@ pub async fn circuit_info(
device.upload_max_mbps as u64 * 1_000_000,
),
};
NoCache::new(Json(result))
NoCache::new(MsgPack(result))
} else {
let result = CircuitInfo {
name: "Nameless".to_string(),
capacity: (1_000_000, 1_000_000),
};
NoCache::new(Json(result))
NoCache::new(MsgPack(result))
}
}
@@ -59,7 +60,7 @@ pub async fn circuit_info(
pub async fn current_circuit_throughput(
circuit_id: String,
_auth: AuthGuard,
) -> NoCache<Json<Vec<(String, u64, u64)>>> {
) -> NoCache<MsgPack<Vec<(String, u64, u64)>>> {
let mut result = Vec::new();
// Get a list of host counts
// This is really inefficient, but I'm struggling to find a better way.
@@ -84,25 +85,29 @@ pub async fn current_circuit_throughput(
}
}
NoCache::new(Json(result))
NoCache::new(MsgPack(result))
}
#[get("/api/raw_queue_by_circuit/<circuit_id>")]
pub async fn raw_queue_by_circuit(
circuit_id: String,
_auth: AuthGuard,
) -> NoCache<RawJson<String>> {
) -> NoCache<MsgPack<QueueStoreTransit>> {
let responses =
bus_request(vec![BusRequest::GetRawQueueData(circuit_id)]).await.unwrap();
let result = match &responses[0] {
BusResponse::RawQueueData(msg) => msg.clone(),
_ => "Unable to request queue".to_string(),
BusResponse::RawQueueData(Some(msg)) => {
*msg.clone()
}
_ => QueueStoreTransit::default()
};
NoCache::new(RawJson(result))
NoCache::new(MsgPack(result))
}
#[get("/api/flows/<ip_list>")]
pub async fn flow_stats(ip_list: String, _auth: AuthGuard) -> NoCache<Json<Vec<(FlowTransport, Option<FlowTransport>)>>> {
pub async fn flow_stats(ip_list: String, _auth: AuthGuard) -> NoCache<MsgPack<Vec<(FlowTransport, Option<FlowTransport>)>>> {
let mut result = Vec::new();
let request: Vec<BusRequest> = ip_list.split(',').map(|ip| BusRequest::GetFlowStats(ip.to_string())).collect();
let responses = bus_request(request).await.unwrap();
@@ -111,7 +116,7 @@ pub async fn flow_stats(ip_list: String, _auth: AuthGuard) -> NoCache<Json<Vec<(
result.extend_from_slice(flow);
}
}
NoCache::new(Json(result))
NoCache::new(MsgPack(result))
}
#[derive(Serialize, Clone)]

757
src/rust/lqos_node_manager/static/circuit_queue.html
View File

@@ -1,5 +1,6 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
@@ -9,15 +10,20 @@
<title>LibreQoS - Local Node Manager</title>
<script src="/lqos.js"></script>
<script src="/vendor/plotly-2.16.1.min.js"></script>
<script src="/vendor/jquery.min.js"></script><script src="/vendor/msgpack.min.js"></script>
<script src="/vendor/jquery.min.js"></script>
<script src="/vendor/msgpack.min.js"></script>
<script defer src="/vendor/bootstrap.bundle.min.js"></script>
</head>
<body class="bg-secondary">
<!-- Navigation -->
<nav class="navbar navbar-expand-lg navbar-dark bg-dark">
<div class="container-fluid">
<a class="navbar-brand" href="/"><img src="/vendor/tinylogo.svg" alt="LibreQoS SVG Logo" width="25" height="25" />&nbsp;LibreQoS</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<a class="navbar-brand" href="/"><img src="/vendor/tinylogo.svg" alt="LibreQoS SVG Logo" width="25"
height="25" />&nbsp;LibreQoS</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse"
data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false"
aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
@@ -26,10 +32,13 @@
<a class="nav-link" href="/tree?parent=0"><i class="fa fa-tree"></i> Tree</a>
</li>
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="/shaped"><i class="fa fa-users"></i> Shaped Devices <span id="shapedCount" class="badge badge-pill badge-success green-badge">?</span></a>
<a class="nav-link active" aria-current="page" href="/shaped"><i class="fa fa-users"></i> Shaped
Devices <span id="shapedCount"
class="badge badge-pill badge-success green-badge">?</span></a>
</li>
<li class="nav-item">
<a class="nav-link" href="/unknown"><i class="fa fa-address-card"></i> Unknown IPs <span id="unshapedCount" class="badge badge-warning orange-badge">?</span></a>
<a class="nav-link" href="/unknown"><i class="fa fa-address-card"></i> Unknown IPs <span
id="unshapedCount" class="badge badge-warning orange-badge">?</span></a>
</li>
</ul>
</div>
@@ -37,13 +46,15 @@
<ul class="navbar-nav ms-auto">
<li class="nav-item" id="currentLogin"></li>
<li class="nav-item">
<a class="nav-link" href="#" id="startTest"><i class="fa fa-flag-checkered"></i> Run Bandwidth Test</a>
<a class="nav-link" href="#" id="startTest"><i class="fa fa-flag-checkered"></i> Run Bandwidth
Test</a>
</li>
<li class="nav-item ms-auto">
<a class="nav-link" href="/config"><i class="fa fa-gear"></i> Configuration</a>
</li>
<li>
<a class="nav-link btn btn-small black-txt" href="#" id="btnReload"><i class="fa fa-refresh"></i> Reload LibreQoS</a>
<a class="nav-link btn btn-small black-txt" href="#" id="btnReload"><i class="fa fa-refresh"></i>
Reload LibreQoS</a>
</li>
</ul>
</div>
@@ -60,16 +71,24 @@
<div class="col-sm-6">
<ul class="nav nav-pills mb-3" id="pills-tab" role="tablist">
<li class="nav-item" role="presentation">
<button class="nav-link active" id="pills-home-tab" data-bs-toggle="pill" data-bs-target="#pills-home" type="button" role="tab" aria-controls="pills-home" aria-selected="true">Overview</button>
<button class="nav-link active" id="pills-home-tab" data-bs-toggle="pill"
data-bs-target="#pills-home" type="button" role="tab" aria-controls="pills-home"
aria-selected="true">Overview</button>
</li>
<li class="nav-item" role="presentation">
<button class="nav-link" id="pills-tins-tab" data-bs-toggle="pill" data-bs-target="#pills-tins" type="button" role="tab" aria-controls="pills-profile" aria-selected="false">All Tins</button>
<button class="nav-link" id="pills-tins-tab" data-bs-toggle="pill"
data-bs-target="#pills-tins" type="button" role="tab" aria-controls="pills-profile"
aria-selected="false">All Tins</button>
</li>
<li class="nav-item" role="presentation">
<button class="nav-link" id="pills-funnel-tab" data-bs-toggle="pill" data-bs-target="#pills-funnel" type="button" role="tab" aria-controls="pills-funnel" aria-selected="false">Queue Tree</button>
<button class="nav-link" id="pills-funnel-tab" data-bs-toggle="pill"
data-bs-target="#pills-funnel" type="button" role="tab" aria-controls="pills-funnel"
aria-selected="false">Queue Tree</button>
</li>
<li class="nav-item" role="presentation">
<button class="nav-link" id="pills-flows-tab" data-bs-toggle="pill" data-bs-target="#pills-flows" type="button" role="tab" aria-controls="pills-flows" aria-selected="false">Flows</button>
<button class="nav-link" id="pills-flows-tab" data-bs-toggle="pill"
data-bs-target="#pills-flows" type="button" role="tab" aria-controls="pills-flows"
aria-selected="false">Flows</button>
</li>
</ul>
</div>
@@ -81,7 +100,8 @@
</div>
<div class="tab-content" id="pills-tabContent">
<div class="tab-pane fade show active" id="pills-home" role="tabpanel" aria-labelledby="pills-home-tab" tabindex="0">
<div class="tab-pane fade show active" id="pills-home" role="tabpanel" aria-labelledby="pills-home-tab"
tabindex="0">
<!-- Total Throughput and Backlog -->
<div class="row">
@@ -104,7 +124,7 @@
<div class="col-sm-4">
<div class="card bg-light">
<div class="card-body">
<h5 class="card-title"><i class="fa fa-bar-chart"></i> Capacity Quantile</h5>
<h5 class="card-title"><i class="fa fa-bar-chart"></i> Capacity Quantile (Last 10s)</h5>
<div id="capacityQuantile" class="graph150"></div>
</div>
</div>
@@ -185,7 +205,8 @@
</div>
</div>
<div class="tab-pane fade" id="pills-funnel" role="tabpanel" aria-labelledby="pills-funnel-tab" tabindex="2">
<div class="tab-pane fade" id="pills-funnel" role="tabpanel" aria-labelledby="pills-funnel-tab"
tabindex="2">
</div>
<div class="tab-pane fade" id="pills-flows" role="tabpanel" aria-labelledby="pills-flows-tab" tabindex="3">
@@ -195,7 +216,8 @@
<div class="card-body">
<h5 class="card-title"><i class="fa fa-bar-chart"></i> Flows (Last 30 Seconds)</h5>
<p class="alert alert-warning" role="alert">
<i class="fa fa-warning"></i> Gathering packet data can cause high CPU load during the capture window.
<i class="fa fa-warning"></i> Gathering packet data can cause high CPU load during
the capture window.
</p>
<div id="packetButtons"></div>
<div id="flowList"></div>
@@ -214,305 +236,362 @@
let throughput_head = 0;
let circuit_info = null;
function setX(x, counter) {
for (let i=0; i<x.length; i++) {
x[i].push(counter);
function nameCircuit() {
if (circuit_info == null) {
msgPackGet("/api/circuit_info/" + encodeURI(id), (data) => {
circuit_info = data;
let capacity = scaleNumber(circuit_info[CircuitInfo.capacity][0]) + " / " + scaleNumber(circuit_info[CircuitInfo.capacity][1]);
$("#circuitName").text(redactText(circuit_info[CircuitInfo.name]) + " " + capacity);
});
}
}
function setY(y, i, data, tin) {
if (data[0] == "None" || data[1] == "None") {
for (let j=0; i<y.length; y++) {
y[j].push(0);
function displayMemory(data) {
// Fill Base Information
let total_memory = data[QD.current_download][CT.memory_used] + data[QD.current_upload][CT.memory_used];
$("#memory").text(scaleNumber(total_memory));
}
class CombinedPlot {
constructor(capacity) {
this.y = []
for (let i = 0; i < capacity * 2; ++i) {
this.y.push(0);
}
}
store(x, y, value) {
if (value == 0) value = null;
this.y[(x * 2) + y] = value;
}
}
class TinsPlot {
constructor(capacity) {
this.tins = [
new CombinedPlot(capacity),
new CombinedPlot(capacity),
new CombinedPlot(capacity),
new CombinedPlot(capacity)
];
}
store(tin, x, y, value) {
this.tins[tin].store(x, y, value);
}
}
class QueuePlotter {
constructor(capacity) {
this.capacity = capacity;
this.x_axis = [];
this.backlog = new TinsPlot(capacity);
this.delays = new TinsPlot(capacity);
this.queueLen = new CombinedPlot(capacity);
this.throughput = new TinsPlot(capacity);
this.drops = new TinsPlot(capacity);
this.marks = new TinsPlot(capacity);
for (let i = 0; i < capacity; ++i) {
this.x_axis.push(i);
this.x_axis.push(i);
}
}
ingestBacklog(subData, currentX, tin) {
this.backlog.store(tin, currentX, 0, subData[0][CDT.tins][tin][CDTT.backlog_bytes] * 8);
this.backlog.store(tin, currentX, 1, 0.0 - subData[1][CDT.tins][tin][CDTT.backlog_bytes] * 8);
}
ingestDelays(subData, currentX, tin) {
this.delays.store(tin, currentX, 0, subData[0][CDT.tins][tin][CDTT.avg_delay_us] * 0.001);
this.delays.store(tin, currentX, 1, 0.0 - subData[1][CDT.tins][tin][CDTT.avg_delay_us] * 0.001);
}
ingestQueueLen(subData, currentX) {
this.queueLen.store(currentX, 0, subData[0][CDT.qlen]);
this.queueLen.store(currentX, 1, 0.0 - subData[1][CDT.qlen]);
}
ingestThroughput(subData, currentX, tin) {
this.throughput.store(tin, currentX, 0, subData[0][CDT.tins][tin][CDTT.sent_bytes] * 8);
this.throughput.store(tin, currentX, 1, 0.0 - (subData[1][CDT.tins][tin][CDTT.sent_bytes] * 8));
}
ingestDrops(subData, currentX, tin) {
this.drops.store(tin, currentX, 0, subData[0][CDT.tins][tin][CDTT.drops]);
this.drops.store(tin, currentX, 1, 0.0 - subData[1][CDT.tins][tin][CDTT.drops]);
}
ingestMarks(subData, currentX, tin) {
this.marks.store(tin, currentX, 0, subData[0][CDT.tins][tin][CDTT.marks]);
this.marks.store(tin, currentX, 1, 0.0 - subData[1][CDT.tins][tin][CDTT.marks]);
}
ingest(data, currentX, hi) {
// We're inserting at currentX, from the history entry indexed
// by hi
if (activeTab == "pills-home-tab") {
this.ingestQueueLen(data[QD.history][hi], currentX);
}
for (let tin = 0; tin < 4; ++tin) {
if (data[QD.history][hi][0][3].length == 4 && data[QD.history][hi][1][3].length == 4) {
if (activeTab == "pills-home-tab") {
this.ingestBacklog(data[QD.history][hi], currentX, tin);
this.ingestDelays(data[QD.history][hi], currentX, tin);
} else if (activeTab == "pills-tins-tab") {
this.ingestThroughput(data[QD.history][hi], currentX, tin);
this.ingestDrops(data[QD.history][hi], currentX, tin);
this.ingestMarks(data[QD.history][hi], currentX, tin);
}
}
}
}
update(data) {
// Iterate the whole history ringbuffer
// Note that we're going backwards. reverse() turned out
// to be surprisingly expensive in JS.
let currentX = this.capacity;
for (let hi = data[QD.history_head]; hi < 600; ++hi) {
this.ingest(data, currentX, hi);
currentX--;
}
for (let hi = 0; hi < data[QD.history_head]; ++hi) {
this.ingest(data, currentX, hi);
currentX--;
}
}
plotBacklog() {
let graph = document.getElementById("backlogGraph");
let graphData = [
{ x: this.x_axis, y: this.backlog.tins[0].y, type: 'scattergl', mode: 'markers', name: 'Bulk', marker: { size: 4 } },
{ x: this.x_axis, y: this.backlog.tins[1].y, type: 'scattergl', mode: 'markers', name: 'Best Effort', marker: { size: 4 } },
{ x: this.x_axis, y: this.backlog.tins[2].y, type: 'scattergl', mode: 'markers', name: 'Video', marker: { size: 4 } },
{ x: this.x_axis, y: this.backlog.tins[3].y, type: 'scattergl', mode: 'markers', name: 'Voice', marker: { size: 4 } },
];
if (this.backlogPlotted == null) {
this.backlogPlotted = true;
Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Bytes" }, xaxis: { automargin: true, title: "Time since now" } });
} else {
y[0].push(data[0].Cake.tins[tin].sent_bytes * 8); // Download
y[1].push(0.0 - (data[1].Cake.tins[tin].sent_bytes * 8)); // Upload
y[2].push(data[0].Cake.tins[tin].drops); // Down Drops
y[3].push(data[0].Cake.tins[tin].marks); // Down Marks
y[4].push(0.0 - data[1].Cake.tins[tin].drops); // Up Drops
y[5].push(0.0 - data[1].Cake.tins[tin].marks); // Up Marks
// Backlog
y[6].push(data[0].Cake.tins[tin].backlog_bytes * 8);
y[7].push(0.0 - data[1].Cake.tins[tin].backlog_bytes * 8);
// Delays
y[8].push(data[0].Cake.tins[tin].avg_delay_us);
y[9].push(0.0 - data[1].Cake.tins[tin].avg_delay_us);
Plotly.redraw(graph, graphData);
}
}
plotDelays() {
let graph = document.getElementById("delayGraph");
let graphData = [
{ x: this.x_axis, y: this.delays.tins[0].y, type: 'scattergl', mode: 'markers', name: 'Bulk', marker: { size: 4 } },
{ x: this.x_axis, y: this.delays.tins[1].y, type: 'scattergl', mode: 'markers', name: 'Best Effort', marker: { size: 4 } },
{ x: this.x_axis, y: this.delays.tins[2].y, type: 'scattergl', mode: 'markers', name: 'Video', marker: { size: 4 } },
{ x: this.x_axis, y: this.delays.tins[3].y, type: 'scattergl', mode: 'markers', name: 'Voice', marker: { size: 4 } },
];
if (this.delaysPlotted == null) {
Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "ms" }, xaxis: { automargin: true, title: "Time since now" } });
this.delaysPlotted = true;
} else {
Plotly.redraw(graph, graphData);
}
}
plotQueueLen() {
let graph = document.getElementById("qlenGraph");
let graphData = [
{ x: this.x_axis, y: this.queueLen.y, type: 'scattergl', mode: 'markers', name: 'Queue Length' },
];
if (this.queueLenPlotted == null) {
Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Packets" }, xaxis: { automargin: true, title: "Time since now" } });
this.queueLenPlotted = true;
} else {
Plotly.redraw(graph, graphData);
}
}
plotTinThroughput(tin) {
let graph = document.getElementById("tinTp_" + tin);
let graphData = [
{ x: this.x_axis, y: this.throughput.tins[tin].y, type: 'scattergl', mode: 'markers' }
];
if (this.tinsPlotted == null) {
Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Bits" }, xaxis: { automargin: true, title: "Time since now" } });
} else {
Plotly.redraw(graph, graphData);
}
}
plotMarksDrops(tin) {
let graph = document.getElementById("tinMd_" + tin);
let graphData = [
{ x: this.x_axis, y: this.drops.tins[tin].y, name: 'Drops', type: 'scattergl', mode: 'markers', marker: { size: 4 } },
{ x: this.x_axis, y: this.marks.tins[tin].y, name: 'Marks', type: 'scattergl', mode: 'markers', marker: { size: 4 } },
];
if (this.tinsPlotted == null) {
Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Packets" }, xaxis: { automargin: true, title: "Time since now" } });
} else {
Plotly.redraw(graph, graphData);
}
}
plot() {
if (activeTab == "pills-home-tab") {
this.plotBacklog();
this.plotDelays();
this.plotQueueLen();
} else if (activeTab == "pills-tins-tab") {
for (let tin = 0; tin < 4; ++tin) {
this.plotTinThroughput(tin);
this.plotMarksDrops(tin);
}
this.tinsPlotted = true;
}
}
}
let qp = null;
function pollQueue() {
const params = new Proxy(new URLSearchParams(window.location.search), {
get: (searchParams, prop) => searchParams.get(prop),
});
if (params.id != null) {
if (id != null) {
// Name the circuit
if (circuit_info == null) {
$.get("/api/circuit_info/" + encodeURI(params.id), (data) => {
circuit_info = data;
let capacity = scaleNumber(circuit_info.capacity[0]) + " / " + scaleNumber(circuit_info.capacity[1]);
$("#circuitName").text(redactText(circuit_info.name) + " " + capacity);
});
}
// Fill the raw button
$("#raw").html("<a class='btn btn-sm btn-info' href='/api/raw_queue_by_circuit/" + encodeURI(params.id) + "'><i class='fa fa-search'></i> Raw Data</a>");
nameCircuit();
// Graphs
$.get("/api/raw_queue_by_circuit/" + encodeURI(params.id), (data) => {
// Fill Base Information
let total_memory = data.current_download.Cake.memory_used + data.current_upload.Cake.memory_used;
$("#memory").text(scaleNumber(total_memory));
// Fill Tin Graphs
let backlogX1 = [];
let backlogY1 = [];
let backlogX2 = [];
let backlogY2 = [];
let delaysX1 = [];
let delaysX2 = [];
let delaysY1 = [];
let delaysY2 = [];
let qlenX1 = [];
let qlenY1 = [];
let qlenX2 = [];
let qlenY2 = [];
for (let tin=0; tin<4; tin++) {
let entries = {
x: [[], [], [], [], [], [], [], [], [], []],
y: [[], [], [], [], [], [], [], [], [], []],
}
let counter = 0;
for (let i=data.history_head; i<600; i++) {
setX(entries.x, counter);
setY(entries.y, i, data.history[i], tin);
if (tin == 0) {
qlenX1.push(counter);
qlenX2.push(counter);
if (data.history[i][0].Cake) {
qlenY1.push(data.history[i][0].Cake.qlen);
} else {
qlenY1.push(0);
}
if (data.history[i][1].Cake) {
qlenY2.push(0.0 - data.history[i][1].Cake.qlen);
} else {
qlenY2.push(0.0);
}
}
counter++;
}
for (let i=0; i<data.history_head; i++) {
setX(entries.x, counter);
setY(entries.y, i, data.history[i], tin);
if (tin == 0) {
qlenX1.push(counter);
qlenX2.push(counter);
if (data.history[i][0].Cake) {
qlenY1.push(data.history[i][0].Cake.qlen);
} else {
qlenY1.push(0.0);
}
if (data.history[i][1].Cake) {
qlenY2.push(0.0 - data.history[i][1].Cake.qlen);
} else {
qlenY2.push(0.0);
}
}
counter++;
}
let graph = document.getElementById("tinTp_" + tin);
let graph_data = [
{x: entries.x[0], y:entries.y[0].reverse(), name: 'Download', type: 'scatter'},
{x: entries.x[1], y:entries.y[1].reverse(), name: 'Upload', type: 'scatter'},
];
Plotly.newPlot(graph, graph_data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: "Bits" }, xaxis: {automargin: true, title: "Time since now"} });
// Try to hide zeroes
// Fix the units on delay, convert from usec to ms
for (let i=0; i<entries.y[2].length; i++) {
if (entries.y[2][i] == 0) entries.y[2][i] = null;
if (entries.y[3][i] == 0) entries.y[3][i] = null;
if (entries.y[4][i] == 0) entries.y[4][i] = null;
if (entries.y[5][i] == 0) entries.y[5][i] = null;
entries.y[8][i] *= 0.001; // Scale the delay
entries.y[9][i] *= 0.001;
}
let mdx = zip(entries.x[2], entries.x[4]);
let drop = zip(entries.y[2].reverse(), entries.y[4].reverse());
let mark = zip(entries.y[3].reverse(), entries.y[4].reverse());
graph = document.getElementById("tinMd_" + tin);
graph_data = [
{x: mdx, y:drop, name: 'Drops', mode: 'markers', marker: { size: 4 }},
{x: mdx, y:mark, name: 'Marks', mode: 'markers', marker: { size: 4 }},
//{x: entries.x[4], y:entries.y[4].reverse(), name: 'Up Drops', mode: 'markers', marker: { size: 4 }},
//{x: entries.x[5], y:entries.y[5].reverse(), name: 'Up Marks', mode: 'markers', marker: { size: 4 }},
];
Plotly.newPlot(graph, graph_data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: "Packets" }, xaxis: {automargin: true, title: "Time since now"} });
backlogX1.push(entries.x[6]);
backlogX2.push(entries.x[7]);
backlogY1.push(entries.y[6]);
backlogY2.push(entries.y[7]);
delaysX1.push(entries.x[8]);
delaysX2.push(entries.x[9]);
delaysY1.push(entries.y[8]);
delaysY2.push(entries.y[9]);
}
let graph = document.getElementById("backlogGraph");
let blx = zip(backlogX1[0], backlogX2[0]);
let bly = [
zip(backlogY1[0].reverse(), backlogY2[0].reverse()),
zip(backlogY1[1].reverse(), backlogY2[1].reverse()),
zip(backlogY1[2].reverse(), backlogY2[2].reverse()),
zip(backlogY1[3].reverse(), backlogY2[3].reverse()),
]
for (let n=0; n<bly.length; ++n) {
zero_to_null(bly[n]);
}
let graph_data = [
{x: blx, y:bly[0], type: 'scatter', mode: 'markers', name: 'Bulk', marker: { size: 4 }},
//{x: blx, y:backlogY2[0].reverse(), type: 'scatter', mode: 'markers', name: 'Tin 0 Up'},
{x: blx, y:bly[1], type: 'scatter', mode: 'markers', name: 'Best Effort', marker: { size: 4 }},
//{x: blx, y:backlogY2[1].reverse(), type: 'scatter', mode: 'markers', name: 'Tin 1 Up'},
{x: blx, y:bly[2], type: 'scatter', mode: 'markers', name: 'Video', marker: { size: 4 }},
//{x: blx, y:backlogY2[2].reverse(), type: 'scatter', mode: 'markers', name: 'Tin 2 Up'},
{x: blx, y:bly[3], type: 'scatter', mode: 'markers', name: 'Voice', marker: { size: 4 }},
//{x: blx, y:backlogY2[3].reverse(), type: 'scatter', mode: 'markers', name: 'Tin 3 Up'},
];
Plotly.newPlot(graph, graph_data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: "Bytes" }, xaxis: {automargin: true, title: "Time since now (seconds)"} });
let dlx = zip(delaysX1[0], delaysX2[0]);
let dly = [
zip(delaysY1[0].reverse(), delaysY2[0].reverse()),
zip(delaysY1[1].reverse(), delaysY2[1].reverse()),
zip(delaysY1[2].reverse(), delaysY2[2].reverse()),
zip(delaysY1[3].reverse(), delaysY2[3].reverse()),
]
for (let n=0; n<dly.length; ++n) {
zero_to_null(dly[n]);
}
graph = document.getElementById("delayGraph");
graph_data = [
{x: dlx, y:dly[0], type: 'scatter', mode: 'markers', name: 'Bulk', marker: { size: 4 }},
//{x: delaysX2[0], y:delaysY2[0].reverse(), type: 'scatter', name: 'Tin 0 Up'},
{x: dlx, y:dly[1], type: 'scatter', mode: 'markers', name: 'Best Effort', marker: { size: 4 }},
//{x: delaysX2[1], y:delaysY2[1].reverse(), type: 'scatter', name: 'Tin 1 Up'},
{x: dlx, y:dly[2], type: 'scatter', mode: 'markers', name: 'Video', marker: { size: 4 }},
//{x: delaysX2[2], y:delaysY2[2].reverse(), type: 'scatter', name: 'Tin 2 Up'},
{x: dlx, y:dly[3], type: 'scatter', mode: 'markers', name: 'Voice', marker: { size: 4 }},
//{x: delaysX2[3], y:delaysY2[3].reverse(), type: 'scatter', name: 'Tin 3 Up'},
];
Plotly.newPlot(graph, graph_data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: "Time (ms)" }, xaxis: {automargin: true, title: "Time since now (seconds)"} });
graph = document.getElementById("qlenGraph");
graph_data = [
{x: qlenX1, y:qlenY1.reverse(), type: 'scatter', name: 'Down'},
{x: qlenX2, y:qlenY2.reverse(), type: 'scatter', name: 'Up'},
];
Plotly.newPlot(graph, graph_data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: "Packets" }, xaxis: {automargin: true, title: "Time since now (seconds)"} });
msgPackGet("/api/raw_queue_by_circuit/" + encodeURI(id), (data) => {
if (qp == null) qp = new QueuePlotter(600);
qp.update(data);
qp.plot();
displayMemory(data);
});
}
}
let ips = [];
function getThroughput() {
const params = new Proxy(new URLSearchParams(window.location.search), {
get: (searchParams, prop) => searchParams.get(prop),
});
if (params.id != null) {
$.get("/api/circuit_throughput/" + encodeURI(params.id), (data) => {
let quantiles = [
class ThroughputMonitor {
constructor(capacity) {
this.capacity = capacity;
this.head = 0;
this.per_ip = {};
this.y = {};
this.x_axis = [];
for (let i=0; i<capacity; ++i) {
this.x_axis.push(i);
this.x_axis.push(i);
}
this.quantiles = [
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100],
];
}
clearQuantiles() {
for (let i=0; i<12; ++i) {
this.quantiles[0][i] = 0;
this.quantiles[1][i] = 0;
}
}
ingest(ip, down, up) {
down = down * 8;
up = up * 8;
if (!this.per_ip.hasOwnProperty(ip)) {
this.per_ip[ip] = [];
this.y[ip] = [];
for (let i=0; i<this.capacity; ++i) {
this.per_ip[ip].push(0);
this.per_ip[ip].push(0);
this.y[ip].push(0);
this.y[ip].push(0);
}
}
this.per_ip[ip][this.head] = down;
this.per_ip[ip][this.head + 1] = 0.0 - up;
this.head += 2;
if (this.head > this.capacity*2) {
this.head = 0;
}
}
addQuantile(down, up) {
up = 0 - up;
let down_slot = Math.floor((down / circuit_info[CircuitInfo.capacity][0]) * 10.0);
let up_slot = Math.floor((up / circuit_info[CircuitInfo.capacity][1]) * 10.0);
/*if (down_slot < 0) down_slot = 0;
if (up_slot < 0) up_slot = 0;
if (down_slot > 11) down_slot = 11;
if (up_slot > 11) up_slot = 11;*/
this.quantiles[0][down_slot] += 1;
this.quantiles[1][up_slot] += 1;
//console.log(down_slot, up_slot);
}
prepare() {
this.clearQuantiles();
for (const ip in this.per_ip) {
let counter = this.capacity*2;
for (let i = this.head; i < this.capacity*2; i++) {
this.y[ip][counter] = this.per_ip[ip][i];
counter--;
}
for (let i = 0; i < this.head; i++) {
this.y[ip][counter] = this.per_ip[ip][i];
counter--;
}
for (let i=2; i<22; i+=2) {
this.addQuantile(this.y[ip][i], this.y[ip][i+1]);
}
}
}
plot() {
let graph = document.getElementById("throughputGraph");
let graphData = [];
for (const ip in this.per_ip) {
graphData.push({ x: this.x_axis, y: this.y[ip], name: ip, mode: 'markers', type: 'scattergl', marker: { size: 3 } });
}
if (this.plotted == null) {
Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: "Traffic (bits)" }, xaxis: { automargin: true, title: "Time since now" } });
this.plotted = true;
} else {
Plotly.redraw(graph, graphData);
}
}
plotQuantiles() {
let graph = document.getElementById("capacityQuantile");
let graphData = [
{ x: this.quantiles[2], y: this.quantiles[0], name: 'Download', type: 'bar' },
{ x: this.quantiles[2], y: this.quantiles[1], name: 'Upload', type: 'bar' },
];
if (this.plottedQ == null) {
Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true, title: '# Samples' }, xaxis: { automargin: true, title: '% Utilization' } });
this.plottedQ = true;
} else {
Plotly.redraw(graph, graphData);
}
}
}
let tpData = null;
function getThroughput() {
if (id != null) {
msgPackGet("/api/circuit_throughput/" + encodeURI(id), (data) => {
if (tpData == null) tpData = new ThroughputMonitor(300);
ips = [];
for (let i=0; i<data.length; i++) {
for (let i=0; i < data.length; i++) {
let ip = data[i][0];
ips.push(ip);
let down = data[i][1];
let up = data[i][2];
if (throughput[ip] == null) {
throughput[ip] = {
down: [],
up: [],
};
for (let j=0; j<300; j++) {
throughput[ip].down.push(0);
throughput[ip].up.push(0);
tpData.ingest(ip, down, up);
}
}
throughput[ip].down[throughput_head] = down * 8;
throughput[ip].up[throughput_head] = up * 8;
}
// Build the graph
let graph = document.getElementById("throughputGraph");
let graph_data = [];
for (const ip in throughput) {
//console.log(ip);
let xUp = [];
let xDown = [];
let yUp = [];
let yDown = [];
let counter = 0;
for (let i=throughput_head; i<300; i++) {
xUp.push(counter);
xDown.push(counter);
yUp.push(0.0 - throughput[ip].up[i]);
yDown.push(throughput[ip].down[i]);
counter++;
}
for (let i=0; i<throughput_head; i++) {
xUp.push(counter);
xDown.push(counter);
yUp.push(0.0 - throughput[ip].up[i]);
yDown.push(throughput[ip].down[i]);
counter++;
}
// Build the quantiles graph
if (circuit_info != null) {
for (let i=0; i<yDown.length; i++) {
let down = yDown[i];
let up = 0.0 - yUp[i];
let down_slot = Math.floor((down / circuit_info.capacity[0]) * 10.0);
let up_slot = Math.floor((up / circuit_info.capacity[1]) * 10.0);
if (down_slot > 0) quantiles[0][down_slot] += 1;
if (up_slot > 0) quantiles[1][up_slot] += 1;
}
}
let xd = zip(xDown, xUp);
let yd = zip(yDown.reverse(), yUp.reverse());
zero_to_null(yd);
graph_data.push({x: xd, y: yd, name: ip, mode: 'markers', type: 'scatter', marker: { size: 4 }});
//graph_data.push({x: xUp, y: yUp.reverse(), name: ip + " Up", mode: 'markers', type: 'scatter'});
}
Plotly.newPlot(graph, graph_data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: "Traffic (bits)" }, xaxis: {automargin: true, title: "Time since now (seconds)"} });
throughput_head += 1;
if (throughput_head >= 300) {
throughput_head = 0;
}
graph = document.getElementById("capacityQuantile");
graph_data = [
{x: quantiles[2], y: quantiles[0], name: 'Download', type: 'bar'},
{x: quantiles[2], y: quantiles[1], name: 'Upload', type: 'bar'},
];
Plotly.newPlot(graph, graph_data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: '# Samples' }, xaxis: {automargin: true, title: '% Utilization'} });
tpData.prepare();
tpData.plot();
tpData.plotQuantiles();
});
}
}
@@ -520,27 +599,32 @@
let funnels = new MultiRingBuffer(300);
let rtts = {};
let circuitId = "";
let builtFunnelDivs = false;
function getFunnel(c) {
circuitId = encodeURI(c);
$.get("/api/funnel_for_queue/" + circuitId, (data) => {
function getFunnel() {
if (builtFunnelDivs) {
plotFunnels();
return;
}
circuitId = encodeURI(id);
msgPackGet("/api/funnel_for_queue/" + circuitId, (data) => {
let html = "";
for (let i=0; i<data.length; ++i) {
funnels.push(data[i][0], data[i][1].current_throughput[0]*8, data[i][1].current_throughput[1]*8);
for (let i = 0; i < data.length; ++i) {
funnels.push(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
rtts[data[i][0]] = new RttHistogram();
let row = "<div class='row row220'>";
row += "<div class='col-sm-6'>";
row += "<div class='card bg-light'>";
row += "<h5 class='card-title'><i class='fa fa-hourglass'></i> <a class='redact' href='/tree?parent=" + data[i][0] + "'>" + redactText(data[i][1].name) + " Throughput</a></h5>";
row += "<h5 class='card-title'><i class='fa fa-hourglass'></i> <a class='redact' href='/tree?parent=" + data[i][0] + "'>" + redactText(data[i][1][NetTrans.name]) + " Throughput</a></h5>";
row += "<div id='tp" + data[i][0] + "' class='graph98 graph150'></div>";
row += "</div>";
row += "</div>";
row += "<div class='col-sm-6'>";
row += "<div class='card bg-light'>";
row += "<h5 class='card-title redact'><i class='fa fa-bar-chart'></i> " + redactText(data[i][1].name) + " TCP RTT</h5>";
row += "<h5 class='card-title redact'><i class='fa fa-bar-chart'></i> " + redactText(data[i][1][NetTrans.name]) + " TCP RTT</h5>";
row += "<div id='rtt" + data[i][0] + "' class='graph98 graph150'></div>";
row += "</div>";
row += "</div>";
@@ -549,24 +633,31 @@
html += row;
}
$("#pills-funnel").html(html);
setTimeout(plotFunnels, 1000);
builtFunnelDivs = true;
setTimeout(plotFunnels, 10);
});
}
let plottedFunnels = {};
function plotFunnels() {
$.get("/api/funnel_for_queue/" + encodeURI(circuitId), (data) => {
for (let i=0; i<data.length; ++i) {
funnels.push(data[i][0], data[i][1].current_throughput[0]*8, data[i][1].current_throughput[1]*8);
msgPackGet("/api/funnel_for_queue/" + encodeURI(circuitId), (data) => {
for (let i = 0; i < data.length; ++i) {
funnels.push(data[i][0], data[i][1][NetTrans.current_throughput][0] * 8, data[i][1][NetTrans.current_throughput][1] * 8);
for (const [k, v] of Object.entries(funnels.data)) {
let target_div = "tp" + k;
let graphData = v.toScatterGraphData();
let graph = document.getElementById(target_div);
Plotly.newPlot(graph, graphData, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true }, xaxis: {automargin: true, title: "Time since now (seconds)"} }, { responsive: true });
if (!plotFunnels.hasOwnProperty(target_div)) {
Plotly.newPlot(graph, graphData, { margin: { l: 0, r: 0, b: 0, t: 0, pad: 4 }, yaxis: { automargin: true }, xaxis: { automargin: true, title: "Time since now" } }, { responsive: true });
} else {
Plotly.redraw(graph, graphData);
}
}
rtts[data[i][0]].clear();
for (let j=0; j<data[i][1].rtts.length; j++) {
rtts[data[i][0]].push(data[i][1].rtts[j]);
for (let j = 0; j < data[i][1][NetTrans.rtts].length; j++) {
rtts[data[i][0]].push(data[i][1][NetTrans.rtts][j]);
}
rtts[data[i][0]].plot("rtt" + data[i][0]);
}
@@ -627,7 +718,7 @@
function getFlows() {
let ip_list = "";
let ip_btns = "";
for (let i=0; i<ips.length; ++i) {
for (let i = 0; i < ips.length; ++i) {
ip_list += ips[i] + ",";
if (circuit_info != null) {
ip_btns += "<a id='dumpBtn_" + i + "' href='#' onclick='analyze(\"" + i + "\")' class='btn btn-info'><i class='fa fa-search'></i> Analyze: " + ips[i] + "</a> "
@@ -638,9 +729,9 @@
madeButtons = true;
$("#packetButtons").html(ip_btns);
}
ip_list = ip_list.substring(0, ip_list.length-1);
ip_list = ip_list.substring(0, ip_list.length - 1);
if (ip_list == "") return;
$.get("/api/flows/" + ip_list, (data) => {
msgPackGet("/api/flows/" + ip_list, (data) => {
//console.log(data);
let html = "<table class='table table-striped'>";
html += "<thead>";
@@ -658,42 +749,42 @@
html += "<th>ECN In</th>";
html += "<th>ECN Out</th>";
html += "</thead>";
for (let i=0; i<data.length; i++) {
for (let i = 0; i < data.length; i++) {
let rpackets = "-";
let rbytes = "-";
let rdscp = "-";
let rcongestion = "-";
if (data[i][1] != null) {
rpackets = data[i][1].packets;
rbytes = scaleNumber(data[i][1].bytes);
rdscp = "0x" + data[i][1].dscp.toString(16);
rcongestion = ecn(data[i][1].ecn);
rpackets = data[i][1][FlowTrans.packets];
rbytes = scaleNumber(data[i][1][FlowTrans.bytes]);
rdscp = "0x" + data[i][1][FlowTrans.dscp].toString(16);
rcongestion = ecn(data[i][1][FlowTrans.ecn]);
}
html += "<tr>";
html += "<td>" + data[i][0].proto + "</td>";
html += "<td>" + data[i][0].src + "</td>";
html += "<td>" + data[i][0][FlowTrans.proto] + "</td>";
html += "<td>" + data[i][0][FlowTrans.src] + "</td>";
if (data[i][0].proto == "ICMP") {
html += "<td>" + icmpType(data[i][0].src_port) + "</td>";
html += "<td>" + icmpType(data[i][0][FlowTrans.src_port]) + "</td>";
} else {
html += "<td>" + data[i][0].src_port + "</td>";
html += "<td>" + data[i][0][FlowTrans.src_port] + "</td>";
}
html += "<td>" + data[i][0].dst + "</td>";
if (data[i][0].proto == "ICMP") {
html += "<td>" + data[i][0][FlowTrans.dst] + "</td>";
if (data[i][0][FlowTrans.proto] == "ICMP") {
if (data[i][1] != null) {
html += "<td>" + icmpType(data[i][1].src_port) + "</td>";
html += "<td>" + icmpType(data[i][1][FlowTrans.src_port]) + "</td>";
} else {
html += "<td></td>";
}
} else {
html += "<td>" + data[i][0].dst_port + "</td>";
html += "<td>" + data[i][0][FlowTrans.dst_port] + "</td>";
}
html += "<td>" + data[i][0].packets + "</td>";
html += "<td>" + data[i][0][FlowTrans.packets] + "</td>";
html += "<td>" + rpackets + "</td>";
html += "<td>" + scaleNumber(data[i][0].bytes) + "</td>";
html += "<td>" + scaleNumber(data[i][0][FlowTrans.bytes]) + "</td>";
html += "<td>" + rbytes + "</td>";
html += "<td>0x" + data[i][0].dscp.toString(16) + "</td>";
html += "<td>0x" + data[i][0][FlowTrans.dscp].toString(16) + "</td>";
html += "<td>" + rdscp + "</td>";
html += "<td>" + ecn(data[i][0].ecn) + "</td>";
html += "<td>" + ecn(data[i][0][FlowTrans.ecn]) + "</td>";
html += "<td>" + rcongestion + "</td>";
html += "</tr>";
}
@@ -703,16 +794,35 @@
}
let id = 0;
let activeTab = "pills-home-tab";
function oneSecondCadence() {
//console.log(activeTab);
switch (activeTab) {
case "pills-funnel-tab": {
getFunnel();
} break;
case "pills-flows-tab": {
getFlows();
} break;
default: {
pollQueue();
getThroughput();
getFunnel(id);
getFlows();
}
}
setTimeout(oneSecondCadence, 1000);
}
function wireUpTabEvents() {
// Fire events when the active tab changes
$(document).on('shown.bs.tab', 'button[data-bs-toggle="pill"]', function (e) {
activeTab = e.target.id;
//console.log(activeTab);
});
}
function start() {
wireUpTabEvents();
colorReloadButton();
updateHostCounts();
const params = new Proxy(new URLSearchParams(window.location.search), {
@@ -721,10 +831,6 @@
id = params.id;
$.get("/api/watch_circuit/" + params.id, () => {
oneSecondCadence();
pollQueue();
getThroughput();
getFunnel(params.id);
getFlows();
});
}
@@ -732,4 +838,5 @@
</script>
</body>
</html>

96
src/rust/lqos_node_manager/static/lqos.js
View File

@@ -36,6 +36,49 @@ const IpStats = {
"plan": 6,
}
const FlowTrans = {
"src": 0,
"dst": 1,
"proto": 2,
"src_port": 3,
"dst_port": 4,
"bytes": 5,
"packets": 6,
"dscp": 7,
"ecn": 8
}
const CircuitInfo = {
"name" : 0,
"capacity" : 1,
}
const QD = { // Queue data
"history": 0,
"history_head": 1,
"current_download": 2,
"current_upload": 3,
}
const CT = { // Cake transit
"memory_used": 0,
}
const CDT = { // Cake Diff Transit
"bytes": 0,
"packets": 1,
"qlen": 2,
"tins": 3,
}
const CDTT = { // Cake Diff Tin Transit
"sent_bytes": 0,
"backlog_bytes": 1,
"drops": 2,
"marks": 3,
"avg_delay_us": 4,
}
function metaverse_color_ramp(n) {
if (n <= 9) {
return "#32b08c";
@@ -271,18 +314,23 @@ class MultiRingBuffer {
plotTotalThroughput(target_div) {
let graph = document.getElementById(target_div);
let total = this.data['total'].sortedY();
let shaped = this.data['shaped'].sortedY();
this.data['total'].prepare();
this.data['shaped'].prepare();
let x = this.data['total'].x_axis;
let data = [
{x: x, y:total.down, name: 'Download', type: 'scatter', marker: {color: 'rgb(255,160,122)'}},
{x: x, y:total.up, name: 'Upload', type: 'scatter', marker: {color: 'rgb(255,160,122)'}},
{x: x, y:shaped.down, name: 'Shaped Download', type: 'scatter', fill: 'tozeroy', marker: {color: 'rgb(124,252,0)'}},
{x: x, y:shaped.up, name: 'Shaped Upload', type: 'scatter', fill: 'tozeroy', marker: {color: 'rgb(124,252,0)'}},
let graphData = [
{x: x, y:this.data['total'].sortedY[0], name: 'Download', type: 'scatter', marker: {color: 'rgb(255,160,122)'}},
{x: x, y:this.data['total'].sortedY[1], name: 'Upload', type: 'scatter', marker: {color: 'rgb(255,160,122)'}},
{x: x, y:this.data['shaped'].sortedY[0], name: 'Shaped Download', type: 'scatter', fill: 'tozeroy', marker: {color: 'rgb(124,252,0)'}},
{x: x, y:this.data['shaped'].sortedY[1], name: 'Shaped Upload', type: 'scatter', fill: 'tozeroy', marker: {color: 'rgb(124,252,0)'}},
];
Plotly.newPlot(graph, data, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: "Traffic (bits)" }, xaxis: {automargin: true, title: "Time since now (seconds)"} }, { responsive: true });
if (this.plotted == null) {
Plotly.newPlot(graph, graphData, { margin: { l:0,r:0,b:0,t:0,pad:4 }, yaxis: { automargin: true, title: "Traffic (bits)" }, xaxis: {automargin: true, title: "Time since now (seconds)"} }, { responsive: true });
this.plotted = true;
} else {
Plotly.redraw(graph, graphData);
}
}
}
@@ -293,10 +341,13 @@ class RingBuffer {
this.download = [];
this.upload = [];
this.x_axis = [];
this.sortedY = [ [], [] ];
for (var i = 0; i < capacity; ++i) {
this.download.push(0.0);
this.upload.push(0.0);
this.x_axis.push(capacity - i);
this.sortedY[0].push(0);
this.sortedY[1].push(0);
}
}
@@ -307,27 +358,25 @@ class RingBuffer {
this.head %= this.capacity;
}
sortedY() {
let result = {
down: [],
up: [],
};
prepare() {
let counter = 0;
for (let i=this.head; i<this.capacity; i++) {
result.down.push(this.download[i]);
result.up.push(this.upload[i]);
this.sortedY[0][counter] = this.download[i];
this.sortedY[1][counter] = this.upload[i];
counter++;
}
for (let i=0; i < this.head; i++) {
result.down.push(this.download[i]);
result.up.push(this.upload[i]);
this.sortedY[0][counter] = this.download[i];
this.sortedY[1][counter] = this.upload[i];
counter++;
}
return result;
}
toScatterGraphData() {
let y = this.sortedY();
this.prepare();
let GraphData = [
{ x: this.x_axis, y: y.down, name: 'Download', type: 'scatter' },
{ x: this.x_axis, y: y.up, name: 'Upload', type: 'scatter' },
{ x: this.x_axis, y: this.sortedY[0], name: 'Download', type: 'scatter' },
{ x: this.x_axis, y: this.sortedY[1], name: 'Upload', type: 'scatter' },
];
return GraphData;
}
@@ -366,7 +415,12 @@ class RttHistogram {
{ x: this.x, y: this.entries, type: 'bar', marker: { color: this.x, colorscale: 'RdBu' } }
]
let graph = document.getElementById(target_div);
if (this.plotted == null) {
Plotly.newPlot(graph, gData, { margin: { l: 40, r: 0, b: 35, t: 0 }, yaxis: { title: "# Hosts" }, xaxis: { title: 'TCP Round-Trip Time (ms)' } }, { responsive: true });
this.plotted = true;
} else {
Plotly.redraw(graph, gData);
}
}
}

14
src/rust/lqos_queue_tracker/src/bus.rs
View File

@@ -1,15 +1,15 @@
use crate::{circuit_to_queue::CIRCUIT_TO_QUEUE, still_watching};
use crate::{
circuit_to_queue::CIRCUIT_TO_QUEUE, queue_store::QueueStore, still_watching,
};
use lqos_bus::BusResponse;
pub fn get_raw_circuit_data(circuit_id: &str) -> BusResponse {
still_watching(circuit_id);
if let Some(circuit) = CIRCUIT_TO_QUEUE.get(circuit_id) {
if let Ok(json) = serde_json::to_string(circuit.value()) {
BusResponse::RawQueueData(json)
let cv: QueueStore = circuit.value().clone();
let transit = Box::new(cv.into());
BusResponse::RawQueueData(Some(transit))
} else {
BusResponse::RawQueueData(String::new())
}
} else {
BusResponse::RawQueueData(String::new())
BusResponse::RawQueueData(None)
}
}

142
src/rust/lqos_queue_tracker/src/queue_store.rs
View File

@@ -1,11 +1,16 @@
use crate::{
queue_diff::{make_queue_diff, QueueDiff},
queue_types::QueueType,
queue_diff::{make_queue_diff, CakeDiffTin, QueueDiff},
queue_types::{
QueueType,
},
NUM_QUEUE_HISTORY,
};
use lqos_bus::{
CakeDiffTinTransit, CakeDiffTransit, CakeTransit, QueueStoreTransit,
};
use serde::Serialize;
#[derive(Debug, Serialize)]
#[derive(Debug, Serialize, Clone)]
pub struct QueueStore {
history: Vec<(QueueDiff, QueueDiff)>,
history_head: usize,
@@ -50,3 +55,134 @@ impl QueueStore {
}
}
}
// Note: I'm overriding the warning because the "from only" behaviour
// is actually what we want here.
#[allow(clippy::from_over_into)]
impl Into<QueueStoreTransit> for QueueStore {
fn into(self) -> QueueStoreTransit {
QueueStoreTransit {
history: self
.history
.iter()
.cloned()
.map(|(a, b)| (a.into(), b.into()))
.collect(),
history_head: self.history_head,
//prev_download: self.prev_download.map(|d| d.into()),
//prev_upload: self.prev_upload.map(|u| u.into()),
current_download: self.current_download.into(),
current_upload: self.current_upload.into(),
}
}
}
#[allow(clippy::from_over_into)]
impl Into<CakeDiffTransit> for QueueDiff {
fn into(self) -> CakeDiffTransit {
if let QueueDiff::Cake(c) = &self {
CakeDiffTransit {
bytes: c.bytes,
packets: c.packets,
qlen: c.qlen,
tins: c.tins.iter().cloned().map(|t| t.into()).collect(),
}
} else {
CakeDiffTransit::default()
}
}
}
#[allow(clippy::from_over_into)]
impl Into<CakeDiffTinTransit> for CakeDiffTin {
fn into(self) -> CakeDiffTinTransit {
CakeDiffTinTransit {
sent_bytes: self.sent_bytes,
backlog_bytes: self.backlog_bytes,
drops: self.drops,
marks: self.marks,
avg_delay_us: self.avg_delay_us,
}
}
}
#[allow(clippy::from_over_into)]
impl Into<CakeTransit> for QueueType {
fn into(self) -> CakeTransit {
if let QueueType::Cake(c) = self {
CakeTransit {
//handle: c.handle,
//parent: c.parent,
//options: c.options.into(),
//bytes: c.bytes,
//packets: c.packets,
//overlimits: c.overlimits,
//requeues: c.requeues,
//backlog: c.backlog,
//qlen: c.qlen,
memory_used: c.memory_used,
//memory_limit: c.memory_limit,
//capacity_estimate: c.capacity_estimate,
//min_network_size: c.min_network_size,
//max_network_size: c.max_network_size,
//min_adj_size: c.min_adj_size,
//max_adj_size: c.max_adj_size,
//avg_hdr_offset: c.avg_hdr_offset,
//tins: c.tins.iter().cloned().map(|t| t.into()).collect(),
//drops: c.drops,
}
} else {
CakeTransit::default()
}
}
}
/*
#[allow(clippy::from_over_into)]
impl Into<CakeOptionsTransit> for TcCakeOptions {
fn into(self) -> CakeOptionsTransit {
CakeOptionsTransit {
rtt: self.rtt,
bandwidth: self.bandwidth as u8,
diffserv: self.diffserv as u8,
flowmode: self.flowmode as u8,
ack_filter: self.ack_filter as u8,
nat: self.nat,
wash: self.wash,
ingress: self.ingress,
split_gso: self.split_gso,
raw: self.raw,
overhead: self.overhead,
fwmark: self.fwmark,
}
}
}
#[allow(clippy::from_over_into)]
impl Into<CakeTinTransit> for TcCakeTin {
fn into(self) -> CakeTinTransit {
CakeTinTransit {
//threshold_rate: self.threshold_rate,
//sent_bytes: self.sent_bytes,
//backlog_bytes: self.backlog_bytes,
//target_us: self.target_us,
//interval_us: self.interval_us,
//peak_delay_us: self.peak_delay_us,
//avg_delay_us: self.avg_delay_us,
//base_delay_us: self.base_delay_us,
//sent_packets: self.sent_packets,
//way_indirect_hits: self.way_indirect_hits,
//way_misses: self.way_misses,
//way_collisions: self.way_collisions,
//drops: self.drops,
//ecn_marks: self.ecn_marks,
//ack_drops: self.ack_drops,
//sparse_flows: self.sparse_flows,
//bulk_flows: self.bulk_flows,
//unresponsive_flows: self.unresponsive_flows,
//max_pkt_len: self.max_pkt_len,
//flow_quantum: self.flow_quantum,
}
}
}
*/

2
src/rust/lqos_queue_tracker/src/queue_types/mod.rs
View File

@@ -1,4 +1,4 @@
mod tc_cake;
pub(crate) mod tc_cake;
mod tc_fq_codel;
mod tc_htb;
mod tc_mq;

80
src/rust/lqos_queue_tracker/src/queue_types/tc_cake.rs
View File

@@ -27,63 +27,63 @@ string_table_enum!(BandWidth, unlimited); // in the present implementation with
pub struct TcCake {
pub(crate) handle: TcHandle,
pub(crate) parent: TcHandle,
options: TcCakeOptions,
pub(crate) options: TcCakeOptions,
pub(crate) bytes: u64,
pub(crate) packets: u32,
overlimits: u32,
requeues: u32,
pub(crate) overlimits: u32,
pub(crate) requeues: u32,
pub(crate) backlog: u32,
pub(crate) qlen: u32,
memory_used: u32,
memory_limit: u32,
capacity_estimate: u32,
min_network_size: u16,
max_network_size: u16,
min_adj_size: u16,
max_adj_size: u16,
avg_hdr_offset: u16,
pub(crate) memory_used: u32,
pub(crate) memory_limit: u32,
pub(crate) capacity_estimate: u32,
pub(crate) min_network_size: u16,
pub(crate) max_network_size: u16,
pub(crate) min_adj_size: u16,
pub(crate) max_adj_size: u16,
pub(crate) avg_hdr_offset: u16,
pub(crate) tins: Vec<TcCakeTin>,
pub(crate) drops: u32,
}
#[derive(Default, Clone, Debug, Serialize, Deserialize)]
struct TcCakeOptions {
rtt: u64,
bandwidth: BandWidth,
diffserv: DiffServ,
flowmode: FlowMode,
ack_filter: AckFilter,
nat: bool,
wash: bool,
ingress: bool,
split_gso: bool,
raw: bool,
overhead: u16,
fwmark: TcHandle,
pub(crate) struct TcCakeOptions {
pub(crate) rtt: u64,
pub(crate) bandwidth: BandWidth,
pub(crate) diffserv: DiffServ,
pub(crate) flowmode: FlowMode,
pub(crate) ack_filter: AckFilter,
pub(crate) nat: bool,
pub(crate) wash: bool,
pub(crate) ingress: bool,
pub(crate) split_gso: bool,
pub(crate) raw: bool,
pub(crate) overhead: u16,
pub(crate) fwmark: TcHandle,
}
#[derive(Default, Clone, Debug, Serialize, Deserialize)]
pub struct TcCakeTin {
threshold_rate: u64,
pub(crate) struct TcCakeTin {
pub(crate) threshold_rate: u64,
pub(crate) sent_bytes: u64,
pub(crate) backlog_bytes: u32,
target_us: u32,
interval_us: u32,
peak_delay_us: u32,
pub(crate) target_us: u32,
pub(crate) interval_us: u32,
pub(crate) peak_delay_us: u32,
pub(crate) avg_delay_us: u32,
base_delay_us: u32,
sent_packets: u32,
way_indirect_hits: u16,
way_misses: u16,
way_collisions: u16,
pub(crate) base_delay_us: u32,
pub(crate) sent_packets: u32,
pub(crate) way_indirect_hits: u16,
pub(crate) way_misses: u16,
pub(crate) way_collisions: u16,
pub(crate) drops: u32,
pub(crate) ecn_marks: u32,
ack_drops: u32,
sparse_flows: u16,
bulk_flows: u16,
unresponsive_flows: u16,
max_pkt_len: u16,
flow_quantum: u16,
pub(crate) ack_drops: u32,
pub(crate) sparse_flows: u16,
pub(crate) bulk_flows: u16,
pub(crate) unresponsive_flows: u16,
pub(crate) max_pkt_len: u16,
pub(crate) flow_quantum: u16,
}
impl TcCake {

4
src/rust/lqos_utils/src/string_table_enum.rs
View File

@@ -3,7 +3,7 @@ macro_rules! string_table_enum {
($enum_name: ident, $($option:ident),*) => {
#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
#[allow(non_camel_case_types)]
enum $enum_name {
pub(crate) enum $enum_name {
$($option, )*
Unknown
}
@@ -41,7 +41,7 @@ macro_rules! dashy_table_enum {
($enum_name: ident, $($option:ident),*) => {
#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
#[allow(non_camel_case_types)]
enum $enum_name {
pub(crate) enum $enum_name {
$($option, )*
Unknown
}

3
src/rust/lqosd/src/shaped_devices_tracker/netjson.rs
View File

@@ -41,6 +41,9 @@ fn load_network_json() {
if let Ok(njs) = njs {
let mut write_lock = NETWORK_JSON.write().unwrap();
*write_lock = njs;
std::mem::drop(write_lock);
crate::throughput_tracker::THROUGHPUT_TRACKER
.refresh_circuit_ids();
} else {
warn!("Unable to load network.json");
}