Basic display of total statistics.

src/rust/Cargo.lock

@@ -2155,12 +2155,15 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "axum",
+ "chrono",
  "dryoc",
  "env_logger",
  "futures",
  "influxdb2",
+ "influxdb2-structmap",
  "log",
  "lqos_bus",
+ "num-traits",
  "once_cell",
  "pgdb",
  "serde",

src/rust/long_term_stats/lts_node/Cargo.toml

@@ -17,9 +17,12 @@ pgdb = { path = "../pgdb" }
 dryoc = { version = "0.5", features = ["serde"] }
 once_cell = "1"
 influxdb2 = "0"
+influxdb2-structmap = "0"
+num-traits = "0"
 futures = "0"
 tokio-tungstenite = "0.18"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 tower = { version = "0.4", features = ["util"] }
 tower-http = { version = "0.4.0", features = ["fs", "trace"] }
+chrono = "0"

src/rust/long_term_stats/lts_node/run.sh

@@ -2,4 +2,4 @@
 pushd ../site_build
 ./esbuild.mjs
 popd
-RUST_LOG=info cargo run
+RUST_LOG=info RUST_BACKTRACE=1 cargo run

src/rust/long_term_stats/lts_node/src/submissions/mod.rs

@@ -1,4 +1,5 @@
 mod submission_server;
 mod submission_queue;
 pub use submission_server::submissions_server;
-pub use submission_queue::submissions_queue;
+pub use submission_queue::submissions_queue;
+pub use submission_queue::get_org_details;

src/rust/long_term_stats/lts_node/src/submissions/submission_queue/mod.rs

@@ -1,4 +1,5 @@
 mod queue;
 mod host_totals;
 mod organization_cache;
-pub use queue::{submissions_queue, SubmissionType};
+pub use queue::{submissions_queue, SubmissionType};
+pub use organization_cache::get_org_details;

src/rust/long_term_stats/lts_node/src/web/wss/dashboard.rs

@@ -0,0 +1,175 @@
+use std::f32::consts::E;
+
+use crate::submissions::get_org_details;
+use axum::extract::ws::{WebSocket, Message};
+use chrono::{DateTime, FixedOffset, Utc};
+use influxdb2::{models::Query, Client, FromDataPoint, FromMap};
+use pgdb::sqlx::{Pool, Postgres};
+use serde::Serialize;
+
+#[derive(Debug, FromDataPoint)]
+pub struct BitsAndPackets {
+    direction: String,
+    host_id: String,
+    min: f64,
+    max: f64,
+    avg: f64,
+    time: DateTime<FixedOffset>,
+}
+
+impl Default for BitsAndPackets {
+    fn default() -> Self {
+        Self {
+            direction: "".to_string(),
+            host_id: "".to_string(),
+            min: 0.0,
+            max: 0.0,
+            avg: 0.0,
+            time: DateTime::<Utc>::MIN_UTC.into(),
+        }
+    }
+}
+
+
+#[derive(Serialize)]
+struct Packets {
+    value: f64,
+    date: String,
+    l: f64,
+    u: f64,
+}
+
+#[derive(Serialize)]
+struct PacketChart {
+    msg: String,
+    down: Vec<Packets>,
+    up: Vec<Packets>,
+}
+
+pub async fn packets(cnn: Pool<Postgres>, socket: &mut WebSocket, key: &str) {
+    if let Some(org) = get_org_details(cnn, key).await {
+        let influx_url = format!("http://{}:8086", org.influx_host);
+        let client = Client::new(influx_url, &org.influx_org, &org.influx_token);
+
+        let qs = format!(
+            "from(bucket: \"{}\")
+        |> range(start: -5m)
+        |> filter(fn: (r) => r[\"_measurement\"] == \"packets\")
+        |> filter(fn: (r) => r[\"organization_id\"] == \"{}\")
+        |> aggregateWindow(every: 10s, fn: mean, createEmpty: false)
+        |> yield(name: \"last\")",
+            org.influx_bucket, org.key
+        );
+
+        let query = Query::new(qs);
+        let rows = client.query::<BitsAndPackets>(Some(query)).await;
+        match rows {
+            Err(e) => {
+                tracing::error!("Error querying InfluxDB: {}", e);
+            }
+            Ok(rows) => {
+                // Parse and send the data
+                //println!("{rows:?}");
+
+                let mut down = Vec::new();
+                let mut up = Vec::new();
+
+                // Fill download
+                for row in rows
+                    .iter()
+                    .filter(|r| r.direction == "down")
+                {
+                    down.push(Packets {
+                        value: row.avg,
+                        date: row.time.format("%H:%M:%S").to_string(),
+                        l: row.min,
+                        u: row.max,
+                    });
+                }
+
+                // Fill upload
+                for row in rows
+                    .iter()
+                    .filter(|r| r.direction == "up")
+                {
+                    up.push(Packets {
+                        value: row.avg,
+                        date: row.time.format("%H:%M:%S").to_string(),
+                        l: row.min,
+                        u: row.max,
+                    });
+                }
+
+
+                // Send it
+                let chart = PacketChart { msg: "packetChart".to_string(), down, up };
+                let json = serde_json::to_string(&chart).unwrap();
+                socket.send(Message::Text(json)).await.unwrap();
+            }
+        }
+    }
+}
+
+pub async fn bits(cnn: Pool<Postgres>, socket: &mut WebSocket, key: &str) {
+    if let Some(org) = get_org_details(cnn, key).await {
+        let influx_url = format!("http://{}:8086", org.influx_host);
+        let client = Client::new(influx_url, &org.influx_org, &org.influx_token);
+
+        let qs = format!(
+            "from(bucket: \"{}\")
+        |> range(start: -5m)
+        |> filter(fn: (r) => r[\"_measurement\"] == \"bits\")
+        |> filter(fn: (r) => r[\"organization_id\"] == \"{}\")
+        |> aggregateWindow(every: 10s, fn: mean, createEmpty: false)
+        |> yield(name: \"last\")",
+            org.influx_bucket, org.key
+        );
+
+        let query = Query::new(qs);
+        let rows = client.query::<BitsAndPackets>(Some(query)).await;
+        match rows {
+            Err(e) => {
+                tracing::error!("Error querying InfluxDB: {}", e);
+            }
+            Ok(rows) => {
+                // Parse and send the data
+                //println!("{rows:?}");
+
+                let mut down = Vec::new();
+                let mut up = Vec::new();
+
+                // Fill download
+                for row in rows
+                    .iter()
+                    .filter(|r| r.direction == "down")
+                {
+                    down.push(Packets {
+                        value: row.avg,
+                        date: row.time.format("%H:%M:%S").to_string(),
+                        l: row.min,
+                        u: row.max,
+                    });
+                }
+
+                // Fill upload
+                for row in rows
+                    .iter()
+                    .filter(|r| r.direction == "up")
+                {
+                    up.push(Packets {
+                        value: row.avg,
+                        date: row.time.format("%H:%M:%S").to_string(),
+                        l: row.min,
+                        u: row.max,
+                    });
+                }
+
+
+                // Send it
+                let chart = PacketChart { msg: "bitsChart".to_string(), down, up };
+                let json = serde_json::to_string(&chart).unwrap();
+                socket.send(Message::Text(json)).await.unwrap();
+            }
+        }
+    }
+}

src/rust/long_term_stats/lts_node/src/web/wss/mod.rs

@@ -6,6 +6,7 @@ use pgdb::sqlx::{Pool, Postgres};
 use serde_json::Value;
 mod login;
 mod nodes;
+mod dashboard;
 
 pub async fn ws_handler(ws: WebSocketUpgrade, State(state): State<Pool<Postgres>>) -> impl IntoResponse {
     ws.on_upgrade(move |sock| handle_socket(sock, state))
@@ -50,6 +51,20 @@ async fn handle_socket(mut socket: WebSocket, cnn: Pool<Postgres>) {
                                 log::info!("Node status requested but no credentials provided");
                             }
                         }
+                        "packetChart" => {
+                            if let Some(credentials) = &credentials {
+                                dashboard::packets(cnn.clone(), &mut socket, &credentials.license_key).await;
+                            } else {
+                                log::info!("Throughput requested but no credentials provided");
+                            }
+                        }
+                        "throughputChart" => {
+                            if let Some(credentials) = &credentials {
+                                dashboard::bits(cnn.clone(), &mut socket, &credentials.license_key).await;
+                            } else {
+                                log::info!("Throughput requested but no credentials provided");
+                            }
+                        }
                         _ => {
                             log::warn!("Unknown message type: {msg_type}");
                         }

src/rust/long_term_stats/site_build/README.md

@@ -13,7 +13,7 @@ install `esbuild` and `npm` (ugh). You can do this with:
 ```bash
 (change directory to site_build folder)
 sudo apt-get install npm
-npm install --save-exact esbuild
+npm install
 ````
 
 You can run the build manually by running `./esbuild.sh` in this

src/rust/long_term_stats/site_build/package.json

@@ -1,7 +1,9 @@
 {
   "dependencies": {
     "@types/bootstrap": "^5.2.6",
+    "@types/echarts": "^4.9.17",
     "bootstrap": "^5.2.3",
+    "echarts": "^5.4.2",
     "esbuild": "^0.17.17"
   }
 }

src/rust/long_term_stats/site_build/src/bus.ts

@@ -25,7 +25,7 @@ export class Bus {
         };
         this.ws.onerror = (e) => { console.log("error", e) };
         this.ws.onmessage = (e) => { 
-            console.log("message", e.data)
+            //console.log("message", e.data)
             let json = JSON.parse(e.data);
             if (json.msg && json.msg == "authOk") {
                 window.auth.hasCredentials = true;
@@ -51,6 +51,14 @@ export class Bus {
     requestNodeStatus() {
         this.ws.send("{ \"msg\": \"nodeStatus\" }");
     }
+
+    requestPacketChart() {
+        this.ws.send("{ \"msg\": \"packetChart\" }");
+    }
+
+    requestThroughputChart() {
+        this.ws.send("{ \"msg\": \"throughputChart\" }");
+    }
 }
 
 function formatToken(token: string) {

src/rust/long_term_stats/site_build/src/components/node_status.ts

@@ -1,4 +1,4 @@
-import { Component } from "../component";
+import { Component } from "./component";
 
 export class NodeStatus implements Component {
     wireup(): void {

src/rust/long_term_stats/site_build/src/components/packets.ts

@@ -0,0 +1,142 @@
+import { scaleNumber } from "../helpers";
+import { Component } from "./component";
+import * as echarts from 'echarts';
+
+export class PacketsChart implements Component {
+    div: HTMLElement;
+    myChart: echarts.ECharts;
+    download: any;
+    downloadMin: any;
+    downloadMax: any;
+    upload: any;
+    uploadMin: any;
+    uploadMax: any;
+    x: any;
+    chartMade: boolean = false;
+
+    constructor() {
+        this.div = document.getElementById("packetsChart") as HTMLElement;
+        this.myChart = echarts.init(this.div);
+        this.myChart.showLoading();
+    }
+
+    wireup(): void {
+    }
+
+    ontick(): void {
+        window.bus.requestPacketChart();
+    }
+
+    onmessage(event: any): void {
+        if (event.msg == "packetChart") {
+            //console.log(event);
+            this.download = [];
+            this.downloadMin = [];
+            this.downloadMax = [];
+            this.upload = [];
+            this.uploadMin = [];
+            this.uploadMax = [];
+            this.x = [];
+            for (let i = 0; i < event.down.length; i++) {
+                this.download.push(event.down[i].value);
+                this.downloadMin.push(event.down[i].l);
+                this.downloadMax.push(event.down[i].u);
+                this.upload.push(0.0 - event.up[i].value);
+                this.uploadMin.push(0.0 - event.up[i].l);
+                this.uploadMax.push(0.0 - event.up[i].u);
+                this.x.push(event.down[i].date);
+            }
+
+            if (!this.chartMade) {
+                this.myChart.hideLoading();
+                var option: echarts.EChartsOption;
+                this.myChart.setOption<echarts.EChartsOption>(
+                    (option = {
+                        title: { text: "Packets" },
+                        xAxis: {
+                            type: 'category',
+                            data: this.x,
+                        },
+                        yAxis: {
+                            type: 'value',
+                            axisLabel: {
+                                formatter: function (val: number) {
+                                    return scaleNumber(Math.abs(val));
+                                }
+                            }
+                        },
+                        series: [
+                            {
+                                name: "L",
+                                type: "line",
+                                data: this.downloadMin,
+                                symbol: 'none',
+                                stack: 'confidence-band',
+                                lineStyle: {
+                                    opacity: 0
+                                },
+                            },
+                            {
+                                name: "U",
+                                type: "line",
+                                data: this.downloadMax,
+                                symbol: 'none',
+                                stack: 'confidence-band',
+                                lineStyle: {
+                                    opacity: 0
+                                },
+                                areaStyle: {
+                                    color: '#ccc'
+                                },
+                            },
+                            {
+                                name: "Download",
+                                type: "line",
+                                data: this.download,
+                                symbol: 'none',
+                                itemStyle: {
+                                    color: '#333'
+                                },
+                            },
+                            // Upload
+                            {
+                                name: "LU",
+                                type: "line",
+                                data: this.uploadMin,
+                                symbol: 'none',
+                                stack: 'confidence-band',
+                                lineStyle: {
+                                    opacity: 0
+                                },
+                            },
+                            {
+                                name: "UU",
+                                type: "line",
+                                data: this.uploadMax,
+                                symbol: 'none',
+                                stack: 'confidence-band',
+                                lineStyle: {
+                                    opacity: 0
+                                },
+                                areaStyle: {
+                                    color: '#ccc'
+                                },
+                            },
+                            {
+                                name: "Upload",
+                                type: "line",
+                                data: this.upload,
+                                symbol: 'none',
+                                itemStyle: {
+                                    color: '#333'
+                                },
+                            },
+                        ]
+                    })
+                );
+                option && this.myChart.setOption(option);
+                // this.chartMade = true;
+            }
+        }
+    }
+}

src/rust/long_term_stats/site_build/src/components/throughput.ts

@@ -0,0 +1,142 @@
+import { scaleNumber } from "../helpers";
+import { Component } from "./component";
+import * as echarts from 'echarts';
+
+export class ThroughputChart implements Component {
+    div: HTMLElement;
+    myChart: echarts.ECharts;
+    download: any;
+    downloadMin: any;
+    downloadMax: any;
+    upload: any;
+    uploadMin: any;
+    uploadMax: any;
+    x: any;
+    chartMade: boolean = false;
+
+    constructor() {
+        this.div = document.getElementById("throughputChart") as HTMLElement;
+        this.myChart = echarts.init(this.div);
+        this.myChart.showLoading();
+    }
+
+    wireup(): void {
+    }
+
+    ontick(): void {
+        window.bus.requestThroughputChart();
+    }
+
+    onmessage(event: any): void {
+        if (event.msg == "bitsChart") {
+            //console.log(event);
+            this.download = [];
+            this.downloadMin = [];
+            this.downloadMax = [];
+            this.upload = [];
+            this.uploadMin = [];
+            this.uploadMax = [];
+            this.x = [];
+            for (let i = 0; i < event.down.length; i++) {
+                this.download.push(event.down[i].value);
+                this.downloadMin.push(event.down[i].l);
+                this.downloadMax.push(event.down[i].u);
+                this.upload.push(0.0 - event.up[i].value);
+                this.uploadMin.push(0.0 - event.up[i].l);
+                this.uploadMax.push(0.0 - event.up[i].u);
+                this.x.push(event.down[i].date);
+            }
+
+            if (!this.chartMade) {
+                this.myChart.hideLoading();
+                var option: echarts.EChartsOption;
+                this.myChart.setOption<echarts.EChartsOption>(
+                    (option = {
+                        title: { text: "Bits" },
+                        xAxis: {
+                            type: 'category',
+                            data: this.x,
+                        },
+                        yAxis: {
+                            type: 'value',
+                            axisLabel: {
+                                formatter: function (val: number) {
+                                    return scaleNumber(Math.abs(val));
+                                }
+                            }
+                        },
+                        series: [
+                            {
+                                name: "L",
+                                type: "line",
+                                data: this.downloadMin,
+                                symbol: 'none',
+                                stack: 'confidence-band',
+                                lineStyle: {
+                                    opacity: 0
+                                },
+                            },
+                            {
+                                name: "U",
+                                type: "line",
+                                data: this.downloadMax,
+                                symbol: 'none',
+                                stack: 'confidence-band',
+                                lineStyle: {
+                                    opacity: 0
+                                },
+                                areaStyle: {
+                                    color: '#ccc'
+                                },
+                            },
+                            {
+                                name: "Download",
+                                type: "line",
+                                data: this.download,
+                                symbol: 'none',
+                                itemStyle: {
+                                    color: '#333'
+                                },
+                            },
+                            // Upload
+                            {
+                                name: "LU",
+                                type: "line",
+                                data: this.uploadMin,
+                                symbol: 'none',
+                                stack: 'confidence-band',
+                                lineStyle: {
+                                    opacity: 0
+                                },
+                            },
+                            {
+                                name: "UU",
+                                type: "line",
+                                data: this.uploadMax,
+                                symbol: 'none',
+                                stack: 'confidence-band',
+                                lineStyle: {
+                                    opacity: 0
+                                },
+                                areaStyle: {
+                                    color: '#ccc'
+                                },
+                            },
+                            {
+                                name: "Upload",
+                                type: "line",
+                                data: this.upload,
+                                symbol: 'none',
+                                itemStyle: {
+                                    color: '#333'
+                                },
+                            },
+                        ]
+                    })
+                );
+                option && this.myChart.setOption(option);
+                // this.chartMade = true;
+            }
+        }
+    }
+}

src/rust/long_term_stats/site_build/src/dashboard/dashboard.ts

@@ -2,7 +2,9 @@ import html from './template.html';
 import { Page } from '../page'
 import { MenuPage } from '../menu/menu';
 import { Component } from '../components/component';
-import { NodeStatus } from '../components/node_status/node_status';
+import { NodeStatus } from '../components/node_status';
+import { PacketsChart } from '../components/packets';
+import { ThroughputChart } from '../components/throughput';
 
 export class DashboardPage implements Page {
     menu: MenuPage;
@@ -15,7 +17,9 @@ export class DashboardPage implements Page {
             container.innerHTML = html;
         }
         this.components = [
-            new NodeStatus()
+            new NodeStatus(),
+            new PacketsChart(),
+            new ThroughputChart(),
         ];
     }
 

src/rust/long_term_stats/site_build/src/dashboard/template.html

@@ -1,5 +1,43 @@
-<div class="d-flex flex-nowrap flex-row flex-fill">
-    <div class="d-flex flex-column flex-fill p-3" id="nodeStatus">
+<div class="container">
+    <div class="row">
+        <div class="col-12" id="nodeStatus">
+            <h1>Dashboard</h1>
+        </div>
+    </div>
+    <div class="row">
+        <div class="col-6">
+            <div class="card">
+                <div class="card-body">
+                    <div id="packetsChart" style="height: 250px"></div>
+                </div>
+            </div>
+        </div>
+
+        <div class="col-6">
+            <div class="card">
+                <div class="card-body">
+                    <div id="throughputChart" style="height: 250px"></div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+<!--<div class="d-flex flex-row mb-12">
+    <div class="d-flex flex-column flex-fill p-10" id="nodeStatus">
         <i class="fa-solid fa-spinner fa-spin"></i>
     </div>
-</div>
+</div>
+<div class="break"></div>
+<div class="d-flex flex-row mb-3">
+
+    <div class="d-flex flex-column flex-fill p-3">
+        <div class="card">
+            <div class="card-body">
+                <h5 class="card-title">Throughput</h5>
+                <i class="fa-solid fa-spinner fa-spin"></i>
+            </div>
+        </div>
+    </div>
+</div>
+-->

src/rust/long_term_stats/site_build/src/helpers.ts

@@ -4,4 +4,17 @@ export function getValueFromForm(id: string): string {
         return input.value;
     }
     return "";
+}
+
+export function scaleNumber(n: any): string {
+    if (n > 1000000000000) {
+        return (n / 1000000000000).toFixed(2) + "T";
+    } else if (n > 1000000000) {
+        return (n / 1000000000).toFixed(2) + "G";
+    } else if (n > 1000000) {
+        return (n / 1000000).toFixed(2) + "M";
+    } else if (n > 1000) {
+        return (n / 1000).toFixed(2) + "K";
+    }
+    return n;
 }

src/rust/long_term_stats/site_build/src/style.css

@@ -18,4 +18,13 @@
     flex-shrink: 0;
     width: 0.5rem;
     height: 100vh;
+  }
+
+  .break {
+    flex-basis: 100%;
+    height: 0;
+  }
+
+  .row {
+    margin-bottom: 2px;
   }