IntenseWebs/LibreQoS
3
0
Fork 0
mirror of https://github.com/LibreQoE/LibreQoS.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #393 from LibreQoE/long_term_stats

Browse Source 
Long term stats > Develop
This commit is contained in:
Robert Chacón
2023-08-16 20:34:13 -06:00
committed by GitHub
parent 79010628c5 e0cc705c19
commit b824a1471a
177 changed files with 42 additions and 15899 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/integrationCommon.py
View File

@@ -356,6 +356,10 @@ class NetworkGraph:
def createShapedDevices(self):
import csv
from ispConfig import bandwidthOverheadFactor
try:
from ispConfig import committedBandwidthMultiplier
except:
committedBandwidthMultiplier = 0.98
# Builds ShapedDevices.csv from the network tree.
circuits = []
for (i, node) in enumerate(self.nodes):
@@ -416,8 +420,8 @@ class NetworkGraph:
device["mac"],
device["ipv4"],
device["ipv6"],
int(float(circuit["download"]) * 0.98),
int(float(circuit["upload"]) * 0.98),
int(float(circuit["download"]) * committedBandwidthMultiplier),
int(float(circuit["upload"]) * committedBandwidthMultiplier),
int(float(circuit["download"]) * bandwidthOverheadFactor),
int(float(circuit["upload"]) * bandwidthOverheadFactor),
""

2
src/ispConfig.example.py
View File

@@ -115,6 +115,8 @@ findIPv6usingMikrotik = False
# If you want to provide a safe cushion for speed test results to prevent customer complains, you can set this to
# 1.15 (15% above plan rate). If not, you can leave as 1.0
bandwidthOverheadFactor = 1.0
# Number to multiply the maximum/ceiling bandwidth with to determine the minimum bandwidth.
committedBandwidthMultiplier = 0.98
# For edge cases, set the respective ParentNode for these CPEs
exceptionCPEs = {}
# exceptionCPEs = {

1197
src/rust/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

9
src/rust/Cargo.toml
View File

@@ -28,14 +28,7 @@ members = [
"lqos_heimdall", # Library for managing Heimdall flow watching
"lqos_map_perf", # A CLI tool for testing eBPF map performance
"lqstats", # A CLI utility for retrieving long-term statistics
"long_term_stats/license_server", # Licensing Server for LibreQoS Long-term stats
"long_term_stats/lts_node", # Long-term stats cluster node (web interface)
"long_term_stats/lts_ingestor", # Long-term stats data ingestor (feeding databases)
"long_term_stats/pgdb", # PostgreSQL interface for the LTS system
"long_term_stats/licman", # A CLI tool for managing the licensing server
"long_term_stats/lts_client", # Shared data and client-side code for long-term stats
"long_term_stats/wasm_pipe", # Provides a WebAssembly tight/compressed data pipeline
"long_term_stats/wasm_pipe_types", # Common types between the WASM conduit and the WASM server
"lts_client", # Shared data and client-side code for long-term stats
"lqos_map_perf", # A CLI tool for testing eBPF map performance
"uisp", # REST support for the UISP API
]

23
src/rust/long_term_stats/README.md
View File

@@ -1,23 +0,0 @@
# Long Term Stats
We'd really rather you let us host your long-term statistics. It's a lot
of work, and gives us a revenue stream to keep building LibreQoS.
If you really want to self-host, setup is a bit convoluted - but we won't
stop you.
## PostgreSQL
* Install PostgreSQL somewhere on your network. You only want one PostgreSQL host per long-term node stats cluster.
* Setup the database schema (TBD).
* Put the connection string for your database in `/etc/lqdb` on each host.
* Install the `sqlx` tool with `cargo install sqlx-cli --no-default-features --features rustls,postgres`
## For each stats node in the cluster
* Install InfluxDB.
* Install lts_node.
* Setup `/etc/lqdb`.
* Copy `lts_keys.bin` from the license server to the `lts_node` directory.
* Run the process.
* Login to the licensing server, and run `licman host add <ip of the new host>`

14
src/rust/long_term_stats/license_server/Cargo.toml
View File

@@ -1,14 +0,0 @@
[package]
name = "license_server"
version = "0.1.0"
edition = "2021"
[dependencies]
tokio = { version = "1.25.0", features = ["full"] }
anyhow = "1"
env_logger = "0"
log = "0"
serde = { version = "1.0", features = ["derive"] }
lts_client = { path = "../lts_client" }
pgdb = { path = "../pgdb" }
once_cell = "1"

3
src/rust/long_term_stats/license_server/README.md
View File

@@ -1,3 +0,0 @@
# License Server
Runs at LibreQoS and matches license keys with an "is valid" list. If you're running your very own licensing server, then you will need to set this up on your server to accept your key. Details will be provided later.

14
src/rust/long_term_stats/license_server/src/main.rs
View File

@@ -1,14 +0,0 @@
mod server;
mod pki;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// Start the logger
env_logger::init_from_env(
env_logger::Env::default()
.filter_or(env_logger::DEFAULT_FILTER_ENV, "warn"),
);
let _ = server::start().await;
Ok(())
}

6
src/rust/long_term_stats/license_server/src/pki.rs
View File

@@ -1,6 +0,0 @@
use lts_client::{dryoc::dryocbox::*, pki::generate_new_keypair};
use once_cell::sync::Lazy;
use tokio::sync::RwLock;
pub(crate) static LIBREQOS_KEYPAIR: Lazy<RwLock<KeyPair>> = Lazy::new(|| RwLock::new(generate_new_keypair(KEY_PATH)));
const KEY_PATH: &str = "lqkeys.bin"; // Store in the working directory

149
src/rust/long_term_stats/license_server/src/server.rs
View File

@@ -1,149 +0,0 @@
use lts_client::transport_data::{LicenseReply, LicenseRequest};
use pgdb::sqlx::{Pool, Postgres};
use std::net::SocketAddr;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
spawn,
};
use crate::pki::LIBREQOS_KEYPAIR;
pub async fn start() -> anyhow::Result<()> {
let listener = TcpListener::bind(":::9126").await?;
log::info!("Listening on :::9126");
let pool = pgdb::get_connection_pool(5).await;
if pool.is_err() {
log::error!("Unable to connect to the database");
log::error!("{pool:?}");
return Err(anyhow::Error::msg("Unable to connect to the database"));
}
let pool = pool.unwrap();
loop {
let (mut socket, address) = listener.accept().await?;
log::info!("Connection from {address:?}");
let pool = pool.clone();
spawn(async move {
let mut buf = vec![0u8; 10240];
if let Ok(bytes) = socket.read(&mut buf).await {
log::info!("Received {bytes} bytes from {address:?}");
match decode(&buf, address, pool).await {
Err(e) => log::error!("{e:?}"),
Ok(reply) => {
let bytes = build_reply(&reply);
match bytes {
Ok(bytes) => {
log::info!("Submitting {} bytes to network", bytes.len());
if let Err(e) = socket.write_all(&bytes).await {
log::error!("Write error: {e:?}");
}
}
Err(e) => {
log::error!("{e:?}");
}
}
}
}
}
});
}
}
async fn decode(
buf: &[u8],
address: SocketAddr,
pool: Pool<Postgres>,
) -> anyhow::Result<LicenseReply> {
const U64SIZE: usize = std::mem::size_of::<u64>();
let version_buf = &buf[0..2].try_into()?;
let version = u16::from_be_bytes(*version_buf);
let size_buf = &buf[2..2 + U64SIZE].try_into()?;
let size = u64::from_be_bytes(*size_buf);
log::info!("Received a version {version} payload of serialized size {size} from {address:?}");
match version {
1 => {
let start = 2 + U64SIZE;
let end = start + size as usize;
let payload: LicenseRequest = lts_client::cbor::from_slice(&buf[start..end])?;
let license = check_license(&payload, address, pool).await?;
Ok(license)
}
_ => {
log::error!("Unknown version of statistics: {version}, dumped {size} bytes");
Err(anyhow::Error::msg("Version error"))
}
}
}
async fn check_license(
request: &LicenseRequest,
address: SocketAddr,
pool: Pool<Postgres>,
) -> anyhow::Result<LicenseReply> {
match request {
LicenseRequest::LicenseCheck { key } => {
log::info!("Checking license from {address:?}, key: {key}");
if key == "test" {
log::info!("License is valid");
Ok(LicenseReply::Valid {
expiry: 0, // Temporary value
stats_host: "127.0.0.1:9127".to_string(), // Also temporary
})
} else {
match pgdb::get_stats_host_for_key(pool, key).await {
Ok(host) => {
log::info!("License is valid");
return Ok(LicenseReply::Valid {
expiry: 0, // Temporary value
stats_host: host,
});
}
Err(e) => {
log::warn!("Unable to get stats host for key: {e:?}");
}
}
log::info!("License is denied");
Ok(LicenseReply::Denied)
}
}
LicenseRequest::KeyExchange { node_id, node_name, license_key, public_key } => {
log::info!("Public key exchange requested by {node_id}");
// Check if the node_id / license key combination exists
// If it does, update it to the current last-seen and the new public key
// If it doesn't, insert it
let public_key = lts_client::cbor::to_vec(&public_key).unwrap();
let result = pgdb::insert_or_update_node_public_key(pool, node_id, node_name, license_key, &public_key).await;
if result.is_err() {
log::warn!("Unable to insert or update node public key: {result:?}");
return Err(anyhow::Error::msg("Unable to insert or update node public key"));
}
let public_key = LIBREQOS_KEYPAIR.read().await.public_key.clone();
Ok(LicenseReply::MyPublicKey { public_key })
}
}
}
fn build_reply(reply: &LicenseReply) -> anyhow::Result<Vec<u8>> {
let mut result = Vec::new();
let payload = lts_client::cbor::to_vec(reply);
if let Err(e) = payload {
log::warn!("Unable to serialize statistics. Not sending them.");
log::warn!("{e:?}");
return Err(anyhow::Error::msg("Unable to serialize"));
}
let payload = payload.unwrap();
// Store the version as network order
result.extend(1u16.to_be_bytes());
// Store the payload size as network order
result.extend((payload.len() as u64).to_be_bytes());
// Store the payload itself
result.extend(payload);
Ok(result)
}

12
src/rust/long_term_stats/licman/Cargo.toml
View File

@@ -1,12 +0,0 @@
[package]
name = "licman"
version = "0.1.0"
edition = "2021"
[dependencies]
clap = { version = "4", features = ["derive"] }
anyhow = "1"
pgdb = { path = "../pgdb" }
tokio = { version = "1", features = [ "rt", "macros", "net", "io-util", "time" ] }
env_logger = "0"
log = "0"

112
src/rust/long_term_stats/licman/src/main.rs
View File

@@ -1,112 +0,0 @@
use anyhow::Result;
use clap::{Parser, Subcommand};
use pgdb::create_free_trial;
use std::process::exit;
#[derive(Parser)]
#[command()]
struct Args {
#[command(subcommand)]
command: Option<Commands>,
}
#[derive(Subcommand)]
enum Commands {
/// Manage stats hosts
Hosts {
#[command(subcommand)]
command: Option<HostsCommands>,
},
/// Manage licenses
License {
#[command(subcommand)]
command: Option<LicenseCommands>,
},
/// Manage users
Users {
#[command(subcommand)]
command: Option<UsersCommands>,
},
}
#[derive(Subcommand)]
enum HostsCommands {
/// Add a host to the list of available stats storing hosts
Add { hostname: String, influx_host: String, api_key: String },
}
#[derive(Subcommand)]
enum LicenseCommands {
/// Create a new free trial license
FreeTrial { organization: String },
}
#[derive(Subcommand)]
enum UsersCommands {
/// Add a new user
Add { key: String, username: String, password: String, nicename: String },
/// Delete a user
Delete { key: String, username: String },
}
#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<()> {
env_logger::init_from_env(
env_logger::Env::default()
.filter_or(env_logger::DEFAULT_FILTER_ENV, "warn"),
);
// Get the database connection pool
let pool = pgdb::get_connection_pool(5).await;
if pool.is_err() {
log::error!("Unable to connect to the database");
log::error!("{pool:?}");
return Err(anyhow::Error::msg("Unable to connect to the database"));
}
let pool = pool.unwrap();
let cli = Args::parse();
match cli.command {
Some(Commands::Hosts {
command: Some(HostsCommands::Add { hostname, influx_host, api_key }),
}) => {
match pgdb::add_stats_host(pool, hostname, influx_host, api_key).await {
Err(e) => {
log::error!("Unable to add stats host: {e:?}");
exit(1);
}
Ok(new_id) => {
println!("Added stats host with id {}", new_id);
}
}
}
Some(Commands::License{command: Some(LicenseCommands::FreeTrial { organization })}) => {
match create_free_trial(pool, &organization).await {
Err(e) => {
log::error!("Unable to create free trial: {e:?}");
exit(1);
}
Ok(key) => {
println!("Your new license key is: {}", key);
}
}
}
Some(Commands::Users{command: Some(UsersCommands::Add { key, username, password, nicename })}) => {
match pgdb::add_user(pool, &key, &username, &password, &nicename).await {
Err(e) => {
log::error!("Unable to add user: {e:?}");
exit(1);
}
Ok(_) => {
println!("Added user {}", username);
}
}
}
_ => {
println!("Run with --help to see instructions");
exit(0);
}
}
Ok(())
}

20
src/rust/long_term_stats/lts_ingestor/Cargo.toml
View File

@@ -1,20 +0,0 @@
[package]
name = "lts_ingestor"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
pgdb = { path = "../pgdb" }
lts_client = { path = "../lts_client" }
lqos_config = { path = "../../lqos_config" }
tokio = { version = "1.25.0", features = ["full"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
anyhow = "1"
influxdb2 = "0"
influxdb2-structmap = "0"
futures = "0"
once_cell = "1"
miniz_oxide = "0.7.1"

BIN
src/rust/long_term_stats/lts_ingestor/lqkeys.bin
View File

Binary file not shown.

33
src/rust/long_term_stats/lts_ingestor/src/main.rs
View File

@@ -1,33 +0,0 @@
use tracing::{error, info};
mod submissions;
mod pki;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// install global collector configured based on RUST_LOG env var.
tracing_subscriber::fmt::init();
// Get the database connection pool
let pool = pgdb::get_connection_pool(5).await;
if pool.is_err() {
error!("Unable to connect to the database");
error!("{pool:?}");
return Err(anyhow::Error::msg("Unable to connect to the database"));
}
let pool = pool.unwrap();
// Start the submission queue
let submission_sender = {
info!("Starting the submission queue");
submissions::submissions_queue(pool.clone()).await?
};
// Start the submissions serer
info!("Starting the submissions server");
if let Err(e) = tokio::spawn(submissions::submissions_server(pool.clone(), submission_sender)).await {
error!("Server exited with error: {}", e);
}
Ok(())
}

6
src/rust/long_term_stats/lts_ingestor/src/pki.rs
View File

@@ -1,6 +0,0 @@
use std::sync::RwLock;
use once_cell::sync::Lazy;
use lts_client::{pki::generate_new_keypair, dryoc::dryocbox::KeyPair};
pub(crate) static LIBREQOS_KEYPAIR: Lazy<RwLock<KeyPair>> = Lazy::new(|| RwLock::new(generate_new_keypair(KEY_PATH)));
const KEY_PATH: &str = "lqkeys.bin"; // Store in the working directory

5
src/rust/long_term_stats/lts_ingestor/src/submissions/mod.rs
View File

@@ -1,5 +0,0 @@
mod submission_server;
mod submission_queue;
pub use submission_server::submissions_server;
pub use submission_queue::submissions_queue;
pub use submission_queue::get_org_details;

85
src/rust/long_term_stats/lts_ingestor/src/submissions/submission_queue/devices.rs
View File

@@ -1,85 +0,0 @@
use lqos_config::ShapedDevice;
use pgdb::{OrganizationDetails, sqlx::{Pool, Postgres}};
use tracing::{warn, error};
pub async fn ingest_shaped_devices(
cnn: Pool<Postgres>,
org: &OrganizationDetails,
node_id: &str,
devices: &[ShapedDevice],
) -> anyhow::Result<()> {
let mut trans = cnn.begin().await?;
// Clear existing data from shaped devices
pgdb::sqlx::query("DELETE FROM shaped_devices WHERE key=$1 AND node_id=$2")
.bind(org.key.to_string())
.bind(node_id)
.execute(&mut trans)
.await?;
// Clear existing data from shaped devices IP lists
pgdb::sqlx::query("DELETE FROM shaped_device_ip WHERE key=$1 AND node_id=$2")
.bind(org.key.to_string())
.bind(node_id)
.execute(&mut trans)
.await?;
const SQL_INSERT: &str = "INSERT INTO shaped_devices
(key, node_id, circuit_id, device_id, circuit_name, device_name, parent_node, mac, download_min_mbps, upload_min_mbps, download_max_mbps, upload_max_mbps, comment)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)";
const SQL_IP_INSERT: &str = "INSERT INTO public.shaped_device_ip
(key, node_id, circuit_id, ip_range, subnet)
VALUES
($1, $2, $3, $4, $5)
ON CONFLICT (key, node_id, circuit_id, ip_range, subnet) DO NOTHING;";
for device in devices.iter() {
pgdb::sqlx::query(SQL_INSERT)
.bind(org.key.to_string())
.bind(node_id)
.bind(device.circuit_id.clone())
.bind(device.device_id.clone())
.bind(device.circuit_name.clone())
.bind(device.device_name.clone())
.bind(device.parent_node.clone())
.bind(device.mac.clone())
.bind(device.download_min_mbps as i32)
.bind(device.upload_min_mbps as i32)
.bind(device.download_max_mbps as i32)
.bind(device.upload_max_mbps as i32)
.bind(device.comment.clone())
.execute(&mut trans)
.await?;
for ip in device.ipv4.iter() {
pgdb::sqlx::query(SQL_IP_INSERT)
.bind(org.key.to_string())
.bind(node_id)
.bind(device.circuit_id.clone())
.bind(ip.0.to_string())
.bind(ip.1 as i32)
.execute(&mut trans)
.await?;
}
for ip in device.ipv6.iter() {
pgdb::sqlx::query(SQL_IP_INSERT)
.bind(org.key.to_string())
.bind(node_id)
.bind(device.circuit_id.clone())
.bind(ip.0.to_string())
.bind(ip.1 as i32)
.execute(&mut trans)
.await?;
}
}
let result = trans.commit().await;
warn!("Transaction committed");
if let Err(e) = result {
error!("Error committing transaction: {}", e);
}
Ok(())
}

83
src/rust/long_term_stats/lts_ingestor/src/submissions/submission_queue/host_totals.rs
View File

@@ -1,83 +0,0 @@
use futures::prelude::*;
use influxdb2::models::DataPoint;
use influxdb2::Client;
use lts_client::transport_data::StatsTotals;
use pgdb::OrganizationDetails;
pub async fn collect_host_totals(
org: &OrganizationDetails,
node_id: &str,
timestamp: i64,
totals: &Option<StatsTotals>,
) -> anyhow::Result<()> {
if let Some(totals) = totals {
let influx_url = format!("http://{}:8086", org.influx_host);
let client = Client::new(&influx_url, &org.influx_org, &org.influx_token);
let points = vec![
DataPoint::builder("packets")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("direction", "down".to_string())
.timestamp(timestamp)
.field("min", totals.packets.min.0 as i64)
.field("max", totals.packets.max.0 as i64)
.field("avg", totals.packets.avg.0 as i64)
.build()?,
DataPoint::builder("packets")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("direction", "up".to_string())
.timestamp(timestamp)
.field("min", totals.packets.min.1 as i64)
.field("max", totals.packets.max.1 as i64)
.field("avg", totals.packets.avg.1 as i64)
.build()?,
DataPoint::builder("bits")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("direction", "down".to_string())
.timestamp(timestamp)
.field("min", totals.bits.min.0 as i64)
.field("max", totals.bits.max.0 as i64)
.field("avg", totals.bits.avg.0 as i64)
.build()?,
DataPoint::builder("bits")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("direction", "up".to_string())
.timestamp(timestamp)
.field("min", totals.bits.min.1 as i64)
.field("max", totals.bits.max.1 as i64)
.field("avg", totals.bits.avg.1 as i64)
.build()?,
DataPoint::builder("shaped_bits")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("direction", "down".to_string())
.timestamp(timestamp)
.field("min", totals.shaped_bits.min.0 as i64)
.field("max", totals.shaped_bits.max.0 as i64)
.field("avg", totals.shaped_bits.avg.0 as i64)
.build()?,
DataPoint::builder("shaped_bits")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("direction", "up".to_string())
.timestamp(timestamp)
.field("min", totals.shaped_bits.min.1 as i64)
.field("max", totals.shaped_bits.max.1 as i64)
.field("avg", totals.shaped_bits.avg.1 as i64)
.build()?,
];
//client.write(&org.influx_bucket, stream::iter(points)).await?;
client
.write_with_precision(
&org.influx_bucket,
stream::iter(points),
influxdb2::api::write::TimestampPrecision::Seconds,
)
.await?;
}
Ok(())
}

10
src/rust/long_term_stats/lts_ingestor/src/submissions/submission_queue/mod.rs
View File

@@ -1,10 +0,0 @@
mod queue;
mod devices;
mod host_totals;
mod organization_cache;
mod per_host;
mod tree;
mod node_perf;
mod uisp_devices;
pub use queue::{submissions_queue, SubmissionType};
pub use organization_cache::get_org_details;

35
src/rust/long_term_stats/lts_ingestor/src/submissions/submission_queue/node_perf.rs
View File

@@ -1,35 +0,0 @@
use futures::prelude::*;
use influxdb2::{models::DataPoint, Client};
use pgdb::OrganizationDetails;
pub async fn collect_node_perf(
org: &OrganizationDetails,
node_id: &str,
timestamp: i64,
cpu: &Option<Vec<u32>>,
ram: &Option<u32>,
) -> anyhow::Result<()> {
if let (Some(cpu), Some(ram)) = (cpu, ram) {
let influx_url = format!("http://{}:8086", org.influx_host);
let client = Client::new(&influx_url, &org.influx_org, &org.influx_token);
let cpu_sum = cpu.iter().sum::<u32>();
let cpu_avg = cpu_sum / cpu.len() as u32;
let cpu_max = *cpu.iter().max().unwrap();
let points = vec![DataPoint::builder("perf")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.timestamp(timestamp)
.field("ram", *ram as i64)
.field("cpu", cpu_avg as i64)
.field("cpu_max", cpu_max as i64)
.build()?];
client
.write_with_precision(
&org.influx_bucket,
stream::iter(points),
influxdb2::api::write::TimestampPrecision::Seconds,
)
.await?;
}
Ok(())
}

25
src/rust/long_term_stats/lts_ingestor/src/submissions/submission_queue/organization_cache.rs
View File

@@ -1,25 +0,0 @@
use std::collections::HashMap;
use once_cell::sync::Lazy;
use pgdb::{OrganizationDetails, sqlx::{Pool, Postgres}};
use tokio::sync::RwLock;
static ORG_CACHE: Lazy<RwLock<HashMap<String, OrganizationDetails>>> = Lazy::new(|| {
RwLock::new(HashMap::new())
});
pub async fn get_org_details(cnn: &Pool<Postgres>, key: &str) -> Option<OrganizationDetails> {
{ // Safety scope - lock is dropped on exit
let cache = ORG_CACHE.read().await;
if let Some(org) = cache.get(key) {
return Some(org.clone());
}
}
// We can be certain that we don't have a dangling lock now.
// Upgrade to a write lock and try to fetch the org details.
let mut cache = ORG_CACHE.write().await;
if let Ok(org) = pgdb::get_organization(cnn, key).await {
cache.insert(key.to_string(), org.clone());
return Some(org);
}
None
}

68
src/rust/long_term_stats/lts_ingestor/src/submissions/submission_queue/per_host.rs
View File

@@ -1,68 +0,0 @@
use influxdb2::{Client, models::DataPoint};
use lts_client::transport_data::StatsHost;
use pgdb::OrganizationDetails;
use futures::prelude::*;
use tracing::info;
pub async fn collect_per_host(
org: &OrganizationDetails,
node_id: &str,
timestamp: i64,
totals: &Option<Vec<StatsHost>>,
) -> anyhow::Result<()> {
if let Some(hosts) = totals {
let influx_url = format!("http://{}:8086", org.influx_host);
let client = Client::new(&influx_url, &org.influx_org, &org.influx_token);
let mut points: Vec<DataPoint> = Vec::new();
info!("Received per-host stats, {} hosts", hosts.len());
for host in hosts.iter() {
let circuit_id = if let Some(cid) = &host.circuit_id {
cid.clone()
} else {
"unknown".to_string()
};
points.push(DataPoint::builder("host_bits")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("direction", "down".to_string())
.tag("circuit_id", &circuit_id)
.tag("ip", host.ip_address.to_string())
.timestamp(timestamp)
.field("min", host.bits.min.0 as i64)
.field("max", host.bits.max.0 as i64)
.field("avg", host.bits.avg.0 as i64)
.build()?);
points.push(DataPoint::builder("host_bits")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("direction", "up".to_string())
.tag("circuit_id", &circuit_id)
.tag("ip", host.ip_address.to_string())
.timestamp(timestamp)
.field("min", host.bits.min.1 as i64)
.field("max", host.bits.max.1 as i64)
.field("avg", host.bits.avg.1 as i64)
.build()?);
points.push(DataPoint::builder("rtt")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("circuit_id", &circuit_id)
.tag("ip", host.ip_address.to_string())
.timestamp(timestamp)
.field("min", host.rtt.avg as f64 / 100.0)
.field("max", host.rtt.max as f64 / 100.0)
.field("avg", host.rtt.avg as f64 / 100.0)
.build()?);
}
client
.write_with_precision(
&org.influx_bucket,
stream::iter(points),
influxdb2::api::write::TimestampPrecision::Seconds,
)
.await?;
}
Ok(())
}

91
src/rust/long_term_stats/lts_ingestor/src/submissions/submission_queue/queue.rs
View File

@@ -1,91 +0,0 @@
//! Provides a queue of submissions to be processed by the long-term storage.
//! This is a "fan in" pattern: multi-producer, single-consumer messages
//! send data into the queue, which is managed by a single consumer
//! thread. The consumer thread spawns tokio tasks to actually
//! perform the processing.
use crate::submissions::submission_queue::{
devices::ingest_shaped_devices, host_totals::collect_host_totals, node_perf::collect_node_perf,
organization_cache::get_org_details, tree::collect_tree, per_host::collect_per_host, uisp_devices::collect_uisp_devices,
};
use lts_client::transport_data::{LtsCommand, NodeIdAndLicense};
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::mpsc::{Receiver, Sender};
use tracing::{info, error, warn};
const SUBMISSION_QUEUE_SIZE: usize = 100;
pub type SubmissionType = (NodeIdAndLicense, LtsCommand);
pub async fn submissions_queue(cnn: Pool<Postgres>) -> anyhow::Result<Sender<SubmissionType>> {
// Create a channel to send data to the consumer thread
let (tx, rx) = tokio::sync::mpsc::channel::<SubmissionType>(SUBMISSION_QUEUE_SIZE);
tokio::spawn(run_queue(cnn, rx)); // Note that'we *moving* rx into the spawned task
Ok(tx)
}
async fn run_queue(cnn: Pool<Postgres>, mut rx: Receiver<SubmissionType>) -> anyhow::Result<()> {
while let Some(message) = rx.recv().await {
info!("Received a message from the submission queue");
let (node_id, command) = message;
tokio::spawn(ingest_stats(cnn.clone(), node_id, command));
}
Ok(())
}
//#[tracing::instrument]
async fn ingest_stats(
cnn: Pool<Postgres>,
node_id: NodeIdAndLicense,
command: LtsCommand,
) -> anyhow::Result<()> {
info!("Ingesting stats for node {}", node_id.node_id);
if let Some(org) = get_org_details(&cnn, &node_id.license_key).await {
//println!("{:?}", command);
match command {
LtsCommand::Devices(devices) => {
info!("Ingesting Shaped Devices");
update_last_seen(cnn.clone(), &node_id).await;
if let Err(e) = ingest_shaped_devices(cnn, &org, &node_id.node_id, &devices).await {
error!("Error ingesting shaped devices: {}", e);
}
}
LtsCommand::Submit(stats) => {
//println!("Submission: {:?}", submission);
info!("Ingesting statistics dump");
let ts = stats.timestamp as i64;
let _ = tokio::join!(
update_last_seen(cnn.clone(), &node_id),
collect_host_totals(&org, &node_id.node_id, ts, &stats.totals),
collect_node_perf(
&org,
&node_id.node_id,
ts,
&stats.cpu_usage,
&stats.ram_percent
),
collect_tree(cnn.clone(), &org, &node_id.node_id, ts, &stats.tree),
collect_per_host(&org, &node_id.node_id, ts, &stats.hosts),
collect_uisp_devices(cnn.clone(), &org, &stats.uisp_devices, ts),
);
}
}
} else {
warn!(
"Unable to find organization for license {}",
node_id.license_key
);
}
Ok(())
}
async fn update_last_seen(cnn: Pool<Postgres>, details: &NodeIdAndLicense) {
let res = pgdb::new_stats_arrived(cnn, &details.license_key, &details.node_id).await;
if res.is_err() {
error!(
"Unable to update last seen for node {}: {}",
details.node_id,
res.unwrap_err()
);
}
}

122
src/rust/long_term_stats/lts_ingestor/src/submissions/submission_queue/tree.rs
View File

@@ -1,122 +0,0 @@
use futures::prelude::*;
use influxdb2::{models::DataPoint, Client};
use lts_client::transport_data::StatsTreeNode;
use pgdb::{
sqlx::{Pool, Postgres},
OrganizationDetails,
};
use tracing::{info, error};
const SQL: &str = "INSERT INTO site_tree (key, host_id, site_name, index, parent, site_type, max_up, max_down, current_up, current_down, current_rtt) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (key, host_id, site_name) DO NOTHING";
pub async fn collect_tree(
cnn: Pool<Postgres>,
org: &OrganizationDetails,
node_id: &str,
timestamp: i64,
totals: &Option<Vec<StatsTreeNode>>,
) -> anyhow::Result<()> {
if let Some(tree) = totals {
//info!("{tree:?}");
let influx_url = format!("http://{}:8086", org.influx_host);
let client = Client::new(&influx_url, &org.influx_org, &org.influx_token);
let mut points: Vec<DataPoint> = Vec::new();
let mut trans = cnn.begin().await?;
pgdb::sqlx::query("DELETE FROM site_tree WHERE key=$1 AND host_id=$2")
.bind(org.key.to_string())
.bind(node_id)
.execute(&mut trans)
.await?;
for node in tree.iter() {
let mut parents = format!("S{}S", node.parents.iter().map(|p| p.to_string()).collect::<Vec<String>>().join("S"));
if parents.is_empty() {
parents = "0S".to_string();
}
let my_id = node.index.to_string();
//let parent = node.immediate_parent.unwrap_or(0).to_string();
//warn!("{}: {}", node.name, parents);
//warn!("{parent}");
points.push(
DataPoint::builder("tree")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("node_name", node.name.to_string())
.tag("direction", "down".to_string())
.tag("node_parents", parents.clone())
.tag("node_index", my_id.clone())
.timestamp(timestamp)
.field("bits_min", node.current_throughput.min.0 as i64)
.field("bits_max", node.current_throughput.max.0 as i64)
.field("bits_avg", node.current_throughput.avg.0 as i64)
.build()?,
);
points.push(
DataPoint::builder("tree")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("node_name", node.name.to_string())
.tag("direction", "up".to_string())
.tag("node_parents", parents.clone())
.tag("node_index", my_id.clone())
.timestamp(timestamp)
.field("bits_min", node.current_throughput.min.1 as i64)
.field("bits_max", node.current_throughput.max.1 as i64)
.field("bits_avg", node.current_throughput.avg.1 as i64)
.build()?,
);
points.push(
DataPoint::builder("tree")
.tag("host_id", node_id.to_string())
.tag("organization_id", org.key.to_string())
.tag("node_name", node.name.to_string())
.tag("node_parents", parents)
.tag("node_index", my_id.clone())
.timestamp(timestamp)
.field("rtt_min", node.rtt.min as i64 / 100)
.field("rtt_max", node.rtt.max as i64 / 100)
.field("rtt_avg", node.rtt.avg as i64 / 100)
.build()?,
);
let result = pgdb::sqlx::query(SQL)
.bind(org.key.to_string())
.bind(node_id)
.bind(&node.name)
.bind(node.index as i32)
.bind(node.immediate_parent.unwrap_or(0) as i32)
.bind(node.node_type.as_ref().unwrap_or(&String::new()).clone())
.bind(node.max_throughput.1 as i64)
.bind(node.max_throughput.0 as i64)
.bind(node.current_throughput.max.1 as i64)
.bind(node.current_throughput.max.0 as i64)
.bind(node.rtt.avg as i64)
.execute(&mut trans)
.await;
if let Err(e) = result {
error!("Error inserting tree node: {}", e);
}
}
let result = trans.commit().await;
info!("Transaction committed");
if let Err(e) = result {
error!("Error committing transaction: {}", e);
}
if let Err(e) = client
.write_with_precision(
&org.influx_bucket,
stream::iter(points),
influxdb2::api::write::TimestampPrecision::Seconds,
)
.await {
error!("Error committing tree to Influx: {}", e);
}
info!("Wrote tree to InfluxDB");
}
Ok(())
}

115
src/rust/long_term_stats/lts_ingestor/src/submissions/submission_queue/uisp_devices.rs
View File

@@ -1,115 +0,0 @@
use futures::prelude::*;
use influxdb2::{models::DataPoint, Client};
use lts_client::transport_data::UispExtDevice;
use pgdb::{
sqlx::{Pool, Postgres},
OrganizationDetails,
};
pub async fn collect_uisp_devices(
cnn: Pool<Postgres>,
org: &OrganizationDetails,
devices: &Option<Vec<UispExtDevice>>,
ts: i64,
) {
let (sql, influx) = tokio::join!(uisp_sql(cnn, org, devices), uisp_influx(org, devices, ts),);
if let Err(e) = sql {
tracing::error!("Error writing uisp sql: {:?}", e);
}
if let Err(e) = influx {
tracing::error!("Error writing uisp influx: {:?}", e);
}
}
async fn uisp_sql(
cnn: Pool<Postgres>,
org: &OrganizationDetails,
devices: &Option<Vec<UispExtDevice>>,
) -> anyhow::Result<()> {
if let Some(devices) = devices {
let mut trans = cnn.begin().await.unwrap();
// Handle the SQL portion (things that don't need to be graphed, just displayed)
pgdb::sqlx::query("DELETE FROM uisp_devices_ext WHERE key=$1")
.bind(org.key.to_string())
.execute(&mut trans)
.await?;
pgdb::sqlx::query("DELETE FROM uisp_devices_interfaces WHERE key=$1")
.bind(org.key.to_string())
.execute(&mut trans)
.await?;
for device in devices.iter() {
pgdb::sqlx::query("INSERT INTO uisp_devices_ext (key, device_id, name, model, firmware, status, mode) VALUES ($1, $2, $3, $4, $5, $6, $7)")
.bind(org.key.to_string())
.bind(&device.device_id)
.bind(&device.name)
.bind(&device.model)
.bind(&device.firmware)
.bind(&device.status)
.bind(&device.mode)
.execute(&mut trans)
.await?;
for interface in device.interfaces.iter() {
let mut ip_list = String::new();
for ip in interface.ip.iter() {
ip_list.push_str(&format!("{} ", ip));
}
pgdb::sqlx::query("INSERT INTO uisp_devices_interfaces (key, device_id, name, mac, status, speed, ip_list) VALUES ($1, $2, $3, $4, $5, $6, $7)")
.bind(org.key.to_string())
.bind(&device.device_id)
.bind(&interface.name)
.bind(&interface.mac)
.bind(&interface.status)
.bind(&interface.speed)
.bind(ip_list)
.execute(&mut trans)
.await?;
}
}
trans.commit().await?;
}
Ok(())
}
async fn uisp_influx(
org: &OrganizationDetails,
devices: &Option<Vec<UispExtDevice>>,
timestamp: i64,
) -> anyhow::Result<()> {
if let Some(devices) = devices {
let influx_url = format!("http://{}:8086", org.influx_host);
let client = Client::new(&influx_url, &org.influx_org, &org.influx_token);
let mut points: Vec<DataPoint> = Vec::new();
for device in devices.iter() {
points.push(
DataPoint::builder("device_ext")
.tag("device_id", &device.device_id)
.tag("organization_id", org.key.to_string())
.tag("direction", "down".to_string())
.timestamp(timestamp)
.field("rx_signal", device.rx_signal as i64)
.field("noise_floor", device.noise_floor as i64)
.field("dl_capacity", device.downlink_capacity_mbps as i64)
.field("ul_capacity", device.uplink_capacity_mbps as i64)
.build()?,
);
}
client
.write_with_precision(
&org.influx_bucket,
stream::iter(points),
influxdb2::api::write::TimestampPrecision::Seconds,
)
.await?;
}
Ok(())
}

102
src/rust/long_term_stats/lts_ingestor/src/submissions/submission_server.rs
View File

@@ -1,102 +0,0 @@
//! Provides a TCP handler server, listening on port 9128. Connections
//! are expected in the encrypted LTS format (see the `lq_bus` crate).
//! If everything checks out, they are sent to the submission queue
//! for storage.
use super::submission_queue::SubmissionType;
use crate::pki::LIBREQOS_KEYPAIR;
use lts_client::{
dryoc::dryocbox::{DryocBox, PublicKey},
transport_data::{LtsCommand, NodeIdAndLicense},
};
use pgdb::sqlx::{Pool, Postgres};
use tokio::{io::AsyncReadExt, net::{TcpListener, TcpStream}, spawn, sync::mpsc::Sender};
use tracing::{info, error, warn};
/// Starts the submission server, listening on port 9128.
/// The server runs in the background.
pub async fn submissions_server(
cnn: Pool<Postgres>,
sender: Sender<SubmissionType>,
) -> anyhow::Result<()> {
let listener = TcpListener::bind(":::9128").await?;
info!("Listening for stats submissions on :::9128");
loop {
let (mut socket, address) = listener.accept().await?;
info!("Connection from {address:?}");
let pool = cnn.clone();
let my_sender = sender.clone();
spawn(async move {
loop {
if let Ok(message) = read_message(&mut socket, pool.clone()).await {
my_sender.send(message).await.unwrap();
} else {
error!("Read failed. Dropping socket.");
std::mem::drop(socket);
break;
}
}
});
}
}
#[tracing::instrument]
async fn read_message(socket: &mut TcpStream, pool: Pool<Postgres>) -> anyhow::Result<SubmissionType> {
read_version(socket).await?;
let header_size = read_size(socket).await?;
let header = read_header(socket, header_size as usize).await?;
let body_size = read_size(socket).await?;
let message = read_body(socket, pool.clone(), body_size as usize, &header).await?;
Ok((header, message))
}
async fn read_version(stream: &mut TcpStream) -> anyhow::Result<()> {
let version = stream.read_u16().await?;
if version != 1 {
warn!("Received a version {version} header.");
return Err(anyhow::Error::msg("Received an unknown version header"));
}
Ok(())
}
async fn read_size(stream: &mut TcpStream) -> anyhow::Result<u64> {
let size = stream.read_u64().await?;
Ok(size)
}
async fn read_header(stream: &mut TcpStream, size: usize) -> anyhow::Result<NodeIdAndLicense> {
let mut buffer = vec![0u8; size];
let _bytes_read = stream.read(&mut buffer).await?;
let header: NodeIdAndLicense = lts_client::cbor::from_slice(&buffer)?;
Ok(header)
}
async fn read_body(stream: &mut TcpStream, pool: Pool<Postgres>, size: usize, header: &NodeIdAndLicense) -> anyhow::Result<LtsCommand> {
info!("Reading body of size {size}");
info!("{header:?}");
let mut buffer = vec![0u8; size];
let bytes_read = stream.read_exact(&mut buffer).await?;
if bytes_read != size {
warn!("Received a body of size {bytes_read}, expected {size}");
return Err(anyhow::Error::msg("Received a body of unexpected size"));
}
// Check the header against the database and retrieve the current
// public key
let public_key = pgdb::fetch_public_key(pool, &header.license_key, &header.node_id).await?;
let public_key: PublicKey = lts_client::cbor::from_slice(&public_key)?;
let private_key = LIBREQOS_KEYPAIR.read().unwrap().secret_key.clone();
// Decrypt
let dryocbox = DryocBox::from_bytes(&buffer).expect("failed to read box");
let decrypted = dryocbox
.decrypt_to_vec(&header.nonce.into(), &public_key, &private_key)?;
let decrypted = miniz_oxide::inflate::decompress_to_vec(&decrypted).expect("failed to decompress");
// Try to deserialize
let payload = lts_client::cbor::from_slice(&decrypted)?;
Ok(payload)
}

35
src/rust/long_term_stats/lts_node/Cargo.toml
View File

@@ -1,35 +0,0 @@
[package]
name = "lts_node"
version = "0.1.0"
edition = "2021"
license = "GPL-2.0-only"
[features]
default = []
tokio-console = ["console-subscriber"]
[dependencies]
tokio = { version = "1.25.0", features = ["full"] }
anyhow = "1"
serde = { version = "1.0", features = ["derive"] }
axum = {version = "0.6", features = ["ws", "headers"] }
lts_client = { path = "../lts_client" }
lqos_config = { path = "../../lqos_config" }
serde_json = "1"
pgdb = { path = "../pgdb" }
once_cell = "1"
influxdb2 = "0"
influxdb2-structmap = "0"
num-traits = "0"
futures = "0"
tracing = "0.1"
tracing-subscriber = { version = "0.3" }
tower = { version = "0.4", features = ["util"] }
tower-http = { version = "0.4.0", features = ["fs", "trace"] }
chrono = "0"
miniz_oxide = "0.7.1"
tokio-util = { version = "0.7.8", features = ["io"] }
wasm_pipe_types = { path = "../wasm_pipe_types" }
console-subscriber = {version = "0.1.10", optional = true }
itertools = "0.11.0"
urlencoding = "2.1.3"

13
src/rust/long_term_stats/lts_node/run.sh
View File

@@ -1,13 +0,0 @@
#!/bin/bash
pushd ../wasm_pipe
./build.sh
popd
pushd ../site_build
./esbuild.mjs
popd
pushd web
cp ../../site_build/output/* .
cp ../../site_build/src/main.html .
cp ../../site_build/wasm/wasm_pipe_bg.wasm .
popd
RUST_LOG=info RUST_BACKTRACE=1 cargo run

64
src/rust/long_term_stats/lts_node/src/main.rs
View File

@@ -1,64 +0,0 @@
mod web;
use tracing::{error, info};
use tracing_subscriber::fmt::format::FmtSpan;
#[cfg(not(feature="tokio-console"))]
fn set_console_logging() -> anyhow::Result<()> {
// install global collector configured based on RUST_LOG env var.
let subscriber = tracing_subscriber::fmt()
// Use a more compact, abbreviated log format
.compact()
// Display source code file paths
.with_file(true)
// Display source code line numbers
.with_line_number(true)
// Display the thread ID an event was recorded on
.with_thread_ids(true)
// Don't display the event's target (module path)
.with_target(false)
// Include per-span timings
.with_span_events(FmtSpan::CLOSE)
// Build the subscriber
.finish();
// Set the subscriber as the default
tracing::subscriber::set_global_default(subscriber)?;
Ok(())
}
#[cfg(feature="tokio-console")]
fn set_tokio_console() {
// Initialize the Tokio Console subscription
console_subscriber::init();
}
#[cfg(not(feature="tokio-console"))]
fn setup_tracing() {
set_console_logging().unwrap();
}
#[cfg(feature="tokio-console")]
fn setup_tracing() {
set_tokio_console();
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
setup_tracing();
// Get the database connection pool
let pool = pgdb::get_connection_pool(5).await;
if pool.is_err() {
error!("Unable to connect to the database");
error!("{pool:?}");
return Err(anyhow::Error::msg("Unable to connect to the database"));
}
let pool = pool.unwrap();
// Start the webserver
info!("Starting the webserver");
let _ = tokio::spawn(web::webserver(pool)).await;
Ok(())
}

92
src/rust/long_term_stats/lts_node/src/web/mod.rs
View File

@@ -1,92 +0,0 @@
//! The webserver listens on port 9127, but it is intended that this only
//! listen on localhost and have a reverse proxy in front of it. The proxy
//! should provide HTTPS.
mod wss;
use crate::web::wss::ws_handler;
use axum::body::StreamBody;
use axum::http::{header, HeaderMap};
use axum::response::IntoResponse;
use axum::{response::Html, routing::get, Router};
use pgdb::sqlx::Pool;
use pgdb::sqlx::Postgres;
use tokio_util::io::ReaderStream;
use tower_http::trace::TraceLayer;
use tower_http::trace::DefaultMakeSpan;
const JS_BUNDLE: &str = include_str!("../../web/app.js");
const JS_MAP: &str = include_str!("../../web/app.js.map");
const CSS: &str = include_str!("../../web/style.css");
const CSS_MAP: &str = include_str!("../../web/style.css.map");
const HTML_MAIN: &str = include_str!("../../web/main.html");
const WASM_BODY: &[u8] = include_bytes!("../../web/wasm_pipe_bg.wasm");
pub async fn webserver(cnn: Pool<Postgres>) {
let app = Router::new()
.route("/", get(index_page))
.route("/app.js", get(js_bundle))
.route("/app.js.map", get(js_map))
.route("/style.css", get(css))
.route("/style.css.map", get(css_map))
.route("/ws", get(ws_handler))
.route("/wasm_pipe_bg.wasm", get(wasm_file))
.with_state(cnn)
.layer(
TraceLayer::new_for_http()
.make_span_with(DefaultMakeSpan::default().include_headers(true)),
);
tracing::info!("Listening for web traffic on 0.0.0.0:9127");
axum::Server::bind(&"0.0.0.0:9127".parse().unwrap())
.serve(app.into_make_service())
.await
.unwrap();
}
async fn index_page() -> Html<String> {
Html(HTML_MAIN.to_string())
}
async fn js_bundle() -> axum::response::Response<String> {
axum::response::Response::builder()
.header("Content-Type", "text/javascript")
.body(JS_BUNDLE.to_string())
.unwrap()
}
async fn js_map() -> axum::response::Response<String> {
axum::response::Response::builder()
.header("Content-Type", "text/json")
.body(JS_MAP.to_string())
.unwrap()
}
async fn css() -> axum::response::Response<String> {
axum::response::Response::builder()
.header("Content-Type", "text/css")
.body(CSS.to_string())
.unwrap()
}
async fn css_map() -> axum::response::Response<String> {
axum::response::Response::builder()
.header("Content-Type", "text/json")
.body(CSS_MAP.to_string())
.unwrap()
}
async fn wasm_file() -> impl IntoResponse {
let mut headers = HeaderMap::new();
headers.insert(
header::CONTENT_TYPE,
header::HeaderValue::from_static("application/wasm"),
);
headers.insert(
header::CONTENT_DISPOSITION,
header::HeaderValue::from_static("attachment; filename=wasm_pipe_bg.wasm"),
);
axum::response::Response::builder()
.header(header::CONTENT_TYPE, header::HeaderValue::from_static("application/wasm"))
.header(header::CONTENT_DISPOSITION, header::HeaderValue::from_static("attachment; filename=wasm_pipe_bg.wasm"))
.body(StreamBody::new(ReaderStream::new(WASM_BODY)))
.unwrap()
}

58
src/rust/long_term_stats/lts_node/src/web/wss/login.rs
View File

@@ -1,58 +0,0 @@
use pgdb::sqlx::{Pool, Postgres};
use serde::Serialize;
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use wasm_pipe_types::WasmResponse;
#[derive(Debug, Serialize, Clone)]
pub struct LoginResult {
pub msg: String,
pub token: String,
pub name: String,
pub license_key: String,
}
#[instrument(skip(license, username, password, tx, cnn))]
pub async fn on_login(license: &str, username: &str, password: &str, tx: Sender<WasmResponse>, cnn: Pool<Postgres>) -> Option<LoginResult> {
let login = pgdb::try_login(cnn, license, username, password).await;
if let Ok(login) = login {
let lr = WasmResponse::LoginOk {
token: login.token.clone(),
name: login.name.clone(),
license_key: license.to_string(),
};
tx.send(lr).await.unwrap();
return Some(LoginResult {
msg: "Login Ok".to_string(),
token: login.token.to_string(),
name: login.name.to_string(),
license_key: license.to_string(),
});
} else {
let lr = WasmResponse::LoginFail;
tx.send(lr).await.unwrap();
}
None
}
#[instrument(skip(token_id, tx, cnn))]
pub async fn on_token_auth(token_id: &str, tx: Sender<WasmResponse>, cnn: Pool<Postgres>) -> Option<LoginResult> {
let login = pgdb::token_to_credentials(cnn, token_id).await;
if let Ok(login) = login {
let lr = WasmResponse::AuthOk {
token: login.token.clone(),
name: login.name.clone(),
license_key: login.license.clone(),
};
tx.send(lr).await.unwrap();
return Some(LoginResult {
msg: "Login Ok".to_string(),
token: login.token.to_string(),
name: login.name.to_string(),
license_key: login.license.to_string(),
});
} else {
tx.send(WasmResponse::AuthFail).await.unwrap();
}
None
}

396
src/rust/long_term_stats/lts_node/src/web/wss/mod.rs
View File

@@ -1,396 +0,0 @@
use std::sync::Arc;
use crate::web::wss::{
nodes::node_status,
queries::{
ext_device::{
send_extended_device_capacity_graph, send_extended_device_info,
send_extended_device_snr_graph,
},
omnisearch, root_heat_map, send_circuit_info, send_circuit_parents,
send_packets_for_all_nodes, send_packets_for_node, send_perf_for_node, send_root_parents,
send_rtt_for_all_nodes, send_rtt_for_all_nodes_circuit, send_rtt_for_all_nodes_site,
send_rtt_for_node, send_rtt_histogram_for_all_nodes, send_site_info, send_site_parents,
send_site_stack_map, send_throughput_for_all_nodes,
send_throughput_for_all_nodes_by_circuit, send_throughput_for_all_nodes_by_site,
send_throughput_for_node, site_heat_map,
site_tree::send_site_tree,
},
};
use axum::{
extract::{
ws::{Message, WebSocket, WebSocketUpgrade},
State,
},
response::IntoResponse,
};
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::{mpsc::Sender, Mutex};
use tracing::instrument;
use wasm_pipe_types::{WasmRequest, WasmResponse};
use self::queries::InfluxTimePeriod;
mod login;
mod nodes;
mod queries;
pub async fn ws_handler(
ws: WebSocketUpgrade,
State(state): State<Pool<Postgres>>,
) -> impl IntoResponse {
ws.on_upgrade(move |sock| handle_socket(sock, state))
}
#[instrument(skip(socket, cnn), name = "handle_wss")]
async fn handle_socket(mut socket: WebSocket, cnn: Pool<Postgres>) {
tracing::info!("WebSocket Connected");
let credentials: Arc<Mutex<Option<login::LoginResult>>> = Arc::new(Mutex::new(None));
// Setup the send/receive channel
let (tx, mut rx) = tokio::sync::mpsc::channel::<WasmResponse>(10);
loop {
tokio::select! {
msg = socket.recv() => {
match msg {
Some(msg) => {
tokio::spawn(
handle_socket_message(msg.unwrap(), cnn.clone(), credentials.clone(), tx.clone())
);
}
None => {
tracing::info!("WebSocket Disconnected");
break;
}
}
},
msg = rx.recv() => {
match msg {
Some(msg) => {
let serialized = serialize_response(msg);
socket.send(Message::Binary(serialized)).await.unwrap();
}
None => {
tracing::info!("WebSocket Disconnected");
break;
}
}
},
}
}
}
#[instrument(skip(credentials, cnn))]
async fn update_token_credentials(
credentials: Arc<Mutex<Option<login::LoginResult>>>,
cnn: Pool<Postgres>,
) {
let mut credentials = credentials.lock().await;
if let Some(credentials) = &mut *credentials {
let _ = pgdb::refresh_token(cnn, &credentials.token).await;
}
}
async fn set_credentials(
credentials: Arc<Mutex<Option<login::LoginResult>>>,
result: login::LoginResult,
) {
let mut credentials = credentials.lock().await;
*credentials = Some(result);
}
fn extract_message(msg: Message) -> WasmRequest {
let raw = msg.into_data();
let uncompressed = miniz_oxide::inflate::decompress_to_vec(&raw).unwrap();
lts_client::cbor::from_slice::<WasmRequest>(&uncompressed).unwrap()
}
async fn handle_auth_message(
msg: &WasmRequest,
credentials: Arc<Mutex<Option<login::LoginResult>>>,
tx: Sender<WasmResponse>,
cnn: Pool<Postgres>,
) {
match msg {
// Handle login with just a token
WasmRequest::Auth { token } => {
let result = login::on_token_auth(token, tx, cnn).await;
if let Some(result) = result {
set_credentials(credentials, result).await;
}
}
// Handle a full login
WasmRequest::Login { license, username, password } => {
let result = login::on_login(license, username, password, tx, cnn).await;
if let Some(result) = result {
set_credentials(credentials, result).await;
}
}
_ => {}
}
}
async fn handle_socket_message(
msg: Message,
cnn: Pool<Postgres>,
credentials: Arc<Mutex<Option<login::LoginResult>>>,
tx: Sender<WasmResponse>,
) {
// Get the binary message and decompress it
let msg = extract_message(msg);
update_token_credentials(credentials.clone(), cnn.clone()).await;
// Handle the message by type
handle_auth_message(&msg, credentials.clone(), tx.clone(), cnn.clone()).await;
let my_credentials = {
let lock = credentials.lock().await;
lock.clone()
};
let matcher = (&msg, &my_credentials);
match matcher {
// Node status for dashboard
(WasmRequest::GetNodeStatus, Some(credentials)) => {
node_status(&cnn, tx, &credentials.license_key).await;
}
// Packet chart for dashboard
(WasmRequest::PacketChart { period }, Some(credentials)) => {
let _ =
send_packets_for_all_nodes(&cnn, tx, &credentials.license_key, period.into()).await;
}
// Packet chart for individual node
(
WasmRequest::PacketChartSingle {
period,
node_id,
node_name,
},
Some(credentials),
) => {
let _ = send_packets_for_node(
&cnn,
tx,
&credentials.license_key,
period.into(),
node_id,
node_name,
)
.await;
}
// Throughput chart for the dashboard
(WasmRequest::ThroughputChart { period }, Some(credentials)) => {
let _ = send_throughput_for_all_nodes(
&cnn,
tx,
&credentials.license_key,
InfluxTimePeriod::new(period),
)
.await;
}
// Throughput chart for a single shaper node
(
WasmRequest::ThroughputChartSingle {
period,
node_id,
node_name,
},
Some(credentials),
) => {
let _ = send_throughput_for_node(
&cnn,
tx,
&credentials.license_key,
InfluxTimePeriod::new(period),
node_id.to_string(),
node_name.to_string(),
)
.await;
}
(WasmRequest::ThroughputChartSite { period, site_id }, Some(credentials)) => {
let site_id = urlencoding::decode(site_id).unwrap();
let _ = send_throughput_for_all_nodes_by_site(
&cnn,
tx,
&credentials.license_key,
site_id.to_string(),
InfluxTimePeriod::new(period),
)
.await;
}
(WasmRequest::ThroughputChartCircuit { period, circuit_id }, Some(credentials)) => {
let _ = send_throughput_for_all_nodes_by_circuit(
&cnn,
tx,
&credentials.license_key,
circuit_id.to_string(),
InfluxTimePeriod::new(period),
)
.await;
}
// Rtt Chart
(WasmRequest::RttChart { period }, Some(credentials)) => {
let _ = send_rtt_for_all_nodes(
&cnn,
tx,
&credentials.license_key,
InfluxTimePeriod::new(period),
)
.await;
}
(WasmRequest::RttHistogram { period }, Some(credentials)) => {
let _ = send_rtt_histogram_for_all_nodes(
&cnn,
tx,
&credentials.license_key,
InfluxTimePeriod::new(period),
)
.await;
}
(WasmRequest::RttChartSite { period, site_id }, Some(credentials)) => {
let site_id = urlencoding::decode(site_id).unwrap();
let _ = send_rtt_for_all_nodes_site(
&cnn,
tx,
&credentials.license_key,
site_id.to_string(),
InfluxTimePeriod::new(period),
)
.await;
}
(
WasmRequest::RttChartSingle {
period,
node_id,
node_name,
},
Some(credentials),
) => {
let _ = send_rtt_for_node(
&cnn,
tx,
&credentials.license_key,
InfluxTimePeriod::new(period),
node_id.to_string(),
node_name.to_string(),
)
.await;
}
(WasmRequest::RttChartCircuit { period, circuit_id }, Some(credentials)) => {
let _ = send_rtt_for_all_nodes_circuit(
&cnn,
tx,
&credentials.license_key,
circuit_id.to_string(),
InfluxTimePeriod::new(period),
)
.await;
}
// Site Stack
(WasmRequest::SiteStack { period, site_id }, Some(credentials)) => {
let site_id = urlencoding::decode(site_id).unwrap();
let _ = send_site_stack_map(
&cnn,
tx,
&credentials.license_key,
InfluxTimePeriod::new(period),
site_id.to_string(),
)
.await;
}
(WasmRequest::RootHeat { period }, Some(credentials)) => {
let _ = root_heat_map(
&cnn,
tx,
&credentials.license_key,
InfluxTimePeriod::new(period),
)
.await;
}
(WasmRequest::SiteHeat { period, site_id }, Some(credentials)) => {
let _ = site_heat_map(
&cnn,
tx,
&credentials.license_key,
site_id,
InfluxTimePeriod::new(period),
)
.await;
}
(
WasmRequest::NodePerfChart {
period,
node_id,
node_name,
},
Some(credentials),
) => {
let _ = send_perf_for_node(
&cnn,
tx,
&credentials.license_key,
InfluxTimePeriod::new(period),
node_id.to_string(),
node_name.to_string(),
)
.await;
}
(WasmRequest::Tree { parent }, Some(credentials)) => {
send_site_tree(&cnn, tx, &credentials.license_key, parent).await;
}
(WasmRequest::SiteInfo { site_id }, Some(credentials)) => {
send_site_info(&cnn, tx, &credentials.license_key, site_id).await;
}
(WasmRequest::SiteParents { site_id }, Some(credentials)) => {
let site_id = urlencoding::decode(site_id).unwrap();
send_site_parents(&cnn, tx, &credentials.license_key, &site_id).await;
}
(WasmRequest::CircuitParents { circuit_id }, Some(credentials)) => {
let circuit_id = urlencoding::decode(circuit_id).unwrap();
send_circuit_parents(&cnn, tx, &credentials.license_key, &circuit_id).await;
}
(WasmRequest::RootParents, Some(credentials)) => {
send_root_parents(&cnn, tx, &credentials.license_key).await;
}
(WasmRequest::Search { term }, Some(credentials)) => {
let _ = omnisearch(&cnn, tx, &credentials.license_key, term).await;
}
(WasmRequest::CircuitInfo { circuit_id }, Some(credentials)) => {
send_circuit_info(&cnn, tx, &credentials.license_key, circuit_id).await;
}
(WasmRequest::ExtendedDeviceInfo { circuit_id }, Some(credentials)) => {
send_extended_device_info(&cnn, tx, &credentials.license_key, circuit_id).await;
}
(WasmRequest::SignalNoiseChartExt { period, device_id }, Some(credentials)) => {
let _ = send_extended_device_snr_graph(
&cnn,
tx,
&credentials.license_key,
device_id,
&InfluxTimePeriod::new(period),
)
.await;
}
(WasmRequest::DeviceCapacityChartExt { period, device_id }, Some(credentials)) => {
let _ = send_extended_device_capacity_graph(
&cnn,
tx,
&credentials.license_key,
device_id,
&InfluxTimePeriod::new(period),
)
.await;
}
(WasmRequest::ApSignalExt { period, site_name }, Some(credentials)) => {}
(WasmRequest::ApCapacityExt { period, site_name }, Some(credentials)) => {}
(_, None) => {
tracing::error!("No credentials");
}
_ => {
let error = format!("Unknown message: {msg:?}");
tracing::error!(error);
}
}
}
fn serialize_response(response: WasmResponse) -> Vec<u8> {
let cbor = lts_client::cbor::to_vec(&response).unwrap();
miniz_oxide::deflate::compress_to_vec(&cbor, 8)
}

27
src/rust/long_term_stats/lts_node/src/web/wss/nodes.rs
View File

@@ -1,27 +0,0 @@
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use wasm_pipe_types::{Node, WasmResponse};
fn convert(ns: pgdb::NodeStatus) -> Node {
Node {
node_id: ns.node_id,
node_name: ns.node_name,
last_seen: ns.last_seen,
}
}
#[instrument(skip(cnn, tx, key))]
pub async fn node_status(cnn: &Pool<Postgres>, tx: Sender<WasmResponse>, key: &str) {
tracing::info!("Fetching node status, {key}");
let nodes = pgdb::node_status(cnn, key).await;
match nodes {
Ok(nodes) => {
let nodes: Vec<Node> = nodes.into_iter().map(convert).collect();
tx.send(wasm_pipe_types::WasmResponse::NodeStatus { nodes }).await.unwrap();
},
Err(e) => {
tracing::error!("Unable to obtain node status: {}", e);
}
}
}

28
src/rust/long_term_stats/lts_node/src/web/wss/queries/circuit_info.rs
View File

@@ -1,28 +0,0 @@
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::mpsc::Sender;
use wasm_pipe_types::{CircuitList, WasmResponse};
fn from(circuit: pgdb::CircuitInfo) -> CircuitList {
CircuitList {
circuit_name: circuit.circuit_name,
device_id: circuit.device_id,
device_name: circuit.device_name,
parent_node: circuit.parent_node,
mac: circuit.mac,
download_min_mbps: circuit.download_min_mbps,
download_max_mbps: circuit.download_max_mbps,
upload_min_mbps: circuit.upload_min_mbps,
upload_max_mbps: circuit.upload_max_mbps,
comment: circuit.comment,
ip_range: circuit.ip_range,
subnet: circuit.subnet,
}
}
#[tracing::instrument(skip(cnn, tx, key, circuit_id))]
pub async fn send_circuit_info(cnn: &Pool<Postgres>, tx: Sender<WasmResponse>, key: &str, circuit_id: &str) {
if let Ok(hosts) = pgdb::get_circuit_info(cnn, key, circuit_id).await {
let hosts = hosts.into_iter().map(from).collect::<Vec<_>>();
tx.send(WasmResponse::CircuitInfo { data: hosts }).await.unwrap();
}
}

164
src/rust/long_term_stats/lts_node/src/web/wss/queries/ext_device.rs
View File

@@ -1,164 +0,0 @@
use std::collections::HashSet;
use axum::extract::ws::WebSocket;
use chrono::{DateTime, FixedOffset};
use influxdb2::FromDataPoint;
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::mpsc::Sender;
use wasm_pipe_types::{WasmResponse, SignalNoiseChartExt, CapacityChartExt};
use super::{influx::InfluxTimePeriod, QueryBuilder};
#[tracing::instrument(skip(cnn, tx, key, circuit_id))]
pub async fn send_extended_device_info(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
circuit_id: &str,
) {
// Get devices for circuit
if let Ok(hosts_list) = pgdb::get_circuit_info(cnn, key, circuit_id).await {
tracing::error!("Got hosts list: {:?}", hosts_list);
// Get the hosts known to be in this circuit
let mut hosts = HashSet::new();
hosts_list.into_iter().for_each(|h| {
hosts.insert(h.device_id);
});
if hosts.is_empty() {
return;
}
println!("{hosts:?}");
// Get extended data
let mut extended_data = Vec::new();
for host in hosts.iter() {
let ext = pgdb::get_device_info_ext(cnn, key, host).await;
if let Ok(ext) = ext {
let mut ext_wasm = wasm_pipe_types::ExtendedDeviceInfo {
device_id: ext.device_id.clone(),
name: ext.name.clone(),
model: ext.model.clone(),
firmware: ext.firmware.clone(),
status: ext.status.clone(),
mode: ext.mode.clone(),
channel_width: ext.channel_width,
tx_power: ext.tx_power,
interfaces: Vec::new(),
};
if let Ok(interfaces) = pgdb::get_device_interfaces_ext(cnn, key, host).await {
for ed in interfaces {
let edw = wasm_pipe_types::ExtendedDeviceInterface {
name: ed.name,
mac: ed.mac,
status: ed.status,
speed: ed.speed,
ip_list: ed.ip_list,
};
ext_wasm.interfaces.push(edw);
}
}
extended_data.push(ext_wasm);
} else {
tracing::error!("Error getting extended device info: {:?}", ext);
}
}
// If there is any, send it
println!("{extended_data:?}");
if !extended_data.is_empty() {
tx.send(WasmResponse::DeviceExt { data: extended_data }).await.unwrap();
}
}
}
#[tracing::instrument(skip(cnn, tx, key, device_id, period))]
pub async fn send_extended_device_snr_graph(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
device_id: &str,
period: &InfluxTimePeriod,
) -> anyhow::Result<()> {
let rows = QueryBuilder::new()
.with_period(period)
.derive_org(cnn, key)
.await
.bucket()
.range()
.measure_fields_org("device_ext", &["noise_floor", "rx_signal"])
.filter(&format!("r[\"device_id\"] == \"{}\"", device_id))
.aggregate_window()
.execute::<SnrRow>()
.await?
.into_iter()
.map(|row| {
wasm_pipe_types::SignalNoiseChartExt {
noise: row.noise_floor,
signal: row.rx_signal,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
}
})
.collect::<Vec<SignalNoiseChartExt>>();
tx.send(WasmResponse::DeviceExtSnr { data: rows, device_id: device_id.to_string() }).await?;
Ok(())
}
pub async fn send_ap_snr(
cnn: &Pool<Postgres>,
socket: &mut WebSocket,
key: &str,
site_name: &str,
period: InfluxTimePeriod,
) -> anyhow::Result<()> {
// Get list of child devices
let hosts = pgdb::get_host_list_for_site(cnn, key, site_name).await?;
let host_filter = pgdb::device_list_to_influx_filter(&hosts);
Ok(())
}
#[derive(Debug, FromDataPoint, Default)]
pub struct SnrRow {
pub device_id: String,
pub noise_floor: f64,
pub rx_signal: f64,
pub time: DateTime<FixedOffset>,
}
#[tracing::instrument(skip(cnn, tx, key, device_id, period))]
pub async fn send_extended_device_capacity_graph(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
device_id: &str,
period: &InfluxTimePeriod,
) -> anyhow::Result<()> {
let rows = QueryBuilder::new()
.with_period(period)
.derive_org(cnn, key)
.await
.bucket()
.range()
.measure_fields_org("device_ext", &["dl_capacity", "ul_capacity"])
.filter(&format!("r[\"device_id\"] == \"{}\"", device_id))
.aggregate_window()
.execute::<CapacityRow>()
.await?
.into_iter()
.map(|row| {
wasm_pipe_types::CapacityChartExt {
dl: row.dl_capacity,
ul: row.ul_capacity,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
}
})
.collect::<Vec<CapacityChartExt>>();
tx.send(WasmResponse::DeviceExtCapacity { data: rows, device_id: device_id.to_string() }).await?;
Ok(())
}
#[derive(Debug, FromDataPoint, Default)]
pub struct CapacityRow {
pub device_id: String,
pub dl_capacity: f64,
pub ul_capacity: f64,
pub time: DateTime<FixedOffset>,
}

220
src/rust/long_term_stats/lts_node/src/web/wss/queries/influx/influx_query_builder.rs
View File

@@ -1,220 +0,0 @@
#![allow(dead_code)]
use influxdb2::{Client, models::Query};
use influxdb2_structmap::FromMap;
use pgdb::{sqlx::{Pool, Postgres}, organization_cache::get_org_details, OrganizationDetails};
use anyhow::{Result, Error};
use tracing::instrument;
use super::InfluxTimePeriod;
#[derive(Debug)]
pub struct InfluxQueryBuilder {
imports: Vec<String>,
fields: Vec<String>,
period: InfluxTimePeriod,
measurement: Option<String>,
group_by: Vec<String>,
aggregate_window: bool,
yield_as: Option<String>,
host_id: Option<String>,
filters: Vec<String>,
sample_after_org: bool,
fill_empty: bool,
}
impl InfluxQueryBuilder {
pub fn new(period: InfluxTimePeriod) -> Self {
Self {
fields: Vec::new(),
imports: Vec::new(),
group_by: Vec::new(),
period,
measurement: None,
aggregate_window: true,
yield_as: Some("last".to_string()),
host_id: None,
filters: Vec::new(),
sample_after_org: false,
fill_empty: false,
}
}
pub fn with_measurement<S: ToString>(mut self, measurement: S) -> Self {
self.measurement = Some(measurement.to_string());
self
}
pub fn with_import<S: ToString>(mut self, import: S) -> Self {
self.imports.push(import.to_string());
self
}
pub fn with_field<S: ToString>(mut self, field: S) -> Self {
self.fields.push(field.to_string());
self
}
pub fn with_fields<S: ToString>(mut self, fields: &[S]) -> Self {
for field in fields.iter() {
self.fields.push(field.to_string());
}
self
}
pub fn with_group<S: ToString>(mut self, group: S) -> Self {
self.group_by.push(group.to_string());
self
}
pub fn with_groups<S: ToString>(mut self, group: &[S]) -> Self {
for group in group.iter() {
self.group_by.push(group.to_string());
}
self
}
pub fn with_host_id<S: ToString>(mut self, host_id: S) -> Self {
self.host_id = Some(host_id.to_string());
self
}
pub fn sample_no_window(mut self) -> Self {
self.aggregate_window = false;
self
}
pub fn with_filter<S: ToString>(mut self, filter: S) -> Self {
self.filters.push(filter.to_string());
self
}
pub fn sample_after_org(mut self) -> Self {
self.sample_after_org = true;
self
}
pub fn fill_empty(mut self) -> Self {
self.fill_empty = true;
self
}
fn build_query(&self, org: &OrganizationDetails) -> String {
let mut lines = Vec::<String>::with_capacity(10);
// Add any import stanzas
self.imports.iter().for_each(|i| lines.push(format!("import \"{i}\"")));
// Add the bucket
lines.push(format!("from(bucket: \"{}\")", org.influx_bucket));
// Add a range limit
lines.push(format!("|> {}", self.period.range()));
// Add the measurement filter
if let Some(measurement) = &self.measurement {
lines.push(format!("|> filter(fn: (r) => r[\"_measurement\"] == \"{}\")", measurement));
}
// Add fields filters
if !self.fields.is_empty() {
let mut fields = String::new();
for field in self.fields.iter() {
if !fields.is_empty() {
fields.push_str(" or ");
}
fields.push_str(&format!("r[\"_field\"] == \"{}\"", field));
}
lines.push(format!("|> filter(fn: (r) => {})", fields));
}
// Filter by organization id
lines.push(format!("|> filter(fn: (r) => r[\"organization_id\"] == \"{}\")", org.key));
if self.sample_after_org {
lines.push(format!("|> {}", self.period.sample()));
}
// Filter by host_id
if let Some(host_id) = &self.host_id {
lines.push(format!("|> filter(fn: (r) => r[\"host_id\"] == \"{}\")", host_id));
}
// Add any other filters
for filter in self.filters.iter().filter(|f| !f.is_empty()) {
lines.push(format!("|> filter(fn: (r) => {})", filter));
}
// Group by
if !self.group_by.is_empty() {
let mut group_by = String::new();
for group in self.group_by.iter() {
if !group_by.is_empty() {
group_by.push_str(", ");
}
group_by.push_str(&format!("\"{}\"", group));
}
lines.push(format!("|> group(columns: [{}])", group_by));
}
// Aggregate Window
if self.aggregate_window {
if self.fill_empty {
lines.push(format!("|> {}", self.period.aggregate_window_empty()));
} else {
lines.push(format!("|> {}", self.period.aggregate_window()));
}
} else {
lines.push(format!("|> {}", self.period.sample()));
}
// Yield as
if let Some(yield_as) = &self.yield_as {
lines.push(format!("|> yield(name: \"{}\")", yield_as));
}
// Combine
lines.join("\n")
}
#[instrument(skip(self, cnn, key))]
pub async fn execute<T>(&self, cnn: &Pool<Postgres>, key: &str) -> Result<Vec<T>>
where T: FromMap + std::fmt::Debug
{
if let Some(org) = get_org_details(cnn, key).await {
let influx_url = format!("http://{}:8086", org.influx_host);
let client = Client::new(influx_url, &org.influx_org, &org.influx_token);
let query_string = self.build_query(&org);
tracing::info!("{query_string}");
let query = Query::new(query_string);
let rows = client.query::<T>(Some(query)).await;
if let Ok(rows) = rows {
Ok(rows)
} else {
tracing::error!("InfluxDb query error: {rows:?}");
Err(Error::msg("Influx query error"))
}
} else {
Err(Error::msg("Organization not found"))
}
}
#[instrument(skip(cnn, key, query))]
pub async fn raw<T>(cnn: &Pool<Postgres>, key: &str, query: String) -> Result<Vec<T>>
where T: FromMap + std::fmt::Debug
{
if let Some(org) = get_org_details(cnn, key).await {
let influx_url = format!("http://{}:8086", org.influx_host);
let client = Client::new(influx_url, &org.influx_org, &org.influx_token);
tracing::info!("{query}");
let query = Query::new(query);
let rows = client.query::<T>(Some(query)).await;
if let Ok(rows) = rows {
Ok(rows)
} else {
tracing::error!("InfluxDb query error: {rows:?}");
Err(Error::msg("Influx query error"))
}
} else {
Err(Error::msg("Organization not found"))
}
}
}

8
src/rust/long_term_stats/lts_node/src/web/wss/queries/influx/mod.rs
View File

@@ -1,8 +0,0 @@
//! InfluxDB query builder and support code.
mod influx_query_builder;
pub use influx_query_builder::*;
mod time_period;
pub use time_period::*;
mod query_builder2;
pub use query_builder2::*;

155
src/rust/long_term_stats/lts_node/src/web/wss/queries/influx/query_builder2.rs
View File

@@ -1,155 +0,0 @@
use super::InfluxTimePeriod;
use influxdb2::{models::Query, Client};
use influxdb2_structmap::FromMap;
use pgdb::{
organization_cache::get_org_details,
sqlx::{Pool, Postgres},
OrganizationDetails,
};
pub struct QueryBuilder<'a> {
lines: Vec<String>,
period: Option<&'a InfluxTimePeriod>,
org: Option<OrganizationDetails>,
}
#[allow(dead_code)]
impl<'a> QueryBuilder<'a> {
/// Construct a new, completely empty query.
pub fn new() -> Self {
Self {
lines: Vec::new(),
period: None,
org: None,
}
}
pub fn with_period(mut self, period: &'a InfluxTimePeriod) -> Self {
self.period = Some(period);
self
}
pub fn with_org(mut self, org: OrganizationDetails) -> Self {
self.org = Some(org);
self
}
pub async fn derive_org(mut self, cnn: &Pool<Postgres>, key: &str) -> QueryBuilder<'a> {
let org = get_org_details(cnn, key).await;
self.org = org;
self
}
pub fn add_line(mut self, line: &str) -> Self {
self.lines.push(line.to_string());
self
}
pub fn add_lines(mut self, lines: &[&str]) -> Self {
for line in lines.iter() {
self.lines.push(line.to_string());
}
self
}
pub fn bucket(mut self) -> Self {
if let Some(org) = &self.org {
self.lines
.push(format!("from(bucket: \"{}\")", org.influx_bucket));
} else {
tracing::warn!("No organization in query, cannot add bucket");
}
self
}
pub fn range(mut self) -> Self {
if let Some(period) = &self.period {
self.lines.push(format!("|> {}", period.range()));
} else {
tracing::warn!("No period in query, cannot add range");
}
self
}
pub fn filter(mut self, filter: &str) -> Self {
if !filter.is_empty() {
self.lines.push(format!("|> filter(fn: (r) => {})", filter));
}
self
}
pub fn filter_and(mut self, filters: &[&str]) -> Self {
let all_filters = filters.join(" and ");
self.lines
.push(format!("|> filter(fn: (r) => {})", all_filters));
self
}
pub fn measure_field_org(mut self, measurement: &str, field: &str) -> Self {
if let Some(org) = &self.org {
self.lines.push(format!("|> filter(fn: (r) => r[\"_field\"] == \"{}\" and r[\"_measurement\"] == \"{}\" and r[\"organization_id\"] == \"{}\")", field, measurement, org.key));
} else {
tracing::warn!("No organization in query, cannot add measure_field_org");
}
self
}
pub fn measure_fields_org(mut self, measurement: &str, fields: &[&str]) -> Self {
if let Some(org) = &self.org {
let mut filters = Vec::new();
for field in fields.iter() {
filters.push(format!("r[\"_field\"] == \"{}\"", field));
}
self.lines.push(format!("|> filter(fn: (r) => r[\"_measurement\"] == \"{}\" and r[\"organization_id\"] == \"{}\" and ({}))", measurement, org.key, filters.join(" or ")));
} else {
tracing::warn!("No organization in query, cannot add measure_field_org");
}
self
}
pub fn aggregate_window(mut self) -> Self {
if let Some(period) = &self.period {
self.lines.push(format!("|> {}", period.aggregate_window()));
} else {
tracing::warn!("No period in query, cannot add aggregate_window");
}
self
}
pub fn group(mut self, columns: &[&str]) -> Self {
let group_by = columns.join(", ");
self.lines
.push(format!("|> group(columns: [\"{}\"])", group_by));
self
}
pub fn with_host_id(mut self, host_id: &str) -> Self {
self.lines.push(format!(
"|> filter(fn: (r) => r[\"host_id\"] == \"{}\")",
host_id
));
self
}
pub async fn execute<T>(&self) -> anyhow::Result<Vec<T>>
where
T: FromMap + std::fmt::Debug,
{
let qs = self.lines.join("\n");
tracing::info!("Query:\n{}", qs);
if let Some(org) = &self.org {
let influx_url = format!("http://{}:8086", org.influx_host);
let client = Client::new(influx_url, &org.influx_org, &org.influx_token);
let query = Query::new(qs.clone());
let rows = client.query::<T>(Some(query)).await;
if let Ok(rows) = rows {
Ok(rows)
} else {
tracing::error!("InfluxDb query error: {rows:?} for: {qs}");
anyhow::bail!("Influx query error");
}
} else {
anyhow::bail!("No organization in query, cannot execute");
}
}
}

122
src/rust/long_term_stats/lts_node/src/web/wss/queries/influx/time_period.rs
View File

@@ -1,122 +0,0 @@
#![allow(dead_code)]
#[derive(Clone, Debug)]
pub struct InfluxTimePeriod {
pub start: String,
pub aggregate: String,
sample: i32,
}
const fn minutes_to_seconds(minutes: i32) -> i32 {
minutes * 60
}
const fn hours_to_seconds(hours: i32) -> i32 {
minutes_to_seconds(hours * 60)
}
const fn days_to_seconds(days: i32) -> i32 {
hours_to_seconds(days * 24)
}
const SAMPLES_PER_GRAPH: i32 = 30;
const fn aggregate_window(seconds: i32) -> i32 {
seconds / SAMPLES_PER_GRAPH
}
fn period_string_to_seconds(period: &str) -> i32 {
let last_char = period.chars().last().unwrap();
let number_part = &period[..period.len() - 1];
let number = number_part.parse::<i32>().unwrap_or(5);
let start_seconds = match last_char {
's' => number,
'm' => minutes_to_seconds(number),
'h' => hours_to_seconds(number),
'd' => days_to_seconds(number),
_ => {
tracing::warn!("Unknown time unit: {last_char}");
minutes_to_seconds(5)
}
};
start_seconds
}
impl InfluxTimePeriod {
pub fn new(period: &str) -> Self {
let last_char = period.chars().last().unwrap();
let number_part = &period[..period.len() - 1];
let number = number_part.parse::<i32>().unwrap_or(5);
let start_seconds = match last_char {
's' => number,
'm' => minutes_to_seconds(number),
'h' => hours_to_seconds(number),
'd' => days_to_seconds(number),
_ => {
tracing::warn!("Unknown time unit: {last_char}");
minutes_to_seconds(5)
}
};
let start = format!("-{}s", start_seconds);
let aggregate_seconds = aggregate_window(start_seconds);
let aggregate = format!("{}s", aggregate_seconds);
let sample = start_seconds / 100;
Self {
start: start.to_string(),
aggregate: aggregate.to_string(),
sample
}
}
pub fn range(&self) -> String {
format!("range(start: {})", self.start)
}
pub fn aggregate_window(&self) -> String {
format!(
"aggregateWindow(every: {}, fn: mean, createEmpty: false)",
self.aggregate
)
}
pub fn aggregate_window_empty(&self) -> String {
format!(
"aggregateWindow(every: {}, fn: mean, createEmpty: true)",
self.aggregate
)
}
pub fn aggregate_window_fn(&self, mode: &str) -> String {
format!(
"aggregateWindow(every: {}, fn: {mode}, createEmpty: false)",
self.aggregate
)
}
pub fn sample(&self) -> String {
format!("sample(n: {}, pos: 1)", self.sample)
}
}
impl From<&String> for InfluxTimePeriod {
fn from(period: &String) -> Self {
Self::new(period)
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_period_to_seconds() {
assert_eq!(period_string_to_seconds("5s"), 5);
assert_eq!(period_string_to_seconds("5m"), 300);
assert_eq!(period_string_to_seconds("5h"), 18000);
assert_eq!(period_string_to_seconds("5d"), 432000);
// Test that an unknown returns the default
assert_eq!(period_string_to_seconds("5x"), 300);
}
}

31
src/rust/long_term_stats/lts_node/src/web/wss/queries/mod.rs
View File

@@ -1,31 +0,0 @@
//! Provides pre-packaged queries for obtaining data, that will
//! then be used by the web server to respond to requests.
mod influx;
pub(crate) use influx::*;
mod circuit_info;
pub mod ext_device;
mod node_perf;
mod packet_counts;
mod rtt;
mod search;
mod site_heat_map;
mod site_info;
mod site_parents;
pub mod site_tree;
mod throughput;
pub use circuit_info::send_circuit_info;
pub use node_perf::send_perf_for_node;
pub use packet_counts::{send_packets_for_all_nodes, send_packets_for_node};
pub use rtt::{
send_rtt_for_all_nodes, send_rtt_for_all_nodes_circuit, send_rtt_for_all_nodes_site,
send_rtt_for_node, send_rtt_histogram_for_all_nodes,
};
pub use search::omnisearch;
pub use site_heat_map::{root_heat_map, site_heat_map};
pub use site_info::send_site_info;
pub use site_parents::{send_circuit_parents, send_root_parents, send_site_parents};
pub use throughput::{
send_site_stack_map, send_throughput_for_all_nodes, send_throughput_for_all_nodes_by_circuit,
send_throughput_for_all_nodes_by_site, send_throughput_for_node,
};

79
src/rust/long_term_stats/lts_node/src/web/wss/queries/node_perf.rs
View File

@@ -1,79 +0,0 @@
use chrono::{DateTime, FixedOffset, Utc};
use influxdb2::FromDataPoint;
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::mpsc::Sender;
use wasm_pipe_types::{Perf, PerfHost, WasmResponse};
use super::{influx::InfluxTimePeriod, QueryBuilder};
#[derive(Debug, FromDataPoint)]
pub struct PerfRow {
pub host_id: String,
pub time: DateTime<FixedOffset>,
pub cpu: f64,
pub cpu_max: f64,
pub ram: f64,
}
impl Default for PerfRow {
fn default() -> Self {
Self {
host_id: "".to_string(),
time: DateTime::<Utc>::MIN_UTC.into(),
cpu: 0.0,
cpu_max: 0.0,
ram: 0.0,
}
}
}
pub async fn send_perf_for_node(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
period: InfluxTimePeriod,
node_id: String,
node_name: String,
) -> anyhow::Result<()> {
let node = get_perf_for_node(cnn, key, node_id, node_name, &period).await?;
tx.send(WasmResponse::NodePerfChart { nodes: vec![node] })
.await?;
Ok(())
}
pub async fn get_perf_for_node(
cnn: &Pool<Postgres>,
key: &str,
node_id: String,
node_name: String,
period: &InfluxTimePeriod,
) -> anyhow::Result<PerfHost> {
let rows = QueryBuilder::new()
.with_period(period)
.derive_org(cnn, key)
.await
.bucket()
.range()
.measure_fields_org("perf", &["cpu", "cpu_max", "ram"])
.filter(&format!("r[\"host_id\"] == \"{}\"", node_id))
.aggregate_window()
.execute::<PerfRow>()
.await?;
let mut stats = Vec::new();
// Fill download
for row in rows.iter() {
stats.push(Perf {
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
cpu: row.cpu,
cpu_max: row.cpu_max,
ram: row.ram,
});
}
Ok(PerfHost {
node_id,
node_name,
stats,
})
}

159
src/rust/long_term_stats/lts_node/src/web/wss/queries/packet_counts/mod.rs
View File

@@ -1,159 +0,0 @@
//! Packet-per-second data queries
mod packet_row;
use self::packet_row::PacketRow;
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use wasm_pipe_types::{PacketHost, Packets, WasmResponse};
use super::{influx::{InfluxTimePeriod, InfluxQueryBuilder}, QueryBuilder};
fn add_by_direction(direction: &str, down: &mut Vec<Packets>, up: &mut Vec<Packets>, row: &PacketRow) {
match direction {
"down" => {
down.push(Packets {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
});
}
"up" => {
up.push(Packets {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
});
}
_ => {}
}
}
#[instrument(skip(cnn, tx, key, period))]
pub async fn send_packets_for_all_nodes(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
period: InfluxTimePeriod,
) -> anyhow::Result<()> {
let node_status = pgdb::node_status(cnn, key).await?;
let mut nodes = Vec::<PacketHost>::new();
InfluxQueryBuilder::new(period.clone())
.with_measurement("packets")
.with_fields(&["min", "max", "avg"])
.with_groups(&["host_id", "min", "max", "avg", "direction", "_field"])
.execute::<PacketRow>(cnn, key)
.await?
.into_iter()
.for_each(|row| {
if let Some(node) = nodes.iter_mut().find(|n| n.node_id == row.host_id) {
add_by_direction(&row.direction, &mut node.down, &mut node.up, &row);
} else {
let mut down = Vec::new();
let mut up = Vec::new();
add_by_direction(&row.direction, &mut down, &mut up, &row);
let node_name = if let Some(node) = node_status.iter().find(|n| n.node_id == row.host_id) {
node.node_name.clone()
} else {
row.host_id.clone()
};
nodes.push(PacketHost {
node_id: row.host_id,
node_name,
down,
up,
});
}
});
tx.send(wasm_pipe_types::WasmResponse::PacketChart { nodes }).await?;
Ok(())
}
#[instrument(skip(cnn, tx, key, period))]
pub async fn send_packets_for_node(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
period: InfluxTimePeriod,
node_id: &str,
node_name: &str,
) -> anyhow::Result<()> {
let node =
get_packets_for_node(cnn, key, node_id.to_string(), node_name.to_string(), period).await?;
tx.send(wasm_pipe_types::WasmResponse::PacketChart { nodes: vec![node] }).await?;
Ok(())
}
/// Requests packet-per-second data for a single shaper node.
///
/// # Arguments
/// * `cnn` - A connection pool to the database
/// * `key` - The organization's license key
/// * `node_id` - The ID of the node to query
/// * `node_name` - The name of the node to query
pub async fn get_packets_for_node(
cnn: &Pool<Postgres>,
key: &str,
node_id: String,
node_name: String,
period: InfluxTimePeriod,
) -> anyhow::Result<PacketHost> {
let rows = QueryBuilder::new()
.with_period(&period)
.derive_org(cnn, key)
.await
.bucket()
.range()
.measure_fields_org("packets", &["min", "max", "avg"])
.with_host_id(&node_id)
.aggregate_window()
.execute::<PacketRow>()
.await;
match rows {
Err(e) => {
tracing::error!("Error querying InfluxDB (packets by node): {}", e);
Err(anyhow::Error::msg("Unable to query influx"))
}
Ok(rows) => {
// Parse and send the data
//println!("{rows:?}");
let mut down = Vec::new();
let mut up = Vec::new();
// Fill download
for row in rows.iter().filter(|r| r.direction == "down") {
down.push(Packets {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
});
}
// Fill upload
for row in rows.iter().filter(|r| r.direction == "up") {
up.push(Packets {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
});
}
Ok(PacketHost {
node_id,
node_name,
down,
up,
})
}
}
}

25
src/rust/long_term_stats/lts_node/src/web/wss/queries/packet_counts/packet_row.rs
View File

@@ -1,25 +0,0 @@
use chrono::{DateTime, FixedOffset, Utc};
use influxdb2::FromDataPoint;
#[derive(Debug, FromDataPoint)]
pub struct PacketRow {
pub direction: String,
pub host_id: String,
pub min: f64,
pub max: f64,
pub avg: f64,
pub time: DateTime<FixedOffset>,
}
impl Default for PacketRow {
fn default() -> Self {
Self {
direction: "".to_string(),
host_id: "".to_string(),
min: 0.0,
max: 0.0,
avg: 0.0,
time: DateTime::<Utc>::MIN_UTC.into(),
}
}
}

152
src/rust/long_term_stats/lts_node/src/web/wss/queries/rtt/mod.rs
View File

@@ -1,152 +0,0 @@
mod per_node;
pub use per_node::*;
mod per_site;
pub use per_site::*;
use self::rtt_row::RttRow;
use super::{influx::InfluxTimePeriod, QueryBuilder};
use crate::web::wss::queries::rtt::rtt_row::RttCircuitRow;
use futures::future::join_all;
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use wasm_pipe_types::{Rtt, RttHost, WasmResponse};
mod rtt_row;
#[instrument(skip(cnn, tx, key, site_id, period))]
pub async fn send_rtt_for_all_nodes_circuit(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
site_id: String,
period: InfluxTimePeriod,
) -> anyhow::Result<()> {
let nodes = get_rtt_for_all_nodes_circuit(cnn, key, &site_id, period).await?;
let mut histogram = vec![0; 20];
for node in nodes.iter() {
for rtt in node.rtt.iter() {
let bucket = usize::min(19, (rtt.value / 200.0) as usize);
histogram[bucket] += 1;
}
}
tx.send(WasmResponse::RttChartCircuit { nodes, histogram })
.await?;
Ok(())
}
pub async fn send_rtt_for_node(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
period: InfluxTimePeriod,
node_id: String,
node_name: String,
) -> anyhow::Result<()> {
let node = get_rtt_for_node(cnn, key, node_id, node_name, &period).await?;
let nodes = vec![node];
tx.send(WasmResponse::RttChart { nodes }).await?;
Ok(())
}
pub async fn get_rtt_for_all_nodes_circuit(
cnn: &Pool<Postgres>,
key: &str,
circuit_id: &str,
period: InfluxTimePeriod,
) -> anyhow::Result<Vec<RttHost>> {
let node_status = pgdb::node_status(cnn, key).await?;
let mut futures = Vec::new();
for node in node_status {
futures.push(get_rtt_for_node_circuit(
cnn,
key,
node.node_id.to_string(),
node.node_name.to_string(),
circuit_id.to_string(),
&period,
));
}
let all_nodes: anyhow::Result<Vec<RttHost>> = join_all(futures).await.into_iter().collect();
all_nodes
}
pub async fn get_rtt_for_node(
cnn: &Pool<Postgres>,
key: &str,
node_id: String,
node_name: String,
period: &InfluxTimePeriod,
) -> anyhow::Result<RttHost> {
let rows = QueryBuilder::new()
.with_period(period)
.derive_org(cnn, key)
.await
.bucket()
.range()
.measure_fields_org("rtt", &["avg", "min", "max"])
.filter(&format!("r[\"host_id\"] == \"{}\"", node_id))
.aggregate_window()
.execute::<RttRow>()
.await?;
let mut rtt = Vec::new();
// Fill RTT
for row in rows.iter() {
rtt.push(Rtt {
value: f64::min(200.0, row.avg),
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: f64::min(200.0, row.min),
u: f64::min(200.0, row.max) - f64::min(200.0, row.min),
});
}
Ok(RttHost {
node_id,
node_name,
rtt,
})
}
pub async fn get_rtt_for_node_circuit(
cnn: &Pool<Postgres>,
key: &str,
node_id: String,
node_name: String,
circuit_id: String,
period: &InfluxTimePeriod,
) -> anyhow::Result<RttHost> {
let rows = QueryBuilder::new()
.with_period(period)
.derive_org(cnn, key)
.await
.bucket()
.range()
.measure_fields_org("rtt", &["avg", "min", "max"])
.filter(&format!("r[\"host_id\"] == \"{}\"", node_id))
.filter(&format!("r[\"circuit_id\"] == \"{}\"", circuit_id))
.aggregate_window()
.execute::<RttCircuitRow>()
.await?;
let mut rtt = Vec::new();
// Fill download
for row in rows.iter() {
rtt.push(Rtt {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
});
}
Ok(RttHost {
node_id,
node_name,
rtt,
})
}

92
src/rust/long_term_stats/lts_node/src/web/wss/queries/rtt/per_node.rs
View File

@@ -1,92 +0,0 @@
use pgdb::{
sqlx::{Pool, Postgres},
NodeStatus
};
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use wasm_pipe_types::{Rtt, RttHost, WasmResponse};
use crate::web::wss::queries::influx::{InfluxTimePeriod, InfluxQueryBuilder};
use super::rtt_row::{RttRow, RttHistoRow};
#[instrument(skip(cnn, tx, key, period))]
pub async fn send_rtt_for_all_nodes(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
period: InfluxTimePeriod,
) -> anyhow::Result<()> {
let rows = InfluxQueryBuilder::new(period.clone())
.with_measurement("rtt")
.with_fields(&["avg", "min", "max"])
.with_groups(&["host_id", "_field"])
.execute::<RttRow>(cnn, key)
.await?;
let node_status = pgdb::node_status(cnn, key).await?;
let nodes = rtt_rows_to_result(rows, node_status);
tx.send(WasmResponse::RttChart { nodes }).await?;
Ok(())
}
#[instrument(skip(cnn, tx, key, period))]
pub async fn send_rtt_histogram_for_all_nodes(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
period: InfluxTimePeriod,
) -> anyhow::Result<()> {
let rows = InfluxQueryBuilder::new(period.clone())
.with_measurement("rtt")
.with_field("avg")
.sample_no_window()
.execute::<RttHistoRow>(cnn, key)
.await?;
let mut histo = vec![0u32; 20];
rows.iter().for_each(|row| {
let rtt = f64::min(row.avg, 200.);
let bucket = usize::min((rtt / 10.0) as usize, 19);
histo[bucket] += 1;
});
tx.send(WasmResponse::RttHistogram { histogram: histo }).await?;
Ok(())
}
pub(crate) fn rtt_rows_to_result(rows: Vec<RttRow>, node_status: Vec<NodeStatus>) -> Vec<RttHost> {
let mut result = Vec::<RttHost>::new();
for row in rows.into_iter() {
if let Some(host) = result.iter_mut().find(|h| h.node_id == row.host_id) {
// We found one - add to it
host.rtt.push(Rtt {
value: f64::min(200.0, row.avg),
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: f64::min(200.0, row.min),
u: f64::min(200.0, row.max) - f64::min(200.0, row.min),
});
} else {
let rtt = vec![Rtt {
value: f64::min(200.0, row.avg),
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: f64::min(200.0, row.min),
u: f64::min(200.0, row.max) - f64::min(200.0, row.min),
}];
let node_name = node_status
.iter()
.filter(|n| n.node_id == row.host_id)
.map(|n| n.node_name.clone())
.next()
.unwrap_or("".to_string());
let new_host = RttHost {
node_id: row.host_id,
node_name,
rtt,
};
result.push(new_host);
}
}
result
}

64
src/rust/long_term_stats/lts_node/src/web/wss/queries/rtt/per_site.rs
View File

@@ -1,64 +0,0 @@
use pgdb::{
sqlx::{Pool, Postgres},
NodeStatus
};
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use wasm_pipe_types::{Rtt, RttHost, WasmResponse};
use crate::web::wss::queries::influx::{InfluxTimePeriod, InfluxQueryBuilder};
use super::rtt_row::RttSiteRow;
#[instrument(skip(cnn, tx, key, period))]
pub async fn send_rtt_for_all_nodes_site(
cnn: &Pool<Postgres>, tx: Sender<WasmResponse>, key: &str, site_name: String, period: InfluxTimePeriod
) -> anyhow::Result<()> {
let rows = InfluxQueryBuilder::new(period.clone())
.with_measurement("tree")
.with_fields(&["rtt_avg", "rtt_min", "rtt_max"])
.with_filter(format!("r[\"node_name\"] == \"{}\"", site_name))
.with_groups(&["host_id", "_field"])
.execute::<RttSiteRow>(cnn, key)
.await?;
let node_status = pgdb::node_status(cnn, key).await?;
let nodes = rtt_rows_to_result(rows, node_status);
tx.send(WasmResponse::RttChartSite { nodes }).await?;
Ok(())
}
fn rtt_rows_to_result(rows: Vec<RttSiteRow>, node_status: Vec<NodeStatus>) -> Vec<RttHost> {
let mut result = Vec::<RttHost>::new();
for row in rows.into_iter() {
if let Some(host) = result.iter_mut().find(|h| h.node_id == row.host_id) {
// We found one - add to it
host.rtt.push(Rtt {
value: f64::min(200.0, row.rtt_avg),
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: f64::min(200.0, row.rtt_min),
u: f64::min(200.0, row.rtt_max) - f64::min(200.0, row.rtt_min),
});
} else {
let rtt = vec![Rtt {
value: f64::min(200.0, row.rtt_avg),
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: f64::min(200.0, row.rtt_min),
u: f64::min(200.0, row.rtt_max) - f64::min(200.0, row.rtt_min),
}];
let node_name = node_status
.iter()
.filter(|n| n.node_id == row.host_id)
.map(|n| n.node_name.clone())
.next()
.unwrap_or("".to_string());
let new_host = RttHost {
node_id: row.host_id,
node_name,
rtt,
};
result.push(new_host);
}
}
result
}

87
src/rust/long_term_stats/lts_node/src/web/wss/queries/rtt/rtt_row.rs
View File

@@ -1,87 +0,0 @@
use chrono::{DateTime, FixedOffset, Utc};
use influxdb2::FromDataPoint;
#[derive(Debug, FromDataPoint)]
pub struct RttRow {
pub host_id: String,
pub min: f64,
pub max: f64,
pub avg: f64,
pub time: DateTime<FixedOffset>,
}
impl Default for RttRow {
fn default() -> Self {
Self {
host_id: "".to_string(),
min: 0.0,
max: 0.0,
avg: 0.0,
time: DateTime::<Utc>::MIN_UTC.into(),
}
}
}
#[derive(Debug, FromDataPoint)]
pub struct RttValue {
pub host_id: String,
pub avg: f64,
pub time: DateTime<FixedOffset>,
}
impl Default for RttValue {
fn default() -> Self {
Self {
host_id: "".to_string(),
avg: 0.0,
time: DateTime::<Utc>::MIN_UTC.into(),
}
}
}
#[derive(Debug, FromDataPoint, Default)]
pub struct RttHistoRow {
pub avg: f64,
}
#[derive(Debug, FromDataPoint)]
pub struct RttSiteRow {
pub host_id: String,
pub rtt_min: f64,
pub rtt_max: f64,
pub rtt_avg: f64,
pub time: DateTime<FixedOffset>,
}
impl Default for RttSiteRow {
fn default() -> Self {
Self {
host_id: "".to_string(),
rtt_min: 0.0,
rtt_max: 0.0,
rtt_avg: 0.0,
time: DateTime::<Utc>::MIN_UTC.into(),
}
}
}
#[derive(Debug, FromDataPoint)]
pub struct RttCircuitRow {
pub host_id: String,
pub min: f64,
pub max: f64,
pub avg: f64,
pub time: DateTime<FixedOffset>,
}
impl Default for RttCircuitRow {
fn default() -> Self {
Self {
host_id: "".to_string(),
min: 0.0,
max: 0.0,
avg: 0.0,
time: DateTime::<Utc>::MIN_UTC.into(),
}
}
}

87
src/rust/long_term_stats/lts_node/src/web/wss/queries/search.rs
View File

@@ -1,87 +0,0 @@
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::mpsc::Sender;
use wasm_pipe_types::{SearchResult, WasmResponse};
#[tracing::instrument(skip(cnn, tx, key, term))]
pub async fn omnisearch(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
term: &str,
) -> anyhow::Result<()> {
tracing::warn!("Searching for {term}");
let hits = search_devices(cnn, key, term).await;
if let Err(e) = &hits {
tracing::error!("{e:?}");
}
let mut hits = hits.unwrap();
hits.extend(search_ips(cnn, key, term).await?);
hits.extend(search_sites(cnn, key, term).await?);
hits.sort_by(|a,b| a.name.cmp(&b.name));
hits.dedup_by(|a,b| a.name == b.name && a.url == b.url);
hits.sort_by(|a,b| a.score.partial_cmp(&b.score).unwrap());
tx.send(WasmResponse::SearchResult { hits }).await?;
Ok(())
}
async fn search_devices(
cnn: &Pool<Postgres>,
key: &str,
term: &str,
) -> anyhow::Result<Vec<SearchResult>> {
let hits = pgdb::search_devices(cnn, key, term).await?;
Ok(hits
.iter()
.map(|hit| SearchResult {
name: hit.circuit_name.to_string(),
url: format!("circuit:{}", hit.circuit_id),
score: hit.score,
icon: "circuit".to_string(),
})
.collect())
}
async fn search_ips(
cnn: &Pool<Postgres>,
key: &str,
term: &str,
) -> anyhow::Result<Vec<SearchResult>> {
let hits = pgdb::search_ip(cnn, key, term).await?;
Ok(hits
.iter()
.map(|hit| SearchResult {
name: hit.circuit_name.to_string(),
url: format!("circuit:{}", hit.circuit_id),
score: hit.score,
icon: "circuit".to_string(),
})
.collect())
}
async fn search_sites(
cnn: &Pool<Postgres>,
key: &str,
term: &str,
) -> anyhow::Result<Vec<SearchResult>> {
let hits = pgdb::search_sites(cnn, key, term).await?;
Ok(hits
.iter()
.map(|hit| {
let t = if hit.site_type.is_empty() {
"site".to_string()
} else {
hit.site_type.to_string()
};
SearchResult {
name: hit.site_name.to_string(),
url: format!("{t}:{}", hit.site_name),
score: hit.score,
icon: t,
}})
.collect())
}

197
src/rust/long_term_stats/lts_node/src/web/wss/queries/site_heat_map.rs
View File

@@ -1,197 +0,0 @@
use chrono::{DateTime, FixedOffset, Utc};
use influxdb2::FromDataPoint;
use pgdb::sqlx::{Pool, Postgres};
use serde::Serialize;
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use std::collections::HashMap;
use wasm_pipe_types::WasmResponse;
use itertools::Itertools;
use super::influx::{InfluxTimePeriod, InfluxQueryBuilder};
fn headings_sorter<T: HeatMapData>(rows: Vec<T>) -> HashMap<String, Vec<(DateTime<FixedOffset>, f64)>> {
let mut headings = rows.iter().map(|r| r.time()).collect::<Vec<_>>();
headings.sort();
let headings: Vec<DateTime<FixedOffset>> = headings.iter().dedup().cloned().collect();
//println!("{headings:#?}");
let defaults = headings.iter().map(|h| (*h, 0.0)).collect::<Vec<_>>();
let mut sorter: HashMap<String, Vec<(DateTime<FixedOffset>, f64)>> = HashMap::new();
for row in rows.into_iter() {
let entry = sorter.entry(row.name()).or_insert(defaults.clone());
if let Some(idx) = headings.iter().position(|h| h == &row.time()) {
entry[idx] = (row.time(), row.avg());
}
}
sorter
}
#[instrument(skip(cnn,tx,key,period))]
pub async fn root_heat_map(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
period: InfluxTimePeriod,
) -> anyhow::Result<()> {
let rows: Vec<HeatRow> = InfluxQueryBuilder::new(period.clone())
.with_import("strings")
.with_measurement("tree")
.with_fields(&["rtt_avg"])
.sample_after_org()
.with_filter("exists(r[\"node_parents\"])")
.with_filter("strings.hasSuffix(suffix: \"S0S\" + r[\"node_index\"] + \"S\", v: r[\"node_parents\"])")
.with_filter("r[\"_value\"] > 0.0")
.with_groups(&["_field", "node_name"])
.execute(cnn, key)
.await?;
let sorter = headings_sorter(rows);
tx.send(WasmResponse::RootHeat { data: sorter }).await?;
Ok(())
}
#[instrument(skip(cnn, key, site_name, period))]
async fn site_circuits_heat_map(
cnn: &Pool<Postgres>,
key: &str,
site_name: &str,
period: InfluxTimePeriod,
) -> anyhow::Result<Vec<HeatCircuitRow>> {
// List all hosts with this site as exact parent
let hosts = pgdb::get_circuit_list_for_site(cnn, key, site_name).await?;
let host_filter = pgdb::circuit_list_to_influx_filter(&hosts);
let rows: Vec<HeatCircuitRow> = InfluxQueryBuilder::new(period.clone())
.with_measurement("rtt")
.with_fields(&["avg"])
.sample_after_org()
.with_filter(host_filter)
.with_filter("r[\"_value\"] > 0.0")
.with_groups(&["_field", "circuit_id"])
.execute(cnn, key)
.await?
.into_iter()
.map(|row: HeatCircuitRow| HeatCircuitRow {
circuit_id: hosts.iter().find(|h| h.0 == row.circuit_id).unwrap().1.clone(),
..row
})
.collect();
Ok(rows)
}
#[instrument(skip(cnn, tx, key, period))]
pub async fn site_heat_map(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
site_name: &str,
period: InfluxTimePeriod,
) -> anyhow::Result<()> {
let (site_id, circuits) = tokio::join!(
pgdb::get_site_id_from_name(cnn, key, site_name),
site_circuits_heat_map(cnn, key, site_name, period.clone()),
);
let (site_id, circuits) = (site_id?, circuits?);
let mut rows: Vec<HeatRow> = InfluxQueryBuilder::new(period.clone())
.with_import("strings")
.with_measurement("tree")
.with_fields(&["rtt_avg"])
.sample_after_org()
.with_filter("exists(r[\"node_parents\"])")
.with_filter(format!("strings.containsStr(substr: \"S{site_id}S\" + r[\"node_index\"] + \"S\", v: r[\"node_parents\"])"))
.with_filter("r[\"_value\"] > 0.0")
.with_groups(&["_field", "node_name"])
.execute(cnn, key)
.await?;
circuits.iter().for_each(|c| {
rows.push(HeatRow {
node_name: c.circuit_id.clone(),
rtt_avg: c.avg,
time: c.time,
})
});
let sorter = headings_sorter(rows);
tx.send(WasmResponse::SiteHeat { data: sorter }).await?;
Ok(())
}
trait HeatMapData {
fn avg(&self) -> f64;
fn time(&self) -> DateTime<FixedOffset>;
fn name(&self) -> String;
}
#[derive(Serialize)]
struct HeatMessage {
msg: String,
data: HashMap<String, Vec<(DateTime<FixedOffset>, f64)>>,
}
#[derive(Debug, FromDataPoint)]
pub struct HeatRow {
pub node_name: String,
pub rtt_avg: f64,
pub time: DateTime<FixedOffset>,
}
impl Default for HeatRow {
fn default() -> Self {
Self {
node_name: "".to_string(),
rtt_avg: 0.0,
time: DateTime::<Utc>::MIN_UTC.into(),
}
}
}
impl HeatMapData for HeatRow {
fn avg(&self) -> f64 {
self.rtt_avg
}
fn time(&self) -> DateTime<FixedOffset> {
self.time
}
fn name(&self) -> String {
self.node_name.clone()
}
}
#[derive(Debug, FromDataPoint)]
pub struct HeatCircuitRow {
pub circuit_id: String,
pub avg: f64,
pub time: DateTime<FixedOffset>,
}
impl Default for HeatCircuitRow {
fn default() -> Self {
Self {
circuit_id: "".to_string(),
avg: 0.0,
time: DateTime::<Utc>::MIN_UTC.into(),
}
}
}
impl HeatMapData for HeatCircuitRow {
fn avg(&self) -> f64 {
self.avg
}
fn time(&self) -> DateTime<FixedOffset> {
self.time
}
fn name(&self) -> String {
self.circuit_id.clone()
}
}

39
src/rust/long_term_stats/lts_node/src/web/wss/queries/site_info.rs
View File

@@ -1,39 +0,0 @@
use super::site_tree::tree_to_host;
use pgdb::sqlx::{Pool, Postgres};
use serde::Serialize;
use tokio::sync::mpsc::Sender;
use wasm_pipe_types::{SiteTree, WasmResponse, SiteOversubscription};
#[derive(Serialize)]
struct SiteInfoMessage {
msg: String,
data: SiteTree,
}
pub async fn send_site_info(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
site_id: &str,
) {
let (host, oversub) = tokio::join!(
pgdb::get_site_info(cnn, key, site_id),
pgdb::get_oversubscription(cnn, key, site_id)
);
if let Ok(host) = host {
if let Ok(oversub) = oversub {
let host = tree_to_host(host);
let oversubscription = SiteOversubscription {
dlmax: oversub.dlmax,
dlmin: oversub.dlmin,
devicecount: oversub.devicecount,
};
tx.send(WasmResponse::SiteInfo { data: host, oversubscription }).await.unwrap();
} else {
tracing::error!("{oversub:?}");
}
} else {
tracing::error!("{host:?}");
}
}

47
src/rust/long_term_stats/lts_node/src/web/wss/queries/site_parents.rs
View File

@@ -1,47 +0,0 @@
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::mpsc::Sender;
use wasm_pipe_types::WasmResponse;
#[tracing::instrument(skip(cnn, tx, key, site_name))]
pub async fn send_site_parents(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
site_name: &str,
) {
if let Ok(parents) = pgdb::get_parent_list(cnn, key, site_name).await {
tx.send(WasmResponse::SiteParents { data: parents }).await.unwrap();
}
let child_result = pgdb::get_child_list(cnn, key, site_name).await;
if let Ok(children) = child_result {
tx.send(WasmResponse::SiteChildren { data: children }).await.unwrap();
} else {
tracing::error!("Error getting children: {:?}", child_result);
}
}
pub async fn send_circuit_parents(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
circuit_id: &str,
) {
if let Ok(parents) = pgdb::get_circuit_parent_list(cnn, key, circuit_id).await {
tx.send(WasmResponse::SiteParents { data: parents }).await.unwrap();
}
}
pub async fn send_root_parents(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
) {
let site_name = "Root";
let child_result = pgdb::get_child_list(cnn, key, site_name).await;
if let Ok(children) = child_result {
tx.send(WasmResponse::SiteChildren { data: children }).await.unwrap();
} else {
tracing::error!("Error getting children: {:?}", child_result);
}
}

31
src/rust/long_term_stats/lts_node/src/web/wss/queries/site_tree.rs
View File

@@ -1,31 +0,0 @@
use pgdb::{
sqlx::{Pool, Postgres},
TreeNode,
};
use tokio::sync::mpsc::Sender;
use wasm_pipe_types::{SiteTree, WasmResponse};
#[tracing::instrument(skip(cnn, tx, key, parent))]
pub async fn send_site_tree(cnn: &Pool<Postgres>, tx: Sender<WasmResponse>, key: &str, parent: &str) {
let tree = pgdb::get_site_tree(cnn, key, parent).await.unwrap();
let tree = tree
.into_iter()
.map(tree_to_host)
.collect::<Vec<SiteTree>>();
tx.send(WasmResponse::SiteTree { data: tree }).await.unwrap();
}
pub(crate) fn tree_to_host(row: TreeNode) -> SiteTree {
SiteTree {
index: row.index,
site_name: row.site_name,
site_type: row.site_type,
parent: row.parent,
max_down: row.max_down,
max_up: row.max_up,
current_down: row.current_down * 8,
current_up: row.current_up * 8,
current_rtt: row.current_rtt,
}
}

334
src/rust/long_term_stats/lts_node/src/web/wss/queries/throughput/mod.rs
View File

@@ -1,334 +0,0 @@
use std::collections::HashMap;
mod site_stack;
use self::throughput_row::{ThroughputRow, ThroughputRowByCircuit, ThroughputRowBySite};
use futures::future::join_all;
use pgdb::sqlx::{Pool, Postgres};
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use wasm_pipe_types::{Throughput, ThroughputHost, WasmResponse};
mod throughput_row;
use super::{
influx::{InfluxQueryBuilder, InfluxTimePeriod},
QueryBuilder,
};
pub use site_stack::send_site_stack_map;
fn add_by_direction(
direction: &str,
down: &mut Vec<Throughput>,
up: &mut Vec<Throughput>,
row: &ThroughputRow,
) {
match direction {
"down" => {
down.push(Throughput {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
});
}
"up" => {
up.push(Throughput {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
});
}
_ => {}
}
}
fn add_by_direction_site(
direction: &str,
down: &mut Vec<Throughput>,
up: &mut Vec<Throughput>,
row: &ThroughputRowBySite,
) {
match direction {
"down" => {
down.push(Throughput {
value: row.bits_avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.bits_min,
u: row.bits_max - row.bits_min,
});
}
"up" => {
up.push(Throughput {
value: row.bits_avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.bits_min,
u: row.bits_max - row.bits_min,
});
}
_ => {}
}
}
#[instrument(skip(cnn, tx, key, period))]
pub async fn send_throughput_for_all_nodes(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
period: InfluxTimePeriod,
) -> anyhow::Result<()> {
let node_status = pgdb::node_status(cnn, key).await?;
let mut nodes = Vec::<ThroughputHost>::new();
InfluxQueryBuilder::new(period.clone())
.with_measurement("bits")
.with_fields(&["min", "max", "avg"])
.with_groups(&["host_id", "direction", "_field"])
.execute::<ThroughputRow>(cnn, key)
.await?
.into_iter()
.for_each(|row| {
if let Some(node) = nodes.iter_mut().find(|n| n.node_id == row.host_id) {
add_by_direction(&row.direction, &mut node.down, &mut node.up, &row);
} else {
let mut down = Vec::new();
let mut up = Vec::new();
add_by_direction(&row.direction, &mut down, &mut up, &row);
let node_name =
if let Some(node) = node_status.iter().find(|n| n.node_id == row.host_id) {
node.node_name.clone()
} else {
row.host_id.clone()
};
nodes.push(ThroughputHost {
node_id: row.host_id,
node_name,
down,
up,
});
}
});
tx.send(WasmResponse::BitsChart { nodes }).await?;
Ok(())
}
#[instrument(skip(cnn, tx, key, period, site_name))]
pub async fn send_throughput_for_all_nodes_by_site(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
site_name: String,
period: InfluxTimePeriod,
) -> anyhow::Result<()> {
let node_status = pgdb::node_status(cnn, key).await?;
let mut nodes = Vec::<ThroughputHost>::new();
InfluxQueryBuilder::new(period.clone())
.with_measurement("tree")
.with_fields(&["bits_min", "bits_max", "bits_avg"])
.with_filter(format!("r[\"node_name\"] == \"{}\"", site_name))
.with_groups(&["host_id", "direction", "_field"])
.execute::<ThroughputRowBySite>(cnn, key)
.await?
.into_iter()
.for_each(|row| {
if let Some(node) = nodes.iter_mut().find(|n| n.node_id == row.host_id) {
add_by_direction_site(&row.direction, &mut node.down, &mut node.up, &row);
} else {
let mut down = Vec::new();
let mut up = Vec::new();
add_by_direction_site(&row.direction, &mut down, &mut up, &row);
let node_name =
if let Some(node) = node_status.iter().find(|n| n.node_id == row.host_id) {
node.node_name.clone()
} else {
row.host_id.clone()
};
nodes.push(ThroughputHost {
node_id: row.host_id,
node_name,
down,
up,
});
}
});
tx.send(WasmResponse::BitsChart { nodes }).await?;
Ok(())
}
/*#[instrument(skip(cnn, socket, key, period, site_name))]
pub async fn send_throughput_for_all_nodes_by_site(cnn: &Pool<Postgres>, socket: &mut WebSocket, key: &str, site_name: String, period: InfluxTimePeriod) -> anyhow::Result<()> {
let nodes = get_throughput_for_all_nodes_by_site(cnn, key, period, &site_name).await?;
send_response(socket, wasm_pipe_types::WasmResponse::BitsChart { nodes }).await;
Ok(())
}*/
pub async fn send_throughput_for_all_nodes_by_circuit(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
circuit_id: String,
period: InfluxTimePeriod,
) -> anyhow::Result<()> {
let nodes = get_throughput_for_all_nodes_by_circuit(cnn, key, period, &circuit_id).await?;
tx.send(WasmResponse::BitsChart { nodes }).await?;
Ok(())
}
pub async fn send_throughput_for_node(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
period: InfluxTimePeriod,
node_id: String,
node_name: String,
) -> anyhow::Result<()> {
let node = get_throughput_for_node(cnn, key, node_id, node_name, &period).await?;
tx.send(WasmResponse::BitsChart { nodes: vec![node] })
.await?;
Ok(())
}
pub async fn get_throughput_for_all_nodes_by_circuit(
cnn: &Pool<Postgres>,
key: &str,
period: InfluxTimePeriod,
circuit_id: &str,
) -> anyhow::Result<Vec<ThroughputHost>> {
let node_status = pgdb::node_status(cnn, key).await?;
let mut futures = Vec::new();
for node in node_status {
futures.push(get_throughput_for_node_by_circuit(
cnn,
key,
node.node_id.to_string(),
node.node_name.to_string(),
circuit_id.to_string(),
&period,
));
}
let mut all_nodes = Vec::new();
for node in (join_all(futures).await).into_iter().flatten() {
all_nodes.extend(node);
}
Ok(all_nodes)
}
pub async fn get_throughput_for_node(
cnn: &Pool<Postgres>,
key: &str,
node_id: String,
node_name: String,
period: &InfluxTimePeriod,
) -> anyhow::Result<ThroughputHost> {
let rows = QueryBuilder::new()
.with_period(period)
.derive_org(cnn, key)
.await
.bucket()
.range()
.measure_fields_org("bits", &["avg", "min", "max"])
.aggregate_window()
.execute::<ThroughputRow>()
.await?;
let mut down = Vec::new();
let mut up = Vec::new();
// Fill download
for row in rows.iter().filter(|r| r.direction == "down") {
down.push(Throughput {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
});
}
// Fill upload
for row in rows.iter().filter(|r| r.direction == "up") {
up.push(Throughput {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
});
}
Ok(ThroughputHost {
node_id,
node_name,
down,
up,
})
}
pub async fn get_throughput_for_node_by_circuit(
cnn: &Pool<Postgres>,
key: &str,
node_id: String,
node_name: String,
circuit_id: String,
period: &InfluxTimePeriod,
) -> anyhow::Result<Vec<ThroughputHost>> {
let rows = QueryBuilder::new()
.with_period(period)
.derive_org(cnn, key)
.await
.bucket()
.range()
.measure_fields_org("host_bits", &["avg", "min", "max"])
.with_host_id(&node_id)
.filter(&format!("r[\"circuit_id\"] == \"{}\"", circuit_id))
.aggregate_window()
.execute::<ThroughputRowByCircuit>()
.await?;
let mut sorter: HashMap<String, (Vec<Throughput>, Vec<Throughput>)> =
HashMap::new();
// Fill download
for row in rows.iter().filter(|r| r.direction == "down") {
let tp = Throughput {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
};
if let Some(hat) = sorter.get_mut(&row.ip) {
hat.0.push(tp);
} else {
sorter.insert(row.ip.clone(), (vec![tp], Vec::new()));
}
}
// Fill upload
for row in rows.iter().filter(|r| r.direction == "up") {
let tp = Throughput {
value: row.avg,
date: row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
l: row.min,
u: row.max - row.min,
};
if let Some(hat) = sorter.get_mut(&row.ip) {
hat.1.push(tp);
} else {
sorter.insert(row.ip.clone(), (Vec::new(), vec![tp]));
}
}
let mut result = Vec::new();
for (ip, (down, up)) in sorter.iter() {
result.push(ThroughputHost {
node_id: node_id.clone(),
node_name: format!("{ip} {node_name}"),
down: down.clone(),
up: up.clone(),
});
}
Ok(result)
}

221
src/rust/long_term_stats/lts_node/src/web/wss/queries/throughput/site_stack.rs
View File

@@ -1,221 +0,0 @@
use crate::web::wss::queries::{influx::InfluxTimePeriod, QueryBuilder};
use pgdb::{
organization_cache::get_org_details,
sqlx::{Pool, Postgres},
OrganizationDetails,
};
use tokio::sync::mpsc::Sender;
use tracing::{error, instrument};
use wasm_pipe_types::{SiteStackHost, WasmResponse};
#[derive(Debug, influxdb2::FromDataPoint)]
pub struct SiteStackRow {
pub node_name: String,
pub node_parents: String,
pub bits_max: f64,
pub time: chrono::DateTime<chrono::FixedOffset>,
pub direction: String,
}
impl Default for SiteStackRow {
fn default() -> Self {
Self {
node_name: "".to_string(),
node_parents: "".to_string(),
bits_max: 0.0,
time: chrono::DateTime::<chrono::Utc>::MIN_UTC.into(),
direction: "".to_string(),
}
}
}
#[derive(Debug, influxdb2::FromDataPoint)]
pub struct CircuitStackRow {
pub circuit_id: String,
pub max: f64,
pub time: chrono::DateTime<chrono::FixedOffset>,
pub direction: String,
}
impl Default for CircuitStackRow {
fn default() -> Self {
Self {
circuit_id: "".to_string(),
max: 0.0,
time: chrono::DateTime::<chrono::Utc>::MIN_UTC.into(),
direction: "".to_string(),
}
}
}
#[instrument(skip(cnn, tx, key, period))]
pub async fn send_site_stack_map(
cnn: &Pool<Postgres>,
tx: Sender<WasmResponse>,
key: &str,
period: InfluxTimePeriod,
site_id: String,
) -> anyhow::Result<()> {
let site_index = pgdb::get_site_id_from_name(cnn, key, &site_id).await?;
if let Some(org) = get_org_details(cnn, key).await {
// Determine child hosts
let hosts = pgdb::get_circuit_list_for_site(cnn, key, &site_id).await?;
let host_filter = pgdb::circuit_list_to_influx_filter(&hosts);
let (circuits, rows) = tokio::join!(
query_circuits_influx(&org, &period, &hosts, &host_filter),
query_site_stack_influx(&org, &period, site_index)
);
match rows {
Err(e) => error!("Influxdb tree query error: {e}"),
Ok(mut rows) => {
if let Ok(circuits) = circuits {
rows.extend(circuits);
}
let mut result = site_rows_to_hosts(rows);
reduce_to_x_entries(&mut result);
// Send the reply
tx.send(WasmResponse::SiteStack { nodes: result }).await?;
}
}
}
Ok(())
}
#[instrument(skip(org, period, hosts, host_filter))]
async fn query_circuits_influx(
org: &OrganizationDetails,
period: &InfluxTimePeriod,
hosts: &[(String, String)],
host_filter: &str,
) -> anyhow::Result<Vec<SiteStackRow>> {
if host_filter.is_empty() {
return Ok(Vec::new());
}
let rows = QueryBuilder::new()
.with_period(period)
.with_org(org.clone())
.bucket()
.range()
.measure_field_org("host_bits", "max")
.aggregate_window()
.filter(host_filter)
.group(&["circuit_id", "_field", "direction"])
.execute::<CircuitStackRow>()
.await?
.into_iter()
.map(|row| SiteStackRow {
node_name: hosts
.iter()
.find(|h| h.0 == row.circuit_id)
.unwrap()
.1
.clone(),
node_parents: "".to_string(),
bits_max: row.max / 8.0,
time: row.time,
direction: row.direction,
})
.collect();
Ok(rows)
}
#[instrument(skip(org, period, site_index))]
async fn query_site_stack_influx(
org: &OrganizationDetails,
period: &InfluxTimePeriod,
site_index: i32,
) -> anyhow::Result<Vec<SiteStackRow>> {
Ok(QueryBuilder::new()
.add_line("import \"strings\"")
.with_period(period)
.with_org(org.clone())
.bucket()
.range()
.measure_field_org("tree", "bits_max")
.filter_and(&["exists r[\"node_parents\"]", "exists r[\"node_index\"]"])
.aggregate_window()
.filter(&format!("strings.hasSuffix(v: r[\"node_parents\"], suffix: \"S{}S\" + r[\"node_index\"] + \"S\")", site_index))
.group(&["node_name", "node_parents", "_field", "node_index", "direction"])
.execute::<SiteStackRow>()
.await?
)
}
fn site_rows_to_hosts(rows: Vec<SiteStackRow>) -> Vec<SiteStackHost> {
let mut result: Vec<SiteStackHost> = Vec::new();
for row in rows.iter() {
if let Some(r) = result.iter_mut().find(|r| r.node_name == row.node_name) {
if row.direction == "down" {
r.download.push((
row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
row.bits_max as i64,
));
} else {
r.upload.push((
row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
row.bits_max as i64,
));
}
} else if row.direction == "down" {
result.push(SiteStackHost {
node_name: row.node_name.clone(),
download: vec![(
row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
row.bits_max as i64,
)],
upload: vec![],
});
} else {
result.push(SiteStackHost {
node_name: row.node_name.clone(),
upload: vec![(
row.time.format("%Y-%m-%d %H:%M:%S").to_string(),
row.bits_max as i64,
)],
download: vec![],
});
}
}
result
}
fn reduce_to_x_entries(result: &mut Vec<SiteStackHost>) {
// Sort descending by total
result.sort_by(|a, b| {
b.total()
.partial_cmp(&a.total())
.unwrap_or(std::cmp::Ordering::Equal)
});
const MAX_HOSTS: usize = 8;
if result.len() > MAX_HOSTS {
let mut others = SiteStackHost {
node_name: "others".to_string(),
download: Vec::new(),
upload: Vec::new(),
};
result[0].download.iter().for_each(|x| {
others.download.push((x.0.clone(), 0));
others.upload.push((x.0.clone(), 0));
});
result.iter().skip(MAX_HOSTS).for_each(|row| {
row.download.iter().enumerate().for_each(|(i, x)| {
if i < others.download.len() {
others.download[i].1 += x.1;
}
});
row.upload.iter().enumerate().for_each(|(i, x)| {
if i < others.upload.len() {
others.upload[i].1 += x.1;
}
});
});
result.truncate(MAX_HOSTS - 1);
result.push(others);
}
}

71
src/rust/long_term_stats/lts_node/src/web/wss/queries/throughput/throughput_row.rs
View File

@@ -1,71 +0,0 @@
use chrono::{DateTime, FixedOffset, Utc};
use influxdb2::FromDataPoint;
#[derive(Debug, FromDataPoint)]
pub struct ThroughputRow {
pub direction: String,
pub host_id: String,
pub min: f64,
pub max: f64,
pub avg: f64,
pub time: DateTime<FixedOffset>,
}
impl Default for ThroughputRow {
fn default() -> Self {
Self {
direction: "".to_string(),
host_id: "".to_string(),
min: 0.0,
max: 0.0,
avg: 0.0,
time: DateTime::<Utc>::MIN_UTC.into(),
}
}
}
#[derive(Debug, FromDataPoint)]
pub struct ThroughputRowBySite {
pub direction: String,
pub host_id: String,
pub bits_min: f64,
pub bits_max: f64,
pub bits_avg: f64,
pub time: DateTime<FixedOffset>,
}
impl Default for ThroughputRowBySite {
fn default() -> Self {
Self {
direction: "".to_string(),
host_id: "".to_string(),
bits_min: 0.0,
bits_max: 0.0,
bits_avg: 0.0,
time: DateTime::<Utc>::MIN_UTC.into(),
}
}
}
#[derive(Debug, FromDataPoint)]
pub struct ThroughputRowByCircuit {
pub direction: String,
pub ip: String,
pub min: f64,
pub max: f64,
pub avg: f64,
pub time: DateTime<FixedOffset>,
}
impl Default for ThroughputRowByCircuit {
fn default() -> Self {
Self {
direction: "".to_string(),
ip: "".to_string(),
min: 0.0,
max: 0.0,
avg: 0.0,
time: DateTime::<Utc>::MIN_UTC.into(),
}
}
}

20
src/rust/long_term_stats/lts_node/web/README.md
View File

@@ -1,20 +0,0 @@
# Site Build
This folder compiles and packages the website used by `lts_node`. It
needs to be compiled and made available to the `lts_node` process.
Steps: TBA
## Requirements
To run the build (as opposed to shipping pre-built files), you need to
install `esbuild` and `npm` (ugh). You can do this with:
```bash
(change directory to site_build folder)
sudo apt-get install npm
npm install
````
You can run the build manually by running `./esbuild.sh` in this
directory.

11
src/rust/long_term_stats/lts_node/web/app.css
View File

File diff suppressed because one or more lines are too long

7
src/rust/long_term_stats/lts_node/web/app.css.map
View File

File diff suppressed because one or more lines are too long

2092
src/rust/long_term_stats/lts_node/web/app.js
View File

File diff suppressed because one or more lines are too long

7
src/rust/long_term_stats/lts_node/web/app.js.map
View File

File diff suppressed because one or more lines are too long

440
src/rust/long_term_stats/lts_node/web/main.html
View File

@@ -1,440 +0,0 @@
<!doctype html>
<html lang="en" data-bs-theme="auto">
<head>
<script>
/*!
* Color mode toggler for Bootstrap's docs (https://getbootstrap.com/)
* Copyright 2011-2023 The Bootstrap Authors
* Licensed under the Creative Commons Attribution 3.0 Unported License.
*/
(() => {
'use strict'
const getStoredTheme = () => localStorage.getItem('theme')
const setStoredTheme = theme => localStorage.setItem('theme', theme)
const getPreferredTheme = () => {
const storedTheme = getStoredTheme()
if (storedTheme) {
return storedTheme
}
return window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light'
}
const setTheme = theme => {
if (theme === 'auto' && window.matchMedia('(prefers-color-scheme: dark)').matches) {
document.documentElement.setAttribute('data-bs-theme', 'dark')
} else {
document.documentElement.setAttribute('data-bs-theme', theme)
}
}
setTheme(getPreferredTheme())
const showActiveTheme = (theme, focus = false) => {
const themeSwitcher = document.querySelector('#bd-theme')
if (!themeSwitcher) {
return
}
const themeSwitcherText = document.querySelector('#bd-theme-text')
const activeThemeIcon = document.querySelector('.theme-icon-active use')
const btnToActive = document.querySelector(`[data-bs-theme-value="${theme}"]`)
const svgOfActiveBtn = btnToActive.querySelector('svg use').getAttribute('href')
document.querySelectorAll('[data-bs-theme-value]').forEach(element => {
element.classList.remove('active')
element.setAttribute('aria-pressed', 'false')
})
btnToActive.classList.add('active')
btnToActive.setAttribute('aria-pressed', 'true')
activeThemeIcon.setAttribute('href', svgOfActiveBtn)
const themeSwitcherLabel = `${themeSwitcherText.textContent} (${btnToActive.dataset.bsThemeValue})`
themeSwitcher.setAttribute('aria-label', themeSwitcherLabel)
if (focus) {
themeSwitcher.focus()
}
}
window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', () => {
const storedTheme = getStoredTheme()
if (storedTheme !== 'light' && storedTheme !== 'dark') {
setTheme(getPreferredTheme())
}
})
window.addEventListener('DOMContentLoaded', () => {
showActiveTheme(getPreferredTheme())
document.querySelectorAll('[data-bs-theme-value]')
.forEach(toggle => {
toggle.addEventListener('click', () => {
const theme = toggle.getAttribute('data-bs-theme-value')
setStoredTheme(theme)
setTheme(theme)
showActiveTheme(theme, true)
})
})
})
})()
</script>
<!-- Import color switcher here -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>LibreQoS Long-Term Statistics</title>
<link rel="shortcut icon" href="#" />
<!-- From Theme -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@docsearch/css@3">
<!-- From BootStrap -->
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.1/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-4bw+/aepP/YC94hEpVNVgiZdgIC5+VKNBQNGCHeKRQN+PtmoHDEXuppvnDJzQIu9" crossorigin="anonymous">
<style>
.bd-placeholder-img {
font-size: 1.125rem;
text-anchor: middle;
-webkit-user-select: none;
-moz-user-select: none;
user-select: none;
}
@media (min-width: 768px) {
.bd-placeholder-img-lg {
font-size: 3.5rem;
}
}
.b-example-divider {
width: 100%;
height: 3rem;
background-color: rgba(0, 0, 0, .1);
border: solid rgba(0, 0, 0, .15);
border-width: 1px 0;
box-shadow: inset 0 .5em 1.5em rgba(0, 0, 0, .1), inset 0 .125em .5em rgba(0, 0, 0, .15);
}
.b-example-vr {
flex-shrink: 0;
width: 1.5rem;
height: 100vh;
}
.bi {
vertical-align: -.125em;
fill: currentColor;
}
.nav-scroller {
position: relative;
z-index: 2;
height: 2.75rem;
overflow-y: hidden;
}
.nav-scroller .nav {
display: flex;
flex-wrap: nowrap;
padding-bottom: 1rem;
margin-top: -1px;
overflow-x: auto;
text-align: center;
white-space: nowrap;
-webkit-overflow-scrolling: touch;
}
.btn-bd-primary {
--bd-violet-bg: #712cf9;
--bd-violet-rgb: 112.520718, 44.062154, 249.437846;
--bs-btn-font-weight: 600;
--bs-btn-color: var(--bs-white);
--bs-btn-bg: var(--bd-violet-bg);
--bs-btn-border-color: var(--bd-violet-bg);
--bs-btn-hover-color: var(--bs-white);
--bs-btn-hover-bg: #6528e0;
--bs-btn-hover-border-color: #6528e0;
--bs-btn-focus-shadow-rgb: var(--bd-violet-rgb);
--bs-btn-active-color: var(--bs-btn-hover-color);
--bs-btn-active-bg: #5a23c8;
--bs-btn-active-border-color: #5a23c8;
}
.bd-mode-toggle {
z-index: 1500;
}
</style>
<link rel="stylesheet" href="style.css" />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css" />
</head>
<body>
<svg xmlns="http://www.w3.org/2000/svg" class="d-none">
<symbol id="check2" viewBox="0 0 16 16">
<path
d="M13.854 3.646a.5.5 0 0 1 0 .708l-7 7a.5.5 0 0 1-.708 0l-3.5-3.5a.5.5 0 1 1 .708-.708L6.5 10.293l6.646-6.647a.5.5 0 0 1 .708 0z" />
</symbol>
<symbol id="circle-half" viewBox="0 0 16 16">
<path d="M8 15A7 7 0 1 0 8 1v14zm0 1A8 8 0 1 1 8 0a8 8 0 0 1 0 16z" />
</symbol>
<symbol id="moon-stars-fill" viewBox="0 0 16 16">
<path
d="M6 .278a.768.768 0 0 1 .08.858 7.208 7.208 0 0 0-.878 3.46c0 4.021 3.278 7.277 7.318 7.277.527 0 1.04-.055 1.533-.16a.787.787 0 0 1 .81.316.733.733 0 0 1-.031.893A8.349 8.349 0 0 1 8.344 16C3.734 16 0 12.286 0 7.71 0 4.266 2.114 1.312 5.124.06A.752.752 0 0 1 6 .278z" />
<path
d="M10.794 3.148a.217.217 0 0 1 .412 0l.387 1.162c.173.518.579.924 1.097 1.097l1.162.387a.217.217 0 0 1 0 .412l-1.162.387a1.734 1.734 0 0 0-1.097 1.097l-.387 1.162a.217.217 0 0 1-.412 0l-.387-1.162A1.734 1.734 0 0 0 9.31 6.593l-1.162-.387a.217.217 0 0 1 0-.412l1.162-.387a1.734 1.734 0 0 0 1.097-1.097l.387-1.162zM13.863.099a.145.145 0 0 1 .274 0l.258.774c.115.346.386.617.732.732l.774.258a.145.145 0 0 1 0 .274l-.774.258a1.156 1.156 0 0 0-.732.732l-.258.774a.145.145 0 0 1-.274 0l-.258-.774a1.156 1.156 0 0 0-.732-.732l-.774-.258a.145.145 0 0 1 0-.274l.774-.258c.346-.115.617-.386.732-.732L13.863.1z" />
</symbol>
<symbol id="sun-fill" viewBox="0 0 16 16">
<path
d="M8 12a4 4 0 1 0 0-8 4 4 0 0 0 0 8zM8 0a.5.5 0 0 1 .5.5v2a.5.5 0 0 1-1 0v-2A.5.5 0 0 1 8 0zm0 13a.5.5 0 0 1 .5.5v2a.5.5 0 0 1-1 0v-2A.5.5 0 0 1 8 13zm8-5a.5.5 0 0 1-.5.5h-2a.5.5 0 0 1 0-1h2a.5.5 0 0 1 .5.5zM3 8a.5.5 0 0 1-.5.5h-2a.5.5 0 0 1 0-1h2A.5.5 0 0 1 3 8zm10.657-5.657a.5.5 0 0 1 0 .707l-1.414 1.415a.5.5 0 1 1-.707-.708l1.414-1.414a.5.5 0 0 1 .707 0zm-9.193 9.193a.5.5 0 0 1 0 .707L3.05 13.657a.5.5 0 0 1-.707-.707l1.414-1.414a.5.5 0 0 1 .707 0zm9.193 2.121a.5.5 0 0 1-.707 0l-1.414-1.414a.5.5 0 0 1 .707-.707l1.414 1.414a.5.5 0 0 1 0 .707zM4.464 4.465a.5.5 0 0 1-.707 0L2.343 3.05a.5.5 0 1 1 .707-.707l1.414 1.414a.5.5 0 0 1 0 .708z" />
</symbol>
</svg>
<!-- Theme Switcher -->
<div class="dropdown position-fixed bottom-0 end-0 mb-3 me-3 bd-mode-toggle">
<button class="btn btn-bd-primary py-2 dropdown-toggle d-flex align-items-center" id="bd-theme" type="button"
aria-expanded="false" data-bs-toggle="dropdown" aria-label="Toggle theme (auto)">
<svg class="bi my-1 theme-icon-active" width="1em" height="1em">
<use href="#circle-half"></use>
</svg>
<span class="visually-hidden" id="bd-theme-text">Toggle theme</span>
</button>
<ul class="dropdown-menu dropdown-menu-end shadow" aria-labelledby="bd-theme-text">
<li>
<button type="button" class="dropdown-item d-flex align-items-center" data-bs-theme-value="light"
aria-pressed="false">
<svg class="bi me-2 opacity-50 theme-icon" width="1em" height="1em">
<use href="#sun-fill"></use>
</svg>
Light
<svg class="bi ms-auto d-none" width="1em" height="1em">
<use href="#check2"></use>
</svg>
</button>
</li>
<li>
<button type="button" class="dropdown-item d-flex align-items-center" data-bs-theme-value="dark"
aria-pressed="false">
<svg class="bi me-2 opacity-50 theme-icon" width="1em" height="1em">
<use href="#moon-stars-fill"></use>
</svg>
Dark
<svg class="bi ms-auto d-none" width="1em" height="1em">
<use href="#check2"></use>
</svg>
</button>
</li>
<li>
<button type="button" class="dropdown-item d-flex align-items-center active" data-bs-theme-value="auto"
aria-pressed="true">
<svg class="bi me-2 opacity-50 theme-icon" width="1em" height="1em">
<use href="#circle-half"></use>
</svg>
Auto
<svg class="bi ms-auto d-none" width="1em" height="1em">
<use href="#check2"></use>
</svg>
</button>
</li>
</ul>
</div>
<svg xmlns="http://www.w3.org/2000/svg" class="d-none">
<symbol id="calendar3" viewBox="0 0 16 16">
<path
d="M14 0H2a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V2a2 2 0 0 0-2-2zM1 3.857C1 3.384 1.448 3 2 3h12c.552 0 1 .384 1 .857v10.286c0 .473-.448.857-1 .857H2c-.552 0-1-.384-1-.857V3.857z" />
<path
d="M6.5 7a1 1 0 1 0 0-2 1 1 0 0 0 0 2zm3 0a1 1 0 1 0 0-2 1 1 0 0 0 0 2zm3 0a1 1 0 1 0 0-2 1 1 0 0 0 0 2zm-9 3a1 1 0 1 0 0-2 1 1 0 0 0 0 2zm3 0a1 1 0 1 0 0-2 1 1 0 0 0 0 2zm3 0a1 1 0 1 0 0-2 1 1 0 0 0 0 2zm3 0a1 1 0 1 0 0-2 1 1 0 0 0 0 2zm-9 3a1 1 0 1 0 0-2 1 1 0 0 0 0 2zm3 0a1 1 0 1 0 0-2 1 1 0 0 0 0 2zm3 0a1 1 0 1 0 0-2 1 1 0 0 0 0 2z" />
</symbol>
<symbol id="cart" viewBox="0 0 16 16">
<path
d="M0 1.5A.5.5 0 0 1 .5 1H2a.5.5 0 0 1 .485.379L2.89 3H14.5a.5.5 0 0 1 .49.598l-1 5a.5.5 0 0 1-.465.401l-9.397.472L4.415 11H13a.5.5 0 0 1 0 1H4a.5.5 0 0 1-.491-.408L2.01 3.607 1.61 2H.5a.5.5 0 0 1-.5-.5zM3.102 4l.84 4.479 9.144-.459L13.89 4H3.102zM5 12a2 2 0 1 0 0 4 2 2 0 0 0 0-4zm7 0a2 2 0 1 0 0 4 2 2 0 0 0 0-4zm-7 1a1 1 0 1 1 0 2 1 1 0 0 1 0-2zm7 0a1 1 0 1 1 0 2 1 1 0 0 1 0-2z" />
</symbol>
<symbol id="chevron-right" viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M4.646 1.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1 0 .708l-6 6a.5.5 0 0 1-.708-.708L10.293 8 4.646 2.354a.5.5 0 0 1 0-.708z" />
</symbol>
<symbol id="door-closed" viewBox="0 0 16 16">
<path d="M3 2a1 1 0 0 1 1-1h8a1 1 0 0 1 1 1v13h1.5a.5.5 0 0 1 0 1h-13a.5.5 0 0 1 0-1H3V2zm1 13h8V2H4v13z" />
<path d="M9 9a1 1 0 1 0 2 0 1 1 0 0 0-2 0z" />
</symbol>
<symbol id="file-earmark" viewBox="0 0 16 16">
<path
d="M14 4.5V14a2 2 0 0 1-2 2H4a2 2 0 0 1-2-2V2a2 2 0 0 1 2-2h5.5L14 4.5zm-3 0A1.5 1.5 0 0 1 9.5 3V1H4a1 1 0 0 0-1 1v12a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1V4.5h-2z" />
</symbol>
<symbol id="file-earmark-text" viewBox="0 0 16 16">
<path
d="M5.5 7a.5.5 0 0 0 0 1h5a.5.5 0 0 0 0-1h-5zM5 9.5a.5.5 0 0 1 .5-.5h5a.5.5 0 0 1 0 1h-5a.5.5 0 0 1-.5-.5zm0 2a.5.5 0 0 1 .5-.5h2a.5.5 0 0 1 0 1h-2a.5.5 0 0 1-.5-.5z" />
<path
d="M9.5 0H4a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V4.5L9.5 0zm0 1v2A1.5 1.5 0 0 0 11 4.5h2V14a1 1 0 0 1-1 1H4a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1h5.5z" />
</symbol>
<symbol id="gear-wide-connected" viewBox="0 0 16 16">
<path
d="M7.068.727c.243-.97 1.62-.97 1.864 0l.071.286a.96.96 0 0 0 1.622.434l.205-.211c.695-.719 1.888-.03 1.613.931l-.08.284a.96.96 0 0 0 1.187 1.187l.283-.081c.96-.275 1.65.918.931 1.613l-.211.205a.96.96 0 0 0 .434 1.622l.286.071c.97.243.97 1.62 0 1.864l-.286.071a.96.96 0 0 0-.434 1.622l.211.205c.719.695.03 1.888-.931 1.613l-.284-.08a.96.96 0 0 0-1.187 1.187l.081.283c.275.96-.918 1.65-1.613.931l-.205-.211a.96.96 0 0 0-1.622.434l-.071.286c-.243.97-1.62.97-1.864 0l-.071-.286a.96.96 0 0 0-1.622-.434l-.205.211c-.695.719-1.888.03-1.613-.931l.08-.284a.96.96 0 0 0-1.186-1.187l-.284.081c-.96.275-1.65-.918-.931-1.613l.211-.205a.96.96 0 0 0-.434-1.622l-.286-.071c-.97-.243-.97-1.62 0-1.864l.286-.071a.96.96 0 0 0 .434-1.622l-.211-.205c-.719-.695-.03-1.888.931-1.613l.284.08a.96.96 0 0 0 1.187-1.186l-.081-.284c-.275-.96.918-1.65 1.613-.931l.205.211a.96.96 0 0 0 1.622-.434l.071-.286zM12.973 8.5H8.25l-2.834 3.779A4.998 4.998 0 0 0 12.973 8.5zm0-1a4.998 4.998 0 0 0-7.557-3.779l2.834 3.78h4.723zM5.048 3.967c-.03.021-.058.043-.087.065l.087-.065zm-.431.355A4.984 4.984 0 0 0 3.002 8c0 1.455.622 2.765 1.615 3.678L7.375 8 4.617 4.322zm.344 7.646.087.065-.087-.065z" />
</symbol>
<symbol id="graph-up" viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M0 0h1v15h15v1H0V0Zm14.817 3.113a.5.5 0 0 1 .07.704l-4.5 5.5a.5.5 0 0 1-.74.037L7.06 6.767l-3.656 5.027a.5.5 0 0 1-.808-.588l4-5.5a.5.5 0 0 1 .758-.06l2.609 2.61 4.15-5.073a.5.5 0 0 1 .704-.07Z" />
</symbol>
<symbol id="house-fill" viewBox="0 0 16 16">
<path
d="M8.707 1.5a1 1 0 0 0-1.414 0L.646 8.146a.5.5 0 0 0 .708.708L8 2.207l6.646 6.647a.5.5 0 0 0 .708-.708L13 5.793V2.5a.5.5 0 0 0-.5-.5h-1a.5.5 0 0 0-.5.5v1.293L8.707 1.5Z" />
<path d="m8 3.293 6 6V13.5a1.5 1.5 0 0 1-1.5 1.5h-9A1.5 1.5 0 0 1 2 13.5V9.293l6-6Z" />
</symbol>
<symbol id="list" viewBox="0 0 16 16">
<path fill-rule="evenodd"
d="M2.5 12a.5.5 0 0 1 .5-.5h10a.5.5 0 0 1 0 1H3a.5.5 0 0 1-.5-.5zm0-4a.5.5 0 0 1 .5-.5h10a.5.5 0 0 1 0 1H3a.5.5 0 0 1-.5-.5zm0-4a.5.5 0 0 1 .5-.5h10a.5.5 0 0 1 0 1H3a.5.5 0 0 1-.5-.5z" />
</symbol>
<symbol id="people" viewBox="0 0 16 16">
<path
d="M15 14s1 0 1-1-1-4-5-4-5 3-5 4 1 1 1 1h8Zm-7.978-1A.261.261 0 0 1 7 12.996c.001-.264.167-1.03.76-1.72C8.312 10.629 9.282 10 11 10c1.717 0 2.687.63 3.24 1.276.593.69.758 1.457.76 1.72l-.008.002a.274.274 0 0 1-.014.002H7.022ZM11 7a2 2 0 1 0 0-4 2 2 0 0 0 0 4Zm3-2a3 3 0 1 1-6 0 3 3 0 0 1 6 0ZM6.936 9.28a5.88 5.88 0 0 0-1.23-.247A7.35 7.35 0 0 0 5 9c-4 0-5 3-5 4 0 .667.333 1 1 1h4.216A2.238 2.238 0 0 1 5 13c0-1.01.377-2.042 1.09-2.904.243-.294.526-.569.846-.816ZM4.92 10A5.493 5.493 0 0 0 4 13H1c0-.26.164-1.03.76-1.724.545-.636 1.492-1.256 3.16-1.275ZM1.5 5.5a3 3 0 1 1 6 0 3 3 0 0 1-6 0Zm3-2a2 2 0 1 0 0 4 2 2 0 0 0 0-4Z" />
</symbol>
<symbol id="plus-circle" viewBox="0 0 16 16">
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z" />
<path d="M8 4a.5.5 0 0 1 .5.5v3h3a.5.5 0 0 1 0 1h-3v3a.5.5 0 0 1-1 0v-3h-3a.5.5 0 0 1 0-1h3v-3A.5.5 0 0 1 8 4z" />
</symbol>
<symbol id="puzzle" viewBox="0 0 16 16">
<path
d="M3.112 3.645A1.5 1.5 0 0 1 4.605 2H7a.5.5 0 0 1 .5.5v.382c0 .696-.497 1.182-.872 1.469a.459.459 0 0 0-.115.118.113.113 0 0 0-.012.025L6.5 4.5v.003l.003.01c.004.01.014.028.036.053a.86.86 0 0 0 .27.194C7.09 4.9 7.51 5 8 5c.492 0 .912-.1 1.19-.24a.86.86 0 0 0 .271-.194.213.213 0 0 0 .039-.063v-.009a.112.112 0 0 0-.012-.025.459.459 0 0 0-.115-.118c-.375-.287-.872-.773-.872-1.469V2.5A.5.5 0 0 1 9 2h2.395a1.5 1.5 0 0 1 1.493 1.645L12.645 6.5h.237c.195 0 .42-.147.675-.48.21-.274.528-.52.943-.52.568 0 .947.447 1.154.862C15.877 6.807 16 7.387 16 8s-.123 1.193-.346 1.638c-.207.415-.586.862-1.154.862-.415 0-.733-.246-.943-.52-.255-.333-.48-.48-.675-.48h-.237l.243 2.855A1.5 1.5 0 0 1 11.395 14H9a.5.5 0 0 1-.5-.5v-.382c0-.696.497-1.182.872-1.469a.459.459 0 0 0 .115-.118.113.113 0 0 0 .012-.025L9.5 11.5v-.003a.214.214 0 0 0-.039-.064.859.859 0 0 0-.27-.193C8.91 11.1 8.49 11 8 11c-.491 0-.912.1-1.19.24a.859.859 0 0 0-.271.194.214.214 0 0 0-.039.063v.003l.001.006a.113.113 0 0 0 .012.025c.016.027.05.068.115.118.375.287.872.773.872 1.469v.382a.5.5 0 0 1-.5.5H4.605a1.5 1.5 0 0 1-1.493-1.645L3.356 9.5h-.238c-.195 0-.42.147-.675.48-.21.274-.528.52-.943.52-.568 0-.947-.447-1.154-.862C.123 9.193 0 8.613 0 8s.123-1.193.346-1.638C.553 5.947.932 5.5 1.5 5.5c.415 0 .733.246.943.52.255.333.48.48.675.48h.238l-.244-2.855zM4.605 3a.5.5 0 0 0-.498.55l.001.007.29 3.4A.5.5 0 0 1 3.9 7.5h-.782c-.696 0-1.182-.497-1.469-.872a.459.459 0 0 0-.118-.115.112.112 0 0 0-.025-.012L1.5 6.5h-.003a.213.213 0 0 0-.064.039.86.86 0 0 0-.193.27C1.1 7.09 1 7.51 1 8c0 .491.1.912.24 1.19.07.14.14.225.194.271a.213.213 0 0 0 .063.039H1.5l.006-.001a.112.112 0 0 0 .025-.012.459.459 0 0 0 .118-.115c.287-.375.773-.872 1.469-.872H3.9a.5.5 0 0 1 .498.542l-.29 3.408a.5.5 0 0 0 .497.55h1.878c-.048-.166-.195-.352-.463-.557-.274-.21-.52-.528-.52-.943 0-.568.447-.947.862-1.154C6.807 10.123 7.387 10 8 10s1.193.123 1.638.346c.415.207.862.586.862 1.154 0 .415-.246.733-.52.943-.268.205-.415.39-.463.557h1.878a.5.5 0 0 0 .498-.55l-.001-.007-.29-3.4A.5.5 0 0 1 12.1 8.5h.782c.696 0 1.182.497 1.469.872.05.065.091.099.118.115.013.008.021.01.025.012a.02.02 0 0 0 .006.001h.003a.214.214 0 0 0 .064-.039.86.86 0 0 0 .193-.27c.14-.28.24-.7.24-1.191 0-.492-.1-.912-.24-1.19a.86.86 0 0 0-.194-.271.215.215 0 0 0-.063-.039H14.5l-.006.001a.113.113 0 0 0-.025.012.459.459 0 0 0-.118.115c-.287.375-.773.872-1.469.872H12.1a.5.5 0 0 1-.498-.543l.29-3.407a.5.5 0 0 0-.497-.55H9.517c.048.166.195.352.463.557.274.21.52.528.52.943 0 .568-.447.947-.862 1.154C9.193 5.877 8.613 6 8 6s-1.193-.123-1.638-.346C5.947 5.447 5.5 5.068 5.5 4.5c0-.415.246-.733.52-.943.268-.205.415-.39.463-.557H4.605z" />
</symbol>
<symbol id="search" viewBox="0 0 16 16">
<path
d="M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0z" />
</symbol>
</svg>
<header class="navbar sticky-top bg-dark flex-md-nowrap p-0 shadow" data-bs-theme="dark">
<a class="navbar-brand col-md-3 col-lg-2 me-0 px-3 fs-6 text-white" href="#"
onclick="window.router.goto('dashboard')">LibreQoS</a>
<ul class="navbar-nav flex-row">
<li class="nav-item text-nowrap">
<div class="dropdown" style="padding: 6px;">
<button class="btn btn-secondary dropdown-toggle" type="button" data-bs-toggle="dropdown"
aria-expanded="false" id="graphPeriodBtn">
Graph Period
</button>
<ul class="dropdown-menu">
<li><a class="dropdown-item" href="#" onclick="window.changeGraphPeriod('5m')">5 minutes</a></li>
<li><a class="dropdown-item" href="#" onclick="window.changeGraphPeriod('15m')">15 minutes</a></li>
<li><a class="dropdown-item" href="#" onclick="window.changeGraphPeriod('1h')">1 hour</a></li>
<li><a class="dropdown-item" href="#" onclick="window.changeGraphPeriod('6h')">6 hours</a></li>
<li><a class="dropdown-item" href="#" onclick="window.changeGraphPeriod('12h')">12 hours</a></li>
<li><a class="dropdown-item" href="#" onclick="window.changeGraphPeriod('24h')">24 hours</a></li>
<li><a class="dropdown-item" href="#" onclick="window.changeGraphPeriod('7d')">7 days</a></li>
<li><a class="dropdown-item" href="#" onclick="window.changeGraphPeriod('28d')">28 days</a></li>
</ul>
</div>
</li>
<li class="nav-item text-nowrap">
<form class="d-flex" role="search" onkeydown="return event.key != 'Enter';">
<input id="txtSearch" class="form-control me-2" type="search" placeholder="Search" aria-label="Search"
autocomplete="off">
<div id="searchResults"
style="position: fixed; display: none; color: black; background-color: #eee; top:55px; left: 0x; width: 300px; height: auto; z-index: 2000;">
Blah</div>
</form>
</li>
<li class="nav-item text-nowrap">
<div id="connStatus" class="fs-4" style="color: red; padding: 6px;"><i class="fa-sharp fa-solid fa-plug"></i>
</div>
</li>
</ul>
<ul class="navbar-nav flex-row d-md-none">
<li class="nav-item text-nowrap">
<button class="nav-link px-3 text-white" type="button" data-bs-toggle="offcanvas" data-bs-target="#sidebarMenu"
aria-controls="sidebarMenu" aria-expanded="false" aria-label="Toggle navigation">
<svg class="bi">
<use xlink:href="#list" />
</svg>
</button>
</li>
</ul>
<div id="navbarSearch" class="navbar-search w-100 collapse">
<input class="form-control w-100 rounded-0 border-0" type="text" placeholder="Search" aria-label="Search">
</div>
</header>
<div class="container-fluid">
<div class="row">
<div class="sidebar border border-right col-md-3 col-lg-2 p-0 bg-body-tertiary">
<div class="offcanvas-md offcanvas-end bg-body-tertiary" tabindex="-1" id="sidebarMenu"
aria-labelledby="sidebarMenuLabel">
<div class="offcanvas-header">
<h5 class="offcanvas-title" id="sidebarMenuLabel">LibreQoS</h5>
<button type="button" class="btn-close" data-bs-dismiss="offcanvas" data-bs-target="#sidebarMenu"
aria-label="Close"></button>
</div>
<div class="offcanvas-body d-md-flex flex-column p-0 pt-lg-3 overflow-y-auto">
<ul class="nav nav-pills flex-column mb-auto">
<li class="nav-item text-white">
<span class="nav-link" aria-current="page" id="menuDash" onclick="window.router.goto('dashboard')">
<i class="fa-solid fa-gauge"></i> Dashboard
</span>
</li>
<li class="nav-item">
<span class="nav-link" aria-current="page" id="nodesDash" onclick="window.router.goto('shapernodes')">
<i class="fa-solid fa-server"></i> Shaper Nodes
</span>
</li>
<li class="nav-item">
<span class="nav-link" aria-current="page" id="sitetreeDash" onclick="window.router.goto('sitetree')">
<i class="fa-solid fa-tree"></i> Site Tree
</span>
</li>
<li class="nav-item">
<a href="#" class="nav-link" aria-current="page" id="menuUser">
Username
</a>
</li>
</ul>
<div id="nodeStatus">
</div>
</div>
</div>
</div>
<main class="col-md-9 ms-sm-auto col-lg-10 px-md-4">
<div
class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom"
id="mainContent">
<i class="fa-solid fa-spinner fa-spin"></i>
</div>
</div>
</main>
</div>
</div>
<div id="SpinLoad"
style="z-index: 2000; position: absolute; left: 0px; right: 0px; top: 0px; bottom: 0px; background: #ddd; font-size: 48px; color: #555; text-align: center;">
<p>
Loading, Please Wait...
<i class="fa-solid fa-spinner fa-spin"></i>
</p>
</div>
<footer>Copyright &copy; 2023 LibreQoS</footer>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.1/dist/js/bootstrap.bundle.min.js"
integrity="sha384-HwwvtgBNo3bZJJLYd8oVXjrBZt8cqVSpeBNS5n7C8IVInixGAoxmnlMuBnhbgrkm"
crossorigin="anonymous"></script>
<script type="module" src="/app.js"></script>
</body>
</html>

2
src/rust/long_term_stats/lts_node/web/style.css
View File

@@ -1,2 +0,0 @@
.b-example-divider{height:3rem;background-color:#0000001a;border:solid rgba(0,0,0,.15);border-width:1px 0;box-shadow:inset 0 .5em 1.5em #0000001a,inset 0 .125em .5em #00000026}.b-example-vr{flex-shrink:0;width:.5rem;height:100vh}.break{flex-basis:100%;height:0}.row{margin-bottom:2px}.bi{display:inline-block;width:1rem;height:1rem}@media (min-width: 768px){.sidebar .offcanvas-lg{position:-webkit-sticky;position:sticky;top:48px}.navbar-search{display:block}}.sidebar .nav-link{font-size:.875rem;font-weight:500}.sidebar .nav-link.active{color:#2470dc}.sidebar-heading{font-size:.75rem}.navbar-brand{padding-top:.75rem;padding-bottom:.75rem;background-color:#00000040;box-shadow:inset -1px 0 #00000040}.navbar .form-control{padding:.75rem 1rem}
/*# sourceMappingURL=style.css.map */

7
src/rust/long_term_stats/lts_node/web/style.css.map
View File

@@ -1,7 +0,0 @@
{
"version": 3,
"sources": ["../src/style.css"],
"sourcesContent": ["/*@import 'bootstrap/dist/css/bootstrap.css';*/\n\n.b-example-divider {\n height: 3rem;\n background-color: rgba(0, 0, 0, .1);\n border: solid rgba(0, 0, 0, .15);\n border-width: 1px 0;\n box-shadow: inset 0 .5em 1.5em rgba(0, 0, 0, .1), inset 0 .125em .5em rgba(0, 0, 0, .15);\n}\n\n.b-example-vr {\n flex-shrink: 0;\n width: 0.5rem;\n height: 100vh;\n}\n\n.break {\n flex-basis: 100%;\n height: 0;\n}\n\n.row {\n margin-bottom: 2px;\n}\n\n.bi {\n display: inline-block;\n width: 1rem;\n height: 1rem;\n}\n\n/*\n * Sidebar\n */\n\n@media (min-width: 768px) {\n .sidebar .offcanvas-lg {\n position: -webkit-sticky;\n position: sticky;\n top: 48px;\n }\n\n .navbar-search {\n display: block;\n }\n}\n\n.sidebar .nav-link {\n font-size: .875rem;\n font-weight: 500;\n}\n\n.sidebar .nav-link.active {\n color: #2470dc;\n}\n\n.sidebar-heading {\n font-size: .75rem;\n}\n\n/*\n * Navbar\n */\n\n.navbar-brand {\n padding-top: .75rem;\n padding-bottom: .75rem;\n background-color: rgba(0, 0, 0, .25);\n box-shadow: inset -1px 0 0 rgba(0, 0, 0, .25);\n}\n\n.navbar .form-control {\n padding: .75rem 1rem;\n}"],
"mappings": "AAEA,mBACE,YACA,2BACA,6BACA,mBACA,sEAGF,cACE,cACA,YACA,aAGF,OACE,gBACA,SAGF,KACE,kBAGF,IACE,qBACA,WACA,YAOF,0BACE,uBACE,wBACA,gBACA,SAGF,eACE,eAIJ,mBACE,kBACA,gBAGF,0BACE,cAGF,iBACE,iBAOF,cACE,mBACA,sBACA,2BACA,kCAGF,sBAvEA",
"names": []
}

BIN
src/rust/long_term_stats/lts_node/web/wasm_pipe_bg.wasm
View File

Binary file not shown.

19
src/rust/long_term_stats/pgdb/Cargo.toml
View File

@@ -1,19 +0,0 @@
[package]
name = "pgdb"
version = "0.1.0"
edition = "2021"
[dependencies]
once_cell = "1"
thiserror = "1"
env_logger = "0"
log = "0"
lqos_bus = { path = "../../lqos_bus" }
sqlx = { version = "0.6.3", features = [ "runtime-tokio-rustls", "postgres" ] }
futures = "0"
uuid = { version = "1", features = ["v4", "fast-rng" ] }
influxdb2 = "0"
sha2 = "0"
dashmap = "5"
lqos_utils = { path = "../../lqos_utils" }
itertools = "0.11.0"

184
src/rust/long_term_stats/pgdb/migrations/20230420154634_initial-tables.sql
View File

@@ -1,184 +0,0 @@
-- Creates the initial tables for the license server
-- We're using Trigrams for faster text search
CREATE EXTENSION pg_trgm;
CREATE TABLE public.licenses (
key character varying(254) NOT NULL,
stats_host integer NOT NULL
);
CREATE TABLE public.organizations (
key character varying(254) NOT NULL,
name character varying(254) NOT NULL,
influx_host character varying(254) NOT NULL,
influx_org character varying(254) NOT NULL,
influx_token character varying(254) NOT NULL,
influx_bucket character varying(254) NOT NULL
);
CREATE TABLE public.shaper_nodes (
license_key character varying(254) NOT NULL,
node_id character varying(254) NOT NULL,
node_name character varying(254) NOT NULL,
last_seen timestamp without time zone DEFAULT now() NOT NULL,
public_key bytea
);
CREATE TABLE public.site_tree
(
key character varying(254) NOT NULL,
site_name character varying(254) NOT NULL,
host_id character varying(254) NOT NULL,
index integer NOT NULL,
parent integer NOT NULL,
site_type character varying(32),
max_up integer NOT NULL DEFAULT 0,
max_down integer NOT NULL DEFAULT 0,
current_up integer NOT NULL DEFAULT 0,
current_down integer NOT NULL DEFAULT 0,
current_rtt integer NOT NULL DEFAULT 0,
PRIMARY KEY (key, site_name, host_id)
);
CREATE TABLE public.shaped_devices
(
key character varying(254) NOT NULL,
node_id character varying(254) NOT NULL,
circuit_id character varying(254) NOT NULL,
device_id character varying(254) NOT NULL,
circuit_name character varying(254) NOT NULL DEFAULT '',
device_name character varying(254) NOT NULL DEFAULT '',
parent_node character varying(254) NOT NULL DEFAULT '',
mac character varying(254) NOT NULL DEFAULT '',
download_min_mbps integer NOT NULL DEFAULT 0,
upload_min_mbps integer NOT NULL DEFAULT 0,
download_max_mbps integer NOT NULL DEFAULT 0,
upload_max_mbps integer NOT NULL DEFAULT 0,
comment text,
PRIMARY KEY (key, node_id, circuit_id, device_id)
);
CREATE TABLE public.shaped_device_ip
(
key character varying(254) COLLATE pg_catalog."default" NOT NULL,
node_id character varying(254) COLLATE pg_catalog."default" NOT NULL,
circuit_id character varying(254) COLLATE pg_catalog."default" NOT NULL,
ip_range character varying(254) COLLATE pg_catalog."default" NOT NULL,
subnet integer NOT NULL,
CONSTRAINT shaped_device_ip_pkey PRIMARY KEY (key, node_id, circuit_id, ip_range, subnet)
);
CREATE TABLE public.stats_hosts (
id integer NOT NULL,
ip_address character varying(128) NOT NULL,
can_accept_new_clients boolean NOT NULL DEFAULT true,
influx_host character varying(128) NOT NULL,
api_key character varying(255) NOT NULL
);
CREATE SEQUENCE public.stats_hosts_id_seq
AS integer
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER TABLE ONLY public.stats_hosts
ALTER COLUMN id SET DEFAULT nextval('public.stats_hosts_id_seq'::regclass);
ALTER TABLE ONLY public.licenses
ADD CONSTRAINT licenses_pkey PRIMARY KEY (key);
ALTER TABLE ONLY public.organizations
ADD CONSTRAINT pk_organizations PRIMARY KEY (key);
ALTER TABLE ONLY public.shaper_nodes
ADD CONSTRAINT shaper_nodes_pk PRIMARY KEY (license_key, node_id);
ALTER TABLE ONLY public.stats_hosts
ADD CONSTRAINT stats_hosts_pkey PRIMARY KEY (id);
ALTER TABLE ONLY public.organizations
ADD CONSTRAINT organizations_license_fk FOREIGN KEY (key) REFERENCES public.licenses(key);
ALTER TABLE ONLY public.licenses
ADD CONSTRAINT stats_host_fk FOREIGN KEY (stats_host) REFERENCES public.stats_hosts(id) NOT VALID;
CREATE TABLE public.logins
(
key character varying(254) NOT NULL,
username character varying(64) NOT NULL,
password_hash character varying(64) NOT NULL,
nicename character varying(64) NOT NULL,
CONSTRAINT pk_logins_licenses PRIMARY KEY (key, username),
CONSTRAINT fk_login_licenses FOREIGN KEY (key)
REFERENCES public.licenses (key) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
NOT VALID
);
CREATE TABLE public.active_tokens
(
key character varying(254) NOT NULL,
token character varying(254) NOT NULL,
username character varying(64) NOT NULL,
expires timestamp without time zone NOT NULL DEFAULT NOW() + interval '2 hours',
PRIMARY KEY (token)
);
CREATE TABLE public.uisp_devices_ext
(
key character varying(254) NOT NULL,
device_id character varying(254) NOT NULL,
name character varying(254) NOT NULL DEFAULT '',
model character varying(254) NOT NULL DEFAULT '',
firmware character varying(64) NOT NULL DEFAULT '',
status character varying(64) NOT NULL DEFAULT '',
mode character varying(64) NOT NULL DEFAULT '',
channel_width integer NOT NULL DEFAULT 0,
tx_power integer NOT NULL DEFAULT 0,
PRIMARY KEY (key, device_id)
);
CREATE TABLE public.uisp_devices_interfaces
(
key character varying(254) NOT NULL,
device_id character varying(254) NOT NULL,
id serial NOT NULL,
name character varying(64) NOT NULL DEFAULT '',
mac character varying(64) NOT NULL DEFAULT '',
status character varying(64) NOT NULL DEFAULT '',
speed character varying(64) NOT NULL DEFAULT '',
ip_list character varying(254) NOT NULL DEFAULT '',
PRIMARY KEY (key, device_id, id)
);
---- Indices
CREATE INDEX site_tree_key
ON public.site_tree USING btree
(key ASC NULLS LAST)
;
CREATE INDEX site_tree_key_parent
ON public.site_tree USING btree
(key ASC NULLS LAST, parent ASC NULLS LAST)
;
CREATE INDEX shaped_devices_key_circuit_id
ON public.shaped_devices USING btree
(key ASC NULLS LAST, circuit_id ASC NULLS LAST)
;
CREATE INDEX stats_host_ip
ON public.stats_hosts USING btree
(ip_address ASC NULLS LAST)
;
CREATE INDEX shaper_nodes_license_key_idx
ON public.shaper_nodes USING btree
(license_key ASC NULLS LAST)
;

132
src/rust/long_term_stats/pgdb/src/circuit.rs
View File

@@ -1,132 +0,0 @@
use itertools::Itertools;
use sqlx::{Pool, Postgres, FromRow, Row};
use crate::license::StatsHostError;
#[derive(Debug, FromRow)]
pub struct CircuitInfo {
pub circuit_name: String,
pub device_id: String,
pub device_name: String,
pub parent_node: String,
pub mac: String,
pub download_min_mbps: i32,
pub download_max_mbps: i32,
pub upload_min_mbps: i32,
pub upload_max_mbps: i32,
pub comment: String,
pub ip_range: String,
pub subnet: i32,
}
pub async fn get_circuit_info(
cnn: &Pool<Postgres>,
key: &str,
circuit_id: &str,
) -> Result<Vec<CircuitInfo>, StatsHostError> {
const SQL: &str = "SELECT circuit_name, device_id, device_name, parent_node, mac, download_min_mbps, download_max_mbps, upload_min_mbps, upload_max_mbps, comment, ip_range, subnet FROM shaped_devices INNER JOIN shaped_device_ip ON shaped_device_ip.key = shaped_devices.key AND shaped_device_ip.circuit_id = shaped_devices.circuit_id WHERE shaped_devices.key=$1 AND shaped_devices.circuit_id=$2";
sqlx::query_as::<_, CircuitInfo>(SQL)
.bind(key)
.bind(circuit_id)
.fetch_all(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))
}
#[derive(Debug, FromRow)]
pub struct DeviceInfoExt {
pub device_id: String,
pub name: String,
pub model: String,
pub firmware: String,
pub status: String,
pub mode: String,
pub channel_width: i32,
pub tx_power: i32,
}
pub async fn get_device_info_ext(
cnn: &Pool<Postgres>,
key: &str,
device_id: &str,
) -> Result<DeviceInfoExt, StatsHostError> {
sqlx::query_as::<_, DeviceInfoExt>("SELECT device_id, name, model, firmware, status, mode, channel_width, tx_power FROM uisp_devices_ext WHERE key=$1 AND device_id=$2")
.bind(key)
.bind(device_id)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))
}
#[derive(Debug, FromRow)]
pub struct DeviceInterfaceExt {
pub name: String,
pub mac: String,
pub status: String,
pub speed: String,
pub ip_list: String,
}
pub async fn get_device_interfaces_ext(
cnn: &Pool<Postgres>,
key: &str,
device_id: &str,
) -> Result<Vec<DeviceInterfaceExt>, StatsHostError>
{
sqlx::query_as::<_, DeviceInterfaceExt>("SELECT name, mac, status, speed, ip_list FROM uisp_devices_interfaces WHERE key=$1 AND device_id=$2")
.bind(key)
.bind(device_id)
.fetch_all(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))
}
pub async fn get_circuit_list_for_site(
cnn: &Pool<Postgres>,
key: &str,
site_name: &str,
) -> Result<Vec<(String, String)>, StatsHostError> {
let hosts = sqlx::query("SELECT DISTINCT circuit_id, circuit_name FROM shaped_devices WHERE key=$1 AND parent_node=$2")
.bind(key)
.bind(site_name)
.fetch_all(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?
.iter()
.map(|row| (row.try_get("circuit_id").unwrap(), row.try_get("circuit_name").unwrap()))
.collect();
Ok(hosts)
}
pub async fn get_host_list_for_site(
cnn: &Pool<Postgres>,
key: &str,
site_name: &str,
) -> Result<Vec<(String, String)>, StatsHostError> {
let hosts = sqlx::query("SELECT DISTINCT device_id, circuit_name FROM shaped_devices WHERE key=$1 AND parent_node=$2")
.bind(key)
.bind(site_name)
.fetch_all(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?
.iter()
.map(|row| (row.try_get("device_id").unwrap(), row.try_get("circuit_name").unwrap()))
.collect();
Ok(hosts)
}
pub fn circuit_list_to_influx_filter(hosts: &[(String, String)]) -> String {
hosts.iter().map(|(id, _name)| format!("r[\"circuit_id\"] == \"{id}\"", id=id)).join(" or ")
}
pub fn host_list_to_influx_filter(hosts: &[(String, String)]) -> String {
hosts.iter().map(|(id, _name)| format!("r[\"host_id\"] == \"{id}\"", id=id)).join(" or ")
}
pub fn device_list_to_influx_filter(hosts: &[(String, String)]) -> String {
hosts.iter().map(|(id, _name)| format!("r[\"device_id\"] == \"{id}\"", id=id)).join(" or ")
}

37
src/rust/long_term_stats/pgdb/src/connection/connection_string.rs
View File

@@ -1,37 +0,0 @@
//! Manages access to the safely stored connection string, in `/etc/lqdb`.
//! Failure to obtain a database connection is a fatal error.
//! The connection string is read once, on the first call to `get_connection_string()`.
//! Please be careful to never include `/etc/lqdb` in any git commits.
use std::path::Path;
use std::fs::File;
use std::io::Read;
use once_cell::sync::Lazy;
pub static CONNECTION_STRING: Lazy<String> = Lazy::new(read_connection_string);
/// Read the connection string from /etc/lqdb
/// Called by the `Lazy` on CONNECTION_STRING
fn read_connection_string() -> String {
let path = Path::new("/etc/lqdb");
if !path.exists() {
log::error!("{} does not exist", path.display());
panic!("{} does not exist", path.display());
}
match File::open(path) {
Ok(mut file) => {
let mut buf = String::new();
if let Ok(_size) = file.read_to_string(&mut buf) {
buf
} else {
log::error!("Could not read {}", path.display());
panic!("Could not read {}", path.display());
}
}
Err(e) => {
log::error!("Could not open {}: {e:?}", path.display());
panic!("Could not open {}: {e:?}", path.display());
}
}
}

4
src/rust/long_term_stats/pgdb/src/connection/mod.rs
View File

@@ -1,4 +0,0 @@
mod connection_string;
mod pool;
pub use pool::get_connection_pool;

13
src/rust/long_term_stats/pgdb/src/connection/pool.rs
View File

@@ -1,13 +0,0 @@
use sqlx::{postgres::PgPoolOptions, Postgres, Pool};
use super::connection_string::CONNECTION_STRING;
/// Obtain a connection pool to the database.
///
/// # Arguments
/// * `max_connections` - The maximum number of connections to the database.
pub async fn get_connection_pool(max_connections: u32) -> Result<Pool<Postgres>, sqlx::Error> {
PgPoolOptions::new()
.max_connections(max_connections)
.connect(&CONNECTION_STRING)
.await
}

59
src/rust/long_term_stats/pgdb/src/hosts.rs
View File

@@ -1,59 +0,0 @@
use sqlx::{Pool, Postgres, Row};
use crate::license::StatsHostError;
pub async fn add_stats_host(cnn: Pool<Postgres>, hostname: String, influx_host: String, api_key: String) -> Result<i64, StatsHostError> {
// Does the stats host already exist? We don't want duplicates
let row = sqlx::query("SELECT COUNT(*) AS count FROM stats_hosts WHERE ip_address=$1")
.bind(&hostname)
.fetch_one(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let count: i64 = row.try_get("count").map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
if count != 0 {
return Err(StatsHostError::HostAlreadyExists);
}
// Get the new primary key
log::info!("Getting new primary key for stats host");
let row = sqlx::query("SELECT NEXTVAL('stats_hosts_id_seq') AS id")
.fetch_one(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let new_id: i64 = row.try_get("id").map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
// Insert the stats host
log::info!("Inserting new stats host: {} ({})", hostname, new_id);
sqlx::query("INSERT INTO stats_hosts (id, ip_address, can_accept_new_clients, influx_host, api_key) VALUES ($1, $2, $3, $4, $5)")
.bind(new_id)
.bind(&hostname)
.bind(true)
.bind(&influx_host)
.bind(&api_key)
.execute(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
Ok(new_id)
}
const FIND_STATS_HOST: &str = "SELECT a.id AS id, a.influx_host AS influx_host, a.api_key AS api_key
FROM stats_hosts a
WHERE can_accept_new_clients = true
ORDER BY (SELECT COUNT(organizations.\"key\") FROM organizations WHERE a.influx_host = influx_host)
LIMIT 1";
pub async fn find_emptiest_stats_host(cnn: Pool<Postgres>) -> Result<(i32, String, String), StatsHostError> {
let row = sqlx::query(FIND_STATS_HOST)
.fetch_one(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let id: i32 = row.try_get("id").map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let influx_host: String = row.try_get("influx_host").map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let api_key: String = row.try_get("api_key").map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
Ok((id, influx_host, api_key))
}

26
src/rust/long_term_stats/pgdb/src/lib.rs
View File

@@ -1,26 +0,0 @@
mod connection;
mod license;
mod organization;
mod hosts;
mod orchestrator;
mod logins;
mod nodes;
mod search;
mod tree;
mod circuit;
pub mod organization_cache;
pub mod sqlx {
pub use sqlx::*;
}
pub use connection::get_connection_pool;
pub use license::{get_stats_host_for_key, insert_or_update_node_public_key, fetch_public_key};
pub use organization::{OrganizationDetails, get_organization};
pub use hosts::add_stats_host;
pub use orchestrator::create_free_trial;
pub use logins::{try_login, delete_user, add_user, refresh_token, token_to_credentials};
pub use nodes::{new_stats_arrived, node_status, NodeStatus};
pub use search::*;
pub use tree::*;
pub use circuit::*;

87
src/rust/long_term_stats/pgdb/src/license.rs
View File

@@ -1,87 +0,0 @@
//! Handles license checks from the `license_server`.
use sqlx::{Pool, Postgres, Row};
use thiserror::Error;
pub async fn get_stats_host_for_key(cnn: Pool<Postgres>, key: &str) -> Result<String, StatsHostError> {
let row = sqlx::query("SELECT ip_address FROM licenses INNER JOIN stats_hosts ON stats_hosts.id = licenses.stats_host WHERE key=$1")
.bind(key)
.fetch_one(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let ip_address: &str = row.try_get("ip_address").map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
log::info!("Found stats host for key: {}", ip_address);
Ok(ip_address.to_string())
}
pub async fn insert_or_update_node_public_key(cnn: Pool<Postgres>, node_id: &str, node_name: &str, license_key: &str, public_key: &[u8]) -> Result<(), StatsHostError> {
let row = sqlx::query("SELECT COUNT(*) AS count FROM shaper_nodes WHERE node_id=$1 AND license_key=$2")
.bind(node_id)
.bind(license_key)
.fetch_one(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let count: i64 = row.try_get("count").map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
match count {
0 => {
// Insert
log::info!("Inserting new node: {} {}", node_id, license_key);
sqlx::query("INSERT INTO shaper_nodes (license_key, node_id, public_key, node_name) VALUES ($1, $2, $3, $4)")
.bind(license_key)
.bind(node_id)
.bind(public_key)
.bind(node_name)
.execute(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
}
1 => {
// Update
log::info!("Updating node: {} {}", node_id, license_key);
sqlx::query("UPDATE shaper_nodes SET public_key=$1, last_seen=NOW(), node_name=$4 WHERE node_id=$2 AND license_key=$3")
.bind(public_key)
.bind(node_id)
.bind(license_key)
.bind(node_name)
.execute(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
}
_ => {
log::error!("Found multiple nodes with the same node_id and license_key");
return Err(StatsHostError::DatabaseError("Found multiple nodes with the same node_id and license_key".to_string()));
}
}
Ok(())
}
pub async fn fetch_public_key(cnn: Pool<Postgres>, license_key: &str, node_id: &str) -> Result<Vec<u8>, StatsHostError> {
let row = sqlx::query("SELECT public_key FROM shaper_nodes WHERE license_key=$1 AND node_id=$2")
.bind(license_key)
.bind(node_id)
.fetch_one(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let public_key: Vec<u8> = row.try_get("public_key").map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
Ok(public_key)
}
#[derive(Debug, Error)]
pub enum StatsHostError {
#[error("Database error occurred")]
DatabaseError(String),
#[error("Host already exists")]
HostAlreadyExists,
#[error("Organization already exists")]
OrganizationAlreadyExists,
#[error("No available stats hosts")]
NoStatsHostsAvailable,
#[error("InfluxDB Error")]
InfluxError(String),
#[error("No such login")]
InvalidLogin,
}

28
src/rust/long_term_stats/pgdb/src/logins/add_del.rs
View File

@@ -1,28 +0,0 @@
use sqlx::{Pool, Postgres};
use crate::license::StatsHostError;
use super::hasher::hash_password;
pub async fn delete_user(cnn: Pool<Postgres>, key: &str, username: &str) -> Result<(), StatsHostError> {
sqlx::query("DELETE FROM logins WHERE key = $1 AND username = $2")
.bind(key)
.bind(username)
.execute(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
Ok(())
}
pub async fn add_user(cnn: Pool<Postgres>, key: &str, username: &str, password: &str, nicename: &str) -> Result<(), StatsHostError> {
let password = hash_password(password);
sqlx::query("INSERT INTO logins (key, username, password_hash, nicename) VALUES ($1, $2, $3, $4)")
.bind(key)
.bind(username)
.bind(password)
.bind(nicename)
.execute(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
Ok(())
}

9
src/rust/long_term_stats/pgdb/src/logins/hasher.rs
View File

@@ -1,9 +0,0 @@
use sha2::Sha256;
use sha2::Digest;
pub(crate) fn hash_password(password: &str) -> String {
let salted = format!("!x{password}_SaltIsGoodForYou");
let mut sha256 = Sha256::new();
sha256.update(salted);
format!("{:X}", sha256.finalize())
}

35
src/rust/long_term_stats/pgdb/src/logins/login.rs
View File

@@ -1,35 +0,0 @@
use sqlx::{Pool, Postgres, Row};
use uuid::Uuid;
use crate::license::StatsHostError;
use super::{hasher::hash_password, token_cache::create_token};
#[derive(Debug, Clone)]
pub struct LoginDetails {
pub token: String,
pub license: String,
pub name: String,
}
pub async fn try_login(cnn: Pool<Postgres>, key: &str, username: &str, password: &str) -> Result<LoginDetails, StatsHostError> {
let password = hash_password(password);
let row = sqlx::query("SELECT nicename FROM logins WHERE key = $1 AND username = $2 AND password_hash = $3")
.bind(key)
.bind(username)
.bind(password)
.fetch_one(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let nicename: String = row.try_get("nicename").map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let uuid = Uuid::new_v4().to_string();
let details = LoginDetails {
token: uuid,
name: nicename,
license: key.to_string(),
};
create_token(&cnn, &details, key, username).await?;
Ok(details)
}

8
src/rust/long_term_stats/pgdb/src/logins/mod.rs
View File

@@ -1,8 +0,0 @@
mod hasher;
mod login;
mod add_del;
mod token_cache;
pub use login::{LoginDetails, try_login};
pub use add_del::{add_user, delete_user};
pub use token_cache::{refresh_token, token_to_credentials};

96
src/rust/long_term_stats/pgdb/src/logins/token_cache.rs
View File

@@ -1,96 +0,0 @@
use super::LoginDetails;
use crate::license::StatsHostError;
use dashmap::DashMap;
use lqos_utils::unix_time::unix_now;
use once_cell::sync::Lazy;
use sqlx::{Pool, Postgres, Row};
static TOKEN_CACHE: Lazy<DashMap<String, TokenDetails>> = Lazy::new(DashMap::new);
struct TokenDetails {
last_seen: u64,
last_refreshed: u64,
}
pub async fn create_token(
cnn: &Pool<Postgres>,
details: &LoginDetails,
key: &str,
username: &str,
) -> Result<(), StatsHostError> {
sqlx::query("INSERT INTO active_tokens (token, key, username) VALUES ($1, $2, $3)")
.bind(&details.token)
.bind(key)
.bind(username)
.execute(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let now = unix_now().unwrap_or(0);
TOKEN_CACHE.insert(
details.token.clone(),
TokenDetails {
last_seen: now,
last_refreshed: now,
},
);
Ok(())
}
pub async fn refresh_token(cnn: Pool<Postgres>, token_id: &str) -> Result<(), StatsHostError> {
if let Some(mut token) = TOKEN_CACHE.get_mut(token_id) {
let now = unix_now().unwrap_or(0);
token.last_seen = now;
let age = now - token.last_refreshed;
if age > 300 {
token.last_refreshed = now;
sqlx::query("UPDATE active_tokens SET last_seen = NOW() WHERE token = $1")
.bind(token_id)
.execute(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
}
Ok(())
} else {
Err(StatsHostError::DatabaseError("Unauthorized".to_string()))
}
}
pub async fn token_to_credentials(
cnn: Pool<Postgres>,
token_id: &str,
) -> Result<LoginDetails, StatsHostError> {
let row = sqlx::query("SELECT key, username FROM active_tokens WHERE token = $1")
.bind(token_id)
.fetch_one(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let key: String = row
.try_get("key")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let username: String = row
.try_get("username")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let row = sqlx::query("SELECT nicename FROM logins WHERE key = $1 AND username = $2")
.bind(&key)
.bind(username)
.fetch_one(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let nicename: String = row
.try_get("nicename")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let details = LoginDetails {
token: token_id.to_string(),
name: nicename,
license: key,
};
Ok(details)
}

36
src/rust/long_term_stats/pgdb/src/nodes.rs
View File

@@ -1,36 +0,0 @@
use sqlx::{Pool, Postgres};
use crate::license::StatsHostError;
pub async fn new_stats_arrived(cnn: Pool<Postgres>, license: &str, node: &str) -> Result<(), StatsHostError> {
// Does the node exist?
sqlx::query("UPDATE shaper_nodes SET last_seen=NOW() WHERE license_key=$1 AND node_id=$2")
.bind(license)
.bind(node)
.execute(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
Ok(())
}
#[derive(Clone, sqlx::FromRow, Debug)]
pub struct NodeStatus {
pub node_id: String,
pub node_name: String,
pub last_seen: i32,
}
pub async fn node_status(cnn: &Pool<Postgres>, license: &str) -> Result<Vec<NodeStatus>, StatsHostError> {
let res = sqlx::query_as::<_, NodeStatus>("SELECT node_id, node_name, extract('epoch' from NOW()-last_seen)::integer AS last_seen FROM shaper_nodes WHERE license_key=$1")
.bind(license)
.fetch_all(cnn)
.await;
match res {
Err(e) => {
log::error!("Unable to get node status: {}", e);
Err(StatsHostError::DatabaseError(e.to_string()))
}
Ok(rows) => Ok(rows)
}
}

107
src/rust/long_term_stats/pgdb/src/orchestrator.rs
View File

@@ -1,107 +0,0 @@
use crate::{
hosts::find_emptiest_stats_host, license::StatsHostError,
organization::does_organization_name_exist,
};
use influxdb2::{
models::{PostBucketRequest, RetentionRule, Status},
Client,
};
use sqlx::{Pool, Postgres};
use uuid::Uuid;
pub async fn create_free_trial(
cnn: Pool<Postgres>,
organization_name: &str,
) -> Result<String, StatsHostError> {
// Check that no organization of this name exists already (error if they exist)
if does_organization_name_exist(cnn.clone(), organization_name).await? {
return Err(StatsHostError::OrganizationAlreadyExists);
}
// Find the most empty, available stats host (error if none)
let (stats_host_id, influx_url, api_key) = find_emptiest_stats_host(cnn.clone()).await?;
// Generate a new license key
let uuid = Uuid::new_v4().to_string();
// Connect to Influx, and create a new bucket and API token
create_bucket(&influx_url, &api_key, organization_name).await?;
// As a transaction:
// - Insert into licenses
// - Insert into organizations
let mut tx = cnn.begin().await.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
sqlx::query("INSERT INTO licenses (key, stats_host) VALUES ($1, $2);")
.bind(&uuid)
.bind(stats_host_id)
.execute(&mut tx)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
sqlx::query("INSERT INTO organizations (key, name, influx_host, influx_org, influx_token, influx_bucket) VALUES ($1, $2, $3, $4, $5, $6);")
.bind(&uuid)
.bind(organization_name)
.bind(&influx_url)
.bind("LibreQoS")
.bind(api_key)
.bind(organization_name)
.execute(&mut tx)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
tx.commit().await.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
Ok(uuid)
}
async fn create_bucket(
influx_host: &str,
api_key: &str,
org_name: &str,
) -> Result<(), StatsHostError> {
let influx_url = format!("http://{influx_host}:8086");
let client = Client::new(influx_url, "LibreQoS", api_key);
// Is Influx alive and well?
match client.health().await {
Err(e) => return Err(StatsHostError::InfluxError(e.to_string())),
Ok(health) => {
if health.status == Status::Fail {
return Err(StatsHostError::InfluxError(
"Influx health check failed".to_string(),
));
}
}
}
// Translate the organization name into an id
let org = client.list_organizations(influxdb2::api::organization::ListOrganizationRequest {
descending: None,
limit: None,
offset: None,
org: None,
org_id: None,
user_id: None
}).await.map_err(|e| StatsHostError::InfluxError(e.to_string()))?;
let org_id = org.orgs[0].id.as_ref().unwrap();
// Let's make the bucket
if let Err(e) = client
.create_bucket(Some(PostBucketRequest {
org_id: org_id.to_string(),
name: org_name.to_string(),
description: None,
rp: None,
retention_rules: vec![RetentionRule::new(
influxdb2::models::retention_rule::Type::Expire,
604800,
)], // 1 Week
}))
.await
{
log::error!("Error creating bucket: {}", e);
return Err(StatsHostError::InfluxError(e.to_string()));
}
Ok(())
}

38
src/rust/long_term_stats/pgdb/src/organization.rs
View File

@@ -1,38 +0,0 @@
use sqlx::{Pool, Postgres, Row};
use crate::license::StatsHostError;
#[derive(Clone, sqlx::FromRow, Debug)]
pub struct OrganizationDetails {
pub key: String,
pub name: String,
pub influx_host: String,
pub influx_org: String,
pub influx_token: String,
pub influx_bucket: String,
}
pub async fn get_organization(cnn: &Pool<Postgres>, key: &str) -> Result<OrganizationDetails, StatsHostError> {
let mut row = sqlx::query_as::<_, OrganizationDetails>("SELECT * FROM organizations WHERE key=$1")
.bind(key)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
// For local development - comment out
if row.influx_host == "127.0.0.1" {
row.influx_host = "146.190.156.69".to_string();
}
Ok(row)
}
pub async fn does_organization_name_exist(cnn: Pool<Postgres>, name: &str) -> Result<bool, StatsHostError> {
let row = sqlx::query("SELECT COUNT(*) AS count FROM organizations WHERE name=$1")
.bind(name)
.fetch_one(&cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let count: i64 = row.try_get("count").map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
Ok(count > 0)
}

25
src/rust/long_term_stats/pgdb/src/organization_cache.rs
View File

@@ -1,25 +0,0 @@
use std::{collections::HashMap, sync::RwLock};
use once_cell::sync::Lazy;
use sqlx::{Pool, Postgres};
use crate::{OrganizationDetails, get_organization};
static ORG_CACHE: Lazy<RwLock<HashMap<String, OrganizationDetails>>> = Lazy::new(|| {
RwLock::new(HashMap::new())
});
pub async fn get_org_details(cnn: &Pool<Postgres>, key: &str) -> Option<OrganizationDetails> {
{ // Safety scope - lock is dropped on exit
let cache = ORG_CACHE.read().unwrap();
if let Some(org) = cache.get(key) {
return Some(org.clone());
}
}
// We can be certain that we don't have a dangling lock now.
// Upgrade to a write lock and try to fetch the org details.
if let Ok(org) = get_organization(cnn, key).await {
let mut cache = ORG_CACHE.write().unwrap();
cache.insert(key.to_string(), org.clone());
return Some(org);
}
None
}

101
src/rust/long_term_stats/pgdb/src/search.rs
View File

@@ -1,101 +0,0 @@
use sqlx::{Pool, Postgres, FromRow};
use crate::license::StatsHostError;
#[derive(Debug, FromRow)]
pub struct DeviceHit {
pub circuit_id: String,
pub circuit_name: String,
pub score: f64,
}
#[derive(Debug, FromRow)]
pub struct SiteHit {
pub site_name: String,
pub site_type: String,
pub score: f64,
}
pub async fn search_devices(
cnn: &Pool<Postgres>,
key: &str,
term: &str,
) -> Result<Vec<DeviceHit>, StatsHostError> {
const SQL: &str = "with input as (select $1 as q)
select circuit_id, circuit_name, 1 - (input.q <<-> (circuit_name || ' ' || device_name || ' ' || mac)) as score
from shaped_devices, input
where
key = $2 AND
(input.q <<-> (circuit_name || ' ' || device_name || ' ' || mac)) < 0.15
order by input.q <<-> (circuit_name || ' ' || device_name || ' ' || mac)";
let rows = sqlx::query_as::<_, DeviceHit>(SQL)
.bind(term)
.bind(key)
.fetch_all(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()));
if let Err(e) = &rows {
log::error!("{e:?}");
}
rows
}
pub async fn search_ip(
cnn: &Pool<Postgres>,
key: &str,
term: &str,
) -> Result<Vec<DeviceHit>, StatsHostError> {
const SQL: &str = "with input as (select $1 as q)
select shaped_device_ip.circuit_id AS circuit_id,
circuit_name || ' (' || shaped_device_ip.ip_range || '/' || shaped_device_ip.subnet || ')' AS circuit_name,
1 - (input.q <<-> shaped_device_ip.ip_range) AS score
FROM shaped_device_ip INNER JOIN shaped_devices
ON (shaped_devices.circuit_id = shaped_device_ip.circuit_id AND shaped_devices.key = shaped_device_ip.key), input
WHERE shaped_device_ip.key = $2
AND (input.q <<-> shaped_device_ip.ip_range) < 0.15
ORDER BY (input.q <<-> shaped_device_ip.ip_range)";
let rows = sqlx::query_as::<_, DeviceHit>(SQL)
.bind(term)
.bind(key)
.fetch_all(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()));
if let Err(e) = &rows {
log::error!("{e:?}");
}
rows
}
pub async fn search_sites(
cnn: &Pool<Postgres>,
key: &str,
term: &str,
) -> Result<Vec<SiteHit>, StatsHostError> {
const SQL: &str = "with input as (select $1 as q)
select site_name, site_type, 1 - (input.q <<-> site_name) as score
from site_tree, input
where
key = $2 AND
(input.q <<-> site_name) < 0.15
order by input.q <<-> site_name";
let rows = sqlx::query_as::<_, SiteHit>(SQL)
.bind(term)
.bind(key)
.fetch_all(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()));
if let Err(e) = &rows {
log::error!("{e:?}");
}
rows
}

273
src/rust/long_term_stats/pgdb/src/tree.rs
View File

@@ -1,273 +0,0 @@
use crate::license::StatsHostError;
use sqlx::{FromRow, Pool, Postgres, Row};
use itertools::Itertools;
#[derive(Debug, FromRow)]
pub struct TreeNode {
pub site_name: String,
pub index: i32,
pub parent: i32,
pub site_type: String,
pub max_down: i32,
pub max_up: i32,
pub current_down: i32,
pub current_up: i32,
pub current_rtt: i32,
}
pub async fn get_site_tree(
cnn: &Pool<Postgres>,
key: &str,
host_id: &str,
) -> Result<Vec<TreeNode>, StatsHostError> {
sqlx::query_as::<_, TreeNode>("SELECT site_name, index, parent, site_type, max_down, max_up, current_down, current_up, current_rtt FROM site_tree WHERE key = $1 AND host_id=$2")
.bind(key)
.bind(host_id)
.fetch_all(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))
}
pub async fn get_site_info(
cnn: &Pool<Postgres>,
key: &str,
site_name: &str,
) -> Result<TreeNode, StatsHostError> {
sqlx::query_as::<_, TreeNode>("SELECT site_name, index, parent, site_type, max_down, max_up, current_down, current_up, current_rtt FROM site_tree WHERE key = $1 AND site_name=$2")
.bind(key)
.bind(site_name)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))
}
pub async fn get_site_id_from_name(
cnn: &Pool<Postgres>,
key: &str,
site_name: &str,
) -> Result<i32, StatsHostError> {
if site_name == "root" {
return Ok(0);
}
let site_id_db = sqlx::query("SELECT index FROM site_tree WHERE key = $1 AND site_name=$2")
.bind(key)
.bind(site_name)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let site_id: i32 = site_id_db
.try_get("index")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
Ok(site_id)
}
pub async fn get_parent_list(
cnn: &Pool<Postgres>,
key: &str,
site_name: &str,
) -> Result<Vec<(String, String)>, StatsHostError> {
let mut result = Vec::new();
// Get the site index
let site_id_db = sqlx::query("SELECT index FROM site_tree WHERE key = $1 AND site_name=$2")
.bind(key)
.bind(site_name)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let mut site_id: i32 = site_id_db
.try_get("index")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
// Get the parent list
while site_id != 0 {
let parent_db = sqlx::query(
"SELECT site_name, parent, site_type FROM site_tree WHERE key = $1 AND index=$2",
)
.bind(key)
.bind(site_id)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let parent: String = parent_db
.try_get("site_name")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let site_type: String = parent_db
.try_get("site_type")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
site_id = parent_db
.try_get("parent")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
result.push((site_type, parent));
}
Ok(result)
}
pub async fn get_child_list(
cnn: &Pool<Postgres>,
key: &str,
site_name: &str,
) -> Result<Vec<(String, String, String)>, StatsHostError> {
let mut result = Vec::new();
// Get the site index
let site_id_db = sqlx::query("SELECT index FROM site_tree WHERE key = $1 AND site_name=$2")
.bind(key)
.bind(site_name)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let site_id: i32 = site_id_db
.try_get("index")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
// Add child sites
let child_sites = sqlx::query(
"SELECT DISTINCT site_name, parent, site_type FROM site_tree WHERE key=$1 AND parent=$2",
)
.bind(key)
.bind(site_id)
.fetch_all(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
for child in child_sites {
let child_name: String = child
.try_get("site_name")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let child_type: String = child
.try_get("site_type")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
result.push((child_type, child_name.clone(), child_name));
}
// Add child shaper nodes
let child_circuits = sqlx::query(
"SELECT circuit_id, circuit_name FROM shaped_devices WHERE key=$1 AND parent_node=$2",
)
.bind(key)
.bind(site_name)
.fetch_all(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
for child in child_circuits {
let child_name: String = child
.try_get("circuit_name")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let child_id: String = child
.try_get("circuit_id")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
result.push(("circuit".to_string(), child_id, child_name));
}
result.sort_by(|a, b| a.2.cmp(&b.2));
let result = result.into_iter().dedup_by(|a,b| a.2 == b.2).collect();
Ok(result)
}
pub async fn get_circuit_parent_list(
cnn: &Pool<Postgres>,
key: &str,
circuit_id: &str,
) -> Result<Vec<(String, String)>, StatsHostError> {
let mut result = Vec::new();
// Get the site name to start at
let site_name: String =
sqlx::query("SELECT parent_node FROM shaped_devices WHERE key = $1 AND circuit_id= $2")
.bind(key)
.bind(circuit_id)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?
.get(0);
// Get the site index
let site_id_db = sqlx::query("SELECT index FROM site_tree WHERE key = $1 AND site_name=$2")
.bind(key)
.bind(site_name)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let mut site_id: i32 = site_id_db
.try_get("index")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
// Get the parent list
while site_id != 0 {
let parent_db = sqlx::query(
"SELECT site_name, parent, site_type FROM site_tree WHERE key = $1 AND index=$2",
)
.bind(key)
.bind(site_id)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let parent: String = parent_db
.try_get("site_name")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
let site_type: String = parent_db
.try_get("site_type")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
site_id = parent_db
.try_get("parent")
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
result.push((site_type, parent));
}
Ok(result)
}
#[derive(Debug, FromRow)]
pub struct SiteOversubscription {
pub dlmax: i64,
pub dlmin: i64,
pub devicecount: i64,
}
pub async fn get_oversubscription(cnn: &Pool<Postgres>, key: &str, site_name: &str) -> Result<SiteOversubscription, StatsHostError> {
let site_id = get_site_id_from_name(cnn, key, site_name).await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
const SQL: &str = "WITH RECURSIVE children
(index, site_name, level, parent) AS (
SELECT index, site_name, 0, parent FROM site_tree WHERE key=$1 and index = $2
UNION ALL
SELECT
st.index,
st.site_name,
children.level + 1,
children.parent
FROM site_tree st, children
WHERE children.index = st.parent AND children.level < 5 AND key=$3
),
devices (circuit_id, download_max_mbps, download_min_mbps) AS (
SELECT DISTINCT
circuit_id,
download_max_mbps,
download_min_mbps
FROM shaped_devices WHERE key=$4
AND parent_node IN (SELECT site_name FROM children)
AND circuit_name NOT LIKE '%(site)'
)
SELECT
SUM(download_max_mbps) AS dlmax,
SUM(download_min_mbps) AS dlmin,
COUNT(circuit_id) AS devicecount
FROM devices;";
let rows = sqlx::query_as::<_, SiteOversubscription>(SQL)
.bind(key)
.bind(site_id)
.bind(key)
.bind(key)
.fetch_one(cnn)
.await
.map_err(|e| StatsHostError::DatabaseError(e.to_string()))?;
Ok(rows)
}

20
src/rust/long_term_stats/site_build/README.md
View File

@@ -1,20 +0,0 @@
# Site Build
This folder compiles and packages the website used by `lts_node`. It
needs to be compiled and made available to the `lts_node` process.
Steps: TBA
## Requirements
To run the build (as opposed to shipping pre-built files), you need to
install `esbuild` and `npm` (ugh). You can do this with:
```bash
(change directory to site_build folder)
sudo apt-get install npm
npm install
````
You can run the build manually by running `./esbuild.sh` in this
directory.

13
src/rust/long_term_stats/site_build/esbuild.mjs
View File

@@ -1,13 +0,0 @@
#!/usr/bin/env node
import * as esbuild from 'esbuild'
await esbuild.build({
entryPoints: ['src/app.ts', 'src/style.css'],
bundle: true,
minify: true,
sourcemap: true,
// target: ['chrome58', 'firefox57', 'safari11', 'edge16'],
outdir: 'output/',
loader: { '.html': 'text'},
format: 'esm',
})

8
src/rust/long_term_stats/site_build/package.json
View File

@@ -1,8 +0,0 @@
{
"dependencies": {
"@types/echarts": "^4.9.17",
"echarts": "^5.4.2",
"esbuild": "^0.17.17",
"mermaid": "^10.1.0"
}
}

60
src/rust/long_term_stats/site_build/src/ap/ap.ts
View File

@@ -1,60 +0,0 @@
import html from './template.html';
import { Page } from '../page'
import { MenuPage } from '../menu/menu';
import { Component } from '../components/component';
import { ThroughputSiteChart } from '../components/throughput_site';
import { SiteInfo } from '../components/site_info';
import { RttChartSite } from '../components/rtt_site';
import { RttHistoSite } from '../components/rtt_histo_site';
import { SiteBreadcrumbs } from '../components/site_breadcrumbs';
import { SiteHeat } from '../components/site_heat';
import { SiteStackChart } from '../components/site_stack';
import { request_ext_device_info } from '../../wasm/wasm_pipe';
export class AccessPointPage implements Page {
menu: MenuPage;
components: Component[];
siteId: string;
constructor(siteId: string) {
this.siteId = siteId;
this.menu = new MenuPage("sitetreeDash");
let container = document.getElementById('mainContent');
if (container) {
container.innerHTML = html;
}
this.components = [
new SiteInfo(siteId),
new SiteBreadcrumbs(siteId),
new ThroughputSiteChart(siteId),
new RttChartSite(siteId, 1.0),
new SiteHeat(siteId),
new SiteStackChart(siteId),
];
}
wireup() {
this.components.forEach(component => {
component.wireup();
});
}
ontick(): void {
this.menu.ontick();
this.components.forEach(component => {
component.ontick();
});
}
onmessage(event: any) {
if (event.msg) {
this.menu.onmessage(event);
this.components.forEach(component => {
component.onmessage(event);
});
}
}
}

62
src/rust/long_term_stats/site_build/src/ap/template.html
View File

@@ -1,62 +0,0 @@
<div class="container">
<div class="row">
<div class="col-12" id="siteName">
</div>
</div>
<div class="row">
<div class="col-6">
<div class="card">
<div class="card-body" id="siteInfo">
Details
</div>
</div>
</div>
<div class="col-6">
<div class="card">
<div class="card-body">
<div id="oversub"></div>
</div>
</div>
</div>
</div>
<div id="ext"></div>
<div class="row">
<div class="col-6">
<div class="card">
<div class="card-body">
<div id="throughputChart" style="height: 250px"></div>
</div>
</div>
</div>
<div class="col-6">
<div class="card">
<div class="card-body">
<div id="rttChart" style="height: 250px"></div>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-12">
<div class="card">
<div class="card-body">
<div id="siteStack" style="height: 250px"></div>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-12">
<div class="card">
<div class="card-body">
<div id="rootHeat" style="height: 900px"></div>
</div>
</div>
</div>
</div>
</div>

69
src/rust/long_term_stats/site_build/src/app.ts
View File

@@ -1,69 +0,0 @@
//import 'bootstrap/dist/css/bootstrap.css';
//import 'bootstrap/dist/js/bootstrap.js';
import { SiteRouter } from './router';
import { Bus, onAuthFail, onAuthOk, onMessage } from './bus';
import { Auth } from './auth';
import init from '../wasm/wasm_pipe.js';
await init();
console.log("WASM loaded");
declare global {
interface Window {
router: SiteRouter;
bus: Bus;
auth: Auth;
login: any;
graphPeriod: string;
changeGraphPeriod: any;
toggleThroughput: any;
toggleLatency: any;
}
}
(window as any).onAuthFail = onAuthFail;
(window as any).onAuthOk = onAuthOk;
(window as any).onMessage = onMessage;
window.auth = new Auth;
window.bus = new Bus();
window.router = new SiteRouter();
//window.bus.connect();
window.router.initialRoute();
let graphPeriod = localStorage.getItem('graphPeriod');
if (!graphPeriod) {
graphPeriod = "5m";
localStorage.setItem('graphPeriod', graphPeriod);
}
window.graphPeriod = graphPeriod;
window.changeGraphPeriod = (period: string) => changeGraphPeriod(period);
// 10 Second interval for refreshing the page
window.setInterval(() => {
window.bus.updateConnected();
window.router.ontick();
}, 10000);
// Faster interval for tracking the WSS connection
window.setInterval(() => {
updateDisplayedInterval();
window.bus.updateConnected();
try {
window.bus.sendQueue();
} catch (e) {
//console.log("Error sending queue: " + e);
}
}, 500);
function updateDisplayedInterval() {
let btn = document.getElementById("graphPeriodBtn") as HTMLButtonElement | null;
if (!btn) {
return;
}
btn.innerText = window.graphPeriod;
}
function changeGraphPeriod(period: string) {
window.graphPeriod = period;
localStorage.setItem('graphPeriod', period);
updateDisplayedInterval();
}

15
src/rust/long_term_stats/site_build/src/auth.ts
View File

@@ -1,15 +0,0 @@
export class Auth {
hasCredentials: boolean;
token: string | undefined;
constructor() {
let token = localStorage.getItem("token");
if (token) {
this.hasCredentials = true;
this.token = token;
} else {
this.hasCredentials = false;
this.token = undefined;
}
}
}

152
src/rust/long_term_stats/site_build/src/bus.ts
View File

@@ -1,152 +0,0 @@
import { is_wasm_connected, send_wss_queue, initialize_wss } from "../wasm/wasm_pipe";
import { Auth } from "./auth";
import { SiteRouter } from "./router";
export class Bus {
ws: WebSocket;
constructor() {
const currentUrlWithoutAnchors = window.location.href.split('#')[0].replace("https://", "").replace("http://", "");
const url = "ws://" + currentUrlWithoutAnchors + "ws";
initialize_wss(url);
}
updateConnected() {
//console.log("Connection via WASM: " + is_wasm_connected());
let indicator = document.getElementById("connStatus");
if (indicator && is_wasm_connected()) {
indicator.style.color = "green";
// Clear the loader
let loader = document.getElementById('SpinLoad');
if (loader) {
loader.style.display = "none";
}
} else if (indicator) {
indicator.style.color = "red";
}
}
sendQueue() {
send_wss_queue();
}
getToken(): string {
if (window.auth.hasCredentials && window.auth.token) {
return window.auth.token;
} else {
return "";
}
}
requestThroughputChartCircuit(circuit_id: string) {
let request = {
msg: "throughputChartCircuit",
period: window.graphPeriod,
circuit_id: decodeURI(circuit_id),
};
let json = JSON.stringify(request);
this.ws.send(json);
}
requestThroughputChartSite(site_id: string) {
let request = {
msg: "throughputChartSite",
period: window.graphPeriod,
site_id: decodeURI(site_id),
};
let json = JSON.stringify(request);
this.ws.send(json);
}
requestRttChartSite(site_id: string) {
let request = {
msg: "rttChartSite",
period: window.graphPeriod,
site_id: decodeURI(site_id),
};
let json = JSON.stringify(request);
this.ws.send(json);
}
requestRttChartCircuit(circuit_id: string) {
let request = {
msg: "rttChartCircuit",
period: window.graphPeriod,
circuit_id: decodeURI(circuit_id),
};
let json = JSON.stringify(request);
this.ws.send(json);
}
requestSiteHeat(site_id: string) {
let request = {
msg: "siteHeat",
period: window.graphPeriod,
site_id: decodeURI(site_id),
};
let json = JSON.stringify(request);
this.ws.send(json);
}
sendSearch(term: string) {
let request = {
msg: "search",
term: term,
};
let json = JSON.stringify(request);
this.ws.send(json);
}
requestSiteInfo(site_id: string) {
let request = {
msg: "siteInfo",
site_id: decodeURI(site_id),
};
let json = JSON.stringify(request);
this.ws.send(json);
}
requestCircuitInfo(circuit_id: string) {
let request = {
msg: "circuitInfo",
circuit_id: decodeURI(circuit_id),
};
let json = JSON.stringify(request);
this.ws.send(json);
}
requestSiteParents(site_id: string) {
let request = {
msg: "siteParents",
site_id: decodeURI(site_id),
};
let json = JSON.stringify(request);
this.ws.send(json);
}
}
// WASM callback
export function onAuthFail() {
window.auth.hasCredentials = false;
window.login = null;
window.auth.token = null;
localStorage.removeItem("token");
window.router.goto("login");
}
// WASM callback
export function onAuthOk(token: string, name: string, license_key: string) {
window.auth.hasCredentials = true;
window.login = { msg: "authOk", token: token, name: name, license_key: license_key };
window.auth.token = token;
}
// WASM Callback
export function onMessage(rawJson: string) {
let json = JSON.parse(rawJson);
//console.log(json);
//console.log(Object.keys(json));
json.msg = Object.keys(json)[0];
window.router.onMessage(json);
}

245
src/rust/long_term_stats/site_build/src/circuit/circuit.ts
View File

@@ -1,245 +0,0 @@
import html from './template.html';
import { Page } from '../page'
import { MenuPage } from '../menu/menu';
import { Component } from '../components/component';
import { CircuitInfo } from '../components/circuit_info';
import { ThroughputCircuitChart } from '../components/throughput_circuit';
import { RttChartCircuit } from '../components/rtt_circuit';
import { request_ext_device_info, request_ext_snr_graph, request_ext_capacity_graph } from "../../wasm/wasm_pipe";
import * as echarts from 'echarts';
import { scaleNumber } from '../helpers';
import { CircuitBreadcrumbs } from '../components/circuit_breadcrumbs';
export class CircuitPage implements Page {
menu: MenuPage;
components: Component[];
circuitId: string;
constructor(circuitId: string) {
this.circuitId = circuitId;
this.menu = new MenuPage("sitetreeDash");
let container = document.getElementById('mainContent');
if (container) {
container.innerHTML = html;
}
this.components = [
new CircuitInfo(this.circuitId),
new ThroughputCircuitChart(this.circuitId),
new RttChartCircuit(this.circuitId),
new CircuitBreadcrumbs(this.circuitId),
];
}
wireup() {
this.components.forEach(component => {
component.wireup();
});
request_ext_device_info(this.circuitId);
}
ontick(): void {
this.menu.ontick();
this.components.forEach(component => {
component.ontick();
});
}
onmessage(event: any) {
if (event.msg) {
this.menu.onmessage(event);
this.components.forEach(component => {
component.onmessage(event);
});
if (event.msg == "DeviceExt") {
//console.log(event.DeviceExt.data);
let div = document.getElementById("ext") as HTMLDivElement;
let html = "";
for (let i=0; i<event.DeviceExt.data.length; i++) {
let d = event.DeviceExt.data[i];
html += "<div class='row'>";
let name = d.name;
name += " (" + formatStatus(d.status);
name += ", " + d.model;
name += ", " + d.mode + " mode";
name += ", " + d.firmware;
name += ")";
html += "<div class='col-12'>";
html += "<div class='card'>";
html += "<div class='card-body'>";
html += "<h5 class='card-title'><i class='fa fa-wifi'></i> " + name + "</h5>";
console.log(d);
html += "<table class='table table-striped'>";
for (let j=0; j<d.interfaces.length; j++) {
html += "<tr>";
let iface = d.interfaces[j];
html += "<td>" + iface.name + "</td>";
html += "<td>" + iface.mac + "</td>";
html += "<td>" + iface.ip_list + "</td>";
html += "<td>" + iface.status + "</td>";
html += "<td>" + iface.speed + "</td>";
html += "</tr>";
}
html += "</table>";
html += "<div class='card-body' id='extdev_" + d.device_id + "' style='height: 250px'></div>";
html += "<div class='card-body' id='extdev_cap_" + d.device_id + "' style='height: 250px'></div>";
request_ext_snr_graph(window.graphPeriod, d.device_id);
request_ext_capacity_graph(window.graphPeriod, d.device_id);
/*
html += "</div>";
html += "</div>";
html += "</div>";
html += "</div><div class='row'>";
html += "<div class='col-4'>";
html += "<div class='card'>";
html += "<div class='card-body' id='extdev_" + d.device_id + "' style='height: 250px'>";
html += "<p>Signal/noise graph</p>";
html += "</div>";
html += "</div>";
html += "</div>";
request_ext_snr_graph(window.graphPeriod, d.device_id);
html += "<div class='col-4'>";
html += "<div class='card'>";
html += "<div class='card-body' id='extdev_cap_" + d.device_id + "' style='height: 250px'>";
html += "<p>Capacity Graph</p>";
html += "</div>";
html += "</div>";
html += "</div>";
request_ext_capacity_graph(window.graphPeriod, d.device_id);
// End row
html += "</div>";*/
}
div.outerHTML = html;
} else if (event.msg == "DeviceExtSnr") {
//console.log(event);
let div = document.getElementById("extdev_" + event.DeviceExtSnr.device_id) as HTMLDivElement;
let sig: number[] = [];
let n: number[] = [];
let x: any[] = [];
for (let i=0; i<event.DeviceExtSnr.data.length; i++) {
let d = event.DeviceExtSnr.data[i];
sig.push(d.signal);
n.push(d.noise);
x.push(d.date);
}
let series: echarts.SeriesOption[] = [];
let signal: echarts.SeriesOption = {
name: "Signal",
type: "line",
data: sig,
symbol: 'none',
};
let noise: echarts.SeriesOption = {
name: "Noise",
type: "line",
data: n,
symbol: 'none',
};
series.push(signal);
series.push(noise);
let myChart: echarts.ECharts = echarts.init(div);
var option: echarts.EChartsOption;
myChart.setOption<echarts.EChartsOption>(
(option = {
title: { text: "Signal/Noise" },
legend: {
orient: "horizontal",
right: 10,
top: "bottom",
},
xAxis: {
type: 'category',
data: x,
},
yAxis: {
type: 'value',
name: 'dB',
},
series: series
})
);
option && myChart.setOption(option);
} else if (event.msg == "DeviceExtCapacity") {
//console.log(event);
let div = document.getElementById("extdev_cap_" + event.DeviceExtCapacity.device_id) as HTMLDivElement;
let down: number[] = [];
let up: number[] = [];
let x: any[] = [];
for (let i=0; i<event.DeviceExtCapacity.data.length; i++) {
let d = event.DeviceExtCapacity.data[i];
down.push(d.dl);
up.push(d.ul);
x.push(d.date);
}
let series: echarts.SeriesOption[] = [];
let signal: echarts.SeriesOption = {
name: "Download",
type: "line",
data: down,
symbol: 'none',
};
let noise: echarts.SeriesOption = {
name: "Upload",
type: "line",
data: up,
symbol: 'none',
};
series.push(signal);
series.push(noise);
let myChart: echarts.ECharts = echarts.init(div);
var option: echarts.EChartsOption;
myChart.setOption<echarts.EChartsOption>(
(option = {
title: { text: "Estimated Capacity" },
legend: {
orient: "horizontal",
right: 10,
top: "bottom",
},
xAxis: {
type: 'category',
data: x,
},
yAxis: {
type: 'value',
name: 'Mbps',
axisLabel: {
formatter: function (val: number) {
return scaleNumber(Math.abs(val), 0);
}
}
},
series: series
})
);
option && myChart.setOption(option);
}
}
}
}
function formatStatus(status: String): String {
switch (status) {
case "active": return "<i class='fa fa-plug' style='color: green'></i> Active";
case "disconnected" : return "<i class='fa fa-times' style='color: red'></i> Disconnected";
default: return status;
}
}

40
src/rust/long_term_stats/site_build/src/circuit/template.html
View File

@@ -1,40 +0,0 @@
<div class="container">
<div class="row">
<div class="col-12" id="siteName">
</div>
</div>
<div class="row">
<div class="col-6">
<div class="card">
<div class="card-body" id="circuitInfo">
</div>
</div>
</div>
<div class="col-6">
<div class="card">
<div class="card-body" id="circuitDevices">
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-6">
<div class="card">
<div class="card-body">
<div id="throughputChart" style="height: 250px"></div>
</div>
</div>
</div>
<div class="col-6">
<div class="card">
<div class="card-body">
<div id="rttChart" style="height: 250px"></div>
</div>
</div>
</div>
</div>
<div id="ext"></div>
</div>

33
src/rust/long_term_stats/site_build/src/components/circuit_breadcrumbs.ts
View File

@@ -1,33 +0,0 @@
import { request_circuit_parents } from "../../wasm/wasm_pipe";
import { makeUrl } from "../helpers";
import { Component } from "./component";
export class CircuitBreadcrumbs implements Component {
circuitId: string;
constructor(siteId: string) {
this.circuitId = siteId;
}
wireup(): void {
request_circuit_parents(this.circuitId);
}
ontick(): void {
}
onmessage(event: any): void {
if (event.msg == "SiteParents") {
//console.log(event.data);
let div = document.getElementById("siteName") as HTMLDivElement;
let html = "";
let crumbs = event.SiteParents.data.reverse();
for (let i = 0; i < crumbs.length; i++) {
let url = makeUrl(crumbs[i][0], crumbs[i][1]);
html += "<a href='#" + url + "' onclick='window.router.goto(\"" + url + "\")'>" + crumbs[i][1] + "</a> | ";
}
html = html.substring(0, html.length - 3);
div.innerHTML = html;
}
}
}

50
src/rust/long_term_stats/site_build/src/components/circuit_info.ts
View File

@@ -1,50 +0,0 @@
import { scaleNumber } from "../helpers";
import { mbps_to_bps } from "../site_tree/site_tree";
import { Component } from "./component";
import { request_circuit_info } from "../../wasm/wasm_pipe";
export class CircuitInfo implements Component {
circuitId: string;
count: number = 0;
constructor(siteId: string) {
this.circuitId = decodeURI(siteId);
}
wireup(): void {
request_circuit_info(this.circuitId);
}
ontick(): void {
this.count++;
if (this.count % 10 == 0) {
request_circuit_info(this.circuitId);
}
}
onmessage(event: any): void {
if (event.msg == "CircuitInfo") {
//console.log(event.CircuitInfo.data);
let div = document.getElementById("circuitInfo") as HTMLDivElement;
let html = "";
html += "<table class='table table-striped'>";
html += "<tr><td>Circuit Name:</td><td>" + event.CircuitInfo.data[0].circuit_name + "</td></tr>";
html += "<tr><td>Min (CIR) Limits:</td><td>" + event.CircuitInfo.data[0].download_min_mbps + " / " + event.CircuitInfo.data[0].upload_min_mbps + " Mbps</td></tr>";
html += "<tr><td>Max (Ceiling) Limits:</td><td>" + event.CircuitInfo.data[0].download_max_mbps + " / " + event.CircuitInfo.data[0].upload_max_mbps + " Mbps</td></tr>";
html += "</table>";
div.innerHTML = html;
div = document.getElementById("circuitDevices") as HTMLDivElement;
html = "";
html += "<table class='table table-striped'>";
for (let i=0; i<event.CircuitInfo.data.length; i++) {
html += "<tr>";
html += "<td>Device:</td><td>" + event.CircuitInfo.data[i].device_name + "</td>";
html += "<td>IP:</td><td>" + event.CircuitInfo.data[i].ip_range + "/" + event.CircuitInfo.data[i].subnet + "</td>";
html += "</tr>";
}
html += "</table>";
div.innerHTML = html;
}
}
}

5
src/rust/long_term_stats/site_build/src/components/component.ts
View File

@@ -1,5 +0,0 @@
export interface Component {
wireup(): void;
ontick(): void;
onmessage(event: any): void;
}

97
src/rust/long_term_stats/site_build/src/components/node_cpu.ts
View File

@@ -1,97 +0,0 @@
import { scaleNumber } from "../helpers";
import { Component } from "./component";
import * as echarts from 'echarts';
export class NodeCpuChart implements Component {
div: HTMLElement;
myChart: echarts.ECharts;
chartMade: boolean = false;
node_id: string;
node_name: string;
constructor(node_id: string, node_name: string) {
this.node_id = node_id;
this.node_name = node_name;
this.div = document.getElementById("cpuChart") as HTMLElement;
this.myChart = echarts.init(this.div);
this.myChart.showLoading();
}
wireup(): void {
}
ontick(): void {
// Requested by the RAM chart
}
onmessage(event: any): void {
if (event.msg == "NodePerfChart") {
let series: echarts.SeriesOption[] = [];
// Iterate all provides nodes and create a set of series for each,
// providing upload and download banding per node.
let x: any[] = [];
let first = true;
let legend: string[] = [];
for (let i=0; i<event.NodePerfChart.nodes.length; i++) {
let node = event.NodePerfChart.nodes[i];
legend.push(node.node_name + " CPU %");
legend.push(node.node_name + " Single Core Peak");
//console.log(node);
let cpu: number[] = [];
let cpu_max: number[] = [];
for (let j=0; j<node.stats.length; j++) {
if (first) x.push(node.stats[j].date);
cpu.push(node.stats[j].cpu);
cpu_max.push(node.stats[j].cpu_max);
}
if (first) first = false;
let val: echarts.SeriesOption = {
name: node.node_name + " CPU %",
type: "line",
data: cpu,
symbol: 'none',
};
let val2: echarts.SeriesOption = {
name: node.node_name + " Single Core Peak",
type: "line",
data: cpu_max,
symbol: 'none',
};
series.push(val);
series.push(val2);
}
if (!this.chartMade) {
this.myChart.hideLoading();
var option: echarts.EChartsOption;
this.myChart.setOption<echarts.EChartsOption>(
(option = {
title: { text: "CPU Usage" },
tooltip: { trigger: "axis" },
legend: {
orient: "horizontal",
right: 10,
top: "bottom",
data: legend,
},
xAxis: {
type: 'category',
data: x,
},
yAxis: {
type: 'value',
name: '%',
},
series: series
})
);
option && this.myChart.setOption(option);
// this.chartMade = true;
}
}
}
}

36
src/rust/long_term_stats/site_build/src/components/node_list.ts
View File

@@ -1,36 +0,0 @@
import { Component } from "./component";
import { request_node_status } from "../../wasm/wasm_pipe";
export class NodeList implements Component {
wireup(): void {
}
ontick(): void {
request_node_status();
}
onmessage(event: any): void {
if (event.msg == "NodeStatus") {
let status = document.getElementById("nodeList");
let html = "";
if (status) {
html += "<table class='table table-striped'>";
html += "<thead>";
html += "<th>Node ID</th><th>Node Name</th><th>Last Seen</th>";
html += "</thead><tbody>";
for (let i = 0; i < event.NodeStatus.nodes.length; i++) {
let node = event.NodeStatus.nodes[i];
let url = "\"shaperNode:" + node.node_id + ":" + node.node_name.replace(':', '_') + "\"";
let oc = "onclick='window.router.goto(" + url + ")'";
html += "<tr>";
html += "<td><span " + oc + ">" + node.node_id + "</span></td>";
html += "<td><span " + oc + ">" + node.node_name + "</span></td>";
html += "<td><span " + oc + ">" + node.last_seen + " seconds ago</span></td>";
}
html += "</tbody></table>";
status.innerHTML = html;
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More