IntenseWebs/LibreQoS
3
0
Fork 0
mirror of https://github.com/LibreQoE/LibreQoS.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

Truly minimal stats server, but it works.

Browse Source
This commit is contained in:
Herbert Wolverson
2023-03-17 18:19:47 +00:00
parent ef5ffa4e64
commit b19b8b356b
5 changed files with 430 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

290
src/rust/Cargo.lock generated
View File

@@ -67,6 +67,15 @@ dependencies = [
"alloc-no-stdlib",
]
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "anes"
version = "0.1.6"
@@ -152,6 +161,55 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "axum"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13d8068b6ccb8b34db9de397c7043f91db8b4c66414952c6db944f238c4d3db3"
dependencies = [
"async-trait",
"axum-core",
"bitflags",
"bytes",
"futures-util",
"http",
"http-body",
"hyper",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tower",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2f958c80c248b34b9a877a643811be8dbca03ca5ba827f2b63baf3a81e5fc4e"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"mime",
"rustversion",
"tower-layer",
"tower-service",
]
[[package]]
name = "base64"
version = "0.20.0"
@@ -282,6 +340,21 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b"
dependencies = [
"iana-time-zone",
"js-sys",
"num-integer",
"num-traits",
"time 0.1.45",
"wasm-bindgen",
"winapi",
]
[[package]]
name = "ciborium"
version = "0.2.0"
@@ -388,6 +461,16 @@ dependencies = [
"os_str_bytes",
]
[[package]]
name = "codespan-reporting"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
dependencies = [
"termcolor",
"unicode-width",
]
[[package]]
name = "colored"
version = "2.0.0"
@@ -413,7 +496,7 @@ dependencies = [
"rand",
"sha2",
"subtle",
"time",
"time 0.3.20",
"version_check",
]
@@ -615,6 +698,50 @@ dependencies = [
"cipher",
]
[[package]]
name = "cxx"
version = "1.0.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72"
dependencies = [
"cc",
"cxxbridge-flags",
"cxxbridge-macro",
"link-cplusplus",
]
[[package]]
name = "cxx-build"
version = "1.0.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613"
dependencies = [
"cc",
"codespan-reporting",
"once_cell",
"proc-macro2",
"quote",
"scratch",
"syn",
]
[[package]]
name = "cxxbridge-flags"
version = "1.0.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97"
[[package]]
name = "cxxbridge-macro"
version = "1.0.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "dashmap"
version = "5.4.0"
@@ -784,6 +911,15 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "form_urlencoded"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
dependencies = [
"percent-encoding",
]
[[package]]
name = "futures"
version = "0.3.26"
@@ -904,7 +1040,7 @@ checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
dependencies = [
"cfg-if",
"libc",
"wasi",
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
@@ -1066,6 +1202,30 @@ dependencies = [
"want",
]
[[package]]
name = "iana-time-zone"
version = "0.1.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"winapi",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
dependencies = [
"cxx",
"cxx-build",
]
[[package]]
name = "indexmap"
version = "1.9.2"
@@ -1273,6 +1433,15 @@ dependencies = [
"winapi",
]
[[package]]
name = "link-cplusplus"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
dependencies = [
"cc",
]
[[package]]
name = "linux-raw-sys"
version = "0.1.4"
@@ -1327,6 +1496,8 @@ name = "lqos_anonymous_stats_server"
version = "0.1.0"
dependencies = [
"anyhow",
"axum",
"chrono",
"env_logger",
"log",
"lqos_bus",
@@ -1509,6 +1680,12 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"
[[package]]
name = "memalloc"
version = "0.1.0"
@@ -1568,7 +1745,7 @@ checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
dependencies = [
"libc",
"log",
"wasi",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.45.0",
]
@@ -1651,6 +1828,16 @@ dependencies = [
"winapi",
]
[[package]]
name = "num-integer"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.15"
@@ -1764,6 +1951,26 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]]
name = "pin-project"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pin-project-lite"
version = "0.2.9"
@@ -2103,7 +2310,7 @@ dependencies = [
"serde_json",
"state",
"tempfile",
"time",
"time 0.3.20",
"tokio",
"tokio-stream",
"tokio-util",
@@ -2164,7 +2371,7 @@ dependencies = [
"smallvec",
"stable-pattern",
"state",
"time",
"time 0.3.20",
"tokio",
"uncased",
"uuid",
@@ -2223,6 +2430,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "scratch"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1"
[[package]]
name = "serde"
version = "1.0.153"
@@ -2264,6 +2477,15 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db0969fff533976baadd92e08b1d102c5a3d8a8049eadfd69d4d1e3c5b2ed189"
dependencies = [
"serde",
]
[[package]]
name = "serde_spanned"
version = "0.6.1"
@@ -2273,6 +2495,18 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
dependencies = [
"form_urlencoded",
"itoa",
"ryu",
"serde",
]
[[package]]
name = "sha2"
version = "0.10.6"
@@ -2437,6 +2671,12 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "sysinfo"
version = "0.28.2"
@@ -2537,6 +2777,17 @@ dependencies = [
"once_cell",
]
[[package]]
name = "time"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
dependencies = [
"libc",
"wasi 0.10.0+wasi-snapshot-preview1",
"winapi",
]
[[package]]
name = "time"
version = "0.3.20"
@@ -2673,6 +2924,28 @@ dependencies = [
"winnow",
]
[[package]]
name = "tower"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
dependencies = [
"futures-core",
"futures-util",
"pin-project",
"pin-project-lite",
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower-layer"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
[[package]]
name = "tower-service"
version = "0.3.2"
@@ -2686,6 +2959,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
dependencies = [
"cfg-if",
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@@ -2869,6 +3143,12 @@ dependencies = [
"try-lock",
]
[[package]]
name = "wasi"
version = "0.10.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"

2
src/rust/lqos_anonymous_stats_server/Cargo.toml
View File

@@ -12,3 +12,5 @@ lqos_bus = { path = "../lqos_bus" }
serde = { version = "1.0", features = ["derive"] }
serde_cbor = "0" # For RFC8949/7409 format C binary objects
sqlite = "0.30.4"
chrono = "0.4.24"
axum = "0.6"

121
src/rust/lqos_anonymous_stats_server/src/db.rs
View File

@@ -1,4 +1,7 @@
use std::path::Path;
use std::{path::Path, sync::atomic::AtomicI64, time::SystemTime};
use lqos_bus::anonymous::AnonymousUsageV1;
use sqlite::Value;
use chrono::prelude::{DateTime, Utc};
const DBPATH: &str = "anonymous.sqlite";
const SETUP_QUERY: &str =
@@ -43,6 +46,8 @@ CREATE TABLE nics (
);
";
static SUBMISSION_ID: AtomicI64 = AtomicI64::new(0);
pub fn create_if_not_exist() {
let path = Path::new(DBPATH);
if !path.exists() {
@@ -56,4 +61,118 @@ pub fn create_if_not_exist() {
panic!("Unable to create database");
}
}
}
pub fn check_id() {
log::info!("Checking primary keys");
if let Ok(cn) = sqlite::open(DBPATH) {
cn.iterate("SELECT MAX(id) as id FROM submissions;", |pairs| {
for &(name, value) in pairs.iter() {
if name == "id" {
if let Some(val) = value {
if let Ok(n) = val.parse::<i64>() {
log::info!("Last id: {n}");
SUBMISSION_ID.store(n+1, std::sync::atomic::Ordering::Relaxed);
}
}
}
}
true
}).unwrap();
} else {
panic!("Unable to connect to database");
}
}
const INSERT_STATS: &str =
"INSERT INTO submissions VALUES (
:id, :date, :node_id, :ip_address, :git_hash, :using_xdp_bridge, :on_a_stick,
:total_memory, :available_memory, :kernel_version, :distro, :usable_cores,
:cpu_brand, :cpu_vendor, :cpu_frequency, :sqm, :monitor_mode, :capcity_down,
:capacity_up, :genereated_pdn_down, :generated_pdn_up, :shaped_device_count,
:net_json_len
);";
const INSERT_NIC: &str =
"INSERT INTO nics
(parent, description, product, vendor, clock, capacity)
VALUES (
:parent, :description, :product, :vendor, :clock, :capacity
);";
fn iso8601(st: std::time::SystemTime) -> String {
let dt: DateTime<Utc> = st.into();
format!("{}", dt.format("%+"))
// formats like "2001-07-08T00:34:60.026490+09:30"
}
fn bool_to_n(x: bool) -> i64 {
if x {
1
} else {
0
}
}
pub fn insert_stats_dump(stats: &AnonymousUsageV1, ip: &str) -> anyhow::Result<()> {
let date = iso8601(SystemTime::now());
let new_id = SUBMISSION_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let cn = sqlite::open(DBPATH)?;
let mut statement = cn.prepare(INSERT_STATS)?;
statement.bind_iter::<_, (_, Value)>([
(":id", new_id.into()),
(":date", date.into()),
(":node_id", stats.node_id.clone().into()),
(":ip_address", ip.into()),
(":git_hash", stats.git_hash.clone().into()),
(":using_xdp_bridge", bool_to_n(stats.using_xdp_bridge).into()),
(":on_a_stick", bool_to_n(stats.on_a_stick).into()),
(":total_memory", (stats.total_memory as i64).into()),
(":available_memory", (stats.available_memory as i64).into()),
(":kernel_version", stats.kernel_version.clone().into()),
(":distro", stats.distro.clone().into()),
(":usable_cores", (stats.usable_cores as i64).into()),
(":cpu_brand", stats.cpu_brand.clone().into()),
(":cpu_vendor", stats.cpu_vendor.clone().into()),
(":cpu_frequency", (stats.cpu_frequency as i64).into()),
(":sqm", stats.sqm.clone().into()),
(":monitor_mode", bool_to_n(stats.monitor_mode).into()),
(":capcity_down", (stats.total_capacity.0 as i64).into()),
(":capacity_up", (stats.total_capacity.1 as i64).into()),
(":genereated_pdn_down", (stats.generated_pdn_capacity.0 as i64).into()),
(":generated_pdn_up", (stats.generated_pdn_capacity.1 as i64).into()),
(":shaped_device_count", (stats.shaped_device_count as i64).into()),
(":net_json_len", (stats.net_json_len as i64).into()),
])?;
statement.next()?;
for nic in stats.nics.iter() {
let mut statement = cn.prepare(INSERT_NIC)?;
statement.bind_iter::<_, (_, Value)>([
(":parent", new_id.into()),
(":description", nic.description.clone().into()),
(":product", nic.product.clone().into()),
(":vendor", nic.vendor.clone().into()),
(":clock", nic.clock.clone().into()),
(":capacity", nic.capacity.clone().into()),
])?;
statement.next()?;
}
log::info!("Submitted");
Ok(())
}
// Not a great idea, this is for test data
pub fn dump_all_to_string() -> anyhow::Result<String> {
let mut result = String::new();
let cn = sqlite::open(DBPATH)?;
cn.iterate("SELECT * FROM submissions;", |pairs| {
for &(name, value) in pairs.iter() {
result += &format!(";{name}={value:?}");
}
result += "\n";
true
}).unwrap();
Ok(result)
}

20
src/rust/lqos_anonymous_stats_server/src/main.rs
View File

@@ -1,5 +1,22 @@
mod stats_server;
mod db;
use axum::{routing::get, Router};
use db::dump_all_to_string;
use tokio::spawn;
async fn stats_viewer() -> anyhow::Result<()> {
let app = Router::new().route("/", get(handler));
axum::Server::bind(&"0.0.0.0:3000".parse().unwrap())
.serve(app.into_make_service())
.await?;
Ok(())
}
async fn handler() -> String {
dump_all_to_string().unwrap()
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
@@ -10,6 +27,9 @@ async fn main() -> anyhow::Result<()> {
);
db::create_if_not_exist();
db::check_id();
spawn(stats_viewer());
let _ = stats_server::gather_stats().await;
Ok(())

4
src/rust/lqos_anonymous_stats_server/src/stats_server.rs
View File

@@ -2,6 +2,8 @@ use lqos_bus::anonymous::AnonymousUsageV1;
use std::net::SocketAddr;
use tokio::{io::AsyncReadExt, net::TcpListener, spawn};
use crate::db::insert_stats_dump;
pub async fn gather_stats() -> anyhow::Result<()> {
let listener = TcpListener::bind(":::9125").await?;
log::info!("Listening on :::9125");
@@ -60,6 +62,6 @@ async fn store_stats_v1(
payload: &AnonymousUsageV1,
address: SocketAddr,
) -> anyhow::Result<()> {
println!("{payload:?} {address:?}");
insert_stats_dump(payload, &address.to_string())?;
Ok(())
}