Proof of concept with a tiny webserver

2025-01-05 04:04:56 -06:00 · 2023-03-17 19:48:30 +00:00 · 2023-03-17 19:48:30 +00:00 · 26efcdb208
commit 26efcdb208
parent b19b8b356b
7 changed files with 219 additions and 177 deletions
--- a/.gitignore
+++ b/.gitignore
@ -53,6 +53,7 @@ src/liblqos_python.so
 src/webusers.toml
 src/lqusers.toml
 src/dist
+src/rust/lqos_anonymous_stats_server/anonymous.sqlite

 # Ignore Rust build artifacts
 src/rust/target
--- a/src/rust/Cargo.lock
+++ b/src/rust/Cargo.lock
@ -67,15 +67,6 @@ dependencies = [
 "alloc-no-stdlib",
 ]

-[[package]]
-name = "android_system_properties"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "anes"
 version = "0.1.6"
@ -340,21 +331,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

-[[package]]
-name = "chrono"
-version = "0.4.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b"
-dependencies = [
- "iana-time-zone",
- "js-sys",
- "num-integer",
- "num-traits",
- "time 0.1.45",
- "wasm-bindgen",
- "winapi",
-]
-
 [[package]]
 name = "ciborium"
 version = "0.2.0"
@ -461,16 +437,6 @@ dependencies = [
 "os_str_bytes",
 ]

-[[package]]
-name = "codespan-reporting"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
-dependencies = [
- "termcolor",
- "unicode-width",
-]
-
 [[package]]
 name = "colored"
 version = "2.0.0"
@ -496,7 +462,7 @@ dependencies = [
 "rand",
 "sha2",
 "subtle",
- "time 0.3.20",
+ "time",
 "version_check",
 ]

@ -698,50 +664,6 @@ dependencies = [
 "cipher",
 ]

-[[package]]
-name = "cxx"
-version = "1.0.92"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72"
-dependencies = [
- "cc",
- "cxxbridge-flags",
- "cxxbridge-macro",
- "link-cplusplus",
-]
-
-[[package]]
-name = "cxx-build"
-version = "1.0.92"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613"
-dependencies = [
- "cc",
- "codespan-reporting",
- "once_cell",
- "proc-macro2",
- "quote",
- "scratch",
- "syn",
-]
-
-[[package]]
-name = "cxxbridge-flags"
-version = "1.0.92"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97"
-
-[[package]]
-name = "cxxbridge-macro"
-version = "1.0.92"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "dashmap"
 version = "5.4.0"
@ -1040,7 +962,7 @@ checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
 dependencies = [
 "cfg-if",
 "libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi",
 ]

 [[package]]
@ -1202,30 +1124,6 @@ dependencies = [
 "want",
 ]

-[[package]]
-name = "iana-time-zone"
-version = "0.1.53"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
-dependencies = [
- "android_system_properties",
- "core-foundation-sys",
- "iana-time-zone-haiku",
- "js-sys",
- "wasm-bindgen",
- "winapi",
-]
-
-[[package]]
-name = "iana-time-zone-haiku"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
-dependencies = [
- "cxx",
- "cxx-build",
-]
-
 [[package]]
 name = "indexmap"
 version = "1.9.2"
@ -1433,15 +1331,6 @@ dependencies = [
 "winapi",
 ]

-[[package]]
-name = "link-cplusplus"
-version = "1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.1.4"
@ -1497,7 +1386,6 @@ version = "0.1.0"
 dependencies = [
 "anyhow",
 "axum",
- "chrono",
 "env_logger",
 "log",
 "lqos_bus",
@ -1745,7 +1633,7 @@ checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
 dependencies = [
 "libc",
 "log",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi",
 "windows-sys 0.45.0",
 ]

@ -1828,16 +1716,6 @@ dependencies = [
 "winapi",
 ]

-[[package]]
-name = "num-integer"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
-dependencies = [
- "autocfg",
- "num-traits",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.15"
@ -2310,7 +2188,7 @@ dependencies = [
 "serde_json",
 "state",
 "tempfile",
- "time 0.3.20",
+ "time",
 "tokio",
 "tokio-stream",
 "tokio-util",
@ -2371,7 +2249,7 @@ dependencies = [
 "smallvec",
 "stable-pattern",
 "state",
- "time 0.3.20",
+ "time",
 "tokio",
 "uncased",
 "uuid",
@ -2430,12 +2308,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

-[[package]]
-name = "scratch"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1"
-
 [[package]]
 name = "serde"
 version = "1.0.153"
@ -2777,17 +2649,6 @@ dependencies = [
 "once_cell",
 ]

-[[package]]
-name = "time"
-version = "0.1.45"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
-dependencies = [
- "libc",
- "wasi 0.10.0+wasi-snapshot-preview1",
- "winapi",
-]
-
 [[package]]
 name = "time"
 version = "0.3.20"
@ -3143,12 +3004,6 @@ dependencies = [
 "try-lock",
 ]

-[[package]]
-name = "wasi"
-version = "0.10.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
-
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
--- a/src/rust/lqos_anonymous_stats_server/Cargo.toml
+++ b/src/rust/lqos_anonymous_stats_server/Cargo.toml
@ -11,6 +11,5 @@ log = "0"
 lqos_bus = { path = "../lqos_bus" }
 serde = { version = "1.0", features = ["derive"] }
 serde_cbor = "0" # For RFC8949/7409 format C binary objects
-sqlite = "0.30.4"
-chrono = "0.4.24"
+sqlite = "0.30"
 axum = "0.6"
--- a/src/rust/lqos_anonymous_stats_server/src/db.rs
+++ b/src/rust/lqos_anonymous_stats_server/src/db.rs
@ -1,13 +1,12 @@
 use std::{path::Path, sync::atomic::AtomicI64, time::SystemTime};
 use lqos_bus::anonymous::AnonymousUsageV1;
 use sqlite::Value;
-use chrono::prelude::{DateTime, Utc};
 const DBPATH: &str = "anonymous.sqlite";

 const SETUP_QUERY: &str = 
 "CREATE TABLE submissions (
    id INTEGER PRIMARY KEY,
-    date TEXT,
+    date INTEGER,
    node_id TEXT,
    ip_address TEXT,
    git_hash TEXT,
@ -100,12 +99,6 @@ VALUES (
    :parent, :description, :product, :vendor, :clock, :capacity
 );";

-fn iso8601(st: std::time::SystemTime) -> String {
-    let dt: DateTime<Utc> = st.into();
-    format!("{}", dt.format("%+"))
-    // formats like "2001-07-08T00:34:60.026490+09:30"
-}
-
 fn bool_to_n(x: bool) -> i64 {
    if x {
        1
@ -114,8 +107,15 @@ fn bool_to_n(x: bool) -> i64 {
    }
 }

+fn get_sys_time_in_secs() -> u64 {
+    match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) {
+        Ok(n) => n.as_secs(),
+        Err(_) => panic!("SystemTime before UNIX EPOCH!"),
+    }
+}
+
 pub fn insert_stats_dump(stats: &AnonymousUsageV1, ip: &str) -> anyhow::Result<()> {
-    let date = iso8601(SystemTime::now());
+    let date = get_sys_time_in_secs() as i64;
    let new_id = SUBMISSION_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    let cn = sqlite::open(DBPATH)?;
    let mut statement = cn.prepare(INSERT_STATS)?;
@ -175,4 +175,85 @@ pub fn dump_all_to_string() -> anyhow::Result<String> {
        true
    }).unwrap();
    Ok(result)
+}
+
+pub fn count_unique_node_ids() -> anyhow::Result<u64> {
+    let mut result = 0;
+    let cn = sqlite::open(DBPATH)?;
+    cn.iterate("SELECT COUNT(DISTINCT node_id) AS count FROM submissions;", |pairs| {
+        for &(_name, value) in pairs.iter() {
+            if let Some(val) = value {
+                if let Ok(val) = val.parse::<u64>() {
+                    result = val;
+                }
+            }
+        }
+        true
+    }).unwrap();
+    Ok(result)
+}
+
+pub fn count_unique_node_ids_this_week() -> anyhow::Result<u64> {
+    let mut result = 0;
+    let cn = sqlite::open(DBPATH)?;
+    let last_week = (get_sys_time_in_secs() - 604800).to_string();
+    cn.iterate(format!("SELECT COUNT(DISTINCT node_id) AS count FROM submissions WHERE date > {last_week};"), |pairs| {
+        for &(_name, value) in pairs.iter() {
+            if let Some(val) = value {
+                if let Ok(val) = val.parse::<u64>() {
+                    result = val;
+                }
+            }
+        }
+        true
+    }).unwrap();
+    Ok(result)
+}
+
+pub fn shaped_devices() -> anyhow::Result<u64> {
+    let mut result = 0;
+    let cn = sqlite::open(DBPATH)?;
+    cn.iterate("SELECT SUM(shaped_device_count) AS total FROM (SELECT DISTINCT node_id, shaped_device_count FROM submissions);", |pairs| {
+        for &(_name, value) in pairs.iter() {
+            if let Some(val) = value {
+                if let Ok(val) = val.parse::<u64>() {
+                    result = val;
+                }
+            }
+        }
+        true
+    }).unwrap();
+    Ok(result)
+}
+
+pub fn net_json_nodes() -> anyhow::Result<u64> {
+    let mut result = 0;
+    let cn = sqlite::open(DBPATH)?;
+    cn.iterate("SELECT SUM(net_json_len) AS total FROM (SELECT DISTINCT node_id, net_json_len FROM submissions);", |pairs| {
+        for &(_name, value) in pairs.iter() {
+            if let Some(val) = value {
+                if let Ok(val) = val.parse::<u64>() {
+                    result = val;
+                }
+            }
+        }
+        true
+    }).unwrap();
+    Ok(result)
+}
+
+pub fn bandwidth() -> anyhow::Result<u64> {
+    let mut result = 0;
+    let cn = sqlite::open(DBPATH)?;
+    cn.iterate("SELECT SUM(capacity_down) AS total FROM (SELECT DISTINCT node_id, capacity_down FROM submissions);", |pairs| {
+        for &(_name, value) in pairs.iter() {
+            if let Some(val) = value {
+                if let Ok(val) = val.parse::<u64>() {
+                    result = val;
+                }
+            }
+        }
+        true
+    }).unwrap();
+    Ok(result)
 }
--- a/src/rust/lqos_anonymous_stats_server/src/main.rs
+++ b/src/rust/lqos_anonymous_stats_server/src/main.rs
@ -1,22 +1,8 @@
 mod stats_server;
 mod db;
-use axum::{routing::get, Router};
-use db::dump_all_to_string;
+mod webserver;
 use tokio::spawn;

-async fn stats_viewer() -> anyhow::Result<()> {
-  let app = Router::new().route("/", get(handler));
-
-  axum::Server::bind(&"0.0.0.0:3000".parse().unwrap())
-      .serve(app.into_make_service())
-      .await?;
-
-  Ok(())
-}
-
-async fn handler() -> String {
-  dump_all_to_string().unwrap()
-}

 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
@ -29,7 +15,7 @@ async fn main() -> anyhow::Result<()> {
  db::create_if_not_exist();
  db::check_id();

-  spawn(stats_viewer());
+  spawn(webserver::stats_viewer());

  let _ = stats_server::gather_stats().await;
  Ok(())
--- a/src/rust/lqos_anonymous_stats_server/src/webserver/index.html
+++ b/src/rust/lqos_anonymous_stats_server/src/webserver/index.html
@ -0,0 +1,91 @@
+<!doctype html>
+<html lang="en">
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>LibreQoS Installation Statistics</title>
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/css/bootstrap.min.css" rel="stylesheet"
+    integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
+</head>
+
+<body style="margin: 8px;">
+  <header>
+    <div class="container py-3">
+      <div class="d-flex flex-column flex-md-row align-items-center pb-3 mb-4 border-bottom">
+        <a href="/" class="d-flex align-items-center text-dark text-decoration-none">
+          <svg xmlns="http://www.w3.org/2000/svg" width="40" height="32" class="me-2" viewBox="0 0 118 94" role="img">
+            <title>Bootstrap</title>
+            <path fill-rule="evenodd" clip-rule="evenodd"
+              d="M24.509 0c-6.733 0-11.715 5.893-11.492 12.284.214 6.14-.064 14.092-2.066 20.577C8.943 39.365 5.547 43.485 0 44.014v5.972c5.547.529 8.943 4.649 10.951 11.153 2.002 6.485 2.28 14.437 2.066 20.577C12.794 88.106 17.776 94 24.51 94H93.5c6.733 0 11.714-5.893 11.491-12.284-.214-6.14.064-14.092 2.066-20.577 2.009-6.504 5.396-10.624 10.943-11.153v-5.972c-5.547-.529-8.934-4.649-10.943-11.153-2.002-6.484-2.28-14.437-2.066-20.577C105.214 5.894 100.233 0 93.5 0H24.508zM80 57.863C80 66.663 73.436 72 62.543 72H44a2 2 0 01-2-2V24a2 2 0 012-2h18.437c9.083 0 15.044 4.92 15.044 12.474 0 5.302-4.01 10.049-9.119 10.88v.277C75.317 46.394 80 51.21 80 57.863zM60.521 28.34H49.948v14.934h8.905c6.884 0 10.68-2.772 10.68-7.727 0-4.643-3.264-7.207-9.012-7.207zM49.948 49.2v16.458H60.91c7.167 0 10.964-2.876 10.964-8.281 0-5.406-3.903-8.178-11.425-8.178H49.948z"
+              fill="currentColor"></path>
+          </svg>
+          <span class="fs-4">LibreQoS Installation Statistics</span>
+        </a>
+
+        <!--
+                  <nav class="d-inline-flex mt-2 mt-md-0 ms-md-auto">
+                    <a class="me-3 py-2 text-dark text-decoration-none" href="#">Features</a>
+                    <a class="me-3 py-2 text-dark text-decoration-none" href="#">Enterprise</a>
+                    <a class="me-3 py-2 text-dark text-decoration-none" href="#">Support</a>
+                    <a class="py-2 text-dark text-decoration-none" href="#">Pricing</a>
+                  </nav>
+                  -->
+      </div>
+
+      <div class="pricing-header p-3 pb-md-4 mx-auto text-center">
+        <h1 class="display-4 fw-normal">Installation Statistics</h1>
+        <p class="fs-5 text-muted">LibreQoS is fixing the Internet, one ISP at a time.</p>
+      </div>
+    </div>
+    </div>
+  </header>
+  <main>
+    <div class="row row-cols-1 row-cols-md-3 mb-3 text-center">
+      <div class="col">
+        <div class="card mb-4 rounded-3 shadow-sm">
+          <div class="card-header py-3">
+            <h4 class="my-0 fw-normal">Deployments</h4>
+          </div>
+          <div class="card-body">
+            $$UNIQUE_HOSTS$$<br />
+            $$NEW_HOSTS$$
+          </div>
+        </div>
+      </div>
+
+      <div class="col">
+        <div class="card mb-4 rounded-3 shadow-sm">
+          <div class="card-header py-3">
+            <h4 class="my-0 fw-normal">Connections Debloated</h4>
+          </div>
+          <div class="card-body">
+            $$TOTAL_SHAPED$$<br />
+            $$NODES$$<br />
+          </div>
+        </div>
+      </div>
+
+      <div class="col">
+        <div class="card mb-4 rounded-3 shadow-sm">
+          <div class="card-header py-3">
+            <h4 class="my-0 fw-normal">Bandwidth</h4>
+          </div>
+          <div class="card-body">
+            $$BW$$
+          </div>
+        </div>
+      </div>
+    </div>
+    </div>
+  </main>
+  <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/js/bootstrap.bundle.min.js"
+    integrity="sha384-kenU1KFdBIe4zVF0s0G1M5b4hcpxyD9F7jL+jjXkk+Q2h455rYXK/7HAuoJl+0I4"
+    crossorigin="anonymous"></script>
+
+  <script>
+
+  </script>
+</body>
+
+</html>
--- a/src/rust/lqos_anonymous_stats_server/src/webserver/mod.rs
+++ b/src/rust/lqos_anonymous_stats_server/src/webserver/mod.rs
@ -0,0 +1,29 @@
+use axum::{routing::get, Router, response::Html};
+
+use crate::db::{count_unique_node_ids, count_unique_node_ids_this_week, shaped_devices, net_json_nodes, bandwidth};
+
+pub async fn stats_viewer() -> anyhow::Result<()> {
+    let app = Router::new().route("/", get(index_page));
+  
+    log::info!("Listening for web traffic on 0.0.0.0:3000");
+    axum::Server::bind(&"0.0.0.0:3000".parse().unwrap())
+        .serve(app.into_make_service())
+        .await?;
+  
+    Ok(())
+  }
+  
+  async fn index_page() -> Html<String> {
+    let result = include_str!("./index.html");
+    let unique = count_unique_node_ids().unwrap_or(0);
+    let new = count_unique_node_ids_this_week().unwrap_or(0);
+    let total_shaped = shaped_devices().unwrap_or(0);
+    let net_json_nodes = net_json_nodes().unwrap_or(0);
+    let bw = bandwidth().unwrap_or(0) as f64 / 1024.0 / 1024.0;
+    let result = result.replace("$$UNIQUE_HOSTS$$", &format!("<strong>{unique}</strong> Total LQOS Installs"));
+    let result = result.replace("$$NEW_HOSTS$$", &format!("<strong>{new}</strong> New Installs This Week"));
+    let result = result.replace("$$TOTAL_SHAPED$$", &format!("<strong>{total_shaped}</strong> Shaped Devices"));
+    let result = result.replace("$$NODES$$", &format!("<strong>{net_json_nodes}</strong> Network Hierarchy Nodes"));
+    let result = result.replace("$$BW$$", &format!("<strong>{bw:.2}</strong> Tbits/s Monitored"));
+    Html(result)
+  }