Change build script to use compiled results of site_build in the web folder (should fix CI), use miniz_oxide to compress data before encryption (and decompress the encrypted result), reducing network traffic by a factor of 5.

src/rust/Cargo.lock

@@ -2106,6 +2106,7 @@ dependencies = [
  "log",
  "lqos_config",
  "lqos_utils",
+ "miniz_oxide",
  "num-traits",
  "once_cell",
  "serde",
@@ -2129,6 +2130,7 @@ dependencies = [
  "log",
  "lqos_config",
  "lts_client",
+ "miniz_oxide",
  "num-traits",
  "once_cell",
  "pgdb",

src/rust/long_term_stats/lts_client/Cargo.toml

@@ -18,3 +18,4 @@ bincode = "1"
 once_cell = "1"
 sysinfo = "0"
 num-traits = "0.2"
+miniz_oxide = "0.7.1"

src/rust/long_term_stats/lts_client/src/submission_queue/comm_channel/encode.rs

@@ -21,6 +21,9 @@ pub(crate) async fn encode_submission(submission: &LtsCommand) -> Result<Vec<u8>
     // Pack the submission body into bytes
     let payload_bytes = serde_cbor::to_vec(&submission).map_err(|_| QueueError::SendFail)?;
 
+    // TODO: Compress it?
+    let payload_bytes = miniz_oxide::deflate::compress_to_vec(&payload_bytes, 8);
+    
     // Encrypt it
     let remote_public = SERVER_PUBLIC_KEY.read().await.clone().unwrap();
     let my_private = KEYPAIR.read().await.secret_key.clone();

src/rust/long_term_stats/lts_client/src/submission_queue/comm_channel/mod.rs

@@ -18,7 +18,7 @@ pub(crate) async fn start_communication_channel(mut rx: Receiver<SenderChannelMe
         match rx.try_recv() {
             Ok(SenderChannelMessage::QueueReady) => {
                 // If not connected, see if we are allowed to connect and get a target
-                if !connected {
+                if !connected || stream.is_none() {
                     log::info!("Establishing LTS TCP channel.");
                     stream = connect_if_permitted().await;
                     if stream.is_some() {

src/rust/long_term_stats/lts_node/Cargo.toml

@@ -25,3 +25,4 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 tower = { version = "0.4", features = ["util"] }
 tower-http = { version = "0.4.0", features = ["fs", "trace"] }
 chrono = "0"
+miniz_oxide = "0.7.1"

src/rust/long_term_stats/lts_node/run.sh

@@ -2,4 +2,8 @@
 pushd ../site_build
 ./esbuild.mjs
 popd
+pushd web
+cp ../../site_build/output/* .
+cp ../../site_build/src/main.html .
+popd
 RUST_LOG=warn RUST_BACKTRACE=1 cargo run

src/rust/long_term_stats/lts_node/src/submissions/submission_server.rs

@@ -32,6 +32,7 @@ pub async fn submissions_server(
                     my_sender.send(message).await.unwrap();
                 } else {
                     log::error!("Read failed. Dropping socket.");
+                    std::mem::drop(socket);
                     break;
                 }
             }
@@ -91,6 +92,8 @@ async fn read_body(stream: &mut TcpStream, pool: Pool<Postgres>, size: usize, he
     let decrypted = dryocbox
         .decrypt_to_vec(&header.nonce.into(), &public_key, &private_key)?;
 
+    let decrypted = miniz_oxide::inflate::decompress_to_vec(&decrypted).expect("failed to decompress");
+
     // Try to deserialize
     let payload = lts_client::cbor::from_slice(&decrypted)?;
     Ok(payload)

src/rust/long_term_stats/lts_node/src/web/mod.rs

@@ -9,11 +9,11 @@ use pgdb::sqlx::Postgres;
 use tower_http::trace::TraceLayer;
 use tower_http::trace::DefaultMakeSpan;
 
-const JS_BUNDLE: &str = include_str!("../../../site_build/output/app.js");
-const JS_MAP: &str = include_str!("../../../site_build/output/app.js.map");
-const CSS: &str = include_str!("../../../site_build/output/style.css");
-const CSS_MAP: &str = include_str!("../../../site_build/output/style.css.map");
-const HTML_MAIN: &str = include_str!("../../../site_build/src/main.html");
+const JS_BUNDLE: &str = include_str!("../../web/app.js");
+const JS_MAP: &str = include_str!("../../web/app.js.map");
+const CSS: &str = include_str!("../../web/style.css");
+const CSS_MAP: &str = include_str!("../../web/style.css.map");
+const HTML_MAIN: &str = include_str!("../../web/main.html");
 
 pub async fn webserver(cnn: Pool<Postgres>) {
     let app = Router::new()

src/rust/long_term_stats/lts_node/web/README.md

@@ -0,0 +1,20 @@
+# Site Build
+
+This folder compiles and packages the website used by `lts_node`. It
+needs to be compiled and made available to the `lts_node` process.
+
+Steps: TBA
+
+## Requirements
+
+To run the build (as opposed to shipping pre-built files), you need to
+install `esbuild` and `npm` (ugh). You can do this with:
+
+```bash
+(change directory to site_build folder)
+sudo apt-get install npm
+npm install
+````
+
+You can run the build manually by running `./esbuild.sh` in this
+directory.

src/rust/long_term_stats/lts_node/web/main.html

@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta http-equiv="content-type" content="text/html; charset=utf-8" />
+    <title>LibreQoS Long-Term Statistics</title>
+    <link rel="shortcut icon" href="#" />
+
+    <script type="module" src="/app.js"></script>
+    <link rel="stylesheet" href="style.css" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css" />
+  </head>
+  <body>
+    <div id="main"></div>
+    <footer>Copyright &copy; 2023 LibreQoS</footer>
+  </body>
+</html>