From 7b0715756f396e43488cee4c21b114a00f34f612 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Wed, 11 Jan 2023 18:16:30 +0000 Subject: [PATCH] Real-time reloading of lqosd tunables and queue check frequency Extends the bus to include a "reload lqosd" command that reloads the tunables and queue frequency in real-time, without requiring a bridge restart. --- src/rust/Cargo.lock | 54 ++++------- src/rust/lqos_bus/Cargo.toml | 1 + src/rust/lqos_bus/src/bus/request.rs | 4 + src/rust/lqos_config/src/etc.rs | 2 +- .../lqos_node_manager/src/config_control.rs | 29 +++++- src/rust/lqos_node_manager/src/main.rs | 1 + src/rust/lqos_node_manager/static/config.html | 94 ++++++++++++++----- src/rust/lqosd/Cargo.toml | 1 + src/rust/lqosd/src/main.rs | 15 ++- src/rust/lqosd/src/queue_tracker/mod.rs | 12 ++- 10 files changed, 144 insertions(+), 69 deletions(-) diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index be90d57d..9afe53f3 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -205,15 +205,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.3" @@ -386,7 +377,7 @@ dependencies = [ "hmac", "percent-encoding", "rand", - "sha2 0.10.6", + "sha2", "subtle", "time", "version_check", @@ -408,6 +399,17 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "cow_struct" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780363852334e8717416de3477ac2ce707bdf91c086daae09b80f0b354b5a6b5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "cpufeatures" version = "0.2.5" @@ -607,22 +609,13 @@ dependencies = [ "syn", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ - "block-buffer 0.10.3", + "block-buffer", "crypto-common", "subtle", ] @@ -930,7 +923,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.6", + "digest", ] [[package]] @@ -1222,6 +1215,7 @@ dependencies = [ "anyhow", "bincode", "cc", + "lqos_config", "serde", ] @@ -1234,7 +1228,7 @@ dependencies = [ "ip_network", "ip_network_table", "serde", - "sha2 0.9.9", + "sha2", "toml", "uuid", ] @@ -1287,6 +1281,7 @@ name = "lqosd" version = "0.1.0" dependencies = [ "anyhow", + "cow_struct", "env_logger", "lazy_static", "log", @@ -2098,19 +2093,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "sha2" version = "0.10.6" @@ -2119,7 +2101,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.6", + "digest", ] [[package]] diff --git a/src/rust/lqos_bus/Cargo.toml b/src/rust/lqos_bus/Cargo.toml index f3c61e74..28b642d0 100644 --- a/src/rust/lqos_bus/Cargo.toml +++ b/src/rust/lqos_bus/Cargo.toml @@ -11,6 +11,7 @@ equinix_tests = [] serde = { version = "1.0", features = ["derive"] } bincode = "1" anyhow = "1" +lqos_config = { path = "../lqos_config" } [build-dependencies] cc = "1.0" diff --git a/src/rust/lqos_bus/src/bus/request.rs b/src/rust/lqos_bus/src/bus/request.rs index 58f80241..92357b15 100644 --- a/src/rust/lqos_bus/src/bus/request.rs +++ b/src/rust/lqos_bus/src/bus/request.rs @@ -1,3 +1,4 @@ +use lqos_config::Tunables; use serde::{Serialize, Deserialize}; use crate::TcHandle; @@ -83,6 +84,9 @@ pub enum BusRequest { /// Retrieve raw queue data for a given circuit ID. GetRawQueueData(String), // The string is the circuit ID + /// Requests a real-time adjustment of the `lqosd` tuning settings + UpdateLqosDTuning(u64, Tunables), + /// If running on Equinix (the `equinix_test` feature is enabled), /// display a "run bandwidht test" link. #[cfg(feature = "equinix_tests")] diff --git a/src/rust/lqos_config/src/etc.rs b/src/rust/lqos_config/src/etc.rs index c1fc66ac..de8b97ff 100644 --- a/src/rust/lqos_config/src/etc.rs +++ b/src/rust/lqos_config/src/etc.rs @@ -25,7 +25,7 @@ pub struct EtcLqos { /// Represents a set of `sysctl` and `ethtool` tweaks that may be /// applied (in place of the previous version's offload service) -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] pub struct Tunables { /// Should the `irq_balance` system service be stopped? pub stop_irq_balance: bool, diff --git a/src/rust/lqos_node_manager/src/config_control.rs b/src/rust/lqos_node_manager/src/config_control.rs index 196c628a..c66306ce 100644 --- a/src/rust/lqos_node_manager/src/config_control.rs +++ b/src/rust/lqos_node_manager/src/config_control.rs @@ -1,7 +1,8 @@ use crate::{auth_guard::AuthGuard, cache_control::NoCache}; use default_net::get_interfaces; -use lqos_config::{EtcLqos, LibreQoSConfig}; -use rocket::{fs::NamedFile, serde::json::Json}; +use lqos_bus::{BUS_BIND_ADDRESS, BusSession, BusRequest, encode_request, decode_response}; +use lqos_config::{EtcLqos, LibreQoSConfig, Tunables}; +use rocket::{fs::NamedFile, serde::json::Json, tokio::{net::TcpStream, io::{AsyncReadExt, AsyncWriteExt}}}; // Note that NoCache can be replaced with a cache option // once the design work is complete. @@ -43,3 +44,27 @@ pub async fn update_python_config(_auth: AuthGuard, config: Json config.save().unwrap(); Json("OK".to_string()) } + +#[post("/api/lqos_tuning/", data = "")] +pub async fn update_lqos_tuning(auth: AuthGuard, period: u64, tuning: Json) -> Json { + if auth != AuthGuard::Admin { + return Json("Error: Not authorized".to_string()); + } + + // Send the update to the server + let mut stream = TcpStream::connect(BUS_BIND_ADDRESS).await.unwrap(); + let test = BusSession { + auth_cookie: 1234, + requests: vec![BusRequest::UpdateLqosDTuning(period, (*tuning).clone())], + }; + let msg = encode_request(&test).unwrap(); + stream.write(&msg).await.unwrap(); + + // Receive reply + let mut buf = Vec::new(); + let _ = stream.read_to_end(&mut buf).await.unwrap(); + let _reply = decode_response(&buf).unwrap(); + // For now, ignore the reply. + + Json("OK".to_string()) +} \ No newline at end of file diff --git a/src/rust/lqos_node_manager/src/main.rs b/src/rust/lqos_node_manager/src/main.rs index b3d5c1ee..11d11b2a 100644 --- a/src/rust/lqos_node_manager/src/main.rs +++ b/src/rust/lqos_node_manager/src/main.rs @@ -58,6 +58,7 @@ fn rocket() -> _ { config_control::get_current_python_config, config_control::get_current_lqosd_config, config_control::update_python_config, + config_control::update_lqos_tuning, auth_guard::create_first_user, auth_guard::login, auth_guard::admin_check, diff --git a/src/rust/lqos_node_manager/static/config.html b/src/rust/lqos_node_manager/static/config.html index 114c3b1e..ed153b6f 100644 --- a/src/rust/lqos_node_manager/static/config.html +++ b/src/rust/lqos_node_manager/static/config.html @@ -58,9 +58,7 @@
@@ -127,7 +125,12 @@ -

Bifrost XDP-Accelerated Bridge

+ +

Bifrost XDP-Accelerated Bridge

+ + @@ -138,7 +141,7 @@ - + @@ -230,20 +233,6 @@

Server Settings

- - - - - - - + + + + + + +
- Queue Check Frequency (ms) - - -
Disabling actual shell commands stops LibreQoS from actually doing anything. Simulated @@ -292,6 +281,22 @@

These settings can drastically affect performance of your server, including rendering it non-functional.
+ Queue Check Frequency (ms) + + +
IRQ Balancing should generally be disabled.
@@ -345,6 +350,12 @@
+ + + Apply Tuning Settings
Spylnx Settings @@ -440,13 +451,13 @@ html += ""; for (let i=0; i"; - html += "" + buildNICList('bfOut_' + i, lqosd_config.bridge.interface_mapping[i].redirect_to) + ""; + html += "" + buildNICList('bfIn_' + i, lqosd_config.bridge.interface_mapping[i].name, true) + ""; + html += "" + buildNICList('bfOut_' + i, lqosd_config.bridge.interface_mapping[i].redirect_to, true) + ""; html += ""; - html += ""; - html += ""; + html += "" + buildNICList('bfvlanif_' + i, lqosd_config.bridge.vlan_mapping[i].parent, true) + ""; + html += ""; + html += ""; html += ""; } html += ""; @@ -494,6 +505,7 @@ // User management if (is_admin) { userManager(); + tuning(); } }); }); @@ -506,6 +518,34 @@ $("#userManager").html(html); } + function tuning() { + $("#btnApplyTuning").on('click', () => { + let period = Number($("#queuecheckms").val()); + let new_config = { + stop_irq_balance: $("#stopIrqBalance").prop('checked'), + netdev_budget_usecs: Number($("#netDevUsec").val()), + netdev_budget_packets: Number($("#netDevPackets").val()), + rx_usecs: Number($("#rxUsecs").val()), + tx_usecs: Number($("#txUsecs").val()), + disable_rxvlan: $("#disableRxVlan").prop('checked'), + disable_txvlan: $("#disableTxVlan").prop('checked'), + disable_offload: $("#disableOffloadList").val().split(' ') + }; + $.ajax({ + type: "POST", + url: "/api/lqos_tuning/" + period, + data: JSON.stringify(new_config), + success: (data) => { + if (data == "ERROR") { + alert("Unable to apply settings.") + } else { + alert("Settings Applied"); + } + } + }) + }); + } + function fillNicList(id, selected) { let select = $("#" + id); let html = ""; @@ -520,8 +560,10 @@ select.html(html); } - function buildNICList(id, selected) { - let html = " Result<()> { BusRequest::GetRawQueueData(circuit_id) => { queue_tracker::get_raw_circuit_data(&circuit_id) } + BusRequest::UpdateLqosDTuning(interval, tuning) => { + // Real-time tuning changes. Probably dangerous. + if let Ok(config) = LibreQoSConfig::load() { + if tuning.stop_irq_balance { + offloads::stop_irq_balance().await; + } + offloads::netdev_budget(tuning.netdev_budget_usecs, tuning.netdev_budget_packets).await; + offloads::ethtool_tweaks(&config.internet_interface, tuning).await; + offloads::ethtool_tweaks(&config.isp_interface, tuning).await; + } + QUEUE_MONITOR_INTERVAL.store(*interval, std::sync::atomic::Ordering::Relaxed); + lqos_bus::BusResponse::Ack + } #[cfg(feature = "equinix_tests")] BusRequest::RequestLqosEquinixTest => { lqos_daht_test::lqos_daht_test().await diff --git a/src/rust/lqosd/src/queue_tracker/mod.rs b/src/rust/lqosd/src/queue_tracker/mod.rs index ede6180d..a67a8739 100644 --- a/src/rust/lqosd/src/queue_tracker/mod.rs +++ b/src/rust/lqosd/src/queue_tracker/mod.rs @@ -5,7 +5,7 @@ use lqos_config::LibreQoSConfig; use serde::Serialize; use std::{ collections::HashMap, - time::{Duration, Instant}, + time::{Duration, Instant}, sync::atomic::AtomicU64, }; use tokio::{join, task, time}; mod queue_reader; @@ -154,11 +154,17 @@ async fn track_queues() { } } +lazy_static! { + pub(crate) static ref QUEUE_MONITOR_INTERVAL: AtomicU64 = AtomicU64::new(1000); +} + pub async fn spawn_queue_monitor() { let _ = task::spawn(async { - let queue_check_period_ms = lqos_config::EtcLqos::load().unwrap().queue_check_period_ms; - let mut interval = time::interval(Duration::from_millis(queue_check_period_ms)); + QUEUE_MONITOR_INTERVAL.store(lqos_config::EtcLqos::load().unwrap().queue_check_period_ms, std::sync::atomic::Ordering::Relaxed); loop { + let queue_check_period_ms = QUEUE_MONITOR_INTERVAL.load(std::sync::atomic::Ordering::Relaxed); + let mut interval = time::interval(Duration::from_millis(queue_check_period_ms)); + let now = Instant::now(); let _ = track_queues().await; let elapsed = now.elapsed();