Add Criterion-based benchmarks to lqosd

* Add a secondary build option (as library) to lqosd, by including
  a `lib.rs` file that mirrors `main.rs`.
* Add the boilerplate required for a Criterion-based benchmark
  setup.
* Add support for `cargo bench` launching a set of tests that
  benchmark deserializing TC queues, and benchmarks retrieving
  queue statistics from `tc`.
This commit is contained in:
Herbert Wolverson 2023-01-17 14:05:28 +00:00
parent 6403ad21e4
commit 1a0b20aa11
12 changed files with 485 additions and 13 deletions

191
src/rust/Cargo.lock generated
View File

@ -247,6 +247,12 @@ dependencies = [
"serde",
]
[[package]]
name = "bumpalo"
version = "3.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
[[package]]
name = "byteorder"
version = "1.4.3"
@ -265,6 +271,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.0.77"
@ -417,6 +429,42 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "criterion"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f"
dependencies = [
"atty",
"cast",
"clap 2.34.0",
"criterion-plot",
"csv",
"itertools",
"lazy_static",
"num-traits",
"oorandom",
"plotters",
"rayon",
"regex",
"serde",
"serde_cbor",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.6"
@ -867,6 +915,12 @@ dependencies = [
"tracing",
]
[[package]]
name = "half"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
[[package]]
name = "hashbrown"
version = "0.12.3"
@ -1087,6 +1141,15 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "itertools"
version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "0.4.8"
@ -1099,6 +1162,15 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
[[package]]
name = "js-sys"
version = "0.3.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "kqueue"
version = "1.0.7"
@ -1271,6 +1343,7 @@ name = "lqosd"
version = "0.1.0"
dependencies = [
"anyhow",
"criterion",
"env_logger",
"lazy_static",
"log",
@ -1499,6 +1572,12 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
[[package]]
name = "oorandom"
version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "opaque-debug"
version = "0.3.0"
@ -1624,6 +1703,34 @@ version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]]
name = "plotters"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97"
dependencies = [
"num-traits",
"plotters-backend",
"plotters-svg",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "plotters-backend"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
[[package]]
name = "plotters-svg"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f"
dependencies = [
"plotters-backend",
]
[[package]]
name = "polyval"
version = "0.6.0"
@ -2062,6 +2169,16 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde_cbor"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
dependencies = [
"half",
"serde",
]
[[package]]
name = "serde_derive"
version = "1.0.148"
@ -2339,6 +2456,16 @@ dependencies = [
"time-core",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "tokio"
version = "1.22.0"
@ -2612,6 +2739,70 @@ version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
dependencies = [
"cfg-if 1.0.0",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
[[package]]
name = "web-sys"
version = "0.3.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webusers"
version = "0.1.0"

View File

@ -23,3 +23,10 @@ env_logger = "0"
log = "0"
nix = "0"
rayon = "1"
[dev-dependencies]
criterion = { version = "0.3", features = [ "html_reports"] }
[[bench]]
name = "json"
harness = false

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,102 @@
//! Benchmarks for JSON serialization and gathering data from TC.
//! Please select an interface in `test_interface.txt` (no enter character
//! at the end). This benchmark will destructively clear and then create
//! TC queues - so don't select an interface that you need!
use criterion::{criterion_group, criterion_main, Criterion, black_box};
use lqosd::*;
use std::process::Command;
const EXAMPLE_JSON: &str = include_str!("./example_json.txt");
const TC: &str = "/sbin/tc";
const SUDO: &str = "/bin/sudo";
fn clear_queues(interface: &str) {
Command::new(SUDO)
.args([TC, "qdisc", "delete", "dev", interface, "root"])
.output()
.unwrap();
}
fn setup_mq(interface: &str) {
Command::new(SUDO)
.args([TC, "qdisc", "replace", "dev", interface, "root", "handle", "7FFF:", "mq"])
.output()
.unwrap();
}
fn setup_parent_htb(interface: &str) {
Command::new(SUDO)
.args([TC, "qdisc", "add", "dev", interface, "parent", "7FFF:0x1", "handle", "0x1:", "htb", "default", "2"])
.output()
.unwrap();
Command::new(SUDO)
.args([TC, "class", "add", "dev", interface, "parent", "0x1:", "classid", "0x1:1", "htb", "rate", "10000mbit", "ceil", "10000mbit"])
.output()
.unwrap();
Command::new(SUDO)
.args([TC, "qdisc", "add", "dev", interface, "parent", "0x1:1", "cake", "diffserv4"])
.output()
.unwrap();
}
fn add_client_pair(interface: &str, queue_number: u32) {
let class_id = format!("0x1:{:x}", queue_number);
Command::new(SUDO)
.args([TC, "class", "add", "dev", interface, "parent", "0x1:1", "classid", &class_id, "htb", "rate", "2500mbit", "ceil", "9999mbit", "prio", "5"])
.output()
.unwrap();
Command::new(SUDO)
.args([TC, "qdisc", "add", "dev", interface, "parent", &class_id, "cake", "diffserv4"])
.output()
.unwrap();
}
pub fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("deserialize_cake", |b| {
b.iter(|| {
deserialize_tc_tree(EXAMPLE_JSON).unwrap();
});
});
const INTERFACE: &str = include_str!("test_interface.txt");
const QUEUE_COUNTS: [u32; 4] = [10, 100, 1000, 2000];
for queue_count in QUEUE_COUNTS.iter() {
let no_stdbuf = format!("NO-STBUF, {queue_count} queues: tc qdisc show -s -j");
let stdbuf = format!("STBUF -i1024, {queue_count} queues: tc qdisc show -s -j");
clear_queues(INTERFACE);
setup_mq(INTERFACE);
setup_parent_htb(INTERFACE);
for i in 0 .. *queue_count {
let queue_handle = (i+1) * 2;
add_client_pair(INTERFACE, queue_handle);
}
c.bench_function(&no_stdbuf, |b| {
b.iter(|| {
let command_output = Command::new("/sbin/tc")
.args(["-s", "-j", "qdisc", "show", "dev", "eth1"])
.output().unwrap();
let json = String::from_utf8(command_output.stdout).unwrap();
black_box(json);
});
});
c.bench_function(&stdbuf, |b| {
b.iter(|| {
let command_output = Command::new("/usr/bin/stdbuf")
.args(["-i0", "-o1024M", "-e0", TC, "-s", "-j", "qdisc", "show", "dev", "eth1"])
.output().unwrap();
let json = String::from_utf8(command_output.stdout).unwrap();
black_box(json);
});
});
}
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);

View File

@ -0,0 +1 @@
eth1

163
src/rust/lqosd/src/lib.rs Normal file
View File

@ -0,0 +1,163 @@
//! Provides a secondary option to build this crate as a library,
//! utilized by the benchmarking system.
//!
//! This file includes "main.rs"'s contents, but isn't used. The objective
//! is to ensure that it uses the same codepoints as `main.rs` - to avoid
//! confusing the unused code warning system.
// Export for the benchmark program
pub use queue_tracker::deserialize_tc_tree;
mod ip_mapping;
mod libreqos_tracker;
#[cfg(feature = "equinix_tests")]
mod lqos_daht_test;
mod tuning;
mod program_control;
mod queue_tracker;
mod throughput_tracker;
use crate::{ip_mapping::{clear_ip_flows, del_ip_flow, list_mapped_ips, map_ip_to_flow}};
use anyhow::Result;
use log::{info, warn};
use lqos_bus::{
cookie_value, decode_request, encode_response, BusReply, BusRequest, BUS_BIND_ADDRESS,
};
use lqos_config::LibreQoSConfig;
use lqos_sys::LibreQoSKernels;
use signal_hook::{consts::{SIGINT, SIGHUP, SIGTERM }, iterator::Signals};
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
join,
net::{TcpListener, TcpStream}
};
//#[tokio::main]
pub async fn lib_main() -> Result<()> {
// Configure log level with RUST_LOG environment variable,
// defaulting to "info"
env_logger::init_from_env(
env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "warn")
);
info!("LibreQoS Daemon Starting");
let config = LibreQoSConfig::load()?;
tuning::tune_lqosd_from_config_file(&config)?;
// Start the XDP/TC kernels
let kernels = if config.on_a_stick_mode {
LibreQoSKernels::on_a_stick_mode(
&config.internet_interface,
config.stick_vlans.1,
config.stick_vlans.0,
)?
} else {
LibreQoSKernels::new(&config.internet_interface, &config.isp_interface)?
};
// Spawn tracking sub-systems
join!(
throughput_tracker::spawn_throughput_monitor(),
queue_tracker::spawn_queue_monitor(),
//libreqos_tracker::spawn_shaped_devices_monitor(),
libreqos_tracker::spawn_queue_structure_monitor(),
);
// Handle signals
let mut signals = Signals::new(&[SIGINT, SIGHUP, SIGTERM ])?;
std::thread::spawn(move || {
for sig in signals.forever() {
match sig {
SIGINT | SIGTERM => {
match sig {
SIGINT => warn!("Terminating on SIGINT"),
SIGTERM => warn!("Terminating on SIGTERM"),
_ => warn!("This should never happen - terminating on unknown signal"),
}
std::mem::drop(kernels);
std::process::exit(0);
}
SIGHUP => {
warn!("Reloading configuration because of SIGHUP");
if let Ok(config) = LibreQoSConfig::load() {
let result = tuning::tune_lqosd_from_config_file(&config);
match result {
Err(err) => { warn!("Unable to HUP tunables: {:?}", err) },
Ok(..) => {}
}
} else {
warn!("Unable to reload configuration");
}
}
_ => warn!("No handler for signal: {sig}"),
}
}
});
// Main bus listen loop
let listener = TcpListener::bind(BUS_BIND_ADDRESS).await?;
warn!("Listening on: {}", BUS_BIND_ADDRESS);
loop {
let (mut socket, _) = listener.accept().await?;
tokio::spawn(async move {
let mut buf = vec![0; 1024];
let _ = socket
.read(&mut buf)
.await
.expect("failed to read data from socket");
if let Ok(request) = decode_request(&buf) {
if request.auth_cookie == cookie_value() {
let mut response = BusReply {
auth_cookie: request.auth_cookie,
responses: Vec::new(),
};
for req in request.requests.iter() {
//println!("Request: {:?}", req);
response.responses.push(match req {
BusRequest::Ping => lqos_bus::BusResponse::Ack,
BusRequest::GetCurrentThroughput => {
throughput_tracker::current_throughput()
}
BusRequest::GetHostCounter => throughput_tracker::host_counters(),
BusRequest::GetTopNDownloaders(n) => throughput_tracker::top_n(*n),
BusRequest::GetWorstRtt(n) => throughput_tracker::worst_n(*n),
BusRequest::MapIpToFlow {
ip_address,
tc_handle,
cpu,
upload,
} => map_ip_to_flow(ip_address, tc_handle, *cpu, *upload),
BusRequest::DelIpFlow { ip_address, upload } => {
del_ip_flow(&ip_address, *upload)
}
BusRequest::ClearIpFlow => clear_ip_flows(),
BusRequest::ListIpFlow => list_mapped_ips(),
BusRequest::XdpPping => throughput_tracker::xdp_pping_compat(),
BusRequest::RttHistogram => throughput_tracker::rtt_histogram(),
BusRequest::HostCounts => throughput_tracker::host_counts(),
BusRequest::AllUnknownIps => throughput_tracker::all_unknown_ips(),
BusRequest::ReloadLibreQoS => program_control::reload_libre_qos(),
BusRequest::GetRawQueueData(circuit_id) => {
queue_tracker::get_raw_circuit_data(&circuit_id)
}
BusRequest::UpdateLqosDTuning(..) => {
tuning::tune_lqosd_from_bus(&req).await
}
#[cfg(feature = "equinix_tests")]
BusRequest::RequestLqosEquinixTest => {
lqos_daht_test::lqos_daht_test().await
}
});
}
//println!("{:?}", response);
let _ = reply(&encode_response(&response).unwrap(), &mut socket).await;
}
}
});
}
}
async fn reply(response: &[u8], socket: &mut TcpStream) -> Result<()> {
socket.write_all(&response).await?;
Ok(())
}

View File

@ -12,6 +12,7 @@ mod queue_reader;
use lazy_static::*;
use parking_lot::RwLock;
use anyhow::Result;
pub use queue_reader::deserialize_tc_tree;
const NUM_QUEUE_HISTORY: usize = 600;

View File

@ -11,7 +11,7 @@ pub use queue_diff::QueueDiff;
use tokio::process::Command;
#[derive(Debug, Clone, Serialize)]
pub(crate) enum QueueType {
pub enum QueueType {
Mq(tc_mq::TcMultiQueue),
Htb(tc_htb::TcHtb),
FqCodel(tc_fq_codel::TcFqCodel),
@ -32,14 +32,10 @@ impl QueueType {
}
}
pub(crate) async fn read_tc_queues(interface: &str) -> Result<Vec<QueueType>> {
/// Separated into a separate function for cleaner benchmark code
pub fn deserialize_tc_tree(json: &str) -> Result<Vec<QueueType>> {
let mut result = Vec::new();
let command_output = Command::new("/sbin/tc")
.args(["-s", "-j", "qdisc", "show", "dev", interface])
.output()
.await?;
let json = String::from_utf8(command_output.stdout)?;
let json: Value = serde_json::from_str(&json)?;
let json: Value = serde_json::from_str(json)?;
if let Value::Array(array) = &json {
for entry in array.iter() {
match entry {
@ -60,3 +56,13 @@ pub(crate) async fn read_tc_queues(interface: &str) -> Result<Vec<QueueType>> {
Ok(result)
}
pub(crate) async fn read_tc_queues(interface: &str) -> Result<Vec<QueueType>> {
let command_output = Command::new("/sbin/tc")
.args(["-s", "-j", "qdisc", "show", "dev", interface])
.output()
.await?;
let json = String::from_utf8(command_output.stdout)?;
let result = deserialize_tc_tree(&json)?;
Ok(result)
}

View File

@ -133,7 +133,7 @@ use serde::Serialize;
use serde_json::Value;
#[derive(Default, Clone, Debug, Serialize)]
pub(crate) struct TcCake {
pub struct TcCake {
pub(crate) handle: TcHandle,
pub(crate) parent: TcHandle,
options: TcCakeOptions,
@ -172,7 +172,7 @@ struct TcCakeOptions {
}
#[derive(Default, Clone, Debug, Serialize)]
pub(crate) struct TcCakeTin {
pub struct TcCakeTin {
threshold_rate: u64,
pub(crate) sent_bytes: u64,
pub(crate) backlog_bytes: u32,

View File

@ -11,7 +11,7 @@ use serde::Serialize;
use serde_json::Value;
#[derive(Default, Clone, Debug, Serialize)]
pub(crate) struct TcFqCodel {
pub struct TcFqCodel {
handle: TcHandle,
pub(crate) parent: TcHandle,
options: TcFqCodelOptions,

View File

@ -9,7 +9,7 @@ use serde::Serialize;
use serde_json::Value;
#[derive(Default, Clone, Debug, Serialize)]
pub(crate) struct TcHtb {
pub struct TcHtb {
handle: TcHandle,
parent: TcHandle,
options: TcHtbOptions,

View File

@ -8,7 +8,7 @@ use serde::Serialize;
use serde_json::Value;
#[derive(Default, Clone, Debug, Serialize)]
pub(crate) struct TcMultiQueue {
pub struct TcMultiQueue {
handle: TcHandle,
root: bool,
bytes: u64,