From 1a0b20aa115d1bc530d776fef2e75308ace3dfdd Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Tue, 17 Jan 2023 14:05:28 +0000 Subject: [PATCH] Add Criterion-based benchmarks to lqosd * Add a secondary build option (as library) to lqosd, by including a `lib.rs` file that mirrors `main.rs`. * Add the boilerplate required for a Criterion-based benchmark setup. * Add support for `cargo bench` launching a set of tests that benchmark deserializing TC queues, and benchmarks retrieving queue statistics from `tc`. --- src/rust/Cargo.lock | 191 ++++++++++++++++++ src/rust/lqosd/Cargo.toml | 7 + src/rust/lqosd/benches/example_json.txt | 1 + src/rust/lqosd/benches/json.rs | 102 ++++++++++ src/rust/lqosd/benches/test_interface.txt | 1 + src/rust/lqosd/src/lib.rs | 163 +++++++++++++++ src/rust/lqosd/src/queue_tracker/mod.rs | 1 + .../src/queue_tracker/queue_reader/mod.rs | 22 +- .../src/queue_tracker/queue_reader/tc_cake.rs | 4 +- .../queue_tracker/queue_reader/tc_fq_codel.rs | 2 +- .../src/queue_tracker/queue_reader/tc_htb.rs | 2 +- .../src/queue_tracker/queue_reader/tc_mq.rs | 2 +- 12 files changed, 485 insertions(+), 13 deletions(-) create mode 100644 src/rust/lqosd/benches/example_json.txt create mode 100644 src/rust/lqosd/benches/json.rs create mode 100644 src/rust/lqosd/benches/test_interface.txt create mode 100644 src/rust/lqosd/src/lib.rs diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 6b209e20..2a721c9e 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -247,6 +247,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bumpalo" +version = "3.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" + [[package]] name = "byteorder" version = "1.4.3" @@ -265,6 +271,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.77" @@ -417,6 +429,42 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "criterion" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +dependencies = [ + "atty", + "cast", + "clap 2.34.0", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -867,6 +915,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.12.3" @@ -1087,6 +1141,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.8" @@ -1099,6 +1162,15 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kqueue" version = "1.0.7" @@ -1271,6 +1343,7 @@ name = "lqosd" version = "0.1.0" dependencies = [ "anyhow", + "criterion", "env_logger", "lazy_static", "log", @@ -1499,6 +1572,12 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -1624,6 +1703,34 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "plotters" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" + +[[package]] +name = "plotters-svg" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +dependencies = [ + "plotters-backend", +] + [[package]] name = "polyval" version = "0.6.0" @@ -2062,6 +2169,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.148" @@ -2339,6 +2456,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.22.0" @@ -2612,6 +2739,70 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webusers" version = "0.1.0" diff --git a/src/rust/lqosd/Cargo.toml b/src/rust/lqosd/Cargo.toml index 6dc33f87..5c85679e 100644 --- a/src/rust/lqosd/Cargo.toml +++ b/src/rust/lqosd/Cargo.toml @@ -23,3 +23,10 @@ env_logger = "0" log = "0" nix = "0" rayon = "1" + +[dev-dependencies] +criterion = { version = "0.3", features = [ "html_reports"] } + +[[bench]] +name = "json" +harness = false diff --git a/src/rust/lqosd/benches/example_json.txt b/src/rust/lqosd/benches/example_json.txt new file mode 100644 index 00000000..41cafe31 --- /dev/null +++ b/src/rust/lqosd/benches/example_json.txt @@ -0,0 +1 @@ +[{"kind":"mq","handle":"7fff:","root":true,"options":{},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0},{"kind":"htb","handle":"2:","parent":"7fff:2","options":{"r2q":10,"default":"0x2","direct_packets_stat":0,"direct_qlen":1000},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0},{"kind":"htb","handle":"4:","parent":"7fff:4","options":{"r2q":10,"default":"0x2","direct_packets_stat":0,"direct_qlen":1000},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0},{"kind":"cake","handle":"8011:","parent":"1:4","options":{"bandwidth":"unlimited","diffserv":"diffserv4","flowmode":"triple-isolate","nat":false,"wash":false,"ingress":false,"ack-filter":"disabled","split_gso":true,"rtt":100000,"raw":true,"overhead":0,"fwmark":"0"},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0,"memory_used":0,"memory_limit":15503360,"capacity_estimate":0,"min_network_size":65535,"max_network_size":0,"min_adj_size":65535,"max_adj_size":0,"avg_hdr_offset":0,"tins":[{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514}]},{"kind":"cake","handle":"8002:","parent":"1:2","options":{"bandwidth":"unlimited","diffserv":"diffserv4","flowmode":"triple-isolate","nat":false,"wash":false,"ingress":false,"ack-filter":"disabled","split_gso":true,"rtt":100000,"raw":true,"overhead":0,"fwmark":"0"},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0,"memory_used":0,"memory_limit":15503360,"capacity_estimate":0,"min_network_size":65535,"max_network_size":0,"min_adj_size":65535,"max_adj_size":0,"avg_hdr_offset":0,"tins":[{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514}]},{"kind":"htb","handle":"1:","parent":"7fff:1","options":{"r2q":10,"default":"0x2","direct_packets_stat":0,"direct_qlen":1000},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0},{"kind":"htb","handle":"3:","parent":"7fff:3","options":{"r2q":10,"default":"0x2","direct_packets_stat":0,"direct_qlen":1000},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0},{"kind":"cake","handle":"8004:","parent":"2:2","options":{"bandwidth":"unlimited","diffserv":"diffserv4","flowmode":"triple-isolate","nat":false,"wash":false,"ingress":false,"ack-filter":"disabled","split_gso":true,"rtt":100000,"raw":true,"overhead":0,"fwmark":"0"},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0,"memory_used":0,"memory_limit":15503360,"capacity_estimate":0,"min_network_size":65535,"max_network_size":0,"min_adj_size":65535,"max_adj_size":0,"avg_hdr_offset":0,"tins":[{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514}]},{"kind":"cake","handle":"8006:","parent":"3:2","options":{"bandwidth":"unlimited","diffserv":"diffserv4","flowmode":"triple-isolate","nat":false,"wash":false,"ingress":false,"ack-filter":"disabled","split_gso":true,"rtt":100000,"raw":true,"overhead":0,"fwmark":"0"},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0,"memory_used":0,"memory_limit":15503360,"capacity_estimate":0,"min_network_size":65535,"max_network_size":0,"min_adj_size":65535,"max_adj_size":0,"avg_hdr_offset":0,"tins":[{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514}]},{"kind":"cake","handle":"8008:","parent":"4:2","options":{"bandwidth":"unlimited","diffserv":"diffserv4","flowmode":"triple-isolate","nat":false,"wash":false,"ingress":false,"ack-filter":"disabled","split_gso":true,"rtt":100000,"raw":true,"overhead":0,"fwmark":"0"},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0,"memory_used":0,"memory_limit":15503360,"capacity_estimate":0,"min_network_size":65535,"max_network_size":0,"min_adj_size":65535,"max_adj_size":0,"avg_hdr_offset":0,"tins":[{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514},{"threshold_rate":0,"sent_bytes":0,"backlog_bytes":0,"target_us":5000,"interval_us":100000,"peak_delay_us":0,"avg_delay_us":0,"base_delay_us":0,"sent_packets":0,"way_indirect_hits":0,"way_misses":0,"way_collisions":0,"drops":0,"ecn_mark":0,"ack_drops":0,"sparse_flows":0,"bulk_flows":0,"unresponsive_flows":0,"max_pkt_len":0,"flow_quantum":1514}]},{"kind":"clsact","handle":"ffff:","parent":"ffff:fff1","options":{},"bytes":0,"packets":0,"drops":0,"overlimits":0,"requeues":0,"backlog":0,"qlen":0}] \ No newline at end of file diff --git a/src/rust/lqosd/benches/json.rs b/src/rust/lqosd/benches/json.rs new file mode 100644 index 00000000..6081487e --- /dev/null +++ b/src/rust/lqosd/benches/json.rs @@ -0,0 +1,102 @@ +//! Benchmarks for JSON serialization and gathering data from TC. +//! Please select an interface in `test_interface.txt` (no enter character +//! at the end). This benchmark will destructively clear and then create +//! TC queues - so don't select an interface that you need! + +use criterion::{criterion_group, criterion_main, Criterion, black_box}; +use lqosd::*; +use std::process::Command; + +const EXAMPLE_JSON: &str = include_str!("./example_json.txt"); +const TC: &str = "/sbin/tc"; +const SUDO: &str = "/bin/sudo"; + +fn clear_queues(interface: &str) { + Command::new(SUDO) + .args([TC, "qdisc", "delete", "dev", interface, "root"]) + .output() + .unwrap(); +} + +fn setup_mq(interface: &str) { + Command::new(SUDO) + .args([TC, "qdisc", "replace", "dev", interface, "root", "handle", "7FFF:", "mq"]) + .output() + .unwrap(); +} + +fn setup_parent_htb(interface: &str) { + Command::new(SUDO) + .args([TC, "qdisc", "add", "dev", interface, "parent", "7FFF:0x1", "handle", "0x1:", "htb", "default", "2"]) + .output() + .unwrap(); + + Command::new(SUDO) + .args([TC, "class", "add", "dev", interface, "parent", "0x1:", "classid", "0x1:1", "htb", "rate", "10000mbit", "ceil", "10000mbit"]) + .output() + .unwrap(); + + Command::new(SUDO) + .args([TC, "qdisc", "add", "dev", interface, "parent", "0x1:1", "cake", "diffserv4"]) + .output() + .unwrap(); +} + +fn add_client_pair(interface: &str, queue_number: u32) { + let class_id = format!("0x1:{:x}", queue_number); + Command::new(SUDO) + .args([TC, "class", "add", "dev", interface, "parent", "0x1:1", "classid", &class_id, "htb", "rate", "2500mbit", "ceil", "9999mbit", "prio", "5"]) + .output() + .unwrap(); + + Command::new(SUDO) + .args([TC, "qdisc", "add", "dev", interface, "parent", &class_id, "cake", "diffserv4"]) + .output() + .unwrap(); +} + +pub fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("deserialize_cake", |b| { + b.iter(|| { + deserialize_tc_tree(EXAMPLE_JSON).unwrap(); + }); + }); + + const INTERFACE: &str = include_str!("test_interface.txt"); + const QUEUE_COUNTS: [u32; 4] = [10, 100, 1000, 2000]; + for queue_count in QUEUE_COUNTS.iter() { + let no_stdbuf = format!("NO-STBUF, {queue_count} queues: tc qdisc show -s -j"); + let stdbuf = format!("STBUF -i1024, {queue_count} queues: tc qdisc show -s -j"); + + clear_queues(INTERFACE); + setup_mq(INTERFACE); + setup_parent_htb(INTERFACE); + for i in 0 .. *queue_count { + let queue_handle = (i+1) * 2; + add_client_pair(INTERFACE, queue_handle); + } + + c.bench_function(&no_stdbuf, |b| { + b.iter(|| { + let command_output = Command::new("/sbin/tc") + .args(["-s", "-j", "qdisc", "show", "dev", "eth1"]) + .output().unwrap(); + let json = String::from_utf8(command_output.stdout).unwrap(); + black_box(json); + }); + }); + + c.bench_function(&stdbuf, |b| { + b.iter(|| { + let command_output = Command::new("/usr/bin/stdbuf") + .args(["-i0", "-o1024M", "-e0", TC, "-s", "-j", "qdisc", "show", "dev", "eth1"]) + .output().unwrap(); + let json = String::from_utf8(command_output.stdout).unwrap(); + black_box(json); + }); + }); + } +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); \ No newline at end of file diff --git a/src/rust/lqosd/benches/test_interface.txt b/src/rust/lqosd/benches/test_interface.txt new file mode 100644 index 00000000..d0608be2 --- /dev/null +++ b/src/rust/lqosd/benches/test_interface.txt @@ -0,0 +1 @@ +eth1 \ No newline at end of file diff --git a/src/rust/lqosd/src/lib.rs b/src/rust/lqosd/src/lib.rs new file mode 100644 index 00000000..a71add2f --- /dev/null +++ b/src/rust/lqosd/src/lib.rs @@ -0,0 +1,163 @@ +//! Provides a secondary option to build this crate as a library, +//! utilized by the benchmarking system. +//! +//! This file includes "main.rs"'s contents, but isn't used. The objective +//! is to ensure that it uses the same codepoints as `main.rs` - to avoid +//! confusing the unused code warning system. + +// Export for the benchmark program +pub use queue_tracker::deserialize_tc_tree; + +mod ip_mapping; +mod libreqos_tracker; +#[cfg(feature = "equinix_tests")] +mod lqos_daht_test; +mod tuning; +mod program_control; +mod queue_tracker; +mod throughput_tracker; +use crate::{ip_mapping::{clear_ip_flows, del_ip_flow, list_mapped_ips, map_ip_to_flow}}; +use anyhow::Result; +use log::{info, warn}; +use lqos_bus::{ + cookie_value, decode_request, encode_response, BusReply, BusRequest, BUS_BIND_ADDRESS, +}; +use lqos_config::LibreQoSConfig; +use lqos_sys::LibreQoSKernels; +use signal_hook::{consts::{SIGINT, SIGHUP, SIGTERM }, iterator::Signals}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + join, + net::{TcpListener, TcpStream} +}; + +//#[tokio::main] +pub async fn lib_main() -> Result<()> { + // Configure log level with RUST_LOG environment variable, + // defaulting to "info" + env_logger::init_from_env( + env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "warn") + ); + info!("LibreQoS Daemon Starting"); + let config = LibreQoSConfig::load()?; + tuning::tune_lqosd_from_config_file(&config)?; + + // Start the XDP/TC kernels + let kernels = if config.on_a_stick_mode { + LibreQoSKernels::on_a_stick_mode( + &config.internet_interface, + config.stick_vlans.1, + config.stick_vlans.0, + )? + } else { + LibreQoSKernels::new(&config.internet_interface, &config.isp_interface)? + }; + + // Spawn tracking sub-systems + join!( + throughput_tracker::spawn_throughput_monitor(), + queue_tracker::spawn_queue_monitor(), + //libreqos_tracker::spawn_shaped_devices_monitor(), + libreqos_tracker::spawn_queue_structure_monitor(), + ); + + // Handle signals + let mut signals = Signals::new(&[SIGINT, SIGHUP, SIGTERM ])?; + std::thread::spawn(move || { + for sig in signals.forever() { + match sig { + SIGINT | SIGTERM => { + match sig { + SIGINT => warn!("Terminating on SIGINT"), + SIGTERM => warn!("Terminating on SIGTERM"), + _ => warn!("This should never happen - terminating on unknown signal"), + } + std::mem::drop(kernels); + std::process::exit(0); + } + SIGHUP => { + warn!("Reloading configuration because of SIGHUP"); + if let Ok(config) = LibreQoSConfig::load() { + let result = tuning::tune_lqosd_from_config_file(&config); + match result { + Err(err) => { warn!("Unable to HUP tunables: {:?}", err) }, + Ok(..) => {} + } + } else { + warn!("Unable to reload configuration"); + } + } + _ => warn!("No handler for signal: {sig}"), + } + } + }); + + // Main bus listen loop + let listener = TcpListener::bind(BUS_BIND_ADDRESS).await?; + warn!("Listening on: {}", BUS_BIND_ADDRESS); + loop { + let (mut socket, _) = listener.accept().await?; + tokio::spawn(async move { + let mut buf = vec![0; 1024]; + + let _ = socket + .read(&mut buf) + .await + .expect("failed to read data from socket"); + + if let Ok(request) = decode_request(&buf) { + if request.auth_cookie == cookie_value() { + let mut response = BusReply { + auth_cookie: request.auth_cookie, + responses: Vec::new(), + }; + for req in request.requests.iter() { + //println!("Request: {:?}", req); + response.responses.push(match req { + BusRequest::Ping => lqos_bus::BusResponse::Ack, + BusRequest::GetCurrentThroughput => { + throughput_tracker::current_throughput() + } + BusRequest::GetHostCounter => throughput_tracker::host_counters(), + BusRequest::GetTopNDownloaders(n) => throughput_tracker::top_n(*n), + BusRequest::GetWorstRtt(n) => throughput_tracker::worst_n(*n), + BusRequest::MapIpToFlow { + ip_address, + tc_handle, + cpu, + upload, + } => map_ip_to_flow(ip_address, tc_handle, *cpu, *upload), + BusRequest::DelIpFlow { ip_address, upload } => { + del_ip_flow(&ip_address, *upload) + } + BusRequest::ClearIpFlow => clear_ip_flows(), + BusRequest::ListIpFlow => list_mapped_ips(), + BusRequest::XdpPping => throughput_tracker::xdp_pping_compat(), + BusRequest::RttHistogram => throughput_tracker::rtt_histogram(), + BusRequest::HostCounts => throughput_tracker::host_counts(), + BusRequest::AllUnknownIps => throughput_tracker::all_unknown_ips(), + BusRequest::ReloadLibreQoS => program_control::reload_libre_qos(), + BusRequest::GetRawQueueData(circuit_id) => { + queue_tracker::get_raw_circuit_data(&circuit_id) + } + BusRequest::UpdateLqosDTuning(..) => { + tuning::tune_lqosd_from_bus(&req).await + } + #[cfg(feature = "equinix_tests")] + BusRequest::RequestLqosEquinixTest => { + lqos_daht_test::lqos_daht_test().await + } + }); + } + //println!("{:?}", response); + let _ = reply(&encode_response(&response).unwrap(), &mut socket).await; + } + } + }); + } +} + +async fn reply(response: &[u8], socket: &mut TcpStream) -> Result<()> { + socket.write_all(&response).await?; + Ok(()) +} diff --git a/src/rust/lqosd/src/queue_tracker/mod.rs b/src/rust/lqosd/src/queue_tracker/mod.rs index 7c7a6dd9..756c229e 100644 --- a/src/rust/lqosd/src/queue_tracker/mod.rs +++ b/src/rust/lqosd/src/queue_tracker/mod.rs @@ -12,6 +12,7 @@ mod queue_reader; use lazy_static::*; use parking_lot::RwLock; use anyhow::Result; +pub use queue_reader::deserialize_tc_tree; const NUM_QUEUE_HISTORY: usize = 600; diff --git a/src/rust/lqosd/src/queue_tracker/queue_reader/mod.rs b/src/rust/lqosd/src/queue_tracker/queue_reader/mod.rs index 50057297..cdd41aa7 100644 --- a/src/rust/lqosd/src/queue_tracker/queue_reader/mod.rs +++ b/src/rust/lqosd/src/queue_tracker/queue_reader/mod.rs @@ -11,7 +11,7 @@ pub use queue_diff::QueueDiff; use tokio::process::Command; #[derive(Debug, Clone, Serialize)] -pub(crate) enum QueueType { +pub enum QueueType { Mq(tc_mq::TcMultiQueue), Htb(tc_htb::TcHtb), FqCodel(tc_fq_codel::TcFqCodel), @@ -32,14 +32,10 @@ impl QueueType { } } -pub(crate) async fn read_tc_queues(interface: &str) -> Result> { +/// Separated into a separate function for cleaner benchmark code +pub fn deserialize_tc_tree(json: &str) -> Result> { let mut result = Vec::new(); - let command_output = Command::new("/sbin/tc") - .args(["-s", "-j", "qdisc", "show", "dev", interface]) - .output() - .await?; - let json = String::from_utf8(command_output.stdout)?; - let json: Value = serde_json::from_str(&json)?; + let json: Value = serde_json::from_str(json)?; if let Value::Array(array) = &json { for entry in array.iter() { match entry { @@ -60,3 +56,13 @@ pub(crate) async fn read_tc_queues(interface: &str) -> Result> { Ok(result) } + +pub(crate) async fn read_tc_queues(interface: &str) -> Result> { + let command_output = Command::new("/sbin/tc") + .args(["-s", "-j", "qdisc", "show", "dev", interface]) + .output() + .await?; + let json = String::from_utf8(command_output.stdout)?; + let result = deserialize_tc_tree(&json)?; + Ok(result) +} diff --git a/src/rust/lqosd/src/queue_tracker/queue_reader/tc_cake.rs b/src/rust/lqosd/src/queue_tracker/queue_reader/tc_cake.rs index 085dc7a5..1ccfb486 100644 --- a/src/rust/lqosd/src/queue_tracker/queue_reader/tc_cake.rs +++ b/src/rust/lqosd/src/queue_tracker/queue_reader/tc_cake.rs @@ -133,7 +133,7 @@ use serde::Serialize; use serde_json::Value; #[derive(Default, Clone, Debug, Serialize)] -pub(crate) struct TcCake { +pub struct TcCake { pub(crate) handle: TcHandle, pub(crate) parent: TcHandle, options: TcCakeOptions, @@ -172,7 +172,7 @@ struct TcCakeOptions { } #[derive(Default, Clone, Debug, Serialize)] -pub(crate) struct TcCakeTin { +pub struct TcCakeTin { threshold_rate: u64, pub(crate) sent_bytes: u64, pub(crate) backlog_bytes: u32, diff --git a/src/rust/lqosd/src/queue_tracker/queue_reader/tc_fq_codel.rs b/src/rust/lqosd/src/queue_tracker/queue_reader/tc_fq_codel.rs index d657bbe8..b14b4a36 100644 --- a/src/rust/lqosd/src/queue_tracker/queue_reader/tc_fq_codel.rs +++ b/src/rust/lqosd/src/queue_tracker/queue_reader/tc_fq_codel.rs @@ -11,7 +11,7 @@ use serde::Serialize; use serde_json::Value; #[derive(Default, Clone, Debug, Serialize)] -pub(crate) struct TcFqCodel { +pub struct TcFqCodel { handle: TcHandle, pub(crate) parent: TcHandle, options: TcFqCodelOptions, diff --git a/src/rust/lqosd/src/queue_tracker/queue_reader/tc_htb.rs b/src/rust/lqosd/src/queue_tracker/queue_reader/tc_htb.rs index 31f84d84..863fd2f6 100644 --- a/src/rust/lqosd/src/queue_tracker/queue_reader/tc_htb.rs +++ b/src/rust/lqosd/src/queue_tracker/queue_reader/tc_htb.rs @@ -9,7 +9,7 @@ use serde::Serialize; use serde_json::Value; #[derive(Default, Clone, Debug, Serialize)] -pub(crate) struct TcHtb { +pub struct TcHtb { handle: TcHandle, parent: TcHandle, options: TcHtbOptions, diff --git a/src/rust/lqosd/src/queue_tracker/queue_reader/tc_mq.rs b/src/rust/lqosd/src/queue_tracker/queue_reader/tc_mq.rs index a51e9869..6f355591 100644 --- a/src/rust/lqosd/src/queue_tracker/queue_reader/tc_mq.rs +++ b/src/rust/lqosd/src/queue_tracker/queue_reader/tc_mq.rs @@ -8,7 +8,7 @@ use serde::Serialize; use serde_json::Value; #[derive(Default, Clone, Debug, Serialize)] -pub(crate) struct TcMultiQueue { +pub struct TcMultiQueue { handle: TcHandle, root: bool, bytes: u64,