Merge pull request #468 from LibreQoE/per_flow

Per-flow tracking system, out-of-kernel ringbuffer for RTT events, add TCP retransmissions

src/rust/lqos_bus/src/bus/mod.rs

@@ -10,7 +10,7 @@ pub use client::bus_request;
 use log::error;
 pub use persistent_client::BusClient;
 pub use reply::BusReply;
-pub use request::{BusRequest, StatsRequest};
+pub use request::{BusRequest, StatsRequest, TopFlowType};
 pub use response::BusResponse;
 pub use session::BusSession;
 use thiserror::Error;

src/rust/lqos_bus/src/bus/request.rs

@@ -30,6 +30,14 @@ pub enum BusRequest {
     end: u32,
   },
 
+  /// Retrieves the TopN hosts with the worst Retransmits, sorted by Retransmits descending.
+  GetWorstRetransmits {
+    /// First row to retrieve (usually 0 unless you are paging)
+    start: u32,
+    /// Last row to retrieve (10 for top-10 starting at 0)
+    end: u32,
+  },
+
   /// Retrieves the TopN hosts with the best RTT, sorted by RTT descending.
   GetBestRtt {
     /// First row to retrieve (usually 0 unless you are paging)
@@ -133,9 +141,6 @@ pub enum BusRequest {
   /// Obtain the lqosd statistics
   GetLqosStats,
 
-  /// Tell me flow stats for a given IP address
-  GetFlowStats(String),
-
   /// Tell Heimdall to hyper-focus on an IP address for a bit
   GatherPacketData(String),
 
@@ -152,6 +157,51 @@ pub enum BusRequest {
   /// display a "run bandwidht test" link.
   #[cfg(feature = "equinix_tests")]
   RequestLqosEquinixTest,
+
+  /// Request a dump of all active flows. This can be a lot of data.
+  /// so this is intended for debugging
+  DumpActiveFlows,
+
+  /// Count the nubmer of active flows.
+  CountActiveFlows,
+
+  /// Top Flows Reports
+  TopFlows{ 
+    /// The type of top report to request
+    flow_type: TopFlowType,
+    /// The number of flows to return
+    n: u32 
+  },
+
+  /// Flows by IP Address
+  FlowsByIp(String),
+
+  /// Current Endpoints by Country
+  CurrentEndpointsByCountry,
+
+  /// Lat/Lon of Endpoints
+  CurrentEndpointLatLon,
+
+  /// Ether Protocol Summary
+  EtherProtocolSummary,
+
+  /// IP Protocol Summary
+  IpProtocolSummary,
+}
+
+/// Defines the type of "top" flow being requested
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Copy)]
+pub enum TopFlowType {
+  /// Top flows by current estimated bandwidth use
+  RateEstimate,
+  /// Top flows by total bytes transferred
+  Bytes,
+  /// Top flows by total packets transferred
+  Packets,
+  /// Top flows by total drops
+  Drops,
+  /// Top flows by round-trip time estimate
+  RoundTripTime,
 }
 
 /// Specific requests from the long-term stats system

src/rust/lqos_bus/src/bus/response.rs

@@ -1,6 +1,6 @@
 use super::QueueStoreTransit;
 use crate::{
-  ip_stats::PacketHeader, FlowTransport, IpMapping, IpStats, XdpPpingResult,
+  ip_stats::{FlowbeeSummaryData, PacketHeader}, IpMapping, IpStats, XdpPpingResult,
 };
 use lts_client::transport_data::{StatsTotals, StatsHost, StatsTreeNode};
 use serde::{Deserialize, Serialize};
@@ -43,6 +43,9 @@ pub enum BusResponse {
   /// Provides the worst N RTT scores, sorted in descending order.
   WorstRtt(Vec<IpStats>),
 
+  /// Provides the worst N Retransmit scores, sorted in descending order.
+  WorstRetransmits(Vec<IpStats>),
+
   /// Provides the best N RTT scores, sorted in descending order.
   BestRtt(Vec<IpStats>),
 
@@ -89,11 +92,10 @@ pub enum BusResponse {
     high_watermark: (u64, u64),
     /// Number of flows tracked
     tracked_flows: u64,
+    /// RTT events per second
+    rtt_events_per_second: u64,
   },
 
-  /// Flow Data
-  FlowData(Vec<(FlowTransport, Option<FlowTransport>)>),
-
   /// The index of the new packet collection session
   PacketCollectionSession {
     /// The identifier of the capture session
@@ -116,4 +118,41 @@ pub enum BusResponse {
 
   /// Long-term stats tree
   LongTermTree(Vec<StatsTreeNode>),
+
+  /// All Active Flows (Not Recommended - Debug Use)
+  AllActiveFlows(Vec<FlowbeeSummaryData>),
+
+  /// Count active flows
+  CountActiveFlows(u64),
+
+  /// Top Flopws
+  TopFlows(Vec<FlowbeeSummaryData>),
+
+  /// Flows by IP
+  FlowsByIp(Vec<FlowbeeSummaryData>),
+
+  /// Current endpoints by country
+  CurrentEndpointsByCountry(Vec<(String, [u64; 2], [f32; 2])>),
+
+  /// Current Lat/Lon of endpoints
+  CurrentLatLon(Vec<(f64, f64, String, u64, f32)>),
+
+  /// Summary of Ether Protocol
+  EtherProtocols{
+    /// Number of IPv4 Bytes
+    v4_bytes: [u64; 2],
+    /// Number of IPv6 Bytes
+    v6_bytes: [u64; 2],
+    /// Number of IPv4 Packets
+    v4_packets: [u64; 2],
+    /// Number of IPv6 Packets
+    v6_packets: [u64; 2],
+    /// Number of IPv4 Flows
+    v4_rtt: [u64; 2],
+    /// Number of IPv6 Flows
+    v6_rtt: [u64; 2],
+  },
+  
+  /// Summary of IP Protocols
+  IpProtocols(Vec<(String, (u64, u64))>),
 }

src/rust/lqos_bus/src/ip_stats.rs

@@ -5,142 +5,176 @@ use serde::{Deserialize, Serialize};
 /// with a host.
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct IpStats {
-  /// The host's IP address, as detected by the XDP program.
-  pub ip_address: String,
+    /// The host's IP address, as detected by the XDP program.
+    pub ip_address: String,
 
-  /// The host's mapped circuit ID
-  pub circuit_id: String,
+    /// The host's mapped circuit ID
+    pub circuit_id: String,
 
-  /// The current bits-per-second passing through this host. Tuple
-  /// 0 is download, tuple 1 is upload.
-  pub bits_per_second: (u64, u64),
+    /// The current bits-per-second passing through this host. Tuple
+    /// 0 is download, tuple 1 is upload.
+    pub bits_per_second: (u64, u64),
 
-  /// The current packets-per-second passing through this host. Tuple
-  /// 0 is download, tuple 1 is upload.
-  pub packets_per_second: (u64, u64),
+    /// The current packets-per-second passing through this host. Tuple
+    /// 0 is download, tuple 1 is upload.
+    pub packets_per_second: (u64, u64),
 
-  /// Median TCP round-trip-time for this host at the current time.
-  pub median_tcp_rtt: f32,
+    /// Median TCP round-trip-time for this host at the current time.
+    pub median_tcp_rtt: f32,
 
-  /// Associated TC traffic control handle.
-  pub tc_handle: TcHandle,
+    /// Associated TC traffic control handle.
+    pub tc_handle: TcHandle,
+
+    /// TCP Retransmits for this host at the current time.
+    pub tcp_retransmits: (u64, u64),
 }
 
 /// Represents an IP Mapping in the XDP IP to TC/CPU mapping system.
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 pub struct IpMapping {
-  /// The mapped IP address. May be IPv4, or IPv6.
-  pub ip_address: String,
+    /// The mapped IP address. May be IPv4, or IPv6.
+    pub ip_address: String,
 
-  /// The CIDR prefix length of the host. Equivalent to the CIDR value
-  /// after the /. e.g. `/24`.
-  pub prefix_length: u32,
+    /// The CIDR prefix length of the host. Equivalent to the CIDR value
+    /// after the /. e.g. `/24`.
+    pub prefix_length: u32,
 
-  /// The current TC traffic control handle.
-  pub tc_handle: TcHandle,
+    /// The current TC traffic control handle.
+    pub tc_handle: TcHandle,
 
-  /// The CPU index associated with this IP mapping.
-  pub cpu: u32,
+    /// The CPU index associated with this IP mapping.
+    pub cpu: u32,
 }
 
 /// Provided for backwards compatibility with `xdp_pping`, with the intent
 /// to retire it eventually.
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct XdpPpingResult {
-  /// The TC handle in text format. e.g. "1:12"
-  pub tc: String,
+    /// The TC handle in text format. e.g. "1:12"
+    pub tc: String,
 
-  /// The average (mean) RTT value for the current sample.
-  pub avg: f32,
+    /// The average (mean) RTT value for the current sample.
+    pub avg: f32,
 
-  /// The minimum RTT value for the current sample.
-  pub min: f32,
+    /// The minimum RTT value for the current sample.
+    pub min: f32,
 
-  /// The maximum RTT value for the current sample.
-  pub max: f32,
+    /// The maximum RTT value for the current sample.
+    pub max: f32,
 
-  /// The median RTT value for the current sample.
-  pub median: f32,
+    /// The median RTT value for the current sample.
+    pub median: f32,
 
-  /// The number of samples from which these values were
-  /// derived. If 0, the other values are invalid.
-  pub samples: u32,
-}
-
-/// Defines an IP protocol for display in the flow
-/// tracking (Heimdall) system.
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub enum FlowProto {
-  /// A TCP flow
-  TCP, 
-  /// A UDP flow
-  UDP, 
-  /// An ICMP flow
-  ICMP
-}
-
-/// Defines the display data for a flow in Heimdall.
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub struct FlowTransport {
-  /// The Source IP address
-  pub src: String,
-  /// The Destination IP address
-  pub dst: String,
-  /// The flow protocol (see `FlowProto`)
-  pub proto: FlowProto,
-  /// The source port, which is overridden to ICMP code on ICMP flows.
-  pub src_port: u16,
-  /// The destination port, which isn't useful at all on ICMP flows.
-  pub dst_port: u16,
-  /// The number of bytes since we started tracking this flow.
-  pub bytes: u64,
-  /// The number of packets since we started tracking this flow.
-  pub packets: u64,
-  /// Detected DSCP code if any
-  pub dscp: u8,
-  /// Detected ECN bit status (0-3)
-  pub ecn: u8,
+    /// The number of samples from which these values were
+    /// derived. If 0, the other values are invalid.
+    pub samples: u32,
 }
 
 /// Extract the 6-bit DSCP and 2-bit ECN code from a TOS field
 /// in an IP header.
 pub fn tos_parser(tos: u8) -> (u8, u8) {
-  // Format: 2 bits of ECN, 6 bits of DSCP
-  const ECN: u8 = 0b00000011;
-  const DSCP: u8 = 0b11111100;
+    // Format: 2 bits of ECN, 6 bits of DSCP
+    const ECN: u8 = 0b00000011;
+    const DSCP: u8 = 0b11111100;
 
-  let ecn = tos & ECN;
-  let dscp = (tos & DSCP) >> 2;
-  (dscp, ecn)
+    let ecn = tos & ECN;
+    let dscp = (tos & DSCP) >> 2;
+    (dscp, ecn)
 }
 
 /// Packet header dump
 #[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
 pub struct PacketHeader {
-  /// Timestamp (ns since boot)
-  pub timestamp: u64,
-  /// Source IP
-  pub src: String,
-  /// Destination IP
-  pub dst: String,
-  /// Source Port
-  pub src_port : u16,
-  /// Destination Port
-  pub dst_port: u16,
-  /// Ip Protocol (see Linux kernel docs)
-  pub ip_protocol: u8,
-  /// ECN Flag
-  pub ecn: u8,
-  /// DSCP code
-  pub dscp: u8,
-  /// Packet Size
-  pub size: u32,
-  /// TCP Flag Bitset
-  pub tcp_flags: u8,
-  /// TCP Window Size
-  pub tcp_window: u16,
-  /// TCP TSVal
-  pub tcp_tsval: u32,
-  /// TCP ECR val
-  pub tcp_tsecr: u32,
-}
+    /// Timestamp (ns since boot)
+    pub timestamp: u64,
+    /// Source IP
+    pub src: String,
+    /// Destination IP
+    pub dst: String,
+    /// Source Port
+    pub src_port: u16,
+    /// Destination Port
+    pub dst_port: u16,
+    /// Ip Protocol (see Linux kernel docs)
+    pub ip_protocol: u8,
+    /// ECN Flag
+    pub ecn: u8,
+    /// DSCP code
+    pub dscp: u8,
+    /// Packet Size
+    pub size: u32,
+    /// TCP Flag Bitset
+    pub tcp_flags: u8,
+    /// TCP Window Size
+    pub tcp_window: u16,
+    /// TCP TSVal
+    pub tcp_tsval: u32,
+    /// TCP ECR val
+    pub tcp_tsecr: u32,
+}
+
+/// Flowbee protocol enumeration
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
+pub enum FlowbeeProtocol {
+    /// TCP (type 6)
+    TCP,
+    /// UDP (type 17)
+    UDP,
+    /// ICMP (type 1)
+    ICMP,
+}
+
+impl From<u8> for FlowbeeProtocol {
+    fn from(value: u8) -> Self {
+        match value {
+            6 => Self::TCP,
+            17 => Self::UDP,
+            _ => Self::ICMP,
+        }
+    }
+}
+
+/// Flowbee: a complete flow data, combining key and data.
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
+pub struct FlowbeeSummaryData {
+    /// Mapped `XdpIpAddress` source for the flow.
+    pub remote_ip: String,
+    /// Mapped `XdpIpAddress` destination for the flow
+    pub local_ip: String,
+    /// Source port number, or ICMP type.
+    pub src_port: u16,
+    /// Destination port number.
+    pub dst_port: u16,
+    /// IP protocol (see the Linux kernel!)
+    pub ip_protocol: FlowbeeProtocol,
+    /// Padding to align the structure to 16 bytes.
+    /// Time (nanos) when the connection was established
+    pub start_time: u64,
+    /// Time (nanos) when the connection was last seen
+    pub last_seen: u64,
+    /// Bytes transmitted
+    pub bytes_sent: [u64; 2],
+    /// Packets transmitted
+    pub packets_sent: [u64; 2],
+    /// Rate estimate
+    pub rate_estimate_bps: [u32; 2],
+    /// TCP Retransmission count (also counts duplicates)
+    pub tcp_retransmits: [u16; 2],
+    /// Has the connection ended?
+    /// 0 = Alive, 1 = FIN, 2 = RST
+    pub end_status: u8,
+    /// Raw IP TOS
+    pub tos: u8,
+    /// Raw TCP flags
+    pub flags: u8,
+    /// Recent RTT median
+    pub rtt_nanos: [u64; 2],
+    /// Remote ASN
+    pub remote_asn: u32,
+    /// Remote ASN Name
+    pub remote_asn_name: String,
+    /// Remote ASN Country
+    pub remote_asn_country: String,
+    /// Analysis
+    pub analysis: String,
+}

src/rust/lqos_bus/src/lib.rs

@@ -13,15 +13,15 @@
 mod bus;
 mod ip_stats;
 pub use ip_stats::{
-  tos_parser, FlowProto, FlowTransport, IpMapping, IpStats, PacketHeader,
-  XdpPpingResult,
+  tos_parser, IpMapping, IpStats, PacketHeader,
+  XdpPpingResult, FlowbeeSummaryData, FlowbeeProtocol
 };
 mod tc_handle;
 pub use bus::{
   bus_request, decode_request, decode_response, encode_request,
   encode_response, BusClient, BusReply, BusRequest, BusResponse, BusSession,
   CakeDiffTinTransit, CakeDiffTransit, CakeTransit, QueueStoreTransit,
-  UnixSocketServer, BUS_SOCKET_PATH, StatsRequest
+  UnixSocketServer, BUS_SOCKET_PATH, StatsRequest, TopFlowType
 };
 pub use tc_handle::TcHandle;
 

src/rust/lqos_config/src/etc/mod.rs

@@ -45,7 +45,6 @@ pub fn load_config() -> Result<Config, LibreQoSConfigError> {
         *lock = Some(config_result.unwrap());
     }
 
-    log::info!("Returning cached config");
     Ok(lock.as_ref().unwrap().clone())
 }
 

src/rust/lqos_config/src/etc/v15/flows.rs

@@ -0,0 +1,28 @@
+//! Provides netflow support for tracking network flows.
+//!
+//! You can enable them by adding a `[flows]` section to your configuration file.
+
+use serde::{Serialize, Deserialize};
+
+#[derive(Clone, Serialize, Deserialize, Debug)]
+pub struct FlowConfig {
+    pub flow_timeout_seconds: u64,
+    pub netflow_enabled: bool,
+    pub netflow_port: Option<u16>,
+    pub netflow_ip: Option<String>,
+    pub netflow_version: Option<u8>,
+    pub do_not_track_subnets: Option<Vec<String>>,
+}
+
+impl Default for FlowConfig {
+    fn default() -> Self {
+        Self {
+            flow_timeout_seconds: 30,
+            netflow_enabled: false,
+            netflow_port: None,
+            netflow_ip: None,
+            netflow_version: None,
+            do_not_track_subnets: None,
+        }
+    }
+}

src/rust/lqos_config/src/etc/v15/mod.rs

@@ -14,6 +14,7 @@ mod uisp_integration;
 mod powercode_integration;
 mod sonar_integration;
 mod influxdb;
+mod flows;
 pub use bridge::*;
 pub use long_term_stats::LongTermStats;
 pub use tuning::Tunables;

src/rust/lqos_config/src/etc/v15/top_config.rs

@@ -51,6 +51,9 @@ pub struct Config {
     /// IP Range definitions
     pub ip_ranges: super::ip_ranges::IpRanges,
 
+    /// Network flows configuration
+    pub flows: Option<super::flows::FlowConfig>,
+
     /// Integration Common Variables
     pub integration_common: super::integration_common::IntegrationConfig,
 
@@ -133,6 +136,7 @@ impl Default for Config {
             influxdb: super::influxdb::InfluxDbConfig::default(),
             packet_capture_time: 10,
             queue_check_period_ms: 1000,
+            flows: None,
         }
     }
 }

src/rust/lqos_heimdall/src/flows.rs

@@ -1,165 +0,0 @@
-use crate::{timeline::expire_timeline, FLOW_EXPIRE_SECS};
-use dashmap::DashMap;
-use lqos_bus::{tos_parser, BusResponse, FlowTransport};
-use lqos_sys::heimdall_data::{HeimdallKey, HeimdallData};
-use lqos_utils::{unix_time::time_since_boot, XdpIpAddress};
-use once_cell::sync::Lazy;
-use std::{collections::HashSet, time::Duration};
-
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
-struct FlowKey {
-  src: XdpIpAddress,
-  dst: XdpIpAddress,
-  proto: u8,
-  src_port: u16,
-  dst_port: u16,
-}
-
-#[derive(Clone, Debug, Default)]
-struct FlowData {
-  last_seen: u64,
-  bytes: u64,
-  packets: u64,
-  tos: u8,
-}
-
-impl From<&HeimdallKey> for FlowKey {
-  fn from(value: &HeimdallKey) -> Self {
-    Self {
-      src: value.src_ip,
-      dst: value.dst_ip,
-      proto: value.ip_protocol,
-      src_port: value.src_port,
-      dst_port: value.dst_port,
-    }
-  }
-}
-
-static FLOW_DATA: Lazy<DashMap<FlowKey, FlowData>> = Lazy::new(DashMap::new);
-
-/*pub(crate) fn record_flow(event: &HeimdallEvent) {
-  let key: FlowKey = event.into();
-  if let Some(mut data) = FLOW_DATA.get_mut(&key) {
-    data.last_seen = event.timestamp;
-    data.packets += 1;
-    data.bytes += event.size as u64;
-    data.tos = event.tos;
-  } else {
-    FLOW_DATA.insert(
-      key,
-      FlowData {
-        last_seen: event.timestamp,
-        bytes: event.size.into(),
-        packets: 1,
-        tos: event.tos,
-      },
-    );
-  }
-}*/
-
-
-/// Iterates through all throughput entries, and sends them in turn to `callback`.
-/// This elides the need to clone or copy data.
-fn heimdall_for_each(
-  callback: &mut dyn FnMut(&HeimdallKey, &[HeimdallData]),
-) {
-  /*if let Ok(heimdall) = BpfPerCpuMap::<HeimdallKey, HeimdallData>::from_path(
-    "/sys/fs/bpf/heimdall",
-  ) {
-    heimdall.for_each(callback);
-  }*/
-  lqos_sys::iterate_heimdall(callback);
-}
-
-
-fn combine_flows(values: &[HeimdallData]) -> FlowData {
-  let mut result = FlowData::default();
-  let mut ls = 0;
-  values.iter().for_each(|v| {
-    result.bytes += v.bytes;
-    result.packets += v.packets;
-    result.tos += v.tos;
-    if v.last_seen > ls {
-      ls = v.last_seen;
-    }
-  });
-  result.last_seen = ls;
-  result
-}
-
-pub fn read_flows() {
-  heimdall_for_each(&mut |key, value| {
-    let flow_key = key.into();
-    let combined = combine_flows(value);
-    if let Some(mut flow) = FLOW_DATA.get_mut(&flow_key) {
-      flow.last_seen = combined.last_seen;
-      flow.bytes = combined.bytes;
-      flow.packets = combined.packets;
-      flow.tos = combined.tos;
-    } else {
-      FLOW_DATA.insert(flow_key, combined);
-    }
-  });
-}
-
-/// Expire flows that have not been seen in a while.
-pub fn expire_heimdall_flows() {
-  if let Ok(now) = time_since_boot() {
-    let since_boot = Duration::from(now);
-    let expire = (since_boot - Duration::from_secs(FLOW_EXPIRE_SECS)).as_nanos() as u64;
-    FLOW_DATA.retain(|_k, v| v.last_seen > expire);
-    expire_timeline();
-  }
-}
-
-/// Get the flow stats for a given IP address.
-pub fn get_flow_stats(ip: XdpIpAddress) -> BusResponse {
-  let mut result = Vec::new();
-
-  // Obtain all the flows
-  let mut all_flows = Vec::new();
-  for value in FLOW_DATA.iter() {
-    let key = value.key();
-    if key.src == ip || key.dst == ip {
-      let (dscp, ecn) = tos_parser(value.tos);
-      all_flows.push(FlowTransport {
-        src: key.src.as_ip().to_string(),
-        dst: key.dst.as_ip().to_string(),
-        src_port: key.src_port,
-        dst_port: key.dst_port,
-        proto: match key.proto {
-          6 => lqos_bus::FlowProto::TCP,
-          17 => lqos_bus::FlowProto::UDP,
-          _ => lqos_bus::FlowProto::ICMP,
-        },
-        bytes: value.bytes,
-        packets: value.packets,
-        dscp,
-        ecn,
-      });
-    }
-  }
-
-  // Turn them into reciprocal pairs
-  let mut done = HashSet::new();
-  for (i, flow) in all_flows.iter().enumerate() {
-    if !done.contains(&i) {
-      let flow_a = flow.clone();
-      let flow_b = if let Some(flow_b) = all_flows
-        .iter()
-        .position(|f| f.src == flow_a.dst && f.src_port == flow_a.dst_port)
-      {
-        done.insert(flow_b);
-        Some(all_flows[flow_b].clone())
-      } else {
-        None
-      };
-
-      result.push((flow_a, flow_b));
-    }
-  }
-
-  result.sort_by(|a, b| b.0.bytes.cmp(&a.0.bytes));
-
-  BusResponse::FlowData(result)
-}

src/rust/lqos_heimdall/src/lib.rs

@@ -7,8 +7,6 @@ mod config;
 pub mod perf_interface;
 pub mod stats;
 pub use config::{HeimdalConfig, HeimdallMode};
-mod flows;
-pub use flows::{expire_heimdall_flows, get_flow_stats};
 mod timeline;
 pub use timeline::{n_second_packet_dump, n_second_pcap, hyperfocus_on_target};
 mod pcap;
@@ -16,7 +14,7 @@ mod watchlist;
 use lqos_utils::fdtimer::periodic;
 pub use watchlist::{heimdall_expire, heimdall_watch_ip, set_heimdall_mode};
 
-use crate::flows::read_flows;
+use crate::timeline::expire_timeline;
 
 /// How long should Heimdall keep watching a flow after being requested
 /// to do so? Setting this to a long period increases CPU load after the
@@ -24,9 +22,6 @@ use crate::flows::read_flows;
 /// collections if the client hasn't maintained the 1s request cadence.
 const EXPIRE_WATCHES_SECS: u64 = 5;
 
-/// How long should Heimdall retain flow summary data?
-const FLOW_EXPIRE_SECS: u64 = 10;
-
 /// How long should Heimdall retain packet timeline data?
 const TIMELINE_EXPIRE_SECS: u64 = 10;
 
@@ -48,9 +43,8 @@ pub async fn start_heimdall() {
 
   std::thread::spawn(move || {
     periodic(interval_ms, "Heimdall Packet Watcher", &mut || {
-      read_flows();
-      expire_heimdall_flows();
       heimdall_expire();
+      expire_timeline();
     });
   });
 }

src/rust/lqos_map_perf/src/main.rs

@@ -1,6 +1,6 @@
 use std::time::Instant;
 
-use lqos_sys::{rtt_for_each, throughput_for_each};
+use lqos_sys::{iterate_flows, throughput_for_each};
 
 fn main() {
   println!("LibreQoS Map Performance Tool");
@@ -8,7 +8,7 @@ fn main() {
   // Test the RTT map
   let mut rtt_count = 0;
   let now = Instant::now();
-  rtt_for_each(&mut |_rtt, _tracker| {
+  iterate_flows(&mut |_rtt, _tracker| {
     rtt_count += 1;
   });
   let elapsed = now.elapsed();

src/rust/lqos_node_manager/src/config_control.rs

@@ -66,17 +66,19 @@ pub struct LqosStats {
   pub time_to_poll_hosts_us: u64,
   pub high_watermark: (u64, u64),
   pub tracked_flows: u64,
+  pub rtt_events_per_second: u64,
 }
 
 #[get("/api/stats")]
 pub async fn stats() -> NoCache<Json<LqosStats>> {
   for msg in bus_request(vec![BusRequest::GetLqosStats]).await.unwrap() {
-    if let BusResponse::LqosdStats { bus_requests, time_to_poll_hosts, high_watermark, tracked_flows } = msg {
+    if let BusResponse::LqosdStats { bus_requests, time_to_poll_hosts, high_watermark, tracked_flows, rtt_events_per_second } = msg {
       return NoCache::new(Json(LqosStats {
         bus_requests_since_start: bus_requests,
         time_to_poll_hosts_us: time_to_poll_hosts,
         high_watermark,
         tracked_flows,
+        rtt_events_per_second,
       }));
     }
   }

src/rust/lqos_node_manager/src/flow_monitor.rs

@@ -0,0 +1,93 @@
+use lqos_bus::{bus_request, BusRequest, BusResponse, FlowbeeSummaryData};
+use rocket::serde::json::Json;
+use crate::cache_control::NoCache;
+
+#[get("/api/flows/dump_all")]
+pub async fn all_flows_debug_dump() -> NoCache<Json<Vec<FlowbeeSummaryData>>> {
+  let responses =
+    bus_request(vec![BusRequest::DumpActiveFlows]).await.unwrap();
+  let result = match &responses[0] {
+    BusResponse::AllActiveFlows(flowbee) => flowbee.to_owned(),
+    _ => Vec::new(),
+  };
+
+  NoCache::new(Json(result))
+}
+
+#[get("/api/flows/count")]
+pub async fn count_flows() -> NoCache<Json<u64>> {
+  let responses =
+    bus_request(vec![BusRequest::CountActiveFlows]).await.unwrap();
+  let result = match &responses[0] {
+    BusResponse::CountActiveFlows(count) => *count,
+    _ => 0,
+  };
+
+  NoCache::new(Json(result))
+}
+
+#[get("/api/flows/top/<top_n>/<flow_type>")]
+pub async fn top_5_flows(top_n: u32, flow_type: String) -> NoCache<Json<Vec<FlowbeeSummaryData>>> {
+  let flow_type = match flow_type.as_str() {
+    "rate" => lqos_bus::TopFlowType::RateEstimate,
+    "bytes" => lqos_bus::TopFlowType::Bytes,
+    "packets" => lqos_bus::TopFlowType::Packets,
+    "drops" => lqos_bus::TopFlowType::Drops,
+    "rtt" => lqos_bus::TopFlowType::RoundTripTime,
+    _ => lqos_bus::TopFlowType::RateEstimate,
+  };
+
+  let responses =
+    bus_request(vec![BusRequest::TopFlows { n: top_n, flow_type }]).await.unwrap();
+  let result = match &responses[0] {
+    BusResponse::TopFlows(flowbee) => flowbee.to_owned(),
+    _ => Vec::new(),
+  };
+
+  NoCache::new(Json(result))
+}
+
+#[get("/api/flows/by_country")]
+pub async fn flows_by_country() -> NoCache<Json<Vec<(String, [u64; 2], [f32; 2])>>> {
+  let responses =
+    bus_request(vec![BusRequest::CurrentEndpointsByCountry]).await.unwrap();
+  let result = match &responses[0] {
+    BusResponse::CurrentEndpointsByCountry(country_summary) => country_summary.to_owned(),
+    _ => Vec::new(),
+  };
+
+  NoCache::new(Json(result))
+}
+
+#[get("/api/flows/lat_lon")]
+pub async fn flows_lat_lon() -> NoCache<Json<Vec<(f64, f64, String, u64, f32)>>> {
+  let responses =
+    bus_request(vec![BusRequest::CurrentEndpointLatLon]).await.unwrap();
+  let result = match &responses[0] {
+    BusResponse::CurrentLatLon(lat_lon) => lat_lon.to_owned(),
+    _ => Vec::new(),
+  };
+
+  NoCache::new(Json(result))
+}
+
+#[get("/api/flows/ether_protocol")]
+pub async fn flows_ether_protocol() -> NoCache<Json<BusResponse>> {
+  let responses =
+    bus_request(vec![BusRequest::EtherProtocolSummary]).await.unwrap();
+  let result = responses[0].to_owned();
+
+  NoCache::new(Json(result))
+}
+
+#[get("/api/flows/ip_protocol")]
+pub async fn flows_ip_protocol() -> NoCache<Json<Vec<(String, (u64, u64))>>> {
+  let responses =
+    bus_request(vec![BusRequest::IpProtocolSummary]).await.unwrap();
+  let result = match &responses[0] {
+    BusResponse::IpProtocols(ip_protocols) => ip_protocols.to_owned(),
+    _ => Vec::new(),
+  };
+
+  NoCache::new(Json(result))
+}

src/rust/lqos_node_manager/src/main.rs

@@ -12,6 +12,7 @@ mod config_control;
 mod network_tree;
 mod queue_info;
 mod toasts;
+mod flow_monitor;
 
 // Use JemAllocator only on supported platforms
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
@@ -43,6 +44,7 @@ fn rocket() -> _ {
         static_pages::shaped_devices_add_page,
         static_pages::unknown_devices_page,
         static_pages::circuit_queue,
+        static_pages::pretty_map_graph,
         config_control::config_page,
         network_tree::tree_page,
         static_pages::ip_dump,
@@ -57,6 +59,7 @@ fn rocket() -> _ {
         tracker::ram_usage,
         tracker::top_10_downloaders,
         tracker::worst_10_rtt,
+        tracker::worst_10_tcp,
         tracker::rtt_histogram,
         tracker::host_counts,
         shaped_devices::all_shaped_devices,
@@ -109,6 +112,14 @@ fn rocket() -> _ {
         // Front page toast checks
         toasts::version_check,
         toasts::stats_check,
+        // Flowbee System
+        flow_monitor::all_flows_debug_dump,
+        flow_monitor::count_flows,
+        flow_monitor::top_5_flows,
+        flow_monitor::flows_by_country,
+        flow_monitor::flows_lat_lon,
+        flow_monitor::flows_ether_protocol,
+        flow_monitor::flows_ip_protocol,
       ],
     );
 

src/rust/lqos_node_manager/src/queue_info.rs

@@ -1,7 +1,7 @@
 use crate::auth_guard::AuthGuard;
 use crate::cache_control::NoCache;
 use crate::tracker::{SHAPED_DEVICES, lookup_dns};
-use lqos_bus::{bus_request, BusRequest, BusResponse, FlowTransport, PacketHeader, QueueStoreTransit};
+use lqos_bus::{bus_request, BusRequest, BusResponse, FlowbeeSummaryData, PacketHeader, QueueStoreTransit};
 use rocket::fs::NamedFile;
 use rocket::http::Status;
 use rocket::response::content::RawJson;
@@ -107,6 +107,19 @@ pub async fn raw_queue_by_circuit(
 }
 
 #[get("/api/flows/<ip_list>")]
+pub async fn flow_stats(ip_list: String, _auth: AuthGuard) -> NoCache<Json<Vec<FlowbeeSummaryData>>> {
+  let mut result = Vec::new();
+  let request: Vec<BusRequest> = ip_list.split(',').map(|ip| BusRequest::FlowsByIp(ip.to_string())).collect();
+  let responses = bus_request(request).await.unwrap();
+  for r in responses.iter() {
+    if let BusResponse::FlowsByIp(flow) = r {
+      result.extend_from_slice(flow);
+    }
+  }
+  NoCache::new(Json(result))
+}
+
+/*#[get("/api/flows/<ip_list>")]
 pub async fn flow_stats(ip_list: String, _auth: AuthGuard) -> NoCache<MsgPack<Vec<(FlowTransport, Option<FlowTransport>)>>> {
   let mut result = Vec::new();
   let request: Vec<BusRequest> = ip_list.split(',').map(|ip| BusRequest::GetFlowStats(ip.to_string())).collect();
@@ -117,7 +130,7 @@ pub async fn flow_stats(ip_list: String, _auth: AuthGuard) -> NoCache<MsgPack<Ve
     }
   }
   NoCache::new(MsgPack(result))
-}
+}*/
 
 #[derive(Serialize, Clone)]
 #[serde(crate = "rocket::serde")]

src/rust/lqos_node_manager/src/static_pages.rs

@@ -75,6 +75,14 @@ pub async fn shaped_devices_add_page<'a>(
   NoCache::new(NamedFile::open("static/shaped-add.html").await.ok())
 }
 
+// Temporary for funsies
+#[get("/showoff")]
+pub async fn pretty_map_graph<'a>(
+  _auth: AuthGuard,
+) -> NoCache<Option<NamedFile>> {
+  NoCache::new(NamedFile::open("static/showoff.html").await.ok())
+}
+
 #[get("/vendor/bootstrap.min.css")]
 pub async fn bootsrap_css<'a>() -> LongCache<Option<NamedFile>> {
   LongCache::new(NamedFile::open("static/vendor/bootstrap.min.css").await.ok())

src/rust/lqos_node_manager/src/tracker/mod.rs

@@ -22,6 +22,7 @@ pub struct IpStatsWithPlan {
   pub tc_handle: TcHandle,
   pub circuit_id: String,
   pub plan: (u32, u32),
+  pub tcp_retransmits: (u64, u64),
 }
 
 impl From<&IpStats> for IpStatsWithPlan {
@@ -34,6 +35,7 @@ impl From<&IpStats> for IpStatsWithPlan {
       tc_handle: i.tc_handle,
       circuit_id: i.circuit_id.clone(),
       plan: (0, 0),
+      tcp_retransmits: i.tcp_retransmits,
     };
 
     if !result.circuit_id.is_empty() {
@@ -131,6 +133,21 @@ pub async fn worst_10_rtt(_auth: AuthGuard) -> NoCache<MsgPack<Vec<IpStatsWithPl
   NoCache::new(MsgPack(Vec::new()))
 }
 
+#[get("/api/worst_10_tcp")]
+pub async fn worst_10_tcp(_auth: AuthGuard) -> NoCache<MsgPack<Vec<IpStatsWithPlan>>> {
+  if let Ok(messages) = bus_request(vec![BusRequest::GetWorstRetransmits { start: 0, end: 10 }]).await
+  {
+    for msg in messages {
+      if let BusResponse::WorstRetransmits(stats) = msg {
+        let result = stats.iter().map(|tt| tt.into()).collect();
+        return NoCache::new(MsgPack(result));
+      }
+    }
+  }
+
+  NoCache::new(MsgPack(Vec::new()))
+}
+
 #[get("/api/rtt_histogram")]
 pub async fn rtt_histogram(_auth: AuthGuard) -> NoCache<MsgPack<Vec<u32>>> {
   if let Ok(messages) = bus_request(vec![BusRequest::RttHistogram]).await

src/rust/lqos_node_manager/static/circuit_queue.html

@@ -215,10 +215,6 @@
                         <div class="card bg-light">
                             <div class="card-body">
                                 <h5 class="card-title"><i class="fa fa-bar-chart"></i> Flows (Last 30 Seconds)</h5>
-                                <p class="alert alert-warning" role="alert">
-                                    <i class="fa fa-warning"></i> Gathering packet data can cause high CPU load during
-                                    the capture window.
-                                </p>
                                 <div id="packetButtons"></div>
                                 <div id="flowList"></div>
                             </div>
@@ -763,6 +759,19 @@
             }
         }
 
+        function parse_rtts(data, idx) {
+            let n = [];
+            for (let i=0; i<data.rtt_ringbuffer[idx].length; i++) {
+                n.push(data.rtt_ringbuffer[idx][i]);
+            }
+            if (n.length == 0) {
+                return 0.0;
+            }
+            n.sort();
+            // Median
+            return n[Math.floor(n.length / 2)];
+        }
+
         function getFlows() {
             let ip_list = "";
             let ip_btns = "";
@@ -779,10 +788,36 @@
             }
             ip_list = ip_list.substring(0, ip_list.length - 1);
             if (ip_list == "") return;
-            msgPackGet("/api/flows/" + ip_list, (data) => {
+            $.get("/api/flows/" + ip_list, (data) => {
+            //msgPackGet("/api/flows/" + ip_list, (data) => {
                 //console.log(data);
                 let html = "<table class='table table-striped'>";
+                
                 html += "<thead>";
+                html += "<th>Connection</th>";
+                html += "<th>Bytes</th>";
+                html += "<th>Packets</th>";
+                html += "<th>TCP Retransmits</th>";
+                html += "<th>TCP RTT</th>";
+                html += "<th>ASN</th>";
+                html += "<th>ASN Country</th>";
+                html += "</thead>";
+                html += "<tbody>";
+                for (var i=0; i<data.length; i++) {
+                    console.log(data[i]);
+                    html += "<tr>";
+                    html += "<td>" + data[i].analysis + "</td>";
+                    html += "<td>" + scaleNumber(data[i].bytes_sent[0]) + " / " + scaleNumber(data[i].bytes_sent[1]) + "</td>";
+                    html += "<td>" + scaleNumber(data[i].packets_sent[0]) + " / " + scaleNumber(data[i].packets_sent[1]) + "</td>";
+                    html += "<td>" + data[i].tcp_retransmits[0] + " / " + data[i].tcp_retransmits[1] + "</td>";
+                    html += "<td>" + scaleNanos(data[i].rtt_nanos[0]) + " / " + scaleNanos(data[i].rtt_nanos[1]) + "</td>";
+                    html += "<td>(" + data[i].remote_asn + ") " + data[i].remote_asn_name + "</td>";
+                    html += "<td>" + data[i].remote_asn_country + "</td>";
+                    html += "</tr>";
+                }
+                html += "</tbody>";
+
+                /*html += "<thead>";
                 html += "<th>Protocol</th>";
                 html += "<th>Src</th>";
                 html += "<th>Src Port</th>";
@@ -837,6 +872,7 @@
                     html += "</tr>";
                 }
                 html += "</tbody></table>";
+                */
                 $("#flowList").html(html);
             })
         }

src/rust/lqos_node_manager/static/ip_dump.html

@@ -332,6 +332,7 @@ if (hdr->cwr) flags |= 128;
 
             target = params.id;
             $.get("/api/packet_dump/" + params.id, (data) => {
+                console.log(data);
                 data.sort((a,b) => a.timestamp - b.timestamp);
 
                 // Find the minimum timestamp

src/rust/lqos_node_manager/static/lqos.js

@@ -34,6 +34,7 @@ const IpStats = {
     "tc_handle": 4,
     "circuit_id": 5,
     "plan": 6,
+    "tcp_retransmits": 7,
 }
 
 const FlowTrans = {
@@ -165,7 +166,7 @@ function updateHostCounts() {
     });*/
     // LTS Check
     $.get("/api/stats_check", (data) => {
-        console.log(data);
+        //console.log(data);
         let template = "<a class='nav-link' href='$URL$'><i class='fa fa-dashboard'></i> $TEXT$</a>";
         switch (data.action) {
             case "Disabled": {
@@ -272,6 +273,18 @@ function scaleNumber(n) {
     return n;
 }
 
+function scaleNanos(n) {
+    if (n == 0) return "";
+    if (n > 1000000000) {
+        return (n / 1000000000).toFixed(2) + "s";
+    } else if (n > 1000000) {
+        return (n / 1000000).toFixed(2) + "ms";
+    } else if (n > 1000) {
+        return (n / 1000).toFixed(2) + "µs";
+    }
+    return n + "ns";
+}
+
 const reloadModal = `
 <div class='modal fade' id='reloadModal' tabindex='-1' aria-labelledby='reloadModalLabel' aria-hidden='true'>
     <div class='modal-dialog modal-fullscreen'>

src/rust/lqos_node_manager/static/main.html

@@ -66,7 +66,7 @@
             <div class="col-sm-4">
                 <div class="card bg-light">
                     <div class="card-body">
-                        <h5 class="card-title"><i class="fa fa-bolt"></i> Current Throughput</h5>
+                        <h5 class="card-title"><i class="fa fa-bolt"></i> Current Throughput <span class="badge badge-pill green-badge" id="flowCount">?</span></h5>
                         <table class="table">
                             <tr>
                                 <td class="bold">Packets/Second</td>
@@ -143,8 +143,19 @@
             <div class="col-sm-6">
                 <div class="card bg-light">
                     <div class="card-body">
-                        <h5 class="card-title"><i class='fa fa-arrow-down'></i> Top 10 Downloaders</h5>
-                        <div id="top10dl"></div>
+                        <h5 class="card-title">
+                            <i class='fa fa-arrow-down'></i> Top 10 Downloaders
+                            <button id="btntop10dl" class="btn btn-small btn-success" href="/top10" onclick="showCircuits()">Circuits</button>
+                            <button id="btntop10flows" class="btn btn-small btn-primary" href="/top10" onclick="showFlows()">Flows</button>
+                            <button id="btntop10ep" class="btn btn-small btn-primary" href="/top10" onclick="showEndpoints()">Geo Endpoints</button>
+                            <button id="btntop10pro" class="btn btn-small btn-primary" href="/top10" onclick="showProtocols()">Protocols</button>
+                            <button id="btntop10eth" class="btn btn-small btn-primary" href="/top10" onclick="showEthertypes()">Ethertypes</button>
+                        </h5>
+                        <div id="top10dl" style="display:block;"></div>
+                        <div id="top10flows" style="display: none;"></div>
+                        <div id="top10ep" style="display: none;"></div>
+                        <div id="top10eth" style="display: none;"></div>
+                        <div id="top10pro" style="display: none;"></div>
                     </div>
                 </div>
             </div>
@@ -153,8 +164,12 @@
             <div class="col-sm-6">
                 <div class="card bg-light">
                     <div class="card-body">
-                        <h5 class="card-title"><i class='fa fa-exclamation'></i> Worst 10 RTT</h5>
+                        <h5 class="card-title"><i class='fa fa-exclamation'></i> Worst 10
+                            <button id="btnworstRtt" class="btn btn-small btn-success" href="/top10" onclick="showWorstRtt()">RTT</button>
+                            <button id="btnworstTcp" class="btn btn-small btn-primary" href="/top10" onclick="showWorstTcp()">TCP Retransmits</button>
+                        </h5>
                         <div id="worstRtt"></div>
+                        <div id="worstTcp" style="display: none;"></div>
                     </div>
                 </div>
             </div>
@@ -164,7 +179,7 @@
 
     <footer>&copy; 2022-2023, LibreQoE LLC</footer>
 
-    <script>
+    <script>        
         var throughput = new MultiRingBuffer(300);
 
         // Loads the complete ringbuffer for initial display
@@ -190,6 +205,12 @@
             });
         }
 
+        function updateFlowCounter() {
+            $.get("/api/flows/count", (data) => {
+                $("#flowCount").text(data + " flows");
+            });
+        }
+
         function updateCurrentThroughput() {
             msgPackGet("/api/current_throughput", (tp) => {
                 const bits = 0;
@@ -211,7 +232,7 @@
 
         function updateSiteFunnel() {
             msgPackGet("/api/network_tree_summary/", (data) => {
-                let table = "<table class='table' style='font-size: 8pt;'>";
+                let table = "<table class='table table-striped' style='font-size: 8pt;'>";
                 for (let i = 0; i < data.length; ++i) {
                     let id = data[i][0];
                     let name = data[i][1][NetTrans.name];
@@ -263,11 +284,12 @@
         }
 
         function updateNTable(target, tt) {
-            let html = "<table class='table'>";
-            html += "<thead><th>IP Address</th><th>DL ⬇️</th><th>UL ⬆️</th><th>RTT (ms)</th><th>Shaped</th></thead>";
+            let html = "<table class='table table-striped' style='font-size: 8pt'>";
+            html += "<thead><th></th><th>IP Address</th><th>DL ⬇️</th><th>UL ⬆️</th><th>RTT (ms)</th><th>TCP Retransmits</th><th>Shaped</th></thead>";
             for (let i = 0; i < tt.length; i++) {
                 let color = color_ramp(tt[i][IpStats.median_tcp_rtt]);
-                html += "<tr style='background-color: " + color + "'>";
+                html += "<tr>";
+                html += "<td style='color: " + color + "'>⬤</td>";
                 if (tt[i][IpStats.circuit_id] != "") {
                     html += "<td><a class='redact' href='/circuit_queue?id=" + encodeURI(tt[i][IpStats.circuit_id]) + "'>" + redactText(tt[i][IpStats.ip_address]) + "</td>";
                 } else {
@@ -276,6 +298,7 @@
                 html += "<td>" + scaleNumber(tt[i][IpStats.bits_per_second][0]) + "</td>";
                 html += "<td>" + scaleNumber(tt[i][IpStats.bits_per_second][1]) + "</td>";
                 html += "<td>" + tt[i][IpStats.median_tcp_rtt].toFixed(2) + "</td>";
+                html += "<td>" + tt[i][IpStats.tcp_retransmits][0] + "/" + tt[i][IpStats.tcp_retransmits][1] + "</td>";
                 if (tt[i].tc_handle != 0) {
                     html += "<td><i class='fa fa-check-circle'></i> (" + tt[i][IpStats.plan][0] + "/" + tt[i][IpStats.plan][1] + ")</td>";
                 } else {
@@ -300,6 +323,191 @@
             });
         }
 
+        function updateWorstTcp() {
+            msgPackGet("/api/worst_10_tcp", (tt) => {
+                //console.log(tt);
+                updateNTable('#worstTcp', tt);
+            });
+        }
+
+        function updateTop10Flows() {
+            $.get("/api/flows/top/10/rate", data => {
+                let html = "<table class='table table-striped' style='font-size: 8pt'>";
+                html += "<thead>";
+                html += "<th>Protocol</th>";
+                html += "<th>Local IP</th>";
+                html += "<th>Remote IP</th>";
+                html += "<th>UL ⬆️</th>";
+                html += "<th>DL ⬇️</th>";
+                html += "<th>UL RTT</th>";
+                html += "<th>DL RTT</th>";
+                html += "<th>TCP Retransmits</th>";
+                html += "<th>Remote ASN</th>";
+                html += "<th>Country</th>";
+                html += "</thead><tbody>";
+                for (var i = 0; i<data.length; i++) {
+                    //console.log(data[i]);
+                    html += "<tr>";
+                    html += "<td>" + data[i].analysis + "</td>";
+                    html += "<td>" + data[i].local_ip + "</td>";
+                    html += "<td>" + data[i].remote_ip + "</td>";
+                    // TODO: Check scaling
+                    html += "<td>" + scaleNumber(data[i].rate_estimate_bps[0]) + "</td>";
+                    html += "<td>" + scaleNumber(data[i].rate_estimate_bps[1]) + "</td>";
+                    html += "<td>" + scaleNanos(data[i].rtt_nanos[0]) + "</td>";
+                    html += "<td>" + scaleNanos(data[i].rtt_nanos[1]) + "</td>";
+                    html += "<td>" + data[i].tcp_retransmits[0] + "/" + data[i].tcp_retransmits[1] + "</td>";
+                    html += "<td>" + data[i].remote_asn_name + "</td>";
+                    html += "<td>" + data[i].remote_asn_country + "</td>";
+                    html += "</tr>";
+                }
+                html += "</tbody></table>";
+                $("#top10flows").html(html);
+            });
+        }
+
+        function updateTop10Endpoints() {
+            $.get("/api/flows/by_country", data => {
+                //console.log(data);
+                let html = "<table class='table table-striped' style='font-size: 8pt'>";
+                html += "<thead>";
+                html += "<th>Country</th>";
+                html += "<th>UL ⬆️</th>";
+                html += "<th>DL ⬇️</th>";
+                html += "<th>UL RTT</th>";
+                html += "<th>DL RTT</th>";
+                html += "</thead></tbody>";
+                let i = 0;
+                while (i < data.length && i < 10) {
+                    html += "<tr>";
+                    html += "<td>" + data[i][0] + "</td>";
+                    html += "<td>" + scaleNumber(data[i][1][0]) + "</td>";
+                    html += "<td>" + scaleNumber(data[i][1][1]) + "</td>";
+                    html += "<td>" + scaleNanos(data[i][2][0]) + "</td>";
+                    html += "<td>" + scaleNanos(data[i][2][1]) + "</td>";
+                    html += "</tr>";
+                    i += 1;
+                }
+                html += "</tbody></table>";
+                $("#top10ep").html(html);
+            });
+        }
+
+        function updateTop10Ethertypes() {
+            $.get("/api/flows/ether_protocol", data => {
+                let html = "<table class='table' style='font-size: 8pt'>";
+                html += "<thead>";
+                html += "<th>Protocol</th>";
+                html += "<th>UL ⬆️</th>";
+                html += "<th>DL ⬇️</th>";
+                html += "<th>UL RTT</th>";
+                html += "<th>DL RTT</th>";
+                html += "</thead></tbody>";
+                let row = data.EtherProtocols;
+                html += "<tr>";
+                html += "<td>IPv4</td>";
+                html += "<td>" + scaleNumber(row.v4_bytes[0]) + "</td>";
+                html += "<td>" + scaleNumber(row.v4_bytes[1]) + "</td>";
+                html += "<td>" + scaleNanos(row.v4_rtt[0]) + "</td>";
+                html += "<td>" + scaleNanos(row.v4_rtt[1]) + "</td>";
+                html += "</tr>";
+
+                html += "<tr>";
+                html += "<td>IPv6</td>";
+                html += "<td>" + scaleNumber(row.v6_bytes[0]) + "</td>";
+                html += "<td>" + scaleNumber(row.v6_bytes[1]) + "</td>";
+                html += "<td>" + scaleNanos(row.v6_rtt[0]) + "</td>";
+                html += "<td>" + scaleNanos(row.v6_rtt[1]) + "</td>";
+                html += "</tr>";
+
+                html += "</tbody></table>";
+                $("#top10eth").html(html);
+            });
+        }
+
+        function updateTop10Protocols() {
+            $.get("/api/flows/ip_protocol", data => {
+                let html = "<table class='table' style='font-size: 8pt'>";
+                html += "<thead>";
+                html += "<th>Protocol</th>";
+                html += "<th>UL ⬆️</th>";
+                html += "<th>DL ⬇️</th>";
+                html += "</thead></tbody>";
+
+                for (i=0; i<data.length; i++) {
+                    html += "<tr>";
+                    html += "<td>" + data[i][0] + "</td>";
+                    html += "<td>" + scaleNumber(data[i][1][0]) + "</td>";
+                    html += "<td>" + scaleNumber(data[i][1][1]) + "</td>";
+                    html += "</tr>";
+                }
+                html += "</tbody></table>";
+                $("#top10pro").html(html);
+            });
+        }
+
+        let top10view = "circuits";
+        let worst10view = "rtt";
+
+        function changeBottom10(visible) {
+            const bottom10 = ["worstRtt", "worstTcp"];
+            for (let i=0; i<bottom10.length; i++) {
+                $("#" + bottom10[i]).hide();
+                $("#btn" + bottom10[i]).removeClass("btn-success");
+                $("#btn" + bottom10[i]).addClass("btn-primary");
+            }
+            $("#" + visible).show();
+            $("#btn" + visible).removeClass("btn-primary");
+            $("#btn" + visible).addClass("btn-success");
+        }
+
+        function showWorstRtt() {
+            changeBottom10("worstRtt");
+            worst10view = "rtt";
+        }
+
+        function showWorstTcp() {
+            changeBottom10("worstTcp");
+            worst10view = "tcp";
+        }
+
+        function changeTop10(visible) {
+            const top10 = ["top10dl", "top10flows", "top10ep", "top10eth", "top10pro"];
+            for (let i=0; i<top10.length; i++) {
+                $("#" + top10[i]).hide();
+                $("#btn" + top10[i]).removeClass("btn-success");
+                $("#btn" + top10[i]).addClass("btn-primary");
+            }
+            $("#" + visible).show();
+            $("#btn" + visible).removeClass("btn-primary");
+            $("#btn" + visible).addClass("btn-success");
+        }
+
+        function showCircuits() {
+            changeTop10("top10dl");
+            top10view = "circuits";
+        }
+
+        function showFlows() {
+            changeTop10("top10flows");
+            top10view = "flows";
+        }
+
+        function showEndpoints() {
+            changeTop10("top10ep");
+            top10view = "endpoints";
+        }
+
+        function showProtocols() {
+            changeTop10("top10pro");
+            top10view = "protocols";
+        }
+
+        function showEthertypes() {
+            changeTop10("top10eth");
+            top10view = "ethertypes";
+        }
+
         var rttGraph = new RttHistogram();
 
         function updateHistogram() {
@@ -316,12 +524,27 @@
 
         function OneSecondCadence() {
             updateCurrentThroughput();
+            updateFlowCounter();
             updateSiteFunnel();
 
             if (tickCount % 5 == 0) {
                 updateHistogram();
-                updateWorst10();
-                updateTop10();
+                if (worst10view == "rtt") {
+                    updateWorst10();
+                } else if (worst10view == "tcp") {
+                    updateWorstTcp();
+                }
+                if (top10view == "circuits") {
+                    updateTop10();
+                } else if (top10view == "flows") {
+                    updateTop10Flows();
+                } else if (top10view == "endpoints") {
+                    updateTop10Endpoints();
+                } else if (top10view == "protocols") {
+                    updateTop10Protocols();
+                } else if (top10view == "ethertypes") {
+                    updateTop10Ethertypes();
+                }
             }
 
             if (tickCount % 10 == 0) {
@@ -342,6 +565,7 @@
 
             colorReloadButton();
             fillCurrentThroughput();
+            updateFlowCounter();
             updateCpu();
             updateRam();
             updateTop10();

src/rust/lqos_node_manager/static/showoff.html

@@ -0,0 +1,157 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+    <meta name="viewport" content="width=device-width,initial-scale=1,user-scalable=no">
+
+    <script src="/vendor/plotly-2.16.1.min.js"></script>
+    <script src="/vendor/jquery.min.js"></script>
+
+</head>
+
+<body style="background: black; font-family: Arial, Helvetica, sans-serif; height: 100%; margin: 0;">
+    <p style="font-size: 20pt; text-align: center; color: #dddddd" id="heading"></p>
+    <p style="font-size: 12pt; text-align: center; color: #dddddd">    
+        <a href="#" onclick="init()">Refresh</a>
+    </p>
+    <div id="chart" style="width:100vw; height: 100bh; background: black;">
+    </div>
+
+        <script>
+            var option;
+            var routes = [];
+
+            function lerpColor(color1, color2, weight) {
+                var r = Math.round(color1[0] + (color2[0] - color1[0]) * weight);
+                var g = Math.round(color1[1] + (color2[1] - color1[1]) * weight);
+                var b = Math.round(color1[2] + (color2[2] - color1[2]) * weight);
+                return `rgb(${r}, ${g}, ${b})`;
+            }
+
+            function getColorForWeight(weight) {
+                // Define our colors as [R, G, B]
+                const green = [0, 128, 0];
+                const orange = [255, 165, 0];
+                const red = [255, 0, 0];
+
+                if (weight <= 0.5) {
+                    // Scale weight to be from 0 to 1 for the green to orange transition
+                    const adjustedWeight = weight * 2;
+                    return lerpColor(green, orange, adjustedWeight);
+                } else {
+                    // Scale weight to be from 0 to 1 for the orange to red transition
+                    const adjustedWeight = (weight - 0.5) * 2;
+                    return lerpColor(orange, red, adjustedWeight);
+                }
+            }
+
+            function init() {
+                $.get("/api/flows/lat_lon", (worldData) => {
+                    let condensed = {};
+                    let totalBytes = 0;
+                    for (let i = 0; i < worldData.length; i++) {
+                        let label = worldData[i][2];
+                        if (label in condensed) {
+                            condensed[label][3] += worldData[i][3]; // Bytes
+                            totalBytes += worldData[i][3];
+                            if (worldData[i][4] != 0) {
+                                condensed[label][4].push(worldData[i][4]); // RTT
+                            }
+                        } else {
+                            condensed[label] = worldData[i];
+                            worldData[i][4] = [worldData[i][4]];
+                            totalBytes += worldData[i][3];
+                        }
+                    }
+
+                    let entries = [];
+                    for (const [key, value] of Object.entries(condensed)) {
+                        value[3] = value[3] / totalBytes;
+                        entries.push(value);
+                    }
+
+                    $("#heading").text("World Data. Now tracking " + worldData.length + " flows and " + entries.length + " locations.");
+
+                    var data = [{
+                        type: 'scattergeo',
+                        //locationmode: 'world',
+                        lat: [],
+                        lon: [],
+                        hoverinfo: 'text',
+                        text: [],
+                        marker: {
+                            size: [],
+                            color: [],
+                            line: {
+                                color: [],
+                                width: 2
+                            },
+                        }
+                    }];
+
+                    for (let i = 0; i < entries.length; i++) {
+                        var flow = entries[i];
+                        if (flow[4].length != 0) {
+                            var lat = flow[0];
+                            var lon = flow[1];
+                            var text = flow[2];
+
+                            var bytes = flow[3] * 20;
+                            if (bytes < 5) bytes = 5;
+
+                            let middle = flow[4].length -1;
+                            var rtt = flow[4][middle] / 1000000;
+                            rtt = rtt / 200;
+                            if (rtt > 1) rtt = 1;
+                            var color = getColorForWeight(rtt);
+                            data[0].lat.push(lat);
+                            data[0].lon.push(lon);
+                            data[0].text.push(text);
+                            data[0].marker.size.push(bytes);
+                            data[0].marker.line.color.push(color);
+                            data[0].marker.color.push(color);
+                        }
+                    }
+
+                    var layout = {
+                        autosize: true,
+                        margin: {
+                            l: 0,
+                            r: 0,
+                            b: 0,
+                            t: 0,
+                            pad: 0
+                        },
+                        paper_bgcolor: 'black',
+                        geo: {
+                            scope: 'world',
+                            projection: {
+                                type: 'natural earth'
+                            },
+                            showland: true,
+                            showocean: true,
+                            showlakes: true,
+                            showrivers: true,
+                            showcountries: true,
+                            landcolor: 'rgb(217, 217, 217)',
+                            subunitwidth: 1,
+                            countrywidth: 1,
+                            subunitcolor: 'rgb(255,255,255)',
+                            countrycolor: 'rgb(255,255,255)',
+                            framecolor: 'black',
+                            bgcolor: 'black',
+                        },
+                    };
+
+                    Plotly.newPlot('chart', data, layout, { responsive: true, displayModeBar: false });
+                });
+            }
+
+            $(document).ready(function () {
+                init()
+            });
+        </script>
+
+</html>

src/rust/lqos_sys/build.rs

@@ -123,7 +123,7 @@ fn main() {
     .header(&wrapper_target)
     // Tell cargo to invalidate the built crate whenever any of the
     // included header files changed.
-    .parse_callbacks(Box::new(bindgen::CargoCallbacks))
+    .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
     // Finish the builder and generate the bindings.
     .generate()
     // Unwrap the Result and panic on failure.

src/rust/lqos_sys/src/bpf/common/dissector.h

@@ -52,6 +52,9 @@ struct dissector_t
     __u16 window;
     __u32 tsval;
     __u32 tsecr;
+    __u32 sequence;
+    __u32 ack_seq;
+    __u64 now;
 };
 
 // Representation of the VLAN header type.
@@ -114,6 +117,9 @@ static __always_inline bool dissector_new(
     dissector->src_port = 0;
     dissector->dst_port = 0;
     dissector->tos = 0;
+    dissector->sequence = 0;
+    dissector->ack_seq = 0;
+    dissector->now = bpf_ktime_get_boot_ns();
 
     // Check that there's room for an ethernet header
     if SKB_OVERFLOW (dissector->start, dissector->end, ethhdr)
@@ -278,11 +284,11 @@ static __always_inline bool dissector_find_l3_offset(
 
 static __always_inline struct tcphdr *get_tcp_header(struct dissector_t *dissector)
 {
-    if (dissector->eth_type == ETH_P_IP)
+    if (dissector->eth_type == ETH_P_IP && dissector->ip_header.iph->protocol == IPPROTO_TCP)
     {
         return (struct tcphdr *)((char *)dissector->ip_header.iph + (dissector->ip_header.iph->ihl * 4));
     }
-    else if (dissector->eth_type == ETH_P_IPV6)
+    else if (dissector->eth_type == ETH_P_IPV6 && dissector->ip_header.ip6h->nexthdr == IPPROTO_TCP)
     {
         return (struct tcphdr *)(dissector->ip_header.ip6h + 1);
     }
@@ -315,6 +321,17 @@ static __always_inline struct icmphdr *get_icmp_header(struct dissector_t *disse
     return NULL;
 }
 
+#define DIS_TCP_FIN 1
+#define DIS_TCP_SYN 2
+#define DIS_TCP_RST 4
+#define DIS_TCP_PSH 8
+#define DIS_TCP_ACK 16
+#define DIS_TCP_URG 32
+#define DIS_TCP_ECE 64
+#define DIS_TCP_CWR 128
+
+#define BITCHECK(flag) (dissector->tcp_flags & flag)
+
 static __always_inline void snoop(struct dissector_t *dissector)
 {
     switch (dissector->ip_protocol)
@@ -331,17 +348,19 @@ static __always_inline void snoop(struct dissector_t *dissector)
             dissector->src_port = hdr->source;
             dissector->dst_port = hdr->dest;
             __u8 flags = 0;
-            if (hdr->fin) flags |= 1;
-            if (hdr->syn) flags |= 2;
-            if (hdr->rst) flags |= 4;
-            if (hdr->psh) flags |= 8;
-            if (hdr->ack) flags |= 16;
-            if (hdr->urg) flags |= 32;
-            if (hdr->ece) flags |= 64;
-            if (hdr->cwr) flags |= 128;
+            if (hdr->fin) flags |= DIS_TCP_FIN;
+            if (hdr->syn) flags |= DIS_TCP_SYN;
+            if (hdr->rst) flags |= DIS_TCP_RST;
+            if (hdr->psh) flags |= DIS_TCP_PSH;
+            if (hdr->ack) flags |= DIS_TCP_ACK;
+            if (hdr->urg) flags |= DIS_TCP_URG;
+            if (hdr->ece) flags |= DIS_TCP_ECE;
+            if (hdr->cwr) flags |= DIS_TCP_CWR;
 
             dissector->tcp_flags = flags;
             dissector->window = hdr->window;
+            dissector->sequence = hdr->seq;
+            dissector->ack_seq = hdr->ack_seq;
 
             parse_tcp_ts(hdr, dissector->end, &dissector->tsval, &dissector->tsecr);
         }
@@ -399,6 +418,7 @@ static __always_inline bool dissector_find_ip_header(
         dissector->ip_protocol = dissector->ip_header.iph->protocol;
         dissector->tos = dissector->ip_header.iph->tos;
         snoop(dissector);
+
         return true;
     }
     break;
@@ -416,7 +436,7 @@ static __always_inline bool dissector_find_ip_header(
         encode_ipv6(&dissector->ip_header.ip6h->saddr, &dissector->src_ip);
         encode_ipv6(&dissector->ip_header.ip6h->daddr, &dissector->dst_ip);
         dissector->ip_protocol = dissector->ip_header.ip6h->nexthdr;
-        dissector->ip_header.ip6h->flow_lbl[0]; // Is this right?
+        dissector->tos = dissector->ip_header.ip6h->flow_lbl[0]; // Is this right?
         snoop(dissector);
         return true;
     }
@@ -424,4 +444,4 @@ static __always_inline bool dissector_find_ip_header(
     default:
         return false;
     }
-}
+}

src/rust/lqos_sys/src/bpf/common/flows.h

@@ -0,0 +1,359 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+// TCP flow monitor system
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "dissector.h"
+#include "debug.h"
+
+
+#define SECOND_IN_NANOS 1000000000
+#define TWO_SECONDS_IN_NANOS 2000000000
+#define MS_IN_NANOS_T10 10000
+#define HALF_MBPS_IN_BYTES_PER_SECOND 62500
+#define RTT_RING_SIZE 4
+//#define TIMESTAMP_INTERVAL_NANOS 10000000
+
+// Some helpers to make understanding direction easier
+// for readability.
+#define TO_INTERNET 2
+#define FROM_INTERNET 1
+#define TO_LOCAL 1
+#define FROM_LOCAL 2
+
+// Defines a TCP connection flow key
+struct flow_key_t {
+    struct in6_addr src;
+    struct in6_addr dst;
+    __u16 src_port;
+    __u16 dst_port;
+    __u8 protocol;
+    __u8 pad;
+    __u8 pad1;
+    __u8 pad2;
+};
+
+// TCP connection flow entry
+struct flow_data_t {
+    // Time (nanos) when the connection was established
+    __u64 start_time;
+    // Time (nanos) when the connection was last seen
+    __u64 last_seen;
+    // Bytes transmitted
+    __u64 bytes_sent[2];
+    // Packets transmitted
+    __u64 packets_sent[2];
+    // Clock for the next rate estimate
+    __u64 next_count_time[2];
+    // Clock for the previous rate estimate
+    __u64 last_count_time[2];
+    // Bytes at the next rate estimate
+    __u64 next_count_bytes[2];
+    // Rate estimate
+    __u32 rate_estimate_bps[2];
+    // Sequence number of the last packet
+    __u32 last_sequence[2];
+    // Acknowledgement number of the last packet
+    __u32 last_ack[2];
+    // Retransmit Counters (Also catches duplicates and out-of-order packets)
+    __u16 tcp_retransmits[2];
+    // Timestamp values
+    __u32 tsval[2];
+    __u32 tsecr[2];
+    // When did the timestamp change?
+    __u64 ts_change_time[2];
+    // Has the connection ended?
+    // 0 = Alive, 1 = FIN, 2 = RST
+    __u8 end_status;
+    // TOS
+    __u8 tos;
+    // IP Flags
+    __u8 ip_flags;
+    // Padding
+    __u8 pad;
+};
+
+// Map for tracking TCP flow progress.
+// This is pinned and not per-CPU, because half the data appears on either side of the bridge.
+struct
+{
+    __uint(type, BPF_MAP_TYPE_HASH); // TODO: BPF_MAP_TYPE_LRU_PERCPU_HASH?
+    __type(key, struct flow_key_t);
+    __type(value, struct flow_data_t);
+    __uint(max_entries, MAX_FLOWS);
+    __uint(pinning, LIBBPF_PIN_BY_NAME);
+} flowbee SEC(".maps");
+
+// Ringbuffer to userspace for recording RTT events
+struct {
+	__uint(type, BPF_MAP_TYPE_RINGBUF);
+	__uint(max_entries, 256 * 1024 /* 256 KB */);
+} flowbee_events SEC(".maps");
+
+// Event structure we send for events.
+struct flowbee_event {
+    struct flow_key_t key;
+    __u64 round_trip_time;
+    __u32 effective_direction;
+};
+
+// Construct an empty flow_data_t structure, using default values.
+static __always_inline struct flow_data_t new_flow_data(
+    // The packet dissector from the previous step
+    struct dissector_t *dissector
+) {
+    struct flow_data_t data = {
+        .start_time = dissector->now,
+        .bytes_sent = { 0, 0 },
+        .packets_sent = { 0, 0 },
+        // Track flow rates at an MS scale rather than per-second
+        // to minimize rounding errors.
+        .next_count_time = { dissector->now + SECOND_IN_NANOS, dissector->now + SECOND_IN_NANOS },
+        .last_count_time = { dissector->now, dissector->now },
+        .next_count_bytes = { 0, 0 }, // Should be packet size, that isn't working?
+        .rate_estimate_bps = { 0, 0 },
+        .last_sequence = { 0, 0 },
+        .last_ack = { 0, 0 },
+        .tcp_retransmits = { 0, 0 },
+        .tsval = { 0, 0 },
+        .tsecr = { 0, 0 },
+        .ts_change_time = { 0, 0 },
+        .end_status = 0,
+        .tos = 0,
+        .ip_flags = 0,
+    };
+    return data;
+}
+
+// Construct a flow_key_t structure from a dissector_t. This represents the
+// unique key for a flow in the flowbee map.
+static __always_inline struct flow_key_t build_flow_key(
+    struct dissector_t *dissector, // The packet dissector from the previous step
+    u_int8_t direction // The direction of the packet (1 = to internet, 2 = to local network)
+) {
+    __u16 src_port = direction == FROM_INTERNET ? bpf_htons(dissector->src_port) : bpf_htons(dissector->dst_port);
+    __u16 dst_port = direction == FROM_INTERNET ? bpf_htons(dissector->dst_port) : bpf_htons(dissector->src_port);
+    struct in6_addr src = direction == FROM_INTERNET ? dissector->src_ip : dissector->dst_ip;
+    struct in6_addr dst = direction == FROM_INTERNET ? dissector->dst_ip : dissector->src_ip;
+
+    return (struct flow_key_t) {
+        .src = src,
+        .dst = dst,
+        .src_port = src_port,
+        .dst_port = dst_port,
+        .protocol = dissector->ip_protocol,
+        .pad = 0,
+        .pad1 = 0,
+        .pad2 = 0
+    };
+}
+
+// Update the flow data with the current packet's information.
+// * Update the timestamp of the last seen packet
+// * Update the bytes and packets sent
+// * Update the rate estimate (if it is time to do so)
+static __always_inline void update_flow_rates(
+    // The packet dissector from the previous step
+    struct dissector_t *dissector,
+    // The rate index (0 = to internet, 1 = to local network)
+    u_int8_t rate_index,
+    // The flow data structure to update
+    struct flow_data_t *data
+) {
+    data->last_seen = dissector->now;
+    data->end_status = 0; // Reset the end status
+
+    // Update bytes and packets sent
+    data->bytes_sent[rate_index] += dissector->skb_len;
+    data->packets_sent[rate_index]++;
+
+    if (dissector->now > data->next_count_time[rate_index]) {
+        // Calculate the rate estimate
+        __u64 bits = (data->bytes_sent[rate_index] - data->next_count_bytes[rate_index])*8;
+        __u64 time = (dissector->now - data->last_count_time[rate_index]) / SECOND_IN_NANOS; // 1 Second
+        data->rate_estimate_bps[rate_index] = (bits/time); // bits per second
+        data->next_count_time[rate_index] = dissector->now + SECOND_IN_NANOS;
+        data->next_count_bytes[rate_index] = data->bytes_sent[rate_index];
+        data->last_count_time[rate_index] = dissector->now;
+        //bpf_debug("[FLOWS] Rate Estimate: %llu", data->rate_estimate_bps[rate_index]);
+    }
+}
+
+// Handle Per-Flow ICMP Analysis
+static __always_inline void process_icmp(
+    struct dissector_t *dissector,
+    u_int8_t direction,
+    u_int8_t rate_index,
+    u_int8_t other_rate_index
+) {
+    struct flow_key_t key = build_flow_key(dissector, direction);
+    struct flow_data_t *data = bpf_map_lookup_elem(&flowbee, &key);
+    if (data == NULL) {
+        // There isn't a flow, so we need to make one
+        struct flow_data_t new_data = new_flow_data(dissector);
+        if (bpf_map_update_elem(&flowbee, &key, &new_data, BPF_ANY) != 0) {
+            bpf_debug("[FLOWS] Failed to add new flow to map");
+            return;
+        }
+        data = bpf_map_lookup_elem(&flowbee, &key);
+        if (data == NULL) return;
+    }
+    update_flow_rates(dissector, rate_index, data);
+}
+
+// Handle Per-Flow UDP Analysis
+static __always_inline void process_udp(
+    struct dissector_t *dissector,
+    u_int8_t direction,
+    u_int8_t rate_index,
+    u_int8_t other_rate_index
+) {
+    struct flow_key_t key = build_flow_key(dissector, direction);
+    struct flow_data_t *data = bpf_map_lookup_elem(&flowbee, &key);
+    if (data == NULL) {
+        // There isn't a flow, so we need to make one
+        struct flow_data_t new_data = new_flow_data(dissector);
+        if (bpf_map_update_elem(&flowbee, &key, &new_data, BPF_ANY) != 0) {
+            bpf_debug("[FLOWS] Failed to add new flow to map");
+            return;
+        }
+        data = bpf_map_lookup_elem(&flowbee, &key);
+        if (data == NULL) return;
+    }
+    update_flow_rates(dissector, rate_index, data);
+}
+
+// Store the most recent sequence and ack numbers, and detect retransmissions.
+// This will also trigger on duplicate packets, and out-of-order - but those
+// are both an indication that you have issues anyway. So that's ok by me!
+static __always_inline void detect_retries(
+    struct dissector_t *dissector,
+    u_int8_t rate_index,
+    struct flow_data_t *data
+) {
+    __u32 sequence = bpf_ntohl(dissector->sequence);
+    __u32 ack_seq = bpf_ntohl(dissector->ack_seq);
+    if (
+        data->last_sequence[rate_index] != 0 && // We have a previous sequence number
+        sequence < data->last_sequence[rate_index] && // This is a retransmission
+        (
+            data->last_sequence[rate_index] > 0x10000 && // Wrap around possible
+            sequence > data->last_sequence[rate_index] - 0x10000 // Wrap around didn't occur            
+        ) 
+    ) {
+        // This is a retransmission
+        data->tcp_retransmits[rate_index]++;
+    }
+
+    // Store the sequence and ack numbers for the next packet
+    data->last_sequence[rate_index] = sequence;
+    data->last_ack[rate_index] = ack_seq;
+}
+
+// Handle Per-Flow TCP Analysis
+static __always_inline void process_tcp(
+    struct dissector_t *dissector,
+    u_int8_t direction,
+    u_int8_t rate_index,
+    u_int8_t other_rate_index
+) {
+    // SYN packet indicating the start of a conversation. We are explicitly ignoring
+    // SYN-ACK packets, we just want to catch the opening of a new connection.
+    if ((BITCHECK(DIS_TCP_SYN) && !BITCHECK(DIS_TCP_ACK) && direction == TO_INTERNET) || 
+        (BITCHECK(DIS_TCP_SYN) && !BITCHECK(DIS_TCP_ACK) && direction == FROM_INTERNET)) {
+        // A customer is requesting a new TCP connection. That means
+        // we need to start tracking this flow.
+        #ifdef VERBOSE
+        bpf_debug("[FLOWS] New TCP Connection Detected (%u)", direction);
+        #endif
+        struct flow_key_t key = build_flow_key(dissector, direction);
+        struct flow_data_t data = new_flow_data(dissector);
+        data.tos = dissector->tos;
+        data.ip_flags = 0; // Obtain these
+        if (bpf_map_update_elem(&flowbee, &key, &data, BPF_ANY) != 0) {
+            bpf_debug("[FLOWS] Failed to add new flow to map");
+        }
+        return;
+    }
+
+    // Build the flow key to uniquely identify this flow
+    struct flow_key_t key = build_flow_key(dissector, direction);
+    struct flow_data_t *data = bpf_map_lookup_elem(&flowbee, &key);
+    if (data == NULL) {
+        // If it isn't a flow we're tracking, bail out now
+        #ifdef VERBOSE
+        bpf_debug("Bailing");
+        #endif
+        return;
+    }
+
+    // Update the flow data with the current packet's information
+    update_flow_rates(dissector, rate_index, data);
+
+    // Sequence and Acknowledgement numbers
+    detect_retries(dissector, rate_index, data);
+
+    // Timestamps to calculate RTT
+    if (dissector->tsval != 0) {
+        //bpf_debug("[FLOWS][%d] TSVAL: %u, TSECR: %u", direction, tsval, tsecr);
+        if (dissector->tsval != data->tsval[rate_index] && dissector->tsecr != data->tsecr[rate_index]) {
+
+            if (
+                dissector->tsecr == data->tsval[other_rate_index] &&
+                (data->rate_estimate_bps[rate_index] > 0 ||
+                data->rate_estimate_bps[other_rate_index] > 0 )
+            ) {
+                __u64 elapsed = dissector->now - data->ts_change_time[other_rate_index];
+                if (elapsed < TWO_SECONDS_IN_NANOS) {
+                    struct flowbee_event event = { 0 };
+                    event.key = key;
+                    event.round_trip_time = elapsed;
+                    event.effective_direction = rate_index;
+                    bpf_ringbuf_output(&flowbee_events, &event, sizeof(event), 0);
+                }
+            }
+
+            data->ts_change_time[rate_index] = dissector->now;
+            data->tsval[rate_index] = dissector->tsval;
+            data->tsecr[rate_index] = dissector->tsecr;
+        }
+    }
+
+    // Has the connection ended?
+    if (BITCHECK(DIS_TCP_FIN)) {
+        data->end_status = 1;
+    } else if (BITCHECK(DIS_TCP_RST)) {
+        data->end_status = 2;
+    }
+}
+
+// Note that this duplicates a lot of what we do for "snoop" - we're hoping
+// to replace both it and the old RTT system.
+static __always_inline void track_flows(
+    struct dissector_t *dissector, // The packet dissector from the previous step
+    u_int8_t direction // The direction of the packet (1 = to internet, 2 = to local network)
+) {
+    u_int8_t rate_index;
+    u_int8_t other_rate_index;
+    if (direction == TO_INTERNET) {
+        rate_index = 0;
+        other_rate_index = 1;
+    } else {
+        rate_index = 1;
+        other_rate_index = 0;
+    }
+
+    // Pass to the appropriate protocol handler
+    switch (dissector->ip_protocol)
+    {
+        case IPPROTO_TCP: process_tcp(dissector, direction, rate_index, other_rate_index); break;
+        case IPPROTO_UDP: process_udp(dissector, direction, rate_index, other_rate_index); break;
+        case IPPROTO_ICMP: process_icmp(dissector, direction, rate_index, other_rate_index); break;
+        default: {
+            #ifdef VERBOSE
+            bpf_debug("[FLOWS] Unsupported protocol: %d", dissector->ip_protocol);
+            #endif
+        }
+    }
+}

src/rust/lqos_sys/src/bpf/common/heimdall.h

@@ -60,33 +60,6 @@ struct heimdall_event {
     __u8 dump[PACKET_OCTET_SIZE];
 };
 
-struct heimdall_key
-{
-    struct in6_addr src;
-    struct in6_addr dst;
-    __u8 ip_protocol;
-    __u16 src_port;
-    __u16 dst_port;
-    __u8 pad;
-};
-
-struct heimdall_data {
-    __u64 last_seen;
-    __u64 bytes;
-    __u64 packets;
-    __u8 tos;
-};
-
-// Map for tracking flow information in-kernel for watched IPs
-struct
-{
-    __uint(type, BPF_MAP_TYPE_LRU_PERCPU_HASH);
-    __type(key, struct heimdall_key);
-    __type(value, struct heimdall_data);
-    __uint(max_entries, MAX_FLOWS);
-    __uint(pinning, LIBBPF_PIN_BY_NAME);
-} heimdall SEC(".maps");
-
 static __always_inline __u8 get_heimdall_mode()
 {
     __u32 index = 0;
@@ -122,45 +95,9 @@ static __always_inline bool is_heimdall_watching(struct dissector_t *dissector,
 
 static __always_inline void update_heimdall(struct dissector_t *dissector, __u32 size, __u8 mode)
 {
-    if (mode == 1) {
-        // Don't report any non-ICMP without ports
-        if (dissector->ip_protocol != 1 && (dissector->src_port == 0 || dissector->dst_port == 0))
-            return;
-        // Don't report ICMP with invalid numbers
-        if (dissector->ip_protocol == 1 && dissector->src_port > 18) return;
-        struct heimdall_key key = {0};
-        key.src = dissector->src_ip;
-        key.dst = dissector->dst_ip;
-        key.ip_protocol = dissector->ip_protocol;
-        key.src_port = bpf_ntohs(dissector->src_port);
-        key.dst_port = bpf_ntohs(dissector->dst_port);
-        struct heimdall_data *counter = (struct heimdall_data *)bpf_map_lookup_elem(&heimdall, &key);
-        if (counter)
-        {
-            counter->last_seen = bpf_ktime_get_boot_ns();
-            counter->packets += 1;
-            counter->bytes += size;
-            if (dissector->tos != 0)
-            {
-                counter->tos = dissector->tos;
-            }
-        }
-        else
-        {
-            struct heimdall_data counter = {0};
-            counter.last_seen = bpf_ktime_get_boot_ns();
-            counter.bytes = size;
-            counter.packets = 1;
-            counter.tos = dissector->tos;
-            if (bpf_map_update_elem(&heimdall, &key, &counter, BPF_NOEXIST) != 0)
-            {
-                bpf_debug("Failed to insert tracking");
-            }
-            //bpf_debug("Inserted tracking");
-        }
-    } else if (mode == 2) {
+    if (mode == 2) {
         struct heimdall_event event = {0};
-        event.timetamp = bpf_ktime_get_boot_ns();
+        event.timetamp = dissector->now;
         event.src = dissector->src_ip;
         event.dst = dissector->dst_ip;
         event.src_port = dissector->src_port;

src/rust/lqos_sys/src/bpf/common/lpm.h

@@ -11,7 +11,6 @@
 #include "maximums.h"
 #include "debug.h"
 #include "dissector.h"
-#include "dissector_tc.h"
 
 // Data structure used for map_ip_hash
 struct ip_hash_info {
@@ -47,60 +46,39 @@ struct {
 	__uint(map_flags, BPF_F_NO_PREALLOC);
 } map_ip_to_cpu_and_tc_recip SEC(".maps");
 
+// Determine the effective direction of a packet
+static __always_inline u_int8_t determine_effective_direction(int direction, __be16 internet_vlan, struct dissector_t * dissector) {
+    if (direction < 3) {
+        return direction;
+    } else {
+        if (dissector->current_vlan == internet_vlan) {
+            return 1;
+        } else {
+            return 2;
+        }
+    }
+}
+
 // Performs an LPM lookup for an `ip_hash.h` encoded address, taking
 // into account redirection and "on a stick" setup.
 static __always_inline struct ip_hash_info * setup_lookup_key_and_tc_cpu(
-    // The "direction" constant from the main program. 1 = Internet,
-    // 2 = LAN, 3 = Figure it out from VLAN tags
-    int direction,
+    // This must have been pre-calculated by `determine_effective_direction`.
+    u_int8_t direction,
     // Pointer to the "lookup key", which should contain the IP address
     // to search for. Prefix length will be set for you.
     struct ip_hash_key * lookup_key,
     // Pointer to the traffic dissector.
-    struct dissector_t * dissector,
-    // Which VLAN represents the Internet, in redirection scenarios? (i.e.
-    // when direction == 3)
-    __be16 internet_vlan,
-    // Out variable setting the real "direction" of traffic when it has to
-    // be calculated.
-    int * out_effective_direction
+    struct dissector_t * dissector
 ) 
 {
     lookup_key->prefixlen = 128;
-    // Normal preset 2-interface setup, no need to calculate any direction
-    // related VLANs.
-    if (direction < 3) {
-        lookup_key->address = (direction == 1) ? dissector->dst_ip : 
-            dissector->src_ip;
-        *out_effective_direction = direction;
-        struct ip_hash_info * ip_info = bpf_map_lookup_elem(
-            &map_ip_to_cpu_and_tc, 
-            lookup_key
-        );
-        return ip_info;
-    } else {
-        if (dissector->current_vlan == internet_vlan) {
-            // Packet is coming IN from the Internet.
-            // Therefore it is download.
-            lookup_key->address = dissector->dst_ip;
-            *out_effective_direction = 1;
-            struct ip_hash_info * ip_info = bpf_map_lookup_elem(
-                &map_ip_to_cpu_and_tc, 
-                lookup_key
-            );
-            return ip_info;
-        } else {
-            // Packet is coming IN from the ISP.
-            // Therefore it is UPLOAD.
-            lookup_key->address = dissector->src_ip;
-            *out_effective_direction = 2;
-            struct ip_hash_info * ip_info = bpf_map_lookup_elem(
-                &map_ip_to_cpu_and_tc_recip, 
-                lookup_key
-            );
-            return ip_info;
-        }
-    }
+    lookup_key->address = (direction == 1) ? dissector->dst_ip : 
+        dissector->src_ip;
+    struct ip_hash_info * ip_info = bpf_map_lookup_elem(
+        &map_ip_to_cpu_and_tc, 
+        lookup_key
+    );
+    return ip_info;
 }
 
 // For the TC side, the dissector is different. Operates similarly to

src/rust/lqos_sys/src/bpf/common/tcp_rtt.h

@@ -1,718 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
-Based on the GPLv2 xdp-pping project 
-(https://github.com/xdp-project/bpf-examples/tree/master/pping)
-
-xdp_pping is based on the ideas in Dr. Kathleen Nichols' pping
-utility: https://github.com/pollere/pping
-   and the papers around "Listening to Networks":
-http://www.pollere.net/Pdfdocs/ListeningGoog.pdf
-
-My modifications are Copyright 2022, Herbert Wolverson
-(Bracket Productions)
-*/
-/* Shared structures between userspace and kernel space
- */
- 
-/* Implementation of pping inside the kernel
- * classifier
- */
-#ifndef __TC_CLASSIFY_KERN_PPING_H
-#define __TC_CLASSIFY_KERN_PPING_H
-
-#include <linux/bpf.h>
-#include <bpf/bpf_helpers.h>
-#include <linux/pkt_cls.h>
-#include <linux/in.h>
-#include <linux/in6.h>
-#include <linux/if_ether.h>
-#include <linux/ip.h>
-#include <linux/ipv6.h>
-#include <linux/tcp.h>
-#include <bpf/bpf_endian.h>
-#include <stdbool.h>
-#include "tc_classify_kern_pping_common.h"
-#include "maximums.h"
-#include "debug.h"
-#include "ip_hash.h"
-#include "dissector_tc.h"
-#include "tcp_opts.h"
-
-#define MAX_MEMCMP_SIZE 128
-
-struct parsing_context
-{
-    struct tcphdr *tcp;
-    __u64 now;
-    struct tc_dissector_t * dissector;
-    struct in6_addr * active_host;
-};
-
-/* Event type recorded for a packet flow */
-enum __attribute__((__packed__)) flow_event_type
-{
-    FLOW_EVENT_NONE,
-    FLOW_EVENT_OPENING,
-    FLOW_EVENT_CLOSING,
-    FLOW_EVENT_CLOSING_BOTH
-};
-
-enum __attribute__((__packed__)) connection_state
-{
-    CONNECTION_STATE_EMPTY,
-    CONNECTION_STATE_WAITOPEN,
-    CONNECTION_STATE_OPEN,
-    CONNECTION_STATE_CLOSED
-};
-
-struct flow_state
-{
-    __u64 last_timestamp;
-    __u32 last_id;
-    __u32 outstanding_timestamps;
-    enum connection_state conn_state;
-    __u8 reserved[2];
-};
-
-/*
- * Stores flowstate for both direction (src -> dst and dst -> src) of a flow
- *
- * Uses two named members instead of array of size 2 to avoid hassels with
- * convincing verifier that member access is not out of bounds
- */
-struct dual_flow_state
-{
-    struct flow_state dir1;
-    struct flow_state dir2;
-};
-
-/*
- * Struct filled in by parse_packet_id.
- *
- * Note: As long as parse_packet_id is successful, the flow-parts of pid
- * and reply_pid should be valid, regardless of value for pid_valid and
- * reply_pid valid. The *pid_valid members are there to indicate that the
- * identifier part of *pid are valid and can be used for timestamping/lookup.
- * The reason for not keeping the flow parts as an entirely separate members
- * is to save some performance by avoid doing a copy for lookup/insertion
- * in the packet_ts map.
- */
-struct packet_info
-{
-    __u64 time; // Arrival time of packet
-    //__u32 payload;              // Size of packet data (excluding headers)
-    struct packet_id pid;       // flow + identifier to timestamp (ex. TSval)
-    struct packet_id reply_pid; // rev. flow + identifier to match against (ex. TSecr)
-    //__u32 ingress_ifindex;      // Interface packet arrived on (if is_ingress, otherwise not valid)    
-    bool pid_flow_is_dfkey;              // Used to determine which member of dualflow state to use for forward direction
-    bool pid_valid;                      // identifier can be used to timestamp packet
-    bool reply_pid_valid;                // reply_identifier can be used to match packet
-    enum flow_event_type event_type;     // flow event triggered by packet
-};
-
-/*
- * Struct filled in by protocol id parsers (ex. parse_tcp_identifier)
- */
-struct protocol_info
-{
-    __u32 pid;
-    __u32 reply_pid;
-    bool pid_valid;
-    bool reply_pid_valid;
-    enum flow_event_type event_type;
-};
-
-
-
-/* Map Definitions */
-struct
-{
-    __uint(type, BPF_MAP_TYPE_LRU_HASH);
-    __type(key, struct packet_id);
-    __type(value, __u64);
-    __uint(max_entries, MAX_PACKETS);
-    __uint(pinning, LIBBPF_PIN_BY_NAME);
-//    __uint(map_flags, BPF_F_NO_PREALLOC);
-} packet_ts SEC(".maps");
-
-struct
-{
-    __uint(type, BPF_MAP_TYPE_LRU_HASH);
-    __type(key, struct network_tuple);
-    __type(value, struct dual_flow_state);
-    __uint(max_entries, MAX_FLOWS);
-    __uint(pinning, LIBBPF_PIN_BY_NAME);
-//    __uint(map_flags, BPF_F_NO_PREALLOC);
-} flow_state SEC(".maps");
-
-struct
-{
-    __uint(type, BPF_MAP_TYPE_LRU_HASH);
-    __type(key, struct in6_addr); // Keyed to the IP address
-    __type(value, struct rotating_performance);
-    __uint(max_entries, IP_HASH_ENTRIES_MAX);
-    __uint(pinning, LIBBPF_PIN_BY_NAME);
-//    __uint(map_flags, BPF_F_NO_PREALLOC);
-
-} rtt_tracker SEC(".maps");
-
-// Mask for IPv6 flowlabel + traffic class -  used in fib lookup
-#define IPV6_FLOWINFO_MASK __cpu_to_be32(0x0FFFFFFF)
-
-#ifndef AF_INET
-#define AF_INET 2
-#endif
-#ifndef AF_INET6
-#define AF_INET6 10
-#endif
-
-#define MAX_TCP_OPTIONS 10
-
-/* Functions */
-
-/*
- * Convenience function for getting the corresponding reverse flow.
- * PPing needs to keep track of flow in both directions, and sometimes
- * also needs to reverse the flow to report the "correct" (consistent
- * with Kathie's PPing) src and dest address.
- */
-static __always_inline void reverse_flow(
-    struct network_tuple *dest, 
-    struct network_tuple *src
-) {
-    dest->ipv = src->ipv;
-    dest->proto = src->proto;
-    dest->saddr = src->daddr;
-    dest->daddr = src->saddr;
-    dest->reserved = 0;
-}
-
-/*
- * Can't seem to get __builtin_memcmp to work, so hacking my own
- *
- * Based on https://githubhot.com/repo/iovisor/bcc/issues/3559,
- * __builtin_memcmp should work constant size but I still get the "failed to
- * find BTF for extern" error.
- */
-static __always_inline int my_memcmp(
-    const void *s1_, 
-    const void *s2_, 
-    __u32 size
-) {
-    const __u8 *s1 = (const __u8 *)s1_, *s2 = (const __u8 *)s2_;
-    int i;
-
-    for (i = 0; i < MAX_MEMCMP_SIZE && i < size; i++)
-    {
-        if (s1[i] != s2[i])
-            return s1[i] > s2[i] ? 1 : -1;
-    }
-
-    return 0;
-}
-
-static __always_inline bool is_dualflow_key(struct network_tuple *flow)
-{
-    return my_memcmp(&flow->saddr, &flow->daddr, sizeof(flow->saddr)) <= 0;
-}
-
-static __always_inline struct flow_state *fstate_from_dfkey(
-    struct dual_flow_state *df_state,
-    bool is_dfkey
-) {
-    if (!df_state) {
-        return (struct flow_state *)NULL;
-    }
-
-    return is_dfkey ? &df_state->dir1 : &df_state->dir2;
-}
-
-/*
- * Attempts to fetch an identifier for TCP packets, based on the TCP timestamp
- * option.
- *
- * Will use the TSval as pid and TSecr as reply_pid, and the TCP source and dest
- * as port numbers.
- *
- * If successful, tcph, sport, dport and proto_info will be set
- * appropriately and 0 will be returned.
- * On failure -1 will be returned (and arguments will not be set).
- */
-static __always_inline int parse_tcp_identifier(
-    struct parsing_context *context,
-    __u16 *sport,
-    __u16 *dport, 
-    struct protocol_info *proto_info
-) {
-    if (parse_tcp_ts(context->tcp, context->dissector->end, &proto_info->pid, 
-        &proto_info->reply_pid) < 0) {
-        return -1; // Possible TODO, fall back on seq/ack instead
-    }
-
-    // Do not timestamp pure ACKs (no payload)
-    void *nh_pos = (context->tcp + 1) + (context->tcp->doff << 2);
-    proto_info->pid_valid = nh_pos - context->dissector->start < context->dissector->ctx->len || context->tcp->syn;
-
-    // Do not match on non-ACKs (TSecr not valid)
-    proto_info->reply_pid_valid = context->tcp->ack;
-
-    // Check if connection is opening/closing
-    if (context->tcp->rst)
-    {
-        proto_info->event_type = FLOW_EVENT_CLOSING_BOTH;
-    }
-    else if (context->tcp->fin)
-    {
-        proto_info->event_type = FLOW_EVENT_CLOSING;
-    }
-    else if (context->tcp->syn)
-    {
-        proto_info->event_type = FLOW_EVENT_OPENING;
-    }
-    else
-    {
-        proto_info->event_type = FLOW_EVENT_NONE;
-    }
-
-    *sport = bpf_ntohs(context->tcp->dest);
-    *dport = bpf_ntohs(context->tcp->source);
-
-    return 0;
-}
-
-/* This is a bit of a hackjob from the original */
-static __always_inline int parse_packet_identifier(
-    struct parsing_context *context, 
-    struct packet_info *p_info
-) {
-    p_info->time = context->now;
-    if (context->dissector->eth_type == ETH_P_IP)
-    {
-        p_info->pid.flow.ipv = AF_INET;
-        p_info->pid.flow.saddr.ip = context->dissector->src_ip;
-        p_info->pid.flow.daddr.ip = context->dissector->dst_ip;
-    }
-    else if (context->dissector->eth_type == ETH_P_IPV6)
-    {
-        p_info->pid.flow.ipv = AF_INET6;
-        p_info->pid.flow.saddr.ip = context->dissector->src_ip;
-        p_info->pid.flow.daddr.ip = context->dissector->dst_ip;
-    }
-    else
-    {
-        bpf_debug("Unknown protocol");
-        return -1;
-    }
-    //bpf_debug("IPs: %u %u", p_info->pid.flow.saddr.ip.in6_u.u6_addr32[3], p_info->pid.flow.daddr.ip.in6_u.u6_addr32[3]);
-
-    struct protocol_info proto_info;
-    int err = parse_tcp_identifier(context,
-                                   &p_info->pid.flow.saddr.port,
-                                   &p_info->pid.flow.daddr.port,
-                                   &proto_info);
-    if (err)
-        return -1;
-    //bpf_debug("Ports: %u %u", p_info->pid.flow.saddr.port, p_info->pid.flow.daddr.port);
-
-    // Sucessfully parsed packet identifier - fill in remaining members and return
-    p_info->pid.identifier = proto_info.pid;
-    p_info->pid_valid = proto_info.pid_valid;
-    p_info->reply_pid.identifier = proto_info.reply_pid;
-    p_info->reply_pid_valid = proto_info.reply_pid_valid;
-    p_info->event_type = proto_info.event_type;
-
-    if (p_info->pid.flow.ipv == AF_INET && p_info->pid.flow.ipv == AF_INET6) {
-        bpf_debug("Unknown internal protocol");
-        return -1;
-    }
-
-    p_info->pid_flow_is_dfkey = is_dualflow_key(&p_info->pid.flow);
-
-    reverse_flow(&p_info->reply_pid.flow, &p_info->pid.flow);
-
-    return 0;
-}
-
-static __always_inline struct network_tuple *
-get_dualflow_key_from_packet(struct packet_info *p_info)
-{
-    return p_info->pid_flow_is_dfkey ? &p_info->pid.flow : &p_info->reply_pid.flow;
-}
-
-/*
- * Initilizes an "empty" flow state based on the forward direction of the
- * current packet
- */
-static __always_inline void init_flowstate(struct flow_state *f_state,
-                                           struct packet_info *p_info)
-{
-    f_state->conn_state = CONNECTION_STATE_WAITOPEN;
-    f_state->last_timestamp = p_info->time;
-}
-
-static __always_inline void init_empty_flowstate(struct flow_state *f_state)
-{
-    f_state->conn_state = CONNECTION_STATE_EMPTY;
-}
-
-static __always_inline struct flow_state *
-get_flowstate_from_packet(struct dual_flow_state *df_state,
-                          struct packet_info *p_info)
-{
-    return fstate_from_dfkey(df_state, p_info->pid_flow_is_dfkey);
-}
-
-static __always_inline struct flow_state *
-get_reverse_flowstate_from_packet(struct dual_flow_state *df_state,
-                                  struct packet_info *p_info)
-{
-    return fstate_from_dfkey(df_state, !p_info->pid_flow_is_dfkey);
-}
-
-/*
- * Initilize a new (assumed 0-initlized) dual flow state based on the current
- * packet.
- */
-static __always_inline void init_dualflow_state(
-    struct dual_flow_state *df_state,
-    struct packet_info *p_info
-) {
-    struct flow_state *fw_state =
-        get_flowstate_from_packet(df_state, p_info);
-    struct flow_state *rev_state =
-        get_reverse_flowstate_from_packet(df_state, p_info);
-
-    init_flowstate(fw_state, p_info);
-    init_empty_flowstate(rev_state);
-}
-
-static __always_inline struct dual_flow_state *
-create_dualflow_state(
-    struct parsing_context *ctx, 
-    struct packet_info *p_info, 
-    bool *new_flow
-) {
-    struct network_tuple *key = get_dualflow_key_from_packet(p_info);
-    struct dual_flow_state new_state = {0};
-
-    init_dualflow_state(&new_state, p_info);
-    //new_state.dir1.tc_handle.handle = ctx->tc_handle;
-    //new_state.dir2.tc_handle.handle = ctx->tc_handle;
-
-    if (bpf_map_update_elem(&flow_state, key, &new_state, BPF_NOEXIST) ==
-        0)
-    {
-        if (new_flow)
-            *new_flow = true;
-    }
-    else
-    {
-        return (struct dual_flow_state *)NULL;
-    }
-
-    return (struct dual_flow_state *)bpf_map_lookup_elem(&flow_state, key);
-}
-
-static __always_inline struct dual_flow_state *
-lookup_or_create_dualflow_state(
-    struct parsing_context *ctx, 
-    struct packet_info *p_info, 
-    bool *new_flow
-) {
-    struct dual_flow_state *df_state;
-
-    struct network_tuple *key = get_dualflow_key_from_packet(p_info);
-    df_state = (struct dual_flow_state *)bpf_map_lookup_elem(&flow_state, key);
-
-    if (df_state)
-    {
-        return df_state;
-    }
-
-    // Only try to create new state if we have a valid pid
-    if (!p_info->pid_valid || p_info->event_type == FLOW_EVENT_CLOSING ||
-        p_info->event_type == FLOW_EVENT_CLOSING_BOTH)
-        return (struct dual_flow_state *)NULL;
-
-    return create_dualflow_state(ctx, p_info, new_flow);
-}
-
-static __always_inline bool is_flowstate_active(struct flow_state *f_state)
-{
-    return f_state->conn_state != CONNECTION_STATE_EMPTY &&
-           f_state->conn_state != CONNECTION_STATE_CLOSED;
-}
-
-static __always_inline void update_forward_flowstate(
-    struct packet_info *p_info,
-    struct flow_state *f_state, 
-    bool *new_flow
-) {
-    // "Create" flowstate if it's empty
-    if (f_state->conn_state == CONNECTION_STATE_EMPTY &&
-        p_info->pid_valid)
-    {
-        init_flowstate(f_state, p_info);
-        if (new_flow)
-            *new_flow = true;
-    }
-}
-
-static __always_inline void update_reverse_flowstate(
-    void *ctx, 
-    struct packet_info *p_info,
-    struct flow_state *f_state
-) {
-    if (!is_flowstate_active(f_state))
-        return;
-
-    // First time we see reply for flow?
-    if (f_state->conn_state == CONNECTION_STATE_WAITOPEN &&
-        p_info->event_type != FLOW_EVENT_CLOSING_BOTH)
-    {
-        f_state->conn_state = CONNECTION_STATE_OPEN;
-    }
-}
-
-static __always_inline bool is_new_identifier(
-    struct packet_id *pid, 
-    struct flow_state *f_state
-) {
-    if (pid->flow.proto == IPPROTO_TCP)
-        /* TCP timestamps should be monotonically non-decreasing
-         * Check that pid > last_ts (considering wrap around) by
-         * checking 0 < pid - last_ts < 2^31 as specified by
-         * RFC7323 Section 5.2*/
-        return pid->identifier - f_state->last_id > 0 &&
-               pid->identifier - f_state->last_id < 1UL << 31;
-
-    return pid->identifier != f_state->last_id;
-}
-
-static __always_inline bool is_rate_limited(__u64 now, __u64 last_ts)
-{
-    if (now < last_ts)
-        return true;
-
-    // Static rate limit
-    //return now - last_ts < DELAY_BETWEEN_RTT_REPORTS_MS * NS_PER_MS;
-    return false; // Max firehose drinking speed
-}
-
-/*
- * Attempt to create a timestamp-entry for packet p_info for flow in f_state
- */
-static __always_inline void pping_timestamp_packet(
-    struct flow_state *f_state, 
-    void *ctx,
-    struct packet_info *p_info, 
-    bool new_flow
-) {
-    if (!is_flowstate_active(f_state) || !p_info->pid_valid)
-        return;
-
-    // Check if identfier is new
-    if (!new_flow && !is_new_identifier(&p_info->pid, f_state))
-        return;
-    f_state->last_id = p_info->pid.identifier;
-
-    // Check rate-limit
-    if (!new_flow && is_rate_limited(p_info->time, f_state->last_timestamp))
-        return;
-
-    /*
-     * Updates attempt at creating timestamp, even if creation of timestamp
-     * fails (due to map being full). This should make the competition for
-     * the next available map slot somewhat fairer between heavy and sparse
-     * flows.
-     */
-    f_state->last_timestamp = p_info->time;
-
-    if (bpf_map_update_elem(&packet_ts, &p_info->pid, &p_info->time,
-                            BPF_NOEXIST) == 0)
-        __sync_fetch_and_add(&f_state->outstanding_timestamps, 1);
-}
-
-/*
- * Attempt to match packet in p_info with a timestamp from flow in f_state
- */
-static __always_inline void pping_match_packet(struct flow_state *f_state,
-                                               struct packet_info *p_info,
-                                               struct in6_addr *active_host)
-{
-    __u64 *p_ts;
-
-    if (!is_flowstate_active(f_state) || !p_info->reply_pid_valid)
-        return;
-
-    if (f_state->outstanding_timestamps == 0)
-        return;
-
-    p_ts = (__u64 *)bpf_map_lookup_elem(&packet_ts, &p_info->reply_pid);
-    if (!p_ts || p_info->time < *p_ts)
-        return;
-
-    __u64 rtt = (p_info->time - *p_ts) / NS_PER_MS_TIMES_100;
-
-    // Delete timestamp entry as soon as RTT is calculated
-    if (bpf_map_delete_elem(&packet_ts, &p_info->reply_pid) == 0)
-    {
-        __sync_fetch_and_add(&f_state->outstanding_timestamps, -1);
-    }
-
-    // Update the most performance map to include this data
-    struct rotating_performance *perf = 
-        (struct rotating_performance *)bpf_map_lookup_elem(
-            &rtt_tracker, active_host);
-    if (perf == NULL) return;
-    __sync_fetch_and_add(&perf->next_entry, 1);
-    __u32 next_entry = perf->next_entry;
-    if (next_entry < MAX_PERF_SECONDS) {
-        __sync_fetch_and_add(&perf->rtt[next_entry], rtt);
-        perf->has_fresh_data = 1;
-    }
-}
-
-static __always_inline void close_and_delete_flows(
-    void *ctx, 
-    struct packet_info *p_info,
-    struct flow_state *fw_flow,
-    struct flow_state *rev_flow
-) {
-    // Forward flow closing
-    if (p_info->event_type == FLOW_EVENT_CLOSING ||
-        p_info->event_type == FLOW_EVENT_CLOSING_BOTH)
-    {
-        fw_flow->conn_state = CONNECTION_STATE_CLOSED;
-    }
-
-    // Reverse flow closing
-    if (p_info->event_type == FLOW_EVENT_CLOSING_BOTH)
-    {
-        rev_flow->conn_state = CONNECTION_STATE_CLOSED;
-    }
-
-    // Delete flowstate entry if neither flow is open anymore
-    if (!is_flowstate_active(fw_flow) && !is_flowstate_active(rev_flow))
-    {
-        bpf_map_delete_elem(&flow_state, get_dualflow_key_from_packet(p_info));
-    }
-}
-
-/*
- * Contains the actual pping logic that is applied after a packet has been
- * parsed and deemed to contain some valid identifier.
- * Looks up and updates flowstate (in both directions), tries to save a
- * timestamp of the packet, tries to match packet against previous timestamps,
- * calculates RTTs and pushes messages to userspace as appropriate.
- */
-static __always_inline void pping_parsed_packet(
-    struct parsing_context *context, 
-    struct packet_info *p_info
-) {
-    struct dual_flow_state *df_state;
-    struct flow_state *fw_flow, *rev_flow;
-    bool new_flow = false;
-
-    df_state = lookup_or_create_dualflow_state(context, p_info, &new_flow);
-    if (!df_state)
-    {
-        // bpf_debug("No flow state - stop");
-        return;
-    }
-
-    fw_flow = get_flowstate_from_packet(df_state, p_info);
-    update_forward_flowstate(p_info, fw_flow, &new_flow);
-    pping_timestamp_packet(fw_flow, context, p_info, new_flow);
-
-    rev_flow = get_reverse_flowstate_from_packet(df_state, p_info);
-    update_reverse_flowstate(context, p_info, rev_flow);
-    pping_match_packet(rev_flow, p_info, context->active_host);
-
-    close_and_delete_flows(context, p_info, fw_flow, rev_flow);
-}
-
-/* Entry poing for running pping in the tc context */
-static __always_inline void tc_pping_start(struct parsing_context *context)
-{
-    // Check to see if we can store perf info. Bail if we've hit the limit.
-    // Copying occurs because otherwise the validator complains.
-    struct rotating_performance *perf = 
-        (struct rotating_performance *)bpf_map_lookup_elem(
-            &rtt_tracker, context->active_host);
-    if (perf) {
-        if (perf->next_entry >= MAX_PERF_SECONDS-1) {
-            //bpf_debug("Flow has max samples. Not sampling further until next reset.");
-            //for (int i=0; i<MAX_PERF_SECONDS; ++i) {
-            //    bpf_debug("%u", perf->rtt[i]);
-            //}
-            if (context->now > perf->recycle_time) {
-                // If the time-to-live for the sample is exceeded, recycle it to be
-                // usable again.
-                //bpf_debug("Recycling flow, %u > %u", context->now, perf->recycle_time);
-                __builtin_memset(perf->rtt, 0, sizeof(__u32) * MAX_PERF_SECONDS);
-                perf->recycle_time = context->now + RECYCLE_RTT_INTERVAL;
-                perf->next_entry = 0;
-                perf->has_fresh_data = 0;
-            }
-            return;
-        }
-    }
-
-    // Populate the TCP Header
-    if (context->dissector->eth_type == ETH_P_IP)
-    {
-        // If its not TCP, stop
-        if (context->dissector->ip_header.iph + 1 > context->dissector->end)
-            return; // Stops the error checking from crashing
-        if (context->dissector->ip_header.iph->protocol != IPPROTO_TCP)
-        {
-            return;
-        }
-        context->tcp = (struct tcphdr *)((char *)context->dissector->ip_header.iph + (context->dissector->ip_header.iph->ihl * 4));
-    }
-    else if (context->dissector->eth_type == ETH_P_IPV6)
-    {
-        // If its not TCP, stop
-        if (context->dissector->ip_header.ip6h + 1 > context->dissector->end)
-            return; // Stops the error checking from crashing
-        if (context->dissector->ip_header.ip6h->nexthdr != IPPROTO_TCP)
-        {
-            return;
-        }
-        context->tcp = (struct tcphdr *)(context->dissector->ip_header.ip6h + 1);
-    }
-    else
-    {
-        bpf_debug("UNKNOWN PROTOCOL TYPE");
-        return;
-    }
-
-    // Bail out if the packet is incomplete
-    if (context->tcp + 1 > context->dissector->end)
-    {
-        return;
-    }
-
-    // If we didn't get a handle, make one
-    if (perf == NULL)
-    {
-        struct rotating_performance new_perf = {0};
-        new_perf.recycle_time = context->now + RECYCLE_RTT_INTERVAL;
-        new_perf.has_fresh_data = 0;
-        if (bpf_map_update_elem(&rtt_tracker, context->active_host, &new_perf, BPF_NOEXIST) != 0) return;
-    }
-
-
-    // Start the parsing process
-    struct packet_info p_info = {0};
-    if (parse_packet_identifier(context, &p_info) < 0)
-    {
-        //bpf_debug("Unable to parse packet identifier");
-        return;
-    }
-
-    pping_parsed_packet(context, &p_info);
-}
-
-#endif /* __TC_CLASSIFY_KERN_PPING_H */

src/rust/lqos_sys/src/bpf/common/throughput.h

@@ -33,13 +33,14 @@ static __always_inline void track_traffic(
     int direction, 
     struct in6_addr * key, 
     __u32 size, 
-    __u32 tc_handle
+    __u32 tc_handle,
+    __u64 now
 ) {
     // Count the bits. It's per-CPU, so we can't be interrupted - no sync required
     struct host_counter * counter = 
         (struct host_counter *)bpf_map_lookup_elem(&map_traffic, key);
     if (counter) {
-        counter->last_seen = bpf_ktime_get_boot_ns();
+        counter->last_seen = now;
         counter->tc_handle = tc_handle;
         if (direction == 1) {
             // Download
@@ -53,7 +54,7 @@ static __always_inline void track_traffic(
     } else {
         struct host_counter new_host = {0};
         new_host.tc_handle = tc_handle;
-        new_host.last_seen = bpf_ktime_get_boot_ns();
+        new_host.last_seen = now;
         if (direction == 1) {
             new_host.download_packets = 1;
             new_host.download_bytes = size;

src/rust/lqos_sys/src/bpf/lqos_kern.c

@@ -15,9 +15,10 @@
 #include "common/throughput.h"
 #include "common/lpm.h"
 #include "common/cpu_map.h"
-#include "common/tcp_rtt.h"
+//#include "common/tcp_rtt.h"
 #include "common/bifrost.h"
 #include "common/heimdall.h"
+#include "common/flows.h"
 
 //#define VERBOSE 1
 
@@ -54,6 +55,17 @@ int direction = 255;
 __be16 internet_vlan = 0; // Note: turn these into big-endian
 __be16 isp_vlan = 0;
 
+// Helpers from https://elixir.bootlin.com/linux/v5.4.153/source/tools/testing/selftests/bpf/progs/test_xdp_meta.c#L37
+#define __round_mask(x, y) ((__typeof__(x))((y) - 1))
+#define round_up(x, y) ((((x) - 1) | __round_mask(x, y)) + 1)
+#define ctx_ptr(ctx, mem) (void *)(unsigned long)ctx->mem
+
+
+// Structure for passing metadata from XDP to TC
+struct metadata_pass_t {
+    __u32 tc_handle; // The encoded TC handle
+};
+
 // XDP Entry Point
 SEC("xdp")
 int xdp_prog(struct xdp_md *ctx)
@@ -98,23 +110,26 @@ int xdp_prog(struct xdp_md *ctx)
     // is requested.
     if (!dissector_find_l3_offset(&dissector, vlan_redirect)) return XDP_PASS;
     if (!dissector_find_ip_header(&dissector)) return XDP_PASS;
+    u_int8_t effective_direction = determine_effective_direction(
+        direction, 
+        internet_vlan, 
+        &dissector
+    );
+#ifdef VERBOSE
+    bpf_debug("(XDP) Effective direction: %d", effective_direction);
+#endif
+
 #ifdef VERBOSE
     bpf_debug("(XDP) Spotted VLAN: %u", dissector.current_vlan);
 #endif
 
     // Determine the lookup key by direction
     struct ip_hash_key lookup_key;
-    int effective_direction = 0;
     struct ip_hash_info * ip_info = setup_lookup_key_and_tc_cpu(
-        direction, 
+        effective_direction, 
         &lookup_key, 
-        &dissector, 
-        internet_vlan, 
-        &effective_direction
+        &dissector
     );
-#ifdef VERBOSE
-    bpf_debug("(XDP) Effective direction: %d", effective_direction);
-#endif
 
     // Find the desired TC handle and CPU target
     __u32 tc_handle = 0;
@@ -123,15 +138,19 @@ int xdp_prog(struct xdp_md *ctx)
         tc_handle = ip_info->tc_handle;
         cpu = ip_info->cpu;
     }
+
+    // Per-Flow RTT Tracking
+    track_flows(&dissector, effective_direction);
+
     // Update the traffic tracking buffers
     track_traffic(
         effective_direction, 
         &lookup_key.address, 
         ctx->data_end - ctx->data, // end - data = length
-        tc_handle
+        tc_handle,
+        dissector.now
     );
 
-
     // Send on its way
     if (tc_handle != 0) {
         // Send data to Heimdall
@@ -152,6 +171,34 @@ int xdp_prog(struct xdp_md *ctx)
         }
         __u32 cpu_dest = *cpu_lookup;
 
+        // Can we adjust the metadata? We'll try to do so, and if we can store the
+        // needed info there. Not all drivers support this, so it has to remain
+        // optional. This call invalidates the ctx->data pointer, so it has to be
+        // done last.
+        int ret = bpf_xdp_adjust_meta(ctx, -round_up(ETH_ALEN, sizeof(struct metadata_pass_t)));
+        if (ret < 0) {
+            #ifdef VERBOSE
+            bpf_debug("Error: unable to adjust metadata, ret: %d", ret);
+            #endif
+        } else {
+            #ifdef VERBOSE
+            bpf_debug("Metadata adjusted, ret: %d", ret);
+            #endif
+
+            __u8 *data_meta = ctx_ptr(ctx, data_meta);
+            __u8 *data_end  = ctx_ptr(ctx, data_end);
+            __u8 *data      = ctx_ptr(ctx, data);
+
+            if (data + ETH_ALEN > data_end || data_meta + round_up(ETH_ALEN, 4) > data) {
+                bpf_debug("Bounds error on the metadata");
+                return XDP_DROP;
+            }
+            struct metadata_pass_t meta = (struct metadata_pass_t) {
+                .tc_handle = tc_handle,
+            };
+            __builtin_memcpy(data_meta, &meta, sizeof(struct metadata_pass_t));
+        }
+
         // Redirect based on CPU
 #ifdef VERBOSE
         bpf_debug("(XDP) Zooming to CPU: %u", cpu_dest);
@@ -196,6 +243,43 @@ int tc_iphash_to_cpu(struct __sk_buff *skb)
         }
     } // Scope to remove tcq_cfg when done with it
 
+    // Do we have metadata?
+    if (skb->data != skb->data_meta) {
+        #ifdef VERBOSE
+        bpf_debug("(TC) Metadata is present");
+        #endif
+        int size = skb->data_meta - skb->data;
+        if (size < sizeof(struct metadata_pass_t)) {
+            bpf_debug("(TC) Metadata too small");
+        } else {
+            // Use it here
+            __u8 *data_meta = ctx_ptr(skb, data_meta);
+            __u8 *data_end  = ctx_ptr(skb, data_end);
+            __u8 *data      = ctx_ptr(skb, data);
+
+	        if (data + ETH_ALEN > data_end || data_meta + round_up(ETH_ALEN, 4) > data)
+            {
+                bpf_debug("(TC) Bounds error on the metadata");
+		        return TC_ACT_SHOT;
+            }
+
+            struct metadata_pass_t *meta = (struct metadata_pass_t *)data_meta;
+            #ifdef VERBOSE
+            bpf_debug("(TC) Metadata: CPU: %u, TC: %u", meta->cpu, meta->tc_handle);
+            #endif
+            if (meta->tc_handle != 0) {
+                // We can short-circuit the redirect and bypass the second
+                // LPM lookup! Yay!
+                skb->priority = meta->tc_handle;
+                return TC_ACT_OK;
+            }
+        }
+    } else {
+        #ifdef VERBOSE
+        bpf_debug("(TC) No metadata present");
+        #endif
+    }
+
     // Once again parse the packet
     // Note that we are returning OK on failure, which is a little odd.
     // The reasoning being that if its a packet we don't know how to handle,
@@ -220,14 +304,6 @@ int tc_iphash_to_cpu(struct __sk_buff *skb)
     bpf_debug("(TC) effective direction: %d", effective_direction);
 #endif
 
-    // Call pping to obtain RTT times
-    struct parsing_context context = {0};
-    context.now = bpf_ktime_get_ns();
-    context.tcp = NULL;
-    context.dissector = &dissector;
-    context.active_host = &lookup_key.address;
-    tc_pping_start(&context);
-
     if (ip_info && ip_info->tc_handle != 0) {
         // We found a matching mapped TC flow
 #ifdef VERBOSE
@@ -368,12 +444,12 @@ int throughput_reader(struct bpf_iter__bpf_map_elem *ctx)
 }
 
 SEC("iter/bpf_map_elem")
-int rtt_reader(struct bpf_iter__bpf_map_elem *ctx)
+int flow_reader(struct bpf_iter__bpf_map_elem *ctx)
 {
     // The sequence file
     struct seq_file *seq = ctx->meta->seq;
-    struct rotating_performance *counter = ctx->value;
-    struct in6_addr *ip = ctx->key;
+    struct flow_data_t *counter = ctx->value;
+    struct flow_key_t *ip = ctx->key;
 
     // Bail on end
     if (counter == NULL || ip == NULL) {
@@ -381,36 +457,8 @@ int rtt_reader(struct bpf_iter__bpf_map_elem *ctx)
     }
 
     //BPF_SEQ_PRINTF(seq, "%d %d\n", counter->next_entry, counter->rtt[0]);
-    bpf_seq_write(seq, ip, sizeof(struct in6_addr));
-    bpf_seq_write(seq, counter, sizeof(struct rotating_performance));
-    return 0;
-}
-
-SEC("iter/bpf_map_elem")
-int heimdall_reader(struct bpf_iter__bpf_map_elem *ctx) {
-    // The sequence file
-    struct seq_file *seq = ctx->meta->seq;
-    void *counter = ctx->value;
-    struct heimdall_key *ip = ctx->key;
-    __u32 num_cpus = NUM_CPUS;
-
-    if (ctx->meta->seq_num == 0) {
-        bpf_seq_write(seq, &num_cpus, sizeof(__u32));
-        bpf_seq_write(seq, &num_cpus, sizeof(__u32)); // Repeat for padding
-    }
-
-    // Bail on end
-    if (counter == NULL || ip == NULL) {
-        return 0;
-    }
-
-    bpf_seq_write(seq, ip, sizeof(struct heimdall_key));
-    for (__u32 i=0; i<NUM_CPUS; i++) {
-        struct heimdall_data * content = counter+(i*sizeof(struct heimdall_data));
-        bpf_seq_write(seq, content, sizeof(struct heimdall_data));
-    }
-
-    //BPF_SEQ_PRINTF(seq, "%d %d\n", counter->download_bytes, counter->upload_bytes);
+    bpf_seq_write(seq, ip, sizeof(struct flow_key_t));
+    bpf_seq_write(seq, counter, sizeof(struct flow_data_t));
     return 0;
 }
 

src/rust/lqos_sys/src/bpf_iterator.rs

@@ -1,6 +1,5 @@
 use crate::{
-  kernel_wrapper::BPF_SKELETON, lqos_kernel::bpf, HostCounter,
-  RttTrackingEntry, heimdall_data::{HeimdallKey, HeimdallData},
+  bpf_map::BpfMap, flowbee_data::{FlowbeeData, FlowbeeKey}, kernel_wrapper::BPF_SKELETON, lqos_kernel::bpf, HostCounter
 };
 use lqos_utils::XdpIpAddress;
 use once_cell::sync::Lazy;
@@ -149,7 +148,17 @@ where
           let (_head, values, _tail) =
             unsafe { &value_slice.align_to::<VALUE>() };
 
-          callback(&key[0], &values[0]);
+          if !key.is_empty() && !values.is_empty() {
+            callback(&key[0], &values[0]);
+          } else {
+            log::error!("Empty key or value found in iterator");
+            if key.is_empty() {
+              log::error!("Empty key");
+            }
+            if values.is_empty() {
+              log::error!("Empty value");
+            }
+          }
 
           index += Self::KEY_SIZE + Self::VALUE_SIZE;
         }
@@ -183,12 +192,8 @@ static mut MAP_TRAFFIC: Lazy<
   Option<BpfMapIterator<XdpIpAddress, HostCounter>>,
 > = Lazy::new(|| None);
 
-static mut RTT_TRACKER: Lazy<
-  Option<BpfMapIterator<XdpIpAddress, RttTrackingEntry>>,
-> = Lazy::new(|| None);
-
-static mut HEIMDALL_TRACKER: Lazy<
-  Option<BpfMapIterator<HeimdallKey, HeimdallData>>,
+static mut FLOWBEE_TRACKER: Lazy<
+  Option<BpfMapIterator<FlowbeeKey, FlowbeeData>>,
 > = Lazy::new(|| None);
 
 pub unsafe fn iterate_throughput(
@@ -214,51 +219,41 @@ pub unsafe fn iterate_throughput(
   }
 }
 
-pub unsafe fn iterate_rtt(
-  callback: &mut dyn FnMut(&XdpIpAddress, &RttTrackingEntry),
-) {
-  if RTT_TRACKER.is_none() {
-    let lock = BPF_SKELETON.lock().unwrap();
-    if let Some(skeleton) = lock.as_ref() {
-      let skeleton = skeleton.get_ptr();
-      if let Ok(iter) = unsafe {
-        BpfMapIterator::new(
-          (*skeleton).progs.rtt_reader,
-          (*skeleton).maps.rtt_tracker,
-        )
-      } {
-        *RTT_TRACKER = Some(iter);
-      }
-    }
-  }
-
-  if let Some(iter) = RTT_TRACKER.as_mut() {
-    let _ = iter.for_each(callback);
-  }
-}
-
-/// Iterate through the heimdall map and call the callback for each entry.
-pub fn iterate_heimdall(
-  callback: &mut dyn FnMut(&HeimdallKey, &[HeimdallData]),
+/// Iterate through the Flows 2 system tracker, retrieving all flows
+pub fn iterate_flows(
+  callback: &mut dyn FnMut(&FlowbeeKey, &FlowbeeData)
 ) {
   unsafe {
-    if HEIMDALL_TRACKER.is_none() {
+    if FLOWBEE_TRACKER.is_none() {
       let lock = BPF_SKELETON.lock().unwrap();
       if let Some(skeleton) = lock.as_ref() {
         let skeleton = skeleton.get_ptr();
         if let Ok(iter) = {
           BpfMapIterator::new(
-            (*skeleton).progs.heimdall_reader,
-            (*skeleton).maps.heimdall,
+            (*skeleton).progs.flow_reader,
+            (*skeleton).maps.flowbee,
           )
         } {
-          *HEIMDALL_TRACKER = Some(iter);
+          *FLOWBEE_TRACKER = Some(iter);
         }
       }
     }
-
-    if let Some(iter) = HEIMDALL_TRACKER.as_mut() {
-      let _ = iter.for_each_per_cpu(callback);
+  
+    if let Some(iter) = FLOWBEE_TRACKER.as_mut() {
+      let _ = iter.for_each(callback);
     }
   }
+}
+
+/// Adjust flows to have status 2 - already processed
+///
+// Arguments: the list of flow keys to expire
+pub fn end_flows(flows: &mut [FlowbeeKey]) -> anyhow::Result<()> {
+  let mut map = BpfMap::<FlowbeeKey, FlowbeeData>::from_path("/sys/fs/bpf/flowbee")?;
+
+  for flow in flows {
+    map.delete(flow)?;
+  }
+
+  Ok(())
 }

src/rust/lqos_sys/src/flowbee_data.rs

@@ -0,0 +1,67 @@
+//! Data structures for the Flowbee eBPF program.
+
+use lqos_utils::XdpIpAddress;
+use zerocopy::FromBytes;
+
+/// Representation of the eBPF `flow_key_t` type.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Hash, FromBytes)]
+#[repr(C)]
+pub struct FlowbeeKey {
+    /// Mapped `XdpIpAddress` source for the flow.
+    pub remote_ip: XdpIpAddress,
+    /// Mapped `XdpIpAddress` destination for the flow
+    pub local_ip: XdpIpAddress,
+    /// Source port number, or ICMP type.
+    pub src_port: u16,
+    /// Destination port number.
+    pub dst_port: u16,
+    /// IP protocol (see the Linux kernel!)
+    pub ip_protocol: u8,
+    /// Padding to align the structure to 16 bytes.
+    padding: u8,
+    padding1: u8,
+    padding2: u8,
+}
+
+/// Mapped representation of the eBPF `flow_data_t` type.
+#[derive(Debug, Clone, Default, FromBytes)]
+#[repr(C)]
+pub struct FlowbeeData {
+    /// Time (nanos) when the connection was established
+    pub start_time: u64,
+    /// Time (nanos) when the connection was last seen
+    pub last_seen: u64,
+    /// Bytes transmitted
+    pub bytes_sent: [u64; 2],
+    /// Packets transmitted
+    pub packets_sent: [u64; 2],
+    /// Clock for the next rate estimate
+    pub next_count_time: [u64; 2],
+    /// Clock for the previous rate estimate
+    pub last_count_time: [u64; 2],
+    /// Bytes at the next rate estimate
+    pub next_count_bytes: [u64; 2],
+    /// Rate estimate
+    pub rate_estimate_bps: [u32; 2],
+    /// Sequence number of the last packet
+    pub last_sequence: [u32; 2],
+    /// Acknowledgement number of the last packet
+    pub last_ack: [u32; 2],
+    /// TCP Retransmission count (also counts duplicates)
+    pub tcp_retransmits: [u16; 2],
+    /// Timestamp values
+    pub tsval: [u32; 2],
+    /// Timestamp echo values
+    pub tsecr: [u32; 2],
+    /// When did the timestamp change?
+    pub ts_change_time: [u64; 2],
+    /// Has the connection ended?
+    /// 0 = Alive, 1 = FIN, 2 = RST
+    pub end_status: u8,
+    /// Raw IP TOS
+    pub tos: u8,
+    /// Raw TCP flags
+    pub flags: u8,
+    /// Padding.
+    pub padding: u8,
+}

src/rust/lqos_sys/src/heimdall_data.rs

@@ -1,33 +0,0 @@
-use lqos_utils::XdpIpAddress;
-use zerocopy::FromBytes;
-
-/// Representation of the eBPF `heimdall_key` type.
-#[derive(Debug, Clone, Default, PartialEq, Eq, Hash, FromBytes)]
-#[repr(C)]
-pub struct HeimdallKey {
-  /// Mapped `XdpIpAddress` source for the flow.
-  pub src_ip: XdpIpAddress,
-  /// Mapped `XdpIpAddress` destination for the flow
-  pub dst_ip: XdpIpAddress,
-  /// IP protocol (see the Linux kernel!)
-  pub ip_protocol: u8,
-  /// Source port number, or ICMP type.
-  pub src_port: u16,
-  /// Destination port number.
-  pub dst_port: u16,
-  _padding: u8,
-}
-
-/// Mapped representation of the eBPF `heimdall_data` type.
-#[derive(Debug, Clone, Default, FromBytes)]
-#[repr(C)]
-pub struct HeimdallData {
-  /// Last seen, in nanoseconds (since boot time).
-  pub last_seen: u64,
-  /// Number of bytes since the flow started being tracked
-  pub bytes: u64,
-  /// Number of packets since the flow started being tracked
-  pub packets: u64,
-  /// IP header TOS value
-  pub tos: u8,
-}

src/rust/lqos_sys/src/kernel_wrapper.rs

@@ -45,7 +45,7 @@ impl LibreQoSKernels {
   /// * `to_isp` - the name of the ISP-network facing interface (e.g. `eth2`).
   /// * `heimdall_event_handler` - C function pointer to the ringbuffer
   ///    event handler exported by Heimdall.
-  pub fn new<S: ToString>(to_internet: S, to_isp: S, heimdall_event_handler: ring_buffer_sample_fn) -> anyhow::Result<Self> {
+  pub fn new<S: ToString>(to_internet: S, to_isp: S, heimdall_event_handler: ring_buffer_sample_fn, flowbee_event_handler: ring_buffer_sample_fn) -> anyhow::Result<Self> {
     let kernel = Self {
       to_internet: to_internet.to_string(),
       to_isp: to_isp.to_string(),
@@ -55,11 +55,13 @@ impl LibreQoSKernels {
       &kernel.to_internet,
       InterfaceDirection::Internet,
       heimdall_event_handler,
+      flowbee_event_handler,
     )?;
     attach_xdp_and_tc_to_interface(
       &kernel.to_isp,
       InterfaceDirection::IspNetwork,
       heimdall_event_handler,
+      flowbee_event_handler,
     )?;
     BPF_SKELETON.lock().unwrap().replace(LqosKernBpfWrapper { ptr: skeleton });
     Ok(kernel)
@@ -81,6 +83,7 @@ impl LibreQoSKernels {
     internet_vlan: u16,
     isp_vlan: u16,
     heimdall_event_handler: ring_buffer_sample_fn,
+    flowbee_event_handler: ring_buffer_sample_fn,
   ) -> anyhow::Result<Self> {
     let kernel = Self {
       to_internet: stick_interface.to_string(),
@@ -91,6 +94,7 @@ impl LibreQoSKernels {
       &kernel.to_internet,
       InterfaceDirection::OnAStick(internet_vlan, isp_vlan),
       heimdall_event_handler,
+      flowbee_event_handler,
     )?;
     BPF_SKELETON.lock().unwrap().replace(LqosKernBpfWrapper { ptr: skeleton });
     Ok(kernel)

src/rust/lqos_sys/src/lib.rs

@@ -15,13 +15,12 @@ mod cpu_map;
 mod ip_mapping;
 mod kernel_wrapper;
 mod lqos_kernel;
-mod tcp_rtt;
 mod throughput;
 mod linux;
 mod bpf_iterator;
 /// Data shared between eBPF and Heimdall that needs local access
 /// for map control.
-pub mod heimdall_data;
+pub mod flowbee_data;
 
 pub use ip_mapping::{
   add_ip_to_tc, clear_ips_from_tc, del_ip_from_tc, list_mapped_ips,
@@ -29,6 +28,5 @@ pub use ip_mapping::{
 pub use kernel_wrapper::LibreQoSKernels;
 pub use linux::num_possible_cpus;
 pub use lqos_kernel::max_tracked_ips;
-pub use tcp_rtt::{rtt_for_each, RttTrackingEntry};
 pub use throughput::{throughput_for_each, HostCounter};
-pub use bpf_iterator::iterate_heimdall;
+pub use bpf_iterator::{iterate_flows, end_flows};

src/rust/lqos_sys/src/lqos_kernel.rs

@@ -99,7 +99,8 @@ unsafe fn open_kernel() -> Result<*mut bpf::lqos_kern> {
 unsafe fn load_kernel(skeleton: *mut bpf::lqos_kern) -> Result<()> {
   let error = bpf::lqos_kern_load(skeleton);
   if error != 0 {
-    Err(Error::msg("Unable to load the XDP/TC kernel"))
+    let error = format!("Unable to load the XDP/TC kernel ({error})");
+    Err(Error::msg(error))
   } else {
     Ok(())
   }
@@ -116,6 +117,7 @@ pub fn attach_xdp_and_tc_to_interface(
   interface_name: &str,
   direction: InterfaceDirection,
   heimdall_event_handler: bpf::ring_buffer_sample_fn,
+  flowbee_event_handler: bpf::ring_buffer_sample_fn,
 ) -> Result<*mut lqos_kern> {
   check_root()?;
   // Check the interface is valid
@@ -183,6 +185,28 @@ pub fn attach_xdp_and_tc_to_interface(
   let handle = PerfBufferHandle(heimdall_perf_buffer);
   std::thread::spawn(|| poll_perf_events(handle));
 
+  // Find and attach the Flowbee handler
+  let flowbee_events_name = CString::new("flowbee_events").unwrap();
+  let flowbee_events_map = unsafe { bpf::bpf_object__find_map_by_name((*skeleton).obj, flowbee_events_name.as_ptr()) };
+  let flowbee_events_fd = unsafe { bpf::bpf_map__fd(flowbee_events_map) };
+  if flowbee_events_fd < 0 {
+    log::error!("Unable to load Flowbee Events FD");
+    return Err(anyhow::Error::msg("Unable to load Flowbee Events FD"));
+  }
+  let opts: *const bpf::ring_buffer_opts = std::ptr::null();
+  let flowbee_perf_buffer = unsafe {
+    bpf::ring_buffer__new(
+      flowbee_events_fd, 
+      flowbee_event_handler, 
+      opts as *mut c_void, opts)
+  };
+  if unsafe { bpf::libbpf_get_error(flowbee_perf_buffer as *mut c_void) != 0 } {
+    log::error!("Failed to create Flowbee event buffer");
+    return Err(anyhow::Error::msg("Failed to create Flowbee event buffer"));
+  }
+  let handle = PerfBufferHandle(flowbee_perf_buffer);
+  std::thread::spawn(|| poll_perf_events(handle));
+
   // Remove any previous entry
   let _r = Command::new("tc")
     .args(["qdisc", "del", "dev", interface_name, "clsact"])

src/rust/lqos_sys/src/tcp_rtt.rs

@@ -1,38 +0,0 @@
-use lqos_utils::XdpIpAddress;
-use zerocopy::FromBytes;
-use crate::bpf_iterator::iterate_rtt;
-
-/// Entry from the XDP rtt_tracker map.
-#[repr(C)]
-#[derive(Clone, Copy, Debug, FromBytes)]
-pub struct RttTrackingEntry {
-  /// Array containing TCP round-trip times. Convert to an `f32` and divide by `100.0` for actual numbers.
-  pub rtt: [u32; 60],
-
-  /// Used internally by the XDP program to store the current position in the storage array. Do not modify.
-  next_entry: u32,
-
-  /// Used internally by the XDP program to determine when it is time to recycle and reuse a record. Do not modify.
-  recycle_time: u64,
-
-  /// Flag indicating that an entry has been updated recently (last 30 seconds by default).
-  pub has_fresh_data: u32,
-}
-
-impl Default for RttTrackingEntry {
-  fn default() -> Self {
-    Self { rtt: [0; 60], next_entry: 0, recycle_time: 0, has_fresh_data: 0 }
-  }
-}
-
-/// Queries the active XDP/TC programs for TCP round-trip time tracking
-/// data (from the `rtt_tracker` pinned eBPF map).
-///
-/// Only IP addresses facing the ISP Network side are tracked.
-///
-/// Executes `callback` for each entry.
-pub fn rtt_for_each(callback: &mut dyn FnMut(&XdpIpAddress, &RttTrackingEntry)) {
-  unsafe {
-    iterate_rtt(callback);
-  }
-}

src/rust/lqos_utils/src/xdp_ip_address.rs

@@ -42,7 +42,8 @@ impl XdpIpAddress {
     result
   }
 
-  fn is_v4(&self) -> bool {
+  /// Checks if the `XdpIpAddress` is an IPv4 address
+  pub fn is_v4(&self) -> bool {
     self.0[0] == 0xFF
       && self.0[1] == 0xFF
       && self.0[2] == 0xFF

src/rust/lqosd/Cargo.toml

@@ -29,6 +29,15 @@ sysinfo = "0"
 dashmap = "5"
 num-traits = "0.2"
 thiserror = "1"
+itertools = "0.12.1"
+csv = "1"
+reqwest = { version = "0.11.24", features = ["blocking"] }
+flate2 = "1.0"
+bincode = "1"
+ip_network_table = "0"
+ip_network = "0"
+zerocopy = {version = "0.6.1", features = [ "simd" ] }
+fxhash = "0.2.1"
 
 # Support JemAlloc on supported platforms
 [target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies]

src/rust/lqosd/src/main.rs

@@ -12,7 +12,7 @@ mod long_term_stats;
 use std::net::IpAddr;
 use crate::{
   file_lock::FileLock,
-  ip_mapping::{clear_ip_flows, del_ip_flow, list_mapped_ips, map_ip_to_flow},
+  ip_mapping::{clear_ip_flows, del_ip_flow, list_mapped_ips, map_ip_to_flow}, throughput_tracker::flow_data::{flowbee_handle_events, setup_netflow_tracker},
 };
 use anyhow::Result;
 use log::{info, warn};
@@ -29,7 +29,7 @@ use signal_hook::{
   iterator::Signals,
 };
 use stats::{BUS_REQUESTS, TIME_TO_POLL_HOSTS, HIGH_WATERMARK_DOWN, HIGH_WATERMARK_UP, FLOWS_TRACKED};
-use throughput_tracker::get_flow_stats;
+use throughput_tracker::flow_data::get_rtt_events_per_second;
 use tokio::join;
 mod stats;
 
@@ -66,20 +66,28 @@ async fn main() -> Result<()> {
       config.stick_vlans().1 as u16,
       config.stick_vlans().0 as u16,
       Some(heimdall_handle_events),
+      Some(flowbee_handle_events),
     )?
   } else {
-    LibreQoSKernels::new(&config.internet_interface(), &config.isp_interface(), Some(heimdall_handle_events))?
+    LibreQoSKernels::new(
+      &config.internet_interface(), 
+      &config.isp_interface(), 
+      Some(heimdall_handle_events), 
+      Some(flowbee_handle_events)
+    )?
   };
 
   // Spawn tracking sub-systems
   let long_term_stats_tx = start_long_term_stats().await;
+  let flow_tx = setup_netflow_tracker();
+  let _ = throughput_tracker::flow_data::setup_flow_analysis();
   join!(
     start_heimdall(),
     spawn_queue_structure_monitor(),
     shaped_devices_tracker::shaped_devices_watcher(),
     shaped_devices_tracker::network_json_watcher(),
     anonymous_usage::start_anonymous_usage(),
-    throughput_tracker::spawn_throughput_monitor(long_term_stats_tx.clone()),
+    throughput_tracker::spawn_throughput_monitor(long_term_stats_tx.clone(), flow_tx),
   );
   spawn_queue_monitor();
 
@@ -143,6 +151,9 @@ fn handle_bus_requests(
       BusRequest::GetWorstRtt { start, end } => {
         throughput_tracker::worst_n(*start, *end)
       }
+      BusRequest::GetWorstRetransmits { start, end } => {
+        throughput_tracker::worst_n_retransmits(*start, *end)
+      }
       BusRequest::GetBestRtt { start, end } => {
         throughput_tracker::best_n(*start, *end)
       }
@@ -193,9 +204,9 @@ fn handle_bus_requests(
             HIGH_WATERMARK_UP.load(std::sync::atomic::Ordering::Relaxed),
           ),
           tracked_flows: FLOWS_TRACKED.load(std::sync::atomic::Ordering::Relaxed),
+          rtt_events_per_second: get_rtt_events_per_second(),
         }
       }
-      BusRequest::GetFlowStats(ip) => get_flow_stats(ip),
       BusRequest::GetPacketHeaderDump(id) => {
         BusResponse::PacketDump(n_second_packet_dump(*id))
       }
@@ -223,6 +234,18 @@ fn handle_bus_requests(
       BusRequest::GetLongTermStats(StatsRequest::Tree) => {
         long_term_stats::get_stats_tree()
       }
+      BusRequest::DumpActiveFlows => {
+        throughput_tracker::dump_active_flows()
+      }
+      BusRequest::CountActiveFlows => {
+        throughput_tracker::count_active_flows()
+      }
+      BusRequest::TopFlows { n, flow_type } => throughput_tracker::top_flows(*n, *flow_type),
+      BusRequest::FlowsByIp(ip) => throughput_tracker::flows_by_ip(ip),
+      BusRequest::CurrentEndpointsByCountry => throughput_tracker::current_endpoints_by_country(),
+      BusRequest::CurrentEndpointLatLon => throughput_tracker::current_lat_lon(),
+      BusRequest::EtherProtocolSummary => throughput_tracker::ether_protocol_summary(),
+      BusRequest::IpProtocolSummary => throughput_tracker::ip_protocol_summary(),
     });
   }
 }

src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/asn.rs

@@ -0,0 +1,263 @@
+//! Obtain ASN and geo mappings from IP addresses for flow
+//! analysis.
+
+
+use std::{io::Read, net::IpAddr, path::Path};
+use serde::Deserialize;
+
+#[derive(Deserialize, Clone, Debug)]
+struct AsnEncoded {
+    network: IpAddr,
+    prefix: u8,
+    pub asn: u32,
+    organization: String,
+}
+
+#[allow(dead_code)]
+#[derive(Deserialize, Debug)]
+struct GeoIpLocation {
+    network: IpAddr,
+    prefix: u8,
+    latitude: f64,
+    longitude: f64,
+    city_and_country: String,
+
+}
+
+#[derive(Deserialize)]
+struct Geobin {
+    asn: Vec<AsnEncoded>,
+    geo: Vec<GeoIpLocation>,
+}
+
+pub struct GeoTable {
+    asn_trie: ip_network_table::IpNetworkTable<AsnEncoded>,
+    geo_trie: ip_network_table::IpNetworkTable<GeoIpLocation>,
+}
+
+impl GeoTable {
+    const FILENAME: &'static str = "geo.bin";
+
+    fn file_path() -> std::path::PathBuf {
+        Path::new(&lqos_config::load_config().unwrap().lqos_directory)
+            .join(Self::FILENAME)
+    }
+
+    fn download() -> anyhow::Result<()> {
+        log::info!("Downloading ASN-IP Table");
+        let file_path = Self::file_path();
+        let url = "https://bfnightly.bracketproductions.com/geo.bin";
+        let response = reqwest::blocking::get(url)?;
+        let content = response.bytes()?;
+        let bytes = &content[0..];
+        std::fs::write(file_path, bytes)?;
+        Ok(())
+    }
+
+    pub fn load() -> anyhow::Result<Self> {
+        let path = Self::file_path();
+        if !path.exists() {
+            log::info!("geo.bin not found - trying to download it");
+            Self::download()?;
+        }
+
+        // Decompress and deserialize
+        let file = std::fs::File::open(path)?;
+        let mut buffer = Vec::new();
+        flate2::read::GzDecoder::new(file).read_to_end(&mut buffer)?;
+        let geobin: Geobin = bincode::deserialize(&buffer)?;
+
+        // Build the ASN trie
+        log::info!("Building ASN trie");
+        let mut asn_trie = ip_network_table::IpNetworkTable::<AsnEncoded>::new();
+        for entry in geobin.asn {
+            let (ip, prefix) = match entry.network {
+                IpAddr::V4(ip) => (ip.to_ipv6_mapped(), entry.prefix+96 ),
+                IpAddr::V6(ip) => (ip, entry.prefix),
+            };
+            if let Ok(ip) = ip_network::Ipv6Network::new(ip, prefix) {
+                asn_trie.insert(ip, entry);
+            }
+        }
+
+        // Build the GeoIP trie
+        log::info!("Building GeoIP trie");
+        let mut geo_trie = ip_network_table::IpNetworkTable::<GeoIpLocation>::new();
+        for entry in geobin.geo {
+            let (ip, prefix) = match entry.network {
+                IpAddr::V4(ip) => (ip.to_ipv6_mapped(), entry.prefix+96 ),
+                IpAddr::V6(ip) => (ip, entry.prefix),
+            };
+            if let Ok(ip) = ip_network::Ipv6Network::new(ip, prefix) {
+                geo_trie.insert(ip, entry);
+            }
+        }
+
+        log::info!("GeoTables loaded, {}-{} records.", asn_trie.len().1, geo_trie.len().1);
+
+        Ok(Self {
+            asn_trie,
+            geo_trie,
+        })
+    }
+
+    pub fn find_asn(&self, ip: IpAddr) -> Option<u32> {
+        log::debug!("Looking up ASN for IP: {:?}", ip);
+        let ip = match ip {
+            IpAddr::V4(ip) => ip.to_ipv6_mapped(),
+            IpAddr::V6(ip) => ip,
+        };
+        if let Some(matched) = self.asn_trie.longest_match(ip) {
+            log::debug!("Matched ASN: {:?}", matched.1.asn);
+            Some(matched.1.asn)
+        } else {
+            log::debug!("No ASN found");
+            None
+        }
+    }
+
+    pub fn find_owners_by_ip(&self, ip: IpAddr) -> (String, String) {
+        log::debug!("Looking up ASN for IP: {:?}", ip);
+        let ip = match ip {
+            IpAddr::V4(ip) => ip.to_ipv6_mapped(),
+            IpAddr::V6(ip) => ip,
+        };
+        let mut owners = String::new();
+        let mut country = String::new();
+
+        if let Some(matched) = self.asn_trie.longest_match(ip) {
+            log::debug!("Matched ASN: {:?}", matched.1.asn);
+            owners = matched.1.organization.clone();
+        }
+        if let Some(matched) = self.geo_trie.longest_match(ip) {
+            log::debug!("Matched Geo: {:?}", matched.1.city_and_country);
+            country = matched.1.city_and_country.clone();
+        }
+
+        (owners, country)
+    }
+
+    pub fn find_lat_lon_by_ip(&self, ip: IpAddr) -> (f64, f64) {
+        log::debug!("Looking up ASN for IP: {:?}", ip);
+        let ip = match ip {
+            IpAddr::V4(ip) => ip.to_ipv6_mapped(),
+            IpAddr::V6(ip) => ip,
+        };
+
+        if let Some(matched) = self.geo_trie.longest_match(ip) {
+            log::debug!("Matched Geo: {:?}", matched.1.city_and_country);
+            return (matched.1.latitude, matched.1.longitude);
+        }
+
+        (0.0, 0.0)
+    }
+}
+
+///////////////////////////////////////////////////////////////////////
+
+/*
+/// Structure to represent the on-disk structure for files
+/// from: https://iptoasn.com/
+/// Specifically: https://iptoasn.com/data/ip2asn-combined.tsv.gz
+#[derive(Deserialize, Debug, Clone)]
+pub struct Ip2AsnRow {
+    pub start_ip: IpAddr,
+    pub end_ip: IpAddr,
+    pub asn: u32,
+    pub country: String,
+    pub owners: String,
+}
+
+pub struct AsnTable {
+    asn_table: Vec<Ip2AsnRow>,
+}
+
+impl AsnTable {
+    pub fn new() -> anyhow::Result<Self> {
+        if !Self::exists() {
+            Self::download()?;
+        }
+        let asn_table = Self::build_asn_table()?;
+        log::info!("Setup ASN Table with {} entries.", asn_table.len());
+        Ok(Self {
+            asn_table,
+        })
+    }
+
+    fn file_path() -> std::path::PathBuf {
+        Path::new(&lqos_config::load_config().unwrap().lqos_directory)
+            .join("ip2asn-combined.tsv")
+    }
+
+    fn download() -> anyhow::Result<()> {
+        log::info!("Downloading ASN-IP Table");
+        let file_path = Self::file_path();
+        let url = "https://iptoasn.com/data/ip2asn-combined.tsv.gz";
+        let response = reqwest::blocking::get(url)?;
+        let content = response.bytes()?;
+        let bytes = &content[0..];
+        let mut decompresser = flate2::read::GzDecoder::new(bytes);
+        let mut buf = Vec::new();
+        decompresser.read_to_end(&mut buf)?;
+        std::fs::write(file_path, buf)?;
+        Ok(())
+    }
+
+    fn exists() -> bool {
+        Self::file_path().exists()
+    }
+
+    fn build_asn_table() -> anyhow::Result<Vec<Ip2AsnRow>> {
+        let file_path = Self::file_path();
+    
+        if !file_path.exists() {
+            let mut retries = 0;
+            while retries < 3 {
+                if file_path.exists() {
+                    break;
+                }
+                Self::download()?;
+                retries += 1;
+            }
+        }
+
+        if !file_path.exists() {
+            anyhow::bail!("IP to ASN file not found: {:?}", file_path);
+        }
+        let in_file = std::fs::File::open(file_path)?;
+    
+        let mut rdr = csv::ReaderBuilder::new()
+            .has_headers(false)
+            .delimiter(b'\t')
+            .double_quote(false)
+            .escape(Some(b'\\'))
+            .flexible(true)
+            .comment(Some(b'#'))
+            .from_reader(in_file);
+        
+        let mut output = Vec::new();
+        for result in rdr.deserialize() {
+            let record: Ip2AsnRow = result?;
+            output.push(record);
+        }
+        output.sort_by(|a, b| a.start_ip.cmp(&b.start_ip));
+        Ok(output)
+    }
+
+    pub fn find_asn(&self, ip: IpAddr) -> Option<Ip2AsnRow> {
+        self.asn_table.binary_search_by(|probe| {
+            if ip < probe.start_ip {
+                std::cmp::Ordering::Greater
+            } else if ip > probe.end_ip {
+                std::cmp::Ordering::Less
+            } else {
+                std::cmp::Ordering::Equal
+            }
+        }).map(|idx| self.asn_table[idx].clone()).ok()
+    }
+
+    pub fn find_asn_by_id(&self, asn: u32) -> Option<Ip2AsnRow> {
+        self.asn_table.iter().find(|row| row.asn == asn).map(|row| row.clone())
+    }
+}
+*/

src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/finished_flows.rs

@@ -0,0 +1,269 @@
+use super::{get_asn_lat_lon, get_asn_name_and_country, FlowAnalysis};
+use crate::throughput_tracker::flow_data::{FlowbeeLocalData, FlowbeeRecipient};
+use fxhash::FxHashMap;
+use lqos_bus::BusResponse;
+use lqos_sys::flowbee_data::FlowbeeKey;
+use once_cell::sync::Lazy;
+use std::sync::{Arc, Mutex};
+
+pub struct TimeBuffer {
+    buffer: Mutex<Vec<TimeEntry>>,
+}
+
+struct TimeEntry {
+    time: u64,
+    data: (FlowbeeKey, FlowbeeLocalData, FlowAnalysis),
+}
+
+impl TimeBuffer {
+    fn new() -> Self {
+        Self {
+            buffer: Mutex::new(Vec::new()),
+        }
+    }
+
+    fn expire_over_five_minutes(&self) {
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+        let mut buffer = self.buffer.lock().unwrap();
+        buffer.retain(|v| now - v.time < 300);
+    }
+
+    fn push(&self, entry: TimeEntry) {
+        let mut buffer = self.buffer.lock().unwrap();
+        buffer.push(entry);
+    }
+
+    pub fn lat_lon_endpoints(&self) -> Vec<(f64, f64, String, u64, f32)> {
+        let buffer = self.buffer.lock().unwrap();
+        let mut my_buffer = buffer
+            .iter()
+            .map(|v| {
+                let (key, data, _analysis) = &v.data;
+                let (lat, lon) = get_asn_lat_lon(key.remote_ip.as_ip());
+                let (_name, country) = get_asn_name_and_country(key.remote_ip.as_ip());
+                (lat, lon, country, data.bytes_sent[1], data.rtt[1].as_nanos() as f32)
+            })
+            .filter(|(lat, lon, ..)| *lat != 0.0 && *lon != 0.0)
+            .collect::<Vec<(f64, f64, String, u64, f32)>>();
+
+        // Sort by lat/lon
+        my_buffer.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
+
+        // Depuplicate
+        my_buffer.dedup();
+
+        my_buffer
+    }
+
+    pub fn country_summary(&self) -> Vec<(String, [u64; 2], [f32; 2])> {
+        let buffer = self.buffer.lock().unwrap();
+        let mut my_buffer = buffer
+            .iter()
+            .map(|v| {
+                let (key, data, _analysis) = &v.data;
+                let (_name, country) = get_asn_name_and_country(key.remote_ip.as_ip());
+                let rtt = [
+                    data.rtt[0].as_nanos() as f32,
+                    data.rtt[1].as_nanos() as f32,
+                ];
+                (country, data.bytes_sent, rtt)
+            })
+            .collect::<Vec<(String, [u64; 2], [f32; 2])>>();
+
+        // Sort by country
+        my_buffer.sort_by(|a, b| a.0.cmp(&b.0));
+
+        // Summarize by country
+        let mut country_summary = Vec::new();
+        let mut last_country = String::new();
+        let mut total_bytes = [0, 0];
+        let mut total_rtt = [0.0f64, 0.0f64];
+        let mut rtt_count = [0, 0];
+        for (country, bytes, rtt) in my_buffer {
+            if last_country != country {
+                if !last_country.is_empty() {
+                    // Store the country
+                    if rtt_count[0] > 0 {
+                        total_rtt[0] = (total_rtt[0] / rtt_count[0] as f64) as f64;
+                    }
+                    if rtt_count[1] > 0 {
+                        total_rtt[1] = (total_rtt[1] / rtt_count[1] as f64) as f64;
+                    }
+
+                    let rtt = [
+                        total_rtt[0] as f32,
+                        total_rtt[1] as f32,
+                    ];
+
+                    country_summary.push((last_country, total_bytes, rtt));
+                }
+
+                last_country = country.to_string();
+                total_bytes = [0, 0];
+                total_rtt = [0.0, 0.0];
+                rtt_count = [0, 0];
+            }
+            total_bytes[0] += bytes[0];
+            total_bytes[1] += bytes[1];
+            if rtt[0] > 0.0 {
+                total_rtt[0] += rtt[0] as f64;
+                rtt_count[0] += 1;
+            }
+            if rtt[1] > 0.0 {
+                total_rtt[1] += rtt[1] as f64;
+                rtt_count[1] += 1;
+            }
+        }
+
+        // Store the last country
+        let rtt = [
+            if total_rtt[0] > 0.0 {
+                (total_rtt[0] / rtt_count[0] as f64) as f32
+            } else {
+                0.0
+            },
+            if total_rtt[1] > 0.0 {
+                (total_rtt[1] / rtt_count[1] as f64) as f32
+            } else {
+                0.0
+            },
+        ];
+
+        country_summary.push((last_country, total_bytes, rtt));
+
+        // Sort by bytes downloaded descending
+        country_summary.sort_by(|a, b| b.1[1].cmp(&a.1[1]));
+
+        country_summary
+    }
+
+    fn median(slice: &[u64]) -> u64 {
+        if slice.is_empty() {
+            return 0;
+        }
+        let mut slice = slice.to_vec();
+        slice.sort_by(|a, b| a.partial_cmp(b).unwrap());
+        let mid = slice.len() / 2;
+        if slice.len() % 2 == 0 {
+            (slice[mid] + slice[mid - 1]) / 2
+        } else {
+            slice[mid]
+        }
+    }
+
+    pub fn ether_protocol_summary(&self) -> BusResponse {
+        let buffer = self.buffer.lock().unwrap();
+
+        let mut v4_bytes_sent = [0,0];
+        let mut v4_packets_sent = [0,0];
+        let mut v6_bytes_sent = [0,0];
+        let mut v6_packets_sent = [0,0];
+        let mut v4_rtt = [Vec::new(), Vec::new()];
+        let mut v6_rtt = [Vec::new(), Vec::new()];
+
+        buffer
+            .iter()
+            .for_each(|v| {
+                let (key, data, _analysis) = &v.data;
+                if key.local_ip.is_v4() {
+                    // It's V4
+                    v4_bytes_sent[0] += data.bytes_sent[0];
+                    v4_bytes_sent[1] += data.bytes_sent[1];
+                    v4_packets_sent[0] += data.packets_sent[0];
+                    v4_packets_sent[1] += data.packets_sent[1];
+                    if data.rtt[0].as_nanos() > 0 {
+                        v4_rtt[0].push(data.rtt[0].as_nanos());
+                    }
+                    if data.rtt[1].as_nanos() > 0 {
+                        v4_rtt[1].push(data.rtt[1].as_nanos());
+                    }
+                } else {
+                    // It's V6
+                    v6_bytes_sent[0] += data.bytes_sent[0];
+                    v6_bytes_sent[1] += data.bytes_sent[1];
+                    v6_packets_sent[0] += data.packets_sent[0];
+                    v6_packets_sent[1] += data.packets_sent[1];
+                    if data.rtt[0].as_nanos() > 0 {
+                        v6_rtt[0].push(data.rtt[0].as_nanos());
+                    }
+                    if data.rtt[1].as_nanos() > 0 {
+                        v6_rtt[1].push(data.rtt[1].as_nanos());
+                    }
+
+                }
+            });
+        
+        let v4_rtt = [
+            Self::median(&v4_rtt[0]),
+            Self::median(&v4_rtt[1]),
+        ];
+        let v6_rtt = [
+            Self::median(&v6_rtt[0]),
+            Self::median(&v6_rtt[1]),
+        ];
+
+        BusResponse::EtherProtocols {
+            v4_bytes: v4_bytes_sent,
+            v6_bytes: v6_bytes_sent,
+            v4_packets: v4_packets_sent,
+            v6_packets: v6_packets_sent,
+            v4_rtt,
+            v6_rtt,
+        }
+    }
+
+    pub fn ip_protocol_summary(&self) -> Vec<(String, (u64, u64))> {
+        let buffer = self.buffer.lock().unwrap();
+
+        let mut results = FxHashMap::default();
+
+        buffer
+            .iter()
+            .for_each(|v| {
+                let (_key, data, analysis) = &v.data;
+                let proto = analysis.protocol_analysis.to_string();
+                let entry = results.entry(proto).or_insert((0, 0));
+                entry.0 += data.bytes_sent[0];
+                entry.1 += data.bytes_sent[1];
+            });
+
+        let mut results = results.into_iter().collect::<Vec<(String, (u64, u64))>>();
+        results.sort_by(|a, b| b.1.1.cmp(&a.1.1));
+        // Keep only the top 10
+        results.truncate(10);
+        results
+    }
+}
+
+pub static RECENT_FLOWS: Lazy<TimeBuffer> = Lazy::new(|| TimeBuffer::new());
+
+pub struct FinishedFlowAnalysis {}
+
+impl FinishedFlowAnalysis {
+    pub fn new() -> Arc<Self> {
+        log::debug!("Created Flow Analysis Endpoint");
+
+        std::thread::spawn(|| loop {
+            RECENT_FLOWS.expire_over_five_minutes();
+            std::thread::sleep(std::time::Duration::from_secs(60 * 5));
+        });
+
+        Arc::new(Self {})
+    }
+}
+
+impl FlowbeeRecipient for FinishedFlowAnalysis {
+    fn enqueue(&self, key: FlowbeeKey, data: FlowbeeLocalData, analysis: FlowAnalysis) {
+        log::debug!("Finished flow analysis");
+        RECENT_FLOWS.push(TimeEntry {
+            time: std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap()
+                .as_secs(),
+            data: (key, data, analysis),
+        });
+    }
+}

src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/kernel_ringbuffer.rs

@@ -0,0 +1,227 @@
+//! Connects to the flows.h "flowbee_events" ring buffer and processes the events.
+use crate::throughput_tracker::flow_data::flow_analysis::rtt_types::RttData;
+use fxhash::FxHashMap;
+use lqos_sys::flowbee_data::FlowbeeKey;
+use lqos_utils::unix_time::time_since_boot;
+use once_cell::sync::Lazy;
+use std::{
+    ffi::c_void, net::{IpAddr, Ipv4Addr, Ipv6Addr}, slice, sync::{atomic::AtomicU64, Mutex}, time::Duration
+};
+use zerocopy::FromBytes;
+
+static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
+static EVENTS_PER_SECOND: AtomicU64 = AtomicU64::new(0);
+
+const BUFFER_SIZE: usize = 1024;
+
+struct RttBuffer {
+    index: usize,
+    buffer: [[RttData; BUFFER_SIZE]; 2],
+    last_seen: u64,
+    has_new_data: [bool; 2],
+}
+
+impl RttBuffer {
+    fn new(reading: u64, direction: u32, last_seen: u64) -> Self {
+        let empty = [RttData::from_nanos(0); BUFFER_SIZE];
+        let mut filled = [RttData::from_nanos(0); BUFFER_SIZE];
+        filled[0] = RttData::from_nanos(reading);
+
+        if direction == 0 {
+            Self {
+                index: 1,
+                buffer: [empty, filled],
+                last_seen,
+                has_new_data: [false, true],
+            }
+        } else {
+            Self {
+                index: 0,
+                buffer: [filled, empty],
+                last_seen,
+                has_new_data: [true, false],
+            }
+        }
+    }
+
+    fn push(&mut self, reading: u64, direction: u32, last_seen: u64) {
+        self.buffer[direction as usize][self.index] = RttData::from_nanos(reading);
+        self.index = (self.index + 1) % BUFFER_SIZE;
+        self.last_seen = last_seen;
+        self.has_new_data[direction as usize] = true;
+    }
+
+    fn median_new_data(&self, direction: usize) -> RttData {
+        if !self.has_new_data[direction] {
+            // Reject with no new data
+            return RttData::from_nanos(0);
+        }
+        let mut sorted = self.buffer[direction].iter().filter(|x| x.as_nanos() > 0).collect::<Vec<_>>();
+        if sorted.is_empty() {
+            return RttData::from_nanos(0);
+        }
+        sorted.sort_unstable();
+        let mid = sorted.len() / 2;
+        *sorted[mid]
+    }
+}
+
+struct FlowTracker {
+    flow_rtt: FxHashMap<FlowbeeKey, RttBuffer>,
+    ignore_subnets: ip_network_table::IpNetworkTable<bool>,
+}
+
+impl FlowTracker {
+    fn new() -> Self {
+        let config = lqos_config::load_config().unwrap();
+        let mut ignore_subnets = ip_network_table::IpNetworkTable::new();
+        if let Some(flows) = &config.flows {
+            if let Some(subnets) = &flows.do_not_track_subnets {
+                // Subnets are in CIDR notation
+                for subnet in subnets.iter() {
+                    let mut mask;
+                    if subnet.contains('/') {
+                        let split = subnet.split('/').collect::<Vec<_>>();
+                        println!("{:?}", split);
+                        if split.len() != 2 {
+                            log::error!("Invalid subnet: {}", subnet);
+                            continue;
+                        }
+                        let ip = if split[0].contains(":") {
+                            // It's IPv6
+                            mask = split[1].parse().unwrap_or(128);
+                            let ip: Ipv6Addr = split[0].parse().unwrap();
+                            ip
+                        } else {
+                            // It's IPv4
+                            mask = split[1].parse().unwrap_or(32);
+                            let ip: Ipv4Addr = split[0].parse().unwrap();
+                            mask += 96;
+                            ip.to_ipv6_mapped()
+                        };
+                        println!("{:?} {:?}", ip, mask);
+
+                        let addr = ip_network::IpNetwork::new(ip, mask).unwrap();
+                        ignore_subnets.insert(addr, true);
+                    } else {
+                        log::error!("Invalid subnet: {}", subnet);
+                        continue;                    
+                    }
+
+                }
+            }
+        }
+
+        Self {
+            flow_rtt: FxHashMap::default(),
+            ignore_subnets,
+        }
+    }
+}
+
+static FLOW_RTT: Lazy<Mutex<FlowTracker>> =
+    Lazy::new(|| Mutex::new(FlowTracker::new()));
+
+#[repr(C)]
+#[derive(FromBytes, Debug, Clone, PartialEq, Eq, Hash)]
+pub struct FlowbeeEvent {
+    key: FlowbeeKey,
+    rtt: u64,
+    effective_direction: u32,
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn flowbee_handle_events(
+    _ctx: *mut c_void,
+    data: *mut c_void,
+    data_size: usize,
+) -> i32 {
+    const EVENT_SIZE: usize = std::mem::size_of::<FlowbeeEvent>();
+    if data_size < EVENT_SIZE {
+        log::warn!("Warning: incoming data too small in Flowbee buffer");
+        return 0;
+    }
+
+    let data_u8 = data as *const u8;
+    let data_slice: &[u8] = slice::from_raw_parts(data_u8, EVENT_SIZE);
+    if let Some(incoming) = FlowbeeEvent::read_from(data_slice) {
+        EVENT_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+
+        if let Ok(now) = time_since_boot() {
+            let since_boot = Duration::from(now);
+            if incoming.rtt == 0 {
+                return 0;
+            }
+            let mut lock = FLOW_RTT.lock().unwrap();
+            // Check if it should be ignored
+            let ip = incoming.key.remote_ip.as_ip();
+            let ip = match ip {
+                IpAddr::V4(ip) => {
+                    ip.to_ipv6_mapped()
+                }
+                IpAddr::V6(ip) => {
+                    ip
+                }
+            };
+
+            if lock.ignore_subnets.longest_match(ip).is_some() {
+                return 0;
+            }
+
+            // Insert it
+            if let Some(entry) = lock.flow_rtt.get_mut(&incoming.key) {
+                entry.push(
+                    incoming.rtt,
+                    incoming.effective_direction,
+                    since_boot.as_nanos() as u64,
+                );
+            } else {
+                lock.flow_rtt.insert(
+                    incoming.key,
+                    RttBuffer::new(
+                        incoming.rtt,
+                        incoming.effective_direction,
+                        since_boot.as_nanos() as u64,
+                    ),
+                );
+            }
+        }
+    } else {
+        log::error!("Failed to decode Flowbee Event");
+    }
+
+    0
+}
+
+pub fn get_flowbee_event_count_and_reset() -> u64 {
+    let count = EVENT_COUNT.swap(0, std::sync::atomic::Ordering::Relaxed);
+    EVENTS_PER_SECOND.store(count, std::sync::atomic::Ordering::Relaxed);
+    count
+}
+
+pub fn expire_rtt_flows() {
+    if let Ok(now) = time_since_boot() {
+        let since_boot = Duration::from(now);
+        let expire = (since_boot - Duration::from_secs(30)).as_nanos() as u64;
+        let mut lock = FLOW_RTT.lock().unwrap();
+        lock.flow_rtt.retain(|_, v| v.last_seen > expire);
+    }
+}
+
+pub fn flowbee_rtt_map() -> FxHashMap<FlowbeeKey, [RttData; 2]> {
+    let mut lock = FLOW_RTT.lock().unwrap();
+    let result = lock.flow_rtt.iter()
+        .map(|(k, v)| (k.clone(), [v.median_new_data(0), v.median_new_data(1)]))
+        .collect();
+
+    // Clear all fresh data labeling
+    lock.flow_rtt.iter_mut().for_each(|(_, v)| {
+        v.has_new_data = [false, false];
+    });
+
+    result
+}
+
+pub fn get_rtt_events_per_second() -> u64 {
+    EVENTS_PER_SECOND.swap(0, std::sync::atomic::Ordering::Relaxed)
+}

src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/mod.rs

@@ -0,0 +1,97 @@
+use std::{net::IpAddr, sync::Mutex};
+use lqos_sys::flowbee_data::FlowbeeKey;
+use once_cell::sync::Lazy;
+mod asn;
+mod protocol;
+pub use protocol::FlowProtocol;
+use super::AsnId;
+mod finished_flows;
+pub use finished_flows::FinishedFlowAnalysis;
+pub use finished_flows::RECENT_FLOWS;
+mod kernel_ringbuffer;
+pub use kernel_ringbuffer::*;
+mod rtt_types;
+pub use rtt_types::RttData;
+
+static ANALYSIS: Lazy<FlowAnalysisSystem> = Lazy::new(|| FlowAnalysisSystem::new());
+
+pub struct FlowAnalysisSystem {
+    asn_table: Mutex<Option<asn::GeoTable>>,
+}
+
+impl FlowAnalysisSystem {
+    pub fn new() -> Self {
+        // Periodically update the ASN table
+        std::thread::spawn(|| {
+            loop {
+                let result = asn::GeoTable::load();
+                match result {
+                    Ok(table) => {
+                        ANALYSIS.asn_table.lock().unwrap().replace(table);
+                    }
+                    Err(e) => {
+                        log::error!("Failed to update ASN table: {e}");
+                    }
+                }
+                std::thread::sleep(std::time::Duration::from_secs(60 * 60 * 24));
+            }
+        });
+
+        Self {
+            asn_table: Mutex::new(None),
+        }
+    }
+}
+
+pub fn setup_flow_analysis() -> anyhow::Result<()> {
+    let e = ANALYSIS.asn_table.lock();
+    if e.is_err() {
+        anyhow::bail!("Failed to lock ASN table");
+    }
+    Ok(())
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct FlowAnalysis {
+    pub asn_id: AsnId,
+    pub protocol_analysis: FlowProtocol,
+}
+
+impl FlowAnalysis {
+    pub fn new(key: &FlowbeeKey) -> Self {
+        let asn_id = lookup_asn_id(key.remote_ip.as_ip());
+        let protocol_analysis = FlowProtocol::new(key);
+        Self {
+            asn_id: AsnId(asn_id.unwrap_or(0)),
+            protocol_analysis,
+        }
+    }
+}
+
+
+pub fn lookup_asn_id(ip: IpAddr) -> Option<u32> {
+    if let Ok(table_lock) = ANALYSIS.asn_table.lock() {
+        if let Some(table) = table_lock.as_ref() {
+            return table.find_asn(ip);
+        }
+    }
+    None
+}
+
+pub fn get_asn_name_and_country(ip: IpAddr) -> (String, String) {
+    if let Ok(table_lock) = ANALYSIS.asn_table.lock() {
+        if let Some(table) = table_lock.as_ref() {
+            return table.find_owners_by_ip(ip);
+        }
+    }
+    (String::new(), String::new())
+}
+
+pub fn get_asn_lat_lon(ip: IpAddr) -> (f64, f64) {
+    if let Ok(table_lock) = ANALYSIS.asn_table.lock() {
+        if let Some(table) = table_lock.as_ref() {
+            return table.find_lat_lon_by_ip(ip);
+        }
+    }
+    (0.0, 0.0)
+}

src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/protocol.rs

@@ -0,0 +1,93 @@
+use std::fmt::Display;
+
+use lqos_sys::flowbee_data::FlowbeeKey;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum FlowProtocol {
+    Smtp,
+    Ftp,
+    Http,
+    Https,
+    Ssh,
+    Telnet,
+    Imap,
+    Rdp,
+    Dns,
+    Pop3,
+    Quic,
+    Other { proto: u8, src_port: u16, dst_port: u16 }
+}
+
+impl FlowProtocol {
+    pub fn new(key: &FlowbeeKey) -> Self {
+        match key.ip_protocol {
+            6 => Self::tcp(key),
+            17 => Self::udp(key),
+            _ => Self::Other { 
+                proto: key.ip_protocol, 
+                src_port: key.src_port, 
+                dst_port: key.dst_port, 
+            }
+        }
+    }
+
+    fn tcp(key: &FlowbeeKey) -> Self {
+        match key.src_port {
+            25 => Self::Smtp,
+            80 => Self::Http,
+            443 => Self::Https,
+            21 | 20 => Self::Ftp,
+            22 => Self::Ssh,
+            23 => Self::Telnet,
+            3389 => Self::Rdp,
+            143 => Self::Imap,
+            53 => Self::Dns,
+            110 => Self::Pop3,
+            _ => Self::Other { 
+                proto: key.ip_protocol, 
+                src_port: key.src_port, 
+                dst_port: key.dst_port, 
+            }
+        }
+    }
+
+    fn udp(key: &FlowbeeKey) -> Self {
+        match key.src_port {
+            53 => Self::Dns,
+            80 | 443 => Self::Quic,            
+            _ => Self::Other { 
+                proto: key.ip_protocol, 
+                src_port: key.src_port, 
+                dst_port: key.dst_port, 
+            }
+        }
+    }
+}
+
+impl Display for FlowProtocol {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Smtp => write!(f, "SMTP"),
+            Self::Ftp => write!(f, "FTP"),
+            Self::Http => write!(f, "HTTP"),
+            Self::Https => write!(f, "HTTPS"),
+            Self::Ssh => write!(f, "SSH"),
+            Self::Telnet => write!(f, "Telnet"),
+            Self::Imap => write!(f, "IMAP"),
+            Self::Rdp => write!(f, "RDP"),
+            Self::Dns => write!(f, "DNS"),
+            Self::Pop3 => write!(f, "POP3"),
+            Self::Quic => write!(f, "QUIC"),
+            Self::Other { proto, src_port, dst_port } => write!(f, "{} {}/{}", proto_name(proto), src_port, dst_port),
+        }
+    }
+}
+
+fn proto_name(proto: &u8) -> &'static str {
+    match proto {
+        6 => "TCP",
+        17 => "UDP",
+        1 => "ICMP",
+        _ => "Other",
+    }
+}

src/rust/lqosd/src/throughput_tracker/flow_data/flow_analysis/rtt_types.rs

@@ -0,0 +1,38 @@
+//! Provides a set of types for representing round-trip time (RTT) data,
+//! as produced by the eBPF system and consumed in different ways.
+//! 
+//! Adopting strong-typing is an attempt to reduce confusion with
+//! multipliers, divisors, etc. It is intended to become pervasive
+//! throughout the system.
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct RttData {
+    nanoseconds: u64,
+}
+
+#[allow(dead_code)]
+impl RttData {
+    pub fn from_nanos(nanoseconds: u64) -> Self {
+        Self { nanoseconds }
+    }
+
+    pub fn as_nanos(&self) -> u64 {
+        self.nanoseconds
+    }
+
+    pub fn as_micros(&self) -> f64 {
+        self.nanoseconds as f64 / 1_000.0
+    }
+
+    pub fn as_millis(&self) -> f64 {
+        self.nanoseconds as f64 / 1_000_000.0
+    }
+
+    pub fn as_millis_times_100(&self) -> f64 {
+        self.nanoseconds as f64 / 10_000.0
+    }
+
+    pub fn as_seconds(&self) -> f64 {
+        self.nanoseconds as f64 / 1_000_000_000.0
+    }
+}

src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs

@@ -0,0 +1,59 @@
+//! Provides a globally accessible vector of all flows. This is used to store
+//! all flows for the purpose of tracking and data-services.
+
+use super::{flow_analysis::FlowAnalysis, RttData};
+use lqos_sys::flowbee_data::{FlowbeeData, FlowbeeKey};
+use once_cell::sync::Lazy;
+use std::{collections::HashMap, sync::Mutex};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct AsnId(pub u32);
+
+pub static ALL_FLOWS: Lazy<Mutex<HashMap<FlowbeeKey, (FlowbeeLocalData, FlowAnalysis)>>> =
+    Lazy::new(|| Mutex::new(HashMap::new()));
+
+
+/// Condensed representation of the FlowbeeData type. This contains
+/// only the information we want to keep locally for analysis purposes,
+/// adds RTT data, and uses Rust-friendly typing.
+#[derive(Debug, Clone)]
+pub struct FlowbeeLocalData {
+    /// Time (nanos) when the connection was established
+    pub start_time: u64,
+    /// Time (nanos) when the connection was last seen
+    pub last_seen: u64,
+    /// Bytes transmitted
+    pub bytes_sent: [u64; 2],
+    /// Packets transmitted
+    pub packets_sent: [u64; 2],
+    /// Rate estimate
+    pub rate_estimate_bps: [u32; 2],
+    /// TCP Retransmission count (also counts duplicates)
+    pub tcp_retransmits: [u16; 2],
+    /// Has the connection ended?
+    /// 0 = Alive, 1 = FIN, 2 = RST
+    pub end_status: u8,
+    /// Raw IP TOS
+    pub tos: u8,
+    /// Raw TCP flags
+    pub flags: u8,
+    /// Recent RTT median
+    pub rtt: [RttData; 2],
+}
+
+impl From<&FlowbeeData> for FlowbeeLocalData {
+    fn from(data: &FlowbeeData) -> Self {
+        Self {
+            start_time: data.start_time,
+            last_seen: data.last_seen,
+            bytes_sent: data.bytes_sent,
+            packets_sent: data.packets_sent,
+            rate_estimate_bps: data.rate_estimate_bps,
+            tcp_retransmits: data.tcp_retransmits,
+            end_status: data.end_status,
+            tos: data.tos,
+            flags: data.flags,
+            rtt: [RttData::from_nanos(0); 2],
+        }
+    }
+}

src/rust/lqosd/src/throughput_tracker/flow_data/mod.rs

@@ -0,0 +1,74 @@
+//! Provides tracking and data-services for per-flow data. Includes implementations
+//! of netflow protocols.
+
+mod flow_tracker;
+mod netflow5;
+mod netflow9;
+mod flow_analysis;
+
+use crate::throughput_tracker::flow_data::{flow_analysis::FinishedFlowAnalysis, netflow5::Netflow5, netflow9::Netflow9};
+pub(crate) use flow_tracker::{ALL_FLOWS, AsnId, FlowbeeLocalData};
+use lqos_sys::flowbee_data::FlowbeeKey;
+use std::sync::{
+    mpsc::{channel, Sender},
+    Arc,
+};
+pub(crate) use flow_analysis::{setup_flow_analysis, get_asn_name_and_country, 
+    FlowAnalysis, RECENT_FLOWS, flowbee_handle_events, get_flowbee_event_count_and_reset,
+    expire_rtt_flows, flowbee_rtt_map, RttData, get_rtt_events_per_second,
+};
+
+
+trait FlowbeeRecipient {
+    fn enqueue(&self, key: FlowbeeKey, data: FlowbeeLocalData, analysis: FlowAnalysis);
+}
+
+// Creates the netflow tracker and returns the sender
+pub fn setup_netflow_tracker() -> Sender<(FlowbeeKey, (FlowbeeLocalData, FlowAnalysis))> {
+    let (tx, rx) = channel::<(FlowbeeKey, (FlowbeeLocalData, FlowAnalysis))>();
+    let config = lqos_config::load_config().unwrap();
+
+    std::thread::spawn(move || {
+        log::info!("Starting the network flow tracker back-end");
+
+        // Build the endpoints list
+        let mut endpoints: Vec<Arc<dyn FlowbeeRecipient>> = Vec::new();
+        endpoints.push(FinishedFlowAnalysis::new());
+
+        if let Some(flow_config) = config.flows {
+            if let (Some(ip), Some(port), Some(version)) = (
+                flow_config.netflow_ip,
+                flow_config.netflow_port,
+                flow_config.netflow_version,
+            ) {
+                log::info!("Setting up netflow target: {ip}:{port}, version: {version}");
+                let target = format!("{ip}:{port}", ip = ip, port = port);
+                match version {
+                    5 => {
+                        let endpoint = Netflow5::new(target).unwrap();
+                        endpoints.push(endpoint);
+                        log::info!("Netflow 5 endpoint added");
+                    }
+                    9 => {
+                        let endpoint = Netflow9::new(target).unwrap();
+                        endpoints.push(endpoint);
+                        log::info!("Netflow 9 endpoint added");
+                    }
+                    _ => log::error!("Unsupported netflow version: {version}"),
+                }
+            }
+        }
+        log::info!("Flow Endpoints: {}", endpoints.len());
+
+        // Send to all endpoints upon receipt
+        while let Ok((key, (value, analysis))) = rx.recv() {
+            endpoints.iter_mut().for_each(|f| {
+                //log::debug!("Enqueueing flow data for {key:?}");
+                f.enqueue(key.clone(), value.clone(), analysis.clone());
+            });
+        }
+        log::info!("Network flow tracker back-end has stopped")
+    });
+
+    tx
+}

src/rust/lqosd/src/throughput_tracker/flow_data/netflow5/mod.rs

@@ -0,0 +1,90 @@
+//! Support for the Netflow 5 protocol
+//! Mostly taken from: https://netflow.caligare.com/netflow_v5.htm
+mod protocol;
+use super::{FlowAnalysis, FlowbeeLocalData, FlowbeeRecipient};
+use lqos_sys::flowbee_data::FlowbeeKey;
+pub(crate) use protocol::*;
+use std::{
+    net::UdpSocket,
+    sync::{atomic::AtomicU32, Arc, Mutex},
+};
+
+pub(crate) struct Netflow5 {
+    socket: UdpSocket,
+    sequence: AtomicU32,
+    target: String,
+    send_queue: Mutex<Vec<(FlowbeeKey, FlowbeeLocalData)>>,
+}
+
+impl Netflow5 {
+    pub(crate) fn new(target: String) -> anyhow::Result<Arc<Self>> {
+        let socket = UdpSocket::bind("0.0.0.0:12212")?;
+        let result = Arc::new(Self {
+            socket,
+            sequence: AtomicU32::new(0),
+            target,
+            send_queue: Mutex::new(Vec::new()),
+        });
+        let thread_result = result.clone();
+        std::thread::spawn(move || thread_result.queue_handler());
+        Ok(result)
+    }
+
+    fn queue_handler(&self) {
+        loop {
+            let mut lock = self.send_queue.lock().unwrap();
+            if lock.is_empty() {
+                std::thread::sleep(std::time::Duration::from_millis(100));
+                continue;
+            }
+
+            let send_chunks = lock.chunks(15);
+            for to_send in send_chunks {
+                let num_records = (to_send.len() * 2) as u16;
+                let sequence = self.sequence.load(std::sync::atomic::Ordering::Relaxed);
+                let header = Netflow5Header::new(sequence, num_records);
+                let header_bytes = unsafe {
+                    std::slice::from_raw_parts(
+                        &header as *const _ as *const u8,
+                        std::mem::size_of::<Netflow5Header>(),
+                    )
+                };
+    
+                let mut buffer = Vec::with_capacity(
+                    header_bytes.len() + to_send.len() * 2 * std::mem::size_of::<Netflow5Record>(),
+                );
+    
+                buffer.extend_from_slice(header_bytes);
+                for (key, data) in to_send {
+                    if let Ok((packet1, packet2)) = to_netflow_5(key, data) {
+                        let packet1_bytes = unsafe {
+                            std::slice::from_raw_parts(
+                                &packet1 as *const _ as *const u8,
+                                std::mem::size_of::<Netflow5Record>(),
+                            )
+                        };
+                        let packet2_bytes = unsafe {
+                            std::slice::from_raw_parts(
+                                &packet2 as *const _ as *const u8,
+                                std::mem::size_of::<Netflow5Record>(),
+                            )
+                        };
+                        buffer.extend_from_slice(packet1_bytes);
+                        buffer.extend_from_slice(packet2_bytes);
+                    }
+                }
+
+                self.socket.send_to(&buffer, &self.target).unwrap();
+                self.sequence.fetch_add(num_records as u32, std::sync::atomic::Ordering::Relaxed);    
+            }
+            lock.clear();
+        }
+    }
+}
+
+impl FlowbeeRecipient for Netflow5 {
+    fn enqueue(&self, key: FlowbeeKey, data: FlowbeeLocalData, _analysis: FlowAnalysis) {
+        let mut lock = self.send_queue.lock().unwrap();
+        lock.push((key, data));
+    }
+}

src/rust/lqosd/src/throughput_tracker/flow_data/netflow5/protocol.rs

@@ -0,0 +1,132 @@
+//! Definitions for the actual netflow 5 protocol
+
+use std::net::IpAddr;
+use lqos_sys::flowbee_data::FlowbeeKey;
+use lqos_utils::unix_time::time_since_boot;
+use nix::sys::time::TimeValLike;
+
+use crate::throughput_tracker::flow_data::FlowbeeLocalData;
+
+/// Standard Netflow 5 header
+#[repr(C)]
+pub(crate) struct Netflow5Header {
+    pub(crate) version: u16,
+    pub(crate) count: u16,
+    pub(crate) sys_uptime: u32,
+    pub(crate) unix_secs: u32,
+    pub(crate) unix_nsecs: u32,
+    pub(crate) flow_sequence: u32,
+    pub(crate) engine_type: u8,
+    pub(crate) engine_id: u8,
+    pub(crate) sampling_interval: u16,
+}
+
+impl Netflow5Header {
+    /// Create a new Netflow 5 header
+    pub(crate) fn new(flow_sequence: u32, num_records: u16) -> Self {
+        let uptime = time_since_boot().unwrap();
+
+        Self {
+            version: (5u16).to_be(),
+            count: num_records.to_be(),
+            sys_uptime: (uptime.num_milliseconds() as u32).to_be(),
+            unix_secs: (uptime.num_seconds() as u32).to_be(),
+            unix_nsecs: 0,
+            flow_sequence,
+            engine_type: 0,
+            engine_id: 0,
+            sampling_interval: 0,
+        }
+    }
+}
+
+/// Standard Netflow 5 record
+#[repr(C)]
+pub(crate) struct Netflow5Record {
+    pub(crate) src_addr: u32,
+    pub(crate) dst_addr: u32,
+    pub(crate) next_hop: u32,
+    pub(crate) input: u16,
+    pub(crate) output: u16,
+    pub(crate) d_pkts: u32,
+    pub(crate) d_octets: u32,
+    pub(crate) first: u32,
+    pub(crate) last: u32,
+    pub(crate) src_port: u16,
+    pub(crate) dst_port: u16,
+    pub(crate) pad1: u8,
+    pub(crate) tcp_flags: u8,
+    pub(crate) prot: u8,
+    pub(crate) tos: u8,
+    pub(crate) src_as: u16,
+    pub(crate) dst_as: u16,
+    pub(crate) src_mask: u8,
+    pub(crate) dst_mask: u8,
+    pub(crate) pad2: u16,
+}
+
+/// Convert a Flowbee key and data to a pair of Netflow 5 records
+pub(crate) fn to_netflow_5(key: &FlowbeeKey, data: &FlowbeeLocalData) -> anyhow::Result<(Netflow5Record, Netflow5Record)> {
+    // TODO: Detect overflow
+    let local = key.local_ip.as_ip();
+    let remote = key.remote_ip.as_ip();
+    if let (IpAddr::V4(local), IpAddr::V4(remote)) = (local, remote) {
+        let src_ip = u32::from_ne_bytes(local.octets());
+        let dst_ip = u32::from_ne_bytes(remote.octets());
+        // Convert d_pkts to network order
+        let d_pkts = (data.packets_sent[0] as u32).to_be();
+        let d_octets = (data.bytes_sent[0] as u32).to_be();
+        let d_pkts2 = (data.packets_sent[1] as u32).to_be();
+        let d_octets2 = (data.bytes_sent[1] as u32).to_be();
+
+        let record = Netflow5Record {
+            src_addr: src_ip,
+            dst_addr: dst_ip,
+            next_hop: 0,
+            input: (0u16).to_be(),
+            output: (1u16).to_be(),
+            d_pkts,
+            d_octets,
+            first: ((data.start_time  / 1_000_000) as u32).to_be(), // Convert to milliseconds
+            last: ((data.last_seen / 1_000_000) as u32).to_be(), // Convert to milliseconds
+            src_port: key.src_port.to_be(),
+            dst_port: key.dst_port.to_be(),
+            pad1: 0,
+            tcp_flags: 0,
+            prot: key.ip_protocol.to_be(),
+            tos: 0,
+            src_as: 0,
+            dst_as: 0,
+            src_mask: 0,
+            dst_mask: 0,
+            pad2: 0,
+        };
+
+        let record2 = Netflow5Record {
+            src_addr: dst_ip,
+            dst_addr: src_ip,
+            next_hop: 0,
+            input: 1,
+            output: 0,
+            d_pkts: d_pkts2,
+            d_octets: d_octets2,
+            first: data.start_time as u32, // Convert to milliseconds
+            last: data.last_seen as u32, // Convert to milliseconds
+            src_port: key.dst_port.to_be(),
+            dst_port: key.src_port.to_be(),
+            pad1: 0,
+            tcp_flags: 0,
+            prot: key.ip_protocol.to_be(),
+            tos: 0,
+            src_as: 0,
+            dst_as: 0,
+            src_mask: 0,
+            dst_mask: 0,
+            pad2: 0,
+        };
+
+        Ok((record, record2))
+    } else {
+        Err(anyhow::anyhow!("Only IPv4 is supported"))
+    }
+}

src/rust/lqosd/src/throughput_tracker/flow_data/netflow9/mod.rs

@@ -0,0 +1,73 @@
+use crate::throughput_tracker::flow_data::netflow9::protocol::{
+    header::Netflow9Header, template_ipv4::template_data_ipv4, template_ipv6::template_data_ipv6,
+};
+use lqos_sys::flowbee_data::FlowbeeKey;
+use std::{net::UdpSocket, sync::{atomic::AtomicU32, Arc, Mutex}};
+
+use self::protocol::to_netflow_9;
+use super::{FlowAnalysis, FlowbeeLocalData, FlowbeeRecipient};
+mod protocol;
+
+pub(crate) struct Netflow9 {
+    socket: UdpSocket,
+    sequence: AtomicU32,
+    target: String,
+    send_queue: Mutex<Vec<(FlowbeeKey, FlowbeeLocalData)>>,
+}
+
+impl Netflow9 {
+    pub(crate) fn new(target: String) -> anyhow::Result<Arc<Self>> {
+        let socket = UdpSocket::bind("0.0.0.0:12212")?;
+        let result = Arc::new(Self {
+            socket,
+            sequence: AtomicU32::new(0),
+            target,
+            send_queue: Mutex::new(Vec::new()),
+        });
+        let thread_result = result.clone();
+        std::thread::spawn(move || thread_result.queue_handler());
+        Ok(result)
+    }
+
+    fn queue_handler(&self) {
+        loop {
+            let mut lock = self.send_queue.lock().unwrap();
+            if lock.is_empty() {
+                std::thread::sleep(std::time::Duration::from_millis(100));
+                continue;
+            }
+
+            let send_chunks = lock.chunks(14);            
+            for to_send in send_chunks {
+                let num_records = (to_send.len() * 2) as u16 + 2; // +2 to include templates
+                let sequence = self.sequence.load(std::sync::atomic::Ordering::Relaxed);
+                let header = Netflow9Header::new(sequence, num_records);
+                let header_bytes = unsafe { std::slice::from_raw_parts(&header as *const _ as *const u8, std::mem::size_of::<Netflow9Header>()) };
+                let template1 = template_data_ipv4();
+                let template2 = template_data_ipv6();
+                let mut buffer = Vec::with_capacity(header_bytes.len() + template1.len() + template2.len() + (num_records as usize) * 140);
+                buffer.extend_from_slice(header_bytes);
+                buffer.extend_from_slice(&template1);
+                buffer.extend_from_slice(&template2);
+
+                for (key, data) in to_send {
+                    if let Ok((packet1, packet2)) = to_netflow_9(key, data) {
+                        buffer.extend_from_slice(&packet1);
+                        buffer.extend_from_slice(&packet2);
+                    }
+                }
+                self.socket.send_to(&buffer, &self.target).unwrap();
+                self.sequence.fetch_add(num_records as u32, std::sync::atomic::Ordering::Relaxed);
+            }
+            lock.clear();
+        }
+
+    }
+}
+
+impl FlowbeeRecipient for Netflow9 {
+    fn enqueue(&self, key: FlowbeeKey, data: FlowbeeLocalData, _analysis: FlowAnalysis) {
+        let mut lock = self.send_queue.lock().unwrap();
+        lock.push((key, data));
+    }
+}

src/rust/lqosd/src/throughput_tracker/flow_data/netflow9/protocol/field_encoder.rs

@@ -0,0 +1,70 @@
+use std::net::IpAddr;
+use lqos_sys::flowbee_data::FlowbeeKey;
+use crate::throughput_tracker::flow_data::FlowbeeLocalData;
+use super::field_types::*;
+
+pub(crate) fn encode_fields_from_template(template: &[(u16, u16)], direction: usize, key: &FlowbeeKey, data: &FlowbeeLocalData) -> anyhow::Result<Vec<u8>> {
+    let src_port = if direction == 0 { key.src_port } else { key.dst_port };
+    let dst_port = if direction == 0 { key.dst_port } else { key.src_port };
+
+    let total_size: u16 = template.iter().map(|(_, size)| size).sum();
+    let mut result = Vec::with_capacity(total_size as usize);
+    for (field_type, field_length) in template.iter() {
+        match (*field_type, *field_length) {
+            IN_BYTES => encode_u64(data.bytes_sent[direction], &mut result),
+            IN_PKTS => encode_u64(data.packets_sent[direction], &mut result),
+            PROTOCOL => result.push(key.ip_protocol),
+            L4_SRC_PORT => encode_u16(src_port, &mut result),
+            L4_DST_PORT => encode_u16(dst_port, &mut result),
+            DST_TOS => result.push(data.tos),
+            IPV4_SRC_ADDR => encode_ipv4(0, key, &mut result)?,
+            IPV4_DST_ADDR => encode_ipv4(1, key, &mut result)?,
+            IPV6_SRC_ADDR => encode_ipv6(0, key, &mut result)?,
+            IPV6_DST_ADDR => encode_ipv6(1, key, &mut result)?,
+            _ => anyhow::bail!("Don't know how to encode field type {} yet", field_type),
+        }
+    }
+    Ok(result)
+}
+
+fn encode_u64(value: u64, target: &mut Vec<u8>) {
+    target.extend_from_slice(&value.to_be_bytes());
+}
+
+fn encode_u16(value: u16, target: &mut Vec<u8>) {
+    target.extend_from_slice(&value.to_be_bytes());
+}
+
+fn encode_ipv4(direction: usize, key: &FlowbeeKey, target: &mut Vec<u8>) -> anyhow::Result<()> {
+    let local = key.local_ip.as_ip();
+    let remote = key.remote_ip.as_ip();
+    if let (IpAddr::V4(local), IpAddr::V4(remote)) = (local, remote) {
+        let src_ip = u32::from_ne_bytes(local.octets());
+        let dst_ip = u32::from_ne_bytes(remote.octets());
+        if direction == 0 {
+            target.extend_from_slice(&src_ip.to_be_bytes());
+        } else {
+            target.extend_from_slice(&dst_ip.to_be_bytes());
+        }
+    } else {
+        anyhow::bail!("Expected IPv4 addresses, got {:?}", (local, remote));
+    }
+    Ok(())
+}
+
+fn encode_ipv6(direction: usize, key: &FlowbeeKey, target: &mut Vec<u8>) -> anyhow::Result<()> {
+    let local = key.local_ip.as_ip();
+    let remote = key.remote_ip.as_ip();
+    if let (IpAddr::V6(local), IpAddr::V6(remote)) = (local, remote) {
+        let src_ip = local.octets();
+        let dst_ip = remote.octets();
+        if direction == 0 {
+            target.extend_from_slice(&src_ip);
+        } else {
+            target.extend_from_slice(&dst_ip);
+        }
+    } else {
+        anyhow::bail!("Expected IPv6 addresses, got {:?}", (local, remote));
+    }
+    Ok(())
+}

src/rust/lqosd/src/throughput_tracker/flow_data/netflow9/protocol/field_types.rs

@@ -0,0 +1,82 @@
+// Extracted from https://netflow.caligare.com/netflow_v9.htm
+#![allow(dead_code)]
+
+pub(crate) const IN_BYTES:(u16, u16) = (1, 8);
+pub(crate) const IN_PKTS:(u16, u16) = (2, 8);
+pub(crate) const FLOWS:(u16, u16) = (3, 4);
+pub(crate) const PROTOCOL:(u16, u16) = (4, 1);
+pub(crate) const SRC_TOS:(u16, u16) = (5, 1);
+pub(crate) const TCP_FLAGS:(u16, u16) = (6, 1);
+pub(crate) const L4_SRC_PORT:(u16, u16) = (7, 2);
+pub(crate) const IPV4_SRC_ADDR:(u16, u16) = (8, 4);
+pub(crate) const SRC_MASK:(u16, u16) = (9, 1);
+pub(crate) const INPUT_SNMP:(u16, u16) = (10, 2);
+pub(crate) const L4_DST_PORT:(u16, u16) = (11, 2);
+pub(crate) const IPV4_DST_ADDR:(u16, u16) = (12, 4);
+pub(crate) const DST_MASK:(u16, u16) = (13, 1);
+pub(crate) const OUTPUT_SNMP:(u16, u16) = (14, 2);
+pub(crate) const IPV4_NEXT_HOP:(u16, u16) = (15, 4);
+pub(crate) const SRC_AS:(u16, u16) = (16, 2);
+pub(crate) const DST_AS:(u16, u16) = (17, 2);
+pub(crate) const BGP_IPV4_NEXT_HOP:(u16, u16) = (18, 4);
+pub(crate) const MUL_DST_PKTS:(u16, u16) = (19, 4);
+pub(crate) const MUL_DST_BYTES:(u16, u16) = (20, 4);
+pub(crate) const LAST_SWITCHED:(u16, u16) = (21, 4);
+pub(crate) const FIRST_SWITCHED:(u16, u16) = (22, 4);
+pub(crate) const OUT_BYTES:(u16, u16) = (23, 4);
+pub(crate) const OUT_PKTS:(u16, u16) = (24, 4);
+pub(crate) const MIN_PKT_LNGTH:(u16, u16) = (25, 2);
+pub(crate) const MAX_PKT_LNGTH:(u16, u16) = (26, 2);
+pub(crate) const IPV6_SRC_ADDR:(u16, u16) = (27, 16);
+pub(crate) const IPV6_DST_ADDR:(u16, u16) = (28, 16);
+pub(crate) const IPV6_SRC_MASK:(u16, u16) = (29, 1);
+pub(crate) const IPV6_DST_MASK:(u16, u16) = (30, 1);
+pub(crate) const IPV6_FLOW_LABEL:(u16, u16) = (31, 3);
+pub(crate) const ICMP_TYPE:(u16, u16) = (32, 2);
+pub(crate) const MUL_IGMP_TYPE:(u16, u16) = (33, 1);
+pub(crate) const SAMPLING_INTERVAL:(u16, u16) = (34, 4);
+pub(crate) const SAMPLING_ALGORITHM:(u16, u16) = (35, 1);
+pub(crate) const FLOW_ACTIVE_TIMEOUT:(u16, u16) = (36, 2);
+pub(crate) const FLOW_INACTIVE_TIMEOUT:(u16, u16) = (37, 2);
+pub(crate) const ENGINE_TYPE:(u16, u16) = (38, 1);
+pub(crate) const ENGINE_ID:(u16, u16) = (39, 1);
+pub(crate) const TOTAL_BYTES_EXP:(u16, u16) = (40, 4);
+pub(crate) const TOTAL_PKTS_EXP:(u16, u16) = (41, 4);
+pub(crate) const TOTAL_FLOWS_EXP:(u16, u16) = (42, 4);
+pub(crate) const IPV4_SRC_PREFIX:(u16, u16) = (44, 4);
+pub(crate) const IPV4_DST_PREFIX:(u16, u16) = (45, 4);
+pub(crate) const MPLS_TOP_LABEL_TYPE:(u16, u16) = (46, 1);
+pub(crate) const MPLS_TOP_LABEL_IP_ADDR:(u16, u16) = (47, 4);
+pub(crate) const FLOW_SAMPLER_ID:(u16, u16) = (48, 1);
+pub(crate) const FLOW_SAMPLER_MODE:(u16, u16) = (49, 1);
+pub(crate) const FLOW_SAMPLER_RANDOM_INTERVAL:(u16, u16) = (50, 4);
+pub(crate) const MIN_TTL:(u16, u16) = (52, 1);
+pub(crate) const MAX_TTL:(u16, u16) = (53, 1);
+pub(crate) const IPV4_IDENT:(u16, u16) = (54, 2);
+pub(crate) const DST_TOS:(u16, u16) = (55, 1);
+pub(crate) const IN_SRC_MAC:(u16, u16) = (56, 6);
+pub(crate) const OUT_DST_MAC:(u16, u16) = (57, 6);
+pub(crate) const SRC_VLAN:(u16, u16) = (58, 2);
+pub(crate) const DST_VLAN:(u16, u16) = (59, 2);
+pub(crate) const IP_PROTOCOL_VERSION:(u16, u16) = (60, 1);
+pub(crate) const DIRECTION:(u16, u16) = (61, 1);
+pub(crate) const IPV6_NEXT_HOP:(u16, u16) = (62, 16);
+pub(crate) const BPG_IPV6_NEXT_HOP:(u16, u16) = (63, 16);
+pub(crate) const IPV6_OPTION_HEADERS:(u16, u16) = (64, 4);
+pub(crate) const MPLS_LABEL_1:(u16, u16) = (70, 3);
+pub(crate) const MPLS_LABEL_2:(u16, u16) = (71, 3);
+pub(crate) const MPLS_LABEL_3:(u16, u16) = (72, 3);
+pub(crate) const MPLS_LABEL_4:(u16, u16) = (73, 3);
+pub(crate) const MPLS_LABEL_5:(u16, u16) = (74, 3);
+pub(crate) const MPLS_LABEL_6:(u16, u16) = (75, 3);
+pub(crate) const MPLS_LABEL_7:(u16, u16) = (76, 3);
+pub(crate) const MPLS_LABEL_8:(u16, u16) = (77, 3);
+pub(crate) const MPLS_LABEL_9:(u16, u16) = (78, 3);
+pub(crate) const MPLS_LABEL_10:(u16, u16) = (79, 3);
+pub(crate) const IN_DST_MAC:(u16, u16) = (80, 6);
+pub(crate) const OUT_SRC_MAC:(u16, u16) = (81, 6);
+pub(crate) const IF_NAME:(u16, u16) = (82, 0);
+pub(crate) const IF_DESC:(u16, u16) = (83, 0);
+pub(crate) const SAMPLER_NAME:(u16, u16) = (84, 0);
+pub(crate) const IN_PERMANENT_BYTES:(u16, u16) = (85, 4);
+pub(crate) const IN_PERMANENT_PKTS:(u16, u16) = (86, 4);

src/rust/lqosd/src/throughput_tracker/flow_data/netflow9/protocol/header.rs

@@ -0,0 +1,28 @@
+use lqos_utils::unix_time::time_since_boot;
+use nix::sys::time::TimeValLike;
+
+#[repr(C)]
+pub(crate) struct Netflow9Header {
+    pub(crate) version: u16,
+    pub(crate) count: u16,
+    pub(crate) sys_uptime: u32,
+    pub(crate) unix_secs: u32,
+    pub(crate) package_sequence: u32,
+    pub(crate) source_id: u32,
+}
+
+impl Netflow9Header {
+    /// Create a new Netflow 9 header
+    pub(crate) fn new(flow_sequence: u32, record_count_including_templates: u16) -> Self {
+        let uptime = time_since_boot().unwrap();
+
+        Self {
+            version: (9u16).to_be(),
+            count: record_count_including_templates.to_be(),
+            sys_uptime: (uptime.num_milliseconds() as u32).to_be(),
+            unix_secs: (uptime.num_seconds() as u32).to_be(),
+            package_sequence: flow_sequence.to_be(),
+            source_id: 0,
+        }
+    }
+}

src/rust/lqosd/src/throughput_tracker/flow_data/netflow9/protocol/mod.rs

@@ -0,0 +1,96 @@
+//! Protocol definitions for Netflow v9 Data.
+//! Mostly derived from https://netflow.caligare.com/netflow_v9.htm
+
+use lqos_sys::flowbee_data::FlowbeeKey;
+mod field_types;
+use field_types::*;
+
+use crate::throughput_tracker::flow_data::FlowbeeLocalData;
+pub(crate) mod field_encoder;
+pub(crate) mod header;
+pub(crate) mod template_ipv4;
+pub(crate) mod template_ipv6;
+
+fn add_field(bytes: &mut Vec<u8>, field_type: u16, field_length: u16) {
+    bytes.extend_from_slice(field_type.to_be_bytes().as_ref());
+    bytes.extend_from_slice(field_length.to_be_bytes().as_ref());
+}
+
+pub(crate) fn to_netflow_9(
+    key: &FlowbeeKey,
+    data: &FlowbeeLocalData,
+) -> anyhow::Result<(Vec<u8>, Vec<u8>)> {
+    if key.local_ip.is_v4() && key.remote_ip.is_v4() {
+        // Return IPv4 records
+        Ok((ipv4_record(key, data, 0)?, ipv4_record(key, data, 1)?))
+    } else if (!key.local_ip.is_v4()) && (!key.remote_ip.is_v4()) {
+        // Return IPv6 records
+        Ok((ipv6_record(key, data, 0)?, ipv6_record(key, data, 1)?))
+    } else {
+        anyhow::bail!("Mixing IPv4 and IPv6 is not supported");
+    }
+}
+
+fn ipv4_record(key: &FlowbeeKey, data: &FlowbeeLocalData, direction: usize) -> anyhow::Result<Vec<u8>> {
+    let field_bytes = field_encoder::encode_fields_from_template(
+        &template_ipv4::FIELDS_IPV4,
+        direction,
+        key,
+        data,
+    )?;
+
+    // Build the actual record
+    let mut bytes = Vec::new();
+    // Add the flowset_id. Template ID is 256
+    bytes.extend_from_slice(&(256u16).to_be_bytes());
+
+    // Add the length. Length includes 2 bytes for flowset and 2 bytes for the length field
+    // itself. That's odd.
+    let padding = (field_bytes.len() + 4) % 4;
+    let size = (bytes.len() + field_bytes.len() + padding + 2) as u16;
+    bytes.extend_from_slice(&size.to_be_bytes());
+
+    // Add the data itself
+    bytes.extend_from_slice(&field_bytes);
+
+    println!("Padding: {}", padding);
+    println!("IPv4 data {} = {}", bytes.len(), size);
+    println!("Field bytes was: {}", field_bytes.len());
+
+    // Pad to 32-bits
+    for _ in 0..padding {
+        bytes.push(0);
+    }
+
+    Ok(bytes)
+}
+
+fn ipv6_record(key: &FlowbeeKey, data: &FlowbeeLocalData, direction: usize) -> anyhow::Result<Vec<u8>> {
+    let field_bytes = field_encoder::encode_fields_from_template(
+        &template_ipv6::FIELDS_IPV6,
+        direction,
+        key,
+        data,
+    )?;
+
+    // Build the actual record
+    let mut bytes = Vec::new();
+    // Add the flowset_id. Template ID is 257
+    bytes.extend_from_slice(&(257u16).to_be_bytes());
+
+    // Add the length. Length includes 2 bytes for flowset and 2 bytes for the length field
+    // itself. That's odd.
+    let padding = (field_bytes.len() + 4) % 4;
+    let size = (bytes.len() + field_bytes.len() + padding + 2) as u16;
+    bytes.extend_from_slice(&size.to_be_bytes());
+
+    // Add the data itself
+    bytes.extend_from_slice(&field_bytes);
+
+    // Pad to 32-bits
+    while bytes.len() % 4 != 0 {
+        bytes.push(0);
+    }
+
+    Ok(bytes)
+}

src/rust/lqosd/src/throughput_tracker/flow_data/netflow9/protocol/template_ipv4.rs

@@ -0,0 +1,42 @@
+use crate::throughput_tracker::flow_data::netflow9::protocol::*;
+
+pub(crate) const FIELDS_IPV4: [(u16, u16); 8] = [
+    IN_BYTES,
+    IN_PKTS,
+    PROTOCOL,
+    L4_SRC_PORT,
+    IPV4_SRC_ADDR,
+    L4_DST_PORT,
+    IPV4_DST_ADDR,
+    DST_TOS,
+];
+
+pub fn template_data_ipv4() -> Vec<u8> {
+    // Build the header
+    let mut bytes = Vec::new();
+
+    // Add the flowset_id, id is zero. (See https://netflow.caligare.com/netflow_v9.htm)
+    // 16
+    bytes.push(0);
+    bytes.push(0);
+
+    // Add the length of the flowset, 4 bytes
+    const LENGTH: u16 = 8 + (FIELDS_IPV4.len() * 4) as u16; // TODO: Fixme
+    bytes.extend_from_slice(LENGTH.to_be_bytes().as_ref());
+
+    // Add the TemplateID. We're going to use 256 for IPv4.
+    const TEMPLATE_ID: u16 = 256;
+    bytes.extend_from_slice(TEMPLATE_ID.to_be_bytes().as_ref());
+
+    // Add the number of fields in the template
+    const FIELD_COUNT: u16 = FIELDS_IPV4.len() as u16;
+    bytes.extend_from_slice(FIELD_COUNT.to_be_bytes().as_ref());
+
+    for (field_type, field_length) in FIELDS_IPV4.iter() {
+        add_field(&mut bytes, *field_type, *field_length);
+    }
+
+    println!("Templatev4 Size {} = {}", bytes.len(), 8 + (FIELDS_IPV4.len() * 2));
+
+    bytes
+}

src/rust/lqosd/src/throughput_tracker/flow_data/netflow9/protocol/template_ipv6.rs

@@ -0,0 +1,40 @@
+use crate::throughput_tracker::flow_data::netflow9::protocol::*;
+
+pub(crate) const FIELDS_IPV6: [(u16, u16); 8] = [
+    IN_BYTES,
+    IN_PKTS,
+    PROTOCOL,
+    L4_SRC_PORT,
+    IPV6_SRC_ADDR,
+    L4_DST_PORT,
+    IPV6_DST_ADDR,
+    DST_TOS,
+];
+
+pub fn template_data_ipv6() -> Vec<u8> {
+    // Build the header
+    let mut bytes = Vec::new();
+
+    // Add the flowset_id, id is zero. (See https://netflow.caligare.com/netflow_v9.htm)
+    // 16
+    bytes.push(0);
+    bytes.push(0);
+
+    // Add the length of the flowset, 4 bytes
+    const LENGTH: u16 = 8 + (FIELDS_IPV6.len() * 4) as u16; // TODO: Fixme
+    bytes.extend_from_slice(LENGTH.to_be_bytes().as_ref());
+
+    // Add the TemplateID. We're going to use 257 for IPv6.
+    const TEMPLATE_ID: u16 = 257;
+    bytes.extend_from_slice(TEMPLATE_ID.to_be_bytes().as_ref());
+
+    // Add the number of fields in the template
+    const FIELD_COUNT: u16 = FIELDS_IPV6.len() as u16;
+    bytes.extend_from_slice(FIELD_COUNT.to_be_bytes().as_ref());
+
+    for (field_type, field_length) in FIELDS_IPV6.iter() {
+        add_field(&mut bytes, *field_type, *field_length);
+    }
+
+    bytes
+}

src/rust/lqosd/src/throughput_tracker/heimdall_data.rs

@@ -1,14 +0,0 @@
-use std::net::IpAddr;
-use lqos_bus::BusResponse;
-use lqos_heimdall::heimdall_watch_ip;
-use lqos_utils::XdpIpAddress;
-
-pub fn get_flow_stats(ip: &str) -> BusResponse {
-  let ip = ip.parse::<IpAddr>();
-  if let Ok(ip) = ip {
-    let ip = XdpIpAddress::from_ip(ip);
-    heimdall_watch_ip(ip);
-    return lqos_heimdall::get_flow_stats(ip);
-  }
-  BusResponse::Fail("No Stats or bad IP".to_string())
-}

src/rust/lqosd/src/throughput_tracker/mod.rs

@@ -1,15 +1,20 @@
-mod heimdall_data;
+pub mod flow_data;
 mod throughput_entry;
 mod tracking_data;
+use std::net::IpAddr;
+
+use self::flow_data::{get_asn_name_and_country, FlowAnalysis, FlowbeeLocalData, ALL_FLOWS};
 use crate::{
-    shaped_devices_tracker::{NETWORK_JSON, STATS_NEEDS_NEW_SHAPED_DEVICES, SHAPED_DEVICES}, stats::TIME_TO_POLL_HOSTS,
-    throughput_tracker::tracking_data::ThroughputTracker, long_term_stats::get_network_tree,
+    long_term_stats::get_network_tree,
+    shaped_devices_tracker::{NETWORK_JSON, SHAPED_DEVICES, STATS_NEEDS_NEW_SHAPED_DEVICES},
+    stats::TIME_TO_POLL_HOSTS,
+    throughput_tracker::tracking_data::ThroughputTracker,
 };
-pub use heimdall_data::get_flow_stats;
 use log::{info, warn};
-use lqos_bus::{BusResponse, IpStats, TcHandle, XdpPpingResult};
+use lqos_bus::{BusResponse, FlowbeeProtocol, IpStats, TcHandle, TopFlowType, XdpPpingResult};
+use lqos_sys::flowbee_data::FlowbeeKey;
 use lqos_utils::{unix_time::time_since_boot, XdpIpAddress};
-use lts_client::collector::{StatsUpdateMessage, ThroughputSummary, HostSummary};
+use lts_client::collector::{HostSummary, StatsUpdateMessage, ThroughputSummary};
 use once_cell::sync::Lazy;
 use tokio::{
     sync::mpsc::Sender,
@@ -27,44 +32,82 @@ pub static THROUGHPUT_TRACKER: Lazy<ThroughputTracker> = Lazy::new(ThroughputTra
 ///
 /// * `long_term_stats_tx` - an optional MPSC sender to notify the
 ///   collection thread that there is fresh data.
-pub async fn spawn_throughput_monitor(long_term_stats_tx: Sender<StatsUpdateMessage>) {
+pub async fn spawn_throughput_monitor(
+    long_term_stats_tx: Sender<StatsUpdateMessage>,
+    netflow_sender: std::sync::mpsc::Sender<(FlowbeeKey, (FlowbeeLocalData, FlowAnalysis))>,
+) {
     info!("Starting the bandwidth monitor thread.");
     let interval_ms = 1000; // 1 second
     info!("Bandwidth check period set to {interval_ms} ms.");
-    tokio::spawn(throughput_task(interval_ms, long_term_stats_tx));
+    tokio::spawn(throughput_task(
+        interval_ms,
+        long_term_stats_tx,
+        netflow_sender,
+    ));
 }
 
-async fn throughput_task(interval_ms: u64, long_term_stats_tx: Sender<StatsUpdateMessage>) {
+async fn throughput_task(
+    interval_ms: u64,
+    long_term_stats_tx: Sender<StatsUpdateMessage>,
+    netflow_sender: std::sync::mpsc::Sender<(FlowbeeKey, (FlowbeeLocalData, FlowAnalysis))>,
+) {
+    // Obtain the flow timeout from the config, default to 30 seconds
+    let timeout_seconds = if let Ok(config) = lqos_config::load_config() {
+        if let Some(flow_config) = config.flows {
+            flow_config.flow_timeout_seconds
+        } else {
+            30
+        }
+    } else {
+        30
+    };
+
+    // Obtain the netflow_enabled from the config, default to false
+    let netflow_enabled = if let Ok(config) = lqos_config::load_config() {
+        if let Some(flow_config) = config.flows {
+            flow_config.netflow_enabled
+        } else {
+            false
+        }
+    } else {
+        false
+    };
+
     loop {
         let start = Instant::now();
 
         // Perform the stats collection in a blocking thread, ensuring that
         // the tokio runtime is not blocked.
+        let my_netflow_sender = netflow_sender.clone();
         if let Err(e) = tokio::task::spawn_blocking(move || {
-
-          {
-              let net_json = NETWORK_JSON.read().unwrap();
-              net_json.zero_throughput_and_rtt();
-          } // Scope to end the lock
-          THROUGHPUT_TRACKER.copy_previous_and_reset_rtt();
-          THROUGHPUT_TRACKER.apply_new_throughput_counters();
-          THROUGHPUT_TRACKER.apply_rtt_data();
-          THROUGHPUT_TRACKER.update_totals();
-          THROUGHPUT_TRACKER.next_cycle();
-          let duration_ms = start.elapsed().as_micros();
-          TIME_TO_POLL_HOSTS.store(duration_ms as u64, std::sync::atomic::Ordering::Relaxed);
-
-        }).await {
+            {
+                let net_json = NETWORK_JSON.read().unwrap();
+                net_json.zero_throughput_and_rtt();
+            } // Scope to end the lock
+            THROUGHPUT_TRACKER.copy_previous_and_reset_rtt();
+            THROUGHPUT_TRACKER.apply_new_throughput_counters();
+            THROUGHPUT_TRACKER.apply_flow_data(
+                timeout_seconds,
+                netflow_enabled,
+                my_netflow_sender.clone(),
+            );
+            THROUGHPUT_TRACKER.update_totals();
+            THROUGHPUT_TRACKER.next_cycle();
+            let duration_ms = start.elapsed().as_micros();
+            TIME_TO_POLL_HOSTS.store(duration_ms as u64, std::sync::atomic::Ordering::Relaxed);
+        })
+        .await
+        {
             log::error!("Error polling network. {e:?}");
         }
         tokio::spawn(submit_throughput_stats(long_term_stats_tx.clone()));
 
         let elapsed = start.elapsed();
         if elapsed.as_secs_f32() < 1.0 {
-          let sleep_duration = Duration::from_millis(interval_ms) - start.elapsed();
-          tokio::time::sleep(sleep_duration).await;
+            let sleep_duration = Duration::from_millis(interval_ms) - start.elapsed();
+            tokio::time::sleep(sleep_duration).await;
         } else {
-          log::error!("Throughput monitor thread is running behind. It took {elapsed} to poll the network.", elapsed=elapsed.as_secs_f32());
+            log::error!("Throughput monitor thread is running behind. It took {elapsed} to poll the network.", elapsed=elapsed.as_secs_f32());
         }
     }
 }
@@ -110,12 +153,15 @@ async fn submit_throughput_stats(long_term_stats_tx: Sender<StatsUpdateMessage>)
         })
         .collect();
 
-    let summary = Box::new((ThroughputSummary{
-        bits_per_second,
-        shaped_bits_per_second,
-        packets_per_second,
-        hosts,
-    }, get_network_tree()));
+    let summary = Box::new((
+        ThroughputSummary {
+            bits_per_second,
+            shaped_bits_per_second,
+            packets_per_second,
+            hosts,
+        },
+        get_network_tree(),
+    ));
 
     // Send the stats
     let result = long_term_stats_tx
@@ -156,145 +202,219 @@ fn retire_check(cycle: u64, recent_cycle: u64) -> bool {
     cycle < recent_cycle + RETIRE_AFTER_SECONDS
 }
 
-type TopList = (XdpIpAddress, (u64, u64), (u64, u64), f32, TcHandle, String);
+type TopList = (XdpIpAddress, (u64, u64), (u64, u64), f32, TcHandle, String, (u64, u64));
 
 pub fn top_n(start: u32, end: u32) -> BusResponse {
     let mut full_list: Vec<TopList> = {
-      let tp_cycle = THROUGHPUT_TRACKER.cycle.load(std::sync::atomic::Ordering::Relaxed);
-      THROUGHPUT_TRACKER.raw_data
-        .iter()
-        .filter(|v| !v.key().as_ip().is_loopback())
-        .filter(|d| retire_check(tp_cycle, d.most_recent_cycle))
-        .map(|te| {
-          (
-            *te.key(),
-            te.bytes_per_second,
-            te.packets_per_second,
-            te.median_latency().unwrap_or(0.0),
-            te.tc_handle,
-            te.circuit_id.as_ref().unwrap_or(&String::new()).clone(),
-          )
-        })
-        .collect()
+        let tp_cycle = THROUGHPUT_TRACKER
+            .cycle
+            .load(std::sync::atomic::Ordering::Relaxed);
+        THROUGHPUT_TRACKER
+            .raw_data
+            .iter()
+            .filter(|v| !v.key().as_ip().is_loopback())
+            .filter(|d| retire_check(tp_cycle, d.most_recent_cycle))
+            .map(|te| {
+                (
+                    *te.key(),
+                    te.bytes_per_second,
+                    te.packets_per_second,
+                    te.median_latency().unwrap_or(0.0),
+                    te.tc_handle,
+                    te.circuit_id.as_ref().unwrap_or(&String::new()).clone(),
+                    te.tcp_retransmits,
+                )
+            })
+            .collect()
     };
     full_list.sort_by(|a, b| b.1 .0.cmp(&a.1 .0));
     let result = full_list
-      .iter()
-      .skip(start as usize)
-      .take((end as usize) - (start as usize))
-      .map(
-        |(
-          ip,
-          (bytes_dn, bytes_up),
-          (packets_dn, packets_up),
-          median_rtt,
-          tc_handle,
-          circuit_id,
-        )| IpStats {
-          ip_address: ip.as_ip().to_string(),
-          circuit_id: circuit_id.clone(),
-          bits_per_second: (bytes_dn * 8, bytes_up * 8),
-          packets_per_second: (*packets_dn, *packets_up),
-          median_tcp_rtt: *median_rtt,
-          tc_handle: *tc_handle,
-        },
-      )
-      .collect();
-    BusResponse::TopDownloaders(result)
-  }
-
-  pub fn worst_n(start: u32, end: u32) -> BusResponse {
-    let mut full_list: Vec<TopList> = {
-      let tp_cycle = THROUGHPUT_TRACKER.cycle.load(std::sync::atomic::Ordering::Relaxed);
-      THROUGHPUT_TRACKER.raw_data
         .iter()
-        .filter(|v| !v.key().as_ip().is_loopback())
-        .filter(|d| retire_check(tp_cycle, d.most_recent_cycle))
-        .filter(|te| te.median_latency().is_some())
-        .map(|te| {
-          (
-            *te.key(),
-            te.bytes_per_second,
-            te.packets_per_second,
-            te.median_latency().unwrap_or(0.0),
-            te.tc_handle,
-            te.circuit_id.as_ref().unwrap_or(&String::new()).clone(),
-          )
-        })
-        .collect()
+        .skip(start as usize)
+        .take((end as usize) - (start as usize))
+        .map(
+            |(
+                ip,
+                (bytes_dn, bytes_up),
+                (packets_dn, packets_up),
+                median_rtt,
+                tc_handle,
+                circuit_id,
+                tcp_retransmits,      
+            )| IpStats {
+                ip_address: ip.as_ip().to_string(),
+                circuit_id: circuit_id.clone(),
+                bits_per_second: (bytes_dn * 8, bytes_up * 8),
+                packets_per_second: (*packets_dn, *packets_up),
+                median_tcp_rtt: *median_rtt,
+                tc_handle: *tc_handle,
+                tcp_retransmits: *tcp_retransmits,
+            },
+        )
+        .collect();
+    BusResponse::TopDownloaders(result)
+}
+
+pub fn worst_n(start: u32, end: u32) -> BusResponse {
+    let mut full_list: Vec<TopList> = {
+        let tp_cycle = THROUGHPUT_TRACKER
+            .cycle
+            .load(std::sync::atomic::Ordering::Relaxed);
+        THROUGHPUT_TRACKER
+            .raw_data
+            .iter()
+            .filter(|v| !v.key().as_ip().is_loopback())
+            .filter(|d| retire_check(tp_cycle, d.most_recent_cycle))
+            .filter(|te| te.median_latency().is_some())
+            .map(|te| {
+                (
+                    *te.key(),
+                    te.bytes_per_second,
+                    te.packets_per_second,
+                    te.median_latency().unwrap_or(0.0),
+                    te.tc_handle,
+                    te.circuit_id.as_ref().unwrap_or(&String::new()).clone(),
+                    te.tcp_retransmits,
+                )
+            })
+            .collect()
     };
     full_list.sort_by(|a, b| b.3.partial_cmp(&a.3).unwrap());
     let result = full_list
-      .iter()
-      .skip(start as usize)
-      .take((end as usize) - (start as usize))
-      .map(
-        |(
-          ip,
-          (bytes_dn, bytes_up),
-          (packets_dn, packets_up),
-          median_rtt,
-          tc_handle,
-          circuit_id,
-        )| IpStats {
-          ip_address: ip.as_ip().to_string(),
-          circuit_id: circuit_id.clone(),
-          bits_per_second: (bytes_dn * 8, bytes_up * 8),
-          packets_per_second: (*packets_dn, *packets_up),
-          median_tcp_rtt: *median_rtt,
-          tc_handle: *tc_handle,
-        },
-      )
-      .collect();
-    BusResponse::WorstRtt(result)
-  }
-
-  pub fn best_n(start: u32, end: u32) -> BusResponse {
-    let mut full_list: Vec<TopList> = {
-      let tp_cycle = THROUGHPUT_TRACKER.cycle.load(std::sync::atomic::Ordering::Relaxed);
-      THROUGHPUT_TRACKER.raw_data
         .iter()
-        .filter(|v| !v.key().as_ip().is_loopback())
-        .filter(|d| retire_check(tp_cycle, d.most_recent_cycle))
-        .filter(|te| te.median_latency().is_some())
-        .map(|te| {
-          (
-            *te.key(),
-            te.bytes_per_second,
-            te.packets_per_second,
-            te.median_latency().unwrap_or(0.0),
-            te.tc_handle,
-            te.circuit_id.as_ref().unwrap_or(&String::new()).clone(),
-          )
-        })
-        .collect()
+        .skip(start as usize)
+        .take((end as usize) - (start as usize))
+        .map(
+            |(
+                ip,
+                (bytes_dn, bytes_up),
+                (packets_dn, packets_up),
+                median_rtt,
+                tc_handle,
+                circuit_id,
+                tcp_retransmits,
+            )| IpStats {
+                ip_address: ip.as_ip().to_string(),
+                circuit_id: circuit_id.clone(),
+                bits_per_second: (bytes_dn * 8, bytes_up * 8),
+                packets_per_second: (*packets_dn, *packets_up),
+                median_tcp_rtt: *median_rtt,
+                tc_handle: *tc_handle,
+                tcp_retransmits: *tcp_retransmits,
+            },
+        )
+        .collect();
+    BusResponse::WorstRtt(result)
+}
+
+pub fn worst_n_retransmits(start: u32, end: u32) -> BusResponse {
+    let mut full_list: Vec<TopList> = {
+        let tp_cycle = THROUGHPUT_TRACKER
+            .cycle
+            .load(std::sync::atomic::Ordering::Relaxed);
+        THROUGHPUT_TRACKER
+            .raw_data
+            .iter()
+            .filter(|v| !v.key().as_ip().is_loopback())
+            .filter(|d| retire_check(tp_cycle, d.most_recent_cycle))
+            .filter(|te| te.median_latency().is_some())
+            .map(|te| {
+                (
+                    *te.key(),
+                    te.bytes_per_second,
+                    te.packets_per_second,
+                    te.median_latency().unwrap_or(0.0),
+                    te.tc_handle,
+                    te.circuit_id.as_ref().unwrap_or(&String::new()).clone(),
+                    te.tcp_retransmits,
+                )
+            })
+            .collect()
+    };
+    full_list.sort_by(|a, b| {
+        let total_a = a.6 .0 + a.6 .1;
+        let total_b = b.6 .0 + b.6 .1;
+        total_b.cmp(&total_a)
+    });
+    let result = full_list
+        .iter()
+        .skip(start as usize)
+        .take((end as usize) - (start as usize))
+        .map(
+            |(
+                ip,
+                (bytes_dn, bytes_up),
+                (packets_dn, packets_up),
+                median_rtt,
+                tc_handle,
+                circuit_id,
+                tcp_retransmits,
+            )| IpStats {
+                ip_address: ip.as_ip().to_string(),
+                circuit_id: circuit_id.clone(),
+                bits_per_second: (bytes_dn * 8, bytes_up * 8),
+                packets_per_second: (*packets_dn, *packets_up),
+                median_tcp_rtt: *median_rtt,
+                tc_handle: *tc_handle,
+                tcp_retransmits: *tcp_retransmits,
+            },
+        )
+        .collect();
+    BusResponse::WorstRetransmits(result)
+}
+
+pub fn best_n(start: u32, end: u32) -> BusResponse {
+    let mut full_list: Vec<TopList> = {
+        let tp_cycle = THROUGHPUT_TRACKER
+            .cycle
+            .load(std::sync::atomic::Ordering::Relaxed);
+        THROUGHPUT_TRACKER
+            .raw_data
+            .iter()
+            .filter(|v| !v.key().as_ip().is_loopback())
+            .filter(|d| retire_check(tp_cycle, d.most_recent_cycle))
+            .filter(|te| te.median_latency().is_some())
+            .map(|te| {
+                (
+                    *te.key(),
+                    te.bytes_per_second,
+                    te.packets_per_second,
+                    te.median_latency().unwrap_or(0.0),
+                    te.tc_handle,
+                    te.circuit_id.as_ref().unwrap_or(&String::new()).clone(),
+                    te.tcp_retransmits,
+                )
+            })
+            .collect()
     };
     full_list.sort_by(|a, b| b.3.partial_cmp(&a.3).unwrap());
     full_list.reverse();
     let result = full_list
-      .iter()
-      .skip(start as usize)
-      .take((end as usize) - (start as usize))
-      .map(
-        |(
-          ip,
-          (bytes_dn, bytes_up),
-          (packets_dn, packets_up),
-          median_rtt,
-          tc_handle,
-          circuit_id,
-        )| IpStats {
-          ip_address: ip.as_ip().to_string(),
-          circuit_id: circuit_id.clone(),
-          bits_per_second: (bytes_dn * 8, bytes_up * 8),
-          packets_per_second: (*packets_dn, *packets_up),
-          median_tcp_rtt: *median_rtt,
-          tc_handle: *tc_handle,
-        },
-      )
-      .collect();
+        .iter()
+        .skip(start as usize)
+        .take((end as usize) - (start as usize))
+        .map(
+            |(
+                ip,
+                (bytes_dn, bytes_up),
+                (packets_dn, packets_up),
+                median_rtt,
+                tc_handle,
+                circuit_id,
+                tcp_retransmits,
+            )| IpStats {
+                ip_address: ip.as_ip().to_string(),
+                circuit_id: circuit_id.clone(),
+                bits_per_second: (bytes_dn * 8, bytes_up * 8),
+                packets_per_second: (*packets_dn, *packets_up),
+                median_tcp_rtt: *median_rtt,
+                tc_handle: *tc_handle,
+                tcp_retransmits: *tcp_retransmits,
+            },
+        )
+        .collect();
     BusResponse::BestRtt(result)
-  }  
+}
 
 pub fn xdp_pping_compat() -> BusResponse {
     let raw_cycle = THROUGHPUT_TRACKER
@@ -309,8 +429,8 @@ pub fn xdp_pping_compat() -> BusResponse {
                 let mut valid_samples: Vec<u32> = data
                     .recent_rtt_data
                     .iter()
-                    .filter(|d| **d > 0)
-                    .copied()
+                    .filter(|d| d.as_millis_times_100() > 0.0)
+                    .map(|d| d.as_millis_times_100() as u32)
                     .collect();
                 let samples = valid_samples.len() as u32;
                 if samples > 0 {
@@ -350,17 +470,17 @@ pub fn rtt_histogram() -> BusResponse {
         .iter()
         .filter(|d| retire_check(reader_cycle, d.most_recent_cycle))
     {
-        let valid_samples: Vec<u32> = data
+        let valid_samples: Vec<f64> = data
             .recent_rtt_data
             .iter()
-            .filter(|d| **d > 0)
-            .copied()
+            .filter(|d| d.as_millis() > 0.0)
+            .map(|d| d.as_millis())
             .collect();
         let samples = valid_samples.len() as u32;
         if samples > 0 {
-            let median = valid_samples[valid_samples.len() / 2] as f32 / 100.0;
+            let median = valid_samples[valid_samples.len() / 2] as f32 / 10.0;
             let median = f32::min(200.0, median);
-            let column = (median / 10.0) as usize;
+            let column = median as usize;
             result[usize::min(column, 19)] += 1;
         }
     }
@@ -392,54 +512,242 @@ type FullList = (XdpIpAddress, (u64, u64), (u64, u64), f32, TcHandle, u64);
 pub fn all_unknown_ips() -> BusResponse {
     let boot_time = time_since_boot();
     if boot_time.is_err() {
-      warn!("The Linux system clock isn't available to provide time since boot, yet.");
-      warn!("This only happens immediately after a reboot.");
-      return BusResponse::NotReadyYet;
+        warn!("The Linux system clock isn't available to provide time since boot, yet.");
+        warn!("This only happens immediately after a reboot.");
+        return BusResponse::NotReadyYet;
     }
     let boot_time = boot_time.unwrap();
     let time_since_boot = Duration::from(boot_time);
-    let five_minutes_ago =
-      time_since_boot.saturating_sub(Duration::from_secs(300));
+    let five_minutes_ago = time_since_boot.saturating_sub(Duration::from_secs(300));
     let five_minutes_ago_nanoseconds = five_minutes_ago.as_nanos();
-  
+
     let mut full_list: Vec<FullList> = {
-      THROUGHPUT_TRACKER.raw_data
-        .iter()
-        .filter(|v| !v.key().as_ip().is_loopback())
-        .filter(|d| d.tc_handle.as_u32() == 0)
-        .filter(|d| d.last_seen as u128 > five_minutes_ago_nanoseconds)
-        .map(|te| {
-          (
-            *te.key(),
-            te.bytes,
-            te.packets,
-            te.median_latency().unwrap_or(0.0),
-            te.tc_handle,
-            te.most_recent_cycle,
-          )
-        })
-        .collect()
+        THROUGHPUT_TRACKER
+            .raw_data
+            .iter()
+            .filter(|v| !v.key().as_ip().is_loopback())
+            .filter(|d| d.tc_handle.as_u32() == 0)
+            .filter(|d| d.last_seen as u128 > five_minutes_ago_nanoseconds)
+            .map(|te| {
+                (
+                    *te.key(),
+                    te.bytes,
+                    te.packets,
+                    te.median_latency().unwrap_or(0.0),
+                    te.tc_handle,
+                    te.most_recent_cycle,
+                )
+            })
+            .collect()
     };
     full_list.sort_by(|a, b| b.5.partial_cmp(&a.5).unwrap());
     let result = full_list
-      .iter()
-      .map(
-        |(
-          ip,
-          (bytes_dn, bytes_up),
-          (packets_dn, packets_up),
-          median_rtt,
-          tc_handle,
-          _last_seen,
-        )| IpStats {
-          ip_address: ip.as_ip().to_string(),
-          circuit_id: String::new(),
-          bits_per_second: (bytes_dn * 8, bytes_up * 8),
-          packets_per_second: (*packets_dn, *packets_up),
-          median_tcp_rtt: *median_rtt,
-          tc_handle: *tc_handle,
-        },
-      )
-      .collect();
+        .iter()
+        .map(
+            |(
+                ip,
+                (bytes_dn, bytes_up),
+                (packets_dn, packets_up),
+                median_rtt,
+                tc_handle,
+                _last_seen,
+            )| IpStats {
+                ip_address: ip.as_ip().to_string(),
+                circuit_id: String::new(),
+                bits_per_second: (bytes_dn * 8, bytes_up * 8),
+                packets_per_second: (*packets_dn, *packets_up),
+                median_tcp_rtt: *median_rtt,
+                tc_handle: *tc_handle,
+                tcp_retransmits: (0, 0),
+            },
+        )
+        .collect();
     BusResponse::AllUnknownIps(result)
-  }
+}
+
+/// For debugging: dump all active flows!
+pub fn dump_active_flows() -> BusResponse {
+    let lock = ALL_FLOWS.lock().unwrap();
+    let result: Vec<lqos_bus::FlowbeeSummaryData> = lock
+        .iter()
+        .map(|(key, row)| {
+            let (remote_asn_name, remote_asn_country) =
+                get_asn_name_and_country(key.remote_ip.as_ip());
+
+            lqos_bus::FlowbeeSummaryData {
+                remote_ip: key.remote_ip.as_ip().to_string(),
+                local_ip: key.local_ip.as_ip().to_string(),
+                src_port: key.src_port,
+                dst_port: key.dst_port,
+                ip_protocol: FlowbeeProtocol::from(key.ip_protocol),
+                bytes_sent: row.0.bytes_sent,
+                packets_sent: row.0.packets_sent,
+                rate_estimate_bps: row.0.rate_estimate_bps,
+                tcp_retransmits: row.0.tcp_retransmits,
+                end_status: row.0.end_status,
+                tos: row.0.tos,
+                flags: row.0.flags,
+                remote_asn: row.1.asn_id.0,
+                remote_asn_name,
+                remote_asn_country,
+                analysis: row.1.protocol_analysis.to_string(),
+                last_seen: row.0.last_seen,
+                start_time: row.0.start_time,
+                rtt_nanos: [row.0.rtt[0].as_nanos(), row.0.rtt[1].as_nanos()],
+            }
+        })
+        .collect();
+
+    BusResponse::AllActiveFlows(result)
+}
+
+/// Count active flows
+pub fn count_active_flows() -> BusResponse {
+    let lock = ALL_FLOWS.lock().unwrap();
+    BusResponse::CountActiveFlows(lock.len() as u64)
+}
+
+/// Top Flows Report
+pub fn top_flows(n: u32, flow_type: TopFlowType) -> BusResponse {
+    let lock = ALL_FLOWS.lock().unwrap();
+    let mut table: Vec<(FlowbeeKey, (FlowbeeLocalData, FlowAnalysis))> = lock
+        .iter()
+        .map(|(key, value)| (key.clone(), value.clone()))
+        .collect();
+    std::mem::drop(lock); // Early lock release
+
+    match flow_type {
+        TopFlowType::RateEstimate => {
+            table.sort_by(|a, b| {
+                let a_total = a.1 .0.rate_estimate_bps[0] + a.1 .0.rate_estimate_bps[1];
+                let b_total = b.1 .0.rate_estimate_bps[0] + b.1 .0.rate_estimate_bps[1];
+                b_total.cmp(&a_total)
+            });
+        }
+        TopFlowType::Bytes => {
+            table.sort_by(|a, b| {
+                let a_total = a.1 .0.bytes_sent[0] + a.1 .0.bytes_sent[1];
+                let b_total = b.1 .0.bytes_sent[0] + b.1 .0.bytes_sent[1];
+                b_total.cmp(&a_total)
+            });
+        }
+        TopFlowType::Packets => {
+            table.sort_by(|a, b| {
+                let a_total = a.1 .0.packets_sent[0] + a.1 .0.packets_sent[1];
+                let b_total = b.1 .0.packets_sent[0] + b.1 .0.packets_sent[1];
+                b_total.cmp(&a_total)
+            });
+        }
+        TopFlowType::Drops => {
+            table.sort_by(|a, b| {
+                let a_total = a.1 .0.tcp_retransmits[0] + a.1 .0.tcp_retransmits[1];
+                let b_total = b.1 .0.tcp_retransmits[0] + b.1 .0.tcp_retransmits[1];
+                b_total.cmp(&a_total)
+            });
+        }
+        TopFlowType::RoundTripTime => {
+            table.sort_by(|a, b| {
+                let a_total = a.1 .0.rtt;
+                let b_total = b.1 .0.rtt;
+                a_total.cmp(&b_total)
+            });
+        }
+    }
+
+    let result = table
+        .iter()
+        .take(n as usize)
+        .map(|(ip, flow)| {
+            let (remote_asn_name, remote_asn_country) =
+                get_asn_name_and_country(ip.remote_ip.as_ip());
+            lqos_bus::FlowbeeSummaryData {
+                remote_ip: ip.remote_ip.as_ip().to_string(),
+                local_ip: ip.local_ip.as_ip().to_string(),
+                src_port: ip.src_port,
+                dst_port: ip.dst_port,
+                ip_protocol: FlowbeeProtocol::from(ip.ip_protocol),
+                bytes_sent: flow.0.bytes_sent,
+                packets_sent: flow.0.packets_sent,
+                rate_estimate_bps: flow.0.rate_estimate_bps,
+                tcp_retransmits: flow.0.tcp_retransmits,
+                end_status: flow.0.end_status,
+                tos: flow.0.tos,
+                flags: flow.0.flags,
+                remote_asn: flow.1.asn_id.0,
+                remote_asn_name,
+                remote_asn_country,
+                analysis: flow.1.protocol_analysis.to_string(),
+                last_seen: flow.0.last_seen,
+                start_time: flow.0.start_time,
+                rtt_nanos: [flow.0.rtt[0].as_nanos(), flow.0.rtt[1].as_nanos()],
+            }
+        })
+        .collect();
+
+    BusResponse::TopFlows(result)
+}
+
+/// Flows by IP
+pub fn flows_by_ip(ip: &str) -> BusResponse {
+    if let Ok(ip) = ip.parse::<IpAddr>() {
+        let ip = XdpIpAddress::from_ip(ip);
+        let lock = ALL_FLOWS.lock().unwrap();
+        let matching_flows: Vec<_> = lock
+            .iter()
+            .filter(|(key, _)| key.local_ip == ip)
+            .map(|(key, row)| {
+                let (remote_asn_name, remote_asn_country) =
+                    get_asn_name_and_country(key.remote_ip.as_ip());
+
+                lqos_bus::FlowbeeSummaryData {
+                    remote_ip: key.remote_ip.as_ip().to_string(),
+                    local_ip: key.local_ip.as_ip().to_string(),
+                    src_port: key.src_port,
+                    dst_port: key.dst_port,
+                    ip_protocol: FlowbeeProtocol::from(key.ip_protocol),
+                    bytes_sent: row.0.bytes_sent,
+                    packets_sent: row.0.packets_sent,
+                    rate_estimate_bps: row.0.rate_estimate_bps,
+                    tcp_retransmits: row.0.tcp_retransmits,
+                    end_status: row.0.end_status,
+                    tos: row.0.tos,
+                    flags: row.0.flags,
+                    remote_asn: row.1.asn_id.0,
+                    remote_asn_name,
+                    remote_asn_country,
+                    analysis: row.1.protocol_analysis.to_string(),
+                    last_seen: row.0.last_seen,
+                    start_time: row.0.start_time,
+                    rtt_nanos: [row.0.rtt[0].as_nanos(), row.0.rtt[1].as_nanos()],
+                }
+            })
+            .collect();
+
+        return BusResponse::FlowsByIp(matching_flows);
+    }
+    BusResponse::Ack
+}
+
+/// Current endpoints by country
+pub fn current_endpoints_by_country() -> BusResponse {
+    let summary = flow_data::RECENT_FLOWS.country_summary();
+    BusResponse::CurrentEndpointsByCountry(summary)
+}
+
+/// Current endpoint lat/lon
+pub fn current_lat_lon() -> BusResponse {
+    let summary = flow_data::RECENT_FLOWS.lat_lon_endpoints();
+    BusResponse::CurrentLatLon(summary)
+}
+
+/// Ether Protocol Summary
+pub fn ether_protocol_summary() -> BusResponse {
+    flow_data::RECENT_FLOWS.ether_protocol_summary()
+}
+
+/// IP Protocol Summary
+pub fn ip_protocol_summary() -> BusResponse {
+    BusResponse::IpProtocols(
+        flow_data::RECENT_FLOWS.ip_protocol_summary()
+    )
+}

src/rust/lqosd/src/throughput_tracker/throughput_entry.rs

@@ -1,4 +1,5 @@
 use lqos_bus::TcHandle;
+use super::flow_data::RttData;
 
 #[derive(Debug)]
 pub(crate) struct ThroughputEntry {
@@ -13,9 +14,11 @@ pub(crate) struct ThroughputEntry {
   pub(crate) bytes_per_second: (u64, u64),
   pub(crate) packets_per_second: (u64, u64),
   pub(crate) tc_handle: TcHandle,
-  pub(crate) recent_rtt_data: [u32; 60],
+  pub(crate) recent_rtt_data: [RttData; 60],
   pub(crate) last_fresh_rtt_data_cycle: u64,
   pub(crate) last_seen: u64, // Last seen in kernel time since boot
+  pub(crate) tcp_retransmits: (u64, u64),
+  pub(crate) last_tcp_retransmits: (u64, u64),
 }
 
 impl ThroughputEntry {
@@ -33,8 +36,8 @@ impl ThroughputEntry {
     let mut shifted: Vec<f32> = self
       .recent_rtt_data
       .iter()
-      .filter(|n| **n != 0)
-      .map(|n| *n as f32 / 100.0)
+      .filter(|n| n.as_nanos() != 0)
+      .map(|n| n.as_millis() as f32)
       .collect();
     if shifted.len() < 5 {
       return None;

src/rust/lqosd/src/throughput_tracker/tracking_data.rs

@@ -1,10 +1,11 @@
-use std::sync::atomic::AtomicU64;
-use crate::{shaped_devices_tracker::{SHAPED_DEVICES, NETWORK_JSON}, stats::{HIGH_WATERMARK_DOWN, HIGH_WATERMARK_UP}};
-use super::{throughput_entry::ThroughputEntry, RETIRE_AFTER_SECONDS};
+use std::{sync::atomic::AtomicU64, time::Duration};
+use crate::{shaped_devices_tracker::{NETWORK_JSON, SHAPED_DEVICES}, stats::{HIGH_WATERMARK_DOWN, HIGH_WATERMARK_UP}, throughput_tracker::flow_data::{expire_rtt_flows, flowbee_rtt_map}};
+use super::{flow_data::{get_flowbee_event_count_and_reset, FlowAnalysis, FlowbeeLocalData, RttData, ALL_FLOWS}, throughput_entry::ThroughputEntry, RETIRE_AFTER_SECONDS};
 use dashmap::DashMap;
+use fxhash::FxHashMap;
 use lqos_bus::TcHandle;
-use lqos_sys::{rtt_for_each, throughput_for_each};
-use lqos_utils::XdpIpAddress;
+use lqos_sys::{flowbee_data::FlowbeeKey, iterate_flows, throughput_for_each};
+use lqos_utils::{unix_time::time_since_boot, XdpIpAddress};
 
 pub struct ThroughputTracker {
   pub(crate) cycle: AtomicU64,
@@ -49,7 +50,7 @@ impl ThroughputTracker {
       if self_cycle > RETIRE_AFTER_SECONDS
         && v.last_fresh_rtt_data_cycle < self_cycle - RETIRE_AFTER_SECONDS
       {
-        v.recent_rtt_data = [0; 60];
+        v.recent_rtt_data = [RttData::from_nanos(0); 60];
       }
     });
   }
@@ -150,9 +151,11 @@ impl ThroughputTracker {
           bytes_per_second: (0, 0),
           packets_per_second: (0, 0),
           tc_handle: TcHandle::zero(),
-          recent_rtt_data: [0; 60],
+          recent_rtt_data: [RttData::from_nanos(0); 60],
           last_fresh_rtt_data_cycle: 0,
           last_seen: 0,
+          tcp_retransmits: (0, 0),
+          last_tcp_retransmits: (0, 0),
         };
         for c in counts {
           entry.bytes.0 += c.download_bytes;
@@ -168,22 +171,163 @@ impl ThroughputTracker {
     });
   }
 
-  pub(crate) fn apply_rtt_data(&self) {
+  pub(crate) fn apply_flow_data(
+    &self, 
+    timeout_seconds: u64,
+    _netflow_enabled: bool,
+    sender: std::sync::mpsc::Sender<(FlowbeeKey, (FlowbeeLocalData, FlowAnalysis))>,
+  ) {
+    //log::debug!("Flowbee events this second: {}", get_flowbee_event_count_and_reset());
     let self_cycle = self.cycle.load(std::sync::atomic::Ordering::Relaxed);
-    rtt_for_each(&mut |ip, rtt| {
-      if rtt.has_fresh_data != 0 {
-        if let Some(mut tracker) = self.raw_data.get_mut(ip) {
-          tracker.recent_rtt_data = rtt.rtt;
-          tracker.last_fresh_rtt_data_cycle = self_cycle;
-          if let Some(parents) = &tracker.network_json_parents {
-            let net_json = NETWORK_JSON.write().unwrap();
-            if let Some(rtt) = tracker.median_latency() {
-              net_json.add_rtt_cycle(parents, rtt);
+
+    if let Ok(now) = time_since_boot() {
+      let rtt_samples = flowbee_rtt_map();
+      get_flowbee_event_count_and_reset();
+      let since_boot = Duration::from(now);
+      let expire = (since_boot - Duration::from_secs(timeout_seconds)).as_nanos() as u64;
+
+      // Tracker for per-circuit RTT data. We're losing some of the smoothness by sampling
+      // every flow; the idea is to combine them into a single entry for the circuit. This
+      // should limit outliers.
+      let mut rtt_circuit_tracker: FxHashMap<XdpIpAddress, [Vec<RttData>; 2]> = FxHashMap::default();
+
+      // Tracker for TCP retries. We're storing these per second.
+      let mut tcp_retries: FxHashMap<XdpIpAddress, [u64; 2]> = FxHashMap::default();
+
+      // Track the expired keys
+      let mut expired_keys = Vec::new();
+
+      let mut all_flows_lock = ALL_FLOWS.lock().unwrap();
+        
+      // Track through all the flows
+      iterate_flows(&mut |key, data| {
+
+        if data.end_status == 3 {
+          // The flow has been handled already and should be ignored.
+          // DO NOT process it again.          
+        } else if data.last_seen < expire {
+          // This flow has expired but not been handled yet. Add it to the list to be cleaned.
+          expired_keys.push(key.clone());
+        } else {
+          // We have a valid flow, so it needs to be tracked
+          if let Some(this_flow) = all_flows_lock.get_mut(&key) {
+            this_flow.0.last_seen = data.last_seen;
+            this_flow.0.bytes_sent = data.bytes_sent;
+            this_flow.0.packets_sent = data.packets_sent;
+            this_flow.0.rate_estimate_bps = data.rate_estimate_bps;
+            this_flow.0.tcp_retransmits = data.tcp_retransmits;
+            this_flow.0.end_status = data.end_status;
+            this_flow.0.tos = data.tos;
+            this_flow.0.flags = data.flags;
+            if let Some([up, down]) = rtt_samples.get(&key) {
+              if up.as_nanos() != 0 {
+                this_flow.0.rtt[0] = *up;              
+              }
+              if down.as_nanos() != 0 {
+                this_flow.0.rtt[1] = *down;
+              }
+            }
+          } else {
+            // Insert it into the map
+            let flow_analysis = FlowAnalysis::new(&key);
+            all_flows_lock.insert(key.clone(), (data.into(), flow_analysis));
+          }
+
+          // TCP - we have RTT data? 6 is TCP
+          if key.ip_protocol == 6 && data.end_status == 0 && self.raw_data.contains_key(&key.local_ip) {
+              if let Some(rtt) = rtt_samples.get(&key) {
+                // Add the RTT data to the per-circuit tracker
+                if let Some(tracker) = rtt_circuit_tracker.get_mut(&key.local_ip) {
+                  if rtt[0].as_nanos() > 0 {
+                    tracker[0].push(rtt[0]);
+                  }
+                  if rtt[1].as_nanos() > 0 {
+                    tracker[1].push(rtt[1]);
+                  }
+                } else if rtt[0].as_nanos() > 0 || rtt[1].as_nanos() > 0 {
+                  rtt_circuit_tracker.insert(key.local_ip, [vec![rtt[0]], vec![rtt[1]]]);
+                }
+              }
+
+              // TCP Retries
+              if let Some(retries) = tcp_retries.get_mut(&key.local_ip) {
+                retries[0] += data.tcp_retransmits[0] as u64;
+                retries[1] += data.tcp_retransmits[1] as u64;
+              } else {
+                tcp_retries.insert(key.local_ip, [data.tcp_retransmits[0] as u64, data.tcp_retransmits[1] as u64]);
+              }
+
+              if data.end_status != 0 {
+                // The flow has ended. We need to remove it from the map.
+                expired_keys.push(key.clone());
+              }
+          }
+        }
+      }); // End flow iterator
+
+      // Merge in the per-flow RTT data into the per-circuit tracker
+      for (local_ip, rtt_data) in rtt_circuit_tracker {
+        let mut rtts = rtt_data[0].iter().filter(|r| r.as_nanos() > 0).collect::<Vec<_>>();
+        rtts.extend(rtt_data[1].iter().filter(|r| r.as_nanos() > 0));
+        if !rtts.is_empty() {
+          rtts.sort();
+          let median = rtts[rtts.len() / 2];
+          if let Some(mut tracker) = self.raw_data.get_mut(&local_ip) {
+            // Only apply if the flow has achieved 1 Mbps or more
+            if tracker.bytes_per_second.0 + tracker.bytes_per_second.1 > 125000 {
+              // Shift left
+              for i in 1..60 {
+                tracker.recent_rtt_data[i] = tracker.recent_rtt_data[i - 1];
+              }
+              tracker.recent_rtt_data[0] = *median;
+              tracker.last_fresh_rtt_data_cycle = self_cycle;
+              if let Some(parents) = &tracker.network_json_parents {
+                let net_json = NETWORK_JSON.write().unwrap();
+                if let Some(rtt) = tracker.median_latency() {
+                  net_json.add_rtt_cycle(parents, rtt);
+                }
+              }
             }
           }
         }
       }
-    });
+
+      // Merge in the TCP retries
+      // Reset all entries in the tracker to 0
+      for mut circuit in self.raw_data.iter_mut() {
+        circuit.tcp_retransmits = (0, 0);
+      }
+      // Apply the new ones
+      for (local_ip, retries) in tcp_retries {
+        if let Some(mut tracker) = self.raw_data.get_mut(&local_ip) {
+          tracker.tcp_retransmits.0 = retries[0].saturating_sub(tracker.last_tcp_retransmits.0);
+          tracker.tcp_retransmits.1 = retries[1].saturating_sub(tracker.last_tcp_retransmits.1);
+          tracker.last_tcp_retransmits.0 = retries[0];
+          tracker.last_tcp_retransmits.1 = retries[1];
+        }
+      }
+
+      // Key Expiration
+      if !expired_keys.is_empty() {
+        for key in expired_keys.iter() {
+          // Send it off to netperf for analysis if we are supporting doing so.
+          if let Some(d) = all_flows_lock.get(&key) {
+            let _ = sender.send((key.clone(), (d.0.clone(), d.1.clone())));
+          }
+          // Remove the flow from circulation
+          all_flows_lock.remove(&key);
+        }
+
+        let ret = lqos_sys::end_flows(&mut expired_keys);
+        if let Err(e) = ret {
+          log::warn!("Failed to end flows: {:?}", e);
+        }
+      }
+
+      // Cleaning run
+      all_flows_lock.retain(|_k,v| v.0.last_seen >= expire);
+      expire_rtt_flows();
+    }
   }
 
   #[inline(always)]

src/rust/remove_pinned_maps.sh

@@ -13,3 +13,4 @@ rm -v /sys/fs/bpf/bifrost_vlan_map
 rm -v /sys/fs/bpf/heimdall
 rm -v /sys/fs/bpf/heimdall_config
 rm -v /sys/fs/bpf/heimdall_watching
+rm -v /sys/fs/bpf/flowbee