ISSUE #518 : Remove the "hot cache clear" call altogether, and instead use a timeout/expiration to clear the cache gracefully. This allowed the map to be unpinned, and never accessed from userspace. Should fix the reload delays, and still give accurate mappings after a complete map rebuild.

src/LibreQoS.py

@@ -986,7 +986,6 @@ def refreshShapers():
 		print("Executing XDP-CPUMAP-TC IP filter commands")
 		numXdpCommands = ipMapBatch.length()
 		if enable_actual_shell_commands():
-			ipMapBatch.finish_ip_mappings()
 			ipMapBatch.submit()
 			#for command in xdpCPUmapCommands:
 			#	logging.info(command)

src/rust/lqos_bus/src/bus/request.rs

@@ -70,11 +70,6 @@ pub enum BusRequest {
     upload: bool,
   },
 
-  /// After a batch of `MapIpToFlow` requests, this command will
-  /// clear the hot cache, forcing the XDP program to re-read the
-  /// mapping table.
-  ClearHotCache,
-
   /// Requests that the XDP program unmap an IP address/subnet from
   /// the traffic management system.
   DelIpFlow {

src/rust/lqos_python/src/lib.rs

@@ -234,12 +234,6 @@ impl BatchedCommands {
     }
   }
 
-  pub fn finish_ip_mappings(&mut self) -> PyResult<()> {
-    let request = BusRequest::ClearHotCache;
-    self.batch.push(request);
-    Ok(())
-  }
-
   pub fn length(&self) -> PyResult<usize> {
     Ok(self.batch.len())
   }

src/rust/lqos_sys/src/bpf/common/lpm.h

@@ -31,12 +31,19 @@ struct ip_hash_key {
 #define USE_HOTCACHE 1
 
 #ifdef USE_HOTCACHE
+
+#define HOT_CACHE_EXPIRY 1000000000 // 1 second
+
+struct hot_cache_data {
+    __u64 expires;
+    struct ip_hash_info info;
+};
+
 struct {
 	__uint(type, BPF_MAP_TYPE_LRU_HASH);
 	__uint(max_entries, HOT_CACHE_SIZE);
 	__type(key, struct in6_addr);
-	__type(value, struct ip_hash_info);
-	__uint(pinning, LIBBPF_PIN_BY_NAME);
+	__type(value, struct hot_cache_data);
 } ip_to_cpu_and_tc_hotcache SEC(".maps");
 #endif
 
@@ -45,7 +52,7 @@ struct {
 	__uint(type, BPF_MAP_TYPE_LPM_TRIE);
 	__uint(max_entries, IP_HASH_ENTRIES_MAX);
 	__type(key, struct ip_hash_key);
-	__type(value, struct ip_hash_info);
+	__type(value, struct hot_cache_data);
 	__uint(pinning, LIBBPF_PIN_BY_NAME);
 	__uint(map_flags, BPF_F_NO_PREALLOC);
 } map_ip_to_cpu_and_tc SEC(".maps");
@@ -94,18 +101,19 @@ static __always_inline struct ip_hash_info * setup_lookup_key_and_tc_cpu(
 
     #ifdef USE_HOTCACHE
     // Try a hot cache search
-    ip_info = bpf_map_lookup_elem(
+    struct hot_cache_data * cached;
+    cached = bpf_map_lookup_elem(
         &ip_to_cpu_and_tc_hotcache,
         &lookup_key->address
     );
-    if (ip_info) {
+    if (cached && cached->expires < dissector->now) {
         // Is it a negative hit?
-        if (ip_info->cpu == NEGATIVE_HIT) {
+        if (cached->info.cpu == NEGATIVE_HIT) {
             return NULL;
         }
 
         // We got a cache hit, so return
-        return ip_info;
+        return &cached->info;
     }
     #endif
 
@@ -116,11 +124,15 @@ static __always_inline struct ip_hash_info * setup_lookup_key_and_tc_cpu(
     );
     #ifdef USE_HOTCACHE
     if (ip_info) {
+        struct hot_cache_data hot_data = {
+            .expires = dissector->now + HOT_CACHE_EXPIRY,
+            .info = *ip_info
+        };
         // We found it, so add it to the cache
         bpf_map_update_elem(
             &ip_to_cpu_and_tc_hotcache,
             &lookup_key->address,
-            ip_info,
+            &hot_data,
             BPF_NOEXIST
         );
     } else {

src/rust/lqos_sys/src/ip_mapping/mod.rs

@@ -58,7 +58,6 @@ pub fn del_ip_from_tc(address: &str, upload: bool) -> Result<()> {
   let ip = XdpIpAddress::from_ip(ip);
   let mut key = IpHashKey { prefixlen: ip_to_add.prefix, address: ip.0 };
   bpf_map.delete(&mut key)?;
-  clear_hot_cache()?;
   Ok(())
 }
 
@@ -74,8 +73,6 @@ pub fn clear_ips_from_tc() -> Result<()> {
   )?;
   bpf_map.clear()?;
 
-  clear_hot_cache()?;
-  
   Ok(())
 }
 
@@ -94,12 +91,3 @@ pub fn list_mapped_ips() -> Result<Vec<(IpHashKey, IpHashData)>> {
 
   Ok(raw)
 }
-
-/// Clears the "hot cache", which should be done whenever you change the IP
-/// mappings - because otherwise cached data will keep going to the previous
-/// destinations.
-pub fn clear_hot_cache() -> Result<()> {
-  let mut bpf_map = BpfMap::<XdpIpAddress, IpHashData>::from_path("/sys/fs/bpf/ip_to_cpu_and_tc_hotcache")?;
-  bpf_map.clear()?;
-  Ok(())
-}

src/rust/lqos_sys/src/lib.rs

@@ -23,7 +23,7 @@ mod bpf_iterator;
 pub mod flowbee_data;
 
 pub use ip_mapping::{
-  add_ip_to_tc, clear_ips_from_tc, del_ip_from_tc, list_mapped_ips, clear_hot_cache,
+  add_ip_to_tc, clear_ips_from_tc, del_ip_from_tc, list_mapped_ips,
 };
 pub use kernel_wrapper::LibreQoSKernels;
 pub use linux::num_possible_cpus;

src/rust/lqosd/src/ip_mapping.rs

@@ -19,10 +19,6 @@ pub(crate) fn map_ip_to_flow(
   expect_ack(lqos_sys::add_ip_to_tc(ip_address, *tc_handle, cpu, upload))
 }
 
-pub(crate) fn clear_hot_cache() -> BusResponse {
-  expect_ack(lqos_sys::clear_hot_cache())
-}
-
 pub(crate) fn del_ip_flow(ip_address: &str, upload: bool) -> BusResponse {
   expect_ack(lqos_sys::del_ip_from_tc(ip_address, upload))
 }

src/rust/lqosd/src/main.rs

@@ -38,7 +38,6 @@ mod node_manager;
 // Use JemAllocator only on supported platforms
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 use jemallocator::Jemalloc;
-use crate::ip_mapping::clear_hot_cache;
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
 #[global_allocator]
@@ -177,7 +176,6 @@ fn handle_bus_requests(
       BusRequest::MapIpToFlow { ip_address, tc_handle, cpu, upload } => {
         map_ip_to_flow(ip_address, tc_handle, *cpu, *upload)
       }
-      BusRequest::ClearHotCache => clear_hot_cache(),
       BusRequest::DelIpFlow { ip_address, upload } => {
         del_ip_flow(ip_address, *upload)
       }

src/rust/xdp_iphash_to_cpu_cmdline/src/main.rs

@@ -44,8 +44,6 @@ enum Commands {
   Clear,
   /// List all mapped IPs.
   List,
-  /// Flushes the Hot Cache (to be used after when you are done making changes).
-  Flush,
 }
 
 async fn talk_to_server(command: BusRequest) -> Result<()> {
@@ -124,7 +122,6 @@ pub async fn main() -> Result<()> {
     }
     Some(Commands::Clear) => talk_to_server(BusRequest::ClearIpFlow).await?,
     Some(Commands::List) => talk_to_server(BusRequest::ListIpFlow).await?,
-    Some(Commands::Flush) => talk_to_server(BusRequest::ClearHotCache).await?,
     None => {
       println!("Run with --help to see instructions");
       exit(0);