Super ugly start to getting perf events working for Heimdall.

src/rust/lqos_sys/src/bpf/common/heimdall.h

@@ -58,6 +58,12 @@ struct
     __uint(pinning, LIBBPF_PIN_BY_NAME);
 } heimdall SEC(".maps");
 
+struct {
+	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
+	__uint(key_size, sizeof(__u32));
+	__uint(value_size, sizeof(__u32));
+} heimdall_events SEC(".maps");
+
 static __always_inline __u8 get_heimdall_mode()
 {
     __u32 index = 0;
@@ -74,10 +80,10 @@ static __always_inline __u8 get_heimdall_mode()
 
 static __always_inline bool is_heimdall_watching(struct dissector_t *dissector)
 {
-    __u32 *watching = bpf_map_lookup_elem(&heimdall_watching, &dissector->src_ip);
+    __u32 *watching = (__u32 *)bpf_map_lookup_elem(&heimdall_watching, &dissector->src_ip);
     if (watching)
         return true;
-    watching = bpf_map_lookup_elem(&heimdall_watching, &dissector->dst_ip);
+    watching = (__u32 *)bpf_map_lookup_elem(&heimdall_watching, &dissector->dst_ip);
     if (watching)
         return true;
     return false;
@@ -85,6 +91,11 @@ static __always_inline bool is_heimdall_watching(struct dissector_t *dissector)
 
 static __always_inline void update_heimdall(struct dissector_t *dissector, __u32 size, int dir)
 {
+    __u32 e = 1;
+    if (bpf_perf_event_output(dissector->ctx, &heimdall_events, BPF_F_CURRENT_CPU, &e, sizeof(e)) != 0) {
+        bpf_debug("Failed to send perf event");
+    }
+
     // Don't report any non-ICMP without ports
     if (dissector->ip_protocol != 1 && (dissector->src_port == 0 || dissector->dst_port == 0))
         return;

src/rust/lqos_sys/src/bpf/lqos_kern.c

@@ -137,11 +137,10 @@ int xdp_prog(struct xdp_md *ctx)
     // Send on its way
     if (tc_handle != 0) {
         // Send data to Heimdall
+        if (heimdall_mode == 2 || (heimdall_mode==1 && is_heimdall_watching(&dissector))) {
 #ifdef VERBOSE
                   bpf_debug("(XDP) Storing Heimdall Data");
-#endif
-        if (heimdall_mode == 2 || (heimdall_mode==1 && is_heimdall_watching(&dissector))) {
-            update_heimdall(&dissector, ctx->data_end - ctx->data, effective_direction);
+#endif            update_heimdall(&dissector, ctx->data_end - ctx->data, effective_direction);
         }
 
         // Handle CPU redirection if there is one specified

src/rust/lqos_sys/src/heimdall_map.rs

@@ -1,4 +1,4 @@
-use std::time::Duration;
+use std::{time::Duration, ffi::c_void};
 use dashmap::DashMap;
 use lqos_utils::unix_time::time_since_boot;
 use once_cell::sync::Lazy;
@@ -130,3 +130,13 @@ pub fn heimdall_watch_ip(ip: XdpIpAddress) {
     HEIMDALL_WATCH_LIST.insert(ip, h);
   }
 }
+
+#[no_mangle]
+pub unsafe extern "C" fn missed_events(ctx: *mut c_void, cpu: i32, lost_count: u64) {
+  log::warn!("Missed {lost_count} Heimdall events on {cpu}");
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn handle_events(ctx: *mut c_void, cpu: i32, data: *mut c_void, data_size: u32) {
+  //log::info!("Received a callback on {cpu}");
+}

src/rust/lqos_sys/src/lqos_kernel.rs

@@ -12,7 +12,7 @@ use libbpf_sys::{
 };
 use log::{info, warn};
 use nix::libc::{geteuid, if_nametoindex};
-use std::{ffi::CString, process::Command};
+use std::{ffi::{CString, c_void}, process::Command, thread::Thread};
 
 pub(crate) mod bpf {
   #![allow(warnings, unused)]
@@ -74,6 +74,7 @@ pub fn unload_xdp_from_interface(interface_name: &str) -> Result<()> {
 
 fn set_strict_mode() -> Result<()> {
   let err = unsafe { libbpf_set_strict_mode(LIBBPF_STRICT_ALL) };
+  #[cfg(not(debug_assertions))]
   unsafe {
     bpf::do_not_print();
   }
@@ -156,6 +157,31 @@ pub fn attach_xdp_and_tc_to_interface(
     )
   }; // Ignoring error, because it's ok to not have something to detach
 
+  // Find the heimdall_events perf map by name
+  let heimdall_events_name = CString::new("heimdall_events").unwrap();
+  let heimdall_events_map = unsafe { bpf::bpf_object__find_map_by_name((*skeleton).obj, heimdall_events_name.as_ptr()) };
+  let heimdall_events_fd = unsafe { bpf::bpf_map__fd(heimdall_events_map) };
+  if heimdall_events_fd < 0 {
+    log::error!("Unable to load Heimdall Events FD");
+    return Err(anyhow::Error::msg("Unable to load Heimdall Events FD"));
+  }
+  let opts: *const bpf::perf_buffer_opts = std::ptr::null();
+  let heimdall_perf_buffer = unsafe {
+    bpf::perf_buffer__new(
+      heimdall_events_fd, 
+      8, 
+      Some(crate::heimdall_map::handle_events), 
+      Some(crate::heimdall_map::missed_events), 
+      opts as *mut c_void, 
+      opts)
+  };
+  if unsafe { bpf::libbpf_get_error(heimdall_perf_buffer as *mut c_void) != 0 } {
+    log::error!("Failed to create Heimdall event buffer");
+    return Err(anyhow::Error::msg("Failed to create Heimdall event buffer"));
+  }
+  let handle = PerfBufferHandle(heimdall_perf_buffer);
+  std::thread::spawn(|| poll_perf_events(handle));
+
   // Remove any previous entry
   let _r = Command::new("tc")
     .args(["qdisc", "del", "dev", interface_name, "clsact"])
@@ -257,3 +283,20 @@ unsafe fn try_xdp_attach(
   }
   Ok(())
 }
+
+// Handle type used to wrap *mut bpf::perf_buffer and indicate
+// that it can be moved. Really unsafe code in theory.
+struct PerfBufferHandle(*mut bpf::perf_buffer);
+unsafe impl Send for PerfBufferHandle {}
+unsafe impl Sync for PerfBufferHandle {}
+
+/// Run this in a thread, or doom will surely hit you
+fn poll_perf_events(heimdall_perf_buffer: PerfBufferHandle) {
+  let heimdall_perf_buffer = heimdall_perf_buffer.0;
+  loop {
+    let err = unsafe { bpf::perf_buffer__poll(heimdall_perf_buffer, 100) };
+    if err < 0 {
+      log::error!("Error polling perfbuffer");
+    }
+  }
+}