Rework the file watcher to restart the watcher after firing.

Sometimes the watcher persists, sometimes it doesn't. This is
awful behavior. Work around by looping the watcher and returning
after it fires - restarting the watch process. This has the
added advantage of handling "file deleted" gracefully.

When ShapedDevices cannot be read, an empty set is loaded so
its obvious on the web UI that there is an issue.

In testing, I've been through a bunch of "break shaped devices",
"unbreak shaped devices" and seen the data come and go correctly
now.

ISSUE #239

src/rust/lqos_config/src/shaped_devices/mod.rs

@@ -87,13 +87,13 @@ impl ConfigShapedDevices {
               expected_len,
               len
             );
-            error!("CSV devode error: {msg}");
+            error!("CSV decode error: {msg}");
             return Err(ShapedDevicesError::UnequalLengths(msg));
           } else {
             let msg = format!(
               "Unknown position. Expected {expected_len} fields, found {len}"
             );
-            error!("CSV devode error: {msg}");
+            error!("CSV decode error: {msg}");
             return Err(ShapedDevicesError::UnequalLengths(msg));
           }
         }

src/rust/lqos_node_manager/src/tracker/cache_manager.rs

@@ -86,10 +86,14 @@ pub async fn update_tracking() {
 }
 
 fn load_shaped_devices() {
+  info!("ShapedDevices.csv has changed. Attempting to load it.");
   let shaped_devices = ConfigShapedDevices::load();
   if let Ok(new_file) = shaped_devices {
     info!("ShapedDevices.csv loaded");
     *SHAPED_DEVICES.write() = new_file;
+  } else {
+    warn!("ShapedDevices.csv failed to load, see previous error messages. Reverting to empty set.");
+    *SHAPED_DEVICES.write() = ConfigShapedDevices::default();
   }
 }
 
@@ -109,8 +113,10 @@ fn watch_for_shaped_devices_changing() -> Result<()> {
   watcher.set_file_exists_callback(load_shaped_devices);
   watcher.set_file_created_callback(load_shaped_devices);
   watcher.set_file_changed_callback(load_shaped_devices);
-  let _ = watcher.watch();
-  Ok(())
+  loop {
+    let result = watcher.watch();
+    info!("ShapedDevices watcher returned: {result:?}");
+  }
 }
 
 /// Requests data from `lqosd` and stores it in local

src/rust/lqos_queue_tracker/src/queue_structure/queing_structure_json_monitor.rs

@@ -66,11 +66,12 @@ fn watch_for_queueing_structure_changing() -> Result<(), QueueWatcherError> {
   let mut watcher = FileWatcher::new("queueingStructure.json", watch_path);
   watcher.set_file_created_callback(update_queue_structure);
   watcher.set_file_changed_callback(update_queue_structure);
-  let retval = watcher.watch();
-  if retval.is_err() {
-    error!("Unable to create queueingStructure.json watcher");
+  loop {
+    let retval = watcher.watch();
+    if retval.is_err() {
+      info!("File watcher returned {retval:?}");
+    }
   }
-  Ok(())
 }
 
 #[derive(Error, Debug)]

src/rust/lqos_utils/src/file_watcher.rs

@@ -9,7 +9,7 @@ use thiserror::Error;
 const SLEEP_UNTIL_EXISTS_SECONDS: u64 = 10;
 const SLEEP_AFTER_CREATION_SECONDS: u64 = 3;
 const SLEEP_AFTER_CHANGE_SECONDS: u64 = 3;
-const SLEEP_DEBOUNCE_DURATION: u64 = 10;
+const SLEEP_DEBOUNCE_DURATION: u64 = 1;
 
 /// Provides a convenient mechanism for watching a file for changes.
 /// On Linux, it uses `inotify` - this varies for other operating systems.
@@ -138,6 +138,7 @@ impl FileWatcher {
         info!("{} changed", self.nice_name);
         if let Some(callback) = &mut self.file_changed_callback {
           callback();
+          return Ok(()); // Bail out to restart
         }
       }
     }