This commit is contained in:
rchac committed 2026-05-25 16:39:17 -06:00
1 parent 11542aa41d
commit 922645dfba
7 files changed
+404 -152

No files matched your search

+2
View File
@@ -107,6 +107,8 @@ sudo systemctl status lqosd
journalctl -u lqosd --since "10 minutes ago"
```
Si el log muestra `LibreQoS failed to attach the XDP/TC kernel` o `Unable to load the XDP/TC kernel`, trate el arranque de `lqosd` como fallido. La WebUI y el bus local no arrancan hasta que el programa del kernel se cargue y se adjunte correctamente. El error de carga incluye el valor de retorno bruto, el número errno y el código errno, por ejemplo `raw=-11, errno=11, code=EAGAIN`. Revise si hay un programa XDP existente, un hook TC ocupado, falta de soporte del driver o mapas BPF fijados obsoletos antes de reiniciar `lqosd`.
### Depuración avanzada de lqosd
```bash
+2
View File
@@ -129,6 +129,8 @@ Press the End key on the keyboard to take you to the bottom of the log to see th
Lqosd will provide specific reasons it failed, such as an interface not being up, an interface lacking multi-queue, or other concerns.
If the log shows `LibreQoS failed to attach the XDP/TC kernel` or `Unable to load the XDP/TC kernel`, treat `lqosd` startup as failed. The WebUI and local bus will not start until the kernel program loads and attaches successfully. The load error includes the raw return value, errno number, and errno code, for example `raw=-11, errno=11, code=EAGAIN`. Check for an existing XDP program, busy TC hook, missing driver support, or stale pinned maps before restarting `lqosd`.
If `journalctl -u lqosd` shows `lqosd host memory pressure` or `lqosd process memory critical`, the daemon detected high memory usage and logged diagnostic context. The watchdog does not restart `lqosd`; it records available memory, total memory, `lqosd` RSS/swap, thread count, flow count, and timing counters that help diagnose the source of memory growth. Host memory pressure is logged when available memory is below 10% of installed RAM. Process memory is logged as critical when `lqosd` RSS plus swap reaches 90% of installed RAM.
You can disable these diagnostics with a systemd environment override during short troubleshooting windows:
+35 -7
View File
@@ -11,7 +11,10 @@ use libbpf_sys::{
libbpf_set_strict_mode,
};
use lqos_utils::XdpIpAddress;
use nix::libc::{close, geteuid, if_nametoindex};
use nix::{
errno::Errno,
libc::{close, geteuid, if_nametoindex},
};
use std::{
ffi::{CString, c_void},
fs,
@@ -214,6 +217,21 @@ mod tests {
assert!(err.to_string().contains("map info failed"));
}
#[test]
fn kernel_load_error_includes_raw_errno_and_code() {
for (raw, errno, code) in [(-11, 11, "EAGAIN"), (-16, 16, "EBUSY")] {
let error = format_kernel_load_error(raw);
assert!(error.contains(&format!("raw={raw}")));
assert!(error.contains(&format!("errno={errno}")));
assert!(error.contains(&format!("code={code}")));
}
let error = format_kernel_load_error(i32::MIN);
assert!(error.contains("raw=-2147483648"));
assert!(error.contains("errno=2147483648"));
}
}
pub fn check_root() -> Result<()> {
@@ -382,15 +400,26 @@ unsafe fn open_kernel() -> Result<*mut bpf::lqos_kern> {
}
unsafe fn load_kernel(skeleton: *mut bpf::lqos_kern) -> Result<()> {
let error = unsafe { bpf::lqos_kern_load(skeleton) };
if error != 0 {
let error = format!("Unable to load the XDP/TC kernel ({error})");
Err(Error::msg(error))
let raw_error = unsafe { bpf::lqos_kern_load(skeleton) };
if raw_error != 0 {
Err(Error::msg(format_kernel_load_error(raw_error)))
} else {
Ok(())
}
}
fn format_kernel_load_error(raw_error: i32) -> String {
let errno_number = raw_error.unsigned_abs();
let errno_code = if let Ok(errno) = i32::try_from(errno_number) {
format!("{:?}", Errno::from_raw(errno))
} else {
"unknown".to_string()
};
format!(
"Unable to load the XDP/TC kernel (raw={raw_error}, errno={errno_number}, code={errno_code})"
)
}
#[derive(PartialEq, Eq)]
pub enum InterfaceDirection {
Internet,
@@ -452,8 +481,7 @@ pub fn attach_xdp_and_tc_to_interface(
(*(*skeleton).bss).isp_vlan = isp.to_be();
(*(*skeleton).bss).stick_offset = stick_offset;
}
// Ensure no lingering XDP programs before loading/attaching
let _ = unload_xdp_from_interface(interface_name);
// Load before detaching so a verifier/load failure leaves the previous XDP state intact.
load_kernel(skeleton)?;
crate::initialize_tc_classify_bypass()?;
let prog_fd = bpf::bpf_program__fd((*skeleton).progs.xdp_prog);
+119 -126
View File
@@ -197,7 +197,7 @@ fn main() -> Result<()> {
// Load config
let config = lqos_config::load_config()?;
let web_config = config.clone();
shaping_runtime::mark_starting("LibreQoS is starting the web UI and shaping services.");
shaping_runtime::mark_starting("LibreQoS is starting shaping services.");
ensure_rustls_crypto_provider()?;
@@ -240,119 +240,21 @@ fn main() -> Result<()> {
// Handle signals
let mut signals = Signals::new([SIGINT, SIGHUP, SIGTERM])?;
let (stormguard_ready_tx, stormguard_ready_rx) = tokio::sync::oneshot::channel::<
Option<crossbeam_channel::Sender<lqos_bakery::BakeryCommands>>,
>();
// Create the socket server
let server = UnixSocketServer::new().expect("Unable to spawn server");
// Memory Debugging
memory_debug();
memory_watchdog::start_memory_watchdog();
let control_tx_for_webserver = control_tx_for_web.clone();
let system_usage_tx_for_web = system_usage_tx.clone();
let handle = std::thread::Builder::new()
.name("Async Bus/Web".to_string())
.spawn(move || {
let Ok(tokio_runtime) = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.thread_stack_size(8 * 1024 * 1024)
.build()
else {
error!("Unable to start Tokio runtime. Not much is going to work");
return;
};
tokio_runtime.block_on(async {
let probe_client =
lqos_probe::ProbeManager::spawn(lqos_probe::ProbeManagerConfig::default());
let probe_client_for_stormguard = probe_client.clone();
probe_provider::install_probe_client(probe_client.clone());
tokio::spawn(async {
lqos_topology::start_topology().await;
});
tokio::spawn(async move {
match lts2_sys::control_channel::start_control_channel(control_channel).await {
Ok(_) => info!("Insight control channel started successfully"),
Err(e) => error!("Insight control channel failed to start: {:#}", e),
}
});
tokio::spawn(async move {
match stormguard_ready_rx.await {
Ok(Some(bakery_sender_for_async)) => {
use tokio::time::{Duration, sleep};
for _ in 0..100u32 {
if tokio::fs::metadata(lqos_bus::BUS_SOCKET_PATH).await.is_ok() {
break;
}
sleep(Duration::from_millis(50)).await;
}
if let Some(sender) = lqos_bakery::BAKERY_SENDER.get() {
let _ = sender.send(lqos_bakery::BakeryCommands::BusReady);
}
match lqos_stormguard::start_stormguard(
bakery_sender_for_async,
shaped_devices_tracker::full_network_map_snapshot,
probe_client_for_stormguard,
)
.await
{
Ok(_) => info!("StormGuard started successfully"),
Err(e) => error!("StormGuard failed to start: {:#}", e),
}
}
Ok(None) => {
warn!("StormGuard not started because shaping is degraded.");
}
Err(_) => {
warn!("StormGuard startup channel closed before shaping was ready.");
}
}
});
let (bus_tx, bus_rx) = tokio::sync::mpsc::channel::<(
tokio::sync::oneshot::Sender<lqos_bus::BusReply>,
BusRequest,
)>(100);
let webserver_disabled = web_config.disable_webserver.unwrap_or(false);
if !webserver_disabled {
let control_tx_for_webserver = control_tx_for_webserver.clone();
let probe_client_for_webserver = probe_client.clone();
tokio::spawn(async move {
if let Err(e) = node_manager::spawn_webserver(
bus_tx,
system_usage_tx_for_web,
control_tx_for_webserver,
probe_client_for_webserver,
)
.await
{
error!("Node Manager Failed: {e:?}");
}
});
} else {
warn!("Webserver disabled by configuration");
}
if let Err(e) = server.listen(handle_bus_requests, bus_rx).await {
error!("Bus stopped: {e:?}");
}
});
})?;
// Prepare the bus path and clear any stale socket before fatal attach checks.
// The bus does not bind or listen until after shaping startup below.
let server = UnixSocketServer::new()
.map_err(|err| anyhow::anyhow!("Unable to spawn bus socket server: {err}"))?;
version_checks::start_version_check()?;
let mut kernels = None;
let stormguard_ready = if bakery_sender.is_none() {
false
} else {
match preflight_checks::preflight_checks() {
let stormguard_bakery_sender = match bakery_sender.clone() {
None => None,
Some(bakery_sender_for_shaping) => match preflight_checks::preflight_checks() {
Ok(()) => match stick::recompute_stick_offset(&config) {
Ok(stick_offset) => match FlowActor::start() {
Ok(()) => {
@@ -387,7 +289,7 @@ fn main() -> Result<()> {
throughput_tracker::spawn_throughput_monitor(
flow_tx,
system_usage_tx.clone(),
bakery_sender.clone().expect("checked is_some"),
bakery_sender_for_shaping.clone(),
)?;
spawn_queue_monitor()?;
@@ -398,56 +300,147 @@ fn main() -> Result<()> {
}
lqos_sys::bpf_garbage_collector();
shaping_runtime::mark_active(
"LibreQoS shaping is active and the web UI is available.",
);
true
Some(bakery_sender_for_shaping)
}
Err(err) => {
let detail = format!("{err:#}");
record_shaping_failure(
shaping_runtime::ShapingRuntimeState::ErrorKernelAttach,
"LibreQoS started the web UI, but kernel attach failed.",
format!("{err:#}"),
"LibreQoS failed to start because XDP/TC kernel attach failed.",
detail,
);
return Err(
err.context("LibreQoS failed to attach the XDP/TC kernel")
);
false
}
}
}
Err(err) => {
record_shaping_failure(
shaping_runtime::ShapingRuntimeState::Degraded,
"LibreQoS started the web UI, but shaping runtime initialization failed.",
"LibreQoS could not start shaping runtime services.",
format!("{err:#}"),
);
false
None
}
},
Err(err) => {
record_shaping_failure(
shaping_runtime::ShapingRuntimeState::ErrorConfig,
"LibreQoS started the web UI, but the shaping configuration is invalid.",
"LibreQoS could not start shaping because the configuration is invalid.",
format!("{err:#}"),
);
false
None
}
},
Err(err) => {
let detail = format!("{err:#}");
record_shaping_failure(
shaping_runtime::classify_preflight_error(&detail),
"LibreQoS started the web UI, but shaping preflight failed.",
"LibreQoS could not start shaping because preflight failed.",
detail,
);
false
None
}
}
},
};
let _ = stormguard_ready_tx.send(if stormguard_ready {
bakery_sender.clone()
} else {
None
});
let control_tx_for_webserver = control_tx_for_web.clone();
let system_usage_tx_for_web = system_usage_tx.clone();
let shaping_started = stormguard_bakery_sender.is_some();
let bakery_sender_for_stormguard = stormguard_bakery_sender;
let handle = std::thread::Builder::new()
.name("Async Bus/Web".to_string())
.spawn(move || {
let Ok(tokio_runtime) = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.thread_stack_size(8 * 1024 * 1024)
.build()
else {
error!("Unable to start Tokio runtime. Not much is going to work");
return;
};
tokio_runtime.block_on(async {
let probe_client =
lqos_probe::ProbeManager::spawn(lqos_probe::ProbeManagerConfig::default());
let probe_client_for_stormguard = probe_client.clone();
probe_provider::install_probe_client(probe_client.clone());
tokio::spawn(async {
lqos_topology::start_topology().await;
});
tokio::spawn(async move {
match lts2_sys::control_channel::start_control_channel(control_channel).await {
Ok(_) => info!("Insight control channel started successfully"),
Err(e) => error!("Insight control channel failed to start: {:#}", e),
}
});
tokio::spawn(async move {
match bakery_sender_for_stormguard {
Some(bakery_sender_for_async) => {
use tokio::time::{Duration, sleep};
for _ in 0..100u32 {
if tokio::fs::metadata(lqos_bus::BUS_SOCKET_PATH).await.is_ok() {
break;
}
sleep(Duration::from_millis(50)).await;
}
if let Some(sender) = lqos_bakery::BAKERY_SENDER.get() {
let _ = sender.send(lqos_bakery::BakeryCommands::BusReady);
}
match lqos_stormguard::start_stormguard(
bakery_sender_for_async,
shaped_devices_tracker::full_network_map_snapshot,
probe_client_for_stormguard,
)
.await
{
Ok(_) => info!("StormGuard started successfully"),
Err(e) => error!("StormGuard failed to start: {:#}", e),
}
}
None => {
warn!("StormGuard not started because shaping is degraded.");
}
}
});
let (bus_tx, bus_rx) = tokio::sync::mpsc::channel::<(
tokio::sync::oneshot::Sender<lqos_bus::BusReply>,
BusRequest,
)>(100);
let webserver_disabled = web_config.disable_webserver.unwrap_or(false);
if !webserver_disabled {
let control_tx_for_webserver = control_tx_for_webserver.clone();
let probe_client_for_webserver = probe_client.clone();
tokio::spawn(async move {
if let Err(e) = node_manager::spawn_webserver(
bus_tx,
system_usage_tx_for_web,
control_tx_for_webserver,
probe_client_for_webserver,
)
.await
{
error!("Node Manager Failed: {e:?}");
}
});
} else {
warn!("Webserver disabled by configuration");
}
if let Err(e) = server.listen(handle_bus_requests, bus_rx).await {
error!("Bus stopped: {e:?}");
}
});
})?;
if shaping_started {
shaping_runtime::mark_active("LibreQoS shaping is active.");
}
std::thread::Builder::new()
.name("Signal Handler".to_string())
+61 -6
View File
@@ -1214,14 +1214,19 @@ async fn receive_channel_message(
Err(_) => (false, "Error".to_string()),
};
let response = WsResponse::LtsSignUpResult { ok, message };
if send_ws_response(&tx, response).await {
return true;
}
if ok {
let close_connection = send_ws_response(&tx, response).await;
maybe_schedule_lts_signup_shutdown(ok, close_connection, || {
tokio::spawn(async {
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
std::process::exit(0);
if let Err(err) = crate::program_control::request_graceful_shutdown(
"Insight signup completed",
) {
tracing::error!("Unable to request graceful lqosd restart: {err}");
}
});
});
if close_connection {
return true;
}
}
WsRequest::LtsShaperStatus => match lts::shaper_status_data().await {
@@ -2470,6 +2475,16 @@ async fn send_ws_response(tx: &Sender<Arc<Vec<u8>>>, response: WsResponse) -> bo
}
}
fn maybe_schedule_lts_signup_shutdown(
signup_ok: bool,
close_connection: bool,
schedule_shutdown: impl FnOnce(),
) {
if signup_ok && !close_connection {
schedule_shutdown();
}
}
fn decode_ws_request(payload: &[u8]) -> Result<WsRequest, String> {
let prefix = payload_prefix_hex(payload, 24);
let hint = payload_hint(payload);
@@ -2525,12 +2540,16 @@ fn payload_hint(payload: &[u8]) -> &'static str {
#[cfg(test)]
mod tests {
use super::decode_ws_request;
use super::{decode_ws_request, maybe_schedule_lts_signup_shutdown};
use crate::node_manager::local_api::urgent::{UrgentList, UrgentStatus};
use crate::node_manager::ws::messages::WsRequest;
use crate::node_manager::ws::messages::{WsResponse, encode_ws_message};
use serde_cbor::Value as CborValue;
use std::collections::BTreeMap;
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
fn text(value: &str) -> CborValue {
CborValue::Text(value.to_string())
@@ -2582,6 +2601,42 @@ mod tests {
.expect("valid create payload")
}
#[test]
fn lts_signup_shutdown_is_scheduled_after_successful_response() {
let called = Arc::new(AtomicBool::new(false));
let called_by_scheduler = called.clone();
maybe_schedule_lts_signup_shutdown(true, false, || {
called_by_scheduler.store(true, Ordering::SeqCst);
});
assert!(called.load(Ordering::SeqCst));
}
#[test]
fn lts_signup_shutdown_is_not_scheduled_when_response_closes_connection() {
let called = Arc::new(AtomicBool::new(false));
let called_by_scheduler = called.clone();
maybe_schedule_lts_signup_shutdown(true, true, || {
called_by_scheduler.store(true, Ordering::SeqCst);
});
assert!(!called.load(Ordering::SeqCst));
}
#[test]
fn lts_signup_shutdown_is_not_scheduled_after_failed_signup() {
let called = Arc::new(AtomicBool::new(false));
let called_by_scheduler = called.clone();
maybe_schedule_lts_signup_shutdown(false, false, || {
called_by_scheduler.store(true, Ordering::SeqCst);
});
assert!(!called.load(Ordering::SeqCst));
}
fn update_shaped_device_payload(device: CborValue) -> Vec<u8> {
serde_cbor::to_vec(&map(vec![(
"UpdateShapedDevice",
+64
View File
@@ -1,4 +1,5 @@
use lqos_bus::BusResponse;
use tracing::info;
pub fn reload_libre_qos() -> BusResponse {
let result = lqos_config::load_libreqos();
@@ -7,3 +8,66 @@ pub fn reload_libre_qos() -> BusResponse {
Err(..) => BusResponse::Fail("Unable to reload LibreQoS".to_string()),
}
}
/// Requests the same graceful shutdown path used by systemd and operator SIGTERM.
///
/// Side effects: this function sends `SIGTERM` to the current process. The
/// signal handler owns cleanup for XDP/TC detach, bus socket cleanup, and file
/// lock release before process exit.
pub fn request_graceful_shutdown(reason: &str) -> anyhow::Result<()> {
request_graceful_shutdown_with(reason, send_sigterm_to_self)
}
fn request_graceful_shutdown_with(
reason: &str,
send_sigterm: impl FnOnce() -> std::io::Result<()>,
) -> anyhow::Result<()> {
info!("Requesting graceful lqosd shutdown: {reason}");
send_sigterm()?;
Ok(())
}
fn send_sigterm_to_self() -> std::io::Result<()> {
let result = unsafe { nix::libc::kill(nix::libc::getpid(), nix::libc::SIGTERM) };
if result == 0 {
Ok(())
} else {
Err(std::io::Error::last_os_error())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
#[test]
fn graceful_shutdown_request_sends_sigterm() {
let called = Arc::new(AtomicBool::new(false));
let called_by_sender = called.clone();
request_graceful_shutdown_with("test", || {
called_by_sender.store(true, Ordering::SeqCst);
Ok(())
})
.expect("shutdown request should succeed");
assert!(called.load(Ordering::SeqCst));
}
#[test]
fn graceful_shutdown_request_returns_signal_errors() {
let err = request_graceful_shutdown_with("test", || {
Err(std::io::Error::from_raw_os_error(nix::libc::EPERM))
})
.expect_err("signal errors should be returned");
let io_error = err
.downcast_ref::<std::io::Error>()
.expect("shutdown request should return the signal IO error");
assert_eq!(io_error.raw_os_error(), Some(nix::libc::EPERM));
}
}
+121 -13
View File
@@ -13,20 +13,50 @@ pub fn start_remote_commands() {
if crate::lts2_sys::remote_command_count() > 0 {
let commands = crate::lts2_sys::remote_commands();
commands.into_iter().for_each(run_command);
process_remote_command_batch(commands, run_command);
}
}
});
}
fn run_command(command: RemoteCommand) {
if !crate::lts2_sys::current_capabilities().can_receive_remote_commands {
fn process_remote_command_batch(
commands: Vec<RemoteCommand>,
mut run: impl FnMut(RemoteCommand) -> RemoteCommandAction,
) {
for command in commands {
if run(command) == RemoteCommandAction::StopBatch {
break;
}
}
}
#[derive(Debug, PartialEq, Eq)]
enum RemoteCommandAction {
Continue,
StopBatch,
}
fn run_command(command: RemoteCommand) -> RemoteCommandAction {
run_command_with(
command,
crate::lts2_sys::current_capabilities().can_receive_remote_commands,
crate::program_control::request_graceful_shutdown,
)
}
fn run_command_with(
command: RemoteCommand,
can_receive_remote_commands: bool,
mut request_shutdown: impl FnMut(&str) -> anyhow::Result<()>,
) -> RemoteCommandAction {
if !can_receive_remote_commands {
warn!("Ignoring remote command because current license tier does not permit it");
return;
return RemoteCommandAction::Continue;
}
match command {
RemoteCommand::Log(msg) => {
warn!("Message from Insight: {}", msg);
RemoteCommandAction::Continue
}
RemoteCommand::SetInsightControlledTopology { enabled } => {
if let Ok(config) = lqos_config::load_config() {
@@ -38,6 +68,7 @@ fn run_command(command: RemoteCommand) {
let _ = crate::scheduler_control::enable_scheduler();
let _ = crate::scheduler_control::restart_scheduler();
}
RemoteCommandAction::Continue
}
RemoteCommand::SetInsightRole { role } => {
if let Ok(config) = lqos_config::load_config() {
@@ -49,21 +80,98 @@ fn run_command(command: RemoteCommand) {
let _ = crate::scheduler_control::enable_scheduler();
let _ = crate::scheduler_control::restart_scheduler();
}
RemoteCommandAction::Continue
}
RemoteCommand::RestartLqosd => {
// Gracefully detach XDP/TC before exiting to avoid stale attachments
if let Ok(cfg) = lqos_config::load_config() {
if cfg.on_a_stick_mode() {
let _ = lqos_sys::unload_xdp_from_interface(&cfg.internet_interface());
} else {
let _ = lqos_sys::unload_xdp_from_interface(&cfg.internet_interface());
let _ = lqos_sys::unload_xdp_from_interface(&cfg.isp_interface());
}
if let Err(err) = request_shutdown("Insight requested lqosd restart") {
tracing::error!("Unable to request graceful lqosd restart: {err}");
}
std::process::exit(0);
RemoteCommandAction::StopBatch
}
RemoteCommand::RestartScheduler => {
let _ = crate::scheduler_control::restart_scheduler();
RemoteCommandAction::Continue
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
#[test]
fn remote_command_batch_stops_after_restart_request() {
let commands = vec![
RemoteCommand::Log("before".to_string()),
RemoteCommand::RestartLqosd,
RemoteCommand::Log("after".to_string()),
];
let mut processed = Vec::new();
process_remote_command_batch(commands, |command| {
processed.push(command.clone());
match command {
RemoteCommand::RestartLqosd => RemoteCommandAction::StopBatch,
_ => RemoteCommandAction::Continue,
}
});
assert_eq!(
processed,
vec![
RemoteCommand::Log("before".to_string()),
RemoteCommand::RestartLqosd,
]
);
}
#[test]
fn remote_command_batch_continues_without_restart_request() {
let commands = vec![
RemoteCommand::Log("first".to_string()),
RemoteCommand::RestartScheduler,
RemoteCommand::Log("last".to_string()),
];
let mut processed = Vec::new();
process_remote_command_batch(commands.clone(), |command| {
processed.push(command);
RemoteCommandAction::Continue
});
assert_eq!(processed, commands);
}
#[test]
fn restart_lqosd_requests_shutdown_and_stops_batch() {
let called = Arc::new(AtomicBool::new(false));
let called_by_shutdown = called.clone();
let action = run_command_with(RemoteCommand::RestartLqosd, true, |reason| {
assert_eq!(reason, "Insight requested lqosd restart");
called_by_shutdown.store(true, Ordering::SeqCst);
Ok(())
});
assert_eq!(action, RemoteCommandAction::StopBatch);
assert!(called.load(Ordering::SeqCst));
}
#[test]
fn restart_lqosd_does_not_shutdown_without_remote_command_capability() {
let called = Arc::new(AtomicBool::new(false));
let called_by_shutdown = called.clone();
let action = run_command_with(RemoteCommand::RestartLqosd, false, |_| {
called_by_shutdown.store(true, Ordering::SeqCst);
Ok(())
});
assert_eq!(action, RemoteCommandAction::Continue);
assert!(!called.load(Ordering::SeqCst));
}
}