diff --git a/docs-es/v2.0/troubleshooting-es.md b/docs-es/v2.0/troubleshooting-es.md index 14f95604..b2befc84 100644 --- a/docs-es/v2.0/troubleshooting-es.md +++ b/docs-es/v2.0/troubleshooting-es.md @@ -117,6 +117,7 @@ Si el scheduler falla inmediatamente después de un reinicio con un mensaje como ### El estado del scheduler en WebUI aparece no saludable Versiones recientes muestran estado/readiness del scheduler en WebUI. +Si el modal del scheduler indica que se agotó el tiempo al cargar detalles, las versiones actuales mantienen visible la última instantánea buena del scheduler con su antigüedad en lugar de convertir ese problema de transporte en un error del scheduler. Trate primero esa advertencia como un problema de comunicación entre WebUI y `lqosd`, y confirme la salud real del scheduler en los logs antes de asumir que falló el shaping. Si aparece caído/desactualizado: diff --git a/docs/v2.0/troubleshooting.md b/docs/v2.0/troubleshooting.md index 127b479f..9c341feb 100644 --- a/docs/v2.0/troubleshooting.md +++ b/docs/v2.0/troubleshooting.md @@ -184,6 +184,7 @@ If an integration subprocess fails, current builds keep the scheduler alive, pub Recent builds expose scheduler readiness/state in the WebUI (Node Manager). If the scheduler is still starting, the sidebar now reports the current startup phase and a coarse progress ring rather than only a spinner. Current builds also treat scheduler progress, output, and error bus messages as proof that the scheduler is alive, so the sidebar should not stay stuck on `Scheduler available: false` while the scheduler is actively reporting work. +If the scheduler modal says scheduler details timed out, current builds keep showing the last good scheduler snapshot with its age instead of turning that transport problem into a scheduler error. Treat that warning as a WebUI or `lqosd` communication issue first, then confirm scheduler health in the service logs before assuming shaping failed. If scheduler status appears down/stale: 1. Verify both services: diff --git a/src/rust/lqosd/src/node_manager/js_build/src/template.js b/src/rust/lqosd/src/node_manager/js_build/src/template.js index 233be5f3..af9c347a 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/template.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/template.js @@ -35,9 +35,11 @@ const SCHEDULER_MODAL_ACTIVE_POLL_MS = 1000; let schedulerStatusPollTimer = null; let schedulerStatusRequestInFlight = false; let schedulerStatusFirstRequestedAt = null; +let cachedSchedulerStatusData = null; let schedulerModalPollTimer = null; let schedulerModalRequestInFlight = false; let schedulerModalInstance = null; +let cachedSchedulerDetailsData = null; let cachedQueueModeConfig = null; function listenOnceWithTimeout(eventName, timeoutMs, handler, onTimeout) { @@ -125,6 +127,42 @@ function schedulerErrorText(data) { return String(error).trim(); } +function schedulerSnapshotUpdatedUnix(data) { + const progressUpdatedUnix = Number(data?.progress?.updated_unix); + if (Number.isFinite(progressUpdatedUnix) && progressUpdatedUnix > 0) { + return progressUpdatedUnix; + } + const statusUpdatedUnix = Number(data?.updated_unix); + if (Number.isFinite(statusUpdatedUnix) && statusUpdatedUnix > 0) { + return statusUpdatedUnix; + } + return null; +} + +function schedulerUpdatedText(data) { + const updatedUnix = schedulerSnapshotUpdatedUnix(data); + if (!updatedUnix) { + return "Update time unavailable"; + } + return schedulerRelativeTime(updatedUnix); +} + +function schedulerTransportWarningText(message, data) { + const updatedText = schedulerUpdatedText(data); + if (updatedText === "Update time unavailable") { + return `${message} Retrying in the background.`; + } + return `${message} Showing cached scheduler data. ${updatedText}.`; +} + +function schedulerStaleStatusLabel(data) { + const updatedText = schedulerUpdatedText(data); + if (updatedText === "Update time unavailable") { + return "Scheduler status is stale"; + } + return `Scheduler status is stale. ${updatedText}.`; +} + function schedulerLooksRecovered(data) { const progress = data?.progress || null; if (!data?.available || !progress || progress.active) { @@ -170,6 +208,17 @@ function schedulerStateDescriptor(data) { icon: "fa-list-check", }; } + if (data?.stale) { + return { + tone: "warning", + badgeClass: "text-bg-warning", + label: "Stale", + title: "Scheduler status is stale", + subtitle: "Showing the last scheduler snapshot older than five minutes", + ringTone: "tone-warning", + icon: "fa-clock", + }; + } if (active) { return { tone: "info", @@ -250,7 +299,7 @@ function schedulerActivityItems(output, error, data) { .reverse(); } -function renderSchedulerDetails(data) { +function renderSchedulerDetails(data, options = {}) { const progress = data?.progress || null; const descriptor = schedulerStateDescriptor(data); const liveError = schedulerHasLiveError(data) ? schedulerErrorText(data) : ""; @@ -262,22 +311,30 @@ function renderSchedulerDetails(data) { const stepText = Number.isFinite(stepIndex) && Number.isFinite(stepCount) && stepCount > 0 ? `Step ${stepIndex} of ${stepCount}` : "No active step reported"; - const updatedText = progress?.updated_unix - ? schedulerRelativeTime(progress.updated_unix) - : "Update time unavailable"; - const availabilityText = data?.setup_required ? "Setup Required" : (data?.available ? "Healthy" : "Unavailable"); + const updatedText = schedulerUpdatedText(data); + const availabilityText = data?.setup_required + ? "Setup Required" + : data?.stale + ? "Stale" + : (data?.available ? "Healthy" : "Unavailable"); const recentResult = schedulerSetupMessage(data) || summarizeSchedulerOutput(data?.output, historicalError || liveError); const activity = schedulerActivityItems(data?.output, historicalError || liveError, data); - const progressMeta = data?.setup_required - ? "Complete runtime setup to enable scheduler work" - : progress?.active - ? `${percent}% complete` - : descriptor.label === "Idle" - ? "Last run complete" - : "Waiting for activity"; + let progressMeta = "Waiting for activity"; + if (options.transportWarning) { + progressMeta = "Showing cached scheduler details while the UI reconnects"; + } else if (data?.setup_required) { + progressMeta = "Complete runtime setup to enable scheduler work"; + } else if (progress?.active) { + progressMeta = `${percent}% complete`; + } else if (descriptor.label === "Idle") { + progressMeta = "Last run complete"; + } const setupAlert = data?.setup_required ? `` : ""; + const transportWarningMarkup = options.transportWarning + ? `` + : ""; const alertMarkup = liveError ? `` : ""; @@ -304,6 +361,7 @@ function renderSchedulerDetails(data) { ${setupAlert} + ${transportWarningMarkup} ${alertMarkup}
@@ -360,7 +418,7 @@ function updateSchedulerModalBody(contentHtml) { } } -function renderSchedulerStatus(container, state, progress) { +function renderSchedulerStatus(container, state, progress, labelOverride = null) { if (!container) return; let color = "text-secondary"; @@ -388,11 +446,18 @@ function renderSchedulerStatus(container, state, progress) { color = "text-danger"; label = "Scheduler has an internal error"; indicator = schedulerRingMarkup(100, "tone-danger", "fa-triangle-exclamation"); + } else if (state === "stale") { + color = "text-warning"; + label = "Scheduler status is stale"; + indicator = schedulerRingMarkup(100, "tone-warning", "fa-clock"); } else if (state === "setup") { color = "text-warning"; label = "Scheduler needs topology setup"; indicator = schedulerRingMarkup(100, "tone-warning", "fa-list-check"); } + if (labelOverride) { + label = labelOverride; + } container.innerHTML = `
`); } -function renderSchedulerModalError(message) { - updateSchedulerModalBody(` - `); -} - function ensureSchedulerModalLifecycle() { const modalEl = document.getElementById('schedulerModal'); if (!modalEl || modalEl.dataset.schedulerLiveBound === "1") { @@ -567,10 +649,24 @@ function refreshSchedulerModalDetails(showLoading = false) { return; } if (!msg || !msg.data) { - renderSchedulerModalError("Failed to load scheduler details"); + if (cachedSchedulerDetailsData) { + updateSchedulerModalBody(renderSchedulerDetails(cachedSchedulerDetailsData, { + transportWarning: schedulerTransportWarningText( + "Scheduler details response was empty.", + cachedSchedulerDetailsData + ), + })); + } else { + updateSchedulerModalBody(` + `); + } scheduleNextSchedulerModalPoll(SCHEDULER_MODAL_IDLE_POLL_MS); return; } + cachedSchedulerDetailsData = msg.data; + cachedSchedulerStatusData = msg.data; updateSchedulerModalBody(renderSchedulerDetails(msg.data)); scheduleNextSchedulerModalPoll(schedulerModalPollDelay(msg.data)); }, () => { @@ -579,7 +675,19 @@ function refreshSchedulerModalDetails(showLoading = false) { stopSchedulerModalPolling(); return; } - renderSchedulerModalError("Timed out while loading scheduler details"); + if (cachedSchedulerDetailsData) { + updateSchedulerModalBody(renderSchedulerDetails(cachedSchedulerDetailsData, { + transportWarning: schedulerTransportWarningText( + "Scheduler details refresh timed out.", + cachedSchedulerDetailsData + ), + })); + } else { + updateSchedulerModalBody(` + `); + } scheduleNextSchedulerModalPoll(SCHEDULER_MODAL_IDLE_POLL_MS); }); wsClient.send({ SchedulerDetails: {} }); diff --git a/src/rust/lqosd/src/node_manager/local_api/scheduler.rs b/src/rust/lqosd/src/node_manager/local_api/scheduler.rs index 377307bc..33062255 100644 --- a/src/rust/lqosd/src/node_manager/local_api/scheduler.rs +++ b/src/rust/lqosd/src/node_manager/local_api/scheduler.rs @@ -4,8 +4,8 @@ use lqos_bus::SchedulerProgressReport; use crate::node_manager::runtime_onboarding::runtime_onboarding_state; use crate::tool_status::{ - is_scheduler_available, scheduler_error_message, scheduler_output_message, - scheduler_progress_state, + is_scheduler_available, scheduler_error_message, scheduler_last_seen_unix, + scheduler_output_message, scheduler_progress_state, }; // Remove ANSI escape sequences (basic CSI/OSC handling) for browser display @@ -64,23 +64,45 @@ fn strip_ansi(input: &str) -> String { out } +/// Snapshot of scheduler availability for the node-manager widget. #[derive(Serialize, Debug, Clone)] pub struct SchedulerStatus { + /// Whether the scheduler has reported in within the last five minutes. pub available: bool, + /// The latest scheduler error, if one was reported. pub error: Option, + /// The scheduler's last reported progress snapshot, if any. pub progress: Option, + /// The best-known Unix timestamp for this scheduler snapshot. + pub updated_unix: Option, + /// Whether this snapshot is older than the scheduler availability window. + pub stale: bool, + /// Whether initial runtime onboarding is blocking scheduler activity. pub setup_required: bool, + /// Operator-facing onboarding guidance when `setup_required` is true. pub setup_message: Option, } +/// Detailed scheduler snapshot for the scheduler modal. #[derive(Serialize, Debug, Clone)] pub struct SchedulerDetails { + /// Whether the scheduler has reported in within the last five minutes. pub available: bool, + /// The latest scheduler error, if one was reported. pub error: Option, + /// The latest non-error scheduler output line, if any. pub output: Option, + /// The scheduler's last reported progress snapshot, if any. pub progress: Option, + /// The best-known Unix timestamp for this scheduler snapshot. + pub updated_unix: Option, + /// Whether this snapshot is older than the scheduler availability window. + pub stale: bool, + /// Whether initial runtime onboarding is blocking scheduler activity. pub setup_required: bool, + /// Operator-facing onboarding guidance when `setup_required` is true. pub setup_message: Option, + /// Human-readable raw details for the operator debug view. pub details: String, } @@ -106,6 +128,19 @@ fn scheduler_output() -> Option { }) } +fn scheduler_updated_unix(progress: Option<&SchedulerProgressReport>) -> Option { + let progress_updated = progress.and_then(|report| report.updated_unix); + match (progress_updated, scheduler_last_seen_unix()) { + (Some(progress_updated), Some(last_seen_updated)) => { + Some(progress_updated.max(last_seen_updated)) + } + (Some(progress_updated), None) => Some(progress_updated), + (None, Some(last_seen_updated)) => Some(last_seen_updated), + (None, None) => None, + } +} + +/// Builds the lightweight scheduler status snapshot for node-manager consumers. pub fn scheduler_status_data() -> SchedulerStatus { let onboarding = runtime_onboarding_state(); if onboarding.required { @@ -113,6 +148,8 @@ pub fn scheduler_status_data() -> SchedulerStatus { available: false, error: None, progress: None, + updated_unix: None, + stale: false, setup_required: true, setup_message: Some(onboarding.summary), }; @@ -121,15 +158,19 @@ pub fn scheduler_status_data() -> SchedulerStatus { let available = is_scheduler_available(); let error = scheduler_error(); let progress = scheduler_progress_state(); + let updated_unix = scheduler_updated_unix(progress.as_ref()); SchedulerStatus { available, error, progress, + updated_unix, + stale: updated_unix.is_some() && !available, setup_required: false, setup_message: None, } } +/// Builds the detailed scheduler snapshot for the scheduler modal. pub fn scheduler_details_data() -> SchedulerDetails { let status = scheduler_status_data(); let output = scheduler_output(); @@ -148,6 +189,8 @@ pub fn scheduler_details_data() -> SchedulerDetails { error: None, output: None, progress: None, + updated_unix: None, + stale: false, setup_required: true, setup_message: Some(message), details: body, @@ -155,6 +198,12 @@ pub fn scheduler_details_data() -> SchedulerDetails { } body.push_str(&format!("Scheduler available: {}\n\n", status.available)); + if let Some(updated_unix) = status.updated_unix { + body.push_str(&format!("Last updated Unix: {updated_unix}\n")); + } else { + body.push_str("Last updated Unix: unavailable\n"); + } + body.push_str(&format!("Snapshot stale: {}\n\n", status.stale)); match status.progress.as_ref() { Some(progress) => { body.push_str("Current progress:\n"); @@ -203,6 +252,8 @@ pub fn scheduler_details_data() -> SchedulerDetails { error: status.error, output, progress: status.progress, + updated_unix: status.updated_unix, + stale: status.stale, setup_required: false, setup_message: None, details: body, @@ -214,7 +265,8 @@ mod tests { use super::{scheduler_details_data, scheduler_status_data}; use crate::test_support::runtime_config_test_lock; use crate::tool_status::{ - scheduler_error, scheduler_output, scheduler_progress, scheduler_seen, + scheduler_error, scheduler_last_seen_unix, scheduler_output, scheduler_progress, + scheduler_seen, }; use lqos_bus::SchedulerProgressReport; use std::ffi::OsString; @@ -324,6 +376,9 @@ mod tests { let status = scheduler_status_data(); let details = scheduler_details_data(); + let expected_updated_unix = scheduler_last_seen_unix() + .map(|last_seen_unix| last_seen_unix.max(1_234_567_890)) + .expect("scheduler last seen should be set"); assert!(status.available); assert_eq!( @@ -337,6 +392,8 @@ mod tests { .map(|progress| progress.phase.as_str()), Some("validation_failed") ); + assert_eq!(status.updated_unix, Some(expected_updated_unix)); + assert!(!status.stale); assert!(!status.setup_required); assert!(details.available); @@ -348,7 +405,27 @@ mod tests { details.output.as_deref(), Some("Scheduled shaping refresh blocked by validation: duplicate IPv4") ); + assert_eq!(details.updated_unix, Some(expected_updated_unix)); + assert!(!details.stale); assert!(details.details.contains("Reported error:")); + assert!(details.details.contains("Snapshot stale: false")); assert!(details.details.contains("Scheduler validation failed")); } + + #[test] + fn scheduler_status_uses_last_seen_when_progress_timestamp_is_missing() { + let _context = SchedulerStatusTestContext::new("last-seen-fallback"); + scheduler_seen(); + + let status = scheduler_status_data(); + let details = scheduler_details_data(); + let last_seen_unix = scheduler_last_seen_unix().expect("scheduler last seen should be set"); + + assert_eq!(status.updated_unix, Some(last_seen_unix)); + assert!(!status.stale); + assert_eq!(details.updated_unix, Some(last_seen_unix)); + assert!(details + .details + .contains(&format!("Last updated Unix: {last_seen_unix}"))); + } } diff --git a/src/rust/lqosd/src/tool_status.rs b/src/rust/lqosd/src/tool_status.rs index 3c15ac58..7528b55e 100644 --- a/src/rust/lqosd/src/tool_status.rs +++ b/src/rust/lqosd/src/tool_status.rs @@ -83,6 +83,16 @@ pub fn is_scheduler_available() -> bool { now.saturating_sub(last) < 300 } +/// Returns the Unix timestamp of the scheduler's last heartbeat, if one has been seen. +pub fn scheduler_last_seen_unix() -> Option { + let last = SCHEDULER_LAST_SEEN.load(Ordering::Relaxed); + if last == 0 { + None + } else { + Some(last) + } +} + /// Returns the current scheduler error message, if any. pub fn scheduler_error_message() -> Option { let guard: parking_lot::lock_api::MutexGuard<'_, parking_lot::RawMutex, Option> =