This commit is contained in:
Name committed 2026-04-23 20:41:01 -06:00
1 parent 24320563e3
commit 3597bd53ba
10 files changed
+585 -88

No files matched your search

+497 -69
View File
@@ -27,6 +27,7 @@ mod utils;
use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
use parking_lot::RwLock;
use std::cmp::Reverse;
use std::collections::VecDeque;
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::path::{Path, PathBuf};
@@ -1742,6 +1743,10 @@ pub struct BakeryStatusSnapshot {
pub reload_required: bool,
/// Operator-facing reason for why a full reload is now required, if any.
pub reload_required_reason: Option<String>,
/// True when Bakery has put traffic into pass-through mode to avoid stale TC classification.
pub passthrough_degraded: bool,
/// Operator-facing reason for pass-through mode, if active.
pub passthrough_degraded_reason: Option<String>,
/// Number of runtime node operations currently marked dirty.
pub dirty_subtree_count: usize,
}
@@ -1789,6 +1794,8 @@ struct BakeryTelemetryState {
preflight: Option<BakeryPreflightSnapshot>,
reload_required: bool,
reload_required_reason: Option<String>,
passthrough_degraded: bool,
passthrough_degraded_reason: Option<String>,
dirty_subtree_count: usize,
runtime_operations_by_site: HashMap<i64, RuntimeNodeOperationSnapshot>,
runtime_branch_states_by_site: HashMap<i64, BakeryRuntimeNodeBranchSnapshot>,
@@ -1904,6 +1911,8 @@ impl Default for BakeryTelemetryState {
preflight: None,
reload_required: false,
reload_required_reason: None,
passthrough_degraded: false,
passthrough_degraded_reason: None,
dirty_subtree_count: 0,
runtime_operations_by_site: HashMap::new(),
runtime_branch_states_by_site: HashMap::new(),
@@ -1950,6 +1959,11 @@ struct BakeryFullReloadBatchResult {
qdisc_commands: usize,
}
struct ProcessBatchOptions<'a> {
extra_reserved_handles: &'a HashMap<String, HashSet<u16>>,
preserved_root_child_majors: &'a HashMap<String, HashSet<u16>>,
}
impl BakeryFullReloadBatchResult {
fn metrics<'a>(&self, summary: &'a str, ok: bool) -> BakeryApplyMetrics<'a> {
BakeryApplyMetrics {
@@ -1978,6 +1992,10 @@ impl TcClassifyBypassGuard {
error
})?;
info!("Bakery full reload enabled TC classify bypass");
mark_passthrough_degraded(
"Bakery full reload is rebuilding TC. Traffic is temporarily passing without shaping to avoid stale class-handle classification.",
false,
);
Ok(Self {
cleanup_on_drop: true,
setter,
@@ -1993,9 +2011,15 @@ impl TcClassifyBypassGuard {
error!(
"Bakery full reload failed to disable TC classify bypass via tc_classify_control: {error}"
);
mark_passthrough_degraded(format!(
"Bakery full reload applied TC commands but could not disable pass-through mode: {error}. Traffic may remain unshaped until operator intervention or a later successful reload."
), true);
error
})?;
info!("Bakery full reload disabled TC classify bypass");
clear_passthrough_degraded(
"Bakery full reload disabled pass-through mode; normal shaping is active.",
);
Ok(())
}
@@ -2005,6 +2029,12 @@ impl TcClassifyBypassGuard {
"Bakery full reload keeping TC classify bypass enabled after failure: {}",
summary
);
mark_passthrough_degraded(
format!(
"{summary}. Traffic is passing without shaping until a successful full reload clears TC classify bypass."
),
true,
);
}
}
@@ -2101,6 +2131,39 @@ fn mark_reload_required(summary: String) {
}
}
fn mark_passthrough_degraded(summary: impl Into<String>, emit_activity: bool) {
let summary = summary.into();
let mut should_emit = false;
{
let mut state = telemetry_state().write();
if !state.passthrough_degraded
|| state.passthrough_degraded_reason.as_deref() != Some(summary.as_str())
{
state.passthrough_degraded = true;
state.passthrough_degraded_reason = Some(summary.clone());
should_emit = true;
}
}
if should_emit && emit_activity {
push_bakery_event("passthrough_degraded", "warning", summary);
}
}
fn clear_passthrough_degraded(summary: &str) {
let mut should_emit = false;
{
let mut state = telemetry_state().write();
if state.passthrough_degraded || state.passthrough_degraded_reason.is_some() {
state.passthrough_degraded = false;
state.passthrough_degraded_reason = None;
should_emit = true;
}
}
if should_emit {
push_bakery_event("passthrough_degraded_cleared", "info", summary.to_string());
}
}
fn is_live_migration_reload_required_reason(reason: &str) -> bool {
reason.starts_with("Bakery live-move ") || reason.starts_with("Bakery live migration ")
}
@@ -2239,6 +2302,8 @@ pub fn bakery_status_snapshot() -> BakeryStatusSnapshot {
preflight: state.preflight,
reload_required: state.reload_required,
reload_required_reason: state.reload_required_reason,
passthrough_degraded: state.passthrough_degraded,
passthrough_degraded_reason: state.passthrough_degraded_reason,
dirty_subtree_count: state.dirty_subtree_count,
}
}
@@ -2749,42 +2814,228 @@ fn root_mq_add_command(interface_name: &str) -> Vec<String> {
]
}
fn prepare_root_mq_for_full_reload(config: &Arc<Config>) -> Result<(), String> {
fn command_token_after<'a>(command: &'a [String], needle: &str) -> Option<&'a str> {
command
.windows(2)
.find(|tokens| tokens[0] == needle)
.map(|tokens| tokens[1].as_str())
}
fn command_handle_after(command: &[String], needle: &str) -> Option<TcHandle> {
command_token_after(command, needle).and_then(|token| TcHandle::from_string(token).ok())
}
fn command_interface(command: &[String]) -> Option<&str> {
command_token_after(command, "dev")
}
fn root_infra_class_handle(handle: TcHandle, root_child_majors: &HashSet<u16>) -> bool {
let (major, minor) = handle.get_major_minor();
root_child_majors.contains(&major) && matches!(minor, 1 | 2)
}
fn root_infra_class_entry(entry: &LiveTcClassEntry, root_child_majors: &HashSet<u16>) -> bool {
let (major, minor) = entry.class_id.get_major_minor();
if !root_child_majors.contains(&major) {
return false;
}
match (minor, entry.parent) {
(1, Some(parent)) => parent == tc_handle_from_major_minor(major, 0),
(2, Some(parent)) => parent == tc_handle_from_major_minor(major, 1),
_ => false,
}
}
fn root_infra_qdisc_entry(entry: &LiveTcQdiscEntry, root_child_majors: &HashSet<u16>) -> bool {
if entry.is_root {
return true;
}
if entry
.parent
.is_some_and(|parent| parent.get_major_minor().0 == ROOT_MQ_MAJOR)
{
return true;
}
entry.parent.is_some_and(|parent| {
let (major, minor) = parent.get_major_minor();
root_child_majors.contains(&major) && matches!(minor, 1 | 2)
})
}
fn root_infra_class_command(
command: &[String],
preserved_root_child_majors: &HashMap<String, HashSet<u16>>,
) -> bool {
if !matches!(command.first().map(String::as_str), Some("class")) {
return false;
}
let Some(interface) = command_interface(command) else {
return false;
};
let Some(root_child_majors) = preserved_root_child_majors.get(interface) else {
return false;
};
command_handle_after(command, "classid")
.is_some_and(|handle| root_infra_class_handle(handle, root_child_majors))
}
fn root_infra_qdisc_command(
command: &[String],
preserved_root_child_majors: &HashMap<String, HashSet<u16>>,
) -> bool {
if !matches!(command.first().map(String::as_str), Some("qdisc")) {
return false;
}
let Some(interface) = command_interface(command) else {
return false;
};
let Some(root_child_majors) = preserved_root_child_majors.get(interface) else {
return false;
};
command_handle_after(command, "parent").is_some_and(|parent| {
let (major, minor) = parent.get_major_minor();
major == ROOT_MQ_MAJOR || root_child_majors.contains(&major) && matches!(minor, 1 | 2)
})
}
fn idempotent_full_reload_command(
mut command: Vec<String>,
preserved_root_child_majors: &HashMap<String, HashSet<u16>>,
) -> Vec<String> {
if matches!(command.get(1).map(String::as_str), Some("add"))
&& root_infra_class_command(&command, preserved_root_child_majors)
{
command[1] = "replace".to_string();
}
command
}
fn class_depth(
handle: TcHandle,
snapshot: &HashMap<TcHandle, LiveTcClassEntry>,
seen: &mut HashSet<TcHandle>,
) -> usize {
if !seen.insert(handle) {
return 0;
}
snapshot
.get(&handle)
.and_then(|entry| entry.parent)
.map(|parent| 1 + class_depth(parent, snapshot, seen))
.unwrap_or(0)
}
fn class_delete_commands(
interface: &str,
class_snapshot: &HashMap<TcHandle, LiveTcClassEntry>,
root_child_majors: &HashSet<u16>,
) -> Vec<Vec<String>> {
let mut entries = class_snapshot
.values()
.filter(|entry| {
entry.class_id.get_major_minor().0 != ROOT_MQ_MAJOR
&& !root_infra_class_entry(entry, root_child_majors)
})
.filter_map(|entry| {
entry.parent.map(|parent| {
let mut seen = HashSet::new();
(
class_depth(entry.class_id, class_snapshot, &mut seen),
entry.class_id,
parent,
)
})
})
.collect::<Vec<_>>();
entries.sort_by_key(|entry| Reverse(entry.0));
entries
.into_iter()
.map(|(_, class_id, parent)| {
vec![
"class".to_string(),
"del".to_string(),
"dev".to_string(),
interface.to_string(),
"parent".to_string(),
parent.as_tc_string(),
"classid".to_string(),
class_id.as_tc_string(),
]
})
.collect()
}
fn qdisc_delete_commands(
interface: &str,
qdisc_snapshot: &[LiveTcQdiscEntry],
root_child_majors: &HashSet<u16>,
) -> Vec<Vec<String>> {
qdisc_snapshot
.iter()
.filter(|entry| !root_infra_qdisc_entry(entry, root_child_majors))
.filter_map(|entry| entry.parent)
.map(|parent| {
vec![
"qdisc".to_string(),
"del".to_string(),
"dev".to_string(),
interface.to_string(),
"parent".to_string(),
parent.as_tc_string(),
]
})
.collect()
}
fn drain_managed_tree_to_defaults(
interface: &str,
qdisc_snapshot: &[LiveTcQdiscEntry],
) -> Result<HashSet<u16>, String> {
let root_child_majors = managed_root_child_parent_handles(qdisc_snapshot)
.into_iter()
.map(|parent| parent.get_major_minor().1)
.collect::<HashSet<_>>();
if root_child_majors.is_empty() {
return Ok(root_child_majors);
}
let class_snapshot = read_live_class_snapshot(interface)?;
let mut commands = qdisc_delete_commands(interface, qdisc_snapshot, &root_child_majors);
commands.extend(class_delete_commands(
interface,
&class_snapshot,
&root_child_majors,
));
if commands.is_empty() {
return Ok(root_child_majors);
}
run_root_preflight_commands(
&commands,
&format!("full reload drain managed tree to defaults on {interface}"),
)?;
invalidate_live_tc_snapshots();
Ok(root_child_majors)
}
fn prepare_root_mq_for_full_reload(
config: &Arc<Config>,
) -> Result<HashMap<String, HashSet<u16>>, String> {
let mut preserved_root_child_majors = HashMap::new();
for interface in managed_interfaces_for_config(config) {
let qdisc_snapshot = read_live_qdisc_snapshot(&interface)?;
if retained_root_mq_entry(&qdisc_snapshot).is_some() {
let prune_commands = managed_root_child_parent_handles(&qdisc_snapshot)
.into_iter()
.map(|parent| {
vec![
"qdisc".to_string(),
"del".to_string(),
"dev".to_string(),
interface.clone(),
"parent".to_string(),
parent.as_tc_string(),
]
})
.collect::<Vec<_>>();
if !prune_commands.is_empty() {
run_root_preflight_commands(
&prune_commands,
&format!("full reload retained-root child prune on {interface}"),
)?;
invalidate_live_tc_snapshots();
}
let pruned_qdisc_snapshot = read_live_qdisc_snapshot(&interface)?;
let pruned_class_snapshot = read_live_class_snapshot(&interface)?;
if verify_clean_root_child_tree(
&pruned_qdisc_snapshot,
&pruned_class_snapshot,
&interface,
)
.is_ok()
{
continue;
let has_managed_root_children =
!managed_root_child_parent_handles(&qdisc_snapshot).is_empty();
if has_managed_root_children {
let root_child_majors =
drain_managed_tree_to_defaults(&interface, &qdisc_snapshot)?;
if !root_child_majors.is_empty() {
preserved_root_child_majors.insert(interface.clone(), root_child_majors);
}
}
info!(
"Bakery: preserving existing root mq/default path on {interface} during full reload"
);
continue;
}
let replace_summary = run_root_preflight_commands(
@@ -2832,7 +3083,7 @@ fn prepare_root_mq_for_full_reload(config: &Arc<Config>) -> Result<(), String> {
)?;
}
Ok(())
Ok(preserved_root_child_majors)
}
fn live_tree_mutations_allowed(config: &Arc<Config>) -> bool {
@@ -10562,26 +10813,29 @@ fn full_reload(
let previous_mq_created = MQ_CREATED.load(Ordering::Relaxed);
let previous_shaping_tree_active = SHAPING_TREE_ACTIVE.load(Ordering::Relaxed);
if let Err(error) = prepare_root_mq_for_full_reload(config) {
let summary = format!("Failed to prepare root mq state before full reload: {error}");
error!("{summary}");
tc_bypass_guard.keep_bypass_enabled_after_failure(&summary);
mark_reload_required(format!(
"{summary}. TC classify bypass remains enabled to avoid stale class-handle classification until a successful reload or operator intervention."
));
mark_bakery_action_finished(BakeryApplyMetrics {
apply_type: BakeryApplyType::FullReload,
summary: &summary,
build_duration_ms: 0,
apply_duration_ms: 0,
total_tc_commands: 0,
class_commands: 0,
qdisc_commands: 0,
ok: false,
});
*batch = None;
return;
}
let preserved_root_child_majors = match prepare_root_mq_for_full_reload(config) {
Ok(preserved_root_child_majors) => preserved_root_child_majors,
Err(error) => {
let summary = format!("Failed to prepare root mq state before full reload: {error}");
error!("{summary}");
tc_bypass_guard.keep_bypass_enabled_after_failure(&summary);
mark_reload_required(format!(
"{summary}. TC classify bypass remains enabled to avoid stale class-handle classification until a successful reload or operator intervention."
));
mark_bakery_action_finished(BakeryApplyMetrics {
apply_type: BakeryApplyType::FullReload,
summary: &summary,
build_duration_ms: 0,
apply_duration_ms: 0,
total_tc_commands: 0,
class_commands: 0,
qdisc_commands: 0,
ok: false,
});
*batch = None;
return;
}
};
invalidate_live_tc_snapshots();
MQ_CREATED.store(true, Ordering::Relaxed);
@@ -10625,7 +10879,10 @@ fn full_reload(
&mut working_circuits,
&layout,
&mut working_qdisc_handles,
&live_reserved_handles,
ProcessBatchOptions {
extra_reserved_handles: &live_reserved_handles,
preserved_root_child_majors: &preserved_root_child_majors,
},
);
if batch_result.result.ok {
@@ -10686,7 +10943,7 @@ fn process_batch(
circuits: &mut HashMap<i64, Arc<BakeryCommands>>,
mq_layout: &MqDeviceLayout,
qdisc_handles: &mut QdiscHandleState,
extra_reserved_handles: &HashMap<String, HashSet<u16>>,
options: ProcessBatchOptions<'_>,
) -> BakeryFullReloadBatchResult {
info!("Bakery: Processing batch of {} commands", batch.len());
update_bakery_apply_progress(Some("Building tc command batch"), 0, 0, 0, 0);
@@ -10700,7 +10957,7 @@ fn process_batch(
config,
mq_layout,
qdisc_handles,
extra_reserved_handles,
options.extra_reserved_handles,
)
})
.filter_map(|b| {
@@ -10718,6 +10975,8 @@ fn process_batch(
b.to_commands(config, ExecutionMode::Builder)
})
.flatten()
.map(|command| idempotent_full_reload_command(command, options.preserved_root_child_majors))
.filter(|command| !root_infra_qdisc_command(command, options.preserved_root_child_majors))
.collect::<Vec<Vec<String>>>();
let path = config.debug_state_file_path("linux_tc_rust.txt");
@@ -10814,21 +11073,34 @@ mod tests {
static TC_BYPASS_SETTER_CALLS: Mutex<Vec<bool>> = Mutex::new(Vec::new());
fn recording_tc_bypass_setter(enabled: bool) -> anyhow::Result<()> {
TC_BYPASS_SETTER_CALLS.lock().expect("record setter call").push(enabled);
TC_BYPASS_SETTER_CALLS
.lock()
.expect("record setter call")
.push(enabled);
Ok(())
}
fn failing_tc_bypass_setter(enabled: bool) -> anyhow::Result<()> {
TC_BYPASS_SETTER_CALLS.lock().expect("record setter call").push(enabled);
Err(anyhow::anyhow!("synthetic TC classify bypass setter failure"))
TC_BYPASS_SETTER_CALLS
.lock()
.expect("record setter call")
.push(enabled);
Err(anyhow::anyhow!(
"synthetic TC classify bypass setter failure"
))
}
fn disable_failing_tc_bypass_setter(enabled: bool) -> anyhow::Result<()> {
TC_BYPASS_SETTER_CALLS.lock().expect("record setter call").push(enabled);
TC_BYPASS_SETTER_CALLS
.lock()
.expect("record setter call")
.push(enabled);
if enabled {
Ok(())
} else {
Err(anyhow::anyhow!("synthetic TC classify bypass disable failure"))
Err(anyhow::anyhow!(
"synthetic TC classify bypass disable failure"
))
}
}
@@ -10890,21 +11162,22 @@ mod tests {
reset_bakery_test_state();
{
let _bypass_guard =
TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter)
.expect("enable bypass guard");
let _bypass_guard = TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter)
.expect("enable bypass guard");
assert_eq!(take_tc_bypass_setter_calls(), vec![true]);
}
assert_eq!(take_tc_bypass_setter_calls(), vec![false]);
let status = bakery_status_snapshot();
assert!(!status.passthrough_degraded);
assert!(status.passthrough_degraded_reason.is_none());
}
#[test]
fn tc_classify_bypass_guard_disables_on_early_return() {
fn early_return() -> Result<(), ()> {
let _bypass_guard =
TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter)
.map_err(|_| ())?;
TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter).map_err(|_| ())?;
Err(())
}
@@ -10924,6 +11197,9 @@ mod tests {
assert!(TcClassifyBypassGuard::with_setter(failing_tc_bypass_setter).is_err());
assert_eq!(take_tc_bypass_setter_calls(), vec![true]);
let status = bakery_status_snapshot();
assert!(!status.passthrough_degraded);
assert!(status.passthrough_degraded_reason.is_none());
}
#[test]
@@ -10939,6 +11215,15 @@ mod tests {
}
assert_eq!(take_tc_bypass_setter_calls(), vec![true, false]);
let status = bakery_status_snapshot();
assert!(status.passthrough_degraded);
assert!(
status
.passthrough_degraded_reason
.as_deref()
.unwrap_or_default()
.contains("could not disable pass-through mode")
);
}
#[test]
@@ -10947,13 +11232,21 @@ mod tests {
reset_bakery_test_state();
{
let mut bypass_guard =
TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter)
.expect("enable bypass guard");
let mut bypass_guard = TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter)
.expect("enable bypass guard");
bypass_guard.keep_bypass_enabled_after_failure("test failure after TC mutation");
}
assert_eq!(take_tc_bypass_setter_calls(), vec![true]);
let status = bakery_status_snapshot();
assert!(status.passthrough_degraded);
assert!(
status
.passthrough_degraded_reason
.as_deref()
.unwrap_or_default()
.contains("Traffic is passing without shaping")
);
}
fn mk_add_circuit(hash: i64, ip_addresses: &str) -> Arc<BakeryCommands> {
@@ -13193,6 +13486,141 @@ mod tests {
);
}
#[test]
fn idempotent_full_reload_command_replaces_class_adds() {
let root_child_majors = HashMap::from([("eth0".to_string(), HashSet::from([0x1]))]);
assert_eq!(
idempotent_full_reload_command(
vec![
"class".to_string(),
"add".to_string(),
"dev".to_string(),
"eth0".to_string(),
"classid".to_string(),
"0x1:1".to_string(),
],
&root_child_majors,
),
vec![
"class".to_string(),
"replace".to_string(),
"dev".to_string(),
"eth0".to_string(),
"classid".to_string(),
"0x1:1".to_string(),
]
);
}
#[test]
fn idempotent_full_reload_command_leaves_non_add_commands_unchanged() {
let root_child_majors = HashMap::from([("eth0".to_string(), HashSet::from([0x1]))]);
assert_eq!(
idempotent_full_reload_command(
vec![
"qdisc".to_string(),
"add".to_string(),
"dev".to_string(),
"eth0".to_string(),
],
&root_child_majors,
),
vec![
"qdisc".to_string(),
"add".to_string(),
"dev".to_string(),
"eth0".to_string(),
]
);
assert_eq!(
idempotent_full_reload_command(
vec![
"class".to_string(),
"del".to_string(),
"dev".to_string(),
"eth0".to_string(),
],
&root_child_majors,
),
vec![
"class".to_string(),
"del".to_string(),
"dev".to_string(),
"eth0".to_string(),
]
);
assert_eq!(
idempotent_full_reload_command(
vec![
"filter".to_string(),
"add".to_string(),
"dev".to_string(),
"eth0".to_string(),
],
&root_child_majors,
),
vec![
"filter".to_string(),
"add".to_string(),
"dev".to_string(),
"eth0".to_string(),
]
);
}
#[test]
fn root_infra_qdisc_command_matches_root_and_default_qdiscs() {
let root_child_majors = HashMap::from([("eth0".to_string(), HashSet::from([0x1]))]);
assert!(root_infra_qdisc_command(
&[
"qdisc".to_string(),
"add".to_string(),
"dev".to_string(),
"eth0".to_string(),
"parent".to_string(),
"7FFF:0x1".to_string()
],
&root_child_majors,
));
assert!(root_infra_qdisc_command(
&[
"qdisc".to_string(),
"add".to_string(),
"dev".to_string(),
"eth0".to_string(),
"parent".to_string(),
"0x1:2".to_string()
],
&root_child_majors,
));
assert!(!root_infra_qdisc_command(
&[
"qdisc".to_string(),
"add".to_string(),
"dev".to_string(),
"eth0".to_string(),
"parent".to_string(),
"0x1:0x20".to_string()
],
&root_child_majors,
));
assert!(!root_infra_qdisc_command(
&[
"qdisc".to_string(),
"add".to_string(),
"dev".to_string(),
"eth0".to_string(),
"parent".to_string(),
"0x3:2".to_string()
],
&root_child_majors,
));
assert!(!root_infra_qdisc_command(
&["class".to_string(), "replace".to_string()],
&root_child_majors,
));
}
#[test]
fn grouped_runtime_site_events_retain_human_site_name() {
let _guard = bakery_test_lock().lock().expect("lock");
+3
View File
@@ -109,6 +109,9 @@ int xdp_prog(struct xdp_md *ctx)
bpf_debug("Error: interface direction unspecified, aborting.");
return XDP_PASS;
}
if (tc_classify_bypass_enabled()) {
return XDP_PASS;
}
// Do we need to perform a VLAN redirect?
bool vlan_redirect = false;
+6 -4
View File
@@ -1,4 +1,4 @@
//! Control-map helpers for temporarily bypassing TC classification.
//! Control-map helpers for temporarily bypassing XDP redirection and TC classification.
use crate::lqos_kernel::bpf::libbpf_num_possible_cpus;
use anyhow::{Error, Result};
@@ -19,7 +19,8 @@ const PER_CPU_VALUE_ALIGNMENT: usize = 8;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
#[repr(C)]
pub struct TcClassifyControl {
/// Non-zero tells TC classify to pass packets without setting TC handles.
/// Non-zero tells XDP and TC classify to pass packets without redirecting
/// or setting TC handles.
pub bypass: u32,
}
@@ -109,8 +110,9 @@ pub fn initialize_tc_classify_bypass() -> Result<()> {
/// Enables or disables TC classify bypass.
///
/// This function has side effects: it writes all per-CPU values for the pinned
/// `tc_classify_control` BPF map. When enabled, TC classify returns `TC_ACT_OK`
/// before consuming XDP metadata, flow cache, hot cache, or LPM mappings.
/// `tc_classify_control` BPF map. When enabled, XDP returns `XDP_PASS` before
/// CPU redirection, and TC classify returns `TC_ACT_OK` before consuming XDP
/// metadata, flow cache, hot cache, or LPM mappings.
pub fn set_tc_classify_bypass(enabled: bool) -> Result<()> {
update_tc_classify_control(enabled)
}
@@ -86,12 +86,12 @@ function classifyEvent(entry) {
if (event.endsWith("_started")) {
outcome = "Running";
outcomeClass = "bg-primary-subtle text-primary border border-primary-subtle";
} else if (event === "reload_required") {
} else if (event === "reload_required" || event === "passthrough_degraded") {
stage = "Verify";
scope = "Full Reload";
outcome = "Reload Required";
outcome = event === "passthrough_degraded" ? "Pass-Through" : "Reload Required";
outcomeClass = "bg-danger-subtle text-danger border border-danger-subtle";
} else if (event === "reload_required_cleared") {
} else if (event === "reload_required_cleared" || event === "passthrough_degraded_cleared") {
stage = "Verify";
scope = "Full Reload";
outcome = "Cleared";
@@ -182,6 +182,8 @@ function describeOperation(entry, meta) {
|| event === "full_reload_started"
|| event === "reload_required"
|| event === "reload_required_cleared"
|| event === "passthrough_degraded"
|| event === "passthrough_degraded_cleared"
|| meta.scope === "Full Reload"
) {
return {
@@ -16,12 +16,18 @@ function classifyOutcome(entry) {
const event = (entry?.event ?? "").toString().trim().toLowerCase();
const status = (entry?.status ?? "").toString().trim().toLowerCase();
if (event === "reload_required") {
if (event === "reload_required" || event === "passthrough_degraded") {
return {
label: "Reload Required",
label: event === "passthrough_degraded" ? "Pass-Through" : "Reload Required",
className: "bg-danger-subtle text-danger border border-danger-subtle",
};
}
if (event === "passthrough_degraded_cleared") {
return {
label: "Shaping Restored",
className: "bg-success-subtle text-success border border-success-subtle",
};
}
if (event === "apply_failed" || status === "error") {
return {
label: "Failed",
@@ -58,6 +58,10 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
this.lastFailureSummaryEl = document.createElement("span");
this.lastFailureSummaryEl.style.whiteSpace = "normal";
this.lastFailureSummaryEl.style.wordBreak = "break-word";
this.shapingStateEl = document.createElement("div");
this.passthroughReasonEl = document.createElement("span");
this.passthroughReasonEl.style.whiteSpace = "normal";
this.passthroughReasonEl.style.wordBreak = "break-word";
this.reloadRequiredEl = document.createElement("div");
this.reloadReasonEl = document.createElement("span");
this.reloadReasonEl.style.whiteSpace = "normal";
@@ -65,10 +69,12 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
tbody.appendChild(mkRow("Current Mode", this.modeEl));
tbody.appendChild(mkRow("Current Duration", this.durationEl));
tbody.appendChild(mkRow("Shaping State", this.shapingStateEl));
tbody.appendChild(mkRow("Runtime Drift", this.reloadRequiredEl));
tbody.appendChild(mkRow("Last Success", this.lastSuccessEl));
tbody.appendChild(mkRow("Last Failure", this.lastFailureEl));
tbody.appendChild(mkRow("Failure Summary", this.lastFailureSummaryEl));
tbody.appendChild(mkRow("Pass-Through Reason", this.passthroughReasonEl));
tbody.appendChild(mkRow("Reload Reason", this.reloadReasonEl));
table.appendChild(tbody);
@@ -91,6 +97,12 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
this.modeEl.innerHTML = "";
this.modeEl.appendChild(bakeryModeBadge(status.mode));
this.shapingStateEl.innerHTML = "";
this.shapingStateEl.appendChild(
status.passthroughDegraded
? mkBadge("Pass-Through / Unshaped", "bg-danger-subtle text-danger border border-danger-subtle")
: mkBadge("Normal Shaping", "bg-success-subtle text-success border border-success-subtle"),
);
this.reloadRequiredEl.innerHTML = "";
this.reloadRequiredEl.appendChild(
status.reloadRequired
@@ -104,6 +116,7 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
this.lastSuccessEl.textContent = formatUnixSecondsToLocalDateTime(status.lastSuccessUnix);
this.lastFailureEl.textContent = formatUnixSecondsToLocalDateTime(status.lastFailureUnix);
this.lastFailureSummaryEl.textContent = status.lastFailureSummary || "—";
this.passthroughReasonEl.textContent = status.passthroughDegradedReason || "—";
this.reloadReasonEl.textContent = status.reloadRequiredReason || "—";
}
}
@@ -1159,13 +1159,16 @@ function initUrgentIssues() {
const td = document.createElement('td');
const when = new Date(it.ts * 1000).toLocaleString();
const sev = it.severity === 'Error' ? 'danger' : 'warning';
const clearControl = it.clearable === false
? '<span class="badge bg-secondary-subtle text-secondary border float-end ms-3">Active</span>'
: `<button type="button" class="btn btn-link btn-sm text-secondary float-end ms-3 p-0 urgent-clear" data-id="${it.id}" title="Acknowledge issue" aria-label="Acknowledge issue ${escapeAttr(it.code)}"><i class="fa fa-times" aria-hidden="true"></i></button>`;
td.innerHTML = `
<div>
<span class="badge bg-${sev}">${it.severity}</span>
<strong class="ms-2">${it.code}</strong>
<span class="text-muted ms-2">(${it.source})</span>
<span class="text-muted float-end">${when}</span>
<button type="button" class="btn btn-link btn-sm text-secondary float-end ms-3 p-0 urgent-clear" data-id="${it.id}" title="Acknowledge issue" aria-label="Acknowledge issue ${escapeAttr(it.code)}"><i class="fa fa-times" aria-hidden="true"></i></button>
${clearControl}
</div>
<div class="mt-1" style="white-space: pre-wrap;">${it.message}</div>
${it.context ? `<pre class="mt-2">${it.context}</pre>` : ''}
@@ -73,6 +73,7 @@ pub struct UrgentItem {
pub code: String,
pub message: String,
pub context: Option<String>,
pub clearable: bool,
}
#[derive(Debug, Serialize)]
@@ -80,17 +81,32 @@ pub struct UrgentList {
pub items: Vec<UrgentItem>,
}
pub fn urgent_status_data() -> UrgentStatus {
let items = urgent::list();
UrgentStatus {
has_urgent: !items.is_empty(),
count: items.len(),
fn bakery_passthrough_issue() -> Option<UrgentItem> {
let status = lqos_bakery::bakery_status_snapshot();
if !status.reload_required || !status.passthrough_degraded {
return None;
}
Some(UrgentItem {
id: u64::MAX,
ts: status
.last_failure_unix
.or(status.current_action_started_unix)
.unwrap_or_default(),
source: "System".to_string(),
severity: "Error".to_string(),
code: "BAKERY_PASSTHROUGH_DEGRADED".to_string(),
message: status.passthrough_degraded_reason.unwrap_or_else(|| {
"Bakery full reload failed while pass-through mode was active. Traffic may be unshaped until a successful full reload clears the degraded state.".to_string()
}),
context: status.reload_required_reason.map(|reason| {
format!("Reload required reason: {reason}")
}),
clearable: false,
})
}
pub fn urgent_list_data() -> UrgentList {
let items = urgent::list();
let items: Vec<UrgentItem> = items
fn merged_urgent_items() -> Vec<UrgentItem> {
let mut items: Vec<UrgentItem> = urgent::list()
.into_iter()
.map(|i| UrgentItem {
id: i.id,
@@ -100,9 +116,29 @@ pub fn urgent_list_data() -> UrgentList {
code: i.code,
message: strip_ansi(&i.message),
context: i.context,
clearable: true,
})
.collect();
UrgentList { items }
if let Some(issue) = bakery_passthrough_issue() {
items.push(issue);
}
items.sort_by_key(|item| item.ts);
items.reverse();
items
}
pub fn urgent_status_data() -> UrgentStatus {
let items = merged_urgent_items();
UrgentStatus {
has_urgent: !items.is_empty(),
count: items.len(),
}
}
pub fn urgent_list_data() -> UrgentList {
UrgentList {
items: merged_urgent_items(),
}
}
pub fn urgent_clear_id(id: u64) -> bool {
@@ -447,6 +447,8 @@ pub struct BakeryStatusState {
pub preflight: Option<BakeryPreflightData>,
pub reload_required: bool,
pub reload_required_reason: Option<String>,
pub passthrough_degraded: bool,
pub passthrough_degraded_reason: Option<String>,
pub dirty_subtree_count: usize,
}
@@ -152,6 +152,8 @@ fn map_status(snapshot: BakeryStatusSnapshot) -> BakeryStatusData {
},
reload_required: snapshot.reload_required,
reload_required_reason: snapshot.reload_required_reason,
passthrough_degraded: snapshot.passthrough_degraded,
passthrough_degraded_reason: snapshot.passthrough_degraded_reason,
dirty_subtree_count: snapshot.dirty_subtree_count,
queue_distribution: snapshot
.queue_distribution