mirror of
https://github.com/LibreQoE/LibreQoS.git
synced 2026-10-25 22:30:05 -05:00
changes
This commit is contained in:
1 parent
24320563e3
commit
3597bd53ba
10 files changed
+585
-88
No files matched your search
+497
-69
@@ -27,6 +27,7 @@ mod utils;
|
||||
|
||||
use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
|
||||
use parking_lot::RwLock;
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::VecDeque;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -1742,6 +1743,10 @@ pub struct BakeryStatusSnapshot {
|
||||
pub reload_required: bool,
|
||||
/// Operator-facing reason for why a full reload is now required, if any.
|
||||
pub reload_required_reason: Option<String>,
|
||||
/// True when Bakery has put traffic into pass-through mode to avoid stale TC classification.
|
||||
pub passthrough_degraded: bool,
|
||||
/// Operator-facing reason for pass-through mode, if active.
|
||||
pub passthrough_degraded_reason: Option<String>,
|
||||
/// Number of runtime node operations currently marked dirty.
|
||||
pub dirty_subtree_count: usize,
|
||||
}
|
||||
@@ -1789,6 +1794,8 @@ struct BakeryTelemetryState {
|
||||
preflight: Option<BakeryPreflightSnapshot>,
|
||||
reload_required: bool,
|
||||
reload_required_reason: Option<String>,
|
||||
passthrough_degraded: bool,
|
||||
passthrough_degraded_reason: Option<String>,
|
||||
dirty_subtree_count: usize,
|
||||
runtime_operations_by_site: HashMap<i64, RuntimeNodeOperationSnapshot>,
|
||||
runtime_branch_states_by_site: HashMap<i64, BakeryRuntimeNodeBranchSnapshot>,
|
||||
@@ -1904,6 +1911,8 @@ impl Default for BakeryTelemetryState {
|
||||
preflight: None,
|
||||
reload_required: false,
|
||||
reload_required_reason: None,
|
||||
passthrough_degraded: false,
|
||||
passthrough_degraded_reason: None,
|
||||
dirty_subtree_count: 0,
|
||||
runtime_operations_by_site: HashMap::new(),
|
||||
runtime_branch_states_by_site: HashMap::new(),
|
||||
@@ -1950,6 +1959,11 @@ struct BakeryFullReloadBatchResult {
|
||||
qdisc_commands: usize,
|
||||
}
|
||||
|
||||
struct ProcessBatchOptions<'a> {
|
||||
extra_reserved_handles: &'a HashMap<String, HashSet<u16>>,
|
||||
preserved_root_child_majors: &'a HashMap<String, HashSet<u16>>,
|
||||
}
|
||||
|
||||
impl BakeryFullReloadBatchResult {
|
||||
fn metrics<'a>(&self, summary: &'a str, ok: bool) -> BakeryApplyMetrics<'a> {
|
||||
BakeryApplyMetrics {
|
||||
@@ -1978,6 +1992,10 @@ impl TcClassifyBypassGuard {
|
||||
error
|
||||
})?;
|
||||
info!("Bakery full reload enabled TC classify bypass");
|
||||
mark_passthrough_degraded(
|
||||
"Bakery full reload is rebuilding TC. Traffic is temporarily passing without shaping to avoid stale class-handle classification.",
|
||||
false,
|
||||
);
|
||||
Ok(Self {
|
||||
cleanup_on_drop: true,
|
||||
setter,
|
||||
@@ -1993,9 +2011,15 @@ impl TcClassifyBypassGuard {
|
||||
error!(
|
||||
"Bakery full reload failed to disable TC classify bypass via tc_classify_control: {error}"
|
||||
);
|
||||
mark_passthrough_degraded(format!(
|
||||
"Bakery full reload applied TC commands but could not disable pass-through mode: {error}. Traffic may remain unshaped until operator intervention or a later successful reload."
|
||||
), true);
|
||||
error
|
||||
})?;
|
||||
info!("Bakery full reload disabled TC classify bypass");
|
||||
clear_passthrough_degraded(
|
||||
"Bakery full reload disabled pass-through mode; normal shaping is active.",
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -2005,6 +2029,12 @@ impl TcClassifyBypassGuard {
|
||||
"Bakery full reload keeping TC classify bypass enabled after failure: {}",
|
||||
summary
|
||||
);
|
||||
mark_passthrough_degraded(
|
||||
format!(
|
||||
"{summary}. Traffic is passing without shaping until a successful full reload clears TC classify bypass."
|
||||
),
|
||||
true,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2101,6 +2131,39 @@ fn mark_reload_required(summary: String) {
|
||||
}
|
||||
}
|
||||
|
||||
fn mark_passthrough_degraded(summary: impl Into<String>, emit_activity: bool) {
|
||||
let summary = summary.into();
|
||||
let mut should_emit = false;
|
||||
{
|
||||
let mut state = telemetry_state().write();
|
||||
if !state.passthrough_degraded
|
||||
|| state.passthrough_degraded_reason.as_deref() != Some(summary.as_str())
|
||||
{
|
||||
state.passthrough_degraded = true;
|
||||
state.passthrough_degraded_reason = Some(summary.clone());
|
||||
should_emit = true;
|
||||
}
|
||||
}
|
||||
if should_emit && emit_activity {
|
||||
push_bakery_event("passthrough_degraded", "warning", summary);
|
||||
}
|
||||
}
|
||||
|
||||
fn clear_passthrough_degraded(summary: &str) {
|
||||
let mut should_emit = false;
|
||||
{
|
||||
let mut state = telemetry_state().write();
|
||||
if state.passthrough_degraded || state.passthrough_degraded_reason.is_some() {
|
||||
state.passthrough_degraded = false;
|
||||
state.passthrough_degraded_reason = None;
|
||||
should_emit = true;
|
||||
}
|
||||
}
|
||||
if should_emit {
|
||||
push_bakery_event("passthrough_degraded_cleared", "info", summary.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
fn is_live_migration_reload_required_reason(reason: &str) -> bool {
|
||||
reason.starts_with("Bakery live-move ") || reason.starts_with("Bakery live migration ")
|
||||
}
|
||||
@@ -2239,6 +2302,8 @@ pub fn bakery_status_snapshot() -> BakeryStatusSnapshot {
|
||||
preflight: state.preflight,
|
||||
reload_required: state.reload_required,
|
||||
reload_required_reason: state.reload_required_reason,
|
||||
passthrough_degraded: state.passthrough_degraded,
|
||||
passthrough_degraded_reason: state.passthrough_degraded_reason,
|
||||
dirty_subtree_count: state.dirty_subtree_count,
|
||||
}
|
||||
}
|
||||
@@ -2749,42 +2814,228 @@ fn root_mq_add_command(interface_name: &str) -> Vec<String> {
|
||||
]
|
||||
}
|
||||
|
||||
fn prepare_root_mq_for_full_reload(config: &Arc<Config>) -> Result<(), String> {
|
||||
fn command_token_after<'a>(command: &'a [String], needle: &str) -> Option<&'a str> {
|
||||
command
|
||||
.windows(2)
|
||||
.find(|tokens| tokens[0] == needle)
|
||||
.map(|tokens| tokens[1].as_str())
|
||||
}
|
||||
|
||||
fn command_handle_after(command: &[String], needle: &str) -> Option<TcHandle> {
|
||||
command_token_after(command, needle).and_then(|token| TcHandle::from_string(token).ok())
|
||||
}
|
||||
|
||||
fn command_interface(command: &[String]) -> Option<&str> {
|
||||
command_token_after(command, "dev")
|
||||
}
|
||||
|
||||
fn root_infra_class_handle(handle: TcHandle, root_child_majors: &HashSet<u16>) -> bool {
|
||||
let (major, minor) = handle.get_major_minor();
|
||||
root_child_majors.contains(&major) && matches!(minor, 1 | 2)
|
||||
}
|
||||
|
||||
fn root_infra_class_entry(entry: &LiveTcClassEntry, root_child_majors: &HashSet<u16>) -> bool {
|
||||
let (major, minor) = entry.class_id.get_major_minor();
|
||||
if !root_child_majors.contains(&major) {
|
||||
return false;
|
||||
}
|
||||
match (minor, entry.parent) {
|
||||
(1, Some(parent)) => parent == tc_handle_from_major_minor(major, 0),
|
||||
(2, Some(parent)) => parent == tc_handle_from_major_minor(major, 1),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn root_infra_qdisc_entry(entry: &LiveTcQdiscEntry, root_child_majors: &HashSet<u16>) -> bool {
|
||||
if entry.is_root {
|
||||
return true;
|
||||
}
|
||||
if entry
|
||||
.parent
|
||||
.is_some_and(|parent| parent.get_major_minor().0 == ROOT_MQ_MAJOR)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
entry.parent.is_some_and(|parent| {
|
||||
let (major, minor) = parent.get_major_minor();
|
||||
root_child_majors.contains(&major) && matches!(minor, 1 | 2)
|
||||
})
|
||||
}
|
||||
|
||||
fn root_infra_class_command(
|
||||
command: &[String],
|
||||
preserved_root_child_majors: &HashMap<String, HashSet<u16>>,
|
||||
) -> bool {
|
||||
if !matches!(command.first().map(String::as_str), Some("class")) {
|
||||
return false;
|
||||
}
|
||||
let Some(interface) = command_interface(command) else {
|
||||
return false;
|
||||
};
|
||||
let Some(root_child_majors) = preserved_root_child_majors.get(interface) else {
|
||||
return false;
|
||||
};
|
||||
command_handle_after(command, "classid")
|
||||
.is_some_and(|handle| root_infra_class_handle(handle, root_child_majors))
|
||||
}
|
||||
|
||||
fn root_infra_qdisc_command(
|
||||
command: &[String],
|
||||
preserved_root_child_majors: &HashMap<String, HashSet<u16>>,
|
||||
) -> bool {
|
||||
if !matches!(command.first().map(String::as_str), Some("qdisc")) {
|
||||
return false;
|
||||
}
|
||||
let Some(interface) = command_interface(command) else {
|
||||
return false;
|
||||
};
|
||||
let Some(root_child_majors) = preserved_root_child_majors.get(interface) else {
|
||||
return false;
|
||||
};
|
||||
command_handle_after(command, "parent").is_some_and(|parent| {
|
||||
let (major, minor) = parent.get_major_minor();
|
||||
major == ROOT_MQ_MAJOR || root_child_majors.contains(&major) && matches!(minor, 1 | 2)
|
||||
})
|
||||
}
|
||||
|
||||
fn idempotent_full_reload_command(
|
||||
mut command: Vec<String>,
|
||||
preserved_root_child_majors: &HashMap<String, HashSet<u16>>,
|
||||
) -> Vec<String> {
|
||||
if matches!(command.get(1).map(String::as_str), Some("add"))
|
||||
&& root_infra_class_command(&command, preserved_root_child_majors)
|
||||
{
|
||||
command[1] = "replace".to_string();
|
||||
}
|
||||
command
|
||||
}
|
||||
|
||||
fn class_depth(
|
||||
handle: TcHandle,
|
||||
snapshot: &HashMap<TcHandle, LiveTcClassEntry>,
|
||||
seen: &mut HashSet<TcHandle>,
|
||||
) -> usize {
|
||||
if !seen.insert(handle) {
|
||||
return 0;
|
||||
}
|
||||
snapshot
|
||||
.get(&handle)
|
||||
.and_then(|entry| entry.parent)
|
||||
.map(|parent| 1 + class_depth(parent, snapshot, seen))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn class_delete_commands(
|
||||
interface: &str,
|
||||
class_snapshot: &HashMap<TcHandle, LiveTcClassEntry>,
|
||||
root_child_majors: &HashSet<u16>,
|
||||
) -> Vec<Vec<String>> {
|
||||
let mut entries = class_snapshot
|
||||
.values()
|
||||
.filter(|entry| {
|
||||
entry.class_id.get_major_minor().0 != ROOT_MQ_MAJOR
|
||||
&& !root_infra_class_entry(entry, root_child_majors)
|
||||
})
|
||||
.filter_map(|entry| {
|
||||
entry.parent.map(|parent| {
|
||||
let mut seen = HashSet::new();
|
||||
(
|
||||
class_depth(entry.class_id, class_snapshot, &mut seen),
|
||||
entry.class_id,
|
||||
parent,
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
entries.sort_by_key(|entry| Reverse(entry.0));
|
||||
entries
|
||||
.into_iter()
|
||||
.map(|(_, class_id, parent)| {
|
||||
vec![
|
||||
"class".to_string(),
|
||||
"del".to_string(),
|
||||
"dev".to_string(),
|
||||
interface.to_string(),
|
||||
"parent".to_string(),
|
||||
parent.as_tc_string(),
|
||||
"classid".to_string(),
|
||||
class_id.as_tc_string(),
|
||||
]
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn qdisc_delete_commands(
|
||||
interface: &str,
|
||||
qdisc_snapshot: &[LiveTcQdiscEntry],
|
||||
root_child_majors: &HashSet<u16>,
|
||||
) -> Vec<Vec<String>> {
|
||||
qdisc_snapshot
|
||||
.iter()
|
||||
.filter(|entry| !root_infra_qdisc_entry(entry, root_child_majors))
|
||||
.filter_map(|entry| entry.parent)
|
||||
.map(|parent| {
|
||||
vec![
|
||||
"qdisc".to_string(),
|
||||
"del".to_string(),
|
||||
"dev".to_string(),
|
||||
interface.to_string(),
|
||||
"parent".to_string(),
|
||||
parent.as_tc_string(),
|
||||
]
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn drain_managed_tree_to_defaults(
|
||||
interface: &str,
|
||||
qdisc_snapshot: &[LiveTcQdiscEntry],
|
||||
) -> Result<HashSet<u16>, String> {
|
||||
let root_child_majors = managed_root_child_parent_handles(qdisc_snapshot)
|
||||
.into_iter()
|
||||
.map(|parent| parent.get_major_minor().1)
|
||||
.collect::<HashSet<_>>();
|
||||
if root_child_majors.is_empty() {
|
||||
return Ok(root_child_majors);
|
||||
}
|
||||
let class_snapshot = read_live_class_snapshot(interface)?;
|
||||
let mut commands = qdisc_delete_commands(interface, qdisc_snapshot, &root_child_majors);
|
||||
commands.extend(class_delete_commands(
|
||||
interface,
|
||||
&class_snapshot,
|
||||
&root_child_majors,
|
||||
));
|
||||
if commands.is_empty() {
|
||||
return Ok(root_child_majors);
|
||||
}
|
||||
run_root_preflight_commands(
|
||||
&commands,
|
||||
&format!("full reload drain managed tree to defaults on {interface}"),
|
||||
)?;
|
||||
invalidate_live_tc_snapshots();
|
||||
Ok(root_child_majors)
|
||||
}
|
||||
|
||||
fn prepare_root_mq_for_full_reload(
|
||||
config: &Arc<Config>,
|
||||
) -> Result<HashMap<String, HashSet<u16>>, String> {
|
||||
let mut preserved_root_child_majors = HashMap::new();
|
||||
for interface in managed_interfaces_for_config(config) {
|
||||
let qdisc_snapshot = read_live_qdisc_snapshot(&interface)?;
|
||||
if retained_root_mq_entry(&qdisc_snapshot).is_some() {
|
||||
let prune_commands = managed_root_child_parent_handles(&qdisc_snapshot)
|
||||
.into_iter()
|
||||
.map(|parent| {
|
||||
vec![
|
||||
"qdisc".to_string(),
|
||||
"del".to_string(),
|
||||
"dev".to_string(),
|
||||
interface.clone(),
|
||||
"parent".to_string(),
|
||||
parent.as_tc_string(),
|
||||
]
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if !prune_commands.is_empty() {
|
||||
run_root_preflight_commands(
|
||||
&prune_commands,
|
||||
&format!("full reload retained-root child prune on {interface}"),
|
||||
)?;
|
||||
invalidate_live_tc_snapshots();
|
||||
}
|
||||
|
||||
let pruned_qdisc_snapshot = read_live_qdisc_snapshot(&interface)?;
|
||||
let pruned_class_snapshot = read_live_class_snapshot(&interface)?;
|
||||
if verify_clean_root_child_tree(
|
||||
&pruned_qdisc_snapshot,
|
||||
&pruned_class_snapshot,
|
||||
&interface,
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
continue;
|
||||
let has_managed_root_children =
|
||||
!managed_root_child_parent_handles(&qdisc_snapshot).is_empty();
|
||||
if has_managed_root_children {
|
||||
let root_child_majors =
|
||||
drain_managed_tree_to_defaults(&interface, &qdisc_snapshot)?;
|
||||
if !root_child_majors.is_empty() {
|
||||
preserved_root_child_majors.insert(interface.clone(), root_child_majors);
|
||||
}
|
||||
}
|
||||
info!(
|
||||
"Bakery: preserving existing root mq/default path on {interface} during full reload"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let replace_summary = run_root_preflight_commands(
|
||||
@@ -2832,7 +3083,7 @@ fn prepare_root_mq_for_full_reload(config: &Arc<Config>) -> Result<(), String> {
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
Ok(preserved_root_child_majors)
|
||||
}
|
||||
|
||||
fn live_tree_mutations_allowed(config: &Arc<Config>) -> bool {
|
||||
@@ -10562,26 +10813,29 @@ fn full_reload(
|
||||
let previous_mq_created = MQ_CREATED.load(Ordering::Relaxed);
|
||||
let previous_shaping_tree_active = SHAPING_TREE_ACTIVE.load(Ordering::Relaxed);
|
||||
|
||||
if let Err(error) = prepare_root_mq_for_full_reload(config) {
|
||||
let summary = format!("Failed to prepare root mq state before full reload: {error}");
|
||||
error!("{summary}");
|
||||
tc_bypass_guard.keep_bypass_enabled_after_failure(&summary);
|
||||
mark_reload_required(format!(
|
||||
"{summary}. TC classify bypass remains enabled to avoid stale class-handle classification until a successful reload or operator intervention."
|
||||
));
|
||||
mark_bakery_action_finished(BakeryApplyMetrics {
|
||||
apply_type: BakeryApplyType::FullReload,
|
||||
summary: &summary,
|
||||
build_duration_ms: 0,
|
||||
apply_duration_ms: 0,
|
||||
total_tc_commands: 0,
|
||||
class_commands: 0,
|
||||
qdisc_commands: 0,
|
||||
ok: false,
|
||||
});
|
||||
*batch = None;
|
||||
return;
|
||||
}
|
||||
let preserved_root_child_majors = match prepare_root_mq_for_full_reload(config) {
|
||||
Ok(preserved_root_child_majors) => preserved_root_child_majors,
|
||||
Err(error) => {
|
||||
let summary = format!("Failed to prepare root mq state before full reload: {error}");
|
||||
error!("{summary}");
|
||||
tc_bypass_guard.keep_bypass_enabled_after_failure(&summary);
|
||||
mark_reload_required(format!(
|
||||
"{summary}. TC classify bypass remains enabled to avoid stale class-handle classification until a successful reload or operator intervention."
|
||||
));
|
||||
mark_bakery_action_finished(BakeryApplyMetrics {
|
||||
apply_type: BakeryApplyType::FullReload,
|
||||
summary: &summary,
|
||||
build_duration_ms: 0,
|
||||
apply_duration_ms: 0,
|
||||
total_tc_commands: 0,
|
||||
class_commands: 0,
|
||||
qdisc_commands: 0,
|
||||
ok: false,
|
||||
});
|
||||
*batch = None;
|
||||
return;
|
||||
}
|
||||
};
|
||||
invalidate_live_tc_snapshots();
|
||||
MQ_CREATED.store(true, Ordering::Relaxed);
|
||||
|
||||
@@ -10625,7 +10879,10 @@ fn full_reload(
|
||||
&mut working_circuits,
|
||||
&layout,
|
||||
&mut working_qdisc_handles,
|
||||
&live_reserved_handles,
|
||||
ProcessBatchOptions {
|
||||
extra_reserved_handles: &live_reserved_handles,
|
||||
preserved_root_child_majors: &preserved_root_child_majors,
|
||||
},
|
||||
);
|
||||
|
||||
if batch_result.result.ok {
|
||||
@@ -10686,7 +10943,7 @@ fn process_batch(
|
||||
circuits: &mut HashMap<i64, Arc<BakeryCommands>>,
|
||||
mq_layout: &MqDeviceLayout,
|
||||
qdisc_handles: &mut QdiscHandleState,
|
||||
extra_reserved_handles: &HashMap<String, HashSet<u16>>,
|
||||
options: ProcessBatchOptions<'_>,
|
||||
) -> BakeryFullReloadBatchResult {
|
||||
info!("Bakery: Processing batch of {} commands", batch.len());
|
||||
update_bakery_apply_progress(Some("Building tc command batch"), 0, 0, 0, 0);
|
||||
@@ -10700,7 +10957,7 @@ fn process_batch(
|
||||
config,
|
||||
mq_layout,
|
||||
qdisc_handles,
|
||||
extra_reserved_handles,
|
||||
options.extra_reserved_handles,
|
||||
)
|
||||
})
|
||||
.filter_map(|b| {
|
||||
@@ -10718,6 +10975,8 @@ fn process_batch(
|
||||
b.to_commands(config, ExecutionMode::Builder)
|
||||
})
|
||||
.flatten()
|
||||
.map(|command| idempotent_full_reload_command(command, options.preserved_root_child_majors))
|
||||
.filter(|command| !root_infra_qdisc_command(command, options.preserved_root_child_majors))
|
||||
.collect::<Vec<Vec<String>>>();
|
||||
|
||||
let path = config.debug_state_file_path("linux_tc_rust.txt");
|
||||
@@ -10814,21 +11073,34 @@ mod tests {
|
||||
static TC_BYPASS_SETTER_CALLS: Mutex<Vec<bool>> = Mutex::new(Vec::new());
|
||||
|
||||
fn recording_tc_bypass_setter(enabled: bool) -> anyhow::Result<()> {
|
||||
TC_BYPASS_SETTER_CALLS.lock().expect("record setter call").push(enabled);
|
||||
TC_BYPASS_SETTER_CALLS
|
||||
.lock()
|
||||
.expect("record setter call")
|
||||
.push(enabled);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn failing_tc_bypass_setter(enabled: bool) -> anyhow::Result<()> {
|
||||
TC_BYPASS_SETTER_CALLS.lock().expect("record setter call").push(enabled);
|
||||
Err(anyhow::anyhow!("synthetic TC classify bypass setter failure"))
|
||||
TC_BYPASS_SETTER_CALLS
|
||||
.lock()
|
||||
.expect("record setter call")
|
||||
.push(enabled);
|
||||
Err(anyhow::anyhow!(
|
||||
"synthetic TC classify bypass setter failure"
|
||||
))
|
||||
}
|
||||
|
||||
fn disable_failing_tc_bypass_setter(enabled: bool) -> anyhow::Result<()> {
|
||||
TC_BYPASS_SETTER_CALLS.lock().expect("record setter call").push(enabled);
|
||||
TC_BYPASS_SETTER_CALLS
|
||||
.lock()
|
||||
.expect("record setter call")
|
||||
.push(enabled);
|
||||
if enabled {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(anyhow::anyhow!("synthetic TC classify bypass disable failure"))
|
||||
Err(anyhow::anyhow!(
|
||||
"synthetic TC classify bypass disable failure"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10890,21 +11162,22 @@ mod tests {
|
||||
reset_bakery_test_state();
|
||||
|
||||
{
|
||||
let _bypass_guard =
|
||||
TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter)
|
||||
.expect("enable bypass guard");
|
||||
let _bypass_guard = TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter)
|
||||
.expect("enable bypass guard");
|
||||
assert_eq!(take_tc_bypass_setter_calls(), vec![true]);
|
||||
}
|
||||
|
||||
assert_eq!(take_tc_bypass_setter_calls(), vec![false]);
|
||||
let status = bakery_status_snapshot();
|
||||
assert!(!status.passthrough_degraded);
|
||||
assert!(status.passthrough_degraded_reason.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tc_classify_bypass_guard_disables_on_early_return() {
|
||||
fn early_return() -> Result<(), ()> {
|
||||
let _bypass_guard =
|
||||
TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter)
|
||||
.map_err(|_| ())?;
|
||||
TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter).map_err(|_| ())?;
|
||||
Err(())
|
||||
}
|
||||
|
||||
@@ -10924,6 +11197,9 @@ mod tests {
|
||||
assert!(TcClassifyBypassGuard::with_setter(failing_tc_bypass_setter).is_err());
|
||||
|
||||
assert_eq!(take_tc_bypass_setter_calls(), vec![true]);
|
||||
let status = bakery_status_snapshot();
|
||||
assert!(!status.passthrough_degraded);
|
||||
assert!(status.passthrough_degraded_reason.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -10939,6 +11215,15 @@ mod tests {
|
||||
}
|
||||
|
||||
assert_eq!(take_tc_bypass_setter_calls(), vec![true, false]);
|
||||
let status = bakery_status_snapshot();
|
||||
assert!(status.passthrough_degraded);
|
||||
assert!(
|
||||
status
|
||||
.passthrough_degraded_reason
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains("could not disable pass-through mode")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -10947,13 +11232,21 @@ mod tests {
|
||||
reset_bakery_test_state();
|
||||
|
||||
{
|
||||
let mut bypass_guard =
|
||||
TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter)
|
||||
.expect("enable bypass guard");
|
||||
let mut bypass_guard = TcClassifyBypassGuard::with_setter(recording_tc_bypass_setter)
|
||||
.expect("enable bypass guard");
|
||||
bypass_guard.keep_bypass_enabled_after_failure("test failure after TC mutation");
|
||||
}
|
||||
|
||||
assert_eq!(take_tc_bypass_setter_calls(), vec![true]);
|
||||
let status = bakery_status_snapshot();
|
||||
assert!(status.passthrough_degraded);
|
||||
assert!(
|
||||
status
|
||||
.passthrough_degraded_reason
|
||||
.as_deref()
|
||||
.unwrap_or_default()
|
||||
.contains("Traffic is passing without shaping")
|
||||
);
|
||||
}
|
||||
|
||||
fn mk_add_circuit(hash: i64, ip_addresses: &str) -> Arc<BakeryCommands> {
|
||||
@@ -13193,6 +13486,141 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idempotent_full_reload_command_replaces_class_adds() {
|
||||
let root_child_majors = HashMap::from([("eth0".to_string(), HashSet::from([0x1]))]);
|
||||
assert_eq!(
|
||||
idempotent_full_reload_command(
|
||||
vec![
|
||||
"class".to_string(),
|
||||
"add".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
"classid".to_string(),
|
||||
"0x1:1".to_string(),
|
||||
],
|
||||
&root_child_majors,
|
||||
),
|
||||
vec![
|
||||
"class".to_string(),
|
||||
"replace".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
"classid".to_string(),
|
||||
"0x1:1".to_string(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idempotent_full_reload_command_leaves_non_add_commands_unchanged() {
|
||||
let root_child_majors = HashMap::from([("eth0".to_string(), HashSet::from([0x1]))]);
|
||||
assert_eq!(
|
||||
idempotent_full_reload_command(
|
||||
vec![
|
||||
"qdisc".to_string(),
|
||||
"add".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
],
|
||||
&root_child_majors,
|
||||
),
|
||||
vec![
|
||||
"qdisc".to_string(),
|
||||
"add".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
idempotent_full_reload_command(
|
||||
vec![
|
||||
"class".to_string(),
|
||||
"del".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
],
|
||||
&root_child_majors,
|
||||
),
|
||||
vec![
|
||||
"class".to_string(),
|
||||
"del".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
idempotent_full_reload_command(
|
||||
vec![
|
||||
"filter".to_string(),
|
||||
"add".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
],
|
||||
&root_child_majors,
|
||||
),
|
||||
vec![
|
||||
"filter".to_string(),
|
||||
"add".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn root_infra_qdisc_command_matches_root_and_default_qdiscs() {
|
||||
let root_child_majors = HashMap::from([("eth0".to_string(), HashSet::from([0x1]))]);
|
||||
assert!(root_infra_qdisc_command(
|
||||
&[
|
||||
"qdisc".to_string(),
|
||||
"add".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
"parent".to_string(),
|
||||
"7FFF:0x1".to_string()
|
||||
],
|
||||
&root_child_majors,
|
||||
));
|
||||
assert!(root_infra_qdisc_command(
|
||||
&[
|
||||
"qdisc".to_string(),
|
||||
"add".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
"parent".to_string(),
|
||||
"0x1:2".to_string()
|
||||
],
|
||||
&root_child_majors,
|
||||
));
|
||||
assert!(!root_infra_qdisc_command(
|
||||
&[
|
||||
"qdisc".to_string(),
|
||||
"add".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
"parent".to_string(),
|
||||
"0x1:0x20".to_string()
|
||||
],
|
||||
&root_child_majors,
|
||||
));
|
||||
assert!(!root_infra_qdisc_command(
|
||||
&[
|
||||
"qdisc".to_string(),
|
||||
"add".to_string(),
|
||||
"dev".to_string(),
|
||||
"eth0".to_string(),
|
||||
"parent".to_string(),
|
||||
"0x3:2".to_string()
|
||||
],
|
||||
&root_child_majors,
|
||||
));
|
||||
assert!(!root_infra_qdisc_command(
|
||||
&["class".to_string(), "replace".to_string()],
|
||||
&root_child_majors,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn grouped_runtime_site_events_retain_human_site_name() {
|
||||
let _guard = bakery_test_lock().lock().expect("lock");
|
||||
|
||||
@@ -109,6 +109,9 @@ int xdp_prog(struct xdp_md *ctx)
|
||||
bpf_debug("Error: interface direction unspecified, aborting.");
|
||||
return XDP_PASS;
|
||||
}
|
||||
if (tc_classify_bypass_enabled()) {
|
||||
return XDP_PASS;
|
||||
}
|
||||
|
||||
// Do we need to perform a VLAN redirect?
|
||||
bool vlan_redirect = false;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Control-map helpers for temporarily bypassing TC classification.
|
||||
//! Control-map helpers for temporarily bypassing XDP redirection and TC classification.
|
||||
|
||||
use crate::lqos_kernel::bpf::libbpf_num_possible_cpus;
|
||||
use anyhow::{Error, Result};
|
||||
@@ -19,7 +19,8 @@ const PER_CPU_VALUE_ALIGNMENT: usize = 8;
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
#[repr(C)]
|
||||
pub struct TcClassifyControl {
|
||||
/// Non-zero tells TC classify to pass packets without setting TC handles.
|
||||
/// Non-zero tells XDP and TC classify to pass packets without redirecting
|
||||
/// or setting TC handles.
|
||||
pub bypass: u32,
|
||||
}
|
||||
|
||||
@@ -109,8 +110,9 @@ pub fn initialize_tc_classify_bypass() -> Result<()> {
|
||||
/// Enables or disables TC classify bypass.
|
||||
///
|
||||
/// This function has side effects: it writes all per-CPU values for the pinned
|
||||
/// `tc_classify_control` BPF map. When enabled, TC classify returns `TC_ACT_OK`
|
||||
/// before consuming XDP metadata, flow cache, hot cache, or LPM mappings.
|
||||
/// `tc_classify_control` BPF map. When enabled, XDP returns `XDP_PASS` before
|
||||
/// CPU redirection, and TC classify returns `TC_ACT_OK` before consuming XDP
|
||||
/// metadata, flow cache, hot cache, or LPM mappings.
|
||||
pub fn set_tc_classify_bypass(enabled: bool) -> Result<()> {
|
||||
update_tc_classify_control(enabled)
|
||||
}
|
||||
|
||||
@@ -86,12 +86,12 @@ function classifyEvent(entry) {
|
||||
if (event.endsWith("_started")) {
|
||||
outcome = "Running";
|
||||
outcomeClass = "bg-primary-subtle text-primary border border-primary-subtle";
|
||||
} else if (event === "reload_required") {
|
||||
} else if (event === "reload_required" || event === "passthrough_degraded") {
|
||||
stage = "Verify";
|
||||
scope = "Full Reload";
|
||||
outcome = "Reload Required";
|
||||
outcome = event === "passthrough_degraded" ? "Pass-Through" : "Reload Required";
|
||||
outcomeClass = "bg-danger-subtle text-danger border border-danger-subtle";
|
||||
} else if (event === "reload_required_cleared") {
|
||||
} else if (event === "reload_required_cleared" || event === "passthrough_degraded_cleared") {
|
||||
stage = "Verify";
|
||||
scope = "Full Reload";
|
||||
outcome = "Cleared";
|
||||
@@ -182,6 +182,8 @@ function describeOperation(entry, meta) {
|
||||
|| event === "full_reload_started"
|
||||
|| event === "reload_required"
|
||||
|| event === "reload_required_cleared"
|
||||
|| event === "passthrough_degraded"
|
||||
|| event === "passthrough_degraded_cleared"
|
||||
|| meta.scope === "Full Reload"
|
||||
) {
|
||||
return {
|
||||
|
||||
@@ -16,12 +16,18 @@ function classifyOutcome(entry) {
|
||||
const event = (entry?.event ?? "").toString().trim().toLowerCase();
|
||||
const status = (entry?.status ?? "").toString().trim().toLowerCase();
|
||||
|
||||
if (event === "reload_required") {
|
||||
if (event === "reload_required" || event === "passthrough_degraded") {
|
||||
return {
|
||||
label: "Reload Required",
|
||||
label: event === "passthrough_degraded" ? "Pass-Through" : "Reload Required",
|
||||
className: "bg-danger-subtle text-danger border border-danger-subtle",
|
||||
};
|
||||
}
|
||||
if (event === "passthrough_degraded_cleared") {
|
||||
return {
|
||||
label: "Shaping Restored",
|
||||
className: "bg-success-subtle text-success border border-success-subtle",
|
||||
};
|
||||
}
|
||||
if (event === "apply_failed" || status === "error") {
|
||||
return {
|
||||
label: "Failed",
|
||||
|
||||
@@ -58,6 +58,10 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
|
||||
this.lastFailureSummaryEl = document.createElement("span");
|
||||
this.lastFailureSummaryEl.style.whiteSpace = "normal";
|
||||
this.lastFailureSummaryEl.style.wordBreak = "break-word";
|
||||
this.shapingStateEl = document.createElement("div");
|
||||
this.passthroughReasonEl = document.createElement("span");
|
||||
this.passthroughReasonEl.style.whiteSpace = "normal";
|
||||
this.passthroughReasonEl.style.wordBreak = "break-word";
|
||||
this.reloadRequiredEl = document.createElement("div");
|
||||
this.reloadReasonEl = document.createElement("span");
|
||||
this.reloadReasonEl.style.whiteSpace = "normal";
|
||||
@@ -65,10 +69,12 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
|
||||
|
||||
tbody.appendChild(mkRow("Current Mode", this.modeEl));
|
||||
tbody.appendChild(mkRow("Current Duration", this.durationEl));
|
||||
tbody.appendChild(mkRow("Shaping State", this.shapingStateEl));
|
||||
tbody.appendChild(mkRow("Runtime Drift", this.reloadRequiredEl));
|
||||
tbody.appendChild(mkRow("Last Success", this.lastSuccessEl));
|
||||
tbody.appendChild(mkRow("Last Failure", this.lastFailureEl));
|
||||
tbody.appendChild(mkRow("Failure Summary", this.lastFailureSummaryEl));
|
||||
tbody.appendChild(mkRow("Pass-Through Reason", this.passthroughReasonEl));
|
||||
tbody.appendChild(mkRow("Reload Reason", this.reloadReasonEl));
|
||||
|
||||
table.appendChild(tbody);
|
||||
@@ -91,6 +97,12 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
|
||||
|
||||
this.modeEl.innerHTML = "";
|
||||
this.modeEl.appendChild(bakeryModeBadge(status.mode));
|
||||
this.shapingStateEl.innerHTML = "";
|
||||
this.shapingStateEl.appendChild(
|
||||
status.passthroughDegraded
|
||||
? mkBadge("Pass-Through / Unshaped", "bg-danger-subtle text-danger border border-danger-subtle")
|
||||
: mkBadge("Normal Shaping", "bg-success-subtle text-success border border-success-subtle"),
|
||||
);
|
||||
this.reloadRequiredEl.innerHTML = "";
|
||||
this.reloadRequiredEl.appendChild(
|
||||
status.reloadRequired
|
||||
@@ -104,6 +116,7 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
|
||||
this.lastSuccessEl.textContent = formatUnixSecondsToLocalDateTime(status.lastSuccessUnix);
|
||||
this.lastFailureEl.textContent = formatUnixSecondsToLocalDateTime(status.lastFailureUnix);
|
||||
this.lastFailureSummaryEl.textContent = status.lastFailureSummary || "—";
|
||||
this.passthroughReasonEl.textContent = status.passthroughDegradedReason || "—";
|
||||
this.reloadReasonEl.textContent = status.reloadRequiredReason || "—";
|
||||
}
|
||||
}
|
||||
@@ -1159,13 +1159,16 @@ function initUrgentIssues() {
|
||||
const td = document.createElement('td');
|
||||
const when = new Date(it.ts * 1000).toLocaleString();
|
||||
const sev = it.severity === 'Error' ? 'danger' : 'warning';
|
||||
const clearControl = it.clearable === false
|
||||
? '<span class="badge bg-secondary-subtle text-secondary border float-end ms-3">Active</span>'
|
||||
: `<button type="button" class="btn btn-link btn-sm text-secondary float-end ms-3 p-0 urgent-clear" data-id="${it.id}" title="Acknowledge issue" aria-label="Acknowledge issue ${escapeAttr(it.code)}"><i class="fa fa-times" aria-hidden="true"></i></button>`;
|
||||
td.innerHTML = `
|
||||
<div>
|
||||
<span class="badge bg-${sev}">${it.severity}</span>
|
||||
<strong class="ms-2">${it.code}</strong>
|
||||
<span class="text-muted ms-2">(${it.source})</span>
|
||||
<span class="text-muted float-end">${when}</span>
|
||||
<button type="button" class="btn btn-link btn-sm text-secondary float-end ms-3 p-0 urgent-clear" data-id="${it.id}" title="Acknowledge issue" aria-label="Acknowledge issue ${escapeAttr(it.code)}"><i class="fa fa-times" aria-hidden="true"></i></button>
|
||||
${clearControl}
|
||||
</div>
|
||||
<div class="mt-1" style="white-space: pre-wrap;">${it.message}</div>
|
||||
${it.context ? `<pre class="mt-2">${it.context}</pre>` : ''}
|
||||
|
||||
@@ -73,6 +73,7 @@ pub struct UrgentItem {
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
pub context: Option<String>,
|
||||
pub clearable: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -80,17 +81,32 @@ pub struct UrgentList {
|
||||
pub items: Vec<UrgentItem>,
|
||||
}
|
||||
|
||||
pub fn urgent_status_data() -> UrgentStatus {
|
||||
let items = urgent::list();
|
||||
UrgentStatus {
|
||||
has_urgent: !items.is_empty(),
|
||||
count: items.len(),
|
||||
fn bakery_passthrough_issue() -> Option<UrgentItem> {
|
||||
let status = lqos_bakery::bakery_status_snapshot();
|
||||
if !status.reload_required || !status.passthrough_degraded {
|
||||
return None;
|
||||
}
|
||||
Some(UrgentItem {
|
||||
id: u64::MAX,
|
||||
ts: status
|
||||
.last_failure_unix
|
||||
.or(status.current_action_started_unix)
|
||||
.unwrap_or_default(),
|
||||
source: "System".to_string(),
|
||||
severity: "Error".to_string(),
|
||||
code: "BAKERY_PASSTHROUGH_DEGRADED".to_string(),
|
||||
message: status.passthrough_degraded_reason.unwrap_or_else(|| {
|
||||
"Bakery full reload failed while pass-through mode was active. Traffic may be unshaped until a successful full reload clears the degraded state.".to_string()
|
||||
}),
|
||||
context: status.reload_required_reason.map(|reason| {
|
||||
format!("Reload required reason: {reason}")
|
||||
}),
|
||||
clearable: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn urgent_list_data() -> UrgentList {
|
||||
let items = urgent::list();
|
||||
let items: Vec<UrgentItem> = items
|
||||
fn merged_urgent_items() -> Vec<UrgentItem> {
|
||||
let mut items: Vec<UrgentItem> = urgent::list()
|
||||
.into_iter()
|
||||
.map(|i| UrgentItem {
|
||||
id: i.id,
|
||||
@@ -100,9 +116,29 @@ pub fn urgent_list_data() -> UrgentList {
|
||||
code: i.code,
|
||||
message: strip_ansi(&i.message),
|
||||
context: i.context,
|
||||
clearable: true,
|
||||
})
|
||||
.collect();
|
||||
UrgentList { items }
|
||||
if let Some(issue) = bakery_passthrough_issue() {
|
||||
items.push(issue);
|
||||
}
|
||||
items.sort_by_key(|item| item.ts);
|
||||
items.reverse();
|
||||
items
|
||||
}
|
||||
|
||||
pub fn urgent_status_data() -> UrgentStatus {
|
||||
let items = merged_urgent_items();
|
||||
UrgentStatus {
|
||||
has_urgent: !items.is_empty(),
|
||||
count: items.len(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn urgent_list_data() -> UrgentList {
|
||||
UrgentList {
|
||||
items: merged_urgent_items(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn urgent_clear_id(id: u64) -> bool {
|
||||
|
||||
@@ -447,6 +447,8 @@ pub struct BakeryStatusState {
|
||||
pub preflight: Option<BakeryPreflightData>,
|
||||
pub reload_required: bool,
|
||||
pub reload_required_reason: Option<String>,
|
||||
pub passthrough_degraded: bool,
|
||||
pub passthrough_degraded_reason: Option<String>,
|
||||
pub dirty_subtree_count: usize,
|
||||
}
|
||||
|
||||
|
||||
@@ -152,6 +152,8 @@ fn map_status(snapshot: BakeryStatusSnapshot) -> BakeryStatusData {
|
||||
},
|
||||
reload_required: snapshot.reload_required,
|
||||
reload_required_reason: snapshot.reload_required_reason,
|
||||
passthrough_degraded: snapshot.passthrough_degraded,
|
||||
passthrough_degraded_reason: snapshot.passthrough_degraded_reason,
|
||||
dirty_subtree_count: snapshot.dirty_subtree_count,
|
||||
queue_distribution: snapshot
|
||||
.queue_distribution
|
||||
|
||||
Reference in new issue
Block a user