Merge develop into feat/1056_caddy

This commit is contained in:
Herbert Wolverson committed 2026-05-14 09:04:38 -05:00
commit fd39fa49ad
152 files changed
+10827 -1951

No files matched your search

+195 -111
View File
@@ -51,13 +51,37 @@ from liblqos_python import is_lqosd_alive, clear_ip_mappings, delete_ip_mapping,
shaping_cpu_count, \
Bakery
# Optional: urgent issue submission (available in newer liblqos_python)
# Optional: urgent issue helpers (available in newer liblqos_python)
try:
from liblqos_python import submit_urgent_issue # type: ignore
except Exception:
def submit_urgent_issue(*_args, **_kwargs):
return False
try:
from liblqos_python import clear_urgent_issue_by_identity # type: ignore
except ImportError:
_missing_clear_urgent_issue_by_identity_warned = False
def clear_urgent_issue_by_identity(*_args, **_kwargs):
global _missing_clear_urgent_issue_by_identity_warned
if not _missing_clear_urgent_issue_by_identity_warned:
logging.warning("Unable to clear recovered urgent issues: liblqos_python helper is unavailable")
_missing_clear_urgent_issue_by_identity_warned = True
return False
try:
from liblqos_python import xdp_ip_mapping_ready # type: ignore
except ImportError:
_missing_xdp_ip_mapping_ready_warned = False
def xdp_ip_mapping_ready():
global _missing_xdp_ip_mapping_ready_warned
if not _missing_xdp_ip_mapping_ready_warned:
logging.warning("Unable to check XDP IP mapping readiness: liblqos_python helper is unavailable")
_missing_xdp_ip_mapping_ready_warned = True
return True
try:
from liblqos_python import calculate_topology_source_generation # type: ignore
except Exception:
@@ -78,6 +102,11 @@ class ValidationFailure(RefreshFailure):
pass
XDP_IP_MAPPING_APPLY_FAILED = "XDP_IP_MAPPING_APPLY_FAILED"
XDP_MAPPING_READY_TIMEOUT_SECONDS = 5.0
XDP_MAPPING_READY_INTERVAL_SECONDS = 0.1
def report_refresh_failure(code, message, context=None, dedupe_key=None):
logging.error(message)
print("ERROR: " + message)
@@ -98,6 +127,56 @@ def report_refresh_failure(code, message, context=None, dedupe_key=None):
pass
raise RefreshFailure(message)
def wait_for_xdp_ip_mapping_ready(
ready=xdp_ip_mapping_ready,
sleep=time.sleep,
now=time.monotonic,
timeout_seconds=XDP_MAPPING_READY_TIMEOUT_SECONDS,
interval_seconds=XDP_MAPPING_READY_INTERVAL_SECONDS,
):
deadline = now() + timeout_seconds
while now() < deadline:
if ready():
return
sleep(interval_seconds)
raise TimeoutError("XDP IP mapping BPF maps are not ready")
def apply_xdp_ip_mappings(
ip_map_batch,
context,
ready=xdp_ip_mapping_ready,
sleep=time.sleep,
report_failure=report_refresh_failure,
clear_recovered_issue=clear_urgent_issue_by_identity,
):
try:
wait_for_xdp_ip_mapping_ready(ready=ready, sleep=sleep)
except (TimeoutError, OSError) as exc:
report_failure(
XDP_IP_MAPPING_APPLY_FAILED,
"Failed to apply XDP IP mappings: " + str(exc),
context,
XDP_IP_MAPPING_APPLY_FAILED,
)
return
try:
ip_map_batch.finish_ip_mappings()
ip_map_batch.submit()
except OSError as exc:
report_failure(
XDP_IP_MAPPING_APPLY_FAILED,
"Failed to apply XDP IP mappings: " + str(exc),
context,
XDP_IP_MAPPING_APPLY_FAILED,
)
return
try:
clear_recovered_issue(XDP_IP_MAPPING_APPLY_FAILED, XDP_IP_MAPPING_APPLY_FAILED)
except OSError as exc:
logging.warning("Unable to clear recovered XDP IP mapping urgent issue: %s", exc)
R2Q = 10
#MAX_R2Q = 200_000
MAX_R2Q = 60_000 # See https://lartc.vger.kernel.narkive.com/NKaH1ZNG/htb-quantum-of-class-100001-is-small-consider-r2q-change
@@ -345,6 +424,72 @@ def _attachment_lookup_candidates(node_key, node_data):
return candidates
def _circuit_attachment_name_candidates(circuit):
candidates = []
for candidate_name in (
circuit.get('ParentNode', ''),
circuit.get('effectiveParentNodeName', ''),
):
candidate = str(candidate_name or '').strip()
if candidate and candidate != 'none' and candidate not in candidates:
candidates.append(candidate)
if circuit.get('parentResolvedByShapingInputs') is not True:
logical_parent = str(circuit.get('logicalParentNode', '') or '').strip()
if logical_parent and logical_parent != 'none' and logical_parent not in candidates:
candidates.append(logical_parent)
return candidates
def _normalize_circuit_shaping_parent(circuit, parent_node_ids=None):
parent_name = str(circuit.get('ParentNode', '') or '').strip()
effective_name = str(circuit.get('effectiveParentNodeName', '') or '').strip()
parent_id = str(circuit.get('effectiveParentNodeID', '') or circuit.get('ParentNodeID', '') or '').strip()
resolved_id_parent = ''
if parent_node_ids and parent_id:
resolved_id_parent = str(parent_node_ids.get(parent_id, '') or '').strip()
if not parent_name:
parent_name = effective_name
if (not parent_name or parent_name == 'none') and resolved_id_parent:
parent_name = resolved_id_parent
if not parent_name:
parent_name = 'none'
if is_generated_parent_node_name(parent_name) or (
resolved_id_parent and resolved_id_parent != parent_name
):
parent_id = ''
circuit['shapingParentNode'] = parent_name
circuit['shapingParentNodeID'] = parent_id
circuit['shapingParentKey'] = f"id:{parent_id}" if parent_id else f"name:{parent_name}"
return parent_name, parent_id
def _validate_planned_circuit_attachment(node_name, node_data, circuit, planned_identity):
site_major = _parse_int_token(node_data.get('classMajor'))
site_up_major = _parse_int_token(node_data.get('up_classMajor'))
planned_major = _parse_int_token(planned_identity.get('class_major'))
planned_up_major = _parse_int_token(planned_identity.get('up_class_major'))
if None in (site_major, site_up_major, planned_major, planned_up_major):
raise ValueError(
f"Incomplete class identity while attaching circuit '{circuit.get('circuitID', '')}' to parent '{node_name}'"
)
if site_major != planned_major or site_up_major != planned_up_major:
raise ValueError(
"Planned circuit class majors do not match the selected parent node: "
f"circuit='{circuit.get('circuitID', '')}', parent='{node_name}', "
f"parentMajor={hex(site_major)}, parentUpMajor={hex(site_up_major)}, "
f"plannedMajor={hex(planned_major)}, plannedUpMajor={hex(planned_up_major)}, "
f"ParentNode='{circuit.get('ParentNode', '')}', "
f"logicalParentNode='{circuit.get('logicalParentNode', '')}', "
f"resolutionSource='{circuit.get('parentResolutionSource', '')}'"
)
return planned_major, planned_up_major
def _current_topology_source_generation():
try:
generation = calculate_topology_source_generation()
@@ -461,6 +606,7 @@ def loadSubscriberCircuitsFromShapingInputs(shapingInputsPath):
"circuitID": circuitID,
"circuitName": str(circuit.get('circuit_name', '') or ''),
"ParentNode": parent_node if parent_node != '' else 'none',
"effectiveParentNodeName": parent_node,
"ParentNodeID": parent_node_id,
"AnchorNodeID": anchor_node_id,
"devices": devices,
@@ -623,60 +769,6 @@ def planner_circuit_identity_key(circuit):
return circuit_id
def load_minor_state_from_queuing_structure(path=None):
if path is None:
path = get_queuing_structure_path()
data = _load_json_dict(path)
network = data.get("Network")
if not isinstance(network, dict):
return {"sites": {}, "circuits": {}}
sites = {}
circuits = {}
def walk(node_map, trail=()):
for node_name, node in sorted(node_map.items()):
if not isinstance(node, dict):
continue
node_path = trail + (node_name,)
site_key = "/".join(node_path)
parent_path = "/".join(trail)
queue = _parse_int_token(node.get("cpuNum"))
class_minor = _parse_int_token(node.get("classMinor"))
class_major = _parse_int_token(node.get("classMajor"))
up_class_major = _parse_int_token(node.get("up_classMajor"))
if queue is not None and class_minor is not None:
sites[site_key] = {
"class_minor": class_minor,
"queue": queue + 1,
"parent_path": parent_path,
"class_major": class_major,
"up_class_major": up_class_major,
}
if isinstance(node.get("circuits"), list):
for circuit in node.get("circuits", []):
if not isinstance(circuit, dict):
continue
circuit_id = circuit.get("circuitID")
circuit_minor = _parse_int_token(circuit.get("classMinor"))
if queue is None or circuit_id is None or circuit_minor is None:
continue
circuits[str(circuit_id)] = {
"class_minor": circuit_minor,
"queue": queue + 1,
"parent_node": circuit.get("ParentNode", node_name),
"class_major": _parse_int_token(circuit.get("classMajor")),
"up_class_major": _parse_int_token(circuit.get("up_classMajor")),
}
children = node.get("children")
if isinstance(children, dict):
walk(children, node_path)
walk(network)
return {"sites": sites, "circuits": circuits}
def calculateR2q(maxRateInMbps):
# So we've learned that r2q defaults to 10, and is used to calculate quantum. Quantum is rateInBytes/r2q by
# default. This default gives errors at high rates, and tc clamps the quantum to 200000. Setting a high quantum
@@ -1766,17 +1858,11 @@ def refreshShapers():
circuits_by_parent_id = {}
circuits_by_parent_name = {}
for circuit in subscriberCircuits:
parent_id = str(circuit.get('effectiveParentNodeID', '') or circuit.get('ParentNodeID', '') or '').strip()
parent_name, parent_id = _normalize_circuit_shaping_parent(circuit, parent_node_ids)
if parent_id:
circuits_by_parent_id.setdefault(parent_id, []).append(circuit)
for parent_name in {
str(circuit.get('ParentNode', '') or '').strip(),
str(circuit.get('effectiveParentNodeName', '') or '').strip(),
str(circuit.get('logicalParentNode', '') or '').strip(),
}:
if parent_name:
circuits_by_parent_name.setdefault(parent_name, []).append(circuit)
elif parent_name and parent_name != 'none':
circuits_by_parent_name.setdefault(parent_name, []).append(circuit)
# Parse network structure and add devices from ShapedDevices.csv
print("Parsing network structure and tallying devices")
@@ -1993,9 +2079,8 @@ def refreshShapers():
f"Failed to save planner state at {state_path}: {e}", stacklevel=2
)
# Seed persisted site/circuit minor assignments. When planner state is absent,
# fall back to the previous queuing structure so the first run after an upgrade
# can preserve existing class IDs.
# Load persisted top-level planner state. Full reloads rebuild TC, so
# site/circuit class minors are runtime artifacts and are not reused below.
try:
state # noqa: B018
except NameError:
@@ -2003,24 +2088,6 @@ def refreshShapers():
state_path = get_planner_state_path()
if not isinstance(state, dict) or len(state.keys()) == 0:
state = load_planner_state(state_path, None)
try:
circuit_state_from_disk = state.get("circuits", {}) if isinstance(state, dict) else {}
except Exception:
circuit_state_from_disk = {}
try:
site_state_from_disk = state.get("sites", {}) if isinstance(state, dict) else {}
except Exception:
site_state_from_disk = {}
if not isinstance(circuit_state_from_disk, dict):
circuit_state_from_disk = {}
if not isinstance(site_state_from_disk, dict):
site_state_from_disk = {}
if not circuit_state_from_disk or not site_state_from_disk:
fallback_minor_state = load_minor_state_from_queuing_structure()
if not site_state_from_disk:
site_state_from_disk = fallback_minor_state.get("sites", {}) or {}
if not circuit_state_from_disk:
circuit_state_from_disk = fallback_minor_state.get("circuits", {}) or {}
circuit_state_updated = {}
site_state_updated = {}
planner_site_inputs = []
@@ -2090,8 +2157,8 @@ def refreshShapers():
identity_plan = plan_class_identities(
planner_site_inputs,
planner_circuit_groups,
site_state=site_state_from_disk,
circuit_state=circuit_state_from_disk,
site_state={},
circuit_state={},
stick_offset=stickOffset,
circuit_padding=CIRCUIT_PADDING,
)
@@ -2184,8 +2251,9 @@ def refreshShapers():
for node in sorted_node_keys(data, depth):
node_data = data[node]
queue_token = _parse_int_token(node_data.get('cpuNum'))
major = _parse_int_token(node_data.get('classMajor'))
if queue_token is None or major is None:
site_major = _parse_int_token(node_data.get('classMajor'))
site_up_major = _parse_int_token(node_data.get('up_classMajor'))
if queue_token is None or site_major is None or site_up_major is None:
continue
queue = queue_token + 1
circuitsForThisNetworkNode = []
@@ -2236,11 +2304,15 @@ def refreshShapers():
if planned_identity is None:
raise ValueError(f"Missing planned circuit identity for {planner_key}")
candidate_minor = int(planned_identity["class_minor"])
major = int(planned_identity["class_major"])
up_major = int(planned_identity["up_class_major"])
planned_major, planned_up_major = _validate_planned_circuit_attachment(
node,
node_data,
circuit,
planned_identity,
)
ensure_minor_capacity(queue, candidate_minor)
flowIDstring = hex(major) + ':' + hex(candidate_minor)
upFlowIDstring = hex(up_major) + ':' + hex(candidate_minor)
flowIDstring = hex(planned_major) + ':' + hex(candidate_minor)
upFlowIDstring = hex(planned_up_major) + ':' + hex(candidate_minor)
circuit['classid'] = flowIDstring
circuit['up_classid'] = upFlowIDstring
attached_circuit_ids.add(circuit_id)
@@ -2268,8 +2340,8 @@ def refreshShapers():
"devices": circuit['devices'],
"classid": flowIDstring,
"up_classid": upFlowIDstring,
"classMajor": hex(major),
"up_classMajor": hex(up_major),
"classMajor": hex(planned_major),
"up_classMajor": hex(planned_up_major),
"classMinor": hex(candidate_minor),
"comment": circuit['comment'],
}
@@ -2315,15 +2387,33 @@ def refreshShapers():
f"{circuit_id} ({circuit.get('circuitName', '')}) parent={circuit.get('ParentNode', '')} parent_id={circuit.get('effectiveParentNodeID', circuit.get('ParentNodeID', ''))}"
)
if unattached_circuits:
examples = unattached_circuits[:20]
warnings.warn(
"Some shaped circuits did not attach to the live queue tree. First examples: " + "; ".join(unattached_circuits[:20]),
"Some shaped circuits did not attach to the live queue tree. First examples: " + "; ".join(examples),
stacklevel=2,
)
logging.warning(
"Unattached shaped circuits after queue build: %s total. Examples: %s",
len(unattached_circuits),
"; ".join(unattached_circuits[:20]),
"; ".join(examples),
)
try:
submit_urgent_issue(
"LibreQoS",
"Warning",
"UNATTACHED_SHAPED_CIRCUITS",
(
f"{len(unattached_circuits)} shaped circuit(s) did not attach to the live queue tree. "
"These circuits will not have class IDs or queue stats until topology/shaping inputs are fixed."
),
json.dumps({
"count": len(unattached_circuits),
"examples": examples,
}),
"UNATTACHED_SHAPED_CIRCUITS",
)
except Exception as exc:
logging.debug("Unable to submit unattached shaped circuits urgent issue: %s", exc)
minorByCPU = {
int(queue): int(minor)
@@ -2746,20 +2836,14 @@ def refreshShapers():
print("Observe mode active; skipping XDP-CPUMAP-TC IP filter apply after clearing mappings")
numXdpCommands = 0
else:
ipMapBatch.finish_ip_mappings()
try:
ipMapBatch.submit()
except Exception as e:
report_refresh_failure(
"XDP_IP_MAPPING_APPLY_FAILED",
"Failed to apply XDP IP mappings: " + str(e),
{
"required_ip_mappings": requiredIpMappings,
"queued_requests": numXdpCommands,
"on_a_stick": on_a_stick(),
},
"XDP_IP_MAPPING_APPLY_FAILED",
)
apply_xdp_ip_mappings(
ipMapBatch,
{
"required_ip_mappings": requiredIpMappings,
"queued_requests": numXdpCommands,
"on_a_stick": on_a_stick(),
},
)
#for command in xdpCPUmapCommands:
# logging.info(command)
# commands = command.split(' ')
+3 -1
View File
@@ -5,7 +5,9 @@ Requires=lqosd.service
[Service]
WorkingDirectory=/opt/libreqos/src
ExecStart=/usr/bin/python3 /opt/libreqos/src/scheduler.py
ExecStartPre=/bin/test -x /opt/libreqos/venv/bin/python
ExecStartPre=/opt/libreqos/venv/bin/python -c "import apscheduler, chardet, deepdiff, psutil, requests"
ExecStart=/opt/libreqos/venv/bin/python /opt/libreqos/src/scheduler.py
Restart=always
[Install]
+42
View File
@@ -0,0 +1,42 @@
#!/bin/bash
set -euo pipefail
VENV_DIR="/opt/libreqos/venv"
REQUIREMENTS=""
CONSTRAINTS=""
if [ "$(id -u)" -ne 0 ]; then
echo "rebuild_python_venv.sh must be run as root." >&2
exit 1
fi
for candidate in /opt/libreqos/src/requirements.txt /opt/libreqos/requirements.txt; do
if [ -f "$candidate" ]; then
REQUIREMENTS="$candidate"
break
fi
done
if [ -z "$REQUIREMENTS" ]; then
echo "LibreQoS Python requirements file not found in /opt/libreqos/src or /opt/libreqos." >&2
exit 1
fi
for candidate in /opt/libreqos/src/deb-requirements-constraints.txt /opt/libreqos/deb-requirements-constraints.txt; do
if [ -s "$candidate" ]; then
CONSTRAINTS="$candidate"
break
fi
done
python3 -m venv --clear "$VENV_DIR"
"$VENV_DIR/bin/python" -m pip install --upgrade pip
if [ -n "$CONSTRAINTS" ]; then
"$VENV_DIR/bin/python" -m pip install --upgrade -c "$CONSTRAINTS" -r "$REQUIREMENTS"
else
"$VENV_DIR/bin/python" -m pip install --upgrade -r "$REQUIREMENTS"
fi
chown -R root:root "$VENV_DIR"
chmod -R go-w "$VENV_DIR"
+23 -16
View File
@@ -12,7 +12,7 @@ PACKAGE=libreqos
VERSION=$(cat ./VERSION_STRING).$BUILD_DATE
PKGVERSION="${PACKAGE}_${VERSION}"
DPKG_DIR=dist/$PKGVERSION-1_amd64
APT_DEPENDENCIES="python3-pip, nano, curl, ca-certificates"
APT_DEPENDENCIES="python3-pip, python3-venv, nano, curl, ca-certificates"
DEBIAN_DIR=$DPKG_DIR/DEBIAN
LQOS_DIR=$DPKG_DIR/opt/libreqos/src
LQOS_STATE_DIR=$DPKG_DIR/opt/libreqos/state
@@ -27,6 +27,7 @@ LQOS_FILES=(
integrationNetzur.py
integrationRestHttp.py
integrationSonar.py
integrationUtils.py
integrationSplynx.py
integrationVISP.py
integrationWISPGate.py
@@ -55,6 +56,7 @@ LQOS_BIN_FILES=(
lqosd.service.example
lqos_api.service.example
lqos_setup.service.example
rebuild_python_venv.sh
)
RUSTPROGS=(
@@ -142,16 +144,10 @@ else
fi
}
# Install Python Dependencies
pushd /opt/libreqos > /dev/null
# - Setup Python dependencies as a post-install task. Use --ignore-installed so
# pip does not try to uninstall Debian-managed packages that do not ship a
# pip RECORD file (for example blinker on Ubuntu 24.04).
if [ -s src/deb-requirements-constraints.txt ]; then
PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --ignore-installed -c src/deb-requirements-constraints.txt -r src/requirements.txt
else
PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --ignore-installed -r src/requirements.txt
fi
# - Setup Python dependencies in a root-owned virtual environment. LibreQoS
# services still run as root, but Python packages no longer mix with
# apt-managed system site-packages.
/opt/libreqos/src/bin/rebuild_python_venv.sh
# Ensure folder permissions are correct post-install
set_libreqos_operator_permissions
@@ -163,10 +159,20 @@ install -m 0644 /opt/libreqos/src/bin/lqos_api.service.example /etc/systemd/syst
install -m 0644 /opt/libreqos/src/bin/lqos_setup.service.example /etc/systemd/system/lqos_setup.service
/bin/rm -f /etc/systemd/system/lqos_netplan_helper.service || true
/bin/systemctl daemon-reload || true
/bin/systemctl stop lqos_node_manager || true # In case it's running from a previous release
/bin/systemctl disable lqos_node_manager || true # In case it's running from a previous release
/bin/systemctl stop lqos_netplan_helper || true
/bin/systemctl disable lqos_netplan_helper || true
unit_stop_disable_if_present() {
local unit="$1"
if /bin/systemctl cat "${unit}" >/dev/null 2>&1 \
|| [ -e "/etc/systemd/system/${unit}" ] \
|| [ -e "/lib/systemd/system/${unit}" ] \
|| [ -e "/usr/lib/systemd/system/${unit}" ]; then
/bin/systemctl stop "${unit}" || true
/bin/systemctl disable "${unit}" || true
fi
}
unit_stop_disable_if_present lqos_node_manager.service # In case it's running from a previous release
unit_stop_disable_if_present lqos_netplan_helper.service
case "$(/opt/libreqos/src/bin/lqos_setup postinst-action)" in
activate_runtime)
/opt/libreqos/src/bin/lqos_setup activate-runtime
@@ -193,7 +199,6 @@ HOTFIX
exit 1
;;
esac
popd > /dev/null
EOF
# Uninstall Script
@@ -239,6 +244,8 @@ for file in "${LQOS_BIN_FILES[@]}"; do
cp "bin/$file" "$LQOS_DIR/bin"
done
chmod a+x "$LQOS_DIR/bin/rebuild_python_venv.sh"
# Copy the remove pinned maps
cp rust/remove_pinned_maps.sh "$LQOS_DIR"/rust
+43 -6
View File
@@ -22,8 +22,10 @@ do
esac
done
VENV_PYTHON="/opt/libreqos/venv/bin/python"
# Check Pre-Requisites
sudo apt install python3-pip clang gcc gcc-multilib llvm libelf-dev git nano curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev curl
sudo apt install python3-pip python3-venv clang gcc gcc-multilib llvm libelf-dev git nano curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev curl
if ! rustup -V &> /dev/null
then
@@ -148,7 +150,7 @@ done
# Update the lqos_api binary
echo "Updating lqos_api binary..."
bash ./update_api.sh || echo "Warning: Failed to update lqos_api (continuing)."
bash ./update_api.sh --no-restart || echo "Warning: Failed to update lqos_api (continuing)."
set_libreqos_operator_permissions() {
local runtime_paths=()
@@ -183,6 +185,16 @@ service_is_active() {
systemctl is-active --quiet "$n.service"
}
service_is_enabled() {
local n=$1
systemctl is-enabled --quiet "$n.service" 2>/dev/null
}
service_is_failed() {
local n=$1
systemctl is-failed --quiet "$n.service" 2>/dev/null
}
refresh_service_unit() {
local unit_name=$1
local src="./bin/${unit_name}.service.example"
@@ -214,7 +226,30 @@ clear_pinned_maps_before_lqosd_restart() {
fi
}
rebuild_python_venv() {
local script_path="./bin/rebuild_python_venv.sh"
if [ ! -x "$script_path" ]; then
echo "Expected $script_path to exist and be executable before refreshing Python services."
exit 1
fi
echo "Rebuilding /opt/libreqos/venv before restarting Python services."
if ! sudo "$script_path"; then
echo "Failed to rebuild /opt/libreqos/venv. Skipping service refresh/restarts."
exit 1
fi
if [ ! -x "$VENV_PYTHON" ]; then
echo "Expected $VENV_PYTHON to exist and be executable after rebuilding the Python venv."
echo "Skipping service refresh/restarts."
exit 1
fi
}
SERVICE_UNITS_UPDATED=0
rebuild_python_venv
refresh_service_unit lqosd
refresh_service_unit lqos_scheduler
refresh_service_unit lqos_api
@@ -250,12 +285,13 @@ else
echo "lqosd is active as a service. Restarting it. You may need to enter your sudo password."
sudo systemctl restart lqosd
fi
if service_is_active lqos_scheduler; then
echo "lqos_scheduler is active as a service. Restarting it. You may need to enter your sudo password."
if service_is_active lqos_scheduler || service_is_failed lqos_scheduler || service_is_enabled lqos_scheduler; then
echo "lqos_scheduler is installed as an active, failed, or enabled service. Restarting it. You may need to enter your sudo password."
sudo systemctl reset-failed lqos_scheduler || true
sudo systemctl restart lqos_scheduler
fi
if service_is_active lqos_api; then
echo "lqos_api is active as a service. Restarting it. You may need to enter your sudo password."
if service_is_active lqos_api || service_is_failed lqos_api || service_is_enabled lqos_api; then
echo "lqos_api is installed as an active, failed, or enabled service. Restarting it. You may need to enter your sudo password."
sudo systemctl restart lqos_api
fi
if service_is_active lqos_setup; then
@@ -267,6 +303,7 @@ fi
echo "-----------------------------------------------------------------"
echo "Don't forget to setup /etc/lqos.conf!"
echo "Template .service files can be found in bin/"
echo "Debian package installs create/update /opt/libreqos/venv for Python dependencies."
echo "If src/deb-requirements-constraints.txt exists, Debian package installs use it to constrain Python dependencies."
echo "Use ./systemd_hotfix.sh to evaluate or install the Ubuntu 24.04 networkd hotfix from the LibreQoS APT repo at https://repo.libreqos.com."
echo "The hotfix installer now offers to schedule a reboot after it finishes."
-1
View File
@@ -36,7 +36,6 @@ oldConfig = {
# Common
'circuitNameUseAddress': getIspConfigValue('circuitNameUseAddress', True),
'overwriteNetworkJSONalways': getIspConfigValue('overwriteNetworkJSONalways', False),
'ignoreSubnets': getIspConfigValue('ignoreSubnets', ["192.168.0.0/16"]),
'allowedSubnets': getIspConfigValue('allowedSubnets', ["100.64.0.0/10"]),
'excludeSites': getIspConfigValue('excludeSites', []),
+45 -4
View File
@@ -1,7 +1,7 @@
# Optimized version of integrationCommon.py for better performance with large datasets
# Provides common functionality shared between integrations.
from typing import List, Any, Dict, Set
from typing import List, Any, Dict, Set, Tuple
from liblqos_python import allowed_subnets, ignore_subnets, generated_pn_download_mbps, generated_pn_upload_mbps, \
circuit_name_use_address, upstream_bandwidth_capacity_download_mbps, upstream_bandwidth_capacity_upload_mbps, \
find_ipv6_using_mikrotik, exclude_sites, bandwidth_overhead_factor, committed_bandwidth_multiplier, \
@@ -32,6 +32,46 @@ def _state_directory():
def _state_path(category, filename):
return os.path.join(_state_directory(), category, filename)
def _normalize_exception_cpes(raw_exception_cpes) -> Tuple[Dict[str, str], List[str]]:
normalized = {}
errors = []
if raw_exception_cpes is None:
return normalized, errors
if isinstance(raw_exception_cpes, dict):
for cpe, parent in raw_exception_cpes.items():
cpe = str(cpe).strip()
parent = str(parent).strip()
if cpe and parent:
normalized[cpe] = parent
return normalized, errors
if not isinstance(raw_exception_cpes, (list, tuple, set)):
errors.append(
f"exception_cpes ignored: expected dict or list, got {type(raw_exception_cpes).__name__}"
)
return normalized, errors
for entry in raw_exception_cpes:
cpe = None
parent = None
if isinstance(entry, str):
if ":" in entry:
cpe, parent = entry.split(":", 1)
else:
errors.append(f"exception_cpes entry ignored: {entry!r} is missing ':'")
continue
elif isinstance(entry, dict):
cpe = entry.get("cpe")
parent = entry.get("parent")
else:
cpe = getattr(entry, "cpe", None)
parent = getattr(entry, "parent", None)
cpe = str(cpe or "").strip()
parent = str(parent or "").strip()
if not cpe or not parent:
errors.append(f"exception_cpes entry ignored: {entry!r} is missing cpe or parent")
continue
normalized[cpe] = parent
return normalized, errors
def isInAllowedSubnets(inputIP):
# Check whether an IP address occurs inside the allowedSubnets list
isAllowed = False
@@ -203,8 +243,9 @@ class NetworkGraph:
parentId="", displayName="Shaper Root")
]
self.excludeSites = exclude_sites()
self.exceptionCPEs = exception_cpes()
self.errors: List[str] = []
self.exceptionCPEs, exception_cpe_errors = _normalize_exception_cpes(exception_cpes())
self.errors.extend(exception_cpe_errors)
# Initialize optimization structures
self._id_to_index = {"FakeRoot": 0}
@@ -735,8 +776,8 @@ class NetworkGraph:
device["mac"],
ipv4,
ipv6,
int(1),
int(1),
max(1.0, round(float(circuit["download"]*0.98), 2)),
max(1.0, round(float(circuit["upload"])*0.98, 2)),
max(1.0, round(float(circuit["download"]), 2)),
max(1.0, round(float(circuit["upload"]), 2)),
""
+94 -3
View File
@@ -1,5 +1,6 @@
from pythonCheck import checkPythonVersion
checkPythonVersion()
import ipaddress
import requests
import subprocess
import time
@@ -65,6 +66,10 @@ def sonarRadiusAccountDeviceNodeId(radius_account_id):
return f"sonar:radius-account:{radius_account_id}"
def sonarAccountIpAssignmentDeviceNodeId(account_id):
return f"sonar:account-ip-assignments:{account_id}"
def sonarSession():
global _SONAR_SESSION
if _SONAR_SESSION is None:
@@ -223,6 +228,23 @@ def dedupeSubnets(subnets):
return deduped
def splitIpVersions(subnets):
ipv4 = []
ipv6 = []
invalid = []
for subnet in subnets or []:
try:
network = ipaddress.ip_network(subnet, strict=False)
except ValueError:
invalid.append(subnet)
continue
if network.version == 4:
ipv4.append(subnet)
else:
ipv6.append(subnet)
return ipv4, ipv6, invalid
def normalizeServiceName(name):
return (name or "").strip().casefold()
@@ -236,6 +258,15 @@ def findRadiusAccountIPs(radius_account):
return dedupeSubnets(ips)
def findAccountIPs(account):
ips = []
for ip in account.get('ip_assignments', {}).get('entities', []):
subnet = ip.get('subnet')
if subnet:
ips.append(subnet)
return dedupeSubnets(ips)
def recurringServiceRateRules():
rules = {}
for enabled, service_name, download_mbps, upload_mbps in sonar_recurring_service_rates():
@@ -413,7 +444,8 @@ def buildAccountDevices(account):
'raw_id': item['id'],
'name': item['inventory_model']['name'],
'ips': ips,
'mac': findPrimaryMac(item)
'mac': findPrimaryMac(item),
'source': 'inventory_item',
})
for radius_account in account.get('radius_accounts', {}).get('entities', []):
@@ -429,7 +461,23 @@ def buildAccountDevices(account):
'raw_id': radius_account['id'],
'name': f"Radius Account {radius_account['id']}",
'ips': radius_ips,
'mac': ''
'mac': '',
'source': 'radius_account',
})
account_ips = [
subnet for subnet in findAccountIPs(account)
if subnet not in known_ip_subnets
]
if account_ips:
known_ip_subnets.update(account_ips)
devices.append({
'id': sonarAccountIpAssignmentDeviceNodeId(account['id']),
'raw_id': account['id'],
'name': 'Account IP Assignments',
'ips': account_ips,
'mac': '',
'source': 'account_ip_assignments',
})
return devices
@@ -520,6 +568,11 @@ def getAccounts(sonar_active_status_ids):
account_status_id
id
name
ip_assignments {
entities {
subnet
}
}
account_services (reverse_relation_filters: [$data]) {
entities {
service {
@@ -580,6 +633,11 @@ def getAccounts(sonar_active_status_ids):
entities {
id
name
ip_assignments {
entities {
subnet
}
}
account_services (reverse_relation_filters: [$data]) {
entities {
service {
@@ -729,6 +787,34 @@ def buildMacToApMap(aps):
mac_to_ap[cpe_mac] = ap['id']
return mac_to_ap
def sonarImportSummary(sites, aps, accounts):
total_devices = 0
devices_with_ips = 0
ipv4_assignments = 0
ipv6_assignments = 0
account_level_ip_devices = 0
accounts_with_parent = 0
for account in accounts:
if account.get('parent'):
accounts_with_parent += 1
for device in account.get('devices', []):
total_devices += 1
if len(device.get('ips') or []) > 0:
devices_with_ips += 1
ipv4, ipv6, _invalid = splitIpVersions(device.get('ips', []))
ipv4_assignments += len(ipv4)
ipv6_assignments += len(ipv6)
if device.get('source') == 'account_ip_assignments':
account_level_ip_devices += 1
return (
f"Sonar import summary: sites={len(sites)} aps={len(aps)} "
f"accounts={len(accounts)} accounts_with_ap_parent={accounts_with_parent} "
f"devices={total_devices} devices_with_ips={devices_with_ips} "
f"ipv4_assignments={ipv4_assignments} ipv6_assignments={ipv6_assignments} "
f"account_level_ip_devices={account_level_ip_devices}"
)
def createShaper():
net = NetworkGraph()
@@ -757,6 +843,8 @@ def createShaper():
if account['parent']:
break
print(sonarImportSummary(sites, aps, accounts))
for site in sites:
net.addRawNode(NetworkNode(id=site['id'], displayName=site['name'], parentId="", type=NodeType.site, networkJsonId=f"sonar:site:{site['raw_id']}"))
@@ -772,7 +860,10 @@ def createShaper():
net.addRawNode(customer)
for device in account['devices']:
libre_device = NetworkNode(id=device['id'], displayName=device['name'],parentId=account['id'],type=NodeType.device, ipv4=device['ips'],ipv6=[],mac=device['mac'])
ipv4, ipv6, invalid_ips = splitIpVersions(device.get('ips', []))
if invalid_ips:
print(f"Skipping invalid Sonar IP assignment(s) for {device['id']}: {invalid_ips}")
libre_device = NetworkNode(id=device['id'], displayName=device['name'],parentId=account['id'],type=NodeType.device, ipv4=ipv4,ipv6=ipv6,mac=device['mac'])
net.addRawNode(libre_device)
net.prepareTree()
+1 -44
View File
@@ -90,36 +90,6 @@ def normalize_splynx_graph_id(value):
return str(value)
def splynx_generated_unattached_site_id():
"""
Stable raw graph ID for Splynx full-mode circuits with no inferred parent.
"""
return "splynx_generated_unattached_site"
def splynx_generated_unattached_site_network_id():
"""
Stable exported topology ID for Splynx full-mode fallback grouping.
"""
return "libreqos:generated:splynx:site:unattached"
def splynx_generated_unattached_site_name():
"""
Display name for the explicit Splynx full-mode fallback site.
"""
return "LibreQoS Unattached [Site]"
def splynx_generated_unattached_parent_id(strategy_name: str):
"""
Only full mode should materialize a stable generated site for unresolved parents.
Other modes handle unresolved circuits through their own compiler/runtime fallbacks.
"""
if strategy_name == "full":
return splynx_generated_unattached_site_id()
return None
def supplement_existing_devices_with_online_ips(net, allServices, service_ids_handled, customersOnline, cust_id_to_name, allocated_ipv4s, allocated_ipv6s, device_by_service_id=None):
"""
For services already handled (devices created from static IPs), supplement missing
@@ -424,19 +394,6 @@ def run_splynx_pipeline(strategy_name: str):
# ap_site and full
print("Creating site and AP infrastructure")
createInfrastructureNodes(net, monitoring, hardware_name, hardware_parent, hardware_type, siteBandwidth, hardware_name_extended)
if strategy_name == 'full':
net.addRawNode(
NetworkNode(
id=splynx_generated_unattached_site_id(),
displayName=splynx_generated_unattached_site_name(),
type=NodeType.site,
parentId=None,
download=10000,
upload=10000,
address=splynx_generated_unattached_site_name(),
networkJsonId=splynx_generated_unattached_site_network_id(),
)
)
# Parent selector per strategy
def select_parent(serviceItem):
@@ -450,7 +407,7 @@ def run_splynx_pipeline(strategy_name: str):
return None
if strategy_name == 'ap_only':
return parent_node_id if (parent_node_id in ap_nodes) else None
return parent_node_id or splynx_generated_unattached_parent_id(strategy_name)
return parent_node_id
# Run pipeline
build_infrastructure()
+10
View File
@@ -0,0 +1,10 @@
from typing import Any
def normalize_mac(mac: Any) -> str:
if mac is None:
return ""
value = str(mac).strip()
if not value:
return ""
return value.replace(":", "").replace("-", "").upper()
+5 -14
View File
@@ -34,6 +34,7 @@ from integrationCommon import (
apply_client_bandwidth_multiplier,
isIntegrationOutputIpAllowed,
)
from integrationUtils import normalize_mac
TOKEN_URL = "https://data.visp.net/token"
@@ -71,16 +72,6 @@ def _unit_to_mbps(value: Any, unit: Any) -> Optional[float]:
return v
def _normalize_mac(mac: Any) -> str:
if mac is None:
return ""
s = str(mac).strip()
if not s:
return ""
# VISP sometimes returns MACs without separators. Keep uppercase hex.
return s.replace(":", "").replace("-", "").upper()
def _safe_isp_id_from_token_payload(payload: Dict[str, Any]) -> Optional[int]:
isp_ids = payload.get("ispId")
if isinstance(isp_ids, list) and isp_ids:
@@ -505,7 +496,7 @@ def _extract_mac_candidates(row: Dict[str, Any]) -> List[str]:
if not isinstance(equipment_data, dict):
equipment_data = {}
for key in ("Fiber MAC", "mac_address", "Ethernet MAC", "WAN MAC", "MAC Addr", "radio_mac"):
mac = _normalize_mac(equipment_data.get(key))
mac = normalize_mac(equipment_data.get(key))
if mac and mac not in seen:
seen.add(mac)
candidates.append(mac)
@@ -921,7 +912,7 @@ def createShaper() -> None:
if eq_id > 0:
equipment_ids.add(eq_id)
for key in ("mac_address", "radio_mac"):
mac = _normalize_mac(eq.get(key))
mac = normalize_mac(eq.get(key))
if mac:
preferred_macs.add(mac)
for ws in ws_list:
@@ -947,7 +938,7 @@ def createShaper() -> None:
if not ipv4s:
counters["skipped_no_ip"] += 1
continue
mac = _normalize_mac(ws.get("mac_address"))
mac = normalize_mac(ws.get("mac_address"))
if mac:
preferred_macs.add(mac)
add_service_record(
@@ -1032,7 +1023,7 @@ def createShaper() -> None:
ipv4s = list(ip_cache)
except Exception:
ipv4s = list(ip_cache)
mac = _normalize_mac(service_details.get("mac_address"))
mac = normalize_mac(service_details.get("mac_address"))
if not mac and preferred_macs:
mac = sorted(preferred_macs)[0]
+2
View File
@@ -25,12 +25,14 @@ disable_offload = [ "gso", "tso", "lro", "sg", "gro" ]
use_xdp_bridge = false
to_internet = "eth0"
to_network = "eth1"
# mtu = 1500
# OR:
#[single_interface]
#interface = "eth0"
#internet_vlan = 2
#network_vlan = 3
# mtu = 1500
[queues]
# Global default SQM is a single token list applied to both directions.
+204 -52
View File
@@ -1,12 +1,39 @@
#!/usr/bin/python3
import csv
import ipaddress
import json
import sys
from enum import IntEnum
from typing import NamedTuple
import routeros_api
from integrationUtils import normalize_mac
from liblqos_python import load_mikrotik_ipv6_routers_json, mikrotik_ipv6_config_path
class SourcePriority(IntEnum):
ARP = 0
NEIGHBOR = 1
DHCP = 2
class Ipv6Rank(IntEnum):
INVALID = -1
LINK_LOCAL = 0
HOST = 1
PREFIX = 2
class IpRecord(NamedTuple):
source_priority: SourcePriority
address_rank: int
canonical_address: str
address: str
ambiguous: bool = False
class ClientAddressRecord(NamedTuple):
canonical_address: str
address: str
ambiguous: bool = False
def _load_legacy_router_csv(csv_path):
router_list = []
with open(csv_path) as csv_file:
@@ -32,13 +59,182 @@ def _load_router_list(configPath=None):
return _load_legacy_router_csv(configPath)
raise ValueError("Explicit Mikrotik IPv6 config overrides must currently point to a legacy .csv file")
def pullMikrotikIPv6(configPath=None):
import routeros_api
def _parse_ip_address(address):
try:
return ipaddress.ip_address(address.split('/')[0])
except (AttributeError, ValueError):
return None
def _canonical_ip_text(address):
parsed_address = _parse_ip_address(address)
if parsed_address is None:
return None
return str(parsed_address)
def _canonical_ip_network_text(address):
try:
network = ipaddress.ip_network(address, strict=False)
except (AttributeError, ValueError):
return None
return network.with_prefixlen
def _entry_value(entry, key):
if not isinstance(entry, dict):
return None
return entry.get(key)
def _record_address_by_mac(entry, mac_to_ipv4, mac_to_ipv6):
mac = _entry_value(entry, 'mac-address')
address = _entry_value(entry, 'address')
parsed_address = _parse_ip_address(address)
if mac is None or parsed_address is None:
return
normalized_mac = normalize_mac(mac)
if parsed_address.version == 4:
_record_ipv4_for_mac(mac_to_ipv4, normalized_mac, address, source_priority=SourcePriority.ARP)
else:
_record_ipv6_for_mac(mac_to_ipv6, normalized_mac, address, source_priority=SourcePriority.ARP)
def _record_dhcp4_address_by_mac(entry, mac_to_ipv4):
mac = _entry_value(entry, 'mac-address')
address = _entry_value(entry, 'address')
parsed_address = _parse_ip_address(address)
if mac is None or parsed_address is None or parsed_address.version != 4:
return
_record_ipv4_for_mac(mac_to_ipv4, normalize_mac(mac), address, source_priority=SourcePriority.DHCP)
def _mac_from_dhcpv6_duid(duid):
duid_hex = str(duid).strip().replace(":", "").replace("-", "").lower()
if duid_hex.startswith("0x"):
duid_hex = duid_hex[2:]
try:
duid_bytes = bytes.fromhex(duid_hex)
except ValueError:
return None
if len(duid_bytes) == 14 and duid_bytes[0:2] == b'\x00\x01':
return normalize_mac(duid_bytes[8:14].hex())
if len(duid_bytes) == 10 and duid_bytes[0:2] == b'\x00\x03':
return normalize_mac(duid_bytes[4:10].hex())
return None
def _record_ipv4_for_mac(mac_to_ipv4, mac, address, source_priority):
if not mac or not address:
return
parsed_address = _parse_ip_address(address)
if parsed_address is None:
return
current = mac_to_ipv4.get(mac)
candidate = IpRecord(source_priority, int(parsed_address), str(parsed_address), address)
if current is None or source_priority > current.source_priority:
mac_to_ipv4[mac] = candidate
elif source_priority == current.source_priority and candidate.canonical_address != current.canonical_address:
mac_to_ipv4[mac] = current._replace(ambiguous=True)
def _ipv6_rank(address):
try:
network = ipaddress.ip_network(address, strict=False)
except ValueError:
return Ipv6Rank.INVALID
if network.version != 6:
return Ipv6Rank.INVALID
ip = network.network_address
if ip.is_unspecified or ip.is_loopback or ip.is_multicast:
return Ipv6Rank.INVALID
if ip.is_link_local:
address_priority = Ipv6Rank.LINK_LOCAL
elif network.prefixlen < 128:
address_priority = Ipv6Rank.PREFIX
else:
address_priority = Ipv6Rank.HOST
return address_priority
def _record_ipv6_for_mac(mac_to_ipv6, mac, address, source_priority):
if not mac or not address:
return
address_rank = _ipv6_rank(address)
if address_rank in (Ipv6Rank.INVALID, Ipv6Rank.LINK_LOCAL):
return
canonical_address = _canonical_ip_network_text(address)
if canonical_address is None:
return
current = mac_to_ipv6.get(mac)
candidate = IpRecord(source_priority, address_rank, canonical_address, address)
if current is None or source_priority > current.source_priority:
mac_to_ipv6[mac] = candidate
elif source_priority == current.source_priority:
if address_rank > current.address_rank:
mac_to_ipv6[mac] = candidate
elif address_rank == current.address_rank and canonical_address != current.canonical_address:
mac_to_ipv6[mac] = current._replace(ambiguous=True)
def _record_client_address_binding(client_address_to_ipv6, client_address, address):
if not client_address or not address or _ipv6_rank(address) in (Ipv6Rank.INVALID, Ipv6Rank.LINK_LOCAL):
return
canonical_address = _canonical_ip_network_text(address)
if canonical_address is None:
return
current = client_address_to_ipv6.get(client_address)
if current is None:
client_address_to_ipv6[client_address] = ClientAddressRecord(canonical_address, address)
elif canonical_address != current.canonical_address:
client_address_to_ipv6[client_address] = current._replace(ambiguous=True)
def _build_ipv4_to_ipv6_map(arp_entries, dhcp4_entries, dhcp6_bindings, neighbor6_entries):
ipv4_to_ipv6 = {}
mac_to_ipv4 = {}
mac_to_ipv6 = {}
client_address_to_ipv6 = {}
for entry in arp_entries:
_record_address_by_mac(entry, mac_to_ipv4, mac_to_ipv6)
for entry in dhcp4_entries:
_record_dhcp4_address_by_mac(entry, mac_to_ipv4)
for entry in dhcp6_bindings:
mac = _mac_from_dhcpv6_duid(_entry_value(entry, 'duid') or '')
address = _entry_value(entry, 'address')
if mac is not None and address:
_record_ipv6_for_mac(mac_to_ipv6, mac, address, source_priority=SourcePriority.DHCP)
elif _entry_value(entry, 'client-address') and address:
client_address = _canonical_ip_text(_entry_value(entry, 'client-address'))
if client_address is not None:
_record_client_address_binding(client_address_to_ipv6, client_address, address)
for entry in neighbor6_entries:
mac = _entry_value(entry, 'mac-address')
address = _entry_value(entry, 'address')
if mac is None or _parse_ip_address(address) is None:
continue
normalized_mac = normalize_mac(mac)
neighbor_address = _canonical_ip_text(address)
mapped_ipv6_record = client_address_to_ipv6.get(neighbor_address)
if mapped_ipv6_record is not None and mapped_ipv6_record.ambiguous:
print('Skipped ambiguous DHCPv6 client-address entries for ' + neighbor_address, file=sys.stderr)
_record_ipv6_for_mac(mac_to_ipv6, normalized_mac, address, source_priority=SourcePriority.NEIGHBOR)
elif mapped_ipv6_record is not None:
_record_ipv6_for_mac(mac_to_ipv6, normalized_mac, mapped_ipv6_record.address, source_priority=SourcePriority.DHCP)
else:
_record_ipv6_for_mac(mac_to_ipv6, normalized_mac, address, source_priority=SourcePriority.NEIGHBOR)
for mac, ipv6_record in mac_to_ipv6.items():
if ipv6_record.ambiguous:
print('Skipped ambiguous IPv6 entries for MAC ' + mac, file=sys.stderr)
continue
ipv6 = ipv6_record.address
try:
ipv4_record = mac_to_ipv4[mac]
if ipv4_record.ambiguous:
print('Skipped ambiguous IPv4 entries for MAC ' + mac, file=sys.stderr)
continue
ipv4 = ipv4_record.address
ipv4_to_ipv6[ipv4] = ipv6
except KeyError:
print('Failed to find associated IPv4 for ' + ipv6, file=sys.stderr)
return ipv4_to_ipv6
def pullMikrotikIPv6(configPath=None):
ipv4ToIPv6 = {}
routerList = _load_router_list(configPath)
for router in routerList:
RouterName = router['name']
IP = router['host']
inputUsername = router['username']
inputPassword = router['password']
@@ -47,62 +243,18 @@ def pullMikrotikIPv6(configPath=None):
plaintext_login = bool(router.get('plaintext_login', True))
connection = routeros_api.RouterOsApiPool(IP, username=inputUsername, password=inputPassword, port=apiPort, use_ssl=use_ssl, ssl_verify=False, ssl_verify_hostname=False, plaintext_login=plaintext_login)
api = connection.get_api()
macToIPv4 = {}
macToIPv6 = {}
clientAddressToIPv6 = {}
# list_dhcp4 = api.get_resource('/ip/dhcp-server/lease')
# entries = list_dhcp4.get()
# for entry in entries:
# try:
# macToIPv4[entry['mac-address']] = entry['address']
# except:
# pass
list_arp4 = api.get_resource('/ip/arp')
entries = list_arp4.get()
for entry in entries:
try:
macToIPv4[entry['mac-address']] = entry['address']
except:
pass
arp_entries = list_arp4.get()
list_dhcp4 = api.get_resource('/ip/dhcp-server/lease')
entries = list_dhcp4.get()
for entry in entries:
try:
macToIPv4[entry['mac-address']] = entry['address']
except:
pass
dhcp4_entries = list_dhcp4.get()
list_binding6 = api.get_resource('/ipv6/dhcp-server/binding')
entries = list_binding6.get()
for entry in entries:
if len(entry['duid']) == 14:
mac = entry['duid'][2:14].upper()
macNew = mac[0:2] + ':' + mac[2:4] + ':' + mac[4:6] + ':' + mac[6:8] + ':' + mac[8:10] + ':' + mac[10:12]
macToIPv6[macNew] = entry['address']
else:
try:
clientAddressToIPv6[entry['client-address']] = entry['address']
except:
pass
dhcp6_bindings = list_binding6.get()
list_neighbor6 = api.get_resource('/ipv6/neighbor')
entries = list_neighbor6.get()
for entry in entries:
try:
realIPv6 = clientAddressToIPv6[entry['address']]
macToIPv6[entry['mac-address']] = realIPv6
except:
pass
for mac, ipv6 in macToIPv6.items():
try:
ipv4 = macToIPv4[mac]
ipv4ToIPv6[ipv4] = ipv6
except:
print('Failed to find associated IPv4 for ' + ipv6, file=sys.stderr)
neighbor6_entries = list_neighbor6.get()
ipv4ToIPv6.update(_build_ipv4_to_ipv6_map(arp_entries, dhcp4_entries, dhcp6_bindings, neighbor6_entries))
return json.dumps(ipv4ToIPv6)
def pullMikrotikIPv6_Mock(CsvPath):
return "{\n\"172.29.200.2\": \"2602:fdca:800:1500::/56\"\n}"
if __name__ == '__main__':
# If the first argument is a string, it's treated as a legacy CSV path override.
if len(sys.argv) > 1 and sys.argv[1] == '--show-config':
+9
View File
@@ -8,3 +8,12 @@ username = "admin"
password = "password"
use_ssl = false
plaintext_login = true
[[router]]
name = "secondary"
host = "100.64.0.2"
port = 8728
username = "admin"
password = "password"
use_ssl = false
plaintext_login = true
+2 -2
View File
@@ -3737,9 +3737,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.103.12"
version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
"aws-lc-rs",
"ring",
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -269,7 +269,7 @@ fn memory_guard_failure_summary(
phase: &str,
) -> String {
format!(
"Bakery memory guard stopped {purpose} {phase} chunk {}/{}: available memory {} bytes is below safety floor {} bytes (total RAM {} bytes).",
"Bakery memory guard stopped {purpose} {phase} chunk {}/{}: available memory {} bytes is below required available memory {} bytes (total RAM {} bytes).",
chunk_number,
total_chunks,
snapshot.available_bytes,
+43
View File
@@ -2,9 +2,11 @@
// SPDX-License-Identifier: AGPL-3.0-or-later WITH LicenseRef-LibreQoS-Exception
use crate::{BUS_SOCKET_PATH, BusReply, BusRequest, BusResponse, BusSession, bus::BusClientError};
use std::time::Duration;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::UnixStream,
time::timeout,
};
use tracing::error;
@@ -50,6 +52,16 @@ impl LibreqosBusClient {
})
}
/// Creates a new `LibreqosBusClient`, failing if connection setup exceeds
/// `request_timeout`.
///
/// Side effects: opens a Unix socket connection to the local `lqosd` bus.
pub async fn new_with_timeout(request_timeout: Duration) -> Result<Self, BusClientError> {
timeout(request_timeout, Self::new())
.await
.map_err(|_| BusClientError::TimedOut)?
}
/// Sends a request to the bus and waits for a response.
///
/// ## Arguments
@@ -91,6 +103,21 @@ impl LibreqosBusClient {
}
Ok(response.responses)
}
/// Sends a request to the bus and waits for a bounded response.
///
/// Side effects: writes the request to the connected Unix socket. If this
/// call times out, discard the client because the stream may later receive
/// the timed-out reply.
pub async fn request_with_timeout(
&mut self,
requests: Vec<BusRequest>,
request_timeout: Duration,
) -> Result<Vec<BusResponse>, BusClientError> {
timeout(request_timeout, self.request(requests))
.await
.map_err(|_| BusClientError::TimedOut)?
}
}
/// Convenient wrapper for accessing the bus, for a single request-response cycle. This
@@ -108,3 +135,19 @@ pub async fn bus_request(requests: Vec<BusRequest>) -> Result<Vec<BusResponse>,
};
client.request(requests).await
}
/// Convenient wrapper for a single bus request with an end-to-end timeout.
///
/// Side effects: opens a Unix socket connection, sends the request, and closes
/// the connection when the client is dropped.
pub async fn bus_request_with_timeout(
requests: Vec<BusRequest>,
request_timeout: Duration,
) -> Result<Vec<BusResponse>, BusClientError> {
timeout(request_timeout, async move {
let mut client = LibreqosBusClient::new().await?;
client.request(requests).await
})
.await
.map_err(|_| BusClientError::TimedOut)?
}
+4 -1
View File
@@ -9,7 +9,7 @@ mod request;
pub mod response;
mod session;
mod unix_socket_server;
pub use client::{LibreqosBusClient, bus_request};
pub use client::{LibreqosBusClient, bus_request, bus_request_with_timeout};
pub use queue_data::*;
pub use reply::BusReply;
pub use request::{
@@ -55,6 +55,9 @@ pub enum BusClientError {
/// Reading the reply from the socket failed.
#[error("Cannot read from socket")]
StreamReadError,
/// The socket operation exceeded the caller's deadline.
#[error("Timed out waiting for bus response")]
TimedOut,
/// The socket connection is no longer usable.
#[error("Stream is no longer connected")]
StreamNotConnected,
+13
View File
@@ -171,6 +171,19 @@ mod tests {
assert_eq!(decoded.requests, session.requests);
}
#[test]
fn cbor_round_trip_clear_urgent_issue_by_identity_request() {
let session = BusSession {
requests: vec![BusRequest::ClearUrgentIssueByIdentity {
code: "XDP_IP_MAPPING_APPLY_FAILED".to_string(),
dedupe_key: "XDP_IP_MAPPING_APPLY_FAILED".to_string(),
}],
};
let bytes = encode_session_cbor(&session).expect("encode_session_cbor");
let decoded = decode_session_cbor(&bytes).expect("decode_session_cbor");
assert_eq!(decoded.requests, session.requests);
}
#[test]
fn cbor_round_trip_reply() {
let reply = BusReply {
+8
View File
@@ -539,6 +539,14 @@ pub enum BusRequest {
/// Clear a specific urgent issue by ID
ClearUrgentIssue(u64),
/// Clear urgent issues matching a code and dedupe key
ClearUrgentIssueByIdentity {
/// Machine-readable issue code
code: String,
/// Dedupe key used when the issue was submitted
dedupe_key: String,
},
/// Clear all urgent issues
ClearAllUrgentIssues,
@@ -5,21 +5,63 @@ use crate::{
BUS_SOCKET_PATH, BusReply, BusRequest, BusResponse,
bus::client::{MAGIC_NUMBER, MAGIC_RESPONSE},
};
use std::{ffi::CString, fs::remove_file};
use std::{ffi::CString, fs::remove_file, time::Duration};
use thiserror::Error;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::UnixListener,
task::spawn_blocking,
time::timeout,
};
use tracing::{debug, error, info, warn};
use super::BUS_SOCKET_DIRECTORY;
use super::protocol::{decode_session_cbor, encode_reply_cbor, read_frame, write_frame};
const BUS_HANDLER_TIMEOUT: Duration = Duration::from_secs(30);
fn dropped_reply_response_count(reply: &BusReply) -> usize {
reply.responses.len()
}
fn timeout_reply(request_count: usize) -> BusReply {
BusReply {
responses: (0..request_count)
.map(|_| BusResponse::Fail("Bus request handler timed out".to_string()))
.collect(),
}
}
async fn handle_requests_with_deadline(
handle_bus_requests: fn(&[BusRequest], &mut Vec<BusResponse>),
requests: Vec<BusRequest>,
) -> BusReply {
let request_count = requests.len();
let handler = spawn_blocking(move || {
let mut response = BusReply {
responses: Vec::with_capacity(request_count),
};
handle_bus_requests(&requests, &mut response.responses);
response
});
match timeout(BUS_HANDLER_TIMEOUT, handler).await {
Ok(Ok(response)) => response,
Ok(Err(err)) => {
warn!("Bus request handler task failed: {err}");
BusReply {
responses: (0..request_count)
.map(|_| BusResponse::Fail("Bus request handler failed".to_string()))
.collect(),
}
}
Err(_) => {
warn!("Bus request handler timed out after {BUS_HANDLER_TIMEOUT:?}");
timeout_reply(request_count)
}
}
}
/// Implements a Tokio-friendly server using Unix Sockets and the bus protocol.
/// Requests are handled and then forwarded to the handler.
pub struct UnixSocketServer {}
@@ -113,8 +155,7 @@ impl UnixSocketServer {
ret = bus_rx.recv() => {
// We received a channel-based message
if let Some((reply_channel, msg)) = ret {
let mut response = BusReply { responses: Vec::with_capacity(8) };
handle_bus_requests(&[msg], &mut response.responses);
let response = handle_requests_with_deadline(handle_bus_requests, vec![msg]).await;
if let Err(reply) = reply_channel.send(response) {
warn!(
dropped_response_count = dropped_reply_response_count(&reply),
@@ -182,8 +223,9 @@ impl UnixSocketServer {
debug!("Received request: {:?}", request);
// Handle the request and build the response
let mut response = BusReply { responses: Vec::with_capacity(8) };
handle_bus_requests(&request.requests, &mut response.responses);
let response =
handle_requests_with_deadline(handle_bus_requests, request.requests)
.await;
// Encode the response
let Ok(encoded_response) = encode_reply_cbor(&response) else {
+1 -1
View File
@@ -37,7 +37,7 @@ pub use bus::{
BUS_SOCKET_PATH, BakeryCapacityReportInterface, BlackboardSystem, BusClientError, BusReply,
BusRequest, BusResponse, BusSession, CakeDiffTinTransit, CakeDiffTransit, CakeTransit,
LibreqosBusClient, QueueStoreTransit, SchedulerProgressReport, TopFlowType, UnixSocketServer,
UrgentSeverity, UrgentSource, bus_request,
UrgentSeverity, UrgentSource, bus_request, bus_request_with_timeout,
};
pub use tc_handle::TcHandle;
@@ -182,6 +182,7 @@ fn migrate_bridge(
interface: python_config.interface_a.clone(),
internet_vlan: python_config.stick_vlan_a as u32,
network_vlan: python_config.stick_vlan_b as u32,
mtu: None,
});
} else {
new_config.single_interface = None;
@@ -193,6 +194,7 @@ fn migrate_bridge(
.use_xdp_bridge,
to_internet: python_config.interface_b.clone(),
to_network: python_config.interface_a.clone(),
mtu: None,
});
}
Ok(())
@@ -77,12 +77,6 @@ pub struct PythonMigration {
pub influx_dbtoken: String,
#[serde(rename = "circuitNameUseAddress")]
pub circuit_name_use_address: bool,
#[serde(
default,
rename = "overwriteNetworkJSONalways",
alias = "always_overwrite_network_json"
)]
pub overwrite_network_jsonalways: bool,
#[serde(rename = "ignoreSubnets")]
pub ignore_subnets: Vec<String>,
#[serde(rename = "allowedSubnets")]
@@ -177,89 +171,3 @@ impl PythonMigration {
}
}
}
#[cfg(test)]
mod tests {
use super::PythonMigration;
fn minimal_python_migration_json() -> &'static str {
r#"{
"sqm": "cake diffserv4",
"monitorOnlyMode": false,
"upstreamBandwidthCapacityDownloadMbps": 1000,
"upstreamBandwidthCapacityUploadMbps": 1000,
"generatedPNDownloadMbps": 1000,
"generatedPNUploadMbps": 1000,
"interfaceA": "ens20",
"interfaceB": "ens19",
"queueRefreshIntervalMins": 30,
"OnAStick": false,
"StickVlanA": 0,
"StickVlanB": 0,
"enableActualShellCommands": true,
"runShellCommandsAsSudo": false,
"queuesAvailableOverride": 0,
"useBinPackingToBalanceCPU": false,
"influxEnabled": false,
"influxDBurl": "http://localhost:8086",
"influxDBBucket": "libreqos",
"influxDBOrg": "libreqos",
"influxDBtoken": "",
"circuitNameUseAddress": false,
"ignoreSubnets": ["192.168.0.0/16"],
"allowedSubnets": ["100.64.0.0/10"],
"excludeSites": [],
"findIPv6usingMikrotikAPI": false,
"automaticImportSplynx": false,
"splynx_api_key": "",
"splynx_api_secret": "",
"splynx_api_url": "",
"automaticImportNetzur": false,
"netzur_api_key": "",
"netzur_api_url": "",
"netzur_api_timeout": 60,
"automaticImportUISP": false,
"uispAuthToken": "",
"UISPbaseURL": "",
"uispSite": "",
"uispStrategy": "full",
"uispSuspendedStrategy": "none",
"airMax_capacity": 1.0,
"ltu_capacity": 1.0,
"bandwidthOverheadFactor": 1.0,
"committedBandwidthMultiplier": 0.98,
"exceptionCPEs": {},
"apiUsername": "",
"apiPassword": "",
"apiHostIP": "127.0.0.1",
"apiHostPost": 5000,
"automaticImportPowercode": false,
"powercode_api_key": "",
"powercode_api_url": "",
"automaticImportSonar": false,
"sonar_api_key": "",
"sonar_api_url": "",
"snmp_community": "public",
"sonar_active_status_ids": [],
"sonar_airmax_ap_model_ids": [],
"sonar_ltu_ap_model_ids": []
}"#
}
#[test]
fn python_migration_defaults_missing_deprecated_overwrite_flag() {
let parsed: PythonMigration =
serde_json::from_str(minimal_python_migration_json()).expect("json should parse");
assert!(!parsed.overwrite_network_jsonalways);
}
#[test]
fn python_migration_accepts_snake_case_deprecated_overwrite_flag_alias() {
let raw = minimal_python_migration_json().replace(
"\"circuitNameUseAddress\": false,",
"\"circuitNameUseAddress\": false,\n \"always_overwrite_network_json\": true,",
);
let parsed: PythonMigration = serde_json::from_str(&raw).expect("json should parse");
assert!(parsed.overwrite_network_jsonalways);
}
}
@@ -123,9 +123,6 @@ influxDBtoken = \"\"
# Use Customer Name or Address as Circuit Name
circuitNameUseAddress = True
# Legacy network.json overwrite flag from older configs. Modern integration mode ignores it.
overwriteNetworkJSONalways = False
# If a device shows a WAN IP within these subnets, assume they are behind NAT / un-shapable, and ignore them
ignoreSubnets = ['192.168.0.0/16']
allowedSubnets = ['100.64.0.0/10']
@@ -16,6 +16,10 @@ pub struct BridgeConfig {
/// The name of the second interface, facing the LAN
pub to_network: String,
/// Optional MTU for LibreQoS-managed Linux bridge interfaces.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub mtu: Option<u32>,
}
impl Default for BridgeConfig {
@@ -24,6 +28,7 @@ impl Default for BridgeConfig {
use_xdp_bridge: true,
to_internet: "eth0".to_string(),
to_network: "eth1".to_string(),
mtu: None,
}
}
}
@@ -39,6 +44,10 @@ pub struct SingleInterfaceConfig {
/// The VLAN ID facing the LAN
pub network_vlan: u32,
/// Optional MTU for the LibreQoS-managed trunk interface.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub mtu: Option<u32>,
}
impl Default for SingleInterfaceConfig {
@@ -47,6 +56,7 @@ impl Default for SingleInterfaceConfig {
interface: "eth0".to_string(),
internet_vlan: 2,
network_vlan: 3,
mtu: None,
}
}
}
@@ -207,4 +207,50 @@ mod tests {
1
);
}
#[test]
fn config_roundtrips_with_multiple_dynamic_circuit_ranges() {
let mut config = Config::default();
let ipv4_rule = DynamicCircuitRangeRule {
name: "IPv4 default".to_string(),
ip_range: "0.0.0.0/0"
.parse()
.expect("default ipv4 network should parse"),
download_min_mbps: 10.0,
upload_min_mbps: 10.0,
download_max_mbps: 100.0,
upload_max_mbps: 100.0,
attach_to: "Tower-A".to_string(),
};
let ipv6_rule = DynamicCircuitRangeRule {
name: "IPv6 subscribers".to_string(),
ip_range: "2001:db8::/48"
.parse()
.expect("default ipv6 network should parse"),
download_min_mbps: 25.0,
upload_min_mbps: 5.0,
download_max_mbps: 250.0,
upload_max_mbps: 50.0,
attach_to: String::new(),
};
config.dynamic_circuits = Some(DynamicCircuitsConfig {
enabled: true,
ttl_seconds: 300,
enable_unknown_ip_promotion: true,
ranges: vec![ipv4_rule.clone(), ipv6_rule.clone()],
});
let raw = toml::to_string_pretty(&config).expect("config should serialize to TOML");
let parsed = Config::load_from_string(&raw).expect("config should deserialize");
let ranges = &parsed
.dynamic_circuits
.as_ref()
.expect("dynamic circuits should deserialize")
.ranges;
assert_eq!(ranges.len(), 2);
assert_eq!(ranges[0], ipv4_rule);
assert_eq!(ranges[1], ipv6_rule);
}
}
@@ -26,12 +26,14 @@ disable_offload = [ "gso", "tso", "lro", "sg", "gro" ]
use_xdp_bridge = true
to_internet = "eth0"
to_network = "eth1"
# mtu = 1500
# OR:
#[single_interface]
#interface = "eth0"
#internet_vlan = 2
#network_vlan = 3
# mtu = 1500
#[ssl]
#enabled = false
@@ -15,6 +15,9 @@ fn default_true() -> bool {
true
}
const MIN_INTERFACE_MTU: u32 = 576;
const MAX_INTERFACE_MTU: u32 = 9216;
fn default_rtt_green_ms() -> u32 {
0
}
@@ -31,6 +34,17 @@ fn default_false() -> bool {
false
}
fn validate_interface_mtu(field_name: &str, mtu: Option<u32>) -> Result<(), String> {
if let Some(mtu) = mtu
&& !(MIN_INTERFACE_MTU..=MAX_INTERFACE_MTU).contains(&mtu)
{
return Err(format!(
"{field_name} must be between {MIN_INTERFACE_MTU} and {MAX_INTERFACE_MTU}"
));
}
Ok(())
}
/// RTT color scale thresholds (milliseconds) used by the web UI.
///
/// These represent the color "anchor points" for RTT:
@@ -275,6 +289,12 @@ impl Config {
.to_string(),
);
}
if let Some(bridge) = &self.bridge {
validate_interface_mtu("bridge.mtu", bridge.mtu)?;
}
if let Some(single_interface) = &self.single_interface {
validate_interface_mtu("single_interface.mtu", single_interface.mtu)?;
}
if self.version.trim() != "1.5" {
return Err(format!(
"Configuration file is at version [{}], but this version of lqos only supports version 1.5.0",
@@ -626,6 +646,7 @@ impl Config {
#[cfg(test)]
mod test {
use super::super::bridge::{BridgeConfig, SingleInterfaceConfig};
use super::{Config, RttThresholds, SslConfig, normalize_external_hostname};
fn remove_sections(raw: &str, sections: &[&str]) -> String {
@@ -829,6 +850,86 @@ mod test {
assert!(cfg.validate().is_err());
}
#[test]
fn bridge_mtu_defaults_to_none_when_omitted() {
let config =
Config::load_from_string(include_str!("example.toml")).expect("example should load");
assert_eq!(config.bridge.as_ref().and_then(|bridge| bridge.mtu), None);
}
#[test]
fn interface_mtu_values_deserialize() {
let bridge_raw = include_str!("example.toml")
.replace("to_network = \"eth1\"", "to_network = \"eth1\"\nmtu = 9000");
let bridge_config = Config::load_from_string(&bridge_raw).expect("bridge MTU should load");
assert_eq!(
bridge_config.bridge.as_ref().and_then(|bridge| bridge.mtu),
Some(9000)
);
let single_raw = remove_sections(include_str!("example.toml"), &["bridge"])
+ r#"
[single_interface]
interface = "eth0"
internet_vlan = 2
network_vlan = 3
mtu = 1500
"#;
let single_config =
Config::load_from_string(&single_raw).expect("single-interface MTU should load");
assert_eq!(
single_config
.single_interface
.as_ref()
.and_then(|single| single.mtu),
Some(1500)
);
}
#[test]
fn interface_mtu_validation_rejects_out_of_range_values() {
let mut bridge_config = Config {
bridge: Some(BridgeConfig {
mtu: Some(575),
..BridgeConfig::default()
}),
..Config::default()
};
let bridge_error = bridge_config
.validate()
.expect_err("bridge.mtu below range should fail");
assert!(bridge_error.contains("bridge.mtu"));
bridge_config.bridge.as_mut().expect("bridge").mtu = Some(9217);
let bridge_error = bridge_config
.validate()
.expect_err("bridge.mtu above range should fail");
assert!(bridge_error.contains("bridge.mtu"));
let mut single_config = Config {
bridge: None,
single_interface: Some(SingleInterfaceConfig {
mtu: Some(575),
..SingleInterfaceConfig::default()
}),
..Config::default()
};
let single_error = single_config
.validate()
.expect_err("single_interface.mtu below range should fail");
assert!(single_error.contains("single_interface.mtu"));
single_config
.single_interface
.as_mut()
.expect("single interface")
.mtu = Some(9217);
let single_error = single_config
.validate()
.expect_err("single_interface.mtu above range should fail");
assert!(single_error.contains("single_interface.mtu"));
}
#[test]
fn treeguard_defaults_match_default_on_rollout() {
let cfg = Config::default();
@@ -27,9 +27,6 @@ pub struct UispIntegration {
pub bandwidth_overhead_factor: f32,
pub commit_bandwidth_multiplier: f32,
pub exception_cpes: Vec<ExceptionCpe>,
/// Deprecated legacy importer-side PtMP-parent toggle. Existing values are ignored.
#[serde(default, skip_serializing)]
pub use_ptmp_as_parent: bool,
#[serde(default = "default_ignore_calculated_capacity")]
pub ignore_calculated_capacity: bool,
#[serde(default = "default_infrastructure_transport_caps_enabled")]
@@ -69,7 +66,6 @@ impl Default for UispIntegration {
bandwidth_overhead_factor: 1.0,
commit_bandwidth_multiplier: 1.0,
exception_cpes: vec![],
use_ptmp_as_parent: false,
ignore_calculated_capacity: false,
infrastructure_transport_caps_enabled: default_infrastructure_transport_caps_enabled(),
insecure_ssl: None,
@@ -91,33 +87,4 @@ mod tests {
assert_eq!(config.ltu_capacity, 1.0);
assert!(config.infrastructure_transport_caps_enabled);
}
#[test]
fn deprecated_ptmp_parent_flag_loads_but_does_not_serialize() {
let config: UispIntegration = toml::from_str(
r#"
enable_uisp = true
token = ""
url = ""
site = ""
strategy = ""
suspended_strategy = ""
airmax_capacity = 1.0
ltu_capacity = 1.0
exclude_sites = []
ipv6_with_mikrotik = false
bandwidth_overhead_factor = 1.0
commit_bandwidth_multiplier = 1.0
exception_cpes = []
use_ptmp_as_parent = true
"#,
)
.expect("deprecated use_ptmp_as_parent key should still deserialize");
assert!(config.use_ptmp_as_parent);
let serialized =
toml::to_string(&config).expect("uisp integration config should serialize");
assert!(!serialized.contains("use_ptmp_as_parent"));
}
}
+9 -7
View File
@@ -93,13 +93,15 @@ pub use topology_runtime_state::{
TOPOLOGY_IMPORT_FILENAME, TOPOLOGY_RUNTIME_STATUS_FILENAME, TOPOLOGY_SHAPING_INPUTS_FILENAME,
TopologyAttachmentEndpointStatus, TopologyAttachmentHealthEntry,
TopologyAttachmentHealthStateFile, TopologyEffectiveAttachmentState,
TopologyEffectiveNodeState, TopologyEffectiveStateFile, TopologyRuntimeStateError,
TopologyRuntimeStatusFile, TopologyShapingCircuitInput, TopologyShapingDeviceInput,
TopologyShapingInputsFile, TopologyShapingResolutionSource, active_runtime_shaping_inputs_path,
compute_effective_network_generation, compute_topology_source_generation,
load_active_runtime_shaping_inputs, topology_attachment_health_state_path,
topology_compiled_shaping_path, topology_effective_network_path, topology_effective_state_path,
topology_import_path, topology_runtime_status_path, topology_shaping_inputs_path,
TopologyEffectiveNodeState, TopologyEffectiveStateFile, TopologyRuntimeShapingPayloadIdentity,
TopologyRuntimeStateError, TopologyRuntimeStatusFile, TopologyShapingCircuitInput,
TopologyShapingDeviceInput, TopologyShapingInputsFile, TopologyShapingResolutionSource,
active_runtime_shaping_inputs_path, compute_effective_network_generation,
active_runtime_shaping_inputs_path_from_status, compute_topology_source_generation,
load_active_runtime_shaping_inputs, load_active_runtime_shaping_inputs_from_status,
topology_attachment_health_state_path, topology_compiled_shaping_path,
topology_effective_network_path, topology_effective_state_path, topology_import_path,
topology_runtime_status_path, topology_shaping_inputs_path,
};
/// Returns whether an integration-backed topology ingress is enabled in the
+91
View File
@@ -834,6 +834,97 @@ mod tests {
assert_ne!(result.sites[0].class_minor, 0x29);
}
#[test]
fn class_identity_planner_compacts_when_full_reload_ignores_saved_minors() {
const FRAGMENTED_SITE_MINOR: u16 = 0xff00;
const FRAGMENTED_CHILD_MINOR: u16 = 0xff02;
const FRAGMENTED_CIRCUIT_MINOR: u16 = 0xff10;
const CIRCUIT_PADDING: u32 = 8;
let site_inputs = vec![
SiteIdentityInput {
site_key: "CpueQueue0/site-a".to_string(),
parent_path: "CpueQueue0".to_string(),
queue: 1,
has_children: true,
},
SiteIdentityInput {
site_key: "CpueQueue0/site-a/child".to_string(),
parent_path: "CpueQueue0/site-a".to_string(),
queue: 1,
has_children: false,
},
];
let circuit_groups = vec![CircuitIdentityGroupInput {
parent_node: "child".to_string(),
queue: 1,
circuit_ids: vec!["c1".to_string(), "c2".to_string()],
}];
let fragmented_sites = BTreeMap::from([
(
"CpueQueue0/site-a".to_string(),
PlannerSiteIdentityState {
class_minor: FRAGMENTED_SITE_MINOR,
queue: 1,
parent_path: "CpueQueue0".to_string(),
class_major: 1,
up_class_major: 0x41,
},
),
(
"CpueQueue0/site-a/child".to_string(),
PlannerSiteIdentityState {
class_minor: FRAGMENTED_CHILD_MINOR,
queue: 1,
parent_path: "CpueQueue0/site-a".to_string(),
class_major: 1,
up_class_major: 0x41,
},
),
]);
let fragmented_circuits = BTreeMap::from([(
"c1".to_string(),
PlannerCircuitIdentityState {
class_minor: FRAGMENTED_CIRCUIT_MINOR,
queue: 1,
parent_node: "child".to_string(),
class_major: 1,
up_class_major: 0x41,
},
)]);
let sticky_result = plan_class_identities(
&site_inputs,
&circuit_groups,
&fragmented_sites,
&fragmented_circuits,
0x40,
CIRCUIT_PADDING,
);
assert!(sticky_result.last_used_minor_by_queue[&1] > u32::from(FRAGMENTED_CIRCUIT_MINOR));
let full_reload_result = plan_class_identities(
&site_inputs,
&circuit_groups,
&BTreeMap::new(),
&BTreeMap::new(),
0x40,
CIRCUIT_PADDING,
);
let expected_last_minor = u32::from(full_reload_result.circuits[1].class_minor)
.saturating_add(1)
.saturating_add(CIRCUIT_PADDING);
assert_eq!(full_reload_result.sites[0].class_minor, 3);
assert_eq!(full_reload_result.sites[1].class_minor, 5);
assert_eq!(full_reload_result.circuits[0].class_minor, 6);
assert_eq!(full_reload_result.circuits[1].class_minor, 7);
assert_eq!(
full_reload_result.last_used_minor_by_queue[&1],
expected_last_minor
);
}
#[test]
fn class_identity_planner_runtime_constraints_prefer_runtime_band() {
let site_inputs = vec![SiteIdentityInput {
@@ -439,9 +439,9 @@ fn normalized_insight_node_kind(kind: &str) -> String {
}
}
fn short_node_suffix(node_id: &str) -> &str {
fn short_node_suffix(node_id: &str) -> String {
let short = node_id.rsplit(':').next().unwrap_or(node_id);
&short[..short.len().min(8)]
short.chars().take(8).collect()
}
fn unique_export_name(name: &str, node_id: &str, used_names: &mut HashSet<String>) -> String {
@@ -459,6 +459,87 @@ fn unique_export_name(name: &str, node_id: &str, used_names: &mut HashSet<String
fallback
}
fn duplicate_export_name(name: &str, node_id: &str, used_names: &mut HashSet<String>) -> String {
let short = format!("{name} [{}]", short_node_suffix(node_id));
if used_names.insert(short.clone()) {
return short;
}
let fallback = format!("{name} [{node_id}]");
used_names.insert(fallback.clone());
fallback
}
fn globally_unique_export_names(
nodes: &[TopologyCanonicalNode],
base_names_by_id: &HashMap<String, String>,
) -> HashMap<String, String> {
let mut node_ids_by_name = HashMap::<&str, HashSet<&str>>::new();
let mut site_name_owner_by_base = HashMap::<&str, Option<&str>>::new();
for node in nodes {
let Some(base_name) = base_names_by_id.get(&node.node_id) else {
continue;
};
node_ids_by_name
.entry(base_name.as_str())
.or_default()
.insert(node.node_id.as_str());
if normalized_insight_node_kind(&node.node_kind) == "Site" {
match site_name_owner_by_base.entry(base_name.as_str()) {
std::collections::hash_map::Entry::Vacant(entry) => {
entry.insert(Some(node.node_id.as_str()));
}
std::collections::hash_map::Entry::Occupied(mut entry) => {
if *entry.get() != Some(node.node_id.as_str()) {
entry.insert(None);
}
}
}
}
}
let mut ordered_nodes = nodes.iter().collect::<Vec<_>>();
ordered_nodes.sort_unstable_by(|left, right| {
let left_name = base_names_by_id
.get(&left.node_id)
.map(String::as_str)
.unwrap_or(left.node_name.as_str());
let right_name = base_names_by_id
.get(&right.node_id)
.map(String::as_str)
.unwrap_or(right.node_name.as_str());
left_name
.cmp(right_name)
.then_with(|| left.node_id.cmp(&right.node_id))
});
let mut used_names = HashSet::new();
let mut assigned_node_ids = HashSet::new();
let mut export_names = HashMap::new();
for node in ordered_nodes {
if !assigned_node_ids.insert(node.node_id.as_str()) {
continue;
}
let base_name = base_names_by_id
.get(&node.node_id)
.map(String::as_str)
.unwrap_or(node.node_name.as_str());
let duplicate_count = node_ids_by_name
.get(base_name)
.map(HashSet::len)
.unwrap_or_default();
let plain_site_owner = site_name_owner_by_base.get(base_name).copied().flatten();
let export_name = if duplicate_count > 1 && plain_site_owner != Some(node.node_id.as_str())
{
duplicate_export_name(base_name, &node.node_id, &mut used_names)
} else {
unique_export_name(base_name, &node.node_id, &mut used_names)
};
export_names.insert(node.node_id.clone(), export_name);
}
export_names
}
fn build_insight_logical_entry_json(
entry_id: &str,
entries: &HashMap<String, InsightLogicalNodeEntry>,
@@ -980,14 +1061,24 @@ impl TopologyCanonicalStateFile {
build_network_node_index(network_map, &mut by_id, &mut by_name);
}
let mut entries = HashMap::<String, InsightLogicalNodeEntry>::new();
let mut base_names_by_id = HashMap::<String, String>::new();
for node in &self.nodes {
let (download_mbps, upload_mbps) = canonical_node_rates(node);
let export_name = by_id
.get(&node.node_id)
.or_else(|| by_name.get(&node.node_name))
.map(|snapshot| snapshot.export_name.clone())
.unwrap_or_else(|| node.node_name.clone());
base_names_by_id.insert(node.node_id.clone(), export_name);
}
let export_names_by_id = globally_unique_export_names(&self.nodes, &base_names_by_id);
let mut entries = HashMap::<String, InsightLogicalNodeEntry>::new();
for node in &self.nodes {
let (download_mbps, upload_mbps) = canonical_node_rates(node);
let export_name = export_names_by_id
.get(&node.node_id)
.cloned()
.unwrap_or_else(|| node.node_name.clone());
entries.insert(
node.node_id.clone(),
InsightLogicalNodeEntry {
@@ -1338,6 +1429,89 @@ mod tests {
);
}
#[test]
fn native_integration_export_suffixes_duplicate_node_names() {
let canonical = TopologyCanonicalStateFile {
schema_version: 1,
source: "uisp/ap_site".to_string(),
generated_unix: Some(123),
ingress_identity: None,
ingress_kind: TopologyCanonicalIngressKind::NativeIntegration,
nodes: vec![
TopologyCanonicalNode {
node_id: "uisp:site:site-a".to_string(),
node_name: "Site A".to_string(),
node_kind: "Site".to_string(),
rate_input: super::TopologyCanonicalRateInput {
intrinsic_download_mbps: Some(1000),
intrinsic_upload_mbps: Some(1000),
..Default::default()
},
..Default::default()
},
TopologyCanonicalNode {
node_id: "uisp:site:site-b".to_string(),
node_name: "Site B".to_string(),
node_kind: "Site".to_string(),
rate_input: super::TopologyCanonicalRateInput {
intrinsic_download_mbps: Some(1000),
intrinsic_upload_mbps: Some(1000),
..Default::default()
},
..Default::default()
},
TopologyCanonicalNode {
node_id: "uisp:device:7c57e383-d0c4-47ac-af78-7b626a20d0a3".to_string(),
node_name: "Nanoswitch".to_string(),
node_kind: "AP".to_string(),
current_parent_node_id: Some("uisp:site:site-a".to_string()),
current_parent_node_name: Some("Site A".to_string()),
rate_input: super::TopologyCanonicalRateInput {
intrinsic_download_mbps: Some(95),
intrinsic_upload_mbps: Some(33),
..Default::default()
},
..Default::default()
},
TopologyCanonicalNode {
node_id: "uisp:device:01557d07-8a17-4263-98fb-80606c7dc613".to_string(),
node_name: "Nanoswitch".to_string(),
node_kind: "AP".to_string(),
current_parent_node_id: Some("uisp:site:site-b".to_string()),
current_parent_node_name: Some("Site B".to_string()),
rate_input: super::TopologyCanonicalRateInput {
intrinsic_download_mbps: Some(47),
intrinsic_upload_mbps: Some(28),
..Default::default()
},
..Default::default()
},
],
compatibility_network_json: json!({}),
};
let logical = canonical.insight_topology_network_json();
let site_a_children = logical["Site A"]["children"]
.as_object()
.expect("Site A should have child map");
let site_b_children = logical["Site B"]["children"]
.as_object()
.expect("Site B should have child map");
assert!(site_a_children.get("Nanoswitch").is_none());
assert!(site_b_children.get("Nanoswitch").is_none());
assert!(site_a_children.get("Nanoswitch [7c57e383]").is_some());
assert!(site_b_children.get("Nanoswitch [01557d07]").is_some());
assert_eq!(
site_a_children["Nanoswitch [7c57e383]"]["name"].as_str(),
Some("Nanoswitch [7c57e383]")
);
assert_eq!(
site_b_children["Nanoswitch [01557d07]"]["name"].as_str(),
Some("Nanoswitch [01557d07]")
);
}
#[test]
fn legacy_network_json_import_is_read_only_with_imported_rate_inputs() {
let canonical = TopologyCanonicalStateFile::from_legacy_network_json(&json!({
@@ -356,6 +356,20 @@ pub struct TopologyRuntimeStatusFile {
pub error: Option<String>,
}
/// Shaping-relevant runtime status identity used by consumers that only need
/// to know whether the active shaping payload changed.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct TopologyRuntimeShapingPayloadIdentity {
/// Whether runtime outputs are ready.
pub ready: bool,
/// Stable generation hash of the source topology inputs.
pub source_generation: String,
/// Stable generation hash of the shaping payload.
pub shaping_generation: String,
/// Path to the active shaping inputs payload.
pub shaping_inputs_path: String,
}
fn default_runtime_schema_version() -> u32 {
1
}
@@ -428,20 +442,33 @@ pub fn active_runtime_shaping_inputs_path(
) -> Result<Option<PathBuf>, TopologyRuntimeStateError> {
let current_generation = compute_topology_source_generation(config)?;
let status = TopologyRuntimeStatusFile::load(config)?;
Ok(active_runtime_shaping_inputs_path_from_status(
&status,
&current_generation,
))
}
/// Returns the active runtime shaping-inputs path from an already-loaded status file.
///
/// This function is pure: it has no side effects.
pub fn active_runtime_shaping_inputs_path_from_status(
status: &TopologyRuntimeStatusFile,
current_generation: &str,
) -> Option<PathBuf> {
if !status.ready {
return Ok(None);
return None;
}
if status.shaping_generation.trim().is_empty() {
return Ok(None);
return None;
}
if status.source_generation.trim() != current_generation {
return Ok(None);
return None;
}
let path = status.shaping_inputs_path.trim();
if path.is_empty() {
return Ok(None);
return None;
}
Ok(Some(PathBuf::from(path)))
Some(PathBuf::from(path))
}
/// Loads the currently active runtime shaping-inputs file when the topology
@@ -456,6 +483,20 @@ pub fn load_active_runtime_shaping_inputs(
Ok(Some(serde_json::from_str(&raw)?))
}
/// Loads the active runtime shaping-inputs file from an already-loaded status file.
pub fn load_active_runtime_shaping_inputs_from_status(
config: &Config,
status: &TopologyRuntimeStatusFile,
) -> Result<Option<TopologyShapingInputsFile>, TopologyRuntimeStateError> {
let current_generation = compute_topology_source_generation(config)?;
let Some(path) = active_runtime_shaping_inputs_path_from_status(status, &current_generation)
else {
return Ok(None);
};
let raw = std::fs::read_to_string(&path)?;
Ok(Some(serde_json::from_str(&raw)?))
}
/// Returns the path of the topology runtime readiness status file.
pub fn topology_runtime_status_path(config: &Config) -> PathBuf {
config.topology_state_read_path(TOPOLOGY_RUNTIME_STATUS_FILENAME)
@@ -694,7 +735,7 @@ fn normalized_json_value_for_generation(value: &Value) -> Value {
match value {
Value::Object(map) => {
let mut entries = map.iter().collect::<Vec<_>>();
entries.sort_unstable_by(|(left_key, _), (right_key, _)| left_key.cmp(right_key));
entries.sort_unstable_by_key(|(key, _)| *key);
let mut normalized = serde_json::Map::with_capacity(entries.len());
for (key, child) in entries {
normalized.insert(key.clone(), normalized_json_value_for_generation(child));
@@ -742,6 +783,29 @@ impl TopologyRuntimeStatusFile {
self,
)
}
/// Returns true when runtime status is equal except for the generated timestamp.
pub fn semantic_equals_for_publish(&self, other: &Self) -> bool {
self.schema_version == other.schema_version
&& self.source_generation == other.source_generation
&& self.shaping_generation == other.shaping_generation
&& self.effective_generation == other.effective_generation
&& self.ready == other.ready
&& self.effective_state_path == other.effective_state_path
&& self.effective_network_path == other.effective_network_path
&& self.shaping_inputs_path == other.shaping_inputs_path
&& self.error == other.error
}
/// Returns the shaping payload identity used by cache consumers.
pub fn shaping_payload_identity(&self) -> TopologyRuntimeShapingPayloadIdentity {
TopologyRuntimeShapingPayloadIdentity {
ready: self.ready,
source_generation: self.source_generation.clone(),
shaping_generation: self.shaping_generation.clone(),
shaping_inputs_path: self.shaping_inputs_path.clone(),
}
}
}
#[cfg(test)]
@@ -1300,6 +1364,41 @@ mod tests {
);
}
#[test]
fn topology_runtime_status_publish_compare_ignores_generated_unix() {
let first = TopologyRuntimeStatusFile {
schema_version: 1,
source_generation: "source-1".to_string(),
shaping_generation: "shape-1".to_string(),
effective_generation: "effective-1".to_string(),
ready: true,
generated_unix: Some(1),
effective_state_path: "/tmp/effective.json".to_string(),
effective_network_path: "/tmp/network.effective.json".to_string(),
shaping_inputs_path: "/tmp/shaping_inputs.json".to_string(),
error: None,
};
let second = TopologyRuntimeStatusFile {
generated_unix: Some(2),
..first.clone()
};
let changed = TopologyRuntimeStatusFile {
shaping_generation: "shape-2".to_string(),
..first.clone()
};
assert!(first.semantic_equals_for_publish(&second));
assert!(!first.semantic_equals_for_publish(&changed));
assert_eq!(
first.shaping_payload_identity(),
second.shaping_payload_identity()
);
assert_ne!(
first.shaping_payload_identity(),
changed.shaping_payload_identity()
);
}
#[test]
fn shaping_generation_ignores_timestamp_only_changes() {
let first = TopologyShapingInputsFile {
+132 -10
View File
@@ -68,6 +68,7 @@ enum RequestedMode {
LinuxBridge {
to_internet: String,
to_network: String,
mtu: Option<u32>,
},
XdpBridge {
to_internet: String,
@@ -75,6 +76,7 @@ enum RequestedMode {
},
SingleInterface {
interface: String,
mtu: Option<u32>,
},
Unknown,
}
@@ -307,11 +309,13 @@ fn requested_mode(config: &Config) -> RequestedMode {
RequestedMode::LinuxBridge {
to_internet: bridge.to_internet.trim().to_string(),
to_network: bridge.to_network.trim().to_string(),
mtu: bridge.mtu,
}
}
} else if let Some(single) = &config.single_interface {
RequestedMode::SingleInterface {
interface: single.interface.trim().to_string(),
mtu: single.mtu,
}
} else {
RequestedMode::Unknown
@@ -332,6 +336,7 @@ fn selected_interfaces(mode: &RequestedMode) -> Vec<String> {
RequestedMode::LinuxBridge {
to_internet,
to_network,
..
}
| RequestedMode::XdpBridge {
to_internet,
@@ -340,7 +345,7 @@ fn selected_interfaces(mode: &RequestedMode) -> Vec<String> {
.into_iter()
.filter(|iface| !iface.is_empty())
.collect(),
RequestedMode::SingleInterface { interface } => {
RequestedMode::SingleInterface { interface, .. } => {
if interface.is_empty() {
Vec::new()
} else {
@@ -351,21 +356,38 @@ fn selected_interfaces(mode: &RequestedMode) -> Vec<String> {
}
}
fn mtu_yaml_line(mtu: Option<u32>, indent: &str) -> String {
mtu.map(|value| format!("{indent}mtu: {value}\n"))
.unwrap_or_default()
}
fn managed_linux_bridge_yaml(to_internet: &str, to_network: &str, mtu: Option<u32>) -> String {
let ethernet_mtu = mtu_yaml_line(mtu, " ");
let bridge_mtu = mtu_yaml_line(mtu, " ");
format!(
"network:\n version: 2\n ethernets:\n {to_internet}:\n dhcp4: false\n dhcp6: false\n{ethernet_mtu} {to_network}:\n dhcp4: false\n dhcp6: false\n{ethernet_mtu} bridges:\n br0:\n{bridge_mtu} interfaces:\n - {to_internet}\n - {to_network}\n"
)
}
fn managed_single_interface_yaml(interface: &str, mtu: Option<u32>) -> String {
let mtu_line = mtu_yaml_line(mtu, " ");
format!(
"network:\n version: 2\n ethernets:\n {interface}:\n dhcp4: false\n dhcp6: false\n{mtu_line}"
)
}
fn managed_preview_yaml(mode: &RequestedMode) -> (Option<String>, Option<String>) {
match mode {
RequestedMode::LinuxBridge {
to_internet,
to_network,
mtu,
} if !to_internet.is_empty() && !to_network.is_empty() => (
Some(format!(
"network:\n version: 2\n ethernets:\n {to_internet}:\n dhcp4: false\n dhcp6: false\n {to_network}:\n dhcp4: false\n dhcp6: false\n bridges:\n br0:\n interfaces:\n - {to_internet}\n - {to_network}\n"
)),
Some(managed_linux_bridge_yaml(to_internet, to_network, *mtu)),
None,
),
RequestedMode::SingleInterface { interface } if !interface.is_empty() => (
Some(format!(
"network:\n version: 2\n ethernets:\n {interface}:\n dhcp4: false\n dhcp6: false\n"
)),
RequestedMode::SingleInterface { interface, mtu } if !interface.is_empty() => (
Some(managed_single_interface_yaml(interface, *mtu)),
None,
),
RequestedMode::XdpBridge { .. } => (
@@ -598,6 +620,7 @@ pub(crate) fn adoption_rewrite_for_path(path: &Path, config: &Config) -> Result<
RequestedMode::LinuxBridge {
to_internet,
to_network,
..
} => {
let mut removed_bridge = false;
doc.network.ethernets.remove(&to_internet);
@@ -618,7 +641,7 @@ pub(crate) fn adoption_rewrite_for_path(path: &Path, config: &Config) -> Result<
));
}
}
RequestedMode::SingleInterface { interface } => {
RequestedMode::SingleInterface { interface, .. } => {
if doc.network.ethernets.remove(&interface).is_none() {
return Err(format!(
"Unable to find interface {interface} in {} for adoption.",
@@ -658,6 +681,7 @@ fn assess_file(path: &Path, doc: &NetplanDocument, mode: &RequestedMode) -> File
RequestedMode::LinuxBridge {
to_internet,
to_network,
..
} => {
for iface in [to_internet, to_network] {
if let Some(cfg) = doc.network.ethernets.get(iface) {
@@ -724,7 +748,7 @@ fn assess_file(path: &Path, doc: &NetplanDocument, mode: &RequestedMode) -> File
}
}
}
RequestedMode::SingleInterface { interface } => {
RequestedMode::SingleInterface { interface, .. } => {
if let Some(cfg) = doc.network.ethernets.get(interface) {
relevant.insert(interface.clone());
if cfg.dhcp_disabled() && !cfg.has_l3_config() {
@@ -1113,6 +1137,7 @@ mod tests {
use_xdp_bridge: false,
to_internet: "ens19".to_string(),
to_network: "ens20".to_string(),
mtu: None,
}),
single_interface: None,
..lqos_config::Config::default()
@@ -1126,6 +1151,7 @@ mod tests {
interface: "ens19".to_string(),
internet_vlan: 2,
network_vlan: 3,
mtu: None,
}),
..lqos_config::Config::default()
}
@@ -1144,6 +1170,102 @@ mod tests {
BTreeMap::from([("ens19".to_string(), true), ("ens20".to_string(), true)])
}
#[test]
fn managed_linux_bridge_preview_includes_mtu_on_members_and_bridge() {
let mut config = linux_bridge_config();
config.bridge.as_mut().expect("bridge").mtu = Some(9000);
let root = test_env("bridge-mtu-preview");
let inspection = inspect_network_mode_with_paths(
&config,
&root.join("netplan"),
&root.join("pending"),
&BTreeSet::from(["ens19".to_string(), "ens20".to_string()]),
&queue_caps(),
);
let preview = inspection
.managed_preview_yaml
.expect("managed preview should exist");
assert_eq!(
preview,
"network:\n version: 2\n ethernets:\n ens19:\n dhcp4: false\n dhcp6: false\n mtu: 9000\n ens20:\n dhcp4: false\n dhcp6: false\n mtu: 9000\n bridges:\n br0:\n mtu: 9000\n interfaces:\n - ens19\n - ens20\n"
);
let _ = std::fs::remove_dir_all(root);
}
#[test]
fn managed_single_interface_preview_includes_mtu_on_trunk() {
let mut config = single_interface_config();
config
.single_interface
.as_mut()
.expect("single interface")
.mtu = Some(1500);
let root = test_env("single-mtu-preview");
let inspection = inspect_network_mode_with_paths(
&config,
&root.join("netplan"),
&root.join("pending"),
&BTreeSet::from(["ens19".to_string()]),
&BTreeMap::from([("ens19".to_string(), true)]),
);
let preview = inspection
.managed_preview_yaml
.expect("managed preview should exist");
assert_eq!(
preview,
"network:\n version: 2\n ethernets:\n ens19:\n dhcp4: false\n dhcp6: false\n mtu: 1500\n"
);
let _ = std::fs::remove_dir_all(root);
}
#[test]
fn managed_previews_omit_mtu_when_unset() {
let root = test_env("unset-mtu-preview");
let inspection = inspect_network_mode_with_paths(
&linux_bridge_config(),
&root.join("netplan"),
&root.join("pending"),
&BTreeSet::from(["ens19".to_string(), "ens20".to_string()]),
&queue_caps(),
);
let preview = inspection
.managed_preview_yaml
.expect("managed preview should exist");
assert!(!preview.contains("mtu:"));
let _ = std::fs::remove_dir_all(root);
}
#[test]
fn xdp_bridge_mode_keeps_managed_preview_disabled() {
let mut config = linux_bridge_config();
let bridge = config.bridge.as_mut().expect("bridge");
bridge.use_xdp_bridge = true;
bridge.mtu = Some(9000);
let root = test_env("xdp-mtu-preview");
let inspection = inspect_network_mode_with_paths(
&config,
&root.join("netplan"),
&root.join("pending"),
&BTreeSet::from(["ens19".to_string(), "ens20".to_string()]),
&queue_caps(),
);
assert!(inspection.managed_preview_yaml.is_none());
assert!(
inspection
.preview_note
.as_deref()
.is_some_and(|note| note.contains("manual workflow"))
);
let _ = std::fs::remove_dir_all(root);
}
#[test]
fn detects_external_compatible_linux_bridge() {
let root = test_env("external-compatible");
+56 -11
View File
@@ -170,7 +170,7 @@ fn system_interfaces() -> BTreeSet<String> {
default_net::get_interfaces()
.into_iter()
.map(|iface| iface.name)
.collect()
.collect::<BTreeSet<_>>()
}
fn supports_multi_queue(interface: &str) -> bool {
@@ -195,7 +195,14 @@ fn supports_multi_queue(interface: &str) -> bool {
/// Inspect the requested network mode using the helper's configured paths.
pub fn inspect_with_paths(paths: &HelperPaths, config: &Config) -> NetworkModeInspection {
let system_ifaces = system_interfaces();
inspect_with_paths_and_interfaces(paths, config, system_interfaces())
}
fn inspect_with_paths_and_interfaces(
paths: &HelperPaths,
config: &Config,
system_ifaces: BTreeSet<String>,
) -> NetworkModeInspection {
let queue_caps = system_ifaces
.iter()
.map(|iface| (iface.clone(), supports_multi_queue(iface)))
@@ -750,6 +757,15 @@ pub fn apply_transaction(
paths: &HelperPaths,
pending_children: &mut PendingChildren,
request: ApplyRequest,
) -> Result<ApplyResponse> {
apply_transaction_with_interfaces(paths, pending_children, request, system_interfaces())
}
fn apply_transaction_with_interfaces(
paths: &HelperPaths,
pending_children: &mut PendingChildren,
request: ApplyRequest,
system_ifaces: BTreeSet<String>,
) -> Result<ApplyResponse> {
normalize_pending_children(paths, pending_children)?;
if !list_pending_records(paths)?.is_empty() {
@@ -761,7 +777,7 @@ pub fn apply_transaction(
} else {
Config::default()
};
let inspection = inspect_with_paths(paths, &request.config);
let inspection = inspect_with_paths_and_interfaces(paths, &request.config, system_ifaces);
validate_apply_request(&request, &inspection, &previous_config)?;
let preview_yaml = inspection.managed_preview_yaml.as_ref().ok_or_else(|| {
@@ -1013,7 +1029,7 @@ esac
.find(|iface| iface.as_str() != "lo")
.cloned()
.or_else(|| interfaces.iter().next().cloned())
.expect("test host should expose at least one interface");
.unwrap_or_else(|| "ens19".to_string());
let secondary = interfaces
.iter()
.find(|iface| iface.as_str() != primary && iface.as_str() != "lo")
@@ -1024,10 +1040,19 @@ esac
.find(|iface| iface.as_str() != primary)
.cloned()
})
.expect("test host should expose a second distinct interface");
.unwrap_or_else(|| "ens20".to_string());
(primary, secondary)
}
fn test_system_interfaces() -> BTreeSet<String> {
let interfaces = system_interfaces();
if interfaces.is_empty() {
BTreeSet::from(["ens19".to_string(), "ens20".to_string()])
} else {
interfaces
}
}
fn linux_bridge_config() -> Config {
let (to_internet, to_network) = test_interfaces();
Config {
@@ -1035,6 +1060,7 @@ esac
use_xdp_bridge: false,
to_internet,
to_network,
mtu: None,
}),
single_interface: None,
..Config::default()
@@ -1058,7 +1084,13 @@ esac
confirm_dangerous_changes: true,
};
let apply = apply_transaction(&paths, &mut pending, request).expect("apply should succeed");
let apply = apply_transaction_with_interfaces(
&paths,
&mut pending,
request,
test_system_interfaces(),
)
.expect("apply should succeed");
assert!(apply.ok);
assert!(apply.operation.is_some());
@@ -1082,7 +1114,13 @@ esac
mode: ApplyMode::Apply,
confirm_dangerous_changes: true,
};
let apply = apply_transaction(&paths, &mut pending, request).expect("second apply");
let apply = apply_transaction_with_interfaces(
&paths,
&mut pending,
request,
test_system_interfaces(),
)
.expect("second apply");
let operation_id = apply
.operation
.as_ref()
@@ -1112,10 +1150,11 @@ esac
interface: test_interfaces().0,
internet_vlan: 2,
network_vlan: 3,
mtu: None,
});
next.bridge = None;
let apply = apply_transaction(
let apply = apply_transaction_with_interfaces(
&paths,
&mut pending,
ApplyRequest {
@@ -1125,6 +1164,7 @@ esac
mode: ApplyMode::Apply,
confirm_dangerous_changes: true,
},
test_system_interfaces(),
)
.expect("apply should succeed");
let backup_id = apply
@@ -1147,6 +1187,7 @@ esac
interface: test_interfaces().1,
internet_vlan: 10,
network_vlan: 11,
mtu: None,
}),
bridge: None,
..Config::default()
@@ -1181,7 +1222,7 @@ esac
let existing = linux_bridge_config();
write_config(&paths.config_path, &existing);
let apply = apply_transaction(
let apply = apply_transaction_with_interfaces(
&paths,
&mut pending,
ApplyRequest {
@@ -1191,6 +1232,7 @@ esac
mode: ApplyMode::Apply,
confirm_dangerous_changes: true,
},
test_system_interfaces(),
)
.expect("apply should succeed");
let operation_id = apply
@@ -1228,10 +1270,11 @@ esac
interface: test_interfaces().0,
internet_vlan: 2,
network_vlan: 3,
mtu: None,
});
next.bridge = None;
let apply = apply_transaction(
let apply = apply_transaction_with_interfaces(
&paths,
&mut pending,
ApplyRequest {
@@ -1241,6 +1284,7 @@ esac
mode: ApplyMode::Apply,
confirm_dangerous_changes: true,
},
test_system_interfaces(),
)
.expect("apply should succeed");
let operation_id = apply
@@ -1295,7 +1339,7 @@ esac
let existing = linux_bridge_config();
write_config(&paths.config_path, &existing);
let err = apply_transaction(
let err = apply_transaction_with_interfaces(
&paths,
&mut pending,
ApplyRequest {
@@ -1305,6 +1349,7 @@ esac
mode: ApplyMode::Apply,
confirm_dangerous_changes: true,
},
test_system_interfaces(),
)
.expect_err("apply should fail");
+11 -9
View File
@@ -11,8 +11,8 @@ Centralized, shared access to LibreQoS topology inputs:
In daemon mode, this crate starts:
- A single-thread actor that owns reload/apply commands.
- Watchers for the config directory and topology-state directory that coalesce
filesystem events and request reloads.
- Watchers for the config directory, topology-state directory, and shaping-state
directory that coalesce filesystem events and request reloads.
Callers access state through public accessor functions; the underlying channel sender is kept in a
private static.
@@ -27,7 +27,7 @@ private static.
- `with_network_json_write(|net_json| ...)`: mutable access to in-memory `NetworkJson` (used by runtime counters).
- `resolve_parent_node_reference(parent_node, parent_node_id)`: canonicalize shaped-device parent references against `network.json`.
- `request_reload_shaped_devices(reason)` / `request_reload_network_json(reason)`: ask actor to reload from disk.
- `apply_shaped_devices_snapshot(reason, shaped)`: publish a caller-provided shaped-devices snapshot via actor.
- `apply_shaped_devices_snapshot(reason, shaped, provenance)`: publish a caller-provided shaped-devices snapshot via actor.
- `swap_shaped_devices_snapshot(reason, shaped_arc)`: replace shaped-devices snapshot without actor (intended for tests).
## `ShapedDevicesCatalog` helpers
@@ -48,16 +48,18 @@ Runtime throughput tracking mutates the in-memory `NetworkJson` via `with_networ
## Access & update catalog
Current (2026-04) call sites and update patterns:
Call sites and update patterns:
- Daemon startup: `lqosd` calls `start_daemon_mode()` once at boot and provides `DaemonHooks`.
- Shaped-device updates:
- Directory watcher requests reload when `topology_runtime_status.json`,
`ShapedDevices.csv`, or `ShapedDevices.insight.csv` changes.
- Reload precedence is: ready runtime shaping inputs, then `topology_import.json`,
then `ShapedDevices.csv`.
- Directory watcher requests reload when `shaping_inputs.json`,
`topology_import.json`, `ShapedDevices.csv`, or `ShapedDevices.insight.csv` changes.
- `topology_runtime_status.json` is readiness metadata and does not trigger a shaped-device reload by itself.
- In integration-ingress mode, reload precedence is: ready runtime shaping inputs,
then `topology_import.json`, then an empty snapshot until the topology runtime publishes.
- In manual mode, reloads use `ShapedDevices.csv` or `ShapedDevices.insight.csv`.
- Node manager admin edits write `ShapedDevices.csv` and immediately publish the new snapshot via
`apply_shaped_devices_snapshot()`.
`apply_shaped_devices_snapshot()` with external-snapshot provenance.
- Network topology updates:
- Directory watcher requests reload when `network.json`, `network.insight.json`, or
`network.effective.json` changes, including the runtime topology-state copy.
+336 -67
View File
@@ -3,20 +3,25 @@ use crate::dynamic::{
};
use crate::dynamic_store::{load_dynamic_circuits_from_disk, persist_dynamic_circuits_to_disk};
use crate::state;
use crate::{DaemonHooks, ShapedDevicesCatalog, load_network_json, load_shaped_devices};
use crate::{
DaemonHooks, LoadedShapedDevices, LoadedShapedDevicesSource, ShapedDevicesCatalog,
ShapedDevicesSnapshotProvenance, load_network_json, load_shaped_devices_with_source,
};
use anyhow::{Context, Result, anyhow};
use crossbeam_channel::{Receiver, Sender};
use crossbeam_channel::{Receiver, RecvTimeoutError, SendTimeoutError, Sender, bounded};
use ip_network_table::IpNetworkTable;
use once_cell::sync::OnceCell;
use std::collections::HashSet;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::net::IpAddr;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::oneshot;
use tracing::{debug, error, warn};
const RELOAD_RETRY_DELAY_MS: u64 = 500;
const RELOAD_ATTEMPTS: usize = 2;
const ACTOR_COMMAND_TIMEOUT: Duration = Duration::from_secs(10);
const DYNAMIC_CIRCUITS_MAINTENANCE_TICK_SECONDS: u64 = 60;
const MAX_UNKNOWN_IP_PROMOTIONS_PER_OBSERVATION: usize = 32;
@@ -44,41 +49,69 @@ fn sender() -> Result<Sender<NetworkDevicesCommand>> {
.ok_or_else(|| anyhow!("lqos_network_devices runtime actor is not running"))
}
fn send_actor_command(command: NetworkDevicesCommand, operation: &str) -> Result<()> {
sender()?
.send_timeout(command, ACTOR_COMMAND_TIMEOUT)
.map_err(|err| match err {
SendTimeoutError::Timeout(_) => {
anyhow!("{operation} timed out while queueing lqos_network_devices actor command")
}
SendTimeoutError::Disconnected(_) => {
anyhow!("{operation} failed because lqos_network_devices actor stopped")
}
})
}
fn receive_actor_reply<T>(reply_rx: Receiver<Result<T>>, operation: &str) -> Result<T> {
match reply_rx.recv_timeout(ACTOR_COMMAND_TIMEOUT) {
Ok(result) => result,
Err(RecvTimeoutError::Timeout) => {
Err(anyhow!("{operation} timed out waiting for actor reply"))
}
Err(RecvTimeoutError::Disconnected) => Err(anyhow!("{operation} reply channel closed")),
}
}
pub(crate) fn request_reload_shaped_devices(reason: &str) -> Result<()> {
let (reply_tx, reply_rx) = oneshot::channel();
sender()?.send(NetworkDevicesCommand::ReloadShapedDevices {
reason: reason.to_string(),
reply: reply_tx,
})?;
reply_rx
.blocking_recv()
.map_err(|_| anyhow!("ShapedDevices reload reply channel closed"))?
let (reply_tx, reply_rx) = bounded(1);
send_actor_command(
NetworkDevicesCommand::ReloadShapedDevices {
reason: reason.to_string(),
reply: reply_tx,
},
"ShapedDevices reload",
)?;
receive_actor_reply(reply_rx, "ShapedDevices reload")
}
pub(crate) fn request_reload_network_json(reason: &str) -> Result<()> {
let (reply_tx, reply_rx) = oneshot::channel();
sender()?.send(NetworkDevicesCommand::ReloadNetworkJson {
reason: reason.to_string(),
reply: reply_tx,
})?;
reply_rx
.blocking_recv()
.map_err(|_| anyhow!("NetworkJson reload reply channel closed"))?
let (reply_tx, reply_rx) = bounded(1);
send_actor_command(
NetworkDevicesCommand::ReloadNetworkJson {
reason: reason.to_string(),
reply: reply_tx,
},
"NetworkJson reload",
)?;
receive_actor_reply(reply_rx, "NetworkJson reload")
}
pub(crate) fn apply_shaped_devices_snapshot(
reason: &str,
shaped: lqos_config::ConfigShapedDevices,
provenance: ShapedDevicesSnapshotProvenance,
) -> Result<()> {
let (reply_tx, reply_rx) = oneshot::channel();
sender()?.send(NetworkDevicesCommand::ApplyShapedDevicesSnapshot {
reason: reason.to_string(),
shaped: Box::new(shaped),
reply: reply_tx,
})?;
reply_rx
.blocking_recv()
.map_err(|_| anyhow!("Apply shaped devices reply channel closed"))?
let (reply_tx, reply_rx) = bounded(1);
send_actor_command(
NetworkDevicesCommand::ApplyShapedDevicesSnapshot {
reason: reason.to_string(),
shaped: Box::new(shaped),
provenance,
reply: reply_tx,
},
"Apply shaped devices",
)?;
receive_actor_reply(reply_rx, "Apply shaped devices")
}
pub(crate) fn report_observations(observations: &[CircuitObservation]) {
@@ -95,58 +128,64 @@ pub(crate) fn report_observations(observations: &[CircuitObservation]) {
}
pub(crate) fn upsert_dynamic_circuit(shaped_device: lqos_config::ShapedDevice) -> Result<()> {
let (reply_tx, reply_rx) = oneshot::channel();
sender()?.send(NetworkDevicesCommand::UpsertDynamicCircuit {
shaped_device: Box::new(shaped_device),
reply: reply_tx,
})?;
reply_rx
.blocking_recv()
.map_err(|_| anyhow!("Upsert dynamic circuit reply channel closed"))?
let (reply_tx, reply_rx) = bounded(1);
send_actor_command(
NetworkDevicesCommand::UpsertDynamicCircuit {
shaped_device: Box::new(shaped_device),
reply: reply_tx,
},
"Upsert dynamic circuit",
)?;
receive_actor_reply(reply_rx, "Upsert dynamic circuit")
}
pub(crate) fn remove_dynamic_circuit(circuit_id: &str) -> Result<bool> {
let (reply_tx, reply_rx) = oneshot::channel();
sender()?.send(NetworkDevicesCommand::RemoveDynamicCircuit {
circuit_id: circuit_id.to_string(),
reply: reply_tx,
})?;
reply_rx
.blocking_recv()
.map_err(|_| anyhow!("Remove dynamic circuit reply channel closed"))?
let (reply_tx, reply_rx) = bounded(1);
send_actor_command(
NetworkDevicesCommand::RemoveDynamicCircuit {
circuit_id: circuit_id.to_string(),
reply: reply_tx,
},
"Remove dynamic circuit",
)?;
receive_actor_reply(reply_rx, "Remove dynamic circuit")
}
enum NetworkDevicesCommand {
ReloadShapedDevices {
reason: String,
reply: oneshot::Sender<Result<()>>,
reply: Sender<Result<()>>,
},
ReloadNetworkJson {
reason: String,
reply: oneshot::Sender<Result<()>>,
reply: Sender<Result<()>>,
},
ApplyShapedDevicesSnapshot {
reason: String,
shaped: Box<lqos_config::ConfigShapedDevices>,
reply: oneshot::Sender<Result<()>>,
provenance: ShapedDevicesSnapshotProvenance,
reply: Sender<Result<()>>,
},
ReportObservations {
observations: Vec<CircuitObservation>,
},
UpsertDynamicCircuit {
shaped_device: Box<lqos_config::ShapedDevice>,
reply: oneshot::Sender<Result<()>>,
reply: Sender<Result<()>>,
},
RemoveDynamicCircuit {
circuit_id: String,
reply: oneshot::Sender<Result<bool>>,
reply: Sender<Result<bool>>,
},
}
fn actor_loop(rx: Receiver<NetworkDevicesCommand>, hooks: Option<Arc<dyn DaemonHooks>>) {
debug!("lqos_network_devices actor starting");
let mut active_shaped_key = None;
if let Err(err) = reload_shaped_devices_inner("startup", hooks.as_deref()) {
if let Err(err) =
reload_shaped_devices_inner("startup", hooks.as_deref(), &mut active_shaped_key)
{
warn!("Initial shaped-devices load failed: {err}");
}
if let Err(err) = reload_network_json_inner("startup", hooks.as_deref()) {
@@ -169,7 +208,11 @@ fn actor_loop(rx: Receiver<NetworkDevicesCommand>, hooks: Option<Arc<dyn DaemonH
match command {
NetworkDevicesCommand::ReloadShapedDevices { reason, reply } => {
let result = reload_shaped_devices_inner(&reason, hooks.as_deref());
let result = reload_shaped_devices_inner(
&reason,
hooks.as_deref(),
&mut active_shaped_key,
);
let _ = reply.send(result);
}
NetworkDevicesCommand::ReloadNetworkJson { reason, reply } => {
@@ -179,15 +222,18 @@ fn actor_loop(rx: Receiver<NetworkDevicesCommand>, hooks: Option<Arc<dyn DaemonH
NetworkDevicesCommand::ApplyShapedDevicesSnapshot {
reason,
shaped,
provenance,
reply,
} => {
debug!("Publishing shaped-devices snapshot reason={reason}");
state::publish_shaped_devices(*shaped);
prune_dynamic_circuits_superseded_by_shaped_devices_inner(hooks.as_deref());
if let Some(hooks) = &hooks {
hooks.on_shaped_devices_updated();
}
let _ = reply.send(Ok(()));
let key = shaped_devices_key_for_snapshot(provenance, &shaped);
let result = publish_shaped_devices_if_changed(
&reason,
*shaped,
key,
hooks.as_deref(),
&mut active_shaped_key,
);
let _ = reply.send(result);
}
NetworkDevicesCommand::ReportObservations { observations } => {
handle_observations(&observations, hooks.as_deref());
@@ -224,6 +270,119 @@ fn recompute_hashes(device: &mut lqos_config::ShapedDevice) {
device.parent_hash = lqos_utils::hash_to_i64(&device.parent_node);
}
#[derive(Clone, Debug, PartialEq, Eq)]
enum ShapedDevicesProvenanceKey {
RuntimeTopology {
shaping_generation: String,
},
TopologyImport {
devices_fingerprint: u64,
},
Csv {
devices_fingerprint: u64,
},
ExternalSnapshot {
devices_fingerprint: u64,
},
}
#[derive(Eq, Hash, Ord, PartialEq, PartialOrd)]
struct ShapedDeviceSemanticKey {
circuit_id: String,
circuit_name: String,
device_id: String,
device_name: String,
parent_node: String,
parent_node_id: Option<String>,
anchor_node_id: Option<String>,
mac: String,
ipv4: Vec<(std::net::Ipv4Addr, u32)>,
ipv6: Vec<(std::net::Ipv6Addr, u32)>,
download_min_mbps_bits: u32,
upload_min_mbps_bits: u32,
download_max_mbps_bits: u32,
upload_max_mbps_bits: u32,
comment: String,
sqm_override: Option<String>,
circuit_hash: i64,
device_hash: i64,
parent_hash: i64,
}
fn shaped_device_semantic_key(device: &lqos_config::ShapedDevice) -> ShapedDeviceSemanticKey {
let mut ipv4 = device.ipv4.clone();
ipv4.sort_unstable();
let mut ipv6 = device.ipv6.clone();
ipv6.sort_unstable();
ShapedDeviceSemanticKey {
circuit_id: device.circuit_id.clone(),
circuit_name: device.circuit_name.clone(),
device_id: device.device_id.clone(),
device_name: device.device_name.clone(),
parent_node: device.parent_node.clone(),
parent_node_id: device.parent_node_id.clone(),
anchor_node_id: device.anchor_node_id.clone(),
mac: device.mac.clone(),
ipv4,
ipv6,
download_min_mbps_bits: device.download_min_mbps.to_bits(),
upload_min_mbps_bits: device.upload_min_mbps.to_bits(),
download_max_mbps_bits: device.download_max_mbps.to_bits(),
upload_max_mbps_bits: device.upload_max_mbps.to_bits(),
comment: device.comment.clone(),
sqm_override: device.sqm_override.clone(),
circuit_hash: device.circuit_hash,
device_hash: device.device_hash,
parent_hash: device.parent_hash,
}
}
fn shaped_devices_fingerprint(devices: &[lqos_config::ShapedDevice]) -> u64 {
let mut sorted = devices
.iter()
.map(shaped_device_semantic_key)
.collect::<Vec<_>>();
sorted.sort_unstable();
let mut hasher = DefaultHasher::new();
for key in sorted {
key.hash(&mut hasher);
}
hasher.finish()
}
fn shaped_devices_key_for_loaded(loaded: &LoadedShapedDevices) -> ShapedDevicesProvenanceKey {
match &loaded.source {
LoadedShapedDevicesSource::RuntimeTopology {
shaping_generation,
..
} => ShapedDevicesProvenanceKey::RuntimeTopology {
shaping_generation: shaping_generation.clone(),
},
LoadedShapedDevicesSource::TopologyImport => {
ShapedDevicesProvenanceKey::TopologyImport {
devices_fingerprint: shaped_devices_fingerprint(&loaded.shaped.devices),
}
}
LoadedShapedDevicesSource::Csv => ShapedDevicesProvenanceKey::Csv {
devices_fingerprint: shaped_devices_fingerprint(&loaded.shaped.devices),
},
}
}
fn shaped_devices_key_for_snapshot(
provenance: ShapedDevicesSnapshotProvenance,
shaped: &lqos_config::ConfigShapedDevices,
) -> ShapedDevicesProvenanceKey {
match provenance {
ShapedDevicesSnapshotProvenance::ExternalSnapshot => {
ShapedDevicesProvenanceKey::ExternalSnapshot {
devices_fingerprint: shaped_devices_fingerprint(&shaped.devices),
}
}
}
}
fn reload_dynamic_circuits_inner(reason: &str) {
let circuits = load_dynamic_circuits_from_disk();
debug!("Loaded {} dynamic circuits reason={reason}", circuits.len());
@@ -562,17 +721,45 @@ fn shaped_device_from_unknown_ip(
device
}
fn reload_shaped_devices_inner(reason: &str, hooks: Option<&dyn DaemonHooks>) -> Result<()> {
fn publish_shaped_devices_if_changed(
reason: &str,
shaped: lqos_config::ConfigShapedDevices,
key: ShapedDevicesProvenanceKey,
hooks: Option<&dyn DaemonHooks>,
active_key: &mut Option<ShapedDevicesProvenanceKey>,
) -> Result<()> {
if active_key.as_ref() == Some(&key) {
debug!("Skipping unchanged shaped-devices snapshot reason={reason}");
return Ok(());
}
debug!("Publishing shaped-devices snapshot reason={reason}");
state::publish_shaped_devices(shaped);
*active_key = Some(key);
prune_dynamic_circuits_superseded_by_shaped_devices_inner(hooks);
if let Some(hooks) = hooks {
hooks.on_shaped_devices_updated();
}
Ok(())
}
fn reload_shaped_devices_inner(
reason: &str,
hooks: Option<&dyn DaemonHooks>,
active_key: &mut Option<ShapedDevicesProvenanceKey>,
) -> Result<()> {
for attempt in 1..=RELOAD_ATTEMPTS {
match load_shaped_devices() {
Ok(shaped) => {
match load_shaped_devices_with_source() {
Ok(loaded) => {
debug!("Loaded shaped devices reason={reason}");
state::publish_shaped_devices(shaped);
prune_dynamic_circuits_superseded_by_shaped_devices_inner(hooks);
if let Some(hooks) = hooks {
hooks.on_shaped_devices_updated();
}
return Ok(());
let key = shaped_devices_key_for_loaded(&loaded);
return publish_shaped_devices_if_changed(
reason,
loaded.shaped,
key,
hooks,
active_key,
);
}
Err(err) if attempt < RELOAD_ATTEMPTS => {
warn!(
@@ -682,3 +869,85 @@ fn reload_network_json_inner(reason: &str, hooks: Option<&dyn DaemonHooks>) -> R
}
unreachable!("reload loop must return");
}
#[cfg(test)]
mod tests {
use super::shaped_devices_fingerprint;
fn shaped_device(circuit_id: &str, device_id: &str) -> lqos_config::ShapedDevice {
let mut device = lqos_config::ShapedDevice {
circuit_id: circuit_id.to_string(),
circuit_name: circuit_id.to_string(),
device_id: device_id.to_string(),
device_name: device_id.to_string(),
parent_node: "Tower A".to_string(),
ipv4: vec![("192.0.2.1".parse().expect("valid IP"), 32)],
download_min_mbps: 10.0,
upload_min_mbps: 10.0,
download_max_mbps: 100.0,
upload_max_mbps: 100.0,
..lqos_config::ShapedDevice::default()
};
device.circuit_hash = lqos_utils::hash_to_i64(&device.circuit_id);
device.device_hash = lqos_utils::hash_to_i64(&device.device_id);
device.parent_hash = lqos_utils::hash_to_i64(&device.parent_node);
device
}
#[test]
fn shaped_devices_fingerprint_ignores_row_order() {
let first = vec![
shaped_device("circuit-1", "device-1"),
shaped_device("circuit-2", "device-2"),
];
let second = vec![
shaped_device("circuit-2", "device-2"),
shaped_device("circuit-1", "device-1"),
];
assert_eq!(
shaped_devices_fingerprint(&first),
shaped_devices_fingerprint(&second)
);
}
#[test]
fn shaped_devices_fingerprint_ignores_address_order() {
let first = vec![shaped_device("circuit-1", "device-1")];
let mut second = vec![shaped_device("circuit-1", "device-1")];
second[0].ipv4 = vec![
("192.0.2.2".parse().expect("valid IPv4"), 32),
("192.0.2.1".parse().expect("valid IPv4"), 32),
];
second[0].ipv6 = vec![
("2001:db8::2".parse().expect("valid IPv6"), 128),
("2001:db8::1".parse().expect("valid IPv6"), 128),
];
let mut first = first;
first[0].ipv4 = vec![
("192.0.2.1".parse().expect("valid IPv4"), 32),
("192.0.2.2".parse().expect("valid IPv4"), 32),
];
first[0].ipv6 = vec![
("2001:db8::1".parse().expect("valid IPv6"), 128),
("2001:db8::2".parse().expect("valid IPv6"), 128),
];
assert_eq!(
shaped_devices_fingerprint(&first),
shaped_devices_fingerprint(&second)
);
}
#[test]
fn shaped_devices_fingerprint_detects_payload_changes() {
let first = vec![shaped_device("circuit-1", "device-1")];
let mut second = vec![shaped_device("circuit-1", "device-1")];
second[0].download_max_mbps = 200.0;
assert_ne!(
shaped_devices_fingerprint(&first),
shaped_devices_fingerprint(&second)
);
}
}
@@ -25,6 +25,11 @@ pub(crate) fn start_network_devices_directory_watch() -> Result<()> {
"LibreQoS topology state directory",
config.resolved_state_directory().join("topology"),
)?;
spawn_directory_watch_thread(
"Network Devices Shaping Dir Watcher",
"LibreQoS shaping state directory",
config.resolved_state_directory().join("shaping"),
)?;
Ok(())
}
@@ -88,7 +93,7 @@ fn classify_changed_path(path: &Path) -> Option<DirectoryReloadEvent> {
"ShapedDevices.csv"
| "ShapedDevices.insight.csv"
| "topology_import.json"
| "topology_runtime_status.json" => Some(DirectoryReloadEvent::ShapedDevices),
| "shaping_inputs.json" => Some(DirectoryReloadEvent::ShapedDevices),
"network.effective.json" | "network.insight.json" | "network.json" => {
Some(DirectoryReloadEvent::NetworkJson)
}
@@ -118,8 +123,14 @@ mod tests {
use std::path::PathBuf;
#[test]
fn topology_runtime_status_triggers_shaped_device_reload() {
fn topology_runtime_status_does_not_trigger_shaped_device_reload() {
let event = classify_changed_path(&PathBuf::from("/tmp/topology_runtime_status.json"));
assert_eq!(event, None);
}
#[test]
fn shaping_inputs_triggers_shaped_device_reload() {
let event = classify_changed_path(&PathBuf::from("/tmp/shaping_inputs.json"));
assert_eq!(event, Some(DirectoryReloadEvent::ShapedDevices));
}
@@ -134,7 +145,7 @@ mod tests {
let events = classify_changed_paths(&[
PathBuf::from("/tmp/network.json"),
PathBuf::from("/tmp/network.effective.json"),
PathBuf::from("/tmp/topology_runtime_status.json"),
PathBuf::from("/tmp/shaping_inputs.json"),
]);
assert_eq!(
events,
+61 -11
View File
@@ -20,7 +20,9 @@ mod topology;
mod tests;
use anyhow::{Context, Result};
use lqos_config::{Config, ConfigShapedDevices, NetworkJson, TopologyShapingInputsFile};
use lqos_config::{
Config, ConfigShapedDevices, NetworkJson, TopologyShapingInputsFile,
};
use std::sync::Arc;
use tracing::debug;
@@ -30,6 +32,27 @@ pub use dynamic::{CircuitObservation, DynamicCircuit};
pub use hash_cache::ShapedDeviceHashCache;
pub use topology::{ResolvedParentNode, resolve_parent_node, resolve_parent_node_reference};
/// Provenance for caller-provided shaped-device snapshots.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ShapedDevicesSnapshotProvenance {
/// Snapshot came from an external caller that already constructed a complete payload.
ExternalSnapshot,
}
#[derive(Clone, Debug)]
pub(crate) enum LoadedShapedDevicesSource {
RuntimeTopology {
shaping_generation: String,
},
TopologyImport,
Csv,
}
pub(crate) struct LoadedShapedDevices {
pub(crate) shaped: ConfigShapedDevices,
pub(crate) source: LoadedShapedDevicesSource,
}
/// Hooks that `lqosd` can provide to run side effects after updates.
pub trait DaemonHooks: Send + Sync + 'static {
/// Called after a new `ShapedDevices.csv` snapshot is published.
@@ -67,13 +90,18 @@ pub fn load_runtime_shaping_inputs_for_config(
runtime_inputs::load_ready_runtime_shaping_inputs(config)
}
fn load_shaped_devices_for_config(config: &lqos_config::Config) -> Result<ConfigShapedDevices> {
fn load_shaped_devices_for_config(config: &lqos_config::Config) -> Result<LoadedShapedDevices> {
if lqos_config::integration_ingress_enabled(config) {
if let Some(shaping_inputs) = load_runtime_shaping_inputs_for_config(config)? {
let shaped_devices =
runtime_inputs::shaped_devices_from_runtime_inputs(&shaping_inputs);
if !shaped_devices.devices.is_empty() {
return Ok(shaped_devices);
return Ok(LoadedShapedDevices {
shaped: shaped_devices,
source: LoadedShapedDevicesSource::RuntimeTopology {
shaping_generation: shaping_inputs.shaping_generation,
},
});
}
debug!(
"Active runtime shaping inputs contained 0 shaped devices; checking other integration sources"
@@ -83,13 +111,19 @@ fn load_shaped_devices_for_config(config: &lqos_config::Config) -> Result<Config
debug!(
"topology_import.json is missing or contains 0 shaped devices; keeping integration mode empty until topology is published"
);
return Ok(ConfigShapedDevices::default());
return Ok(LoadedShapedDevices {
shaped: ConfigShapedDevices::default(),
source: LoadedShapedDevicesSource::TopologyImport,
});
}
match lqos_topology_compile::TopologyImportFile::load(config) {
Ok(Some(topology_import)) => {
let shaped_devices = topology_import.into_imported_bundle().shaped_devices;
if !shaped_devices.devices.is_empty() {
return Ok(shaped_devices);
return Ok(LoadedShapedDevices {
shaped: shaped_devices,
source: LoadedShapedDevicesSource::TopologyImport,
});
}
debug!(
"topology_import.json advertised shaped devices but loaded empty; keeping integration mode empty until topology is published"
@@ -106,16 +140,28 @@ fn load_shaped_devices_for_config(config: &lqos_config::Config) -> Result<Config
);
}
}
return Ok(ConfigShapedDevices::default());
return Ok(LoadedShapedDevices {
shaped: ConfigShapedDevices::default(),
source: LoadedShapedDevicesSource::TopologyImport,
});
}
ConfigShapedDevices::load_for_config(config).context("Unable to load ShapedDevices.csv")
Ok(LoadedShapedDevices {
shaped: ConfigShapedDevices::load_for_config(config)
.context("Unable to load ShapedDevices.csv")?,
source: LoadedShapedDevicesSource::Csv,
})
}
/// Loads shaped-device configuration from disk.
///
/// When an integration-backed topology ingress is enabled, this loads shaped devices from the
/// active `topology_import.json` bundle. Otherwise, it loads `ShapedDevices.csv`.
/// When integration-backed topology ingress is enabled, this prefers ready runtime shaping inputs,
/// then `topology_import.json`, then an empty snapshot until the topology runtime publishes.
/// Manual mode loads `ShapedDevices.csv`.
pub fn load_shaped_devices() -> Result<ConfigShapedDevices> {
Ok(load_shaped_devices_with_source()?.shaped)
}
pub(crate) fn load_shaped_devices_with_source() -> Result<LoadedShapedDevices> {
let config = lqos_config::load_config().context("Unable to load /etc/lqos.conf")?;
load_shaped_devices_for_config(config.as_ref())
}
@@ -183,8 +229,12 @@ pub fn request_reload_network_json(reason: &str) -> Result<()> {
}
/// Publishes a caller-provided shaped-devices snapshot through the runtime actor.
pub fn apply_shaped_devices_snapshot(reason: &str, shaped: ConfigShapedDevices) -> Result<()> {
actor::apply_shaped_devices_snapshot(reason, shaped)
pub fn apply_shaped_devices_snapshot(
reason: &str,
shaped: ConfigShapedDevices,
provenance: ShapedDevicesSnapshotProvenance,
) -> Result<()> {
actor::apply_shaped_devices_snapshot(reason, shaped, provenance)
}
/// Reports kernel observations that may indicate dynamic circuit activity.
@@ -5,7 +5,10 @@
//! into the shaped-device snapshot used by `lqosd`.
use anyhow::Result;
use lqos_config::{Config, ConfigShapedDevices, ShapedDevice, TopologyShapingInputsFile};
use lqos_config::{
Config, ConfigShapedDevices, ShapedDevice, TopologyShapingInputsFile,
TopologyShapingResolutionSource,
};
use std::net::{Ipv4Addr, Ipv6Addr};
use tracing::debug;
@@ -58,6 +61,22 @@ fn parse_ipv6_list(values: &[String]) -> Vec<(Ipv6Addr, u32)> {
.collect()
}
fn runtime_parent_for_circuit(
circuit: &lqos_config::TopologyShapingCircuitInput,
) -> (String, Option<String>) {
if circuit.resolution_source == TopologyShapingResolutionSource::RuntimeFallback {
return (String::new(), None);
}
let parent_node = optional_trimmed_string(&circuit.effective_parent_node_name)
.or_else(|| circuit.logical_parent_node_name.clone())
.unwrap_or_default();
let parent_node_id = optional_trimmed_string(&circuit.effective_parent_node_id)
.or_else(|| circuit.logical_parent_node_id.clone());
(parent_node, parent_node_id)
}
/// Loads the currently active runtime shaping inputs when available.
pub(crate) fn load_ready_runtime_shaping_inputs(
config: &Config,
@@ -80,11 +99,7 @@ pub(crate) fn shaped_devices_from_runtime_inputs(
) -> ConfigShapedDevices {
let mut devices = Vec::new();
for circuit in &shaping_inputs.circuits {
let parent_node = optional_trimmed_string(&circuit.effective_parent_node_name)
.or_else(|| circuit.logical_parent_node_name.clone())
.unwrap_or_default();
let parent_node_id = optional_trimmed_string(&circuit.effective_parent_node_id)
.or_else(|| circuit.logical_parent_node_id.clone());
let (parent_node, parent_node_id) = runtime_parent_for_circuit(circuit);
for device in &circuit.devices {
devices.push(ShapedDevice {
circuit_id: circuit.circuit_id.clone(),
+1 -3
View File
@@ -132,9 +132,7 @@ mod tests {
use super::network_json_with_carried_runtime_state;
use lqos_config::NetworkJsonNode;
use lqos_utils::{
qoq_heatmap::TemporalQoqHeatmap,
rtt::RttBuffer,
temporal_heatmap::TemporalHeatmap,
qoq_heatmap::TemporalQoqHeatmap, rtt::RttBuffer, temporal_heatmap::TemporalHeatmap,
units::DownUpOrder,
};
+38 -9
View File
@@ -5,7 +5,7 @@ use crate::{
use lqos_config::{
CircuitAnchorsFile, Config, ConfigShapedDevices, NetworkJsonNode, ShapedDevice,
TOPOLOGY_RUNTIME_STATUS_FILENAME, TopologyShapingCircuitInput, TopologyShapingDeviceInput,
TopologyShapingInputsFile,
TopologyShapingInputsFile, TopologyShapingResolutionSource,
};
use lqos_topology_compile::{ImportedTopologyBundle, TopologyImportFile};
use lqos_utils::rtt::RttBuffer;
@@ -337,6 +337,35 @@ fn runtime_inputs_build_shaped_devices_from_effective_parent_data() {
assert_eq!(shaped.devices[0].circuit_id, "circuit-1");
}
#[test]
fn runtime_inputs_leave_runtime_fallback_circuits_unparented() {
let _guard = TEST_LOCK.lock();
let shaping_inputs = TopologyShapingInputsFile {
circuits: vec![TopologyShapingCircuitInput {
circuit_id: "circuit-1".to_string(),
circuit_name: "Circuit Alpha".to_string(),
logical_parent_node_name: Some("LibreQoS Unattached [Site]".to_string()),
logical_parent_node_id: Some("libreqos:generated:splynx:site:unattached".to_string()),
resolution_source: TopologyShapingResolutionSource::RuntimeFallback,
devices: vec![TopologyShapingDeviceInput {
device_id: "device-1".to_string(),
device_name: "Device Alpha".to_string(),
ipv4: vec!["192.168.1.10/32".to_string()],
..TopologyShapingDeviceInput::default()
}],
..TopologyShapingCircuitInput::default()
}],
..TopologyShapingInputsFile::default()
};
let shaped = runtime_inputs::shaped_devices_from_runtime_inputs(&shaping_inputs);
assert_eq!(shaped.devices.len(), 1);
assert_eq!(shaped.devices[0].parent_node, "");
assert_eq!(shaped.devices[0].parent_node_id, None);
assert_eq!(shaped.devices[0].circuit_id, "circuit-1");
}
#[test]
fn load_shaped_devices_uses_topology_import_when_runtime_inputs_are_empty() {
let _guard = TEST_LOCK.lock();
@@ -402,8 +431,8 @@ fn load_shaped_devices_uses_topology_import_when_runtime_inputs_are_empty() {
let loaded = load_shaped_devices_for_config(&config)
.expect("preferred shaped-device source should load");
assert_eq!(loaded.devices.len(), 1);
assert_eq!(loaded.devices[0].circuit_id, "import-circuit");
assert_eq!(loaded.shaped.devices.len(), 1);
assert_eq!(loaded.shaped.devices[0].circuit_id, "import-circuit");
}
#[test]
@@ -464,8 +493,8 @@ fn load_shaped_devices_uses_topology_import_when_runtime_is_not_ready() {
let loaded =
load_shaped_devices_for_config(&config).expect("topology import fallback should load");
assert_eq!(loaded.devices.len(), 1);
assert_eq!(loaded.devices[0].circuit_id, "import-circuit");
assert_eq!(loaded.shaped.devices.len(), 1);
assert_eq!(loaded.shaped.devices[0].circuit_id, "import-circuit");
}
#[test]
@@ -505,7 +534,7 @@ fn load_shaped_devices_stays_empty_when_topology_import_is_empty() {
let loaded =
load_shaped_devices_for_config(&config).expect("integration mode should stay empty");
assert!(loaded.devices.is_empty());
assert!(loaded.shaped.devices.is_empty());
}
#[test]
@@ -553,8 +582,8 @@ fn load_shaped_devices_ignores_stale_runtime_inputs_in_manual_mode() {
let loaded =
load_shaped_devices_for_config(&config).expect("manual mode should use shaped devices csv");
assert_eq!(loaded.devices.len(), 1);
assert_eq!(loaded.devices[0].circuit_id, "csv-circuit");
assert_eq!(loaded.shaped.devices.len(), 1);
assert_eq!(loaded.shaped.devices[0].circuit_id, "csv-circuit");
}
#[test]
@@ -581,5 +610,5 @@ fn load_shaped_devices_stays_empty_when_runtime_status_is_malformed() {
let loaded =
load_shaped_devices_for_config(&config).expect("malformed status should stay empty");
assert!(loaded.devices.is_empty());
assert!(loaded.shaped.devices.is_empty());
}
+2 -2
View File
@@ -64,7 +64,7 @@ impl FileLock {
}
let unix_path = CString::new(LOCK_PATH)?;
unsafe {
nix::libc::chmod(unix_path.as_ptr(), mode_t::from_le(666));
nix::libc::chmod(unix_path.as_ptr(), mode_t::from_le(0o666));
}
Ok(())
}
@@ -77,7 +77,7 @@ impl FileLock {
std::fs::create_dir(dir_path)?;
let unix_path = CString::new(LOCK_DIR_PERMS)?;
unsafe {
nix::libc::chmod(unix_path.as_ptr(), 777);
nix::libc::chmod(unix_path.as_ptr(), 0o777);
}
Ok(())
}
+46 -3
View File
@@ -1,5 +1,6 @@
use std::{
fs::read_to_string,
io::Write,
path::{Path, PathBuf},
};
@@ -201,6 +202,9 @@ fn treeguard_read_path(config: &lqos_config::Config) -> PathBuf {
fn load_from_path(path: &Path) -> Result<OverrideFile> {
let raw = read_to_string(path)?;
if raw.trim().is_empty() {
anyhow::bail!("Overrides file '{}' was empty", path.display());
}
let as_json = serde_json::from_str(&raw)?;
Ok(as_json)
}
@@ -212,13 +216,40 @@ fn ensure_exists_default(path: &Path) -> Result<()> {
// Create a default empty file
let new_override_file = OverrideFile::default();
let as_json = serde_json::to_string(&new_override_file)?;
std::fs::write(path, as_json.as_bytes())?;
write_atomic(path, as_json.as_bytes())?;
Ok(())
}
fn save_to_path(path: &Path, overrides: &OverrideFile) -> Result<()> {
let as_json = serde_json::to_string_pretty(overrides)?;
std::fs::write(path, as_json.as_bytes())?;
write_atomic(path, as_json.as_bytes())?;
Ok(())
}
fn write_atomic(path: &Path, contents: &[u8]) -> Result<()> {
let parent = path.parent().ok_or_else(|| {
anyhow::anyhow!("Overrides path '{}' has no parent directory", path.display())
})?;
let file_name = path.file_name().ok_or_else(|| {
anyhow::anyhow!("Overrides path '{}' has no file name", path.display())
})?;
let pid = std::process::id();
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0);
let tmp_name = format!("{}.tmp.{}.{}", file_name.to_string_lossy(), pid, nanos);
let tmp_path = parent.join(tmp_name);
let mut file = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&tmp_path)?;
file.write_all(contents)?;
file.sync_all()?;
drop(file);
std::fs::rename(&tmp_path, path)?;
Ok(())
}
@@ -1089,7 +1120,8 @@ impl OverrideStore {
pub fn load_effective(apply_stormguard: bool, apply_treeguard: bool) -> Result<OverrideFile> {
let lock = FileLock::new()?;
let config = lqos_config::load_config()?;
let merged = Self::load_effective_for_config(&config, apply_stormguard, apply_treeguard)?;
let merged =
Self::load_effective_for_config_inner(&config, apply_stormguard, apply_treeguard)?;
drop(lock);
Ok(merged)
}
@@ -1098,10 +1130,21 @@ impl OverrideStore {
///
/// This function reads override files from the config's LibreQoS directory without
/// reloading `/etc/lqos.conf`.
///
/// Side effects: reads override files from disk and may create the operator
/// overrides file if it does not exist.
pub fn load_effective_for_config(
config: &lqos_config::Config,
apply_stormguard: bool,
apply_treeguard: bool,
) -> Result<OverrideFile> {
Self::load_effective_for_config_inner(config, apply_stormguard, apply_treeguard)
}
fn load_effective_for_config_inner(
config: &lqos_config::Config,
apply_stormguard: bool,
apply_treeguard: bool,
) -> Result<OverrideFile> {
let operator_path = overrides_path(config, OverrideLayer::Operator);
ensure_exists_default(&operator_path)?;
+114 -17
View File
@@ -15,8 +15,14 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
use surge_ping::{Client, Config as PingConfig, ICMP, IcmpPacket, PingIdentifier, PingSequence};
use thiserror::Error;
use tokio::sync::{Semaphore, mpsc, oneshot};
use tokio::time::timeout;
use tracing::warn;
const PROBE_DNS_RESOLUTION_TIMEOUT: Duration = Duration::from_secs(2);
const PROBE_WORKER_GRACE: Duration = Duration::from_secs(1);
const PROBE_MANAGER_REPLY_GRACE: Duration = Duration::from_secs(5);
const PROBE_BATCH_CONCURRENCY_ESTIMATE: usize = 256;
/// Logical consumer class for a probe request.
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize, Allocative)]
pub enum ProbeClass {
@@ -135,18 +141,23 @@ impl ProbeClient {
if requests.is_empty() {
return Ok(Vec::new());
}
let wait_timeout = probe_batch_wait_timeout(&requests);
let (reply_tx, reply_rx) = oneshot::channel();
self.tx
.send(ProbeCommand::ProbeBatch {
requests,
max_age,
reply: reply_tx,
})
.await
.map_err(|_| ProbeClientError::ManagerUnavailable)?;
reply_rx
.await
.map_err(|_| ProbeClientError::ManagerUnavailable)
timeout(wait_timeout, async {
self.tx
.send(ProbeCommand::ProbeBatch {
requests,
max_age,
reply: reply_tx,
})
.await
.map_err(|_| ProbeClientError::ManagerUnavailable)?;
reply_rx
.await
.map_err(|_| ProbeClientError::ManagerUnavailable)
})
.await
.map_err(|_| ProbeClientError::TimedOut)?
}
/// Probes a batch of targets for reachability.
@@ -192,6 +203,9 @@ pub enum ProbeClientError {
/// The manager returned no observations for the request.
#[error("shared probe manager returned no observations")]
EmptyResponse,
/// The manager did not answer before the caller's deadline.
#[error("shared probe manager timed out")]
TimedOut,
}
/// Background shared probe manager.
@@ -213,6 +227,27 @@ impl ProbeManager {
}
}
fn probe_worker_timeout(request_timeout: Duration) -> Duration {
PROBE_DNS_RESOLUTION_TIMEOUT
.saturating_add(request_timeout)
.saturating_add(PROBE_WORKER_GRACE)
}
fn probe_batch_wait_timeout(requests: &[ProbeRequest]) -> Duration {
let worker_timeout = requests
.iter()
.map(|request| probe_worker_timeout(request.timeout))
.max()
.unwrap_or(PROBE_MANAGER_REPLY_GRACE);
let wave_count = requests
.len()
.max(1)
.div_ceil(PROBE_BATCH_CONCURRENCY_ESTIMATE) as u32;
worker_timeout
.saturating_mul(wave_count)
.saturating_add(PROBE_MANAGER_REPLY_GRACE)
}
#[derive(Clone, Debug)]
struct ProbeCacheEntry {
normalized_target: String,
@@ -313,7 +348,23 @@ impl ProbeManagerState {
let semaphore = self.semaphore.clone();
join_set.spawn(async move {
let _permit = semaphore.acquire_owned().await.ok();
let observation = execute_probe(&request, &key.normalized_target).await;
let observation = match timeout(
probe_worker_timeout(request.timeout),
execute_probe(&request, &key.normalized_target),
)
.await
{
Ok(observation) => observation,
Err(_) => ProbeCacheEntry {
normalized_target: key.normalized_target.clone(),
resolved_ip: None,
kind: request.kind,
observed_at_unix_ms: now_unix_ms(),
reachable: false,
rtt_ms: None,
error: Some("probe worker timed out".to_string()),
},
};
(key, observation)
});
}
@@ -344,7 +395,7 @@ impl ProbeManagerState {
results
.into_iter()
.zip(requests.into_iter())
.zip(requests)
.map(|(result, request)| {
result.unwrap_or_else(|| {
failed_observation(
@@ -529,9 +580,15 @@ async fn resolve_target(normalized_target: &str) -> Result<(IpAddr, String), Str
return Ok((ip, ip.to_string()));
}
let mut addrs = tokio::net::lookup_host((normalized_target, 80))
.await
.map_err(|err| format!("unable to resolve target '{normalized_target}': {err}"))?;
let lookup = timeout(
PROBE_DNS_RESOLUTION_TIMEOUT,
tokio::net::lookup_host((normalized_target, 80)),
)
.await
.map_err(|_| format!("timed out resolving target '{normalized_target}'"))?;
let mut addrs =
lookup.map_err(|err| format!("unable to resolve target '{normalized_target}': {err}"))?;
let Some(addr) = addrs.next() else {
return Err(format!("unable to resolve target '{normalized_target}'"));
};
@@ -540,7 +597,11 @@ async fn resolve_target(normalized_target: &str) -> Result<(IpAddr, String), Str
#[cfg(test)]
mod tests {
use super::{ProbeKind, ProbeRequest, duration_to_millis, normalize_target};
use super::{
PROBE_DNS_RESOLUTION_TIMEOUT, PROBE_MANAGER_REPLY_GRACE, PROBE_WORKER_GRACE, ProbeKind,
ProbeRequest, duration_to_millis, normalize_target, probe_batch_wait_timeout,
probe_worker_timeout,
};
use std::time::Duration;
#[test]
@@ -576,4 +637,40 @@ mod tests {
assert_eq!(reachability.kind, ProbeKind::Reachability);
assert_eq!(rtt.kind, ProbeKind::RoundTripTime);
}
#[test]
fn probe_deadlines_include_dns_probe_and_reply_grace() {
let request = ProbeRequest::reachability(
"example.com",
super::ProbeClass::UiMonitor,
Duration::from_millis(750),
);
assert_eq!(
probe_worker_timeout(request.timeout),
PROBE_DNS_RESOLUTION_TIMEOUT + Duration::from_millis(750) + PROBE_WORKER_GRACE
);
assert_eq!(
probe_batch_wait_timeout(&[request]),
PROBE_DNS_RESOLUTION_TIMEOUT
+ Duration::from_millis(750)
+ PROBE_WORKER_GRACE
+ PROBE_MANAGER_REPLY_GRACE
);
}
#[test]
fn probe_batch_deadline_scales_for_multiple_probe_waves() {
let requests = vec![
ProbeRequest::reachability(
"example.com",
super::ProbeClass::UiMonitor,
Duration::from_secs(1),
);
257
];
assert_eq!(
probe_batch_wait_timeout(&requests),
probe_worker_timeout(Duration::from_secs(1)) * 2 + PROBE_MANAGER_REPLY_GRACE
);
}
}
+82 -9
View File
@@ -16,7 +16,13 @@ use lqos_bus::{BusRequest, BusResponse};
use lqos_config::{ShapedDevice, load_config};
use pyo3::pyclass;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use std::{
sync::{Mutex, OnceLock},
time::{Duration, Instant},
};
const REMOTE_WEIGHT_SUCCESS_TTL: Duration = Duration::from_secs(300);
const REMOTE_WEIGHT_FAILURE_BACKOFF: Duration = Duration::from_secs(300);
/// This struct is used to send a request to the Long Term Stats API
#[derive(Serialize, Deserialize)]
@@ -30,7 +36,7 @@ pub struct DeviceWeightRequest {
/// This struct is used to receive a response from the Long Term Stats API
/// It contains the circuit_id and the weight of the device
#[derive(Serialize, Deserialize, Debug)]
#[derive(Serialize, Deserialize, Debug, Clone)]
#[pyclass]
pub struct DeviceWeightResponse {
#[pyo3(get)]
@@ -39,6 +45,57 @@ pub struct DeviceWeightResponse {
pub weight: i64,
}
#[derive(Clone)]
struct CachedRemoteWeights {
fetched_at: Instant,
weights: Vec<DeviceWeightResponse>,
}
#[derive(Default)]
struct RemoteWeightCache {
last_success: Option<CachedRemoteWeights>,
last_failure: Option<Instant>,
}
fn remote_weight_cache() -> &'static Mutex<RemoteWeightCache> {
static CACHE: OnceLock<Mutex<RemoteWeightCache>> = OnceLock::new();
CACHE.get_or_init(|| Mutex::new(RemoteWeightCache::default()))
}
fn cached_remote_weights() -> Option<Vec<DeviceWeightResponse>> {
let cache = remote_weight_cache().lock().ok()?;
let cached = cache.last_success.as_ref()?;
if cached.fetched_at.elapsed() <= REMOTE_WEIGHT_SUCCESS_TTL {
return Some(cached.weights.clone());
}
None
}
fn recent_remote_weight_failure() -> bool {
let Ok(cache) = remote_weight_cache().lock() else {
return false;
};
cache
.last_failure
.is_some_and(|failed_at| failed_at.elapsed() <= REMOTE_WEIGHT_FAILURE_BACKOFF)
}
fn remember_remote_weight_success(weights: &[DeviceWeightResponse]) {
if let Ok(mut cache) = remote_weight_cache().lock() {
cache.last_success = Some(CachedRemoteWeights {
fetched_at: Instant::now(),
weights: weights.to_vec(),
});
cache.last_failure = None;
}
}
fn remember_remote_weight_failure() {
if let Ok(mut cache) = remote_weight_cache().lock() {
cache.last_failure = Some(Instant::now());
}
}
/// This function is used to get the device weights from the Long Term Stats API
fn get_weights_from_lts(
org_key: &str,
@@ -147,10 +204,28 @@ pub(crate) fn get_weights_rust() -> Result<Vec<DeviceWeightResponse>> {
let duration_seconds = 60 * 60 * 24; // 1 day
let percentile = 0.95;
eprintln!("Getting weights from Insight");
let weights = get_weights_from_lts(&org_key, &node_id, start, duration_seconds, percentile);
if let Ok(weights) = weights {
eprintln!("Retrieved {} weights from Insight", weights.len());
let remote_weights = if let Some(weights) = cached_remote_weights() {
eprintln!("Using cached Insight weights");
Some(weights)
} else if recent_remote_weight_failure() {
eprintln!("Skipping Insight weight fetch; recent request failed");
None
} else {
eprintln!("Getting weights from Insight");
match get_weights_from_lts(&org_key, &node_id, start, duration_seconds, percentile) {
Ok(weights) => {
eprintln!("Retrieved {} weights from Insight", weights.len());
remember_remote_weight_success(&weights);
Some(weights)
}
Err(err) => {
remember_remote_weight_failure();
eprintln!("Failed to get weights from Insight: {:?}", err);
None
}
}
};
if let Some(weights) = remote_weights {
// Merge them
for weight in weights.iter() {
if let Some(existing) = shaped_devices_weights
@@ -160,8 +235,6 @@ pub(crate) fn get_weights_rust() -> Result<Vec<DeviceWeightResponse>> {
existing.weight = weight.weight;
}
}
} else {
eprintln!("Failed to get weights from Insight: {:?}", weights);
}
}
@@ -238,6 +311,6 @@ pub(crate) fn calculate_tree_weights() -> Result<Vec<NetworkNodeWeight>> {
});
});
result.sort_by(|a, b| b.weight.cmp(&a.weight));
result.sort_by_key(|entry| std::cmp::Reverse(entry.weight));
Ok(result)
}
+78 -17
View File
@@ -942,6 +942,7 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(add_ip_mapping, m)?)?;
m.add_function(wrap_pyfunction!(validate_shaped_devices, m)?)?;
m.add_function(wrap_pyfunction!(wait_for_bus_ready, m)?)?;
m.add_function(wrap_pyfunction!(xdp_ip_mapping_ready, m)?)?;
m.add_function(wrap_pyfunction!(is_libre_already_running, m)?)?;
m.add_function(wrap_pyfunction!(create_lock_file, m)?)?;
m.add_function(wrap_pyfunction!(free_lock_file, m)?)?;
@@ -987,7 +988,6 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(uisp_suspended_strategy, m)?)?;
m.add_function(wrap_pyfunction!(airmax_capacity, m)?)?;
m.add_function(wrap_pyfunction!(ltu_capacity, m)?)?;
m.add_function(wrap_pyfunction!(use_ptmp_as_parent, m)?)?;
m.add_function(wrap_pyfunction!(uisp_base_url, m)?)?;
m.add_function(wrap_pyfunction!(uisp_auth_token, m)?)?;
m.add_function(wrap_pyfunction!(splynx_api_key, m)?)?;
@@ -1085,6 +1085,7 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(scheduler_error, m)?)?;
m.add_function(wrap_pyfunction!(scheduler_output, m)?)?;
m.add_function(wrap_pyfunction!(submit_urgent_issue, m)?)?;
m.add_function(wrap_pyfunction!(clear_urgent_issue_by_identity, m)?)?;
m.add_function(wrap_pyfunction!(xdp_ip_mapping_capacity, m)?)?;
m.add_function(wrap_pyfunction!(is_insight_enabled, m)?)?;
m.add_function(wrap_pyfunction!(log_info, m)?)?;
@@ -1380,6 +1381,13 @@ fn xdp_ip_mapping_capacity() -> PyResult<usize> {
Ok(lqos_sys::ip_mapping_capacity())
}
/// Returns whether the pinned BPF maps required for XDP IP mapping updates are ready.
#[pyfunction]
fn xdp_ip_mapping_ready() -> PyResult<bool> {
lqos_sys::ip_mapping_subsystem_ready()
.map_err(|e| PyOSError::new_err(format!("Unable to inspect XDP IP mapping maps: {e}")))
}
/// Requests Rust-side validation of `ShapedDevices.csv`
#[pyfunction]
fn validate_shaped_devices() -> PyResult<String> {
@@ -2240,8 +2248,10 @@ fn committed_bandwidth_multiplier() -> PyResult<f32> {
/// A UISP exception CPE entry paired with its forced parent.
pub struct PyExceptionCpe {
/// Child CPE site or device identifier.
#[pyo3(get)]
pub cpe: String,
/// Parent identifier assigned to the exception CPE.
#[pyo3(get)]
pub parent: String,
}
@@ -2290,12 +2300,6 @@ fn ltu_capacity() -> PyResult<f32> {
Ok(config.uisp_integration.ltu_capacity)
}
#[pyfunction]
fn use_ptmp_as_parent() -> PyResult<bool> {
let config = lqos_config::load_config().unwrap();
Ok(config.uisp_integration.use_ptmp_as_parent)
}
#[pyfunction]
fn uisp_base_url() -> PyResult<String> {
let config = lqos_config::load_config().unwrap();
@@ -3565,12 +3569,26 @@ impl Bakery {
estimated_memory_bytes: detail.estimated_memory_bytes,
})
.collect::<Vec<_>>();
let qdisc_counts_fit = estimate
.interfaces
.values()
.all(|count| *count <= estimate.safe_budget);
let preflight_status = match (qdisc_counts_fit, estimate.memory_ok) {
(true, true) => "fits preflight",
(false, true) => "exceeds qdisc-count preflight",
(true, false) => "passed qdisc-count preflight but failed memory preflight",
(false, false) => "exceeds qdisc-count preflight and failed memory preflight",
};
let memory_summary = if let Some(snapshot) = estimate.memory_snapshot.as_ref() {
let required_available_bytes = estimate
.memory_guard_min_available_bytes
.saturating_add(estimate.estimated_total_memory_bytes);
format!(
"estimated qdisc memory {} bytes with {} bytes currently available and safety floor {} bytes",
"estimated qdisc memory {} bytes; available memory {} bytes; required minimum {} bytes (safety floor {} bytes plus estimate)",
estimate.estimated_total_memory_bytes,
snapshot.available_bytes,
lqos_bakery::BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES
required_available_bytes,
estimate.memory_guard_min_available_bytes
)
} else {
format!(
@@ -3580,8 +3598,7 @@ impl Bakery {
};
let summary = if interface_reports.is_empty() {
format!(
"Planned queue model {} preflight. No shaping interfaces were queued; {memory_summary}.",
if is_ok { "fits" } else { "exceeds" },
"Planned queue model {preflight_status}. No shaping interfaces were queued; {memory_summary}.",
)
} else {
let interface_summary = interface_reports
@@ -3603,10 +3620,8 @@ impl Bakery {
.collect::<Vec<_>>()
.join(", ");
format!(
"Planned queue model {} preflight. {interface_summary}; safe budget {}, kernel limit {}; {memory_summary}.",
if is_ok { "fits" } else { "exceeds" },
estimate.safe_budget,
estimate.hard_limit,
"Planned queue model {preflight_status}. {interface_summary}; safe budget {}, kernel limit {}; {memory_summary}.",
estimate.safe_budget, estimate.hard_limit,
)
};
let _ = run_query(vec![BusRequest::BakeryReportPreflight {
@@ -3619,7 +3634,7 @@ impl Bakery {
.memory_snapshot
.as_ref()
.map(|snapshot| snapshot.available_bytes),
memory_guard_min_available_bytes: lqos_bakery::BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES,
memory_guard_min_available_bytes: estimate.memory_guard_min_available_bytes,
memory_ok: estimate.memory_ok,
interfaces: interface_reports,
}]);
@@ -3652,7 +3667,7 @@ impl Bakery {
result.set_item("memory_ok", estimate.memory_ok)?;
result.set_item(
"memory_guard_min_available_bytes",
lqos_bakery::BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES,
estimate.memory_guard_min_available_bytes,
)?;
if let Some(snapshot) = estimate.memory_snapshot {
result.set_item("memory_total_bytes", snapshot.total_bytes)?;
@@ -3859,6 +3874,52 @@ fn submit_urgent_issue(
Ok(false)
}
/// Clear urgent issues matching a code and dedupe key.
#[pyfunction]
fn clear_urgent_issue_by_identity(_py: Python, code: String, dedupe_key: String) -> PyResult<bool> {
if let Ok(reply) = run_query(vec![BusRequest::ClearUrgentIssueByIdentity {
code: code.clone(),
dedupe_key: dedupe_key.clone(),
}]) {
for resp in reply.iter() {
if let BusResponse::Ack = resp {
return Ok(true);
}
}
}
let Ok(reply) = run_query(vec![BusRequest::GetUrgentIssues]) else {
return Ok(false);
};
let mut ids_to_clear = Vec::new();
for resp in reply.iter() {
if let BusResponse::UrgentIssues(issues) = resp {
ids_to_clear.extend(
issues
.iter()
.filter(|issue| {
issue.code == code
&& issue.dedupe_key.as_deref().unwrap_or(&issue.code) == dedupe_key
})
.map(|issue| issue.id),
);
}
}
if ids_to_clear.is_empty() {
return Ok(false);
}
let requests = ids_to_clear
.into_iter()
.map(BusRequest::ClearUrgentIssue)
.collect::<Vec<_>>();
let Ok(reply) = run_query(requests) else {
return Ok(false);
};
Ok(reply.iter().any(|resp| matches!(resp, BusResponse::Ack)))
}
/// Log an informational message via the lqosd bus (appears in lqosd logs).
#[pyfunction]
fn log_info(_py: Python, message: String) -> PyResult<bool> {
+67 -9
View File
@@ -1,7 +1,7 @@
use crate::queue_types::QueueType;
use serde::Serialize;
use thiserror::Error;
use tracing::error;
use tracing::warn;
#[derive(Debug, Clone, Serialize)]
pub enum QueueDiff {
@@ -21,27 +21,46 @@ pub(crate) fn make_queue_diff(
QueueType::FqCodel(..) => match current {
QueueType::FqCodel(..) => Ok(fq_codel_diff(previous, current)?),
_ => {
error!(
"Queue diffs are not implemented for FqCodel to {:?}",
current
warn!(
previous = queue_kind(previous),
current = queue_kind(current),
"Queue type changed; resetting queue diff sample"
);
Err(QueueDiffError::NotImplemented)
Ok(QueueDiff::None)
}
},
QueueType::Cake(..) => match current {
QueueType::Cake(..) => Ok(cake_diff(previous, current)?),
_ => {
error!("Queue diffs are not implemented for Cake to {:?}", current);
Err(QueueDiffError::NotImplemented)
warn!(
previous = queue_kind(previous),
current = queue_kind(current),
"Queue type changed; resetting queue diff sample"
);
Ok(QueueDiff::None)
}
},
_ => {
error!("Queue diffs are not implemented for {:?}", current);
Err(QueueDiffError::NotImplemented)
warn!(
previous = queue_kind(previous),
current = queue_kind(current),
"Queue diff unavailable for queue type; resetting queue diff sample"
);
Ok(QueueDiff::None)
}
}
}
fn queue_kind(queue: &QueueType) -> &'static str {
match queue {
QueueType::FqCodel(_) => "fq_codel",
QueueType::Cake(_) => "cake",
QueueType::Mq(_) => "mq",
QueueType::Htb(_) => "htb",
QueueType::ClsAct => "clsact",
}
}
#[derive(Serialize, Clone, Debug)]
pub struct CakeDiff {
pub bytes: u64,
@@ -116,3 +135,42 @@ fn fq_codel_diff(previous: &QueueType, current: &QueueType) -> Result<QueueDiff,
}
Err(QueueDiffError::NotImplemented)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::deserialize_tc_tree;
#[test]
fn queue_type_change_resets_diff_sample() {
let fq_codel = deserialize_tc_tree(
r#"[{
"kind":"fq_codel",
"handle":"9000:",
"parent":"1:2",
"options":{},
"bytes":1000,
"packets":10
}]"#,
)
.expect("fq_codel qdisc should parse")
.remove(0);
let cake = deserialize_tc_tree(
r#"[{
"kind":"cake",
"handle":"9001:",
"parent":"1:2",
"options":{},
"bytes":2000,
"packets":20,
"tins":[]
}]"#,
)
.expect("cake qdisc should parse")
.remove(0);
let diff = make_queue_diff(&fq_codel, &cake).expect("type changes should not error");
assert!(matches!(diff, QueueDiff::None));
}
}
+38 -1
View File
@@ -42,6 +42,11 @@ pub(crate) enum CommitOutcome {
pub(crate) fn build_candidate_config(existing: Option<lqos_config::Config>) -> lqos_config::Config {
let mut config = existing.unwrap_or_default();
let existing_bridge_mtu = config.bridge.as_ref().and_then(|bridge| bridge.mtu);
let existing_single_interface_mtu = config
.single_interface
.as_ref()
.and_then(|single| single.mtu);
let new_config = CURRENT_CONFIG.lock();
config.node_name = new_config.node_name.clone();
config.queues.downlink_bandwidth_mbps = new_config.mbps_to_internet;
@@ -55,6 +60,7 @@ pub(crate) fn build_candidate_config(existing: Option<lqos_config::Config>) -> l
use_xdp_bridge: false,
to_internet: new_config.to_internet.clone(),
to_network: new_config.to_network.clone(),
mtu: existing_bridge_mtu,
});
}
BridgeMode::XDP => {
@@ -63,6 +69,7 @@ pub(crate) fn build_candidate_config(existing: Option<lqos_config::Config>) -> l
use_xdp_bridge: true,
to_internet: new_config.to_internet.clone(),
to_network: new_config.to_network.clone(),
mtu: existing_bridge_mtu,
});
}
BridgeMode::Single => {
@@ -70,6 +77,7 @@ pub(crate) fn build_candidate_config(existing: Option<lqos_config::Config>) -> l
interface: new_config.to_internet.clone(),
internet_vlan: new_config.internet_vlan,
network_vlan: new_config.network_vlan,
mtu: existing_single_interface_mtu,
});
config.bridge = None;
}
@@ -245,7 +253,8 @@ fn load_existing_or_default(event_log: &mut Vec<String>) -> Result<lqos_config::
#[cfg(test)]
mod tests {
use super::load_existing_or_default;
use super::{build_candidate_config, load_existing_or_default};
use crate::config_builder::{BridgeMode, CURRENT_CONFIG};
use once_cell::sync::Lazy;
use parking_lot::Mutex;
use std::fs;
@@ -282,4 +291,32 @@ mod tests {
lqos_config::clear_cached_config();
fs::remove_file(path).expect("remove temp config");
}
#[test]
fn build_candidate_config_preserves_existing_mtu_for_same_mode() {
let _guard = CONFIG_ENV_LOCK.lock();
let previous_builder = {
let mut builder = CURRENT_CONFIG.lock();
let previous = builder.clone();
builder.bridge_mode = BridgeMode::Linux;
builder.to_internet = "wan0".to_string();
builder.to_network = "lan0".to_string();
previous
};
let existing = lqos_config::Config {
bridge: Some(lqos_config::BridgeConfig {
mtu: Some(9000),
..lqos_config::BridgeConfig::default()
}),
..lqos_config::Config::default()
};
let candidate = build_candidate_config(Some(existing));
assert_eq!(
candidate.bridge.as_ref().and_then(|bridge| bridge.mtu),
Some(9000)
);
*CURRENT_CONFIG.lock() = previous_builder;
}
}
+22
View File
@@ -19,6 +19,28 @@ struct ip_hash_info {
__u64 device_id;
};
struct tc_classify_control {
__u32 bypass;
};
struct {
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
__uint(max_entries, 1);
__type(key, __u32);
__type(value, struct tc_classify_control);
__uint(pinning, LIBBPF_PIN_BY_NAME);
} tc_classify_control SEC(".maps");
static __always_inline bool tc_classify_bypass_enabled(void) {
__u32 key = 0;
struct tc_classify_control *control = bpf_map_lookup_elem(&tc_classify_control, &key);
if (!control) {
// Fail safe: bypass classification rather than risk stale TC handles.
return true;
}
return control->bypass != 0;
}
// In on-a-stick mode, upload classes/CPUs are offset by this amount.
// This is configured by userspace at load time.
extern __u32 stick_offset;
+6
View File
@@ -109,6 +109,9 @@ int xdp_prog(struct xdp_md *ctx)
bpf_debug("Error: interface direction unspecified, aborting.");
return XDP_PASS;
}
if (tc_classify_bypass_enabled()) {
return XDP_PASS;
}
// Do we need to perform a VLAN redirect?
bool vlan_redirect = false;
@@ -266,6 +269,9 @@ int tc_iphash_to_cpu(struct __sk_buff *skb)
bpf_debug("(TC) Error: interface direction unspecified, aborting.");
return TC_ACT_OK;
}
if (tc_classify_bypass_enabled()) {
return TC_ACT_OK;
}
#ifdef VERBOSE
bpf_debug("(TC) SKB VLAN TCI: %u", skb->vlan_tci);
#endif
+3
View File
@@ -22,6 +22,7 @@ mod ip_mapping;
mod kernel_wrapper;
mod linux;
mod lqos_kernel;
mod tc_classify_control;
mod throughput;
pub use bpf_iterator::{end_flows, expire_throughput, iterate_flows};
@@ -33,6 +34,8 @@ pub use kernel_wrapper::LibreQoSKernels;
pub use linux::num_possible_cpus;
pub use lqos_kernel::interface_name_to_index;
pub use lqos_kernel::ip_mapping_capacity;
pub use lqos_kernel::ip_mapping_subsystem_ready;
pub use lqos_kernel::max_tracked_ips;
pub use lqos_kernel::unload_xdp_from_interface;
pub use tc_classify_control::{initialize_tc_classify_bypass, set_tc_classify_bypass};
pub use throughput::{HostCounter, throughput_for_each};
+170 -53
View File
@@ -31,6 +31,13 @@ pub(crate) mod bpf {
include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
}
#[derive(Clone, Copy)]
struct PinnedMapAbiSpec {
path: &'static str,
expected_key_size: u32,
expected_value_size: u32,
}
/// Returns the value set in the C XDP system's MAX_TRACKED_IPS
/// constant.
pub fn max_tracked_ips() -> usize {
@@ -42,27 +49,7 @@ fn pinned_map_max_entries(path: &str) -> Result<Option<u32>> {
return Ok(None);
}
let path_c = CString::new(path)?;
let fd = unsafe { bpf_obj_get(path_c.as_ptr()) };
if fd < 0 {
warn!("Unable to open pinned BPF map '{path}' for capacity check (fd={fd})");
return Ok(None);
}
let mut info = MaybeUninit::<bpf_map_info>::zeroed();
let mut len: u32 = std::mem::size_of::<bpf_map_info>() as u32;
let err = unsafe { bpf_obj_get_info_by_fd(fd, info.as_mut_ptr() as *mut c_void, &mut len) };
unsafe {
close(fd);
}
if err != 0 {
return Err(Error::msg(format!(
"Unable to query pinned BPF map '{path}' capacity (err={err})."
)));
}
let info = unsafe { info.assume_init() };
Ok(Some(info.max_entries))
Ok(pinned_map_info(path)?.map(|info| info.max_entries))
}
/// Returns the currently available IP-mapping capacity.
@@ -80,6 +67,155 @@ pub fn ip_mapping_capacity() -> usize {
}
}
fn ip_mapping_pinned_map_abi_specs() -> [PinnedMapAbiSpec; 3] {
let expected_key_size = std::mem::size_of::<crate::ip_mapping::IpHashKey>() as u32;
let expected_value_size = std::mem::size_of::<crate::ip_mapping::IpHashData>() as u32;
[
PinnedMapAbiSpec {
path: "/sys/fs/bpf/map_ip_to_cpu_and_tc",
expected_key_size,
expected_value_size,
},
PinnedMapAbiSpec {
path: "/sys/fs/bpf/ip_to_cpu_and_tc_hotcache",
expected_key_size: std::mem::size_of::<XdpIpAddress>() as u32,
expected_value_size,
},
PinnedMapAbiSpec {
path: "/sys/fs/bpf/ip_mapping_epoch",
expected_key_size: std::mem::size_of::<u32>() as u32,
expected_value_size: std::mem::size_of::<u32>() as u32,
},
]
}
fn pinned_map_info(path: &str) -> Result<Option<bpf_map_info>> {
let path_c = CString::new(path)?;
let fd = unsafe { bpf_obj_get(path_c.as_ptr()) };
if fd < 0 {
return Ok(None);
}
let mut info = MaybeUninit::<bpf_map_info>::zeroed();
let mut len: u32 = std::mem::size_of::<bpf_map_info>() as u32;
let err = unsafe { bpf_obj_get_info_by_fd(fd, info.as_mut_ptr() as *mut c_void, &mut len) };
unsafe {
close(fd);
}
if err != 0 {
return Err(Error::msg(format!(
"Unable to query pinned BPF map '{path}' info (err={err})."
)));
}
let info = unsafe { info.assume_init() };
Ok(Some(info))
}
fn pinned_map_key_value_sizes(path: &str) -> Result<Option<(u32, u32)>> {
Ok(pinned_map_info(path)?.map(|info| (info.key_size, info.value_size)))
}
fn ip_mapping_subsystem_ready_with<F>(mut map_sizes: F) -> Result<bool>
where
F: FnMut(&str) -> Result<Option<(u32, u32)>>,
{
for spec in ip_mapping_pinned_map_abi_specs() {
let Some((key_size, value_size)) = map_sizes(spec.path)? else {
return Ok(false);
};
if key_size != spec.expected_key_size || value_size != spec.expected_value_size {
return Ok(false);
}
}
Ok(true)
}
/// Returns whether the pinned maps required for XDP IP mapping updates are ready.
///
/// This function has side effects: it opens the relevant pinned BPF maps and
/// queries their ABI metadata. It returns `Ok(false)` for maps that are not yet
/// pinned or have an incompatible ABI, and `Err` for unexpected inspection
/// failures.
pub fn ip_mapping_subsystem_ready() -> Result<bool> {
ip_mapping_subsystem_ready_with(pinned_map_key_value_sizes)
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
#[test]
fn ip_mapping_ready_requires_all_expected_maps() {
let specs = ip_mapping_pinned_map_abi_specs();
let sizes: HashMap<&str, (u32, u32)> = specs
.iter()
.map(|spec| {
(
spec.path,
(spec.expected_key_size, spec.expected_value_size),
)
})
.collect();
let ready = ip_mapping_subsystem_ready_with(|path| Ok(sizes.get(path).copied()))
.expect("readiness check should succeed");
assert!(ready);
}
#[test]
fn ip_mapping_ready_is_false_when_a_map_is_missing() {
let ready = ip_mapping_subsystem_ready_with(|path| {
if path == "/sys/fs/bpf/ip_mapping_epoch" {
Ok(None)
} else {
let spec = ip_mapping_pinned_map_abi_specs()
.into_iter()
.find(|spec| spec.path == path)
.expect("test path should be part of IP mapping ABI specs");
Ok(Some((spec.expected_key_size, spec.expected_value_size)))
}
})
.expect("missing map should not be an inspection error");
assert!(!ready);
}
#[test]
fn ip_mapping_ready_is_false_when_a_map_abi_differs() {
let ready = ip_mapping_subsystem_ready_with(|path| {
let spec = ip_mapping_pinned_map_abi_specs()
.into_iter()
.find(|spec| spec.path == path)
.expect("test path should be part of IP mapping ABI specs");
let value_size = if path == "/sys/fs/bpf/map_ip_to_cpu_and_tc" {
spec.expected_value_size + 1
} else {
spec.expected_value_size
};
Ok(Some((spec.expected_key_size, value_size)))
})
.expect("ABI mismatch should not be an inspection error");
assert!(!ready);
}
#[test]
fn ip_mapping_ready_returns_inspection_error() {
let err = ip_mapping_subsystem_ready_with(|path| {
if path == "/sys/fs/bpf/map_ip_to_cpu_and_tc" {
Err(Error::msg("map info failed"))
} else {
Ok(None)
}
})
.expect_err("inspection errors should be returned to the caller");
assert!(err.to_string().contains("map info failed"));
}
}
pub fn check_root() -> Result<()> {
unsafe {
if geteuid() == 0 {
@@ -99,25 +235,10 @@ fn remove_incompatible_pinned_map(
return Ok(());
}
let path_c = CString::new(path)?;
let fd = unsafe { bpf_obj_get(path_c.as_ptr()) };
if fd < 0 {
warn!("Unable to open pinned BPF map '{path}' for ABI check (fd={fd})");
let Some(info) = pinned_map_info(path)? else {
warn!("Unable to open pinned BPF map '{path}' for ABI check");
return Ok(());
}
let mut info = MaybeUninit::<bpf_map_info>::zeroed();
let mut len: u32 = std::mem::size_of::<bpf_map_info>() as u32;
let err = unsafe { bpf_obj_get_info_by_fd(fd, info.as_mut_ptr() as *mut c_void, &mut len) };
unsafe {
close(fd);
}
if err != 0 {
return Err(Error::msg(format!(
"Unable to query pinned BPF map '{path}' info (err={err})."
)));
}
let info = unsafe { info.assume_init() };
};
if info.key_size != expected_key_size || info.value_size != expected_value_size {
warn!(
@@ -135,22 +256,17 @@ fn remove_incompatible_pinned_map(
}
fn ensure_ip_mapping_maps_abi() -> Result<()> {
let expected_key_size = std::mem::size_of::<crate::ip_mapping::IpHashKey>() as u32;
let expected_value_size = std::mem::size_of::<crate::ip_mapping::IpHashData>() as u32;
for spec in ip_mapping_pinned_map_abi_specs() {
remove_incompatible_pinned_map(
spec.path,
spec.expected_key_size,
spec.expected_value_size,
)?;
}
remove_incompatible_pinned_map(
"/sys/fs/bpf/map_ip_to_cpu_and_tc",
expected_key_size,
expected_value_size,
)?;
remove_incompatible_pinned_map(
"/sys/fs/bpf/ip_to_cpu_and_tc_hotcache",
std::mem::size_of::<XdpIpAddress>() as u32,
expected_value_size,
)?;
remove_incompatible_pinned_map(
"/sys/fs/bpf/ip_mapping_epoch",
std::mem::size_of::<u32>() as u32,
"/sys/fs/bpf/tc_classify_control",
std::mem::size_of::<u32>() as u32,
std::mem::size_of::<crate::tc_classify_control::TcClassifyControl>() as u32,
)?;
remove_incompatible_pinned_map(
"/sys/fs/bpf/map_traffic",
@@ -339,6 +455,7 @@ pub fn attach_xdp_and_tc_to_interface(
// Ensure no lingering XDP programs before loading/attaching
let _ = unload_xdp_from_interface(interface_name);
load_kernel(skeleton)?;
crate::initialize_tc_classify_bypass()?;
let prog_fd = bpf::bpf_program__fd((*skeleton).progs.xdp_prog);
attach_xdp_best_available(interface_index, prog_fd, interface_name)?;
skeleton
@@ -0,0 +1,146 @@
//! Control-map helpers for temporarily bypassing XDP redirection and TC classification.
use crate::lqos_kernel::bpf::libbpf_num_possible_cpus;
use anyhow::{Error, Result};
use libbpf_sys::{bpf_map_update_elem, bpf_obj_get};
use nix::libc::close;
use std::{
ffi::{CString, c_void},
io,
mem::size_of,
};
use tracing::error;
const TC_CLASSIFY_CONTROL_PATH: &str = "/sys/fs/bpf/tc_classify_control";
const TC_CLASSIFY_CONTROL_KEY: u32 = 0;
const PER_CPU_VALUE_ALIGNMENT: usize = 8;
/// Rust mirror of `struct tc_classify_control` in the eBPF program.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
#[repr(C)]
pub struct TcClassifyControl {
/// Non-zero tells XDP and TC classify to pass packets without redirecting
/// or setting TC handles.
pub bypass: u32,
}
fn aligned_per_cpu_value_size() -> usize {
size_of::<TcClassifyControl>().div_ceil(PER_CPU_VALUE_ALIGNMENT) * PER_CPU_VALUE_ALIGNMENT
}
fn per_cpu_payload(enabled: bool, cpu_count: u32) -> Vec<u8> {
let stride = aligned_per_cpu_value_size();
let mut payload = vec![0u8; stride * cpu_count as usize];
let control = TcClassifyControl {
bypass: u32::from(enabled),
};
let bytes = control.bypass.to_ne_bytes();
for cpu in 0..cpu_count as usize {
let offset = cpu * stride;
payload[offset..offset + bytes.len()].copy_from_slice(&bytes);
}
payload
}
fn update_tc_classify_control(enabled: bool) -> Result<()> {
let path_c = CString::new(TC_CLASSIFY_CONTROL_PATH)?;
let fd = unsafe { bpf_obj_get(path_c.as_ptr()) };
if fd < 0 {
let error = io::Error::last_os_error();
let errno = error.raw_os_error().unwrap_or(0);
return Err(Error::msg(format!(
"Unable to open BPF map '{TC_CLASSIFY_CONTROL_PATH}' for TC classify bypass update (fd={fd}, errno={errno}, error={error})"
)));
}
let cpu_count = unsafe { libbpf_num_possible_cpus() };
if cpu_count <= 0 {
unsafe {
close(fd);
}
return Err(Error::msg(format!(
"Unable to determine CPU count for TC classify bypass update: libbpf_num_possible_cpus returned {cpu_count}"
)));
}
let cpu_count = u32::try_from(cpu_count).map_err(|e| {
unsafe {
close(fd);
}
Error::msg(format!(
"Unable to convert CPU count for TC classify bypass update: {e}"
))
})?;
let mut key = TC_CLASSIFY_CONTROL_KEY;
let payload = per_cpu_payload(enabled, cpu_count);
let err = unsafe {
bpf_map_update_elem(
fd,
&mut key as *mut u32 as *mut c_void,
payload.as_ptr() as *mut c_void,
0,
)
};
let update_error = (err != 0).then(io::Error::last_os_error);
unsafe {
close(fd);
}
if err != 0 {
let error = update_error.unwrap_or_else(|| io::Error::from_raw_os_error(0));
let errno = error.raw_os_error().unwrap_or(0);
error!(
"Unable to update BPF map '{}' for TC classify bypass={} (err={}, errno={}, error={})",
TC_CLASSIFY_CONTROL_PATH, enabled, err, errno, error
);
return Err(Error::msg(format!(
"Unable to update BPF map '{TC_CLASSIFY_CONTROL_PATH}' for TC classify bypass={enabled} (err={err}, errno={errno}, error={error})"
)));
}
Ok(())
}
/// Initializes TC classify bypass to disabled.
///
/// This function has side effects: it writes all per-CPU values for the pinned
/// `tc_classify_control` BPF map.
pub fn initialize_tc_classify_bypass() -> Result<()> {
update_tc_classify_control(false)
}
/// Enables or disables TC classify bypass.
///
/// This function has side effects: it writes all per-CPU values for the pinned
/// `tc_classify_control` BPF map. When enabled, XDP returns `XDP_PASS` before
/// CPU redirection, and TC classify returns `TC_ACT_OK` before consuming XDP
/// metadata, flow cache, hot cache, or LPM mappings.
pub fn set_tc_classify_bypass(enabled: bool) -> Result<()> {
update_tc_classify_control(enabled)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn per_cpu_payload_uses_eight_byte_stride() {
let payload = per_cpu_payload(true, 3);
assert_eq!(payload.len(), 3 * PER_CPU_VALUE_ALIGNMENT);
for cpu in 0..3 {
let offset = cpu * PER_CPU_VALUE_ALIGNMENT;
assert_eq!(
&payload[offset..offset + size_of::<u32>()],
&1u32.to_ne_bytes()
);
assert_eq!(
&payload[offset + size_of::<u32>()..offset + PER_CPU_VALUE_ALIGNMENT],
&[0, 0, 0, 0]
);
}
}
#[test]
fn disabled_payload_clears_all_per_cpu_values() {
let payload = per_cpu_payload(false, 2);
assert_eq!(payload, vec![0u8; 2 * PER_CPU_VALUE_ALIGNMENT]);
}
}
File diff suppressed because it is too large. Load diff
+230 -20
View File
@@ -1,25 +1,29 @@
use anyhow::{Context, Result};
use lqos_bus::{BusRequest, BusResponse, bus_request};
use lqos_bus::{BusRequest, BusResponse, bus_request_with_timeout};
use lqos_config::{
TopologyAttachmentEndpointStatus, TopologyAttachmentHealthEntry,
TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus,
TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus, TopologyCanonicalStateFile,
compute_topology_source_generation, load_config,
};
use lqos_overrides::TopologyOverridesFile;
use lqos_probe::{ProbeClass, ProbeRequest};
use std::collections::HashMap;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::net::IpAddr;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tracing::{info, warn};
use crate::{
AttachmentProbeSpec, build_effective_topology_artifacts_from_canonical, is_health_state_fresh,
load_canonical_topology_state, prepare_runtime_topology_editor_state_from_canonical,
probe_specs_from_state, publish_effective_topology_artifacts,
publish_topology_runtime_error_status,
AttachmentProbeSpec,
build_effective_topology_artifacts_from_canonical_with_runtime_queue_context,
is_health_state_fresh, load_canonical_topology_state,
prepare_runtime_topology_editor_state_from_canonical, probe_specs_from_state,
publish_effective_topology_artifacts, publish_topology_runtime_error_status,
};
const TOPOLOGY_PROBE_MAX_AGE_MS: u64 = 250;
const TOPOLOGY_PROBE_BUS_TIMEOUT: Duration = Duration::from_secs(8);
fn now_unix() -> Option<u64> {
SystemTime::now()
@@ -119,10 +123,13 @@ async fn probe_specs(
return Ok(HashMap::new());
}
let responses = bus_request(vec![BusRequest::ProbeBatch {
requests: probe_requests,
max_age_ms: TOPOLOGY_PROBE_MAX_AGE_MS,
}])
let responses = bus_request_with_timeout(
vec![BusRequest::ProbeBatch {
requests: probe_requests,
max_age_ms: TOPOLOGY_PROBE_MAX_AGE_MS,
}],
TOPOLOGY_PROBE_BUS_TIMEOUT,
)
.await
.map_err(|err| anyhow::anyhow!("unable to query shared probe manager: {err}"))?;
let Some(response) = responses.into_iter().next() else {
@@ -312,6 +319,61 @@ fn refresh_health_state(
Ok(true)
}
fn hash_health_status(status: TopologyAttachmentHealthStatus, hasher: &mut impl Hasher) {
match status {
TopologyAttachmentHealthStatus::Healthy => 0_u8.hash(hasher),
TopologyAttachmentHealthStatus::Suppressed => 1_u8.hash(hasher),
TopologyAttachmentHealthStatus::ProbeUnavailable => 2_u8.hash(hasher),
TopologyAttachmentHealthStatus::Disabled => 3_u8.hash(hasher),
}
}
fn hash_health_effective_entry(entry: &TopologyAttachmentHealthEntry, hasher: &mut impl Hasher) {
entry.attachment_pair_id.hash(hasher);
entry.attachment_id.hash(hasher);
entry.child_node_id.hash(hasher);
entry.parent_node_id.hash(hasher);
entry.local_probe_ip.hash(hasher);
entry.remote_probe_ip.hash(hasher);
hash_health_status(entry.status, hasher);
entry.probeable.hash(hasher);
entry.enabled.hash(hasher);
entry.suppressed_until_unix.hash(hasher);
}
fn health_effective_signature(health_state: &TopologyAttachmentHealthStateFile) -> u64 {
let mut hasher = DefaultHasher::new();
for entry in &health_state.attachments {
hash_health_effective_entry(entry, &mut hasher);
}
hasher.finish()
}
#[derive(Clone, Debug, Default)]
struct RuntimeBuildGate {
last_source_generation: Option<String>,
last_overrides_generation: Option<u64>,
last_health_effective_signature: Option<u64>,
cached_probe_specs: Vec<AttachmentProbeSpec>,
publish_completed: bool,
next_error_retry_after_unix: Option<u64>,
}
fn topology_overrides_generation(config: &lqos_config::Config) -> u64 {
let path = TopologyOverridesFile::path_for_config(config);
let mut hasher = DefaultHasher::new();
path.hash(&mut hasher);
if let Ok(contents) = std::fs::read(&path) {
match serde_json::from_slice::<TopologyOverridesFile>(&contents)
.and_then(|overrides| serde_json::to_vec(&overrides))
{
Ok(canonical) => canonical.hash(&mut hasher),
Err(_) => contents.hash(&mut hasher),
}
}
hasher.finish()
}
#[derive(Clone, Copy, Debug, Default)]
struct RoundHints {
probes_enabled: bool,
@@ -320,30 +382,63 @@ struct RoundHints {
async fn run_round(
health_state: &mut TopologyAttachmentHealthStateFile,
last_effective: &mut HashMap<String, Option<String>>,
gate: &mut RuntimeBuildGate,
) -> Result<RoundHints> {
let config = load_config().context("Unable to load config for topology runtime")?;
let source_generation = compute_topology_source_generation(config.as_ref())
.context("Unable to compute topology source generation")?;
let canonical = load_canonical_topology_state(config.as_ref());
let overrides =
TopologyOverridesFile::load().context("Unable to load topology overrides file")?;
let prepared = prepare_runtime_topology_editor_state_from_canonical(&canonical, &overrides);
let specs = probe_specs_from_state(&prepared, &overrides);
let overrides_generation = topology_overrides_generation(config.as_ref());
let source_changed = gate.last_source_generation.as_deref() != Some(source_generation.as_str());
let overrides_changed = gate.last_overrides_generation != Some(overrides_generation);
let mut loaded_inputs: Option<(TopologyCanonicalStateFile, TopologyOverridesFile)> = None;
if source_changed || overrides_changed || gate.cached_probe_specs.is_empty() {
let canonical = load_canonical_topology_state(config.as_ref());
let overrides =
TopologyOverridesFile::load().context("Unable to load topology overrides file")?;
let prepared = prepare_runtime_topology_editor_state_from_canonical(&canonical, &overrides);
gate.cached_probe_specs = probe_specs_from_state(&prepared, &overrides);
loaded_inputs = Some((canonical, overrides));
}
let specs = &gate.cached_probe_specs;
let probes_enabled = specs.iter().any(|spec| spec.enabled);
if probes_enabled {
match probe_specs(&specs, Duration::from_millis(750)).await {
match probe_specs(specs, Duration::from_millis(750)).await {
Ok(probe_results) => {
refresh_health_state(config.as_ref(), health_state, &specs, &probe_results)?;
refresh_health_state(config.as_ref(), health_state, specs, &probe_results)?;
}
Err(err) => {
warn!("Topology probe round could not query shared probe manager: {err:#}");
}
}
} else {
refresh_health_state(config.as_ref(), health_state, &specs, &HashMap::new())?;
refresh_health_state(config.as_ref(), health_state, specs, &HashMap::new())?;
}
let artifacts = build_effective_topology_artifacts_from_canonical(
let next_signature = health_effective_signature(health_state);
let retry_due = gate
.next_error_retry_after_unix
.is_some_and(|deadline| now_unix().is_some_and(|now| now >= deadline));
let retry_pending = gate.next_error_retry_after_unix.is_some() && !retry_due;
let source_or_health_changed = source_changed
|| overrides_changed
|| gate.last_health_effective_signature.as_ref() != Some(&next_signature);
let should_publish =
source_or_health_changed || (!retry_pending && (!gate.publish_completed || retry_due));
if !should_publish {
return Ok(RoundHints { probes_enabled });
}
let (canonical, overrides) = match loaded_inputs {
Some(inputs) => inputs,
None => {
let canonical = load_canonical_topology_state(config.as_ref());
let overrides =
TopologyOverridesFile::load().context("Unable to load topology overrides file")?;
(canonical, overrides)
}
};
let artifacts = build_effective_topology_artifacts_from_canonical_with_runtime_queue_context(
config.as_ref(),
&canonical,
&overrides,
@@ -367,8 +462,21 @@ async fn run_round(
"Unable to publish failed topology runtime status after publish error: {status_err:#}"
);
}
let health = &config.integration_common.topology_attachment_health;
let retry_delay = health
.refresh_debounce_seconds
.max(health.probe_interval_seconds.max(1));
gate.last_source_generation = Some(source_generation.clone());
gate.last_overrides_generation = Some(overrides_generation);
gate.last_health_effective_signature = Some(next_signature);
gate.next_error_retry_after_unix = now_unix().map(|now| now.saturating_add(retry_delay));
return Err(err);
}
gate.last_source_generation = Some(source_generation.clone());
gate.last_overrides_generation = Some(overrides_generation);
gate.last_health_effective_signature = Some(next_signature);
gate.publish_completed = true;
gate.next_error_retry_after_unix = None;
for node in &artifacts.effective.nodes {
let next = node.effective_attachment_id.clone();
@@ -393,9 +501,12 @@ async fn run_round(
pub async fn start_topology() {
let mut health_state = load_starting_health();
let mut last_effective = HashMap::<String, Option<String>>::new();
let mut build_gate = RuntimeBuildGate::default();
loop {
let round_hints = match run_round(&mut health_state, &mut last_effective).await {
let round_hints = match run_round(&mut health_state, &mut last_effective, &mut build_gate)
.await
{
Ok(hints) => hints,
Err(err) => {
if let Ok(config) = load_config()
@@ -433,3 +544,102 @@ pub async fn start_topology() {
tokio::time::sleep(Duration::from_secs(sleep_seconds)).await;
}
}
#[cfg(test)]
mod tests {
use super::{
TopologyAttachmentHealthEntry, health_effective_signature, topology_overrides_generation,
};
use lqos_config::{Config, TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus};
use lqos_overrides::TopologyOverridesFile;
use std::fs;
use std::time::{SystemTime, UNIX_EPOCH};
fn unique_temp_dir(prefix: &str) -> std::path::PathBuf {
let unique = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system time should be valid")
.as_nanos();
std::env::temp_dir().join(format!("{prefix}-{}-{unique}", std::process::id()))
}
fn health_entry() -> TopologyAttachmentHealthEntry {
TopologyAttachmentHealthEntry {
attachment_pair_id: "pair-1".to_string(),
attachment_id: Some("attachment-1".to_string()),
child_node_id: Some("child-1".to_string()),
parent_node_id: Some("parent-1".to_string()),
local_probe_ip: Some("192.0.2.1".to_string()),
remote_probe_ip: Some("192.0.2.2".to_string()),
status: TopologyAttachmentHealthStatus::Healthy,
probeable: true,
enabled: true,
consecutive_misses: 1,
consecutive_successes: 2,
last_success_unix: Some(10),
endpoint_status: vec![],
..TopologyAttachmentHealthEntry::default()
}
}
#[test]
fn health_effective_signature_ignores_probe_counters_and_timestamps() {
let first = TopologyAttachmentHealthStateFile {
generated_unix: Some(1),
attachments: vec![health_entry()],
..TopologyAttachmentHealthStateFile::default()
};
let mut second_entry = health_entry();
second_entry.consecutive_misses = 4;
second_entry.consecutive_successes = 5;
second_entry.last_success_unix = Some(20);
second_entry.last_failure_unix = Some(21);
let second = TopologyAttachmentHealthStateFile {
generated_unix: Some(2),
attachments: vec![second_entry],
..TopologyAttachmentHealthStateFile::default()
};
assert_eq!(
health_effective_signature(&first),
health_effective_signature(&second)
);
}
#[test]
fn health_effective_signature_tracks_suppression_status() {
let first = TopologyAttachmentHealthStateFile {
attachments: vec![health_entry()],
..TopologyAttachmentHealthStateFile::default()
};
let mut suppressed = health_entry();
suppressed.status = TopologyAttachmentHealthStatus::Suppressed;
suppressed.suppressed_until_unix = Some(123);
let second = TopologyAttachmentHealthStateFile {
attachments: vec![suppressed],
..TopologyAttachmentHealthStateFile::default()
};
assert_ne!(
health_effective_signature(&first),
health_effective_signature(&second)
);
}
#[test]
fn topology_overrides_generation_tracks_manual_override_file_changes() {
let lqos_directory = unique_temp_dir("lqos-topology-runtime-overrides");
fs::create_dir_all(&lqos_directory).expect("temp lqos directory should exist");
let config = Config {
lqos_directory: lqos_directory.to_string_lossy().to_string(),
..Config::default()
};
let path = TopologyOverridesFile::path_for_config(&config);
let before = topology_overrides_generation(&config);
fs::write(&path, "{\"schemaVersion\":1}\n").expect("override file should write");
let after = topology_overrides_generation(&config);
assert_ne!(before, after);
}
}
@@ -50,11 +50,6 @@ struct CircuitProjection {
node_name: String,
}
const GENERATED_UNATTACHED_SITE_ID: &str = "libreqos:generated:site:unattached";
const GENERATED_UNATTACHED_SITE_NAME: &str = "LibreQoS Unattached [Site]";
const GENERATED_UNATTACHED_AP_ID: &str = "libreqos:generated:ap:unattached";
const GENERATED_UNATTACHED_AP_NAME: &str = "LibreQoS Unattached [AP]";
fn empty_parent_candidates(source: &str) -> TopologyParentCandidatesFile {
TopologyParentCandidatesFile {
source: source.to_string(),
@@ -330,85 +325,6 @@ where
circuits
}
fn unresolved_circuits(
imported: &ImportedTopologyBundle,
projection: &HashMap<String, CircuitProjection>,
) -> Vec<(String, String)> {
let mut unresolved = Vec::new();
let mut seen = HashSet::new();
for device in &imported.shaped_devices.devices {
if projection.contains_key(&device.circuit_id) || !seen.insert(device.circuit_id.clone()) {
continue;
}
unresolved.push((device.circuit_id.clone(), device.circuit_name.clone()));
}
unresolved
}
fn generated_ap_node() -> ExportedNode {
ExportedNode {
id: GENERATED_UNATTACHED_AP_ID.to_string(),
name: GENERATED_UNATTACHED_AP_NAME.to_string(),
kind: "AP".to_string(),
parent_id: None,
download_mbps: None,
upload_mbps: None,
}
}
fn generated_site_node() -> ExportedNode {
ExportedNode {
id: GENERATED_UNATTACHED_SITE_ID.to_string(),
name: GENERATED_UNATTACHED_SITE_NAME.to_string(),
kind: "Site".to_string(),
parent_id: None,
download_mbps: None,
upload_mbps: None,
}
}
fn exported_unattached_fallback_node(
nodes_by_id: &HashMap<String, ExportedNode>,
) -> Option<ExportedNode> {
let mut preferred_site = None;
let mut preferred_ap = None;
for node in nodes_by_id.values() {
let id = node.id.to_ascii_lowercase();
let name = node.name.to_ascii_lowercase();
let is_unattached = id.contains("unattach")
|| name.contains("unattach")
|| id.contains("orphan")
|| name.contains("orphan");
if !is_unattached {
continue;
}
if node.kind.eq_ignore_ascii_case("site") {
preferred_site = Some(node.clone());
break;
}
if preferred_ap.is_none() && node.kind.eq_ignore_ascii_case("ap") {
preferred_ap = Some(node.clone());
}
}
preferred_site.or(preferred_ap)
}
fn attach_unresolved_to_projection(
projection: &mut HashMap<String, CircuitProjection>,
unresolved: Vec<(String, String)>,
fallback_node: &ExportedNode,
) {
for (circuit_id, _circuit_name) in unresolved {
projection.insert(
circuit_id,
CircuitProjection {
node_id: fallback_node.id.clone(),
node_name: fallback_node.name.clone(),
},
);
}
}
fn set_number(map: &mut Map<String, Value>, key: &str, value: Option<u64>, fallback: u64) {
map.insert(
key.to_string(),
@@ -706,8 +622,6 @@ fn compile_full_like(
) -> Result<CompiledTopologyBundle> {
let source = compiled_source(&imported.source, mode);
let ingress_identity = compiled_ingress_identity(&imported, mode);
let (exported_nodes_by_id, _) = exported_index(&imported.compatibility_network_json);
let unresolved_fallback = exported_unattached_fallback_node(&exported_nodes_by_id);
let mut editor = sanitized_editor_state(&source, imported.generated_unix, &imported);
editor.ingress_identity = ingress_identity.clone();
let mut canonical = TopologyCanonicalStateFile::from_editor_and_network(
@@ -723,18 +637,6 @@ fn compile_full_like(
.unwrap_or_else(|| empty_parent_candidates(&source));
parent_candidates.source = source.clone();
parent_candidates.ingress_identity = ingress_identity.clone();
let mut shaped_devices = imported.shaped_devices.devices;
if let Some(fallback) = unresolved_fallback {
for device in &mut shaped_devices {
let missing_parent = device.parent_node.trim().is_empty()
&& device.parent_node_id.is_none()
&& device.anchor_node_id.is_none();
if missing_parent {
device.parent_node = fallback.name.clone();
device.parent_node_id = Some(fallback.id.clone());
}
}
}
Ok(CompiledTopologyBundle {
source: source.clone(),
generated_unix: imported.generated_unix,
@@ -743,7 +645,7 @@ fn compile_full_like(
editor,
parent_candidates,
compatibility_network_json: imported.compatibility_network_json,
shaped_devices: make_shaped_devices(shaped_devices),
shaped_devices: make_shaped_devices(imported.shaped_devices.devices),
circuit_anchors: CircuitAnchorsFile {
schema_version: imported.circuit_anchors.schema_version,
source,
@@ -767,10 +669,9 @@ fn compile_flat(imported: ImportedTopologyBundle) -> CompiledTopologyBundle {
fn compile_ap_only(imported: ImportedTopologyBundle) -> Result<CompiledTopologyBundle> {
let source = compiled_source(&imported.source, TopologyCompileMode::ApOnly);
let mut projection = circuit_projection_map(&imported, |nodes_by_id, node_id| {
let projection = circuit_projection_map(&imported, |nodes_by_id, node_id| {
nearest_kind(nodes_by_id, node_id, "AP")
});
let unresolved = unresolved_circuits(&imported, &projection);
let (nodes_by_id, _) = native_projection_index(&imported);
let mut root = BTreeMap::<String, Value>::new();
let mut seen = HashSet::new();
@@ -783,11 +684,6 @@ fn compile_ap_only(imported: ImportedTopologyBundle) -> Result<CompiledTopologyB
};
root.insert(node.name.clone(), ap_object(node, None));
}
if !unresolved.is_empty() {
let fallback_ap = generated_ap_node();
attach_unresolved_to_projection(&mut projection, unresolved, &fallback_ap);
root.insert(fallback_ap.name.clone(), ap_object(&fallback_ap, None));
}
Ok(projected_outputs(
imported,
projection,
@@ -799,10 +695,9 @@ fn compile_ap_only(imported: ImportedTopologyBundle) -> Result<CompiledTopologyB
fn compile_ap_site(imported: ImportedTopologyBundle) -> Result<CompiledTopologyBundle> {
let source = compiled_source(&imported.source, TopologyCompileMode::ApSite);
let mut projection = circuit_projection_map(&imported, |nodes_by_id, node_id| {
let projection = circuit_projection_map(&imported, |nodes_by_id, node_id| {
nearest_kind(nodes_by_id, node_id, "AP")
});
let unresolved = unresolved_circuits(&imported, &projection);
let (nodes_by_id, _) = native_projection_index(&imported);
let mut groups = BTreeMap::<String, (ExportedNode, BTreeMap<String, Value>)>::new();
let mut synthetic_counter = 0usize;
@@ -835,21 +730,6 @@ fn compile_ap_site(imported: ImportedTopologyBundle) -> Result<CompiledTopologyB
ap_object(&ap, Some(site_entry.name.as_str())),
);
}
if !unresolved.is_empty() {
let fallback_site = generated_site_node();
let fallback_ap = ExportedNode {
parent_id: Some(fallback_site.id.clone()),
..generated_ap_node()
};
attach_unresolved_to_projection(&mut projection, unresolved, &fallback_ap);
let entry = groups
.entry(fallback_site.id.clone())
.or_insert_with(|| (fallback_site.clone(), BTreeMap::new()));
entry.1.insert(
fallback_ap.name.clone(),
ap_object(&fallback_ap, Some(fallback_site.name.as_str())),
);
}
let mut root = BTreeMap::<String, Value>::new();
for (_site_id, (site, children)) in groups {
root.insert(
@@ -892,10 +772,7 @@ pub fn compile_topology(
#[cfg(test)]
mod tests {
use super::{
GENERATED_UNATTACHED_AP_ID, GENERATED_UNATTACHED_AP_NAME, GENERATED_UNATTACHED_SITE_NAME,
TopologyCompileMode, compile_topology,
};
use super::{TopologyCompileMode, compile_topology};
use crate::bundle::ImportedTopologyBundle;
use lqos_config::{
CircuitAnchor, CircuitAnchorsFile, ConfigShapedDevices, ShapedDevice,
@@ -1044,22 +921,24 @@ mod tests {
}
#[test]
fn ap_only_drops_unresolved_orphan_parents() {
fn ap_only_leaves_unresolved_orphan_parents_unattached() {
let compiled = compile_topology(imported_bundle(), TopologyCompileMode::ApOnly)
.expect("ap_only compilation should succeed");
assert_eq!(compiled.circuit_anchors.anchors.len(), 3);
assert!(compiled.compatibility_network_json[GENERATED_UNATTACHED_AP_NAME].is_object());
assert_eq!(compiled.circuit_anchors.anchors.len(), 2);
assert!(
compiled
.compatibility_network_json
.get("LibreQoS Unattached [AP]")
.is_none()
);
let device = compiled
.shaped_devices
.devices
.iter()
.find(|device| device.circuit_id == "circuit-3")
.expect("unresolved circuit should still be emitted");
assert_eq!(
device.parent_node_id.as_deref(),
Some(GENERATED_UNATTACHED_AP_ID)
);
assert_eq!(device.parent_node, GENERATED_UNATTACHED_AP_NAME);
assert!(device.parent_node.is_empty());
assert!(device.parent_node_id.is_none());
}
#[test]
@@ -1079,27 +958,15 @@ mod tests {
.expect("site should have children");
assert!(children.get("AP North").is_some());
assert!(children.get("AP South").is_some());
let fallback_site = root[GENERATED_UNATTACHED_SITE_NAME]
.as_object()
.expect("fallback site should be present for unresolved circuits");
let fallback_children = fallback_site["children"]
.as_object()
.expect("fallback site should have children");
assert!(
fallback_children
.get(GENERATED_UNATTACHED_AP_NAME)
.is_some()
);
assert!(root.get("LibreQoS Unattached [Site]").is_none());
let device = compiled
.shaped_devices
.devices
.iter()
.find(|device| device.circuit_id == "circuit-3")
.expect("unresolved circuit should still be emitted");
assert_eq!(
device.parent_node_id.as_deref(),
Some(GENERATED_UNATTACHED_AP_ID)
);
assert!(device.parent_node.is_empty());
assert!(device.parent_node_id.is_none());
}
#[test]
@@ -1359,7 +1226,7 @@ mod tests {
}
#[test]
fn full_mode_assigns_blank_parent_circuits_to_exported_unattached_site() {
fn full_mode_leaves_blank_parent_circuits_unattached() {
let mut imported = imported_bundle();
imported.compatibility_network_json = json!({
"Site Alpha": {
@@ -1411,11 +1278,8 @@ mod tests {
.find(|device| device.circuit_id == "circuit-3")
.expect("unresolved circuit should still be emitted");
assert_eq!(device.parent_node, "LibreQoS Unattached [Site]");
assert_eq!(
device.parent_node_id.as_deref(),
Some("libreqos:generated:splynx:site:unattached")
);
assert!(device.parent_node.is_empty());
assert!(device.parent_node_id.is_none());
}
#[test]
+110 -57
View File
@@ -1,5 +1,5 @@
use anyhow::{Result, bail};
use futures_util::{StreamExt, sink::SinkExt};
use futures_util::{StreamExt, sink::SinkExt, stream::SplitSink};
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, time::Duration};
use tokio::net::TcpStream;
@@ -23,6 +23,11 @@ pub use messages::{
};
const SHAPER_VERSION_STRING: &str = include_str!("../../../../VERSION_STRING");
const CONTROL_CHANNEL_QUEUE_DEPTH: usize = 256;
const CONNECTION_COMMAND_QUEUE_DEPTH: usize = 1024;
const SOCKET_SENDER_QUEUE_DEPTH: usize = 32;
type ControlSocketWriter = SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>;
fn current_shaper_version() -> Option<String> {
let version = SHAPER_VERSION_STRING.trim();
@@ -133,7 +138,7 @@ pub struct ControlChannelBuilder {
pub fn init_control_channel() -> Result<ControlChannelBuilder> {
// Doing this two-step: make the channel here and then spawn the task
let (tx, rx) = tokio::sync::mpsc::channel(256);
let (tx, rx) = tokio::sync::mpsc::channel(CONTROL_CHANNEL_QUEUE_DEPTH);
Ok(ControlChannelBuilder { tx, rx })
}
@@ -148,7 +153,7 @@ pub async fn start_control_channel(builder: ControlChannelBuilder) -> Result<()>
async fn control_channel_loop(mut builder: ControlChannelBuilder) -> Result<()> {
// Handle the persistent channel to Insight here
let (tx, rx) = tokio::sync::mpsc::channel::<ConnectionCommand>(1024);
let (tx, rx) = tokio::sync::mpsc::channel::<ConnectionCommand>(CONNECTION_COMMAND_QUEUE_DEPTH);
tokio::spawn(persistent_connection(rx));
while let Some(cmd) = builder.rx.recv().await {
@@ -273,6 +278,72 @@ const TCP_TIMEOUT: Duration = Duration::from_secs(30);
// Prevent unbounded growth while waiting for Welcome
const MAX_PENDING_CHATBOT_MESSAGES: usize = 256;
fn encode_ws_binary(
message: &messages::WsMessage,
label: &'static str,
) -> std::result::Result<Message, ()> {
let Ok((_, _, bytes)) = message.to_bytes() else {
error!("Failed to serialize {label} message");
return Err(());
};
Ok(Message::Binary(bytes))
}
async fn send_ws_message_now(
write: &mut ControlSocketWriter,
message: Message,
label: &'static str,
) -> std::result::Result<(), ()> {
let Ok(Ok(_)) = timeout(TCP_TIMEOUT, write.send(message)).await else {
error!("Failed to send {label} message");
return Err(());
};
Ok(())
}
async fn send_ingest_batch(
write: &mut ControlSocketWriter,
serial: usize,
chunks: Vec<Vec<u8>>,
) -> std::result::Result<usize, ()> {
let n_chunks = chunks.len();
let byte_count = chunks.iter().map(|chunk| chunk.len()).sum::<usize>();
let begin = messages::WsMessage::BeginIngest {
unique_id: serial as u64,
n_chunks: n_chunks as u64,
};
send_ws_message_now(
write,
encode_ws_binary(&begin, "BeginIngest")?,
"BeginIngest",
)
.await?;
for (chunk_index, chunk) in chunks.into_iter().enumerate() {
let ingest_chunk = messages::WsMessage::IngestChunk {
unique_id: serial as u64,
chunk: chunk_index as u64,
n_chunks: n_chunks as u64,
data: chunk,
};
send_ws_message_now(
write,
encode_ws_binary(&ingest_chunk, "IngestChunk")?,
"IngestChunk",
)
.await?;
}
let end = messages::WsMessage::EndIngest {
unique_id: serial as u64,
n_chunks: n_chunks as u64,
};
send_ws_message_now(write, encode_ws_binary(&end, "EndIngest")?, "EndIngest").await?;
Ok(byte_count)
}
async fn persistent_connection(
mut rx: tokio::sync::mpsc::Receiver<ConnectionCommand>,
) -> std::result::Result<(), String> {
@@ -303,7 +374,7 @@ async fn persistent_connection(
// Split the socket
let (mut write, mut read) = socket.split();
let (socket_sender_tx, mut socket_sender_rx) =
tokio::sync::mpsc::channel::<Message>(32);
tokio::sync::mpsc::channel::<Message>(SOCKET_SENDER_QUEUE_DEPTH);
let mut ping_interval = tokio::time::interval(Duration::from_secs(10));
let mut license_interval = tokio::time::interval(Duration::from_secs(60 * 15)); // 15 minutes
let mut pending_history: HashMap<
@@ -363,61 +434,11 @@ async fn persistent_connection(
info!("Not permitted to send chunks yet");
continue 'message_pump;
}
let n_chunks = chunks.len();
let byte_count = chunks.iter().map(|c| c.len()).sum::<usize>();
// Send BeginIngest
let Ok((_, _, bytes)) = messages::WsMessage::BeginIngest { unique_id: serial as u64, n_chunks: n_chunks as u64 }.to_bytes() else {
error!("Failed to serialize BeginIngest message");
let Ok(byte_count) =
send_ingest_batch(&mut write, serial, chunks).await
else {
break 'message_pump;
};
if let Err(e) = socket_sender_tx.try_send(Message::Binary(bytes)) {
match e {
TrySendError::Full(_) => {
warn!("Send unavailable: BeginIngest queue full; dropping message");
}
TrySendError::Closed(_) => {
error!("Failed to send BeginIngest message: channel closed");
break 'message_pump;
}
}
}
// Submit Each Chunk
for (i, chunk) in chunks.into_iter().enumerate() {
let Ok((_, _, bytes)) = messages::WsMessage::IngestChunk { unique_id: serial as u64, chunk: i as u64, n_chunks: n_chunks as u64, data: chunk }.to_bytes() else {
error!("Failed to serialize IngestChunk message");
break 'message_pump;
};
if let Err(e) = socket_sender_tx.try_send(Message::Binary(bytes)) {
match e {
TrySendError::Full(_) => {
warn!("Send unavailable: IngestChunk queue full; dropping chunk");
}
TrySendError::Closed(_) => {
error!("Failed to send IngestChunk message: channel closed");
break 'message_pump;
}
}
}
}
// Send EndIngest
let Ok((_, _, bytes)) = messages::WsMessage::EndIngest { unique_id: serial as u64, n_chunks: n_chunks as u64 }.to_bytes() else {
error!("Failed to serialize EndIngest message");
break 'message_pump;
};
if let Err(e) = socket_sender_tx.try_send(Message::Binary(bytes)) {
match e {
TrySendError::Full(_) => {
warn!("Send unavailable: EndIngest queue full; dropping message");
}
TrySendError::Closed(_) => {
error!("Failed to send EndIngest message: channel closed");
break 'message_pump;
}
}
}
debug!("Submitted {} bytes for ingestion", byte_count);
}
Some(ConnectionCommand::FetchHistory { request, responder }) => {
@@ -1736,3 +1757,35 @@ async fn tree_snapshot_streaming(
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn encode_ws_binary_round_trips_begin_ingest() {
let encoded = encode_ws_binary(
&messages::WsMessage::BeginIngest {
unique_id: 42,
n_chunks: 7,
},
"BeginIngest",
)
.expect("BeginIngest should encode");
let Message::Binary(bytes) = encoded else {
panic!("BeginIngest should encode to a binary websocket frame");
};
match messages::WsMessage::from_bytes(&bytes).expect("BeginIngest should decode") {
messages::WsMessage::BeginIngest {
unique_id,
n_chunks,
} => {
assert_eq!(unique_id, 42);
assert_eq!(n_chunks, 7);
}
other => panic!("unexpected decoded message: {other:?}"),
}
}
}
+48 -10
View File
@@ -10,6 +10,7 @@ mod ip_mapping;
#[cfg(feature = "equinix_tests")]
mod lqos_daht_test;
pub mod lts2_sys;
mod memory_watchdog;
mod network_devices_hooks;
mod node_manager;
mod preflight_checks;
@@ -248,6 +249,7 @@ fn main() -> Result<()> {
// Memory Debugging
memory_debug();
memory_watchdog::start_memory_watchdog();
let control_tx_for_webserver = control_tx_for_web.clone();
let system_usage_tx_for_web = system_usage_tx.clone();
@@ -381,7 +383,7 @@ fn main() -> Result<()> {
let _ = throughput_tracker::flow_data::setup_flow_analysis();
start_heimdall()?;
spawn_queue_structure_monitor()?;
shaped_devices_tracker::shaping_inputs_watcher()?;
shaped_devices_tracker::topology_runtime_status_watcher()?;
throughput_tracker::spawn_throughput_monitor(
flow_tx,
system_usage_tx.clone(),
@@ -616,16 +618,18 @@ fn handle_bus_requests(requests: &[BusRequest], responses: &mut Vec<BusResponse>
lqos_bus::BusResponse::Ack
}
BusRequest::UpdateLqosDTuning(..) => tuning::tune_lqosd_from_bus(req),
BusRequest::UpdateLqosdConfig(config) => {
let result = lqos_config::update_config(config);
if result.is_err() {
error!("Error updating config: {:?}", result);
BusRequest::UpdateLqosdConfig(config) => match lqos_config::update_config(config) {
Ok(()) => {
if let Ok(cfg) = lqos_config::load_config() {
let _ = stick::recompute_stick_offset(&cfg);
}
BusResponse::Ack
}
if let Ok(cfg) = lqos_config::load_config() {
let _ = stick::recompute_stick_offset(&cfg);
Err(err) => {
error!("Error updating config: {err:?}");
BusResponse::Fail(err.to_string())
}
BusResponse::Ack
}
},
BusRequest::CreateDynamicCircuit { shaped_device } => {
crate::dynamic_circuits::create_dynamic_circuit((**shaped_device).clone())
}
@@ -1043,6 +1047,10 @@ fn handle_bus_requests(requests: &[BusRequest], responses: &mut Vec<BusResponse>
urgent::clear(*id);
BusResponse::Ack
}
BusRequest::ClearUrgentIssueByIdentity { code, dedupe_key } => {
urgent::clear_by_identity(code, dedupe_key);
BusResponse::Ack
}
BusRequest::ClearAllUrgentIssues => {
urgent::clear_all();
BusResponse::Ack
@@ -1351,7 +1359,7 @@ fn tree_summary_l2_data() -> Vec<(usize, Vec<(usize, lqos_config::NetworkJsonTra
}
}
candidates.sort_by(|a, b| b.3.cmp(&a.3));
candidates.sort_by_key(|candidate| std::cmp::Reverse(candidate.3));
let n: usize = 10;
if candidates.len() > n {
candidates.truncate(n);
@@ -1386,3 +1394,33 @@ fn search_result_to_bus(entry: node_manager::SearchResult) -> lqos_bus::SearchRe
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use lqos_bus::{UrgentSeverity, UrgentSource};
#[test]
fn bus_clear_urgent_issue_by_identity_reaches_urgent_store() {
let code = "TEST_BUS_XDP_IP_MAPPING_APPLY_FAILED";
urgent::clear_by_identity(code, code);
urgent::submit(
UrgentSource::LibreQoS,
UrgentSeverity::Error,
code.to_string(),
"mapping failed".to_string(),
None,
Some(code.to_string()),
);
let requests = [BusRequest::ClearUrgentIssueByIdentity {
code: code.to_string(),
dedupe_key: code.to_string(),
}];
let mut responses = Vec::new();
handle_bus_requests(&requests, &mut responses);
assert!(matches!(responses.as_slice(), [BusResponse::Ack]));
assert!(!urgent::list().iter().any(|issue| issue.code == code));
}
}
+337
View File
@@ -0,0 +1,337 @@
//! Memory pressure watchdog for controlled `lqosd` restarts.
//!
//! The watchdog samples `/proc` and exits the daemon before the kernel OOM
//! path has to choose a victim. `lqosd.service` is configured with
//! `Restart=always`, so this releases process memory while preserving logs that
//! explain why the restart happened.
use crate::stats::{BUS_REQUESTS, FLOWS_TRACKED, HIGH_WATERMARK, TIME_TO_POLL_HOSTS};
use std::collections::HashMap;
use std::sync::atomic::Ordering;
use std::time::Duration;
use tracing::{error, info, warn};
const CHECK_INTERVAL_SECONDS: u64 = 15;
const PRESSURE_WARNING_AVAILABLE_BYTES: u64 = mib(3_072);
const DEFAULT_MIN_AVAILABLE_BYTES: u64 = mib(2_304);
const DEFAULT_MIN_PROCESS_BYTES: u64 = mib(1_024);
const DEFAULT_MAX_PROCESS_BYTES: u64 = gib(4);
const DEFAULT_MAX_SWAP_BYTES: u64 = gib(1);
const WATCHDOG_EXIT_CODE: i32 = 75;
/// Spawns the memory watchdog thread.
///
/// Side effects: this function starts a background thread. The thread reads
/// `/proc/meminfo`, `/proc/self/status`, and exits the process when memory
/// pressure reaches the configured critical threshold.
pub fn start_memory_watchdog() {
if env_flag_enabled("LQOSD_MEMORY_WATCHDOG_DISABLED") {
warn!("lqosd memory watchdog disabled by LQOSD_MEMORY_WATCHDOG_DISABLED");
return;
}
match std::thread::Builder::new()
.name("Memory Watchdog".to_string())
.spawn(memory_watchdog_loop)
{
Ok(_) => info!("lqosd memory watchdog started"),
Err(err) => warn!("Failed to start lqosd memory watchdog: {err:?}"),
}
}
fn memory_watchdog_loop() {
let config = MemoryWatchdogConfig::from_env();
let mut warned_about_pressure = false;
loop {
std::thread::sleep(Duration::from_secs(CHECK_INTERVAL_SECONDS));
let Ok(snapshot) = MemorySnapshot::read() else {
warn!("Unable to sample lqosd memory state from /proc");
continue;
};
if let Some(reason) = config.critical_reason(&snapshot) {
log_critical_memory_snapshot(&snapshot, &config, &reason);
std::thread::sleep(Duration::from_millis(250));
std::process::exit(WATCHDOG_EXIT_CODE);
}
if snapshot.mem_available_bytes < PRESSURE_WARNING_AVAILABLE_BYTES {
if !warned_about_pressure {
warn!(
"lqosd memory pressure warning: mem_available={} process_rss={} process_swap={} process_total={} threads={}",
format_bytes(snapshot.mem_available_bytes),
format_bytes(snapshot.process_rss_bytes),
format_bytes(snapshot.process_swap_bytes),
format_bytes(snapshot.process_total_bytes()),
snapshot.thread_count,
);
warned_about_pressure = true;
}
} else {
warned_about_pressure = false;
}
}
}
fn log_critical_memory_snapshot(
snapshot: &MemorySnapshot,
config: &MemoryWatchdogConfig,
reason: &str,
) {
error!(
"lqosd memory watchdog restarting daemon: reason={} mem_available={} mem_total={} process_rss={} process_swap={} process_total={} threads={} thresholds=min_available:{} min_process:{} max_process:{} max_swap:{}",
reason,
format_bytes(snapshot.mem_available_bytes),
format_bytes(snapshot.mem_total_bytes),
format_bytes(snapshot.process_rss_bytes),
format_bytes(snapshot.process_swap_bytes),
format_bytes(snapshot.process_total_bytes()),
snapshot.thread_count,
format_bytes(config.min_available_bytes),
format_bytes(config.min_process_bytes),
format_bytes(config.max_process_bytes),
format_bytes(config.max_swap_bytes),
);
error!(
"lqosd memory watchdog diagnostics: flows_tracked={} bus_requests={} time_to_poll_hosts_us={} high_watermark_down={} high_watermark_up={}",
FLOWS_TRACKED.load(Ordering::Relaxed),
BUS_REQUESTS.load(Ordering::Relaxed),
TIME_TO_POLL_HOSTS.load(Ordering::Relaxed),
HIGH_WATERMARK.get_down(),
HIGH_WATERMARK.get_up(),
);
}
#[derive(Debug)]
struct MemoryWatchdogConfig {
min_available_bytes: u64,
min_process_bytes: u64,
max_process_bytes: u64,
max_swap_bytes: u64,
}
impl MemoryWatchdogConfig {
fn from_env() -> Self {
Self {
min_available_bytes: env_mib(
"LQOSD_MEMORY_WATCHDOG_MIN_AVAILABLE_MB",
DEFAULT_MIN_AVAILABLE_BYTES,
),
min_process_bytes: env_mib(
"LQOSD_MEMORY_WATCHDOG_MIN_PROCESS_MB",
DEFAULT_MIN_PROCESS_BYTES,
),
max_process_bytes: env_mib(
"LQOSD_MEMORY_WATCHDOG_MAX_PROCESS_MB",
DEFAULT_MAX_PROCESS_BYTES,
),
max_swap_bytes: env_mib("LQOSD_MEMORY_WATCHDOG_MAX_SWAP_MB", DEFAULT_MAX_SWAP_BYTES),
}
}
fn critical_reason(&self, snapshot: &MemorySnapshot) -> Option<String> {
if snapshot.process_swap_bytes >= self.max_swap_bytes {
return Some(format!(
"process swap {} reached limit {}",
format_bytes(snapshot.process_swap_bytes),
format_bytes(self.max_swap_bytes)
));
}
if snapshot.process_total_bytes() >= self.max_process_bytes {
return Some(format!(
"process rss+swap {} reached limit {}",
format_bytes(snapshot.process_total_bytes()),
format_bytes(self.max_process_bytes)
));
}
if snapshot.mem_available_bytes < self.min_available_bytes
&& snapshot.process_total_bytes() >= self.min_process_bytes
{
return Some(format!(
"available memory {} below limit {} while lqosd uses {}",
format_bytes(snapshot.mem_available_bytes),
format_bytes(self.min_available_bytes),
format_bytes(snapshot.process_total_bytes())
));
}
None
}
}
#[derive(Debug, PartialEq, Eq)]
struct MemorySnapshot {
mem_total_bytes: u64,
mem_available_bytes: u64,
process_rss_bytes: u64,
process_swap_bytes: u64,
thread_count: u64,
}
impl MemorySnapshot {
fn read() -> anyhow::Result<Self> {
let meminfo = parse_proc_key_values(&std::fs::read_to_string("/proc/meminfo")?);
let status = parse_proc_key_values(&std::fs::read_to_string("/proc/self/status")?);
Ok(Self {
mem_total_bytes: read_required_kb(&meminfo, "MemTotal")?,
mem_available_bytes: read_required_kb(&meminfo, "MemAvailable")?,
process_rss_bytes: read_required_kb(&status, "VmRSS")?,
process_swap_bytes: read_optional_kb(&status, "VmSwap"),
thread_count: read_optional_raw(&status, "Threads"),
})
}
fn process_total_bytes(&self) -> u64 {
self.process_rss_bytes
.saturating_add(self.process_swap_bytes)
}
}
fn parse_proc_key_values(input: &str) -> HashMap<String, u64> {
input
.lines()
.filter_map(|line| {
let (key, rest) = line.split_once(':')?;
let value = rest.split_whitespace().next()?.parse::<u64>().ok()?;
Some((key.to_string(), value))
})
.collect()
}
fn read_required_kb(values: &HashMap<String, u64>, key: &str) -> anyhow::Result<u64> {
values
.get(key)
.copied()
.map(kib)
.ok_or_else(|| anyhow::anyhow!("{key} missing from /proc memory data"))
}
fn read_optional_kb(values: &HashMap<String, u64>, key: &str) -> u64 {
values.get(key).copied().map(kib).unwrap_or(0)
}
fn read_optional_raw(values: &HashMap<String, u64>, key: &str) -> u64 {
values.get(key).copied().unwrap_or(0)
}
fn env_flag_enabled(name: &str) -> bool {
std::env::var(name)
.map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES"))
.unwrap_or(false)
}
fn env_mib(name: &str, default_bytes: u64) -> u64 {
std::env::var(name)
.ok()
.and_then(|value| value.parse::<u64>().ok())
.map(mib)
.unwrap_or(default_bytes)
}
const fn kib(value: u64) -> u64 {
value * 1024
}
const fn mib(value: u64) -> u64 {
value * 1024 * 1024
}
const fn gib(value: u64) -> u64 {
value * 1024 * 1024 * 1024
}
fn format_bytes(bytes: u64) -> String {
let mib_value = bytes as f64 / mib(1) as f64;
format!("{mib_value:.1} MiB")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn proc_parser_reads_kb_and_raw_values() {
let parsed = parse_proc_key_values(
"MemTotal: 15458992 kB\nMemAvailable: 12843176 kB\nThreads:\t44\n",
);
assert_eq!(
read_required_kb(&parsed, "MemTotal").unwrap(),
15_458_992 * 1024
);
assert_eq!(read_optional_raw(&parsed, "Threads"), 44);
assert_eq!(read_optional_kb(&parsed, "VmSwap"), 0);
}
#[test]
fn watchdog_restarts_on_process_swap_limit() {
let config = MemoryWatchdogConfig {
min_available_bytes: mib(2_304),
min_process_bytes: mib(1_024),
max_process_bytes: gib(4),
max_swap_bytes: gib(1),
};
let snapshot = MemorySnapshot {
mem_total_bytes: gib(16),
mem_available_bytes: gib(12),
process_rss_bytes: mib(900),
process_swap_bytes: gib(2),
thread_count: 44,
};
assert!(
config
.critical_reason(&snapshot)
.unwrap()
.contains("process swap")
);
}
#[test]
fn watchdog_restarts_on_low_available_memory_with_large_lqosd() {
let config = MemoryWatchdogConfig {
min_available_bytes: mib(2_304),
min_process_bytes: mib(1_024),
max_process_bytes: gib(4),
max_swap_bytes: gib(1),
};
let snapshot = MemorySnapshot {
mem_total_bytes: gib(16),
mem_available_bytes: mib(1_900),
process_rss_bytes: mib(1_500),
process_swap_bytes: 0,
thread_count: 44,
};
assert!(
config
.critical_reason(&snapshot)
.unwrap()
.contains("available memory")
);
}
#[test]
fn watchdog_ignores_low_memory_when_lqosd_is_small() {
let config = MemoryWatchdogConfig {
min_available_bytes: mib(2_304),
min_process_bytes: mib(1_024),
max_process_bytes: gib(4),
max_swap_bytes: gib(1),
};
let snapshot = MemorySnapshot {
mem_total_bytes: gib(16),
mem_available_bytes: mib(1_900),
process_rss_bytes: mib(256),
process_swap_bytes: 0,
thread_count: 44,
};
assert!(config.critical_reason(&snapshot).is_none());
}
}
@@ -1580,6 +1580,38 @@ let topAsnSortColumn = 'rate';
let topAsnSortDirection = 'desc';
let trafficCurrentPage = 1;
let trafficPageSize = DEFAULT_TRAFFIC_PAGE_SIZE;
const TOP_ASN_TABLE_COLUMNS = [
{ group: "attribution" },
{ group: "attribution" },
{ group: "rates", numeric: true },
{ group: "rates", numeric: true },
{ group: "quality", numeric: true },
{ group: "quality", numeric: true },
{ group: "quality", numeric: true },
{ group: "quality", numeric: true },
{ group: "quality", numeric: true },
{ group: "quality", numeric: true },
{ group: "flows", numeric: true },
];
const TRAFFIC_TABLE_COLUMNS = [
{ group: "identity" },
{ group: "rates", numeric: true },
{ group: "rates", numeric: true },
{ group: "volume", numeric: true },
{ group: "volume", numeric: true },
{ group: "volume", numeric: true },
{ group: "volume", numeric: true },
{ group: "quality", numeric: true },
{ group: "quality", numeric: true },
{ group: "quality", numeric: true },
{ group: "quality", numeric: true },
{ group: "quality", numeric: true },
{ group: "quality", numeric: true },
{ group: "attribution" },
{ group: "attribution" },
{ group: "attribution" },
{ group: "actions", action: true },
];
function formatQooScore(score0to100, fallback = "-") {
if (score0to100 === null || score0to100 === undefined) {
@@ -1630,6 +1662,106 @@ function truncatedTrafficCell(text, cellClass) {
return td;
}
function applyCircuitTableCellChrome(cell, columns, startIndex, endIndex) {
const descriptor = columns[startIndex];
if (!descriptor) {
return;
}
const group = String(descriptor.group || "default");
cell.classList.add("lqos-circuit-table-cell", `lqos-circuit-colgroup-${group}`);
if (descriptor.numeric) {
cell.classList.add("lqos-circuit-cell-numeric");
}
if (descriptor.action) {
cell.classList.add("lqos-circuit-cell-action");
}
if (startIndex > 0 && columns[startIndex - 1]?.group !== group) {
cell.classList.add("lqos-circuit-col-start");
}
if (endIndex < columns.length - 1 && columns[endIndex + 1]?.group !== group) {
cell.classList.add("lqos-circuit-col-end");
}
cell.dataset.columnStart = String(startIndex);
cell.dataset.columnEnd = String(endIndex);
}
function annotateCircuitDataTable(table, columns) {
const headRow = table.tHead?.rows?.[0];
if (headRow) {
let columnIndex = 0;
Array.from(headRow.cells).forEach((cell) => {
const span = Math.max(1, Number(cell.colSpan) || 1);
const startIndex = columnIndex;
const endIndex = Math.min(columns.length - 1, startIndex + span - 1);
applyCircuitTableCellChrome(cell, columns, startIndex, endIndex);
columnIndex += span;
});
}
Array.from(table.tBodies).forEach((tbody) => {
Array.from(tbody.rows).forEach((row) => {
if (row.cells.length !== columns.length) {
return;
}
Array.from(row.cells).forEach((cell, index) => {
applyCircuitTableCellChrome(cell, columns, index, index);
});
});
});
wireCircuitDataTableHover(table);
}
function wireCircuitDataTableHover(table) {
let activeStart = null;
let activeEnd = null;
const paintRange = (start, end) => {
Array.from(table.querySelectorAll("[data-column-start][data-column-end]")).forEach((cell) => {
const cellStart = Number.parseInt(cell.dataset.columnStart || "", 10);
const cellEnd = Number.parseInt(cell.dataset.columnEnd || "", 10);
const shouldHighlight = Number.isFinite(cellStart)
&& Number.isFinite(cellEnd)
&& cellStart <= end
&& cellEnd >= start;
cell.classList.toggle("is-column-hover", shouldHighlight);
});
};
const clearRange = () => {
activeStart = null;
activeEnd = null;
Array.from(table.querySelectorAll(".is-column-hover")).forEach((cell) => {
cell.classList.remove("is-column-hover");
});
};
table.addEventListener("pointerover", (event) => {
const cell = event.target.closest("th, td");
if (!cell || !table.contains(cell)) {
return;
}
const start = Number.parseInt(cell.dataset.columnStart || "", 10);
const end = Number.parseInt(cell.dataset.columnEnd || "", 10);
if (!Number.isFinite(start) || !Number.isFinite(end)) {
return;
}
if (activeStart === start && activeEnd === end) {
return;
}
activeStart = start;
activeEnd = end;
paintRange(start, end);
});
table.addEventListener("pointerleave", () => {
clearRange();
});
}
function visibleTrafficRows() {
return Array.isArray(latestTrafficPage?.rows) ? latestTrafficPage.rows : [];
}
@@ -1735,7 +1867,7 @@ function renderTopAsnTab() {
tableWrap.classList.add("lqos-table-wrap");
const table = document.createElement("table");
table.classList.add("lqos-table", "lqos-table-tight", "lqos-circuit-traffic-table");
table.classList.add("lqos-table", "lqos-table-tight", "lqos-circuit-traffic-table", "lqos-circuit-data-table");
const thead = document.createElement("thead", "small");
thead.style.fontSize = "0.8em";
@@ -1797,6 +1929,7 @@ function renderTopAsnTab() {
}
table.appendChild(tbody);
annotateCircuitDataTable(table, TOP_ASN_TABLE_COLUMNS);
tableWrap.appendChild(table);
clearDiv(target);
target.appendChild(tableWrap);
@@ -1818,7 +1951,7 @@ function renderTrafficTab() {
tableWrap.classList.add("lqos-table-wrap");
let table = document.createElement("table");
table.classList.add("lqos-table", "lqos-table-tight", "lqos-circuit-traffic-table");
table.classList.add("lqos-table", "lqos-table-tight", "lqos-circuit-traffic-table", "lqos-circuit-data-table");
let thead = document.createElement("thead", "small");
thead.style.fontSize = "0.8em";
@@ -1903,6 +2036,7 @@ function renderTrafficTab() {
}
table.appendChild(tbody);
annotateCircuitDataTable(table, TRAFFIC_TABLE_COLUMNS);
tableWrap.appendChild(table);
clearDiv(target);
target.appendChild(tableWrap);
@@ -2483,6 +2617,7 @@ function cleanupCircuitPage() {
channelLink = null;
}
if (cakeChannel) {
wsClient.send({ Private: { StopCakeWatcher: null } });
cakeChannel.close();
cakeChannel = null;
}
@@ -2,27 +2,39 @@ import { get_ws_client } from "../pubsub/ws";
const wsClient = get_ws_client();
const secretBindings = [];
const CONFIG_SAVE_TIMEOUT_MS = 10000;
export function sendWsRequest(responseEvent, request, onComplete, onError) {
export function sendWsRequest(responseEvent, request, onComplete, onError, options = {}) {
let done = false;
const responseHandler = (msg) => {
let timeoutId = null;
const finish = (callback, msg) => {
if (done) return;
done = true;
if (timeoutId !== null) {
clearTimeout(timeoutId);
timeoutId = null;
}
wsClient.off(responseEvent, responseHandler);
wsClient.off("Error", errorHandler);
onComplete(msg);
if (callback) {
callback(msg);
}
};
const responseHandler = (msg) => {
finish(onComplete, msg);
};
const errorHandler = (msg) => {
if (done) return;
done = true;
wsClient.off(responseEvent, responseHandler);
wsClient.off("Error", errorHandler);
if (onError) {
onError(msg);
}
finish(onError, msg);
};
wsClient.on(responseEvent, responseHandler);
wsClient.on("Error", errorHandler);
if (Number.isFinite(options.timeoutMs) && options.timeoutMs > 0) {
timeoutId = setTimeout(() => {
finish(onError, {
message: options.timeoutMessage || `Timed out waiting for ${responseEvent}`,
});
}, options.timeoutMs);
}
wsClient.send(request);
}
@@ -34,12 +46,18 @@ function ensureOptionalConfigSections(config) {
if (!config.splynx_integration || typeof config.splynx_integration !== "object") {
config.splynx_integration = {};
}
if (!config.integration_common || typeof config.integration_common !== "object") {
config.integration_common = {};
}
if (!config.topology || typeof config.topology !== "object") {
config.topology = {};
}
if (!config.ssl || typeof config.ssl !== "object") {
config.ssl = {};
}
if (!config.uisp_integration || typeof config.uisp_integration !== "object") {
config.uisp_integration = {};
}
if (!config.sonar_integration || typeof config.sonar_integration !== "object") {
config.sonar_integration = {};
@@ -154,6 +172,17 @@ export function loadConfig(onComplete, onError) {
export function saveConfig(onComplete, onError) {
const clearSecrets = window.configSecretClears || {};
const reportError = (msg) => {
const errorMsg = (msg && msg.message) ? msg.message : "Request failed";
const detail = (msg && typeof msg === "object")
? { ...msg, message: errorMsg }
: { message: errorMsg };
if (onError) {
onError(detail);
} else {
alert("Error saving configuration: " + errorMsg);
}
};
sendWsRequest(
"UpdateConfigResult",
{
@@ -163,29 +192,29 @@ export function saveConfig(onComplete, onError) {
},
},
(msg) => {
if (msg && msg.ok) {
secretBindings.forEach((binding) => {
const sectionState = secretFieldMap(window.configSecretState || {}, binding.section);
const inputValue = binding.input.value.trim();
sectionState[binding.field] = clearSecrets?.[binding.section]?.[binding.field]
? false
: (inputValue.length > 0 || sectionState[binding.field]);
if (window.config?.[binding.section] && typeof window.config[binding.section] === "object") {
window.config[binding.section][binding.field] = "";
}
binding.input.value = "";
setSecretClear(binding.section, binding.field, false);
binding.updateStatus();
});
if (!msg || !msg.ok) {
reportError(msg);
return;
}
secretBindings.forEach((binding) => {
const sectionState = secretFieldMap(window.configSecretState || {}, binding.section);
const inputValue = binding.input.value.trim();
sectionState[binding.field] = clearSecrets?.[binding.section]?.[binding.field]
? false
: (inputValue.length > 0 || sectionState[binding.field]);
if (window.config?.[binding.section] && typeof window.config[binding.section] === "object") {
window.config[binding.section][binding.field] = "";
}
binding.input.value = "";
setSecretClear(binding.section, binding.field, false);
binding.updateStatus();
});
if (onComplete) onComplete(msg);
},
(msg) => {
if (onError) {
onError(msg);
} else {
alert("That didn't work");
}
reportError,
{
timeoutMs: CONFIG_SAVE_TIMEOUT_MS,
timeoutMessage: "Timed out waiting for the config save result. The websocket may have disconnected.",
},
);
}
@@ -0,0 +1,46 @@
import assert from "node:assert/strict";
import test from "node:test";
import {hydrateDraftMtu, parseOptionalMtu} from "./network_mode_mtu.mjs";
test("accepts blank and valid MTU values", () => {
assert.deepEqual(parseOptionalMtu(""), { ok: true, value: null, error: "" });
assert.deepEqual(parseOptionalMtu(" "), { ok: true, value: null, error: "" });
assert.deepEqual(parseOptionalMtu("1500"), { ok: true, value: 1500, error: "" });
assert.deepEqual(parseOptionalMtu("9000"), { ok: true, value: 9000, error: "" });
});
test("rejects invalid MTU values", () => {
assert.equal(parseOptionalMtu("abc").ok, false);
assert.equal(parseOptionalMtu("1500.5").ok, false);
assert.equal(parseOptionalMtu("575").ok, false);
assert.equal(parseOptionalMtu("9217").ok, false);
});
test("returns normalized candidate values", () => {
assert.equal(parseOptionalMtu("9000").value, 9000);
assert.equal(parseOptionalMtu("").value, null);
});
test("hydrates pre-MTU drafts from live config", () => {
const liveConfig = {
bridge: { mtu: 9000 },
single_interface: { mtu: 1500 },
};
assert.equal(hydrateDraftMtu({ bridge: { to_internet: "eth0" } }, liveConfig).bridge.mtu, 9000);
assert.equal(
hydrateDraftMtu({ single_interface: { interface: "eth0" } }, liveConfig).single_interface.mtu,
1500
);
});
test("keeps explicit draft MTU values", () => {
const liveConfig = {
bridge: { mtu: 9000 },
single_interface: { mtu: 1500 },
};
assert.equal(hydrateDraftMtu({ bridge: { mtu: null } }, liveConfig).bridge.mtu, null);
assert.equal(hydrateDraftMtu({ single_interface: { mtu: 1400 } }, liveConfig).single_interface.mtu, 1400);
});
@@ -0,0 +1,29 @@
export const MTU_MIN = 576;
export const MTU_MAX = 9216;
export function parseOptionalMtu(rawValue) {
const trimmed = String(rawValue ?? "").trim();
if (!trimmed) {
return { ok: true, value: null, error: "" };
}
if (!/^\d+$/.test(trimmed)) {
return { ok: false, value: null, error: `MTU must be a whole number from ${MTU_MIN} through ${MTU_MAX}.` };
}
const value = Number.parseInt(trimmed, 10);
if (value < MTU_MIN || value > MTU_MAX) {
return { ok: false, value: null, error: `MTU must be from ${MTU_MIN} through ${MTU_MAX}.` };
}
return { ok: true, value, error: "" };
}
export function hydrateDraftMtu(draft, liveConfig) {
if (!draft) return null;
const hydrated = JSON.parse(JSON.stringify(draft));
if (hydrated.bridge && hydrated.bridge.mtu === undefined) {
hydrated.bridge.mtu = liveConfig?.bridge?.mtu ?? null;
}
if (hydrated.single_interface && hydrated.single_interface.mtu === undefined) {
hydrated.single_interface.mtu = liveConfig?.single_interface?.mtu ?? null;
}
return hydrated;
}
@@ -54,6 +54,7 @@ function validateConfig() {
function updateConfig() {
// Update only the integration_common section
window.config.integration_common = {
...(window.config.integration_common || {}),
circuit_name_as_address: document.getElementById("circuitNameAsAddress").checked,
use_mikrotik_ipv6: document.getElementById("useMikrotikIpv6").checked,
queue_refresh_interval_mins: parseInt(document.getElementById("queueRefreshInterval").value),
@@ -1,9 +1,14 @@
import {loadConfig, renderConfigMenu} from "./config/config_helper";
import {hydrateDraftMtu, MTU_MAX, MTU_MIN, parseOptionalMtu} from "./config/network_mode_mtu.mjs";
const DRAFT_KEY = "lqos-network-mode-draft";
const PENDING_OPERATION_KEY = "lqos-network-mode-pending";
const NETPLAN_TRY_TIMEOUT_MS = 30_000;
const RECONNECT_POLL_INTERVAL_MS = 2_000;
const BRIDGE_MTU_HELP = "Applies to the bridge members and br0.";
const SINGLE_INTERFACE_MTU_HELP = "Applies to the trunk interface.";
const MTU_LOCKED_HELP = "MTU cannot be changed while a network-mode operation is pending.";
const XDP_MTU_HELP = "XDP mode keeps this value for later, but LibreQoS will not apply it.";
let currentHelperStatus = null;
let currentInspection = null;
let currentPendingOperation = null;
@@ -71,7 +76,94 @@ function postJson(url, body) {
});
}
function setMtuFieldError(inputId, errorId, message) {
const input = document.getElementById(inputId);
const error = document.getElementById(errorId);
if (!input || !error) return;
input.setAttribute("aria-invalid", "true");
input.classList.add("is-invalid");
error.textContent = message;
}
function clearMtuFieldError(inputId, errorId) {
const input = document.getElementById(inputId);
const error = document.getElementById(errorId);
if (!input || !error) return;
input.removeAttribute("aria-invalid");
input.classList.remove("is-invalid");
error.textContent = "";
}
function clearMtuErrors() {
clearMtuFieldError("bridgeMtu", "bridgeMtuError");
clearMtuFieldError("singleInterfaceMtu", "singleInterfaceMtuError");
}
function applyMtuBounds() {
["bridgeMtu", "singleInterfaceMtu"].forEach((id) => {
const input = document.getElementById(id);
if (!input) return;
input.min = String(MTU_MIN);
input.max = String(MTU_MAX);
});
}
function validateMtuField(inputId, errorId, parser) {
const input = document.getElementById(inputId);
if (!input) return true;
const parsed = parser(input.value);
if (parsed.ok) {
clearMtuFieldError(inputId, errorId);
return true;
}
setMtuFieldError(inputId, errorId, parsed.error);
input.focus();
return false;
}
function currentBridgeMtuValue() {
const parsed = parseOptionalMtu(document.getElementById("bridgeMtu").value);
if (parsed.ok) return parsed.value;
return window.config?.bridge?.mtu ?? null;
}
function updateBridgeMtuState() {
const useXdpBridge = document.getElementById("useXdpBridge")?.checked ?? false;
const bridgeMtu = document.getElementById("bridgeMtu");
const bridgeMtuHelp = document.getElementById("bridgeMtuHelp");
if (!bridgeMtu || !bridgeMtuHelp) return;
const editingLocked = Boolean(currentInspection?.editing_locked);
bridgeMtu.disabled = useXdpBridge || editingLocked;
if (editingLocked) {
bridgeMtuHelp.textContent = MTU_LOCKED_HELP;
return;
}
if (useXdpBridge) {
clearMtuFieldError("bridgeMtu", "bridgeMtuError");
bridgeMtuHelp.textContent = XDP_MTU_HELP;
} else {
bridgeMtuHelp.textContent = BRIDGE_MTU_HELP;
}
}
function updateSingleInterfaceMtuState() {
const singleInterfaceMtu = document.getElementById("singleInterfaceMtu");
const singleInterfaceMtuHelp = document.getElementById("singleInterfaceMtuHelp");
if (!singleInterfaceMtu || !singleInterfaceMtuHelp) return;
if (currentInspection?.editing_locked) {
singleInterfaceMtuHelp.textContent = MTU_LOCKED_HELP;
} else {
singleInterfaceMtuHelp.textContent = SINGLE_INTERFACE_MTU_HELP;
}
}
function updateMtuState() {
updateBridgeMtuState();
updateSingleInterfaceMtuState();
}
function validateConfig() {
clearMtuErrors();
if (document.getElementById("bridgeMode").checked) {
const toInternet = document.getElementById("toInternet").value.trim();
const toNetwork = document.getElementById("toNetwork").value.trim();
@@ -83,6 +175,11 @@ function validateConfig() {
alert("Bridge mode requires two different interfaces");
return false;
}
if (document.getElementById("useXdpBridge").checked) {
clearMtuFieldError("bridgeMtu", "bridgeMtuError");
} else if (!validateMtuField("bridgeMtu", "bridgeMtuError", parseOptionalMtu)) {
return false;
}
} else if (document.getElementById("singleInterfaceMode").checked) {
const interfaceName = document.getElementById("interface").value.trim();
const internetVlan = parseInt(document.getElementById("internetVlan").value, 10);
@@ -99,6 +196,9 @@ function validateConfig() {
alert("Network VLAN must be between 1 and 4094");
return false;
}
if (!validateMtuField("singleInterfaceMtu", "singleInterfaceMtuError", parseOptionalMtu)) {
return false;
}
} else {
alert("Please select either bridge or single interface mode");
return false;
@@ -116,12 +216,14 @@ function buildCandidateConfig() {
use_xdp_bridge: document.getElementById("useXdpBridge").checked,
to_internet: document.getElementById("toInternet").value.trim(),
to_network: document.getElementById("toNetwork").value.trim(),
mtu: currentBridgeMtuValue(),
};
} else {
next.single_interface = {
interface: document.getElementById("interface").value.trim(),
internet_vlan: parseInt(document.getElementById("internetVlan").value, 10),
network_vlan: parseInt(document.getElementById("networkVlan").value, 10),
mtu: parseOptionalMtu(document.getElementById("singleInterfaceMtu").value).value,
};
}
@@ -247,15 +349,18 @@ function populateFormFromConfig(config) {
document.getElementById("useXdpBridge").checked = config.bridge.use_xdp_bridge ?? true;
document.getElementById("toInternet").value = config.bridge.to_internet ?? "";
document.getElementById("toNetwork").value = config.bridge.to_network ?? "";
document.getElementById("bridgeMtu").value = config.bridge.mtu ?? "";
} else if (config?.single_interface) {
document.getElementById("singleInterfaceMode").checked = true;
document.getElementById("interface").value = config.single_interface.interface ?? "";
document.getElementById("internetVlan").value = config.single_interface.internet_vlan ?? 2;
document.getElementById("networkVlan").value = config.single_interface.network_vlan ?? 3;
document.getElementById("singleInterfaceMtu").value = config.single_interface.mtu ?? "";
}
const event = new Event("change");
document.querySelector('input[name="networkMode"]:checked')?.dispatchEvent(event);
updateMtuState();
}
function loadDraft() {
@@ -590,9 +695,11 @@ function renderInspection(inspection) {
"useXdpBridge",
"toInternet",
"toNetwork",
"bridgeMtu",
"interface",
"internetVlan",
"networkVlan",
"singleInterfaceMtu",
"saveButton",
].forEach((id) => {
const element = document.getElementById(id);
@@ -603,6 +710,7 @@ function renderInspection(inspection) {
applyButton.disabled = !inspection?.can_apply;
adoptButton.disabled = !inspection?.can_adopt;
takeoverButton.disabled = !inspection?.can_take_over;
updateMtuState();
}
function renderHelperStatus(status) {
@@ -820,6 +928,7 @@ function retryShaping() {
}
function wireActions() {
applyMtuBounds();
wireReviewTabs();
[
"bridgeMode",
@@ -832,6 +941,21 @@ function wireActions() {
renderInterfaceSelectors();
});
});
document.getElementById("useXdpBridge").addEventListener("change", updateMtuState);
[
"bridgeMtu",
"singleInterfaceMtu",
].forEach((id) => {
document.getElementById(id).addEventListener("input", () => {
if (id === "bridgeMtu") {
const parsed = parseOptionalMtu(document.getElementById("bridgeMtu").value);
if (parsed.ok) clearMtuFieldError("bridgeMtu", "bridgeMtuError");
} else {
const parsed = parseOptionalMtu(document.getElementById("singleInterfaceMtu").value);
if (parsed.ok) clearMtuFieldError("singleInterfaceMtu", "singleInterfaceMtuError");
}
});
});
document.getElementById("saveButton").addEventListener("click", () => {
if (!validateConfig()) return;
saveDraft();
@@ -867,7 +991,7 @@ loadConfig((msg) => {
renderShapingStatus(payload.shaping_status || {});
renderInspection(payload.network_mode_inspection || {});
const draft = loadDraft();
const draft = hydrateDraftMtu(loadDraft(), window.config);
populateFormFromConfig(draft || window.config);
renderRecoveryPanel();
wireActions();
@@ -1,11 +1,11 @@
import {
loadConfig,
loadNetworkJson,
loadNodeDirectory,
renderConfigMenu,
saveConfig,
} from "./config/config_helper";
let networkData = null;
let siteRows = [];
let selectedTargets = [];
let excludedSites = [];
@@ -75,26 +75,31 @@ function updateStrategyUi() {
}
}
// Load network.json for site dropdown
function nodeType(entry) {
return (entry?.node_type || "").toString().trim().toLowerCase();
}
function displayName(entry) {
return (entry?.node_name || "").toString().trim();
}
// Load the active runtime node directory for site dropdowns.
function loadNetworkData() {
return new Promise((resolve, reject) => {
loadNetworkJson(
loadNodeDirectory(
(data) => {
// Check if we got the "Not done yet" response
if (typeof data === 'string' && data === 'Not done yet') {
console.error('Network.json file not found on server');
alert('Network configuration not found. Please ensure network.json exists.');
resolve();
return;
}
networkData = data;
console.log('Network data loaded:', networkData);
const rows = Array.isArray(data) ? data : [];
const siteCandidates = rows.filter((entry) => nodeType(entry) === "site");
const sourceRows = siteCandidates.length > 0 ? siteCandidates : rows;
siteRows = sourceRows
.filter((entry) => displayName(entry).length > 0);
siteRows.sort((a, b) => displayName(a).localeCompare(displayName(b)));
populateSiteSelectors();
resolve();
},
(err) => {
console.error('Error loading network data:', err);
alert('Failed to load network sites. Please refresh the page.');
console.error('Error loading node directory:', err);
alert('Topology data is not ready yet. Run the integration or scheduler refresh and try again.');
reject(err);
},
);
@@ -105,32 +110,13 @@ function setSelectorOptions(selectorId) {
const selector = document.getElementById(selectorId);
selector.innerHTML = '<option value="">Select a site...</option>';
function iterate(data, level = 0) {
if (typeof data !== 'object' || data === null) {
return;
}
for (const [key, value] of Object.entries(data)) {
const option = document.createElement('option');
option.value = key;
let prefix = '';
for (let i = 0; i < level; i++) {
prefix += '- ';
}
option.textContent = prefix + key;
selector.appendChild(option);
if (value && typeof value === 'object' && value.children != null) {
iterate(value.children, level + 1);
}
}
}
if (networkData) {
iterate(networkData);
}
siteRows.forEach((entry) => {
const name = displayName(entry);
const option = document.createElement('option');
option.value = name;
option.textContent = name;
selector.appendChild(option);
});
}
function populateSiteSelectors() {
@@ -1,13 +1,13 @@
import {
loadAllCircuitDirectoryRows,
loadConfig,
loadNetworkJson,
loadNodeDirectory,
renderConfigMenu,
saveConfig,
} from "./config/config_helper";
import {defaultTreeguardConfig, ensureTreeguardConfig} from "./config/treeguard_defaults";
let networkData = null;
let nodeRows = [];
let circuitRows = null;
let selectedNodes = [];
let selectedCircuits = [];
@@ -98,21 +98,22 @@ function updateCircuitsEnrollmentUi() {
function loadNetworkData() {
return new Promise((resolve, reject) => {
loadNetworkJson(
loadNodeDirectory(
(data) => {
if (typeof data === "string" && data === "Not done yet") {
console.error("Network.json file not found on server");
alert("Network configuration not found. Please ensure network.json exists.");
resolve();
return;
}
networkData = data;
nodeRows = Array.isArray(data)
? data.filter((entry) => (entry?.node_name || "").toString().trim().length > 0)
: [];
nodeRows.sort((a, b) => {
const left = (a?.node_name || "").toString();
const right = (b?.node_name || "").toString();
return left.localeCompare(right);
});
populateNodeSelector();
resolve();
},
(err) => {
console.error("Error loading network data:", err);
alert("Failed to load network nodes. You can still add nodes manually.");
console.error("Error loading node directory:", err);
alert("Topology data is not ready yet. Run the integration or scheduler refresh and try again. You can still add nodes manually.");
reject(err);
},
);
@@ -145,31 +146,17 @@ function populateNodeSelector() {
const selector = document.getElementById("nodeSelector");
selector.innerHTML = '<option value="">Select a node...</option>';
function iterate(data, level = 0) {
if (typeof data !== "object" || data === null) {
nodeRows.forEach((entry) => {
const name = (entry?.node_name || "").toString().trim();
if (!name) {
return;
}
for (const [key, value] of Object.entries(data)) {
const option = document.createElement("option");
option.value = key;
let prefix = "";
for (let i = 0; i < level; i++) {
prefix += "- ";
}
option.textContent = prefix + key;
selector.appendChild(option);
if (value && typeof value === "object" && value.children != null) {
iterate(value.children, level + 1);
}
}
}
if (networkData) {
iterate(networkData);
}
const type = (entry?.node_type || "").toString().trim();
const option = document.createElement("option");
option.value = name;
option.textContent = type ? `${name} (${type})` : name;
selector.appendChild(option);
});
}
function populateCircuitSelector() {
@@ -6,6 +6,30 @@ import {
secretWillExistAfterSave,
} from "./config/config_helper";
function currentTopologyMode() {
return (
window.config?.topology?.compile_mode
?? document.getElementById("uispStrategy").value.trim()
);
}
function rootSiteRequiredForMode(mode) {
return mode === "full" || mode === "full2";
}
function updateRootSiteRequirement() {
const required = rootSiteRequiredForMode(currentTopologyMode());
const siteInput = document.getElementById("uispSite");
const siteLabel = document.getElementById("uispSiteLabel");
const siteHelp = document.getElementById("uispSiteHelp");
siteInput.required = required;
siteLabel.textContent = required ? "Root UISP Site" : "Root UISP Site (Optional)";
siteHelp.textContent = required
? "Required for full topology mode. Enter the exact UISP site name at the root of the network."
: "Optional in this topology mode. Leave blank to let LibreQoS choose the import root.";
}
function validateConfig() {
// Validate required fields when enabled
if (document.getElementById("enableUisp").checked) {
@@ -27,8 +51,8 @@ function validateConfig() {
}
const site = document.getElementById("uispSite").value.trim();
if (!site) {
alert("UISP Site is required when UISP integration is enabled");
if (rootSiteRequiredForMode(currentTopologyMode()) && !site) {
alert("Root UISP Site is required when full topology mode is selected");
return false;
}
@@ -88,7 +112,6 @@ function updateConfig() {
// Parse comma-separated strings into arrays
const existingUisp = { ...(window.config.uisp_integration || {}) };
delete existingUisp.enable_squashing;
delete existingUisp.use_ptmp_as_parent;
const excludeSites = document.getElementById("uispExcludeSites").value.trim();
const excludeSitesArray = excludeSites ? excludeSites.split(',').map(s => s.trim()) : [];
@@ -149,6 +172,7 @@ loadConfig(() => {
document.getElementById("uispSite").value = uisp.site ?? "";
document.getElementById("uispStrategy").value = window.config.topology?.compile_mode ?? uisp.strategy ?? "ap_site";
document.getElementById("uispSuspendedStrategy").value = uisp.suspended_strategy ?? "none";
updateRootSiteRequirement();
// Numeric fields
document.getElementById("uispAirmaxCapacity").value = uisp.airmax_capacity ?? 1.0;
@@ -171,6 +195,8 @@ loadConfig(() => {
clearButtonId: "clearUispToken",
});
document.getElementById("uispStrategy").addEventListener("change", updateRootSiteRequirement);
// Add save button click handler
document.getElementById('saveButton').addEventListener('click', () => {
if (validateConfig()) {
@@ -86,12 +86,12 @@ function classifyEvent(entry) {
if (event.endsWith("_started")) {
outcome = "Running";
outcomeClass = "bg-primary-subtle text-primary border border-primary-subtle";
} else if (event === "reload_required") {
} else if (event === "reload_required" || event === "passthrough_degraded") {
stage = "Verify";
scope = "Full Reload";
outcome = "Reload Required";
outcome = event === "passthrough_degraded" ? "Pass-Through" : "Reload Required";
outcomeClass = "bg-danger-subtle text-danger border border-danger-subtle";
} else if (event === "reload_required_cleared") {
} else if (event === "reload_required_cleared" || event === "passthrough_degraded_cleared") {
stage = "Verify";
scope = "Full Reload";
outcome = "Cleared";
@@ -182,6 +182,8 @@ function describeOperation(entry, meta) {
|| event === "full_reload_started"
|| event === "reload_required"
|| event === "reload_required_cleared"
|| event === "passthrough_degraded"
|| event === "passthrough_degraded_cleared"
|| meta.scope === "Full Reload"
) {
return {
@@ -16,12 +16,18 @@ function classifyOutcome(entry) {
const event = (entry?.event ?? "").toString().trim().toLowerCase();
const status = (entry?.status ?? "").toString().trim().toLowerCase();
if (event === "reload_required") {
if (event === "reload_required" || event === "passthrough_degraded") {
return {
label: "Reload Required",
label: event === "passthrough_degraded" ? "Pass-Through" : "Reload Required",
className: "bg-danger-subtle text-danger border border-danger-subtle",
};
}
if (event === "passthrough_degraded_cleared") {
return {
label: "Shaping Restored",
className: "bg-success-subtle text-success border border-success-subtle",
};
}
if (event === "apply_failed" || status === "error") {
return {
label: "Failed",
@@ -58,6 +58,10 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
this.lastFailureSummaryEl = document.createElement("span");
this.lastFailureSummaryEl.style.whiteSpace = "normal";
this.lastFailureSummaryEl.style.wordBreak = "break-word";
this.shapingStateEl = document.createElement("div");
this.passthroughReasonEl = document.createElement("span");
this.passthroughReasonEl.style.whiteSpace = "normal";
this.passthroughReasonEl.style.wordBreak = "break-word";
this.reloadRequiredEl = document.createElement("div");
this.reloadReasonEl = document.createElement("span");
this.reloadReasonEl.style.whiteSpace = "normal";
@@ -65,10 +69,12 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
tbody.appendChild(mkRow("Current Mode", this.modeEl));
tbody.appendChild(mkRow("Current Duration", this.durationEl));
tbody.appendChild(mkRow("Shaping State", this.shapingStateEl));
tbody.appendChild(mkRow("Runtime Drift", this.reloadRequiredEl));
tbody.appendChild(mkRow("Last Success", this.lastSuccessEl));
tbody.appendChild(mkRow("Last Failure", this.lastFailureEl));
tbody.appendChild(mkRow("Failure Summary", this.lastFailureSummaryEl));
tbody.appendChild(mkRow("Pass-Through Reason", this.passthroughReasonEl));
tbody.appendChild(mkRow("Reload Reason", this.reloadReasonEl));
table.appendChild(tbody);
@@ -91,6 +97,12 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
this.modeEl.innerHTML = "";
this.modeEl.appendChild(bakeryModeBadge(status.mode));
this.shapingStateEl.innerHTML = "";
this.shapingStateEl.appendChild(
status.passthroughDegraded
? mkBadge("Pass-Through / Unshaped", "bg-danger-subtle text-danger border border-danger-subtle")
: mkBadge("Normal Shaping", "bg-success-subtle text-success border border-success-subtle"),
);
this.reloadRequiredEl.innerHTML = "";
this.reloadRequiredEl.appendChild(
status.reloadRequired
@@ -104,6 +116,7 @@ export class BakeryStatusSummaryDashlet extends BaseDashlet {
this.lastSuccessEl.textContent = formatUnixSecondsToLocalDateTime(status.lastSuccessUnix);
this.lastFailureEl.textContent = formatUnixSecondsToLocalDateTime(status.lastFailureUnix);
this.lastFailureSummaryEl.textContent = status.lastFailureSummary || "—";
this.passthroughReasonEl.textContent = status.passthroughDegradedReason || "—";
this.reloadReasonEl.textContent = status.reloadRequiredReason || "—";
}
}
@@ -1,17 +1,10 @@
import {listenOnceMatchingWithTimeout} from "./pubsub/listeners";
import {get_ws_client, subscribeWS} from "./pubsub/ws";
const wsClient = get_ws_client();
const DEFAULT_EXECUTIVE_REFRESH_MS = 5000;
let executiveRequestCounter = 0;
const listenOnce = (eventName, handler) => {
const wrapped = (msg) => {
wsClient.off(eventName, wrapped);
handler(msg);
};
wsClient.on(eventName, wrapped);
};
function normalizeEntityKinds(entityKinds) {
return [...new Set((entityKinds || []).map((kind) => String(kind)))].sort();
}
@@ -178,7 +171,7 @@ export function getNodeIdMap() {
siteIdMap = map;
resolve(map);
};
listenOnce("NetworkTreeLite", (msg) => {
listenOnceMatchingWithTimeout(wsClient, "NetworkTreeLite", 5000, () => true, (msg) => {
const data = msg && msg.data ? msg.data : [];
const map = new Map();
(data || []).forEach((entry) => {
@@ -190,11 +183,10 @@ export function getNodeIdMap() {
}
});
resolveOnce(map);
}, () => {
resolveOnce(new Map());
});
wsClient.send({ NetworkTreeLite: {} });
setTimeout(() => {
resolveOnce(new Map());
}, 5000);
});
return siteIdMapPromise;
}
@@ -5,6 +5,9 @@ import {heading5Icon} from "../helpers/content_builders";
import {openDashboardEditor} from "./dashboard_editor";
const DIAGNOSTIC_CHANNELS = new Set(["Cpu", "Ram", "RttHistogram"]);
const DASHBOARD_DATA_STALE_MS = 15000;
const DASHBOARD_SUBSCRIPTION_REFRESH_COOLDOWN_MS = 30000;
const DASHBOARD_WATCHDOG_INTERVAL_MS = 5000;
export class Dashboard {
// Takes the target DIV in which to build the dashboard,
@@ -51,6 +54,9 @@ export class Dashboard {
this.channelDisposers = new Map();
this.subscribedChannels = new Set();
this.lastMessages = new Map();
this.lastDashboardMessageAt = Date.now();
this.lastSubscriptionRefreshAt = 0;
this.watchdogTimer = null;
// Auto Refresh Handling
this.paused = false;
@@ -256,10 +262,12 @@ export class Dashboard {
}
this.#queueImmediateDashlets(renderableDashlets);
this.#syncWebSocketChannels();
this.#startWatchdog();
this.#replayCachedMessages(renderableDashlets);
}
destroy() {
this.#stopWatchdog();
this.#disposeWebSocketChannels();
if (window.__lqosDashboardDashletHook) {
delete window.__lqosDashboardDashletHook;
@@ -680,12 +688,52 @@ export class Dashboard {
});
}
#startWatchdog() {
this.#stopWatchdog();
this.lastDashboardMessageAt = Date.now();
this.watchdogTimer = window.setInterval(() => {
this.#checkDashboardDataFlow();
}, DASHBOARD_WATCHDOG_INTERVAL_MS);
}
#stopWatchdog() {
if (this.watchdogTimer !== null) {
window.clearInterval(this.watchdogTimer);
this.watchdogTimer = null;
}
}
#checkDashboardDataFlow() {
if (this.paused || this.subscribedChannels.size === 0) {
return;
}
const now = Date.now();
const ageMs = now - this.lastDashboardMessageAt;
if (ageMs < DASHBOARD_DATA_STALE_MS) {
return;
}
if (now - this.lastSubscriptionRefreshAt < DASHBOARD_SUBSCRIPTION_REFRESH_COOLDOWN_MS) {
return;
}
const activeChannels = Array.from(this.subscribedChannels);
this.lastSubscriptionRefreshAt = now;
this.lastDashboardMessageAt = now;
this.#debug("stale-data-refresh", {
ageMs,
channels: activeChannels,
wsConnected: !!this.wsClient.ws,
handshakeDone: !!this.wsClient.handshake_done,
});
this.wsClient.refreshSubscriptions("dashboard-data-stale");
}
#handleWebSocketMessage(msg) {
if (!msg || !msg.event || this.paused) {
return;
}
this.lastMessages.set(msg.event, msg);
this.lastDashboardMessageAt = Date.now();
this.tickCounter++;
this.tickCounter %= this.cadence;
const renderableDashlets = this.#renderableDashlets();
@@ -0,0 +1,23 @@
export function listenOnceMatchingWithTimeout(wsClient, eventName, timeoutMs, matches, handler, onTimeout) {
let done = false;
const cleanup = wsClient.on(eventName, (msg) => {
if (done) return;
if (!matches(msg)) return;
done = true;
clearTimeout(timer);
cleanup();
handler(msg);
});
const timer = setTimeout(() => {
if (done) return;
done = true;
cleanup();
onTimeout();
}, timeoutMs);
return () => {
if (done) return;
done = true;
clearTimeout(timer);
cleanup();
};
}
@@ -3,6 +3,7 @@ import { Encoder, decode } from "../lq_js_common/helpers/cbor-x";
const ACK_TEXT = "I accept that this is an unstable, internal API and is unsupported";
const EXPECTED_UI_VERSION = (window.LQOS_UI_VERSION || "").trim() || null;
const USER_TOKEN_COOKIE = "User-Token";
const VERSION_RELOAD_KEY = "lqosWsVersionReload";
const encoder = new Encoder({ useRecords: false, variableMapSize: true });
const DIAGNOSTIC_CHANNELS = new Set(["Cpu", "Ram", "RttHistogram"]);
@@ -78,6 +79,28 @@ function get_user_token() {
return get_cookie_value(USER_TOKEN_COOKIE);
}
function versionReloadKey(serverVersion) {
const clientVersion = EXPECTED_UI_VERSION || "missing";
const wsVersion = (serverVersion || "missing").toString().trim() || "missing";
return `${clientVersion}->${wsVersion}`;
}
function hasReloadedForVersionMismatch(mismatchKey) {
try {
if (window.sessionStorage.getItem(VERSION_RELOAD_KEY) === mismatchKey) {
return true;
}
window.sessionStorage.setItem(VERSION_RELOAD_KEY, mismatchKey);
return false;
} catch (_) {
if (window.__lqosWsVersionReload === mismatchKey) {
return true;
}
window.__lqosWsVersionReload = mismatchKey;
return false;
}
}
export function ws_proto() {
if (window.location.protocol.startsWith("https")) {
return "wss://";
@@ -95,11 +118,15 @@ export class WsClient {
this.reconnectTimer = null;
this.reconnectDelayMs = 1000;
this.manualClose = false;
this.versionReloading = false;
}
connect() {
if (this.ws) {
return;
if (this.ws.readyState === WebSocket.CONNECTING || this.ws.readyState === WebSocket.OPEN) {
return;
}
this._dropSocket("stale-socket");
}
if (this.reconnectTimer) {
clearTimeout(this.reconnectTimer);
@@ -159,7 +186,9 @@ export class WsClient {
});
this.ws = null;
this.handshake_done = false;
this._scheduleReconnect();
if (!this.versionReloading) {
this._scheduleReconnect();
}
};
socket.onerror = () => {
@@ -171,7 +200,14 @@ export class WsClient {
});
this.ws = null;
this.handshake_done = false;
this._scheduleReconnect();
try {
socket.close();
} catch (_) {
// The browser may already have closed the socket.
}
if (!this.versionReloading) {
this._scheduleReconnect();
}
};
}
@@ -194,6 +230,54 @@ export class WsClient {
this.handlers.clear();
}
_dropSocket(reason = "unspecified") {
dashboardWsDebug("drop-socket", {
reason,
desiredChannels: Array.from(this.desiredChannels.keys()),
});
const oldSocket = this.ws;
this.ws = null;
this.handshake_done = false;
if (oldSocket) {
try {
oldSocket.close();
} catch (_) {
// Ignore close errors; reconnect/reload handling continues below.
}
}
}
_reloadForVersionMismatch(serverVersion) {
const mismatchKey = versionReloadKey(serverVersion);
dashboardWsDebug("version-mismatch", {
expected: EXPECTED_UI_VERSION || "missing",
actual: serverVersion || "missing",
mismatchKey,
});
if (!hasReloadedForVersionMismatch(mismatchKey)) {
this.versionReloading = true;
this._dropSocket("version-mismatch");
window.location.reload();
window.setTimeout(() => {
if (this.versionReloading) {
this.versionReloading = false;
this._scheduleReconnect();
}
}, 3000);
return true;
}
console.error(
"Websocket version mismatch persisted after reload:",
`client=${EXPECTED_UI_VERSION || "missing"}`,
`server=${serverVersion || "missing"}`,
);
this._dropSocket("version-mismatch-persistent");
this._scheduleReconnect();
return true;
}
refreshSubscriptions(reason = "unspecified") {
dashboardWsDebug("refresh-subscriptions", {
reason,
@@ -236,7 +320,7 @@ export class WsClient {
dashboardWsInteresting("subscribe-wire-interesting", [channel], {
desiredChannels: Array.from(this.desiredChannels.keys()),
});
this.ws.send(encoder.encode({ Subscribe: { channel } }));
this._sendControl({ Subscribe: { channel } });
}
}
if (channels.length > 0) {
@@ -263,7 +347,7 @@ export class WsClient {
dashboardWsInteresting("unsubscribe-wire-interesting", [channel], {
desiredChannels: Array.from(this.desiredChannels.keys()),
});
this.ws.send(encoder.encode({ Unsubscribe: { channel } }));
this._sendControl({ Unsubscribe: { channel } });
}
} else {
this.desiredChannels.set(channel, current - 1);
@@ -279,11 +363,16 @@ export class WsClient {
if (!this.ws) {
this.connect();
}
if (!this.handshake_done) {
if (!this.handshake_done || !this._socketIsOpen()) {
this.pending.push(normalized);
if (this.handshake_done && this.ws && !this._socketIsOpen()) {
this._retireSocket();
}
return;
}
this.ws.send(encoder.encode(normalized));
if (!this._sendControl(normalized)) {
this.pending.push(normalized);
}
}
on(event_name, handler) {
@@ -318,35 +407,70 @@ export class WsClient {
"Websocket version mismatch:",
hello ? hello.version : "missing",
);
this.close();
this._reloadForVersionMismatch(hello ? hello.version : null);
return;
}
try {
window.sessionStorage.removeItem(VERSION_RELOAD_KEY);
} catch (_) {
// Storage cleanup is best-effort.
}
this.handshake_done = true;
dashboardWsDebug("handshake-complete", {
desiredChannels: Array.from(this.desiredChannels.keys()),
});
this.ws.send(
encoder.encode({
HelloReply: {
ack: ACK_TEXT,
token: get_user_token(),
},
}),
);
for (let i = 0; i < this.pending.length; i++) {
this.ws.send(encoder.encode(this.pending[i]));
if (!this._sendControl({
HelloReply: {
ack: ACK_TEXT,
token: get_user_token(),
},
})) {
return;
}
const pending = this.pending;
this.pending = [];
for (let i = 0; i < pending.length; i++) {
if (!this._sendControl(pending[i])) {
this.pending = pending.slice(i).concat(this.pending);
return;
}
}
for (const channel of this.desiredChannels.keys()) {
dashboardWsInteresting("subscribe-wire-interesting", [channel], {
desiredChannels: Array.from(this.desiredChannels.keys()),
fromHandshake: true,
});
this.ws.send(encoder.encode({ Subscribe: { channel } }));
if (!this._sendControl({ Subscribe: { channel } })) {
return;
}
}
this.reconnectDelayMs = 1000;
}
_socketIsOpen() {
return this.ws && this.ws.readyState === WebSocket.OPEN;
}
_retireSocket() {
this._dropSocket("retire-socket");
this._scheduleReconnect();
}
_sendControl(request_obj) {
if (!this._socketIsOpen()) {
this._retireSocket();
return false;
}
try {
this.ws.send(encoder.encode(request_obj));
return true;
} catch (err) {
console.warn("Websocket send failed; reconnecting", err);
this._retireSocket();
return false;
}
}
_dispatch(msg) {
if (!msg || !msg.event) {
return;
@@ -356,7 +480,15 @@ export class WsClient {
return;
}
for (const handler of Array.from(list)) {
handler(msg);
try {
handler(msg);
} catch (err) {
console.error(`Websocket handler failed for ${msg.event}`, err);
dashboardWsDebug("handler-error", {
eventName: msg.event,
message: err && err.message ? err.message : String(err),
});
}
}
}
@@ -0,0 +1,103 @@
import assert from "node:assert/strict";
import test from "node:test";
globalThis.window = {
LQOS_UI_VERSION: "test",
location: { search: "", protocol: "http:", host: "localhost" },
localStorage: { getItem: () => null },
sessionStorage: {
getItem: () => null,
setItem: () => {},
removeItem: () => {},
},
};
globalThis.document = { cookie: "" };
const { listenOnceMatchingWithTimeout } = await import("./listeners.js");
function makeWsClient() {
const handlers = new Map();
return {
on(eventName, handler) {
if (!handlers.has(eventName)) {
handlers.set(eventName, new Set());
}
handlers.get(eventName).add(handler);
return () => this.off(eventName, handler);
},
off(eventName, handler) {
handlers.get(eventName)?.delete(handler);
},
emit(eventName, message) {
for (const handler of Array.from(handlers.get(eventName) || [])) {
handler(message);
}
},
handlerCount(eventName) {
return handlers.get(eventName)?.size || 0;
},
};
}
test("timeout listener ignores stale responses and resolves matching response", () => {
const wsClient = makeWsClient();
const seen = [];
listenOnceMatchingWithTimeout(
wsClient,
"UrgentClearResult",
1000,
(message) => message.request_id === 2,
(message) => seen.push(message.ok),
() => seen.push("timeout"),
);
wsClient.emit("UrgentClearResult", { request_id: 1, ok: false });
assert.deepEqual(seen, []);
assert.equal(wsClient.handlerCount("UrgentClearResult"), 1);
wsClient.emit("UrgentClearResult", { request_id: 2, ok: true });
assert.deepEqual(seen, [true]);
assert.equal(wsClient.handlerCount("UrgentClearResult"), 0);
});
test("timeout listener disposer removes pending handler", () => {
const wsClient = makeWsClient();
const seen = [];
const dispose = listenOnceMatchingWithTimeout(
wsClient,
"UrgentList",
1000,
() => true,
() => seen.push("response"),
() => seen.push("timeout"),
);
assert.equal(wsClient.handlerCount("UrgentList"), 1);
dispose();
wsClient.emit("UrgentList", { request_id: 1 });
assert.deepEqual(seen, []);
assert.equal(wsClient.handlerCount("UrgentList"), 0);
});
test("timeout listener fires timeout and removes pending handler", async () => {
const wsClient = makeWsClient();
const seen = [];
listenOnceMatchingWithTimeout(
wsClient,
"UrgentList",
5,
() => false,
() => seen.push("response"),
() => seen.push("timeout"),
);
assert.equal(wsClient.handlerCount("UrgentList"), 1);
await new Promise((resolve) => setTimeout(resolve, 20));
assert.deepEqual(seen, ["timeout"]);
assert.equal(wsClient.handlerCount("UrgentList"), 0);
});
@@ -21,6 +21,8 @@ let totalRows = 0;
let totalCircuits = 0;
let circuitMetricsWatchSignature = null;
const wsClient = get_ws_client();
const DYNAMIC_VIEW_MODE_STORAGE_KEY = "lqosShapedDevicesDynamicViewMode";
let dynamicViewMode = loadDynamicViewMode();
const QOO_TOOLTIP_HTML =
"<h5>Quality of Outcome (QoO)</h5>" +
@@ -75,6 +77,38 @@ function formatPlanValue(value) {
return formatted;
}
function normalizeDynamicViewMode(mode) {
return mode === "list" ? "list" : "cards";
}
function loadDynamicViewMode() {
try {
if (window.localStorage) {
return normalizeDynamicViewMode(window.localStorage.getItem(DYNAMIC_VIEW_MODE_STORAGE_KEY));
}
} catch (_error) {
// Ignore storage failures and fall back to the default cards view.
}
return "cards";
}
function persistDynamicViewMode(mode) {
try {
if (window.localStorage) {
window.localStorage.setItem(
DYNAMIC_VIEW_MODE_STORAGE_KEY,
normalizeDynamicViewMode(mode),
);
}
} catch (_error) {
// Ignore storage failures and keep the current in-memory preference.
}
}
function isDynamicListView() {
return shapedDevicesKind === "Dynamic" && dynamicViewMode === "list";
}
function countCircuits() {
return totalCircuits;
}
@@ -125,6 +159,16 @@ function setShapedDevicesKind(kind) {
requestShapedDevicesPage();
}
function setDynamicViewMode(mode) {
const normalized = normalizeDynamicViewMode(mode);
if (dynamicViewMode === normalized) {
return;
}
dynamicViewMode = normalized;
persistDynamicViewMode(dynamicViewMode);
renderDevices();
}
function filterDevices() {
page = 0;
requestShapedDevicesPage();
@@ -304,6 +348,96 @@ function metricTableRow(labelEl, downEl, upEl) {
return tr;
}
function buildDeviceIdentity(device, options = {}) {
const {
circuitClasses = [],
deviceClasses = [],
} = options;
const wrapper = document.createElement("div");
wrapper.style.minWidth = "0";
if (device.circuit_id) {
const circuitLink = document.createElement("a");
circuitLink.href = "circuit.html?id=" + encodeURI(device.circuit_id);
circuitClasses.forEach((className) => circuitLink.classList.add(className));
circuitLink.innerText = device.circuit_name || "(Unknown circuit)";
wrapper.appendChild(circuitLink);
if (device.device_name) {
const deviceLink = document.createElement("a");
deviceLink.href = "circuit.html?id=" + encodeURI(device.circuit_id);
deviceClasses.forEach((className) => deviceLink.classList.add(className));
deviceLink.innerText = device.device_name;
wrapper.appendChild(deviceLink);
}
} else {
const circuitName = document.createElement("div");
circuitClasses.forEach((className) => circuitName.classList.add(className));
circuitName.innerText = device.circuit_name || "(Unknown circuit)";
wrapper.appendChild(circuitName);
if (device.device_name) {
const deviceName = document.createElement("div");
deviceClasses.forEach((className) => deviceName.classList.add(className));
deviceName.innerText = device.device_name;
wrapper.appendChild(deviceName);
}
}
return wrapper;
}
function buildPlanBadge(device, extraClasses = []) {
const badge = document.createElement("span");
badge.classList.add("badge", "text-bg-secondary", "exec-badge", ...extraClasses);
badge.innerText =
formatPlanValue(device.download_max_mbps) + " / " + formatPlanValue(device.upload_max_mbps) + " Mbps";
return badge;
}
function buildMetricValueEl() {
const span = document.createElement("span");
span.innerHTML = "-";
return span;
}
function buildMetricPairCell(circuitId, downMetricName, upMetricName) {
const wrapper = document.createElement("div");
wrapper.classList.add("lqos-direction-metric");
const addLine = (label, metricName) => {
const line = document.createElement("div");
line.classList.add("lqos-direction-metric-line");
const labelEl = document.createElement("span");
labelEl.classList.add("lqos-direction-metric-label");
labelEl.innerText = label;
line.appendChild(labelEl);
const valueEl = buildMetricValueEl();
line.appendChild(valueEl);
wrapper.appendChild(line);
registerMetricEl(circuitId, metricName, valueEl);
};
addLine("DL", downMetricName);
addLine("UL", upMetricName);
return wrapper;
}
function appendListCell(row, content, classNames = []) {
const td = document.createElement("td");
classNames.forEach((className) => td.classList.add(className));
if (typeof content === "string") {
td.innerText = content;
} else if (content) {
td.appendChild(content);
}
row.appendChild(td);
return td;
}
function buildDeviceCard(device) {
const card = document.createElement("div");
card.classList.add("executive-card", "h-100");
@@ -312,37 +446,12 @@ function buildDeviceCard(device) {
const header = document.createElement("div");
header.classList.add("d-flex", "justify-content-between", "align-items-start", "gap-2");
const titleWrap = document.createElement("div");
titleWrap.style.minWidth = "0";
const titleWrap = buildDeviceIdentity(device, {
circuitClasses: ["redactable", "fw-semibold", "text-decoration-none"],
deviceClasses: ["redactable", "d-block", "small", "text-body-secondary", "text-decoration-none"],
});
if (device.circuit_id) {
const circuitLink = document.createElement("a");
circuitLink.href = "circuit.html?id=" + encodeURI(device.circuit_id);
circuitLink.classList.add("redactable", "fw-semibold", "text-decoration-none");
circuitLink.innerText = device.circuit_name || "(Unknown circuit)";
titleWrap.appendChild(circuitLink);
const deviceLink = document.createElement("a");
deviceLink.href = "circuit.html?id=" + encodeURI(device.circuit_id);
deviceLink.classList.add("redactable", "d-block", "small", "text-body-secondary", "text-decoration-none");
deviceLink.innerText = device.device_name || "";
titleWrap.appendChild(deviceLink);
} else {
const circuitName = document.createElement("div");
circuitName.classList.add("fw-semibold", "redactable");
circuitName.innerText = device.circuit_name || "(Unknown circuit)";
titleWrap.appendChild(circuitName);
const deviceName = document.createElement("div");
deviceName.classList.add("small", "text-body-secondary", "redactable");
deviceName.innerText = device.device_name || "";
titleWrap.appendChild(deviceName);
}
const planBadge = document.createElement("span");
planBadge.classList.add("badge", "text-bg-secondary", "exec-badge", "ms-auto");
planBadge.innerText =
formatPlanValue(device.download_max_mbps) + " / " + formatPlanValue(device.upload_max_mbps) + " Mbps";
const planBadge = buildPlanBadge(device, ["ms-auto"]);
header.appendChild(titleWrap);
header.appendChild(planBadge);
@@ -446,19 +555,57 @@ function buildDeviceCard(device) {
return card;
}
function buildDeviceListRow(device) {
const row = document.createElement("tr");
row.classList.add("align-middle");
const identity = buildDeviceIdentity(device, {
circuitClasses: ["redactable", "fw-semibold", "text-decoration-none"],
deviceClasses: ["redactable", "d-block", "small", "text-body-secondary", "text-decoration-none"],
});
appendListCell(row, identity);
const parent = document.createElement("div");
parent.classList.add("small", "redactable");
parent.innerText = device.parent_node || "-";
appendListCell(row, parent);
appendListCell(row, buildPlanBadge(device));
appendListCell(row, buildMetricPairCell(device.circuit_id, "tpDown", "tpUp"), ["text-nowrap"]);
appendListCell(row, buildMetricPairCell(device.circuit_id, "rttDown", "rttUp"), ["text-nowrap"]);
appendListCell(row, buildMetricPairCell(device.circuit_id, "qooDown", "qooUp"), ["text-nowrap"]);
appendListCell(row, buildMetricPairCell(device.circuit_id, "reXmitDown", "reXmitUp"), ["text-nowrap"]);
const lastSeenValue = document.createElement("span");
lastSeenValue.innerText = "-";
appendListCell(row, lastSeenValue, ["text-nowrap"]);
registerMetricEl(device.circuit_id, "lastSeen", lastSeenValue);
appendListCell(row, buildIpListEl(device), ["text-nowrap"]);
return row;
}
function ensureLayout() {
const target = document.getElementById("deviceTable");
if (!target) return null;
let toolbar = document.getElementById("sdToolbar");
let grid = document.getElementById("sdCardsGrid");
if (toolbar && grid) {
let listWrap = document.getElementById("sdListWrap");
let listBody = document.getElementById("sdListBody");
if (toolbar && grid && listWrap && listBody) {
return {
target,
toolbar,
grid,
listWrap,
listBody,
searchInput: document.getElementById("sdSearch"),
perPageSelect: document.getElementById("sdPerPage"),
viewToggleWrap: document.getElementById("sdViewToggleWrap"),
viewCardsButton: document.getElementById("sdViewCards"),
viewListButton: document.getElementById("sdViewList"),
prevButton: document.getElementById("sdPrevPage"),
nextButton: document.getElementById("sdNextPage"),
pageCounter: document.getElementById("sdPageCounter"),
@@ -529,6 +676,41 @@ function ensureLayout() {
perPageWrap.appendChild(perPageSelect);
toolbar.appendChild(perPageWrap);
const viewToggleWrap = document.createElement("div");
viewToggleWrap.id = "sdViewToggleWrap";
viewToggleWrap.classList.add("d-none", "d-flex", "align-items-center", "gap-1");
const viewLabel = document.createElement("span");
viewLabel.classList.add("small", "text-body-secondary");
viewLabel.innerText = "View";
const viewToggle = document.createElement("div");
viewToggle.classList.add("btn-group", "btn-group-sm", "lqos-shaped-devices-view-toggle");
viewToggle.setAttribute("role", "group");
viewToggle.setAttribute("aria-label", "Dynamic circuits view");
const viewCards = document.createElement("button");
viewCards.id = "sdViewCards";
viewCards.type = "button";
viewCards.classList.add("btn", "btn-outline-secondary");
viewCards.innerHTML = "<i class='fa fa-th-large me-1'></i>Cards";
viewCards.onclick = () => {
setDynamicViewMode("cards");
};
const viewList = document.createElement("button");
viewList.id = "sdViewList";
viewList.type = "button";
viewList.classList.add("btn", "btn-outline-secondary");
viewList.innerHTML = "<i class='fa fa-list me-1'></i>List";
viewList.onclick = () => {
setDynamicViewMode("list");
};
viewToggle.appendChild(viewCards);
viewToggle.appendChild(viewList);
viewToggleWrap.appendChild(viewLabel);
viewToggleWrap.appendChild(viewToggle);
toolbar.appendChild(viewToggleWrap);
const pagerWrap = document.createElement("div");
pagerWrap.classList.add("d-flex", "align-items-center", "gap-2");
const pager = document.createElement("div");
@@ -580,12 +762,50 @@ function ensureLayout() {
grid.classList.add("row", "row-cols-1", "row-cols-md-2", "row-cols-xl-3", "g-3");
target.appendChild(grid);
listWrap = document.createElement("div");
listWrap.id = "sdListWrap";
listWrap.classList.add("table-responsive", "lqos-table-wrap", "d-none");
const listTable = document.createElement("table");
listTable.classList.add("lqos-table", "lqos-table-tight", "mb-0", "lqos-shaped-devices-list-table");
const listHead = document.createElement("thead");
const listHeadRow = document.createElement("tr");
[
"Circuit",
"Parent",
"Plan",
"Throughput",
"RTT",
"QoO",
"Retransmits",
"Last Seen",
"IPs",
].forEach((label) => {
const th = document.createElement("th");
th.innerText = label;
listHeadRow.appendChild(th);
});
listHead.appendChild(listHeadRow);
listTable.appendChild(listHead);
listBody = document.createElement("tbody");
listBody.id = "sdListBody";
listTable.appendChild(listBody);
listWrap.appendChild(listTable);
target.appendChild(listWrap);
return {
target,
toolbar,
grid,
listWrap,
listBody,
searchInput,
perPageSelect,
viewToggleWrap,
viewCardsButton: viewCards,
viewListButton: viewList,
prevButton: prev,
nextButton: next,
pageCounter,
@@ -656,19 +876,36 @@ async function requestShapedDevicesPage() {
current.up = Math.max(toNumber(current.up, 0), toNumber(device.upload_max_mbps, 0));
planByCircuitId.set(device.circuit_id, current);
});
renderCards();
renderDevices();
requestCircuitMetricsWatch(true);
renderCounts();
} catch (_error) {
shapedDevices = [];
totalRows = 0;
totalCircuits = 0;
renderCards();
renderDevices();
renderCounts();
}
}
function renderCards() {
function updateViewToggle(layout) {
const showToggle = shapedDevicesKind === "Dynamic";
if (layout.viewToggleWrap) {
layout.viewToggleWrap.classList.toggle("d-none", !showToggle);
layout.viewToggleWrap.classList.toggle("d-flex", showToggle);
}
const cardsActive = !isDynamicListView();
if (layout.viewCardsButton) {
layout.viewCardsButton.classList.toggle("active", cardsActive);
layout.viewCardsButton.setAttribute("aria-pressed", cardsActive.toString());
}
if (layout.viewListButton) {
layout.viewListButton.classList.toggle("active", !cardsActive);
layout.viewListButton.setAttribute("aria-pressed", (!cardsActive).toString());
}
}
function renderDevices() {
const layout = ensureLayout();
if (!layout) return;
@@ -679,6 +916,7 @@ function renderCards() {
if (layout.perPageSelect && String(devicesPerPage) !== layout.perPageSelect.value) {
layout.perPageSelect.value = String(devicesPerPage);
}
updateViewToggle(layout);
if (layout.prevButton) layout.prevButton.disabled = page <= 0;
if (layout.nextButton) layout.nextButton.disabled = page >= totalPages - 1;
if (layout.pageCounter) layout.pageCounter.innerText = "Page " + (page + 1) + " / " + totalPages;
@@ -688,21 +926,36 @@ function renderCards() {
} else {
const start = page * devicesPerPage + 1;
const end = Math.min((page + 1) * devicesPerPage, totalRows);
layout.summary.innerText = "Showing " + start + "–" + end + " of " + totalRows;
let summary = "Showing " + start + "–" + end + " of " + totalRows;
if (shapedDevicesKind === "Dynamic") {
summary += " (highest throughput first)";
}
layout.summary.innerText = summary;
}
}
metricElsByCircuitId = new Map();
clearDiv(layout.grid);
clearDiv(layout.listBody);
shapedDevices.forEach((device) => {
const col = document.createElement("div");
col.classList.add("col");
col.appendChild(buildDeviceCard(device));
layout.grid.appendChild(col);
});
const showList = isDynamicListView();
layout.grid.classList.toggle("d-none", showList);
layout.listWrap.classList.toggle("d-none", !showList);
// Fill visible cards from cached live data immediately.
if (showList) {
shapedDevices.forEach((device) => {
layout.listBody.appendChild(buildDeviceListRow(device));
});
} else {
shapedDevices.forEach((device) => {
const col = document.createElement("div");
col.classList.add("col");
col.appendChild(buildDeviceCard(device));
layout.grid.appendChild(col);
});
}
// Fill visible metrics from cached live data immediately.
metricElsByCircuitId.forEach((_metrics, circuitId) => {
const latest = latestByCircuitId.get(circuitId);
if (latest) {
@@ -2,6 +2,7 @@ import {clearDiv} from "./helpers/builders";
import {initRedact} from "./helpers/redact";
import {initDayNightMode} from "./helpers/dark_mode";
import {initColorBlind} from "./helpers/colorblind";
import {listenOnceMatchingWithTimeout as listenWsOnceMatchingWithTimeout} from "./pubsub/listeners";
import {get_ws_client} from "./pubsub/ws";
const wsClient = get_ws_client();
@@ -29,35 +30,20 @@ function escapeHtml(text) {
const SCHEDULER_STATUS_POLL_MS = 2000;
const SCHEDULER_STATUS_TIMEOUT_MS = 2500;
const URGENT_ISSUES_TIMEOUT_MS = 2500;
const SCHEDULER_STATUS_STARTING_GRACE_MS = 30000;
const SCHEDULER_MODAL_IDLE_POLL_MS = 3000;
const SCHEDULER_MODAL_ACTIVE_POLL_MS = 1000;
let schedulerStatusPollTimer = null;
let schedulerStatusRequestInFlight = false;
let schedulerStatusFirstRequestedAt = null;
let cachedSchedulerStatusData = null;
let schedulerModalPollTimer = null;
let schedulerModalRequestInFlight = false;
let schedulerModalInstance = null;
let cachedSchedulerDetailsData = null;
let cachedQueueModeConfig = null;
function listenOnceWithTimeout(eventName, timeoutMs, handler, onTimeout) {
let done = false;
const wrapped = (msg) => {
if (done) return;
done = true;
clearTimeout(timer);
wsClient.off(eventName, wrapped);
handler(msg);
};
const timer = setTimeout(() => {
if (done) return;
done = true;
wsClient.off(eventName, wrapped);
onTimeout();
}, timeoutMs);
wsClient.on(eventName, wrapped);
}
function clampSchedulerPercent(value) {
const parsed = Number(value);
if (!Number.isFinite(parsed)) return 0;
@@ -119,12 +105,79 @@ function schedulerRelativeTime(updatedUnix) {
return `Updated ${deltaDays}d ago`;
}
function schedulerErrorText(data) {
const error = data?.error;
if (!error) return "";
return String(error).trim();
}
function schedulerSnapshotUpdatedUnix(data) {
const progressUpdatedUnix = Number(data?.progress?.updated_unix);
if (Number.isFinite(progressUpdatedUnix) && progressUpdatedUnix > 0) {
return progressUpdatedUnix;
}
const statusUpdatedUnix = Number(data?.updated_unix);
if (Number.isFinite(statusUpdatedUnix) && statusUpdatedUnix > 0) {
return statusUpdatedUnix;
}
return null;
}
function schedulerUpdatedText(data) {
const updatedUnix = schedulerSnapshotUpdatedUnix(data);
if (!updatedUnix) {
return "Update time unavailable";
}
return schedulerRelativeTime(updatedUnix);
}
function schedulerTransportWarningText(message, data) {
const updatedText = schedulerUpdatedText(data);
if (updatedText === "Update time unavailable") {
return `${message} Retrying in the background.`;
}
return `${message} Showing cached scheduler data. ${updatedText}.`;
}
function schedulerStaleStatusLabel(data) {
const updatedText = schedulerUpdatedText(data);
if (updatedText === "Update time unavailable") {
return "Scheduler status is stale";
}
return `Scheduler status is stale. ${updatedText}.`;
}
function schedulerLooksRecovered(data) {
const progress = data?.progress || null;
if (!data?.available || !progress || progress.active) {
return false;
}
return String(progress.phase || "").trim() === "ready";
}
function schedulerHasLiveError(data) {
const errorText = schedulerErrorText(data);
if (!errorText) {
return false;
}
return !schedulerLooksRecovered(data);
}
function schedulerWaitsForTopologyRuntime(data) {
const progress = data?.progress || null;
if (!progress) {
return false;
}
return String(progress.phase || "").trim() === "waiting_for_topology_runtime";
}
function schedulerStateDescriptor(data) {
const progress = data?.progress || null;
const hasError = !!(data?.error && String(data.error).trim().length > 0);
const hasError = schedulerHasLiveError(data);
const available = !!data?.available;
const active = !!progress?.active;
const setupRequired = !!data?.setup_required;
const topologyRuntimeWait = schedulerWaitsForTopologyRuntime(data);
if (hasError) {
return {
@@ -148,6 +201,28 @@ function schedulerStateDescriptor(data) {
icon: "fa-list-check",
};
}
if (data?.stale) {
return {
tone: "warning",
badgeClass: "text-bg-warning",
label: "Stale",
title: "Scheduler status is stale",
subtitle: "Showing the last scheduler snapshot older than five minutes",
ringTone: "tone-warning",
icon: "fa-clock",
};
}
if (topologyRuntimeWait) {
return {
tone: "warning",
badgeClass: "text-bg-warning",
label: "Deferred",
title: progress?.phase_label || "Waiting for topology runtime",
subtitle: "Scheduler is waiting for topology runtime outputs",
ringTone: "tone-warning",
icon: "fa-hourglass-half",
};
}
if (active) {
return {
tone: "info",
@@ -228,9 +303,11 @@ function schedulerActivityItems(output, error, data) {
.reverse();
}
function renderSchedulerDetails(data) {
function renderSchedulerDetails(data, options = {}) {
const progress = data?.progress || null;
const descriptor = schedulerStateDescriptor(data);
const liveError = schedulerHasLiveError(data) ? schedulerErrorText(data) : "";
const historicalError = !liveError ? schedulerErrorText(data) : "";
const percent = progress ? schedulerProgressPercent(progress) : (data?.available ? 100 : 0);
const phaseLabel = progress?.phase_label || progress?.phase || descriptor.title;
const stepCount = Number(progress?.step_count);
@@ -238,24 +315,36 @@ function renderSchedulerDetails(data) {
const stepText = Number.isFinite(stepIndex) && Number.isFinite(stepCount) && stepCount > 0
? `Step ${stepIndex} of ${stepCount}`
: "No active step reported";
const updatedText = progress?.updated_unix
? schedulerRelativeTime(progress.updated_unix)
: "Update time unavailable";
const availabilityText = data?.setup_required ? "Setup Required" : (data?.available ? "Healthy" : "Unavailable");
const recentResult = schedulerSetupMessage(data) || summarizeSchedulerOutput(data?.output, data?.error);
const activity = schedulerActivityItems(data?.output, data?.error, data);
const progressMeta = data?.setup_required
? "Complete runtime setup to enable scheduler work"
: progress?.active
? `${percent}% complete`
: descriptor.label === "Idle"
? "Last run complete"
: "Waiting for activity";
const updatedText = schedulerUpdatedText(data);
const availabilityText = data?.setup_required
? "Setup Required"
: data?.stale
? "Stale"
: schedulerWaitsForTopologyRuntime(data)
? "Deferred"
: (data?.available ? "Healthy" : "Unavailable");
const recentResult = schedulerSetupMessage(data) || summarizeSchedulerOutput(data?.output, historicalError || liveError);
const activity = schedulerActivityItems(data?.output, historicalError || liveError, data);
let progressMeta = "Waiting for activity";
if (options.transportWarning) {
progressMeta = "Showing cached scheduler details while the UI reconnects";
} else if (data?.setup_required) {
progressMeta = "Complete runtime setup to enable scheduler work";
} else if (schedulerWaitsForTopologyRuntime(data)) {
progressMeta = "The next shaping update is waiting for topology runtime outputs";
} else if (progress?.active) {
progressMeta = `${percent}% complete`;
} else if (descriptor.label === "Idle") {
progressMeta = "Last run complete";
}
const setupAlert = data?.setup_required
? `<div class="alert alert-warning mt-3 mb-0" role="alert"><i class="fa fa-list-check me-2"></i>${escapeHtml(schedulerSetupMessage(data) || "Choose a topology source in Complete Setup before expecting scheduler activity.")}</div>`
: "";
const alertMarkup = data?.error
? `<div class="alert alert-danger mt-3 mb-0" role="alert"><i class="fa fa-triangle-exclamation me-2"></i>${escapeHtml(String(data.error).trim())}</div>`
const transportWarningMarkup = options.transportWarning
? `<div class="alert alert-warning mt-3 mb-0" role="alert"><i class="fa fa-wifi me-2"></i>${escapeHtml(options.transportWarning)}</div>`
: "";
const alertMarkup = liveError
? `<div class="alert alert-danger mt-3 mb-0" role="alert"><i class="fa fa-triangle-exclamation me-2"></i>${escapeHtml(liveError)}</div>`
: "";
const activityMarkup = activity.length
? activity.map(line => `
@@ -280,6 +369,7 @@ function renderSchedulerDetails(data) {
</div>
</div>
${setupAlert}
${transportWarningMarkup}
${alertMarkup}
<div class="lqos-scheduler-progress-card">
<div class="lqos-scheduler-progress-topline">
@@ -336,7 +426,7 @@ function updateSchedulerModalBody(contentHtml) {
}
}
function renderSchedulerStatus(container, state, progress) {
function renderSchedulerStatus(container, state, progress, labelOverride = null) {
if (!container) return;
let color = "text-secondary";
@@ -364,11 +454,26 @@ function renderSchedulerStatus(container, state, progress) {
color = "text-danger";
label = "Scheduler has an internal error";
indicator = schedulerRingMarkup(100, "tone-danger", "fa-triangle-exclamation");
} else if (state === "stale") {
color = "text-warning";
label = "Scheduler status is stale";
indicator = schedulerRingMarkup(100, "tone-warning", "fa-clock");
} else if (state === "deferred") {
color = "text-warning";
label = labelOverride || "Scheduler is waiting for topology runtime";
indicator = schedulerRingMarkup(
schedulerProgressPercent(progress),
"tone-warning",
"fa-hourglass-half"
);
} else if (state === "setup") {
color = "text-warning";
label = "Scheduler needs topology setup";
indicator = schedulerRingMarkup(100, "tone-warning", "fa-list-check");
}
if (labelOverride) {
label = labelOverride;
}
container.innerHTML = `
<button class="nav-link btn btn-link text-start w-100 p-0 border-0 ${color} lqos-scheduler-link" type="button" id="schedulerStatusLink" aria-label="${escapeAttr(label)}" title="${escapeAttr(label)}">
@@ -405,19 +510,30 @@ function loadSchedulerStatus(force = false) {
}
schedulerStatusRequestInFlight = true;
listenOnceWithTimeout("SchedulerStatus", SCHEDULER_STATUS_TIMEOUT_MS, (msg) => {
listenWsOnceMatchingWithTimeout(wsClient, "SchedulerStatus", SCHEDULER_STATUS_TIMEOUT_MS, () => true, (msg) => {
schedulerStatusRequestInFlight = false;
if (!msg || !msg.data) {
renderSchedulerStatus(container, "unavailable", null);
if (cachedSchedulerStatusData) {
renderSchedulerStatus(
container,
"stale",
cachedSchedulerStatusData.progress || null,
schedulerTransportWarningText("Scheduler status response was empty.", cachedSchedulerStatusData)
);
} else {
renderSchedulerStatus(container, "unavailable", null);
}
scheduleNextSchedulerStatusPoll();
return;
}
const data = msg.data;
const hasError = !!(data.error && String(data.error).trim().length > 0);
cachedSchedulerStatusData = data;
const hasError = schedulerHasLiveError(data);
const isHealthy = !!data.available && !hasError;
const progress = data.progress || null;
const progressActive = !!(progress && progress.active);
const setupRequired = !!data.setup_required;
const stale = !!data.stale;
const elapsed = schedulerStatusFirstRequestedAt === null ? 0 : (Date.now() - schedulerStatusFirstRequestedAt);
if (hasError) {
@@ -432,6 +548,23 @@ function loadSchedulerStatus(force = false) {
return;
}
if (stale) {
renderSchedulerStatus(container, "stale", progress, schedulerStaleStatusLabel(data));
scheduleNextSchedulerStatusPoll();
return;
}
if (schedulerWaitsForTopologyRuntime(data)) {
renderSchedulerStatus(
container,
"deferred",
progress,
progress?.phase_label || "Scheduler is waiting for topology runtime"
);
scheduleNextSchedulerStatusPoll();
return;
}
if (progressActive) {
renderSchedulerStatus(container, "progress", progress);
scheduleNextSchedulerStatusPoll();
@@ -451,7 +584,14 @@ function loadSchedulerStatus(force = false) {
}, () => {
schedulerStatusRequestInFlight = false;
const elapsed = schedulerStatusFirstRequestedAt === null ? 0 : (Date.now() - schedulerStatusFirstRequestedAt);
if (elapsed < SCHEDULER_STATUS_STARTING_GRACE_MS) {
if (cachedSchedulerStatusData) {
renderSchedulerStatus(
container,
"stale",
cachedSchedulerStatusData.progress || null,
schedulerTransportWarningText("Scheduler status refresh timed out.", cachedSchedulerStatusData)
);
} else if (elapsed < SCHEDULER_STATUS_STARTING_GRACE_MS) {
renderSchedulerStatus(container, "loading", null);
} else {
renderSchedulerStatus(container, "unavailable", null);
@@ -510,13 +650,6 @@ function renderSchedulerModalLoading() {
</div>`);
}
function renderSchedulerModalError(message) {
updateSchedulerModalBody(`
<div class="alert alert-danger mb-0" role="alert">
<i class="fa fa-triangle-exclamation me-2"></i>${escapeHtml(message)}
</div>`);
}
function ensureSchedulerModalLifecycle() {
const modalEl = document.getElementById('schedulerModal');
if (!modalEl || modalEl.dataset.schedulerLiveBound === "1") {
@@ -536,17 +669,31 @@ function refreshSchedulerModalDetails(showLoading = false) {
renderSchedulerModalLoading();
}
schedulerModalRequestInFlight = true;
listenOnceWithTimeout("SchedulerDetails", SCHEDULER_STATUS_TIMEOUT_MS, (msg) => {
listenWsOnceMatchingWithTimeout(wsClient, "SchedulerDetails", SCHEDULER_STATUS_TIMEOUT_MS, () => true, (msg) => {
schedulerModalRequestInFlight = false;
if (!isSchedulerModalOpen()) {
stopSchedulerModalPolling();
return;
}
if (!msg || !msg.data) {
renderSchedulerModalError("Failed to load scheduler details");
if (cachedSchedulerDetailsData) {
updateSchedulerModalBody(renderSchedulerDetails(cachedSchedulerDetailsData, {
transportWarning: schedulerTransportWarningText(
"Scheduler details response was empty.",
cachedSchedulerDetailsData
),
}));
} else {
updateSchedulerModalBody(`
<div class="alert alert-warning mb-0" role="alert">
<i class="fa fa-wifi me-2"></i>${escapeHtml("Failed to load scheduler details. Retrying in the background.")}
</div>`);
}
scheduleNextSchedulerModalPoll(SCHEDULER_MODAL_IDLE_POLL_MS);
return;
}
cachedSchedulerDetailsData = msg.data;
cachedSchedulerStatusData = msg.data;
updateSchedulerModalBody(renderSchedulerDetails(msg.data));
scheduleNextSchedulerModalPoll(schedulerModalPollDelay(msg.data));
}, () => {
@@ -555,7 +702,19 @@ function refreshSchedulerModalDetails(showLoading = false) {
stopSchedulerModalPolling();
return;
}
renderSchedulerModalError("Timed out while loading scheduler details");
if (cachedSchedulerDetailsData) {
updateSchedulerModalBody(renderSchedulerDetails(cachedSchedulerDetailsData, {
transportWarning: schedulerTransportWarningText(
"Scheduler details refresh timed out.",
cachedSchedulerDetailsData
),
}));
} else {
updateSchedulerModalBody(`
<div class="alert alert-warning mb-0" role="alert">
<i class="fa fa-wifi me-2"></i>${escapeHtml("Timed out while loading scheduler details. Retrying in the background.")}
</div>`);
}
scheduleNextSchedulerModalPoll(SCHEDULER_MODAL_IDLE_POLL_MS);
});
wsClient.send({ SchedulerDetails: {} });
@@ -928,6 +1087,15 @@ function initUrgentIssues() {
const containerId = 'urgentStatus';
const linkId = 'urgentStatusLink';
const badgeId = 'urgentBadge';
let urgentClearState = null;
let urgentStatusPending = false;
let urgentStatusRefreshQueued = false;
let urgentListPending = false;
let urgentListActiveRequestId = null;
let urgentListCancel = null;
let urgentRequestId = 0;
let pendingUrgentActionError = null;
let pendingUrgentRetryMessage = null;
function ensurePlaceholder() {
return document.getElementById(containerId) !== null;
@@ -951,65 +1119,299 @@ function initUrgentIssues() {
function poll() {
if (!ensurePlaceholder()) return;
listenOnce("UrgentStatus", (msg) => {
const count = msg && msg.data ? msg.data.count : 0;
renderStatus(count || 0);
if (urgentStatusPending) {
urgentStatusRefreshQueued = true;
return;
}
urgentStatusPending = true;
const requestId = ++urgentRequestId;
listenWsOnceMatchingWithTimeout(
wsClient,
"UrgentStatus",
URGENT_ISSUES_TIMEOUT_MS,
(msg) => msg.request_id === requestId,
(msg) => {
urgentStatusPending = false;
const count = msg && msg.data ? msg.data.count : 0;
renderStatus(count || 0);
runQueuedUrgentStatusRefresh();
},
() => {
urgentStatusPending = false;
runQueuedUrgentStatusRefresh();
}
);
wsClient.send({ UrgentStatus: { request_id: requestId } });
}
function runQueuedUrgentStatusRefresh() {
if (!urgentStatusRefreshQueued) return false;
urgentStatusRefreshQueued = false;
poll();
return true;
}
function clearActionInFlight() {
return urgentClearState !== null;
}
function clearActionBlocked() {
return clearActionInFlight() || urgentListPending;
}
function setClearControlsDisabled(disabled) {
document.querySelectorAll('.urgent-clear').forEach((button) => {
button.disabled = disabled;
});
wsClient.send({ UrgentStatus: {} });
const clearAllButton = document.getElementById('urgentClearAll');
if (clearAllButton) {
clearAllButton.disabled = disabled;
}
}
function showUrgentBusyState(message) {
const holder = document.getElementById('urgentListContainer');
if (!holder || !urgentModalIsOpen()) return;
holder.setAttribute('aria-busy', 'true');
holder.querySelector('.urgent-busy-status')?.remove();
holder.insertAdjacentHTML('afterbegin', `
<div class="alert alert-info mb-2 urgent-busy-status" role="status" aria-live="polite" tabindex="-1">
<i class="fa fa-spinner fa-spin" aria-hidden="true"></i>
${escapeHtml(message)}
</div>`);
holder.querySelector('.urgent-busy-status')?.focus();
}
function clearUrgentBusyState(holder) {
const target = holder || document.getElementById('urgentListContainer');
if (!target) return;
target.removeAttribute('aria-busy');
target.querySelector('.urgent-busy-status')?.remove();
}
function renderUrgentWarning(holder, message, { focus = false, retry = false } = {}) {
if (!holder) return;
holder.querySelector('.urgent-action-error')?.remove();
const retryButton = retry
? '<button type="button" class="btn btn-link btn-sm urgent-refresh">Retry</button>'
: '';
holder.insertAdjacentHTML('afterbegin', `
<div class="alert alert-warning mb-2 urgent-action-error" role="alert" tabindex="-1">
${escapeHtml(message)}
${retryButton}
</div>`);
if (focus) {
holder.querySelector(retry ? '.urgent-refresh' : '.urgent-action-error')?.focus();
}
}
function showUrgentActionError(message) {
pendingUrgentActionError = message;
if (!urgentModalIsOpen()) return;
const holder = document.getElementById('urgentListContainer');
if (!holder) return;
clearUrgentBusyState(holder);
renderUrgentWarning(holder, message, { focus: true });
pendingUrgentActionError = null;
}
function attachPendingUrgentActionError(holder) {
if (!pendingUrgentActionError || !urgentModalIsOpen() || !holder) return;
renderUrgentWarning(holder, pendingUrgentActionError);
pendingUrgentActionError = null;
}
function showUrgentRetryState(message) {
pendingUrgentRetryMessage = message;
const holder = document.getElementById('urgentListContainer');
if (!holder || !urgentModalIsOpen()) return;
pendingUrgentRetryMessage = null;
setClearControlsDisabled(true);
clearUrgentBusyState(holder);
holder.innerHTML = '';
renderUrgentWarning(holder, message, { focus: true, retry: true });
$(holder).off('click').on('click', '.urgent-refresh', function (e) {
e.preventDefault();
showModal();
});
}
function showPendingUrgentRetryState() {
if (!pendingUrgentRetryMessage || !urgentModalIsOpen()) return false;
showUrgentRetryState(pendingUrgentRetryMessage);
return true;
}
function urgentModalIsOpen() {
return document.getElementById('urgentModal')?.classList.contains('show') === true;
}
function focusUrgentModalFallback() {
const modalEl = document.getElementById('urgentModal');
if (!modalEl || !urgentModalIsOpen()) return;
modalEl
.querySelector('.urgent-clear:not(:disabled), #urgentClearAll:not(:disabled), .btn-close:not(:disabled), [data-bs-dismiss="modal"]:not(:disabled)')
?.focus();
}
function refreshUrgentIssues() {
showModal();
poll();
}
function completeClearAction() {
urgentClearState = null;
setClearControlsDisabled(clearActionInFlight());
refreshUrgentIssues();
}
function startUrgentClear({
responseEvent,
requestPayload,
busyMessage,
failureMessage,
timeoutMessage,
}) {
if (clearActionBlocked()) {
return;
}
const requestId = ++urgentRequestId;
setClearControlsDisabled(true);
showUrgentBusyState(busyMessage);
urgentClearState = {
busyMessage,
cancel: null,
};
urgentClearState.cancel = listenWsOnceMatchingWithTimeout(
wsClient,
responseEvent,
URGENT_ISSUES_TIMEOUT_MS,
(msg) => msg.request_id === requestId,
(msg) => {
urgentClearState = null;
if (msg && msg.ok === false) {
setClearControlsDisabled(false);
showUrgentActionError(failureMessage);
return;
}
completeClearAction();
},
() => {
urgentClearState = null;
setClearControlsDisabled(false);
showUrgentActionError(timeoutMessage);
}
);
wsClient.send(requestPayload(requestId));
}
function showModal() {
const modalEl = document.getElementById('urgentModal');
if (!modalEl) return;
new bootstrap.Modal(modalEl, { focus: true }).show();
if (!modalEl) {
return;
}
bootstrap.Modal.getOrCreateInstance(modalEl, { focus: true }).show();
const holder = document.getElementById('urgentListContainer');
if (!holder) return;
holder.innerHTML = `<div class="text-center text-muted"><i class='fa fa-spinner fa-spin'></i> Loading...</div>`;
listenOnce("UrgentList", (msg) => {
const items = msg && msg.data ? msg.data.items || [] : [];
if (items.length === 0) {
holder.innerHTML = '<div class="text-center text-success">No urgent issues.</div>';
return;
}
const table = document.createElement('table');
table.className = 'lqos-table lqos-table-compact mb-0';
const tbody = document.createElement('tbody');
items.forEach((it) => {
const tr = document.createElement('tr');
const td = document.createElement('td');
const when = new Date(it.ts * 1000).toLocaleString();
const sev = it.severity === 'Error' ? 'danger' : 'warning';
td.innerHTML = `
if (!holder) {
return;
}
if (showPendingUrgentRetryState()) {
return;
}
if (clearActionInFlight()) {
setClearControlsDisabled(true);
showUrgentBusyState(urgentClearState.busyMessage);
return;
}
if (urgentListPending) {
setClearControlsDisabled(true);
return;
}
urgentListPending = true;
const requestId = ++urgentRequestId;
urgentListActiveRequestId = requestId;
setClearControlsDisabled(true);
clearUrgentBusyState(holder);
holder.innerHTML = `<div class="text-center text-muted urgent-loading" role="status" aria-live="polite" tabindex="-1"><i class='fa fa-spinner fa-spin'></i> Loading...</div>`;
holder.querySelector('.urgent-loading')?.focus();
urgentListCancel = listenWsOnceMatchingWithTimeout(
wsClient,
"UrgentList",
URGENT_ISSUES_TIMEOUT_MS,
(msg) => msg.request_id === requestId && urgentListActiveRequestId === requestId,
(msg) => {
urgentListPending = false;
urgentListActiveRequestId = null;
urgentListCancel = null;
const items = msg && msg.data ? msg.data.items || [] : [];
clearUrgentBusyState(holder);
if (items.length === 0) {
holder.innerHTML = '<div class="text-center text-success" role="status" aria-live="polite" tabindex="-1">No urgent issues.</div>';
attachPendingUrgentActionError(holder);
setClearControlsDisabled(true);
holder.querySelector('[role="status"]')?.focus();
return;
}
const table = document.createElement('table');
table.className = 'lqos-table lqos-table-compact mb-0';
const tbody = document.createElement('tbody');
items.forEach((it) => {
const tr = document.createElement('tr');
const td = document.createElement('td');
const when = new Date(it.ts * 1000).toLocaleString();
const sev = it.severity === 'Error' ? 'danger' : 'warning';
const clearControl = it.clearable === false
? '<span class="badge bg-secondary-subtle text-secondary border float-end ms-3">Active</span>'
: `<button type="button" class="btn btn-link btn-sm text-secondary float-end ms-3 p-0 urgent-clear" data-id="${it.id}" title="Acknowledge issue" aria-label="Acknowledge issue ${escapeAttr(it.code)}"><i class="fa fa-times" aria-hidden="true"></i></button>`;
td.innerHTML = `
<div>
<span class="badge bg-${sev}">${it.severity}</span>
<strong class="ms-2">${it.code}</strong>
<span class="text-muted ms-2">(${it.source})</span>
<span class="text-muted float-end">${when}</span>
<button type="button" class="btn btn-link btn-sm text-secondary float-end ms-3 p-0 urgent-clear" data-id="${it.id}" title="Acknowledge issue" aria-label="Acknowledge issue ${escapeAttr(it.code)}"><i class="fa fa-times" aria-hidden="true"></i></button>
${clearControl}
</div>
<div class="mt-1" style="white-space: pre-wrap;">${it.message}</div>
${it.context ? `<pre class="mt-2">${it.context}</pre>` : ''}
`;
tr.appendChild(td);
tbody.appendChild(tr);
});
table.appendChild(tbody);
holder.innerHTML = '';
const tableWrap = document.createElement('div');
tableWrap.className = 'table-responsive lqos-table-wrap';
tableWrap.appendChild(table);
holder.appendChild(tableWrap);
$(holder).off('click').on('click', '.urgent-clear', function (e) {
e.preventDefault();
const id = $(this).data('id');
listenOnce("UrgentClearResult", () => {
showModal();
poll();
tr.appendChild(td);
tbody.appendChild(tr);
});
wsClient.send({ UrgentClear: { id } });
});
});
wsClient.send({ UrgentList: {} });
table.appendChild(tbody);
holder.innerHTML = '';
const tableWrap = document.createElement('div');
tableWrap.className = 'table-responsive lqos-table-wrap';
tableWrap.appendChild(table);
holder.appendChild(tableWrap);
attachPendingUrgentActionError(holder);
$(holder).off('click').on('click', '.urgent-clear', function (e) {
e.preventDefault();
const id = $(this).data('id');
startUrgentClear({
responseEvent: 'UrgentClearResult',
requestPayload: (requestId) => ({ UrgentClear: { id, request_id: requestId } }),
busyMessage: 'Acknowledging urgent issue...',
failureMessage: 'Unable to acknowledge urgent issue.',
timeoutMessage: 'Timed out while acknowledging urgent issue.',
});
});
setClearControlsDisabled(clearActionInFlight());
focusUrgentModalFallback();
},
() => {
if (urgentListActiveRequestId !== requestId) return;
urgentListPending = false;
urgentListActiveRequestId = null;
urgentListCancel = null;
if (!urgentModalIsOpen()) {
setClearControlsDisabled(clearActionInFlight());
return;
}
showUrgentRetryState('Unable to refresh urgent issues.');
setClearControlsDisabled(true);
}
);
wsClient.send({ UrgentList: { request_id: requestId } });
}
if (!document.getElementById(containerId)) {
@@ -1023,11 +1425,31 @@ function initUrgentIssues() {
}
$(document).off('click', '#urgentClearAll').on('click', '#urgentClearAll', () => {
listenOnce("UrgentClearAllResult", () => {
showModal();
poll();
startUrgentClear({
responseEvent: 'UrgentClearAllResult',
requestPayload: (requestId) => ({ UrgentClearAll: { request_id: requestId } }),
busyMessage: 'Acknowledging urgent issues...',
failureMessage: 'Unable to acknowledge urgent issues.',
timeoutMessage: 'Timed out while acknowledging urgent issues.',
});
wsClient.send({ UrgentClearAll: {} });
});
$(document).off('hidden.bs.modal', '#urgentModal').on('hidden.bs.modal', '#urgentModal', () => {
pendingUrgentRetryMessage = null;
if (urgentClearState?.cancel) {
urgentClearState.cancel();
urgentClearState = null;
}
if (urgentListPending) {
if (urgentListCancel) {
urgentListCancel();
urgentListCancel = null;
}
urgentListPending = false;
urgentListActiveRequestId = null;
}
clearUrgentBusyState();
setClearControlsDisabled(clearActionInFlight());
});
poll();
@@ -9,7 +9,10 @@ if ! command -v node >/dev/null 2>&1; then
fi
echo "[test-node-manager] Running frontend contract tests..."
node --test "${SCRIPT_DIR}/src/config/shaped_device_wire.test.mjs"
node --test \
"${SCRIPT_DIR}/src/config/shaped_device_wire.test.mjs" \
"${SCRIPT_DIR}/src/config/config_interface_mtu.test.mjs" \
"${SCRIPT_DIR}/src/pubsub/ws.test.mjs"
echo "[test-node-manager] Building bundles..."
"${SCRIPT_DIR}/esbuild.sh"
@@ -7,21 +7,24 @@ use axum::body::Bytes;
use axum::http::StatusCode;
use axum::http::header;
use default_net::get_interfaces;
use lqos_bus::{BusRequest, bus_request};
use lqos_config::{Config, ConfigShapedDevices, ShapedDevice, UserRole, WebUser, WebUsers};
use lqos_bus::{BusRequest, BusResponse, bus_request_with_timeout};
use lqos_config::{
Config, ConfigShapedDevices, NetworkJson, ShapedDevice, UserRole, WebUser, WebUsers,
};
use lqos_utils::hash_to_i64;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::io::{Cursor, ErrorKind, Write};
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
const COBRAND_FILE_NAME: &str = "cobrand.png";
const COBRAND_DISPLAY_HEIGHT_PX: u64 = 48;
const COBRAND_MAX_DISPLAY_WIDTH_PX: u64 = 176;
const COBRAND_MAX_DECODE_BYTES: usize = 64 * 1024 * 1024;
const PNG_SIGNATURE: &[u8; 8] = b"\x89PNG\r\n\x1a\n";
const CONFIG_BUS_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum CobrandUploadValidationError {
@@ -524,10 +527,10 @@ pub fn list_nics_data(login: LoginResult) -> Result<Vec<(String, String, String)
pub fn network_json_data() -> Value {
if let Ok(config) = lqos_config::load_config() {
let path = std::path::Path::new(&config.lqos_directory).join("network.json");
if path.exists() {
let raw = std::fs::read_to_string(path).expect("Unable to read network json");
let json: Value = serde_json::from_str(&raw).expect("Unable to read network json");
let path = NetworkJson::path_for_config(config.as_ref());
if let Ok(raw) = std::fs::read_to_string(path)
&& let Ok(json) = serde_json::from_str(&raw)
{
return json;
}
}
@@ -783,6 +786,7 @@ fn persist_shaped_devices(mut devices: Vec<ShapedDevice>) -> Result<(), String>
lqos_network_devices::apply_shaped_devices_snapshot(
"node_manager:persist_shaped_devices",
copied,
lqos_network_devices::ShapedDevicesSnapshotProvenance::ExternalSnapshot,
)
.map_err(|e| format!("Unable to publish ShapedDevices.csv snapshot: {e}"))?;
@@ -793,16 +797,26 @@ pub async fn update_lqosd_config_data(
login: LoginResult,
mut config: Config,
clear_secrets: ConfigSecretClearRequest,
) -> Result<(), StatusCode> {
) -> Result<(), String> {
if login != LoginResult::Admin {
return Err(StatusCode::FORBIDDEN);
return Err("Unauthorized".to_string());
}
let existing = lqos_config::load_config().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let existing =
lqos_config::load_config().map_err(|_| "Unable to load the current config".to_string())?;
apply_secret_updates(existing.as_ref(), &mut config, &clear_secrets);
bus_request(vec![BusRequest::UpdateLqosdConfig(Box::new(config))])
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
Ok(())
let mut responses = bus_request_with_timeout(
vec![BusRequest::UpdateLqosdConfig(Box::new(config))],
CONFIG_BUS_REQUEST_TIMEOUT,
)
.await
.map_err(|err| format!("Unable to update config: {err}"))?;
match responses.pop() {
Some(BusResponse::Ack) => Ok(()),
Some(BusResponse::Fail(message)) => Err(message),
Some(other) => Err(format!("Unexpected config update response: {other:?}")),
None => Err("No response received for config update".to_string()),
}
}
/// Persists both `network.json` and `ShapedDevices.csv` for administrative
@@ -352,7 +352,7 @@ fn build_top_asn_rows(entities: &[ExecutiveEntitySnapshot]) -> Vec<ExecutiveTopA
median_retransmit_pct: median_value(&entity.heatmap.retransmit),
})
.collect::<Vec<_>>();
rows.sort_by(|left, right| right.total_bytes_15m.cmp(&left.total_bytes_15m));
rows.sort_by_key(|row| std::cmp::Reverse(row.total_bytes_15m));
rows.truncate(EXECUTIVE_TOP_LIMIT);
rows
}
@@ -7,7 +7,7 @@ use crate::node_manager::local_api::circuit_count;
use axum::http::StatusCode;
pub use last_24_hours::*;
use lqos_bus::LtsCapabilitiesSummary;
use lqos_bus::{BusRequest, bus_request};
use lqos_bus::{BusRequest, bus_request_with_timeout};
use lqos_config::load_config;
use serde::{Deserialize, Serialize};
pub use shaper_status::ShaperStatus;
@@ -18,6 +18,8 @@ use tracing::{debug, info, warn};
use uuid::Uuid;
const SIGNUP_POLL_INTERVAL: Duration = Duration::from_secs(10);
const CONFIG_BUS_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
const SERVICE_RESTART_TIMEOUT: Duration = Duration::from_secs(10);
const INSIGHT_FREE_TRIAL_STATUS_CODE: i32 = 2;
const INSIGHT_TRIAL_DAYS_HINT: i32 = 30;
@@ -132,9 +134,15 @@ async fn apply_insight_license(
.clone();
cfg.long_term_stats.gather_stats = true;
cfg.long_term_stats.license_key = Some(license_key.clone());
bus_request(vec![BusRequest::UpdateLqosdConfig(Box::new(cfg))])
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
bus_request_with_timeout(
vec![BusRequest::UpdateLqosdConfig(Box::new(cfg))],
CONFIG_BUS_REQUEST_TIMEOUT,
)
.await
.map_err(|err| {
warn!("Unable to update config with Insight license: {err}");
StatusCode::GATEWAY_TIMEOUT
})?;
crate::lts2_sys::capabilities::set_signup_bootstrap_active(false);
crate::lts2_sys::capabilities::clear_bootstrap_suppression();
crate::lts2_sys::capabilities::wake_control_channel();
@@ -147,9 +155,21 @@ async fn apply_insight_license(
info!("LQOSD configuration updated with new Insight license key.");
if restart_lqos_api {
let _ = Command::new("/bin/systemctl")
.args(["restart", "lqos_api"])
.output();
let restart = tokio::time::timeout(
SERVICE_RESTART_TIMEOUT,
tokio::task::spawn_blocking(|| {
Command::new("/bin/systemctl")
.args(["restart", "lqos_api"])
.output()
}),
)
.await;
match restart {
Ok(Ok(Ok(_))) => {}
Ok(Ok(Err(err))) => warn!("Unable to restart lqos_api: {err}"),
Ok(Err(err)) => warn!("lqos_api restart task failed: {err}"),
Err(_) => warn!("Timed out restarting lqos_api"),
}
}
Ok(())
@@ -1,8 +1,13 @@
use crate::node_manager::shaper_queries_actor::ShaperQueryCommand;
use axum::http::StatusCode;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use tokio::time::timeout;
use tracing::warn;
const SHAPER_QUERY_RESPONSE_TIMEOUT: Duration = Duration::from_secs(35);
const SHAPER_QUERY_SEND_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Serialize, Deserialize, Copy, Clone, Debug)]
pub struct ThroughputData {
time: i64, // Unix timestamp
@@ -136,20 +141,54 @@ pub struct RecentMedians {
pub last_week: (i64, i64),
}
async fn wait_for_shaper_response<T>(
context: &'static str,
rx: tokio::sync::oneshot::Receiver<Vec<T>>,
) -> Result<Vec<T>, StatusCode> {
match timeout(SHAPER_QUERY_RESPONSE_TIMEOUT, rx).await {
Ok(Ok(response)) => Ok(response),
Ok(Err(err)) => {
warn!("Error getting {context}: {err:?}");
Err(StatusCode::INTERNAL_SERVER_ERROR)
}
Err(_) => {
warn!("Timed out waiting for {context}");
Err(StatusCode::GATEWAY_TIMEOUT)
}
}
}
async fn send_shaper_query(
context: &'static str,
shaper_query: &tokio::sync::mpsc::Sender<ShaperQueryCommand>,
command: ShaperQueryCommand,
) -> Result<(), StatusCode> {
match timeout(SHAPER_QUERY_SEND_TIMEOUT, shaper_query.send(command)).await {
Ok(Ok(())) => Ok(()),
Ok(Err(_)) => {
warn!("Error sending {context} query");
Err(StatusCode::INTERNAL_SERVER_ERROR)
}
Err(_) => {
warn!("Timed out queueing {context} query");
Err(StatusCode::GATEWAY_TIMEOUT)
}
}
}
pub async fn throughput_period_data(
shaper_query: tokio::sync::mpsc::Sender<ShaperQueryCommand>,
seconds: i32,
) -> Result<Vec<ThroughputData>, StatusCode> {
super::insight_gate().await?;
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::ShaperThroughput { seconds, reply: tx })
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let throughput = rx.await.map_err(|e| {
warn!("Error getting total throughput: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"throughput",
&shaper_query,
ShaperQueryCommand::ShaperThroughput { seconds, reply: tx },
)
.await?;
let throughput = wait_for_shaper_response("total throughput", rx).await?;
Ok(throughput)
}
@@ -159,17 +198,13 @@ pub async fn packets_period_data(
) -> Result<Vec<FullPacketData>, StatusCode> {
super::insight_gate().await?;
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::ShaperPackets { seconds, reply: tx })
.await
.map_err(|_| {
warn!("Error sending packets period");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let throughput = rx.await.map_err(|e| {
warn!("Error getting packets period: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"packets period",
&shaper_query,
ShaperQueryCommand::ShaperPackets { seconds, reply: tx },
)
.await?;
let throughput = wait_for_shaper_response("packets period", rx).await?;
Ok(throughput)
}
@@ -179,17 +214,13 @@ pub async fn percent_shaped_period_data(
) -> Result<Vec<PercentShapedWeb>, StatusCode> {
super::insight_gate().await?;
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::ShaperPercent { seconds, reply: tx })
.await
.map_err(|_| {
warn!("Error sending percent shaped period");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let throughput = rx.await.map_err(|e| {
warn!("Error getting percent shaped: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"percent shaped period",
&shaper_query,
ShaperQueryCommand::ShaperPercent { seconds, reply: tx },
)
.await?;
let throughput = wait_for_shaper_response("percent shaped", rx).await?;
Ok(throughput)
}
@@ -199,17 +230,13 @@ pub async fn percent_flows_period_data(
) -> Result<Vec<FlowCountViewWeb>, StatusCode> {
super::insight_gate().await?;
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::ShaperFlows { seconds, reply: tx })
.await
.map_err(|_| {
warn!("Error sending flows period");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let throughput = rx.await.map_err(|e| {
warn!("Error getting flows: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"flows period",
&shaper_query,
ShaperQueryCommand::ShaperFlows { seconds, reply: tx },
)
.await?;
let throughput = wait_for_shaper_response("flows", rx).await?;
Ok(throughput)
}
@@ -218,19 +245,14 @@ pub async fn rtt_histo_period_data(
seconds: i32,
) -> Result<Vec<ShaperRttHistogramEntry>, StatusCode> {
super::insight_gate().await?;
tracing::error!("rtt_histo_period");
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::ShaperRttHistogram { seconds, reply: tx })
.await
.map_err(|_| {
warn!("Error sending flows period");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let throughput = rx.await.map_err(|e| {
warn!("Error getting flows: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"RTT histogram period",
&shaper_query,
ShaperQueryCommand::ShaperRttHistogram { seconds, reply: tx },
)
.await?;
let throughput = wait_for_shaper_response("RTT histogram", rx).await?;
Ok(throughput)
}
@@ -239,19 +261,14 @@ pub async fn top10_downloaders_period_data(
seconds: i32,
) -> Result<Vec<Top10Circuit>, StatusCode> {
super::insight_gate().await?;
tracing::error!("rtt_histo_period");
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::ShaperTopDownloaders { seconds, reply: tx })
.await
.map_err(|_| {
warn!("Error sending flows period");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let throughput = rx.await.map_err(|e| {
warn!("Error getting flows: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"top downloaders period",
&shaper_query,
ShaperQueryCommand::ShaperTopDownloaders { seconds, reply: tx },
)
.await?;
let throughput = wait_for_shaper_response("top downloaders", rx).await?;
Ok(throughput)
}
@@ -260,19 +277,14 @@ pub async fn worst10_rtt_period_data(
seconds: i32,
) -> Result<Vec<Worst10RttCircuit>, StatusCode> {
super::insight_gate().await?;
tracing::error!("rtt_histo_period");
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::ShaperWorstRtt { seconds, reply: tx })
.await
.map_err(|_| {
warn!("Error sending flows period");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let throughput = rx.await.map_err(|e| {
warn!("Error getting flows: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"worst RTT period",
&shaper_query,
ShaperQueryCommand::ShaperWorstRtt { seconds, reply: tx },
)
.await?;
let throughput = wait_for_shaper_response("worst RTT", rx).await?;
Ok(throughput)
}
@@ -281,19 +293,14 @@ pub async fn worst10_rxmit_period_data(
seconds: i32,
) -> Result<Vec<Worst10RxmitCircuit>, StatusCode> {
super::insight_gate().await?;
tracing::error!("rtt_histo_period");
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::ShaperWorstRxmit { seconds, reply: tx })
.await
.map_err(|_| {
warn!("Error sending flows period");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let throughput = rx.await.map_err(|e| {
warn!("Error getting flows: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"worst retransmits period",
&shaper_query,
ShaperQueryCommand::ShaperWorstRxmit { seconds, reply: tx },
)
.await?;
let throughput = wait_for_shaper_response("worst retransmits", rx).await?;
Ok(throughput)
}
@@ -302,19 +309,14 @@ pub async fn top10_flows_period_data(
seconds: i32,
) -> Result<Vec<AsnFlowSizeWeb>, StatusCode> {
super::insight_gate().await?;
tracing::error!("rtt_histo_period");
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::ShaperTopFlows { seconds, reply: tx })
.await
.map_err(|_| {
warn!("Error sending flows period");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let throughput = rx.await.map_err(|e| {
warn!("Error getting flows: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"top flows period",
&shaper_query,
ShaperQueryCommand::ShaperTopFlows { seconds, reply: tx },
)
.await?;
let throughput = wait_for_shaper_response("top flows", rx).await?;
Ok(throughput)
}
@@ -322,19 +324,14 @@ pub async fn recent_medians_data(
shaper_query: tokio::sync::mpsc::Sender<ShaperQueryCommand>,
) -> Result<Vec<RecentMedians>, StatusCode> {
super::insight_gate().await?;
tracing::debug!("rtt_histo_period");
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::ShaperRecentMedian { reply: tx })
.await
.map_err(|_| {
warn!("Error sending flows period");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let throughput = rx.await.map_err(|e| {
warn!("Error getting flows: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"recent median",
&shaper_query,
ShaperQueryCommand::ShaperRecentMedian { reply: tx },
)
.await?;
let throughput = wait_for_shaper_response("recent medians", rx).await?;
Ok(throughput)
}
@@ -344,16 +341,12 @@ pub async fn cake_period_data(
) -> Result<Vec<CakeData>, StatusCode> {
super::insight_gate().await?;
let (tx, rx) = tokio::sync::oneshot::channel();
shaper_query
.send(ShaperQueryCommand::CakeTotals { seconds, reply: tx })
.await
.map_err(|_| {
warn!("Error sending cake stats period");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let response = rx.await.map_err(|e| {
warn!("Error getting cake stats: {:?}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
send_shaper_query(
"cake stats period",
&shaper_query,
ShaperQueryCommand::CakeTotals { seconds, reply: tx },
)
.await?;
let response = wait_for_shaper_response("cake stats", rx).await?;
Ok(response)
}
@@ -77,7 +77,7 @@ fn unauthorized() -> (StatusCode, Json<ApplyResponse>) {
}
fn helper_validation_error_response(message: String) -> (StatusCode, Json<ApplyResponse>) {
warn!("Bridge helper request failed: {message}");
warn!("Network-mode request failed: {message}");
(
StatusCode::BAD_REQUEST,
Json(ApplyResponse {
@@ -90,7 +90,7 @@ fn helper_validation_error_response(message: String) -> (StatusCode, Json<ApplyR
}
fn helper_internal_error_response(message: String) -> (StatusCode, Json<ApplyResponse>) {
error!("Bridge helper internal error: {message}");
error!("Network-mode internal error: {message}");
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(ApplyResponse {
@@ -102,11 +102,20 @@ fn helper_internal_error_response(message: String) -> (StatusCode, Json<ApplyRes
)
}
fn merge_network_mode(candidate: Config) -> Result<Config, StatusCode> {
let live = lqos_config::load_config().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
#[derive(Debug, Clone, PartialEq, Eq)]
enum MergeNetworkModeError {
LoadLiveConfig,
InvalidCandidate(String),
}
fn merge_network_mode(candidate: Config) -> Result<Config, MergeNetworkModeError> {
let live = lqos_config::load_config().map_err(|_| MergeNetworkModeError::LoadLiveConfig)?;
let mut merged = (*live).clone();
merged.bridge = candidate.bridge;
merged.single_interface = candidate.single_interface;
merged
.validate()
.map_err(MergeNetworkModeError::InvalidCandidate)?;
Ok(merged)
}
@@ -156,13 +165,23 @@ pub async fn inspect(
State(state): State<NetworkModeApiState>,
Extension(login): Extension<LoginResult>,
Json(body): Json<NetworkModeInspectRequest>,
) -> Result<Json<NetworkModeInspection>, StatusCode> {
) -> Result<Json<NetworkModeInspection>, (StatusCode, Json<ApplyResponse>)> {
if login != LoginResult::Admin {
return Err(StatusCode::FORBIDDEN);
return Err(unauthorized());
}
let merged = merge_network_mode(body.config)?;
Ok(Json(run_network_mode_inspection(&state, merged).await?))
let merged = merge_network_mode(body.config).map_err(|error| match error {
MergeNetworkModeError::InvalidCandidate(message) => {
helper_validation_error_response(message)
}
MergeNetworkModeError::LoadLiveConfig => helper_internal_error_response(
"Unable to load the live LibreQoS configuration".to_string(),
),
})?;
run_network_mode_inspection(&state, merged)
.await
.map(Json)
.map_err(|_| helper_internal_error_response("Unable to inspect network mode".to_string()))
}
pub async fn apply(
@@ -178,16 +197,17 @@ pub async fn apply(
let username = get_username(&jar).await;
let merged = match merge_network_mode(body.config) {
Ok(config) => config,
Err(status) => {
return (
status,
Json(ApplyResponse {
ok: false,
message: "Unable to load the live LibreQoS configuration".to_string(),
operation: None,
last_backup_id: None,
}),
);
Err(error) => {
return match error {
MergeNetworkModeError::InvalidCandidate(message) => {
helper_validation_error_response(message)
}
MergeNetworkModeError::LoadLiveConfig => {
helper_internal_error_response(
"Unable to load the live LibreQoS configuration".to_string(),
)
}
};
}
};
let result = tokio::task::spawn_blocking({
@@ -212,7 +232,9 @@ pub async fn apply(
match result {
Ok(Ok(response)) => (StatusCode::OK, Json(response)),
Ok(Err(err)) => helper_validation_error_response(err.to_string()),
Err(err) => helper_internal_error_response(format!("Bridge helper task failed: {err}")),
Err(err) => {
helper_internal_error_response(format!("Network-mode helper task failed: {err}"))
}
}
}
@@ -238,7 +260,9 @@ pub async fn confirm(
match result {
Ok(Ok(response)) => (StatusCode::OK, Json(response)),
Ok(Err(err)) => helper_validation_error_response(err.to_string()),
Err(err) => helper_internal_error_response(format!("Bridge helper task failed: {err}")),
Err(err) => {
helper_internal_error_response(format!("Network-mode helper task failed: {err}"))
}
}
}
@@ -264,7 +288,9 @@ pub async fn revert(
match result {
Ok(Ok(response)) => (StatusCode::OK, Json(response)),
Ok(Err(err)) => helper_validation_error_response(err.to_string()),
Err(err) => helper_internal_error_response(format!("Bridge helper task failed: {err}")),
Err(err) => {
helper_internal_error_response(format!("Network-mode helper task failed: {err}"))
}
}
}
@@ -290,7 +316,9 @@ pub async fn rollback(
match result {
Ok(Ok(response)) => (StatusCode::OK, Json(response)),
Ok(Err(err)) => helper_validation_error_response(err.to_string()),
Err(err) => helper_internal_error_response(format!("Bridge helper task failed: {err}")),
Err(err) => {
helper_internal_error_response(format!("Network-mode helper task failed: {err}"))
}
}
}
@@ -314,6 +342,88 @@ pub async fn retry_shaping(
match result {
Ok(Ok(response)) => (StatusCode::OK, Json(response)),
Ok(Err(err)) => helper_internal_error_response(err.to_string()),
Err(err) => helper_internal_error_response(format!("Bridge helper task failed: {err}")),
Err(err) => {
helper_internal_error_response(format!("Network-mode helper task failed: {err}"))
}
}
}
#[cfg(test)]
mod tests {
use super::{MergeNetworkModeError, merge_network_mode};
use lqos_config::{BridgeConfig, Config, SingleInterfaceConfig};
use std::sync::{Mutex, OnceLock};
fn env_lock() -> &'static Mutex<()> {
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| Mutex::new(()))
}
fn with_config_env<T>(test_fn: impl FnOnce() -> T) -> T {
let _guard = env_lock().lock().expect("network-mode env lock");
let old_lqos_config = std::env::var_os("LQOS_CONFIG");
let config_path = std::env::temp_dir().join(format!(
"lqosd-network-mode-test-{}.toml",
std::process::id()
));
let raw = include_str!("../../../../lqos_config/src/etc/v15/example.toml");
std::fs::write(&config_path, raw).expect("write config");
lqos_config::clear_cached_config();
unsafe {
std::env::set_var("LQOS_CONFIG", &config_path);
}
let result = test_fn();
match old_lqos_config {
Some(value) => unsafe {
std::env::set_var("LQOS_CONFIG", value);
},
None => unsafe {
std::env::remove_var("LQOS_CONFIG");
},
}
lqos_config::clear_cached_config();
let _ = std::fs::remove_file(config_path);
result
}
#[test]
fn merge_network_mode_rejects_invalid_bridge_mtu() {
let mut candidate = Config::default();
candidate.bridge = Some(BridgeConfig {
mtu: Some(9217),
..BridgeConfig::default()
});
let error =
with_config_env(|| merge_network_mode(candidate).expect_err("invalid MTU should fail"));
assert_eq!(
error,
MergeNetworkModeError::InvalidCandidate(
"bridge.mtu must be between 576 and 9216".to_string()
)
);
}
#[test]
fn merge_network_mode_rejects_invalid_single_interface_mtu() {
let mut candidate = Config::default();
candidate.bridge = None;
candidate.single_interface = Some(SingleInterfaceConfig {
mtu: Some(575),
..SingleInterfaceConfig::default()
});
let error =
with_config_env(|| merge_network_mode(candidate).expect_err("invalid MTU should fail"));
assert_eq!(
error,
MergeNetworkModeError::InvalidCandidate(
"single_interface.mtu must be between 576 and 9216".to_string()
)
);
}
}
@@ -4,8 +4,8 @@ use lqos_bus::SchedulerProgressReport;
use crate::node_manager::runtime_onboarding::runtime_onboarding_state;
use crate::tool_status::{
is_scheduler_available, scheduler_error_message, scheduler_output_message,
scheduler_progress_state,
is_scheduler_available, scheduler_error_message, scheduler_last_seen_unix,
scheduler_output_message, scheduler_progress_state,
};
// Remove ANSI escape sequences (basic CSI/OSC handling) for browser display
@@ -64,23 +64,45 @@ fn strip_ansi(input: &str) -> String {
out
}
/// Snapshot of scheduler availability for the node-manager widget.
#[derive(Serialize, Debug, Clone)]
pub struct SchedulerStatus {
/// Whether the scheduler has reported in within the last five minutes.
pub available: bool,
/// The latest scheduler error, if one was reported.
pub error: Option<String>,
/// The scheduler's last reported progress snapshot, if any.
pub progress: Option<SchedulerProgressReport>,
/// The best-known Unix timestamp for this scheduler snapshot.
pub updated_unix: Option<u64>,
/// Whether this snapshot is older than the scheduler availability window.
pub stale: bool,
/// Whether initial runtime onboarding is blocking scheduler activity.
pub setup_required: bool,
/// Operator-facing onboarding guidance when `setup_required` is true.
pub setup_message: Option<String>,
}
/// Detailed scheduler snapshot for the scheduler modal.
#[derive(Serialize, Debug, Clone)]
pub struct SchedulerDetails {
/// Whether the scheduler has reported in within the last five minutes.
pub available: bool,
/// The latest scheduler error, if one was reported.
pub error: Option<String>,
/// The latest non-error scheduler output line, if any.
pub output: Option<String>,
/// The scheduler's last reported progress snapshot, if any.
pub progress: Option<SchedulerProgressReport>,
/// The best-known Unix timestamp for this scheduler snapshot.
pub updated_unix: Option<u64>,
/// Whether this snapshot is older than the scheduler availability window.
pub stale: bool,
/// Whether initial runtime onboarding is blocking scheduler activity.
pub setup_required: bool,
/// Operator-facing onboarding guidance when `setup_required` is true.
pub setup_message: Option<String>,
/// Human-readable raw details for the operator debug view.
pub details: String,
}
@@ -106,6 +128,19 @@ fn scheduler_output() -> Option<String> {
})
}
fn scheduler_updated_unix(progress: Option<&SchedulerProgressReport>) -> Option<u64> {
let progress_updated = progress.and_then(|report| report.updated_unix);
match (progress_updated, scheduler_last_seen_unix()) {
(Some(progress_updated), Some(last_seen_updated)) => {
Some(progress_updated.max(last_seen_updated))
}
(Some(progress_updated), None) => Some(progress_updated),
(None, Some(last_seen_updated)) => Some(last_seen_updated),
(None, None) => None,
}
}
/// Builds the lightweight scheduler status snapshot for node-manager consumers.
pub fn scheduler_status_data() -> SchedulerStatus {
let onboarding = runtime_onboarding_state();
if onboarding.required {
@@ -113,6 +148,8 @@ pub fn scheduler_status_data() -> SchedulerStatus {
available: false,
error: None,
progress: None,
updated_unix: None,
stale: false,
setup_required: true,
setup_message: Some(onboarding.summary),
};
@@ -121,15 +158,19 @@ pub fn scheduler_status_data() -> SchedulerStatus {
let available = is_scheduler_available();
let error = scheduler_error();
let progress = scheduler_progress_state();
let updated_unix = scheduler_updated_unix(progress.as_ref());
SchedulerStatus {
available,
error,
progress,
updated_unix,
stale: updated_unix.is_some() && !available,
setup_required: false,
setup_message: None,
}
}
/// Builds the detailed scheduler snapshot for the scheduler modal.
pub fn scheduler_details_data() -> SchedulerDetails {
let status = scheduler_status_data();
let output = scheduler_output();
@@ -148,6 +189,8 @@ pub fn scheduler_details_data() -> SchedulerDetails {
error: None,
output: None,
progress: None,
updated_unix: None,
stale: false,
setup_required: true,
setup_message: Some(message),
details: body,
@@ -155,6 +198,12 @@ pub fn scheduler_details_data() -> SchedulerDetails {
}
body.push_str(&format!("Scheduler available: {}\n\n", status.available));
if let Some(updated_unix) = status.updated_unix {
body.push_str(&format!("Last updated Unix: {updated_unix}\n"));
} else {
body.push_str("Last updated Unix: unavailable\n");
}
body.push_str(&format!("Snapshot stale: {}\n\n", status.stale));
match status.progress.as_ref() {
Some(progress) => {
body.push_str("Current progress:\n");
@@ -203,6 +252,8 @@ pub fn scheduler_details_data() -> SchedulerDetails {
error: status.error,
output,
progress: status.progress,
updated_unix: status.updated_unix,
stale: status.stale,
setup_required: false,
setup_message: None,
details: body,
@@ -214,7 +265,8 @@ mod tests {
use super::{scheduler_details_data, scheduler_status_data};
use crate::test_support::runtime_config_test_lock;
use crate::tool_status::{
scheduler_error, scheduler_output, scheduler_progress, scheduler_seen,
scheduler_error, scheduler_last_seen_unix, scheduler_output, scheduler_progress,
scheduler_seen,
};
use lqos_bus::SchedulerProgressReport;
use std::ffi::OsString;
@@ -324,6 +376,9 @@ mod tests {
let status = scheduler_status_data();
let details = scheduler_details_data();
let expected_updated_unix = scheduler_last_seen_unix()
.map(|last_seen_unix| last_seen_unix.max(1_234_567_890))
.expect("scheduler last seen should be set");
assert!(status.available);
assert_eq!(
@@ -337,6 +392,8 @@ mod tests {
.map(|progress| progress.phase.as_str()),
Some("validation_failed")
);
assert_eq!(status.updated_unix, Some(expected_updated_unix));
assert!(!status.stale);
assert!(!status.setup_required);
assert!(details.available);
@@ -348,7 +405,29 @@ mod tests {
details.output.as_deref(),
Some("Scheduled shaping refresh blocked by validation: duplicate IPv4")
);
assert_eq!(details.updated_unix, Some(expected_updated_unix));
assert!(!details.stale);
assert!(details.details.contains("Reported error:"));
assert!(details.details.contains("Snapshot stale: false"));
assert!(details.details.contains("Scheduler validation failed"));
}
#[test]
fn scheduler_status_uses_last_seen_when_progress_timestamp_is_missing() {
let _context = SchedulerStatusTestContext::new("last-seen-fallback");
scheduler_seen();
let status = scheduler_status_data();
let details = scheduler_details_data();
let last_seen_unix = scheduler_last_seen_unix().expect("scheduler last seen should be set");
assert_eq!(status.updated_unix, Some(last_seen_unix));
assert!(!status.stale);
assert_eq!(details.updated_unix, Some(last_seen_unix));
assert!(
details
.details
.contains(&format!("Last updated Unix: {last_seen_unix}"))
);
}
}
@@ -1,5 +1,9 @@
use crate::shaped_devices_tracker::circuit_live::{
CircuitLiveRollup, CircuitLiveSnapshot, fresh_circuit_live_snapshot,
};
use lqos_config::ShapedDevice;
use serde::{Deserialize, Deserializer, Serialize};
use std::cmp::Ordering;
use std::collections::HashSet;
const DEFAULT_SHAPED_DEVICES_PAGE_SIZE: usize = 24;
@@ -57,7 +61,49 @@ fn normalized_page_size(query: &ShapedDevicesPageQuery) -> usize {
.clamp(1, MAX_SHAPED_DEVICES_PAGE_SIZE)
}
fn alphabetic_inventory_order(left: &ShapedDevice, right: &ShapedDevice) -> Ordering {
left.circuit_name
.cmp(&right.circuit_name)
.then_with(|| left.device_name.cmp(&right.device_name))
.then_with(|| left.device_id.cmp(&right.device_id))
}
fn live_sort_rank(snapshot: &CircuitLiveSnapshot, device: &ShapedDevice) -> (u64, u64, u64, u64) {
let live = snapshot.by_circuit_id.get(&device.circuit_id);
let bytes_per_second = live
.map(|row: &CircuitLiveRollup| row.bytes_per_second)
.unwrap_or_default();
let total_bytes_per_second = bytes_per_second.down.saturating_add(bytes_per_second.up);
let last_seen_nanos = live.map(|row| row.last_seen_nanos).unwrap_or(u64::MAX);
(
total_bytes_per_second,
bytes_per_second.down,
bytes_per_second.up,
last_seen_nanos,
)
}
fn dynamic_inventory_order(
snapshot: &CircuitLiveSnapshot,
left: &ShapedDevice,
right: &ShapedDevice,
) -> Ordering {
let left_rank = live_sort_rank(snapshot, left);
let right_rank = live_sort_rank(snapshot, right);
right_rank
.0
.cmp(&left_rank.0)
.then_with(|| right_rank.1.cmp(&left_rank.1))
.then_with(|| right_rank.2.cmp(&left_rank.2))
.then_with(|| left_rank.3.cmp(&right_rank.3))
.then_with(|| alphabetic_inventory_order(left, right))
}
/// Returns one filtered, sorted page of shaped-device rows.
///
/// Static inventory rows are ordered alphabetically. Dynamic inventory rows are
/// ordered by current observed throughput before pagination so the busiest
/// circuits stay at the top of the page.
pub fn shaped_devices_page(query: ShapedDevicesPageQuery) -> ShapedDevicesPage {
let page = query.page.unwrap_or(0);
let page_size = normalized_page_size(&query);
@@ -103,12 +149,13 @@ pub fn shaped_devices_page(query: ShapedDevicesPageQuery) -> ShapedDevicesPage {
.cloned()
.collect(),
};
filtered.sort_by(|left, right| {
left.circuit_name
.cmp(&right.circuit_name)
.then_with(|| left.device_name.cmp(&right.device_name))
.then_with(|| left.device_id.cmp(&right.device_id))
});
match kind {
ShapedDevicesPageKind::Static => filtered.sort_by(alphabetic_inventory_order),
ShapedDevicesPageKind::Dynamic => {
let snapshot = fresh_circuit_live_snapshot();
filtered.sort_by(|left, right| dynamic_inventory_order(snapshot.as_ref(), left, right));
}
}
let total_rows = filtered.len();
let total_circuits = filtered
@@ -140,3 +187,84 @@ pub fn shaped_devices_page(query: ShapedDevicesPageQuery) -> ShapedDevicesPage {
rows,
}
}
#[cfg(test)]
mod tests {
use super::*;
use fxhash::FxHashMap;
use lqos_utils::units::DownUpOrder;
fn test_device(circuit_id: &str, circuit_name: &str, device_name: &str) -> ShapedDevice {
ShapedDevice {
circuit_id: circuit_id.to_string(),
circuit_name: circuit_name.to_string(),
device_id: format!("{circuit_id}-device"),
device_name: device_name.to_string(),
..Default::default()
}
}
fn test_snapshot(entries: &[(&str, u64, u64, u64)]) -> CircuitLiveSnapshot {
let mut by_circuit_id = FxHashMap::default();
for (circuit_id, down, up, last_seen_nanos) in entries {
by_circuit_id.insert(
(*circuit_id).to_string(),
CircuitLiveRollup {
circuit_id: (*circuit_id).to_string(),
circuit_name: String::new(),
parent_node: String::new(),
device_names: Vec::new(),
ip_addrs: Vec::new(),
plan_mbps: DownUpOrder::default(),
bytes_per_second: DownUpOrder::new(*down, *up),
rtt_current_p50_nanos: DownUpOrder::default(),
qoo: DownUpOrder::default(),
tcp_retransmit_sample: DownUpOrder::default(),
last_seen_nanos: *last_seen_nanos,
},
);
}
CircuitLiveSnapshot { by_circuit_id }
}
#[test]
fn dynamic_inventory_order_prefers_higher_combined_throughput() {
let snapshot = test_snapshot(&[
("slow", 10, 20, 3_000),
("fast", 40, 90, 2_000),
("idle", 0, 0, u64::MAX),
]);
let mut rows = [
test_device("slow", "Slow Circuit", "Slow Device"),
test_device("idle", "Idle Circuit", "Idle Device"),
test_device("fast", "Fast Circuit", "Fast Device"),
];
rows.sort_by(|left, right| dynamic_inventory_order(&snapshot, left, right));
assert_eq!(
rows.iter()
.map(|row| row.circuit_id.as_str())
.collect::<Vec<_>>(),
vec!["fast", "slow", "idle"]
);
}
#[test]
fn dynamic_inventory_order_uses_alphabetic_tie_breakers() {
let snapshot = test_snapshot(&[("alpha", 25, 25, 100), ("beta", 25, 25, 100)]);
let mut rows = [
test_device("beta", "Beta Circuit", "Second Device"),
test_device("alpha", "Alpha Circuit", "First Device"),
];
rows.sort_by(|left, right| dynamic_inventory_order(&snapshot, left, right));
assert_eq!(
rows.iter()
.map(|row| row.circuit_id.as_str())
.collect::<Vec<_>>(),
vec!["alpha", "beta"]
);
}
}
@@ -73,6 +73,7 @@ pub struct UrgentItem {
pub code: String,
pub message: String,
pub context: Option<String>,
pub clearable: bool,
}
#[derive(Debug, Serialize)]
@@ -80,17 +81,32 @@ pub struct UrgentList {
pub items: Vec<UrgentItem>,
}
pub fn urgent_status_data() -> UrgentStatus {
let items = urgent::list();
UrgentStatus {
has_urgent: !items.is_empty(),
count: items.len(),
fn bakery_passthrough_issue() -> Option<UrgentItem> {
let status = lqos_bakery::bakery_status_snapshot();
if !status.reload_required || !status.passthrough_degraded {
return None;
}
Some(UrgentItem {
id: u64::MAX,
ts: status
.last_failure_unix
.or(status.current_action_started_unix)
.unwrap_or_default(),
source: "System".to_string(),
severity: "Error".to_string(),
code: "BAKERY_PASSTHROUGH_DEGRADED".to_string(),
message: status.passthrough_degraded_reason.unwrap_or_else(|| {
"Bakery full reload failed while pass-through mode was active. Traffic may be unshaped until a successful full reload clears the degraded state.".to_string()
}),
context: status.reload_required_reason.map(|reason| {
format!("Reload required reason: {reason}")
}),
clearable: false,
})
}
pub fn urgent_list_data() -> UrgentList {
let items = urgent::list();
let items: Vec<UrgentItem> = items
fn merged_urgent_items() -> Vec<UrgentItem> {
let mut items: Vec<UrgentItem> = urgent::list()
.into_iter()
.map(|i| UrgentItem {
id: i.id,
@@ -100,9 +116,29 @@ pub fn urgent_list_data() -> UrgentList {
code: i.code,
message: strip_ansi(&i.message),
context: i.context,
clearable: true,
})
.collect();
UrgentList { items }
if let Some(issue) = bakery_passthrough_issue() {
items.push(issue);
}
items.sort_by_key(|item| item.ts);
items.reverse();
items
}
pub fn urgent_status_data() -> UrgentStatus {
let items = merged_urgent_items();
UrgentStatus {
has_urgent: !items.is_empty(),
count: items.len(),
}
}
pub fn urgent_list_data() -> UrgentList {
UrgentList {
items: merged_urgent_items(),
}
}
pub fn urgent_clear_id(id: u64) -> bool {
@@ -9,8 +9,11 @@ use crate::node_manager::shaper_queries_actor::ShaperQueryCommand;
use crate::node_manager::shaper_queries_actor::caches::Caches;
use std::time::Duration;
use tokio::sync::oneshot;
use tokio::time::timeout;
use tracing::{info, warn};
const HISTORY_QUERY_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30);
#[derive(Clone)]
struct HistoryClient {
control_tx: tokio::sync::mpsc::Sender<ControlChannelCommand>,
@@ -26,21 +29,30 @@ impl HistoryClient {
request: RemoteInsightRequest,
) -> Result<HistoryQueryResultPayload, ()> {
let (tx, rx) = oneshot::channel();
if self
.control_tx
.send(ControlChannelCommand::FetchHistory {
let send_result = timeout(
HISTORY_QUERY_ATTEMPT_TIMEOUT,
self.control_tx.send(ControlChannelCommand::FetchHistory {
request,
responder: tx,
})
.await
.is_err()
{
return Err(());
}),
)
.await;
match send_result {
Ok(Ok(())) => {}
Ok(Err(_)) => return Err(()),
Err(_) => {
warn!("Timed out queueing Insight history request");
return Err(());
}
}
match rx.await {
Ok(result) => result,
Err(_) => Err(()),
match timeout(HISTORY_QUERY_ATTEMPT_TIMEOUT, rx).await {
Ok(Ok(result)) => result,
Ok(Err(_)) => Err(()),
Err(_) => {
warn!("Timed out waiting for Insight history response");
Err(())
}
}
}
}
@@ -89,6 +101,9 @@ macro_rules! shaper_query {
break;
}
Err(()) => {
if my_reply.is_closed() {
break;
}
attempts += 1;
if attempts >= 3 {
warn!("History request exhausted retries");
@@ -55,12 +55,12 @@
<section class="lqos-config-panel">
<div class="lqos-config-section">
<h6 class="lqos-config-section-title">Bridge Mode Configuration</h6>
<div class="lqos-config-section-subtitle">Choose the two shaping interfaces for the managed Linux bridge flow, or leave XDP enabled if you are intentionally handling Netplan outside this page.</div>
<div class="lqos-config-section-subtitle">Choose the two shaping interfaces. Linux bridge mode can manage Netplan; XDP mode leaves Netplan alone.</div>
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="useXdpBridge">
<input type="checkbox" class="form-check-input" id="useXdpBridge" aria-describedby="useXdpBridgeHelp">
<label class="form-check-label" for="useXdpBridge">Use XDP Bridge</label>
<div class="form-text">Use the XDP bridge instead of the Linux Bridge. This remains a manual/power-user netplan workflow.</div>
<div id="useXdpBridgeHelp" class="form-text">XDP does not write Netplan or apply Bridge MTU.</div>
</div>
<div class="mb-3">
@@ -78,6 +78,13 @@
</select>
<div class="form-text">Pick a different eligible interface for the LAN side.</div>
</div>
<div class="mb-2">
<label for="bridgeMtu" class="form-label">Bridge MTU</label>
<input type="number" class="form-control" id="bridgeMtu" aria-describedby="bridgeMtuHelp bridgeMtuError">
<div id="bridgeMtuHelp" class="form-text">Applies to the bridge members and br0.</div>
<div id="bridgeMtuError" class="invalid-feedback" role="alert"></div>
</div>
<div id="bridgeInterfaceHelp" class="small text-secondary"></div>
</div>
</section>
@@ -87,7 +94,7 @@
<section class="lqos-config-panel">
<div class="lqos-config-section">
<h6 class="lqos-config-section-title">Single Interface Mode Configuration</h6>
<div class="lqos-config-section-subtitle">Pick the trunk interface and define the WAN and LAN VLAN IDs that LibreQoS should prepare in managed Netplan.</div>
<div class="lqos-config-section-subtitle">Pick the trunk interface and VLAN IDs for managed Netplan.</div>
<div class="mb-3">
<label for="interface" class="form-label">Interface Name</label>
@@ -108,6 +115,13 @@
<input type="number" class="form-control" id="networkVlan" min="1" max="4094">
<div class="form-text">VLAN ID for LAN traffic.</div>
</div>
<div class="mb-0 mt-3">
<label for="singleInterfaceMtu" class="form-label">Single Interface MTU</label>
<input type="number" class="form-control" id="singleInterfaceMtu" aria-describedby="singleInterfaceMtuHelp singleInterfaceMtuError">
<div id="singleInterfaceMtuHelp" class="form-text">Applies to the trunk interface.</div>
<div id="singleInterfaceMtuError" class="invalid-feedback" role="alert"></div>
</div>
<div id="singleInterfaceHelp" class="small text-secondary mt-2"></div>
</div>
</section>
@@ -39,9 +39,9 @@
</div>
<div class="mb-3">
<label for="uispSite" class="form-label">UISP Site</label>
<label for="uispSite" class="form-label" id="uispSiteLabel">Root UISP Site</label>
<input type="text" class="form-control" id="uispSite">
<div class="form-text">Optional site identifier in UISP.</div>
<div class="form-text" id="uispSiteHelp">Required only for full topology mode. Leave blank in other modes to let LibreQoS choose the import root.</div>
</div>
<div class="mb-3 form-check">
@@ -1567,6 +1567,74 @@ h5 {
max-width: 100%;
}
.lqos-circuit-data-table th,
.lqos-circuit-data-table td {
--lqos-circuit-group-tint: transparent;
--lqos-circuit-hover-tint: transparent;
background-image:
linear-gradient(var(--lqos-circuit-hover-tint), var(--lqos-circuit-hover-tint)),
linear-gradient(var(--lqos-circuit-group-tint), var(--lqos-circuit-group-tint));
background-clip: padding-box;
transition:
background-color 140ms ease,
border-color 140ms ease,
box-shadow 140ms ease;
}
.lqos-circuit-data-table thead th + th,
.lqos-circuit-data-table tbody td + td {
border-left: 1px solid color-mix(in srgb, var(--lqos-border) 42%, transparent);
}
.lqos-circuit-data-table .lqos-circuit-col-start {
border-left-color: color-mix(in srgb, var(--lqos-border) 92%, transparent);
}
.lqos-circuit-data-table .lqos-circuit-colgroup-identity,
.lqos-circuit-data-table .lqos-circuit-colgroup-actions {
--lqos-circuit-group-tint: color-mix(in srgb, var(--lqos-surface) 14%, transparent);
}
.lqos-circuit-data-table .lqos-circuit-colgroup-rates {
--lqos-circuit-group-tint: color-mix(in srgb, var(--bs-info) 7%, transparent);
}
.lqos-circuit-data-table .lqos-circuit-colgroup-volume {
--lqos-circuit-group-tint: color-mix(in srgb, #d39f37 7%, transparent);
}
.lqos-circuit-data-table .lqos-circuit-colgroup-quality {
--lqos-circuit-group-tint: color-mix(in srgb, #2b8a57 7%, transparent);
}
.lqos-circuit-data-table .lqos-circuit-colgroup-attribution,
.lqos-circuit-data-table .lqos-circuit-colgroup-flows {
--lqos-circuit-group-tint: color-mix(in srgb, var(--bs-secondary-color) 8%, transparent);
}
.lqos-circuit-data-table .lqos-circuit-cell-numeric {
text-align: right;
font-variant-numeric: tabular-nums;
}
.lqos-circuit-data-table .lqos-circuit-cell-action {
text-align: center;
}
.lqos-circuit-data-table .is-column-hover {
--lqos-circuit-hover-tint: rgba(255, 255, 255, 0.035);
box-shadow:
inset 0 1px 0 rgba(255, 255, 255, 0.05),
inset 0 0 0 1px color-mix(in srgb, var(--lqos-border) 70%, transparent);
}
[data-bs-theme="light"] .lqos-circuit-data-table .is-column-hover {
--lqos-circuit-hover-tint: rgba(17, 29, 45, 0.045);
box-shadow:
inset 0 1px 0 rgba(255, 255, 255, 0.28),
inset 0 0 0 1px color-mix(in srgb, var(--lqos-border) 74%, transparent);
}
.lqos-tree-stack {
display: flex;
flex-direction: column;
@@ -2433,6 +2501,41 @@ table tr td a {
background: var(--lqos-surface-solid);
box-shadow: var(--lqos-shadow-sm);
}
.lqos-shaped-devices-view-toggle .btn {
border-color: color-mix(in srgb, var(--lqos-border) 80%, transparent);
}
.lqos-shaped-devices-view-toggle .btn.active,
.lqos-shaped-devices-view-toggle .btn:active {
color: var(--bs-emphasis-color);
border-color: color-mix(in srgb, var(--lqos-border) 72%, transparent);
background: color-mix(in srgb, var(--lqos-surface) 86%, transparent);
box-shadow:
inset 0 1px 0 rgba(255, 255, 255, 0.08),
0 0 0 1px color-mix(in srgb, var(--lqos-border) 30%, transparent);
}
.lqos-shaped-devices-list-table td {
vertical-align: middle;
}
.lqos-shaped-devices-list-table td .badge {
white-space: nowrap;
}
.lqos-direction-metric {
display: flex;
flex-direction: column;
gap: 0.15rem;
min-width: 7rem;
}
.lqos-direction-metric-line {
display: flex;
align-items: center;
justify-content: space-between;
gap: 0.75rem;
}
.lqos-direction-metric-label {
font-size: 0.74rem;
font-weight: 600;
color: var(--bs-secondary-color);
}
.exec-icon {
display: inline-flex;
align-items: center;
+5 -4
View File
@@ -287,10 +287,11 @@ mod tests {
}
fn test_config(runtime_dir: &std::path::Path, display_cobrand: bool) -> Config {
let mut config = Config::default();
config.lqos_directory = runtime_dir.display().to_string();
config.display_cobrand = display_cobrand;
config
Config {
lqos_directory: runtime_dir.display().to_string(),
display_cobrand,
..Default::default()
}
}
#[test]
+267 -69
View File
@@ -7,7 +7,9 @@
use std::collections::{BTreeMap, HashSet};
use std::sync::Arc;
use std::time::Duration;
use crate::lts2_sys::control_channel::ControlChannelCommand;
use crate::node_manager::auth::{LoginResult, login_from_token};
use crate::node_manager::local_api::{
circuit, circuit_count, config, cpu_affinity, dashboard_themes, device_counts, directories,
@@ -55,6 +57,22 @@ mod ticker;
const WS_VERSION: &str = include_str!("../../../../VERSION_STRING");
const HANDSHAKE_TIMEOUT_SECS: u64 = 10;
const CONTROL_CHANNEL_SEND_TIMEOUT: Duration = Duration::from_secs(5);
const REQUEST_TIMEOUT_SECS: u64 = 15;
async fn send_control_command(
control_tx: &tokio::sync::mpsc::Sender<ControlChannelCommand>,
command: ControlChannelCommand,
) -> bool {
match tokio::time::timeout(CONTROL_CHANNEL_SEND_TIMEOUT, control_tx.send(command)).await {
Ok(Ok(())) => true,
Ok(Err(_)) => false,
Err(_) => {
warn!("Timed out queueing websocket control-channel command");
false
}
}
}
/// Provides an Axum router for the websocket system. Exposes a single /ws route that supports
/// pubsub subscriptions and private commands.
@@ -182,19 +200,29 @@ async fn handle_socket(
// Received a websocket message
match inbound {
Some(Ok(msg)) => {
let should_close = receive_channel_message(
msg,
channels.clone(),
tx.clone(),
&mut subscribed_channels,
&mut handshake_complete,
&mut WsRequestState {
private_state: &mut private_state,
login: &mut login,
shaper_query: shaper_query.clone(),
},
let should_close = match tokio::time::timeout(
std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS),
receive_channel_message(
msg,
channels.clone(),
tx.clone(),
&mut subscribed_channels,
&mut handshake_complete,
&mut WsRequestState {
private_state: &mut private_state,
login: &mut login,
shaper_query: shaper_query.clone(),
},
),
)
.await;
.await
{
Ok(should_close) => should_close,
Err(_) => {
warn!("Websocket request timed out; closing connection");
true
}
};
if should_close {
break;
}
@@ -212,6 +240,11 @@ async fn handle_socket(
}
}
}
for channel in subscribed_channels.drain() {
channels.unsubscribe(channel, tx.clone()).await;
}
drop(private_state);
drop(tx);
outbound_handle.abort();
let _ = outbound_handle.await;
info!("Websocket disconnected");
@@ -660,32 +693,37 @@ async fn receive_channel_message(
return true;
}
}
WsRequest::UrgentStatus => {
WsRequest::UrgentStatus { request_id } => {
let response = WsResponse::UrgentStatus {
data: urgent::urgent_status_data(),
request_id,
};
if send_ws_response(&tx, response).await {
return true;
}
}
WsRequest::UrgentList => {
WsRequest::UrgentList { request_id } => {
let response = WsResponse::UrgentList {
data: urgent::urgent_list_data(),
request_id,
};
if send_ws_response(&tx, response).await {
return true;
}
}
WsRequest::UrgentClear { id } => {
WsRequest::UrgentClear { id, request_id } => {
let ok = urgent::urgent_clear_id(id);
let response = WsResponse::UrgentClearResult { ok };
let response = WsResponse::UrgentClearResult { ok, request_id };
if send_ws_response(&tx, response).await {
return true;
}
}
WsRequest::UrgentClearAll => {
WsRequest::UrgentClearAll { request_id } => {
urgent::urgent_clear_all_data();
let response = WsResponse::UrgentClearAllResult { ok: true };
let response = WsResponse::UrgentClearAllResult {
ok: true,
request_id,
};
if send_ws_response(&tx, response).await {
return true;
}
@@ -742,14 +780,13 @@ async fn receive_channel_message(
_ => {
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
let control_tx = request_state.private_state.control_tx();
if control_tx
.send(
crate::lts2_sys::control_channel::ControlChannelCommand::SupportTicketList {
responder: reply_tx,
},
)
.await
.is_err()
if !send_control_command(
&control_tx,
ControlChannelCommand::SupportTicketList {
responder: reply_tx,
},
)
.await
{
if send_ws_response(
&tx,
@@ -766,8 +803,11 @@ async fn receive_channel_message(
tokio::time::timeout(std::time::Duration::from_secs(30), reply_rx).await;
match result {
Ok(Ok(Ok(tickets))) => {
if send_ws_response(&tx, WsResponse::SupportTicketListResult { tickets })
.await
if send_ws_response(
&tx,
WsResponse::SupportTicketListResult { tickets },
)
.await
{
return true;
}
@@ -816,15 +856,14 @@ async fn receive_channel_message(
_ => {
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
let control_tx = request_state.private_state.control_tx();
if control_tx
.send(
crate::lts2_sys::control_channel::ControlChannelCommand::SupportTicketGet {
ticket_id,
responder: reply_tx,
},
)
.await
.is_err()
if !send_control_command(
&control_tx,
ControlChannelCommand::SupportTicketGet {
ticket_id,
responder: reply_tx,
},
)
.await
{
if send_ws_response(
&tx,
@@ -899,18 +938,17 @@ async fn receive_channel_message(
} else {
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
let control_tx = request_state.private_state.control_tx();
if control_tx
.send(
crate::lts2_sys::control_channel::ControlChannelCommand::SupportTicketCreate {
subject,
priority,
body,
commentor,
responder: reply_tx,
},
)
.await
.is_err()
if !send_control_command(
&control_tx,
ControlChannelCommand::SupportTicketCreate {
subject,
priority,
body,
commentor,
responder: reply_tx,
},
)
.await
{
if send_ws_response(
&tx,
@@ -923,12 +961,15 @@ async fn receive_channel_message(
return true;
}
} else {
let result = tokio::time::timeout(std::time::Duration::from_secs(30), reply_rx)
.await;
let result =
tokio::time::timeout(std::time::Duration::from_secs(30), reply_rx).await;
match result {
Ok(Ok(Ok(ticket))) => {
if send_ws_response(&tx, WsResponse::SupportTicketCreateResult { ticket })
.await
if send_ws_response(
&tx,
WsResponse::SupportTicketCreateResult { ticket },
)
.await
{
return true;
}
@@ -988,18 +1029,17 @@ async fn receive_channel_message(
.unwrap_or(0);
let (reply_tx, reply_rx) = tokio::sync::oneshot::channel();
let control_tx = request_state.private_state.control_tx();
if control_tx
.send(
crate::lts2_sys::control_channel::ControlChannelCommand::SupportTicketAddComment {
ticket_id,
commentor,
body,
date,
responder: reply_tx,
},
)
.await
.is_err()
if !send_control_command(
&control_tx,
ControlChannelCommand::SupportTicketAddComment {
ticket_id,
commentor,
body,
date,
responder: reply_tx,
},
)
.await
{
if send_ws_response(
&tx,
@@ -1012,8 +1052,8 @@ async fn receive_channel_message(
return true;
}
} else {
let result = tokio::time::timeout(std::time::Duration::from_secs(30), reply_rx)
.await;
let result =
tokio::time::timeout(std::time::Duration::from_secs(30), reply_rx).await;
match result {
Ok(Ok(Ok(()))) => {
if send_ws_response(
@@ -1579,8 +1619,7 @@ async fn receive_channel_message(
config::update_lqosd_config_data(*request_state.login, cfg, clear_secrets).await;
let (ok, message) = match result {
Ok(()) => (true, "Ok".to_string()),
Err(StatusCode::FORBIDDEN) => (false, "Unauthorized".to_string()),
Err(_) => (false, "Error".to_string()),
Err(message) => (false, message),
};
let response = WsResponse::UpdateConfigResult { ok, message };
if send_ws_response(&tx, response).await {
@@ -2487,7 +2526,9 @@ fn payload_hint(payload: &[u8]) -> &'static str {
#[cfg(test)]
mod tests {
use super::decode_ws_request;
use crate::node_manager::local_api::urgent::{UrgentList, UrgentStatus};
use crate::node_manager::ws::messages::WsRequest;
use crate::node_manager::ws::messages::{WsResponse, encode_ws_message};
use serde_cbor::Value as CborValue;
use std::collections::BTreeMap;
@@ -2552,6 +2593,163 @@ mod tests {
.expect("valid update payload")
}
fn encode_response_value(response: &WsResponse) -> CborValue {
let payload = encode_ws_message(response).expect("response should encode");
serde_cbor::from_slice(payload.as_ref()).expect("response should decode as CBOR value")
}
fn response_request_id(response: &WsResponse, event_name: &str) -> Option<u64> {
let CborValue::Map(entries) = encode_response_value(response) else {
panic!("response should encode as a CBOR map");
};
let event_key = text("event");
let request_id_key = text("request_id");
assert_eq!(entries.get(&event_key), Some(&text(event_name)));
match entries.get(&request_id_key) {
Some(CborValue::Integer(value)) => {
Some(u64::try_from(*value).expect("request_id is u64"))
}
Some(other) => panic!("unexpected request_id value: {other:?}"),
None => None,
}
}
fn response_has_request_id(response: &WsResponse) -> bool {
let CborValue::Map(entries) = encode_response_value(response) else {
panic!("response should encode as a CBOR map");
};
entries.contains_key(&text("request_id"))
}
#[test]
fn decodes_urgent_status_with_request_id() {
let payload = serde_cbor::to_vec(&map(vec![(
"UrgentStatus",
map(vec![("request_id", integer(42))]),
)]))
.expect("valid urgent status payload");
let request = decode_ws_request(&payload).expect("urgent status request should decode");
match request {
WsRequest::UrgentStatus { request_id } => assert_eq!(request_id, Some(42)),
other => panic!("unexpected request: {other:?}"),
}
}
#[test]
fn decodes_urgent_list_without_request_id() {
let payload = serde_cbor::to_vec(&map(vec![("UrgentList", map(vec![]))]))
.expect("valid urgent list payload");
let request = decode_ws_request(&payload).expect("urgent list request should decode");
match request {
WsRequest::UrgentList { request_id } => assert_eq!(request_id, None),
other => panic!("unexpected request: {other:?}"),
}
}
#[test]
fn decodes_urgent_clear_with_request_id() {
let payload = serde_cbor::to_vec(&map(vec![(
"UrgentClear",
map(vec![("id", integer(7)), ("request_id", integer(42))]),
)]))
.expect("valid urgent clear payload");
let request = decode_ws_request(&payload).expect("urgent clear request should decode");
match request {
WsRequest::UrgentClear { id, request_id } => {
assert_eq!(id, 7);
assert_eq!(request_id, Some(42));
}
other => panic!("unexpected request: {other:?}"),
}
}
#[test]
fn decodes_urgent_clear_all_without_request_id() {
let payload = serde_cbor::to_vec(&map(vec![("UrgentClearAll", map(vec![]))]))
.expect("valid urgent clear all payload");
let request = decode_ws_request(&payload).expect("urgent clear all request should decode");
match request {
WsRequest::UrgentClearAll { request_id } => assert_eq!(request_id, None),
other => panic!("unexpected request: {other:?}"),
}
}
#[test]
fn urgent_responses_echo_request_id() {
assert_eq!(
response_request_id(
&WsResponse::UrgentStatus {
data: UrgentStatus {
has_urgent: true,
count: 1,
},
request_id: Some(42),
},
"UrgentStatus",
),
Some(42),
);
assert_eq!(
response_request_id(
&WsResponse::UrgentList {
data: UrgentList { items: Vec::new() },
request_id: Some(43),
},
"UrgentList",
),
Some(43),
);
assert_eq!(
response_request_id(
&WsResponse::UrgentClearResult {
ok: true,
request_id: Some(44),
},
"UrgentClearResult",
),
Some(44),
);
assert_eq!(
response_request_id(
&WsResponse::UrgentClearAllResult {
ok: true,
request_id: Some(45),
},
"UrgentClearAllResult",
),
Some(45),
);
}
#[test]
fn urgent_responses_omit_missing_request_id() {
assert!(!response_has_request_id(&WsResponse::UrgentStatus {
data: UrgentStatus {
has_urgent: false,
count: 0,
},
request_id: None,
}));
assert!(!response_has_request_id(&WsResponse::UrgentList {
data: UrgentList { items: Vec::new() },
request_id: None,
}));
assert!(!response_has_request_id(&WsResponse::UrgentClearResult {
ok: true,
request_id: None,
}));
assert!(!response_has_request_id(
&WsResponse::UrgentClearAllResult {
ok: true,
request_id: None,
}
));
}
#[test]
fn decodes_create_shaped_device_with_browser_style_ipv4_bytes() {
let payload = create_shaped_device_payload(device_value(
+21 -3
View File
@@ -92,6 +92,7 @@ pub enum PrivateRequest {
StopCircuitWatcher,
StopPingMonitorWatch,
CakeWatcher { circuit: String },
StopCakeWatcher,
Chatbot { browser_ts_ms: Option<f64> },
ChatbotUserInput { text: String },
WatchTreeAttachedCircuits { query: TreeAttachedCircuitsQuery },
@@ -323,12 +324,19 @@ pub enum WsRequest {
ProtocolFlowTimeline {
protocol: String,
},
UrgentStatus,
UrgentList,
UrgentStatus {
request_id: Option<u64>,
},
UrgentList {
request_id: Option<u64>,
},
UrgentClear {
id: u64,
request_id: Option<u64>,
},
UrgentClearAll {
request_id: Option<u64>,
},
UrgentClearAll,
UnknownIps,
UnknownIpsClear,
UnknownIpsCsv,
@@ -447,6 +455,8 @@ pub struct BakeryStatusState {
pub preflight: Option<BakeryPreflightData>,
pub reload_required: bool,
pub reload_required_reason: Option<String>,
pub passthrough_degraded: bool,
pub passthrough_degraded_reason: Option<String>,
pub dirty_subtree_count: usize,
}
@@ -623,15 +633,23 @@ pub enum WsResponse {
},
UrgentStatus {
data: UrgentStatus,
#[serde(skip_serializing_if = "Option::is_none")]
request_id: Option<u64>,
},
UrgentList {
data: UrgentList,
#[serde(skip_serializing_if = "Option::is_none")]
request_id: Option<u64>,
},
UrgentClearResult {
ok: bool,
#[serde(skip_serializing_if = "Option::is_none")]
request_id: Option<u64>,
},
UrgentClearAllResult {
ok: bool,
#[serde(skip_serializing_if = "Option::is_none")]
request_id: Option<u64>,
},
UnknownIps {
data: Vec<UnknownIp>,
@@ -6,6 +6,7 @@ pub(crate) mod flows_by_circuit;
mod ping_monitor;
mod tree_attached_circuits;
use crate::lts2_sys::control_channel::ControlChannelCommand;
use crate::node_manager::ws::messages::{PrivateRequest, WsResponse, encode_ws_message};
use crate::node_manager::ws::single_user_channels::cake_watcher::cake_watcher;
use crate::node_manager::ws::single_user_channels::circuit::circuit_watcher;
@@ -13,22 +14,58 @@ use crate::node_manager::ws::single_user_channels::circuit_metrics::watch_circui
use crate::node_manager::ws::single_user_channels::ping_monitor::ping_monitor;
use crate::node_manager::ws::single_user_channels::tree_attached_circuits::watch_tree_attached_circuits;
use lqos_probe::ProbeClient;
use std::{sync::Arc, time::Duration};
use tokio::spawn;
use tokio::sync::mpsc::Sender;
use tracing::info;
use tokio::sync::mpsc::{Sender, error::TrySendError};
use tokio::time::timeout;
use tracing::{debug, info};
const CONTROL_CHANNEL_SEND_TIMEOUT: Duration = Duration::from_secs(5);
/// Sends a private watcher payload without waiting behind a slow websocket.
pub(super) fn try_send_private_payload(
tx: &Sender<Arc<Vec<u8>>>,
payload: Arc<Vec<u8>>,
channel_name: &'static str,
) -> bool {
match tx.try_send(payload) {
Ok(()) => true,
Err(TrySendError::Full(_)) => {
debug!("{channel_name} outbound queue full; stopping watcher");
false
}
Err(TrySendError::Closed(_)) => {
debug!("{channel_name} channel closed");
false
}
}
}
async fn send_control_command(
control_tx: &tokio::sync::mpsc::Sender<ControlChannelCommand>,
command: ControlChannelCommand,
context: &'static str,
) {
match timeout(CONTROL_CHANNEL_SEND_TIMEOUT, control_tx.send(command)).await {
Ok(Ok(())) => {}
Ok(Err(_)) => debug!("{context} control channel is closed"),
Err(_) => debug!("{context} timed out queueing control channel command"),
}
}
pub struct PrivateState {
tx: Sender<std::sync::Arc<Vec<u8>>>,
tx: Sender<Arc<Vec<u8>>>,
bus_tx: Sender<(
tokio::sync::oneshot::Sender<lqos_bus::BusReply>,
lqos_bus::BusRequest,
)>,
control_tx: tokio::sync::mpsc::Sender<crate::lts2_sys::control_channel::ControlChannelCommand>,
control_tx: tokio::sync::mpsc::Sender<ControlChannelCommand>,
probe_client: ProbeClient,
browser_language: Option<String>,
chatbot_request: Option<u64>,
circuit_watch: Option<tokio::task::JoinHandle<()>>,
ping_monitor_watch: Option<tokio::task::JoinHandle<()>>,
cake_watch: Option<tokio::task::JoinHandle<()>>,
tree_attached_circuits_watch: Option<tokio::task::JoinHandle<()>>,
circuit_metrics_watch: Option<tokio::task::JoinHandle<()>>,
}
@@ -40,9 +77,7 @@ impl PrivateState {
tokio::sync::oneshot::Sender<lqos_bus::BusReply>,
lqos_bus::BusRequest,
)>,
control_tx: tokio::sync::mpsc::Sender<
crate::lts2_sys::control_channel::ControlChannelCommand,
>,
control_tx: tokio::sync::mpsc::Sender<ControlChannelCommand>,
probe_client: ProbeClient,
browser_language: Option<String>,
) -> Self {
@@ -55,14 +90,13 @@ impl PrivateState {
chatbot_request: None,
circuit_watch: None,
ping_monitor_watch: None,
cake_watch: None,
tree_attached_circuits_watch: None,
circuit_metrics_watch: None,
}
}
pub fn control_tx(
&self,
) -> tokio::sync::mpsc::Sender<crate::lts2_sys::control_channel::ControlChannelCommand> {
pub fn control_tx(&self) -> tokio::sync::mpsc::Sender<ControlChannelCommand> {
self.control_tx.clone()
}
@@ -90,7 +124,10 @@ impl PrivateState {
self.abort_ping_monitor_watch();
}
PrivateRequest::CakeWatcher { circuit } => {
spawn(cake_watcher(circuit, self.tx.clone()));
self.replace_cake_watch(circuit);
}
PrivateRequest::StopCakeWatcher => {
self.abort_cake_watch();
}
PrivateRequest::Chatbot { browser_ts_ms } => {
self.start_chatbot(normalize_browser_ts_ms(browser_ts_ms))
@@ -144,6 +181,17 @@ impl PrivateState {
}
}
fn replace_cake_watch(&mut self, circuit: String) {
self.abort_cake_watch();
self.cake_watch = Some(spawn(cake_watcher(circuit, self.tx.clone())));
}
fn abort_cake_watch(&mut self) {
if let Some(handle) = self.cake_watch.take() {
handle.abort();
}
}
fn replace_tree_attached_circuits_watch(
&mut self,
query: crate::node_manager::local_api::tree_attached_circuits::TreeAttachedCircuitsQuery,
@@ -189,7 +237,7 @@ impl PrivateState {
message: message.to_string(),
};
if let Ok(payload) = encode_ws_message(&response) {
let _ = self.tx.send(payload).await;
let _ = try_send_private_payload(&self.tx, payload, "Chatbot");
}
return;
}
@@ -204,7 +252,7 @@ impl PrivateState {
let text = String::from_utf8_lossy(&chunk).to_string();
let response = WsResponse::ChatbotChunk { text };
if let Ok(payload) = encode_ws_message(&response) {
if to_client.send(payload).await.is_err() {
if !try_send_private_payload(&to_client, payload, "Chatbot") {
break;
}
} else {
@@ -213,17 +261,17 @@ impl PrivateState {
}
});
let _ = self
.control_tx
.send(
crate::lts2_sys::control_channel::ControlChannelCommand::StartChat {
request_id,
browser_ts_ms,
browser_language: self.browser_language.clone(),
stream: stream_tx,
},
)
.await;
send_control_command(
&self.control_tx,
ControlChannelCommand::StartChat {
request_id,
browser_ts_ms,
browser_language: self.browser_language.clone(),
stream: stream_tx,
},
"StartChat",
)
.await;
info!(
"[chatbot] starting session request_id={} browser_ts_ms={:?}",
request_id, browser_ts_ms
@@ -234,15 +282,12 @@ impl PrivateState {
let Some(request_id) = self.chatbot_request else {
return;
};
let _ = self
.control_tx
.send(
crate::lts2_sys::control_channel::ControlChannelCommand::ChatSend {
request_id,
text,
},
)
.await;
send_control_command(
&self.control_tx,
ControlChannelCommand::ChatSend { request_id, text },
"ChatSend",
)
.await;
}
}
@@ -250,6 +295,7 @@ impl Drop for PrivateState {
fn drop(&mut self) {
self.abort_circuit_watch();
self.abort_ping_monitor_watch();
self.abort_cake_watch();
self.abort_tree_attached_circuits_watch();
self.abort_circuit_metrics_watch();
}
@@ -1,4 +1,5 @@
use crate::node_manager::ws::messages::{WsResponse, encode_ws_message};
use crate::node_manager::ws::single_user_channels::try_send_private_payload;
use lqos_queue_tracker::{add_watched_queue, get_raw_circuit_data, still_watching};
pub(super) async fn cake_watcher(
@@ -16,11 +17,10 @@ pub(super) async fn cake_watcher(
if let lqos_bus::BusResponse::RawQueueData(Some(msg)) = get_raw_circuit_data(&circuit_id) {
let response = WsResponse::CakeWatcher { data: *msg };
if let Ok(payload) = encode_ws_message(&response) {
let send_result = tx.send(payload).await;
if send_result.is_err() {
break;
}
if let Ok(payload) = encode_ws_message(&response)
&& !try_send_private_payload(&tx, payload, "CakeWatcher")
{
break;
}
}
}
@@ -1,5 +1,6 @@
use crate::node_manager::local_api::circuit_activity::{CircuitSummaryData, circuit_flow_counts};
use crate::node_manager::ws::messages::{CircuitDevicesResult, WsResponse, encode_ws_message};
use crate::node_manager::ws::single_user_channels::try_send_private_payload;
use crate::node_manager::ws::ticker::all_circuits;
use crate::rtt_exclusions;
use crate::throughput_tracker::{circuit_current_qoo, circuit_current_rtt_p50_nanos};
@@ -121,7 +122,7 @@ pub(super) async fn circuit_watcher(
};
if let Ok(payload) = encode_ws_message(&response) {
if tx.send(payload).await.is_err() {
if !try_send_private_payload(&tx, payload, "CircuitWatcher") {
info!("CircuitWatcher channel is gone");
break;
}
Loaded 100 of 152 files, more files were not shown because too many files have changed in this diff. Show more