From 4d9e27798f67af5e2f8293d93e1a8f9adddb6587 Mon Sep 17 00:00:00 2001 From: rchac Date: Thu, 30 Apr 2026 13:10:00 -0600 Subject: [PATCH 01/20] changes --- src/systemd_hotfix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/systemd_hotfix.sh b/src/systemd_hotfix.sh index cc5f3c57..1ad9f015 100755 --- a/src/systemd_hotfix.sh +++ b/src/systemd_hotfix.sh @@ -11,7 +11,7 @@ HOTFIX_APT_SOURCE_PATH="${HOTFIX_APT_SOURCE_PATH:-/etc/apt/sources.list.d/libreq HOTFIX_APT_PREFERENCES_PATH="${HOTFIX_APT_PREFERENCES_PATH:-/etc/apt/preferences.d/libreqos-systemd-hotfix}" HOTFIX_APT_PIN_ORIGIN="${HOTFIX_APT_PIN_ORIGIN:-LibreQoS}" HOTFIX_APT_PIN_LABEL="${HOTFIX_APT_PIN_LABEL:-LibreQoS}" -HOTFIX_PACKAGE_VERSION="${HOTFIX_PACKAGE_VERSION:-255.4-1ubuntu8.14+libreqos2}" +HOTFIX_PACKAGE_VERSION="${HOTFIX_PACKAGE_VERSION:-255.4-1ubuntu8.15+libreqos1}" SUPPORTED_UBUNTU_SYSTEMD_VERSION_GLOBS="${SUPPORTED_UBUNTU_SYSTEMD_VERSION_GLOBS:-255.4-1ubuntu8 255.4-1ubuntu8.*}" HOTFIX_MARKER="${HOTFIX_MARKER:-/opt/libreqos/src/.systemd_hotfix_installed}" HOTFIX_SKIP_REBOOT_PROMPT="${HOTFIX_SKIP_REBOOT_PROMPT:-0}" From befbf8d3ef4f495ae671a5189f1a40a52105f9ff Mon Sep 17 00:00:00 2001 From: rchac Date: Fri, 1 May 2026 15:27:59 +0000 Subject: [PATCH 02/20] Install Python dependencies in root-owned venv --- docs-es/v2.0/Quickstart/services-and-run.md | 6 ++-- docs-es/v2.0/TechnicalDocs/git-install.md | 7 +++-- docs-es/v2.0/TechnicalDocs/troubleshooting.md | 14 ++++++++-- docs-es/v2.0/components-es.md | 6 ++-- docs-es/v2.0/git-install-es.md | 7 +++-- docs-es/v2.0/troubleshooting-es.md | 10 ++++++- docs/v2.0/components.md | 7 +++-- docs/v2.0/git-install.md | 7 +++-- docs/v2.0/troubleshooting.md | 24 ++++++++++++++-- src/bin/lqos_scheduler.service.example | 2 +- src/bin/rebuild_python_venv.sh | 28 +++++++++++++++++++ src/build_dpkg.sh | 20 ++++++------- src/build_rust.sh | 16 ++++++++++- 13 files changed, 117 insertions(+), 37 deletions(-) create mode 100755 src/bin/rebuild_python_venv.sh diff --git a/docs-es/v2.0/Quickstart/services-and-run.md b/docs-es/v2.0/Quickstart/services-and-run.md index 1e4c6864..b05fc9af 100644 --- a/docs-es/v2.0/Quickstart/services-and-run.md +++ b/docs-es/v2.0/Quickstart/services-and-run.md @@ -48,19 +48,19 @@ sudo systemctl stop lqos_scheduler For one-time runs of LibreQoS.py, use ```shell -sudo ./LibreQoS.py +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/LibreQoS.py ``` - To use the debug mode with more verbose output, use: ```shell -sudo ./LibreQoS.py --debug +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/LibreQoS.py --debug ``` To confirm that lqos_scheduler (scheduler.py) is able to work correctly, run: ```shell -sudo python3 scheduler.py +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/scheduler.py ``` Once you have any errors eliminated, restart lqos_scheduler with diff --git a/docs-es/v2.0/TechnicalDocs/git-install.md b/docs-es/v2.0/TechnicalDocs/git-install.md index 154e2127..1b3a4405 100644 --- a/docs-es/v2.0/TechnicalDocs/git-install.md +++ b/docs-es/v2.0/TechnicalDocs/git-install.md @@ -21,15 +21,16 @@ By specifying `libreqos` at the end, git will ensure the folder name is lowercas You need to have a few packages from `apt` installed: ```shell -sudo apt-get install -y python3-pip clang gcc gcc-multilib llvm libelf-dev git nano graphviz curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev +sudo apt-get install -y python3-pip python3-venv clang gcc gcc-multilib llvm libelf-dev git nano graphviz curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev ``` Then you need to install some Python dependencies: ```shell cd /opt/libreqos -PIP_BREAK_SYSTEM_PACKAGES=1 pip install -r requirements.txt -sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip install -r requirements.txt +sudo python3 -m venv /opt/libreqos/venv +sudo /opt/libreqos/venv/bin/python -m pip install --upgrade pip +sudo /opt/libreqos/venv/bin/python -m pip install --upgrade -r requirements.txt ``` ## Install the Rust development system diff --git a/docs-es/v2.0/TechnicalDocs/troubleshooting.md b/docs-es/v2.0/TechnicalDocs/troubleshooting.md index 5bd88b72..31155216 100644 --- a/docs-es/v2.0/TechnicalDocs/troubleshooting.md +++ b/docs-es/v2.0/TechnicalDocs/troubleshooting.md @@ -57,7 +57,17 @@ At the command-line, run ```sudo RUST_LOG=info /opt/libreqos/src/bin/lqosd``` wh This tends to show up when the MQ qdisc cannot be added correctly to the NIC interface. This would suggest the NIC has insufficient RX/TX queues. Please make sure you are using the [recommended NICs](../../SystemRequirements/Compute.md#network-interface-requirements). -### Python ModuleNotFoundError in Ubuntu 24.04 +### Python dependency or virtual environment errors + +Packaged installs keep LibreQoS Python dependencies in `/opt/libreqos/venv`. The services still run as root, but Python packages do not mix with apt-managed system packages. If the scheduler reports missing Python modules, or package configuration was interrupted while installing Python dependencies, rebuild the virtual environment: + +```bash +sudo /opt/libreqos/src/bin/rebuild_python_venv.sh +sudo dpkg --configure -a +sudo systemctl restart lqosd lqos_scheduler +``` + +### Python ModuleNotFoundError in older Ubuntu 24.04 installs ``` pip uninstall binpacking --break-system-packages --yes sudo pip uninstall binpacking --break-system-packages --yes @@ -73,7 +83,7 @@ sudo pip install deepdiff --break-system-packages ``` cd /opt/libreqos/src sudo systemctl stop lqos_scheduler -sudo python3 LibreQoS.py +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/LibreQoS.py ``` The console output from running LibreQoS.py directly provides more specific errors regarding issues with ShapedDevices.csv and network.json diff --git a/docs-es/v2.0/components-es.md b/docs-es/v2.0/components-es.md index 0eff075d..7dcbed04 100644 --- a/docs-es/v2.0/components-es.md +++ b/docs-es/v2.0/components-es.md @@ -51,15 +51,15 @@ sudo journalctl -u lqos_scheduler -b ### Depuración de lqos_scheduler -`lqos_scheduler` ejecuta `scheduler.py`, que a su vez invoca `LibreQoS.py`. +`lqos_scheduler` ejecuta `scheduler.py`, que a su vez invoca `LibreQoS.py`. En instalaciones empaquetadas, Python se ejecuta desde el entorno virtual propiedad de root en `/opt/libreqos/venv`. Flujo de depuración recomendado: ```bash sudo systemctl stop lqos_scheduler cd /opt/libreqos/src -sudo ./LibreQoS.py --debug -sudo python3 scheduler.py +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/LibreQoS.py --debug +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/scheduler.py sudo systemctl start lqos_scheduler ``` diff --git a/docs-es/v2.0/git-install-es.md b/docs-es/v2.0/git-install-es.md index 9f155ef8..e1670714 100644 --- a/docs-es/v2.0/git-install-es.md +++ b/docs-es/v2.0/git-install-es.md @@ -20,15 +20,16 @@ Al especificar `libreqos` al final, Git asegurará que el nombre del directorio Es necesario instalar ciertos paquetes mediante `apt`: ```shell -sudo apt-get install -y python3-pip clang mold esbuild gcc gcc-multilib llvm libelf-dev git nano graphviz curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev +sudo apt-get install -y python3-pip python3-venv clang mold esbuild gcc gcc-multilib llvm libelf-dev git nano graphviz curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev ``` Posteriormente, debe instalar algunas dependencias de Python: ```shell cd /opt/libreqos -PIP_BREAK_SYSTEM_PACKAGES=1 pip install -r requirements.txt -sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip install -r requirements.txt +sudo python3 -m venv /opt/libreqos/venv +sudo /opt/libreqos/venv/bin/python -m pip install --upgrade pip +sudo /opt/libreqos/venv/bin/python -m pip install --upgrade -r requirements.txt ``` ## Instalar el entorno de desarrollo de Rust diff --git a/docs-es/v2.0/troubleshooting-es.md b/docs-es/v2.0/troubleshooting-es.md index db4a348d..1afcd98e 100644 --- a/docs-es/v2.0/troubleshooting-es.md +++ b/docs-es/v2.0/troubleshooting-es.md @@ -112,6 +112,14 @@ sudo RUST_LOG=info /opt/libreqos/src/bin/lqosd sudo journalctl -u lqos_scheduler --since "1 day ago" --no-pager > lqos_sched_log.txt ``` +Las instalaciones empaquetadas mantienen las dependencias Python de LibreQoS en `/opt/libreqos/venv`. Los servicios siguen ejecutándose como root, pero los paquetes Python no se mezclan con los paquetes administrados por apt. Si el scheduler informa módulos faltantes, o si la configuración del paquete se interrumpió al instalar dependencias Python, reconstruya el entorno virtual: + +```bash +sudo /opt/libreqos/src/bin/rebuild_python_venv.sh +sudo dpkg --configure -a +sudo systemctl restart lqosd lqos_scheduler +``` + Si el scheduler falla inmediatamente después de un reinicio con un mensaje como `Socket (typically /run/lqos/bus) not found`, eso indica que `lqosd` todavía no había terminado de enlazar el bus local. Los builds actuales esperan brevemente la disponibilidad del bus al arrancar el scheduler en lugar de abortar de inmediato, por lo que ya no deberían aparecer panics repetidos de arranque tras un reinicio. ### El estado del scheduler en WebUI aparece no saludable @@ -174,7 +182,7 @@ sudo pip install deepdiff --break-system-packages ```bash cd /opt/libreqos/src sudo systemctl stop lqos_scheduler -sudo python3 LibreQoS.py +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/LibreQoS.py ``` Corrija errores en `ShapedDevices.csv` y/o `network.json`, luego: diff --git a/docs/v2.0/components.md b/docs/v2.0/components.md index bc4bdf7b..a43fec72 100644 --- a/docs/v2.0/components.md +++ b/docs/v2.0/components.md @@ -31,6 +31,7 @@ flowchart LR ### lqos_scheduler - lqos_scheduler performs continuous refreshes of LibreQoS' shapers, including pulling from any enabled CRM Integrations (UISP, Splynx, Netzur). +- Packaged installs run scheduler Python from the root-owned virtual environment at `/opt/libreqos/venv`. - Actions: - On start: Run a full setup of queues - Current builds wait briefly for `lqosd` to finish binding the local bus before the first scheduler run. @@ -89,19 +90,19 @@ sudo systemctl stop lqos_scheduler For one-time runs of LibreQoS.py, use ```shell -sudo ./LibreQoS.py +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/LibreQoS.py ``` - To use the debug mode with more verbose output, use: ```shell -sudo ./LibreQoS.py --debug +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/LibreQoS.py --debug ``` To confirm that lqos_scheduler (scheduler.py) is able to work correctly, run: ```shell -sudo python3 scheduler.py +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/scheduler.py ``` If an integration fails during a scheduler run, current builds keep the scheduler alive, surface a shortened output preview in scheduler status/error reporting, and save the full captured output to a timestamped `/tmp/lqos_scheduler__*.log` file. diff --git a/docs/v2.0/git-install.md b/docs/v2.0/git-install.md index 4c4ad0ba..0c5d2287 100644 --- a/docs/v2.0/git-install.md +++ b/docs/v2.0/git-install.md @@ -20,15 +20,16 @@ By specifying `libreqos` at the end, git will ensure the folder name is lowercas You need to have a few packages from `apt` installed: ```shell -sudo apt-get install -y python3-pip clang mold esbuild gcc gcc-multilib llvm libelf-dev git nano graphviz curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev +sudo apt-get install -y python3-pip python3-venv clang mold esbuild gcc gcc-multilib llvm libelf-dev git nano graphviz curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev ``` Then you need to install some Python dependencies: ```shell cd /opt/libreqos -PIP_BREAK_SYSTEM_PACKAGES=1 pip install -r requirements.txt -sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip install -r requirements.txt +sudo python3 -m venv /opt/libreqos/venv +sudo /opt/libreqos/venv/bin/python -m pip install --upgrade pip +sudo /opt/libreqos/venv/bin/python -m pip install --upgrade -r requirements.txt ``` ## Install the Rust development system diff --git a/docs/v2.0/troubleshooting.md b/docs/v2.0/troubleshooting.md index 609287fe..edb1f60d 100644 --- a/docs/v2.0/troubleshooting.md +++ b/docs/v2.0/troubleshooting.md @@ -214,7 +214,27 @@ If status repeatedly oscillates between ready/error, collect both logs and confi This tends to show up when the MQ qdisc cannot be added correctly to the NIC interface. This would suggest the NIC has insufficient RX/TX queues. Please make sure you are using the [recommended NICs](requirements.md). -### Python ModuleNotFoundError in Ubuntu 24.04 +### Python dependency or virtual environment errors + +Packaged installs keep LibreQoS Python dependencies in `/opt/libreqos/venv`. The services still run as root, but Python packages do not mix with apt-managed system packages. If the scheduler reports missing Python modules, or package configuration was interrupted while installing Python dependencies, rebuild the virtual environment: + +```bash +sudo /opt/libreqos/src/bin/rebuild_python_venv.sh +sudo dpkg --configure -a +sudo systemctl restart lqosd lqos_scheduler +``` + +For manual shaping tests, use the same interpreter as the service: + +```bash +sudo systemctl stop lqos_scheduler +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/LibreQoS.py +sudo systemctl start lqos_scheduler +``` + +Older installs that predate the virtual environment may show `ModuleNotFoundError` and suggest system `pip` commands. Upgrade to a package that creates `/opt/libreqos/venv`, then use the repair command above. + +### Python ModuleNotFoundError in older Ubuntu 24.04 installs ``` pip uninstall binpacking --break-system-packages --yes sudo pip uninstall binpacking --break-system-packages --yes @@ -230,7 +250,7 @@ sudo pip install deepdiff --break-system-packages ``` cd /opt/libreqos/src sudo systemctl stop lqos_scheduler -sudo python3 LibreQoS.py +sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/LibreQoS.py ``` The console output from running LibreQoS.py directly provides more specific errors regarding issues with ShapedDevices.csv and network.json diff --git a/src/bin/lqos_scheduler.service.example b/src/bin/lqos_scheduler.service.example index c1e766dc..78a42d74 100644 --- a/src/bin/lqos_scheduler.service.example +++ b/src/bin/lqos_scheduler.service.example @@ -5,7 +5,7 @@ Requires=lqosd.service [Service] WorkingDirectory=/opt/libreqos/src -ExecStart=/usr/bin/python3 /opt/libreqos/src/scheduler.py +ExecStart=/opt/libreqos/venv/bin/python /opt/libreqos/src/scheduler.py Restart=always [Install] diff --git a/src/bin/rebuild_python_venv.sh b/src/bin/rebuild_python_venv.sh new file mode 100755 index 00000000..247ef4db --- /dev/null +++ b/src/bin/rebuild_python_venv.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euo pipefail + +VENV_DIR="/opt/libreqos/venv" +REQUIREMENTS="/opt/libreqos/src/requirements.txt" +CONSTRAINTS="/opt/libreqos/src/deb-requirements-constraints.txt" + +if [ "$(id -u)" -ne 0 ]; then + echo "rebuild_python_venv.sh must be run as root." >&2 + exit 1 +fi + +if [ ! -f "$REQUIREMENTS" ]; then + echo "LibreQoS Python requirements file not found: $REQUIREMENTS" >&2 + exit 1 +fi + +python3 -m venv --clear "$VENV_DIR" +"$VENV_DIR/bin/python" -m pip install --upgrade pip + +if [ -s "$CONSTRAINTS" ]; then + "$VENV_DIR/bin/python" -m pip install --upgrade -c "$CONSTRAINTS" -r "$REQUIREMENTS" +else + "$VENV_DIR/bin/python" -m pip install --upgrade -r "$REQUIREMENTS" +fi + +chown -R root:root "$VENV_DIR" +chmod -R go-w "$VENV_DIR" diff --git a/src/build_dpkg.sh b/src/build_dpkg.sh index 66388a1d..45e2c74c 100755 --- a/src/build_dpkg.sh +++ b/src/build_dpkg.sh @@ -12,7 +12,7 @@ PACKAGE=libreqos VERSION=$(cat ./VERSION_STRING).$BUILD_DATE PKGVERSION="${PACKAGE}_${VERSION}" DPKG_DIR=dist/$PKGVERSION-1_amd64 -APT_DEPENDENCIES="python3-pip, nano, curl, ca-certificates" +APT_DEPENDENCIES="python3-pip, python3-venv, nano, curl, ca-certificates" DEBIAN_DIR=$DPKG_DIR/DEBIAN LQOS_DIR=$DPKG_DIR/opt/libreqos/src LQOS_STATE_DIR=$DPKG_DIR/opt/libreqos/state @@ -53,6 +53,7 @@ LQOS_BIN_FILES=( lqosd.service.example lqos_api.service.example lqos_setup.service.example + rebuild_python_venv.sh ) RUSTPROGS=( @@ -140,16 +141,10 @@ else fi } -# Install Python Dependencies -pushd /opt/libreqos > /dev/null -# - Setup Python dependencies as a post-install task. Use --ignore-installed so -# pip does not try to uninstall Debian-managed packages that do not ship a -# pip RECORD file (for example blinker on Ubuntu 24.04). -if [ -s src/deb-requirements-constraints.txt ]; then -PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --ignore-installed -c src/deb-requirements-constraints.txt -r src/requirements.txt -else -PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --ignore-installed -r src/requirements.txt -fi +# - Setup Python dependencies in a root-owned virtual environment. LibreQoS +# services still run as root, but Python packages no longer mix with +# apt-managed system site-packages. +/opt/libreqos/src/bin/rebuild_python_venv.sh # Ensure folder permissions are correct post-install set_libreqos_operator_permissions @@ -191,7 +186,6 @@ HOTFIX exit 1 ;; esac -popd > /dev/null EOF # Uninstall Script @@ -235,6 +229,8 @@ for file in "${LQOS_BIN_FILES[@]}"; do cp "bin/$file" "$LQOS_DIR/bin" done +chmod a+x "$LQOS_DIR/bin/rebuild_python_venv.sh" + # Copy the remove pinned maps cp rust/remove_pinned_maps.sh "$LQOS_DIR"/rust diff --git a/src/build_rust.sh b/src/build_rust.sh index d61c9407..234dc8fe 100755 --- a/src/build_rust.sh +++ b/src/build_rust.sh @@ -23,7 +23,7 @@ do done # Check Pre-Requisites -sudo apt install python3-pip clang gcc gcc-multilib llvm libelf-dev git nano curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev curl +sudo apt install python3-pip python3-venv clang gcc gcc-multilib llvm libelf-dev git nano curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev curl if ! rustup -V &> /dev/null then @@ -214,6 +214,17 @@ clear_pinned_maps_before_lqosd_restart() { fi } +rebuild_python_venv() { + local script_path="./bin/rebuild_python_venv.sh" + if [ ! -x "$script_path" ]; then + echo "Expected $script_path to exist and be executable before refreshing Python services." + exit 1 + fi + + echo "Rebuilding /opt/libreqos/venv before restarting Python services." + sudo "$script_path" +} + SERVICE_UNITS_UPDATED=0 refresh_service_unit lqosd refresh_service_unit lqos_scheduler @@ -240,6 +251,8 @@ if service_unit_exists lqos_netplan_helper; then fi fi +rebuild_python_venv + if hotfix_blocks_service_restart; then echo "Ubuntu 24.04 systemd hotfix is still required. Skipping LibreQoS service restarts." echo "Install it with: sudo ./systemd_hotfix.sh install" @@ -267,6 +280,7 @@ fi echo "-----------------------------------------------------------------" echo "Don't forget to setup /etc/lqos.conf!" echo "Template .service files can be found in bin/" +echo "Debian package installs create/update /opt/libreqos/venv for Python dependencies." echo "If src/deb-requirements-constraints.txt exists, Debian package installs use it to constrain Python dependencies." echo "Use ./systemd_hotfix.sh to evaluate or install the Ubuntu 24.04 networkd hotfix from the LibreQoS APT repo at https://repo.libreqos.com." echo "The hotfix installer now offers to schedule a reboot after it finishes." From 85fe3db2b2ca478c079c6acf92f7d81a91dfcd40 Mon Sep 17 00:00:00 2001 From: rchac Date: Fri, 1 May 2026 15:52:32 +0000 Subject: [PATCH 03/20] changes --- docs-es/v2.0/TechnicalDocs/troubleshooting.md | 2 ++ docs-es/v2.0/troubleshooting-es.md | 2 ++ docs/v2.0/troubleshooting.md | 2 ++ src/build_rust.sh | 5 +++-- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs-es/v2.0/TechnicalDocs/troubleshooting.md b/docs-es/v2.0/TechnicalDocs/troubleshooting.md index 31155216..c53a57f6 100644 --- a/docs-es/v2.0/TechnicalDocs/troubleshooting.md +++ b/docs-es/v2.0/TechnicalDocs/troubleshooting.md @@ -67,6 +67,8 @@ sudo dpkg --configure -a sudo systemctl restart lqosd lqos_scheduler ``` +Git-based installs must rebuild the virtual environment before restarting `lqos_scheduler` after pulling updates that change the service file. If systemd reports `status=203/EXEC`, rebuild the virtual environment with the command above and restart `lqos_scheduler`. + ### Python ModuleNotFoundError in older Ubuntu 24.04 installs ``` pip uninstall binpacking --break-system-packages --yes diff --git a/docs-es/v2.0/troubleshooting-es.md b/docs-es/v2.0/troubleshooting-es.md index 1afcd98e..2f661992 100644 --- a/docs-es/v2.0/troubleshooting-es.md +++ b/docs-es/v2.0/troubleshooting-es.md @@ -120,6 +120,8 @@ sudo dpkg --configure -a sudo systemctl restart lqosd lqos_scheduler ``` +Las instalaciones basadas en git deben reconstruir el entorno virtual antes de reiniciar `lqos_scheduler` después de actualizar cambios que modifiquen el archivo de servicio. Si systemd informa `status=203/EXEC`, reconstruya el entorno virtual con el comando anterior y reinicie `lqos_scheduler`. + Si el scheduler falla inmediatamente después de un reinicio con un mensaje como `Socket (typically /run/lqos/bus) not found`, eso indica que `lqosd` todavía no había terminado de enlazar el bus local. Los builds actuales esperan brevemente la disponibilidad del bus al arrancar el scheduler en lugar de abortar de inmediato, por lo que ya no deberían aparecer panics repetidos de arranque tras un reinicio. ### El estado del scheduler en WebUI aparece no saludable diff --git a/docs/v2.0/troubleshooting.md b/docs/v2.0/troubleshooting.md index edb1f60d..8ef8588f 100644 --- a/docs/v2.0/troubleshooting.md +++ b/docs/v2.0/troubleshooting.md @@ -224,6 +224,8 @@ sudo dpkg --configure -a sudo systemctl restart lqosd lqos_scheduler ``` +Git-based installs must rebuild the virtual environment before restarting `lqos_scheduler` after pulling updates that change the service file. If systemd reports `status=203/EXEC`, rebuild the virtual environment with the command above and restart `lqos_scheduler`. + For manual shaping tests, use the same interpreter as the service: ```bash diff --git a/src/build_rust.sh b/src/build_rust.sh index 234dc8fe..3182e4cb 100755 --- a/src/build_rust.sh +++ b/src/build_rust.sh @@ -226,6 +226,9 @@ rebuild_python_venv() { } SERVICE_UNITS_UPDATED=0 + +rebuild_python_venv + refresh_service_unit lqosd refresh_service_unit lqos_scheduler refresh_service_unit lqos_api @@ -251,8 +254,6 @@ if service_unit_exists lqos_netplan_helper; then fi fi -rebuild_python_venv - if hotfix_blocks_service_restart; then echo "Ubuntu 24.04 systemd hotfix is still required. Skipping LibreQoS service restarts." echo "Install it with: sudo ./systemd_hotfix.sh install" From f9dc6e0047f6b407e805aad195e3d9cf64454170 Mon Sep 17 00:00:00 2001 From: rchac Date: Fri, 1 May 2026 15:58:07 +0000 Subject: [PATCH 04/20] changes --- docs-es/v2.0/TechnicalDocs/troubleshooting.md | 2 +- docs-es/v2.0/troubleshooting-es.md | 2 +- docs/v2.0/troubleshooting.md | 2 +- src/build_rust.sh | 13 ++++++++++++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs-es/v2.0/TechnicalDocs/troubleshooting.md b/docs-es/v2.0/TechnicalDocs/troubleshooting.md index c53a57f6..e91e2600 100644 --- a/docs-es/v2.0/TechnicalDocs/troubleshooting.md +++ b/docs-es/v2.0/TechnicalDocs/troubleshooting.md @@ -67,7 +67,7 @@ sudo dpkg --configure -a sudo systemctl restart lqosd lqos_scheduler ``` -Git-based installs must rebuild the virtual environment before restarting `lqos_scheduler` after pulling updates that change the service file. If systemd reports `status=203/EXEC`, rebuild the virtual environment with the command above and restart `lqos_scheduler`. +Git-based installs should use `./build_rust.sh` after pulling updates. It rebuilds the virtual environment before refreshing service files or restarting services. If systemd reports `status=203/EXEC`, rebuild the virtual environment with the command above and restart `lqos_scheduler`. ### Python ModuleNotFoundError in older Ubuntu 24.04 installs ``` diff --git a/docs-es/v2.0/troubleshooting-es.md b/docs-es/v2.0/troubleshooting-es.md index 2f661992..c9883eec 100644 --- a/docs-es/v2.0/troubleshooting-es.md +++ b/docs-es/v2.0/troubleshooting-es.md @@ -120,7 +120,7 @@ sudo dpkg --configure -a sudo systemctl restart lqosd lqos_scheduler ``` -Las instalaciones basadas en git deben reconstruir el entorno virtual antes de reiniciar `lqos_scheduler` después de actualizar cambios que modifiquen el archivo de servicio. Si systemd informa `status=203/EXEC`, reconstruya el entorno virtual con el comando anterior y reinicie `lqos_scheduler`. +Las instalaciones basadas en git deben usar `./build_rust.sh` después de actualizar. Ese script reconstruye el entorno virtual antes de actualizar los archivos de servicio o reiniciar servicios. Si systemd informa `status=203/EXEC`, reconstruya el entorno virtual con el comando anterior y reinicie `lqos_scheduler`. Si el scheduler falla inmediatamente después de un reinicio con un mensaje como `Socket (typically /run/lqos/bus) not found`, eso indica que `lqosd` todavía no había terminado de enlazar el bus local. Los builds actuales esperan brevemente la disponibilidad del bus al arrancar el scheduler en lugar de abortar de inmediato, por lo que ya no deberían aparecer panics repetidos de arranque tras un reinicio. diff --git a/docs/v2.0/troubleshooting.md b/docs/v2.0/troubleshooting.md index 8ef8588f..e4da9fc3 100644 --- a/docs/v2.0/troubleshooting.md +++ b/docs/v2.0/troubleshooting.md @@ -224,7 +224,7 @@ sudo dpkg --configure -a sudo systemctl restart lqosd lqos_scheduler ``` -Git-based installs must rebuild the virtual environment before restarting `lqos_scheduler` after pulling updates that change the service file. If systemd reports `status=203/EXEC`, rebuild the virtual environment with the command above and restart `lqos_scheduler`. +Git-based installs should use `./build_rust.sh` after pulling updates. It rebuilds the virtual environment before refreshing service files or restarting services. If systemd reports `status=203/EXEC`, rebuild the virtual environment with the command above and restart `lqos_scheduler`. For manual shaping tests, use the same interpreter as the service: diff --git a/src/build_rust.sh b/src/build_rust.sh index 3182e4cb..b6c92891 100755 --- a/src/build_rust.sh +++ b/src/build_rust.sh @@ -22,6 +22,8 @@ do esac done +VENV_PYTHON="/opt/libreqos/venv/bin/python" + # Check Pre-Requisites sudo apt install python3-pip python3-venv clang gcc gcc-multilib llvm libelf-dev git nano curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev curl @@ -222,7 +224,16 @@ rebuild_python_venv() { fi echo "Rebuilding /opt/libreqos/venv before restarting Python services." - sudo "$script_path" + if ! sudo "$script_path"; then + echo "Failed to rebuild /opt/libreqos/venv. Skipping service refresh/restarts." + exit 1 + fi + + if [ ! -x "$VENV_PYTHON" ]; then + echo "Expected $VENV_PYTHON to exist and be executable after rebuilding the Python venv." + echo "Skipping service refresh/restarts." + exit 1 + fi } SERVICE_UNITS_UPDATED=0 From a426c9d4d88c53245d31cb8c5c2514da46699a86 Mon Sep 17 00:00:00 2001 From: rchac Date: Fri, 1 May 2026 15:59:56 +0000 Subject: [PATCH 05/20] changes --- src/bin/rebuild_python_venv.sh | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/bin/rebuild_python_venv.sh b/src/bin/rebuild_python_venv.sh index 247ef4db..b3909d11 100755 --- a/src/bin/rebuild_python_venv.sh +++ b/src/bin/rebuild_python_venv.sh @@ -2,23 +2,37 @@ set -euo pipefail VENV_DIR="/opt/libreqos/venv" -REQUIREMENTS="/opt/libreqos/src/requirements.txt" -CONSTRAINTS="/opt/libreqos/src/deb-requirements-constraints.txt" +REQUIREMENTS="" +CONSTRAINTS="" if [ "$(id -u)" -ne 0 ]; then echo "rebuild_python_venv.sh must be run as root." >&2 exit 1 fi -if [ ! -f "$REQUIREMENTS" ]; then - echo "LibreQoS Python requirements file not found: $REQUIREMENTS" >&2 +for candidate in /opt/libreqos/src/requirements.txt /opt/libreqos/requirements.txt; do + if [ -f "$candidate" ]; then + REQUIREMENTS="$candidate" + break + fi +done + +if [ -z "$REQUIREMENTS" ]; then + echo "LibreQoS Python requirements file not found in /opt/libreqos/src or /opt/libreqos." >&2 exit 1 fi +for candidate in /opt/libreqos/src/deb-requirements-constraints.txt /opt/libreqos/deb-requirements-constraints.txt; do + if [ -s "$candidate" ]; then + CONSTRAINTS="$candidate" + break + fi +done + python3 -m venv --clear "$VENV_DIR" "$VENV_DIR/bin/python" -m pip install --upgrade pip -if [ -s "$CONSTRAINTS" ]; then +if [ -n "$CONSTRAINTS" ]; then "$VENV_DIR/bin/python" -m pip install --upgrade -c "$CONSTRAINTS" -r "$REQUIREMENTS" else "$VENV_DIR/bin/python" -m pip install --upgrade -r "$REQUIREMENTS" From 3cd14ed4a627390be84af624321138a23e59bb68 Mon Sep 17 00:00:00 2001 From: rchac Date: Fri, 1 May 2026 16:03:16 +0000 Subject: [PATCH 06/20] changes --- src/build_rust.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/build_rust.sh b/src/build_rust.sh index b6c92891..36f15f02 100755 --- a/src/build_rust.sh +++ b/src/build_rust.sh @@ -275,8 +275,9 @@ else echo "lqosd is active as a service. Restarting it. You may need to enter your sudo password." sudo systemctl restart lqosd fi - if service_is_active lqos_scheduler; then - echo "lqos_scheduler is active as a service. Restarting it. You may need to enter your sudo password." + if service_is_active lqos_scheduler || service_is_failed lqos_scheduler || service_is_enabled lqos_scheduler; then + echo "lqos_scheduler is installed as an active, failed, or enabled service. Restarting it. You may need to enter your sudo password." + sudo systemctl reset-failed lqos_scheduler || true sudo systemctl restart lqos_scheduler fi if service_is_active lqos_api || service_is_failed lqos_api || service_is_enabled lqos_api; then From e5a721b4ec258f3f7f8a91a404b875e076e60aa6 Mon Sep 17 00:00:00 2001 From: rchac Date: Fri, 1 May 2026 21:08:44 +0000 Subject: [PATCH 07/20] changes --- src/rust/lqos_python/src/device_weights.rs | 89 +++++++++++++++++-- src/rust/lqos_queue_tracker/src/queue_diff.rs | 76 ++++++++++++++-- .../node_manager/js_build/src/pubsub/ws.js | 75 ++++++++++++---- src/rust/lqosd/src/node_manager/ws.rs | 40 ++++++--- 4 files changed, 235 insertions(+), 45 deletions(-) diff --git a/src/rust/lqos_python/src/device_weights.rs b/src/rust/lqos_python/src/device_weights.rs index 770975ae..7d0076ae 100644 --- a/src/rust/lqos_python/src/device_weights.rs +++ b/src/rust/lqos_python/src/device_weights.rs @@ -16,7 +16,13 @@ use lqos_bus::{BusRequest, BusResponse}; use lqos_config::{ShapedDevice, load_config}; use pyo3::pyclass; use serde::{Deserialize, Serialize}; -use std::time::Duration; +use std::{ + sync::{Mutex, OnceLock}, + time::{Duration, Instant}, +}; + +const REMOTE_WEIGHT_SUCCESS_TTL: Duration = Duration::from_secs(300); +const REMOTE_WEIGHT_FAILURE_BACKOFF: Duration = Duration::from_secs(300); /// This struct is used to send a request to the Long Term Stats API #[derive(Serialize, Deserialize)] @@ -30,7 +36,7 @@ pub struct DeviceWeightRequest { /// This struct is used to receive a response from the Long Term Stats API /// It contains the circuit_id and the weight of the device -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[pyclass] pub struct DeviceWeightResponse { #[pyo3(get)] @@ -39,6 +45,57 @@ pub struct DeviceWeightResponse { pub weight: i64, } +#[derive(Clone)] +struct CachedRemoteWeights { + fetched_at: Instant, + weights: Vec, +} + +#[derive(Default)] +struct RemoteWeightCache { + last_success: Option, + last_failure: Option, +} + +fn remote_weight_cache() -> &'static Mutex { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(RemoteWeightCache::default())) +} + +fn cached_remote_weights() -> Option> { + let cache = remote_weight_cache().lock().ok()?; + let cached = cache.last_success.as_ref()?; + if cached.fetched_at.elapsed() <= REMOTE_WEIGHT_SUCCESS_TTL { + return Some(cached.weights.clone()); + } + None +} + +fn recent_remote_weight_failure() -> bool { + let Ok(cache) = remote_weight_cache().lock() else { + return false; + }; + cache + .last_failure + .is_some_and(|failed_at| failed_at.elapsed() <= REMOTE_WEIGHT_FAILURE_BACKOFF) +} + +fn remember_remote_weight_success(weights: &[DeviceWeightResponse]) { + if let Ok(mut cache) = remote_weight_cache().lock() { + cache.last_success = Some(CachedRemoteWeights { + fetched_at: Instant::now(), + weights: weights.to_vec(), + }); + cache.last_failure = None; + } +} + +fn remember_remote_weight_failure() { + if let Ok(mut cache) = remote_weight_cache().lock() { + cache.last_failure = Some(Instant::now()); + } +} + /// This function is used to get the device weights from the Long Term Stats API fn get_weights_from_lts( org_key: &str, @@ -147,10 +204,28 @@ pub(crate) fn get_weights_rust() -> Result> { let duration_seconds = 60 * 60 * 24; // 1 day let percentile = 0.95; - eprintln!("Getting weights from Insight"); - let weights = get_weights_from_lts(&org_key, &node_id, start, duration_seconds, percentile); - if let Ok(weights) = weights { - eprintln!("Retrieved {} weights from Insight", weights.len()); + let remote_weights = if let Some(weights) = cached_remote_weights() { + eprintln!("Using cached Insight weights"); + Some(weights) + } else if recent_remote_weight_failure() { + eprintln!("Skipping Insight weight fetch; recent request failed"); + None + } else { + eprintln!("Getting weights from Insight"); + match get_weights_from_lts(&org_key, &node_id, start, duration_seconds, percentile) { + Ok(weights) => { + eprintln!("Retrieved {} weights from Insight", weights.len()); + remember_remote_weight_success(&weights); + Some(weights) + } + Err(err) => { + remember_remote_weight_failure(); + eprintln!("Failed to get weights from Insight: {:?}", err); + None + } + } + }; + if let Some(weights) = remote_weights { // Merge them for weight in weights.iter() { if let Some(existing) = shaped_devices_weights @@ -160,8 +235,6 @@ pub(crate) fn get_weights_rust() -> Result> { existing.weight = weight.weight; } } - } else { - eprintln!("Failed to get weights from Insight: {:?}", weights); } } diff --git a/src/rust/lqos_queue_tracker/src/queue_diff.rs b/src/rust/lqos_queue_tracker/src/queue_diff.rs index c343c0e0..d5e3c207 100644 --- a/src/rust/lqos_queue_tracker/src/queue_diff.rs +++ b/src/rust/lqos_queue_tracker/src/queue_diff.rs @@ -1,7 +1,7 @@ use crate::queue_types::QueueType; use serde::Serialize; use thiserror::Error; -use tracing::error; +use tracing::warn; #[derive(Debug, Clone, Serialize)] pub enum QueueDiff { @@ -21,27 +21,46 @@ pub(crate) fn make_queue_diff( QueueType::FqCodel(..) => match current { QueueType::FqCodel(..) => Ok(fq_codel_diff(previous, current)?), _ => { - error!( - "Queue diffs are not implemented for FqCodel to {:?}", - current + warn!( + previous = queue_kind(previous), + current = queue_kind(current), + "Queue type changed; resetting queue diff sample" ); - Err(QueueDiffError::NotImplemented) + Ok(QueueDiff::None) } }, QueueType::Cake(..) => match current { QueueType::Cake(..) => Ok(cake_diff(previous, current)?), _ => { - error!("Queue diffs are not implemented for Cake to {:?}", current); - Err(QueueDiffError::NotImplemented) + warn!( + previous = queue_kind(previous), + current = queue_kind(current), + "Queue type changed; resetting queue diff sample" + ); + Ok(QueueDiff::None) } }, _ => { - error!("Queue diffs are not implemented for {:?}", current); - Err(QueueDiffError::NotImplemented) + warn!( + previous = queue_kind(previous), + current = queue_kind(current), + "Queue diff unavailable for queue type; resetting queue diff sample" + ); + Ok(QueueDiff::None) } } } +fn queue_kind(queue: &QueueType) -> &'static str { + match queue { + QueueType::FqCodel(_) => "fq_codel", + QueueType::Cake(_) => "cake", + QueueType::Mq(_) => "mq", + QueueType::Htb(_) => "htb", + QueueType::ClsAct => "clsact", + } +} + #[derive(Serialize, Clone, Debug)] pub struct CakeDiff { pub bytes: u64, @@ -116,3 +135,42 @@ fn fq_codel_diff(previous: &QueueType, current: &QueueType) -> Result 0) { @@ -339,7 +347,7 @@ export class WsClient { dashboardWsInteresting("unsubscribe-wire-interesting", [channel], { desiredChannels: Array.from(this.desiredChannels.keys()), }); - this.ws.send(encoder.encode({ Unsubscribe: { channel } })); + this._sendControl({ Unsubscribe: { channel } }); } } else { this.desiredChannels.set(channel, current - 1); @@ -355,11 +363,16 @@ export class WsClient { if (!this.ws) { this.connect(); } - if (!this.handshake_done) { + if (!this.handshake_done || !this._socketIsOpen()) { this.pending.push(normalized); + if (this.handshake_done && this.ws && !this._socketIsOpen()) { + this._retireSocket(); + } return; } - this.ws.send(encoder.encode(normalized)); + if (!this._sendControl(normalized)) { + this.pending.push(normalized); + } } on(event_name, handler) { @@ -406,28 +419,58 @@ export class WsClient { dashboardWsDebug("handshake-complete", { desiredChannels: Array.from(this.desiredChannels.keys()), }); - this.ws.send( - encoder.encode({ - HelloReply: { - ack: ACK_TEXT, - token: get_user_token(), - }, - }), - ); - for (let i = 0; i < this.pending.length; i++) { - this.ws.send(encoder.encode(this.pending[i])); + if (!this._sendControl({ + HelloReply: { + ack: ACK_TEXT, + token: get_user_token(), + }, + })) { + return; } + const pending = this.pending; this.pending = []; + for (let i = 0; i < pending.length; i++) { + if (!this._sendControl(pending[i])) { + this.pending = pending.slice(i).concat(this.pending); + return; + } + } for (const channel of this.desiredChannels.keys()) { dashboardWsInteresting("subscribe-wire-interesting", [channel], { desiredChannels: Array.from(this.desiredChannels.keys()), fromHandshake: true, }); - this.ws.send(encoder.encode({ Subscribe: { channel } })); + if (!this._sendControl({ Subscribe: { channel } })) { + return; + } } this.reconnectDelayMs = 1000; } + _socketIsOpen() { + return this.ws && this.ws.readyState === WebSocket.OPEN; + } + + _retireSocket() { + this._dropSocket("retire-socket"); + this._scheduleReconnect(); + } + + _sendControl(request_obj) { + if (!this._socketIsOpen()) { + this._retireSocket(); + return false; + } + try { + this.ws.send(encoder.encode(request_obj)); + return true; + } catch (err) { + console.warn("Websocket send failed; reconnecting", err); + this._retireSocket(); + return false; + } + } + _dispatch(msg) { if (!msg || !msg.event) { return; diff --git a/src/rust/lqosd/src/node_manager/ws.rs b/src/rust/lqosd/src/node_manager/ws.rs index 4e359bd8..6b5bb6eb 100644 --- a/src/rust/lqosd/src/node_manager/ws.rs +++ b/src/rust/lqosd/src/node_manager/ws.rs @@ -55,6 +55,7 @@ mod ticker; const WS_VERSION: &str = include_str!("../../../../VERSION_STRING"); const HANDSHAKE_TIMEOUT_SECS: u64 = 10; +const REQUEST_TIMEOUT_SECS: u64 = 15; /// Provides an Axum router for the websocket system. Exposes a single /ws route that supports /// pubsub subscriptions and private commands. @@ -182,19 +183,29 @@ async fn handle_socket( // Received a websocket message match inbound { Some(Ok(msg)) => { - let should_close = receive_channel_message( - msg, - channels.clone(), - tx.clone(), - &mut subscribed_channels, - &mut handshake_complete, - &mut WsRequestState { - private_state: &mut private_state, - login: &mut login, - shaper_query: shaper_query.clone(), - }, + let should_close = match tokio::time::timeout( + std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS), + receive_channel_message( + msg, + channels.clone(), + tx.clone(), + &mut subscribed_channels, + &mut handshake_complete, + &mut WsRequestState { + private_state: &mut private_state, + login: &mut login, + shaper_query: shaper_query.clone(), + }, + ), ) - .await; + .await + { + Ok(should_close) => should_close, + Err(_) => { + warn!("Websocket request timed out; closing connection"); + true + } + }; if should_close { break; } @@ -212,6 +223,11 @@ async fn handle_socket( } } } + for channel in subscribed_channels.drain() { + channels.unsubscribe(channel, tx.clone()).await; + } + drop(private_state); + drop(tx); outbound_handle.abort(); let _ = outbound_handle.await; info!("Websocket disconnected"); From 45a641a3595c84181067e9903b3a139e4da60370 Mon Sep 17 00:00:00 2001 From: rchac Date: Sun, 3 May 2026 19:21:43 -0600 Subject: [PATCH 08/20] changes --- .../v2.0/libreqos-backend-architecture-es.md | 9 +- docs-es/v2.0/troubleshooting-es.md | 2 +- docs/v2.0/libreqos-backend-architecture.md | 7 +- docs/v2.0/troubleshooting.md | 2 +- src/rust/lqos_bakery/src/lib.rs | 222 ++++++++++++++++-- src/rust/lqos_bakery/src/utils.rs | 2 +- src/rust/lqos_python/src/lib.rs | 12 +- 7 files changed, 214 insertions(+), 42 deletions(-) diff --git a/docs-es/v2.0/libreqos-backend-architecture-es.md b/docs-es/v2.0/libreqos-backend-architecture-es.md index 741bceff..1ca79e39 100644 --- a/docs-es/v2.0/libreqos-backend-architecture-es.md +++ b/docs-es/v2.0/libreqos-backend-architecture-es.md @@ -270,14 +270,15 @@ flowchart TD 3. Esperar mayor riesgo cuando coinciden muchos circuitos y muchos cambios estructurales. 4. Diseñar la cadencia operativa para priorizar cambios incrementalmente seguros. -### 7.4 Guardrails de seguridad para recargas completas +### 7.4 Salvaguardas de seguridad para recargas completas -Las recargas completas actuales de Bakery aplican dos verificaciones conservadoras de seguridad antes y durante reconstrucciones grandes de colas: +Las recargas completas actuales de Bakery aplican verificaciones conservadoras de seguridad antes y durante reconstrucciones grandes de colas: 1. Un preflight de qdisc estima los qdisc planificados por interfaz y además separa qdisc de infraestructura, hojas `cake` y hojas `fq_codel`. 2. Ese mismo preflight aplica una proyección conservadora de memoria y bloquea de forma estricta las recargas completas claramente inseguras antes de arrancar `tc -batch`. -3. Durante la aplicación chunked de una recarga completa, Bakery vuelve a revisar la memoria del host en los límites de chunk y aborta el resto de la aplicación si la memoria disponible cae por debajo de su piso de seguridad. -4. Estos guardrails están sesgados intencionalmente hacia falsos positivos en recargas grandes para fallar temprano con diagnósticos en lugar de entrar en una espiral OOM. +3. El piso de memoria de Bakery escala con la RAM del host: mantiene al menos 2 GiB disponibles, o un octavo de la RAM total en sistemas más grandes. +4. Durante la aplicación por fragmentos de una recarga completa, Bakery vuelve a revisar la memoria del host en los límites de cada fragmento y aborta el resto de la aplicación si la memoria disponible cae por debajo del piso escalado más la memoria qdisc proyectada para el lote. +5. Estas salvaguardas están sesgadas intencionalmente hacia falsos positivos en recargas grandes para fallar temprano con diagnósticos en lugar de entrar en una espiral OOM. ## 8) Límites de Diseño para Operadores diff --git a/docs-es/v2.0/troubleshooting-es.md b/docs-es/v2.0/troubleshooting-es.md index c9883eec..c5daff47 100644 --- a/docs-es/v2.0/troubleshooting-es.md +++ b/docs-es/v2.0/troubleshooting-es.md @@ -253,7 +253,7 @@ WebUI muestra códigos legibles por máquina para triage rápido. | `MAPPED_CIRCUIT_LIMIT` | Bakery está forzando límite de circuitos mapeados. | Estado de licencia Insight y `journalctl -u lqosd` con requested/allowed/dropped. | Reducir circuitos mapeados o actualizar licencia/límites. | | `TC_U16_OVERFLOW` | IDs minor de clases/colas excedieron rango u16 de tc en una cola CPU. | `journalctl -u lqos_scheduler -u lqosd`, profundidad topológica y distribución por colas. | Aumentar paralelismo de colas y/o simplificar/rebalancear jerarquía. | | `TC_QDISC_CAPACITY` | Los qdisc autoasignados planificados exceden el presupuesto seguro por interfaz o el preflight conservador de seguridad de memoria de Bakery antes de aplicar. | Conteos estimados por interfaz, desglose por tipo de qdisc y campos de memoria en el contexto del urgent issue, `journalctl -u lqos_scheduler -u lqosd`, configuración `on_a_stick` y `monitor_only`. | Reducir la carga planificada de qdisc para esta ejecución (por ejemplo menos circuitos/dispositivos en la forma de prueba) antes de reintentar; no confiar en una aplicación parcial. | -| `BAKERY_MEMORY_GUARD` | Un full reload chunked de Bakery fue detenido a mitad de aplicación porque la memoria disponible del host cayó por debajo del piso de seguridad. | `journalctl -u lqosd`, memoria disponible/total en el contexto del urgent issue y progreso reciente de aplicación de Bakery. | Tratar la ejecución como fallida, reducir presión de memoria o huella de colas y reintentar solo cuando el host esté estable. | +| `BAKERY_MEMORY_GUARD` | Una recarga completa por fragmentos de Bakery fue detenida a mitad de aplicación porque la memoria disponible del host cayó por debajo del piso de seguridad escalado. | `journalctl -u lqosd`, memoria disponible/total en el contexto del urgent issue y progreso reciente de aplicación de Bakery. | Tratar la ejecución como fallida, reducir presión de memoria o huella de colas y reintentar solo cuando el host esté estable. | | `XDP_IP_MAPPING_CAPACITY` | Los mapeos IP requeridos exceden la capacidad actual del mapa XDP en el kernel. | Forma de `ShapedDevices.csv`, mezcla IPv4/IPv6, supuesto de un dispositivo frente a varios dispositivos, `journalctl -u lqos_scheduler -u lqosd`. | Reducir mapeos requeridos de inmediato (por ejemplo menos dispositivos o prueba IPv4-only), o aumentar la capacidad del mapa del kernel en un cambio coordinado. | | `XDP_IP_MAPPING_APPLY_FAILED` | Uno o más inserts de mapeo IP fallaron durante la aplicación. | `journalctl -u lqos_scheduler -u lqosd` para ejemplos resumidos y conteos de fallo. | Corregir la causa del fallo y volver a ejecutar; no confiar en shaping parcial. | diff --git a/docs/v2.0/libreqos-backend-architecture.md b/docs/v2.0/libreqos-backend-architecture.md index 13e58fbb..195317b7 100644 --- a/docs/v2.0/libreqos-backend-architecture.md +++ b/docs/v2.0/libreqos-backend-architecture.md @@ -343,12 +343,13 @@ This retained-root strategy reduces avoidable root-level churn, while still pref ### 8.6 Full-reload safety guards -Current Bakery full reloads apply two conservative safety checks before and during large queue rebuilds: +Current Bakery full reloads apply conservative safety checks before and during large queue rebuilds: 1. A qdisc preflight estimates planned qdiscs per interface and also separates infrastructure, `cake`, and `fq_codel` leaf qdiscs. 2. That same preflight applies a conservative memory forecast and hard-blocks clearly unsafe full reloads before `tc -batch` starts. -3. During chunked full reload apply, Bakery re-checks host memory at chunk boundaries and aborts the remaining apply if available memory drops below its safety floor. -4. These guards are intentionally biased toward false positives on large reloads so the system fails early with diagnostics instead of spiraling into an OOM event. +3. Bakery's memory floor scales with host RAM: it keeps at least 2 GiB available, or one eighth of total RAM on larger systems. +4. During chunked full reload apply, Bakery re-checks host memory at chunk boundaries and aborts the remaining apply if available memory drops below the scaled floor plus the projected qdisc memory for the batch. +5. These guards are intentionally biased toward false positives on large reloads so the system fails early with diagnostics instead of spiraling into an OOM event. ### 8.7 Runtime safety model diff --git a/docs/v2.0/troubleshooting.md b/docs/v2.0/troubleshooting.md index e4da9fc3..058d56a2 100644 --- a/docs/v2.0/troubleshooting.md +++ b/docs/v2.0/troubleshooting.md @@ -330,7 +330,7 @@ WebUI urgent issues include machine-readable codes. Use them to triage quickly. | `MAPPED_CIRCUIT_LIMIT` | Bakery is enforcing a mapped-circuit limit. | Insight license status, `journalctl -u lqosd` for requested/allowed/dropped counts. | Reduce mapped circuits immediately or update license/limits. | | `TC_U16_OVERFLOW` | Queue/class minor IDs exceeded the Linux tc u16 range on a CPU queue. | `journalctl -u lqos_scheduler -u lqosd`, topology depth/queue distribution. | Increase queue count and/or simplify/rebalance hierarchy (for example with integration strategy or root promotion changes). | | `TC_QDISC_CAPACITY` | Planned auto-allocated qdiscs exceed the per-interface safe budget or Bakery's conservative memory-safety preflight before apply. | Estimated per-interface qdisc counts, qdisc-kind breakdown, and memory fields in the urgent issue context, `journalctl -u lqos_scheduler -u lqosd`, `on_a_stick` and `queue_mode` config. | Reduce the planned qdisc load for this run (for example fewer circuits/devices in the test shape) before retrying; do not trust partial apply. | -| `BAKERY_MEMORY_GUARD` | A chunked Bakery full reload was stopped mid-apply because available host memory fell below the safety floor. | `journalctl -u lqosd`, available/total memory in the urgent issue context, and recent Bakery apply progress. | Treat the run as failed, reduce memory pressure or queue footprint, and retry only after the host is stable. | +| `BAKERY_MEMORY_GUARD` | A chunked Bakery full reload was stopped mid-apply because available host memory fell below the scaled safety floor. | `journalctl -u lqosd`, available/total memory in the urgent issue context, and recent Bakery apply progress. | Treat the run as failed, reduce memory pressure or queue footprint, and retry only after the host is stable. | | `XDP_IP_MAPPING_CAPACITY` | Required IP mappings exceed the current XDP kernel map capacity. | `ShapedDevices.csv` row shape, IPv4/IPv6 mix, one-device-vs-many-device assumptions, `journalctl -u lqos_scheduler -u lqosd`. | Reduce required mappings immediately (for example fewer devices or IPv4-only test shape), or raise kernel map capacity in a coordinated change. | | `XDP_IP_MAPPING_APPLY_FAILED` | One or more IP mapping inserts failed during apply. | `journalctl -u lqos_scheduler -u lqosd` for summarized failure examples and counts. | Fix the underlying mapping failure, then rerun; do not trust partial shaping. | diff --git a/src/rust/lqos_bakery/src/lib.rs b/src/rust/lqos_bakery/src/lib.rs index 9264b986..7d4c34f1 100644 --- a/src/rust/lqos_bakery/src/lib.rs +++ b/src/rust/lqos_bakery/src/lib.rs @@ -1513,7 +1513,14 @@ pub const FQ_CODEL_QDISC_ESTIMATED_MEMORY_BYTES: u64 = 64 * 1024; /// substantially exceeding the earlier heuristic during busy periods. pub const CAKE_QDISC_ESTIMATED_MEMORY_BYTES: u64 = 512 * 1024; /// Minimum memory headroom Bakery tries to leave unused after a projected or in-flight apply. -pub const BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES: u64 = 768 * 1024 * 1024; +pub const BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES: u64 = 2 * 1024 * 1024 * 1024; +/// Divisor used to scale Bakery's memory guard floor with installed RAM. +pub const BAKERY_MEMORY_GUARD_TOTAL_RAM_DIVISOR: u64 = 8; + +/// Returns Bakery's effective memory guard floor for a host with `total_bytes` RAM. +pub fn bakery_memory_guard_min_available_bytes(total_bytes: u64) -> u64 { + BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES.max(total_bytes / BAKERY_MEMORY_GUARD_TOTAL_RAM_DIVISOR) +} /// Maximum number of mapped circuits allowed without Insight. const DEFAULT_MAPPED_CIRCUITS_LIMIT: usize = 1000; /// Minimum interval between repeated mapped-circuit-limit urgent issues. @@ -2055,22 +2062,69 @@ fn telemetry_state() -> &'static RwLock { BAKERY_TELEMETRY.get_or_init(|| RwLock::new(BakeryTelemetryState::default())) } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TcCommandType { + Class, + Qdisc, +} + +fn tc_command_type(argv: &[String]) -> Option { + match argv.first()?.as_str() { + "class" => Some(TcCommandType::Class), + "qdisc" => Some(TcCommandType::Qdisc), + _ => None, + } +} + fn count_tc_command_types(commands: &[Vec]) -> (usize, usize, usize) { let total = commands.len(); let mut class_count = 0usize; let mut qdisc_count = 0usize; for argv in commands { - if let Some(kind) = argv.first() { - match kind.as_str() { - "class" => class_count += 1, - "qdisc" => qdisc_count += 1, - _ => {} - } + match tc_command_type(argv) { + Some(TcCommandType::Class) => class_count += 1, + Some(TcCommandType::Qdisc) => qdisc_count += 1, + None => {} } } (total, class_count, qdisc_count) } +fn count_tc_command_types_and_estimate_qdisc_memory( + commands: &[Vec], +) -> (usize, usize, usize, u64) { + let total = commands.len(); + let mut class_count = 0usize; + let mut qdisc_count = 0usize; + let mut estimated_qdisc_memory_bytes = 0u64; + let mut seen_qdiscs = HashSet::new(); + + for argv in commands { + match tc_command_type(argv) { + Some(TcCommandType::Class) => class_count += 1, + Some(TcCommandType::Qdisc) => qdisc_count += 1, + None => {} + } + + let Some(qdisc_identity) = planned_qdisc_identity(argv) else { + continue; + }; + if !seen_qdiscs.insert(qdisc_identity) { + continue; + } + let kind = planned_qdisc_kind(argv).unwrap_or(PlannedQdiscKind::Infra); + estimated_qdisc_memory_bytes = + estimated_qdisc_memory_bytes.saturating_add(qdisc_kind_estimated_memory_bytes(kind)); + } + + ( + total, + class_count, + qdisc_count, + estimated_qdisc_memory_bytes, + ) +} + fn push_bakery_event(event: &str, status: &str, summary: String) { push_bakery_event_with_site(event, status, None, summary); } @@ -2505,6 +2559,8 @@ pub struct QdiscBudgetEstimate { pub estimated_total_memory_bytes: u64, /// Current host memory snapshot used for preflight, if available. pub memory_snapshot: Option, + /// Effective memory floor used for the memory preflight. + pub memory_guard_min_available_bytes: u64, /// Whether the memory preflight passed. pub memory_ok: bool, } @@ -2649,10 +2705,14 @@ pub fn estimate_full_reload_auto_qdisc_budget( acc.saturating_add(detail.estimated_memory_bytes) }); let memory_snapshot = read_memory_snapshot().ok(); + let memory_guard_min_available_bytes = memory_snapshot + .as_ref() + .map(|snapshot| bakery_memory_guard_min_available_bytes(snapshot.total_bytes)) + .unwrap_or(BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES); let memory_ok = memory_snapshot.as_ref().is_none_or(|snapshot| { snapshot .available_bytes - .saturating_sub(BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES) + .saturating_sub(memory_guard_min_available_bytes) >= estimated_total_memory_bytes }); @@ -2663,6 +2723,7 @@ pub fn estimate_full_reload_auto_qdisc_budget( hard_limit: HARD_QDISC_HANDLE_LIMIT_PER_INTERFACE, estimated_total_memory_bytes, memory_snapshot, + memory_guard_min_available_bytes, memory_ok, } } @@ -3889,16 +3950,26 @@ fn maybe_emit_mapped_circuit_limit_urgent(stats: &MappedLimitStats) { }); } -fn maybe_emit_memory_guard_urgent(summary: &str) { +fn maybe_emit_memory_guard_urgent( + summary: &str, + memory_guard_required_available_bytes: u64, + estimated_qdisc_memory_bytes: u64, +) { if !summary.contains("Bakery memory guard stopped") { return; } let message = summary.to_string(); let context = read_memory_snapshot().ok().map(|snapshot| { + let memory_guard_floor_bytes = + bakery_memory_guard_min_available_bytes(snapshot.total_bytes); format!( - "{{\"available_bytes\":{},\"total_bytes\":{},\"memory_guard_floor_bytes\":{}}}", - snapshot.available_bytes, snapshot.total_bytes, BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES + "{{\"available_bytes\":{},\"total_bytes\":{},\"memory_guard_floor_bytes\":{},\"estimated_qdisc_memory_bytes\":{},\"memory_guard_required_available_bytes\":{}}}", + snapshot.available_bytes, + snapshot.total_bytes, + memory_guard_floor_bytes, + estimated_qdisc_memory_bytes, + memory_guard_required_available_bytes ) }); @@ -9059,6 +9130,34 @@ fn site_commands_observed_live( }) } +fn read_optional_restore_class_snapshots( + config: &Arc, + site_label: &str, +) -> Option<( + HashMap, + HashMap, +)> { + let down_snapshot = match read_live_class_snapshot(&config.isp_interface()) { + Ok(snapshot) => snapshot, + Err(summary) => { + debug!( + "Bakery: skipping live restore shortcut for {site_label}; unable to snapshot downlink classes: {summary}" + ); + return None; + } + }; + let up_snapshot = match read_live_class_snapshot(&config.internet_interface()) { + Ok(snapshot) => snapshot, + Err(summary) => { + debug!( + "Bakery: skipping live restore shortcut for {site_label}; unable to snapshot uplink classes: {summary}" + ); + return None; + } + }; + Some((down_snapshot, up_snapshot)) +} + fn live_parent_matches(observed: Option, expected: TcHandle) -> bool { if observed == Some(expected) { return true; @@ -10593,28 +10692,27 @@ fn handle_treeguard_set_node_virtual_live( } let reversible_standby = saved_state.active_branch_hides_original_site(); - let live_restore_snapshots = if reversible_standby { - Some(( - read_live_class_snapshot(&config.isp_interface())?, - read_live_class_snapshot(&config.internet_interface())?, - )) - } else { - None - }; + let mut live_restore_snapshots = None; if !saved_state.pending_prune && let Some(cmds) = saved_state .site .to_commands(&config, ExecutionMode::Builder) { - let root_already_live = + let root_already_live = if reversible_standby { live_restore_snapshots + .get_or_insert_with(|| { + read_optional_restore_class_snapshots(&config, &site_label) + }) .as_ref() .is_some_and(|(down_snapshot, up_snapshot)| { let mut only_root = HashMap::new(); only_root.insert(site_hash, saved_state.site.clone()); site_commands_observed_live(&only_root, down_snapshot, up_snapshot) - }); + }) + } else { + false + }; if !root_already_live { let result = execute_and_record_live_change(&cmds, "TreeGuard runtime hidden site restore"); @@ -10643,8 +10741,9 @@ fn handle_treeguard_set_node_virtual_live( ) }) .collect(); - let originals_already_live = + let originals_already_live = if reversible_standby && !saved_state.saved_sites.is_empty() { live_restore_snapshots + .get_or_insert_with(|| read_optional_restore_class_snapshots(&config, &site_label)) .as_ref() .is_some_and(|(down_snapshot, up_snapshot)| { site_commands_observed_live( @@ -10652,7 +10751,10 @@ fn handle_treeguard_set_node_virtual_live( down_snapshot, up_snapshot, ) - }); + }) + } else { + false + }; if reversible_standby && originals_already_live { for (hash, command) in &saved_state.saved_sites { sites.insert(*hash, Arc::clone(command)); @@ -11037,7 +11139,12 @@ fn process_batch( let path = config.debug_state_file_path("linux_tc_rust.txt"); write_command_file(&path, &commands); let build_duration_ms = build_started.elapsed().as_millis() as u64; - let (total_tc_commands, class_commands, qdisc_commands) = count_tc_command_types(&commands); + let (total_tc_commands, class_commands, qdisc_commands, estimated_qdisc_memory_bytes) = + count_tc_command_types_and_estimate_qdisc_memory(&commands); + let memory_guard_required_available_bytes = read_memory_snapshot() + .map(|snapshot| bakery_memory_guard_min_available_bytes(snapshot.total_bytes)) + .unwrap_or(BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES) + .saturating_add(estimated_qdisc_memory_bytes); let total_chunks = if total_tc_commands == 0 { 0 } else { @@ -11054,7 +11161,7 @@ fn process_batch( &commands, "processing batch", FULL_RELOAD_TC_CHUNK_SIZE, - Some(BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES), + Some(memory_guard_required_available_bytes), |completed_tc_commands, total_tc_commands, completed_chunks, total_chunks| { update_bakery_apply_progress( Some("Applying tc command chunks"), @@ -11066,7 +11173,11 @@ fn process_batch( }, ); let summary = summarize_apply_result("processing batch", &result); - maybe_emit_memory_guard_urgent(&summary); + maybe_emit_memory_guard_urgent( + &summary, + memory_guard_required_available_bytes, + estimated_qdisc_memory_bytes, + ); BakeryFullReloadBatchResult { result, summary, @@ -13363,6 +13474,65 @@ mod tests { assert!(estimate.ok()); } + #[test] + fn bakery_memory_guard_floor_scales_with_host_ram() { + assert_eq!( + bakery_memory_guard_min_available_bytes(8 * 1024 * 1024 * 1024), + BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES + ); + assert_eq!( + bakery_memory_guard_min_available_bytes(64 * 1024 * 1024 * 1024), + 8 * 1024 * 1024 * 1024 + ); + } + + #[test] + fn command_qdisc_memory_estimate_deduplicates_qdisc_identity() { + let commands = vec![ + vec![ + "qdisc".to_string(), + "add".to_string(), + "dev".to_string(), + "eth0".to_string(), + "parent".to_string(), + "1:10".to_string(), + "cake".to_string(), + ], + vec![ + "qdisc".to_string(), + "replace".to_string(), + "dev".to_string(), + "eth0".to_string(), + "parent".to_string(), + "1:10".to_string(), + "cake".to_string(), + ], + vec![ + "qdisc".to_string(), + "add".to_string(), + "dev".to_string(), + "eth1".to_string(), + "parent".to_string(), + "1:10".to_string(), + "cake".to_string(), + ], + vec![ + "qdisc".to_string(), + "add".to_string(), + "dev".to_string(), + "eth0".to_string(), + "parent".to_string(), + "1:11".to_string(), + "fq_codel".to_string(), + ], + ]; + + assert_eq!( + count_tc_command_types_and_estimate_qdisc_memory(&commands).3, + (CAKE_QDISC_ESTIMATED_MEMORY_BYTES * 2) + FQ_CODEL_QDISC_ESTIMATED_MEMORY_BYTES + ); + } + #[test] fn qdisc_budget_estimate_counts_only_root_mq_in_observe_mode() { let _guard = bakery_test_lock().lock().expect("lock"); diff --git a/src/rust/lqos_bakery/src/utils.rs b/src/rust/lqos_bakery/src/utils.rs index a0aa61d2..350fc435 100644 --- a/src/rust/lqos_bakery/src/utils.rs +++ b/src/rust/lqos_bakery/src/utils.rs @@ -269,7 +269,7 @@ fn memory_guard_failure_summary( phase: &str, ) -> String { format!( - "Bakery memory guard stopped {purpose} {phase} chunk {}/{}: available memory {} bytes is below safety floor {} bytes (total RAM {} bytes).", + "Bakery memory guard stopped {purpose} {phase} chunk {}/{}: available memory {} bytes is below required available memory {} bytes (total RAM {} bytes).", chunk_number, total_chunks, snapshot.available_bytes, diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index aafb385d..4c2c9484 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -3571,15 +3571,15 @@ impl Bakery { (false, false) => "exceeds qdisc-count preflight and failed memory preflight", }; let memory_summary = if let Some(snapshot) = estimate.memory_snapshot.as_ref() { - let memory_floor = lqos_bakery::BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES; - let required_available_bytes = - memory_floor.saturating_add(estimate.estimated_total_memory_bytes); + let required_available_bytes = estimate + .memory_guard_min_available_bytes + .saturating_add(estimate.estimated_total_memory_bytes); format!( "estimated qdisc memory {} bytes; available memory {} bytes; required minimum {} bytes (safety floor {} bytes plus estimate)", estimate.estimated_total_memory_bytes, snapshot.available_bytes, required_available_bytes, - memory_floor + estimate.memory_guard_min_available_bytes ) } else { format!( @@ -3625,7 +3625,7 @@ impl Bakery { .memory_snapshot .as_ref() .map(|snapshot| snapshot.available_bytes), - memory_guard_min_available_bytes: lqos_bakery::BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES, + memory_guard_min_available_bytes: estimate.memory_guard_min_available_bytes, memory_ok: estimate.memory_ok, interfaces: interface_reports, }]); @@ -3658,7 +3658,7 @@ impl Bakery { result.set_item("memory_ok", estimate.memory_ok)?; result.set_item( "memory_guard_min_available_bytes", - lqos_bakery::BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES, + estimate.memory_guard_min_available_bytes, )?; if let Some(snapshot) = estimate.memory_snapshot { result.set_item("memory_total_bytes", snapshot.total_bytes)?; From 8e9b14d46eb033a773adc9ae65d34dec4cabc88b Mon Sep 17 00:00:00 2001 From: Name <2718612@2178972821991.net> Date: Tue, 5 May 2026 14:27:23 -0600 Subject: [PATCH 09/20] changes --- docs-es/v2.0/troubleshooting-es.md | 10 + docs/v2.0/troubleshooting.md | 10 + src/rust/lqosd/src/main.rs | 2 + src/rust/lqosd/src/memory_watchdog.rs | 337 ++++++++++++++++++++++++++ 4 files changed, 359 insertions(+) create mode 100644 src/rust/lqosd/src/memory_watchdog.rs diff --git a/docs-es/v2.0/troubleshooting-es.md b/docs-es/v2.0/troubleshooting-es.md index c5daff47..f95ea6a0 100644 --- a/docs-es/v2.0/troubleshooting-es.md +++ b/docs-es/v2.0/troubleshooting-es.md @@ -124,6 +124,16 @@ Las instalaciones basadas en git deben usar `./build_rust.sh` después de actual Si el scheduler falla inmediatamente después de un reinicio con un mensaje como `Socket (typically /run/lqos/bus) not found`, eso indica que `lqosd` todavía no había terminado de enlazar el bus local. Los builds actuales esperan brevemente la disponibilidad del bus al arrancar el scheduler en lugar de abortar de inmediato, por lo que ya no deberían aparecer panics repetidos de arranque tras un reinicio. +Si `journalctl -u lqosd` muestra `lqosd memory watchdog restarting daemon`, el daemon salió intencionalmente antes de que la presión de memoria del host llegara al camino OOM del kernel. Systemd debería reiniciar `lqosd` automáticamente. Capture esa línea de log antes de cambiar ajustes; incluye memoria disponible, RSS/swap de `lqosd`, cantidad de hilos, cantidad de flujos y contadores de tiempo que ayudan a diagnosticar el origen del crecimiento de memoria. + +El watchdog se puede ajustar con overrides de entorno de systemd: + +```bash +sudo systemctl edit lqosd +``` + +Las variables comunes son `LQOSD_MEMORY_WATCHDOG_MIN_AVAILABLE_MB`, `LQOSD_MEMORY_WATCHDOG_MAX_PROCESS_MB` y `LQOSD_MEMORY_WATCHDOG_MAX_SWAP_MB`. Use `LQOSD_MEMORY_WATCHDOG_DISABLED=1` solo durante ventanas cortas de diagnóstico en las que esté observando activamente la presión de memoria. + ### El estado del scheduler en WebUI aparece no saludable Versiones recientes muestran estado/readiness del scheduler en WebUI. diff --git a/docs/v2.0/troubleshooting.md b/docs/v2.0/troubleshooting.md index 058d56a2..fb5c21b7 100644 --- a/docs/v2.0/troubleshooting.md +++ b/docs/v2.0/troubleshooting.md @@ -122,6 +122,16 @@ Press the End key on the keyboard to take you to the bottom of the log to see th Lqosd will provide specific reasons it failed, such as an interface not being up, an interface lacking multi-queue, or other concerns. +If `journalctl -u lqosd` shows `lqosd memory watchdog restarting daemon`, the daemon intentionally exited before host memory pressure reached the kernel OOM path. Systemd should restart `lqosd` automatically. Capture the watchdog log line before changing settings; it includes available memory, `lqosd` RSS/swap, thread count, flow count, and timing counters that help diagnose the source of memory growth. + +The watchdog can be tuned with systemd environment overrides: + +```bash +sudo systemctl edit lqosd +``` + +Common override variables are `LQOSD_MEMORY_WATCHDOG_MIN_AVAILABLE_MB`, `LQOSD_MEMORY_WATCHDOG_MAX_PROCESS_MB`, and `LQOSD_MEMORY_WATCHDOG_MAX_SWAP_MB`. Use `LQOSD_MEMORY_WATCHDOG_DISABLED=1` only for short troubleshooting windows where you are actively watching memory pressure. + ### Advanced lqosd debug At the command-line, run: diff --git a/src/rust/lqosd/src/main.rs b/src/rust/lqosd/src/main.rs index de540d0f..7b856014 100644 --- a/src/rust/lqosd/src/main.rs +++ b/src/rust/lqosd/src/main.rs @@ -10,6 +10,7 @@ mod ip_mapping; #[cfg(feature = "equinix_tests")] mod lqos_daht_test; pub mod lts2_sys; +mod memory_watchdog; mod network_devices_hooks; mod node_manager; mod preflight_checks; @@ -248,6 +249,7 @@ fn main() -> Result<()> { // Memory Debugging memory_debug(); + memory_watchdog::start_memory_watchdog(); let control_tx_for_webserver = control_tx_for_web.clone(); let system_usage_tx_for_web = system_usage_tx.clone(); diff --git a/src/rust/lqosd/src/memory_watchdog.rs b/src/rust/lqosd/src/memory_watchdog.rs new file mode 100644 index 00000000..6e202137 --- /dev/null +++ b/src/rust/lqosd/src/memory_watchdog.rs @@ -0,0 +1,337 @@ +//! Memory pressure watchdog for controlled `lqosd` restarts. +//! +//! The watchdog samples `/proc` and exits the daemon before the kernel OOM +//! path has to choose a victim. `lqosd.service` is configured with +//! `Restart=always`, so this releases process memory while preserving logs that +//! explain why the restart happened. + +use crate::stats::{BUS_REQUESTS, FLOWS_TRACKED, HIGH_WATERMARK, TIME_TO_POLL_HOSTS}; +use std::collections::HashMap; +use std::sync::atomic::Ordering; +use std::time::Duration; +use tracing::{error, info, warn}; + +const CHECK_INTERVAL_SECONDS: u64 = 15; +const PRESSURE_WARNING_AVAILABLE_BYTES: u64 = mib(3_072); +const DEFAULT_MIN_AVAILABLE_BYTES: u64 = mib(2_304); +const DEFAULT_MIN_PROCESS_BYTES: u64 = mib(1_024); +const DEFAULT_MAX_PROCESS_BYTES: u64 = gib(4); +const DEFAULT_MAX_SWAP_BYTES: u64 = gib(1); +const WATCHDOG_EXIT_CODE: i32 = 75; + +/// Spawns the memory watchdog thread. +/// +/// Side effects: this function starts a background thread. The thread reads +/// `/proc/meminfo`, `/proc/self/status`, and exits the process when memory +/// pressure reaches the configured critical threshold. +pub fn start_memory_watchdog() { + if env_flag_enabled("LQOSD_MEMORY_WATCHDOG_DISABLED") { + warn!("lqosd memory watchdog disabled by LQOSD_MEMORY_WATCHDOG_DISABLED"); + return; + } + + match std::thread::Builder::new() + .name("Memory Watchdog".to_string()) + .spawn(memory_watchdog_loop) + { + Ok(_) => info!("lqosd memory watchdog started"), + Err(err) => warn!("Failed to start lqosd memory watchdog: {err:?}"), + } +} + +fn memory_watchdog_loop() { + let config = MemoryWatchdogConfig::from_env(); + let mut warned_about_pressure = false; + + loop { + std::thread::sleep(Duration::from_secs(CHECK_INTERVAL_SECONDS)); + + let Ok(snapshot) = MemorySnapshot::read() else { + warn!("Unable to sample lqosd memory state from /proc"); + continue; + }; + + if let Some(reason) = config.critical_reason(&snapshot) { + log_critical_memory_snapshot(&snapshot, &config, &reason); + std::thread::sleep(Duration::from_millis(250)); + std::process::exit(WATCHDOG_EXIT_CODE); + } + + if snapshot.mem_available_bytes < PRESSURE_WARNING_AVAILABLE_BYTES { + if !warned_about_pressure { + warn!( + "lqosd memory pressure warning: mem_available={} process_rss={} process_swap={} process_total={} threads={}", + format_bytes(snapshot.mem_available_bytes), + format_bytes(snapshot.process_rss_bytes), + format_bytes(snapshot.process_swap_bytes), + format_bytes(snapshot.process_total_bytes()), + snapshot.thread_count, + ); + warned_about_pressure = true; + } + } else { + warned_about_pressure = false; + } + } +} + +fn log_critical_memory_snapshot( + snapshot: &MemorySnapshot, + config: &MemoryWatchdogConfig, + reason: &str, +) { + error!( + "lqosd memory watchdog restarting daemon: reason={} mem_available={} mem_total={} process_rss={} process_swap={} process_total={} threads={} thresholds=min_available:{} min_process:{} max_process:{} max_swap:{}", + reason, + format_bytes(snapshot.mem_available_bytes), + format_bytes(snapshot.mem_total_bytes), + format_bytes(snapshot.process_rss_bytes), + format_bytes(snapshot.process_swap_bytes), + format_bytes(snapshot.process_total_bytes()), + snapshot.thread_count, + format_bytes(config.min_available_bytes), + format_bytes(config.min_process_bytes), + format_bytes(config.max_process_bytes), + format_bytes(config.max_swap_bytes), + ); + error!( + "lqosd memory watchdog diagnostics: flows_tracked={} bus_requests={} time_to_poll_hosts_us={} high_watermark_down={} high_watermark_up={}", + FLOWS_TRACKED.load(Ordering::Relaxed), + BUS_REQUESTS.load(Ordering::Relaxed), + TIME_TO_POLL_HOSTS.load(Ordering::Relaxed), + HIGH_WATERMARK.get_down(), + HIGH_WATERMARK.get_up(), + ); +} + +#[derive(Debug)] +struct MemoryWatchdogConfig { + min_available_bytes: u64, + min_process_bytes: u64, + max_process_bytes: u64, + max_swap_bytes: u64, +} + +impl MemoryWatchdogConfig { + fn from_env() -> Self { + Self { + min_available_bytes: env_mib( + "LQOSD_MEMORY_WATCHDOG_MIN_AVAILABLE_MB", + DEFAULT_MIN_AVAILABLE_BYTES, + ), + min_process_bytes: env_mib( + "LQOSD_MEMORY_WATCHDOG_MIN_PROCESS_MB", + DEFAULT_MIN_PROCESS_BYTES, + ), + max_process_bytes: env_mib( + "LQOSD_MEMORY_WATCHDOG_MAX_PROCESS_MB", + DEFAULT_MAX_PROCESS_BYTES, + ), + max_swap_bytes: env_mib("LQOSD_MEMORY_WATCHDOG_MAX_SWAP_MB", DEFAULT_MAX_SWAP_BYTES), + } + } + + fn critical_reason(&self, snapshot: &MemorySnapshot) -> Option { + if snapshot.process_swap_bytes >= self.max_swap_bytes { + return Some(format!( + "process swap {} reached limit {}", + format_bytes(snapshot.process_swap_bytes), + format_bytes(self.max_swap_bytes) + )); + } + + if snapshot.process_total_bytes() >= self.max_process_bytes { + return Some(format!( + "process rss+swap {} reached limit {}", + format_bytes(snapshot.process_total_bytes()), + format_bytes(self.max_process_bytes) + )); + } + + if snapshot.mem_available_bytes < self.min_available_bytes + && snapshot.process_total_bytes() >= self.min_process_bytes + { + return Some(format!( + "available memory {} below limit {} while lqosd uses {}", + format_bytes(snapshot.mem_available_bytes), + format_bytes(self.min_available_bytes), + format_bytes(snapshot.process_total_bytes()) + )); + } + + None + } +} + +#[derive(Debug, PartialEq, Eq)] +struct MemorySnapshot { + mem_total_bytes: u64, + mem_available_bytes: u64, + process_rss_bytes: u64, + process_swap_bytes: u64, + thread_count: u64, +} + +impl MemorySnapshot { + fn read() -> anyhow::Result { + let meminfo = parse_proc_key_values(&std::fs::read_to_string("/proc/meminfo")?); + let status = parse_proc_key_values(&std::fs::read_to_string("/proc/self/status")?); + + Ok(Self { + mem_total_bytes: read_required_kb(&meminfo, "MemTotal")?, + mem_available_bytes: read_required_kb(&meminfo, "MemAvailable")?, + process_rss_bytes: read_required_kb(&status, "VmRSS")?, + process_swap_bytes: read_optional_kb(&status, "VmSwap"), + thread_count: read_optional_raw(&status, "Threads"), + }) + } + + fn process_total_bytes(&self) -> u64 { + self.process_rss_bytes + .saturating_add(self.process_swap_bytes) + } +} + +fn parse_proc_key_values(input: &str) -> HashMap { + input + .lines() + .filter_map(|line| { + let (key, rest) = line.split_once(':')?; + let value = rest.split_whitespace().next()?.parse::().ok()?; + Some((key.to_string(), value)) + }) + .collect() +} + +fn read_required_kb(values: &HashMap, key: &str) -> anyhow::Result { + values + .get(key) + .copied() + .map(kib) + .ok_or_else(|| anyhow::anyhow!("{key} missing from /proc memory data")) +} + +fn read_optional_kb(values: &HashMap, key: &str) -> u64 { + values.get(key).copied().map(kib).unwrap_or(0) +} + +fn read_optional_raw(values: &HashMap, key: &str) -> u64 { + values.get(key).copied().unwrap_or(0) +} + +fn env_flag_enabled(name: &str) -> bool { + std::env::var(name) + .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) + .unwrap_or(false) +} + +fn env_mib(name: &str, default_bytes: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|value| value.parse::().ok()) + .map(mib) + .unwrap_or(default_bytes) +} + +const fn kib(value: u64) -> u64 { + value * 1024 +} + +const fn mib(value: u64) -> u64 { + value * 1024 * 1024 +} + +const fn gib(value: u64) -> u64 { + value * 1024 * 1024 * 1024 +} + +fn format_bytes(bytes: u64) -> String { + let mib_value = bytes as f64 / mib(1) as f64; + format!("{mib_value:.1} MiB") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn proc_parser_reads_kb_and_raw_values() { + let parsed = parse_proc_key_values( + "MemTotal: 15458992 kB\nMemAvailable: 12843176 kB\nThreads:\t44\n", + ); + + assert_eq!( + read_required_kb(&parsed, "MemTotal").unwrap(), + 15_458_992 * 1024 + ); + assert_eq!(read_optional_raw(&parsed, "Threads"), 44); + assert_eq!(read_optional_kb(&parsed, "VmSwap"), 0); + } + + #[test] + fn watchdog_restarts_on_process_swap_limit() { + let config = MemoryWatchdogConfig { + min_available_bytes: mib(2_304), + min_process_bytes: mib(1_024), + max_process_bytes: gib(4), + max_swap_bytes: gib(1), + }; + let snapshot = MemorySnapshot { + mem_total_bytes: gib(16), + mem_available_bytes: gib(12), + process_rss_bytes: mib(900), + process_swap_bytes: gib(2), + thread_count: 44, + }; + + assert!( + config + .critical_reason(&snapshot) + .unwrap() + .contains("process swap") + ); + } + + #[test] + fn watchdog_restarts_on_low_available_memory_with_large_lqosd() { + let config = MemoryWatchdogConfig { + min_available_bytes: mib(2_304), + min_process_bytes: mib(1_024), + max_process_bytes: gib(4), + max_swap_bytes: gib(1), + }; + let snapshot = MemorySnapshot { + mem_total_bytes: gib(16), + mem_available_bytes: mib(1_900), + process_rss_bytes: mib(1_500), + process_swap_bytes: 0, + thread_count: 44, + }; + + assert!( + config + .critical_reason(&snapshot) + .unwrap() + .contains("available memory") + ); + } + + #[test] + fn watchdog_ignores_low_memory_when_lqosd_is_small() { + let config = MemoryWatchdogConfig { + min_available_bytes: mib(2_304), + min_process_bytes: mib(1_024), + max_process_bytes: gib(4), + max_swap_bytes: gib(1), + }; + let snapshot = MemorySnapshot { + mem_total_bytes: gib(16), + mem_available_bytes: mib(1_900), + process_rss_bytes: mib(256), + process_swap_bytes: 0, + thread_count: 44, + }; + + assert!(config.critical_reason(&snapshot).is_none()); + } +} From 6e91d72c4eaea37214b4b7e5866750737bd99458 Mon Sep 17 00:00:00 2001 From: rchac Date: Tue, 5 May 2026 22:08:28 +0000 Subject: [PATCH 10/20] changes --- docs-es/v2.0/integrations-reference-es.md | 2 +- docs/v2.0/integrations-netzur.md | 2 +- docs/v2.0/integrations-reference.md | 2 +- src/build_dpkg.sh | 1 + src/integrationUtils.py | 10 + src/integrationVISP.py | 19 +- src/mikrotikFindIPv6.py | 256 +++++++++++--- src/test_mikrotik_find_ipv6.py | 395 ++++++++++++++++++++++ 8 files changed, 618 insertions(+), 69 deletions(-) create mode 100644 src/integrationUtils.py create mode 100644 src/test_mikrotik_find_ipv6.py diff --git a/docs-es/v2.0/integrations-reference-es.md b/docs-es/v2.0/integrations-reference-es.md index 60b472ef..aa8651fc 100644 --- a/docs-es/v2.0/integrations-reference-es.md +++ b/docs-es/v2.0/integrations-reference-es.md @@ -151,7 +151,7 @@ use_mikrotik_ipv6 = false - `api_key` es el token Bearer generado dentro de Netzur. - `api_url` debe devolver un JSON con los arreglos `zones` (convertidos en sitios) y `customers` (convertidos en circuitos y dispositivos). - `timeout_secs` permite incrementar el tiempo de espera de la petición cuando el API responde lentamente (por defecto 60 segundos). -- `use_mikrotik_ipv6` agrega prefijos IPv6 obtenidos de `/etc/libreqos/mikrotik_ipv6.toml`. +- `use_mikrotik_ipv6` agrega prefijos IPv6 obtenidos de `/etc/libreqos/mikrotik_ipv6.toml`. LibreQoS relaciona IPv4 e IPv6 por dirección MAC usando datos de ARP, DHCPv4, DHCPv6 y vecinos IPv6 de MikroTik. Para una importación manual: diff --git a/docs/v2.0/integrations-netzur.md b/docs/v2.0/integrations-netzur.md index a4fb385b..3e18bd04 100644 --- a/docs/v2.0/integrations-netzur.md +++ b/docs/v2.0/integrations-netzur.md @@ -15,7 +15,7 @@ use_mikrotik_ipv6 = false - `api_key` is the Bearer token generated inside Netzur. - `api_url` must return JSON containing `zones` (mapped to sites) and `customers` (mapped to client circuits and devices). - `timeout_secs` overrides the default HTTP timeout (60 seconds) when the API is slow. -- `use_mikrotik_ipv6` enriches subscriber devices with IPv6 prefixes discovered via `/etc/libreqos/mikrotik_ipv6.toml`. +- `use_mikrotik_ipv6` enriches subscriber devices with IPv6 prefixes discovered via `/etc/libreqos/mikrotik_ipv6.toml`. LibreQoS matches IPv4 and IPv6 by MAC address using MikroTik ARP, DHCPv4, DHCPv6, and IPv6 neighbor data. Run a manual import with: diff --git a/docs/v2.0/integrations-reference.md b/docs/v2.0/integrations-reference.md index 2fd00ebe..11c79949 100644 --- a/docs/v2.0/integrations-reference.md +++ b/docs/v2.0/integrations-reference.md @@ -153,7 +153,7 @@ use_mikrotik_ipv6 = false - `api_key` is the Bearer token generated inside Netzur. - `api_url` must return JSON containing `zones` (mapped to sites) and `customers` (mapped to client circuits and devices). - `timeout_secs` overrides the default HTTP timeout (60 seconds) when the API is slow. -- `use_mikrotik_ipv6` enriches subscriber devices with IPv6 prefixes discovered via `/etc/libreqos/mikrotik_ipv6.toml`. +- `use_mikrotik_ipv6` enriches subscriber devices with IPv6 prefixes discovered via `/etc/libreqos/mikrotik_ipv6.toml`. LibreQoS matches IPv4 and IPv6 by MAC address using MikroTik ARP, DHCPv4, DHCPv6, and IPv6 neighbor data. Run a manual import with: diff --git a/src/build_dpkg.sh b/src/build_dpkg.sh index 45e2c74c..32d8a474 100755 --- a/src/build_dpkg.sh +++ b/src/build_dpkg.sh @@ -27,6 +27,7 @@ LQOS_FILES=( integrationNetzur.py integrationRestHttp.py integrationSonar.py + integrationUtils.py integrationSplynx.py integrationVISP.py integrationWISPGate.py diff --git a/src/integrationUtils.py b/src/integrationUtils.py new file mode 100644 index 00000000..38d3b479 --- /dev/null +++ b/src/integrationUtils.py @@ -0,0 +1,10 @@ +from typing import Any + + +def normalize_mac(mac: Any) -> str: + if mac is None: + return "" + value = str(mac).strip() + if not value: + return "" + return value.replace(":", "").replace("-", "").upper() diff --git a/src/integrationVISP.py b/src/integrationVISP.py index 4eb3ccb4..680bc39b 100644 --- a/src/integrationVISP.py +++ b/src/integrationVISP.py @@ -34,6 +34,7 @@ from integrationCommon import ( apply_client_bandwidth_multiplier, isIntegrationOutputIpAllowed, ) +from integrationUtils import normalize_mac TOKEN_URL = "https://data.visp.net/token" @@ -71,16 +72,6 @@ def _unit_to_mbps(value: Any, unit: Any) -> Optional[float]: return v -def _normalize_mac(mac: Any) -> str: - if mac is None: - return "" - s = str(mac).strip() - if not s: - return "" - # VISP sometimes returns MACs without separators. Keep uppercase hex. - return s.replace(":", "").replace("-", "").upper() - - def _safe_isp_id_from_token_payload(payload: Dict[str, Any]) -> Optional[int]: isp_ids = payload.get("ispId") if isinstance(isp_ids, list) and isp_ids: @@ -505,7 +496,7 @@ def _extract_mac_candidates(row: Dict[str, Any]) -> List[str]: if not isinstance(equipment_data, dict): equipment_data = {} for key in ("Fiber MAC", "mac_address", "Ethernet MAC", "WAN MAC", "MAC Addr", "radio_mac"): - mac = _normalize_mac(equipment_data.get(key)) + mac = normalize_mac(equipment_data.get(key)) if mac and mac not in seen: seen.add(mac) candidates.append(mac) @@ -921,7 +912,7 @@ def createShaper() -> None: if eq_id > 0: equipment_ids.add(eq_id) for key in ("mac_address", "radio_mac"): - mac = _normalize_mac(eq.get(key)) + mac = normalize_mac(eq.get(key)) if mac: preferred_macs.add(mac) for ws in ws_list: @@ -947,7 +938,7 @@ def createShaper() -> None: if not ipv4s: counters["skipped_no_ip"] += 1 continue - mac = _normalize_mac(ws.get("mac_address")) + mac = normalize_mac(ws.get("mac_address")) if mac: preferred_macs.add(mac) add_service_record( @@ -1032,7 +1023,7 @@ def createShaper() -> None: ipv4s = list(ip_cache) except Exception: ipv4s = list(ip_cache) - mac = _normalize_mac(service_details.get("mac_address")) + mac = normalize_mac(service_details.get("mac_address")) if not mac and preferred_macs: mac = sorted(preferred_macs)[0] diff --git a/src/mikrotikFindIPv6.py b/src/mikrotikFindIPv6.py index afc4874d..6682c499 100644 --- a/src/mikrotikFindIPv6.py +++ b/src/mikrotikFindIPv6.py @@ -1,12 +1,39 @@ #!/usr/bin/python3 import csv +import ipaddress import json import sys +from enum import IntEnum +from typing import NamedTuple import routeros_api +from integrationUtils import normalize_mac from liblqos_python import load_mikrotik_ipv6_routers_json, mikrotik_ipv6_config_path +class SourcePriority(IntEnum): + ARP = 0 + NEIGHBOR = 1 + DHCP = 2 + +class Ipv6Rank(IntEnum): + INVALID = -1 + LINK_LOCAL = 0 + HOST = 1 + PREFIX = 2 + +class IpRecord(NamedTuple): + source_priority: SourcePriority + address_rank: int + canonical_address: str + address: str + ambiguous: bool = False + +class ClientAddressRecord(NamedTuple): + canonical_address: str + address: str + ambiguous: bool = False + def _load_legacy_router_csv(csv_path): router_list = [] with open(csv_path) as csv_file: @@ -32,13 +59,182 @@ def _load_router_list(configPath=None): return _load_legacy_router_csv(configPath) raise ValueError("Explicit Mikrotik IPv6 config overrides must currently point to a legacy .csv file") -def pullMikrotikIPv6(configPath=None): - import routeros_api +def _parse_ip_address(address): + try: + return ipaddress.ip_address(address.split('/')[0]) + except (AttributeError, ValueError): + return None +def _canonical_ip_text(address): + parsed_address = _parse_ip_address(address) + if parsed_address is None: + return None + return str(parsed_address) + +def _canonical_ip_network_text(address): + try: + network = ipaddress.ip_network(address, strict=False) + except (AttributeError, ValueError): + return None + return network.with_prefixlen + +def _entry_value(entry, key): + if not isinstance(entry, dict): + return None + return entry.get(key) + +def _record_address_by_mac(entry, mac_to_ipv4, mac_to_ipv6): + mac = _entry_value(entry, 'mac-address') + address = _entry_value(entry, 'address') + parsed_address = _parse_ip_address(address) + if mac is None or parsed_address is None: + return + normalized_mac = normalize_mac(mac) + if parsed_address.version == 4: + _record_ipv4_for_mac(mac_to_ipv4, normalized_mac, address, source_priority=SourcePriority.ARP) + else: + _record_ipv6_for_mac(mac_to_ipv6, normalized_mac, address, source_priority=SourcePriority.ARP) + +def _record_dhcp4_address_by_mac(entry, mac_to_ipv4): + mac = _entry_value(entry, 'mac-address') + address = _entry_value(entry, 'address') + parsed_address = _parse_ip_address(address) + if mac is None or parsed_address is None or parsed_address.version != 4: + return + _record_ipv4_for_mac(mac_to_ipv4, normalize_mac(mac), address, source_priority=SourcePriority.DHCP) + +def _mac_from_dhcpv6_duid(duid): + duid_hex = str(duid).strip().replace(":", "").replace("-", "").lower() + if duid_hex.startswith("0x"): + duid_hex = duid_hex[2:] + try: + duid_bytes = bytes.fromhex(duid_hex) + except ValueError: + return None + if len(duid_bytes) == 14 and duid_bytes[0:2] == b'\x00\x01': + return normalize_mac(duid_bytes[8:14].hex()) + if len(duid_bytes) == 10 and duid_bytes[0:2] == b'\x00\x03': + return normalize_mac(duid_bytes[4:10].hex()) + return None + +def _record_ipv4_for_mac(mac_to_ipv4, mac, address, source_priority): + if not mac or not address: + return + parsed_address = _parse_ip_address(address) + if parsed_address is None: + return + current = mac_to_ipv4.get(mac) + candidate = IpRecord(source_priority, int(parsed_address), str(parsed_address), address) + if current is None or source_priority > current.source_priority: + mac_to_ipv4[mac] = candidate + elif source_priority == current.source_priority and candidate.canonical_address != current.canonical_address: + mac_to_ipv4[mac] = current._replace(ambiguous=True) + +def _ipv6_rank(address): + try: + network = ipaddress.ip_network(address, strict=False) + except ValueError: + return Ipv6Rank.INVALID + if network.version != 6: + return Ipv6Rank.INVALID + ip = network.network_address + if ip.is_unspecified or ip.is_loopback or ip.is_multicast: + return Ipv6Rank.INVALID + if ip.is_link_local: + address_priority = Ipv6Rank.LINK_LOCAL + elif network.prefixlen < 128: + address_priority = Ipv6Rank.PREFIX + else: + address_priority = Ipv6Rank.HOST + return address_priority + +def _record_ipv6_for_mac(mac_to_ipv6, mac, address, source_priority): + if not mac or not address: + return + address_rank = _ipv6_rank(address) + if address_rank in (Ipv6Rank.INVALID, Ipv6Rank.LINK_LOCAL): + return + canonical_address = _canonical_ip_network_text(address) + if canonical_address is None: + return + current = mac_to_ipv6.get(mac) + candidate = IpRecord(source_priority, address_rank, canonical_address, address) + if current is None or source_priority > current.source_priority: + mac_to_ipv6[mac] = candidate + elif source_priority == current.source_priority: + if address_rank > current.address_rank: + mac_to_ipv6[mac] = candidate + elif address_rank == current.address_rank and canonical_address != current.canonical_address: + mac_to_ipv6[mac] = current._replace(ambiguous=True) + +def _record_client_address_binding(client_address_to_ipv6, client_address, address): + if not client_address or not address or _ipv6_rank(address) in (Ipv6Rank.INVALID, Ipv6Rank.LINK_LOCAL): + return + canonical_address = _canonical_ip_network_text(address) + if canonical_address is None: + return + current = client_address_to_ipv6.get(client_address) + if current is None: + client_address_to_ipv6[client_address] = ClientAddressRecord(canonical_address, address) + elif canonical_address != current.canonical_address: + client_address_to_ipv6[client_address] = current._replace(ambiguous=True) + +def _build_ipv4_to_ipv6_map(arp_entries, dhcp4_entries, dhcp6_bindings, neighbor6_entries): + ipv4_to_ipv6 = {} + mac_to_ipv4 = {} + mac_to_ipv6 = {} + client_address_to_ipv6 = {} + + for entry in arp_entries: + _record_address_by_mac(entry, mac_to_ipv4, mac_to_ipv6) + for entry in dhcp4_entries: + _record_dhcp4_address_by_mac(entry, mac_to_ipv4) + for entry in dhcp6_bindings: + mac = _mac_from_dhcpv6_duid(_entry_value(entry, 'duid') or '') + address = _entry_value(entry, 'address') + if mac is not None and address: + _record_ipv6_for_mac(mac_to_ipv6, mac, address, source_priority=SourcePriority.DHCP) + elif _entry_value(entry, 'client-address') and address: + client_address = _canonical_ip_text(_entry_value(entry, 'client-address')) + if client_address is not None: + _record_client_address_binding(client_address_to_ipv6, client_address, address) + for entry in neighbor6_entries: + mac = _entry_value(entry, 'mac-address') + address = _entry_value(entry, 'address') + if mac is None or _parse_ip_address(address) is None: + continue + normalized_mac = normalize_mac(mac) + neighbor_address = _canonical_ip_text(address) + mapped_ipv6_record = client_address_to_ipv6.get(neighbor_address) + if mapped_ipv6_record is not None and mapped_ipv6_record.ambiguous: + print('Skipped ambiguous DHCPv6 client-address entries for ' + neighbor_address, file=sys.stderr) + _record_ipv6_for_mac(mac_to_ipv6, normalized_mac, address, source_priority=SourcePriority.NEIGHBOR) + elif mapped_ipv6_record is not None: + _record_ipv6_for_mac(mac_to_ipv6, normalized_mac, mapped_ipv6_record.address, source_priority=SourcePriority.DHCP) + else: + _record_ipv6_for_mac(mac_to_ipv6, normalized_mac, address, source_priority=SourcePriority.NEIGHBOR) + + for mac, ipv6_record in mac_to_ipv6.items(): + if ipv6_record.ambiguous: + print('Skipped ambiguous IPv6 entries for MAC ' + mac, file=sys.stderr) + continue + ipv6 = ipv6_record.address + try: + ipv4_record = mac_to_ipv4[mac] + if ipv4_record.ambiguous: + print('Skipped ambiguous IPv4 entries for MAC ' + mac, file=sys.stderr) + continue + ipv4 = ipv4_record.address + ipv4_to_ipv6[ipv4] = ipv6 + except KeyError: + print('Failed to find associated IPv4 for ' + ipv6, file=sys.stderr) + + return ipv4_to_ipv6 + +def pullMikrotikIPv6(configPath=None): ipv4ToIPv6 = {} routerList = _load_router_list(configPath) for router in routerList: - RouterName = router['name'] IP = router['host'] inputUsername = router['username'] inputPassword = router['password'] @@ -47,62 +243,18 @@ def pullMikrotikIPv6(configPath=None): plaintext_login = bool(router.get('plaintext_login', True)) connection = routeros_api.RouterOsApiPool(IP, username=inputUsername, password=inputPassword, port=apiPort, use_ssl=use_ssl, ssl_verify=False, ssl_verify_hostname=False, plaintext_login=plaintext_login) api = connection.get_api() - macToIPv4 = {} - macToIPv6 = {} - clientAddressToIPv6 = {} - # list_dhcp4 = api.get_resource('/ip/dhcp-server/lease') - # entries = list_dhcp4.get() - # for entry in entries: - # try: - # macToIPv4[entry['mac-address']] = entry['address'] - # except: - # pass list_arp4 = api.get_resource('/ip/arp') - entries = list_arp4.get() - for entry in entries: - try: - macToIPv4[entry['mac-address']] = entry['address'] - except: - pass + arp_entries = list_arp4.get() list_dhcp4 = api.get_resource('/ip/dhcp-server/lease') - entries = list_dhcp4.get() - for entry in entries: - try: - macToIPv4[entry['mac-address']] = entry['address'] - except: - pass + dhcp4_entries = list_dhcp4.get() list_binding6 = api.get_resource('/ipv6/dhcp-server/binding') - entries = list_binding6.get() - for entry in entries: - if len(entry['duid']) == 14: - mac = entry['duid'][2:14].upper() - macNew = mac[0:2] + ':' + mac[2:4] + ':' + mac[4:6] + ':' + mac[6:8] + ':' + mac[8:10] + ':' + mac[10:12] - macToIPv6[macNew] = entry['address'] - else: - try: - clientAddressToIPv6[entry['client-address']] = entry['address'] - except: - pass + dhcp6_bindings = list_binding6.get() list_neighbor6 = api.get_resource('/ipv6/neighbor') - entries = list_neighbor6.get() - for entry in entries: - try: - realIPv6 = clientAddressToIPv6[entry['address']] - macToIPv6[entry['mac-address']] = realIPv6 - except: - pass - for mac, ipv6 in macToIPv6.items(): - try: - ipv4 = macToIPv4[mac] - ipv4ToIPv6[ipv4] = ipv6 - except: - print('Failed to find associated IPv4 for ' + ipv6, file=sys.stderr) + neighbor6_entries = list_neighbor6.get() + ipv4ToIPv6.update(_build_ipv4_to_ipv6_map(arp_entries, dhcp4_entries, dhcp6_bindings, neighbor6_entries)) return json.dumps(ipv4ToIPv6) -def pullMikrotikIPv6_Mock(CsvPath): - return "{\n\"172.29.200.2\": \"2602:fdca:800:1500::/56\"\n}" - if __name__ == '__main__': # If the first argument is a string, it's treated as a legacy CSV path override. if len(sys.argv) > 1 and sys.argv[1] == '--show-config': diff --git a/src/test_mikrotik_find_ipv6.py b/src/test_mikrotik_find_ipv6.py new file mode 100644 index 00000000..932a52e4 --- /dev/null +++ b/src/test_mikrotik_find_ipv6.py @@ -0,0 +1,395 @@ +import contextlib +import io +import json +import sys +import types +import unittest + + +sys.modules.setdefault("routeros_api", types.SimpleNamespace()) +sys.modules.setdefault( + "liblqos_python", + types.SimpleNamespace( + load_mikrotik_ipv6_routers_json=lambda: "[]", + mikrotik_ipv6_config_path=lambda: "/etc/libreqos/mikrotik_ipv6.toml", + ), +) + +import mikrotikFindIPv6 +from mikrotikFindIPv6 import _build_ipv4_to_ipv6_map, _mac_from_dhcpv6_duid + + +class MikrotikFindIPv6Tests(unittest.TestCase): + def test_pull_mikrotik_ipv6_fetches_resources_and_returns_json(self): + resources = { + "/ip/arp": [ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.30"}, + ], + "/ip/dhcp-server/lease": [], + "/ipv6/dhcp-server/binding": [ + {"duid": "00030001AABBCCDDEEFF", "address": "2001:db8:30::/56"}, + ], + "/ipv6/neighbor": [ + {"mac-address": "00:11:22:33:44:55", "address": "2001:db8::99"}, + ], + } + + class Resource: + def __init__(self, entries): + self.entries = entries + + def get(self): + return self.entries + + class Api: + def get_resource(self, name): + return Resource(resources[name]) + + class Pool: + def __init__(self, *args, **kwargs): + pass + + def get_api(self): + return Api() + + original_pool = getattr(mikrotikFindIPv6.routeros_api, "RouterOsApiPool", None) + original_load_router_list = mikrotikFindIPv6._load_router_list + mikrotikFindIPv6.routeros_api.RouterOsApiPool = Pool + mikrotikFindIPv6._load_router_list = lambda configPath=None: [ + { + "name": "edge", + "host": "198.51.100.1", + "username": "admin", + "password": "secret", + } + ] + stderr = io.StringIO() + try: + with contextlib.redirect_stderr(stderr): + result = json.loads(mikrotikFindIPv6.pullMikrotikIPv6()) + finally: + if original_pool is None: + delattr(mikrotikFindIPv6.routeros_api, "RouterOsApiPool") + else: + mikrotikFindIPv6.routeros_api.RouterOsApiPool = original_pool + mikrotikFindIPv6._load_router_list = original_load_router_list + + self.assertEqual(result, {"192.0.2.30": "2001:db8:30::/56"}) + self.assertIn("Failed to find associated IPv4 for 2001:db8::99", stderr.getvalue()) + + def test_dhcpv6_duid_ll_extracts_mac(self): + self.assertEqual(_mac_from_dhcpv6_duid("00030001AABBCCDDEEFF"), "AABBCCDDEEFF") + self.assertEqual(_mac_from_dhcpv6_duid("0x00030001aabbccddeeff"), "AABBCCDDEEFF") + self.assertEqual(_mac_from_dhcpv6_duid("00:03:00:01:aa:bb:cc:dd:ee:ff"), "AABBCCDDEEFF") + + def test_dhcpv6_duid_llt_extracts_mac(self): + self.assertEqual(_mac_from_dhcpv6_duid("0001000100000000AABBCCDDEEFF"), "AABBCCDDEEFF") + + def test_neighbor_ipv6_matches_ipv4_by_mac(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "aa:bb:cc:dd:ee:ff", "address": "192.0.2.10"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::10"}, + ], + ) + + self.assertEqual(result, {"192.0.2.10": "2001:db8::10"}) + + def test_link_local_neighbor_ipv6_is_not_emitted(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "aa:bb:cc:dd:ee:ff", "address": "192.0.2.11"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "fe80::11"}, + ], + ) + + self.assertEqual(result, {}) + + def test_unspecified_ipv6_prefix_is_not_emitted(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "aa:bb:cc:dd:ee:ff", "address": "192.0.2.12"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[ + {"duid": "00030001AABBCCDDEEFF", "address": "::/64"}, + ], + neighbor6_entries=[], + ) + + self.assertEqual(result, {}) + + def test_neighbor_client_address_preserves_dhcpv6_prefix(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.10"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[ + {"client-address": "fe80:0:0:0:0:0:0:1", "address": "2001:db8:100::/56"}, + ], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "fe80::1"}, + ], + ) + + self.assertEqual(result, {"192.0.2.10": "2001:db8:100::/56"}) + + def test_arp_ipv6_matches_dhcp4_by_mac(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::20"}, + ], + dhcp4_entries=[ + {"mac-address": "aa:bb:cc:dd:ee:ff", "address": "192.0.2.20"}, + ], + dhcp6_bindings=[], + neighbor6_entries=[], + ) + + self.assertEqual(result, {"192.0.2.20": "2001:db8::20"}) + + def test_duplicate_ipv6_entries_prefer_non_link_local_prefix(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.40"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "fe80::1"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8:40::/56"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::40"}, + ], + ) + + self.assertEqual(result, {"192.0.2.40": "2001:db8:40::/56"}) + + def test_duplicate_same_priority_ipv6_entries_are_ambiguous(self): + neighbors = [ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::50"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::9"}, + ] + for neighbor6_entries in (neighbors, list(reversed(neighbors))): + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.50"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[], + neighbor6_entries=neighbor6_entries, + ) + + self.assertEqual(result, {}) + self.assertIn("Skipped ambiguous IPv6 entries for MAC AABBCCDDEEFF", stderr.getvalue()) + + def test_neighbor_ipv6_recovers_from_ambiguous_arp_ipv6(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.51"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::51"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::52"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::53"}, + ], + ) + + self.assertEqual(result, {"192.0.2.51": "2001:db8::53"}) + + def test_neighbor_ipv6_preferred_over_arp_ipv6_for_same_mac(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.52"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::99"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::52"}, + ], + ) + + self.assertEqual(result, {"192.0.2.52": "2001:db8::52"}) + + def test_neighbor_client_address_matches_cidr_text(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.53"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[ + {"client-address": "fe80::53/128", "address": "2001:db8:53::/56"}, + ], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "fe80:0:0:0:0:0:0:53"}, + ], + ) + + self.assertEqual(result, {"192.0.2.53": "2001:db8:53::/56"}) + + def test_bad_duid_falls_back_to_client_address(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.54"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[ + {"duid": "not-hex", "client-address": "fe80::54", "address": "2001:db8:54::/56"}, + ], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "fe80::54"}, + ], + ) + + self.assertEqual(result, {"192.0.2.54": "2001:db8:54::/56"}) + + def test_duplicate_client_address_bindings_are_ambiguous(self): + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.57"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[ + {"client-address": "fe80::57", "address": "2001:db8:57::/56"}, + {"client-address": "fe80::57", "address": "2001:db8:58::/56"}, + ], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "fe80::57/128"}, + ], + ) + + self.assertEqual(result, {}) + self.assertIn("Skipped ambiguous DHCPv6 client-address entries for fe80::57", stderr.getvalue()) + + def test_equivalent_client_address_bindings_are_not_ambiguous(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.58"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[ + {"client-address": "fe80::58", "address": "2001:db8:58::/56"}, + {"client-address": "fe80:0:0:0:0:0:0:58", "address": "2001:db8:58:0::/56"}, + ], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "fe80::58"}, + ], + ) + + self.assertEqual(result, {"192.0.2.58": "2001:db8:58::/56"}) + + def test_dhcpv6_binding_preferred_over_arp_and_neighbor_ipv6(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.55"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::55"}, + ], + dhcp4_entries=[], + dhcp6_bindings=[ + {"duid": "00030001AABBCCDDEEFF", "address": "2001:db8:55::/56"}, + ], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::56"}, + ], + ) + + self.assertEqual(result, {"192.0.2.55": "2001:db8:55::/56"}) + + def test_dhcp4_ipv4_preferred_over_arp_for_same_mac(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.60"}, + ], + dhcp4_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.61"}, + ], + dhcp6_bindings=[], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::60"}, + ], + ) + + self.assertEqual(result, {"192.0.2.61": "2001:db8::60"}) + + def test_dhcp4_ipv4_recovers_from_ambiguous_arp_ipv4(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.62"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.63"}, + ], + dhcp4_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.64"}, + ], + dhcp6_bindings=[], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::64"}, + ], + ) + + self.assertEqual(result, {"192.0.2.64": "2001:db8::64"}) + + def test_duplicate_same_priority_ipv4_entries_are_ambiguous(self): + arp_entries = [ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.9"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.10"}, + ] + for entries in (arp_entries, list(reversed(arp_entries))): + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + result = _build_ipv4_to_ipv6_map( + arp_entries=entries, + dhcp4_entries=[], + dhcp6_bindings=[], + neighbor6_entries=[ + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::60"}, + ], + ) + + self.assertEqual(result, {}) + self.assertIn("Skipped ambiguous IPv4 entries for MAC AABBCCDDEEFF", stderr.getvalue()) + + def test_malformed_routeros_rows_are_ignored_while_valid_rows_match(self): + result = _build_ipv4_to_ipv6_map( + arp_entries=[ + None, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "not an address"}, + {"address": "192.0.2.70"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "192.0.2.70"}, + ], + dhcp4_entries=[ + ["not", "a", "dict"], + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "still not an address"}, + ], + dhcp6_bindings=[ + {"duid": "not-hex", "address": "2001:db8:70::/56"}, + {"duid": "00030001AABBCCDDEEFF", "address": "not an address"}, + {"client-address": "not an address", "address": "2001:db8:71::/56"}, + ], + neighbor6_entries=[ + None, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "also not an address"}, + {"address": "2001:db8::70"}, + {"mac-address": "AA:BB:CC:DD:EE:FF", "address": "2001:db8::70"}, + ], + ) + + self.assertEqual(result, {"192.0.2.70": "2001:db8::70"}) + + +if __name__ == "__main__": + unittest.main() From dc0f37a307bbfd83c52337a5e10a4804a805efe1 Mon Sep 17 00:00:00 2001 From: rchac Date: Tue, 5 May 2026 22:21:18 +0000 Subject: [PATCH 11/20] changes --- docs/v2.0/topology-data-flow.md | 4 + src/rust/lqos_topology/src/lib.rs | 252 +++++++++++++++++++++++++----- 2 files changed, 220 insertions(+), 36 deletions(-) diff --git a/docs/v2.0/topology-data-flow.md b/docs/v2.0/topology-data-flow.md index aedb8ad1..6dffd0e9 100644 --- a/docs/v2.0/topology-data-flow.md +++ b/docs/v2.0/topology-data-flow.md @@ -139,6 +139,10 @@ flowchart LR | `lqos_overrides.json` | operator / WebUI / CLI | scheduler / topology runtime | durable operator intent | Yes | Yes | Yes | | `topology_attachment_health_state.json` | topology probes | Topology Manager / debug pages | runtime attachment health | Runtime state | Runtime state | No | +## Runtime Fallback Warnings + +When a circuit references a parent or anchor that is not present in the queue-visible effective topology, LibreQoS still shapes the circuit under a generated parent node. Scheduler output summarizes those fallbacks by reason and includes a few example circuits instead of printing one warning per circuit. Review the examples in Topology Manager or the source integration when those circuits should attach to a specific real site, AP, or parent node. + ## Consumer Map ```{mermaid} diff --git a/src/rust/lqos_topology/src/lib.rs b/src/rust/lqos_topology/src/lib.rs index 9d773dcd..ae8a6da2 100644 --- a/src/rust/lqos_topology/src/lib.rs +++ b/src/rust/lqos_topology/src/lib.rs @@ -33,10 +33,134 @@ use std::thread; use std::time::{SystemTime, UNIX_EPOCH}; const TOPOLOGY_EFFECTIVE_PUBLISH_LOCK_FILENAME: &str = "topology_effective_publish.lock"; +const TOPOLOGY_WARNING_EXAMPLE_LIMIT: usize = 5; type EffectiveQueueAliasMap = HashMap; pub use runtime::start_topology; +#[derive(Default)] +struct TopologyFallbackWarningSummary { + unresolved_exported_anchor_count: usize, + unresolved_exported_anchor_examples: Vec, + missing_anchor_count: usize, + missing_anchor_examples: Vec, + missing_parent_count: usize, + missing_parent_examples: Vec, + non_exported_parent_count: usize, + non_exported_parent_examples: Vec, +} + +impl TopologyFallbackWarningSummary { + fn record_unresolved_exported_anchor( + &mut self, + circuit_id: &str, + anchor_id: &str, + anchor_name: &str, + ) { + self.unresolved_exported_anchor_count += 1; + push_capped_warning_example( + &mut self.unresolved_exported_anchor_examples, + format!("circuit '{circuit_id}' -> anchor '{anchor_id}' ('{anchor_name}')"), + ); + } + + fn record_missing_anchor( + &mut self, + circuit_id: &str, + anchor_id: &str, + anchor_name: Option<&str>, + ) { + self.missing_anchor_count += 1; + let example = if let Some(anchor_name) = anchor_name.filter(|value| !value.is_empty()) { + format!("circuit '{circuit_id}' -> anchor '{anchor_id}' ('{anchor_name}')") + } else { + format!("circuit '{circuit_id}' -> anchor '{anchor_id}'") + }; + push_capped_warning_example(&mut self.missing_anchor_examples, example); + } + + fn record_missing_parent(&mut self, circuit_id: &str, parent_name: &str, parent_id: &str) { + self.missing_parent_count += 1; + push_capped_warning_example( + &mut self.missing_parent_examples, + format!("circuit '{circuit_id}' -> parent '{parent_name}' ({parent_id})"), + ); + } + + fn record_non_exported_parent(&mut self, circuit_id: &str, parent_name: &str, parent_id: &str) { + self.non_exported_parent_count += 1; + push_capped_warning_example( + &mut self.non_exported_parent_examples, + format!("circuit '{circuit_id}' -> parent '{parent_name}' ({parent_id})"), + ); + } + + fn append_to(self, warnings: &mut Vec) { + push_fallback_summary_warning( + warnings, + self.unresolved_exported_anchor_count, + "anchor(s) that did not resolve to exported effective queue nodes", + &self.unresolved_exported_anchor_examples, + ); + push_fallback_summary_warning( + warnings, + self.missing_anchor_count, + "anchor(s) that were not found in the effective topology", + &self.missing_anchor_examples, + ); + push_fallback_summary_warning( + warnings, + self.missing_parent_count, + "parent reference(s) that were not found in the exported effective topology", + &self.missing_parent_examples, + ); + push_fallback_summary_warning( + warnings, + self.non_exported_parent_count, + "resolved parent(s) that were not exported effective queue nodes", + &self.non_exported_parent_examples, + ); + } +} + +fn push_capped_warning_example(examples: &mut Vec, example: String) { + if examples.len() < TOPOLOGY_WARNING_EXAMPLE_LIMIT { + examples.push(example); + } +} + +fn push_fallback_summary_warning( + warnings: &mut Vec, + count: usize, + reason: &str, + examples: &[String], +) { + if count == 0 { + return; + } + let mut warning = format!( + "{count} circuit(s) referenced {reason}. Falling back to generated parent-node shaping." + ); + if !examples.is_empty() { + warning.push_str(" Examples: "); + warning.push_str(&examples.join("; ")); + warning.push('.'); + let omitted_count = count.saturating_sub(examples.len()); + if omitted_count > 0 { + warning.push_str(&format!(" {omitted_count} more omitted.")); + } + } + warnings.push(warning); +} + +fn push_unresolved_runtime_fallback_summary(warnings: &mut Vec, count: usize) { + if count > 0 { + warnings.push(format!( + "{count} circuit(s) are unresolved in runtime topology and will be shaped under generated parent nodes." + )); + } +} + /// One unique probe pair emitted from topology state plus operator intent. #[derive(Clone, Debug, PartialEq, Eq)] pub struct AttachmentProbeSpec { @@ -547,6 +671,7 @@ fn build_shaping_inputs( let mut errors = Vec::new(); let mut warnings = Vec::new(); + let mut fallback_warning_summary = TopologyFallbackWarningSummary::default(); let mut circuits = Vec::::new(); let mut circuits_by_id = HashMap::::new(); @@ -612,12 +737,11 @@ fn build_shaping_inputs( ) { (Some(ui_node), Some(_effective_node), _) => { anchor_node_name = Some(ui_node.node_name.clone()); - warnings.push(format!( - "Circuit '{}' anchor '{}' ('{}') did not resolve to an exported effective queue node. Falling back to generated parent-node shaping.", - device.circuit_id, + fallback_warning_summary.record_unresolved_exported_anchor( + &device.circuit_id, anchor_id, - ui_node.node_name - )); + &ui_node.node_name, + ); ( String::new(), String::new(), @@ -627,15 +751,11 @@ fn build_shaping_inputs( ) } (None, None, Some(anchor)) => { - warnings.push(format!( - "Circuit '{}' anchor '{}' ('{}') was not found in the effective topology. Falling back to generated parent-node shaping.", - device.circuit_id, + fallback_warning_summary.record_missing_anchor( + &device.circuit_id, anchor_id, - anchor - .anchor_node_name - .as_deref() - .unwrap_or_default() - )); + anchor.anchor_node_name.as_deref(), + ); ( String::new(), String::new(), @@ -645,10 +765,11 @@ fn build_shaping_inputs( ) } _ => { - warnings.push(format!( - "Circuit '{}' anchor '{}' was not found in the effective topology. Falling back to generated parent-node shaping.", - device.circuit_id, anchor_id - )); + fallback_warning_summary.record_missing_anchor( + &device.circuit_id, + anchor_id, + None, + ); ( String::new(), String::new(), @@ -681,12 +802,11 @@ fn build_shaping_inputs( if optional_non_empty(&device.parent_node).is_some() || optional_non_empty_owned(device.parent_node_id.clone()).is_some() { - warnings.push(format!( - "Circuit '{}' parent reference '{}' ({}) was not found in the exported effective topology. Falling back to generated parent-node shaping.", - device.circuit_id, + fallback_warning_summary.record_missing_parent( + &device.circuit_id, device.parent_node.trim(), - device.parent_node_id.clone().unwrap_or_default().trim() - )); + device.parent_node_id.clone().unwrap_or_default().trim(), + ); } ( String::new(), @@ -707,10 +827,11 @@ fn build_shaping_inputs( &exported_effective_nodes, &exported_effective_names, ) { - warnings.push(format!( - "Circuit '{}' resolved to non-exported effective parent '{}' ({}). Falling back to generated parent-node shaping.", - device.circuit_id, effective_parent_node_name, effective_parent_node_id - )); + fallback_warning_summary.record_non_exported_parent( + &device.circuit_id, + &effective_parent_node_name, + &effective_parent_node_id, + ); effective_parent_node_id.clear(); effective_parent_node_name.clear(); effective_attachment_id = None; @@ -781,6 +902,7 @@ fn build_shaping_inputs( .devices .sort_unstable_by(|left, right| left.device_id.cmp(&right.device_id)); } + fallback_warning_summary.append_to(&mut warnings); let unresolved_runtime_fallbacks = circuits .iter() @@ -795,16 +917,7 @@ fn build_shaping_inputs( "Flat topology mode assigned {unresolved_runtime_fallbacks} circuit(s) to generated parent nodes during queue construction." )); } else { - for circuit in &circuits { - if circuit.effective_parent_node_id.trim().is_empty() - && circuit.resolution_source == TopologyShapingResolutionSource::RuntimeFallback - { - warnings.push(format!( - "Circuit '{}' is unresolved in runtime topology and will be shaped under generated parent nodes.", - circuit.circuit_id - )); - } - } + push_unresolved_runtime_fallback_summary(&mut warnings, unresolved_runtime_fallbacks); } } @@ -5399,6 +5512,73 @@ mod tests { ); } + #[test] + fn shaping_inputs_aggregate_missing_parent_warnings() { + let lqos_directory = unique_temp_dir("lqos-topology-missing-parent-summary"); + let config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + state_directory: None, + ..Config::default() + }; + let mut csv = String::from( + "Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment\n", + ); + for circuit_number in 1..=12 { + csv.push_str(&format!( + "\"circuit-{circuit_number}\",\"Circuit {circuit_number}\",\"device-{circuit_number}\",\"Device {circuit_number}\",\"Missing Parent\",\"missing-parent\",\"\",\"aa:bb:cc:dd:ee:{circuit_number:02x}\",\"192.0.2.{circuit_number}/32\",\"\",\"10\",\"10\",\"100\",\"100\",\"\"\n", + )); + } + fs::write(lqos_directory.join("ShapedDevices.csv"), csv) + .expect("ShapedDevices.csv should write"); + + let artifacts = EffectiveTopologyArtifacts { + effective: TopologyEffectiveStateFile { + schema_version: 1, + generated_unix: Some(1), + canonical_generated_unix: Some(1), + health_generated_unix: Some(1), + nodes: Vec::new(), + }, + ui_state: TopologyEditorStateFile { + schema_version: 1, + source: "test".to_string(), + generated_unix: Some(1), + ingress_identity: None, + nodes: Vec::new(), + }, + effective_network: Some(json!({})), + }; + + let shaping_inputs = build_shaping_inputs(&config, &artifacts) + .expect("shaping inputs should build") + .expect("shaping inputs should be present"); + assert_eq!(shaping_inputs.circuits.len(), 12); + assert!(shaping_inputs.circuits.iter().all(|circuit| { + circuit.resolution_source + == lqos_config::TopologyShapingResolutionSource::RuntimeFallback + })); + + let parent_warnings = shaping_inputs + .warnings + .iter() + .filter(|warning| warning.contains("parent reference(s)")) + .collect::>(); + assert_eq!(parent_warnings.len(), 1); + assert!(parent_warnings[0].contains("12 circuit(s)")); + assert!(parent_warnings[0].contains("circuit-1")); + assert!(parent_warnings[0].contains("circuit-5")); + assert!(!parent_warnings[0].contains("circuit-6")); + assert!(parent_warnings[0].contains("7 more omitted")); + assert_eq!( + shaping_inputs + .warnings + .iter() + .filter(|warning| warning.contains("unresolved in runtime topology")) + .count(), + 1 + ); + } + #[test] fn flat_mode_assigns_explicit_generated_parent_buckets() { let lqos_directory = unique_temp_dir("lqos-topology-flat-summary"); From 6fa64c431ee349bd99fb44ba6a1b1ac19c67b292 Mon Sep 17 00:00:00 2001 From: rchac Date: Tue, 5 May 2026 22:25:33 +0000 Subject: [PATCH 12/20] changes --- docs-es/v2.0/integrations-reference-es.md | 28 +++++++++++++++++++++++ docs/v2.0/integrations-netzur.md | 26 +++++++++++++++++++++ docs/v2.0/integrations-reference.md | 28 +++++++++++++++++++++++ src/mikrotik_ipv6.example.toml | 9 ++++++++ 4 files changed, 91 insertions(+) diff --git a/docs-es/v2.0/integrations-reference-es.md b/docs-es/v2.0/integrations-reference-es.md index aa8651fc..1b7cf070 100644 --- a/docs-es/v2.0/integrations-reference-es.md +++ b/docs-es/v2.0/integrations-reference-es.md @@ -153,6 +153,34 @@ use_mikrotik_ipv6 = false - `timeout_secs` permite incrementar el tiempo de espera de la petición cuando el API responde lentamente (por defecto 60 segundos). - `use_mikrotik_ipv6` agrega prefijos IPv6 obtenidos de `/etc/libreqos/mikrotik_ipv6.toml`. LibreQoS relaciona IPv4 e IPv6 por dirección MAC usando datos de ARP, DHCPv4, DHCPv6 y vecinos IPv6 de MikroTik. +### Credenciales de routers MikroTik para IPv6 + +Cuando `use_mikrotik_ipv6` o `ipv6_with_mikrotik` esté habilitado, cree `/etc/libreqos/mikrotik_ipv6.toml`. Agregue un bloque `[[router]]` por cada router MikroTik que LibreQoS deba consultar: + +```toml +version = 1 + +[[router]] +name = "core-1" +host = "100.64.0.1" +port = 8728 +username = "libreqos" +password = "secreto" +use_ssl = false +plaintext_login = true + +[[router]] +name = "core-2" +host = "100.64.0.2" +port = 8728 +username = "libreqos" +password = "secreto" +use_ssl = false +plaintext_login = true +``` + +Agregue más bloques `[[router]]` para más routers. `port`, `use_ssl` y `plaintext_login` pueden omitirse cuando los valores por defecto son correctos: puerto `8728`, SSL deshabilitado y login en texto plano habilitado. + Para una importación manual: ```bash diff --git a/docs/v2.0/integrations-netzur.md b/docs/v2.0/integrations-netzur.md index 3e18bd04..49bf7f3b 100644 --- a/docs/v2.0/integrations-netzur.md +++ b/docs/v2.0/integrations-netzur.md @@ -17,6 +17,32 @@ use_mikrotik_ipv6 = false - `timeout_secs` overrides the default HTTP timeout (60 seconds) when the API is slow. - `use_mikrotik_ipv6` enriches subscriber devices with IPv6 prefixes discovered via `/etc/libreqos/mikrotik_ipv6.toml`. LibreQoS matches IPv4 and IPv6 by MAC address using MikroTik ARP, DHCPv4, DHCPv6, and IPv6 neighbor data. +When `use_mikrotik_ipv6` is enabled, create `/etc/libreqos/mikrotik_ipv6.toml`. Add one `[[router]]` block for each MikroTik router LibreQoS should query: + +```toml +version = 1 + +[[router]] +name = "core-1" +host = "100.64.0.1" +port = 8728 +username = "libreqos" +password = "secret" +use_ssl = false +plaintext_login = true + +[[router]] +name = "core-2" +host = "100.64.0.2" +port = 8728 +username = "libreqos" +password = "secret" +use_ssl = false +plaintext_login = true +``` + +Use additional `[[router]]` blocks for additional routers. `port`, `use_ssl`, and `plaintext_login` can be omitted when the defaults are correct: port `8728`, SSL disabled, and plaintext login enabled. + Run a manual import with: ```bash diff --git a/docs/v2.0/integrations-reference.md b/docs/v2.0/integrations-reference.md index 11c79949..0cffe36f 100644 --- a/docs/v2.0/integrations-reference.md +++ b/docs/v2.0/integrations-reference.md @@ -155,6 +155,34 @@ use_mikrotik_ipv6 = false - `timeout_secs` overrides the default HTTP timeout (60 seconds) when the API is slow. - `use_mikrotik_ipv6` enriches subscriber devices with IPv6 prefixes discovered via `/etc/libreqos/mikrotik_ipv6.toml`. LibreQoS matches IPv4 and IPv6 by MAC address using MikroTik ARP, DHCPv4, DHCPv6, and IPv6 neighbor data. +### MikroTik IPv6 Router Credentials + +When `use_mikrotik_ipv6` or `ipv6_with_mikrotik` is enabled, create `/etc/libreqos/mikrotik_ipv6.toml`. Add one `[[router]]` block for each MikroTik router LibreQoS should query: + +```toml +version = 1 + +[[router]] +name = "core-1" +host = "100.64.0.1" +port = 8728 +username = "libreqos" +password = "secret" +use_ssl = false +plaintext_login = true + +[[router]] +name = "core-2" +host = "100.64.0.2" +port = 8728 +username = "libreqos" +password = "secret" +use_ssl = false +plaintext_login = true +``` + +Use additional `[[router]]` blocks for additional routers. `port`, `use_ssl`, and `plaintext_login` can be omitted when the defaults are correct: port `8728`, SSL disabled, and plaintext login enabled. + Run a manual import with: ```bash diff --git a/src/mikrotik_ipv6.example.toml b/src/mikrotik_ipv6.example.toml index 9f857a52..7b49cae1 100644 --- a/src/mikrotik_ipv6.example.toml +++ b/src/mikrotik_ipv6.example.toml @@ -8,3 +8,12 @@ username = "admin" password = "password" use_ssl = false plaintext_login = true + +[[router]] +name = "secondary" +host = "100.64.0.2" +port = 8728 +username = "admin" +password = "password" +use_ssl = false +plaintext_login = true From d7de13f11e5c63269672d4ebc7430f345a7b1919 Mon Sep 17 00:00:00 2001 From: rchac Date: Tue, 5 May 2026 19:23:05 -0600 Subject: [PATCH 13/20] changes --- src/rust/lqos_overrides/src/overrides_file.rs | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/src/rust/lqos_overrides/src/overrides_file.rs b/src/rust/lqos_overrides/src/overrides_file.rs index 41344f35..e63eb461 100644 --- a/src/rust/lqos_overrides/src/overrides_file.rs +++ b/src/rust/lqos_overrides/src/overrides_file.rs @@ -1,5 +1,6 @@ use std::{ fs::read_to_string, + io::Write, path::{Path, PathBuf}, }; @@ -201,6 +202,9 @@ fn treeguard_read_path(config: &lqos_config::Config) -> PathBuf { fn load_from_path(path: &Path) -> Result { let raw = read_to_string(path)?; + if raw.trim().is_empty() { + anyhow::bail!("Overrides file '{}' was empty", path.display()); + } let as_json = serde_json::from_str(&raw)?; Ok(as_json) } @@ -212,13 +216,40 @@ fn ensure_exists_default(path: &Path) -> Result<()> { // Create a default empty file let new_override_file = OverrideFile::default(); let as_json = serde_json::to_string(&new_override_file)?; - std::fs::write(path, as_json.as_bytes())?; + write_atomic(path, as_json.as_bytes())?; Ok(()) } fn save_to_path(path: &Path, overrides: &OverrideFile) -> Result<()> { let as_json = serde_json::to_string_pretty(overrides)?; - std::fs::write(path, as_json.as_bytes())?; + write_atomic(path, as_json.as_bytes())?; + Ok(()) +} + +fn write_atomic(path: &Path, contents: &[u8]) -> Result<()> { + let parent = path.parent().ok_or_else(|| { + anyhow::anyhow!("Overrides path '{}' has no parent directory", path.display()) + })?; + let file_name = path.file_name().ok_or_else(|| { + anyhow::anyhow!("Overrides path '{}' has no file name", path.display()) + })?; + let pid = std::process::id(); + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0); + let tmp_name = format!("{}.tmp.{}.{}", file_name.to_string_lossy(), pid, nanos); + let tmp_path = parent.join(tmp_name); + + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp_path)?; + file.write_all(contents)?; + file.sync_all()?; + drop(file); + + std::fs::rename(&tmp_path, path)?; Ok(()) } @@ -1089,7 +1120,8 @@ impl OverrideStore { pub fn load_effective(apply_stormguard: bool, apply_treeguard: bool) -> Result { let lock = FileLock::new()?; let config = lqos_config::load_config()?; - let merged = Self::load_effective_for_config(&config, apply_stormguard, apply_treeguard)?; + let merged = + Self::load_effective_for_config_inner(&config, apply_stormguard, apply_treeguard)?; drop(lock); Ok(merged) } @@ -1102,6 +1134,17 @@ impl OverrideStore { config: &lqos_config::Config, apply_stormguard: bool, apply_treeguard: bool, + ) -> Result { + let lock = FileLock::new()?; + let merged = Self::load_effective_for_config_inner(config, apply_stormguard, apply_treeguard)?; + drop(lock); + Ok(merged) + } + + fn load_effective_for_config_inner( + config: &lqos_config::Config, + apply_stormguard: bool, + apply_treeguard: bool, ) -> Result { let operator_path = overrides_path(config, OverrideLayer::Operator); ensure_exists_default(&operator_path)?; From ae9571a5ccfd4684254aa53f86430b47f4395d5c Mon Sep 17 00:00:00 2001 From: rchac Date: Wed, 6 May 2026 11:46:43 +0000 Subject: [PATCH 14/20] changes --- src/rust/lqos_config/src/lib.rs | 16 +- .../lqos_config/src/topology_runtime_state.rs | 109 ++++++- src/rust/lqos_network_devices/README.md | 20 +- src/rust/lqos_network_devices/src/actor.rs | 279 ++++++++++++++++-- .../src/directory_watcher.rs | 17 +- src/rust/lqos_network_devices/src/lib.rs | 72 ++++- src/rust/lqos_network_devices/src/tests.rs | 16 +- src/rust/lqos_topology/src/lib.rs | 7 + src/rust/lqos_topology/src/runtime.rs | 225 +++++++++++++- src/rust/lqosd/src/main.rs | 2 +- .../src/node_manager/local_api/config.rs | 1 + .../lqosd/src/shaped_devices_tracker/mod.rs | 107 +++---- 12 files changed, 735 insertions(+), 136 deletions(-) diff --git a/src/rust/lqos_config/src/lib.rs b/src/rust/lqos_config/src/lib.rs index 2e54fd6b..cdf1f2ed 100644 --- a/src/rust/lqos_config/src/lib.rs +++ b/src/rust/lqos_config/src/lib.rs @@ -93,13 +93,15 @@ pub use topology_runtime_state::{ TOPOLOGY_IMPORT_FILENAME, TOPOLOGY_RUNTIME_STATUS_FILENAME, TOPOLOGY_SHAPING_INPUTS_FILENAME, TopologyAttachmentEndpointStatus, TopologyAttachmentHealthEntry, TopologyAttachmentHealthStateFile, TopologyEffectiveAttachmentState, - TopologyEffectiveNodeState, TopologyEffectiveStateFile, TopologyRuntimeStateError, - TopologyRuntimeStatusFile, TopologyShapingCircuitInput, TopologyShapingDeviceInput, - TopologyShapingInputsFile, TopologyShapingResolutionSource, active_runtime_shaping_inputs_path, - compute_effective_network_generation, compute_topology_source_generation, - load_active_runtime_shaping_inputs, topology_attachment_health_state_path, - topology_compiled_shaping_path, topology_effective_network_path, topology_effective_state_path, - topology_import_path, topology_runtime_status_path, topology_shaping_inputs_path, + TopologyEffectiveNodeState, TopologyEffectiveStateFile, TopologyRuntimeShapingPayloadIdentity, + TopologyRuntimeStateError, TopologyRuntimeStatusFile, TopologyShapingCircuitInput, + TopologyShapingDeviceInput, TopologyShapingInputsFile, TopologyShapingResolutionSource, + active_runtime_shaping_inputs_path, compute_effective_network_generation, + active_runtime_shaping_inputs_path_from_status, compute_topology_source_generation, + load_active_runtime_shaping_inputs, load_active_runtime_shaping_inputs_from_status, + topology_attachment_health_state_path, topology_compiled_shaping_path, + topology_effective_network_path, topology_effective_state_path, topology_import_path, + topology_runtime_status_path, topology_shaping_inputs_path, }; /// Returns whether an integration-backed topology ingress is enabled in the diff --git a/src/rust/lqos_config/src/topology_runtime_state.rs b/src/rust/lqos_config/src/topology_runtime_state.rs index 3e1c8954..02f90a9f 100644 --- a/src/rust/lqos_config/src/topology_runtime_state.rs +++ b/src/rust/lqos_config/src/topology_runtime_state.rs @@ -356,6 +356,20 @@ pub struct TopologyRuntimeStatusFile { pub error: Option, } +/// Shaping-relevant runtime status identity used by consumers that only need +/// to know whether the active shaping payload changed. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TopologyRuntimeShapingPayloadIdentity { + /// Whether runtime outputs are ready. + pub ready: bool, + /// Stable generation hash of the source topology inputs. + pub source_generation: String, + /// Stable generation hash of the shaping payload. + pub shaping_generation: String, + /// Path to the active shaping inputs payload. + pub shaping_inputs_path: String, +} + fn default_runtime_schema_version() -> u32 { 1 } @@ -428,20 +442,33 @@ pub fn active_runtime_shaping_inputs_path( ) -> Result, TopologyRuntimeStateError> { let current_generation = compute_topology_source_generation(config)?; let status = TopologyRuntimeStatusFile::load(config)?; + Ok(active_runtime_shaping_inputs_path_from_status( + &status, + ¤t_generation, + )) +} + +/// Returns the active runtime shaping-inputs path from an already-loaded status file. +/// +/// This function is pure: it has no side effects. +pub fn active_runtime_shaping_inputs_path_from_status( + status: &TopologyRuntimeStatusFile, + current_generation: &str, +) -> Option { if !status.ready { - return Ok(None); + return None; } if status.shaping_generation.trim().is_empty() { - return Ok(None); + return None; } if status.source_generation.trim() != current_generation { - return Ok(None); + return None; } let path = status.shaping_inputs_path.trim(); if path.is_empty() { - return Ok(None); + return None; } - Ok(Some(PathBuf::from(path))) + Some(PathBuf::from(path)) } /// Loads the currently active runtime shaping-inputs file when the topology @@ -456,6 +483,20 @@ pub fn load_active_runtime_shaping_inputs( Ok(Some(serde_json::from_str(&raw)?)) } +/// Loads the active runtime shaping-inputs file from an already-loaded status file. +pub fn load_active_runtime_shaping_inputs_from_status( + config: &Config, + status: &TopologyRuntimeStatusFile, +) -> Result, TopologyRuntimeStateError> { + let current_generation = compute_topology_source_generation(config)?; + let Some(path) = active_runtime_shaping_inputs_path_from_status(status, ¤t_generation) + else { + return Ok(None); + }; + let raw = std::fs::read_to_string(&path)?; + Ok(Some(serde_json::from_str(&raw)?)) +} + /// Returns the path of the topology runtime readiness status file. pub fn topology_runtime_status_path(config: &Config) -> PathBuf { config.topology_state_read_path(TOPOLOGY_RUNTIME_STATUS_FILENAME) @@ -742,6 +783,29 @@ impl TopologyRuntimeStatusFile { self, ) } + + /// Returns true when runtime status is equal except for the generated timestamp. + pub fn semantic_equals_for_publish(&self, other: &Self) -> bool { + self.schema_version == other.schema_version + && self.source_generation == other.source_generation + && self.shaping_generation == other.shaping_generation + && self.effective_generation == other.effective_generation + && self.ready == other.ready + && self.effective_state_path == other.effective_state_path + && self.effective_network_path == other.effective_network_path + && self.shaping_inputs_path == other.shaping_inputs_path + && self.error == other.error + } + + /// Returns the shaping payload identity used by cache consumers. + pub fn shaping_payload_identity(&self) -> TopologyRuntimeShapingPayloadIdentity { + TopologyRuntimeShapingPayloadIdentity { + ready: self.ready, + source_generation: self.source_generation.clone(), + shaping_generation: self.shaping_generation.clone(), + shaping_inputs_path: self.shaping_inputs_path.clone(), + } + } } #[cfg(test)] @@ -1300,6 +1364,41 @@ mod tests { ); } + #[test] + fn topology_runtime_status_publish_compare_ignores_generated_unix() { + let first = TopologyRuntimeStatusFile { + schema_version: 1, + source_generation: "source-1".to_string(), + shaping_generation: "shape-1".to_string(), + effective_generation: "effective-1".to_string(), + ready: true, + generated_unix: Some(1), + effective_state_path: "/tmp/effective.json".to_string(), + effective_network_path: "/tmp/network.effective.json".to_string(), + shaping_inputs_path: "/tmp/shaping_inputs.json".to_string(), + error: None, + }; + let second = TopologyRuntimeStatusFile { + generated_unix: Some(2), + ..first.clone() + }; + let changed = TopologyRuntimeStatusFile { + shaping_generation: "shape-2".to_string(), + ..first.clone() + }; + + assert!(first.semantic_equals_for_publish(&second)); + assert!(!first.semantic_equals_for_publish(&changed)); + assert_eq!( + first.shaping_payload_identity(), + second.shaping_payload_identity() + ); + assert_ne!( + first.shaping_payload_identity(), + changed.shaping_payload_identity() + ); + } + #[test] fn shaping_generation_ignores_timestamp_only_changes() { let first = TopologyShapingInputsFile { diff --git a/src/rust/lqos_network_devices/README.md b/src/rust/lqos_network_devices/README.md index 4f8215e0..ec7e7583 100644 --- a/src/rust/lqos_network_devices/README.md +++ b/src/rust/lqos_network_devices/README.md @@ -11,8 +11,8 @@ Centralized, shared access to LibreQoS topology inputs: In daemon mode, this crate starts: - A single-thread actor that owns reload/apply commands. -- Watchers for the config directory and topology-state directory that coalesce - filesystem events and request reloads. +- Watchers for the config directory, topology-state directory, and shaping-state + directory that coalesce filesystem events and request reloads. Callers access state through public accessor functions; the underlying channel sender is kept in a private static. @@ -27,7 +27,7 @@ private static. - `with_network_json_write(|net_json| ...)`: mutable access to in-memory `NetworkJson` (used by runtime counters). - `resolve_parent_node_reference(parent_node, parent_node_id)`: canonicalize shaped-device parent references against `network.json`. - `request_reload_shaped_devices(reason)` / `request_reload_network_json(reason)`: ask actor to reload from disk. -- `apply_shaped_devices_snapshot(reason, shaped)`: publish a caller-provided shaped-devices snapshot via actor. +- `apply_shaped_devices_snapshot(reason, shaped, provenance)`: publish a caller-provided shaped-devices snapshot via actor. - `swap_shaped_devices_snapshot(reason, shaped_arc)`: replace shaped-devices snapshot without actor (intended for tests). ## `ShapedDevicesCatalog` helpers @@ -48,16 +48,18 @@ Runtime throughput tracking mutates the in-memory `NetworkJson` via `with_networ ## Access & update catalog -Current (2026-04) call sites and update patterns: +Call sites and update patterns: - Daemon startup: `lqosd` calls `start_daemon_mode()` once at boot and provides `DaemonHooks`. - Shaped-device updates: - - Directory watcher requests reload when `topology_runtime_status.json`, - `ShapedDevices.csv`, or `ShapedDevices.insight.csv` changes. - - Reload precedence is: ready runtime shaping inputs, then `topology_import.json`, - then `ShapedDevices.csv`. + - Directory watcher requests reload when `shaping_inputs.json`, + `topology_import.json`, `ShapedDevices.csv`, or `ShapedDevices.insight.csv` changes. + - `topology_runtime_status.json` is readiness metadata and does not trigger a shaped-device reload by itself. + - In integration-ingress mode, reload precedence is: ready runtime shaping inputs, + then `topology_import.json`, then an empty snapshot until the topology runtime publishes. + - In manual mode, reloads use `ShapedDevices.csv` or `ShapedDevices.insight.csv`. - Node manager admin edits write `ShapedDevices.csv` and immediately publish the new snapshot via - `apply_shaped_devices_snapshot()`. + `apply_shaped_devices_snapshot()` with external-snapshot provenance. - Network topology updates: - Directory watcher requests reload when `network.json`, `network.insight.json`, or `network.effective.json` changes, including the runtime topology-state copy. diff --git a/src/rust/lqos_network_devices/src/actor.rs b/src/rust/lqos_network_devices/src/actor.rs index 6b6f3069..a823d20a 100644 --- a/src/rust/lqos_network_devices/src/actor.rs +++ b/src/rust/lqos_network_devices/src/actor.rs @@ -3,12 +3,17 @@ use crate::dynamic::{ }; use crate::dynamic_store::{load_dynamic_circuits_from_disk, persist_dynamic_circuits_to_disk}; use crate::state; -use crate::{DaemonHooks, ShapedDevicesCatalog, load_network_json, load_shaped_devices}; +use crate::{ + DaemonHooks, LoadedShapedDevices, LoadedShapedDevicesSource, ShapedDevicesCatalog, + ShapedDevicesSnapshotProvenance, load_network_json, load_shaped_devices_with_source, +}; use anyhow::{Context, Result, anyhow}; use crossbeam_channel::{Receiver, Sender}; use ip_network_table::IpNetworkTable; use once_cell::sync::OnceCell; use std::collections::HashSet; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use std::net::IpAddr; use std::sync::Arc; use std::time::Duration; @@ -69,11 +74,13 @@ pub(crate) fn request_reload_network_json(reason: &str) -> Result<()> { pub(crate) fn apply_shaped_devices_snapshot( reason: &str, shaped: lqos_config::ConfigShapedDevices, + provenance: ShapedDevicesSnapshotProvenance, ) -> Result<()> { let (reply_tx, reply_rx) = oneshot::channel(); sender()?.send(NetworkDevicesCommand::ApplyShapedDevicesSnapshot { reason: reason.to_string(), shaped: Box::new(shaped), + provenance, reply: reply_tx, })?; reply_rx @@ -128,6 +135,7 @@ enum NetworkDevicesCommand { ApplyShapedDevicesSnapshot { reason: String, shaped: Box, + provenance: ShapedDevicesSnapshotProvenance, reply: oneshot::Sender>, }, ReportObservations { @@ -145,8 +153,11 @@ enum NetworkDevicesCommand { fn actor_loop(rx: Receiver, hooks: Option>) { debug!("lqos_network_devices actor starting"); + let mut active_shaped_key = None; - if let Err(err) = reload_shaped_devices_inner("startup", hooks.as_deref()) { + if let Err(err) = + reload_shaped_devices_inner("startup", hooks.as_deref(), &mut active_shaped_key) + { warn!("Initial shaped-devices load failed: {err}"); } if let Err(err) = reload_network_json_inner("startup", hooks.as_deref()) { @@ -169,7 +180,11 @@ fn actor_loop(rx: Receiver, hooks: Option { - let result = reload_shaped_devices_inner(&reason, hooks.as_deref()); + let result = reload_shaped_devices_inner( + &reason, + hooks.as_deref(), + &mut active_shaped_key, + ); let _ = reply.send(result); } NetworkDevicesCommand::ReloadNetworkJson { reason, reply } => { @@ -179,15 +194,18 @@ fn actor_loop(rx: Receiver, hooks: Option { - debug!("Publishing shaped-devices snapshot reason={reason}"); - state::publish_shaped_devices(*shaped); - prune_dynamic_circuits_superseded_by_shaped_devices_inner(hooks.as_deref()); - if let Some(hooks) = &hooks { - hooks.on_shaped_devices_updated(); - } - let _ = reply.send(Ok(())); + let key = shaped_devices_key_for_snapshot(provenance, &shaped); + let result = publish_shaped_devices_if_changed( + &reason, + *shaped, + key, + hooks.as_deref(), + &mut active_shaped_key, + ); + let _ = reply.send(result); } NetworkDevicesCommand::ReportObservations { observations } => { handle_observations(&observations, hooks.as_deref()); @@ -224,6 +242,119 @@ fn recompute_hashes(device: &mut lqos_config::ShapedDevice) { device.parent_hash = lqos_utils::hash_to_i64(&device.parent_node); } +#[derive(Clone, Debug, PartialEq, Eq)] +enum ShapedDevicesProvenanceKey { + RuntimeTopology { + shaping_generation: String, + }, + TopologyImport { + devices_fingerprint: u64, + }, + Csv { + devices_fingerprint: u64, + }, + ExternalSnapshot { + devices_fingerprint: u64, + }, +} + +#[derive(Eq, Hash, Ord, PartialEq, PartialOrd)] +struct ShapedDeviceSemanticKey { + circuit_id: String, + circuit_name: String, + device_id: String, + device_name: String, + parent_node: String, + parent_node_id: Option, + anchor_node_id: Option, + mac: String, + ipv4: Vec<(std::net::Ipv4Addr, u32)>, + ipv6: Vec<(std::net::Ipv6Addr, u32)>, + download_min_mbps_bits: u32, + upload_min_mbps_bits: u32, + download_max_mbps_bits: u32, + upload_max_mbps_bits: u32, + comment: String, + sqm_override: Option, + circuit_hash: i64, + device_hash: i64, + parent_hash: i64, +} + +fn shaped_device_semantic_key(device: &lqos_config::ShapedDevice) -> ShapedDeviceSemanticKey { + let mut ipv4 = device.ipv4.clone(); + ipv4.sort_unstable(); + let mut ipv6 = device.ipv6.clone(); + ipv6.sort_unstable(); + + ShapedDeviceSemanticKey { + circuit_id: device.circuit_id.clone(), + circuit_name: device.circuit_name.clone(), + device_id: device.device_id.clone(), + device_name: device.device_name.clone(), + parent_node: device.parent_node.clone(), + parent_node_id: device.parent_node_id.clone(), + anchor_node_id: device.anchor_node_id.clone(), + mac: device.mac.clone(), + ipv4, + ipv6, + download_min_mbps_bits: device.download_min_mbps.to_bits(), + upload_min_mbps_bits: device.upload_min_mbps.to_bits(), + download_max_mbps_bits: device.download_max_mbps.to_bits(), + upload_max_mbps_bits: device.upload_max_mbps.to_bits(), + comment: device.comment.clone(), + sqm_override: device.sqm_override.clone(), + circuit_hash: device.circuit_hash, + device_hash: device.device_hash, + parent_hash: device.parent_hash, + } +} + +fn shaped_devices_fingerprint(devices: &[lqos_config::ShapedDevice]) -> u64 { + let mut sorted = devices + .iter() + .map(shaped_device_semantic_key) + .collect::>(); + sorted.sort_unstable(); + let mut hasher = DefaultHasher::new(); + for key in sorted { + key.hash(&mut hasher); + } + hasher.finish() +} + +fn shaped_devices_key_for_loaded(loaded: &LoadedShapedDevices) -> ShapedDevicesProvenanceKey { + match &loaded.source { + LoadedShapedDevicesSource::RuntimeTopology { + shaping_generation, + .. + } => ShapedDevicesProvenanceKey::RuntimeTopology { + shaping_generation: shaping_generation.clone(), + }, + LoadedShapedDevicesSource::TopologyImport => { + ShapedDevicesProvenanceKey::TopologyImport { + devices_fingerprint: shaped_devices_fingerprint(&loaded.shaped.devices), + } + } + LoadedShapedDevicesSource::Csv => ShapedDevicesProvenanceKey::Csv { + devices_fingerprint: shaped_devices_fingerprint(&loaded.shaped.devices), + }, + } +} + +fn shaped_devices_key_for_snapshot( + provenance: ShapedDevicesSnapshotProvenance, + shaped: &lqos_config::ConfigShapedDevices, +) -> ShapedDevicesProvenanceKey { + match provenance { + ShapedDevicesSnapshotProvenance::ExternalSnapshot => { + ShapedDevicesProvenanceKey::ExternalSnapshot { + devices_fingerprint: shaped_devices_fingerprint(&shaped.devices), + } + } + } +} + fn reload_dynamic_circuits_inner(reason: &str) { let circuits = load_dynamic_circuits_from_disk(); debug!("Loaded {} dynamic circuits reason={reason}", circuits.len()); @@ -562,17 +693,45 @@ fn shaped_device_from_unknown_ip( device } -fn reload_shaped_devices_inner(reason: &str, hooks: Option<&dyn DaemonHooks>) -> Result<()> { +fn publish_shaped_devices_if_changed( + reason: &str, + shaped: lqos_config::ConfigShapedDevices, + key: ShapedDevicesProvenanceKey, + hooks: Option<&dyn DaemonHooks>, + active_key: &mut Option, +) -> Result<()> { + if active_key.as_ref() == Some(&key) { + debug!("Skipping unchanged shaped-devices snapshot reason={reason}"); + return Ok(()); + } + + debug!("Publishing shaped-devices snapshot reason={reason}"); + state::publish_shaped_devices(shaped); + *active_key = Some(key); + prune_dynamic_circuits_superseded_by_shaped_devices_inner(hooks); + if let Some(hooks) = hooks { + hooks.on_shaped_devices_updated(); + } + Ok(()) +} + +fn reload_shaped_devices_inner( + reason: &str, + hooks: Option<&dyn DaemonHooks>, + active_key: &mut Option, +) -> Result<()> { for attempt in 1..=RELOAD_ATTEMPTS { - match load_shaped_devices() { - Ok(shaped) => { + match load_shaped_devices_with_source() { + Ok(loaded) => { debug!("Loaded shaped devices reason={reason}"); - state::publish_shaped_devices(shaped); - prune_dynamic_circuits_superseded_by_shaped_devices_inner(hooks); - if let Some(hooks) = hooks { - hooks.on_shaped_devices_updated(); - } - return Ok(()); + let key = shaped_devices_key_for_loaded(&loaded); + return publish_shaped_devices_if_changed( + reason, + loaded.shaped, + key, + hooks, + active_key, + ); } Err(err) if attempt < RELOAD_ATTEMPTS => { warn!( @@ -682,3 +841,85 @@ fn reload_network_json_inner(reason: &str, hooks: Option<&dyn DaemonHooks>) -> R } unreachable!("reload loop must return"); } + +#[cfg(test)] +mod tests { + use super::shaped_devices_fingerprint; + + fn shaped_device(circuit_id: &str, device_id: &str) -> lqos_config::ShapedDevice { + let mut device = lqos_config::ShapedDevice { + circuit_id: circuit_id.to_string(), + circuit_name: circuit_id.to_string(), + device_id: device_id.to_string(), + device_name: device_id.to_string(), + parent_node: "Tower A".to_string(), + ipv4: vec![("192.0.2.1".parse().expect("valid IP"), 32)], + download_min_mbps: 10.0, + upload_min_mbps: 10.0, + download_max_mbps: 100.0, + upload_max_mbps: 100.0, + ..lqos_config::ShapedDevice::default() + }; + device.circuit_hash = lqos_utils::hash_to_i64(&device.circuit_id); + device.device_hash = lqos_utils::hash_to_i64(&device.device_id); + device.parent_hash = lqos_utils::hash_to_i64(&device.parent_node); + device + } + + #[test] + fn shaped_devices_fingerprint_ignores_row_order() { + let first = vec![ + shaped_device("circuit-1", "device-1"), + shaped_device("circuit-2", "device-2"), + ]; + let second = vec![ + shaped_device("circuit-2", "device-2"), + shaped_device("circuit-1", "device-1"), + ]; + + assert_eq!( + shaped_devices_fingerprint(&first), + shaped_devices_fingerprint(&second) + ); + } + + #[test] + fn shaped_devices_fingerprint_ignores_address_order() { + let first = vec![shaped_device("circuit-1", "device-1")]; + let mut second = vec![shaped_device("circuit-1", "device-1")]; + second[0].ipv4 = vec![ + ("192.0.2.2".parse().expect("valid IPv4"), 32), + ("192.0.2.1".parse().expect("valid IPv4"), 32), + ]; + second[0].ipv6 = vec![ + ("2001:db8::2".parse().expect("valid IPv6"), 128), + ("2001:db8::1".parse().expect("valid IPv6"), 128), + ]; + let mut first = first; + first[0].ipv4 = vec![ + ("192.0.2.1".parse().expect("valid IPv4"), 32), + ("192.0.2.2".parse().expect("valid IPv4"), 32), + ]; + first[0].ipv6 = vec![ + ("2001:db8::1".parse().expect("valid IPv6"), 128), + ("2001:db8::2".parse().expect("valid IPv6"), 128), + ]; + + assert_eq!( + shaped_devices_fingerprint(&first), + shaped_devices_fingerprint(&second) + ); + } + + #[test] + fn shaped_devices_fingerprint_detects_payload_changes() { + let first = vec![shaped_device("circuit-1", "device-1")]; + let mut second = vec![shaped_device("circuit-1", "device-1")]; + second[0].download_max_mbps = 200.0; + + assert_ne!( + shaped_devices_fingerprint(&first), + shaped_devices_fingerprint(&second) + ); + } +} diff --git a/src/rust/lqos_network_devices/src/directory_watcher.rs b/src/rust/lqos_network_devices/src/directory_watcher.rs index e2886e4c..47d054ba 100644 --- a/src/rust/lqos_network_devices/src/directory_watcher.rs +++ b/src/rust/lqos_network_devices/src/directory_watcher.rs @@ -25,6 +25,11 @@ pub(crate) fn start_network_devices_directory_watch() -> Result<()> { "LibreQoS topology state directory", config.resolved_state_directory().join("topology"), )?; + spawn_directory_watch_thread( + "Network Devices Shaping Dir Watcher", + "LibreQoS shaping state directory", + config.resolved_state_directory().join("shaping"), + )?; Ok(()) } @@ -88,7 +93,7 @@ fn classify_changed_path(path: &Path) -> Option { "ShapedDevices.csv" | "ShapedDevices.insight.csv" | "topology_import.json" - | "topology_runtime_status.json" => Some(DirectoryReloadEvent::ShapedDevices), + | "shaping_inputs.json" => Some(DirectoryReloadEvent::ShapedDevices), "network.effective.json" | "network.insight.json" | "network.json" => { Some(DirectoryReloadEvent::NetworkJson) } @@ -118,8 +123,14 @@ mod tests { use std::path::PathBuf; #[test] - fn topology_runtime_status_triggers_shaped_device_reload() { + fn topology_runtime_status_does_not_trigger_shaped_device_reload() { let event = classify_changed_path(&PathBuf::from("/tmp/topology_runtime_status.json")); + assert_eq!(event, None); + } + + #[test] + fn shaping_inputs_triggers_shaped_device_reload() { + let event = classify_changed_path(&PathBuf::from("/tmp/shaping_inputs.json")); assert_eq!(event, Some(DirectoryReloadEvent::ShapedDevices)); } @@ -134,7 +145,7 @@ mod tests { let events = classify_changed_paths(&[ PathBuf::from("/tmp/network.json"), PathBuf::from("/tmp/network.effective.json"), - PathBuf::from("/tmp/topology_runtime_status.json"), + PathBuf::from("/tmp/shaping_inputs.json"), ]); assert_eq!( events, diff --git a/src/rust/lqos_network_devices/src/lib.rs b/src/rust/lqos_network_devices/src/lib.rs index 588381c6..6dac496b 100644 --- a/src/rust/lqos_network_devices/src/lib.rs +++ b/src/rust/lqos_network_devices/src/lib.rs @@ -20,7 +20,9 @@ mod topology; mod tests; use anyhow::{Context, Result}; -use lqos_config::{Config, ConfigShapedDevices, NetworkJson, TopologyShapingInputsFile}; +use lqos_config::{ + Config, ConfigShapedDevices, NetworkJson, TopologyShapingInputsFile, +}; use std::sync::Arc; use tracing::debug; @@ -30,6 +32,27 @@ pub use dynamic::{CircuitObservation, DynamicCircuit}; pub use hash_cache::ShapedDeviceHashCache; pub use topology::{ResolvedParentNode, resolve_parent_node, resolve_parent_node_reference}; +/// Provenance for caller-provided shaped-device snapshots. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ShapedDevicesSnapshotProvenance { + /// Snapshot came from an external caller that already constructed a complete payload. + ExternalSnapshot, +} + +#[derive(Clone, Debug)] +pub(crate) enum LoadedShapedDevicesSource { + RuntimeTopology { + shaping_generation: String, + }, + TopologyImport, + Csv, +} + +pub(crate) struct LoadedShapedDevices { + pub(crate) shaped: ConfigShapedDevices, + pub(crate) source: LoadedShapedDevicesSource, +} + /// Hooks that `lqosd` can provide to run side effects after updates. pub trait DaemonHooks: Send + Sync + 'static { /// Called after a new `ShapedDevices.csv` snapshot is published. @@ -67,13 +90,18 @@ pub fn load_runtime_shaping_inputs_for_config( runtime_inputs::load_ready_runtime_shaping_inputs(config) } -fn load_shaped_devices_for_config(config: &lqos_config::Config) -> Result { +fn load_shaped_devices_for_config(config: &lqos_config::Config) -> Result { if lqos_config::integration_ingress_enabled(config) { if let Some(shaping_inputs) = load_runtime_shaping_inputs_for_config(config)? { let shaped_devices = runtime_inputs::shaped_devices_from_runtime_inputs(&shaping_inputs); if !shaped_devices.devices.is_empty() { - return Ok(shaped_devices); + return Ok(LoadedShapedDevices { + shaped: shaped_devices, + source: LoadedShapedDevicesSource::RuntimeTopology { + shaping_generation: shaping_inputs.shaping_generation, + }, + }); } debug!( "Active runtime shaping inputs contained 0 shaped devices; checking other integration sources" @@ -83,13 +111,19 @@ fn load_shaped_devices_for_config(config: &lqos_config::Config) -> Result { let shaped_devices = topology_import.into_imported_bundle().shaped_devices; if !shaped_devices.devices.is_empty() { - return Ok(shaped_devices); + return Ok(LoadedShapedDevices { + shaped: shaped_devices, + source: LoadedShapedDevicesSource::TopologyImport, + }); } debug!( "topology_import.json advertised shaped devices but loaded empty; keeping integration mode empty until topology is published" @@ -106,16 +140,28 @@ fn load_shaped_devices_for_config(config: &lqos_config::Config) -> Result Result { + Ok(load_shaped_devices_with_source()?.shaped) +} + +pub(crate) fn load_shaped_devices_with_source() -> Result { let config = lqos_config::load_config().context("Unable to load /etc/lqos.conf")?; load_shaped_devices_for_config(config.as_ref()) } @@ -183,8 +229,12 @@ pub fn request_reload_network_json(reason: &str) -> Result<()> { } /// Publishes a caller-provided shaped-devices snapshot through the runtime actor. -pub fn apply_shaped_devices_snapshot(reason: &str, shaped: ConfigShapedDevices) -> Result<()> { - actor::apply_shaped_devices_snapshot(reason, shaped) +pub fn apply_shaped_devices_snapshot( + reason: &str, + shaped: ConfigShapedDevices, + provenance: ShapedDevicesSnapshotProvenance, +) -> Result<()> { + actor::apply_shaped_devices_snapshot(reason, shaped, provenance) } /// Reports kernel observations that may indicate dynamic circuit activity. diff --git a/src/rust/lqos_network_devices/src/tests.rs b/src/rust/lqos_network_devices/src/tests.rs index 2d71ade6..3c1f5df0 100644 --- a/src/rust/lqos_network_devices/src/tests.rs +++ b/src/rust/lqos_network_devices/src/tests.rs @@ -431,8 +431,8 @@ fn load_shaped_devices_uses_topology_import_when_runtime_inputs_are_empty() { let loaded = load_shaped_devices_for_config(&config) .expect("preferred shaped-device source should load"); - assert_eq!(loaded.devices.len(), 1); - assert_eq!(loaded.devices[0].circuit_id, "import-circuit"); + assert_eq!(loaded.shaped.devices.len(), 1); + assert_eq!(loaded.shaped.devices[0].circuit_id, "import-circuit"); } #[test] @@ -493,8 +493,8 @@ fn load_shaped_devices_uses_topology_import_when_runtime_is_not_ready() { let loaded = load_shaped_devices_for_config(&config).expect("topology import fallback should load"); - assert_eq!(loaded.devices.len(), 1); - assert_eq!(loaded.devices[0].circuit_id, "import-circuit"); + assert_eq!(loaded.shaped.devices.len(), 1); + assert_eq!(loaded.shaped.devices[0].circuit_id, "import-circuit"); } #[test] @@ -534,7 +534,7 @@ fn load_shaped_devices_stays_empty_when_topology_import_is_empty() { let loaded = load_shaped_devices_for_config(&config).expect("integration mode should stay empty"); - assert!(loaded.devices.is_empty()); + assert!(loaded.shaped.devices.is_empty()); } #[test] @@ -582,8 +582,8 @@ fn load_shaped_devices_ignores_stale_runtime_inputs_in_manual_mode() { let loaded = load_shaped_devices_for_config(&config).expect("manual mode should use shaped devices csv"); - assert_eq!(loaded.devices.len(), 1); - assert_eq!(loaded.devices[0].circuit_id, "csv-circuit"); + assert_eq!(loaded.shaped.devices.len(), 1); + assert_eq!(loaded.shaped.devices[0].circuit_id, "csv-circuit"); } #[test] @@ -610,5 +610,5 @@ fn load_shaped_devices_stays_empty_when_runtime_status_is_malformed() { let loaded = load_shaped_devices_for_config(&config).expect("malformed status should stay empty"); - assert!(loaded.devices.is_empty()); + assert!(loaded.shaped.devices.is_empty()); } diff --git a/src/rust/lqos_topology/src/lib.rs b/src/rust/lqos_topology/src/lib.rs index ae8a6da2..29e698f5 100644 --- a/src/rust/lqos_topology/src/lib.rs +++ b/src/rust/lqos_topology/src/lib.rs @@ -1696,6 +1696,13 @@ pub fn publish_topology_runtime_status( ready, error, ); + if TopologyRuntimeStatusFile::load(config) + .ok() + .as_ref() + .is_some_and(|current| current.semantic_equals_for_publish(&status)) + { + return Ok(()); + } status.save(config).with_context(|| { format!( "Unable to publish topology runtime status at {:?}", diff --git a/src/rust/lqos_topology/src/runtime.rs b/src/rust/lqos_topology/src/runtime.rs index 1a8b763d..50a58a1d 100644 --- a/src/rust/lqos_topology/src/runtime.rs +++ b/src/rust/lqos_topology/src/runtime.rs @@ -2,12 +2,14 @@ use anyhow::{Context, Result}; use lqos_bus::{BusRequest, BusResponse, bus_request}; use lqos_config::{ TopologyAttachmentEndpointStatus, TopologyAttachmentHealthEntry, - TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus, + TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus, TopologyCanonicalStateFile, compute_topology_source_generation, load_config, }; use lqos_overrides::TopologyOverridesFile; use lqos_probe::{ProbeClass, ProbeRequest}; use std::collections::HashMap; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use std::net::IpAddr; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tracing::{info, warn}; @@ -313,6 +315,61 @@ fn refresh_health_state( Ok(true) } +fn hash_health_status(status: TopologyAttachmentHealthStatus, hasher: &mut impl Hasher) { + match status { + TopologyAttachmentHealthStatus::Healthy => 0_u8.hash(hasher), + TopologyAttachmentHealthStatus::Suppressed => 1_u8.hash(hasher), + TopologyAttachmentHealthStatus::ProbeUnavailable => 2_u8.hash(hasher), + TopologyAttachmentHealthStatus::Disabled => 3_u8.hash(hasher), + } +} + +fn hash_health_effective_entry(entry: &TopologyAttachmentHealthEntry, hasher: &mut impl Hasher) { + entry.attachment_pair_id.hash(hasher); + entry.attachment_id.hash(hasher); + entry.child_node_id.hash(hasher); + entry.parent_node_id.hash(hasher); + entry.local_probe_ip.hash(hasher); + entry.remote_probe_ip.hash(hasher); + hash_health_status(entry.status, hasher); + entry.probeable.hash(hasher); + entry.enabled.hash(hasher); + entry.suppressed_until_unix.hash(hasher); +} + +fn health_effective_signature(health_state: &TopologyAttachmentHealthStateFile) -> u64 { + let mut hasher = DefaultHasher::new(); + for entry in &health_state.attachments { + hash_health_effective_entry(entry, &mut hasher); + } + hasher.finish() +} + +#[derive(Clone, Debug, Default)] +struct RuntimeBuildGate { + last_source_generation: Option, + last_overrides_generation: Option, + last_health_effective_signature: Option, + cached_probe_specs: Vec, + publish_completed: bool, + next_error_retry_after_unix: Option, +} + +fn topology_overrides_generation(config: &lqos_config::Config) -> u64 { + let path = TopologyOverridesFile::path_for_config(config); + let mut hasher = DefaultHasher::new(); + path.hash(&mut hasher); + if let Ok(contents) = std::fs::read(&path) { + match serde_json::from_slice::(&contents) + .and_then(|overrides| serde_json::to_vec(&overrides)) + { + Ok(canonical) => canonical.hash(&mut hasher), + Err(_) => contents.hash(&mut hasher), + } + } + hasher.finish() +} + #[derive(Clone, Copy, Debug, Default)] struct RoundHints { probes_enabled: bool, @@ -321,29 +378,62 @@ struct RoundHints { async fn run_round( health_state: &mut TopologyAttachmentHealthStateFile, last_effective: &mut HashMap>, + gate: &mut RuntimeBuildGate, ) -> Result { let config = load_config().context("Unable to load config for topology runtime")?; let source_generation = compute_topology_source_generation(config.as_ref()) .context("Unable to compute topology source generation")?; - let canonical = load_canonical_topology_state(config.as_ref()); - let overrides = - TopologyOverridesFile::load().context("Unable to load topology overrides file")?; - let prepared = prepare_runtime_topology_editor_state_from_canonical(&canonical, &overrides); - let specs = probe_specs_from_state(&prepared, &overrides); + let overrides_generation = topology_overrides_generation(config.as_ref()); + let source_changed = gate.last_source_generation.as_deref() != Some(source_generation.as_str()); + let overrides_changed = gate.last_overrides_generation != Some(overrides_generation); + let mut loaded_inputs: Option<(TopologyCanonicalStateFile, TopologyOverridesFile)> = None; + if source_changed || overrides_changed || gate.cached_probe_specs.is_empty() { + let canonical = load_canonical_topology_state(config.as_ref()); + let overrides = + TopologyOverridesFile::load().context("Unable to load topology overrides file")?; + let prepared = prepare_runtime_topology_editor_state_from_canonical(&canonical, &overrides); + gate.cached_probe_specs = probe_specs_from_state(&prepared, &overrides); + loaded_inputs = Some((canonical, overrides)); + } + let specs = &gate.cached_probe_specs; let probes_enabled = specs.iter().any(|spec| spec.enabled); if probes_enabled { - match probe_specs(&specs, Duration::from_millis(750)).await { + match probe_specs(specs, Duration::from_millis(750)).await { Ok(probe_results) => { - refresh_health_state(config.as_ref(), health_state, &specs, &probe_results)?; + refresh_health_state(config.as_ref(), health_state, specs, &probe_results)?; } Err(err) => { warn!("Topology probe round could not query shared probe manager: {err:#}"); } } } else { - refresh_health_state(config.as_ref(), health_state, &specs, &HashMap::new())?; + refresh_health_state(config.as_ref(), health_state, specs, &HashMap::new())?; } + let next_signature = health_effective_signature(health_state); + let retry_due = gate + .next_error_retry_after_unix + .is_some_and(|deadline| now_unix().is_some_and(|now| now >= deadline)); + let retry_pending = gate.next_error_retry_after_unix.is_some() && !retry_due; + let source_or_health_changed = source_changed + || overrides_changed + || gate.last_health_effective_signature.as_ref() != Some(&next_signature); + let should_publish = + source_or_health_changed || (!retry_pending && (!gate.publish_completed || retry_due)); + if !should_publish { + return Ok(RoundHints { probes_enabled }); + } + + let (canonical, overrides) = match loaded_inputs { + Some(inputs) => inputs, + None => { + let canonical = load_canonical_topology_state(config.as_ref()); + let overrides = + TopologyOverridesFile::load().context("Unable to load topology overrides file")?; + (canonical, overrides) + } + }; + let artifacts = build_effective_topology_artifacts_from_canonical_with_runtime_queue_context( config.as_ref(), &canonical, @@ -368,8 +458,21 @@ async fn run_round( "Unable to publish failed topology runtime status after publish error: {status_err:#}" ); } + let health = &config.integration_common.topology_attachment_health; + let retry_delay = health + .refresh_debounce_seconds + .max(health.probe_interval_seconds.max(1)); + gate.last_source_generation = Some(source_generation.clone()); + gate.last_overrides_generation = Some(overrides_generation); + gate.last_health_effective_signature = Some(next_signature); + gate.next_error_retry_after_unix = now_unix().map(|now| now.saturating_add(retry_delay)); return Err(err); } + gate.last_source_generation = Some(source_generation.clone()); + gate.last_overrides_generation = Some(overrides_generation); + gate.last_health_effective_signature = Some(next_signature); + gate.publish_completed = true; + gate.next_error_retry_after_unix = None; for node in &artifacts.effective.nodes { let next = node.effective_attachment_id.clone(); @@ -394,9 +497,12 @@ async fn run_round( pub async fn start_topology() { let mut health_state = load_starting_health(); let mut last_effective = HashMap::>::new(); + let mut build_gate = RuntimeBuildGate::default(); loop { - let round_hints = match run_round(&mut health_state, &mut last_effective).await { + let round_hints = match run_round(&mut health_state, &mut last_effective, &mut build_gate) + .await + { Ok(hints) => hints, Err(err) => { if let Ok(config) = load_config() @@ -434,3 +540,102 @@ pub async fn start_topology() { tokio::time::sleep(Duration::from_secs(sleep_seconds)).await; } } + +#[cfg(test)] +mod tests { + use super::{ + TopologyAttachmentHealthEntry, health_effective_signature, topology_overrides_generation, + }; + use lqos_config::{Config, TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus}; + use lqos_overrides::TopologyOverridesFile; + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn unique_temp_dir(prefix: &str) -> std::path::PathBuf { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time should be valid") + .as_nanos(); + std::env::temp_dir().join(format!("{prefix}-{}-{unique}", std::process::id())) + } + + fn health_entry() -> TopologyAttachmentHealthEntry { + TopologyAttachmentHealthEntry { + attachment_pair_id: "pair-1".to_string(), + attachment_id: Some("attachment-1".to_string()), + child_node_id: Some("child-1".to_string()), + parent_node_id: Some("parent-1".to_string()), + local_probe_ip: Some("192.0.2.1".to_string()), + remote_probe_ip: Some("192.0.2.2".to_string()), + status: TopologyAttachmentHealthStatus::Healthy, + probeable: true, + enabled: true, + consecutive_misses: 1, + consecutive_successes: 2, + last_success_unix: Some(10), + endpoint_status: vec![], + ..TopologyAttachmentHealthEntry::default() + } + } + + #[test] + fn health_effective_signature_ignores_probe_counters_and_timestamps() { + let first = TopologyAttachmentHealthStateFile { + generated_unix: Some(1), + attachments: vec![health_entry()], + ..TopologyAttachmentHealthStateFile::default() + }; + let mut second_entry = health_entry(); + second_entry.consecutive_misses = 4; + second_entry.consecutive_successes = 5; + second_entry.last_success_unix = Some(20); + second_entry.last_failure_unix = Some(21); + let second = TopologyAttachmentHealthStateFile { + generated_unix: Some(2), + attachments: vec![second_entry], + ..TopologyAttachmentHealthStateFile::default() + }; + + assert_eq!( + health_effective_signature(&first), + health_effective_signature(&second) + ); + } + + #[test] + fn health_effective_signature_tracks_suppression_status() { + let first = TopologyAttachmentHealthStateFile { + attachments: vec![health_entry()], + ..TopologyAttachmentHealthStateFile::default() + }; + let mut suppressed = health_entry(); + suppressed.status = TopologyAttachmentHealthStatus::Suppressed; + suppressed.suppressed_until_unix = Some(123); + let second = TopologyAttachmentHealthStateFile { + attachments: vec![suppressed], + ..TopologyAttachmentHealthStateFile::default() + }; + + assert_ne!( + health_effective_signature(&first), + health_effective_signature(&second) + ); + } + + #[test] + fn topology_overrides_generation_tracks_manual_override_file_changes() { + let lqos_directory = unique_temp_dir("lqos-topology-runtime-overrides"); + fs::create_dir_all(&lqos_directory).expect("temp lqos directory should exist"); + let config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + let path = TopologyOverridesFile::path_for_config(&config); + + let before = topology_overrides_generation(&config); + fs::write(&path, "{\"schemaVersion\":1}\n").expect("override file should write"); + let after = topology_overrides_generation(&config); + + assert_ne!(before, after); + } +} diff --git a/src/rust/lqosd/src/main.rs b/src/rust/lqosd/src/main.rs index 7b856014..6976a159 100644 --- a/src/rust/lqosd/src/main.rs +++ b/src/rust/lqosd/src/main.rs @@ -383,7 +383,7 @@ fn main() -> Result<()> { let _ = throughput_tracker::flow_data::setup_flow_analysis(); start_heimdall()?; spawn_queue_structure_monitor()?; - shaped_devices_tracker::shaping_inputs_watcher()?; + shaped_devices_tracker::topology_runtime_status_watcher()?; throughput_tracker::spawn_throughput_monitor( flow_tx, system_usage_tx.clone(), diff --git a/src/rust/lqosd/src/node_manager/local_api/config.rs b/src/rust/lqosd/src/node_manager/local_api/config.rs index a024764b..0d86926a 100644 --- a/src/rust/lqosd/src/node_manager/local_api/config.rs +++ b/src/rust/lqosd/src/node_manager/local_api/config.rs @@ -785,6 +785,7 @@ fn persist_shaped_devices(mut devices: Vec) -> Result<(), String> lqos_network_devices::apply_shaped_devices_snapshot( "node_manager:persist_shaped_devices", copied, + lqos_network_devices::ShapedDevicesSnapshotProvenance::ExternalSnapshot, ) .map_err(|e| format!("Unable to publish ShapedDevices.csv snapshot: {e}"))?; diff --git a/src/rust/lqosd/src/shaped_devices_tracker/mod.rs b/src/rust/lqosd/src/shaped_devices_tracker/mod.rs index dee9d7b5..20959e14 100644 --- a/src/rust/lqosd/src/shaped_devices_tracker/mod.rs +++ b/src/rust/lqosd/src/shaped_devices_tracker/mod.rs @@ -5,11 +5,13 @@ use arc_swap::ArcSwap; use fxhash::{FxHashMap, FxHashSet}; use lqos_bus::{BusResponse, Circuit}; use lqos_config::{ - NetworkJsonNode, NetworkJsonTransport, TopologyShapingInputsFile, load_config, + NetworkJsonNode, NetworkJsonTransport, TopologyRuntimeShapingPayloadIdentity, + TopologyRuntimeStatusFile, TopologyShapingInputsFile, + load_active_runtime_shaping_inputs_from_status, load_config, topology_runtime_status_path, }; #[cfg(test)] -use lqos_config::{TopologyRuntimeStatusFile, topology_shaping_inputs_path}; +use lqos_config::load_active_runtime_shaping_inputs; use lqos_queue_tracker::EFFECTIVE_NODE_RATES; use lqos_utils::file_watcher::FileWatcher; use lqos_utils::hash_to_i64; @@ -18,8 +20,6 @@ use lqos_utils::units::{DownUpOrder, down_up_retransmit_sample}; use lqos_utils::unix_time::time_since_boot; use once_cell::sync::Lazy; use parking_lot::Mutex; -#[cfg(test)] -use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::time::Duration; @@ -42,6 +42,8 @@ pub static CIRCUIT_LIVE_LAST_REFRESH_SECS: AtomicU64 = AtomicU64::new(0); pub static CIRCUIT_LIVE_REFRESH_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); pub static EFFECTIVE_CIRCUIT_PARENTS: Lazy>> = Lazy::new(|| ArcSwap::new(Arc::new(FxHashMap::default()))); +static LAST_TOPOLOGY_STATUS_IDENTITY: Lazy>> = + Lazy::new(|| Mutex::new(None)); #[derive(Clone, Debug, PartialEq, Eq)] pub struct RuntimeCircuitParent { @@ -104,38 +106,16 @@ fn publish_shaping_inputs(shaping_inputs: TopologyShapingInputsFile) { invalidate_executive_cache_snapshot(); } -#[cfg(test)] -fn active_runtime_shaping_inputs_path(config: &lqos_config::Config) -> Result> { - let status = TopologyRuntimeStatusFile::load(config)?; - if !status.ready { - return Ok(None); - } - let path = status.shaping_inputs_path.trim(); - if path.is_empty() { - return Ok(None); - } - Ok(Some(PathBuf::from(path))) +fn topology_status_identity(status: &TopologyRuntimeStatusFile) -> TopologyRuntimeShapingPayloadIdentity { + status.shaping_payload_identity() } -#[cfg(test)] -fn load_active_runtime_shaping_inputs( - config: &lqos_config::Config, -) -> Result> { - if let Some(active_path) = active_runtime_shaping_inputs_path(config)? - && active_path.exists() - { - let raw = std::fs::read_to_string(&active_path)?; - let shaping_inputs = serde_json::from_str(&raw)?; - return Ok(Some(shaping_inputs)); - } +fn topology_status_identity_changed(identity: &TopologyRuntimeShapingPayloadIdentity) -> bool { + LAST_TOPOLOGY_STATUS_IDENTITY.lock().as_ref() != Some(identity) +} - let fallback_path = topology_shaping_inputs_path(config); - if !fallback_path.exists() { - return Ok(None); - } - let raw = std::fs::read_to_string(&fallback_path)?; - let shaping_inputs = serde_json::from_str(&raw)?; - Ok(Some(shaping_inputs)) +fn remember_topology_status_identity(identity: TopologyRuntimeShapingPayloadIdentity) { + *LAST_TOPOLOGY_STATUS_IDENTITY.lock() = Some(identity); } #[cfg(test)] @@ -257,21 +237,8 @@ fn integration_ingress_enabled(config: &lqos_config::Config) -> bool { fn load_ready_runtime_shaping_inputs( config: &lqos_config::Config, ) -> Result> { - let Some(active_path) = active_runtime_shaping_inputs_path(config)? else { - return Ok(None); - }; - if !active_path.exists() { - return Ok(None); - } - let raw = std::fs::read_to_string(&active_path) - .with_context(|| format!("Unable to read shaping inputs at {}", active_path.display()))?; - let shaping_inputs = serde_json::from_str(&raw).with_context(|| { - format!( - "Unable to decode shaping inputs JSON at {}", - active_path.display() - ) - })?; - Ok(Some(shaping_inputs)) + load_active_runtime_shaping_inputs(config) + .context("Unable to load active runtime shaping inputs") } #[cfg(test)] @@ -311,16 +278,28 @@ fn load_shaped_devices_from_preferred_source( ConfigShapedDevices::load_for_config(config).context("Unable to load ShapedDevices.csv")?; Ok((shaped_devices, ShapedDevicesLoadSource::ShapedDevicesCsv)) } -fn load_shaping_inputs() { +fn load_topology_runtime_status_payload() { let Ok(config) = load_config() else { - warn!("Unable to load LibreQoS config while loading shaping_inputs.json"); - publish_shaping_inputs(TopologyShapingInputsFile::default()); + warn!("Unable to load LibreQoS config while loading topology runtime status"); return; }; - match lqos_network_devices::load_runtime_shaping_inputs_for_config(config.as_ref()) { + let status = match TopologyRuntimeStatusFile::load(config.as_ref()) { + Ok(status) => status, + Err(err) => { + warn!("Unable to load topology_runtime_status.json: {err}; keeping last-known-good effective parent cache"); + return; + } + }; + let identity = topology_status_identity(&status); + if !topology_status_identity_changed(&identity) { + debug!("Topology runtime status changed without shaping payload identity change"); + return; + } + match load_active_runtime_shaping_inputs_from_status(config.as_ref(), &status) { Ok(Some(shaping_inputs)) => { debug!("Loaded shaping inputs from active runtime status"); publish_shaping_inputs(shaping_inputs); + remember_topology_status_identity(identity); } Ok(None) => { if lqos_config::integration_ingress_enabled(config.as_ref()) { @@ -329,9 +308,11 @@ fn load_shaping_inputs() { debug!( "No active runtime shaping inputs published; keeping last-known-good effective parent cache" ); + remember_topology_status_identity(identity); } else { debug!("Integration ingress disabled; clearing effective parent cache"); publish_shaping_inputs(TopologyShapingInputsFile::default()); + remember_topology_status_identity(identity); } } Err(err) => { @@ -342,31 +323,31 @@ fn load_shaping_inputs() { } } -pub fn shaping_inputs_watcher() -> Result<()> { +pub fn topology_runtime_status_watcher() -> Result<()> { std::thread::Builder::new() - .name("Shaping Inputs Watcher".to_string()) + .name("Topology Runtime Status Watcher".to_string()) .spawn(|| { - debug!("Watching for shaping_inputs.json changes"); - if let Err(e) = watch_for_shaping_inputs_changing() { - error!("Error watching for shaping_inputs.json changes: {:?}", e); + debug!("Watching for topology_runtime_status.json changes"); + if let Err(e) = watch_for_topology_runtime_status_changing() { + error!("Error watching topology_runtime_status.json: {:?}", e); } })?; Ok(()) } -fn watch_for_shaping_inputs_changing() -> Result<()> { +fn watch_for_topology_runtime_status_changing() -> Result<()> { let Ok(config) = load_config() else { - error!("Unable to load LibreQoS config to watch shaping_inputs.json"); + error!("Unable to load LibreQoS config to watch topology_runtime_status.json"); return Err(anyhow::Error::msg( - "Unable to load LibreQoS config for shaping_inputs.json", + "Unable to load LibreQoS config for topology_runtime_status.json", )); }; let watch_path = topology_runtime_status_path(config.as_ref()); let mut watcher = FileWatcher::new("topology_runtime_status.json", watch_path); - watcher.set_file_exists_callback(load_shaping_inputs); - watcher.set_file_created_callback(load_shaping_inputs); - watcher.set_file_changed_callback(load_shaping_inputs); + watcher.set_file_exists_callback(load_topology_runtime_status_payload); + watcher.set_file_created_callback(load_topology_runtime_status_payload); + watcher.set_file_changed_callback(load_topology_runtime_status_payload); loop { let result = watcher.watch(); info!("topology_runtime_status.json watcher returned: {result:?}"); From f9921850d1a74bba0a397a46c77e6e60dc8d11cf Mon Sep 17 00:00:00 2001 From: rchac Date: Wed, 6 May 2026 12:34:43 +0000 Subject: [PATCH 15/20] changes --- src/rust/lqos_overrides/src/file_lock.rs | 4 ++-- src/rust/lqos_overrides/src/overrides_file.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/rust/lqos_overrides/src/file_lock.rs b/src/rust/lqos_overrides/src/file_lock.rs index 40cfdae6..14dcc54c 100644 --- a/src/rust/lqos_overrides/src/file_lock.rs +++ b/src/rust/lqos_overrides/src/file_lock.rs @@ -64,7 +64,7 @@ impl FileLock { } let unix_path = CString::new(LOCK_PATH)?; unsafe { - nix::libc::chmod(unix_path.as_ptr(), mode_t::from_le(666)); + nix::libc::chmod(unix_path.as_ptr(), mode_t::from_le(0o666)); } Ok(()) } @@ -77,7 +77,7 @@ impl FileLock { std::fs::create_dir(dir_path)?; let unix_path = CString::new(LOCK_DIR_PERMS)?; unsafe { - nix::libc::chmod(unix_path.as_ptr(), 777); + nix::libc::chmod(unix_path.as_ptr(), 0o777); } Ok(()) } diff --git a/src/rust/lqos_overrides/src/overrides_file.rs b/src/rust/lqos_overrides/src/overrides_file.rs index e63eb461..3d4eaebc 100644 --- a/src/rust/lqos_overrides/src/overrides_file.rs +++ b/src/rust/lqos_overrides/src/overrides_file.rs @@ -1130,15 +1130,15 @@ impl OverrideStore { /// /// This function reads override files from the config's LibreQoS directory without /// reloading `/etc/lqos.conf`. + /// + /// Side effects: reads override files from disk and may create the operator + /// overrides file if it does not exist. pub fn load_effective_for_config( config: &lqos_config::Config, apply_stormguard: bool, apply_treeguard: bool, ) -> Result { - let lock = FileLock::new()?; - let merged = Self::load_effective_for_config_inner(config, apply_stormguard, apply_treeguard)?; - drop(lock); - Ok(merged) + Self::load_effective_for_config_inner(config, apply_stormguard, apply_treeguard) } fn load_effective_for_config_inner( From adbdf603e522b3a467f6303f11db3d891cfc0116 Mon Sep 17 00:00:00 2001 From: rchac Date: Wed, 6 May 2026 10:39:43 -0600 Subject: [PATCH 16/20] changes --- src/build_dpkg.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/build_dpkg.sh b/src/build_dpkg.sh index 32d8a474..06925ca7 100755 --- a/src/build_dpkg.sh +++ b/src/build_dpkg.sh @@ -157,10 +157,20 @@ install -m 0644 /opt/libreqos/src/bin/lqos_api.service.example /etc/systemd/syst install -m 0644 /opt/libreqos/src/bin/lqos_setup.service.example /etc/systemd/system/lqos_setup.service /bin/rm -f /etc/systemd/system/lqos_netplan_helper.service || true /bin/systemctl daemon-reload || true -/bin/systemctl stop lqos_node_manager || true # In case it's running from a previous release -/bin/systemctl disable lqos_node_manager || true # In case it's running from a previous release -/bin/systemctl stop lqos_netplan_helper || true -/bin/systemctl disable lqos_netplan_helper || true + +unit_stop_disable_if_present() { +local unit="$1" +if /bin/systemctl cat "${unit}" >/dev/null 2>&1 \ + || [ -e "/etc/systemd/system/${unit}" ] \ + || [ -e "/lib/systemd/system/${unit}" ] \ + || [ -e "/usr/lib/systemd/system/${unit}" ]; then + /bin/systemctl stop "${unit}" || true + /bin/systemctl disable "${unit}" || true +fi +} + +unit_stop_disable_if_present lqos_node_manager.service # In case it's running from a previous release +unit_stop_disable_if_present lqos_netplan_helper.service case "$(/opt/libreqos/src/bin/lqos_setup postinst-action)" in activate_runtime) /opt/libreqos/src/bin/lqos_setup activate-runtime From 4bea5eb05738dbb1a117785e1364f9ff418a7339 Mon Sep 17 00:00:00 2001 From: rchac Date: Wed, 6 May 2026 18:40:43 +0000 Subject: [PATCH 17/20] changes --- src/LibreQoS.py | 103 ++++++-- src/rust/lqos_bus/src/bus/protocol.rs | 13 + src/rust/lqos_bus/src/bus/request.rs | 8 + src/rust/lqos_python/src/lib.rs | 55 +++++ src/rust/lqos_sys/src/lib.rs | 1 + src/rust/lqos_sys/src/lqos_kernel.rs | 223 +++++++++++++----- src/rust/lqosd/src/main.rs | 34 +++ .../lqosd/src/shaped_devices_tracker/mod.rs | 15 +- src/rust/lqosd/src/urgent.rs | 80 ++++++- src/test_xdp_mapping_recovery.py | 124 ++++++++++ 10 files changed, 577 insertions(+), 79 deletions(-) create mode 100644 src/test_xdp_mapping_recovery.py diff --git a/src/LibreQoS.py b/src/LibreQoS.py index 93f17ac4..67e04899 100755 --- a/src/LibreQoS.py +++ b/src/LibreQoS.py @@ -51,13 +51,37 @@ from liblqos_python import is_lqosd_alive, clear_ip_mappings, delete_ip_mapping, shaping_cpu_count, \ Bakery -# Optional: urgent issue submission (available in newer liblqos_python) +# Optional: urgent issue helpers (available in newer liblqos_python) try: from liblqos_python import submit_urgent_issue # type: ignore except Exception: def submit_urgent_issue(*_args, **_kwargs): return False +try: + from liblqos_python import clear_urgent_issue_by_identity # type: ignore +except ImportError: + _missing_clear_urgent_issue_by_identity_warned = False + + def clear_urgent_issue_by_identity(*_args, **_kwargs): + global _missing_clear_urgent_issue_by_identity_warned + if not _missing_clear_urgent_issue_by_identity_warned: + logging.warning("Unable to clear recovered urgent issues: liblqos_python helper is unavailable") + _missing_clear_urgent_issue_by_identity_warned = True + return False + +try: + from liblqos_python import xdp_ip_mapping_ready # type: ignore +except ImportError: + _missing_xdp_ip_mapping_ready_warned = False + + def xdp_ip_mapping_ready(): + global _missing_xdp_ip_mapping_ready_warned + if not _missing_xdp_ip_mapping_ready_warned: + logging.warning("Unable to check XDP IP mapping readiness: liblqos_python helper is unavailable") + _missing_xdp_ip_mapping_ready_warned = True + return True + try: from liblqos_python import calculate_topology_source_generation # type: ignore except Exception: @@ -78,6 +102,11 @@ class ValidationFailure(RefreshFailure): pass +XDP_IP_MAPPING_APPLY_FAILED = "XDP_IP_MAPPING_APPLY_FAILED" +XDP_MAPPING_READY_TIMEOUT_SECONDS = 5.0 +XDP_MAPPING_READY_INTERVAL_SECONDS = 0.1 + + def report_refresh_failure(code, message, context=None, dedupe_key=None): logging.error(message) print("ERROR: " + message) @@ -98,6 +127,56 @@ def report_refresh_failure(code, message, context=None, dedupe_key=None): pass raise RefreshFailure(message) + +def wait_for_xdp_ip_mapping_ready( + ready=xdp_ip_mapping_ready, + sleep=time.sleep, + now=time.monotonic, + timeout_seconds=XDP_MAPPING_READY_TIMEOUT_SECONDS, + interval_seconds=XDP_MAPPING_READY_INTERVAL_SECONDS, +): + deadline = now() + timeout_seconds + while now() < deadline: + if ready(): + return + sleep(interval_seconds) + raise TimeoutError("XDP IP mapping BPF maps are not ready") + + +def apply_xdp_ip_mappings( + ip_map_batch, + context, + ready=xdp_ip_mapping_ready, + sleep=time.sleep, + report_failure=report_refresh_failure, + clear_recovered_issue=clear_urgent_issue_by_identity, +): + try: + wait_for_xdp_ip_mapping_ready(ready=ready, sleep=sleep) + except (TimeoutError, OSError) as exc: + report_failure( + XDP_IP_MAPPING_APPLY_FAILED, + "Failed to apply XDP IP mappings: " + str(exc), + context, + XDP_IP_MAPPING_APPLY_FAILED, + ) + return + try: + ip_map_batch.finish_ip_mappings() + ip_map_batch.submit() + except OSError as exc: + report_failure( + XDP_IP_MAPPING_APPLY_FAILED, + "Failed to apply XDP IP mappings: " + str(exc), + context, + XDP_IP_MAPPING_APPLY_FAILED, + ) + return + try: + clear_recovered_issue(XDP_IP_MAPPING_APPLY_FAILED, XDP_IP_MAPPING_APPLY_FAILED) + except OSError as exc: + logging.warning("Unable to clear recovered XDP IP mapping urgent issue: %s", exc) + R2Q = 10 #MAX_R2Q = 200_000 MAX_R2Q = 60_000 # See https://lartc.vger.kernel.narkive.com/NKaH1ZNG/htb-quantum-of-class-100001-is-small-consider-r2q-change @@ -2757,20 +2836,14 @@ def refreshShapers(): print("Observe mode active; skipping XDP-CPUMAP-TC IP filter apply after clearing mappings") numXdpCommands = 0 else: - ipMapBatch.finish_ip_mappings() - try: - ipMapBatch.submit() - except Exception as e: - report_refresh_failure( - "XDP_IP_MAPPING_APPLY_FAILED", - "Failed to apply XDP IP mappings: " + str(e), - { - "required_ip_mappings": requiredIpMappings, - "queued_requests": numXdpCommands, - "on_a_stick": on_a_stick(), - }, - "XDP_IP_MAPPING_APPLY_FAILED", - ) + apply_xdp_ip_mappings( + ipMapBatch, + { + "required_ip_mappings": requiredIpMappings, + "queued_requests": numXdpCommands, + "on_a_stick": on_a_stick(), + }, + ) #for command in xdpCPUmapCommands: # logging.info(command) # commands = command.split(' ') diff --git a/src/rust/lqos_bus/src/bus/protocol.rs b/src/rust/lqos_bus/src/bus/protocol.rs index ce68d635..6d48ac66 100644 --- a/src/rust/lqos_bus/src/bus/protocol.rs +++ b/src/rust/lqos_bus/src/bus/protocol.rs @@ -171,6 +171,19 @@ mod tests { assert_eq!(decoded.requests, session.requests); } + #[test] + fn cbor_round_trip_clear_urgent_issue_by_identity_request() { + let session = BusSession { + requests: vec![BusRequest::ClearUrgentIssueByIdentity { + code: "XDP_IP_MAPPING_APPLY_FAILED".to_string(), + dedupe_key: "XDP_IP_MAPPING_APPLY_FAILED".to_string(), + }], + }; + let bytes = encode_session_cbor(&session).expect("encode_session_cbor"); + let decoded = decode_session_cbor(&bytes).expect("decode_session_cbor"); + assert_eq!(decoded.requests, session.requests); + } + #[test] fn cbor_round_trip_reply() { let reply = BusReply { diff --git a/src/rust/lqos_bus/src/bus/request.rs b/src/rust/lqos_bus/src/bus/request.rs index e2561fd9..1209ed21 100644 --- a/src/rust/lqos_bus/src/bus/request.rs +++ b/src/rust/lqos_bus/src/bus/request.rs @@ -539,6 +539,14 @@ pub enum BusRequest { /// Clear a specific urgent issue by ID ClearUrgentIssue(u64), + /// Clear urgent issues matching a code and dedupe key + ClearUrgentIssueByIdentity { + /// Machine-readable issue code + code: String, + /// Dedupe key used when the issue was submitted + dedupe_key: String, + }, + /// Clear all urgent issues ClearAllUrgentIssues, diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index 4c2c9484..788b7cef 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -942,6 +942,7 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(add_ip_mapping, m)?)?; m.add_function(wrap_pyfunction!(validate_shaped_devices, m)?)?; m.add_function(wrap_pyfunction!(wait_for_bus_ready, m)?)?; + m.add_function(wrap_pyfunction!(xdp_ip_mapping_ready, m)?)?; m.add_function(wrap_pyfunction!(is_libre_already_running, m)?)?; m.add_function(wrap_pyfunction!(create_lock_file, m)?)?; m.add_function(wrap_pyfunction!(free_lock_file, m)?)?; @@ -1084,6 +1085,7 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(scheduler_error, m)?)?; m.add_function(wrap_pyfunction!(scheduler_output, m)?)?; m.add_function(wrap_pyfunction!(submit_urgent_issue, m)?)?; + m.add_function(wrap_pyfunction!(clear_urgent_issue_by_identity, m)?)?; m.add_function(wrap_pyfunction!(xdp_ip_mapping_capacity, m)?)?; m.add_function(wrap_pyfunction!(is_insight_enabled, m)?)?; m.add_function(wrap_pyfunction!(log_info, m)?)?; @@ -1379,6 +1381,13 @@ fn xdp_ip_mapping_capacity() -> PyResult { Ok(lqos_sys::ip_mapping_capacity()) } +/// Returns whether the pinned BPF maps required for XDP IP mapping updates are ready. +#[pyfunction] +fn xdp_ip_mapping_ready() -> PyResult { + lqos_sys::ip_mapping_subsystem_ready() + .map_err(|e| PyOSError::new_err(format!("Unable to inspect XDP IP mapping maps: {e}"))) +} + /// Requests Rust-side validation of `ShapedDevices.csv` #[pyfunction] fn validate_shaped_devices() -> PyResult { @@ -3865,6 +3874,52 @@ fn submit_urgent_issue( Ok(false) } +/// Clear urgent issues matching a code and dedupe key. +#[pyfunction] +fn clear_urgent_issue_by_identity(_py: Python, code: String, dedupe_key: String) -> PyResult { + if let Ok(reply) = run_query(vec![BusRequest::ClearUrgentIssueByIdentity { + code: code.clone(), + dedupe_key: dedupe_key.clone(), + }]) { + for resp in reply.iter() { + if let BusResponse::Ack = resp { + return Ok(true); + } + } + } + + let Ok(reply) = run_query(vec![BusRequest::GetUrgentIssues]) else { + return Ok(false); + }; + let mut ids_to_clear = Vec::new(); + for resp in reply.iter() { + if let BusResponse::UrgentIssues(issues) = resp { + ids_to_clear.extend( + issues + .iter() + .filter(|issue| { + issue.code == code + && issue.dedupe_key.as_deref().unwrap_or(&issue.code) == dedupe_key + }) + .map(|issue| issue.id), + ); + } + } + + if ids_to_clear.is_empty() { + return Ok(false); + } + + let requests = ids_to_clear + .into_iter() + .map(BusRequest::ClearUrgentIssue) + .collect::>(); + let Ok(reply) = run_query(requests) else { + return Ok(false); + }; + Ok(reply.iter().any(|resp| matches!(resp, BusResponse::Ack))) +} + /// Log an informational message via the lqosd bus (appears in lqosd logs). #[pyfunction] fn log_info(_py: Python, message: String) -> PyResult { diff --git a/src/rust/lqos_sys/src/lib.rs b/src/rust/lqos_sys/src/lib.rs index f1e10074..a284b0ce 100644 --- a/src/rust/lqos_sys/src/lib.rs +++ b/src/rust/lqos_sys/src/lib.rs @@ -34,6 +34,7 @@ pub use kernel_wrapper::LibreQoSKernels; pub use linux::num_possible_cpus; pub use lqos_kernel::interface_name_to_index; pub use lqos_kernel::ip_mapping_capacity; +pub use lqos_kernel::ip_mapping_subsystem_ready; pub use lqos_kernel::max_tracked_ips; pub use lqos_kernel::unload_xdp_from_interface; pub use tc_classify_control::{initialize_tc_classify_bypass, set_tc_classify_bypass}; diff --git a/src/rust/lqos_sys/src/lqos_kernel.rs b/src/rust/lqos_sys/src/lqos_kernel.rs index 809d05dc..c5ecfaf8 100644 --- a/src/rust/lqos_sys/src/lqos_kernel.rs +++ b/src/rust/lqos_sys/src/lqos_kernel.rs @@ -31,6 +31,13 @@ pub(crate) mod bpf { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); } +#[derive(Clone, Copy)] +struct PinnedMapAbiSpec { + path: &'static str, + expected_key_size: u32, + expected_value_size: u32, +} + /// Returns the value set in the C XDP system's MAX_TRACKED_IPS /// constant. pub fn max_tracked_ips() -> usize { @@ -42,27 +49,7 @@ fn pinned_map_max_entries(path: &str) -> Result> { return Ok(None); } - let path_c = CString::new(path)?; - let fd = unsafe { bpf_obj_get(path_c.as_ptr()) }; - if fd < 0 { - warn!("Unable to open pinned BPF map '{path}' for capacity check (fd={fd})"); - return Ok(None); - } - - let mut info = MaybeUninit::::zeroed(); - let mut len: u32 = std::mem::size_of::() as u32; - let err = unsafe { bpf_obj_get_info_by_fd(fd, info.as_mut_ptr() as *mut c_void, &mut len) }; - unsafe { - close(fd); - } - if err != 0 { - return Err(Error::msg(format!( - "Unable to query pinned BPF map '{path}' capacity (err={err})." - ))); - } - - let info = unsafe { info.assume_init() }; - Ok(Some(info.max_entries)) + Ok(pinned_map_info(path)?.map(|info| info.max_entries)) } /// Returns the currently available IP-mapping capacity. @@ -80,6 +67,155 @@ pub fn ip_mapping_capacity() -> usize { } } +fn ip_mapping_pinned_map_abi_specs() -> [PinnedMapAbiSpec; 3] { + let expected_key_size = std::mem::size_of::() as u32; + let expected_value_size = std::mem::size_of::() as u32; + [ + PinnedMapAbiSpec { + path: "/sys/fs/bpf/map_ip_to_cpu_and_tc", + expected_key_size, + expected_value_size, + }, + PinnedMapAbiSpec { + path: "/sys/fs/bpf/ip_to_cpu_and_tc_hotcache", + expected_key_size: std::mem::size_of::() as u32, + expected_value_size, + }, + PinnedMapAbiSpec { + path: "/sys/fs/bpf/ip_mapping_epoch", + expected_key_size: std::mem::size_of::() as u32, + expected_value_size: std::mem::size_of::() as u32, + }, + ] +} + +fn pinned_map_info(path: &str) -> Result> { + let path_c = CString::new(path)?; + let fd = unsafe { bpf_obj_get(path_c.as_ptr()) }; + if fd < 0 { + return Ok(None); + } + + let mut info = MaybeUninit::::zeroed(); + let mut len: u32 = std::mem::size_of::() as u32; + let err = unsafe { bpf_obj_get_info_by_fd(fd, info.as_mut_ptr() as *mut c_void, &mut len) }; + unsafe { + close(fd); + } + if err != 0 { + return Err(Error::msg(format!( + "Unable to query pinned BPF map '{path}' info (err={err})." + ))); + } + let info = unsafe { info.assume_init() }; + Ok(Some(info)) +} + +fn pinned_map_key_value_sizes(path: &str) -> Result> { + Ok(pinned_map_info(path)?.map(|info| (info.key_size, info.value_size))) +} + +fn ip_mapping_subsystem_ready_with(mut map_sizes: F) -> Result +where + F: FnMut(&str) -> Result>, +{ + for spec in ip_mapping_pinned_map_abi_specs() { + let Some((key_size, value_size)) = map_sizes(spec.path)? else { + return Ok(false); + }; + if key_size != spec.expected_key_size || value_size != spec.expected_value_size { + return Ok(false); + } + } + Ok(true) +} + +/// Returns whether the pinned maps required for XDP IP mapping updates are ready. +/// +/// This function has side effects: it opens the relevant pinned BPF maps and +/// queries their ABI metadata. It returns `Ok(false)` for maps that are not yet +/// pinned or have an incompatible ABI, and `Err` for unexpected inspection +/// failures. +pub fn ip_mapping_subsystem_ready() -> Result { + ip_mapping_subsystem_ready_with(pinned_map_key_value_sizes) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn ip_mapping_ready_requires_all_expected_maps() { + let specs = ip_mapping_pinned_map_abi_specs(); + let sizes: HashMap<&str, (u32, u32)> = specs + .iter() + .map(|spec| { + ( + spec.path, + (spec.expected_key_size, spec.expected_value_size), + ) + }) + .collect(); + + let ready = ip_mapping_subsystem_ready_with(|path| Ok(sizes.get(path).copied())) + .expect("readiness check should succeed"); + + assert!(ready); + } + + #[test] + fn ip_mapping_ready_is_false_when_a_map_is_missing() { + let ready = ip_mapping_subsystem_ready_with(|path| { + if path == "/sys/fs/bpf/ip_mapping_epoch" { + Ok(None) + } else { + let spec = ip_mapping_pinned_map_abi_specs() + .into_iter() + .find(|spec| spec.path == path) + .expect("test path should be part of IP mapping ABI specs"); + Ok(Some((spec.expected_key_size, spec.expected_value_size))) + } + }) + .expect("missing map should not be an inspection error"); + + assert!(!ready); + } + + #[test] + fn ip_mapping_ready_is_false_when_a_map_abi_differs() { + let ready = ip_mapping_subsystem_ready_with(|path| { + let spec = ip_mapping_pinned_map_abi_specs() + .into_iter() + .find(|spec| spec.path == path) + .expect("test path should be part of IP mapping ABI specs"); + let value_size = if path == "/sys/fs/bpf/map_ip_to_cpu_and_tc" { + spec.expected_value_size + 1 + } else { + spec.expected_value_size + }; + Ok(Some((spec.expected_key_size, value_size))) + }) + .expect("ABI mismatch should not be an inspection error"); + + assert!(!ready); + } + + #[test] + fn ip_mapping_ready_returns_inspection_error() { + let err = ip_mapping_subsystem_ready_with(|path| { + if path == "/sys/fs/bpf/map_ip_to_cpu_and_tc" { + Err(Error::msg("map info failed")) + } else { + Ok(None) + } + }) + .expect_err("inspection errors should be returned to the caller"); + + assert!(err.to_string().contains("map info failed")); + } +} + pub fn check_root() -> Result<()> { unsafe { if geteuid() == 0 { @@ -99,25 +235,10 @@ fn remove_incompatible_pinned_map( return Ok(()); } - let path_c = CString::new(path)?; - let fd = unsafe { bpf_obj_get(path_c.as_ptr()) }; - if fd < 0 { - warn!("Unable to open pinned BPF map '{path}' for ABI check (fd={fd})"); + let Some(info) = pinned_map_info(path)? else { + warn!("Unable to open pinned BPF map '{path}' for ABI check"); return Ok(()); - } - - let mut info = MaybeUninit::::zeroed(); - let mut len: u32 = std::mem::size_of::() as u32; - let err = unsafe { bpf_obj_get_info_by_fd(fd, info.as_mut_ptr() as *mut c_void, &mut len) }; - unsafe { - close(fd); - } - if err != 0 { - return Err(Error::msg(format!( - "Unable to query pinned BPF map '{path}' info (err={err})." - ))); - } - let info = unsafe { info.assume_init() }; + }; if info.key_size != expected_key_size || info.value_size != expected_value_size { warn!( @@ -135,23 +256,13 @@ fn remove_incompatible_pinned_map( } fn ensure_ip_mapping_maps_abi() -> Result<()> { - let expected_key_size = std::mem::size_of::() as u32; - let expected_value_size = std::mem::size_of::() as u32; - remove_incompatible_pinned_map( - "/sys/fs/bpf/map_ip_to_cpu_and_tc", - expected_key_size, - expected_value_size, - )?; - remove_incompatible_pinned_map( - "/sys/fs/bpf/ip_to_cpu_and_tc_hotcache", - std::mem::size_of::() as u32, - expected_value_size, - )?; - remove_incompatible_pinned_map( - "/sys/fs/bpf/ip_mapping_epoch", - std::mem::size_of::() as u32, - std::mem::size_of::() as u32, - )?; + for spec in ip_mapping_pinned_map_abi_specs() { + remove_incompatible_pinned_map( + spec.path, + spec.expected_key_size, + spec.expected_value_size, + )?; + } remove_incompatible_pinned_map( "/sys/fs/bpf/tc_classify_control", std::mem::size_of::() as u32, diff --git a/src/rust/lqosd/src/main.rs b/src/rust/lqosd/src/main.rs index 6976a159..5e745fb2 100644 --- a/src/rust/lqosd/src/main.rs +++ b/src/rust/lqosd/src/main.rs @@ -1047,6 +1047,10 @@ fn handle_bus_requests(requests: &[BusRequest], responses: &mut Vec urgent::clear(*id); BusResponse::Ack } + BusRequest::ClearUrgentIssueByIdentity { code, dedupe_key } => { + urgent::clear_by_identity(code, dedupe_key); + BusResponse::Ack + } BusRequest::ClearAllUrgentIssues => { urgent::clear_all(); BusResponse::Ack @@ -1390,3 +1394,33 @@ fn search_result_to_bus(entry: node_manager::SearchResult) -> lqos_bus::SearchRe } } } + +#[cfg(test)] +mod tests { + use super::*; + use lqos_bus::{UrgentSeverity, UrgentSource}; + + #[test] + fn bus_clear_urgent_issue_by_identity_reaches_urgent_store() { + let code = "TEST_BUS_XDP_IP_MAPPING_APPLY_FAILED"; + urgent::clear_by_identity(code, code); + urgent::submit( + UrgentSource::LibreQoS, + UrgentSeverity::Error, + code.to_string(), + "mapping failed".to_string(), + None, + Some(code.to_string()), + ); + + let requests = [BusRequest::ClearUrgentIssueByIdentity { + code: code.to_string(), + dedupe_key: code.to_string(), + }]; + let mut responses = Vec::new(); + handle_bus_requests(&requests, &mut responses); + + assert!(matches!(responses.as_slice(), [BusResponse::Ack])); + assert!(!urgent::list().iter().any(|issue| issue.code == code)); + } +} diff --git a/src/rust/lqosd/src/shaped_devices_tracker/mod.rs b/src/rust/lqosd/src/shaped_devices_tracker/mod.rs index 20959e14..e1a64021 100644 --- a/src/rust/lqosd/src/shaped_devices_tracker/mod.rs +++ b/src/rust/lqosd/src/shaped_devices_tracker/mod.rs @@ -4,14 +4,13 @@ use anyhow::Result; use arc_swap::ArcSwap; use fxhash::{FxHashMap, FxHashSet}; use lqos_bus::{BusResponse, Circuit}; +#[cfg(test)] +use lqos_config::load_active_runtime_shaping_inputs; use lqos_config::{ NetworkJsonNode, NetworkJsonTransport, TopologyRuntimeShapingPayloadIdentity, TopologyRuntimeStatusFile, TopologyShapingInputsFile, - load_active_runtime_shaping_inputs_from_status, load_config, - topology_runtime_status_path, + load_active_runtime_shaping_inputs_from_status, load_config, topology_runtime_status_path, }; -#[cfg(test)] -use lqos_config::load_active_runtime_shaping_inputs; use lqos_queue_tracker::EFFECTIVE_NODE_RATES; use lqos_utils::file_watcher::FileWatcher; use lqos_utils::hash_to_i64; @@ -106,7 +105,9 @@ fn publish_shaping_inputs(shaping_inputs: TopologyShapingInputsFile) { invalidate_executive_cache_snapshot(); } -fn topology_status_identity(status: &TopologyRuntimeStatusFile) -> TopologyRuntimeShapingPayloadIdentity { +fn topology_status_identity( + status: &TopologyRuntimeStatusFile, +) -> TopologyRuntimeShapingPayloadIdentity { status.shaping_payload_identity() } @@ -286,7 +287,9 @@ fn load_topology_runtime_status_payload() { let status = match TopologyRuntimeStatusFile::load(config.as_ref()) { Ok(status) => status, Err(err) => { - warn!("Unable to load topology_runtime_status.json: {err}; keeping last-known-good effective parent cache"); + warn!( + "Unable to load topology_runtime_status.json: {err}; keeping last-known-good effective parent cache" + ); return; } }; diff --git a/src/rust/lqosd/src/urgent.rs b/src/rust/lqosd/src/urgent.rs index d8a76da0..f38b87c4 100644 --- a/src/rust/lqosd/src/urgent.rs +++ b/src/rust/lqosd/src/urgent.rs @@ -17,6 +17,10 @@ fn now_unix() -> u64 { unix_now().unwrap_or_default() } +fn urgent_identity_key(code: &str, dedupe_key: Option<&str>) -> String { + dedupe_key.unwrap_or(code).to_string() +} + fn prune_expired(q: &mut VecDeque) { let now = now_unix(); while let Some(front) = q.front() { @@ -44,8 +48,8 @@ pub fn submit( let mut guard = URGENT.lock(); prune_expired(&mut guard); - // Dedupe: same (code, dedupe_key) within window → update timestamp/message - let key = dedupe_key.clone().unwrap_or_else(|| code.clone()); + // Dedupe: same (code, dedupe_key) within window updates timestamp/message. + let key = urgent_identity_key(&code, dedupe_key.as_deref()); if let Some(existing) = guard.iter_mut().rev().find(|i| { (i.code == code) && (i.dedupe_key.as_deref().unwrap_or("") == key) @@ -90,7 +94,79 @@ pub fn clear(id: u64) -> bool { } } +/// Clears urgent issues matching the same code and dedupe key used for submission. +pub fn clear_by_identity(code: &str, dedupe_key: &str) -> usize { + let key = urgent_identity_key(code, Some(dedupe_key)); + let mut guard = URGENT.lock(); + let before = guard.len(); + guard.retain(|issue| { + issue.code != code || issue.dedupe_key.as_deref().unwrap_or("") != key.as_str() + }); + before.saturating_sub(guard.len()) +} + pub fn clear_all() { let mut guard = URGENT.lock(); guard.clear(); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clear_by_identity_removes_only_matching_issues() { + let code = "TEST_XDP_IP_MAPPING_APPLY_FAILED"; + let other_code = "TEST_OTHER"; + let other_dedupe = "TEST_XDP_IP_MAPPING_APPLY_FAILED_OTHER_DEDUPE"; + clear_by_identity(code, code); + clear_by_identity(code, other_dedupe); + clear_by_identity(other_code, code); + submit( + UrgentSource::LibreQoS, + UrgentSeverity::Error, + code.to_string(), + "mapping failed".to_string(), + None, + Some(code.to_string()), + ); + submit( + UrgentSource::LibreQoS, + UrgentSeverity::Error, + code.to_string(), + "same code different dedupe".to_string(), + None, + Some(other_dedupe.to_string()), + ); + submit( + UrgentSource::LibreQoS, + UrgentSeverity::Error, + other_code.to_string(), + "other failed".to_string(), + None, + Some(code.to_string()), + ); + + let cleared = clear_by_identity(code, code); + let matching_issue_survived = list() + .into_iter() + .any(|issue| issue.code == code && issue.dedupe_key.as_deref() == Some(other_dedupe)); + + assert_eq!(cleared, 1); + assert!(matching_issue_survived); + assert_eq!(clear_by_identity(code, other_dedupe), 1); + assert_eq!(clear_by_identity(other_code, code), 1); + } + + #[test] + fn urgent_identity_key_falls_back_to_code() { + assert_eq!( + urgent_identity_key("CODE_ONLY", None), + "CODE_ONLY".to_string() + ); + assert_eq!( + urgent_identity_key("CODE", Some("DEDUP")), + "DEDUP".to_string() + ); + } +} diff --git a/src/test_xdp_mapping_recovery.py b/src/test_xdp_mapping_recovery.py new file mode 100644 index 00000000..650f4bdd --- /dev/null +++ b/src/test_xdp_mapping_recovery.py @@ -0,0 +1,124 @@ +import unittest + +import LibreQoS + + +class FakeIpMapBatch: + def __init__(self, submit_error=None): + self.finished = False + self.submitted = False + self.submit_error = submit_error + + def finish_ip_mappings(self): + self.finished = True + + def submit(self): + self.submitted = True + if self.submit_error is not None: + raise self.submit_error + + +class XdpMappingRecoveryTests(unittest.TestCase): + def test_ready_on_first_check_does_not_sleep(self): + sleeps = [] + + LibreQoS.wait_for_xdp_ip_mapping_ready( + ready=lambda: True, + sleep=lambda seconds: sleeps.append(seconds), + now=lambda: 0.0, + ) + + self.assertEqual(sleeps, []) + + def test_readiness_after_one_false_check_sleeps_once(self): + checks = iter([False, True]) + sleeps = [] + times = iter([0.0, 0.0, 0.0]) + + LibreQoS.wait_for_xdp_ip_mapping_ready( + ready=lambda: next(checks), + sleep=lambda seconds: sleeps.append(seconds), + now=lambda: next(times), + timeout_seconds=5.0, + interval_seconds=0.1, + ) + + self.assertEqual(sleeps, [0.1]) + + def test_persistent_false_readiness_times_out(self): + times = iter([0.0, 0.0, 0.2]) + + with self.assertRaises(TimeoutError): + LibreQoS.wait_for_xdp_ip_mapping_ready( + ready=lambda: False, + sleep=lambda _seconds: None, + now=lambda: next(times), + timeout_seconds=0.1, + interval_seconds=0.1, + ) + + def test_readiness_os_error_reports_before_submit(self): + batch = FakeIpMapBatch() + reports = [] + + def raise_os_error(): + raise OSError("Unable to inspect XDP IP mapping maps") + + LibreQoS.apply_xdp_ip_mappings( + batch, + {"queued_requests": 1}, + ready=raise_os_error, + sleep=lambda _seconds: None, + report_failure=lambda *args: reports.append(args), + clear_recovered_issue=lambda *_args: True, + ) + + self.assertFalse(batch.finished) + self.assertFalse(batch.submitted) + self.assertEqual(reports[0][0], LibreQoS.XDP_IP_MAPPING_APPLY_FAILED) + + def test_submit_failure_reports_mapping_apply_failure(self): + batch = FakeIpMapBatch(submit_error=OSError("Unable to open BPF map")) + reports = [] + + LibreQoS.apply_xdp_ip_mappings( + batch, + {"queued_requests": 1}, + ready=lambda: True, + sleep=lambda _seconds: None, + report_failure=lambda *args: reports.append(args), + clear_recovered_issue=lambda *_args: True, + ) + + self.assertTrue(batch.finished) + self.assertTrue(batch.submitted) + self.assertEqual(reports[0][0], LibreQoS.XDP_IP_MAPPING_APPLY_FAILED) + + def test_success_clears_recovered_mapping_issue(self): + batch = FakeIpMapBatch() + cleared = [] + + LibreQoS.apply_xdp_ip_mappings( + batch, + {"queued_requests": 1}, + ready=lambda: True, + sleep=lambda _seconds: None, + report_failure=lambda *_args: self.fail("unexpected failure report"), + clear_recovered_issue=lambda code, dedupe_key: cleared.append((code, dedupe_key)) or True, + ) + + self.assertTrue(batch.finished) + self.assertTrue(batch.submitted) + self.assertEqual( + cleared, + [ + ( + LibreQoS.XDP_IP_MAPPING_APPLY_FAILED, + LibreQoS.XDP_IP_MAPPING_APPLY_FAILED, + ) + ], + ) + + +if __name__ == "__main__": + unittest.main() From 5cd7deb825f57d995e44402a10c128a62596fffb Mon Sep 17 00:00:00 2001 From: rchac Date: Wed, 6 May 2026 20:37:39 +0000 Subject: [PATCH 18/20] changes --- .../js_build/src/executive_utils.js | 16 +- .../js_build/src/pubsub/listeners.js | 23 ++ .../js_build/src/pubsub/ws.test.mjs | 103 +++++ .../src/node_manager/js_build/src/template.js | 382 ++++++++++++++---- .../js_build/test-node-manager.sh | 4 +- src/rust/lqosd/src/node_manager/ws.rs | 176 +++++++- .../lqosd/src/node_manager/ws/messages.rs | 21 +- 7 files changed, 634 insertions(+), 91 deletions(-) create mode 100644 src/rust/lqosd/src/node_manager/js_build/src/pubsub/listeners.js create mode 100644 src/rust/lqosd/src/node_manager/js_build/src/pubsub/ws.test.mjs diff --git a/src/rust/lqosd/src/node_manager/js_build/src/executive_utils.js b/src/rust/lqosd/src/node_manager/js_build/src/executive_utils.js index 287808de..41d8e66c 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/executive_utils.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/executive_utils.js @@ -1,17 +1,10 @@ +import {listenOnceMatchingWithTimeout} from "./pubsub/listeners"; import {get_ws_client, subscribeWS} from "./pubsub/ws"; const wsClient = get_ws_client(); const DEFAULT_EXECUTIVE_REFRESH_MS = 5000; let executiveRequestCounter = 0; -const listenOnce = (eventName, handler) => { - const wrapped = (msg) => { - wsClient.off(eventName, wrapped); - handler(msg); - }; - wsClient.on(eventName, wrapped); -}; - function normalizeEntityKinds(entityKinds) { return [...new Set((entityKinds || []).map((kind) => String(kind)))].sort(); } @@ -178,7 +171,7 @@ export function getNodeIdMap() { siteIdMap = map; resolve(map); }; - listenOnce("NetworkTreeLite", (msg) => { + listenOnceMatchingWithTimeout(wsClient, "NetworkTreeLite", 5000, () => true, (msg) => { const data = msg && msg.data ? msg.data : []; const map = new Map(); (data || []).forEach((entry) => { @@ -190,11 +183,10 @@ export function getNodeIdMap() { } }); resolveOnce(map); + }, () => { + resolveOnce(new Map()); }); wsClient.send({ NetworkTreeLite: {} }); - setTimeout(() => { - resolveOnce(new Map()); - }, 5000); }); return siteIdMapPromise; } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/pubsub/listeners.js b/src/rust/lqosd/src/node_manager/js_build/src/pubsub/listeners.js new file mode 100644 index 00000000..cf0ccb8e --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/pubsub/listeners.js @@ -0,0 +1,23 @@ +export function listenOnceMatchingWithTimeout(wsClient, eventName, timeoutMs, matches, handler, onTimeout) { + let done = false; + const cleanup = wsClient.on(eventName, (msg) => { + if (done) return; + if (!matches(msg)) return; + done = true; + clearTimeout(timer); + cleanup(); + handler(msg); + }); + const timer = setTimeout(() => { + if (done) return; + done = true; + cleanup(); + onTimeout(); + }, timeoutMs); + return () => { + if (done) return; + done = true; + clearTimeout(timer); + cleanup(); + }; +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/pubsub/ws.test.mjs b/src/rust/lqosd/src/node_manager/js_build/src/pubsub/ws.test.mjs new file mode 100644 index 00000000..46622629 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/pubsub/ws.test.mjs @@ -0,0 +1,103 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +globalThis.window = { + LQOS_UI_VERSION: "test", + location: { search: "", protocol: "http:", host: "localhost" }, + localStorage: { getItem: () => null }, + sessionStorage: { + getItem: () => null, + setItem: () => {}, + removeItem: () => {}, + }, +}; +globalThis.document = { cookie: "" }; + +const { listenOnceMatchingWithTimeout } = await import("./listeners.js"); + +function makeWsClient() { + const handlers = new Map(); + return { + on(eventName, handler) { + if (!handlers.has(eventName)) { + handlers.set(eventName, new Set()); + } + handlers.get(eventName).add(handler); + return () => this.off(eventName, handler); + }, + off(eventName, handler) { + handlers.get(eventName)?.delete(handler); + }, + emit(eventName, message) { + for (const handler of Array.from(handlers.get(eventName) || [])) { + handler(message); + } + }, + handlerCount(eventName) { + return handlers.get(eventName)?.size || 0; + }, + }; +} + +test("timeout listener ignores stale responses and resolves matching response", () => { + const wsClient = makeWsClient(); + const seen = []; + + listenOnceMatchingWithTimeout( + wsClient, + "UrgentClearResult", + 1000, + (message) => message.request_id === 2, + (message) => seen.push(message.ok), + () => seen.push("timeout"), + ); + + wsClient.emit("UrgentClearResult", { request_id: 1, ok: false }); + assert.deepEqual(seen, []); + assert.equal(wsClient.handlerCount("UrgentClearResult"), 1); + + wsClient.emit("UrgentClearResult", { request_id: 2, ok: true }); + assert.deepEqual(seen, [true]); + assert.equal(wsClient.handlerCount("UrgentClearResult"), 0); +}); + +test("timeout listener disposer removes pending handler", () => { + const wsClient = makeWsClient(); + const seen = []; + + const dispose = listenOnceMatchingWithTimeout( + wsClient, + "UrgentList", + 1000, + () => true, + () => seen.push("response"), + () => seen.push("timeout"), + ); + + assert.equal(wsClient.handlerCount("UrgentList"), 1); + dispose(); + wsClient.emit("UrgentList", { request_id: 1 }); + + assert.deepEqual(seen, []); + assert.equal(wsClient.handlerCount("UrgentList"), 0); +}); + +test("timeout listener fires timeout and removes pending handler", async () => { + const wsClient = makeWsClient(); + const seen = []; + + listenOnceMatchingWithTimeout( + wsClient, + "UrgentList", + 5, + () => false, + () => seen.push("response"), + () => seen.push("timeout"), + ); + + assert.equal(wsClient.handlerCount("UrgentList"), 1); + await new Promise((resolve) => setTimeout(resolve, 20)); + + assert.deepEqual(seen, ["timeout"]); + assert.equal(wsClient.handlerCount("UrgentList"), 0); +}); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/template.js b/src/rust/lqosd/src/node_manager/js_build/src/template.js index 227ea555..cfed7976 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/template.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/template.js @@ -2,6 +2,7 @@ import {clearDiv} from "./helpers/builders"; import {initRedact} from "./helpers/redact"; import {initDayNightMode} from "./helpers/dark_mode"; import {initColorBlind} from "./helpers/colorblind"; +import {listenOnceMatchingWithTimeout as listenWsOnceMatchingWithTimeout} from "./pubsub/listeners"; import {get_ws_client} from "./pubsub/ws"; const wsClient = get_ws_client(); @@ -29,6 +30,7 @@ function escapeHtml(text) { const SCHEDULER_STATUS_POLL_MS = 2000; const SCHEDULER_STATUS_TIMEOUT_MS = 2500; +const URGENT_ISSUES_TIMEOUT_MS = 2500; const SCHEDULER_STATUS_STARTING_GRACE_MS = 30000; const SCHEDULER_MODAL_IDLE_POLL_MS = 3000; const SCHEDULER_MODAL_ACTIVE_POLL_MS = 1000; @@ -42,24 +44,6 @@ let schedulerModalInstance = null; let cachedSchedulerDetailsData = null; let cachedQueueModeConfig = null; -function listenOnceWithTimeout(eventName, timeoutMs, handler, onTimeout) { - let done = false; - const wrapped = (msg) => { - if (done) return; - done = true; - clearTimeout(timer); - wsClient.off(eventName, wrapped); - handler(msg); - }; - const timer = setTimeout(() => { - if (done) return; - done = true; - wsClient.off(eventName, wrapped); - onTimeout(); - }, timeoutMs); - wsClient.on(eventName, wrapped); -} - function clampSchedulerPercent(value) { const parsed = Number(value); if (!Number.isFinite(parsed)) return 0; @@ -526,7 +510,7 @@ function loadSchedulerStatus(force = false) { } schedulerStatusRequestInFlight = true; - listenOnceWithTimeout("SchedulerStatus", SCHEDULER_STATUS_TIMEOUT_MS, (msg) => { + listenWsOnceMatchingWithTimeout(wsClient, "SchedulerStatus", SCHEDULER_STATUS_TIMEOUT_MS, () => true, (msg) => { schedulerStatusRequestInFlight = false; if (!msg || !msg.data) { if (cachedSchedulerStatusData) { @@ -685,7 +669,7 @@ function refreshSchedulerModalDetails(showLoading = false) { renderSchedulerModalLoading(); } schedulerModalRequestInFlight = true; - listenOnceWithTimeout("SchedulerDetails", SCHEDULER_STATUS_TIMEOUT_MS, (msg) => { + listenWsOnceMatchingWithTimeout(wsClient, "SchedulerDetails", SCHEDULER_STATUS_TIMEOUT_MS, () => true, (msg) => { schedulerModalRequestInFlight = false; if (!isSchedulerModalOpen()) { stopSchedulerModalPolling(); @@ -1108,6 +1092,15 @@ function initUrgentIssues() { const containerId = 'urgentStatus'; const linkId = 'urgentStatusLink'; const badgeId = 'urgentBadge'; + let urgentClearState = null; + let urgentStatusPending = false; + let urgentStatusRefreshQueued = false; + let urgentListPending = false; + let urgentListActiveRequestId = null; + let urgentListCancel = null; + let urgentRequestId = 0; + let pendingUrgentActionError = null; + let pendingUrgentRetryMessage = null; function ensurePlaceholder() { return document.getElementById(containerId) !== null; @@ -1131,38 +1124,251 @@ function initUrgentIssues() { function poll() { if (!ensurePlaceholder()) return; - listenOnce("UrgentStatus", (msg) => { - const count = msg && msg.data ? msg.data.count : 0; - renderStatus(count || 0); + if (urgentStatusPending) { + urgentStatusRefreshQueued = true; + return; + } + urgentStatusPending = true; + const requestId = ++urgentRequestId; + listenWsOnceMatchingWithTimeout( + wsClient, + "UrgentStatus", + URGENT_ISSUES_TIMEOUT_MS, + (msg) => msg.request_id === requestId, + (msg) => { + urgentStatusPending = false; + const count = msg && msg.data ? msg.data.count : 0; + renderStatus(count || 0); + runQueuedUrgentStatusRefresh(); + }, + () => { + urgentStatusPending = false; + runQueuedUrgentStatusRefresh(); + } + ); + wsClient.send({ UrgentStatus: { request_id: requestId } }); + } + + function runQueuedUrgentStatusRefresh() { + if (!urgentStatusRefreshQueued) return false; + urgentStatusRefreshQueued = false; + poll(); + return true; + } + + function clearActionInFlight() { + return urgentClearState !== null; + } + + function clearActionBlocked() { + return clearActionInFlight() || urgentListPending; + } + + function setClearControlsDisabled(disabled) { + document.querySelectorAll('.urgent-clear').forEach((button) => { + button.disabled = disabled; }); - wsClient.send({ UrgentStatus: {} }); + const clearAllButton = document.getElementById('urgentClearAll'); + if (clearAllButton) { + clearAllButton.disabled = disabled; + } + } + + function showUrgentBusyState(message) { + const holder = document.getElementById('urgentListContainer'); + if (!holder || !urgentModalIsOpen()) return; + holder.setAttribute('aria-busy', 'true'); + holder.querySelector('.urgent-busy-status')?.remove(); + holder.insertAdjacentHTML('afterbegin', ` +
+ + ${escapeHtml(message)} +
`); + holder.querySelector('.urgent-busy-status')?.focus(); + } + + function clearUrgentBusyState(holder) { + const target = holder || document.getElementById('urgentListContainer'); + if (!target) return; + target.removeAttribute('aria-busy'); + target.querySelector('.urgent-busy-status')?.remove(); + } + + function renderUrgentWarning(holder, message, { focus = false, retry = false } = {}) { + if (!holder) return; + holder.querySelector('.urgent-action-error')?.remove(); + const retryButton = retry + ? '' + : ''; + holder.insertAdjacentHTML('afterbegin', ` + `); + if (focus) { + holder.querySelector(retry ? '.urgent-refresh' : '.urgent-action-error')?.focus(); + } + } + + function showUrgentActionError(message) { + pendingUrgentActionError = message; + if (!urgentModalIsOpen()) return; + const holder = document.getElementById('urgentListContainer'); + if (!holder) return; + clearUrgentBusyState(holder); + renderUrgentWarning(holder, message, { focus: true }); + pendingUrgentActionError = null; + } + + function attachPendingUrgentActionError(holder) { + if (!pendingUrgentActionError || !urgentModalIsOpen() || !holder) return; + renderUrgentWarning(holder, pendingUrgentActionError); + pendingUrgentActionError = null; + } + + function showUrgentRetryState(message) { + pendingUrgentRetryMessage = message; + const holder = document.getElementById('urgentListContainer'); + if (!holder || !urgentModalIsOpen()) return; + pendingUrgentRetryMessage = null; + setClearControlsDisabled(true); + clearUrgentBusyState(holder); + holder.innerHTML = ''; + renderUrgentWarning(holder, message, { focus: true, retry: true }); + $(holder).off('click').on('click', '.urgent-refresh', function (e) { + e.preventDefault(); + showModal(); + }); + } + + function showPendingUrgentRetryState() { + if (!pendingUrgentRetryMessage || !urgentModalIsOpen()) return false; + showUrgentRetryState(pendingUrgentRetryMessage); + return true; + } + + function urgentModalIsOpen() { + return document.getElementById('urgentModal')?.classList.contains('show') === true; + } + + function focusUrgentModalFallback() { + const modalEl = document.getElementById('urgentModal'); + if (!modalEl || !urgentModalIsOpen()) return; + modalEl + .querySelector('.urgent-clear:not(:disabled), #urgentClearAll:not(:disabled), .btn-close:not(:disabled), [data-bs-dismiss="modal"]:not(:disabled)') + ?.focus(); + } + + function refreshUrgentIssues() { + showModal(); + poll(); + } + + function completeClearAction() { + urgentClearState = null; + setClearControlsDisabled(clearActionInFlight()); + refreshUrgentIssues(); + } + + function startUrgentClear({ + responseEvent, + requestPayload, + busyMessage, + failureMessage, + timeoutMessage, + }) { + if (clearActionBlocked()) { + return; + } + const requestId = ++urgentRequestId; + setClearControlsDisabled(true); + showUrgentBusyState(busyMessage); + urgentClearState = { + busyMessage, + cancel: null, + }; + urgentClearState.cancel = listenWsOnceMatchingWithTimeout( + wsClient, + responseEvent, + URGENT_ISSUES_TIMEOUT_MS, + (msg) => msg.request_id === requestId, + (msg) => { + urgentClearState = null; + if (msg && msg.ok === false) { + setClearControlsDisabled(false); + showUrgentActionError(failureMessage); + return; + } + completeClearAction(); + }, + () => { + urgentClearState = null; + setClearControlsDisabled(false); + showUrgentActionError(timeoutMessage); + } + ); + wsClient.send(requestPayload(requestId)); } function showModal() { const modalEl = document.getElementById('urgentModal'); - if (!modalEl) return; - new bootstrap.Modal(modalEl, { focus: true }).show(); + if (!modalEl) { + return; + } + bootstrap.Modal.getOrCreateInstance(modalEl, { focus: true }).show(); const holder = document.getElementById('urgentListContainer'); - if (!holder) return; - holder.innerHTML = `
Loading...
`; - listenOnce("UrgentList", (msg) => { - const items = msg && msg.data ? msg.data.items || [] : []; - if (items.length === 0) { - holder.innerHTML = '
No urgent issues.
'; - return; - } - const table = document.createElement('table'); - table.className = 'lqos-table lqos-table-compact mb-0'; - const tbody = document.createElement('tbody'); - items.forEach((it) => { - const tr = document.createElement('tr'); - const td = document.createElement('td'); - const when = new Date(it.ts * 1000).toLocaleString(); - const sev = it.severity === 'Error' ? 'danger' : 'warning'; - const clearControl = it.clearable === false - ? 'Active' - : ``; - td.innerHTML = ` + if (!holder) { + return; + } + if (showPendingUrgentRetryState()) { + return; + } + if (clearActionInFlight()) { + setClearControlsDisabled(true); + showUrgentBusyState(urgentClearState.busyMessage); + return; + } + if (urgentListPending) { + setClearControlsDisabled(true); + return; + } + urgentListPending = true; + const requestId = ++urgentRequestId; + urgentListActiveRequestId = requestId; + setClearControlsDisabled(true); + clearUrgentBusyState(holder); + holder.innerHTML = `
Loading...
`; + holder.querySelector('.urgent-loading')?.focus(); + urgentListCancel = listenWsOnceMatchingWithTimeout( + wsClient, + "UrgentList", + URGENT_ISSUES_TIMEOUT_MS, + (msg) => msg.request_id === requestId && urgentListActiveRequestId === requestId, + (msg) => { + urgentListPending = false; + urgentListActiveRequestId = null; + urgentListCancel = null; + const items = msg && msg.data ? msg.data.items || [] : []; + clearUrgentBusyState(holder); + if (items.length === 0) { + holder.innerHTML = '
No urgent issues.
'; + attachPendingUrgentActionError(holder); + setClearControlsDisabled(true); + holder.querySelector('[role="status"]')?.focus(); + return; + } + const table = document.createElement('table'); + table.className = 'lqos-table lqos-table-compact mb-0'; + const tbody = document.createElement('tbody'); + items.forEach((it) => { + const tr = document.createElement('tr'); + const td = document.createElement('td'); + const when = new Date(it.ts * 1000).toLocaleString(); + const sev = it.severity === 'Error' ? 'danger' : 'warning'; + const clearControl = it.clearable === false + ? 'Active' + : ``; + td.innerHTML = `
${it.severity} ${it.code} @@ -1173,26 +1379,44 @@ function initUrgentIssues() {
${it.message}
${it.context ? `
${it.context}
` : ''} `; - tr.appendChild(td); - tbody.appendChild(tr); - }); - table.appendChild(tbody); - holder.innerHTML = ''; - const tableWrap = document.createElement('div'); - tableWrap.className = 'table-responsive lqos-table-wrap'; - tableWrap.appendChild(table); - holder.appendChild(tableWrap); - $(holder).off('click').on('click', '.urgent-clear', function (e) { - e.preventDefault(); - const id = $(this).data('id'); - listenOnce("UrgentClearResult", () => { - showModal(); - poll(); + tr.appendChild(td); + tbody.appendChild(tr); }); - wsClient.send({ UrgentClear: { id } }); - }); - }); - wsClient.send({ UrgentList: {} }); + table.appendChild(tbody); + holder.innerHTML = ''; + const tableWrap = document.createElement('div'); + tableWrap.className = 'table-responsive lqos-table-wrap'; + tableWrap.appendChild(table); + holder.appendChild(tableWrap); + attachPendingUrgentActionError(holder); + $(holder).off('click').on('click', '.urgent-clear', function (e) { + e.preventDefault(); + const id = $(this).data('id'); + startUrgentClear({ + responseEvent: 'UrgentClearResult', + requestPayload: (requestId) => ({ UrgentClear: { id, request_id: requestId } }), + busyMessage: 'Acknowledging urgent issue...', + failureMessage: 'Unable to acknowledge urgent issue.', + timeoutMessage: 'Timed out while acknowledging urgent issue.', + }); + }); + setClearControlsDisabled(clearActionInFlight()); + focusUrgentModalFallback(); + }, + () => { + if (urgentListActiveRequestId !== requestId) return; + urgentListPending = false; + urgentListActiveRequestId = null; + urgentListCancel = null; + if (!urgentModalIsOpen()) { + setClearControlsDisabled(clearActionInFlight()); + return; + } + showUrgentRetryState('Unable to refresh urgent issues.'); + setClearControlsDisabled(true); + } + ); + wsClient.send({ UrgentList: { request_id: requestId } }); } if (!document.getElementById(containerId)) { @@ -1206,11 +1430,31 @@ function initUrgentIssues() { } $(document).off('click', '#urgentClearAll').on('click', '#urgentClearAll', () => { - listenOnce("UrgentClearAllResult", () => { - showModal(); - poll(); + startUrgentClear({ + responseEvent: 'UrgentClearAllResult', + requestPayload: (requestId) => ({ UrgentClearAll: { request_id: requestId } }), + busyMessage: 'Acknowledging urgent issues...', + failureMessage: 'Unable to acknowledge urgent issues.', + timeoutMessage: 'Timed out while acknowledging urgent issues.', }); - wsClient.send({ UrgentClearAll: {} }); + }); + + $(document).off('hidden.bs.modal', '#urgentModal').on('hidden.bs.modal', '#urgentModal', () => { + pendingUrgentRetryMessage = null; + if (urgentClearState?.cancel) { + urgentClearState.cancel(); + urgentClearState = null; + } + if (urgentListPending) { + if (urgentListCancel) { + urgentListCancel(); + urgentListCancel = null; + } + urgentListPending = false; + urgentListActiveRequestId = null; + } + clearUrgentBusyState(); + setClearControlsDisabled(clearActionInFlight()); }); poll(); diff --git a/src/rust/lqosd/src/node_manager/js_build/test-node-manager.sh b/src/rust/lqosd/src/node_manager/js_build/test-node-manager.sh index 5d093180..20456c61 100755 --- a/src/rust/lqosd/src/node_manager/js_build/test-node-manager.sh +++ b/src/rust/lqosd/src/node_manager/js_build/test-node-manager.sh @@ -9,7 +9,9 @@ if ! command -v node >/dev/null 2>&1; then fi echo "[test-node-manager] Running frontend contract tests..." -node --test "${SCRIPT_DIR}/src/config/shaped_device_wire.test.mjs" +node --test \ + "${SCRIPT_DIR}/src/config/shaped_device_wire.test.mjs" \ + "${SCRIPT_DIR}/src/pubsub/ws.test.mjs" echo "[test-node-manager] Building bundles..." "${SCRIPT_DIR}/esbuild.sh" diff --git a/src/rust/lqosd/src/node_manager/ws.rs b/src/rust/lqosd/src/node_manager/ws.rs index 6b5bb6eb..02766378 100644 --- a/src/rust/lqosd/src/node_manager/ws.rs +++ b/src/rust/lqosd/src/node_manager/ws.rs @@ -676,32 +676,37 @@ async fn receive_channel_message( return true; } } - WsRequest::UrgentStatus => { + WsRequest::UrgentStatus { request_id } => { let response = WsResponse::UrgentStatus { data: urgent::urgent_status_data(), + request_id, }; if send_ws_response(&tx, response).await { return true; } } - WsRequest::UrgentList => { + WsRequest::UrgentList { request_id } => { let response = WsResponse::UrgentList { data: urgent::urgent_list_data(), + request_id, }; if send_ws_response(&tx, response).await { return true; } } - WsRequest::UrgentClear { id } => { + WsRequest::UrgentClear { id, request_id } => { let ok = urgent::urgent_clear_id(id); - let response = WsResponse::UrgentClearResult { ok }; + let response = WsResponse::UrgentClearResult { ok, request_id }; if send_ws_response(&tx, response).await { return true; } } - WsRequest::UrgentClearAll => { + WsRequest::UrgentClearAll { request_id } => { urgent::urgent_clear_all_data(); - let response = WsResponse::UrgentClearAllResult { ok: true }; + let response = WsResponse::UrgentClearAllResult { + ok: true, + request_id, + }; if send_ws_response(&tx, response).await { return true; } @@ -2502,7 +2507,9 @@ fn payload_hint(payload: &[u8]) -> &'static str { #[cfg(test)] mod tests { use super::decode_ws_request; + use crate::node_manager::local_api::urgent::{UrgentList, UrgentStatus}; use crate::node_manager::ws::messages::WsRequest; + use crate::node_manager::ws::messages::{WsResponse, encode_ws_message}; use serde_cbor::Value as CborValue; use std::collections::BTreeMap; @@ -2567,6 +2574,163 @@ mod tests { .expect("valid update payload") } + fn encode_response_value(response: &WsResponse) -> CborValue { + let payload = encode_ws_message(response).expect("response should encode"); + serde_cbor::from_slice(payload.as_ref()).expect("response should decode as CBOR value") + } + + fn response_request_id(response: &WsResponse, event_name: &str) -> Option { + let CborValue::Map(entries) = encode_response_value(response) else { + panic!("response should encode as a CBOR map"); + }; + let event_key = text("event"); + let request_id_key = text("request_id"); + assert_eq!(entries.get(&event_key), Some(&text(event_name))); + match entries.get(&request_id_key) { + Some(CborValue::Integer(value)) => { + Some(u64::try_from(*value).expect("request_id is u64")) + } + Some(other) => panic!("unexpected request_id value: {other:?}"), + None => None, + } + } + + fn response_has_request_id(response: &WsResponse) -> bool { + let CborValue::Map(entries) = encode_response_value(response) else { + panic!("response should encode as a CBOR map"); + }; + entries.contains_key(&text("request_id")) + } + + #[test] + fn decodes_urgent_status_with_request_id() { + let payload = serde_cbor::to_vec(&map(vec![( + "UrgentStatus", + map(vec![("request_id", integer(42))]), + )])) + .expect("valid urgent status payload"); + + let request = decode_ws_request(&payload).expect("urgent status request should decode"); + match request { + WsRequest::UrgentStatus { request_id } => assert_eq!(request_id, Some(42)), + other => panic!("unexpected request: {other:?}"), + } + } + + #[test] + fn decodes_urgent_list_without_request_id() { + let payload = serde_cbor::to_vec(&map(vec![("UrgentList", map(vec![]))])) + .expect("valid urgent list payload"); + + let request = decode_ws_request(&payload).expect("urgent list request should decode"); + match request { + WsRequest::UrgentList { request_id } => assert_eq!(request_id, None), + other => panic!("unexpected request: {other:?}"), + } + } + + #[test] + fn decodes_urgent_clear_with_request_id() { + let payload = serde_cbor::to_vec(&map(vec![( + "UrgentClear", + map(vec![("id", integer(7)), ("request_id", integer(42))]), + )])) + .expect("valid urgent clear payload"); + + let request = decode_ws_request(&payload).expect("urgent clear request should decode"); + match request { + WsRequest::UrgentClear { id, request_id } => { + assert_eq!(id, 7); + assert_eq!(request_id, Some(42)); + } + other => panic!("unexpected request: {other:?}"), + } + } + + #[test] + fn decodes_urgent_clear_all_without_request_id() { + let payload = serde_cbor::to_vec(&map(vec![("UrgentClearAll", map(vec![]))])) + .expect("valid urgent clear all payload"); + + let request = decode_ws_request(&payload).expect("urgent clear all request should decode"); + match request { + WsRequest::UrgentClearAll { request_id } => assert_eq!(request_id, None), + other => panic!("unexpected request: {other:?}"), + } + } + + #[test] + fn urgent_responses_echo_request_id() { + assert_eq!( + response_request_id( + &WsResponse::UrgentStatus { + data: UrgentStatus { + has_urgent: true, + count: 1, + }, + request_id: Some(42), + }, + "UrgentStatus", + ), + Some(42), + ); + assert_eq!( + response_request_id( + &WsResponse::UrgentList { + data: UrgentList { items: Vec::new() }, + request_id: Some(43), + }, + "UrgentList", + ), + Some(43), + ); + assert_eq!( + response_request_id( + &WsResponse::UrgentClearResult { + ok: true, + request_id: Some(44), + }, + "UrgentClearResult", + ), + Some(44), + ); + assert_eq!( + response_request_id( + &WsResponse::UrgentClearAllResult { + ok: true, + request_id: Some(45), + }, + "UrgentClearAllResult", + ), + Some(45), + ); + } + + #[test] + fn urgent_responses_omit_missing_request_id() { + assert!(!response_has_request_id(&WsResponse::UrgentStatus { + data: UrgentStatus { + has_urgent: false, + count: 0, + }, + request_id: None, + })); + assert!(!response_has_request_id(&WsResponse::UrgentList { + data: UrgentList { items: Vec::new() }, + request_id: None, + })); + assert!(!response_has_request_id(&WsResponse::UrgentClearResult { + ok: true, + request_id: None, + })); + assert!(!response_has_request_id( + &WsResponse::UrgentClearAllResult { + ok: true, + request_id: None, + } + )); + } + #[test] fn decodes_create_shaped_device_with_browser_style_ipv4_bytes() { let payload = create_shaped_device_payload(device_value( diff --git a/src/rust/lqosd/src/node_manager/ws/messages.rs b/src/rust/lqosd/src/node_manager/ws/messages.rs index 37c41c7a..4a851965 100644 --- a/src/rust/lqosd/src/node_manager/ws/messages.rs +++ b/src/rust/lqosd/src/node_manager/ws/messages.rs @@ -324,12 +324,19 @@ pub enum WsRequest { ProtocolFlowTimeline { protocol: String, }, - UrgentStatus, - UrgentList, + UrgentStatus { + request_id: Option, + }, + UrgentList { + request_id: Option, + }, UrgentClear { id: u64, + request_id: Option, + }, + UrgentClearAll { + request_id: Option, }, - UrgentClearAll, UnknownIps, UnknownIpsClear, UnknownIpsCsv, @@ -626,15 +633,23 @@ pub enum WsResponse { }, UrgentStatus { data: UrgentStatus, + #[serde(skip_serializing_if = "Option::is_none")] + request_id: Option, }, UrgentList { data: UrgentList, + #[serde(skip_serializing_if = "Option::is_none")] + request_id: Option, }, UrgentClearResult { ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + request_id: Option, }, UrgentClearAllResult { ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + request_id: Option, }, UnknownIps { data: Vec, From 42175b269a9703084d3f173abf7e55b22328b8c6 Mon Sep 17 00:00:00 2001 From: rchac Date: Wed, 6 May 2026 21:26:24 +0000 Subject: [PATCH 19/20] changes --- docs-es/v2.0/TechnicalDocs/troubleshooting.md | 17 ++++------------- docs-es/v2.0/troubleshooting-es.md | 18 +++--------------- docs/v2.0/troubleshooting.md | 17 +++-------------- src/bin/lqos_scheduler.service.example | 2 ++ 4 files changed, 12 insertions(+), 42 deletions(-) diff --git a/docs-es/v2.0/TechnicalDocs/troubleshooting.md b/docs-es/v2.0/TechnicalDocs/troubleshooting.md index e91e2600..2f03daf7 100644 --- a/docs-es/v2.0/TechnicalDocs/troubleshooting.md +++ b/docs-es/v2.0/TechnicalDocs/troubleshooting.md @@ -67,21 +67,12 @@ sudo dpkg --configure -a sudo systemctl restart lqosd lqos_scheduler ``` -Git-based installs should use `./build_rust.sh` after pulling updates. It rebuilds the virtual environment before refreshing service files or restarting services. If systemd reports `status=203/EXEC`, rebuild the virtual environment with the command above and restart `lqos_scheduler`. +Git-based installs should use `./build_rust.sh` after pulling updates. It rebuilds the virtual environment before refreshing service files or restarting services. If systemd reports `status=203/EXEC` on `/opt/libreqos/venv/bin/python`, or a failed scheduler pre-start check, rebuild the virtual environment with the command above and restart `lqos_scheduler`. + +Older installs that predate the virtual environment may show `ModuleNotFoundError` and suggest system `pip` commands. Do not repair current installs with system `pip` or `--break-system-packages`; those packages are not used by the venv-backed scheduler service. Upgrade to a package that creates `/opt/libreqos/venv`, then use the repair command above. -### Python ModuleNotFoundError in older Ubuntu 24.04 installs -``` -pip uninstall binpacking --break-system-packages --yes -sudo pip uninstall binpacking --break-system-packages --yes -sudo pip install binpacking --break-system-packages -pip uninstall apscheduler --break-system-packages --yes -sudo pip uninstall apscheduler --break-system-packages --yes -sudo pip install apscheduler --break-system-packages -pip uninstall deepdiff --break-system-packages --yes -sudo pip uninstall deepdiff --break-system-packages --yes -sudo pip install deepdiff --break-system-packages -``` ### All customer IPs are listed under Unknown IPs, rather than Shaped Devices in GUI + ``` cd /opt/libreqos/src sudo systemctl stop lqos_scheduler diff --git a/docs-es/v2.0/troubleshooting-es.md b/docs-es/v2.0/troubleshooting-es.md index f95ea6a0..e0d60b2b 100644 --- a/docs-es/v2.0/troubleshooting-es.md +++ b/docs-es/v2.0/troubleshooting-es.md @@ -120,7 +120,9 @@ sudo dpkg --configure -a sudo systemctl restart lqosd lqos_scheduler ``` -Las instalaciones basadas en git deben usar `./build_rust.sh` después de actualizar. Ese script reconstruye el entorno virtual antes de actualizar los archivos de servicio o reiniciar servicios. Si systemd informa `status=203/EXEC`, reconstruya el entorno virtual con el comando anterior y reinicie `lqos_scheduler`. +Las instalaciones basadas en git deben usar `./build_rust.sh` después de actualizar. Ese script reconstruye el entorno virtual antes de actualizar los archivos de servicio o reiniciar servicios. Si systemd informa `status=203/EXEC` en `/opt/libreqos/venv/bin/python`, o una falla en la comprobación previa del scheduler, reconstruya el entorno virtual con el comando anterior y reinicie `lqos_scheduler`. + +Las instalaciones antiguas anteriores al entorno virtual pueden mostrar `ModuleNotFoundError` y recomendar comandos de `pip` del sistema. No repare instalaciones actuales con `pip` del sistema ni con `--break-system-packages`; esos paquetes no los usa el servicio `lqos_scheduler` respaldado por el entorno virtual. Actualice a un paquete que cree `/opt/libreqos/venv` y use el comando de reparación anterior. Si el scheduler falla inmediatamente después de un reinicio con un mensaje como `Socket (typically /run/lqos/bus) not found`, eso indica que `lqosd` todavía no había terminado de enlazar el bus local. Los builds actuales esperan brevemente la disponibilidad del bus al arrancar el scheduler en lugar de abortar de inmediato, por lo que ya no deberían aparecer panics repetidos de arranque tras un reinicio. @@ -175,20 +177,6 @@ Si `journalctl -u lqosd` muestra advertencias repetidas como `BeginIngest queue Suele indicar que no se pudo agregar correctamente qdisc MQ en la NIC (colas RX/TX insuficientes). Verifique [NICs recomendadas](requirements-es.md). -### Python ModuleNotFoundError en Ubuntu 24.04 - -```bash -pip uninstall binpacking --break-system-packages --yes -sudo pip uninstall binpacking --break-system-packages --yes -sudo pip install binpacking --break-system-packages -pip uninstall apscheduler --break-system-packages --yes -sudo pip uninstall apscheduler --break-system-packages --yes -sudo pip install apscheduler --break-system-packages -pip uninstall deepdiff --break-system-packages --yes -sudo pip uninstall deepdiff --break-system-packages --yes -sudo pip install deepdiff --break-system-packages -``` - ### Todas las IPs de clientes aparecen como Unknown IPs ```bash diff --git a/docs/v2.0/troubleshooting.md b/docs/v2.0/troubleshooting.md index fb5c21b7..2f641545 100644 --- a/docs/v2.0/troubleshooting.md +++ b/docs/v2.0/troubleshooting.md @@ -234,7 +234,7 @@ sudo dpkg --configure -a sudo systemctl restart lqosd lqos_scheduler ``` -Git-based installs should use `./build_rust.sh` after pulling updates. It rebuilds the virtual environment before refreshing service files or restarting services. If systemd reports `status=203/EXEC`, rebuild the virtual environment with the command above and restart `lqos_scheduler`. +Git-based installs should use `./build_rust.sh` after pulling updates. It rebuilds the virtual environment before refreshing service files or restarting services. If systemd reports `status=203/EXEC` on `/opt/libreqos/venv/bin/python`, or a failed scheduler pre-start check, rebuild the virtual environment with the command above and restart `lqos_scheduler`. For manual shaping tests, use the same interpreter as the service: @@ -244,21 +244,10 @@ sudo /opt/libreqos/venv/bin/python /opt/libreqos/src/LibreQoS.py sudo systemctl start lqos_scheduler ``` -Older installs that predate the virtual environment may show `ModuleNotFoundError` and suggest system `pip` commands. Upgrade to a package that creates `/opt/libreqos/venv`, then use the repair command above. +Older installs that predate the virtual environment may show `ModuleNotFoundError` and suggest system `pip` commands. Do not repair current installs with system `pip` or `--break-system-packages`; those packages are not used by the venv-backed scheduler service. Upgrade to a package that creates `/opt/libreqos/venv`, then use the repair command above. -### Python ModuleNotFoundError in older Ubuntu 24.04 installs -``` -pip uninstall binpacking --break-system-packages --yes -sudo pip uninstall binpacking --break-system-packages --yes -sudo pip install binpacking --break-system-packages -pip uninstall apscheduler --break-system-packages --yes -sudo pip uninstall apscheduler --break-system-packages --yes -sudo pip install apscheduler --break-system-packages -pip uninstall deepdiff --break-system-packages --yes -sudo pip uninstall deepdiff --break-system-packages --yes -sudo pip install deepdiff --break-system-packages -``` ### All customer IPs are listed under Unknown IPs, rather than Shaped Devices in GUI + ``` cd /opt/libreqos/src sudo systemctl stop lqos_scheduler diff --git a/src/bin/lqos_scheduler.service.example b/src/bin/lqos_scheduler.service.example index 78a42d74..9162c57c 100644 --- a/src/bin/lqos_scheduler.service.example +++ b/src/bin/lqos_scheduler.service.example @@ -5,6 +5,8 @@ Requires=lqosd.service [Service] WorkingDirectory=/opt/libreqos/src +ExecStartPre=/bin/test -x /opt/libreqos/venv/bin/python +ExecStartPre=/opt/libreqos/venv/bin/python -c "import apscheduler, chardet, deepdiff, psutil, requests" ExecStart=/opt/libreqos/venv/bin/python /opt/libreqos/src/scheduler.py Restart=always From 6eac4c10d4bbcdaa3f69ce2f713f0fc208c5b286 Mon Sep 17 00:00:00 2001 From: rchac Date: Wed, 6 May 2026 22:12:01 +0000 Subject: [PATCH 20/20] changes --- src/lqos.example | 2 + src/rust/lqos_bakery/src/lib.rs | 2 + src/rust/lqos_config/src/etc/migration.rs | 2 + src/rust/lqos_config/src/etc/v15/bridge.rs | 10 ++ src/rust/lqos_config/src/etc/v15/example.toml | 2 + .../lqos_config/src/etc/v15/top_config.rs | 101 ++++++++++++ src/rust/lqos_netplan_helper/src/inspect.rs | 142 ++++++++++++++-- .../lqos_netplan_helper/src/transaction.rs | 67 ++++++-- src/rust/lqos_setup/src/setup_actions.rs | 39 ++++- .../src/config/config_interface_mtu.test.mjs | 46 ++++++ .../js_build/src/config/network_mode_mtu.mjs | 29 ++++ .../js_build/src/config_interface.js | 126 +++++++++++++- .../js_build/test-node-manager.sh | 1 + .../node_manager/local_api/network_mode.rs | 156 +++++++++++++++--- .../static2/config_interface.html | 22 ++- 15 files changed, 697 insertions(+), 50 deletions(-) create mode 100644 src/rust/lqosd/src/node_manager/js_build/src/config/config_interface_mtu.test.mjs create mode 100644 src/rust/lqosd/src/node_manager/js_build/src/config/network_mode_mtu.mjs diff --git a/src/lqos.example b/src/lqos.example index 547e0ddb..c3f67847 100644 --- a/src/lqos.example +++ b/src/lqos.example @@ -25,12 +25,14 @@ disable_offload = [ "gso", "tso", "lro", "sg", "gro" ] use_xdp_bridge = false to_internet = "eth0" to_network = "eth1" +# mtu = 1500 # OR: #[single_interface] #interface = "eth0" #internet_vlan = 2 #network_vlan = 3 +# mtu = 1500 [queues] # Global default SQM is a single token list applied to both directions. diff --git a/src/rust/lqos_bakery/src/lib.rs b/src/rust/lqos_bakery/src/lib.rs index 7d4c34f1..624251b3 100644 --- a/src/rust/lqos_bakery/src/lib.rs +++ b/src/rust/lqos_bakery/src/lib.rs @@ -11294,6 +11294,7 @@ mod tests { use_xdp_bridge: false, to_internet: "lo".to_string(), to_network: "lo".to_string(), + mtu: None, }), ..lqos_config::Config::default() }; @@ -13142,6 +13143,7 @@ mod tests { use_xdp_bridge: false, to_internet: "__bakery-missing-wan__".to_string(), to_network: "__bakery-missing-lan__".to_string(), + mtu: None, }), ..Config::default() }; diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs index 9686a1ef..1a6eec6a 100644 --- a/src/rust/lqos_config/src/etc/migration.rs +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -182,6 +182,7 @@ fn migrate_bridge( interface: python_config.interface_a.clone(), internet_vlan: python_config.stick_vlan_a as u32, network_vlan: python_config.stick_vlan_b as u32, + mtu: None, }); } else { new_config.single_interface = None; @@ -193,6 +194,7 @@ fn migrate_bridge( .use_xdp_bridge, to_internet: python_config.interface_b.clone(), to_network: python_config.interface_a.clone(), + mtu: None, }); } Ok(()) diff --git a/src/rust/lqos_config/src/etc/v15/bridge.rs b/src/rust/lqos_config/src/etc/v15/bridge.rs index 2bec3419..aebffa91 100644 --- a/src/rust/lqos_config/src/etc/v15/bridge.rs +++ b/src/rust/lqos_config/src/etc/v15/bridge.rs @@ -16,6 +16,10 @@ pub struct BridgeConfig { /// The name of the second interface, facing the LAN pub to_network: String, + + /// Optional MTU for LibreQoS-managed Linux bridge interfaces. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mtu: Option, } impl Default for BridgeConfig { @@ -24,6 +28,7 @@ impl Default for BridgeConfig { use_xdp_bridge: true, to_internet: "eth0".to_string(), to_network: "eth1".to_string(), + mtu: None, } } } @@ -39,6 +44,10 @@ pub struct SingleInterfaceConfig { /// The VLAN ID facing the LAN pub network_vlan: u32, + + /// Optional MTU for the LibreQoS-managed trunk interface. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mtu: Option, } impl Default for SingleInterfaceConfig { @@ -47,6 +56,7 @@ impl Default for SingleInterfaceConfig { interface: "eth0".to_string(), internet_vlan: 2, network_vlan: 3, + mtu: None, } } } diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index 730f272d..37955997 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -26,12 +26,14 @@ disable_offload = [ "gso", "tso", "lro", "sg", "gro" ] use_xdp_bridge = true to_internet = "eth0" to_network = "eth1" +# mtu = 1500 # OR: #[single_interface] #interface = "eth0" #internet_vlan = 2 #network_vlan = 3 +# mtu = 1500 [queues] default_sqm = "cake diffserv4" diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index 3d942b45..856f1075 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -15,6 +15,9 @@ fn default_true() -> bool { true } +const MIN_INTERFACE_MTU: u32 = 576; +const MAX_INTERFACE_MTU: u32 = 9216; + fn default_rtt_green_ms() -> u32 { 0 } @@ -27,6 +30,17 @@ fn default_rtt_red_ms() -> u32 { 200 } +fn validate_interface_mtu(field_name: &str, mtu: Option) -> Result<(), String> { + if let Some(mtu) = mtu + && !(MIN_INTERFACE_MTU..=MAX_INTERFACE_MTU).contains(&mtu) + { + return Err(format!( + "{field_name} must be between {MIN_INTERFACE_MTU} and {MAX_INTERFACE_MTU}" + )); + } + Ok(()) +} + /// RTT color scale thresholds (milliseconds) used by the web UI. /// /// These represent the color "anchor points" for RTT: @@ -225,6 +239,12 @@ impl Config { .to_string(), ); } + if let Some(bridge) = &self.bridge { + validate_interface_mtu("bridge.mtu", bridge.mtu)?; + } + if let Some(single_interface) = &self.single_interface { + validate_interface_mtu("single_interface.mtu", single_interface.mtu)?; + } if self.version.trim() != "1.5" { return Err(format!( "Configuration file is at version [{}], but this version of lqos only supports version 1.5.0", @@ -569,6 +589,7 @@ impl Config { #[cfg(test)] mod test { + use super::super::bridge::{BridgeConfig, SingleInterfaceConfig}; use super::{Config, RttThresholds}; fn remove_sections(raw: &str, sections: &[&str]) -> String { @@ -772,6 +793,86 @@ mod test { assert!(cfg.validate().is_err()); } + #[test] + fn bridge_mtu_defaults_to_none_when_omitted() { + let config = + Config::load_from_string(include_str!("example.toml")).expect("example should load"); + assert_eq!(config.bridge.as_ref().and_then(|bridge| bridge.mtu), None); + } + + #[test] + fn interface_mtu_values_deserialize() { + let bridge_raw = include_str!("example.toml") + .replace("to_network = \"eth1\"", "to_network = \"eth1\"\nmtu = 9000"); + let bridge_config = Config::load_from_string(&bridge_raw).expect("bridge MTU should load"); + assert_eq!( + bridge_config.bridge.as_ref().and_then(|bridge| bridge.mtu), + Some(9000) + ); + + let single_raw = remove_sections(include_str!("example.toml"), &["bridge"]) + + r#" +[single_interface] +interface = "eth0" +internet_vlan = 2 +network_vlan = 3 +mtu = 1500 +"#; + let single_config = + Config::load_from_string(&single_raw).expect("single-interface MTU should load"); + assert_eq!( + single_config + .single_interface + .as_ref() + .and_then(|single| single.mtu), + Some(1500) + ); + } + + #[test] + fn interface_mtu_validation_rejects_out_of_range_values() { + let mut bridge_config = Config { + bridge: Some(BridgeConfig { + mtu: Some(575), + ..BridgeConfig::default() + }), + ..Config::default() + }; + let bridge_error = bridge_config + .validate() + .expect_err("bridge.mtu below range should fail"); + assert!(bridge_error.contains("bridge.mtu")); + + bridge_config.bridge.as_mut().expect("bridge").mtu = Some(9217); + let bridge_error = bridge_config + .validate() + .expect_err("bridge.mtu above range should fail"); + assert!(bridge_error.contains("bridge.mtu")); + + let mut single_config = Config { + bridge: None, + single_interface: Some(SingleInterfaceConfig { + mtu: Some(575), + ..SingleInterfaceConfig::default() + }), + ..Config::default() + }; + let single_error = single_config + .validate() + .expect_err("single_interface.mtu below range should fail"); + assert!(single_error.contains("single_interface.mtu")); + + single_config + .single_interface + .as_mut() + .expect("single interface") + .mtu = Some(9217); + let single_error = single_config + .validate() + .expect_err("single_interface.mtu above range should fail"); + assert!(single_error.contains("single_interface.mtu")); + } + #[test] fn treeguard_defaults_match_default_on_rollout() { let cfg = Config::default(); diff --git a/src/rust/lqos_netplan_helper/src/inspect.rs b/src/rust/lqos_netplan_helper/src/inspect.rs index fb012324..098ef02f 100644 --- a/src/rust/lqos_netplan_helper/src/inspect.rs +++ b/src/rust/lqos_netplan_helper/src/inspect.rs @@ -68,6 +68,7 @@ enum RequestedMode { LinuxBridge { to_internet: String, to_network: String, + mtu: Option, }, XdpBridge { to_internet: String, @@ -75,6 +76,7 @@ enum RequestedMode { }, SingleInterface { interface: String, + mtu: Option, }, Unknown, } @@ -307,11 +309,13 @@ fn requested_mode(config: &Config) -> RequestedMode { RequestedMode::LinuxBridge { to_internet: bridge.to_internet.trim().to_string(), to_network: bridge.to_network.trim().to_string(), + mtu: bridge.mtu, } } } else if let Some(single) = &config.single_interface { RequestedMode::SingleInterface { interface: single.interface.trim().to_string(), + mtu: single.mtu, } } else { RequestedMode::Unknown @@ -332,6 +336,7 @@ fn selected_interfaces(mode: &RequestedMode) -> Vec { RequestedMode::LinuxBridge { to_internet, to_network, + .. } | RequestedMode::XdpBridge { to_internet, @@ -340,7 +345,7 @@ fn selected_interfaces(mode: &RequestedMode) -> Vec { .into_iter() .filter(|iface| !iface.is_empty()) .collect(), - RequestedMode::SingleInterface { interface } => { + RequestedMode::SingleInterface { interface, .. } => { if interface.is_empty() { Vec::new() } else { @@ -351,21 +356,38 @@ fn selected_interfaces(mode: &RequestedMode) -> Vec { } } +fn mtu_yaml_line(mtu: Option, indent: &str) -> String { + mtu.map(|value| format!("{indent}mtu: {value}\n")) + .unwrap_or_default() +} + +fn managed_linux_bridge_yaml(to_internet: &str, to_network: &str, mtu: Option) -> String { + let ethernet_mtu = mtu_yaml_line(mtu, " "); + let bridge_mtu = mtu_yaml_line(mtu, " "); + format!( + "network:\n version: 2\n ethernets:\n {to_internet}:\n dhcp4: false\n dhcp6: false\n{ethernet_mtu} {to_network}:\n dhcp4: false\n dhcp6: false\n{ethernet_mtu} bridges:\n br0:\n{bridge_mtu} interfaces:\n - {to_internet}\n - {to_network}\n" + ) +} + +fn managed_single_interface_yaml(interface: &str, mtu: Option) -> String { + let mtu_line = mtu_yaml_line(mtu, " "); + format!( + "network:\n version: 2\n ethernets:\n {interface}:\n dhcp4: false\n dhcp6: false\n{mtu_line}" + ) +} + fn managed_preview_yaml(mode: &RequestedMode) -> (Option, Option) { match mode { RequestedMode::LinuxBridge { to_internet, to_network, + mtu, } if !to_internet.is_empty() && !to_network.is_empty() => ( - Some(format!( - "network:\n version: 2\n ethernets:\n {to_internet}:\n dhcp4: false\n dhcp6: false\n {to_network}:\n dhcp4: false\n dhcp6: false\n bridges:\n br0:\n interfaces:\n - {to_internet}\n - {to_network}\n" - )), + Some(managed_linux_bridge_yaml(to_internet, to_network, *mtu)), None, ), - RequestedMode::SingleInterface { interface } if !interface.is_empty() => ( - Some(format!( - "network:\n version: 2\n ethernets:\n {interface}:\n dhcp4: false\n dhcp6: false\n" - )), + RequestedMode::SingleInterface { interface, mtu } if !interface.is_empty() => ( + Some(managed_single_interface_yaml(interface, *mtu)), None, ), RequestedMode::XdpBridge { .. } => ( @@ -598,6 +620,7 @@ pub(crate) fn adoption_rewrite_for_path(path: &Path, config: &Config) -> Result< RequestedMode::LinuxBridge { to_internet, to_network, + .. } => { let mut removed_bridge = false; doc.network.ethernets.remove(&to_internet); @@ -618,7 +641,7 @@ pub(crate) fn adoption_rewrite_for_path(path: &Path, config: &Config) -> Result< )); } } - RequestedMode::SingleInterface { interface } => { + RequestedMode::SingleInterface { interface, .. } => { if doc.network.ethernets.remove(&interface).is_none() { return Err(format!( "Unable to find interface {interface} in {} for adoption.", @@ -658,6 +681,7 @@ fn assess_file(path: &Path, doc: &NetplanDocument, mode: &RequestedMode) -> File RequestedMode::LinuxBridge { to_internet, to_network, + .. } => { for iface in [to_internet, to_network] { if let Some(cfg) = doc.network.ethernets.get(iface) { @@ -724,7 +748,7 @@ fn assess_file(path: &Path, doc: &NetplanDocument, mode: &RequestedMode) -> File } } } - RequestedMode::SingleInterface { interface } => { + RequestedMode::SingleInterface { interface, .. } => { if let Some(cfg) = doc.network.ethernets.get(interface) { relevant.insert(interface.clone()); if cfg.dhcp_disabled() && !cfg.has_l3_config() { @@ -1113,6 +1137,7 @@ mod tests { use_xdp_bridge: false, to_internet: "ens19".to_string(), to_network: "ens20".to_string(), + mtu: None, }), single_interface: None, ..lqos_config::Config::default() @@ -1126,6 +1151,7 @@ mod tests { interface: "ens19".to_string(), internet_vlan: 2, network_vlan: 3, + mtu: None, }), ..lqos_config::Config::default() } @@ -1144,6 +1170,102 @@ mod tests { BTreeMap::from([("ens19".to_string(), true), ("ens20".to_string(), true)]) } + #[test] + fn managed_linux_bridge_preview_includes_mtu_on_members_and_bridge() { + let mut config = linux_bridge_config(); + config.bridge.as_mut().expect("bridge").mtu = Some(9000); + let root = test_env("bridge-mtu-preview"); + + let inspection = inspect_network_mode_with_paths( + &config, + &root.join("netplan"), + &root.join("pending"), + &BTreeSet::from(["ens19".to_string(), "ens20".to_string()]), + &queue_caps(), + ); + let preview = inspection + .managed_preview_yaml + .expect("managed preview should exist"); + + assert_eq!( + preview, + "network:\n version: 2\n ethernets:\n ens19:\n dhcp4: false\n dhcp6: false\n mtu: 9000\n ens20:\n dhcp4: false\n dhcp6: false\n mtu: 9000\n bridges:\n br0:\n mtu: 9000\n interfaces:\n - ens19\n - ens20\n" + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn managed_single_interface_preview_includes_mtu_on_trunk() { + let mut config = single_interface_config(); + config + .single_interface + .as_mut() + .expect("single interface") + .mtu = Some(1500); + + let root = test_env("single-mtu-preview"); + let inspection = inspect_network_mode_with_paths( + &config, + &root.join("netplan"), + &root.join("pending"), + &BTreeSet::from(["ens19".to_string()]), + &BTreeMap::from([("ens19".to_string(), true)]), + ); + let preview = inspection + .managed_preview_yaml + .expect("managed preview should exist"); + + assert_eq!( + preview, + "network:\n version: 2\n ethernets:\n ens19:\n dhcp4: false\n dhcp6: false\n mtu: 1500\n" + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn managed_previews_omit_mtu_when_unset() { + let root = test_env("unset-mtu-preview"); + let inspection = inspect_network_mode_with_paths( + &linux_bridge_config(), + &root.join("netplan"), + &root.join("pending"), + &BTreeSet::from(["ens19".to_string(), "ens20".to_string()]), + &queue_caps(), + ); + let preview = inspection + .managed_preview_yaml + .expect("managed preview should exist"); + + assert!(!preview.contains("mtu:")); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn xdp_bridge_mode_keeps_managed_preview_disabled() { + let mut config = linux_bridge_config(); + let bridge = config.bridge.as_mut().expect("bridge"); + bridge.use_xdp_bridge = true; + bridge.mtu = Some(9000); + + let root = test_env("xdp-mtu-preview"); + let inspection = inspect_network_mode_with_paths( + &config, + &root.join("netplan"), + &root.join("pending"), + &BTreeSet::from(["ens19".to_string(), "ens20".to_string()]), + &queue_caps(), + ); + + assert!(inspection.managed_preview_yaml.is_none()); + assert!( + inspection + .preview_note + .as_deref() + .is_some_and(|note| note.contains("manual workflow")) + ); + let _ = std::fs::remove_dir_all(root); + } + #[test] fn detects_external_compatible_linux_bridge() { let root = test_env("external-compatible"); diff --git a/src/rust/lqos_netplan_helper/src/transaction.rs b/src/rust/lqos_netplan_helper/src/transaction.rs index 0d4b3b2a..a785e6d9 100644 --- a/src/rust/lqos_netplan_helper/src/transaction.rs +++ b/src/rust/lqos_netplan_helper/src/transaction.rs @@ -170,7 +170,7 @@ fn system_interfaces() -> BTreeSet { default_net::get_interfaces() .into_iter() .map(|iface| iface.name) - .collect() + .collect::>() } fn supports_multi_queue(interface: &str) -> bool { @@ -195,7 +195,14 @@ fn supports_multi_queue(interface: &str) -> bool { /// Inspect the requested network mode using the helper's configured paths. pub fn inspect_with_paths(paths: &HelperPaths, config: &Config) -> NetworkModeInspection { - let system_ifaces = system_interfaces(); + inspect_with_paths_and_interfaces(paths, config, system_interfaces()) +} + +fn inspect_with_paths_and_interfaces( + paths: &HelperPaths, + config: &Config, + system_ifaces: BTreeSet, +) -> NetworkModeInspection { let queue_caps = system_ifaces .iter() .map(|iface| (iface.clone(), supports_multi_queue(iface))) @@ -750,6 +757,15 @@ pub fn apply_transaction( paths: &HelperPaths, pending_children: &mut PendingChildren, request: ApplyRequest, +) -> Result { + apply_transaction_with_interfaces(paths, pending_children, request, system_interfaces()) +} + +fn apply_transaction_with_interfaces( + paths: &HelperPaths, + pending_children: &mut PendingChildren, + request: ApplyRequest, + system_ifaces: BTreeSet, ) -> Result { normalize_pending_children(paths, pending_children)?; if !list_pending_records(paths)?.is_empty() { @@ -761,7 +777,7 @@ pub fn apply_transaction( } else { Config::default() }; - let inspection = inspect_with_paths(paths, &request.config); + let inspection = inspect_with_paths_and_interfaces(paths, &request.config, system_ifaces); validate_apply_request(&request, &inspection, &previous_config)?; let preview_yaml = inspection.managed_preview_yaml.as_ref().ok_or_else(|| { @@ -1013,7 +1029,7 @@ esac .find(|iface| iface.as_str() != "lo") .cloned() .or_else(|| interfaces.iter().next().cloned()) - .expect("test host should expose at least one interface"); + .unwrap_or_else(|| "ens19".to_string()); let secondary = interfaces .iter() .find(|iface| iface.as_str() != primary && iface.as_str() != "lo") @@ -1024,10 +1040,19 @@ esac .find(|iface| iface.as_str() != primary) .cloned() }) - .expect("test host should expose a second distinct interface"); + .unwrap_or_else(|| "ens20".to_string()); (primary, secondary) } + fn test_system_interfaces() -> BTreeSet { + let interfaces = system_interfaces(); + if interfaces.is_empty() { + BTreeSet::from(["ens19".to_string(), "ens20".to_string()]) + } else { + interfaces + } + } + fn linux_bridge_config() -> Config { let (to_internet, to_network) = test_interfaces(); Config { @@ -1035,6 +1060,7 @@ esac use_xdp_bridge: false, to_internet, to_network, + mtu: None, }), single_interface: None, ..Config::default() @@ -1058,7 +1084,13 @@ esac confirm_dangerous_changes: true, }; - let apply = apply_transaction(&paths, &mut pending, request).expect("apply should succeed"); + let apply = apply_transaction_with_interfaces( + &paths, + &mut pending, + request, + test_system_interfaces(), + ) + .expect("apply should succeed"); assert!(apply.ok); assert!(apply.operation.is_some()); @@ -1082,7 +1114,13 @@ esac mode: ApplyMode::Apply, confirm_dangerous_changes: true, }; - let apply = apply_transaction(&paths, &mut pending, request).expect("second apply"); + let apply = apply_transaction_with_interfaces( + &paths, + &mut pending, + request, + test_system_interfaces(), + ) + .expect("second apply"); let operation_id = apply .operation .as_ref() @@ -1112,10 +1150,11 @@ esac interface: test_interfaces().0, internet_vlan: 2, network_vlan: 3, + mtu: None, }); next.bridge = None; - let apply = apply_transaction( + let apply = apply_transaction_with_interfaces( &paths, &mut pending, ApplyRequest { @@ -1125,6 +1164,7 @@ esac mode: ApplyMode::Apply, confirm_dangerous_changes: true, }, + test_system_interfaces(), ) .expect("apply should succeed"); let backup_id = apply @@ -1147,6 +1187,7 @@ esac interface: test_interfaces().1, internet_vlan: 10, network_vlan: 11, + mtu: None, }), bridge: None, ..Config::default() @@ -1181,7 +1222,7 @@ esac let existing = linux_bridge_config(); write_config(&paths.config_path, &existing); - let apply = apply_transaction( + let apply = apply_transaction_with_interfaces( &paths, &mut pending, ApplyRequest { @@ -1191,6 +1232,7 @@ esac mode: ApplyMode::Apply, confirm_dangerous_changes: true, }, + test_system_interfaces(), ) .expect("apply should succeed"); let operation_id = apply @@ -1228,10 +1270,11 @@ esac interface: test_interfaces().0, internet_vlan: 2, network_vlan: 3, + mtu: None, }); next.bridge = None; - let apply = apply_transaction( + let apply = apply_transaction_with_interfaces( &paths, &mut pending, ApplyRequest { @@ -1241,6 +1284,7 @@ esac mode: ApplyMode::Apply, confirm_dangerous_changes: true, }, + test_system_interfaces(), ) .expect("apply should succeed"); let operation_id = apply @@ -1295,7 +1339,7 @@ esac let existing = linux_bridge_config(); write_config(&paths.config_path, &existing); - let err = apply_transaction( + let err = apply_transaction_with_interfaces( &paths, &mut pending, ApplyRequest { @@ -1305,6 +1349,7 @@ esac mode: ApplyMode::Apply, confirm_dangerous_changes: true, }, + test_system_interfaces(), ) .expect_err("apply should fail"); diff --git a/src/rust/lqos_setup/src/setup_actions.rs b/src/rust/lqos_setup/src/setup_actions.rs index b27c659a..2c4a398f 100644 --- a/src/rust/lqos_setup/src/setup_actions.rs +++ b/src/rust/lqos_setup/src/setup_actions.rs @@ -42,6 +42,11 @@ pub(crate) enum CommitOutcome { pub(crate) fn build_candidate_config(existing: Option) -> lqos_config::Config { let mut config = existing.unwrap_or_default(); + let existing_bridge_mtu = config.bridge.as_ref().and_then(|bridge| bridge.mtu); + let existing_single_interface_mtu = config + .single_interface + .as_ref() + .and_then(|single| single.mtu); let new_config = CURRENT_CONFIG.lock(); config.node_name = new_config.node_name.clone(); config.queues.downlink_bandwidth_mbps = new_config.mbps_to_internet; @@ -55,6 +60,7 @@ pub(crate) fn build_candidate_config(existing: Option) -> l use_xdp_bridge: false, to_internet: new_config.to_internet.clone(), to_network: new_config.to_network.clone(), + mtu: existing_bridge_mtu, }); } BridgeMode::XDP => { @@ -63,6 +69,7 @@ pub(crate) fn build_candidate_config(existing: Option) -> l use_xdp_bridge: true, to_internet: new_config.to_internet.clone(), to_network: new_config.to_network.clone(), + mtu: existing_bridge_mtu, }); } BridgeMode::Single => { @@ -70,6 +77,7 @@ pub(crate) fn build_candidate_config(existing: Option) -> l interface: new_config.to_internet.clone(), internet_vlan: new_config.internet_vlan, network_vlan: new_config.network_vlan, + mtu: existing_single_interface_mtu, }); config.bridge = None; } @@ -245,7 +253,8 @@ fn load_existing_or_default(event_log: &mut Vec) -> Result { + assert.deepEqual(parseOptionalMtu(""), { ok: true, value: null, error: "" }); + assert.deepEqual(parseOptionalMtu(" "), { ok: true, value: null, error: "" }); + assert.deepEqual(parseOptionalMtu("1500"), { ok: true, value: 1500, error: "" }); + assert.deepEqual(parseOptionalMtu("9000"), { ok: true, value: 9000, error: "" }); +}); + +test("rejects invalid MTU values", () => { + assert.equal(parseOptionalMtu("abc").ok, false); + assert.equal(parseOptionalMtu("1500.5").ok, false); + assert.equal(parseOptionalMtu("575").ok, false); + assert.equal(parseOptionalMtu("9217").ok, false); +}); + +test("returns normalized candidate values", () => { + assert.equal(parseOptionalMtu("9000").value, 9000); + assert.equal(parseOptionalMtu("").value, null); +}); + +test("hydrates pre-MTU drafts from live config", () => { + const liveConfig = { + bridge: { mtu: 9000 }, + single_interface: { mtu: 1500 }, + }; + + assert.equal(hydrateDraftMtu({ bridge: { to_internet: "eth0" } }, liveConfig).bridge.mtu, 9000); + assert.equal( + hydrateDraftMtu({ single_interface: { interface: "eth0" } }, liveConfig).single_interface.mtu, + 1500 + ); +}); + +test("keeps explicit draft MTU values", () => { + const liveConfig = { + bridge: { mtu: 9000 }, + single_interface: { mtu: 1500 }, + }; + + assert.equal(hydrateDraftMtu({ bridge: { mtu: null } }, liveConfig).bridge.mtu, null); + assert.equal(hydrateDraftMtu({ single_interface: { mtu: 1400 } }, liveConfig).single_interface.mtu, 1400); +}); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config/network_mode_mtu.mjs b/src/rust/lqosd/src/node_manager/js_build/src/config/network_mode_mtu.mjs new file mode 100644 index 00000000..193d3a5a --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/config/network_mode_mtu.mjs @@ -0,0 +1,29 @@ +export const MTU_MIN = 576; +export const MTU_MAX = 9216; + +export function parseOptionalMtu(rawValue) { + const trimmed = String(rawValue ?? "").trim(); + if (!trimmed) { + return { ok: true, value: null, error: "" }; + } + if (!/^\d+$/.test(trimmed)) { + return { ok: false, value: null, error: `MTU must be a whole number from ${MTU_MIN} through ${MTU_MAX}.` }; + } + const value = Number.parseInt(trimmed, 10); + if (value < MTU_MIN || value > MTU_MAX) { + return { ok: false, value: null, error: `MTU must be from ${MTU_MIN} through ${MTU_MAX}.` }; + } + return { ok: true, value, error: "" }; +} + +export function hydrateDraftMtu(draft, liveConfig) { + if (!draft) return null; + const hydrated = JSON.parse(JSON.stringify(draft)); + if (hydrated.bridge && hydrated.bridge.mtu === undefined) { + hydrated.bridge.mtu = liveConfig?.bridge?.mtu ?? null; + } + if (hydrated.single_interface && hydrated.single_interface.mtu === undefined) { + hydrated.single_interface.mtu = liveConfig?.single_interface?.mtu ?? null; + } + return hydrated; +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_interface.js b/src/rust/lqosd/src/node_manager/js_build/src/config_interface.js index 9ac5c2b7..696b1d3e 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_interface.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_interface.js @@ -1,9 +1,14 @@ import {loadConfig, renderConfigMenu} from "./config/config_helper"; +import {hydrateDraftMtu, MTU_MAX, MTU_MIN, parseOptionalMtu} from "./config/network_mode_mtu.mjs"; const DRAFT_KEY = "lqos-network-mode-draft"; const PENDING_OPERATION_KEY = "lqos-network-mode-pending"; const NETPLAN_TRY_TIMEOUT_MS = 30_000; const RECONNECT_POLL_INTERVAL_MS = 2_000; +const BRIDGE_MTU_HELP = "Applies to the bridge members and br0."; +const SINGLE_INTERFACE_MTU_HELP = "Applies to the trunk interface."; +const MTU_LOCKED_HELP = "MTU cannot be changed while a network-mode operation is pending."; +const XDP_MTU_HELP = "XDP mode keeps this value for later, but LibreQoS will not apply it."; let currentHelperStatus = null; let currentInspection = null; let currentPendingOperation = null; @@ -71,7 +76,94 @@ function postJson(url, body) { }); } +function setMtuFieldError(inputId, errorId, message) { + const input = document.getElementById(inputId); + const error = document.getElementById(errorId); + if (!input || !error) return; + input.setAttribute("aria-invalid", "true"); + input.classList.add("is-invalid"); + error.textContent = message; +} + +function clearMtuFieldError(inputId, errorId) { + const input = document.getElementById(inputId); + const error = document.getElementById(errorId); + if (!input || !error) return; + input.removeAttribute("aria-invalid"); + input.classList.remove("is-invalid"); + error.textContent = ""; +} + +function clearMtuErrors() { + clearMtuFieldError("bridgeMtu", "bridgeMtuError"); + clearMtuFieldError("singleInterfaceMtu", "singleInterfaceMtuError"); +} + +function applyMtuBounds() { + ["bridgeMtu", "singleInterfaceMtu"].forEach((id) => { + const input = document.getElementById(id); + if (!input) return; + input.min = String(MTU_MIN); + input.max = String(MTU_MAX); + }); +} + +function validateMtuField(inputId, errorId, parser) { + const input = document.getElementById(inputId); + if (!input) return true; + const parsed = parser(input.value); + if (parsed.ok) { + clearMtuFieldError(inputId, errorId); + return true; + } + setMtuFieldError(inputId, errorId, parsed.error); + input.focus(); + return false; +} + +function currentBridgeMtuValue() { + const parsed = parseOptionalMtu(document.getElementById("bridgeMtu").value); + if (parsed.ok) return parsed.value; + return window.config?.bridge?.mtu ?? null; +} + +function updateBridgeMtuState() { + const useXdpBridge = document.getElementById("useXdpBridge")?.checked ?? false; + const bridgeMtu = document.getElementById("bridgeMtu"); + const bridgeMtuHelp = document.getElementById("bridgeMtuHelp"); + if (!bridgeMtu || !bridgeMtuHelp) return; + const editingLocked = Boolean(currentInspection?.editing_locked); + bridgeMtu.disabled = useXdpBridge || editingLocked; + if (editingLocked) { + bridgeMtuHelp.textContent = MTU_LOCKED_HELP; + return; + } + if (useXdpBridge) { + clearMtuFieldError("bridgeMtu", "bridgeMtuError"); + bridgeMtuHelp.textContent = XDP_MTU_HELP; + } else { + bridgeMtuHelp.textContent = BRIDGE_MTU_HELP; + } +} + +function updateSingleInterfaceMtuState() { + const singleInterfaceMtu = document.getElementById("singleInterfaceMtu"); + const singleInterfaceMtuHelp = document.getElementById("singleInterfaceMtuHelp"); + if (!singleInterfaceMtu || !singleInterfaceMtuHelp) return; + if (currentInspection?.editing_locked) { + singleInterfaceMtuHelp.textContent = MTU_LOCKED_HELP; + } else { + singleInterfaceMtuHelp.textContent = SINGLE_INTERFACE_MTU_HELP; + } +} + +function updateMtuState() { + updateBridgeMtuState(); + updateSingleInterfaceMtuState(); +} + function validateConfig() { + clearMtuErrors(); if (document.getElementById("bridgeMode").checked) { const toInternet = document.getElementById("toInternet").value.trim(); const toNetwork = document.getElementById("toNetwork").value.trim(); @@ -83,6 +175,11 @@ function validateConfig() { alert("Bridge mode requires two different interfaces"); return false; } + if (document.getElementById("useXdpBridge").checked) { + clearMtuFieldError("bridgeMtu", "bridgeMtuError"); + } else if (!validateMtuField("bridgeMtu", "bridgeMtuError", parseOptionalMtu)) { + return false; + } } else if (document.getElementById("singleInterfaceMode").checked) { const interfaceName = document.getElementById("interface").value.trim(); const internetVlan = parseInt(document.getElementById("internetVlan").value, 10); @@ -99,6 +196,9 @@ function validateConfig() { alert("Network VLAN must be between 1 and 4094"); return false; } + if (!validateMtuField("singleInterfaceMtu", "singleInterfaceMtuError", parseOptionalMtu)) { + return false; + } } else { alert("Please select either bridge or single interface mode"); return false; @@ -116,12 +216,14 @@ function buildCandidateConfig() { use_xdp_bridge: document.getElementById("useXdpBridge").checked, to_internet: document.getElementById("toInternet").value.trim(), to_network: document.getElementById("toNetwork").value.trim(), + mtu: currentBridgeMtuValue(), }; } else { next.single_interface = { interface: document.getElementById("interface").value.trim(), internet_vlan: parseInt(document.getElementById("internetVlan").value, 10), network_vlan: parseInt(document.getElementById("networkVlan").value, 10), + mtu: parseOptionalMtu(document.getElementById("singleInterfaceMtu").value).value, }; } @@ -247,15 +349,18 @@ function populateFormFromConfig(config) { document.getElementById("useXdpBridge").checked = config.bridge.use_xdp_bridge ?? true; document.getElementById("toInternet").value = config.bridge.to_internet ?? ""; document.getElementById("toNetwork").value = config.bridge.to_network ?? ""; + document.getElementById("bridgeMtu").value = config.bridge.mtu ?? ""; } else if (config?.single_interface) { document.getElementById("singleInterfaceMode").checked = true; document.getElementById("interface").value = config.single_interface.interface ?? ""; document.getElementById("internetVlan").value = config.single_interface.internet_vlan ?? 2; document.getElementById("networkVlan").value = config.single_interface.network_vlan ?? 3; + document.getElementById("singleInterfaceMtu").value = config.single_interface.mtu ?? ""; } const event = new Event("change"); document.querySelector('input[name="networkMode"]:checked')?.dispatchEvent(event); + updateMtuState(); } function loadDraft() { @@ -590,9 +695,11 @@ function renderInspection(inspection) { "useXdpBridge", "toInternet", "toNetwork", + "bridgeMtu", "interface", "internetVlan", "networkVlan", + "singleInterfaceMtu", "saveButton", ].forEach((id) => { const element = document.getElementById(id); @@ -603,6 +710,7 @@ function renderInspection(inspection) { applyButton.disabled = !inspection?.can_apply; adoptButton.disabled = !inspection?.can_adopt; takeoverButton.disabled = !inspection?.can_take_over; + updateMtuState(); } function renderHelperStatus(status) { @@ -820,6 +928,7 @@ function retryShaping() { } function wireActions() { + applyMtuBounds(); wireReviewTabs(); [ "bridgeMode", @@ -832,6 +941,21 @@ function wireActions() { renderInterfaceSelectors(); }); }); + document.getElementById("useXdpBridge").addEventListener("change", updateMtuState); + [ + "bridgeMtu", + "singleInterfaceMtu", + ].forEach((id) => { + document.getElementById(id).addEventListener("input", () => { + if (id === "bridgeMtu") { + const parsed = parseOptionalMtu(document.getElementById("bridgeMtu").value); + if (parsed.ok) clearMtuFieldError("bridgeMtu", "bridgeMtuError"); + } else { + const parsed = parseOptionalMtu(document.getElementById("singleInterfaceMtu").value); + if (parsed.ok) clearMtuFieldError("singleInterfaceMtu", "singleInterfaceMtuError"); + } + }); + }); document.getElementById("saveButton").addEventListener("click", () => { if (!validateConfig()) return; saveDraft(); @@ -867,7 +991,7 @@ loadConfig((msg) => { renderShapingStatus(payload.shaping_status || {}); renderInspection(payload.network_mode_inspection || {}); - const draft = loadDraft(); + const draft = hydrateDraftMtu(loadDraft(), window.config); populateFormFromConfig(draft || window.config); renderRecoveryPanel(); wireActions(); diff --git a/src/rust/lqosd/src/node_manager/js_build/test-node-manager.sh b/src/rust/lqosd/src/node_manager/js_build/test-node-manager.sh index 5d093180..ff966549 100755 --- a/src/rust/lqosd/src/node_manager/js_build/test-node-manager.sh +++ b/src/rust/lqosd/src/node_manager/js_build/test-node-manager.sh @@ -10,6 +10,7 @@ fi echo "[test-node-manager] Running frontend contract tests..." node --test "${SCRIPT_DIR}/src/config/shaped_device_wire.test.mjs" +node --test "${SCRIPT_DIR}/src/config/config_interface_mtu.test.mjs" echo "[test-node-manager] Building bundles..." "${SCRIPT_DIR}/esbuild.sh" diff --git a/src/rust/lqosd/src/node_manager/local_api/network_mode.rs b/src/rust/lqosd/src/node_manager/local_api/network_mode.rs index fbda8395..5ab55238 100644 --- a/src/rust/lqosd/src/node_manager/local_api/network_mode.rs +++ b/src/rust/lqosd/src/node_manager/local_api/network_mode.rs @@ -77,7 +77,7 @@ fn unauthorized() -> (StatusCode, Json) { } fn helper_validation_error_response(message: String) -> (StatusCode, Json) { - warn!("Bridge helper request failed: {message}"); + warn!("Network-mode request failed: {message}"); ( StatusCode::BAD_REQUEST, Json(ApplyResponse { @@ -90,7 +90,7 @@ fn helper_validation_error_response(message: String) -> (StatusCode, Json (StatusCode, Json) { - error!("Bridge helper internal error: {message}"); + error!("Network-mode internal error: {message}"); ( StatusCode::INTERNAL_SERVER_ERROR, Json(ApplyResponse { @@ -102,11 +102,20 @@ fn helper_internal_error_response(message: String) -> (StatusCode, Json Result { - let live = lqos_config::load_config().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; +#[derive(Debug, Clone, PartialEq, Eq)] +enum MergeNetworkModeError { + LoadLiveConfig, + InvalidCandidate(String), +} + +fn merge_network_mode(candidate: Config) -> Result { + let live = lqos_config::load_config().map_err(|_| MergeNetworkModeError::LoadLiveConfig)?; let mut merged = (*live).clone(); merged.bridge = candidate.bridge; merged.single_interface = candidate.single_interface; + merged + .validate() + .map_err(MergeNetworkModeError::InvalidCandidate)?; Ok(merged) } @@ -156,13 +165,23 @@ pub async fn inspect( State(state): State, Extension(login): Extension, Json(body): Json, -) -> Result, StatusCode> { +) -> Result, (StatusCode, Json)> { if login != LoginResult::Admin { - return Err(StatusCode::FORBIDDEN); + return Err(unauthorized()); } - let merged = merge_network_mode(body.config)?; - Ok(Json(run_network_mode_inspection(&state, merged).await?)) + let merged = merge_network_mode(body.config).map_err(|error| match error { + MergeNetworkModeError::InvalidCandidate(message) => { + helper_validation_error_response(message) + } + MergeNetworkModeError::LoadLiveConfig => helper_internal_error_response( + "Unable to load the live LibreQoS configuration".to_string(), + ), + })?; + run_network_mode_inspection(&state, merged) + .await + .map(Json) + .map_err(|_| helper_internal_error_response("Unable to inspect network mode".to_string())) } pub async fn apply( @@ -178,16 +197,17 @@ pub async fn apply( let username = get_username(&jar).await; let merged = match merge_network_mode(body.config) { Ok(config) => config, - Err(status) => { - return ( - status, - Json(ApplyResponse { - ok: false, - message: "Unable to load the live LibreQoS configuration".to_string(), - operation: None, - last_backup_id: None, - }), - ); + Err(error) => { + return match error { + MergeNetworkModeError::InvalidCandidate(message) => { + helper_validation_error_response(message) + } + MergeNetworkModeError::LoadLiveConfig => { + helper_internal_error_response( + "Unable to load the live LibreQoS configuration".to_string(), + ) + } + }; } }; let result = tokio::task::spawn_blocking({ @@ -212,7 +232,9 @@ pub async fn apply( match result { Ok(Ok(response)) => (StatusCode::OK, Json(response)), Ok(Err(err)) => helper_validation_error_response(err.to_string()), - Err(err) => helper_internal_error_response(format!("Bridge helper task failed: {err}")), + Err(err) => { + helper_internal_error_response(format!("Network-mode helper task failed: {err}")) + } } } @@ -238,7 +260,9 @@ pub async fn confirm( match result { Ok(Ok(response)) => (StatusCode::OK, Json(response)), Ok(Err(err)) => helper_validation_error_response(err.to_string()), - Err(err) => helper_internal_error_response(format!("Bridge helper task failed: {err}")), + Err(err) => { + helper_internal_error_response(format!("Network-mode helper task failed: {err}")) + } } } @@ -264,7 +288,9 @@ pub async fn revert( match result { Ok(Ok(response)) => (StatusCode::OK, Json(response)), Ok(Err(err)) => helper_validation_error_response(err.to_string()), - Err(err) => helper_internal_error_response(format!("Bridge helper task failed: {err}")), + Err(err) => { + helper_internal_error_response(format!("Network-mode helper task failed: {err}")) + } } } @@ -290,7 +316,9 @@ pub async fn rollback( match result { Ok(Ok(response)) => (StatusCode::OK, Json(response)), Ok(Err(err)) => helper_validation_error_response(err.to_string()), - Err(err) => helper_internal_error_response(format!("Bridge helper task failed: {err}")), + Err(err) => { + helper_internal_error_response(format!("Network-mode helper task failed: {err}")) + } } } @@ -314,6 +342,88 @@ pub async fn retry_shaping( match result { Ok(Ok(response)) => (StatusCode::OK, Json(response)), Ok(Err(err)) => helper_internal_error_response(err.to_string()), - Err(err) => helper_internal_error_response(format!("Bridge helper task failed: {err}")), + Err(err) => { + helper_internal_error_response(format!("Network-mode helper task failed: {err}")) + } + } +} + +#[cfg(test)] +mod tests { + use super::{MergeNetworkModeError, merge_network_mode}; + use lqos_config::{BridgeConfig, Config, SingleInterfaceConfig}; + use std::sync::{Mutex, OnceLock}; + + fn env_lock() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + } + + fn with_config_env(test_fn: impl FnOnce() -> T) -> T { + let _guard = env_lock().lock().expect("network-mode env lock"); + let old_lqos_config = std::env::var_os("LQOS_CONFIG"); + let config_path = std::env::temp_dir().join(format!( + "lqosd-network-mode-test-{}.toml", + std::process::id() + )); + let raw = include_str!("../../../../lqos_config/src/etc/v15/example.toml"); + std::fs::write(&config_path, raw).expect("write config"); + lqos_config::clear_cached_config(); + + unsafe { + std::env::set_var("LQOS_CONFIG", &config_path); + } + let result = test_fn(); + + match old_lqos_config { + Some(value) => unsafe { + std::env::set_var("LQOS_CONFIG", value); + }, + None => unsafe { + std::env::remove_var("LQOS_CONFIG"); + }, + } + lqos_config::clear_cached_config(); + let _ = std::fs::remove_file(config_path); + result + } + + #[test] + fn merge_network_mode_rejects_invalid_bridge_mtu() { + let mut candidate = Config::default(); + candidate.bridge = Some(BridgeConfig { + mtu: Some(9217), + ..BridgeConfig::default() + }); + + let error = + with_config_env(|| merge_network_mode(candidate).expect_err("invalid MTU should fail")); + + assert_eq!( + error, + MergeNetworkModeError::InvalidCandidate( + "bridge.mtu must be between 576 and 9216".to_string() + ) + ); + } + + #[test] + fn merge_network_mode_rejects_invalid_single_interface_mtu() { + let mut candidate = Config::default(); + candidate.bridge = None; + candidate.single_interface = Some(SingleInterfaceConfig { + mtu: Some(575), + ..SingleInterfaceConfig::default() + }); + + let error = + with_config_env(|| merge_network_mode(candidate).expect_err("invalid MTU should fail")); + + assert_eq!( + error, + MergeNetworkModeError::InvalidCandidate( + "single_interface.mtu must be between 576 and 9216".to_string() + ) + ); } } diff --git a/src/rust/lqosd/src/node_manager/static2/config_interface.html b/src/rust/lqosd/src/node_manager/static2/config_interface.html index 783e185e..aa04f7b1 100644 --- a/src/rust/lqosd/src/node_manager/static2/config_interface.html +++ b/src/rust/lqosd/src/node_manager/static2/config_interface.html @@ -55,12 +55,12 @@
Bridge Mode Configuration
-
Choose the two shaping interfaces for the managed Linux bridge flow, or leave XDP enabled if you are intentionally handling Netplan outside this page.
+
Choose the two shaping interfaces. Linux bridge mode can manage Netplan; XDP mode leaves Netplan alone.
- + -
Use the XDP bridge instead of the Linux Bridge. This remains a manual/power-user netplan workflow.
+
XDP does not write Netplan or apply Bridge MTU.
@@ -78,6 +78,13 @@
Pick a different eligible interface for the LAN side.
+ +
+ + +
Applies to the bridge members and br0.
+ +
@@ -87,7 +94,7 @@
Single Interface Mode Configuration
-
Pick the trunk interface and define the WAN and LAN VLAN IDs that LibreQoS should prepare in managed Netplan.
+
Pick the trunk interface and VLAN IDs for managed Netplan.
@@ -108,6 +115,13 @@
VLAN ID for LAN traffic.
+ +
+ + +
Applies to the trunk interface.
+ +