diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index e4db964e..b3851394 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -4,8 +4,8 @@ jobs: lint_python: runs-on: lq_run steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: 3.x - run: pip install --upgrade pip wheel diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4574ff56..815faaf3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -4,23 +4,41 @@ on: [push] jobs: build: - strategy: - matrix: - os: [ubuntu-latest] runs-on: [ lq_run ] steps: - uses: actions/checkout@v4 - name: Set Version ENV - run: pushd src; echo "::set-env name=VERSION::$(cat VERSION_STRING)" - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' + run: echo "VERSION=$(cat src/VERSION_STRING)" >> "$GITHUB_ENV" - name: Check env var run: | echo ${{ env.VERSION }} - name: Install dependencies - run: sudo /bin/apt-get update; sudo /bin/apt-get install --no-install-recommends python3-pip clang gcc gcc-multilib llvm libelf-dev git nano graphviz esbuild curl screen llvm pkg-config linux-tools-common linux-tools-`/bin/uname r` libbpf-dev mold curl unzip - if: matrix.os == 'ubuntu-latest' + run: | + sudo /bin/apt-get update + sudo /bin/apt-get install --no-install-recommends \ + python3-pip clang gcc gcc-multilib llvm libelf-dev graphviz esbuild \ + pkg-config libbpf-dev mold unzip curl ca-certificates + + BPFTOOL_VERSION="7.7.0" + case "$(uname -m)" in + x86_64) BPFTOOL_ARCH="amd64" ;; + aarch64) BPFTOOL_ARCH="arm64" ;; + *) + echo "Unsupported architecture for upstream bpftool binary: $(uname -m)" >&2 + exit 1 + ;; + esac + + BPFTOOL_TARBALL="bpftool-v${BPFTOOL_VERSION}-${BPFTOOL_ARCH}.tar.gz" + BPFTOOL_BASE_URL="https://github.com/libbpf/bpftool/releases/download/v${BPFTOOL_VERSION}" + curl -fsSLO "${BPFTOOL_BASE_URL}/${BPFTOOL_TARBALL}" + curl -fsSLO "${BPFTOOL_BASE_URL}/${BPFTOOL_TARBALL}.sha256sum" + sha256sum -c "${BPFTOOL_TARBALL}.sha256sum" + tar -xzf "${BPFTOOL_TARBALL}" + sudo install -m 0755 bpftool /usr/local/bin/bpftool + command -v bpftool + bpftool version - name: Build run: pushd src/rust; cargo build --verbose --all; popd - name: Run tests @@ -35,13 +53,10 @@ jobs: path: src/dist/libreqos_${{ env.VERSION }}.-1_amd64.deb audit: - strategy: - matrix: - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install tools run: pushd src/rust; cargo install cargo-audit; popd - name: Audit for CVEs diff --git a/conf.py b/conf.py index dbfb41d5..7b9d2ab8 100644 --- a/conf.py +++ b/conf.py @@ -38,7 +38,7 @@ def ultimateReplace(app, docname, source): source[0] = result ultimate_replacements = { - "{deb_url_v1_5}" : "libreqos_2.0-beta-1.202603181919-1_amd64.deb" + "{deb_url_v1_5}" : "libreqos_2.1.202603311003-1_amd64.deb" } def setup(app): diff --git a/docs-es/v2.0/TechnicalDocs/git-install.md b/docs-es/v2.0/TechnicalDocs/git-install.md index 0c391a53..154e2127 100644 --- a/docs-es/v2.0/TechnicalDocs/git-install.md +++ b/docs-es/v2.0/TechnicalDocs/git-install.md @@ -84,6 +84,8 @@ sudo cp /opt/libreqos/src/bin/lqosd.service.example /etc/systemd/system/lqosd.se sudo cp /opt/libreqos/src/bin/lqos_scheduler.service.example /etc/systemd/system/lqos_scheduler.service ``` +Estas plantillas esperan `network-online.target`, por lo que `lqosd` y `lqos_scheduler` no arrancan antes de que DNS y la ruta por defecto esten listas. + Finally, run ```shell diff --git a/docs-es/v2.0/TechnicalDocs/integrations.md b/docs-es/v2.0/TechnicalDocs/integrations.md index 44c6bce7..43bbb73d 100644 --- a/docs-es/v2.0/TechnicalDocs/integrations.md +++ b/docs-es/v2.0/TechnicalDocs/integrations.md @@ -127,6 +127,8 @@ If a network.json file exists, it will not be overwritten, unless you set ```alw ShapedDevices.csv will be overwritten every time the UISP integration is run. +Cuando varios sitios cliente de UISP comparten el mismo nombre, LibreQoS ahora intenta diferenciar los nombres visibles generados para circuitos/sitios con un sufijo más humano, como el primer segmento de la dirección, recurriendo al nombre del servicio y luego a un ID corto solo cuando hace falta. La identidad estable del circuito sigue viniendo del ID de sitio/servicio de UISP, no del nombre visible. + To ensure the network.json is always overwritten with the newest version pulled in by the integration, please edit `/etc/lqos.conf` with the command `sudo nano /etc/lqos.conf`. Edit the file to set the value of `always_overwrite_network_json` to `true`. Then, run `sudo systemctl restart lqosd`. diff --git a/docs-es/v2.0/api-es.md b/docs-es/v2.0/api-es.md index c07bcdc0..09d68125 100644 --- a/docs-es/v2.0/api-es.md +++ b/docs-es/v2.0/api-es.md @@ -34,6 +34,8 @@ sudo systemctl enable lqos_api sudo systemctl start lqos_api ``` +La plantilla incluida de `lqos_api.service.example` espera `network-online.target`, para que la API no arranque antes de que DNS y la ruta por defecto esten listas. + Actualizar solo el binario de la API: ```bash diff --git a/docs-es/v2.0/configuration-es.md b/docs-es/v2.0/configuration-es.md index 20f2a325..89c7b907 100644 --- a/docs-es/v2.0/configuration-es.md +++ b/docs-es/v2.0/configuration-es.md @@ -25,13 +25,21 @@ Para la mayoría de operadores: La mayoría de cambios operativos diarios se realizan en la WebUI (`http://tu_ip_del_shaper:9123/config_general.html`). +Las versiones actuales usan paneles compartidos de configuración en las páginas General, RTT, Queues, TreeGuard, Network Mode, valores por defecto de integración, Network Layout, Insight, páginas de integración por proveedor, IP Ranges, Flow Tracking y Shaped Devices para que los ajustes del nodo y la afinación operativa sigan la misma presentación visual que el resto de Node Manager. La página de TreeGuard comienza con un resumen del comportamiento actual en lugar de un banner permanente de advertencia. + ### Dónde en la WebUI - Ajustes generales: `Configuration -> General` - Ajustes de integración: `Configuration -> Integrations` +- Editor de layout de red: `Configuration -> Network Layout` - Editor de dispositivos regulados: `Configuration -> Shaped Devices` - Validación operativa: páginas de `WebUI (Node Manager)` (dashboard/tree/flow/scheduler) +Cuando una integración está habilitada y actúa como fuente de verdad, los editores `Network Layout` y `Shaped Devices` siguen siendo visibles pero pasan a modo de solo lectura en la WebUI. + +Nota de topología: +- Los nombres de nodo en `network.json` deben ser globalmente únicos en todo el árbol. Los nombres duplicados ahora fallan la validación y no son aceptados por el guardado de la WebUI ni por `LibreQoS.py`. + ## Modos de operación y fuente de verdad Lea esto primero antes de cambios en producción: diff --git a/docs-es/v2.0/git-install-es.md b/docs-es/v2.0/git-install-es.md index 735a541e..9f155ef8 100644 --- a/docs-es/v2.0/git-install-es.md +++ b/docs-es/v2.0/git-install-es.md @@ -83,6 +83,8 @@ sudo cp /opt/libreqos/src/bin/lqosd.service.example /etc/systemd/system/lqosd.se sudo cp /opt/libreqos/src/bin/lqos_scheduler.service.example /etc/systemd/system/lqos_scheduler.service ``` +Estas plantillas esperan `network-online.target`, por lo que `lqosd` y `lqos_scheduler` no arrancan antes de que DNS y la ruta por defecto esten listas. + Finalmente, ejecute: ```shell diff --git a/docs-es/v2.0/libreqos-backend-architecture-es.md b/docs-es/v2.0/libreqos-backend-architecture-es.md index 5222ef74..741bceff 100644 --- a/docs-es/v2.0/libreqos-backend-architecture-es.md +++ b/docs-es/v2.0/libreqos-backend-architecture-es.md @@ -242,6 +242,7 @@ Efecto práctico: |---|---|---|---| | Cambio solo de IP de circuito | Sí | No | Actualización de mapeo puede aplicarse sin reconstruir árbol | | Cambio de velocidad de circuito/sitio (subconjunto) | Sí | A veces | Depende del impacto estructural y class handles disponibles | +| Virtualización runtime de nodos TreeGuard (subárbol soportado / ruta de rebalanceo top-level) | Sí | No | Bakery ahora la aplica como un plan runtime vivo: nodos no top-level usan reparent/prune/restore, y nodos top-level soportados usan una ruta de rebalanceo y migración entre colas | | Cambios masivos en todos los circuitos | A veces | Frecuente | Límites de escala y cardinalidad de transacciones | | Re-parent/reestructura topológica | Rara vez | Sí | Restricciones de mutación de subárbol HTB | | Alta/baja de circuitos | Sí (pequeño/mediano) | A veces | Límites de handles y fronteras de corrección en el diff | @@ -269,6 +270,15 @@ flowchart TD 3. Esperar mayor riesgo cuando coinciden muchos circuitos y muchos cambios estructurales. 4. Diseñar la cadencia operativa para priorizar cambios incrementalmente seguros. +### 7.4 Guardrails de seguridad para recargas completas + +Las recargas completas actuales de Bakery aplican dos verificaciones conservadoras de seguridad antes y durante reconstrucciones grandes de colas: + +1. Un preflight de qdisc estima los qdisc planificados por interfaz y además separa qdisc de infraestructura, hojas `cake` y hojas `fq_codel`. +2. Ese mismo preflight aplica una proyección conservadora de memoria y bloquea de forma estricta las recargas completas claramente inseguras antes de arrancar `tc -batch`. +3. Durante la aplicación chunked de una recarga completa, Bakery vuelve a revisar la memoria del host en los límites de chunk y aborta el resto de la aplicación si la memoria disponible cae por debajo de su piso de seguridad. +4. Estos guardrails están sesgados intencionalmente hacia falsos positivos en recargas grandes para fallar temprano con diagnósticos en lugar de entrar en una espiral OOM. + ## 8) Límites de Diseño para Operadores ### 8.1 Límites de observabilidad diff --git a/docs-es/v2.0/node-manager-ui-es.md b/docs-es/v2.0/node-manager-ui-es.md index e117e8ac..d219e001 100644 --- a/docs-es/v2.0/node-manager-ui-es.md +++ b/docs-es/v2.0/node-manager-ui-es.md @@ -7,18 +7,43 @@ Esta página documenta las vistas clave de la WebUI (Node Manager) y su comporta ### Dashboard - Resumen por widgets de throughput, retransmisiones, RTT, flujos y actividad de colas. - El contenido puede variar según versión y funciones habilitadas. +- Executive Summary ofrece una vista operativa compacta para redes grandes, con un `Network Snapshot` centrado en throughput, inventario y estado de Insight, además de páginas de detalle para heatmaps y rankings ejecutivos. +- Bakery ofrece una pestaña dedicada para el estado de aplicación de colas, resultados de seguridad/preflight de qdisc, progreso de cambios en vivo sobre circuitos y operaciones recientes de Bakery. +- Las pestañas de Bakery y TreeGuard presentan un resumen de alto nivel del pipeline o control loop antes de las tablas más detalladas. +- El widget `Pipeline` de Bakery muestra las etapas del control de colas, el estado de aplicación, el estado de verificación y la temporización del intervalo de TC. +- `Runtime Operations` resume mutaciones de topología entre TreeGuard y Bakery, trabajo de limpieza diferida, fallos y subárboles que esperan un full reload. +- `Recent Bakery Events` prioriza operaciones agrupadas, con el historial detallado disponible cuando hace falta troubleshooting más profundo. +- `TreeGuard Activity` prioriza operaciones agrupadas, incluyendo lotes de cambios SQM, con el historial detallado disponible cuando hace falta troubleshooting más profundo. +- `TreeGuard Control Loop` muestra el estado actual de observar/evaluar/actuar. +- `TreeGuard Decision Impact` se centra en el impacto actual y en las advertencias o errores activos. +- `TreeGuard State Mix` muestra nodos gestionados, virtualización en runtime, circuitos gestionados y la mezcla de circuitos `cake / mixed / fq_codel`. +- El preflight de qdisc de Bakery resume el uso planificado de qdisc por interfaz y el margen de presupuesto antes de aplicar cambios. +- Algunos gráficos pueden tardar un poco en poblarse al abrir una pestaña por primera vez, especialmente en sistemas ocupados o inmediatamente después de reiniciar servicios. +- Durante un full reload de Bakery, las tarjetas de conteo de colas pueden seguir mostrando temporalmente los últimos valores conocidos de HTB/CAKE/fq-codel y marcarlos como `Reloading`. ### Vista de árbol de red - Vista jerárquica de nodos/circuitos desde la perspectiva del shaper. - Útil para identificar cuellos de botella y patrones de utilización padre/hijo. +- Las páginas de detalle del árbol muestran una ruta tipo breadcrumb, conteos de rama e indicadores de estado para el nodo seleccionado. +- `Node Details` resume el tipo de nodo seleccionado, el tamaño de la rama, las velocidades configuradas y la velocidad efectiva. +- `Node Snapshot` ofrece un resumen visual rápido del throughput y el QoO del nodo seleccionado. +- Los circuitos adjuntos se muestran en una tabla dedicada para el nodo seleccionado. +- La columna de IP de circuitos adjuntos mantiene las filas compactas mostrando una dirección inline y colapsando las adicionales como `+X`, mientras la lista completa sigue disponible al pasar el cursor. +- Los circuitos adjuntos limitados por Ethernet pueden mostrar insignias `10M`, `100M` o `1G` junto al valor de `Plan (Mbps)`; al pasar el cursor se explica el auto-cap y al hacer clic en la insignia se abre la página dedicada de revisión Ethernet. +- Los administradores pueden guardar o limpiar valores de `Operator Override` cuando el nodo admite overrides a nivel de nodo. Los usuarios de solo lectura y los nodos no compatibles siguen mostrando los valores actuales sin controles de edición. ### Site Map - Mapa operativo plano de sitios y APs usando geodatos importados de nodos. - Usa QoO por defecto con un selector alternativo para RTT, mientras el tamaño del marcador refleja el throughput combinado reciente. -- Usa un promedio del lado cliente de 30 segundos a partir de `NetworkTree`, sin agregar trabajo de rollup en el backend. - Los APs pueden heredar coordenadas del sitio padre solo para visualización cuando faltan coordenadas explícitas. -- Usa un mapa base local con estilo LibreQoS con bordes de país/estado, costas, lagos principales, ríos principales, áreas marinas, superposiciones sutiles de regiones físicas y contexto de autopistas principales a mayor zoom para orientación geográfica. -- Site Map utiliza una capa local de carreteras derivada de Natural Earth para ayudar con la orientación, manteniendo el resto del mapa base discreto y operativo. +- Los marcadores de sitios cercanos se agrupan y se expanden al acercar el mapa o seleccionar un grupo. +- Los APs sin coordenadas explícitas se representan a través de su sitio padre y pueden desplegarse temporalmente alrededor del sitio seleccionado para inspección. +- Los sitios visibles y sin agrupar muestran etiquetas al acercar el mapa, y el sitio seleccionado mantiene su etiqueta visible mientras se inspecciona. +- Cuando el modo de redacción del navegador está habilitado, Site Map reemplaza los nombres de sitios mostrados por `[redacted]` sin modificar los datos reales de topología. +- El encuadre inicial del mapa prioriza las coordenadas de los sitios para la vista inicial, usando coordenadas de AP solo cuando todavía no hay sitios mapeados. +- Site Map usa una capa raster de OpenStreetMap alojada por Insight. +- En modo oscuro, el mapa base se atenúa y se tiñe hacia una paleta azul/cian para mantener visibles carreteras y geografía sin el brillo del mapa claro. +- Site Map depende del acceso saliente a `https://insight.libreqos.com` para el arranque inicial del mapa y la obtención de tiles. ### Flow Globe - Visualización geográfica de flujos basada en la geolocalización de endpoints. @@ -31,16 +56,38 @@ Esta página documenta las vistas clave de la WebUI (Node Manager) y su comporta ### ASN Analysis - Página operativa ASN en vivo que combina un ranking top-20 de ASN, gráfico de burbujas latencia-vs-tráfico, franja mínima de KPIs del ASN seleccionado, gráfico de tendencia ASN de 15 minutos y la sección integrada de Flow Evidence. - Soporta modos de ranking `Impact` y `Throughput`, manteniendo la evidencia de flujos ASN en la misma página. -- La ruta heredada `ASN Explorer` ahora redirige aquí para conservar compatibilidad con marcadores antiguos. +- Los marcadores antiguos de `ASN Explorer` siguen funcionando mediante redirección. - Resultados vacíos suelen indicar poco dato reciente, no necesariamente falla. +### Página de circuito +- Las páginas de circuito combinan comportamiento de colas, throughput en vivo, RTT, retransmisiones y troubleshooting por flujo para un suscriptor/circuito individual. +- Cuando los metadatos de la integración informan la velocidad Ethernet negociada del CPE, la fila `Max` puede mostrar una insignia de advertencia como `100M`; al pasar el cursor sobre la insignia se explica cuándo LibreQoS redujo automáticamente el shaping por debajo del plan solicitado para respetar ese límite del puerto, y al hacer clic en la insignia se abre la página de revisión Ethernet. +- `Queue Dynamics` muestra el comportamiento del throughput y RTT del circuito a lo largo del tiempo, incluyendo un KPI de `Active Flows` basado en la misma ventana reciente usada por la tabla `Traffic Flows`. +- `Top ASNs` resume los ASN recientes más activos del circuito a partir de esa misma ventana de flujos en vivo y ordena por tasa actual por defecto. +- `Devices` muestra tablas de detalle por dispositivo y gráficos en vivo de throughput, retransmisiones y latencia. +- `Queue Stats` muestra historial reciente de la cola del circuito, incluyendo backlog, delay, longitud de cola, tráfico, marcas ECN y drops. +- Los gráficos de Queue Stats usan hover sincronizado para inspeccionar el mismo segundo en todos los gráficos de cola al mismo tiempo. +- `Queue Tree` muestra la ruta ascendente de colas del circuito, incluyendo un resumen de ruta y contexto de throughput, retransmisiones y latencia para cada nodo aguas arriba. +- `Traffic Flows` es una tabla operativa de flujos recientes, no una vista de historial a largo plazo. +- `Traffic Flows` incluye paginación y un filtro `Hide Small Flows` para que los circuitos grandes y ocupados sigan siendo utilizables sin intentar renderizar cada fila. +- La tasa actual de `Traffic Flows` se limita a valores plausibles y coherentes con el plan del circuito. +- El texto largo en las columnas `Protocol`, `ASN` y `Country` se recorta con puntos suspensivos para mantener estable la altura de cada fila; el valor completo sigue disponible al pasar el cursor. +- `Flow Sankey` enfatiza los flujos recientes más activos en lugar de todos los flujos retenidos más antiguos. + +### Ethernet Caps +- La página de revisión Ethernet es una tabla ligera para operadores con los circuitos reducidos automáticamente porque la velocidad Ethernet detectada quedó por debajo del plan solicitado. +- Intencionalmente no aparece en la navegación principal; los operadores llegan a ella haciendo clic en las insignias de advertencia Ethernet de la página de circuito o de la tabla de circuitos adjuntos del árbol. +- La página soporta búsqueda, filtro por tier (`10M`, `100M`, `1G+`) y paginación sobre los circuitos auto-capped. + ### Árbol/ponderación de CPU - Muestra distribución de colas/circuitos por núcleo de CPU. - Ayuda a validar comportamiento de binpacking y balance de carga. +- CPU Affinity comienza mostrando solo los CPUs de shaping, y los núcleos excluidos o solo-del-host pueden mostrarse cuando hace falta. ### Editor de Shaped Devices - Editor CRUD para `ShapedDevices.csv`. -- Incluye paginación y filtros en versiones actuales. +- Incluye paginación y filtros. +- En el editor dedicado, las acciones de agregar, editar y eliminar se guardan de inmediato. ### Problemas urgentes - WebUI puede mostrar problemas operativos urgentes emitidos por servicios backend. @@ -60,6 +107,7 @@ Esta página documenta las vistas clave de la WebUI (Node Manager) y su comporta - Se activa con el ícono de máscara en la barra superior. - Es redacción del lado cliente y se guarda en `localStorage` del navegador. - Ayuda a ocultar PII en capturas/demos. +- Site Map reemplaza los nombres de sitios mostrados por `[redacted]` mientras el modo de redacción está activo. - No modifica `ShapedDevices.csv`, `network.json` ni datos backend. ## Comportamiento de vistas vacías diff --git a/docs-es/v2.0/quickstart-es.md b/docs-es/v2.0/quickstart-es.md index 42341086..3b5de29e 100644 --- a/docs-es/v2.0/quickstart-es.md +++ b/docs-es/v2.0/quickstart-es.md @@ -42,10 +42,13 @@ sudo /opt/libreqos/src/systemd_hotfix.sh install sudo reboot ``` +El instalador del hotfix configura el repositorio APT de LibreQoS en `https://repo.libreqos.com`, instala el conjunto parchado de paquetes `systemd` de Noble y fija esos paquetes para futuras actualizaciones. + Después del reinicio, reanude la instalación: ```bash cd /tmp +wget https://download.libreqos.com/{deb_url_v1_5} sudo apt install ./{deb_url_v1_5} ``` diff --git a/docs-es/v2.0/stormguard-es.md b/docs-es/v2.0/stormguard-es.md index d3189d1f..7ffbbe20 100644 --- a/docs-es/v2.0/stormguard-es.md +++ b/docs-es/v2.0/stormguard-es.md @@ -12,6 +12,8 @@ StormGuard es el subsistema de ajuste adaptativo de colas de LibreQoS para event - Aplica ajustes acotados a límites de nodos configurados para proteger calidad bajo estrés. - Expone estado y depuración en WebUI. +Los cambios adaptativos de velocidad por sitio de StormGuard se guardan en la capa de overrides de StormGuard. No se escriben de vuelta en `network.json`. + ## Configuración StormGuard se configura en `/etc/lqos.conf` bajo `[stormguard]`. @@ -41,7 +43,12 @@ Si está probando, comience con `dry_run = true`. ## UI y depuración -- WebUI (Node Manager) incluye vistas de estado y depuración de StormGuard. +- WebUI (Node Manager) incluye una pestaña dedicada de StormGuard además de las vistas de estado y depuración. +- La pestaña del dashboard está pensada para responder "qué está haciendo StormGuard ahora mismo?" con: + - tarjetas resumen para sitios observados, en cooldown y con cambios recientes + - una lista de sitios que funciona tanto con un único sitio observado como con conjuntos más grandes + - un panel de detalle por sitio seleccionado que explica límites actuales, últimas acciones y por qué StormGuard mantiene o cambia velocidades + - un feed de actividad reciente para triage rápido del operador - La página de depuración muestra: - límites efectivos actuales - métricas de evaluación @@ -69,6 +76,7 @@ También verifique: - que los nombres en `targets` aún coincidan con `network.json` - que cambios de integración no hayan renombrado nodos clave - que los pisos mínimos sean razonables para su perfil de tráfico +- que `network.json` siga reflejando sus velocidades planificadas/de origen si está investigando una reducción inesperada de StormGuard - que `log_file` (si se usa) sea escribible por el servicio ## Páginas relacionadas diff --git a/docs-es/v2.0/treeguard-es.md b/docs-es/v2.0/treeguard-es.md index a3438936..11482499 100644 --- a/docs-es/v2.0/treeguard-es.md +++ b/docs-es/v2.0/treeguard-es.md @@ -32,11 +32,19 @@ TreeGuard evalúa utilización, frescura de RTT, guardrails de CPU y guardrails Comportamiento de alto nivel: 1. Bajo condiciones sostenidas de baja carga, TreeGuard puede cambiar una dirección de `cake` a `fq_codel`. -2. Si sube la utilización, los guardrails RTT/QoO no son seguros o se cumplen condiciones de reversión, TreeGuard vuelve a `cake`. +2. Si sube la utilización, los guardrails de QoO no son seguros o se cumplen condiciones de reversión, TreeGuard vuelve hacia la política SQM base del circuito. 3. Las decisiones pueden ser independientes por dirección cuando `independent_directions = true`. Esto crea un perfil dinámico en el que direcciones cargadas favorecen `cake diffserv4`, mientras direcciones de baja carga pueden usar `fq_codel` cuando las condiciones son seguras. +La política SQM base proviene de la intención del operador, no de defaults internos de TreeGuard. En la práctica, TreeGuard parte de la política efectiva configurada para cada circuito y solo persiste su propio overlay temporal cuando necesita diferir de esa base. + +Regla importante de política base: + +1. Si la política SQM base de una dirección es `cake`, TreeGuard puede cambiar temporalmente esa dirección a `fq_codel` y luego volver a la base. +2. Si la política SQM base de una dirección es `fq_codel`, TreeGuard no conmuta esa dirección de circuito hacia `cake`. +3. La virtualización de enlaces sigue disponible independientemente de la política SQM base del circuito. + ## Configuración (`/etc/lqos.conf`) La configuración de TreeGuard vive bajo `[treeguard]` y sus sub-secciones: @@ -55,6 +63,11 @@ enabled = true dry_run = false tick_seconds = 1 +[treeguard.cpu] +mode = "cpu_aware" +cpu_high_pct = 75 +cpu_low_pct = 55 + [treeguard.links] enabled = true all_nodes = true @@ -70,6 +83,12 @@ independent_directions = true enabled = true ``` +La virtualización de nodos en TreeGuard está pensada para ser basada en CPU por defecto. El +tráfico, RTT y QoO siguen siendo señales importantes de seguridad y restauración, pero la nueva +virtualización automática debe ocurrir cuando la presión de CPU indica que el ahorro de HTB vale +la pena. Las instalaciones actualizadas desde defaults antiguos se migran silenciosamente de +`traffic_rtt_only` a `cpu_aware`, con un aviso visible en logs/UI. + ## Patrón de Despliegue Seguro 1. Revise la configuración de TreeGuard temprano en el despliegue en lugar de asumir comportamiento de colas fijo/manual. @@ -80,12 +99,54 @@ enabled = true ## Overrides y Notas Operativas -Cuando está habilitado y no está en dry-run, TreeGuard puede persistir decisiones en: +Cuando está habilitado y no está en dry-run, TreeGuard puede persistir decisiones SQM de circuitos en: - `lqos_overrides.treeguard.json` TreeGuard está diseñado para no pelear con overrides del operador. Si existen overrides del operador para entidades enroladas, TreeGuard omite esas entidades y reporta advertencias. +Las decisiones de virtualización de nodos de TreeGuard son operaciones de Bakery solo en tiempo de +ejecución. El scheduler no las materializa de vuelta en el `network.json` base, y tampoco se +persisten como entradas TreeGuard `set_node_virtual` dentro de la entrada efectiva de shaping. En +v1 son efímeras: un reinicio del daemon devuelve el árbol físico a la topología base definida por +el operador hasta que TreeGuard vuelva a decidir. + +Para verificación y depuración local en tiempo de ejecución, `liblqos_python` ahora expone tanto +el estado actual de la operación del nodo TreeGuard como un snapshot del estado de ramas en tiempo +de ejecución de Bakery. Ese snapshot de ramas es la vista autoritativa del plano de control sobre +qué rama retenida está activa para un nodo, y es más confiable que inferir cambios de parentaje +solo desde la salida de `tc` en casos no top-level. +El flujo local de confianza también usa ahora un fixture sintético de Bakery TreeGuard más grande +para pruebas de escala: 8 nodos top-level, 3 niveles de profundidad y 1.000 circuitos conectados +solo en el nivel más bajo. Cuando la verificación con tráfico está habilitada, el runtime verifier +ahora usa por defecto 10 circuitos rastreados por cada caso exitoso en lugar del smoke test menor +anterior. + +Las decisiones SQM por circuito de TreeGuard también son overrides de tiempo de ejecución. El scheduler no materializa los cambios SQM propiedad de TreeGuard de vuelta en el `ShapedDevices.csv` base, por lo que limpiar TreeGuard no reescribe permanentemente la política SQM definida por el operador. + +TreeGuard también se niega a gestionar nodos que ya estén marcados con `"virtual": true` en el `network.json` base. Si existen overrides legados de TreeGuard para esos nodos, TreeGuard limpia ese estado legado y vuelve a respetar la definición base de la topología. + +Para la gestión SQM por circuito, TreeGuard trata los valores duplicados de `device_id` como colisiones de identidad inseguras. Si el mismo `device_id` aparece en más de un circuito dentro de `ShapedDevices.csv`, TreeGuard omite esos circuitos afectados y limpia cualquier override SQM de TreeGuard asociado a esos `device_id` duplicados. + +Si la telemetría RTT no está disponible temporalmente después de un reinicio, TreeGuard no trata la ausencia de RTT por sí sola como evidencia para revertir direcciones en `fq_codel`. Siguen aplicando otros guardrails como utilización, QoO y presión de CPU. + +TreeGuard también aplica un presupuesto global conservador de cambios SQM por tick. En poblaciones muy grandes de circuitos enrolados, los cambios SQM excedentes se difieren a ticks posteriores en lugar de saturar Bakery en una sola pasada. + +Para escalar mejor, TreeGuard ya no reconstruye la membresía de circuitos desde `ShapedDevices.csv` en cada tick de circuitos. Ahora mantiene un inventario por circuito en caché derivado de `ShapedDevices.csv`, lee la telemetría viva por circuito desde el snapshot compartido de rollup por circuito actualizado una vez por segundo, y reparte las evaluaciones SQM grandes con `all_circuits = true` a lo largo de múltiples ticks en lugar de reescanear todos los circuitos enrolados cada segundo. + +En la práctica esto significa: + +1. La virtualización de enlaces sigue la cadencia normal de ticks de TreeGuard. +2. La evaluación SQM de circuitos para enrolamientos pequeños sigue completándose rápidamente. +3. Los enrolamientos muy grandes con `all_circuits` se recorren de forma incremental a lo largo de varios ticks, con un objetivo de barrido completo de alrededor de 15 segundos en vez de intentar un escaneo completo por segundo. +4. La virtualización de nodos de TreeGuard ahora usa rutas vivas de planificación/aplicación en Bakery en lugar de forzar una recarga completa de LibreQoS o de Bakery. +5. La virtualización runtime soportada de nodos top-level ahora usa un plan de rebalanceo/migración en Bakery que puede promover sitios hijos y circuitos directos entre raíces de cola, preservando la jerarquía lógica para reportes. + +La actividad reciente de TreeGuard está disponible en dos lugares: + +- Las vistas de estado/actividad de TreeGuard en la WebUI. +- El journal de `lqosd`, donde TreeGuard ahora registra cada evento de actividad para que recargas, limpieza de overrides, cambios de SQM y fallos se puedan diagnosticar sin inspeccionar websockets. + ## Páginas Relacionadas - [HTB + fq_codel + CAKE: Comportamiento Detallado de Colas](htb_fq_codel_cake-es.md) diff --git a/docs-es/v2.0/troubleshooting-es.md b/docs-es/v2.0/troubleshooting-es.md index d2053f76..0dc21a88 100644 --- a/docs-es/v2.0/troubleshooting-es.md +++ b/docs-es/v2.0/troubleshooting-es.md @@ -46,14 +46,17 @@ Incluya también: ### La contraseña de usuario no funciona -Elimine el archivo de usuarios: +Los builds actuales: +- migran automáticamente archivos de autenticación antiguos +- redirigen `/login.html` a `/first-run.html` cuando no existen usuarios ```bash -sudo rm /opt/libreqos/src/lqusers.toml -sudo systemctl restart lqosd lqos_scheduler +sudo systemctl restart lqosd ``` -Luego abra: `IP_CAJA:9123/index.html`. +Si el usuario/contraseña correctos siguen fallando, pruebe primero con ese reinicio. + +Elimine `lqusers.toml` solo si desea reiniciar el acceso o si el archivo está corrupto y no se puede reparar. Después de eliminarlo, reinicie `lqosd` y abra `IP_CAJA:9123/login.html`; la WebUI debería redirigirlo automáticamente al flujo de primer inicio. ### No hay WebUI en x.x.x.x:9123 @@ -103,6 +106,8 @@ sudo RUST_LOG=info /opt/libreqos/src/bin/lqosd sudo journalctl -u lqos_scheduler --since "1 day ago" --no-pager > lqos_sched_log.txt ``` +Si el scheduler falla inmediatamente después de un reinicio con un mensaje como `Socket (typically /run/lqos/bus) not found`, eso indica que `lqosd` todavía no había terminado de enlazar el bus local. Los builds actuales esperan brevemente la disponibilidad del bus al arrancar el scheduler en lugar de abortar de inmediato, por lo que ya no deberían aparecer panics repetidos de arranque tras un reinicio. + ### El estado del scheduler en WebUI aparece no saludable Versiones recientes muestran estado/readiness del scheduler en WebUI. @@ -214,6 +219,10 @@ WebUI muestra códigos legibles por máquina para triage rápido. |---|---|---|---| | `MAPPED_CIRCUIT_LIMIT` | Bakery está forzando límite de circuitos mapeados. | Estado de licencia Insight y `journalctl -u lqosd` con requested/allowed/dropped. | Reducir circuitos mapeados o actualizar licencia/límites. | | `TC_U16_OVERFLOW` | IDs minor de clases/colas excedieron rango u16 de tc en una cola CPU. | `journalctl -u lqos_scheduler -u lqosd`, profundidad topológica y distribución por colas. | Aumentar paralelismo de colas y/o simplificar/rebalancear jerarquía. | +| `TC_QDISC_CAPACITY` | Los qdisc autoasignados planificados exceden el presupuesto seguro por interfaz o el preflight conservador de seguridad de memoria de Bakery antes de aplicar. | Conteos estimados por interfaz, desglose por tipo de qdisc y campos de memoria en el contexto del urgent issue, `journalctl -u lqos_scheduler -u lqosd`, configuración `on_a_stick` y `monitor_only`. | Reducir la carga planificada de qdisc para esta ejecución (por ejemplo menos circuitos/dispositivos en la forma de prueba) antes de reintentar; no confiar en una aplicación parcial. | +| `BAKERY_MEMORY_GUARD` | Un full reload chunked de Bakery fue detenido a mitad de aplicación porque la memoria disponible del host cayó por debajo del piso de seguridad. | `journalctl -u lqosd`, memoria disponible/total en el contexto del urgent issue y progreso reciente de aplicación de Bakery. | Tratar la ejecución como fallida, reducir presión de memoria o huella de colas y reintentar solo cuando el host esté estable. | +| `XDP_IP_MAPPING_CAPACITY` | Los mapeos IP requeridos exceden la capacidad actual del mapa XDP en el kernel. | Forma de `ShapedDevices.csv`, mezcla IPv4/IPv6, supuesto de un dispositivo frente a varios dispositivos, `journalctl -u lqos_scheduler -u lqosd`. | Reducir mapeos requeridos de inmediato (por ejemplo menos dispositivos o prueba IPv4-only), o aumentar la capacidad del mapa del kernel en un cambio coordinado. | +| `XDP_IP_MAPPING_APPLY_FAILED` | Uno o más inserts de mapeo IP fallaron durante la aplicación. | `journalctl -u lqos_scheduler -u lqosd` para ejemplos resumidos y conteos de fallo. | Corregir la causa del fallo y volver a ejecutar; no confiar en shaping parcial. | Patrón operativo: diff --git a/docs-es/v2.0/update-es.md b/docs-es/v2.0/update-es.md index 5609b5e1..fb39c2ea 100644 --- a/docs-es/v2.0/update-es.md +++ b/docs-es/v2.0/update-es.md @@ -39,10 +39,13 @@ sudo /opt/libreqos/src/systemd_hotfix.sh install sudo reboot ``` +El instalador del hotfix configura el repositorio APT de LibreQoS en `https://repo.libreqos.com`, instala el conjunto parchado de paquetes `systemd` de Noble y fija esos paquetes para futuras actualizaciones. + Después del reinicio, reanuda la actualización y reinicia los servicios: ```bash cd /tmp +wget https://download.libreqos.com/{deb_url_v1_5} sudo apt install ./{deb_url_v1_5} sudo systemctl restart lqosd lqos_scheduler ``` diff --git a/docs/v2.0/api.md b/docs/v2.0/api.md index 00f91508..56a58e46 100644 --- a/docs/v2.0/api.md +++ b/docs/v2.0/api.md @@ -34,6 +34,8 @@ sudo systemctl enable lqos_api sudo systemctl start lqos_api ``` +The shipped `lqos_api.service.example` waits for `network-online.target`, so the API does not start before DNS and the default route are ready. + Update only the API binary: ```bash diff --git a/docs/v2.0/components.md b/docs/v2.0/components.md index 90378832..cfb3b0b0 100644 --- a/docs/v2.0/components.md +++ b/docs/v2.0/components.md @@ -33,6 +33,7 @@ flowchart LR - lqos_scheduler performs continuous refreshes of LibreQoS' shapers, including pulling from any enabled CRM Integrations (UISP, Splynx, Netzur). - Actions: - On start: Run a full setup of queues + - Current builds wait briefly for `lqosd` to finish binding the local bus before the first scheduler run. - Every X minutes: Update queues, pulling new configuration from CRM integration, if enabled. - The default minute interval is 30, so the refresh occurs every 30 minutes by default. - The minute interval is adjustable with the setting `queue_refresh_interval_mins` in `/etc/lqos.conf`. @@ -84,6 +85,8 @@ To confirm that lqos_scheduler (scheduler.py) is able to work correctly, run: sudo python3 scheduler.py ``` +If an integration fails during a scheduler run, current builds keep the scheduler alive, surface a shortened output preview in scheduler status/error reporting, and save the full captured output to a timestamped `/tmp/lqos_scheduler__*.log` file. + Once you have any errors eliminated, restart lqos_scheduler with ```shell diff --git a/docs/v2.0/configuration-advanced.md b/docs/v2.0/configuration-advanced.md index e4f21e59..f2d8732c 100644 --- a/docs/v2.0/configuration-advanced.md +++ b/docs/v2.0/configuration-advanced.md @@ -81,6 +81,20 @@ If integration mode is enabled, integration refresh cycles typically own `Shaped - Use WebUI/manual edits for short operational adjustments only. - Put permanent changes in your integration system, integration overrides, or declared external source of truth workflow. +#### IP range allow/ignore behavior for integrations + +The `[ip_ranges]` section is also used when integrations generate subscriber/device shaping data. + +- `allow_subnets` defines the address space LibreQoS should consider shapeable. +- `ignore_subnets` removes matching addresses from generated subscriber devices even if they are otherwise present in the source CRM/NMS. +- Current shared integration-output pruning uses `ignore_subnets` to exclude generated subscriber/device rows, but does not newly require all imported integration IPs to be inside `allow_subnets` just to survive generation. +- If an imported device is left with no remaining non-ignored IPs after `ignore_subnets` is applied, that device is omitted from generated `ShapedDevices.csv`. +- If an imported circuit is left with no remaining shaped devices, that circuit is omitted from generated `ShapedDevices.csv`. + +This can be used to exclude entire subscriber populations from LibreQoS shaping and Insight-reported shaped-device inventory. For example, some operators shape wireless subscribers in LibreQoS but exclude FTTH subscribers whose ONTs already enforce service rates. + +Use this carefully: `ignore_subnets` is broader than a billing-only toggle. The same setting also affects other LibreQoS/Insight IP-policy handling. + #### CRM/NMS Integrations Learn more about [configuring integrations here](integrations.md). @@ -125,13 +139,16 @@ How overrides apply: - `lqos_scheduler` applies overrides during refresh cycles. - persistent devices are merged into `ShapedDevices.csv`. - circuit/device/network adjustments are applied on top of imported/manual data. +- operator-owned site bandwidth overrides prefer `node_id` when present and fall back to legacy name-only matching. +- tree-page `Operator Override` writes to the operator override layer in `lqos_overrides.json`, not to legacy integration bandwidth CSV files. +- automated runtime layers such as StormGuard and TreeGuard remain separate from the operator layer and are not written back into operator-authored source files. ### Network Hierarchy #### Network.json Network.json allows ISP operators to define a Hierarchical Network Topology, or Flat Network Topology. -Each topology node may optionally include an `"id"` field. This is intended to carry a stable node identifier from the source CRM/NMS when one exists. LibreQoS currently still matches hierarchy and overrides by node name, not by this ID, so `"id"` is metadata for now. +Each topology node may optionally include an `"id"` field. This is intended to carry a stable node identifier from the source CRM/NMS when one exists. Current builds prefer this ID when matching operator-owned site bandwidth overrides, while still supporting legacy name-only matching as a fallback. Recommended format: @@ -151,6 +168,15 @@ Notes: - Generated LibreQoS-only nodes may use stable generated IDs such as `libreqos:generated:uisp:site:orphans`. - Existing integration-specific metadata fields such as `uisp_site` and `uisp_device` may also appear alongside the generic `id` field. +#### Queue mode (`shape` / `observe`) + +LibreQoS currently uses `queue_mode` in the `[queues]` section to control whether the shaping tree is active: + +- `queue_mode = "shape"`: normal shaping mode +- `queue_mode = "observe"`: remove the subscriber shaping tree for a true baseline while keeping the root MQ in place + +The older `monitor_only` setting is retained as a compatibility alias in some configs and serialized output, but `queue_mode` is the current operator-facing setting and documentation term. + If you plan to use the built-in UISP, Splynx, or Netzur integrations, you do not need to create a network.json file quite yet. If you plan to use the built-in UISP integration, it will create this automatically on its first run (assuming network.json is not already present). diff --git a/docs/v2.0/configuration.md b/docs/v2.0/configuration.md index a0a0149b..03c57788 100644 --- a/docs/v2.0/configuration.md +++ b/docs/v2.0/configuration.md @@ -16,6 +16,8 @@ Notes: After install, sign in to WebUI at `http://your_shaper_ip:9123`. +If no WebUI users exist yet, current builds redirect to first-run setup automatically. + For most operators: 1. Choose your operating mode: [Operating Modes and Source of Truth](operating-modes.md) 2. Configure integration settings in WebUI: [CRM/NMS Integrations](integrations.md) @@ -25,13 +27,25 @@ For most operators: Most day-to-day LibreQoS configuration is handled in WebUI (`http://your_shaper_ip:9123/config_general.html`). +Current builds use shared configuration panels on the General, RTT, Queues, TreeGuard, Network Mode, Integration Defaults, Network Layout, Insight, provider-specific integration pages, IP Ranges, Flow Tracking, and Shaped Devices pages so node settings and operational tuning follow the same surface styling as the rest of Node Manager. The TreeGuard page starts with a live behavior summary rather than a permanent warning banner. + ### Where in WebUI - General settings: `Configuration -> General` - Integration settings: `Configuration -> Integrations` +- Network layout editor: `Configuration -> Network Layout` - Shaped devices editor: `Configuration -> Shaped Devices` - Runtime operational validation: `WebUI (Node Manager)` pages (dashboard/tree/flow/scheduler) +When an integration is enabled and acting as the source of truth, the `Network Layout` and `Shaped Devices` editors remain viewable but become read-only in WebUI. + +Topology note: +- `network.json` node names must be globally unique across the whole tree. Duplicate node names now fail validation and are not accepted by the WebUI save path or `LibreQoS.py`. +- When a node exposes a stable `id`, current builds prefer that ID for operator-owned site bandwidth overrides while keeping legacy name-only matching as a fallback. + +Queue-mode note: +- Current builds use `queue_mode` with `shape` and `observe` values. Older `monitor_only` wording is a compatibility alias rather than the primary operator-facing setting. + ## Operating Modes and Source of Truth Read this first before production changes: diff --git a/docs/v2.0/git-install.md b/docs/v2.0/git-install.md index cee5582e..4c4ad0ba 100644 --- a/docs/v2.0/git-install.md +++ b/docs/v2.0/git-install.md @@ -83,6 +83,8 @@ sudo cp /opt/libreqos/src/bin/lqosd.service.example /etc/systemd/system/lqosd.se sudo cp /opt/libreqos/src/bin/lqos_scheduler.service.example /etc/systemd/system/lqos_scheduler.service ``` +These templates wait for `network-online.target`, so `lqosd` and `lqos_scheduler` do not start before DNS and the default route are ready. + Finally, run ```shell diff --git a/docs/v2.0/integrations-sonar.md b/docs/v2.0/integrations-sonar.md index 1f6f6bed..d5c93ebb 100644 --- a/docs/v2.0/integrations-sonar.md +++ b/docs/v2.0/integrations-sonar.md @@ -2,6 +2,16 @@ First, set the relevant parameters for Sonar (sonar_api_key, sonar_api_url, etc.) in `/etc/lqos.conf`. +Current behavior notes: +- `sonar_api_url` may be either the Sonar base URL or the full GraphQL endpoint. LibreQoS normalizes it to `/api/graphql` automatically. +- Current builds page through Sonar GraphQL results instead of relying on a small first-page sample. +- Paginated Sonar GraphQL requests now use a split connect/read timeout and retry transient read timeouts before failing the import. +- Emitted Sonar identities are namespaced (for example `sonar:account:` and `sonar:device:`) so they remain stable across overrides and downstream tooling. +- Account device discovery now preserves inventory-item IP handling and also imports Radius account IP assignments when they exist. Inventory-backed MACs are still used for AP mapping; Radius-only IPs are added as supplemental shaping devices and overlapping subnets are de-duplicated. +- Sonar `child_accounts` are also imported when they expose their own service and usable IP data. If a child account lacks its own address, LibreQoS falls back to the parent account address so the child can still be emitted as its own circuit. +- Sonar settings now support ISP-specific recurring-service fallback rates plus a recurring-service exclusion list. LibreQoS still prefers active `DATA` services first; recurring mappings are only used when an account has no usable `DATA` service. +- If Sonar returns non-JSON content or GraphQL errors, the integration now raises a more specific error message showing the endpoint and a short response preview. + To test the Sonar Integration, use ```shell diff --git a/docs/v2.0/integrations-splynx.md b/docs/v2.0/integrations-splynx.md index c9492b48..e81be58c 100644 --- a/docs/v2.0/integrations-splynx.md +++ b/docs/v2.0/integrations-splynx.md @@ -130,12 +130,16 @@ If sync looks successful but data is incomplete, recheck API permissions and str You can also modify the the file `integrationSplynxBandwidths.csv` to override the default bandwidths for each Node (Site, AP). +Tree-page `Operator Override` edits are separate from this legacy Splynx file. Current builds write those operator-owned node rate changes to `lqos_overrides.json` and do not rewrite `integrationSplynxBandwidths.csv`. + A template is available in the `/opt/libreqos/src` folder. To utilize the template, copy the file `integrationSplynxBandwidths.template.csv` (removing the `.template` part of the filename) and set the appropriate information inside each file. For example, if you want to change the set bandwidth for a site, you would do: ``` sudo cp /opt/libreqos/src/integrationSplynxBandwidths.template.csv /opt/libreqos/src/integrationSplynxBandwidths.csv ``` And edit the CSV using LibreOffice or your preferred CSV editor. +To avoid conflicting sources of truth, prefer one durable override path per node: either the legacy Splynx CSV workflow or the operator override layer. + ## Related Pages diff --git a/docs/v2.0/integrations-uisp.md b/docs/v2.0/integrations-uisp.md index d0c0e5ef..25001243 100644 --- a/docs/v2.0/integrations-uisp.md +++ b/docs/v2.0/integrations-uisp.md @@ -173,6 +173,8 @@ If a `network.json` file exists, it is only overwritten when `always_overwrite_n ShapedDevices.csv will be overwritten every time the UISP integration is run. +When UISP client sites share the same name, LibreQoS now tries to disambiguate the generated circuit/site display names with a human-friendly suffix such as the first street-address segment, falling back to service name and then a short ID only when needed. Stable circuit identity still comes from the UISP site/service ID, not the display name. + For integration-driven deployments, keep `always_overwrite_network_json = true` so topology stays aligned with UISP on each refresh cycle. You have the option to run `uisp_integration` automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_uisp = true``` in `/etc/lqos.conf`. Once set, run `sudo systemctl restart lqos_scheduler`. @@ -183,6 +185,8 @@ You can also modify the following files to more accurately reflect your network: - integrationUISPbandwidths.csv - integrationUISProutes.csv +Tree-page `Operator Override` edits are separate from these legacy UISP files. Current builds write those operator-owned node rate changes to `lqos_overrides.json` and do not rewrite `integrationUISPbandwidths.csv`. + Each of the files above have templates available in the `/opt/libreqos/src` folder. If you don't find them there, you can navigate [here](https://github.com/LibreQoE/LibreQoS/tree/develop/src). To utilize the template, copy the file (removing the `.template` part of the filename) and set the appropriate information inside each file. For example, if you want to change the set bandwidth for a site, you would do: ``` @@ -190,6 +194,8 @@ sudo cp /opt/libreqos/src/integrationUISPbandwidths.template.csv /opt/libreqos/s ``` And edit the CSV using LibreOffice or your preferred CSV editor. +To avoid conflicting sources of truth, prefer one durable override path per node: either the legacy UISP CSV workflow or the operator override layer. + #### UISP Route Overrides The default cost between nodes is typically 10. The integration creates a dot graph file `/opt/libreqos/src/graph.dot` which can be rendered using [Graphviz](https://dreampuf.github.io/GraphvizOnline/). This renders a map with the associated costs of all links. diff --git a/docs/v2.0/integrations.md b/docs/v2.0/integrations.md index 6733d276..867c9cf1 100644 --- a/docs/v2.0/integrations.md +++ b/docs/v2.0/integrations.md @@ -53,7 +53,12 @@ When integrations are enabled: ## Topology Node ID Support -LibreQoS supports an optional generic `"id"` field on `network.json` nodes. This field is intended to carry stable node identifiers from the integration source where possible. In the current release, the field is informational and is not yet the authoritative match key for shaping or overrides. +LibreQoS supports an optional generic `"id"` field on `network.json` nodes. This field is intended to carry stable node identifiers from the integration source where possible. + +Current behavior: +- node IDs are the preferred match key for operator-owned site bandwidth overrides and tree-page node override editing when an ID is present +- legacy name-only matching is still supported as a fallback for older data +- topology names still need to remain globally unique in `network.json` | Integration | `network.json` node ID support | Notes | |---|---|---| diff --git a/docs/v2.0/libreqos-backend-architecture.md b/docs/v2.0/libreqos-backend-architecture.md index c35c1114..78ac968c 100644 --- a/docs/v2.0/libreqos-backend-architecture.md +++ b/docs/v2.0/libreqos-backend-architecture.md @@ -36,6 +36,7 @@ In production terms: - `XDP`/`eBPF` and lookup maps decide packet identity and CPU path. - Linux traffic control (`tc`) enforces queueing policy. - Bakery manages update deltas and reload boundaries. +- Queue mode (`shape` vs `observe`) controls whether the subscriber shaping tree is active or intentionally removed for baseline measurement. ## 2) Runtime Invariants @@ -48,8 +49,23 @@ These invariants are useful for reasoning about whether backend behavior is heal | Data-plane mapping is stable between XDP and `tc` | Mis-mapped packets cause wrong queue assignment | Unexpected class counters, mis-accounted traffic | Compare expected class IDs vs observed `tc -s` class counters | | Control-plane changes stay within incremental-safe bounds | Reduces disruption from full-tree rebuilds | Frequent packet-impacting reload windows | Review change patterns: structural vs speed/mapping updates | | Queue count stays within CPU/RAM budget | Leaf qdisc scale directly impacts resources | Memory growth, reload slowdowns, jitter under churn | Track queue count, RAM headroom, and update cadence | +| Queue mode and dataplane mappings are sequenced consistently | Packets must not be steered toward removed queue state | Brief outages, stale class targets, mis-accounted traffic | Check queue mode, IP mapping lifecycle, and live `tc` state together | -## 3) End-to-End Packet and Control Path +## 3) Runtime Authority and Configuration Model + +LibreQoS has both on-disk config files and a long-running runtime daemon, but they are not the same authority at every moment. + +1. `lqosd` is the runtime control-plane authority while it is running. +2. UI/config API updates go through `lqosd`, which updates in-memory state and then drives apply/reload behavior. +3. Manual edits to `/etc/lqos.conf` are operator-managed source inputs, but they do not automatically become active runtime state until the daemon reload path consumes them. + +Operational takeaway: + +- treat the UI/config API path as the authoritative live-update workflow +- treat direct file edits as a separate operator action that still needs a runtime reload/apply boundary +- do not assume “file changed” and “runtime desired state changed” are equivalent at the same instant + +## 4) End-to-End Packet and Control Path ```{mermaid} flowchart LR @@ -76,9 +92,9 @@ flowchart LR N -. updates queue state .-> H ``` -## 4) Data Plane Design +## 5) Data Plane Design -### 4.1 XDP and classification pipeline +### 5.1 XDP and classification pipeline LibreQoS performs early packet work in XDP where possible: @@ -93,11 +109,11 @@ A key optimization direction has been reducing repeated lookups: - avoid duplicate work between XDP and `tc` when metadata can be passed forward -### 4.2 Why `cpumap` is central +### 5.2 Why `cpumap` is central `cpumap` is used to spread work across cores so shaping does not bottleneck on a single queue path. This is a major part of scaling from "works" to "works at ISP traffic levels". -### 4.3 Cache, generation, and lock-pressure reduction +### 5.3 Cache, generation, and lock-pressure reduction The high-level progression described in development notes/devblogs: @@ -107,8 +123,19 @@ The high-level progression described in development notes/devblogs: Operationally, this helps stabilize latency and CPU under frequent updates. +### 5.4 Mapping state is part of the data-path contract -## 5) Queue Hierarchy: `mq` -> `HTB` -> leaf qdisc +Queue classes and IP mappings are related but not identical backend state. + +1. The shaping tree defines where packets can land in `tc`. +2. IP mappings define which circuit/class packets are currently steered toward. +3. During ordinary `shape -> shape` updates, LibreQoS tries to preserve stable mappings and stable queue handles where safe. +4. During disruptive transitions, especially `observe <-> shape`, LibreQoS can intentionally clear and later republish mappings so packets are not pointed at queue state that no longer exists. + +This sequencing matters as much as the queue commands themselves. A healthy backend design must reason about queue-tree changes and mapping changes together. + + +## 6) Queue Hierarchy: `mq` -> `HTB` -> leaf qdisc LibreQoS queueing is intentionally layered: @@ -134,7 +161,7 @@ flowchart TD I --> J ``` -### 5.1 HTB internals that matter in production +### 6.1 HTB internals that matter in production Important mechanics: @@ -150,16 +177,16 @@ Why operators care: - HTB is the rate envelope; leaf qdiscs are not a drop-in replacement for HTB policy -## 6) AQM in LibreQoS: `fq_codel` and `CAKE` +## 7) AQM in LibreQoS: `fq_codel` and `CAKE` -### 6.1 Responsibility split +### 7.1 Responsibility split Practical split: 1. HTB: hierarchical bandwidth policy and limits 2. `fq_codel`/`CAKE`: queue fairness and delay control within that policy -### 6.2 Why AQM still helps even below line-rate saturation +### 7.2 Why AQM still helps even below line-rate saturation Even when a link is not saturated, fq_codel and CAKE still affect packet behavior. The drop logic (CoDel/BLUE/COBALT) usually stays idle, but the fair-queueing scheduler continues to interleave flows, preventing bursts from one flow from monopolizing the serialization point of the link. This keeps latency-sensitive traffic responsive. @@ -189,7 +216,7 @@ Even without sustained congestion: So even when the AQM drop logic is mostly idle, the fair-queueing part of SQM is still doing useful work by controlling how packets reach the wire. -### 6.3 CAKE vs fq_codel in LibreQoS terms +### 7.3 CAKE vs fq_codel in LibreQoS terms General pattern: @@ -202,7 +229,7 @@ Resource reality: - both are flow-aware and keep state - CAKE can have higher memory/CPU footprint in large queue populations -### 6.4 When below-line-rate gains may be limited +### 7.4 When below-line-rate gains may be limited Lower latency under mixed load is common, but not guaranteed in every scenario. @@ -218,7 +245,7 @@ Operator takeaway: - Treat AQM gains as a result of queue dynamics and contention control, then validate empirically on your own traffic mix. -## 7) Bakery and Reload Behavior +## 8) Bakery and Reload Behavior Bakery exists to avoid unnecessary queue rebuilds and reduce reload penalties. @@ -227,10 +254,10 @@ High-level flow: 1. Build desired state. 2. Diff desired vs active state. 3. Apply smallest safe delta. -4. Trigger full reload only when update type crosses live-mutation limits. +4. Trigger full reload when a change is outside live-mutation support, or when runtime verification/drift detection marks incremental topology mutation unsafe. -### 7.1 Lazy queueing and expiration +### 8.1 Lazy queueing and expiration Key controls: @@ -242,42 +269,132 @@ Practical effect: - reduced memory overhead for dormant endpoints - lower churn for large but partially active subscriber populations -### 7.2 Incremental vs reload boundary +### 8.2 Incremental vs reload boundary | Change type | Usually incremental-safe | Often requires full reload | Why | |---|---|---|---| -| Circuit IP-only change | Yes | No | Mapping update can often be applied without tree rebuild | -| Circuit/site speed change (subset) | Yes | Sometimes | Depends on structural impact and available class handles | -| Bulk all-circuit changes | Sometimes | Often | Scale and transaction/cardinality limits | -| Topology re-parent/restructure | Rarely | Yes | HTB subtree mutation constraints | -| Add/remove circuits | Yes (small/moderate) | Sometimes | Handle availability and diff correctness boundaries | +| Circuit IP-only change | Yes | No | Mapping updates can usually be applied without rebuilding the queue tree | +| Circuit SQM-only change | Yes | No | Leaf qdisc kind/parameter changes can usually be applied live | +| Circuit/site speed change (subset) | Yes | Sometimes | Depends on structural impact, queue-count pressure, and available class handles | +| Ordinary circuit parent move | Yes | Sometimes | Bakery uses staged live migration for common active parent/class moves, including qdisc-handle rotation and final-state verification, but it still escalates to reload if the migration cannot be applied or verified safely | +| TreeGuard runtime node virtualization (supported subtree/top-level rebalance path) | Yes | Sometimes | Bakery can apply supported runtime virtualization live, but deferred cleanup, live-state verification failures, or accumulated dirty runtime subtrees can mark `reload required` and freeze further incremental topology mutation until a full reload | +| Bulk all-circuit changes | Sometimes | Often | Scale and transaction/cardinality limits still matter, even with better incremental behavior | +| Site add/remove or broader structural topology change | Rarely | Yes | HTB subtree mutation constraints remain much stricter at site/topology level than at per-circuit level | +| Add/remove circuits | Yes (small/moderate) | Sometimes | Handle availability, tree size, and diff correctness boundaries | This table reflects Bakery design behavior and Linux `tc` mutation constraints discussed in the devblog material. ```{mermaid} flowchart TD - A[Config or integration change arrives] --> B[Build desired state and compute diff] + A[Config or runtime change arrives] --> B[Build desired or runtime target state] B --> C{Any effective state change?} C -->|No| D[No-op] - C -->|Yes| E{Structural hierarchy change?} - E -->|Yes| F[Controlled full reload] - E -->|No| G{Within incremental-safe limits?} - G -->|Yes| H[Apply incremental tc updates] - G -->|No| F - H --> I[Verify class/qdisc state and counters] - F --> I + C -->|Yes| E{Supported live mutation path?} + E -->|No| F[Controlled full reload] + E -->|Yes| G[Apply staged incremental/runtime mutation] + G --> H{Live verification and cleanup safe?} + H -->|Yes| I[Keep incremental state authoritative] + H -->|No| J[Mark reload required and freeze further incremental topology mutation] + J --> F + F --> K[Re-establish single authoritative queue model] ``` -### 7.3 Reload boundary quick rules +### 8.3 Queue modes and transition semantics + +LibreQoS currently has two explicit queue modes: + +1. `shape` + - normal shaping mode + - root `mq` present + - HTB hierarchy present + - leaf qdiscs present + - per-circuit IP mappings present +2. `observe` + - true-baseline mode + - root `mq` retained + - subscriber shaping tree removed + - per-circuit IP mappings cleared before teardown + - mappings republished after returning to `shape` + +Important operator boundary: + +- `observe` is intentionally honest, not hitless +- switching `observe <-> shape` can briefly interrupt traffic because the shaping tree really is removed and later rebuilt +- this is different from ordinary `shape -> shape` updates, where LibreQoS tries to preserve stable handles and queue placement where possible + +### 8.4 Retained-root full reload behavior + +Current Bakery full reloads try to avoid unnecessary root churn. + +1. Verify live kernel `tc` state, not just planned state. +2. If the root `mq` is healthy and matches the expected layout, retain it. +3. Prune child qdiscs beneath the retained root and verify the subtree is clean. +4. Rebuild the shaping tree beneath that retained root. +5. Fall back to root recovery only when retained-root reuse is unsafe or verification fails. + +This retained-root strategy reduces avoidable root-level churn, while still preferring explicit recovery when live state is ambiguous. + +### 8.5 Reload boundary quick rules 1. Prefer frequent small mapping/speed deltas over large structural churn. 2. Batch topology surgery into planned windows. 3. Expect higher risk when many circuits and many structure-affecting changes happen together. 4. Build operations cadence around incremental-safe updates by default. -## 8) Design Boundaries for Operators +### 8.6 Full-reload safety guards -### 8.1 Observability boundaries +Current Bakery full reloads apply two conservative safety checks before and during large queue rebuilds: + +1. A qdisc preflight estimates planned qdiscs per interface and also separates infrastructure, `cake`, and `fq_codel` leaf qdiscs. +2. That same preflight applies a conservative memory forecast and hard-blocks clearly unsafe full reloads before `tc -batch` starts. +3. During chunked full reload apply, Bakery re-checks host memory at chunk boundaries and aborts the remaining apply if available memory drops below its safety floor. +4. These guards are intentionally biased toward false positives on large reloads so the system fails early with diagnostics instead of spiraling into an OOM event. + +### 8.7 Runtime safety model + +Bakery's newer runtime-safety direction is intentionally narrow: + +1. Reconcile enough live state to decide whether deferred cleanup is safe. +2. Detect material drift between Bakery's intended state and live kernel `tc` state. +3. Stop trusting incremental/runtime mutation once drift is real. +4. Escalate to a controlled full reload as the recovery path. + +This is intentionally **not** a broad self-healing reconciler. LibreQoS is biased toward: + +- lightweight cleanup gating for expected lag +- explicit `reload required` escalation on material live-state drift +- one controlled full reload to re-establish a single authoritative queue model + +That design keeps failure handling easier to reason about than trying to incrementally repair arbitrary split-brain queue state. + +### 8.8 Explicit qdisc-handle management + +Bakery now treats leaf qdisc handles as persistent runtime state rather than disposable auto-allocation details. + +1. Circuit leaf qdiscs are assigned explicit handle majors and persisted across applies. +2. Handle assignments rotate when a live mutation changes the effective leaf qdisc kind or qdisc parent. +3. Full reload planning reserves live handle majors so rebuilds do not collide with surviving kernel state. +4. Parent-changed live migration is rejected if Bakery detects stale-handle reuse that would make final state ambiguous. + +This handle model is one of the mechanisms that makes common live circuit migration safer than earlier Bakery generations. + +### 8.9 Runtime virtualization limits and operator expectations + +Current runtime virtualization support is intentionally constrained. + +1. Non-top-level runtime virtualization is limited to same-queue / same-major-domain subtree paths. +2. Top-level runtime virtualization uses a separate rebalance/promote path and only applies when Bakery can derive a deterministic split. +3. Runtime operations may remain in `AppliedAwaitingCleanup` while deferred prune work completes. +4. Runtime operations can become `Dirty`; repeated dirty subtree states escalate to `reload required` rather than attempting broad self-healing. + +Operator takeaway: + +- treat runtime virtualization as a narrow live-mutation feature with verification gates +- treat `reload required` as the authoritative signal that Bakery no longer trusts incremental topology mutation + +## 9) Design Boundaries for Operators + +### 9.1 Observability boundaries | Signal | Strong for | Weak for | |---|---|---| @@ -285,32 +402,47 @@ flowchart TD | CAKE/fq_codel drops/marks | Detecting persistent queue pressure and policy effects | Full end-to-end application blame assignment | | CPU/RAM and command timing | Capacity and reload risk planning | Isolating every microburst source | +### 9.2 Metric sample semantics and clock domains -### 8.2 Capacity risk factors and mitigations +Backend metrics do not all come from the same sampling source or clock edge. + +1. Some values are sampled from queue/kernel state. +2. Some values are sampled from flow telemetry. +3. Some rollups are built from canonical raw samples and only later rendered as percentages. + +Practical implication: + +- do not casually combine unrelated numerator/denominator sources and assume they describe the same exact second +- prefer transporting canonical samples/counts and deriving percentages at presentation time +- treat “same field name” and “same clock domain” as separate questions + +This matters especially for retransmit, packet, and rate-derived health metrics. + +### 9.3 Capacity risk factors and mitigations | Risk factor | Typical symptom | Mitigation | |---|---|---| | Very high queue counts with CAKE everywhere | RAM growth and scheduler overhead | Use `lazy_queues`, expiry, selective fq_codel where appropriate | -| Frequent full-tree updates | Brief packet disruption windows | Increase incremental-safe update usage; batch structural changes | +| Frequent full-tree updates | Brief packet disruption windows | Increase incremental-safe update usage; batch structural changes, and let Bakery keep ordinary circuit moves incremental where possible | | Incomplete parent mapping in hierarchy | Subscribers unexpectedly unshaped | Validate parent relationships in `network.json` and input data | | Single-queue/weak NIC virtualization behavior | Poor spread and unstable shaping | Ensure multi-queue NIC path and verify queue mapping assumptions | -## 9) Symptom-to-Cause Troubleshooting Matrix +## 10) Symptom-to-Cause Troubleshooting Matrix | Symptom | Common backend cause | First checks | Typical corrective direction | |---|---|---|---| | Latency spikes but interface throughput is not fully pegged | Microburst queue buildup, poor flow isolation, or direction mismatch | Compare latency vs queue/drop trends; verify both directions are shaped | Tune leaf qdisc strategy and verify directional shaping design | | One CPU runs hot while others are underused | Queue steering imbalance or weak multi-queue path | Inspect CPU utilization and per-class counters by queue branch | Fix queue mapping assumptions and verify `mq`/class structure | | Subscribers intermittently appear unshaped | Parent/hierarchy mapping mismatch | Validate parent node references and resulting class creation | Correct hierarchy mappings, then apply and verify class presence | -| Frequent short disruption during updates | Too many full-reload-triggering changes | Classify recent changes as structural vs incremental | Re-batch operations to favor incremental-safe deltas | +| Frequent short disruption during updates | Too many full-reload-triggering changes or runtime drift escalation | Classify recent changes as structural vs incremental, and check for `reload required` events | Re-batch operations to favor incremental-safe deltas and investigate live-state drift | | RAM growth during scale-up | Too many active leaf qdiscs or aggressive CAKE footprint | Measure queue count and memory trends over update windows | Use lazy queue creation/expiry and consider selective fq_codel use | | Dashboard traffic appears higher than expected user throughput | Counter scope differs from post-drop forwarded traffic | Compare dashboard metrics with `tc` drop/mark context | Align runbooks to metric semantics before escalating | -## 10) Change Validation Workflow +## 11) Change Validation Workflow Use this lightweight workflow for backend-impacting changes. -### 10.1 Pre-change +### 11.1 Pre-change 1. Classify change type: mapping/speed/structure. 2. Estimate impact scope: number of affected circuits/classes. @@ -320,7 +452,7 @@ Use this lightweight workflow for backend-impacting changes. - CPU and RAM headroom - `tc` class/qdisc snapshot -### 10.2 During change +### 11.2 During change 1. Watch control-plane behavior: - incremental apply vs full reload occurrence @@ -330,14 +462,19 @@ Use this lightweight workflow for backend-impacting changes. - directional latency drift - per-class counter discontinuities -### 10.3 Post-change +### 11.3 Post-change 1. Re-check the same baseline signals. 2. Confirm hierarchy/class presence for changed circuits. 3. Verify subscriber-facing latency and throughput expectations. 4. If degraded, rollback or reduce change scope and re-apply in smaller batches. -### 10.4 Minimal command checklist +For `observe <-> shape` transitions, add two specific checks: + +1. confirm the queue mode actually matches the intended state +2. confirm per-circuit mappings were cleared or republished at the right phase of the transition + +### 11.4 Minimal command checklist Adjust device names to your environment. @@ -347,7 +484,7 @@ tc -s class show dev journalctl -u lqosd --since "15 min ago" ``` -## 11) Practical Tuning Sequence +## 12) Practical Tuning Sequence Recommended order: @@ -358,7 +495,7 @@ Recommended order: 5. Tune update cadence to favor incremental-safe changes. 6. Re-test after major speed-plan, topology, or integration-cadence changes. -## 12) Glossary +## 13) Glossary - `XDP`: earliest high-performance packet hook in Linux. - `eBPF`: in-kernel programmable packet processing. diff --git a/docs/v2.0/node-manager-ui.md b/docs/v2.0/node-manager-ui.md index 837a37f7..4f2dec78 100644 --- a/docs/v2.0/node-manager-ui.md +++ b/docs/v2.0/node-manager-ui.md @@ -7,18 +7,46 @@ This page documents key WebUI (Node Manager) views and operational behavior in t ### Dashboard - Widget-based overview of throughput, retransmits, RTT, flow counts, and queue activity. - Dashboard content can vary by version and enabled features. +- Executive Summary provides a compact operational view for large networks, with a `Network Snapshot` focused on throughput, inventory, and Insight state plus drilldown pages for executive heatmaps and leaderboards. +- Bakery provides dedicated status for queue apply state, qdisc safety/preflight results, circuit live-change progress, and recent Bakery operations. +- The Bakery and TreeGuard tabs present a high-level pipeline or control-loop summary ahead of the more detailed tables. +- The Bakery `Pipeline` widget shows queue-control stages, apply state, verification state, and TC interval timing. +- `Runtime Operations` summarizes TreeGuard/Bakery topology mutations, deferred cleanup work, failures, and subtrees waiting for a full reload. +- `Recent Bakery Events` emphasizes grouped operations, with detailed event history available when deeper troubleshooting is needed. +- `TreeGuard Activity` emphasizes grouped operations, including SQM change batches, with detailed event history available when deeper troubleshooting is needed. +- `TreeGuard Control Loop` shows the current observe/evaluate/act state. +- `TreeGuard Decision Impact` focuses on current impact and any active warnings or errors. +- `TreeGuard State Mix` shows managed nodes, runtime virtualization, managed circuits, and the `cake / mixed / fq_codel` circuit SQM split. +- Bakery qdisc preflight summarizes planned per-interface qdisc usage and budget headroom before apply. +- Some charts may take a short time to populate after first opening a tab, especially on busy systems or immediately after a service restart. +- During a Bakery full reload, queue-count cards can temporarily continue showing the last known HTB/CAKE/fq-codel values and mark them as `Reloading`. ### Network Tree Overview - Hierarchical topology view of nodes/circuits from the shaper perspective. - Useful for spotting bottlenecks and parent/child utilization patterns. +- Tree detail pages show a breadcrumb path, branch counts, and status indicators for the selected node. +- `Node Details` summarizes the selected node’s type, branch size, configured rates, and effective rate. +- `Node Snapshot` provides a quick visual summary of throughput and QoO for the selected node. +- Attached circuits are shown in a dedicated table for the selected node. +- The attached-circuits IP column keeps rows compact by showing one address inline and collapsing additional addresses as `+X`, with the full list still available on hover. +- Ethernet-limited attached circuits can show inline `10M`, `100M`, or `1G` warning badges beside the `Plan (Mbps)` value; hovering explains the auto-cap and clicking the badge opens the dedicated Ethernet review page. +- Administrators can save or clear `Operator Override` values where node-level overrides are supported. Read-only users and unsupported nodes continue to display current values without edit controls. +- Tree-page operator rate edits write operator-owned overrides to `lqos_overrides.json`; they do not rewrite legacy integration bandwidth CSV files. +- Tree-page operator rate edits require an administrator session, a stable node ID, and a non-generated node. Generated/integration-synthetic nodes remain read-only in this editor. ### Site Map - Flat operational map of Sites and APs using imported node geodata. - Defaults to QoO coloring with an RTT toggle, while marker size reflects recent combined throughput. -- Uses a 30-second client-side average from `NetworkTree` data rather than adding backend rollup work. - APs can inherit parent site coordinates for display when explicit AP coordinates are missing. -- Uses a local LibreQoS-themed basemap with country/state borders, coastlines, major lakes, major rivers, marine areas, subtle physical-region overlays, and higher-zoom major-highway context for geographic orientation. -- Site Map uses a local Natural Earth-derived roads layer for orientation while keeping the rest of the basemap quiet and operational. +- Nearby site markers cluster and expand as the operator zooms or selects a cluster. +- APs without explicit coordinates are represented through their parent site and can be expanded temporarily around the selected site for inspection. +- Visible unclustered sites show labels as the operator zooms in, and the selected site keeps its label visible while it is being inspected. +- When browser redaction mode is enabled, Site Map replaces displayed site names with `[redacted]` while leaving the underlying topology data unchanged. +- Initial map framing prefers site coordinates for the first view, falling back to AP coordinates when no sites are mapped yet. +- Site Map uses an Insight-hosted OpenStreetMap raster tile cache. +- In dark mode, the raster underlay is muted and tinted client-side toward the same cooler blue/cyan palette used by Flow Globe, so roads and geography stay visible without the bright light-theme basemap glare. +- Site Map depends on outbound access to `https://insight.libreqos.com` for initial bbox/bootstrap and raster tile fetches. +- When tiles are missing from the remote cache, the browser retries automatically for a short period instead of failing immediately, so initial map paint can lag briefly on cold tiles. ### Flow Globe - Geographic flow visualization based on endpoint geolocation. @@ -31,23 +59,38 @@ This page documents key WebUI (Node Manager) views and operational behavior in t ### ASN Analysis - Live ASN operations page combining a top-20 ASN leaderboard, latency-vs-traffic bubble chart, selected ASN KPI strip, 15-minute ASN trend chart, and embedded Flow Evidence. - Supports `Impact` and `Throughput` ranking modes while keeping ASN flow evidence on the same page. -- The legacy `ASN Explorer` route now redirects here so older bookmarks still work. +- Older `ASN Explorer` bookmarks continue to work through a redirect. - Empty results usually indicate low recent data volume rather than failure. ### Circuit page - Circuit pages combine queue behavior, live throughput, RTT, retransmits, and per-flow troubleshooting for an individual subscriber/circuit. +- When integration metadata reports a negotiated CPE Ethernet speed, the `Max` row can show a warning badge such as `100M`; hovering the badge explains when LibreQoS auto-capped shaping below the requested plan to stay within that port speed, and clicking the badge opens the Ethernet review page. - `Queue Dynamics` shows circuit throughput and RTT behavior over time, including an `Active Flows` KPI based on the same recent flow window used by the `Traffic Flows` table. +- `Top ASNs` summarizes the busiest recent ASNs on the circuit from that same live flow window and sorts by current rate by default. +- `Devices` shows per-device detail tables and live charts for throughput, retransmits, and latency. +- `Queue Stats` shows recent live queue history for the circuit, including backlog, delay, queue length, traffic, ECN marks, and drops. +- Queue Stats charts use synchronized hover so operators can inspect the same second across all queue charts together. +- `Queue Tree` shows the circuit's live upstream queue path, including a path summary and per-node throughput, retransmit, and latency context. - `Traffic Flows` is a recent-flow operational table rather than a long-term history view. - `Traffic Flows` includes paging and a `Hide Small Flows` filter so large busy circuits remain usable without trying to render every row at once. +- `Traffic Flows` current-rate display is limited to plausible, plan-aware values for the circuit. +- Long text in the `Protocol`, `ASN`, and `Country` columns is truncated with an ellipsis to keep row height stable; the full value remains available on hover. - `Flow Sankey` emphasizes the hottest recent flows rather than every older retained flow. +### Ethernet Caps +- The Ethernet review page is a lightweight operator table of circuits automatically down-rated because detected Ethernet speed was below the requested plan. +- It is intentionally not in the main navigation; operators reach it by clicking Ethernet warning badges on the Circuit page or Tree attached-circuits table. +- The page supports search, tier filtering (`10M`, `100M`, `1G+`), and paging across auto-capped circuits. + ### CPU Tree / CPU Weights - Shows queue/circuit distribution by CPU core. - Helps evaluate binpacking and load distribution behavior. +- CPU Affinity starts with shaping CPUs only, while excluded or host-only cores can be shown when needed. ### Shaped Devices Editor - CRUD editor for `ShapedDevices.csv`. -- Supports paging and filtering in current builds. +- Supports paging and filtering. +- Add, edit, and delete actions save immediately in the dedicated editor. ### Urgent Issues - WebUI can display urgent operational issues generated by backend services. @@ -67,6 +110,7 @@ This page documents key WebUI (Node Manager) views and operational behavior in t - Toggle with the mask icon in the top navigation. - Redaction is client-side and stored in browser local storage. - Redaction helps hide PII in screenshots/demos. +- Site Map replaces displayed site names with `[redacted]` while redaction mode is enabled. - Redaction does not modify `ShapedDevices.csv`, `network.json`, or any backend data. ## Common Empty-State Behavior @@ -87,6 +131,8 @@ When this happens: journalctl -u lqosd --since "10 minutes ago" ``` +If only Site Map is blank or slow while other pages are healthy, also check management-network reachability to `insight.libreqos.com` and allow a short delay for cold tile-cache retries. + ## Useful Related Pages - [Components](components.md) diff --git a/docs/v2.0/quickstart.md b/docs/v2.0/quickstart.md index 5b83d0cc..82088941 100644 --- a/docs/v2.0/quickstart.md +++ b/docs/v2.0/quickstart.md @@ -1,12 +1,21 @@ # Quickstart: ISP Deployment Path -Use this page to move from install to a safe pilot with minimal ambiguity. +Use this page to move from install to a safe pilot. + +Most ISPs, should use a LibreQoS built-in integration. + +Use this page in order: +1. Complete install and bridge setup. +2. Complete first-run WebUI access. +3. Pass the 10-minute health check. +4. Pick one source of truth owner. +5. Follow the recommended path for that owner. Need definitions for key terms? See the [Glossary](glossary.md). ## 1) Common Install Foundation -Complete this once: +Complete this once before trying to shape live traffic: 1. Review architecture and sizing: - [Deployment Scenarios](design.md) @@ -16,9 +25,11 @@ Complete this once: - [Server Setup - Prerequisites](prereq.md) - [Install Ubuntu Server 24.04](ubuntu-server.md) -3. Configure bridge mode: +3. Configure how traffic will pass through the LibreQoS box: - [Configure Shaping Bridge](bridge.md) +This step is required before production use. If bridge/interface setup is wrong, later WebUI and shaping checks will be misleading. + 4. Install LibreQoS (`.deb` recommended): ```bash @@ -42,15 +53,25 @@ sudo /opt/libreqos/src/systemd_hotfix.sh install sudo reboot ``` +The hotfix installer bootstraps the LibreQoS APT repo at `https://repo.libreqos.com`, installs the patched Noble `systemd` package set, and pins those packages for future updates. + After the reboot, resume the install: ```bash cd /tmp +wget https://download.libreqos.com/{deb_url_v1_5} sudo apt install ./{deb_url_v1_5} ``` 5. Open WebUI at `http://your_shaper_ip:9123`. +If no WebUI users exist yet, current builds redirect you to the first-run setup page automatically instead of presenting a normal login flow. + +6. Complete first-run access: +- Create the initial WebUI user if prompted. +- Confirm you can sign in. +- Confirm the Dashboard opens before doing any integration or topology work. + ## 2) 10-Minute Health Check Run: @@ -64,54 +85,65 @@ Confirm: - WebUI Dashboard loads. - Scheduler Status is healthy. - No urgent/fatal startup errors in logs. +- You can sign in successfully after first-run setup. If this fails, go to [Troubleshooting](troubleshooting.md) before proceeding. -## 3) Decision A: Deployment Stage +## 3) Pick One Source of Truth Owner + +This is the most important early decision. + +Source of truth means: the one place that should own persistent shaping data. + +If you make persistent changes somewhere else, they may be overwritten later. Choose one: -- **Lab first**: validate behavior in a controlled environment before inline traffic. -- **Inline pilot now**: proceed directly with limited production traffic. - -If lab-first: -1. Build a lab topology. -2. Generate test traffic. -3. Validate Dashboard, Tree, Flow, Scheduler Status, and Urgent Issues. -4. Continue to Decision B. - -## 4) Decision B: Source of Truth (Pick One Owner) - | If this describes you | Mode | Owner of durable shaping data | |---|---|---| -| You use a supported CRM/NMS integration | Built-in integration mode | Integration jobs | -| You generate `network.json` and `ShapedDevices.csv` with your own scripts | Custom source of truth mode | Your scripts | -| You maintain files manually for a small/simple network | Manual files mode | Manual edits | +| You use a supported LibreQoS CRM/NMS integration | Built-In Integration Mode | Integration jobs | +| You generate `network.json` and `ShapedDevices.csv` with your own automation | Custom Source of Truth Mode | Your scripts | +| You intentionally maintain files by hand | Manual Files Mode | Manual edits | Rule: keep one owner for persistent shaping inputs. -## 5) Path Cards +Most operators should stop here and choose **Built-In Integration Mode**. -### Built-In Integration Mode +Only choose the other two modes if you already know why. + +## 4) Recommended Path: Built-In Integration Mode + +This is the default path for most ISPs, including most small ISPs. When to choose: - Your CRM/NMS is supported by built-in integrations. Do this now: 1. Configure integration settings in WebUI. -2. Run initial sync and validate imported shaping/topology data. -3. Place LibreQoS inline for pilot traffic. -4. Validate Scheduler Status, Urgent Issues, and topology/flow views. -5. Expand pilot scope after stable operation. +2. Run the initial sync. +3. Confirm outputs were created or refreshed: + - `network.json` + - `ShapedDevices.csv` +4. Confirm Scheduler Status is healthy. +5. Confirm Network Tree reflects the expected hierarchy depth for your chosen integration strategy. +6. Confirm there are no urgent/fatal issues that block shaping. +7. Only then move to limited pilot traffic or inline use. + +Do not do this: +- Do not hand-edit integration-owned files as your normal workflow. +- Do not split ownership between integration syncs and manual file edits unless you intentionally want that complexity. +- Do not change overwrite behavior unless you understand how your integration should own `network.json`. Next: - [CRM/NMS Integrations](integrations.md) - [Troubleshooting](troubleshooting.md) -### Custom Source of Truth (Your Scripts) +## 5) Other Paths + +### Custom Source of Truth Mode When to choose: -- Your CRM/NMS is unsupported and you generate `network.json` + `ShapedDevices.csv` with your own pipeline. +- Your CRM/NMS is unsupported and you already generate `network.json` + `ShapedDevices.csv` with your own pipeline. Do this now: 1. Implement script/process to generate and refresh shaping files. @@ -127,17 +159,22 @@ Next: - [Operating Modes and Source of Truth](operating-modes.md) - [Troubleshooting](troubleshooting.md) -### Manual Files Mode (<100 Subscribers) +### Manual Files Mode When to choose: - You intentionally maintain `network.json` + `ShapedDevices.csv` without CRM/NMS synchronization. +- Your network is small/simple enough that manual discipline is realistic. +- You are doing a temporary pilot or working around an unsupported system. Do this now: -1. Build and maintain shaping files directly. -2. Place LibreQoS inline for pilot traffic. -3. Validate shaping and scheduler status in WebUI. -4. Maintain strict manual change discipline. -5. Plan migration to supported integration or scripts if scale/change volume grows. +1. Build `network.json`. +2. Build `ShapedDevices.csv`. +3. Restart or validate services. +4. Confirm the expected topology and devices appear in WebUI. +5. Validate shaping and scheduler status before adding more complexity. +6. Maintain strict manual change discipline. + +This is not the normal recommendation for operators who already use a supported LibreQoS integration. Next: - [Advanced Configuration Reference](configuration-advanced.md) @@ -146,10 +183,22 @@ Next: ## 6) Common First-Run Mistakes - Unclear source of truth ownership between integration and manual edits. +- Hand-editing files even though your integration is supposed to own them. - Changing topology depth before passing the health gate. - Skipping post-change service/log validation before pilot traffic. +- Treating low-data or cold-start pages as proof that shaping is broken. +- Changing too many things before you have one known-good baseline. -## 7) Related Pages +## 7) Done For Day 1 When + +- You can sign in to WebUI successfully. +- Dashboard loads. +- Scheduler Status is healthy. +- No urgent/fatal startup errors are present. +- The expected topology or devices appear. +- One test subscriber/device behaves as expected. + +## 8) Related Pages - [Operating Modes and Source of Truth](operating-modes.md) - [CRM/NMS Integrations](integrations.md) diff --git a/docs/v2.0/stormguard.md b/docs/v2.0/stormguard.md index be1739ef..2a285867 100644 --- a/docs/v2.0/stormguard.md +++ b/docs/v2.0/stormguard.md @@ -12,6 +12,8 @@ StormGuard is LibreQoS' adaptive queue-adjustment subsystem for congestion and q - Applies bounded adjustments to configured node limits to protect quality under stress. - Exposes status/debug data in the WebUI (Node Manager). +StormGuard adaptive site-speed changes are stored in the StormGuard override layer. They are not written back into `network.json`. + ## Configuration StormGuard is configured in `/etc/lqos.conf` under `[stormguard]`. @@ -41,7 +43,12 @@ If you are testing, start with `dry_run = true` so you can observe decisions bef ## UI and Debugging -- WebUI provides StormGuard status and debug views. +- WebUI provides a dedicated StormGuard dashboard tab plus status and debug views. +- The StormGuard dashboard tab is intended to answer "what is StormGuard doing right now?" with: + - summary cards for watched, cooling-down, and recently changed sites + - a site list that works for single-site and multi-site watched sets + - a selected-site detail panel explaining current limits, last actions, and why StormGuard is holding or changing rates + - a recent activity feed for quick operator triage - The StormGuard debug page shows: - current effective limits - evaluation metrics @@ -71,6 +78,7 @@ Also verify: - target node names still match your current `network.json` hierarchy - integration updates have not renamed key nodes/APs - your minimum percentage floors are reasonable for expected traffic profiles +- `network.json` still reflects your planned/source-of-truth site rates if you are investigating an unexpected StormGuard reduction - `log_file` path (if configured) is writable by the service user ## Related Pages diff --git a/docs/v2.0/treeguard.md b/docs/v2.0/treeguard.md index 505b08fc..cdac9d6c 100644 --- a/docs/v2.0/treeguard.md +++ b/docs/v2.0/treeguard.md @@ -32,11 +32,19 @@ TreeGuard evaluates utilization, RTT freshness, CPU guardrails, and optional QoO High-level behavior: 1. For sustained low-load conditions, TreeGuard may switch a direction from `cake` to `fq_codel`. -2. If utilization rises, RTT/QoO guardrails are unsafe, or revert conditions are met, TreeGuard switches back to `cake`. +2. If utilization rises, QoO guardrails are unsafe, or revert conditions are met, TreeGuard switches back toward the circuit's base SQM policy. 3. Decisions can be independent per direction when `independent_directions = true`. This gives a dynamic profile where loaded directions favor `cake diffserv4`, while low-load directions can use `fq_codel` when conditions are safe. +The base SQM policy comes from operator intent, not from TreeGuard defaults. In practice, TreeGuard starts from the effective configured policy for each circuit and only persists its own temporary overlay when it needs to differ from that base. + +Important base-policy rule: + +1. If a direction's base SQM policy is `cake`, TreeGuard may temporarily switch that direction to `fq_codel` and later return it to base. +2. If a direction's base SQM policy is `fq_codel`, TreeGuard does not circuit-switch that direction to `cake`. +3. Link virtualization remains available regardless of the circuit SQM base policy. + ## Configuration (`/etc/lqos.conf`) TreeGuard config lives under `[treeguard]` and sub-sections: @@ -55,6 +63,11 @@ enabled = true dry_run = false tick_seconds = 1 +[treeguard.cpu] +mode = "cpu_aware" +cpu_high_pct = 75 +cpu_low_pct = 55 + [treeguard.links] enabled = true all_nodes = true @@ -70,6 +83,11 @@ independent_directions = true enabled = true ``` +TreeGuard node virtualization is intended to be CPU-aware by default. Traffic, RTT, and QoO +remain important safety and restore signals, but new automatic virtualization should happen when +CPU pressure suggests HTB savings are worthwhile. Upgraded installs from older defaults are +silently migrated from `traffic_rtt_only` to `cpu_aware`, with a visible notice in logs/UI. + ## Safe Rollout Pattern 1. Review TreeGuard settings early in deployment instead of assuming static/manual queue behavior. @@ -80,12 +98,52 @@ enabled = true ## Overrides and Operational Notes -When enabled and not dry-run, TreeGuard may persist decisions to: +When enabled and not dry-run, TreeGuard may persist circuit SQM decisions to: - `lqos_overrides.treeguard.json` TreeGuard is designed to avoid fighting operator-owned overrides. If operator overrides exist for enrolled entities, TreeGuard skips those entities and reports warnings. +TreeGuard virtual-node decisions are runtime-only Bakery operations. They are not materialized back +into the base `network.json` by the scheduler, and they are not persisted as TreeGuard-owned +`set_node_virtual` entries in the effective shaping input. In v1 they are ephemeral: a daemon +restart returns the physical tree to the base operator-defined topology until TreeGuard decides +again. + +For local runtime verification and debugging, `liblqos_python` now exposes both the current +TreeGuard node-operation status and a Bakery runtime branch-state snapshot. The branch-state +snapshot is the authoritative control-plane view of which retained branch is active for a node, and +is more reliable than inferring parent changes from `tc` output alone for non-top-level cases. +The local confidence workflow also now uses a larger synthetic Bakery TreeGuard fixture for scale +checks: 8 top-level nodes, 3 levels deep, and 1,000 circuits attached only at the lowest level. +When traffic-backed verification is enabled, the runtime verifier now defaults to 10 tracked +circuits per successful case instead of the earlier smaller smoke-test default. + +TreeGuard circuit SQM decisions are also runtime overrides. The scheduler does not materialize TreeGuard-owned SQM changes back into the base `ShapedDevices.csv`, so clearing TreeGuard does not permanently rewrite operator-authored circuit SQM policy. + +TreeGuard also refuses to manage nodes that are already marked `"virtual": true` in the base `network.json`. If stale legacy TreeGuard-owned node-virtualization overrides exist for those nodes, TreeGuard clears that legacy override state and falls back to the base topology definition. + +For circuit SQM management, TreeGuard treats duplicate `device_id` values as unsafe identity collisions. If the same `device_id` appears in more than one circuit in `ShapedDevices.csv`, TreeGuard skips those affected circuits and clears any TreeGuard-owned SQM overrides for those duplicate device IDs. + +If RTT telemetry is temporarily unavailable after a restart, TreeGuard does not treat missing RTT alone as evidence that it should revert `fq_codel` directions. Other guardrails such as utilization, QoO, and CPU pressure still apply. + +TreeGuard also applies a conservative global per-tick circuit SQM change budget. On very large enrolled populations, excess circuit SQM changes are deferred to later ticks instead of stampeding Bakery in one pass. + +For scale, TreeGuard no longer rebuilds circuit membership from `ShapedDevices.csv` on every circuit tick. It now keeps a cached per-circuit inventory derived from `ShapedDevices.csv`, reads per-circuit live telemetry from the shared once-per-second circuit rollup snapshot, and spreads large `all_circuits = true` SQM evaluations across multiple ticks instead of rescanning every enrolled circuit every second. + +In practice, this means: + +1. Link virtualization still follows the normal TreeGuard tick cadence. +2. Circuit SQM evaluation for small enrollments still completes quickly. +3. Very large `all_circuits` enrollments are swept incrementally over multiple ticks, with a target full sweep around 15 seconds instead of attempting a full per-second scan. +4. TreeGuard node virtualization now goes through Bakery live runtime planning/apply paths instead of forcing a LibreQoS reload or Bakery full reload. +5. Supported top-level runtime virtualization now uses a Bakery-side rebalance/migration plan that can promote child sites and direct circuits across queue roots while preserving the logical hierarchy for reporting. + +Recent TreeGuard activity is available in two places: + +- The WebUI TreeGuard status/activity views. +- The `lqosd` journal, where TreeGuard now logs each recorded activity event so reloads, override cleanup, SQM changes, and failures are diagnosable without websocket inspection. + ## Related Pages - [HTB + fq_codel + CAKE: Detailed Queueing Behavior](htb_fq_codel_cake.md) diff --git a/docs/v2.0/troubleshooting.md b/docs/v2.0/troubleshooting.md index 634af4ed..0fd896dd 100644 --- a/docs/v2.0/troubleshooting.md +++ b/docs/v2.0/troubleshooting.md @@ -48,13 +48,16 @@ And include: ### User password not working -Delete the lqusers file: +Current builds will: +- migrate older auth files automatically +- redirect `/login.html` to `/first-run.html` when no users exist + +If the correct username/password still fails, first restart `lqosd` and try again: +```bash +sudo systemctl restart lqosd ``` -sudo rm /opt/libreqos/src/lqusers.toml -sudo systemctl restart lqosd lqos_scheduler -``` -Then visit: BOX_IP:9123/index.html -This will allow you to set up the user again from scratch using the WebUI. + +Only remove `lqusers.toml` if you are intentionally resetting access or if the file is corrupt and cannot be repaired. After removing it, restart `lqosd` and open `BOX_IP:9123/login.html`; the WebUI should redirect you to first-run setup automatically. ### No WebUI at x.x.x.x:9123 @@ -129,6 +132,10 @@ sudo journalctl -u lqos_scheduler --since "1 day ago" --no-pager > lqos_sched_lo ``` This exports a log file to lqos_sched_log.txt. You can review this file to see what caused the scheduler to error out. +If the scheduler fails immediately after a restart with a message like `Socket (typically /run/lqos/bus) not found`, that indicates `lqosd` had not finished binding the local bus yet. Current builds wait briefly for bus readiness at scheduler startup instead of crashing immediately, so repeated startup panics after restart should no longer be expected. + +If an integration subprocess fails, current builds keep the scheduler alive, publish a shortened output preview to the scheduler status/error surfaces, and save the full captured output to a timestamped file under `/tmp` such as `lqos_scheduler_uisp_integration_YYYYMMDD_HHMMSS.log`. + ### Scheduler status in WebUI looks unhealthy Recent builds expose scheduler readiness/state in the WebUI (Node Manager). @@ -194,6 +201,21 @@ journalctl -u lqosd --since "10 minutes ago" If still blank under normal traffic, collect recent logs and open an issue. +### Site Map appears blank or slow + +Site Map has one extra dependency beyond normal WebUI data feeds: current builds fetch bbox/bootstrap data and raster tiles from `https://insight.libreqos.com`. + +If Site Map alone is blank or slow: +1. Confirm `lqosd` is healthy. +2. Confirm the box can reach `insight.libreqos.com` from its management network. +3. Wait briefly and reload the page; the browser retries tile requests automatically while cold tiles are being populated upstream. +4. Check recent `lqosd` logs: +``` +journalctl -u lqosd --since "10 minutes ago" +``` + +If the rest of WebUI is healthy but Site Map continues to fail, treat it as a map/tile dependency issue rather than a general scheduler or shaping failure. + ### Virtual node promotion collision (network.json) If LibreQoS.py fails with an error like `Virtual node promotion collision: 'AP_A' already exists at this level.`, you have a `"virtual": true` node whose children get promoted into a parent level where a node with the same name already exists. @@ -234,6 +256,10 @@ WebUI urgent issues include machine-readable codes. Use them to triage quickly. |---|---|---|---| | `MAPPED_CIRCUIT_LIMIT` | Bakery is enforcing a mapped-circuit limit. | Insight license status, `journalctl -u lqosd` for requested/allowed/dropped counts. | Reduce mapped circuits immediately or update license/limits. | | `TC_U16_OVERFLOW` | Queue/class minor IDs exceeded the Linux tc u16 range on a CPU queue. | `journalctl -u lqos_scheduler -u lqosd`, topology depth/queue distribution. | Increase queue count and/or simplify/rebalance hierarchy (for example with integration strategy or root promotion changes). | +| `TC_QDISC_CAPACITY` | Planned auto-allocated qdiscs exceed the per-interface safe budget or Bakery's conservative memory-safety preflight before apply. | Estimated per-interface qdisc counts, qdisc-kind breakdown, and memory fields in the urgent issue context, `journalctl -u lqos_scheduler -u lqosd`, `on_a_stick` and `queue_mode` config. | Reduce the planned qdisc load for this run (for example fewer circuits/devices in the test shape) before retrying; do not trust partial apply. | +| `BAKERY_MEMORY_GUARD` | A chunked Bakery full reload was stopped mid-apply because available host memory fell below the safety floor. | `journalctl -u lqosd`, available/total memory in the urgent issue context, and recent Bakery apply progress. | Treat the run as failed, reduce memory pressure or queue footprint, and retry only after the host is stable. | +| `XDP_IP_MAPPING_CAPACITY` | Required IP mappings exceed the current XDP kernel map capacity. | `ShapedDevices.csv` row shape, IPv4/IPv6 mix, one-device-vs-many-device assumptions, `journalctl -u lqos_scheduler -u lqosd`. | Reduce required mappings immediately (for example fewer devices or IPv4-only test shape), or raise kernel map capacity in a coordinated change. | +| `XDP_IP_MAPPING_APPLY_FAILED` | One or more IP mapping inserts failed during apply. | `journalctl -u lqos_scheduler -u lqosd` for summarized failure examples and counts. | Fix the underlying mapping failure, then rerun; do not trust partial shaping. | Operational pattern: 1. Open urgent issue details in WebUI (code/message/context). diff --git a/docs/v2.0/update.md b/docs/v2.0/update.md index 5bb995e2..0e09645e 100644 --- a/docs/v2.0/update.md +++ b/docs/v2.0/update.md @@ -39,10 +39,13 @@ sudo /opt/libreqos/src/systemd_hotfix.sh install sudo reboot ``` +The hotfix installer bootstraps the LibreQoS APT repo at `https://repo.libreqos.com`, installs the patched Noble `systemd` package set, and pins those packages for future updates. + After the reboot, resume the upgrade and restart services: ```bash cd /tmp +wget https://download.libreqos.com/{deb_url_v1_5} sudo apt install ./{deb_url_v1_5} sudo systemctl restart lqosd lqos_scheduler ``` diff --git a/src/LibreQoS.py b/src/LibreQoS.py index cfff8e53..207bc340 100755 --- a/src/LibreQoS.py +++ b/src/LibreQoS.py @@ -36,10 +36,13 @@ from shaping_skip_report import ( from liblqos_python import is_lqosd_alive, clear_ip_mappings, delete_ip_mapping, validate_shaped_devices, \ is_libre_already_running, create_lock_file, free_lock_file, add_ip_mapping, BatchedCommands, \ check_config, sqm, upstream_bandwidth_capacity_download_mbps, upstream_bandwidth_capacity_upload_mbps, \ - interface_a, interface_b, enable_actual_shell_commands, use_bin_packing_to_balance_cpu, monitor_mode_only, \ + interface_a, interface_b, enable_actual_shell_commands, use_bin_packing_to_balance_cpu, queue_mode, \ run_shell_commands_as_sudo, generated_pn_download_mbps, generated_pn_upload_mbps, queues_available_override, \ on_a_stick, get_tree_weights, get_weights, is_network_flat, get_libreqos_directory, enable_insight_topology, \ - is_insight_enabled, \ + is_insight_enabled, scheduler_error, xdp_ip_mapping_capacity, \ + overrides_circuit_adjustments_effective, \ + plan_top_level_cpu_bins, \ + plan_class_identities, \ fast_queues_fq_codel, \ shaping_cpu_count, \ Bakery @@ -51,6 +54,31 @@ except Exception: def submit_urgent_issue(*_args, **_kwargs): return False + +class RefreshFailure(Exception): + pass + + +def report_refresh_failure(code, message, context=None, dedupe_key=None): + logging.error(message) + print("ERROR: " + message) + try: + scheduler_error(message) + except Exception: + pass + try: + submit_urgent_issue( + "LibreQoS", + "Error", + code, + message, + json.dumps(context) if context is not None else None, + dedupe_key, + ) + except Exception: + pass + raise RefreshFailure(message) + R2Q = 10 #MAX_R2Q = 200_000 MAX_R2Q = 60_000 # See https://lartc.vger.kernel.narkive.com/NKaH1ZNG/htb-quantum-of-class-100001-is-small-consider-r2q-change @@ -87,6 +115,145 @@ def get_network_json_path(): # Either insight not enabled, or file doesn't exist return os.path.join(base_dir, "network.json") + +def get_planner_state_path(): + return os.path.join(get_libreqos_directory(), "planner_state.json") + + +def observe_mode_enabled(): + return queue_mode() == "observe" + + +def _load_json_dict(path): + try: + with open(path, "r") as infile: + data = json.load(infile) + if isinstance(data, dict): + return data + except Exception: + pass + return {} + + +def load_planner_state(state_path=None, planner_module=None): + if state_path is None: + state_path = get_planner_state_path() + if planner_module is not None: + try: + state = planner_module.load_state(state_path) + if isinstance(state, dict): + return state + except Exception: + pass + return _load_json_dict(state_path) + + +def save_planner_state(state, state_path=None, planner_module=None): + if state_path is None: + state_path = get_planner_state_path() + if planner_module is not None: + planner_module.save_state(state_path, state) + return + + parent = os.path.dirname(state_path) + if parent: + os.makedirs(parent, exist_ok=True) + temp_path = state_path + ".tmp" + with open(temp_path, "w") as outfile: + json.dump(state, outfile, indent=2, sort_keys=True) + os.replace(temp_path, state_path) + + +def _parse_int_token(value): + if value is None: + return None + try: + if isinstance(value, int): + return value + token = str(value).strip() + if not token: + return None + if token.lower().startswith("0x"): + return int(token, 16) + return int(token) + except Exception: + return None + + +def is_generated_parent_node_name(node_name): + return isinstance(node_name, str) and node_name.startswith("Generated_PN_") + + +def generated_parent_node_queue_key(node_name, queues_available): + if not is_generated_parent_node_name(node_name) or queues_available <= 0: + return None + suffix = _parse_int_token(str(node_name).rsplit("_", 1)[-1]) + if suffix is None or suffix <= 0: + return None + return "CpueQueue" + str((suffix - 1) % queues_available) + + +def planner_circuit_identity_key(circuit): + circuit_id = str(circuit.get("circuitID", "") or "").strip() + if not circuit_id: + raise ValueError("Missing circuitID is unsupported for planner identity") + return circuit_id + + +def load_minor_state_from_queuing_structure(path=None): + if path is None: + path = os.path.join(get_libreqos_directory(), "queuingStructure.json") + data = _load_json_dict(path) + network = data.get("Network") + if not isinstance(network, dict): + return {"sites": {}, "circuits": {}} + + sites = {} + circuits = {} + + def walk(node_map, trail=()): + for node_name, node in sorted(node_map.items()): + if not isinstance(node, dict): + continue + node_path = trail + (node_name,) + site_key = "/".join(node_path) + parent_path = "/".join(trail) + queue = _parse_int_token(node.get("cpuNum")) + class_minor = _parse_int_token(node.get("classMinor")) + class_major = _parse_int_token(node.get("classMajor")) + up_class_major = _parse_int_token(node.get("up_classMajor")) + if queue is not None and class_minor is not None: + sites[site_key] = { + "class_minor": class_minor, + "queue": queue + 1, + "parent_path": parent_path, + "class_major": class_major, + "up_class_major": up_class_major, + } + + if isinstance(node.get("circuits"), list): + for circuit in node.get("circuits", []): + if not isinstance(circuit, dict): + continue + circuit_id = circuit.get("circuitID") + circuit_minor = _parse_int_token(circuit.get("classMinor")) + if queue is None or circuit_id is None or circuit_minor is None: + continue + circuits[str(circuit_id)] = { + "class_minor": circuit_minor, + "queue": queue + 1, + "parent_node": circuit.get("ParentNode", node_name), + "class_major": _parse_int_token(circuit.get("classMajor")), + "up_class_major": _parse_int_token(circuit.get("up_classMajor")), + } + + children = node.get("children") + if isinstance(children, dict): + walk(children, node_path) + + walk(network) + return {"sites": sites, "circuits": circuits} + def calculateR2q(maxRateInMbps): # So we've learned that r2q defaults to 10, and is used to calculate quantum. Quantum is rateInBytes/r2q by # default. This default gives errors at high rates, and tc clamps the quantum to 200000. Setting a high quantum @@ -235,19 +402,23 @@ def validateNetworkAndDevices(): data = json.load(file) # put JSON-data to a variable if data != {}: #Traverse - observedNodes = {} # Will not be used later + observedNodes = set() + duplicateNodes = set() def traverseToVerifyValidity(data): for elem in data: if isinstance(elem, str): if (isinstance(data[elem], dict)) and (elem != 'children'): if elem not in observedNodes: - observedNodes[elem] = {'downloadBandwidthMbps': data[elem]['uploadBandwidthMbps'], 'downloadBandwidthMbps': data[elem]['uploadBandwidthMbps']} + observedNodes.add(elem) if 'children' in data[elem]: traverseToVerifyValidity(data[elem]['children']) else: - warnings.warn("Non-unique Node name in network.json: " + elem, stacklevel=2) - networkValidatedOrNot = False + duplicateNodes.add(elem) traverseToVerifyValidity(data) + if len(duplicateNodes) > 0: + for elem in sorted(duplicateNodes): + warnings.warn("Non-unique Node name in network.json: " + elem, stacklevel=2) + networkValidatedOrNot = False if len(observedNodes) < 1: warnings.warn("network.json had 0 valid nodes. Only {} is accepted for that scenario.", stacklevel=2) networkValidatedOrNot = False @@ -282,15 +453,16 @@ def validateNetworkAndDevices(): # And read from the sanitized byte stream with io.StringIO(text_content) as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') - #Remove comments if any - commentsRemoved = [] + header_consumed = False + seenTheseIPsAlready = set() for row in csv_reader: - if not row[0].startswith('#'): - commentsRemoved.append(row) - #Remove header - commentsRemoved.pop(0) - seenTheseIPsAlready = [] - for row in commentsRemoved: + if not row: + continue + if row[0].startswith('#'): + continue + if not header_consumed: + header_consumed = True + continue # Accept optional 14th column 'sqm' but ignore here (validation focuses on core fields) circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row[0:13] # Must have circuitID, it's a unique identifier required for stateful changes to queue structure @@ -311,14 +483,14 @@ def validateNetworkAndDevices(): if ipEntry in seenTheseIPsAlready: warnings.warn("Provided IPv4 '" + ipEntry + "' in ShapedDevices.csv at row " + str(rowNum) + " is duplicate.", stacklevel=2) #devicesValidatedOrNot = False - seenTheseIPsAlready.append(ipEntry) + seenTheseIPsAlready.add(ipEntry) else: if (type(ipaddress.ip_network(ipEntry)) is ipaddress.IPv4Network) or (type(ipaddress.ip_address(ipEntry)) is ipaddress.IPv4Address): ipv4_subnets_and_hosts.extend(ipEntry) else: warnings.warn("Provided IPv4 '" + ipEntry + "' in ShapedDevices.csv at row " + str(rowNum) + " is not valid.", stacklevel=2) devicesValidatedOrNot = False - seenTheseIPsAlready.append(ipEntry) + seenTheseIPsAlready.add(ipEntry) except: warnings.warn("Provided IPv4 '" + ipv4_input + "' in ShapedDevices.csv at row " + str(rowNum) + " is not valid.", stacklevel=2) devicesValidatedOrNot = False @@ -333,14 +505,14 @@ def validateNetworkAndDevices(): if ipEntry in seenTheseIPsAlready: warnings.warn("Provided IPv6 '" + ipEntry + "' in ShapedDevices.csv at row " + str(rowNum) + " is duplicate.", stacklevel=2) devicesValidatedOrNot = False - seenTheseIPsAlready.append(ipEntry) + seenTheseIPsAlready.add(ipEntry) else: if (type(ipaddress.ip_network(ipEntry)) is ipaddress.IPv6Network) or (type(ipaddress.ip_address(ipEntry)) is ipaddress.IPv6Address): ipv6_subnets_and_hosts.extend(ipEntry) else: warnings.warn("Provided IPv6 '" + ipEntry + "' in ShapedDevices.csv at row " + str(rowNum) + " is not valid.", stacklevel=2) devicesValidatedOrNot = False - seenTheseIPsAlready.append(ipEntry) + seenTheseIPsAlready.add(ipEntry) except: warnings.warn("Provided IPv6 '" + ipv6_input + "' in ShapedDevices.csv at row " + str(rowNum) + " is not valid.", stacklevel=2) devicesValidatedOrNot = False @@ -404,19 +576,20 @@ def validateNetworkAndDevices(): def loadSubscriberCircuits(shapedDevicesFile): # Load Subscriber Circuits & Devices subscriberCircuits = [] - knownCircuitIDs = [] + circuitsById = {} counterForCircuitsWithoutParentNodes = 0 dictForCircuitsWithoutParentNodes = {} with open(shapedDevicesFile) as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') - # Remove comments if any - commentsRemoved = [] + header_consumed = False for row in csv_reader: - if not row[0].startswith('#'): - commentsRemoved.append(row) - # Remove header - commentsRemoved.pop(0) - for row in commentsRemoved: + if not row: + continue + if row[0].startswith('#'): + continue + if not header_consumed: + header_consumed = True + continue # Optional per-circuit SQM override in last column sqm_override_token = '' if len(row) > 13: @@ -430,12 +603,6 @@ def loadSubscriberCircuits(shapedDevicesFile): token = left + '/' + right sqm_override_token = token circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row[0:13] - # If in monitorOnlyMode, override bandwidth rates to where no shaping will actually occur - if monitor_mode_only() == True: - downloadMin = 10000 - uploadMin = 10000 - downloadMax = 10000 - uploadMax = 10000 ipv4_subnets_and_hosts = [] # Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s separated by commas. Split them up and parse each if ipv4_input != "": @@ -458,41 +625,37 @@ def loadSubscriberCircuits(shapedDevicesFile): # If there is something in the circuit ID field if circuitID != "": # Seen circuit before - if circuitID in knownCircuitIDs: - for circuit in subscriberCircuits: - if circuit['circuitID'] == circuitID: - if circuit['ParentNode'] != "none": - if circuit['ParentNode'] != ParentNode: - errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID - raise ValueError(errorMessageString) - # Check if bandwidth parameters match other cdevices of this same circuit ID, but only check if monitorOnlyMode is Off - if monitor_mode_only() == False: - if ((circuit['minDownload'] != float(downloadMin)) - or (circuit['minUpload'] != float(uploadMin)) - or (circuit['maxDownload'] != float(downloadMax)) - or (circuit['maxUpload'] != float(uploadMax))): - warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different bandwidth parameters than other devices on this circuit. Will instead use the bandwidth parameters defined by the first device added to its circuit.", stacklevel=2) - # If this row specifies an SQM override, but the circuit already has a different one, warn and keep the first. - if sqm_override_token != '': - if 'sqm' in circuit: - if circuit['sqm'] != sqm_override_token: - warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different SQM override than other devices on this circuit. Will instead use the SQM defined by the first device added to its circuit.", stacklevel=2) - else: - circuit['sqm'] = sqm_override_token - devicesListForCircuit = circuit['devices'] - thisDevice = { - "deviceID": deviceID, - "deviceName": deviceName, - "mac": mac, - "ipv4s": ipv4_subnets_and_hosts, - "ipv6s": ipv6_subnets_and_hosts, - "comment": comment - } - devicesListForCircuit.append(thisDevice) - circuit['devices'] = devicesListForCircuit + circuit = circuitsById.get(circuitID) + if circuit is not None: + if circuit['ParentNode'] != "none": + if circuit['ParentNode'] != ParentNode: + errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID + raise ValueError(errorMessageString) + if ((circuit['minDownload'] != float(downloadMin)) + or (circuit['minUpload'] != float(uploadMin)) + or (circuit['maxDownload'] != float(downloadMax)) + or (circuit['maxUpload'] != float(uploadMax))): + warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different bandwidth parameters than other devices on this circuit. Will instead use the bandwidth parameters defined by the first device added to its circuit.", stacklevel=2) + # If this row specifies an SQM override, but the circuit already has a different one, warn and keep the first. + if sqm_override_token != '': + if 'sqm' in circuit: + if circuit['sqm'] != sqm_override_token: + warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different SQM override than other devices on this circuit. Will instead use the SQM defined by the first device added to its circuit.", stacklevel=2) + else: + circuit['sqm'] = sqm_override_token + devicesListForCircuit = circuit['devices'] + thisDevice = { + "deviceID": deviceID, + "deviceName": deviceName, + "mac": mac, + "ipv4s": ipv4_subnets_and_hosts, + "ipv6s": ipv6_subnets_and_hosts, + "comment": comment + } + devicesListForCircuit.append(thisDevice) + circuit['devices'] = devicesListForCircuit # Have not seen circuit before else: - knownCircuitIDs.append(circuitID) if ParentNode == "": ParentNode = "none" #ParentNode = ParentNode.strip() @@ -525,57 +688,92 @@ def loadSubscriberCircuits(shapedDevicesFile): dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((float(downloadMax))+(float(uploadMax))) counterForCircuitsWithoutParentNodes += 1 subscriberCircuits.append(thisCircuit) - # If there is nothing in the circuit ID field + circuitsById[circuitID] = thisCircuit else: - # Copy deviceName to circuitName if none defined already - if circuitName == "": - circuitName = deviceName - if ParentNode == "": - ParentNode = "none" - #ParentNode = ParentNode.strip() - deviceListForCircuit = [] - thisDevice = { - "deviceID": deviceID, - "deviceName": deviceName, - "mac": mac, - "ipv4s": ipv4_subnets_and_hosts, - "ipv6s": ipv6_subnets_and_hosts, - } - deviceListForCircuit.append(thisDevice) - thisCircuit = { - "circuitID": circuitID, - "circuitName": circuitName, - "ParentNode": ParentNode, - "devices": deviceListForCircuit, - "minDownload": float(downloadMin), - "minUpload": float(uploadMin), - "maxDownload": float(downloadMax), - "maxUpload": float(uploadMax), - "classid": '', - "comment": comment - } - if sqm_override_token != '': - thisCircuit['sqm'] = sqm_override_token - if thisCircuit['ParentNode'] == 'none': - thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes - dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((float(downloadMax))+(float(uploadMax))) - counterForCircuitsWithoutParentNodes += 1 - subscriberCircuits.append(thisCircuit) + raise ValueError( + "Missing circuitID is unsupported in ShapedDevices.csv " + f"(deviceID={deviceID}, deviceName={deviceName}, parent={ParentNode})" + ) return (subscriberCircuits, dictForCircuitsWithoutParentNodes) + +def normalize_sqm_override_token(raw_token): + token = (raw_token or '').strip().lower() + if token == '': + return '' + if '/' in token: + left, right = token.split('/', 1) + token = left.strip() + '/' + right.strip() + return token + + +def apply_effective_runtime_circuit_overrides(subscriberCircuits): + """ + Overlay adaptive runtime circuit adjustments in memory without mutating + ShapedDevices.csv, so rebuilds can honor TreeGuard/StormGuard state. + """ + try: + adjustments = overrides_circuit_adjustments_effective() + except Exception as e: + warnings.warn(f"Unable to load effective runtime circuit overrides: {e}", stacklevel=2) + return 0 + + sqm_by_device_id = {} + for adj in adjustments: + if adj.get('type') != 'device_adjust_sqm': + continue + device_id = (adj.get('device_id') or '').strip() + sqm_override = normalize_sqm_override_token(adj.get('sqm_override')) + if device_id == '' or sqm_override == '': + continue + sqm_by_device_id[device_id] = sqm_override + + if not sqm_by_device_id: + return 0 + + overlay_count = 0 + for circuit in subscriberCircuits: + circuit_override = None + for device in circuit.get('devices', []): + override = sqm_by_device_id.get(device.get('deviceID', '')) + if override is None: + continue + if circuit_override is not None and circuit_override != override: + warnings.warn( + "Effective runtime SQM override conflict on circuit " + + circuit.get('circuitID', 'unknown') + + ". Will instead use the first runtime SQM override discovered.", + stacklevel=2, + ) + continue + circuit_override = override + + if circuit_override is None: + continue + if circuit.get('sqm') != circuit_override: + overlay_count += 1 + circuit['sqm'] = circuit_override + + return overlay_count + def refreshShapers(): # Starting print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + observe_mode = observe_mode_enabled() # Create a single batch of xdp update commands to execute together ipMapBatch = BatchedCommands() + requiredIpMappings = 0 # Warn user if enableActualShellCommands is False, because that would mean no actual commands are executing if enable_actual_shell_commands() == False: warnings.warn("enableActualShellCommands is set to False. None of the commands below will actually be executed. Simulated run.", stacklevel=2) - # Warn user if monitorOnlyMode is True, because that would mean no actual shaping is happening - if monitor_mode_only() == True: - warnings.warn("monitorOnlyMode is set to True. Shaping will not occur.", stacklevel=2) + if observe_mode: + warnings.warn( + "queue_mode is set to observe. LibreQoS will keep the root MQ and remove the shaping tree for a true baseline. " + "Switching between Observe and Shape may briefly interrupt traffic.", + stacklevel=2, + ) # Check if first run since boot @@ -609,6 +807,13 @@ def refreshShapers(): # Load Subscriber Circuits & Devices subscriberCircuits, dictForCircuitsWithoutParentNodes = loadSubscriberCircuits(shapedDevicesFile) + runtime_override_count = apply_effective_runtime_circuit_overrides(subscriberCircuits) + if runtime_override_count > 0: + print( + "Applied " + + str(runtime_override_count) + + " effective runtime circuit SQM override(s) in memory" + ) # Preserve the logical parent (as configured in ShapedDevices.csv) before any shaping-time rewrites. for circuit in subscriberCircuits: @@ -732,27 +937,12 @@ def refreshShapers(): queuesAvailable = math.floor(queuesAvailable / 2) stickOffset = queuesAvailable - # If in monitorOnlyMode, override network.json bandwidth rates to where no shaping will actually occur - if monitor_mode_only() == True: - def overrideNetworkBandwidths(data): - for elem in data: - if 'children' in data[elem]: - overrideNetworkBandwidths(data[elem]['children']) - data[elem]['downloadBandwidthMbpsMin'] = 100000 - data[elem]['uploadBandwidthMbpsMin'] = 100000 - overrideNetworkBandwidths(network) - # Generate Parent Nodes. Spread ShapedDevices.csv which lack defined ParentNode across these (balance across CPUs) print("Generating parent nodes") generatedPNs = [] numberOfGeneratedPNs = queuesAvailable - # If in monitorOnlyMode, override bandwidth rates to where no shaping will actually occur - if monitor_mode_only() == True: - chosenDownloadMbps = 10000 - chosenUploadMbps = 10000 - else: - chosenDownloadMbps = generated_pn_download_mbps() - chosenUploadMbps = generated_pn_upload_mbps() + chosenDownloadMbps = generated_pn_download_mbps() + chosenUploadMbps = generated_pn_upload_mbps() for x in range(numberOfGeneratedPNs): genPNname = "Generated_PN_" + str(x+1) network[genPNname] = { @@ -807,10 +997,10 @@ def refreshShapers(): except ImportError: bin_planner = None # Store planner state directly in lqos_directory (no hidden subdirs) - state_path = os.path.join(get_libreqos_directory(), "planner_state.json") + state_path = get_planner_state_path() state = {} if bin_planner is not None: - state = bin_planner.load_state(state_path) + state = load_planner_state(state_path, bin_planner) now_ts = time.time() prev_assign = {} last_change_ts = {} @@ -834,7 +1024,7 @@ def refreshShapers(): "salt": state.get("salt", "default_salt") if isinstance(state, dict) else "default_salt", "last_change_ts_by_item": last_change_ts, } - if monitor_mode_only() == True: + if observe_mode: params["move_budget_per_run"] = 0 if bin_planner is not None: @@ -876,7 +1066,7 @@ def refreshShapers(): state["assignments"][iid] = b try: print(f"Saving planner state to {state_path} (generated PNs)") - bin_planner.save_state(state_path, state) + save_planner_state(state, state_path, bin_planner) except Exception as e: warnings.warn(f"Failed to save planner state at {state_path}: {e}", stacklevel=2) @@ -1031,26 +1221,34 @@ def refreshShapers(): print("Parsing network structure and tallying devices") parentNodes = [] minorByCPUpreloaded = {} - knownClassIDs = [] nodes_requiring_min_squashing = {} - # Persisted circuit minor assignments for stability across reloads - circuit_state_from_disk = {} - circuit_state_updated = {} - used_minors_by_queue = {cpu + 1: set() for cpu in range(queuesAvailable)} # Track minor counter by CPU. This way we can have > 32000 hosts (htb has u16 limit to minor handle) # Minor numbers start at 3 to reserve 1 for root qdisc and 2 for default class # With CIRCUIT_PADDING, we leave gaps between nodes to allow future circuit additions # without disrupting existing ClassID assignments. This maintains stability across reloads. for x in range(queuesAvailable): minorByCPUpreloaded[x+1] = 3 - def traverseNetwork(data, depth, major, minorByCPU, queue, parentClassID, upParentClassID, parentMaxDL, parentMaxUL, parentMinDL, parentMinUL): - # ClassID Assignment Strategy: - # - Nodes and circuits are processed in alphabetical order for stability - # - Each node gets a unique minor number that increments sequentially - # - After processing all circuits for a node, we add CIRCUIT_PADDING to the minor counter - # - This creates gaps that allow adding new circuits without affecting other ClassIDs - # - Children are also sorted before recursive processing to ensure deterministic traversal - # For top-level binpacked keys (CpueQueueN), enforce numeric ordering of N + def report_minor_overflow(queue, minor): + msg = f"Minor class ID overflow on CPU {queue}: {minor} exceeds TC's u16 limit (65535). Consider increasing queue count or restructuring network hierarchy." + logging.error(msg) + try: + ctx = json.dumps({"cpu": queue, "minor": minor}) + submit_urgent_issue("LibreQoS", "Error", "TC_U16_OVERFLOW", msg, ctx, f"TC_U16_OVERFLOW_CPU_{queue}") + except Exception: + pass + raise ValueError(msg) + + def ensure_minor_capacity(queue, minor): + if minor > 0xFFFF: + report_minor_overflow(queue, minor) + + def next_free_minor(start_minor, reserved): + candidate = max(3, start_minor) + while candidate in reserved: + candidate += 1 + return candidate + + def sorted_node_keys(data, depth): keys = list(data.keys()) if depth == 0 and len(keys) > 0 and all(k.startswith("CpueQueue") for k in keys): try: @@ -1059,227 +1257,7 @@ def refreshShapers(): keys = sorted(keys) else: keys = sorted(keys) - for node in keys: - #if data[node]['type'] == "virtual": - # print(node + " is a virtual node. Skipping.") - # if depth == 0: - # parentClassID = hex(major) + ':' - # upParentClassID = hex(major+stickOffset) + ':' - # data[node]['parentClassID'] = parentClassID - # data[node]['up_parentClassID'] = upParentClassID - # data[node]['classMajor'] = hex(major) - # data[node]['up_classMajor'] = hex(major + stickOffset) - # data[node]['classMinor'] = hex(minorByCPU[queue]) - # data[node]['cpuNum'] = hex(queue-1) - # data[node]['up_cpuNum'] = hex(queue-1+stickOffset) - # traverseNetwork(data[node]['children'], depth, major, minorByCPU, queue, parentClassID, upParentClassID, parentMaxDL, parentMaxUL) - # continue - circuitsForThisNetworkNode = [] - nodeClassID = hex(major) + ':' + hex(minorByCPU[queue]) - upNodeClassID = hex(major+stickOffset) + ':' + hex(minorByCPU[queue]) - data[node]['classid'] = nodeClassID - data[node]['up_classid'] = upNodeClassID - if depth == 0: - parentClassID = hex(major) + ':' - upParentClassID = hex(major+stickOffset) + ':' - data[node]['parentClassID'] = parentClassID - data[node]['up_parentClassID'] = upParentClassID - # If in monitorOnlyMode, override bandwidth rates to where no shaping will actually occur - if monitor_mode_only() == True: - data[node]['downloadBandwidthMbps'] = 100000 - data[node]['uploadBandwidthMbps'] = 100000 - data[node]['downloadBandwidthMbpsMin'] = 100000 - data[node]['uploadBandwidthMbpsMin'] = 100000 - # If not in monitorOnlyMode - else: - # Cap based on this node's max bandwidth, or parent node's max bandwidth, whichever is lower - data[node]['downloadBandwidthMbps'] = min(data[node]['downloadBandwidthMbps'],parentMaxDL) - data[node]['uploadBandwidthMbps'] = min(data[node]['uploadBandwidthMbps'],parentMaxUL) - data[node]['downloadBandwidthMbpsMin'] = min(data[node]['downloadBandwidthMbpsMin'], data[node]['downloadBandwidthMbps'], parentMinDL) - data[node]['uploadBandwidthMbpsMin'] = min(data[node]['uploadBandwidthMbpsMin'], data[node]['uploadBandwidthMbps'], parentMinUL) - # Calculations are done in findBandwidthMins() to determine optimal HTB rates (mins) and ceils (maxs) - data[node]['classMajor'] = hex(major) - data[node]['up_classMajor'] = hex(major + stickOffset) - data[node]['classMinor'] = hex(minorByCPU[queue]) - data[node]['cpuNum'] = hex(queue-1) - data[node]['up_cpuNum'] = hex(queue-1+stickOffset) - used_minors_by_queue.setdefault(queue, set()).add(minorByCPU[queue]) - thisParentNode = { - "parentNodeName": node, - "classID": nodeClassID, - "maxDownload": data[node]['downloadBandwidthMbps'], - "maxUpload": data[node]['uploadBandwidthMbps'], - } - parentNodes.append(thisParentNode) - minorByCPU[queue] = minorByCPU[queue] + 1 - # Check for overflow - TC uses u16 for minor class ID (max 65535) - if minorByCPU[queue] > 0xFFFF: - msg = f"Minor class ID overflow on CPU {queue}: {minorByCPU[queue]} exceeds TC's u16 limit (65535). Consider increasing queue count or restructuring network hierarchy." - logging.error(msg) - try: - ctx = json.dumps({"cpu": queue, "minor": minorByCPU[queue]}) - submit_urgent_issue("LibreQoS", "Error", "TC_U16_OVERFLOW", msg, ctx, f"TC_U16_OVERFLOW_CPU_{queue}") - except Exception: - pass - raise ValueError(f"Minor class ID overflow on CPU {queue}: {minorByCPU[queue]} exceeds limit of 65535") - # If a device from ShapedDevices.csv lists this node as its Parent Node, attach it as a leaf to this node HTB - if node in circuits_by_parent_node: - # If mins of circuits combined exceed min of parent node - set to 1 - override_min_down = None - override_min_up = None - if monitor_mode_only() == False: - if (circuit_min_down_combined_by_parent_node[node] > data[node]['downloadBandwidthMbpsMin']) or (circuit_min_up_combined_by_parent_node[node] > data[node]['uploadBandwidthMbpsMin']): - override_min_down = 1 - override_min_up = 1 - logging.info("The combined minimums of circuits in Parent Node [" + node + "] exceeded that of the parent node. Reducing these circuits' minimums to 1 now.", stacklevel=2) - if ((override_min_down * len(circuits_by_parent_node[node])) > data[node]['downloadBandwidthMbpsMin']) or ((override_min_up * len(circuits_by_parent_node[node])) > data[node]['uploadBandwidthMbpsMin']): - logging.info("Even with this change, minimums will exceed the min rate of the parent node. Using 10 kbps as the minimum for these circuits instead.", stacklevel=2) - nodes_requiring_min_squashing[node] = True - # Sort circuits by name for stable ordering - sorted_circuits = sorted(circuits_by_parent_node[node], - key=lambda c: c.get('circuitName', c.get('circuitID', ''))) - for circuit in sorted_circuits: - if node == circuit['ParentNode']: - if monitor_mode_only() == False: - if circuit['maxDownload'] > data[node]['downloadBandwidthMbps']: - logging.info("downloadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2) - if circuit['maxUpload'] > data[node]['uploadBandwidthMbps']: - logging.info("uploadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2) - parentString = hex(major) + ':' - # Attempt to reuse a stored minor for stability (only if parent/bin unchanged and not colliding) - reuse_minor = None - try: - circuit_id_str = str(circuit.get('circuitID')) - stored_entry = circuit_state_from_disk.get(circuit_id_str, {}) if isinstance(circuit_state_from_disk, dict) else {} - stored_minor = stored_entry.get("class_minor") - stored_queue = stored_entry.get("queue") - stored_parent = stored_entry.get("parent_node") - if stored_minor is not None and stored_queue is not None and stored_parent is not None: - try: - stored_minor = int(stored_minor) - except Exception: - stored_minor = None - try: - stored_queue = int(stored_queue) - except Exception: - stored_queue = None - if stored_minor is not None and stored_queue == queue and stored_parent == circuit['ParentNode']: - if stored_minor not in used_minors_by_queue.setdefault(queue, set()): - reuse_minor = stored_minor - except Exception: - pass - - candidate_minor = reuse_minor if reuse_minor is not None else minorByCPU[queue] - reserved = used_minors_by_queue.setdefault(queue, set()) - while candidate_minor in reserved: - candidate_minor += 1 - - flowIDstring = hex(major) + ':' + hex(candidate_minor) - upFlowIDstring = hex(major + stickOffset) + ':' + hex(candidate_minor) - circuit['classid'] = flowIDstring - circuit['up_classid'] = upFlowIDstring - logging.info("Added up_classid to circuit: " + circuit['up_classid']) - # Create circuit dictionary to be added to network structure, eventually output as queuingStructure.json - maxDownload = min(circuit['maxDownload'],data[node]['downloadBandwidthMbps']) - maxUpload = min(circuit['maxUpload'],data[node]['uploadBandwidthMbps']) - if override_min_down: - circuit['minDownload'] = 1 - if override_min_up: - circuit['minUpload'] = 1 - minDownload = min(circuit['minDownload'],maxDownload) - minUpload = min(circuit['minUpload'],maxUpload) - thisNewCircuitItemForNetwork = { - 'maxDownload' : maxDownload, - 'maxUpload' : maxUpload, - 'minDownload' : minDownload, - 'minUpload' : minUpload, - "circuitID": circuit['circuitID'], - "circuitName": circuit['circuitName'], - "ParentNode": circuit['ParentNode'], - "logicalParentNode": circuit.get('logicalParentNode', circuit['ParentNode']), - "devices": circuit['devices'], - "classid": flowIDstring, - "up_classid" : upFlowIDstring, - "classMajor": hex(major), - "up_classMajor" : hex(major + stickOffset), - "classMinor": hex(candidate_minor), - "comment": circuit['comment'] - } - # Attach the planner weight used by the planner/UI summary - # Priority: explicit weight -> fallback to maxDownload - try: - cid = str(circuit.get('circuitID','')) - w = None - if cid in weight_by_circuit_id: - w = float(weight_by_circuit_id[cid]) - if w is None: - w = float(maxDownload) - # Treat 1000 as a sentinel default from Insight; use maxDownload instead - if abs(w - 1000.0) < 1e-6: - w = float(maxDownload) - # If the circuit's configured rate equals the generated PN default, - # ignore it for weight purposes. - try: - default_rate = float(generated_pn_download_mbps()) - if abs(float(maxDownload) - default_rate) < 1e-6: - w = 0.0 - except Exception: - pass - thisNewCircuitItemForNetwork['planner_weight'] = w - except Exception: - pass - # Preserve optional per-circuit SQM override for downstream bakery call - if 'sqm' in circuit and circuit['sqm']: - thisNewCircuitItemForNetwork['sqm'] = circuit['sqm'] - # Generate TC commands to be executed later - thisNewCircuitItemForNetwork['devices'] = circuit['devices'] - circuitsForThisNetworkNode.append(thisNewCircuitItemForNetwork) - reserved.add(candidate_minor) - minorByCPU[queue] = max(minorByCPU[queue], candidate_minor) + 1 - try: - # Record the assigned minor for future stability - circuit_state_updated[str(circuit.get('circuitID'))] = { - "class_minor": candidate_minor, - "queue": queue, - "parent_node": circuit['ParentNode'], - "class_major": major, - "up_class_major": major + stickOffset, - } - except Exception: - pass - if len(circuitsForThisNetworkNode) > 0: - data[node]['circuits'] = circuitsForThisNetworkNode - - # Add padding for future circuit additions (applies to all nodes) - # This ensures space is reserved even for nodes without circuits - minorByCPU[queue] = minorByCPU[queue] + CIRCUIT_PADDING - - # Recursive call this function for children nodes attached to this node - if 'children' in data[node]: - # Sort children to ensure consistent traversal order - sorted_children = dict(sorted(data[node]['children'].items())) - # We need to keep tabs on the minor counter, because we can't have repeating class IDs. Here, we bring back the minor counter from the recursive function - minorByCPU[queue] = minorByCPU[queue] + 1 - # Check for overflow - TC uses u16 for minor class ID (max 65535) - if minorByCPU[queue] > 0xFFFF: - msg = f"Minor class ID overflow on CPU {queue}: {minorByCPU[queue]} exceeds TC's u16 limit (65535). Consider increasing queue count or restructuring network hierarchy." - logging.error(msg) - try: - ctx = json.dumps({"cpu": queue, "minor": minorByCPU[queue]}) - submit_urgent_issue("LibreQoS", "Error", "TC_U16_OVERFLOW", msg, ctx, f"TC_U16_OVERFLOW_CPU_{queue}") - except Exception: - pass - raise ValueError(f"Minor class ID overflow on CPU {queue}: {minorByCPU[queue]} exceeds limit of 65535") - minorByCPU = traverseNetwork(sorted_children, depth+1, major, minorByCPU, queue, nodeClassID, upNodeClassID, data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps'], data[node]['downloadBandwidthMbpsMin'], data[node]['uploadBandwidthMbpsMin']) - # If top level node, increment to next queue / cpu core - if depth == 0: - if queue >= queuesAvailable: - queue = 1 - major = queue - else: - queue += 1 - major += 1 - return minorByCPU + return keys # If we're in binpacking mode, we need to sort the network structure a bit if use_bin_packing_to_balance_cpu() and not is_network_flat(): @@ -1318,17 +1296,9 @@ def refreshShapers(): except Exception: pass - def round_robin_assign(items_list, bins): - if not bins: - return {}, [] - names = [str(it["id"]) for it in items_list] - names.sort() - assignment_local = {} - for idx, name in enumerate(names): - assignment_local[name] = bins[idx % len(bins)] - return assignment_local, list(assignment_local.keys()) - for node in network: + if is_generated_parent_node_name(node): + continue w = weight_by_name.get(str(node), 1.0) try: w = float(w) @@ -1340,138 +1310,79 @@ def refreshShapers(): w = 1.0 items.append({"id": str(node), "weight": w}) - def greedy_assign(items_list, bins): - bin_loads = {key: 0.0 for key in bins} - pairs = [(str(it["id"]), float(it["weight"])) for it in items_list] - pairs.sort(key=lambda nw: (-nw[1], nw[0])) - assignment_local = {} - for name, wgt in pairs: - target = min(bin_loads.items(), key=lambda kv: (kv[1], kv[0]))[0] - assignment_local[name] = target - bin_loads[target] += wgt - return assignment_local, list(assignment_local.keys()) - - # Prepare bins and capacities cpu_keys = ["CpueQueue" + str(cpu) for cpu in range(queuesAvailable)] - bins_list = [{"id": key} for key in cpu_keys] - capacities = {key: 1.0 for key in cpu_keys} - valid_bins = set(capacities.keys()) + valid_bins = set(cpu_keys) planner_used = False - state_path = os.path.join(get_libreqos_directory(), "planner_state.json") + state_path = get_planner_state_path() state = {} now_ts = time.time() assignment = {} changed = [] - bin_planner = None + try: + state = load_planner_state(state_path, None) + except Exception: + state = {} - # Load planner state and generate assignments when Insight is enabled/licensed - if insight_enabled: - try: - import bin_planner as bin_planner # type: ignore - except ImportError: - bin_planner = None + prev_assign = {} + last_change_ts = {} + if isinstance(state, dict): + prev_assign = state.get("assignments", {}) or {} + last_change_ts = state.get("last_change_ts", {}) or {} + item_ids = {str(it["id"]) for it in items} + prev_assign = { + iid: b for iid, b in prev_assign.items() if iid in item_ids and b in valid_bins + } + last_change_ts = {iid: last_change_ts.get(iid, 0.0) for iid in item_ids} - if bin_planner is not None: - try: - state = bin_planner.load_state(state_path) - except Exception: - state = {} - prev_assign = {} - last_change_ts = {} - if isinstance(state, dict): - prev_assign = state.get("assignments", {}) or {} - last_change_ts = state.get("last_change_ts", {}) or {} - item_ids = {str(it["id"]) for it in items} - prev_assign = { - iid: b for iid, b in prev_assign.items() if iid in item_ids and b in valid_bins - } - last_change_ts = {iid: last_change_ts.get(iid, 0.0) for iid in item_ids} + move_budget = max(1, min(32, int(0.01 * max(1, len(items))))) + if observe_mode: + move_budget = 0 - params = { - "candidate_set_size": 4, - "headroom": 0.05, - "alpha": 0.1, - "hysteresis_threshold": 0.03, - "cooldown_seconds": 3600, - "move_budget_per_run": max(1, min(32, int(0.01 * max(1, len(items))))), - "salt": state.get("salt", "default_salt") if isinstance(state, dict) else "default_salt", - "last_change_ts_by_item": last_change_ts, - } - if monitor_mode_only() == True: - params["move_budget_per_run"] = 0 - - try: - assignment, changed = bin_planner.plan_assignments( - items, bins_list, capacities, prev_assign, now_ts, params - ) - planner_used = True - except Exception as e: - warnings.warn( - f"Planner failed ({e}); falling back to greedy distribution.", - stacklevel=2, - ) - assignment, changed = greedy_assign(items, cpu_keys) - planner_used = False - else: - warnings.warn( - "Binpacking requested, but planner module is unavailable; using greedy distribution.", - stacklevel=2, - ) - assignment, changed = greedy_assign(items, cpu_keys) - planner_used = False - else: - # Insight is unavailable; use deterministic round-robin distribution. - assignment, changed = round_robin_assign(items, cpu_keys) + planner_mode = "stable_greedy" if insight_enabled else "round_robin" + try: + plan_result = plan_top_level_cpu_bins( + items, + queuesAvailable, + prev_assign=prev_assign, + last_change_ts=last_change_ts, + now_ts=now_ts, + mode=planner_mode, + move_budget_per_run=move_budget, + cooldown_seconds=3600.0, + hysteresis_threshold=0.03, + ) + assignment = dict(plan_result.get("assignment", {}) or {}) + changed = list(plan_result.get("changed", []) or []) + planner_used = bool(plan_result.get("planner_used", False)) + except Exception as e: + warnings.warn( + f"Shared Rust planner failed ({e}); falling back to deterministic local assignment.", + stacklevel=2, + ) + assignment = {} + names = sorted(str(it["id"]) for it in items) + if cpu_keys: + for idx, name in enumerate(names): + if insight_enabled: + assignment[name] = cpu_keys[idx % len(cpu_keys)] + else: + assignment[name] = cpu_keys[idx % len(cpu_keys)] + changed = list(assignment.keys()) planner_used = False - # Validate assignment covers all items and uses known bins (planner may return partial results) - try: - if not isinstance(assignment, dict): - assignment, changed = greedy_assign(items, cpu_keys) - except Exception: - assignment, changed = greedy_assign(items, cpu_keys) - try: - item_ids = {str(it["id"]) for it in items} - rr_fallback, _ = round_robin_assign(items, cpu_keys) - for iid in item_ids: - tgt = assignment.get(iid) - if tgt not in valid_bins: - assignment[iid] = rr_fallback.get( - iid, cpu_keys[-1] if cpu_keys else "CpueQueue0" - ) - except Exception: - pass - - # Sanity check: if planner returns a degenerate assignment (everything in one bin), - # fall back to a simple greedy distribution so the physical tree can spread. - if planner_used: - try: - # Exclude Generated_PN_* placeholders from the degeneracy test: they may not - # appear in the UI tree and can mask a collapsed real topology. - interesting_ids = { - str(it["id"]) - for it in items - if not str(it["id"]).startswith("Generated_PN_") - } - used = { - tgt - for iid, tgt in assignment.items() - if iid in interesting_ids and tgt in valid_bins - } - if queuesAvailable > 1 and len(interesting_ids) > 1 and len(used) <= 1: - warnings.warn( - "Planner produced degenerate CPU binning (single-bin result); falling back to greedy distribution.", - stacklevel=2, - ) - assignment, changed = greedy_assign(items, cpu_keys) - planner_used = False - except Exception: - pass + resolved_assignment = {} + for node in network: + tgt = generated_parent_node_queue_key(node, queuesAvailable) + if tgt is None: + tgt = assignment.get(node) + if tgt is None: + tgt = "CpueQueue" + str(queuesAvailable - 1) + resolved_assignment[str(node)] = tgt for x in range(queuesAvailable): key = "CpueQueue" + str(x) - assigned = [name for name, tgt in assignment.items() if tgt == key] + assigned = [name for name, tgt in resolved_assignment.items() if tgt == key] print("Bin " + str(x) + " = ", assigned) # Build the binned network structure @@ -1488,74 +1399,325 @@ def refreshShapers(): 'name': cpuKey } for node in network: - tgt = assignment.get(node) + tgt = resolved_assignment.get(str(node)) if tgt is None: tgt = "CpueQueue" + str(queuesAvailable - 1) binnedNetwork[tgt]['children'][node] = network[node] network = binnedNetwork # Update and save state - if planner_used and bin_planner is not None and isinstance(state, dict): + if planner_used and isinstance(state, dict): if state.get("salt") is None: state["salt"] = "default_salt" if "assignments" not in state or not isinstance(state["assignments"], dict): state["assignments"] = {} if "last_change_ts" not in state or not isinstance(state["last_change_ts"], dict): state["last_change_ts"] = {} + stale_generated = [ + iid for iid in list(state["assignments"].keys()) + if is_generated_parent_node_name(iid) + ] + for iid in stale_generated: + state["assignments"].pop(iid, None) + state["last_change_ts"].pop(iid, None) for iid, b in assignment.items(): if iid in changed: state["last_change_ts"][iid] = now_ts state["assignments"][iid] = b try: print(f"Saving planner state to {state_path} (top-level CPU binning)") - bin_planner.save_state(state_path, state) + save_planner_state(state, state_path, None) except Exception as e: warnings.warn( f"Failed to save planner state at {state_path}: {e}", stacklevel=2 ) - # Seed persisted circuit minor assignments (TTL already enforced in bin_planner.load_state()). + # Seed persisted site/circuit minor assignments. When planner state is absent, + # fall back to the previous queuing structure so the first run after an upgrade + # can preserve existing class IDs. try: state # noqa: B018 except NameError: state = {} - try: - import bin_planner # noqa: F401 - except ImportError: - bin_planner = None - if bin_planner is not None and (not isinstance(state, dict) or len(state.keys()) == 0): - try: - state_path = os.path.join(get_libreqos_directory(), "planner_state.json") - state = bin_planner.load_state(state_path) - except Exception: - state = {} + state_path = get_planner_state_path() + if not isinstance(state, dict) or len(state.keys()) == 0: + state = load_planner_state(state_path, None) try: circuit_state_from_disk = state.get("circuits", {}) if isinstance(state, dict) else {} except Exception: circuit_state_from_disk = {} + try: + site_state_from_disk = state.get("sites", {}) if isinstance(state, dict) else {} + except Exception: + site_state_from_disk = {} + if not isinstance(circuit_state_from_disk, dict): + circuit_state_from_disk = {} + if not isinstance(site_state_from_disk, dict): + site_state_from_disk = {} + if not circuit_state_from_disk or not site_state_from_disk: + fallback_minor_state = load_minor_state_from_queuing_structure() + if not site_state_from_disk: + site_state_from_disk = fallback_minor_state.get("sites", {}) or {} + if not circuit_state_from_disk: + circuit_state_from_disk = fallback_minor_state.get("circuits", {}) or {} circuit_state_updated = {} - used_minors_by_queue = {cpu + 1: set() for cpu in range(queuesAvailable)} + site_state_updated = {} + planner_site_inputs = [] + planner_circuit_groups = [] - # Here is the actual call to the recursive traverseNetwork() function. finalMinor is not used. - minorByCPU = traverseNetwork(network, 0, major=1, minorByCPU=minorByCPUpreloaded, queue=1, parentClassID=None, upParentClassID=None, parentMaxDL=upstream_bandwidth_capacity_download_mbps(), parentMaxUL=upstream_bandwidth_capacity_upload_mbps(), parentMinDL=upstream_bandwidth_capacity_download_mbps(), parentMinUL=upstream_bandwidth_capacity_upload_mbps()) + def collect_identity_planner_inputs(data, depth, queue, path=()): + for node in sorted_node_keys(data, depth): + current_queue = queue + node_path = path + (node,) + parent_path = '/'.join(path) + has_children = bool(data[node].get('children')) + planner_site_inputs.append( + { + "site_key": '/'.join(node_path), + "parent_path": parent_path, + "queue": current_queue, + "has_children": has_children, + } + ) + if node in circuits_by_parent_node: + sorted_circuits = sorted( + circuits_by_parent_node[node], + key=lambda c: c.get('circuitName', c.get('circuitID', '')), + ) + planner_circuit_groups.append( + { + "parent_node": node, + "queue": current_queue, + "circuit_ids": [ + planner_circuit_identity_key(circuit) + for circuit in sorted_circuits + if node == circuit['ParentNode'] + ], + } + ) - # Persist the updated circuit minor map for future stability - if 'bin_planner' not in locals(): - bin_planner = None - if bin_planner is not None and isinstance(state, dict): - if 'circuits' not in state or not isinstance(state.get('circuits'), dict): - state['circuits'] = {} - state['circuits'] = circuit_state_updated - try: - state_path = os.path.join(get_libreqos_directory(), "planner_state.json") - except Exception: - state_path = None - if state_path: - try: - print(f"Saving planner state to {state_path} (circuit minors)") - bin_planner.save_state(state_path, state) - except Exception as e: - warnings.warn(f"Failed to save planner circuit state at {state_path}: {e}", stacklevel=2) + if has_children: + sorted_children = dict(sorted(data[node]['children'].items())) + collect_identity_planner_inputs( + sorted_children, + depth + 1, + current_queue, + node_path, + ) + + if depth == 0: + if queue >= queuesAvailable: + queue = 1 + else: + queue += 1 + + collect_identity_planner_inputs(network, 0, 1) + + identity_plan = plan_class_identities( + planner_site_inputs, + planner_circuit_groups, + site_state=site_state_from_disk, + circuit_state=circuit_state_from_disk, + stick_offset=stickOffset, + circuit_padding=CIRCUIT_PADDING, + ) + site_assignment_by_key = { + entry["site_key"]: entry for entry in identity_plan.get("sites", []) + } + circuit_assignment_by_key = { + entry["circuit_id"]: entry for entry in identity_plan.get("circuits", []) + } + site_state_updated = identity_plan.get("site_state", {}) or {} + circuit_state_updated = identity_plan.get("circuit_state", {}) or {} + + def apply_site_assignments(data, depth, queue, parentClassID, upParentClassID, parentMaxDL, parentMaxUL, parentMinDL, parentMinUL, path=()): + for node in sorted_node_keys(data, depth): + current_queue = queue + node_path = path + (node,) + site_key = '/'.join(node_path) + assignment = site_assignment_by_key.get(site_key) + if assignment is None: + raise ValueError(f"Missing planned site identity for {site_key}") + assigned_site_minor = int(assignment["class_minor"]) + major = int(assignment["class_major"]) + up_major = int(assignment["up_class_major"]) + ensure_minor_capacity(current_queue, assigned_site_minor) + nodeClassID = hex(major) + ':' + hex(assigned_site_minor) + upNodeClassID = hex(up_major) + ':' + hex(assigned_site_minor) + data[node]['classid'] = nodeClassID + data[node]['up_classid'] = upNodeClassID + current_parent_classid = parentClassID + current_up_parent_classid = upParentClassID + if depth == 0: + current_parent_classid = hex(major) + ':' + current_up_parent_classid = hex(up_major) + ':' + data[node]['parentClassID'] = current_parent_classid + data[node]['up_parentClassID'] = current_up_parent_classid + data[node]['downloadBandwidthMbps'] = min(data[node]['downloadBandwidthMbps'], parentMaxDL) + data[node]['uploadBandwidthMbps'] = min(data[node]['uploadBandwidthMbps'], parentMaxUL) + data[node]['downloadBandwidthMbpsMin'] = min(data[node]['downloadBandwidthMbpsMin'], data[node]['downloadBandwidthMbps'], parentMinDL) + data[node]['uploadBandwidthMbpsMin'] = min(data[node]['uploadBandwidthMbpsMin'], data[node]['uploadBandwidthMbps'], parentMinUL) + data[node]['classMajor'] = hex(major) + data[node]['up_classMajor'] = hex(up_major) + data[node]['classMinor'] = hex(assigned_site_minor) + data[node]['cpuNum'] = hex(current_queue-1) + data[node]['up_cpuNum'] = hex(current_queue-1+stickOffset) + parentNodes.append( + { + "parentNodeName": node, + "classID": nodeClassID, + "maxDownload": data[node]['downloadBandwidthMbps'], + "maxUpload": data[node]['uploadBandwidthMbps'], + } + ) + + if 'children' in data[node]: + sorted_children = dict(sorted(data[node]['children'].items())) + apply_site_assignments( + sorted_children, + depth+1, + current_queue, + nodeClassID, + upNodeClassID, + data[node]['downloadBandwidthMbps'], + data[node]['uploadBandwidthMbps'], + data[node]['downloadBandwidthMbpsMin'], + data[node]['uploadBandwidthMbpsMin'], + node_path, + ) + + if depth == 0: + if queue >= queuesAvailable: + queue = 1 + else: + queue += 1 + + apply_site_assignments( + network, + 0, + queue=1, + parentClassID=None, + upParentClassID=None, + parentMaxDL=upstream_bandwidth_capacity_download_mbps(), + parentMaxUL=upstream_bandwidth_capacity_upload_mbps(), + parentMinDL=upstream_bandwidth_capacity_download_mbps(), + parentMinUL=upstream_bandwidth_capacity_upload_mbps(), + ) + + def attach_circuits(data, depth, path=()): + for node in sorted_node_keys(data, depth): + node_data = data[node] + queue_token = _parse_int_token(node_data.get('cpuNum')) + major = _parse_int_token(node_data.get('classMajor')) + if queue_token is None or major is None: + continue + queue = queue_token + 1 + circuitsForThisNetworkNode = [] + if node in circuits_by_parent_node: + override_min_down = None + override_min_up = None + if (circuit_min_down_combined_by_parent_node[node] > node_data['downloadBandwidthMbpsMin']) or (circuit_min_up_combined_by_parent_node[node] > node_data['uploadBandwidthMbpsMin']): + override_min_down = 1 + override_min_up = 1 + logging.info("The combined minimums of circuits in Parent Node [" + node + "] exceeded that of the parent node. Reducing these circuits' minimums to 1 now.", stacklevel=2) + if ((override_min_down * len(circuits_by_parent_node[node])) > node_data['downloadBandwidthMbpsMin']) or ((override_min_up * len(circuits_by_parent_node[node])) > node_data['uploadBandwidthMbpsMin']): + logging.info("Even with this change, minimums will exceed the min rate of the parent node. Using 10 kbps as the minimum for these circuits instead.", stacklevel=2) + nodes_requiring_min_squashing[node] = True + sorted_circuits = sorted(circuits_by_parent_node[node], key=lambda c: c.get('circuitName', c.get('circuitID', ''))) + for circuit in sorted_circuits: + if node != circuit['ParentNode']: + continue + if circuit['maxDownload'] > node_data['downloadBandwidthMbps']: + logging.info("downloadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2) + if circuit['maxUpload'] > node_data['uploadBandwidthMbps']: + logging.info("uploadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2) + planner_key = planner_circuit_identity_key(circuit) + planned_identity = circuit_assignment_by_key.get(planner_key) + if planned_identity is None: + raise ValueError(f"Missing planned circuit identity for {planner_key}") + candidate_minor = int(planned_identity["class_minor"]) + major = int(planned_identity["class_major"]) + up_major = int(planned_identity["up_class_major"]) + ensure_minor_capacity(queue, candidate_minor) + flowIDstring = hex(major) + ':' + hex(candidate_minor) + upFlowIDstring = hex(up_major) + ':' + hex(candidate_minor) + circuit['classid'] = flowIDstring + circuit['up_classid'] = upFlowIDstring + logging.info("Added up_classid to circuit: " + circuit['up_classid']) + maxDownload = min(circuit['maxDownload'], node_data['downloadBandwidthMbps']) + maxUpload = min(circuit['maxUpload'], node_data['uploadBandwidthMbps']) + if override_min_down: + circuit['minDownload'] = 1 + if override_min_up: + circuit['minUpload'] = 1 + minDownload = min(circuit['minDownload'], maxDownload) + minUpload = min(circuit['minUpload'], maxUpload) + thisNewCircuitItemForNetwork = { + 'maxDownload': maxDownload, + 'maxUpload': maxUpload, + 'minDownload': minDownload, + 'minUpload': minUpload, + "circuitID": circuit['circuitID'], + "circuitName": circuit['circuitName'], + "ParentNode": circuit['ParentNode'], + "logicalParentNode": circuit.get('logicalParentNode', circuit['ParentNode']), + "devices": circuit['devices'], + "classid": flowIDstring, + "up_classid": upFlowIDstring, + "classMajor": hex(major), + "up_classMajor": hex(up_major), + "classMinor": hex(candidate_minor), + "comment": circuit['comment'], + } + try: + cid = str(circuit.get('circuitID', '')) + w = None + if cid in weight_by_circuit_id: + w = float(weight_by_circuit_id[cid]) + if w is None: + w = float(maxDownload) + if abs(w - 1000.0) < 1e-6: + w = float(maxDownload) + try: + default_rate = float(generated_pn_download_mbps()) + if abs(float(maxDownload) - default_rate) < 1e-6: + w = 0.0 + except Exception: + pass + thisNewCircuitItemForNetwork['planner_weight'] = w + except Exception: + pass + if 'sqm' in circuit and circuit['sqm']: + thisNewCircuitItemForNetwork['sqm'] = circuit['sqm'] + thisNewCircuitItemForNetwork['devices'] = circuit['devices'] + circuitsForThisNetworkNode.append(thisNewCircuitItemForNetwork) + + if len(circuitsForThisNetworkNode) > 0: + node_data['circuits'] = circuitsForThisNetworkNode + else: + node_data.pop('circuits', None) + + if 'children' in node_data: + sorted_children = dict(sorted(node_data['children'].items())) + attach_circuits(sorted_children, depth+1, path + (node,)) + + attach_circuits(network, 0) + minorByCPU = { + int(queue): int(minor) + for queue, minor in (identity_plan.get("last_used_minor_by_queue", {}) or {}).items() + } + for cpu in range(queuesAvailable): + minorByCPU.setdefault(cpu + 1, 3) + + if not isinstance(state, dict): + state = {} + state['circuits'] = circuit_state_updated + state['sites'] = site_state_updated + try: + print(f"Saving planner state to {state_path} (circuit/site minors)") + save_planner_state(state, state_path, None) + except Exception as e: + warnings.warn(f"Failed to save planner circuit state at {state_path}: {e}", stacklevel=2) bakery = Bakery() bakery.start_batch() # Initializes the bakery transaction @@ -1611,6 +1773,7 @@ def refreshShapers(): # Define lists for hash filters print("Preparing TC commands") def traverseNetwork(data): + nonlocal requiredIpMappings # Cake needs help handling rates lower than 5 Mbps def sqmFixupRate(rate:int, sqm:str) -> str: @@ -1707,6 +1870,7 @@ def refreshShapers(): sqm_override = circuit['sqm'] if 'sqm' in circuit else None bakery.add_circuit( circuit_name, + node, data[node]['classid'], data[node]['up_classid'], int(circuit['classMinor'], 16), @@ -1721,57 +1885,52 @@ def refreshShapers(): ) command = 'class add dev ' + interface_a() + ' parent ' + data[node]['classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ format_rate_for_tc(min_down) + ' ceil '+ format_rate_for_tc(circuit['maxDownload']) + ' prio 3' + quantum(circuit['maxDownload']) + tcComment linuxTCcommands.append(command) - # Only add CAKE / fq_codel qdisc if monitorOnlyMode is Off - if monitor_mode_only() == False: - # SQM Fixup for lower rates (and per-circuit override) - def effective_sqm_str(rate, override, direction): - base = sqm() - # Resolve per-direction token from override string - chosen = None - if override: - try: - ov = str(override).strip().lower() - if '/' in ov: - left, right = ov.split('/', 1) - left = left.strip() - right = right.strip() - chosen = left if direction == 'down' else right - else: - chosen = ov - except Exception: - chosen = None - # If no explicit token for this direction, use default behavior - if not chosen: - try: - thresh = fast_queues_fq_codel() - except Exception: - thresh = 1000.0 - if rate >= thresh: - return 'fq_codel' - return sqmFixupRate(rate, base) - if chosen == 'none': - return '' - if chosen == 'fq_codel': + # SQM Fixup for lower rates (and per-circuit override) + def effective_sqm_str(rate, override, direction): + base = sqm() + # Resolve per-direction token from override string + chosen = None + if override: + try: + ov = str(override).strip().lower() + if '/' in ov: + left, right = ov.split('/', 1) + left = left.strip() + right = right.strip() + chosen = left if direction == 'down' else right + else: + chosen = ov + except Exception: + chosen = None + # If no explicit token for this direction, use default behavior + if not chosen: + try: + thresh = fast_queues_fq_codel() + except Exception: + thresh = 1000.0 + if rate >= thresh: return 'fq_codel' - if chosen == 'cake': - cake_base = base if base.startswith('cake') else 'cake diffserv4' - return sqmFixupRate(rate, cake_base) return sqmFixupRate(rate, base) - sqm_override = circuit['sqm'] if 'sqm' in circuit else None - useSqm = effective_sqm_str(circuit['maxDownload'], sqm_override, 'down') - if useSqm != '': - command = 'qdisc add dev ' + interface_a() + ' parent ' + circuit['classMajor'] + ':' + circuit['classMinor'] + ' ' + useSqm - linuxTCcommands.append(command) + if chosen == 'none': + return '' + if chosen == 'fq_codel': + return 'fq_codel' + if chosen == 'cake': + cake_base = base if base.startswith('cake') else 'cake diffserv4' + return sqmFixupRate(rate, cake_base) + return sqmFixupRate(rate, base) + sqm_override = circuit['sqm'] if 'sqm' in circuit else None + useSqm = effective_sqm_str(circuit['maxDownload'], sqm_override, 'down') + if useSqm != '': + command = 'qdisc add dev ' + interface_a() + ' parent ' + circuit['classMajor'] + ':' + circuit['classMinor'] + ' ' + useSqm + linuxTCcommands.append(command) command = 'class add dev ' + interface_b() + ' parent ' + data[node]['up_classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ format_rate_for_tc(min_up) + ' ceil '+ format_rate_for_tc(circuit['maxUpload']) + ' prio 3' + quantum(circuit['maxUpload']) linuxTCcommands.append(command) - # Only add CAKE / fq_codel qdisc if monitorOnlyMode is Off - if monitor_mode_only() == False: - # SQM Fixup for lower rates (and per-circuit override) - sqm_override = circuit['sqm'] if 'sqm' in circuit else None - useSqm = effective_sqm_str(circuit['maxUpload'], sqm_override, 'up') - if useSqm != '': - command = 'qdisc add dev ' + interface_b() + ' parent ' + circuit['up_classMajor'] + ':' + circuit['classMinor'] + ' ' + useSqm - linuxTCcommands.append(command) + sqm_override = circuit['sqm'] if 'sqm' in circuit else None + useSqm = effective_sqm_str(circuit['maxUpload'], sqm_override, 'up') + if useSqm != '': + command = 'qdisc add dev ' + interface_b() + ' parent ' + circuit['up_classMajor'] + ':' + circuit['classMinor'] + ' ' + useSqm + linuxTCcommands.append(command) for device in circuit['devices']: if device['ipv4s']: for ipv4 in device['ipv4s']: @@ -1783,6 +1942,7 @@ def refreshShapers(): circuit.get('circuitID', ''), device.get('deviceID', ''), ) + requiredIpMappings += 1 #xdpCPUmapCommands.append('./bin/xdp_iphash_to_cpu_cmdline add --ip ' + str(ipv4) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['classid']) if on_a_stick(): ipMapBatch.add_ip_mapping( @@ -1804,6 +1964,7 @@ def refreshShapers(): circuit.get('circuitID', ''), device.get('deviceID', ''), ) + requiredIpMappings += 1 #xdpCPUmapCommands.append('./bin/xdp_iphash_to_cpu_cmdline add --ip ' + str(ipv6) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['classid']) if on_a_stick(): ipMapBatch.add_ip_mapping( @@ -1824,6 +1985,59 @@ def refreshShapers(): # Here is the actual call to the recursive traverseNetwork() function. traverseNetwork(network) + if enable_actual_shell_commands(): + ipMappingCapacity = xdp_ip_mapping_capacity() + print( + "Prepared " + + str(requiredIpMappings) + + " unique XDP IP mappings against capacity " + + str(ipMappingCapacity) + ) + if requiredIpMappings > ipMappingCapacity: + report_refresh_failure( + "XDP_IP_MAPPING_CAPACITY", + "Required XDP IP mappings (" + + str(requiredIpMappings) + + ") exceed current kernel map capacity (" + + str(ipMappingCapacity) + + "). Aborting refresh before apply.", + { + "required_ip_mappings": requiredIpMappings, + "kernel_map_capacity": ipMappingCapacity, + "queued_requests": ipMapBatch.length(), + "on_a_stick": on_a_stick(), + "shaped_devices_file": shapedDevicesFile, + "network_json_file": networkJSONfile, + }, + "XDP_IP_MAPPING_CAPACITY", + ) + + qdiscBudgetEstimate = bakery.estimate_qdisc_budget() + if not qdiscBudgetEstimate["ok"]: + interfaceCounts = qdiscBudgetEstimate["interfaces"] + sortedInterfaces = sorted(interfaceCounts.items()) + preflightSummary = qdiscBudgetEstimate.get("summary") or "" + report_refresh_failure( + "TC_QDISC_CAPACITY", + (preflightSummary + " Aborting refresh before apply.").strip(), + { + "interfaces": dict(sortedInterfaces), + "interface_details": qdiscBudgetEstimate.get("interface_details", {}), + "safe_budget": qdiscBudgetEstimate["safe_budget"], + "hard_limit": qdiscBudgetEstimate["hard_limit"], + "estimated_total_memory_bytes": qdiscBudgetEstimate.get("estimated_total_memory_bytes"), + "memory_ok": qdiscBudgetEstimate.get("memory_ok"), + "memory_guard_min_available_bytes": qdiscBudgetEstimate.get("memory_guard_min_available_bytes"), + "memory_total_bytes": qdiscBudgetEstimate.get("memory_total_bytes"), + "memory_available_bytes": qdiscBudgetEstimate.get("memory_available_bytes"), + "on_a_stick": on_a_stick(), + "queue_mode": queue_mode(), + "shaped_devices_file": shapedDevicesFile, + "network_json_file": networkJSONfile, + }, + "TC_QDISC_CAPACITY", + ) + # Save queuingStructure queuingStructure = {} queuingStructure['Network'] = network @@ -1847,7 +2061,25 @@ def refreshShapers(): # Setup XDP and disable XPS regardless of whether it is first run or not (necessary to handle cases where systemctl stop was used) xdpStartTime = datetime.now() #if enable_actual_shell_commands(): - # The bakery will handle stale mapping cleanup; avoid clearing mappings here. + # Observe mode removes the shaping tree entirely, so clear any live + # per-circuit IP mappings before the commit to avoid pointing packets + # at handles that are about to disappear. + if enable_actual_shell_commands() and observe_mode: + logging.info("# Observe Mode Mapping Quiesce") + try: + clear_ip_mappings() + except Exception as e: + report_refresh_failure( + "XDP_IP_MAPPING_CLEAR_FAILED", + "Failed to clear XDP IP mappings before entering Observe mode: " + str(e), + { + "required_ip_mappings": requiredIpMappings, + "queued_requests": ipMapBatch.length(), + "on_a_stick": on_a_stick(), + "queue_mode": queue_mode(), + }, + "XDP_IP_MAPPING_CLEAR_FAILED", + ) # Set up XDP-CPUMAP-TC logging.info("# XDP Setup") # Commented out - the daemon does this @@ -1863,6 +2095,8 @@ def refreshShapers(): # Execute actual Linux TC commands tcStartTime = datetime.now() # print("Executing linux TC class/qdisc commands") + if observe_mode: + linuxTCcommands = [] with open('linux_tc.txt', 'w') as f: for command in linuxTCcommands: logging.info(command) @@ -1881,8 +2115,24 @@ def refreshShapers(): print("Executing XDP-CPUMAP-TC IP filter commands") numXdpCommands = ipMapBatch.length() if enable_actual_shell_commands(): - ipMapBatch.finish_ip_mappings() - ipMapBatch.submit() + if observe_mode: + print("Observe mode active; skipping XDP-CPUMAP-TC IP filter apply after clearing mappings") + numXdpCommands = 0 + else: + ipMapBatch.finish_ip_mappings() + try: + ipMapBatch.submit() + except Exception as e: + report_refresh_failure( + "XDP_IP_MAPPING_APPLY_FAILED", + "Failed to apply XDP IP mappings: " + str(e), + { + "required_ip_mappings": requiredIpMappings, + "queued_requests": numXdpCommands, + "on_a_stick": on_a_stick(), + }, + "XDP_IP_MAPPING_APPLY_FAILED", + ) #for command in xdpCPUmapCommands: # logging.info(command) # commands = command.split(' ') @@ -2086,26 +2336,33 @@ if __name__ == '__main__': if getattr(args, 'planner_reset', False): try: - state_path = os.path.join(get_libreqos_directory(), "planner_state.json") + state_path = get_planner_state_path() if os.path.exists(state_path): os.remove(state_path) print(f"Removed planner state: {state_path}") except Exception as e: print(f"Warning: could not remove planner state: {e}") - if args.validate: - status = validateNetworkAndDevices() - elif args.clearrules: - tearDown(interface_a(), interface_b()) - elif args.updateonly: - # Single-interface updates don't work at all right now. - if on_a_stick(): - print("--updateonly is not supported for single-interface configurations") - os._exit(-1) - refreshShapersUpdateOnly() - else: - # Refresh and/or set up queues - refreshShapers() + exit_code = 0 + try: + if args.validate: + status = validateNetworkAndDevices() + elif args.clearrules: + tearDown(interface_a(), interface_b()) + elif args.updateonly: + # Single-interface updates don't work at all right now. + if on_a_stick(): + print("--updateonly is not supported for single-interface configurations") + exit_code = -1 + else: + refreshShapersUpdateOnly() + else: + # Refresh and/or set up queues + refreshShapers() + except RefreshFailure: + exit_code = 1 + finally: + free_lock_file() - # Free the lock file - free_lock_file() + if exit_code != 0: + os._exit(exit_code) diff --git a/src/VERSION_STRING b/src/VERSION_STRING index cd5ac039..879b416e 100644 --- a/src/VERSION_STRING +++ b/src/VERSION_STRING @@ -1 +1 @@ -2.0 +2.1 diff --git a/src/bakery_integration_test.py b/src/bakery_integration_test.py index db8a481d..bfbeeaa1 100644 --- a/src/bakery_integration_test.py +++ b/src/bakery_integration_test.py @@ -11,18 +11,20 @@ Important maintenance window. It is NOT intended for production service units under systemd. Manual steps (run in two terminals) -1) Start lqosd manually as root with logs to a file: - sudo RUST_LOG=info lqosd 2>&1 | tee /tmp/lqosd.log +1) Either: + - Start lqosd manually as root with logs to a file: + sudo RUST_LOG=info lqosd 2>&1 | tee /tmp/lqosd.log + and run the harness with `--log-file /tmp/lqosd.log`, or + - Run against the systemd-managed daemon and let the harness read `journalctl -u lqosd`. - Ensure /etc/lqos.conf interfaces and bandwidths are set correctly for this host. - - Stop systemd units first if they are enabled (optional): - sudo systemctl stop lqosd lqos_scheduler lqos_api 2) In another terminal from this directory (LibreQoS/src), run the tests: - python3 bakery_integration_test.py --log-file /tmp/lqosd.log - - You can limit scope with --tiered-only or --flat-only. + python3 bakery_integration_test.py + - The default run is the quicker tiered suite. + - Use --full-suite for the whole harness, or --flat-only/--treeguard-only/etc. for a focused run. What this script does - Backs up `network.json` and `ShapedDevices.csv` in the current directory. -- Writes small tiered and flat fixtures and calls LibreQoS.refreshShapers() after each change. +- Writes focused tiered/flat fixtures plus a larger synthetic TreeGuard runtime fixture and calls LibreQoS.refreshShapers() after each change. - Parses lqosd logs to decide whether a full reload occurred or an incremental update happened. - Restores your original files at the end unless --no-restore is used. @@ -30,14 +32,29 @@ Notes - The first commit after (re)starting lqosd will do a full reload (MQ init). The test allows that and asserts the subsequent expected behaviors. - Flat networks do not have explicit sites, so site add/remove checks are skipped there. +- Newer Bakery builds log incremental completion as `Bakery mapped circuit decision (...)` + rather than only the older `Bakery changes: ...` line. This harness accepts both. +- In `cpu_aware` TreeGuard mode, the live TreeGuard suite may legitimately report a + skip if no CPU pressure is induced on the box during the test window. Usage examples -- Tiered + Flat (default): +- Quick default run (tiered only): + python3 bakery_integration_test.py +- Use a tee'd manual log file: python3 bakery_integration_test.py --log-file /tmp/lqosd.log +- Full suite: + python3 bakery_integration_test.py --full-suite - Tiered only: - python3 bakery_integration_test.py --log-file /tmp/lqosd.log --tiered-only + python3 bakery_integration_test.py --tiered-only +- TreeGuard runtime suite only: + python3 bakery_integration_test.py --treeguard-only --treeguard-timeout 90 +- Fault-injection reload escalation suite only: + python3 bakery_integration_test.py --fault-reload-only + This destructive safety-path test is opt-in and is not included in --full-suite. +- Queue mode toggle suite only: + sudo python3 bakery_integration_test.py --queue-mode-only - Flat only: - python3 bakery_integration_test.py --log-file /tmp/lqosd.log --flat-only + python3 bakery_integration_test.py --flat-only Exit code - 0 on success, 1 on assertion failure or precondition failure. @@ -55,7 +72,8 @@ import shutil import sys import time from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple, Union import subprocess @@ -91,6 +109,14 @@ try: from liblqos_python import on_a_stick # type: ignore except Exception: on_a_stick = None # type: ignore + try: + from liblqos_python import queue_mode # type: ignore + except Exception: + queue_mode = None # type: ignore + try: + from liblqos_python import sync_lqosd_config_from_disk # type: ignore + except Exception: + sync_lqosd_config_from_disk = None # type: ignore except Exception: # Provide a soft fallback if binding is unavailable def is_lqosd_alive() -> bool: @@ -122,7 +148,12 @@ MQ_INIT_PAT = re.compile(r"MQ not created, performing full reload\.") CHANGES_PAT = re.compile( r"Bakery changes: sites_speed=(\d+), circuits_added=(\d+), removed=(\d+), speed=(\d+), ip=(\d+)" ) +MAPPED_DECISION_PAT = re.compile( + r"Bakery mapped circuit decision \(([^)]+)\): requested=(\d+), allowed=(\d+), dropped=(\d+)," +) NO_CHANGES_PAT = re.compile(r"No changes detected in batch, skipping processing\.") +# Test-only fault marker path. +TEST_FAULT_ONCE_PATH = "/tmp/lqos_bakery_fail_purpose_once.txt" # nosec B108 @dataclass @@ -130,46 +161,49 @@ class LogResult: full_reload: bool mq_init: bool changes: Optional[Dict[str, int]] + incremental_event: Optional[str] raw_lines: List[str] +SnapshotToken = Union[int, float] + + class LogReader: def __init__(self, path: str): self.path = path - def snapshot(self) -> int: + def snapshot(self) -> SnapshotToken: try: return os.path.getsize(self.path) except Exception: return 0 - def read_since(self, offset: int) -> List[str]: + def read_since(self, offset: SnapshotToken) -> Tuple[SnapshotToken, List[str]]: lines: List[str] = [] try: with open(self.path, "r", errors="replace") as f: - f.seek(offset) + f.seek(int(offset)) for line in f: lines.append(line.rstrip("\n")) except Exception: pass - return lines + next_offset = int(offset) + sum(len(l) + 1 for l in lines) + return next_offset, lines - def wait_for_events(self, offset: int, timeout_s: float = 20.0) -> LogResult: + def wait_for_events(self, offset: SnapshotToken, timeout_s: float = 20.0) -> LogResult: deadline = time.time() + timeout_s print(f"Waiting for Bakery events (timeout {timeout_s:.1f}s)...") collected: List[str] = [] full_reload = False mq_init = False changes: Optional[Dict[str, int]] = None + incremental_event: Optional[str] = None printed_sleep_note = False while time.time() < deadline: - new_lines = self.read_since(offset) + offset, new_lines = self.read_since(offset) if new_lines: collected.extend(new_lines) - # Seek to last byte each time - offset += sum(len(l) + 1 for l in new_lines) - # Parse new chunk for ln in new_lines: if FULL_RELOAD_PAT.search(ln) or FULL_RELOAD_SUMMARY_PAT.search(ln): @@ -185,6 +219,16 @@ class LogReader: "speed": int(m.group(4)), "ip": int(m.group(5)), } + mapped = MAPPED_DECISION_PAT.search(ln) + if mapped: + incremental_event = mapped.group(1) + changes = { + "requested": int(mapped.group(2)), + "allowed": int(mapped.group(3)), + "dropped": int(mapped.group(4)), + } + if "full reload" in incremental_event.lower(): + full_reload = True # Heuristic: stop when we saw an outcome (changes or full reload or no changes) if changes or full_reload or any(NO_CHANGES_PAT.search(x) for x in new_lines): break @@ -194,7 +238,116 @@ class LogReader: printed_sleep_note = True time.sleep(0.4) - return LogResult(full_reload=full_reload, mq_init=mq_init, changes=changes, raw_lines=collected) + return LogResult( + full_reload=full_reload, + mq_init=mq_init, + changes=changes, + incremental_event=incremental_event, + raw_lines=collected, + ) + + +class JournalctlLogReader: + def __init__(self, unit: str = "lqosd"): + self.unit = unit + + def snapshot(self) -> SnapshotToken: + return time.time() + + def _since_arg(self, token: SnapshotToken) -> str: + return f"@{float(token):.6f}" + + def read_since(self, token: SnapshotToken) -> Tuple[SnapshotToken, List[str]]: + start = float(token) + proc = subprocess.run( + [ + "journalctl", + "-u", + self.unit, + "--since", + self._since_arg(start), + "--no-pager", + "-o", + "short-unix", + ], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + return start, [] + + lines: List[str] = [] + next_token = start + for raw_line in proc.stdout.splitlines(): + line = raw_line.rstrip("\n") + if not line: + continue + lines.append(line) + timestamp_token = line.split(" ", 1)[0].strip() + try: + next_token = max(next_token, float(timestamp_token) + 0.000001) + except Exception: + next_token = max(next_token, time.time()) + + return next_token, lines + + def wait_for_events(self, offset: SnapshotToken, timeout_s: float = 20.0) -> LogResult: + deadline = time.time() + timeout_s + print(f"Waiting for Bakery events from journalctl (timeout {timeout_s:.1f}s)...") + collected: List[str] = [] + full_reload = False + mq_init = False + changes: Optional[Dict[str, int]] = None + incremental_event: Optional[str] = None + printed_sleep_note = False + + while time.time() < deadline: + offset, new_lines = self.read_since(offset) + if new_lines: + collected.extend(new_lines) + for ln in new_lines: + if FULL_RELOAD_PAT.search(ln) or FULL_RELOAD_SUMMARY_PAT.search(ln): + full_reload = True + if MQ_INIT_PAT.search(ln): + mq_init = True + m = CHANGES_PAT.search(ln) + if m: + changes = { + "sites_speed": int(m.group(1)), + "circuits_added": int(m.group(2)), + "removed": int(m.group(3)), + "speed": int(m.group(4)), + "ip": int(m.group(5)), + } + mapped = MAPPED_DECISION_PAT.search(ln) + if mapped: + incremental_event = mapped.group(1) + changes = { + "requested": int(mapped.group(2)), + "allowed": int(mapped.group(3)), + "dropped": int(mapped.group(4)), + } + if "full reload" in incremental_event.lower(): + full_reload = True + if changes or full_reload or any(NO_CHANGES_PAT.search(x) for x in new_lines): + break + + if not printed_sleep_note: + print("No new logs yet; sleeping 0.4s...") + printed_sleep_note = True + time.sleep(0.4) + + return LogResult( + full_reload=full_reload, + mq_init=mq_init, + changes=changes, + incremental_event=incremental_event, + raw_lines=collected, + ) + + +AnyLogReader = Union[LogReader, JournalctlLogReader] # ----------------------- @@ -217,6 +370,140 @@ TIERED_NETWORK_BASE = { } +VIRTUALIZED_TIERED_NETWORK_BASE = { + "Site_A": { + "downloadBandwidthMbps": 120, + "uploadBandwidthMbps": 120, + "type": "Site", + "children": { + "Town_V": { + "downloadBandwidthMbps": 80, + "uploadBandwidthMbps": 80, + "type": "Site", + "virtual": True, + "children": { + "AP_V": { + "downloadBandwidthMbps": 40, + "uploadBandwidthMbps": 40, + "type": "AP", + } + }, + }, + "AP_A": { + "downloadBandwidthMbps": 50, + "uploadBandwidthMbps": 50, + "type": "AP", + }, + }, + }, +} + + +TREEGUARD_RUNTIME_REGION_COUNT = 8 +TREEGUARD_RUNTIME_POPS_PER_REGION = 5 +TREEGUARD_RUNTIME_APS_PER_POP = 5 +TREEGUARD_RUNTIME_CIRCUITS_PER_AP = 5 +TREEGUARD_RUNTIME_TOTAL_CIRCUITS = ( + TREEGUARD_RUNTIME_REGION_COUNT + * TREEGUARD_RUNTIME_POPS_PER_REGION + * TREEGUARD_RUNTIME_APS_PER_POP + * TREEGUARD_RUNTIME_CIRCUITS_PER_AP +) + + +def _treeguard_runtime_region_name(region_index: int) -> str: + return f"REGION_{region_index:02d}" + + +def _treeguard_runtime_pop_name(region_index: int, pop_index: int) -> str: + ordinal = (region_index - 1) * TREEGUARD_RUNTIME_POPS_PER_REGION + pop_index + return f"POP_{ordinal:03d}" + + +def _treeguard_runtime_ap_name(region_index: int, pop_index: int, ap_index: int) -> str: + ordinal = ( + (region_index - 1) * TREEGUARD_RUNTIME_POPS_PER_REGION * TREEGUARD_RUNTIME_APS_PER_POP + + (pop_index - 1) * TREEGUARD_RUNTIME_APS_PER_POP + + ap_index + ) + return f"AP_{ordinal:04d}" + + +def _treeguard_runtime_circuit_ordinal(region_index: int, pop_index: int, ap_index: int, circuit_index: int) -> int: + ap_ordinal = ( + (region_index - 1) * TREEGUARD_RUNTIME_POPS_PER_REGION * TREEGUARD_RUNTIME_APS_PER_POP + + (pop_index - 1) * TREEGUARD_RUNTIME_APS_PER_POP + + ap_index + ) + return (ap_ordinal - 1) * TREEGUARD_RUNTIME_CIRCUITS_PER_AP + circuit_index + + +def _treeguard_runtime_circuit_id(region_index: int, pop_index: int, ap_index: int, circuit_index: int) -> str: + return f"{100000 + _treeguard_runtime_circuit_ordinal(region_index, pop_index, ap_index, circuit_index):06d}" + + +def _treeguard_runtime_circuit_ipv4(circuit_ordinal: int) -> str: + # Keep addresses in CGNAT space while avoiding .0 and .255 octets. + third_octet = 1 + ((circuit_ordinal - 1) // 250) + fourth_octet = 1 + ((circuit_ordinal - 1) % 250) + return f"100.90.{third_octet}.{fourth_octet}" + + +def treeguard_runtime_fixture_metadata() -> dict[str, str | int]: + return { + "virtualized_region": _treeguard_runtime_region_name(1), + "promoted_pop": _treeguard_runtime_pop_name(1, 1), + "promoted_leaf": _treeguard_runtime_ap_name(1, 1, 1), + "promoted_leaf_circuit": _treeguard_runtime_circuit_id(1, 1, 1, 1), + "sibling_pop": _treeguard_runtime_pop_name(1, 2), + "sibling_leaf": _treeguard_runtime_ap_name(1, 2, 1), + "sibling_leaf_circuit": _treeguard_runtime_circuit_id(1, 2, 1, 1), + "low_value_sibling": _treeguard_runtime_region_name(TREEGUARD_RUNTIME_REGION_COUNT), + "total_circuits": TREEGUARD_RUNTIME_TOTAL_CIRCUITS, + } + + +def treeguard_runtime_network_base() -> dict: + network: dict[str, dict] = {} + for region_index in range(1, TREEGUARD_RUNTIME_REGION_COUNT + 1): + region_name = _treeguard_runtime_region_name(region_index) + pop_children: dict[str, dict] = {} + for pop_index in range(1, TREEGUARD_RUNTIME_POPS_PER_REGION + 1): + pop_name = _treeguard_runtime_pop_name(region_index, pop_index) + ap_children: dict[str, dict] = {} + for ap_index in range(1, TREEGUARD_RUNTIME_APS_PER_POP + 1): + ap_name = _treeguard_runtime_ap_name(region_index, pop_index, ap_index) + ap_children[ap_name] = { + "downloadBandwidthMbps": max(12, 60 - region_index - pop_index - ap_index), + "uploadBandwidthMbps": max(12, 40 - region_index - pop_index - ap_index), + "type": "AP", + } + pop_children[pop_name] = { + "downloadBandwidthMbps": max(60, 260 - (region_index * 12) - (pop_index * 8)), + "uploadBandwidthMbps": max(40, 180 - (region_index * 8) - (pop_index * 5)), + "type": "Site", + "children": ap_children, + } + + if region_index == 1: + region_download = 1200 + region_upload = 800 + elif region_index == TREEGUARD_RUNTIME_REGION_COUNT: + region_download = 180 + region_upload = 120 + else: + region_download = max(260, 760 - (region_index * 55)) + region_upload = max(180, 520 - (region_index * 35)) + + network[region_name] = { + "downloadBandwidthMbps": region_download, + "uploadBandwidthMbps": region_upload, + "type": "Site", + "children": pop_children, + } + return network + + def write_network_json(obj: dict) -> None: with open("network.json", "w") as f: json.dump(obj, f, indent=2) @@ -286,6 +573,99 @@ def tiered_circuits_base() -> List[Dict[str, str | int | float]]: ] +def virtualized_tiered_circuits_base() -> List[Dict[str, str | int | float]]: + return [ + { + "Circuit ID": "501", + "Circuit Name": "VIRTUAL_DIRECT", + "Device ID": "501", + "Device Name": "VD1", + "Parent Node": "Town_V", + "MAC": "", + "IPv4": "100.64.2.1", + "IPv6": "", + "Download Min Mbps": 1, + "Upload Min Mbps": 1, + "Download Max Mbps": 20, + "Upload Max Mbps": 10, + "Comment": "", + "sqm": "", + }, + { + "Circuit ID": "502", + "Circuit Name": "VIRTUAL_CHILD", + "Device ID": "502", + "Device Name": "VD2", + "Parent Node": "AP_V", + "MAC": "", + "IPv4": "100.64.2.2", + "IPv6": "", + "Download Min Mbps": 1, + "Upload Min Mbps": 1, + "Download Max Mbps": 15, + "Upload Max Mbps": 8, + "Comment": "", + "sqm": "", + }, + { + "Circuit ID": "503", + "Circuit Name": "SIBLING_REAL", + "Device ID": "503", + "Device Name": "VD3", + "Parent Node": "AP_A", + "MAC": "", + "IPv4": "100.64.2.3", + "IPv6": "", + "Download Min Mbps": 1, + "Upload Min Mbps": 1, + "Download Max Mbps": 12, + "Upload Max Mbps": 6, + "Comment": "", + "sqm": "", + }, + ] + + +def treeguard_runtime_circuits_base() -> List[Dict[str, str | int | float]]: + rows: List[Dict[str, str | int | float]] = [] + for region_index in range(1, TREEGUARD_RUNTIME_REGION_COUNT + 1): + for pop_index in range(1, TREEGUARD_RUNTIME_POPS_PER_REGION + 1): + for ap_index in range(1, TREEGUARD_RUNTIME_APS_PER_POP + 1): + ap_name = _treeguard_runtime_ap_name(region_index, pop_index, ap_index) + for circuit_index in range(1, TREEGUARD_RUNTIME_CIRCUITS_PER_AP + 1): + circuit_ordinal = _treeguard_runtime_circuit_ordinal( + region_index, pop_index, ap_index, circuit_index + ) + circuit_id = _treeguard_runtime_circuit_id( + region_index, pop_index, ap_index, circuit_index + ) + rows.append( + { + "Circuit ID": circuit_id, + "Circuit Name": ( + f"TG_R{region_index:02d}_P{pop_index:02d}_A{ap_index:02d}_C{circuit_index:02d}" + ), + "Device ID": circuit_id, + "Device Name": f"TG-D{circuit_ordinal:04d}", + "Parent Node": ap_name, + "MAC": "", + "IPv4": _treeguard_runtime_circuit_ipv4(circuit_ordinal), + "IPv6": "", + "Download Min Mbps": 1, + "Upload Min Mbps": 1, + "Download Max Mbps": 8 + ((circuit_index + ap_index) % 6), + "Upload Max Mbps": 4 + ((circuit_index + pop_index) % 4), + "Comment": "", + "sqm": "", + } + ) + if len(rows) != TREEGUARD_RUNTIME_TOTAL_CIRCUITS: + raise RuntimeError( + f"treeguard runtime fixture generated {len(rows)} circuits, expected {TREEGUARD_RUNTIME_TOTAL_CIRCUITS}" + ) + return rows + + def flat_network_base() -> dict: # Flat = empty object return {} @@ -458,6 +838,8 @@ def realistic_tiered_circuits_base() -> List[Dict[str, str | int | float]]: max_ul = max(1, max_dl - 1) ip_addr = f"100.64.{third}.{j + 1}" + if j == 0: + ip_addr = f"{ip_addr},100.64.{third}.{j + 101}" row: Dict[str, str | int | float] = { "Circuit ID": str(circuit_id), "Circuit Name": f"CIRCUIT_{circuit_id:04d}", @@ -518,7 +900,7 @@ def realistic_tiered_circuits_base() -> List[Dict[str, str | int | float]]: # ----------------------- -def run_refresh_and_wait(log: LogReader, timeout_s: float) -> LogResult: +def run_refresh_and_wait(log: AnyLogReader, timeout_s: float) -> LogResult: offset = log.snapshot() # Call the core refresh function; this does not require running as __main__ LibreQoS.refreshShapers() @@ -528,6 +910,46 @@ def run_refresh_and_wait(log: LogReader, timeout_s: float) -> LogResult: return log.wait_for_events(offset, timeout_s=timeout_s) +def run_refresh_subprocess_and_wait(log: AnyLogReader, timeout_s: float) -> LogResult: + offset = log.snapshot() + proc = subprocess.run( + [sys.executable, "LibreQoS.py"], + capture_output=True, + text=True, + check=False, + ) + if proc.stdout: + print(proc.stdout, end="" if proc.stdout.endswith("\n") else "\n") + if proc.stderr: + print(proc.stderr, end="" if proc.stderr.endswith("\n") else "\n", file=sys.stderr) + if proc.returncode != 0: + raise RuntimeError(f"queue-mode suite: LibreQoS.py subprocess exited {proc.returncode}") + print("Sleeping 0.5s to allow lqosd to commit and log...") + time.sleep(0.5) + return log.wait_for_events(offset, timeout_s=timeout_s) + + +def settle_initial_bakery_logs(log: AnyLogReader, settle_s: float = 1.5) -> None: + offset = log.snapshot() + time.sleep(settle_s) + _ = log.wait_for_events(offset, timeout_s=0.5) + + +def clear_bakery_fault_once() -> None: + try: + os.remove(TEST_FAULT_ONCE_PATH) + except FileNotFoundError: + pass + + +def arm_bakery_fault_once(selector: str) -> None: + with open(TEST_FAULT_ONCE_PATH, "w") as f: + f.write(selector.strip()) + f.write("\n") + f.flush() + os.fsync(f.fileno()) + + # ----------------------- # TC and structure checks # ----------------------- @@ -646,6 +1068,23 @@ def _hex_to_int(s: str) -> Optional[int]: return None +def _tc_id_token_to_int(tok: str) -> Optional[int]: + """Parse tc class/qdisc ID tokens. + + The live `tc` text output commonly emits bare hex tokens like `10f` rather than + `0x10f`. Treat bare tokens as hex so lookups line up with queuingStructure.json. + """ + s = tok.strip() + if not s: + return None + try: + if s.lower().startswith("0x"): + return int(s, 16) + return int(s, 16) + except Exception: + return None + + def _parse_tc_rate_to_mbps(token: str, unit: Optional[str]) -> float: try: val = float(token) @@ -675,24 +1114,11 @@ def _read_htb_rate_ceil_mbps(iface: str, major: int, minor: int) -> Optional[Tup # Match the class line for this major/minor, accepting decimal or hex forms. cls_pat = re.compile(r"^class\s+htb\s+([0-9A-Fa-fx]+):([0-9A-Fa-fx]+)\b.*$", re.MULTILINE) - def _to_int(tok: str) -> Optional[int]: - s = tok.strip() - # Accept decimal, 0x-prefixed hex, or bare hex (e.g. "f") - try: - if s.lower().startswith("0x"): - return int(s, 16) - try: - return int(s) # decimal - except Exception: - return int(s, 16) # bare hex - except Exception: - return None - line = None for m in cls_pat.finditer(out): mj_tok, mn_tok = m.group(1), m.group(2) - mj = _to_int(mj_tok) - mn = _to_int(mn_tok) + mj = _tc_id_token_to_int(mj_tok) + mn = _tc_id_token_to_int(mn_tok) if mj == major and mn == minor: line = m.group(0) break @@ -711,6 +1137,96 @@ def _qdisc_show(iface: str) -> Tuple[int, str, str]: return _tc(["qdisc", "show", "dev", iface]) +def _read_text(path: str) -> str: + with open(path, "r", encoding="utf-8", errors="replace") as handle: + return handle.read() + + +def _write_text(path: str, text: str) -> None: + with open(path, "w", encoding="utf-8") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + + +def _set_queue_mode_in_lqos_conf_text(config_text: str, mode: str) -> str: + if mode not in {"shape", "observe"}: + raise ValueError(f"unsupported queue mode: {mode}") + + section_pat = re.compile(r"(?ms)^(\[queues\]\n)(.*?)(?=^\[|\Z)") + match = section_pat.search(config_text) + if not match: + raise RuntimeError("queue-mode suite: [queues] section not found in /etc/lqos.conf") + + header = match.group(1) + body = match.group(2) + queue_mode_line = f'queue_mode = "{mode}"\n' + if re.search(r'(?m)^queue_mode\s*=\s*".*?"\s*$', body): + new_body = re.sub( + r'(?m)^queue_mode\s*=\s*".*?"\s*$', + queue_mode_line.rstrip("\n"), + body, + count=1, + ) + if not new_body.endswith("\n"): + new_body += "\n" + else: + new_body = queue_mode_line + body + return config_text[:match.start()] + header + new_body + config_text[match.end():] + + +@dataclass +class TcShapeSummary: + iface: str + root_mq: bool + clsact: bool + root_htb_qdiscs: int + htb_classes: int + child_leaf_qdiscs: int + + +def _summarize_iface_tc(iface: str) -> TcShapeSummary: + qdisc_rc, qdisc_out, qdisc_err = _qdisc_show(iface) + if qdisc_rc != 0: + raise RuntimeError(f"queue-mode suite: tc qdisc show dev {iface} failed: {qdisc_err.strip()}") + + class_rc, class_out, class_err = _tc(["class", "show", "dev", iface]) + if class_rc != 0: + raise RuntimeError(f"queue-mode suite: tc class show dev {iface} failed: {class_err.strip()}") + + root_mq = bool(re.search(r"(?m)^qdisc\s+mq\s+7fff:\s+root\b", qdisc_out)) + clsact = bool(re.search(r"(?m)^qdisc\s+clsact\s+ffff:\s+parent\s+ffff:fff1\b", qdisc_out)) + root_htb_qdiscs = len( + re.findall(r"(?m)^qdisc\s+htb\s+[0-9A-Fa-fx]+:\s+parent\s+7fff:[0-9A-Fa-fx]+\b", qdisc_out) + ) + htb_classes = len(re.findall(r"(?m)^class\s+htb\s+[0-9A-Fa-fx]+:[0-9A-Fa-fx]+\b", class_out)) + child_leaf_qdiscs = len( + re.findall( + r"(?m)^qdisc\s+(?!mq\b)(?!htb\b)(?!clsact\b)\S+\s+\S+:\s+parent\s+(?!ffff:)[0-9A-Fa-fx]+:[0-9A-Fa-fx]+\b", + qdisc_out, + ) + ) + + return TcShapeSummary( + iface=iface, + root_mq=root_mq, + clsact=clsact, + root_htb_qdiscs=root_htb_qdiscs, + htb_classes=htb_classes, + child_leaf_qdiscs=child_leaf_qdiscs, + ) + + +def _shape_summary_message(prefix: str, summary: TcShapeSummary) -> str: + return ( + f"{prefix}: {summary.iface}: " + f"root_mq={summary.root_mq}, clsact={summary.clsact}, " + f"root_htb_qdiscs={summary.root_htb_qdiscs}, " + f"htb_classes={summary.htb_classes}, " + f"child_leaf_qdiscs={summary.child_leaf_qdiscs}" + ) + + def check_no_site_circuit_minor_collisions() -> Tuple[bool, List[str]]: """Verify that no circuit under any node shares the same classMinor as its parent node. @@ -760,6 +1276,65 @@ def check_no_site_circuit_minor_collisions() -> Tuple[bool, List[str]]: return ok, msgs +def snapshot_site_class_assignments() -> Tuple[bool, str, Dict[str, Tuple[str, str, str, str]]]: + """Capture current site class assignment state from queuingStructure.json.""" + qs = _load_queuing_structure() + if not isinstance(qs, dict): + return False, "site snapshot: queuingStructure.json not found", {} + net = qs.get("Network") + if not isinstance(net, dict): + return False, "site snapshot: queuingStructure has no 'Network'", {} + + snapshot: Dict[str, Tuple[str, str, str, str]] = {} + + def walk(node_map: Dict[str, dict], trail: Tuple[str, ...] = ()) -> None: + for name, node in node_map.items(): + if not isinstance(node, dict): + continue + path = "/".join(trail + (name,)) + snapshot[path] = ( + str(node.get("classMinor", "")), + str(node.get("classid", "")), + str(node.get("parentClassID", "")), + str(node.get("cpuNum", "")), + ) + ch = node.get("children") + if isinstance(ch, dict): + walk(ch, trail + (name,)) + + walk(net) + return True, "", snapshot + + +def check_site_class_assignments_unchanged( + before: Dict[str, Tuple[str, str, str, str]] +) -> Tuple[bool, List[str]]: + """Verify that site class assignments did not drift between two snapshots.""" + ok, err, after = snapshot_site_class_assignments() + if not ok: + return False, [err] + + msgs: List[str] = [] + changed = [] + for path in sorted(set(before.keys()) | set(after.keys())): + if before.get(path) != after.get(path): + changed.append((path, before.get(path), after.get(path))) + + if changed: + for path, old_value, new_value in changed[:20]: + msgs.append( + f"site assignment stability: {path} changed {old_value} -> {new_value}" + ) + if len(changed) > 20: + msgs.append( + f"site assignment stability: additional changed sites omitted ({len(changed) - 20})" + ) + return False, msgs + + msgs.append("site assignment stability: all site class assignments unchanged") + return True, msgs + + # ----------------------- # IP mapping checks # ----------------------- @@ -780,8 +1355,13 @@ def _list_ip_mappings() -> Tuple[bool, str, List[Tuple[str, int, str]]]: if "Socket" in out or "Socket" in err: return False, "ipmap check: lqosd bus socket not found; skipping", [] entries: List[Tuple[str, int, str]] = [] - # Lines look like: " CPU: TC: " - pat = re.compile(r"^\s*([0-9A-Fa-f:.]+)/\d+\s+CPU:\s+(\d+)\s+TC:\s+([0-9A-Fa-f]+:[0-9A-Fa-f]+)\s*$") + # Lines look like: + # " CPU: TC: " + # or newer output with trailing identity fields: + # " CPU: TC: CIRCUIT: DEVICE: " + pat = re.compile( + r"^\s*([0-9A-Fa-f:.]+)/\d+\s+CPU:\s+(\d+)\s+TC:\s+([0-9A-Fa-f]+:[0-9A-Fa-f]+)(?:\s+.*)?$" + ) for line in out.splitlines(): m = pat.match(line) if m: @@ -820,6 +1400,208 @@ def _find_parent_node(net: Dict[str, dict], name: str) -> Optional[dict]: return None +def _find_node_path(node_map: Dict[str, dict], target_name: str, prefix: Optional[List[str]] = None) -> Optional[List[str]]: + current_prefix = prefix or [] + for name, node in node_map.items(): + if not isinstance(node, dict): + continue + node_path = current_prefix + [name] + if name == target_name: + return node_path + children = node.get("children") + if isinstance(children, dict): + found = _find_node_path(children, target_name, node_path) + if found is not None: + return found + return None + + +def _walk_nodes_for_circuit_path( + node_map: Dict[str, dict], + circuit_id: str, + prefix: Optional[List[str]] = None, +) -> Optional[Tuple[List[str], dict]]: + current_prefix = prefix or [] + for name, node in node_map.items(): + if not isinstance(node, dict): + continue + node_path = current_prefix + [name] + circuits = node.get("circuits") + if isinstance(circuits, list): + for circuit in circuits: + try: + if str(circuit.get("circuitID", "")) == str(circuit_id): + return node_path, circuit + except Exception: + pass + children = node.get("children") + if isinstance(children, dict): + found = _walk_nodes_for_circuit_path(children, circuit_id, node_path) + if found: + return found + return None + + +def check_virtualized_node_branch_promotion( + virtual_node_name: str, + promoted_child_name: str, + promoted_child_expected_path: List[str], + circuit_expectations: List[Tuple[str, str, List[str]]], + *, + expect_virtual_node_listed: bool = True, +) -> Tuple[bool, List[str]]: + msgs: List[str] = [] + qs = _load_queuing_structure() + if not isinstance(qs, dict): + return False, ["virtualized node check: queuingStructure.json not found"] + net = qs.get("Network") + if not isinstance(net, dict): + return False, ["virtualized node check: queuingStructure has no 'Network'"] + + ok = True + if expect_virtual_node_listed: + virtual_nodes = qs.get("virtual_nodes") + if not isinstance(virtual_nodes, list) or virtual_node_name not in virtual_nodes: + msgs.append(f"virtualized node check: '{virtual_node_name}' missing from queuingStructure virtual_nodes") + ok = False + else: + msgs.append(f"virtualized node check: '{virtual_node_name}' listed in queuingStructure virtual_nodes") + + virtual_node = _find_parent_node(net, virtual_node_name) + if virtual_node is not None: + msgs.append(f"virtualized node check: '{virtual_node_name}' still exists in physical queue tree") + ok = False + else: + msgs.append(f"virtualized node check: '{virtual_node_name}' absent from physical queue tree") + + promoted_child_path = _find_node_path(net, promoted_child_name) + if promoted_child_path is None: + msgs.append(f"virtualized node check: promoted child '{promoted_child_name}' not found") + ok = False + elif promoted_child_path != promoted_child_expected_path: + msgs.append( + "virtualized node check: promoted child " + f"'{promoted_child_name}' found at {'/'.join(promoted_child_path)}, " + f"expected {'/'.join(promoted_child_expected_path)}" + ) + ok = False + else: + msgs.append( + f"virtualized node check: promoted child '{promoted_child_name}' found at {'/'.join(promoted_child_path)}" + ) + + for label, circuit_id, expected_parent_path in circuit_expectations: + found = _walk_nodes_for_circuit_path(net, circuit_id) + if not found: + msgs.append(f"virtualized node check: {label} circuit {circuit_id} not found in structure") + ok = False + continue + actual_parent_path, _ = found + if actual_parent_path != expected_parent_path: + msgs.append( + f"virtualized node check: {label} circuit {circuit_id} attached to " + f"{'/'.join(actual_parent_path)}, expected {'/'.join(expected_parent_path)}" + ) + ok = False + else: + msgs.append( + f"virtualized node check: {label} circuit {circuit_id} attached to {'/'.join(actual_parent_path)}" + ) + + return ok, msgs + + +def check_node_present_in_physical_tree(node_name: str) -> Tuple[bool, List[str]]: + qs = _load_queuing_structure() + if not isinstance(qs, dict): + return False, ["treeguard node presence check: queuingStructure.json not found"] + net = qs.get("Network") + if not isinstance(net, dict): + return False, ["treeguard node presence check: queuingStructure has no 'Network'"] + node = _find_parent_node(net, node_name) + if node is None: + return False, [f"treeguard node presence check: '{node_name}' absent from physical queue tree"] + return True, [f"treeguard node presence check: '{node_name}' still present in physical queue tree"] + + +def _log_contains_unexpected_full_reload(lines: List[str]) -> bool: + full_reload = any(FULL_RELOAD_PAT.search(line) or FULL_RELOAD_SUMMARY_PAT.search(line) for line in lines) + mq_init = any(MQ_INIT_PAT.search(line) for line in lines) + return full_reload and not mq_init + + +def wait_for_runtime_virtualized_node( + virtual_node_name: str, + promoted_child_name: str, + promoted_child_expected_path: List[str], + circuit_expectations: List[Tuple[str, str, List[str]]], + timeout_s: float, + poll_s: float = 1.0, +) -> Tuple[bool, List[str]]: + deadline = time.time() + timeout_s + last_msgs: List[str] = [] + while time.time() < deadline: + passed, msgs = check_virtualized_node_branch_promotion( + virtual_node_name, + promoted_child_name, + promoted_child_expected_path, + circuit_expectations, + expect_virtual_node_listed=False, + ) + last_msgs = msgs + if passed: + last_msgs.append( + f"treeguard runtime check: '{virtual_node_name}' virtualized live within {timeout_s:.1f}s" + ) + return True, last_msgs + time.sleep(poll_s) + last_msgs.append( + f"treeguard runtime check: timed out waiting {timeout_s:.1f}s for '{virtual_node_name}' runtime virtualization" + ) + return False, last_msgs + + +def get_treeguard_cpu_mode(config_path: str = "/etc/lqos.conf") -> Optional[str]: + current_section: Optional[str] = None + mode_re = re.compile(r'^mode\s*=\s*"([^"]+)"') + try: + with open(config_path, "r", errors="replace") as f: + for raw in f: + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("[") and line.endswith("]"): + current_section = line.strip("[]").strip() + continue + if current_section != "treeguard.cpu": + continue + m = mode_re.match(line) + if m: + return m.group(1).strip() + except Exception: + return None + return None + + +def wait_for_circuit_direction_ceil( + circuit_id: str, + timeout_s: float = 5.0, + poll_s: float = 0.5, +) -> Tuple[bool, List[str]]: + deadline = time.time() + timeout_s + last_msgs: List[str] = [] + while time.time() < deadline: + passed, msgs = check_circuit_direction_ceil(circuit_id) + last_msgs = msgs + if passed: + return True, msgs + time.sleep(poll_s) + last_msgs.append( + f"direction check: timed out waiting {timeout_s:.1f}s for circuit {circuit_id} direction mapping to settle" + ) + return False, last_msgs + + def check_ip_mappings_for_circuit(circuit_id: str) -> Tuple[bool, List[str]]: """Ensure that all IPv4s for this circuit are mapped to the correct down (and up if on-a-stick) tc handles.""" ok_tool, msg, entries = _list_ip_mappings() @@ -890,6 +1672,144 @@ def check_ip_mapping_absent(ip: str) -> Tuple[bool, List[str]]: return True, [f"ipmap check: {ip} successfully absent"] +def _expected_ips_for_circuit(circuit: dict) -> List[str]: + ips: List[str] = [] + for dev in circuit.get("devices", []): + try: + for ip in dev.get("ipv4s", []): + ips.append(str(ip)) + except Exception: + pass + return ips + + +def check_exact_ip_mappings_for_circuit( + circuit_id: str, + forbidden_tcs: Optional[Set[str]] = None, +) -> Tuple[bool, List[str]]: + ok_tool, msg, entries = _list_ip_mappings() + msgs: List[str] = [] + if not ok_tool: + return True, [msg] # soft-skip + + qs = _load_queuing_structure() + if not isinstance(qs, dict): + return False, ["ipmap exact check: queuingStructure.json not found"] + net = qs.get("Network") + if not isinstance(net, dict): + return False, ["ipmap exact check: queuingStructure has no 'Network'"] + found = _walk_nodes_for_circuit(net, circuit_id) + if not found: + return False, [f"ipmap exact check: circuit {circuit_id} not found in structure"] + parent_name, circuit = found + parent_node = _find_parent_node(net, parent_name) + if not isinstance(parent_node, dict): + return False, [f"ipmap exact check: parent node '{parent_name}' not found"] + + down_tc, up_tc, _cpu = _expect_handles_for_circuit(parent_node, circuit) + allowed_tcs = {down_tc.lower()} + if up_tc: + allowed_tcs.add(up_tc.lower()) + forbidden = {tc.lower() for tc in (forbidden_tcs or set())} + + ips = _expected_ips_for_circuit(circuit) + if not ips: + return True, [f"ipmap exact check: circuit {circuit_id} has no IPv4s; skipping"] + + ok = True + for ip in ips: + ip_entries = [(cpu, tc.lower()) for (mapped_ip, cpu, tc) in entries if mapped_ip == ip] + if not ip_entries: + msgs.append(f"ipmap exact check: missing mapping for {ip}") + ok = False + continue + actual_tcs = {tc for (_cpu, tc) in ip_entries} + if not actual_tcs.issubset(allowed_tcs): + msgs.append( + f"ipmap exact check: {ip} has unexpected handles {sorted(actual_tcs - allowed_tcs)} (allowed={sorted(allowed_tcs)})" + ) + ok = False + if forbidden and actual_tcs.intersection(forbidden): + msgs.append( + f"ipmap exact check: {ip} still mapped to forbidden handles {sorted(actual_tcs.intersection(forbidden))}" + ) + ok = False + if ok: + msgs.append(f"ipmap exact check: {ip} only mapped to expected handles {sorted(actual_tcs)}") + + return ok, msgs + + +def check_exact_ip_mappings_for_circuits( + circuit_ids: List[str], + forbidden_by_circuit: Optional[Dict[str, Set[str]]] = None, +) -> Tuple[bool, List[str]]: + ok_tool, msg, entries = _list_ip_mappings() + msgs: List[str] = [] + if not ok_tool: + return True, [msg] # soft-skip + + qs = _load_queuing_structure() + if not isinstance(qs, dict): + return False, ["ipmap batch exact check: queuingStructure.json not found"] + net = qs.get("Network") + if not isinstance(net, dict): + return False, ["ipmap batch exact check: queuingStructure has no 'Network'"] + + ok = True + expected_ip_count = 0 + for circuit_id in circuit_ids: + found = _walk_nodes_for_circuit(net, circuit_id) + if not found: + msgs.append(f"ipmap batch exact check: circuit {circuit_id} not found in structure") + ok = False + continue + parent_name, circuit = found + parent_node = _find_parent_node(net, parent_name) + if not isinstance(parent_node, dict): + msgs.append(f"ipmap batch exact check: parent node '{parent_name}' not found for circuit {circuit_id}") + ok = False + continue + + down_tc, up_tc, _cpu = _expect_handles_for_circuit(parent_node, circuit) + allowed_tcs = {down_tc.lower()} + if up_tc: + allowed_tcs.add(up_tc.lower()) + forbidden = {tc.lower() for tc in (forbidden_by_circuit or {}).get(circuit_id, set())} + + ips = _expected_ips_for_circuit(circuit) + if not ips: + msgs.append(f"ipmap batch exact check: circuit {circuit_id} has no IPv4s; skipping") + continue + expected_ip_count += len(ips) + + for ip in ips: + ip_entries = [(cpu, tc.lower()) for (mapped_ip, cpu, tc) in entries if mapped_ip == ip] + if not ip_entries: + msgs.append(f"ipmap batch exact check: missing mapping for {ip} (circuit {circuit_id})") + ok = False + continue + actual_tcs = {tc for (_cpu, tc) in ip_entries} + if not actual_tcs.issubset(allowed_tcs): + msgs.append( + f"ipmap batch exact check: {ip} (circuit {circuit_id}) has unexpected handles {sorted(actual_tcs - allowed_tcs)} (allowed={sorted(allowed_tcs)})" + ) + ok = False + if forbidden and actual_tcs.intersection(forbidden): + msgs.append( + f"ipmap batch exact check: {ip} (circuit {circuit_id}) still mapped to forbidden handles {sorted(actual_tcs.intersection(forbidden))}" + ) + ok = False + if actual_tcs.issubset(allowed_tcs) and not (forbidden and actual_tcs.intersection(forbidden)): + msgs.append(f"ipmap batch exact check: {ip} (circuit {circuit_id}) only mapped to expected handles {sorted(actual_tcs)}") + + if ok: + msgs.append( + f"ipmap batch exact check: verified {expected_ip_count} expected IPv4 mappings across {len(circuit_ids)} circuits" + ) + return ok, msgs + + def check_generated_pn_tc_bandwidths() -> Tuple[bool, List[str]]: msgs: List[str] = [] qs = _load_queuing_structure() @@ -1028,6 +1948,175 @@ def _class_has_parent(iface: str, maj: int, mnr: int, pmaj: int, pmnr: int) -> T return (pmj == pmaj and pmn == pmnr), line +def _tc_class_occurrences(iface: str, maj: int, mnr: int) -> Tuple[int, List[str]]: + rc, out, err = _tc(["class", "show", "dev", iface]) + if rc != 0: + return 0, [f"tc class occurrences: 'tc class show dev {iface}' failed: {err.strip()}"] + + matches: List[str] = [] + cls_pat = re.compile(r"^class\s+htb\s+([0-9A-Fa-fx]+):([0-9A-Fa-fx]+)\b.*$", re.MULTILINE) + for found in cls_pat.finditer(out): + mj = _tc_id_token_to_int(found.group(1)) + mn = _tc_id_token_to_int(found.group(2)) + if mj == maj and mn == mnr: + matches.append(found.group(0)) + return len(matches), matches + + +def _tc_qdisc_occurrences_for_parent(iface: str, maj: int, mnr: int) -> Tuple[int, List[str]]: + rc, out, err = _qdisc_show(iface) + if rc != 0: + return 0, [f"tc qdisc occurrences: 'tc qdisc show dev {iface}' failed: {err.strip()}"] + + matches: List[str] = [] + pat = re.compile(r"^qdisc\s+\S+\s+\S+:\s+parent\s+([0-9A-Fa-fx]+):([0-9A-Fa-fx]+)\b.*$", re.MULTILINE) + for found in pat.finditer(out): + mj = _tc_id_token_to_int(found.group(1)) + mn = _tc_id_token_to_int(found.group(2)) + if mj == maj and mn == mnr: + matches.append(found.group(0)) + return len(matches), matches + + +def _handle_to_parts(handle: Optional[str]) -> Optional[Tuple[int, int]]: + if not handle or ":" not in handle: + return None + left, right = handle.split(":", 1) + major = _tc_id_token_to_int(left) + minor = _tc_id_token_to_int(right) + if major is None or minor is None: + return None + return major, minor + + +def _current_circuit_handle_strings(circuit_id: str) -> Tuple[Optional[str], Optional[str], List[str]]: + qs = _load_queuing_structure() + if not isinstance(qs, dict): + return None, None, ["cleanup check: queuingStructure.json not found"] + net = qs.get("Network") + if not isinstance(net, dict): + return None, None, ["cleanup check: queuingStructure has no 'Network'"] + found = _walk_nodes_for_circuit(net, circuit_id) + if not found: + return None, None, [f"cleanup check: circuit {circuit_id} not found in structure"] + _parent_name, circuit = found + down_tc = str(circuit.get("classid", "")).replace("0x", "").lower() + up_tc = str(circuit.get("up_classid", "")).replace("0x", "").lower() + return down_tc or None, up_tc or None, [] + + +def check_circuit_transition_cleanup( + circuit_id: str, + *, + old_down_tc: Optional[str], + old_up_tc: Optional[str], +) -> Tuple[bool, List[str]]: + msgs: List[str] = [] + ok = True + + new_down_tc, new_up_tc, handle_msgs = _current_circuit_handle_strings(circuit_id) + msgs.extend(handle_msgs) + if handle_msgs: + return False, msgs + + if not interface_a or not callable(interface_a): + return False, ["cleanup check: interface_a() binding unavailable"] + try: + ifa = interface_a() + except Exception as e: + return False, [f"cleanup check: failed to get interface_a(): {e}"] + ifb = None + if interface_b and callable(interface_b): + try: + ifb = interface_b() + except Exception: + ifb = None + + if new_down_tc: + down_parts = _handle_to_parts(new_down_tc) + if down_parts is None: + msgs.append(f"cleanup check: could not parse current down handle {new_down_tc} for circuit {circuit_id}") + ok = False + else: + count, lines = _tc_class_occurrences(ifa, *down_parts) + if count != 1: + msgs.append( + f"cleanup check: expected exactly one current down class {new_down_tc} for circuit {circuit_id} on {ifa}, found {count}" + ) + msgs.extend(lines) + ok = False + else: + msgs.append(f"cleanup check: current down class {new_down_tc} present exactly once on {ifa}") + if new_up_tc and ifb: + up_parts = _handle_to_parts(new_up_tc) + if up_parts is None: + msgs.append(f"cleanup check: could not parse current up handle {new_up_tc} for circuit {circuit_id}") + ok = False + else: + count, lines = _tc_class_occurrences(ifb, *up_parts) + if count != 1: + msgs.append( + f"cleanup check: expected exactly one current up class {new_up_tc} for circuit {circuit_id} on {ifb}, found {count}" + ) + msgs.extend(lines) + ok = False + else: + msgs.append(f"cleanup check: current up class {new_up_tc} present exactly once on {ifb}") + + if old_down_tc and old_down_tc.lower() != (new_down_tc or "").lower(): + old_down_parts = _handle_to_parts(old_down_tc) + if old_down_parts is not None: + count, lines = _tc_class_occurrences(ifa, *old_down_parts) + if count != 0: + msgs.append( + f"cleanup check: stale down class {old_down_tc} still present for circuit {circuit_id} on {ifa}" + ) + msgs.extend(lines) + ok = False + else: + msgs.append(f"cleanup check: stale down class {old_down_tc} absent on {ifa}") + + qcount, qlines = _tc_qdisc_occurrences_for_parent(ifa, *old_down_parts) + if qcount != 0: + msgs.append( + f"cleanup check: stale down qdisc parent {old_down_tc} still present for circuit {circuit_id} on {ifa}" + ) + msgs.extend(qlines) + ok = False + else: + msgs.append(f"cleanup check: stale down qdisc parent {old_down_tc} absent on {ifa}") + + if old_up_tc and ifb and old_up_tc.lower() != (new_up_tc or "").lower(): + old_up_parts = _handle_to_parts(old_up_tc) + if old_up_parts is not None: + count, lines = _tc_class_occurrences(ifb, *old_up_parts) + if count != 0: + msgs.append( + f"cleanup check: stale up class {old_up_tc} still present for circuit {circuit_id} on {ifb}" + ) + msgs.extend(lines) + ok = False + else: + msgs.append(f"cleanup check: stale up class {old_up_tc} absent on {ifb}") + + qcount, qlines = _tc_qdisc_occurrences_for_parent(ifb, *old_up_parts) + if qcount != 0: + msgs.append( + f"cleanup check: stale up qdisc parent {old_up_tc} still present for circuit {circuit_id} on {ifb}" + ) + msgs.extend(qlines) + ok = False + else: + msgs.append(f"cleanup check: stale up qdisc parent {old_up_tc} absent on {ifb}") + + forbidden = {tc for tc in [old_down_tc, old_up_tc] if tc} + passed_ip_exact, ip_msgs = check_exact_ip_mappings_for_circuit(circuit_id, forbidden_tcs=forbidden) + msgs.extend(ip_msgs) + ok &= passed_ip_exact + + return ok, msgs + + def check_orphan_tc_attachment(orphan_id: str = "99901") -> Tuple[bool, List[str]]: msgs: List[str] = [] qs = _load_queuing_structure() @@ -1281,6 +2370,70 @@ def check_circuit_direction_ceil(circuit_id: str) -> Tuple[bool, List[str]]: return ok, msgs +def check_circuit_qdisc_kind( + circuit_id: str, + expected_down_kind: Optional[str] = None, + expected_up_kind: Optional[str] = None, +) -> Tuple[bool, List[str]]: + msgs: List[str] = [] + qs = _load_queuing_structure() + if not isinstance(qs, dict): + return False, ["qdisc kind check: queuingStructure.json not found"] + net = qs.get("Network") + if not isinstance(net, dict): + return False, ["qdisc kind check: queuingStructure has no 'Network'"] + found = _walk_nodes_for_circuit(net, circuit_id) + if not found: + return False, [f"qdisc kind check: circuit {circuit_id} not found in structure"] + _parent_name, circuit = found + + maj = _hex_to_int(str(circuit.get("classMajor"))) + mnr = _hex_to_int(str(circuit.get("classMinor"))) + upm = _hex_to_int(str(circuit.get("up_classMajor"))) + if maj is None or mnr is None: + return False, [f"qdisc kind check: circuit {circuit_id} missing downlink class IDs"] + + def _check_iface(iface: str, parent_major: int, parent_minor: int, expected_kind: Optional[str], label: str) -> Tuple[bool, str]: + rc, out, err = _qdisc_show(iface) + if rc != 0: + return False, f"qdisc kind check: 'tc qdisc show dev {iface}' failed: {err.strip()}" + + pat = re.compile(r"^qdisc\s+(\S+)\s+\S+:\s+parent\s+([0-9A-Fa-fx]+):([0-9A-Fa-fx]+)\b.*$", re.MULTILINE) + for match in pat.finditer(out): + kind = match.group(1).lower() + mj = _tc_id_token_to_int(match.group(2)) + mn = _tc_id_token_to_int(match.group(3)) + if mj == parent_major and mn == parent_minor: + if expected_kind is not None and kind != expected_kind.lower(): + return False, f"qdisc kind check: {label} {iface} expected {expected_kind}, found {kind} for circuit {circuit_id} | {match.group(0)}" + return True, f"qdisc kind check: {label} {iface} found {kind} for circuit {circuit_id} | {match.group(0)}" + + return False, f"qdisc kind check: {label} {iface} no qdisc found for parent {parent_major}:{parent_minor} (circuit {circuit_id})" + + ok = True + if expected_down_kind is not None: + if not interface_a or not callable(interface_a): + return False, ["qdisc kind check: interface_a() binding unavailable"] + try: + ifa = interface_a() + except Exception as e: + return False, [f"qdisc kind check: failed to get interface_a(): {e}"] + passed, message = _check_iface(ifa, maj, mnr, expected_down_kind, "down") + msgs.append(message) + ok &= passed + + if expected_up_kind is not None and interface_b and callable(interface_b) and upm is not None: + try: + ifb = interface_b() + except Exception as e: + return False, [f"qdisc kind check: failed to get interface_b(): {e}"] + passed, message = _check_iface(ifb, upm, mnr, expected_up_kind, "up") + msgs.append(message) + ok &= passed + + return ok, msgs + + def _walk_nodes_for_circuit(node_map: Dict[str, dict], circuit_id: str) -> Optional[Tuple[str, dict]]: for name, node in node_map.items(): if not isinstance(node, dict): @@ -1387,6 +2540,190 @@ def assert_full_reload(tag: str, res: LogResult, step_started_at: Optional[float return True, f"{tag}: OK (full reload)" +def _journal_since_lines(since_ts: float) -> List[str]: + proc = subprocess.run( + [ + "journalctl", + "-u", + "lqosd", + "--since", + f"@{since_ts:.6f}", + "--no-pager", + "-o", + "short-unix", + ], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + return [] + return [line.rstrip("\n") for line in proc.stdout.splitlines() if line.strip()] + + +def check_no_hidden_incremental_failures(step_started_at: float) -> Tuple[bool, List[str]]: + lines = _journal_since_lines(step_started_at) + patterns = [ + re.compile(r"reload required", re.IGNORECASE), + re.compile(r"\bRTNETLINK\b", re.IGNORECASE), + re.compile(r"\bDirty\b", re.IGNORECASE), + re.compile(r"Command error for \(", re.IGNORECASE), + re.compile(r"Bakery wrote numbered command failure", re.IGNORECASE), + re.compile(r"full reload is now required", re.IGNORECASE), + ] + + matches = [line for line in lines if any(p.search(line) for p in patterns)] + + # Test-only Bakery artifact path. + last_error_path = "/tmp/lqos_bakery_last_error.txt" # nosec B108 + if os.path.exists(last_error_path): + try: + if os.path.getmtime(last_error_path) >= step_started_at: + matches.append(f"last error file updated: {last_error_path}") + except Exception: + pass + + if matches: + return False, [ + "incremental health: unexpected error/desync indicators detected after step", + *matches[-20:], + ] + + return True, ["incremental health: no hidden Bakery desync/error indicators detected after step"] + + +def check_fault_injection_observed(step_started_at: float) -> Tuple[bool, List[str]]: + lines = _journal_since_lines(step_started_at) + fault_patterns = [ + re.compile(r"synthetic Bakery test fault", re.IGNORECASE), + re.compile(r"Bakery test fault injected", re.IGNORECASE), + ] + + fault_matches = [line for line in lines if any(p.search(line) for p in fault_patterns)] + msgs: List[str] = [] + + if os.path.exists(TEST_FAULT_ONCE_PATH): + msgs.append(f"fault injection file still present and was not consumed: {TEST_FAULT_ONCE_PATH}") + return False, msgs + + if not fault_matches: + return False, [ + "fault reload check: synthetic Bakery fault was not observed in recent logs", + *lines[-20:], + ] + + msgs.append("fault reload check: Bakery observed and consumed the injected synthetic fault") + msgs.extend(fault_matches[-5:]) + return True, msgs + + +def check_reload_required_since(step_started_at: float) -> Tuple[bool, List[str]]: + lines = _journal_since_lines(step_started_at) + reload_patterns = [ + re.compile(r"reload required", re.IGNORECASE), + re.compile(r"full reload is now required", re.IGNORECASE), + re.compile(r"full reload required before further incremental topology mutation", re.IGNORECASE), + ] + + reload_matches = [line for line in lines if any(p.search(line) for p in reload_patterns)] + if not reload_matches: + return False, [ + "fault reload check: Bakery did not report reload-required state after injected fault", + *lines[-20:], + ] + + msgs: List[str] = [] + msgs.append("fault reload check: Bakery entered reload-required state after the injected fault") + msgs.extend(reload_matches[-5:]) + return True, msgs + + +def collect_failure_diagnostics( + tag: str, + *, + step_started_at: Optional[float] = None, + circuit_ids: Optional[List[str]] = None, + node_names: Optional[List[str]] = None, +) -> List[str]: + circuit_ids = circuit_ids or [] + node_names = node_names or [] + msgs = [f"diagnostics: begin for {tag}"] + + if step_started_at is not None: + journal_lines = _journal_since_lines(step_started_at) + msgs.append("diagnostics: recent lqosd journal") + msgs.extend(journal_lines[-20:] if journal_lines else ["diagnostics: no recent journal lines found"]) + + planner = None + try: + with open("planner_state.json", "r") as f: + planner = json.load(f) + except Exception as e: + msgs.append(f"diagnostics: failed to read planner_state.json: {e}") + + qs = _load_queuing_structure() + net = qs.get("Network") if isinstance(qs, dict) else None + + if isinstance(planner, dict): + for circuit_id in circuit_ids: + entry = planner.get("circuits", {}).get(str(circuit_id)) + msgs.append(f"diagnostics: planner circuit {circuit_id} -> {json.dumps(entry, sort_keys=True)}") + + if isinstance(net, dict): + for circuit_id in circuit_ids: + found = _walk_nodes_for_circuit(net, circuit_id) + if found: + parent_name, circuit = found + msgs.append( + f"diagnostics: structure circuit {circuit_id} under {parent_name} -> {json.dumps(circuit, sort_keys=True)}" + ) + else: + msgs.append(f"diagnostics: structure circuit {circuit_id} not found") + for node_name in node_names: + node = _find_parent_node(net, node_name) + msgs.append( + f"diagnostics: structure node {node_name} -> {json.dumps(node, sort_keys=True) if node is not None else 'missing'}" + ) + + if interface_a and callable(interface_a): + try: + ifa = interface_a() + rc, out, err = _tc(["class", "show", "dev", ifa]) + msgs.append(f"diagnostics: tc class show dev {ifa} rc={rc}") + msgs.extend(out.splitlines()[-20:] if rc == 0 else [err.strip()]) + rc, out, err = _qdisc_show(ifa) + msgs.append(f"diagnostics: tc qdisc show dev {ifa} rc={rc}") + msgs.extend(out.splitlines()[-20:] if rc == 0 else [err.strip()]) + except Exception as e: + msgs.append(f"diagnostics: failed tc dump for interface_a: {e}") + + if interface_b and callable(interface_b): + try: + ifb = interface_b() + rc, out, err = _tc(["class", "show", "dev", ifb]) + msgs.append(f"diagnostics: tc class show dev {ifb} rc={rc}") + msgs.extend(out.splitlines()[-20:] if rc == 0 else [err.strip()]) + rc, out, err = _qdisc_show(ifb) + msgs.append(f"diagnostics: tc qdisc show dev {ifb} rc={rc}") + msgs.extend(out.splitlines()[-20:] if rc == 0 else [err.strip()]) + except Exception as e: + msgs.append(f"diagnostics: failed tc dump for interface_b: {e}") + + # Test-only Bakery artifact path. + last_error_path = "/tmp/lqos_bakery_last_error.txt" # nosec B108 + if os.path.exists(last_error_path): + try: + with open(last_error_path, "r", errors="replace") as f: + lines = f.read().splitlines() + msgs.append(f"diagnostics: {last_error_path}") + msgs.extend(lines[-20:] if lines else ["diagnostics: last error file empty"]) + except Exception as e: + msgs.append(f"diagnostics: failed reading {last_error_path}: {e}") + + msgs.append(f"diagnostics: end for {tag}") + return msgs + + def with_backups(paths: List[str]): class _Ctx: def __enter__(self): @@ -1420,7 +2757,154 @@ def with_backups(paths: List[str]): # ----------------------- -def run_tiered_suite(log: LogReader, timeout_s: float, results: List[str]) -> bool: +def run_queue_mode_suite(log: AnyLogReader, timeout_s: float, results: List[str]) -> bool: + if os.geteuid() != 0: + results.append("queue-mode: must run as root to edit /etc/lqos.conf") + return False + if not queue_mode or not callable(queue_mode): # type: ignore[name-defined] + results.append("queue-mode: queue_mode() binding unavailable") + return False + if not sync_lqosd_config_from_disk or not callable(sync_lqosd_config_from_disk): # type: ignore[name-defined] + results.append("queue-mode: sync_lqosd_config_from_disk() binding unavailable") + return False + if not interface_a or not callable(interface_a): # type: ignore[name-defined] + results.append("queue-mode: interface_a() binding unavailable") + return False + + iface_names: List[str] = [] + try: + iface_names.append(interface_a()) # type: ignore[misc] + except Exception as exc: + results.append(f"queue-mode: failed to resolve interface_a(): {exc}") + return False + if interface_b and callable(interface_b): # type: ignore[name-defined] + try: + iface_b_name = interface_b() # type: ignore[misc] + if iface_b_name and iface_b_name not in iface_names: + iface_names.append(iface_b_name) + except Exception as exc: + results.append(f"queue-mode: failed to resolve interface_b(): {exc}") + return False + + config_path = "/etc/lqos.conf" + original_config_text = _read_text(config_path) + try: + original_mode = queue_mode() # type: ignore[misc] + except Exception as exc: + results.append(f"queue-mode: failed to read current queue_mode(): {exc}") + return False + + last_applied_mode = original_mode + + def _set_mode_and_refresh(mode: str, step_name: str) -> Tuple[bool, LogResult]: + nonlocal last_applied_mode + _mark_step(step_name) + _write_text(config_path, _set_queue_mode_in_lqos_conf_text(_read_text(config_path), mode)) + sync_lqosd_config_from_disk() # type: ignore[misc] + step_started_at = time.time() + res = run_refresh_subprocess_and_wait(log, timeout_s) + last_applied_mode = mode + results.append( + f"{step_name}: bakery outcome full_reload={res.full_reload} mq_init={res.mq_init} incremental={res.incremental_event or 'none'}" + ) + if not res.raw_lines: + results.append(f"{step_name}: no Bakery logs observed after refresh (started {step_started_at:.6f})") + return False, res + return True, res + + def _check_shape_state(tag: str, baseline: Optional[Dict[str, TcShapeSummary]] = None) -> Tuple[bool, Dict[str, TcShapeSummary]]: + ok = True + summaries: Dict[str, TcShapeSummary] = {} + for iface in iface_names: + summary = _summarize_iface_tc(iface) + summaries[iface] = summary + results.append(_shape_summary_message(tag, summary)) + if not summary.root_mq: + results.append(f"{tag}: {iface}: expected root mq to remain present") + ok = False + if summary.root_htb_qdiscs <= 0: + results.append(f"{tag}: {iface}: expected HTB qdiscs in shape mode") + ok = False + if summary.htb_classes <= 0: + results.append(f"{tag}: {iface}: expected HTB classes in shape mode") + ok = False + if summary.child_leaf_qdiscs <= 0: + results.append(f"{tag}: {iface}: expected leaf qdiscs in shape mode") + ok = False + if baseline is not None: + prior = baseline.get(iface) + if prior is None: + results.append(f"{tag}: {iface}: missing baseline summary for comparison") + ok = False + elif summary != prior: + results.append( + f"{tag}: {iface}: note restored shape counts differ from baseline " + f"(baseline root_htb_qdiscs={prior.root_htb_qdiscs}, htb_classes={prior.htb_classes}, " + f"child_leaf_qdiscs={prior.child_leaf_qdiscs}); " + "shape rebuild appears to have normalized pre-existing stale TC state" + ) + return ok, summaries + + def _check_observe_state(tag: str) -> Tuple[bool, Dict[str, TcShapeSummary]]: + ok = True + summaries: Dict[str, TcShapeSummary] = {} + for iface in iface_names: + summary = _summarize_iface_tc(iface) + summaries[iface] = summary + results.append(_shape_summary_message(tag, summary)) + if not summary.root_mq: + results.append(f"{tag}: {iface}: expected root mq to remain present") + ok = False + if summary.root_htb_qdiscs != 0: + results.append(f"{tag}: {iface}: expected no root HTB qdiscs in observe mode") + ok = False + if summary.htb_classes != 0: + results.append(f"{tag}: {iface}: expected no HTB classes in observe mode") + ok = False + if summary.child_leaf_qdiscs != 0: + results.append(f"{tag}: {iface}: expected no child leaf qdiscs in observe mode") + ok = False + return ok, summaries + + overall_ok = True + baseline_shape: Optional[Dict[str, TcShapeSummary]] = None + + try: + mode_ok, _ = _set_mode_and_refresh("shape", "queue-mode: baseline shape sync") + overall_ok &= mode_ok + + shape_ok, baseline_shape = _check_shape_state("queue-mode: baseline shape") + overall_ok &= shape_ok + + mode_ok, _ = _set_mode_and_refresh("observe", "queue-mode: switch to observe") + overall_ok &= mode_ok + observe_ok, _ = _check_observe_state("queue-mode: observe state") + overall_ok &= observe_ok + + mode_ok, _ = _set_mode_and_refresh("shape", "queue-mode: switch back to shape") + overall_ok &= mode_ok + restore_ok, _ = _check_shape_state("queue-mode: restored shape", baseline_shape) + overall_ok &= restore_ok + finally: + current_config_text = _read_text(config_path) + if current_config_text != original_config_text: + _write_text(config_path, original_config_text) + if last_applied_mode != original_mode: + try: + _mark_step("queue-mode: restore original runtime mode") + sync_lqosd_config_from_disk() # type: ignore[misc] + _ = run_refresh_subprocess_and_wait(log, timeout_s) + results.append(f"queue-mode: restored original runtime mode {original_mode}") + except Exception as exc: + results.append(f"queue-mode: failed to restore original runtime mode {original_mode}: {exc}") + overall_ok = False + else: + results.append("queue-mode: restored /etc/lqos.conf to original text") + + return overall_ok + + +def run_tiered_suite(log: AnyLogReader, timeout_s: float, results: List[str]) -> bool: ok = True # Baseline @@ -1588,24 +3072,11 @@ def run_tiered_suite(log: LogReader, timeout_s: float, results: List[str]) -> bo else: pat = re.compile(r"^qdisc\s+\S+\s+\S+:\s+parent\s+([0-9A-Fa-fx]+):([0-9A-Fa-fx]+)\b.*$", re.MULTILINE) - def _to_int(tok: str) -> Optional[int]: - s = tok.strip() - # Accept decimal, 0x-prefixed hex, or bare hex (e.g. "f") - try: - if s.lower().startswith("0x"): - return int(s, 16) - try: - return int(s) # decimal - except Exception: - return int(s, 16) # bare hex - except Exception: - return None - line = None for m in pat.finditer(out): mj_tok, mn_tok = m.group(1), m.group(2) - mj = _to_int(mj_tok) - mn = _to_int(mn_tok) + mj = _tc_id_token_to_int(mj_tok) + mn = _tc_id_token_to_int(mn_tok) if mj == maj and mn == mnr: line = m.group(0) break @@ -1630,24 +3101,11 @@ def run_tiered_suite(log: LogReader, timeout_s: float, results: List[str]) -> bo else: pat2 = re.compile(r"^qdisc\s+\S+\s+\S+:\s+parent\s+([0-9A-Fa-fx]+):([0-9A-Fa-fx]+)\b.*$", re.MULTILINE) - def _to_int2(tok: str) -> Optional[int]: - s = tok.strip() - # Accept decimal, 0x-prefixed hex, or bare hex (e.g. "f") - try: - if s.lower().startswith("0x"): - return int(s, 16) - try: - return int(s) # decimal - except Exception: - return int(s, 16) # bare hex - except Exception: - return None - line2 = None for m2 in pat2.finditer(out2): mj_tok, mn_tok = m2.group(1), m2.group(2) - mj = _to_int2(mj_tok) - mn = _to_int2(mn_tok) + mj = _tc_id_token_to_int(mj_tok) + mn = _tc_id_token_to_int(mn_tok) if mj == upm and mn == mnr: line2 = m2.group(0) break @@ -1763,9 +3221,150 @@ def run_tiered_suite(log: LogReader, timeout_s: float, results: List[str]) -> bo return ok -def run_realistic_tiered_suite(log: LogReader, timeout_s: float, results: List[str]) -> bool: +def run_virtualized_tiered_suite(log: AnyLogReader, timeout_s: float, results: List[str]) -> bool: ok = True + write_network_json(VIRTUALIZED_TIERED_NETWORK_BASE) + rows = virtualized_tiered_circuits_base() + write_circuits(rows) + _ = run_refresh_and_wait(log, timeout_s) + settle_initial_bakery_logs(log) + results.append("virtualized: baseline: ok (initial run)") + + _mark_step("virtualized: operator-authored virtual node promotion") + passed, msgs = check_virtualized_node_branch_promotion( + "Town_V", + "AP_V", + ["Site_A", "AP_V"], + [ + ("direct", "501", ["Site_A"]), + ("promoted-child", "502", ["Site_A", "AP_V"]), + ], + ) + results.extend(msgs) + ok &= passed + + _mark_step("virtualized: direct circuit ip mappings") + passed_map_direct, msgs_map_direct = check_ip_mappings_for_circuit("501") + results.extend(msgs_map_direct) + ok &= passed_map_direct + + _mark_step("virtualized: promoted child circuit ip mappings") + passed_map_child, msgs_map_child = check_ip_mappings_for_circuit("502") + results.extend(msgs_map_child) + ok &= passed_map_child + + _mark_step("virtualized: promoted child circuit rate change") + rows2 = json.loads(json.dumps(rows)) + rows2[1]["Download Max Mbps"] = 22 + rows2[1]["Upload Max Mbps"] = 11 + write_circuits(rows2) + step_t0 = time.time() + res = run_refresh_and_wait(log, timeout_s) + passed, msg = assert_no_full_reload("virtualized: promoted child circuit rate change", res, step_t0) + results.append(msg) + ok &= passed + + _mark_step("virtualized: promoted child circuit direction mapping") + passed_dir, msgs_dir = check_circuit_direction_ceil("502") + results.extend(msgs_dir) + ok &= passed_dir + + _mark_step("virtualized: promoted child circuit IP change") + rows3 = json.loads(json.dumps(rows2)) + old_ip = str(rows3[1]["IPv4"]) + rows3[1]["IPv4"] = "100.64.2.22" + write_circuits(rows3) + step_t0 = time.time() + res = run_refresh_and_wait(log, timeout_s) + passed, msg = assert_no_full_reload("virtualized: promoted child circuit IP change", res, step_t0) + results.append(msg) + ok &= passed + + passed_map_new, msgs_map_new = check_ip_mappings_for_circuit("502") + results.extend(msgs_map_new) + ok &= passed_map_new + + passed_map_old, msgs_map_old = check_ip_mapping_absent(old_ip) + results.extend(msgs_map_old) + ok &= passed_map_old + + _mark_step("virtualized: promoted child circuit parent move") + rows4 = json.loads(json.dumps(rows3)) + rows4[1]["Parent Node"] = "AP_A" + write_circuits(rows4) + step_t0 = time.time() + res = run_refresh_and_wait(log, timeout_s) + passed, msg = assert_no_full_reload("virtualized: promoted child circuit parent move", res, step_t0) + results.append(msg) + ok &= passed + + _mark_step("virtualized: parent-moved circuit direction mapping") + passed_parent_dir, msgs_parent_dir = check_circuit_direction_ceil("502") + results.extend(msgs_parent_dir) + ok &= passed_parent_dir + + _mark_step("virtualized: parent-moved circuit ip mappings") + passed_parent_map, msgs_parent_map = check_ip_mappings_for_circuit("502") + results.extend(msgs_parent_map) + ok &= passed_parent_map + + return ok + + +def run_realistic_tiered_suite(log: AnyLogReader, timeout_s: float, results: List[str]) -> bool: + ok = True + + def _first_row_index( + rows_local: List[Dict[str, str | int | float]], + parent_name: str, + exclude_ids: Optional[set[str]] = None, + ) -> Optional[int]: + excluded = exclude_ids or set() + for idx, row in enumerate(rows_local): + if str(row.get("Parent Node")) == parent_name and str(row.get("Circuit ID")) not in excluded: + return idx + return None + + def _record_check( + tag: str, + passed: bool, + msgs: List[str], + *, + step_started_at: Optional[float] = None, + circuit_ids: Optional[List[str]] = None, + node_names: Optional[List[str]] = None, + ) -> None: + nonlocal ok + results.extend(msgs) + ok &= passed + if not passed: + results.extend( + collect_failure_diagnostics( + tag, + step_started_at=step_started_at, + circuit_ids=circuit_ids, + node_names=node_names, + ) + ) + + def _record_hidden_health( + tag: str, + step_started_at: float, + *, + circuit_ids: Optional[List[str]] = None, + node_names: Optional[List[str]] = None, + ) -> None: + passed, msgs = check_no_hidden_incremental_failures(step_started_at) + _record_check( + f"{tag} health", + passed, + msgs, + step_started_at=step_started_at, + circuit_ids=circuit_ids, + node_names=node_names, + ) + # Baseline: write realistic network and circuits (including orphans) _mark_step("realistic: baseline") write_network_json(REALISTIC_TIERED_NETWORK) @@ -1818,11 +3417,11 @@ def run_realistic_tiered_suite(log: LogReader, timeout_s: float, results: List[s passed, msg = assert_no_full_reload("realistic: site speed change", res, step_t0) results.append(msg) ok &= passed + _record_hidden_health("realistic: site speed change", step_t0, node_names=["NET2-1"]) _mark_step("realistic: site direction mapping (NET2-1)") passed_site, msgs_site = check_site_direction_ceil("NET2-1", expected_dl=360.0, expected_ul=240.0) - results.extend(msgs_site) - ok &= passed_site + _record_check("realistic: site direction mapping (NET2-1)", passed_site, msgs_site, circuit_ids=[], node_names=["NET2-1"]) # Circuit speed change for a deep leaf circuit (under NET1-1-1) _mark_step("realistic: circuit speed change") @@ -1850,20 +3449,340 @@ def run_realistic_tiered_suite(log: LogReader, timeout_s: float, results: List[s passed, msg = assert_no_full_reload("realistic: circuit speed change", res, step_t0) results.append(msg) ok &= passed + _record_hidden_health("realistic: circuit speed change", step_t0, circuit_ids=[target_id]) _mark_step("realistic: circuit direction mapping (ceil)") passed_dir, msgs_dir = check_circuit_direction_ceil(target_id) - results.extend(msgs_dir) - ok &= passed_dir + _record_check("realistic: circuit direction mapping (ceil)", passed_dir, msgs_dir, circuit_ids=[target_id]) _mark_step("realistic: circuit ip mappings (no change)") passed_map, msgs_map = check_ip_mappings_for_circuit(target_id) - results.extend(msgs_map) - ok &= passed_map + _record_check("realistic: circuit ip mappings (no change)", passed_map, msgs_map, circuit_ids=[target_id]) + + passed_site_snapshot, site_snapshot_msg, site_snapshot = snapshot_site_class_assignments() + if not passed_site_snapshot: + results.append(site_snapshot_msg) + ok = False + + # Circuit parent-node move between existing nodes (should not trigger full reload) + _mark_step("realistic: circuit parent move") + rows_parent = json.loads(json.dumps(rows2)) + old_down_tc, old_up_tc, _old_handle_msgs = _current_circuit_handle_strings(target_id) + old_parent = str(rows_parent[target_idx]["Parent Node"]) + new_parent = "NET1-1-2" if old_parent != "NET1-1-2" else "NET1-2" + rows_parent[target_idx]["Parent Node"] = new_parent + write_circuits(rows_parent) + step_t0 = time.time() + res = run_refresh_and_wait(log, timeout_s) + passed, msg = assert_no_full_reload("realistic: circuit parent move", res, step_t0) + results.append(msg) + ok &= passed + _record_hidden_health("realistic: circuit parent move", step_t0, circuit_ids=[target_id]) + + _mark_step("realistic: site class assignment stability after parent move") + passed_sites_stable, msgs_sites_stable = check_site_class_assignments_unchanged(site_snapshot) + _record_check("realistic: site class assignment stability after parent move", passed_sites_stable, msgs_sites_stable, circuit_ids=[target_id]) + + _mark_step("realistic: parent-moved circuit direction mapping (ceil)") + passed_parent_dir, msgs_parent_dir = wait_for_circuit_direction_ceil(target_id) + _record_check("realistic: parent-moved circuit direction mapping (ceil)", passed_parent_dir, msgs_parent_dir, circuit_ids=[target_id], step_started_at=step_t0) + + _mark_step("realistic: parent-moved circuit ip mappings") + passed_parent_map, msgs_parent_map = check_ip_mappings_for_circuit(target_id) + _record_check("realistic: parent-moved circuit ip mappings", passed_parent_map, msgs_parent_map, circuit_ids=[target_id]) + + _mark_step("realistic: parent-moved circuit cleanup") + passed_parent_cleanup, msgs_parent_cleanup = check_circuit_transition_cleanup( + target_id, + old_down_tc=old_down_tc, + old_up_tc=old_up_tc, + ) + _record_check("realistic: parent-moved circuit cleanup", passed_parent_cleanup, msgs_parent_cleanup, circuit_ids=[target_id]) + + _mark_step("realistic: parent-moved circuit batch exact IP mappings") + passed_parent_batch_ip, msgs_parent_batch_ip = check_exact_ip_mappings_for_circuits( + [target_id], + forbidden_by_circuit={target_id: {tc for tc in [old_down_tc, old_up_tc] if tc}}, + ) + _record_check( + "realistic: parent-moved circuit batch exact IP mappings", + passed_parent_batch_ip, + msgs_parent_batch_ip, + circuit_ids=[target_id], + ) + + # Multiple simultaneous parent-node moves in one incremental commit + _mark_step("realistic: multiple circuit parent moves") + rows_multi = json.loads(json.dumps(rows_parent)) + multi_specs = [ + ("NET2-1-1", "NET2-2"), + ("NET3-1", "NET3-2"), + ("NET1-2", "NET1-1-1"), + ] + moved_ids: List[str] = [] + old_handles_by_circuit: Dict[str, Tuple[Optional[str], Optional[str]]] = {} + excluded_ids = {target_id} + for old_parent_name, new_parent_name in multi_specs: + idx = _first_row_index(rows_multi, old_parent_name, excluded_ids) + if idx is None: + results.append( + f"realistic: multiple circuit parent moves: no circuit found under {old_parent_name}" + ) + return False + moved_id = str(rows_multi[idx]["Circuit ID"]) + old_handles_by_circuit[moved_id] = _current_circuit_handle_strings(moved_id)[:2] + rows_multi[idx]["Parent Node"] = new_parent_name + moved_ids.append(moved_id) + excluded_ids.add(moved_id) + write_circuits(rows_multi) + step_t0 = time.time() + res = run_refresh_and_wait(log, timeout_s) + passed, msg = assert_no_full_reload("realistic: multiple circuit parent moves", res, step_t0) + results.append(msg) + ok &= passed + _record_hidden_health("realistic: multiple circuit parent moves", step_t0, circuit_ids=moved_ids) + + _mark_step("realistic: site class assignment stability after multiple parent moves") + passed_sites_stable_multi, msgs_sites_stable_multi = check_site_class_assignments_unchanged(site_snapshot) + _record_check("realistic: site class assignment stability after multiple parent moves", passed_sites_stable_multi, msgs_sites_stable_multi, circuit_ids=moved_ids) + + for moved_id in moved_ids: + _mark_step(f"realistic: moved circuit {moved_id} direction mapping") + passed_move_dir, msgs_move_dir = wait_for_circuit_direction_ceil(moved_id) + _record_check(f"realistic: moved circuit {moved_id} direction mapping", passed_move_dir, msgs_move_dir, circuit_ids=[moved_id], step_started_at=step_t0) + + _mark_step(f"realistic: moved circuit {moved_id} ip mappings") + passed_move_map, msgs_move_map = check_ip_mappings_for_circuit(moved_id) + _record_check(f"realistic: moved circuit {moved_id} ip mappings", passed_move_map, msgs_move_map, circuit_ids=[moved_id]) + + _mark_step(f"realistic: moved circuit {moved_id} cleanup") + old_down_tc_multi, old_up_tc_multi = old_handles_by_circuit.get(moved_id, (None, None)) + passed_move_cleanup, msgs_move_cleanup = check_circuit_transition_cleanup( + moved_id, + old_down_tc=old_down_tc_multi, + old_up_tc=old_up_tc_multi, + ) + _record_check(f"realistic: moved circuit {moved_id} cleanup", passed_move_cleanup, msgs_move_cleanup, circuit_ids=[moved_id]) + + _mark_step("realistic: multiple parent moves batch exact IP mappings") + forbidden_multi = { + circuit_id: {tc for tc in old_handles_by_circuit.get(circuit_id, (None, None)) if tc} + for circuit_id in moved_ids + } + passed_multi_batch_ip, msgs_multi_batch_ip = check_exact_ip_mappings_for_circuits( + moved_ids, + forbidden_by_circuit=forbidden_multi, + ) + _record_check( + "realistic: multiple parent moves batch exact IP mappings", + passed_multi_batch_ip, + msgs_multi_batch_ip, + circuit_ids=moved_ids, + ) + + # Same-circuit mixed change: parent move + rate + IP + SQM in one incremental commit + _mark_step("realistic: same-circuit mixed change") + rows_same = json.loads(json.dumps(rows_multi)) + same_idx = _first_row_index(rows_same, "NET3-2", {target_id, *moved_ids}) + if same_idx is None: + results.append("realistic: same-circuit mixed change: no candidate circuit found under NET3-2") + return False + same_id = str(rows_same[same_idx]["Circuit ID"]) + same_old_ip = str(rows_same[same_idx]["IPv4"]) + same_old_down_tc, same_old_up_tc, _ = _current_circuit_handle_strings(same_id) + rows_same[same_idx]["Parent Node"] = "NET3-1" + rows_same[same_idx]["Download Max Mbps"] = float(rows_same[same_idx]["Download Max Mbps"]) + 19.0 + rows_same[same_idx]["Upload Max Mbps"] = float(rows_same[same_idx]["Upload Max Mbps"]) + 7.0 + rows_same[same_idx]["IPv4"] = "100.64.220.1" + rows_same[same_idx]["sqm"] = "cake/fq_codel" + write_circuits(rows_same) + step_t0 = time.time() + res = run_refresh_and_wait(log, timeout_s) + passed, msg = assert_no_full_reload("realistic: same-circuit mixed change", res, step_t0) + results.append(msg) + ok &= passed + _record_hidden_health("realistic: same-circuit mixed change", step_t0, circuit_ids=[same_id]) + + _mark_step("realistic: same-circuit mixed direction mapping") + passed_same_dir, msgs_same_dir = wait_for_circuit_direction_ceil(same_id) + _record_check("realistic: same-circuit mixed direction mapping", passed_same_dir, msgs_same_dir, circuit_ids=[same_id], step_started_at=step_t0) + + _mark_step("realistic: same-circuit mixed IP mappings") + passed_same_ip, msgs_same_ip = check_ip_mappings_for_circuit(same_id) + _record_check("realistic: same-circuit mixed IP mappings", passed_same_ip, msgs_same_ip, circuit_ids=[same_id]) + passed_same_old_ip, msgs_same_old_ip = check_ip_mapping_absent(same_old_ip) + _record_check("realistic: same-circuit mixed old IP absent", passed_same_old_ip, msgs_same_old_ip, circuit_ids=[same_id]) + + _mark_step("realistic: same-circuit mixed exact IP mappings") + passed_same_exact_ip, msgs_same_exact_ip = check_exact_ip_mappings_for_circuit( + same_id, + forbidden_tcs={tc for tc in [same_old_down_tc, same_old_up_tc] if tc}, + ) + _record_check("realistic: same-circuit mixed exact IP mappings", passed_same_exact_ip, msgs_same_exact_ip, circuit_ids=[same_id]) + + _mark_step("realistic: same-circuit mixed SQM kind") + passed_same_sqm, msgs_same_sqm = check_circuit_qdisc_kind( + same_id, + expected_down_kind="cake", + expected_up_kind="fq_codel", + ) + _record_check("realistic: same-circuit mixed SQM kind", passed_same_sqm, msgs_same_sqm, circuit_ids=[same_id]) + + _mark_step("realistic: same-circuit mixed cleanup") + passed_same_cleanup, msgs_same_cleanup = check_circuit_transition_cleanup( + same_id, + old_down_tc=same_old_down_tc, + old_up_tc=same_old_up_tc, + ) + _record_check("realistic: same-circuit mixed cleanup", passed_same_cleanup, msgs_same_cleanup, circuit_ids=[same_id]) + + _mark_step("realistic: same-circuit mixed batch exact IP mappings") + passed_same_batch_ip, msgs_same_batch_ip = check_exact_ip_mappings_for_circuits( + [same_id], + forbidden_by_circuit={same_id: {tc for tc in [same_old_down_tc, same_old_up_tc] if tc}}, + ) + _record_check( + "realistic: same-circuit mixed batch exact IP mappings", + passed_same_batch_ip, + msgs_same_batch_ip, + circuit_ids=[same_id], + ) + + # Mixed incremental batch: site speed, circuit speed, IP, SQM, and parent move together + _mark_step("realistic: mixed incremental batch") + net_mixed = json.loads(json.dumps(net2)) + try: + net_mixed["NET3"]["children"]["NET3-1"]["downloadBandwidthMbps"] = 275 + net_mixed["NET3"]["children"]["NET3-1"]["uploadBandwidthMbps"] = 155 + except Exception: + results.append("realistic: mixed incremental batch: failed to modify NET3-1 speeds in fixture") + return False + + rows_mixed = json.loads(json.dumps(rows_same)) + excluded_mixed = {target_id, *moved_ids, same_id} + + speed_idx = _first_row_index(rows_mixed, "NET2-2", excluded_mixed) + if speed_idx is None: + results.append("realistic: mixed incremental batch: no speed-change circuit found under NET2-2") + return False + speed_id = str(rows_mixed[speed_idx]["Circuit ID"]) + rows_mixed[speed_idx]["Download Max Mbps"] = float(rows_mixed[speed_idx]["Download Max Mbps"]) + 17.0 + rows_mixed[speed_idx]["Upload Max Mbps"] = float(rows_mixed[speed_idx]["Upload Max Mbps"]) + 6.0 + excluded_mixed.add(speed_id) + + ip_idx = _first_row_index(rows_mixed, "NET3-2", excluded_mixed) + if ip_idx is None: + results.append("realistic: mixed incremental batch: no IP-change circuit found under NET3-2") + return False + ip_id = str(rows_mixed[ip_idx]["Circuit ID"]) + old_ip = str(rows_mixed[ip_idx]["IPv4"]) + rows_mixed[ip_idx]["IPv4"] = "100.64.210.1" + excluded_mixed.add(ip_id) + + sqm_idx = _first_row_index(rows_mixed, "NET1-1-2", excluded_mixed) + if sqm_idx is None: + results.append("realistic: mixed incremental batch: no SQM-change circuit found under NET1-1-2") + return False + sqm_id = str(rows_mixed[sqm_idx]["Circuit ID"]) + rows_mixed[sqm_idx]["sqm"] = "fq_codel/cake" + excluded_mixed.add(sqm_id) + + mixed_move_idx: Optional[int] = None + mixed_move_target: Optional[str] = None + for source_parent, target_parent in [ + ("NET2-2", "NET2-1-1"), + ("NET3-2", "NET3-1"), + ("NET1-1-2", "NET1-2"), + ]: + idx = _first_row_index(rows_mixed, source_parent, excluded_mixed) + if idx is not None: + mixed_move_idx = idx + mixed_move_target = target_parent + break + if mixed_move_idx is None or mixed_move_target is None: + results.append("realistic: mixed incremental batch: no parent-move circuit found in fallback candidate set") + return False + mixed_move_id = str(rows_mixed[mixed_move_idx]["Circuit ID"]) + mixed_old_down_tc, mixed_old_up_tc, _ = _current_circuit_handle_strings(mixed_move_id) + rows_mixed[mixed_move_idx]["Parent Node"] = mixed_move_target + + write_network_json(net_mixed) + write_circuits(rows_mixed) + step_t0 = time.time() + res = run_refresh_and_wait(log, timeout_s) + passed, msg = assert_no_full_reload("realistic: mixed incremental batch", res, step_t0) + results.append(msg) + ok &= passed + _record_hidden_health( + "realistic: mixed incremental batch", + step_t0, + circuit_ids=[speed_id, ip_id, sqm_id, mixed_move_id], + node_names=["NET3-1"], + ) + + _mark_step("realistic: site class assignment stability after mixed batch") + passed_sites_stable_mixed, msgs_sites_stable_mixed = check_site_class_assignments_unchanged(site_snapshot) + _record_check("realistic: site class assignment stability after mixed batch", passed_sites_stable_mixed, msgs_sites_stable_mixed, circuit_ids=[speed_id, ip_id, sqm_id, mixed_move_id], node_names=["NET3-1"]) + + _mark_step("realistic: mixed batch site direction mapping (NET3-1)") + passed_site_mixed, msgs_site_mixed = check_site_direction_ceil("NET3-1", expected_dl=275.0, expected_ul=155.0) + _record_check("realistic: mixed batch site direction mapping (NET3-1)", passed_site_mixed, msgs_site_mixed, node_names=["NET3-1"]) + + _mark_step("realistic: mixed batch circuit speed direction mapping") + passed_speed_dir, msgs_speed_dir = wait_for_circuit_direction_ceil(speed_id) + _record_check("realistic: mixed batch circuit speed direction mapping", passed_speed_dir, msgs_speed_dir, circuit_ids=[speed_id], step_started_at=step_t0) + + _mark_step("realistic: mixed batch circuit IP mappings") + passed_ip_map, msgs_ip_map = check_ip_mappings_for_circuit(ip_id) + _record_check("realistic: mixed batch circuit IP mappings", passed_ip_map, msgs_ip_map, circuit_ids=[ip_id]) + passed_old_ip_absent, msgs_old_ip_absent = check_ip_mapping_absent(old_ip) + _record_check("realistic: mixed batch old IP absent", passed_old_ip_absent, msgs_old_ip_absent, circuit_ids=[ip_id]) + + _mark_step("realistic: mixed batch exact IP mappings") + passed_ip_exact, msgs_ip_exact = check_exact_ip_mappings_for_circuit(ip_id) + _record_check("realistic: mixed batch exact IP mappings", passed_ip_exact, msgs_ip_exact, circuit_ids=[ip_id]) + + _mark_step("realistic: mixed batch circuit SQM kind") + passed_sqm_kind, msgs_sqm_kind = check_circuit_qdisc_kind( + sqm_id, + expected_down_kind="fq_codel", + expected_up_kind="cake", + ) + _record_check("realistic: mixed batch circuit SQM kind", passed_sqm_kind, msgs_sqm_kind, circuit_ids=[sqm_id]) + + _mark_step("realistic: mixed batch parent-moved circuit direction mapping") + passed_mixed_move_dir, msgs_mixed_move_dir = wait_for_circuit_direction_ceil(mixed_move_id) + _record_check("realistic: mixed batch parent-moved circuit direction mapping", passed_mixed_move_dir, msgs_mixed_move_dir, circuit_ids=[mixed_move_id], step_started_at=step_t0) + + _mark_step("realistic: mixed batch parent-moved circuit ip mappings") + passed_mixed_move_map, msgs_mixed_move_map = check_ip_mappings_for_circuit(mixed_move_id) + _record_check("realistic: mixed batch parent-moved circuit ip mappings", passed_mixed_move_map, msgs_mixed_move_map, circuit_ids=[mixed_move_id]) + + _mark_step("realistic: mixed batch parent-moved circuit cleanup") + passed_mixed_move_cleanup, msgs_mixed_move_cleanup = check_circuit_transition_cleanup( + mixed_move_id, + old_down_tc=mixed_old_down_tc, + old_up_tc=mixed_old_up_tc, + ) + _record_check("realistic: mixed batch parent-moved circuit cleanup", passed_mixed_move_cleanup, msgs_mixed_move_cleanup, circuit_ids=[mixed_move_id]) + + _mark_step("realistic: mixed batch exact IP mappings") + passed_mixed_batch_ip, msgs_mixed_batch_ip = check_exact_ip_mappings_for_circuits( + [speed_id, ip_id, sqm_id, mixed_move_id], + forbidden_by_circuit={ + mixed_move_id: {tc for tc in [mixed_old_down_tc, mixed_old_up_tc] if tc}, + }, + ) + _record_check( + "realistic: mixed batch exact IP mappings", + passed_mixed_batch_ip, + msgs_mixed_batch_ip, + circuit_ids=[speed_id, ip_id, sqm_id, mixed_move_id], + ) # Add circuit under a different leaf (NET3-2) _mark_step("realistic: add circuit") - rows3 = json.loads(json.dumps(rows2)) + rows3 = json.loads(json.dumps(rows_mixed)) new_circuit_id = "99901" new_circuit_ip = "100.64.200.1" rows3.append({ @@ -1889,11 +3808,11 @@ def run_realistic_tiered_suite(log: LogReader, timeout_s: float, results: List[s results.append("realistic: add circuit: full reload (allowed for larger tree)") else: results.append("realistic: add circuit: OK (no full reload)") + _record_hidden_health("realistic: add circuit", step_t0, circuit_ids=[new_circuit_id]) _mark_step("realistic: added circuit ip mappings") passed_add_map, msgs_add_map = check_ip_mappings_for_circuit(new_circuit_id) - results.extend(msgs_add_map) - ok &= passed_add_map + _record_check("realistic: added circuit ip mappings", passed_add_map, msgs_add_map, circuit_ids=[new_circuit_id]) # Remove the added circuit _mark_step("realistic: remove circuit") @@ -1905,16 +3824,273 @@ def run_realistic_tiered_suite(log: LogReader, timeout_s: float, results: List[s results.append("realistic: remove circuit: full reload (allowed for larger tree)") else: results.append("realistic: remove circuit: OK (no full reload)") + _record_hidden_health("realistic: remove circuit", step_t0, circuit_ids=[new_circuit_id]) _mark_step("realistic: removed circuit ip unmapped") passed_unmap, msgs_unmap = check_ip_mapping_absent(new_circuit_ip) - results.extend(msgs_unmap) - ok &= passed_unmap + _record_check("realistic: removed circuit ip unmapped", passed_unmap, msgs_unmap, circuit_ids=[new_circuit_id]) return ok -def run_flat_suite(log: LogReader, timeout_s: float, results: List[str]) -> bool: +def run_fault_reload_suite(log: AnyLogReader, timeout_s: float, results: List[str]) -> bool: + ok = True + + def _record_check( + tag: str, + passed: bool, + msgs: List[str], + *, + step_started_at: Optional[float] = None, + circuit_ids: Optional[List[str]] = None, + node_names: Optional[List[str]] = None, + ) -> None: + nonlocal ok + results.extend(msgs) + ok &= passed + if not passed: + results.extend( + collect_failure_diagnostics( + tag, + step_started_at=step_started_at, + circuit_ids=circuit_ids, + node_names=node_names, + ) + ) + + clear_bakery_fault_once() + try: + _mark_step("fault-reload: baseline") + write_network_json(REALISTIC_TIERED_NETWORK) + rows = realistic_tiered_circuits_base() + write_circuits(rows) + _ = run_refresh_and_wait(log, timeout_s) + results.append("fault-reload: baseline: ok (initial run)") + + target_idx: Optional[int] = None + for idx, row in enumerate(rows): + if str(row.get("Parent Node")) == "NET1-1-1": + target_idx = idx + break + if target_idx is None: + results.append("fault-reload: no candidate circuit found under NET1-1-1") + return False + + target_id = str(rows[target_idx]["Circuit ID"]) + old_down_tc, old_up_tc, _ = _current_circuit_handle_strings(target_id) + + rows_fault = json.loads(json.dumps(rows)) + rows_fault[target_idx]["Parent Node"] = "NET1-1-2" + arm_bakery_fault_once("migrating circuits between parent nodes") + + _mark_step("fault-reload: injected parent-move failure") + write_circuits(rows_fault) + step_t0 = time.time() + res = run_refresh_and_wait(log, timeout_s) + if res.full_reload and not res.mq_init: + results.append("fault-reload: injected failure triggered an immediate full reload") + else: + results.append("fault-reload: injected failure commit completed without immediate full reload") + + passed_reload_required, msgs_reload_required = check_fault_injection_observed(step_t0) + _record_check( + "fault-reload: fault injection observed", + passed_reload_required, + msgs_reload_required, + step_started_at=step_t0, + circuit_ids=[target_id], + ) + + rows_rebuild = json.loads(json.dumps(rows_fault)) + rows_rebuild[target_idx]["Download Max Mbps"] = float(rows_rebuild[target_idx]["Download Max Mbps"]) + 7.0 + rows_rebuild[target_idx]["Upload Max Mbps"] = float(rows_rebuild[target_idx]["Upload Max Mbps"]) + 3.0 + + _mark_step("fault-reload: follow-up commit forces full reload") + write_circuits(rows_rebuild) + step_t1 = time.time() + res = run_refresh_and_wait(log, timeout_s) + passed_full, msg_full = assert_full_reload( + "fault-reload: follow-up commit forces full reload", + res, + step_started_at=step_t1, + ) + results.append(msg_full) + ok &= passed_full + if not passed_full: + results.extend( + collect_failure_diagnostics( + "fault-reload: follow-up commit forces full reload", + step_started_at=step_t1, + circuit_ids=[target_id], + ) + ) + + passed_reload_gate, msgs_reload_gate = check_reload_required_since(step_t0) + _record_check( + "fault-reload: reload-required gate observed", + passed_reload_gate, + msgs_reload_gate, + step_started_at=step_t0, + circuit_ids=[target_id], + ) + + _mark_step("fault-reload: post-reload circuit direction mapping") + passed_dir, msgs_dir = wait_for_circuit_direction_ceil(target_id) + _record_check( + "fault-reload: post-reload circuit direction mapping", + passed_dir, + msgs_dir, + circuit_ids=[target_id], + step_started_at=step_t1, + ) + + _mark_step("fault-reload: post-reload exact IP mappings") + passed_ip, msgs_ip = check_exact_ip_mappings_for_circuit( + target_id, + forbidden_tcs={tc for tc in [old_down_tc, old_up_tc] if tc}, + ) + _record_check( + "fault-reload: post-reload exact IP mappings", + passed_ip, + msgs_ip, + circuit_ids=[target_id], + ) + + _mark_step("fault-reload: post-reload cleanup") + passed_cleanup, msgs_cleanup = check_circuit_transition_cleanup( + target_id, + old_down_tc=old_down_tc, + old_up_tc=old_up_tc, + ) + _record_check( + "fault-reload: post-reload cleanup", + passed_cleanup, + msgs_cleanup, + circuit_ids=[target_id], + ) + + rows_recovered = json.loads(json.dumps(rows_rebuild)) + rows_recovered[target_idx]["Download Max Mbps"] = float(rows_recovered[target_idx]["Download Max Mbps"]) + 5.0 + rows_recovered[target_idx]["Upload Max Mbps"] = float(rows_recovered[target_idx]["Upload Max Mbps"]) + 2.0 + + _mark_step("fault-reload: post-reload incremental recovery") + write_circuits(rows_recovered) + step_t2 = time.time() + res = run_refresh_and_wait(log, timeout_s) + passed_recovery, msg_recovery = assert_no_full_reload( + "fault-reload: post-reload incremental recovery", + res, + step_started_at=step_t2, + ) + results.append(msg_recovery) + ok &= passed_recovery + if not passed_recovery: + results.extend( + collect_failure_diagnostics( + "fault-reload: post-reload incremental recovery", + step_started_at=step_t2, + circuit_ids=[target_id], + ) + ) + + passed_health, msgs_health = check_no_hidden_incremental_failures(step_t2) + _record_check( + "fault-reload: post-reload incremental recovery health", + passed_health, + msgs_health, + step_started_at=step_t2, + circuit_ids=[target_id], + ) + + return ok + finally: + clear_bakery_fault_once() + + +def run_treeguard_runtime_suite( + log: AnyLogReader, timeout_s: float, treeguard_timeout_s: float, results: List[str] +) -> bool: + ok = True + cpu_mode = get_treeguard_cpu_mode() + fixture = treeguard_runtime_fixture_metadata() + + _mark_step("treeguard: baseline") + write_network_json(treeguard_runtime_network_base()) + rows = treeguard_runtime_circuits_base() + write_circuits(rows) + _ = run_refresh_and_wait(log, timeout_s) + results.append( + "treeguard: baseline: ok " + f"({fixture['total_circuits']} circuits across {TREEGUARD_RUNTIME_REGION_COUNT} top-level nodes)" + ) + + after_baseline_offset = log.snapshot() + + _mark_step("treeguard: wait for runtime virtualization") + passed, msgs = wait_for_runtime_virtualized_node( + str(fixture["virtualized_region"]), + str(fixture["promoted_pop"]), + [str(fixture["promoted_pop"])], + [ + ( + "promoted-branch", + str(fixture["promoted_leaf_circuit"]), + [str(fixture["promoted_pop"]), str(fixture["promoted_leaf"])], + ), + ( + "sibling-branch", + str(fixture["sibling_leaf_circuit"]), + [str(fixture["sibling_pop"]), str(fixture["sibling_leaf"])], + ), + ], + treeguard_timeout_s, + ) + if passed: + results.extend(msgs) + ok &= passed + elif cpu_mode == "cpu_aware": + still_physical, still_physical_msgs = check_node_present_in_physical_tree( + str(fixture["virtualized_region"]) + ) + results.extend(still_physical_msgs) + results.append( + "treeguard runtime check: SKIP under cpu_aware because no runtime virtualization occurred without induced CPU pressure" + ) + ok &= still_physical + else: + results.extend(msgs) + ok &= passed + + _, new_lines = log.read_since(after_baseline_offset) + if _log_contains_unexpected_full_reload(new_lines): + results.append("treeguard: runtime virtualization: unexpected full reload detected") + ok = False + else: + results.append("treeguard: runtime virtualization: OK (no full reload)") + + _mark_step("treeguard: promoted branch circuit ip mappings") + passed_map_promoted, msgs_map_promoted = check_ip_mappings_for_circuit( + str(fixture["promoted_leaf_circuit"]) + ) + results.extend(msgs_map_promoted) + ok &= passed_map_promoted + + _mark_step("treeguard: sibling branch circuit ip mappings") + passed_map_sibling, msgs_map_sibling = check_ip_mappings_for_circuit( + str(fixture["sibling_leaf_circuit"]) + ) + results.extend(msgs_map_sibling) + ok &= passed_map_sibling + + _mark_step("treeguard: low-value sibling remains physical") + passed_small, msgs_small = check_node_present_in_physical_tree(str(fixture["low_value_sibling"])) + results.extend(msgs_small) + ok &= passed_small + + return ok + + +def run_flat_suite(log: AnyLogReader, timeout_s: float, results: List[str]) -> bool: ok = True # Baseline @@ -1982,6 +4158,12 @@ def run_flat_suite(log: LogReader, timeout_s: float, results: List[str]) -> bool return ok +def write_json_report(path: str, payload: dict) -> None: + report_path = Path(path) + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + + # ----------------------- # Main # ----------------------- @@ -1989,29 +4171,56 @@ def run_flat_suite(log: LogReader, timeout_s: float, results: List[str]) -> bool def main() -> int: ap = argparse.ArgumentParser(description="LibreQoS/Bakery integration tests") - ap.add_argument("--log-file", required=True, help="Path to lqosd log file (stdout/tee capture)") - ap.add_argument("--timeout", type=float, default=25.0, help="Wait time per step (seconds)") + ap.add_argument( + "--log-file", + help="Path to lqosd log file (stdout/tee capture). When omitted, the harness reads journalctl -u lqosd.", + ) + ap.add_argument("--timeout", type=float, default=8.0, help="Wait time per step (seconds)") + ap.add_argument( + "--treeguard-timeout", + type=float, + default=90.0, + help="Max wait time for TreeGuard runtime virtualization in the dedicated TreeGuard suite", + ) g = ap.add_mutually_exclusive_group() + g.add_argument("--full-suite", action="store_true", help="Run the full harness (slower)") g.add_argument("--tiered-only", action="store_true", help="Run tiered cases only") g.add_argument("--flat-only", action="store_true", help="Run flat cases only") g.add_argument("--realistic-only", action="store_true", help="Run realistic tiered cases only") + g.add_argument("--virtualized-only", action="store_true", help="Run virtualized-node tiered cases only") + g.add_argument("--treeguard-only", action="store_true", help="Run live TreeGuard runtime cases only") + g.add_argument("--queue-mode-only", action="store_true", help="Run live Observe/Shape queue-mode toggle checks only") + g.add_argument( + "--fault-reload-only", + action="store_true", + help="Run the opt-in Bakery fault-injection reload-escalation suite only", + ) ap.add_argument("--no-restore", action="store_true", help="Do not restore original files (debugging)") + ap.add_argument( + "--json-report", + help="Write a machine-readable summary JSON report to this path", + ) args = ap.parse_args() - # Pre-flight guidance - if not os.path.exists(args.log_file): - print("ERROR: Log file not found. Please run lqosd with logs, e.g.:") - print(" sudo RUST_LOG=info lqosd 2>&1 | tee /tmp/lqosd.log") - return 1 - if not is_lqosd_alive(): print("WARNING: lqosd does not appear to be running. This test will not observe Bakery logs.") print("Start lqosd and re-run, or continue if already started but lib binding cannot detect it.") - log = LogReader(args.log_file) + log: AnyLogReader + if args.log_file: + if not os.path.exists(args.log_file): + print(f"ERROR: Log file not found: {args.log_file}") + print("Either point --log-file at an existing tee'd log, or omit it to use journalctl -u lqosd.") + return 1 + print(f"Using tee'd lqosd log file: {args.log_file}") + log = LogReader(args.log_file) + else: + print("Using journalctl -u lqosd for Bakery event detection") + log = JournalctlLogReader("lqosd") results: List[str] = [] overall_ok = True + selected_suite = "tiered-only" files = ["network.json", "ShapedDevices.csv"] ctx = contextlib.nullcontext() @@ -2020,24 +4229,65 @@ def main() -> int: with ctx: if args.tiered_only: + selected_suite = "tiered-only" ok = run_tiered_suite(log, args.timeout, results) overall_ok &= ok + elif args.virtualized_only: + selected_suite = "virtualized-only" + ok = run_virtualized_tiered_suite(log, args.timeout, results) + overall_ok &= ok elif args.flat_only: + selected_suite = "flat-only" ok = run_flat_suite(log, args.timeout, results) overall_ok &= ok elif args.realistic_only: + selected_suite = "realistic-only" ok = run_realistic_tiered_suite(log, args.timeout, results) overall_ok &= ok - else: + elif args.treeguard_only: + selected_suite = "treeguard-only" + ok = run_treeguard_runtime_suite( + log, args.timeout, args.treeguard_timeout, results + ) + overall_ok &= ok + elif args.queue_mode_only: + selected_suite = "queue-mode-only" + ok = run_queue_mode_suite(log, args.timeout, results) + overall_ok &= ok + elif args.fault_reload_only: + selected_suite = "fault-reload-only" + ok = run_fault_reload_suite(log, args.timeout, results) + overall_ok &= ok + elif args.full_suite: + selected_suite = "full-suite" ok = run_tiered_suite(log, args.timeout, results) overall_ok &= ok + ok = run_virtualized_tiered_suite(log, args.timeout, results) + overall_ok &= ok + ok = run_flat_suite(log, args.timeout, results) overall_ok &= ok ok = run_realistic_tiered_suite(log, args.timeout, results) overall_ok &= ok + else: + selected_suite = "tiered-only" + ok = run_tiered_suite(log, args.timeout, results) + overall_ok &= ok + + report_payload = { + "suite": selected_suite, + "log_source": args.log_file or "journalctl -u lqosd", + "timeout_seconds": args.timeout, + "treeguard_timeout_seconds": args.treeguard_timeout, + "results": results, + "overall_ok": overall_ok, + } + if args.json_report: + write_json_report(args.json_report, report_payload) + print("Results:") for r in results: print(f"- {r}") diff --git a/src/bin/lqos_api.service.example b/src/bin/lqos_api.service.example index ef3b2ed7..3e84f52c 100644 --- a/src/bin/lqos_api.service.example +++ b/src/bin/lqos_api.service.example @@ -1,5 +1,6 @@ [Unit] -After=network.service lqosd.service +Wants=network-online.target +After=network-online.target systemd-resolved.service lqosd.service Requires=lqosd.service [Service] @@ -9,4 +10,3 @@ Restart=always [Install] WantedBy=default.target - diff --git a/src/bin/lqos_scheduler.service.example b/src/bin/lqos_scheduler.service.example index 569c64ba..c1e766dc 100644 --- a/src/bin/lqos_scheduler.service.example +++ b/src/bin/lqos_scheduler.service.example @@ -1,5 +1,6 @@ [Unit] -After=network.service lqosd.service +Wants=network-online.target +After=network-online.target systemd-resolved.service lqosd.service Requires=lqosd.service [Service] diff --git a/src/bin/lqosd.service.example b/src/bin/lqosd.service.example index 18e60681..003baad7 100644 --- a/src/bin/lqosd.service.example +++ b/src/bin/lqosd.service.example @@ -1,5 +1,6 @@ [Unit] -After=network.service +Wants=network-online.target +After=network-online.target systemd-resolved.service [Service] WorkingDirectory=/opt/libreqos/src/bin diff --git a/src/build_dpkg.sh b/src/build_dpkg.sh index 42e4f798..276e528b 100755 --- a/src/build_dpkg.sh +++ b/src/build_dpkg.sh @@ -45,7 +45,6 @@ LQOS_FILES=( manualNetwork.template.csv integrationUISProutes.template.csv integrationSplynxBandwidths.template.csv - deb-requirements-constraints.txt ../requirements.txt update_api.sh ) @@ -125,7 +124,7 @@ LibreQoS detected the Ubuntu 24.04 systemd-networkd hotfix requirement on this h Run: sudo /opt/libreqos/src/systemd_hotfix.sh install -The hotfix installer downloads from https://download.libreqos.com and will offer to schedule a reboot. +The hotfix installer bootstraps the LibreQoS APT repo at https://repo.libreqos.com and will offer to schedule a reboot. After the reboot, finish the LibreQoS package configuration with: sudo dpkg --configure -a @@ -136,7 +135,11 @@ fi # Install Python Dependencies pushd /opt/libreqos > /dev/null # - Setup Python dependencies as a post-install task +if [ -s src/deb-requirements-constraints.txt ]; then PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install -c src/deb-requirements-constraints.txt -r src/requirements.txt +else +PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install -r src/requirements.txt +fi # - Setup Python dependencies as a post-install task - handle issue with packages on Ubuntu Server 24.04 PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip uninstall apscheduler deepdiff --yes PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install apscheduler deepdiff @@ -147,14 +150,14 @@ sudo chown -R root:root /opt/libreqos # - Run lqsetup /opt/libreqos/src/bin/lqos_setup # - Setup the services -cp /opt/libreqos/src/bin/lqosd.service.example /etc/systemd/system/lqosd.service -cp /opt/libreqos/src/bin/lqos_scheduler.service.example /etc/systemd/system/lqos_scheduler.service -cp /opt/libreqos/src/bin/lqos_api.service.example /etc/systemd/system/lqos_api.service +install -m 0644 /opt/libreqos/src/bin/lqosd.service.example /etc/systemd/system/lqosd.service +install -m 0644 /opt/libreqos/src/bin/lqos_scheduler.service.example /etc/systemd/system/lqos_scheduler.service +install -m 0644 /opt/libreqos/src/bin/lqos_api.service.example /etc/systemd/system/lqos_api.service /bin/systemctl daemon-reload || true /bin/systemctl stop lqos_node_manager || true # In case it's running from a previous release /bin/systemctl disable lqos_node_manager || true # In case it's running from a previous release /bin/systemctl enable lqosd lqos_scheduler lqos_api || true -/bin/systemctl start lqosd lqos_scheduler lqos_api || true +/bin/systemctl restart lqosd lqos_scheduler lqos_api || true popd > /dev/null EOF @@ -175,6 +178,10 @@ for file in "${LQOS_FILES[@]}"; do cp "$file" "$LQOS_DIR" || echo "Error copying $file" done +if [ -f deb-requirements-constraints.txt ]; then + cp deb-requirements-constraints.txt "$LQOS_DIR" || echo "Error copying deb-requirements-constraints.txt" +fi + # Ensure helper scripts are executable in the package if [ -f "$LQOS_DIR/update_api.sh" ]; then chmod a+x "$LQOS_DIR/update_api.sh" || true diff --git a/src/build_rust.sh b/src/build_rust.sh index 1a996269..dc8e9411 100755 --- a/src/build_rust.sh +++ b/src/build_rust.sh @@ -114,12 +114,48 @@ service_exists() { fi } +refresh_service_unit() { + local unit_name=$1 + local src="./bin/${unit_name}.service.example" + local dst="/etc/systemd/system/${unit_name}.service" + if [ -f "$src" ] && [ -f "$dst" ]; then + echo "Refreshing $dst from $src" + sudo cp "$src" "$dst" + SERVICE_UNITS_UPDATED=1 + fi +} + +clear_pinned_maps_before_lqosd_restart() { + local script_path="./rust/remove_pinned_maps.sh" + if [ ! -x "$script_path" ]; then + echo "Expected $script_path to exist and be executable before restarting lqosd." + exit 1 + fi + + echo "Removing pinned BPF maps before restarting lqosd." + if ! sudo "$script_path"; then + echo "Failed to remove pinned maps. Skipping service restarts to avoid restarting lqosd with stale map state." + exit 1 + fi +} + +SERVICE_UNITS_UPDATED=0 +refresh_service_unit lqosd +refresh_service_unit lqos_scheduler +refresh_service_unit lqos_api + +if [ "$SERVICE_UNITS_UPDATED" -eq 1 ]; then + echo "Reloading systemd unit definitions." + sudo systemctl daemon-reload +fi + if service_exists lqos_node_manager; then echo "lqos_node_manager is running as a service. It's not needed anymore. Killing it." sudo systemctl stop lqos_node_manager sudo systemctl disable lqos_node_manager fi if service_exists lqosd; then + clear_pinned_maps_before_lqosd_restart echo "lqosd is running as a service. Restarting it. You may need to enter your sudo password." sudo systemctl restart lqosd fi @@ -135,10 +171,7 @@ fi echo "-----------------------------------------------------------------" echo "Don't forget to setup /etc/lqos.conf!" echo "Template .service files can be found in bin/" -echo "Debian package installs constrain Python dependencies with src/deb-requirements-constraints.txt to stay compatible with Ubuntu's distro Python packages." -echo "Use ./systemd_hotfix.sh to evaluate or install the Ubuntu 24.04 networkd hotfix bundle from download.libreqos.com." +echo "If src/deb-requirements-constraints.txt exists, Debian package installs use it to constrain Python dependencies." +echo "Use ./systemd_hotfix.sh to evaluate or install the Ubuntu 24.04 networkd hotfix from the LibreQoS APT repo at https://repo.libreqos.com." echo "The hotfix installer now offers to schedule a reboot after it finishes." echo "LibreQoS package installs on affected Ubuntu 24.04 hosts stop until the hotfix is installed; finish with sudo dpkg --configure -a after the reboot." -echo "" -echo "Run sudo rust/remove_pinned_maps.sh before you restart lqosd" -echo "This ensures that any data-format changes will apply correctly." diff --git a/src/integrationCommon.py b/src/integrationCommon.py index 9ea9e6d9..5e5f9d4a 100644 --- a/src/integrationCommon.py +++ b/src/integrationCommon.py @@ -41,6 +41,18 @@ def isIpv4Permitted(inputIP): return isInIgnoredSubnets(inputIP) == False and isInAllowedSubnets(inputIP) +def isIpPermitted(inputIP): + # Generic form of subnet permission checking used by integrations. + # The configured allow/ignore ranges may include either IPv4 or IPv6. + return isIpv4Permitted(inputIP) + + +def isIntegrationOutputIpAllowed(inputIP): + # Shared integration-output pruning is intentionally narrower than + # full "permitted" checks: only explicitly ignored subnets are removed. + return not isInIgnoredSubnets(inputIP) + + def fixSubnet(inputIP): # If an IP address has a CIDR other than /32 (e.g. 192.168.1.1/24), # but doesn't appear as a network address (e.g. 192.168.1.0/24) @@ -252,6 +264,7 @@ class NetworkGraph: # O(1) lookup instead of O(n) search parentIdx = self._id_to_index.get(parent, 0) node.parentIndex = parentIdx + node.parentId = parent idx = len(self.nodes) self.nodes.append(node) @@ -269,6 +282,7 @@ class NetworkGraph: root_list_set = set(cached_root_list) for child in self.nodes: + child.parentIndex = 0 if child.parentId != "": # O(1) lookup instead of O(n) search parent_idx = self._id_to_index.get(child.parentId, -1) @@ -330,26 +344,28 @@ class NetworkGraph: if not self._cache_valid: self._buildChildrenCache() - for i, node in enumerate(self.nodes): - if node.type == NodeType.clientWithChildren: - siteNode = NetworkNode( - id=str(node.id) + "_gen", - displayName="(Generated Site) " + node.displayName, - type=NodeType.site, - networkJsonId=syntheticNetworkJsonId("graph", "site", node.displayName), - ) - siteNode.parentIndex = node.parentIndex - node.parentId = siteNode.id - node.type = NodeType.client - - # Store reparenting operations for batch processing - children = self._children_cache.get(i, []) - for child_idx in children: - child = self.nodes[child_idx] - if child.type in (NodeType.client, NodeType.clientWithChildren, NodeType.site): - reparent_map[child_idx] = siteNode.id - - toAdd.append(siteNode) + for i, node in enumerate(self.nodes): + if node.type == NodeType.clientWithChildren: + original_parent_id = node.parentId + siteNode = NetworkNode( + id=str(node.id) + "_gen", + displayName="(Generated Site) " + node.displayName, + type=NodeType.site, + networkJsonId=syntheticNetworkJsonId("graph", "site", node.displayName), + ) + siteNode.parentIndex = node.parentIndex + siteNode.parentId = original_parent_id + node.parentId = siteNode.id + node.type = NodeType.client + + # Store reparenting operations for batch processing + children = self._children_cache.get(i, []) + for child_idx in children: + child = self.nodes[child_idx] + if child.type in (NodeType.client, NodeType.clientWithChildren, NodeType.site): + reparent_map[child_idx] = siteNode.id + + toAdd.append(siteNode) # Batch add new nodes for n in toAdd: @@ -362,12 +378,12 @@ class NetworkGraph: self.__reparentById() def __findUnconnectedNodes(self) -> List: - # OPTIMIZED: Uses set for O(1) membership testing + # Rebuild the cache each time because callers may have adjusted + # parentIndex directly as part of repair/testing flows. visited = set() next = [0] - if not self._cache_valid: - self._buildChildrenCache() + self._buildChildrenCache() while next: current = next.pop() @@ -422,6 +438,8 @@ class NetworkGraph: self.__clientsWithChildrenToSites() print("PrepareTree: Reconnecting unconnected nodes...") self.__reconnectUnconnected() + print("PrepareTree: Pruning ignored-only devices and empty circuits...") + self.__pruneIgnoredCircuits() print("PrepareTree: Complete") def doesNetworkJsonExist(self): @@ -502,6 +520,55 @@ class NetworkGraph: if ip in self.ipv4ToIPv6.keys(): ipv6.append(self.ipv4ToIPv6[ip]) + def __rebuildIndexes(self) -> None: + self._id_to_index = {} + self._name_to_index = {} + for idx, node in enumerate(self.nodes): + self._id_to_index[node.id] = idx + self._name_to_index[node.displayName] = idx + self._cache_valid = False + + def __filterPermittedIps(self, ip_list: List[str]) -> List[str]: + return [ip for ip in ip_list if isIntegrationOutputIpAllowed(ip)] + + def __pruneIgnoredCircuits(self) -> None: + for node in self.nodes: + if node.type == NodeType.device: + node.ipv4 = self.__filterPermittedIps(node.ipv4) + node.ipv6 = self.__filterPermittedIps(node.ipv6) + + if not self._cache_valid: + self._buildChildrenCache() + + removable = set() + for idx, node in enumerate(self.nodes): + if node.type == NodeType.device and (len(node.ipv4) + len(node.ipv6) == 0): + removable.add(idx) + + for idx, node in enumerate(self.nodes): + if node.type != NodeType.client: + continue + has_shapable_device = False + for child_idx in self._children_cache.get(idx, []): + if child_idx in removable: + continue + child = self.nodes[child_idx] + if child.type == NodeType.device and (len(child.ipv4) + len(child.ipv6) > 0): + has_shapable_device = True + break + if not has_shapable_device: + removable.add(idx) + + if not removable: + return + + self.nodes = [ + node for idx, node in enumerate(self.nodes) + if idx not in removable + ] + self.__rebuildIndexes() + self.__reparentById() + def createShapedDevices(self): import csv # Builds ShapedDevices.csv from the network tree. diff --git a/src/integrationSonar.py b/src/integrationSonar.py index 143bcfc7..581e45b6 100644 --- a/src/integrationSonar.py +++ b/src/integrationSonar.py @@ -2,8 +2,11 @@ from pythonCheck import checkPythonVersion checkPythonVersion() import requests import subprocess +import time +from urllib.parse import urlsplit, urlunsplit from liblqos_python import sonar_api_key, sonar_api_url, snmp_community, sonar_airmax_ap_model_ids, \ - sonar_ltu_ap_model_ids, sonar_active_status_ids + sonar_ltu_ap_model_ids, sonar_active_status_ids, sonar_recurring_service_rates, \ + sonar_recurring_excluded_service_names all_models = sonar_airmax_ap_model_ids() + sonar_ltu_ap_model_ids() from integrationCommon import NetworkGraph, NetworkNode, NodeType, apply_client_bandwidth_multiplier from multiprocessing.pool import ThreadPool @@ -24,18 +27,161 @@ from multiprocessing.pool import ThreadPool # There should probably be a log file somewhere and output status to the Web UI. # If snmp fails to get the name of the AP then it's just called "Not found via snmp" We won't see that happen unless it is able to get the connected cpe and not the name. +_SONAR_SESSION = None +SONAR_CONNECT_TIMEOUT_SECONDS = 10 +SONAR_READ_TIMEOUT_SECONDS = 60 +SONAR_REQUEST_TIMEOUT = (SONAR_CONNECT_TIMEOUT_SECONDS, SONAR_READ_TIMEOUT_SECONDS) +SONAR_PAGINATED_MAX_ATTEMPTS = 3 +SONAR_PAGINATED_RETRY_BACKOFF_SECONDS = (2, 5) -def sonarRequest(query,variables={}): - r = requests.post(sonar_api_url(), json={'query': query, 'variables': variables}, headers={'Authorization': 'Bearer ' + sonar_api_key()}, timeout=10) - r_json = r.json() +def sonarGraphqlUrl(): + raw_url = sonar_api_url().strip() + if not raw_url: + return raw_url + + parsed = urlsplit(raw_url) + path = parsed.path.rstrip('/') + + if path.endswith('/api/graphql'): + normalized_path = path + elif path in ('', '/'): + normalized_path = '/api/graphql' + else: + normalized_path = path + '/api/graphql' + + return urlunsplit((parsed.scheme, parsed.netloc, normalized_path, parsed.query, parsed.fragment)) + + +def sonarAccountNodeId(account_id): + return f"sonar:account:{account_id}" + + +def sonarDeviceNodeId(device_id): + return f"sonar:device:{device_id}" + + +def sonarRadiusAccountDeviceNodeId(radius_account_id): + return f"sonar:radius-account:{radius_account_id}" + + +def sonarSession(): + global _SONAR_SESSION + if _SONAR_SESSION is None: + _SONAR_SESSION = requests.Session() + _SONAR_SESSION.headers.update({ + 'Authorization': 'Bearer ' + sonar_api_key() + }) + return _SONAR_SESSION + + +def sonarPostWithRetry(session, graphql_url, payload, timeout, request_label, max_attempts=1, retry_backoffs=()): + last_error = None + for attempt in range(1, max_attempts + 1): + try: + return session.post( + graphql_url, + json=payload, + timeout=timeout + ) + except requests.exceptions.ReadTimeout as error: + last_error = error + if attempt >= max_attempts: + break + backoff_seconds = 0 + if retry_backoffs: + backoff_index = min(attempt - 1, len(retry_backoffs) - 1) + backoff_seconds = retry_backoffs[backoff_index] + if backoff_seconds > 0: + time.sleep(backoff_seconds) + + raise RuntimeError( + f"Sonar API at {graphql_url} timed out while reading {request_label} " + f"after {max_attempts} attempts" + ) from last_error + + +def sonarRequest(query, variables=None): + if variables is None: + variables = {} + + graphql_url = sonarGraphqlUrl() + r = sonarPostWithRetry( + sonarSession(), + graphql_url, + {'query': query, 'variables': variables}, + SONAR_REQUEST_TIMEOUT, + "Sonar request" + ) + try: + r_json = r.json() + except requests.exceptions.JSONDecodeError as e: + body_preview = r.text[:300].strip() + raise RuntimeError( + f"Sonar API at {graphql_url} returned non-JSON content " + f"(status={r.status_code}, content-type={r.headers.get('content-type')}, body-preview={body_preview!r})" + ) from e # Sonar responses look like this: {"data": {"accounts": {"entities": [{"id": '1'},{"id": 2}]}}} # I just want to return the list so I need to find what field we're querying. + if 'errors' in r_json: + raise RuntimeError(f"Sonar GraphQL error from {graphql_url}: {r_json['errors']}") + if 'data' not in r_json: + raise RuntimeError(f"Sonar API response from {graphql_url} missing 'data': {r_json}") field = list(r_json['data'].keys())[0] sonar_list = r_json['data'][field]['entities'] return sonar_list + +def sonarPaginatedRequest(query, variables=None, paginator_key='pages', records_per_page=100): + if variables is None: + variables = {} + + session = sonarSession() + graphql_url = sonarGraphqlUrl() + page = 1 + entities = [] + + while True: + request_variables = dict(variables) + request_variables[paginator_key] = { + 'records_per_page': records_per_page, + 'page': page + } + r = sonarPostWithRetry( + session, + graphql_url, + {'query': query, 'variables': request_variables}, + SONAR_REQUEST_TIMEOUT, + f"Sonar page {page}", + max_attempts=SONAR_PAGINATED_MAX_ATTEMPTS, + retry_backoffs=SONAR_PAGINATED_RETRY_BACKOFF_SECONDS, + ) + try: + r_json = r.json() + except requests.exceptions.JSONDecodeError as e: + body_preview = r.text[:300].strip() + raise RuntimeError( + f"Sonar API at {graphql_url} returned non-JSON content " + f"(status={r.status_code}, content-type={r.headers.get('content-type')}, body-preview={body_preview!r})" + ) from e + + if 'errors' in r_json: + raise RuntimeError(f"Sonar GraphQL error from {graphql_url}: {r_json['errors']}") + if 'data' not in r_json: + raise RuntimeError(f"Sonar API response from {graphql_url} missing 'data': {r_json}") + + field = list(r_json['data'].keys())[0] + connection = r_json['data'][field] + entities.extend(connection['entities']) + page_info = connection.get('page_info') or {} + total_pages = page_info.get('total_pages', page) + if page >= total_pages: + break + page += 1 + + return entities + def getActiveStatuses(): if not sonar_active_status_ids(): query = """query getActiveStatuses { @@ -58,15 +204,139 @@ def getActiveStatuses(): # Sometimes the IP will be under the field data for an item and sometimes it will be assigned to the inventory item itself. def findIPs(inventory_item): ips = [] - for ip in inventory_item['inventory_model_field_data']['entities'][0]['ip_assignments']['entities']: - ips.append(ip['subnet']) - for ip in inventory_item['ip_assignments']['entities']: + for field_data in inventory_item.get('inventory_model_field_data', {}).get('entities', []): + for ip in field_data.get('ip_assignments', {}).get('entities', []): + ips.append(ip['subnet']) + for ip in inventory_item.get('ip_assignments', {}).get('entities', []): ips.append(ip['subnet']) return ips + +def dedupeSubnets(subnets): + deduped = [] + seen = set() + for subnet in subnets: + if not subnet or subnet in seen: + continue + seen.add(subnet) + deduped.append(subnet) + return deduped + + +def normalizeServiceName(name): + return (name or "").strip().casefold() + + +def findRadiusAccountIPs(radius_account): + ips = [] + for ip in radius_account.get('ip_assignments', {}).get('entities', []): + subnet = ip.get('subnet') + if subnet: + ips.append(subnet) + return dedupeSubnets(ips) + + +def recurringServiceRateRules(): + rules = {} + for enabled, service_name, download_mbps, upload_mbps in sonar_recurring_service_rates(): + normalized_name = normalizeServiceName(service_name) + if not enabled or not normalized_name: + continue + rules[normalized_name] = ( + float(download_mbps), + float(upload_mbps), + ) + return rules + + +def recurringServiceExclusions(): + return { + normalizeServiceName(name) + for name in sonar_recurring_excluded_service_names() + if normalizeServiceName(name) + } + + +def resolveAccountBandwidth(account, recurring_rules=None, excluded_names=None): + account_services = account.get('account_services', {}).get('entities', []) + if account_services: + service = account_services[0].get('service') or {} + service_detail = service.get('data_service_detail') + if service_detail: + return ( + float(service_detail['download_speed_kilobits_per_second']) / 1000, + float(service_detail['upload_speed_kilobits_per_second']) / 1000, + ) + + if recurring_rules is None: + recurring_rules = recurringServiceRateRules() + if not recurring_rules: + return None + if excluded_names is None: + excluded_names = recurringServiceExclusions() + all_services = account.get('all_account_services', {}).get('entities', []) + for row in all_services: + service = row.get('service') or {} + if service.get('type') != 'RECURRING': + continue + normalized_name = normalizeServiceName(service.get('name')) + if not normalized_name or normalized_name in excluded_names: + continue + if normalized_name in recurring_rules: + return recurring_rules[normalized_name] + return None + + +def findPrimaryMac(inventory_item): + field_data_entities = inventory_item.get('inventory_model_field_data', {}).get('entities', []) + + def mac_candidates(primary_only): + for field_data in field_data_entities: + field = field_data.get('inventory_model_field') or {} + value = (field_data.get('value') or '').strip() + field_name = (field.get('name') or '').strip().lower() + if not value: + continue + if 'mac' not in field_name: + continue + if primary_only and not field.get('primary', False): + continue + yield value + + for value in mac_candidates(primary_only=True): + return value + for value in mac_candidates(primary_only=False): + return value + for field_data in field_data_entities: + value = (field_data.get('value') or '').strip() + if value: + return value + return "" + + +def formatAccountAddress(account): + addresses = account.get('addresses', {}).get('entities', []) + if not addresses: + return "" + primary_address = addresses[0] + line1 = primary_address.get('line1') or "" + line2 = primary_address.get('line2') or "" + city = primary_address.get('city') or "" + subdivision = primary_address.get('subdivision') or "" + state = subdivision[-2:] if subdivision else "" + + address = f"{line1},{f' {line2},' if line2 else ''} {city}, {state}".strip() + return address.strip(', ') + def getSitesAndAps(): query = """query getSitesAndAps($pages: Paginator, $rr_ap_models: ReverseRelationFilter, $ap_models: Search){ network_sites (paginator: $pages,reverse_relation_filters: [$rr_ap_models]) { + page_info { + page + records_per_page + total_pages + total_count + } entities { name id @@ -102,12 +372,7 @@ def getSitesAndAps(): "search_value": ap_id }) - variables = {"pages": - { - "records_per_page": 5, - "page": 1 - }, - "rr_ap_models": { + variables = {"rr_ap_models": { "relation": "inventory_items", "search": [{ "integer_fields": search_aps @@ -118,7 +383,7 @@ def getSitesAndAps(): } } - sites_and_aps = sonarRequest(query,variables) + sites_and_aps = sonarPaginatedRequest(query,variables) # This should only return sites that have equipment on them that is in the list sonar_ubiquiti_ap_model_ids in lqos.conf sites = [] aps = [] @@ -134,9 +399,123 @@ def getSitesAndAps(): sites.append({'id': f"site_{site['id']}", 'raw_id': site['id'], 'name': site['name']}) return sites, aps + +def buildAccountDevices(account): + devices = [] + known_ip_subnets = set() + + for address in account.get('addresses', {}).get('entities', []): + for item in address.get('inventory_items', {}).get('entities', []): + ips = dedupeSubnets(findIPs(item)) + known_ip_subnets.update(ips) + devices.append({ + 'id': sonarDeviceNodeId(item['id']), + 'raw_id': item['id'], + 'name': item['inventory_model']['name'], + 'ips': ips, + 'mac': findPrimaryMac(item) + }) + + for radius_account in account.get('radius_accounts', {}).get('entities', []): + radius_ips = [ + subnet for subnet in findRadiusAccountIPs(radius_account) + if subnet not in known_ip_subnets + ] + if not radius_ips: + continue + known_ip_subnets.update(radius_ips) + devices.append({ + 'id': sonarRadiusAccountDeviceNodeId(radius_account['id']), + 'raw_id': radius_account['id'], + 'name': f"Radius Account {radius_account['id']}", + 'ips': radius_ips, + 'mac': '' + }) + + return devices + + +def buildAccountRecord(account, fallback_address="", recurring_rules=None, excluded_names=None): + address = formatAccountAddress(account) or fallback_address + if not address: + return None + + devices = buildAccountDevices(account) + if not devices: + return None + + bandwidth = resolveAccountBandwidth( + account, + recurring_rules=recurring_rules, + excluded_names=excluded_names, + ) + if bandwidth is None: + return None + + download_raw, upload_raw = bandwidth + download = apply_client_bandwidth_multiplier(download_raw) + upload = apply_client_bandwidth_multiplier(upload_raw) + if download < 2: + download = 2 + if upload < 2: + upload = 2 + + return { + 'id': sonarAccountNodeId(account['id']), + 'raw_id': account['id'], + 'name': account['name'], + 'address': address, + 'download': download, + 'upload': upload, + 'devices': devices + } + + +def buildAccountsFromSonarEntities(accounts_from_sonar, recurring_rules=None, excluded_names=None): + accounts = [] + seen_account_ids = set() + child_candidates = [] + + for account in accounts_from_sonar: + account_record = buildAccountRecord( + account, + recurring_rules=recurring_rules, + excluded_names=excluded_names, + ) + if account_record: + accounts.append(account_record) + seen_account_ids.add(account_record['raw_id']) + + parent_address = formatAccountAddress(account) + for child_account in account.get('child_accounts', {}).get('entities', []): + child_candidates.append((child_account, parent_address)) + + for child_account, parent_address in child_candidates: + child_id = child_account['id'] + if child_id in seen_account_ids: + continue + child_record = buildAccountRecord( + child_account, + fallback_address=parent_address, + recurring_rules=recurring_rules, + excluded_names=excluded_names, + ) + if child_record: + accounts.append(child_record) + seen_account_ids.add(child_record['raw_id']) + + return accounts + + def getAccounts(sonar_active_status_ids): - query = """query getAccounts ($pages: Paginator, $account_search: Search, $data: ReverseRelationFilter,$primary: ReverseRelationFilter) { + query = """query getAccounts ($pages: Paginator, $account_search: Search, $data: ReverseRelationFilter) { accounts (paginator: $pages,search: [$account_search]) { + page_info { + page + records_per_page + total_pages + total_count + } entities { account_status_id id @@ -163,7 +542,7 @@ def getAccounts(sonar_active_status_ids): inventory_model { name } - inventory_model_field_data (reverse_relation_filters: [$primary]) { + inventory_model_field_data { entities { value ip_assignments { @@ -171,6 +550,11 @@ def getAccounts(sonar_active_status_ids): subnet } } + inventory_model_field { + id + name + primary + } } } ip_assignments { @@ -182,24 +566,119 @@ def getAccounts(sonar_active_status_ids): } } } + radius_accounts { + entities { + id + ip_assignments { + entities { + subnet + } + } + } + } + child_accounts { + entities { + id + name + account_services (reverse_relation_filters: [$data]) { + entities { + service { + data_service_detail { + download_speed_kilobits_per_second + upload_speed_kilobits_per_second + } + } + } + } + addresses { + entities { + line1 + line2 + city + subdivision + inventory_items { + entities { + id + inventory_model { + name + } + inventory_model_field_data { + entities { + value + ip_assignments { + entities { + subnet + } + } + inventory_model_field { + id + name + primary + } + } + } + ip_assignments { + entities { + subnet + } + } + } + } + } + } + radius_accounts { + entities { + id + ip_assignments { + entities { + subnet + } + } + } + } + all_account_services: account_services { + entities { + service(enabled: true) { + id + name + type + enabled + data_service_detail { + download_speed_kilobits_per_second + upload_speed_kilobits_per_second + } + } + } + } + } + } + all_account_services: account_services { + entities { + service(enabled: true) { + id + name + type + enabled + data_service_detail { + download_speed_kilobits_per_second + upload_speed_kilobits_per_second + } + } + } + } } } }""" active_status_ids = [] - for status_id in sonar_active_status_ids(): + for status_id in sonar_active_status_ids: active_status_ids.append({ "attribute": "account_status_id", "operator": "EQ", "search_value": status_id }) - variables = {"pages": - { - "records_per_page": 5, - "page": 1 - }, - "account_search": { + variables = {"account_search": { "integer_fields": active_status_ids }, "data": { @@ -211,40 +690,17 @@ def getAccounts(sonar_active_status_ids): "search_value": "DATA" }] }] - }, - "primary": { - "relation": "inventory_model_field", - "search": [{ - "boolean_fields": [{ - "attribute": "primary", - "search_value": True - }] - }] - } + } } - accounts_from_sonar = sonarRequest(query,variables) - accounts = [] - for account in accounts_from_sonar: - # We need to make sure the account has an address because Sonar assignments go account -> address (only 1 per account) -> equipment -> ip assignments unless the IP is assigned to the account directly. - if account['addresses']['entities']: - line1 = account['addresses']['entities'][0]['line1'] - line2 = account['addresses']['entities'][0]['line2'] - city = account['addresses']['entities'][0]['city'] - state = account['addresses']['entities'][0]['subdivision'][-2:] - address = f"{line1},{f' {line2},' if line2 else ''} {city}, {state}" - devices = [] - for item in account['addresses']['entities'][0]['inventory_items']['entities']: - devices.append({'id': item['id'], 'name': item['inventory_model']['name'], 'ips': findIPs(item), 'mac': item['inventory_model_field_data']['entities'][0]['value']}) - if account['account_services']['entities'] and devices: # Make sure there is a data plan and devices on the account. - download = apply_client_bandwidth_multiplier(float(account['account_services']['entities'][0]['service']['data_service_detail']['download_speed_kilobits_per_second'])/1000) - upload = apply_client_bandwidth_multiplier(float(account['account_services']['entities'][0]['service']['data_service_detail']['upload_speed_kilobits_per_second'])/1000) - if download < 2: - download = 2 - if upload < 2: - upload = 2 - accounts.append({'id': account['id'],'name': account['name'], 'address': address, 'download': download, 'upload': upload ,'devices': devices}) - return accounts + accounts_from_sonar = sonarPaginatedRequest(query,variables) + recurring_rules = recurringServiceRateRules() + excluded_names = recurringServiceExclusions() if recurring_rules else set() + return buildAccountsFromSonarEntities( + accounts_from_sonar, + recurring_rules=recurring_rules, + excluded_names=excluded_names, + ) def mapApCpeMacs(ap): macs = [] @@ -266,12 +722,12 @@ def mapApCpeMacs(ap): return ap -def mapMacAP(mac,aps): +def buildMacToApMap(aps): + mac_to_ap = {} for ap in aps: for cpe_mac in ap['cpe_macs']: - if cpe_mac == mac: - return ap['id'] - return None + mac_to_ap[cpe_mac] = ap['id'] + return mac_to_ap def createShaper(): net = NetworkGraph() @@ -292,10 +748,12 @@ def createShaper(): pool.close() pool.join() + mac_to_ap = buildMacToApMap(aps) + # Update customers with the AP to which they are connected. for account in accounts: for device in account['devices']: - account['parent'] = mapMacAP(device['mac'],aps) + account['parent'] = mac_to_ap.get(device['mac']) if account['parent']: break diff --git a/src/lqos.example b/src/lqos.example index 1e5b486a..133bae13 100644 --- a/src/lqos.example +++ b/src/lqos.example @@ -35,7 +35,7 @@ to_network = "eth1" # Per-circuit overrides in ShapedDevices.csv may specify directional tokens # as "down_sqm/up_sqm" (see ShapedDevices.example.csv and INTEGRATION_HOWTO.md). default_sqm = "cake diffserv4" -monitor_only = false +queue_mode = "shape" uplink_bandwidth_mbps = 100000 downlink_bandwidth_mbps = 100000 generated_pn_download_mbps = 100000 @@ -120,6 +120,8 @@ snmp_community = "public" airmax_model_ids = [ "" ] ltu_model_ids = [ "" ] active_status_ids = [ "" ] +recurring_excluded_service_names = [] +recurring_service_rates = [] [influxdb] enable_influxdb = false diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 315210ef..fb8ccf7f 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -182,6 +182,18 @@ dependencies = [ "rustversion", ] +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures", + "password-hash", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -207,9 +219,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.15.4" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-sys", "zeroize", @@ -217,9 +229,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.37.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" +checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" dependencies = [ "cc", "cmake", @@ -315,6 +327,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bincode" version = "1.3.3" @@ -356,6 +374,15 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -1027,6 +1054,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -1462,6 +1490,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "http" version = "1.4.0" @@ -1989,7 +2026,10 @@ dependencies = [ "lqos_config", "lqos_sys", "parking_lot", + "serde", + "serde_json", "tokio", + "toml", "tracing", ] @@ -2016,6 +2056,7 @@ dependencies = [ "allocative", "allocative_derive", "arc-swap", + "argon2", "csv", "encoding_rs", "ip_network", @@ -2023,6 +2064,7 @@ dependencies = [ "lqos_utils", "nix", "once_cell", + "rand_core 0.6.4", "serde", "serde_json", "sha2", @@ -2075,9 +2117,11 @@ dependencies = [ "anyhow", "base64", "chrono", + "lqos_bakery", "lqos_bus", "lqos_config", "lqos_overrides", + "lqos_sys", "lqos_utils", "miniz_oxide", "nix", @@ -2099,6 +2143,7 @@ dependencies = [ "arc-swap", "criterion", "dashmap", + "lqos_bakery", "lqos_bus", "lqos_config", "lqos_sys", @@ -2199,6 +2244,7 @@ dependencies = [ "arc-swap", "axum", "axum-extra", + "base64", "bincode", "crossbeam-channel", "crossbeam-queue", @@ -2208,6 +2254,7 @@ dependencies = [ "flate2", "futures-util", "fxhash", + "hmac", "ip_network", "ip_network_table", "itertools 0.13.0", @@ -2230,6 +2277,7 @@ dependencies = [ "serde", "serde_cbor", "serde_json", + "sha2", "sha256", "signal-hook", "smallvec", @@ -2270,7 +2318,9 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "lqos_bus", "lqos_config", + "tokio", ] [[package]] @@ -2739,6 +2789,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "paste" version = "1.0.15" @@ -3233,9 +3294,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "aws-lc-rs", "log", @@ -3297,9 +3358,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "aws-lc-rs", "ring", diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml index 895e4bad..d0a1a253 100644 --- a/src/rust/Cargo.toml +++ b/src/rust/Cargo.toml @@ -54,6 +54,10 @@ csv = "1" ip_network_table = "0" ip_network = "0" sha2 = "0" +hmac = "0.12" +base64 = "0.22" +argon2 = "0.5" +rand_core = { version = "0.6", features = ["getrandom"] } uuid = { version = "1", features = ["v4", "fast-rng" ] } dashmap = "^5.5.3" toml = "0.8.8" diff --git a/src/rust/lqos_bakery/Cargo.toml b/src/rust/lqos_bakery/Cargo.toml index b229c4dc..93ca5c97 100644 --- a/src/rust/lqos_bakery/Cargo.toml +++ b/src/rust/lqos_bakery/Cargo.toml @@ -13,3 +13,8 @@ crossbeam-channel.workspace = true allocative.workspace = true parking_lot.workspace = true tokio.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +toml.workspace = true diff --git a/src/rust/lqos_bakery/src/commands.rs b/src/rust/lqos_bakery/src/commands.rs index 93b73ab4..33ecaab5 100644 --- a/src/rust/lqos_bakery/src/commands.rs +++ b/src/rust/lqos_bakery/src/commands.rs @@ -1,4 +1,5 @@ use crate::MQ_CREATED; +use crate::qdisc_handles::{InfraQdiscSlot, infra_qdisc_handle}; use crate::queue_math::{ format_rate_for_tc, format_rate_for_tc_f32, quantum, r2q, sqm_as_vec, sqm_tokens_for, }; @@ -7,8 +8,71 @@ use lqos_bus::TcHandle; use lqos_config::LazyQueueMode; use std::collections::HashSet; use std::sync::Arc; +use std::sync::mpsc::Sender as ReplySender; use tracing::{debug, info}; +/// Runtime TreeGuard node-operation action tracked by Bakery. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Allocative)] +pub enum RuntimeNodeOperationAction { + /// Virtualize the target node/subtree. + Virtualize, + /// Restore the target node/subtree to the physical tree. + Restore, +} + +/// Runtime TreeGuard node-operation status tracked by Bakery. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Allocative)] +pub enum RuntimeNodeOperationStatus { + /// Operation was accepted but has not started applying yet. + Submitted, + /// Operation was not started because Bakery runtime-node capacity is currently saturated. + Deferred, + /// Operation is currently applying reparent/restore work. + Applying, + /// Traffic-carrying work succeeded; cleanup is still pending. + AppliedAwaitingCleanup, + /// Operation completed successfully. + Completed, + /// Operation failed and can be retried. + Failed, + /// Operation could not be reconciled safely and the subtree is now frozen. + Dirty, +} + +/// Structured failure reason for a Bakery-tracked TreeGuard runtime node operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Allocative)] +pub enum RuntimeNodeOperationFailureReason { + /// The node has no direct child sites or direct circuits to promote. + StructuralIneligibleNoPromotableChildren, + /// The node has only one promotable direct child, so v1 flattening has no deterministic split. + StructuralIneligibleSinglePromotableChild, +} + +/// Snapshot of a Bakery-tracked TreeGuard runtime node operation. +#[derive(Debug, Clone, PartialEq, Eq, Allocative)] +pub struct RuntimeNodeOperationSnapshot { + /// Monotonic Bakery-local operation identifier. + pub operation_id: u64, + /// Stable Bakery site hash derived from the node name. + pub site_hash: i64, + /// Requested runtime action. + pub action: RuntimeNodeOperationAction, + /// Current operation status. + pub status: RuntimeNodeOperationStatus, + /// Number of retry/apply attempts performed so far. + pub attempt_count: u32, + /// Unix timestamp when the operation was submitted. + pub submitted_at_unix: u64, + /// Unix timestamp when the operation last changed state. + pub updated_at_unix: u64, + /// Optional unix timestamp for the next retry, if waiting. + pub next_retry_at_unix: Option, + /// Last error observed by Bakery for this operation, if any. + pub last_error: Option, + /// Structured failure reason observed by Bakery for this operation, if any. + pub failure_reason: Option, +} + #[derive(Debug, Clone, Copy, Allocative)] struct AddSiteParams { site_hash: i64, @@ -33,6 +97,8 @@ struct AddCircuitParams { upload_bandwidth_max: f32, class_major: u16, up_class_major: u16, + down_qdisc_handle: Option, + up_qdisc_handle: Option, // Optional per-circuit SQM override: "cake" or "fq_codel" sqm_override: Option, } @@ -131,6 +197,10 @@ pub enum BakeryCommands { AddCircuit { /// Unique identifier for the circuit. circuit_hash: i64, + /// Human-readable circuit name when known at queue-build time. + circuit_name: Option, + /// Human-readable parent site/node name when known at queue-build time. + site_name: Option, /// Parent class handle on the ISP-facing interface (downlink side). parent_class_id: TcHandle, /// Parent class handle on the Internet-facing interface (uplink side). @@ -149,6 +219,10 @@ pub enum BakeryCommands { class_major: u16, /// Major class ID (uplink) used when attaching SQM/HTB. up_class_major: u16, + /// Explicit qdisc handle major for the downlink leaf qdisc, if assigned. + down_qdisc_handle: Option, + /// Explicit qdisc handle major for the uplink leaf qdisc, if assigned. + up_qdisc_handle: Option, /// Concatenated list of all IPs for this circuit. ip_addresses: String, // Concatenated list of all IPs for this circuit /// Optional per-circuit SQM override: "cake" or "fq_codel" @@ -165,6 +239,16 @@ pub enum BakeryCommands { /// New class ceiling rate in Mbps (the handler sets ceil and rate-1). new_rate: u64, }, + /// Runtime TreeGuard request to virtualize or restore a non-top-level site without a full reload. + TreeGuardSetNodeVirtual { + /// Stable Bakery site hash derived from the node name. + site_hash: i64, + /// Whether the site should be virtualized (`true`) or restored (`false`). + virtualized: bool, + /// Optional synchronous reply channel for immediate operation-state reporting. + #[allocative(skip)] + reply: Option>, + }, } impl BakeryCommands { @@ -217,6 +301,8 @@ impl BakeryCommands { ), BakeryCommands::AddCircuit { circuit_hash, + circuit_name: _, + site_name: _, parent_class_id, up_parent_class_id, class_minor, @@ -226,6 +312,8 @@ impl BakeryCommands { upload_bandwidth_max, class_major, up_class_major, + down_qdisc_handle, + up_qdisc_handle, ip_addresses: _, sqm_override, } => Self::add_circuit( @@ -242,6 +330,8 @@ impl BakeryCommands { upload_bandwidth_max: *upload_bandwidth_max, class_major: *class_major, up_class_major: *up_class_major, + down_qdisc_handle: *down_qdisc_handle, + up_qdisc_handle: *up_qdisc_handle, sqm_override: sqm_override.clone(), }, ), @@ -255,38 +345,27 @@ impl BakeryCommands { stick_offset: usize, ) -> Option>> { let mut result = Vec::new(); - info!("Clearing prior settings"); - if config.on_a_stick_mode() { - // Clear just the MQ on the ISP-facing interface - result.push(vec![ - "qdisc".to_string(), - "del".to_string(), - "dev".to_string(), - config.isp_interface(), - "root".to_string(), - ]); - } else { - result.push(vec![ - "qdisc".to_string(), - "del".to_string(), - "dev".to_string(), - config.isp_interface(), - "root".to_string(), - ]); - result.push(vec![ - "qdisc".to_string(), - "del".to_string(), - "dev".to_string(), - config.internet_interface(), - "root".to_string(), - ]); - } + let observe_only = config.queues.queue_mode.is_observe(); + let mq_already_created = MQ_CREATED.load(std::sync::atomic::Ordering::Relaxed); + if observe_only { + if !mq_already_created { + result.push(root_mq_add_command(config.isp_interface())); + if !config.on_a_stick_mode() { + result.push(root_mq_add_command(config.internet_interface())); + } + } + MQ_CREATED.store(true, std::sync::atomic::Ordering::Relaxed); + return Some(result); + } info!( "Setting up MQ with {} queues and stick offset {}", queues_available, stick_offset ); - // command = 'qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq' + // Root MQ preflight is handled outside the strict builder batch during + // full reloads. The builder path only bootstraps root mq when runtime + // state is empty and otherwise rebuilds child HTB/SQM state beneath + // the prepared root. let sqm_strings = sqm_as_vec(config); let r2q = r2q(u64::max( config.queues.uplink_bandwidth_mbps, @@ -294,16 +373,9 @@ impl BakeryCommands { )); // ISP-facing interface (interface_a in Python) - result.push(vec![ - "qdisc".to_string(), - "replace".to_string(), - "dev".to_string(), - config.isp_interface(), - "root".to_string(), - "handle".to_string(), - "7FFF:".to_string(), - "mq".to_string(), - ]); + if !mq_already_created { + result.push(root_mq_add_command(config.isp_interface())); + } /* for queue in range(queuesAvailable): @@ -364,6 +436,11 @@ impl BakeryCommands { config.isp_interface(), "parent".to_string(), format!("0x{:x}:1", queue + 1), + "handle".to_string(), + format!( + "0x{:x}:", + infra_qdisc_handle((queue + 1) as u16, InfraQdiscSlot::Primary) + ), ]; class.extend(sqm_strings.clone()); result.push(class); @@ -401,23 +478,19 @@ impl BakeryCommands { config.isp_interface(), "parent".to_string(), format!("0x{:x}:2", queue + 1), + "handle".to_string(), + format!( + "0x{:x}:", + infra_qdisc_handle((queue + 1) as u16, InfraQdiscSlot::Default) + ), ]; default_class.extend(sqm_strings.clone()); result.push(default_class); } // Internet-facing interface (interface_b in Python) - if !config.on_a_stick_mode() { - result.push(vec![ - "qdisc".to_string(), - "replace".to_string(), - "dev".to_string(), - config.internet_interface(), - "root".to_string(), - "handle".to_string(), - "7FFF:".to_string(), - "mq".to_string(), - ]); + if !config.on_a_stick_mode() && !mq_already_created { + result.push(root_mq_add_command(config.internet_interface())); } /* @@ -478,6 +551,11 @@ impl BakeryCommands { config.internet_interface(), "parent".to_string(), format!("0x{:x}:1", queue + stick_offset + 1), + "handle".to_string(), + format!( + "0x{:x}:", + infra_qdisc_handle((queue + stick_offset + 1) as u16, InfraQdiscSlot::Primary) + ), ]; class.extend(sqm_strings.clone()); result.push(class); @@ -514,6 +592,11 @@ impl BakeryCommands { config.internet_interface(), "parent".to_string(), format!("0x{:x}:2", queue + stick_offset + 1), + "handle".to_string(), + format!( + "0x{:x}:", + infra_qdisc_handle((queue + stick_offset + 1) as u16, InfraQdiscSlot::Default) + ), ]; default_class.extend(sqm_strings.clone()); result.push(default_class); @@ -527,6 +610,10 @@ impl BakeryCommands { config: &Arc, params: AddSiteParams, ) -> Option>> { + if config.queues.queue_mode.is_observe() { + return None; + } + let mut result = Vec::new(); // Derive major IDs from parent handles so classids are fully qualified // and consistent with queuingStructure.json (classMajor/classMinor). @@ -597,6 +684,10 @@ impl BakeryCommands { config: &Arc, params: AddCircuitParams, ) -> Option>> { + if config.queues.queue_mode.is_observe() { + return None; + } + if let Some(ref s) = params.sqm_override && s.eq_ignore_ascii_case("fq_codel") { @@ -706,7 +797,7 @@ impl BakeryCommands { ), ]); } - if !config.queues.monitor_only + if !config.queues.queue_mode.is_observe() && do_sqm && !matches!(down_override_opt.as_deref(), Some(s) if s.eq_ignore_ascii_case("none")) { @@ -718,6 +809,10 @@ impl BakeryCommands { "parent".to_string(), format!("0x{:x}:0x{:x}", params.class_major, params.class_minor), ]; + if let Some(handle) = params.down_qdisc_handle { + sqm_command.push("handle".to_string()); + sqm_command.push(format!("0x{:x}:", handle)); + } sqm_command.extend(sqm_tokens_for( params.download_bandwidth_max, config, @@ -753,7 +848,7 @@ impl BakeryCommands { ]); } - if !config.queues.monitor_only + if !config.queues.queue_mode.is_observe() && do_sqm && !config.on_a_stick_mode() && !matches!(up_override_opt.as_deref(), Some(s) if s.eq_ignore_ascii_case("none")) @@ -766,6 +861,10 @@ impl BakeryCommands { "parent".to_string(), format!("0x{:x}:0x{:x}", params.up_class_major, params.class_minor), ]; + if let Some(handle) = params.up_qdisc_handle { + sqm_command.push("handle".to_string()); + sqm_command.push(format!("0x{:x}:", handle)); + } sqm_command.extend(sqm_tokens_for( params.upload_bandwidth_max, config, @@ -923,3 +1022,331 @@ impl BakeryCommands { Some(result) } } + +fn root_mq_add_command(interface_name: String) -> Vec { + vec![ + "qdisc".to_string(), + "add".to_string(), + "dev".to_string(), + interface_name, + "root".to_string(), + "handle".to_string(), + "7FFF:".to_string(), + "mq".to_string(), + ] +} + +#[cfg(test)] +mod tests { + use super::{BakeryCommands, ExecutionMode}; + use crate::MQ_CREATED; + use crate::test_state_lock; + use lqos_config::{Config, LazyQueueMode, SingleInterfaceConfig}; + use std::sync::Arc; + + fn is_root_delete(cmd: &[String], interface: &str) -> bool { + cmd.len() == 5 + && cmd[0] == "qdisc" + && cmd[1] == "del" + && cmd[2] == "dev" + && cmd[3] == interface + && cmd[4] == "root" + } + + fn is_root_add_mq(cmd: &[String], interface: &str) -> bool { + cmd.len() == 8 + && cmd[0] == "qdisc" + && cmd[1] == "add" + && cmd[2] == "dev" + && cmd[3] == interface + && cmd[4] == "root" + && cmd[5] == "handle" + && cmd[6] == "7FFF:" + && cmd[7] == "mq" + } + + fn assert_qdisc_add_replace_commands_use_explicit_handles(commands: &[Vec]) { + for cmd in commands { + let is_qdisc_add_or_replace = cmd.first().is_some_and(|part| part == "qdisc") + && cmd + .get(1) + .is_some_and(|part| part == "add" || part == "replace"); + if !is_qdisc_add_or_replace { + continue; + } + assert!( + cmd.iter().any(|part| part == "handle"), + "qdisc add/replace command omitted explicit handle: {:?}", + cmd + ); + } + } + + #[test] + fn mq_setup_bridge_mode_bootstraps_root_add_commands_when_runtime_state_is_empty() { + let _guard = test_state_lock().lock().expect("lock"); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + let config = Arc::new(Config::default()); + let commands = BakeryCommands::MqSetup { + queues_available: 1, + stick_offset: 0, + } + .to_commands(&config, ExecutionMode::Builder) + .expect("mq setup should emit commands"); + + assert!(is_root_add_mq(&commands[0], &config.isp_interface())); + let internet_add_idx = commands + .iter() + .position(|cmd| is_root_add_mq(cmd, &config.internet_interface())) + .expect("expected internet-interface root add"); + assert!( + internet_add_idx > 0, + "internet-interface root add should occur after the ISP-interface root add" + ); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + } + + #[test] + fn mq_setup_on_a_stick_emits_single_root_add_when_runtime_state_is_empty() { + let _guard = test_state_lock().lock().expect("lock"); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + let config = Arc::new(Config { + bridge: None, + single_interface: Some(SingleInterfaceConfig::default()), + ..Config::default() + }); + + let commands = BakeryCommands::MqSetup { + queues_available: 1, + stick_offset: 3, + } + .to_commands(&config, ExecutionMode::Builder) + .expect("mq setup should emit commands"); + + assert!(is_root_add_mq(&commands[0], &config.isp_interface())); + assert_eq!( + commands + .iter() + .filter(|cmd| is_root_add_mq(cmd, &config.isp_interface())) + .count(), + 1, + "expected a single root add on the shared interface" + ); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + } + + #[test] + fn mq_setup_shape_mode_skips_root_add_when_runtime_state_is_ready() { + let _guard = test_state_lock().lock().expect("lock"); + MQ_CREATED.store(true, std::sync::atomic::Ordering::Relaxed); + let config = Arc::new(Config::default()); + let commands = BakeryCommands::MqSetup { + queues_available: 1, + stick_offset: 0, + } + .to_commands(&config, ExecutionMode::Builder) + .expect("mq setup should emit commands"); + + assert!( + commands + .iter() + .all(|cmd| !is_root_add_mq(cmd, &config.isp_interface())) + ); + assert!( + commands + .iter() + .all(|cmd| !is_root_add_mq(cmd, &config.internet_interface())) + ); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + } + + #[test] + fn mq_setup_qdisc_add_replace_commands_use_explicit_handles() { + let _guard = test_state_lock().lock().expect("lock"); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + let config = Arc::new(Config::default()); + let commands = BakeryCommands::MqSetup { + queues_available: 2, + stick_offset: 0, + } + .to_commands(&config, ExecutionMode::Builder) + .expect("mq setup should emit commands"); + + assert_qdisc_add_replace_commands_use_explicit_handles(&commands); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + } + + #[test] + fn mq_setup_observe_mode_bootstraps_root_mq_when_runtime_state_is_empty() { + let _guard = test_state_lock().lock().expect("lock"); + let mut cfg = Config::default(); + cfg.queues.set_queue_mode(lqos_config::QueueMode::Observe); + let config = Arc::new(cfg); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + + let commands = BakeryCommands::MqSetup { + queues_available: 1, + stick_offset: 0, + } + .to_commands(&config, ExecutionMode::Builder) + .expect("mq setup should emit commands in observe mode"); + + assert!( + commands + .iter() + .all(|cmd| !is_root_delete(cmd, &config.isp_interface())) + ); + assert!( + commands + .iter() + .all(|cmd| !is_root_delete(cmd, &config.internet_interface())) + ); + assert!( + commands + .iter() + .any(|cmd| is_root_add_mq(cmd, &config.isp_interface())) + ); + assert!( + commands + .iter() + .any(|cmd| is_root_add_mq(cmd, &config.internet_interface())) + ); + assert_eq!( + commands.len(), + 2, + "observe mode should bootstrap root mq when runtime state is empty" + ); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + } + + #[test] + fn mq_setup_observe_mode_reuses_retained_root_when_mq_exists() { + let _guard = test_state_lock().lock().expect("lock"); + let mut cfg = Config::default(); + cfg.queues.set_queue_mode(lqos_config::QueueMode::Observe); + let config = Arc::new(cfg); + MQ_CREATED.store(true, std::sync::atomic::Ordering::Relaxed); + + let commands = BakeryCommands::MqSetup { + queues_available: 2, + stick_offset: 0, + } + .to_commands(&config, ExecutionMode::Builder) + .expect("mq setup should emit commands in observe mode"); + + assert!( + commands + .iter() + .all(|cmd| !is_root_delete(cmd, &config.isp_interface())) + ); + assert!( + commands + .iter() + .all(|cmd| !is_root_delete(cmd, &config.internet_interface())) + ); + assert!( + commands + .iter() + .all(|cmd| !is_root_add_mq(cmd, &config.isp_interface())) + ); + assert!( + commands + .iter() + .all(|cmd| !is_root_add_mq(cmd, &config.internet_interface())) + ); + assert_eq!( + commands.len(), + 0, + "observe mode should rely on full-reload preflight when the root mq already exists" + ); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + } + + #[test] + fn add_circuit_qdisc_replace_commands_use_explicit_handles_when_enriched() { + let config = Arc::new(Config::default()); + let builder_commands = BakeryCommands::AddCircuit { + circuit_hash: 42, + circuit_name: None, + site_name: None, + parent_class_id: crate::TcHandle::from_u32(0x10020), + up_parent_class_id: crate::TcHandle::from_u32(0x20020), + class_minor: 0x21, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x1, + up_class_major: 0x2, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + ip_addresses: "192.0.2.42/32".to_string(), + sqm_override: None, + } + .to_commands(&config, ExecutionMode::Builder) + .expect("builder add_circuit should emit commands"); + assert_qdisc_add_replace_commands_use_explicit_handles(&builder_commands); + + let mut live_cfg = Config::default(); + live_cfg.queues.lazy_queues = Some(LazyQueueMode::Full); + let live_config = Arc::new(live_cfg); + let live_commands = BakeryCommands::AddCircuit { + circuit_hash: 42, + circuit_name: None, + site_name: None, + parent_class_id: crate::TcHandle::from_u32(0x10020), + up_parent_class_id: crate::TcHandle::from_u32(0x20020), + class_minor: 0x21, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x1, + up_class_major: 0x2, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + ip_addresses: "192.0.2.42/32".to_string(), + sqm_override: None, + } + .to_commands(&live_config, ExecutionMode::LiveUpdate) + .expect("live add_circuit should emit commands"); + assert_qdisc_add_replace_commands_use_explicit_handles(&live_commands); + } + + #[test] + fn mq_setup_observe_mode_reuses_retained_root_on_a_stick() { + let _guard = test_state_lock().lock().expect("lock"); + let mut cfg = Config { + bridge: None, + single_interface: Some(SingleInterfaceConfig::default()), + ..Config::default() + }; + cfg.queues.set_queue_mode(lqos_config::QueueMode::Observe); + let config = Arc::new(cfg); + MQ_CREATED.store(true, std::sync::atomic::Ordering::Relaxed); + + let commands = BakeryCommands::MqSetup { + queues_available: 2, + stick_offset: 2, + } + .to_commands(&config, ExecutionMode::Builder) + .expect("mq setup should emit commands in observe on-a-stick mode"); + + assert!( + commands + .iter() + .all(|cmd| !is_root_delete(cmd, &config.isp_interface())) + ); + assert!( + commands + .iter() + .all(|cmd| !is_root_add_mq(cmd, &config.isp_interface())) + ); + assert_eq!( + commands.len(), + 0, + "observe mode should rely on full-reload preflight for retained on-a-stick roots" + ); + MQ_CREATED.store(false, std::sync::atomic::Ordering::Relaxed); + } +} diff --git a/src/rust/lqos_bakery/src/diff/circuits.rs b/src/rust/lqos_bakery/src/diff/circuits.rs index 272152f5..00d67493 100644 --- a/src/rust/lqos_bakery/src/diff/circuits.rs +++ b/src/rust/lqos_bakery/src/diff/circuits.rs @@ -8,6 +8,7 @@ pub(crate) struct CircuitDiffCategories<'a> { pub removed_circuits: Vec, pub speed_changed: Vec<&'a Arc>, pub ip_changed: Vec<&'a Arc>, + pub migrated: Vec<&'a Arc>, pub structural_changed: Vec<&'a Arc>, } @@ -35,7 +36,8 @@ pub(crate) fn diff_circuits<'a>( let mut removed_circuits = Vec::new(); let mut speed_changed = Vec::new(); let mut ip_changed = Vec::new(); - let mut structural_changed = Vec::new(); + let mut migrated = Vec::new(); + let structural_changed = Vec::new(); for (circuit_hash, new_cmd) in &new_circuits { if !old_circuits.contains_key(circuit_hash) { @@ -53,7 +55,7 @@ pub(crate) fn diff_circuits<'a>( if let Some(new_cmd) = new_circuits.get(circuit_hash) { match classify_circuit_change(old_cmd.as_ref(), new_cmd.as_ref()) { CircuitChange::None => {} - CircuitChange::Structural => structural_changed.push(*new_cmd), + CircuitChange::Migration => migrated.push(*new_cmd), CircuitChange::Speed => speed_changed.push(*new_cmd), CircuitChange::Ip => ip_changed.push(*new_cmd), CircuitChange::SpeedAndIp => { @@ -68,6 +70,7 @@ pub(crate) fn diff_circuits<'a>( || !removed_circuits.is_empty() || !speed_changed.is_empty() || !ip_changed.is_empty() + || !migrated.is_empty() || !structural_changed.is_empty() { CircuitDiffResult::Categorized(CircuitDiffCategories { @@ -75,6 +78,7 @@ pub(crate) fn diff_circuits<'a>( removed_circuits, speed_changed, ip_changed, + migrated, structural_changed, }) } else { @@ -84,7 +88,7 @@ pub(crate) fn diff_circuits<'a>( enum CircuitChange { None, - Structural, + Migration, Speed, Ip, SpeedAndIp, @@ -141,7 +145,7 @@ fn classify_circuit_change(a: &BakeryCommands, b: &BakeryCommands) -> CircuitCha || up_class_major != other_up_class_major; if structural { - return CircuitChange::Structural; + return CircuitChange::Migration; } let speed = download_bandwidth_min != other_download_bandwidth_min @@ -159,3 +163,133 @@ fn classify_circuit_change(a: &BakeryCommands, b: &BakeryCommands) -> CircuitCha (true, true) => CircuitChange::SpeedAndIp, } } + +#[cfg(test)] +mod tests { + use super::{CircuitDiffResult, diff_circuits}; + use crate::BakeryCommands; + use lqos_bus::TcHandle; + use std::collections::HashMap; + use std::sync::Arc; + + #[allow(clippy::too_many_arguments)] + fn add_circuit( + circuit_hash: i64, + parent_class_id: u32, + up_parent_class_id: u32, + class_minor: u16, + class_major: u16, + up_class_major: u16, + dl_min: f32, + ul_min: f32, + dl_max: f32, + ul_max: f32, + ) -> Arc { + Arc::new(BakeryCommands::AddCircuit { + circuit_hash, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(parent_class_id), + up_parent_class_id: TcHandle::from_u32(up_parent_class_id), + class_minor, + download_bandwidth_min: dl_min, + upload_bandwidth_min: ul_min, + download_bandwidth_max: dl_max, + upload_bandwidth_max: ul_max, + class_major, + up_class_major, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + ip_addresses: "192.0.2.1/32".to_string(), + sqm_override: None, + }) + } + + #[test] + fn parent_move_is_categorized_as_migration() { + let old = add_circuit(1, 0x10020, 0x20020, 0x21, 0x1, 0x2, 1.0, 1.0, 10.0, 10.0); + let new = add_circuit(1, 0x10034, 0x20034, 0x35, 0x1, 0x2, 1.0, 1.0, 10.0, 10.0); + let old_circuits = HashMap::from([(1, Arc::clone(&old))]); + let batch = vec![new]; + + let CircuitDiffResult::Categorized(categories) = diff_circuits(&batch, &old_circuits) + else { + panic!("expected categorized diff"); + }; + + assert_eq!(categories.migrated.len(), 1); + assert!(categories.structural_changed.is_empty()); + assert!(categories.speed_changed.is_empty()); + } + + #[test] + fn rate_change_stays_speed_only() { + let old = add_circuit(1, 0x10020, 0x20020, 0x21, 0x1, 0x2, 1.0, 1.0, 10.0, 10.0); + let new = add_circuit(1, 0x10020, 0x20020, 0x21, 0x1, 0x2, 2.0, 1.5, 20.0, 15.0); + let old_circuits = HashMap::from([(1, Arc::clone(&old))]); + let batch = vec![new]; + + let CircuitDiffResult::Categorized(categories) = diff_circuits(&batch, &old_circuits) + else { + panic!("expected categorized diff"); + }; + + assert_eq!(categories.speed_changed.len(), 1); + assert!(categories.migrated.is_empty()); + assert!(categories.structural_changed.is_empty()); + } + + #[test] + fn ip_change_stays_ip_only() { + let old = add_circuit(1, 0x10020, 0x20020, 0x21, 0x1, 0x2, 1.0, 1.0, 10.0, 10.0); + let BakeryCommands::AddCircuit { + circuit_hash, + parent_class_id, + up_parent_class_id, + class_minor, + download_bandwidth_min, + upload_bandwidth_min, + download_bandwidth_max, + upload_bandwidth_max, + class_major, + up_class_major, + down_qdisc_handle, + up_qdisc_handle, + sqm_override, + .. + } = old.as_ref() + else { + panic!("expected add circuit"); + }; + let new = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: *circuit_hash, + circuit_name: None, + site_name: None, + parent_class_id: *parent_class_id, + up_parent_class_id: *up_parent_class_id, + class_minor: *class_minor, + download_bandwidth_min: *download_bandwidth_min, + upload_bandwidth_min: *upload_bandwidth_min, + download_bandwidth_max: *download_bandwidth_max, + upload_bandwidth_max: *upload_bandwidth_max, + class_major: *class_major, + up_class_major: *up_class_major, + down_qdisc_handle: *down_qdisc_handle, + up_qdisc_handle: *up_qdisc_handle, + ip_addresses: "198.51.100.10/32".to_string(), + sqm_override: sqm_override.clone(), + }); + let old_circuits = HashMap::from([(1, Arc::clone(&old))]); + let batch = vec![new]; + + let CircuitDiffResult::Categorized(categories) = diff_circuits(&batch, &old_circuits) + else { + panic!("expected categorized diff"); + }; + + assert_eq!(categories.ip_changed.len(), 1); + assert!(categories.speed_changed.is_empty()); + assert!(categories.migrated.is_empty()); + assert!(categories.structural_changed.is_empty()); + } +} diff --git a/src/rust/lqos_bakery/src/diff/sites.rs b/src/rust/lqos_bakery/src/diff/sites.rs index e8ca179c..3c4cd142 100644 --- a/src/rust/lqos_bakery/src/diff/sites.rs +++ b/src/rust/lqos_bakery/src/diff/sites.rs @@ -3,8 +3,10 @@ use std::collections::HashMap; use std::sync::Arc; use tracing::{debug, warn}; +const SITE_DELTA_PREVIEW_THRESHOLD: usize = 8; + pub(crate) enum SiteDiffResult { - RebuildRequired, + RebuildRequired { summary: String }, SpeedChanges { changes: Vec }, NoChange, } @@ -29,6 +31,7 @@ pub(crate) fn diff_sites( // Therefore, we must rebuild the entire site configuration. let old_keys: Vec = old_sites.keys().cloned().collect(); let new_keys: Vec = new_sites.keys().copied().collect(); + let (added_site_hashes, removed_site_hashes) = site_set_delta(old_sites, &new_sites); warn!( "Site count mismatch: old {} vs new {}", old_sites.len(), @@ -36,7 +39,18 @@ pub(crate) fn diff_sites( ); debug!("Old site hashes: {:?}", old_keys); debug!("New site hashes: {:?}", new_keys); - return SiteDiffResult::RebuildRequired; + debug!("Added site hashes: {:?}", added_site_hashes); + debug!("Removed site hashes: {:?}", removed_site_hashes); + return SiteDiffResult::RebuildRequired { + summary: format_site_delta_summary( + "site_count_mismatch", + None, + old_sites.len(), + new_sites.len(), + &added_site_hashes, + &removed_site_hashes, + ), + }; } // Compare each site in the old and new maps for structural differences. @@ -83,7 +97,12 @@ pub(crate) fn diff_sites( "Site hash {} change detail: parent={}→{}, up_parent={}→{}, minor=0x{:x}→0x{:x}", site_hash, opar, npar, oup, nup, omin, nmin ); - return SiteDiffResult::RebuildRequired; + return SiteDiffResult::RebuildRequired { + summary: format!( + "Bakery full reload triggered by site diff: site_hash={} parent={}→{} up_parent={}→{} minor=0x{:x}→0x{:x}", + site_hash, opar, npar, oup, nup, omin, nmin + ), + }; } // If the speeds have changed, we need to store the change. if let Some(speed_change) = site_speeds_changed(old_cmd.as_ref(), new_cmd.as_ref()) { @@ -93,7 +112,19 @@ pub(crate) fn diff_sites( } else { // If a site is missing in the new batch, we need to rebuild. debug!("Site hash {} is missing in the new batch", site_hash); - return SiteDiffResult::RebuildRequired; + let (added_site_hashes, removed_site_hashes) = site_set_delta(old_sites, &new_sites); + debug!("Added site hashes: {:?}", added_site_hashes); + debug!("Removed site hashes: {:?}", removed_site_hashes); + return SiteDiffResult::RebuildRequired { + summary: format_site_delta_summary( + "site_set_changed", + Some(*site_hash), + old_sites.len(), + new_sites.len(), + &added_site_hashes, + &removed_site_hashes, + ), + }; } } @@ -107,6 +138,67 @@ pub(crate) fn diff_sites( SiteDiffResult::NoChange } +fn site_set_delta( + old_sites: &HashMap>, + new_sites: &HashMap>, +) -> (Vec, Vec) { + let mut added_site_hashes: Vec = new_sites + .keys() + .filter(|hash| !old_sites.contains_key(hash)) + .copied() + .collect(); + added_site_hashes.sort_unstable(); + + let mut removed_site_hashes: Vec = old_sites + .keys() + .filter(|hash| !new_sites.contains_key(hash)) + .copied() + .collect(); + removed_site_hashes.sort_unstable(); + + (added_site_hashes, removed_site_hashes) +} + +fn format_site_hash_preview(label: &str, hashes: &[i64], total_changed: usize) -> Option { + if hashes.is_empty() + || hashes.len() > SITE_DELTA_PREVIEW_THRESHOLD + || total_changed > SITE_DELTA_PREVIEW_THRESHOLD + { + return None; + } + Some(format!(" {label}={hashes:?}")) +} + +fn format_site_delta_summary( + reason: &str, + missing_old_site_hash: Option, + old_count: usize, + new_count: usize, + added_site_hashes: &[i64], + removed_site_hashes: &[i64], +) -> String { + let mut summary = format!( + "Bakery full reload triggered by site diff: {reason} old_count={old_count} new_count={new_count} added_count={} removed_count={}", + added_site_hashes.len(), + removed_site_hashes.len() + ); + let total_changed = added_site_hashes.len() + removed_site_hashes.len(); + if let Some(site_hash) = missing_old_site_hash { + summary.push_str(&format!(" missing_old_site_hash={site_hash}")); + } + if let Some(preview) = + format_site_hash_preview("added_preview", added_site_hashes, total_changed) + { + summary.push_str(&preview); + } + if let Some(preview) = + format_site_hash_preview("removed_preview", removed_site_hashes, total_changed) + { + summary.push_str(&preview); + } + summary +} + fn is_structurally_different(a: &BakeryCommands, b: &BakeryCommands) -> bool { let BakeryCommands::AddSite { site_hash, @@ -211,3 +303,104 @@ fn site_speeds_changed(a: &BakeryCommands, b: &BakeryCommands) -> Option Arc { + Arc::new(BakeryCommands::AddSite { + site_hash, + parent_class_id: TcHandle::from_u32(parent_class_id), + up_parent_class_id: TcHandle::from_u32(up_parent_class_id), + class_minor, + download_bandwidth_min: dl_min, + upload_bandwidth_min: ul_min, + download_bandwidth_max: dl_max, + upload_bandwidth_max: ul_max, + }) + } + + #[test] + fn site_count_mismatch_reports_added_and_removed_hashes() { + let old_site = add_site(10, 0x10010, 0x20010, 0x11, 1.0, 1.0, 10.0, 10.0); + let new_site = add_site(20, 0x10020, 0x20020, 0x21, 1.0, 1.0, 10.0, 10.0); + let old_sites = HashMap::from([(10, old_site)]); + let batch = vec![ + new_site, + add_site(30, 0x10030, 0x20030, 0x31, 1.0, 1.0, 10.0, 10.0), + ]; + + let SiteDiffResult::RebuildRequired { summary } = diff_sites(&batch, &old_sites) else { + panic!("expected rebuild-required site diff"); + }; + + assert!(summary.contains("site_count_mismatch")); + assert!(summary.contains("added_count=2")); + assert!(summary.contains("removed_count=1")); + assert!(summary.contains("added_preview=[20, 30]")); + assert!(summary.contains("removed_preview=[10]")); + } + + #[test] + fn large_site_count_mismatch_stays_counts_only() { + let old_sites = HashMap::from([( + 10, + add_site(10, 0x10010, 0x20010, 0x11, 1.0, 1.0, 10.0, 10.0), + )]); + let mut batch = Vec::new(); + for index in 20..30 { + batch.push(add_site( + index, + 0x10000 | index as u32, + 0x20000 | index as u32, + index as u16, + 1.0, + 1.0, + 10.0, + 10.0, + )); + } + + let SiteDiffResult::RebuildRequired { summary } = diff_sites(&batch, &old_sites) else { + panic!("expected rebuild-required site diff"); + }; + + assert!(summary.contains("site_count_mismatch")); + assert!(summary.contains("added_count=10")); + assert!(summary.contains("removed_count=1")); + assert!(!summary.contains("added_preview=")); + assert!(!summary.contains("removed_preview=")); + } + + #[test] + fn structural_site_change_reports_parent_transition() { + let old_site = add_site(10, 0x10010, 0x20010, 0x11, 1.0, 1.0, 10.0, 10.0); + let new_site = add_site(10, 0x10020, 0x20020, 0x21, 1.0, 1.0, 10.0, 10.0); + let old_sites = HashMap::from([(10, old_site)]); + let batch = vec![new_site]; + + let SiteDiffResult::RebuildRequired { summary } = diff_sites(&batch, &old_sites) else { + panic!("expected rebuild-required site diff"); + }; + + assert!(summary.contains("site_hash=10")); + assert!(summary.contains("parent=0x1:0x10→0x1:0x20")); + assert!(summary.contains("up_parent=0x2:0x10→0x2:0x20")); + assert!(summary.contains("minor=0x11→0x21")); + } +} diff --git a/src/rust/lqos_bakery/src/lib.rs b/src/rust/lqos_bakery/src/lib.rs index 1cd1d287..bc913c83 100644 --- a/src/rust/lqos_bakery/src/lib.rs +++ b/src/rust/lqos_bakery/src/lib.rs @@ -21,48 +21,89 @@ mod commands; mod diff; +mod qdisc_handles; mod queue_math; mod utils; -use crossbeam_channel::{Receiver, Sender}; -use std::collections::{HashMap, HashSet}; +use crossbeam_channel::{Receiver, RecvTimeoutError, Sender}; +use parking_lot::RwLock; +use std::collections::VecDeque; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::Path; use std::sync::atomic::Ordering::Relaxed; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, OnceLock}; +use std::time::Duration; use tracing::{debug, error, info, warn}; use utils::current_timestamp; pub(crate) const CHANNEL_CAPACITY: usize = 65536; // 64k capacity for Bakery commands -use crate::commands::ExecutionMode; +use crate::commands::{ + ExecutionMode, RuntimeNodeOperationAction, RuntimeNodeOperationFailureReason, + RuntimeNodeOperationSnapshot, RuntimeNodeOperationStatus, +}; use crate::diff::{CircuitDiffResult, SiteDiffResult, diff_circuits, diff_sites}; -use crate::queue_math::format_rate_for_tc_f32; -use crate::utils::{execute_in_memory, write_command_file}; -pub use commands::BakeryCommands; +use crate::qdisc_handles::QdiscHandleState; +use crate::queue_math::{SqmKind, effective_sqm_kind, format_rate_for_tc_f32, quantum, r2q}; +use crate::utils::{ + ExecuteResult, LiveTcClassEntry, LiveTcQdiscEntry, MemorySnapshot, execute_in_memory, + execute_in_memory_chunked, invalidate_live_tc_snapshots, read_live_class_snapshot, + read_live_qdisc_handle_majors, read_live_qdisc_snapshot, read_memory_snapshot, + tc_io_cadence_snapshot, write_command_file, +}; +pub use commands::{ + BakeryCommands, RuntimeNodeOperationAction as BakeryRuntimeNodeOperationAction, + RuntimeNodeOperationFailureReason as BakeryRuntimeNodeOperationFailureReason, + RuntimeNodeOperationSnapshot as BakeryRuntimeNodeOperationSnapshot, + RuntimeNodeOperationStatus as BakeryRuntimeNodeOperationStatus, +}; use lqos_bus::{ BusRequest, BusResponse, InsightLicenseSummary, LibreqosBusClient, TcHandle, UrgentSeverity, UrgentSource, }; -use lqos_config::{Config, LazyQueueMode}; +use lqos_config::{ + CircuitIdentityGroupInput, ClassIdentityPlannerConstraints, Config, LazyQueueMode, + PlannerCircuitIdentityState, PlannerMinorReservations, PlannerSiteIdentityState, + SiteIdentityInput, TopLevelPlannerItem, TopLevelPlannerMode, TopLevelPlannerParams, + build_class_identity_reservations, plan_class_identities_with_constraints, + plan_top_level_assignments, +}; +use qdisc_handles::MqDeviceLayout; +use serde_json::{Map, Value}; + +const TEST_FAULT_ONCE_PATH: &str = "/tmp/lqos_bakery_fail_purpose_once.txt"; +const ACTIVE_RUNTIME_MINOR_START: u32 = 0x1000; +const MIGRATION_VERIFICATION_MAX_RETRIES: u8 = 3; // ---------------------- Live-Move Types and Helpers (module scope) ---------------------- #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum MigrationStage { PrepareShadow, + VerifyShadowReady, SwapToShadow, BuildFinal, + VerifyFinalReady, SwapToFinal, - TeardownShadow, + TeardownMigrationScaffold, Done, } #[derive(Clone, Debug)] struct Migration { circuit_hash: i64, - // Parent handles and majors + circuit_name: Option, + site_name: Option, + // Old majors and qdisc handles + old_class_major: u16, + old_up_class_major: u16, + old_down_qdisc_handle: Option, + old_up_qdisc_handle: Option, + // New parent handles and majors parent_class_id: TcHandle, up_parent_class_id: TcHandle, class_major: u16, up_class_major: u16, + down_qdisc_handle: Option, + up_qdisc_handle: Option, // Old and new rates old_down_min: f32, old_down_max: f32, @@ -80,7 +121,76 @@ struct Migration { ips: Vec, // Per-circuit SQM override ("cake" or "fq_codel"), if any sqm_override: Option, + // Desired final circuit state. This is published to the effective circuit map only after + // live TC and IP cutover reach the final class successfully. + desired_cmd: Arc, stage: MigrationStage, + shadow_verify_attempts: u8, + final_verify_attempts: u8, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct MigrationDirectionVerification { + interface_name: String, + expected_handle: TcHandle, + expected_parent: TcHandle, + observed_parent: Option, + observed_leaf_qdisc_major: Option, +} + +impl MigrationDirectionVerification { + fn ready(&self) -> bool { + self.observed_parent == Some(self.expected_parent) + && self.observed_leaf_qdisc_major.is_some() + } + + fn summary(&self, direction: &str) -> String { + let observed = match (self.observed_parent, self.observed_leaf_qdisc_major) { + (Some(parent), Some(leaf_major)) => format!( + "observed parent {} with leaf qdisc 0x{:x}:", + parent.as_tc_string(), + leaf_major + ), + (Some(parent), None) => { + format!( + "observed parent {} with no leaf qdisc", + parent.as_tc_string() + ) + } + (None, _) => "observed missing".to_string(), + }; + format!( + "{direction} {} expected class {} under parent {} with a leaf qdisc, {observed}", + self.interface_name, + self.expected_handle.as_tc_string(), + self.expected_parent.as_tc_string() + ) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct MigrationBranchVerification { + down: MigrationDirectionVerification, + up: Option, +} + +impl MigrationBranchVerification { + fn ready(&self) -> bool { + self.down.ready() + && self + .up + .as_ref() + .map(MigrationDirectionVerification::ready) + .unwrap_or(true) + } + + fn summary(&self) -> String { + let mut parts = vec![self.down.summary("down")]; + if let Some(up) = &self.up { + parts.push(up.summary("up")); + } + parts.join("; ") + } } #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -89,6 +199,235 @@ struct StormguardOverrideKey { class: TcHandle, } +#[derive(Clone, Debug)] +struct VirtualizedSiteQdiscHandles { + down: Option, + up: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RuntimeVirtualizedActiveBranch { + Shadow, + Original, +} + +#[allow(dead_code)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RuntimeVirtualizedBranchLifecycle { + PhysicalActive, + FlattenBuildPending, + CutoverPending, + FlattenedActive, + RestorePending, + PhysicalActiveCleanupPending, + Failed, +} + +#[derive(Clone, Debug)] +struct VirtualizedSiteState { + site_name: String, + site: Arc, + saved_sites: HashMap>, + saved_circuits: HashMap>, + active_sites: HashMap>, + active_circuits: HashMap>, + prune_sites: HashMap>, + prune_circuits: HashMap>, + qdisc_handles: VirtualizedSiteQdiscHandles, + active_branch: RuntimeVirtualizedActiveBranch, + lifecycle: RuntimeVirtualizedBranchLifecycle, + pending_prune: bool, + next_prune_attempt_unix: u64, +} + +impl VirtualizedSiteState { + fn active_branch_hides_original_site(&self) -> bool { + matches!(self.active_branch, RuntimeVirtualizedActiveBranch::Shadow) + } +} + +fn lifecycle_from_active_branch( + active_branch: RuntimeVirtualizedActiveBranch, + pending_prune: bool, +) -> RuntimeVirtualizedBranchLifecycle { + match (active_branch, pending_prune) { + (RuntimeVirtualizedActiveBranch::Shadow, _) => { + RuntimeVirtualizedBranchLifecycle::FlattenedActive + } + (RuntimeVirtualizedActiveBranch::Original, true) => { + RuntimeVirtualizedBranchLifecycle::PhysicalActiveCleanupPending + } + (RuntimeVirtualizedActiveBranch::Original, false) => { + RuntimeVirtualizedBranchLifecycle::PhysicalActive + } + } +} + +fn preserve_cutover_pending_or_lifecycle_from_active_branch( + state: &VirtualizedSiteState, +) -> RuntimeVirtualizedBranchLifecycle { + if state.lifecycle == RuntimeVirtualizedBranchLifecycle::CutoverPending { + RuntimeVirtualizedBranchLifecycle::CutoverPending + } else { + lifecycle_from_active_branch(state.active_branch, state.pending_prune) + } +} + +fn runtime_status_for_virtualized_state( + state: &VirtualizedSiteState, +) -> RuntimeNodeOperationStatus { + match state.lifecycle { + RuntimeVirtualizedBranchLifecycle::Failed => RuntimeNodeOperationStatus::Dirty, + RuntimeVirtualizedBranchLifecycle::FlattenedActive + | RuntimeVirtualizedBranchLifecycle::PhysicalActiveCleanupPending + if state.pending_prune => + { + RuntimeNodeOperationStatus::AppliedAwaitingCleanup + } + RuntimeVirtualizedBranchLifecycle::FlattenBuildPending + | RuntimeVirtualizedBranchLifecycle::CutoverPending + | RuntimeVirtualizedBranchLifecycle::RestorePending => RuntimeNodeOperationStatus::Applying, + RuntimeVirtualizedBranchLifecycle::PhysicalActive + | RuntimeVirtualizedBranchLifecycle::PhysicalActiveCleanupPending + | RuntimeVirtualizedBranchLifecycle::FlattenedActive => { + RuntimeNodeOperationStatus::Completed + } + } +} + +fn runtime_lifecycle_label(lifecycle: RuntimeVirtualizedBranchLifecycle) -> &'static str { + match lifecycle { + RuntimeVirtualizedBranchLifecycle::PhysicalActive => "PhysicalActive", + RuntimeVirtualizedBranchLifecycle::FlattenBuildPending => "FlattenBuildPending", + RuntimeVirtualizedBranchLifecycle::CutoverPending => "CutoverPending", + RuntimeVirtualizedBranchLifecycle::FlattenedActive => "FlattenedActive", + RuntimeVirtualizedBranchLifecycle::RestorePending => "RestorePending", + RuntimeVirtualizedBranchLifecycle::PhysicalActiveCleanupPending => { + "PhysicalActiveCleanupPending" + } + RuntimeVirtualizedBranchLifecycle::Failed => "Failed", + } +} + +fn runtime_active_branch_label(active_branch: RuntimeVirtualizedActiveBranch) -> &'static str { + match active_branch { + RuntimeVirtualizedActiveBranch::Shadow => "Shadow", + RuntimeVirtualizedActiveBranch::Original => "Original", + } +} + +const RUNTIME_SITE_PRUNE_RETRY_SECONDS: u64 = 30; +const RUNTIME_SITE_PRUNE_MAX_ATTEMPTS: u32 = 5; +const RUNTIME_CUTOVER_RETRY_SECONDS: u64 = 1; +const BAKERY_BACKGROUND_INTERVAL_MS: u64 = 250; +const RUNTIME_DIRTY_SUBTREE_RELOAD_THRESHOLD: usize = 3; +const RUNTIME_NODE_OPERATION_CAPACITY: usize = 32; +const RUNTIME_NODE_OPERATION_DEFERRED_RETRY_SECONDS: u64 = 60; +const BAKERY_GROUPED_EVENT_DETAIL_LIMIT: usize = 3; + +#[derive(Clone, Debug)] +struct RuntimeNodeOperation { + operation_id: u64, + site_hash: i64, + site_name: Option, + action: RuntimeNodeOperationAction, + status: RuntimeNodeOperationStatus, + attempt_count: u32, + submitted_at_unix: u64, + updated_at_unix: u64, + next_retry_at_unix: Option, + last_error: Option, + failure_reason: Option, +} + +impl RuntimeNodeOperation { + fn new( + operation_id: u64, + site_hash: i64, + site_name: Option, + action: RuntimeNodeOperationAction, + now_unix: u64, + ) -> Self { + Self { + operation_id, + site_hash, + site_name, + action, + status: RuntimeNodeOperationStatus::Submitted, + attempt_count: 0, + submitted_at_unix: now_unix, + updated_at_unix: now_unix, + next_retry_at_unix: None, + last_error: None, + failure_reason: None, + } + } + + fn snapshot(&self) -> RuntimeNodeOperationSnapshot { + RuntimeNodeOperationSnapshot { + operation_id: self.operation_id, + site_hash: self.site_hash, + action: self.action, + status: self.status, + attempt_count: self.attempt_count, + submitted_at_unix: self.submitted_at_unix, + updated_at_unix: self.updated_at_unix, + next_retry_at_unix: self.next_retry_at_unix, + last_error: self.last_error.clone(), + failure_reason: self.failure_reason, + } + } + + fn update_status( + &mut self, + status: RuntimeNodeOperationStatus, + now_unix: u64, + last_error: Option, + next_retry_at_unix: Option, + ) { + self.update_status_with_reason(status, now_unix, last_error, None, next_retry_at_unix); + } + + fn update_status_with_reason( + &mut self, + status: RuntimeNodeOperationStatus, + now_unix: u64, + last_error: Option, + failure_reason: Option, + next_retry_at_unix: Option, + ) { + self.status = status; + self.updated_at_unix = now_unix; + self.last_error = last_error; + self.failure_reason = failure_reason; + self.next_retry_at_unix = next_retry_at_unix; + } +} + +fn mark_migration_reload_required(circuit_hash: i64, purpose: &str, result: &ExecuteResult) { + let summary = summarize_apply_result(purpose, result); + mark_reload_required(format!( + "Bakery live migration failed for circuit {} during {}: {}. A full reload is now required before further incremental topology mutations.", + circuit_hash, purpose, summary + )); +} + +fn migration_stage_apply_succeeded( + migration: &mut Migration, + purpose: &str, + result: &ExecuteResult, + next_stage: MigrationStage, +) -> bool { + if result.ok { + migration.stage = next_stage; + return true; + } + + mark_migration_reload_required(migration.circuit_hash, purpose, result); + migration.stage = MigrationStage::Done; + false +} + fn parse_ip_list(s: &str) -> Vec { s.split(',') .map(|x| x.trim().to_string()) @@ -109,32 +448,95 @@ fn tc_handle_from_major_minor(major: u16, minor: u16) -> TcHandle { TcHandle::from_u32(((major as u32) << 16) | (minor as u32)) } -fn used_minors_for_parent( - circuits: &HashMap>, - parent: &TcHandle, -) -> HashSet { - let mut set = HashSet::new(); - for (_k, v) in circuits.iter() { - if let BakeryCommands::AddCircuit { +fn used_site_minors_for_majors( + sites: &HashMap>, + down_major: u16, + up_major: u16, +) -> (HashSet, HashSet) { + let mut used_down = HashSet::new(); + let mut used_up = HashSet::new(); + for site in sites.values() { + if let BakeryCommands::AddSite { parent_class_id, + up_parent_class_id, class_minor, .. - } = v.as_ref() - && parent_class_id == parent + } = site.as_ref() { - set.insert(*class_minor); + if parent_class_id.get_major_minor().0 == down_major { + used_down.insert(*class_minor); + } + if up_parent_class_id.get_major_minor().0 == up_major { + used_up.insert(*class_minor); + } } } - set + (used_down, used_up) } -fn find_free_minor( +fn used_circuit_minors_for_majors( circuits: &HashMap>, - down_parent: &TcHandle, - up_parent: &TcHandle, + down_major: u16, + up_major: u16, +) -> (HashSet, HashSet) { + let mut used_down = HashSet::new(); + let mut used_up = HashSet::new(); + for circuit in circuits.values() { + if let BakeryCommands::AddCircuit { + class_minor, + class_major, + up_class_major, + .. + } = circuit.as_ref() + { + if *class_major == down_major { + used_down.insert(*class_minor); + } + if *up_class_major == up_major { + used_up.insert(*class_minor); + } + } + } + (used_down, used_up) +} + +fn used_pending_migration_shadow_minors_for_majors( + migrations: &HashMap, + down_major: u16, + up_major: u16, +) -> (HashSet, HashSet) { + let mut used_down = HashSet::new(); + let mut used_up = HashSet::new(); + for migration in migrations.values() { + if migration.stage == MigrationStage::Done { + continue; + } + if migration.class_major == down_major { + used_down.insert(migration.shadow_minor); + } + if migration.up_class_major == up_major { + used_up.insert(migration.shadow_minor); + } + } + (used_down, used_up) +} + +fn find_free_circuit_shadow_minor( + sites: &HashMap>, + circuits: &HashMap>, + migrations: &HashMap, + down_major: u16, + up_major: u16, ) -> Option { - let used_down = used_minors_for_parent(circuits, down_parent); - let used_up = used_minors_for_parent(circuits, up_parent); + let (mut used_down, mut used_up) = used_site_minors_for_majors(sites, down_major, up_major); + let (used_circuit_down, used_circuit_up) = + used_circuit_minors_for_majors(circuits, down_major, up_major); + used_down.extend(used_circuit_down); + used_up.extend(used_circuit_up); + let (used_shadow_down, used_shadow_up) = + used_pending_migration_shadow_minors_for_majors(migrations, down_major, up_major); + used_down.extend(used_shadow_down); + used_up.extend(used_shadow_up); for start in [0x2000u16, 0x4000, 0x6000, 0x8000, 0xA000, 0xC000, 0xE000] { let end = start.saturating_add(0x1FFF); for m in start..=end.min(0xFFFE) { @@ -146,6 +548,73 @@ fn find_free_minor( (1..=0xFFFEu16).find(|&m| !used_down.contains(&m) && !used_up.contains(&m)) } +fn find_free_site_shadow_minor( + sites: &HashMap>, + circuits: &HashMap>, + migrations: &HashMap, + planned_sites: &HashMap, + planned_circuits: &HashMap, + down_parent: &TcHandle, + up_parent: &TcHandle, +) -> Option { + let down_major = down_parent.get_major_minor().0; + let up_major = up_parent.get_major_minor().0; + let (mut used_down, mut used_up) = used_site_minors_for_majors(sites, down_major, up_major); + let (used_circuit_down, used_circuit_up) = + used_circuit_minors_for_majors(circuits, down_major, up_major); + used_down.extend(used_circuit_down); + used_up.extend(used_circuit_up); + let (used_shadow_down, used_shadow_up) = + used_pending_migration_shadow_minors_for_majors(migrations, down_major, up_major); + used_down.extend(used_shadow_down); + used_up.extend(used_shadow_up); + + for update in planned_sites.values() { + if let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + class_minor, + .. + } = update.command.as_ref() + { + if parent_class_id.get_major_minor().0 == down_major { + used_down.insert(*class_minor); + } + if up_parent_class_id.get_major_minor().0 == up_major { + used_up.insert(*class_minor); + } + } + } + + for update in planned_circuits.values() { + if let BakeryCommands::AddCircuit { + class_minor, + class_major, + up_class_major, + .. + } = update.command.as_ref() + { + if *class_major == down_major { + used_down.insert(*class_minor); + } + if *up_class_major == up_major { + used_up.insert(*class_minor); + } + } + } + + for start in [0x2000u16, 0x4000, 0x6000, 0x8000, 0xA000, 0xC000, 0xE000] { + let end = start.saturating_add(0x1FFF); + for minor in start..=end.min(0xFFFE) { + if !used_down.contains(&minor) && !used_up.contains(&minor) { + return Some(minor); + } + } + } + + (1..=0xFFFEu16).find(|minor| !used_down.contains(minor) && !used_up.contains(minor)) +} + fn add_commands_for_circuit( cmd: &BakeryCommands, config: &Arc, @@ -161,13 +630,18 @@ fn build_temp_add_cmd( down_max: f32, up_min: f32, up_max: f32, + preserve_qdisc_handles: bool, ) -> Option { if let BakeryCommands::AddCircuit { circuit_hash, + circuit_name, + site_name, parent_class_id, up_parent_class_id, class_major, up_class_major, + down_qdisc_handle, + up_qdisc_handle, ip_addresses, sqm_override, .. @@ -175,6 +649,8 @@ fn build_temp_add_cmd( { Some(BakeryCommands::AddCircuit { circuit_hash: *circuit_hash, + circuit_name: circuit_name.clone(), + site_name: site_name.clone(), parent_class_id: *parent_class_id, up_parent_class_id: *up_parent_class_id, class_minor: minor, @@ -184,6 +660,10 @@ fn build_temp_add_cmd( upload_bandwidth_max: up_max, class_major: *class_major, up_class_major: *up_class_major, + down_qdisc_handle: preserve_qdisc_handles + .then_some(*down_qdisc_handle) + .flatten(), + up_qdisc_handle: preserve_qdisc_handles.then_some(*up_qdisc_handle).flatten(), ip_addresses: ip_addresses.clone(), sqm_override: sqm_override.clone(), }) @@ -192,22 +672,2007 @@ fn build_temp_add_cmd( } } +fn shadow_qdisc_allocation_key(circuit_hash: i64, uplink: bool) -> i64 { + let base = circuit_hash.wrapping_mul(2).wrapping_neg(); + if uplink { base.wrapping_sub(1) } else { base } +} + +fn assign_shadow_qdisc_handles( + migration: &Migration, + config: &Arc, + qdisc_handles: &QdiscHandleState, + live_reserved_handles: &HashMap>, +) -> (Option, Option) { + let mut shadow_handles = qdisc_handles.clone(); + let isp_interface = config.isp_interface(); + let mut down_reserved = live_reserved_handles + .get(&isp_interface) + .cloned() + .unwrap_or_default(); + if let Some(handle) = migration.old_down_qdisc_handle { + down_reserved.insert(handle); + } + if let Some(handle) = migration.down_qdisc_handle { + down_reserved.insert(handle); + } + let down_handle = migration.down_qdisc_handle.and_then(|_| { + shadow_handles.assign_circuit_handle( + &isp_interface, + shadow_qdisc_allocation_key(migration.circuit_hash, false), + &down_reserved, + ) + }); + + if config.on_a_stick_mode() { + return (down_handle, None); + } + + let internet_interface = config.internet_interface(); + let mut up_reserved = live_reserved_handles + .get(&internet_interface) + .cloned() + .unwrap_or_default(); + if let Some(handle) = migration.old_up_qdisc_handle { + up_reserved.insert(handle); + } + if let Some(handle) = migration.up_qdisc_handle { + up_reserved.insert(handle); + } + let up_handle = migration.up_qdisc_handle.and_then(|_| { + shadow_handles.assign_circuit_handle( + &internet_interface, + shadow_qdisc_allocation_key(migration.circuit_hash, true), + &up_reserved, + ) + }); + + (down_handle, up_handle) +} + +fn build_shadow_add_cmd( + migration: &Migration, + config: &Arc, + qdisc_handles: &QdiscHandleState, + live_reserved_handles: &HashMap>, +) -> Option { + let mut temp = build_temp_add_cmd( + &BakeryCommands::AddCircuit { + circuit_hash: migration.circuit_hash, + circuit_name: migration.circuit_name.clone(), + site_name: migration.site_name.clone(), + parent_class_id: migration.parent_class_id, + up_parent_class_id: migration.up_parent_class_id, + class_minor: migration.shadow_minor, + download_bandwidth_min: migration.old_down_min, + upload_bandwidth_min: migration.old_up_min, + download_bandwidth_max: migration.old_down_max, + upload_bandwidth_max: migration.old_up_max, + class_major: migration.class_major, + up_class_major: migration.up_class_major, + down_qdisc_handle: migration.down_qdisc_handle, + up_qdisc_handle: migration.up_qdisc_handle, + ip_addresses: String::new(), + sqm_override: migration.sqm_override.clone(), + }, + migration.shadow_minor, + migration.old_down_min, + migration.old_down_max, + migration.old_up_min, + migration.old_up_max, + false, + )?; + let (down_qdisc_handle, up_qdisc_handle) = + assign_shadow_qdisc_handles(migration, config, qdisc_handles, live_reserved_handles); + let BakeryCommands::AddCircuit { + down_qdisc_handle: shadow_down_qdisc_handle, + up_qdisc_handle: shadow_up_qdisc_handle, + .. + } = &mut temp + else { + return None; + }; + *shadow_down_qdisc_handle = down_qdisc_handle; + *shadow_up_qdisc_handle = up_qdisc_handle; + Some(temp) +} + +fn queue_runtime_migration( + old_cmd: &BakeryCommands, + new_cmd: &Arc, + sites: &HashMap>, + circuits: &mut HashMap>, + live_circuits: &HashMap, + migrations: &mut HashMap, + require_live_circuit: bool, +) -> bool { + let BakeryCommands::AddCircuit { + circuit_hash, + circuit_name, + site_name, + parent_class_id, + up_parent_class_id, + class_minor, + download_bandwidth_min, + upload_bandwidth_min, + download_bandwidth_max, + upload_bandwidth_max, + class_major, + up_class_major, + down_qdisc_handle, + up_qdisc_handle, + ip_addresses, + sqm_override, + .. + } = new_cmd.as_ref() + else { + return false; + }; + + if require_live_circuit && !live_circuits.contains_key(circuit_hash) { + return false; + } + + let BakeryCommands::AddCircuit { + class_minor: old_minor, + download_bandwidth_min: old_down_min, + upload_bandwidth_min: old_up_min, + download_bandwidth_max: old_down_max, + upload_bandwidth_max: old_up_max, + class_major: old_class_major, + up_class_major: old_up_class_major, + down_qdisc_handle: old_down_qdisc_handle, + up_qdisc_handle: old_up_qdisc_handle, + .. + } = old_cmd + else { + return false; + }; + + let Some(shadow_minor) = + find_free_circuit_shadow_minor(sites, circuits, migrations, *class_major, *up_class_major) + else { + return false; + }; + + let mig = Migration { + circuit_hash: *circuit_hash, + circuit_name: circuit_name.clone(), + site_name: site_name.clone(), + old_class_major: *old_class_major, + old_up_class_major: *old_up_class_major, + old_down_qdisc_handle: *old_down_qdisc_handle, + old_up_qdisc_handle: *old_up_qdisc_handle, + parent_class_id: *parent_class_id, + up_parent_class_id: *up_parent_class_id, + class_major: *class_major, + up_class_major: *up_class_major, + down_qdisc_handle: *down_qdisc_handle, + up_qdisc_handle: *up_qdisc_handle, + old_down_min: *old_down_min, + old_down_max: *old_down_max, + old_up_min: *old_up_min, + old_up_max: *old_up_max, + new_down_min: *download_bandwidth_min, + new_down_max: *download_bandwidth_max, + new_up_min: *upload_bandwidth_min, + new_up_max: *upload_bandwidth_max, + old_minor: *old_minor, + shadow_minor, + final_minor: *class_minor, + ips: parse_ip_list(ip_addresses), + sqm_override: sqm_override.clone(), + desired_cmd: Arc::clone(new_cmd), + stage: MigrationStage::PrepareShadow, + shadow_verify_attempts: 0, + final_verify_attempts: 0, + }; + + migrations.insert(*circuit_hash, mig); + true +} + +fn queue_live_migration( + old_cmd: &BakeryCommands, + new_cmd: &Arc, + sites: &HashMap>, + circuits: &mut HashMap>, + live_circuits: &HashMap, + migrations: &mut HashMap, +) -> bool { + queue_runtime_migration( + old_cmd, + new_cmd, + sites, + circuits, + live_circuits, + migrations, + true, + ) +} + +fn queue_top_level_runtime_migration( + old_cmd: &BakeryCommands, + new_cmd: &Arc, + sites: &HashMap>, + circuits: &mut HashMap>, + live_circuits: &HashMap, + migrations: &mut HashMap, +) -> bool { + queue_runtime_migration( + old_cmd, + new_cmd, + sites, + circuits, + live_circuits, + migrations, + false, + ) +} + +fn circuits_with_pending_migration_targets( + circuits: &HashMap>, + migrations: &HashMap, +) -> HashMap> { + let mut overlaid = circuits.clone(); + for (circuit_hash, migration) in migrations { + overlaid.insert(*circuit_hash, Arc::clone(&migration.desired_cmd)); + } + overlaid +} + /// Count of Bakery-Managed circuits that are currently active. pub static ACTIVE_CIRCUITS: AtomicUsize = AtomicUsize::new(0); +/// True while Bakery is applying a full reload batch to `tc`. +static FULL_RELOAD_IN_PROGRESS: AtomicBool = AtomicBool::new(false); +#[cfg(test)] +pub(crate) fn test_state_lock() -> &'static std::sync::Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| std::sync::Mutex::new(())) +} +/// Hard kernel limit for auto-allocated qdisc handles on a single network interface. +pub const HARD_QDISC_HANDLE_LIMIT_PER_INTERFACE: usize = 65_534; +/// Conservative operational limit used to fail a full reload before qdisc handle exhaustion. +pub const SAFE_QDISC_BUDGET_PER_INTERFACE: usize = 65_000; +const LIVE_CAPACITY_REFRESH_SECONDS: u64 = 30 * 60; +/// Conservative estimated kernel-memory cost of a non-leaf infrastructure qdisc. +/// +/// These weights are not kernel ABI sizes. They are deliberately conservative +/// safety estimates used to block or stop obviously risky full reloads before +/// the host reaches OOM pressure. +pub const INFRA_QDISC_ESTIMATED_MEMORY_BYTES: u64 = 16 * 1024; +/// Conservative estimated kernel-memory cost of an `fq_codel` leaf qdisc. +/// +/// This remains intentionally lower than CAKE, but is biased high enough to +/// prefer false-positive safety blocks over OOM risk on large reloads. +pub const FQ_CODEL_QDISC_ESTIMATED_MEMORY_BYTES: u64 = 64 * 1024; +/// Conservative estimated kernel-memory cost of a `cake` leaf qdisc. +/// +/// Tuned upward after live production capture showed summed CAKE runtime memory +/// substantially exceeding the earlier heuristic during busy periods. +pub const CAKE_QDISC_ESTIMATED_MEMORY_BYTES: u64 = 512 * 1024; +/// Minimum memory headroom Bakery tries to leave unused after a projected or in-flight apply. +pub const BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES: u64 = 768 * 1024 * 1024; /// Maximum number of mapped circuits allowed without Insight. const DEFAULT_MAPPED_CIRCUITS_LIMIT: usize = 1000; /// Minimum interval between repeated mapped-circuit-limit urgent issues. const CIRCUIT_LIMIT_URGENT_INTERVAL_SECONDS: u64 = 30 * 60; /// Last timestamp at which we emitted a mapped-circuit-limit urgent issue. static LAST_CIRCUIT_LIMIT_URGENT_TS: AtomicU64 = AtomicU64::new(0); +const BAKERY_EVENT_LIMIT: usize = 50; +const FULL_RELOAD_TC_CHUNK_SIZE: usize = 2_500; + +/// High-level Bakery execution mode for operator-facing status. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BakeryMode { + /// Bakery is not currently applying queue changes. + Idle, + /// Bakery is applying a structural full reload. + ApplyingFullReload, + /// Bakery is applying an incremental or live change. + ApplyingLiveChange, +} + +/// Type of the most recent apply action recorded by Bakery. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BakeryApplyType { + /// No apply has been recorded yet. + None, + /// A full reload apply. + FullReload, + /// A live/incremental apply. + LiveChange, +} + +/// Per-interface qdisc budget usage snapshot for Bakery UI/status. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BakeryCapacityInterfaceSnapshot { + /// Interface name, e.g. `ens19`. + pub name: String, + /// Planned qdisc count for that interface. + pub planned_qdiscs: usize, + /// Planned infrastructure qdiscs (`mq`, `htb`, etc.) for that interface. + pub infra_qdiscs: usize, + /// Planned `cake` leaf qdiscs for that interface. + pub cake_qdiscs: usize, + /// Planned `fq_codel` leaf qdiscs for that interface. + pub fq_codel_qdiscs: usize, + /// Estimated kernel memory cost for that interface's planned qdiscs. + pub estimated_memory_bytes: u64, +} + +/// Per-interface live qdisc usage snapshot for Bakery UI/status. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BakeryLiveCapacityInterfaceSnapshot { + /// Interface name, e.g. `ens19`. + pub name: String, + /// Current live qdisc handle count observed on the interface. + pub live_qdiscs: usize, +} + +/// Per-queue-root queue layout summary for Bakery UI/status. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BakeryQueueDistributionSnapshot { + /// Queue/root number, e.g. `1` for `Q1`. + pub queue: u32, + /// Number of top-level sites currently assigned to this queue. + pub top_level_site_count: usize, + /// Total sites currently assigned to this queue. + pub site_count: usize, + /// Total circuits currently assigned to this queue. + pub circuit_count: usize, + /// Aggregate configured downstream max Mbps for circuits on this queue. + pub download_mbps: u64, + /// Aggregate configured upstream max Mbps for circuits on this queue. + pub upload_mbps: u64, +} + +/// Latest Bakery-tracked runtime node operation for UI/status. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BakeryRuntimeOperationHeadlineSnapshot { + /// Monotonic Bakery-local operation identifier. + pub operation_id: u64, + /// Stable Bakery site hash derived from the node name. + pub site_hash: i64, + /// Human site name retained by Bakery when the runtime operation was created. + pub site_name: Option, + /// Requested runtime action. + pub action: RuntimeNodeOperationAction, + /// Current operation status. + pub status: RuntimeNodeOperationStatus, + /// Number of attempts performed so far. + pub attempt_count: u32, + /// Unix timestamp when the operation last changed state. + pub updated_at_unix: u64, + /// Optional unix timestamp for the next retry, if waiting. + pub next_retry_at_unix: Option, + /// Last error observed by Bakery for this operation, if any. + pub last_error: Option, +} + +/// Latest Bakery-tracked runtime branch-state snapshot for a single node. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BakeryRuntimeNodeBranchSnapshot { + /// Stable Bakery site hash derived from the node name. + pub site_hash: i64, + /// Human site name retained by Bakery when the runtime branch was created. + pub site_name: String, + /// Which retained branch is currently active for this node. + pub active_branch: String, + /// Current runtime lifecycle label for this node. + pub lifecycle: String, + /// Whether Bakery is still waiting to prune the inactive branch. + pub pending_prune: bool, + /// Optional unix timestamp for the next cleanup retry, if one is pending. + pub next_prune_attempt_unix: Option, + /// Hashes of branch sites currently marked active. + pub active_site_hashes: Vec, + /// Hashes of original/saved sites retained for restore logic. + pub saved_site_hashes: Vec, + /// Hashes of inactive-branch sites queued for pruning. + pub prune_site_hashes: Vec, + /// Observed downlink qdisc major for the runtime root, if known. + pub qdisc_down_major: Option, + /// Observed uplink qdisc major for the runtime root, if known. + pub qdisc_up_major: Option, +} + +/// Compact runtime-operation summary for Bakery UI/status. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BakeryRuntimeOperationsSnapshot { + /// Number of submitted operations waiting to start. + pub submitted_count: usize, + /// Number of operations deferred because runtime-node capacity is currently saturated. + pub deferred_count: usize, + /// Number of operations currently applying. + pub applying_count: usize, + /// Number of operations awaiting deferred cleanup. + pub awaiting_cleanup_count: usize, + /// Number of failed operations that may be retried. + pub failed_count: usize, + /// Number of operations marked Dirty. + pub dirty_count: usize, + /// Most recently updated runtime operation, if any. + pub latest: Option, +} + +/// Last qdisc preflight result retained for Bakery UI/status. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BakeryPreflightSnapshot { + /// Whether the last preflight was within budget. + pub ok: bool, + /// Short operator-facing summary. + pub message: String, + /// Safe operational budget used during the preflight. + pub safe_budget: usize, + /// Hard kernel limit used for reference. + pub hard_limit: usize, + /// Estimated total kernel memory cost of the planned qdisc model. + pub estimated_total_memory_bytes: u64, + /// Current host memory available during the preflight, if known. + pub memory_available_bytes: Option, + /// Memory floor that must remain available before the apply proceeds. + pub memory_guard_min_available_bytes: u64, + /// Whether the memory preflight passed. + pub memory_ok: bool, + /// Per-interface planned qdisc counts. + pub interfaces: Vec, +} + +/// Operator-facing Bakery status snapshot. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BakeryStatusSnapshot { + /// Current Bakery mode. + pub mode: BakeryMode, + /// Unix timestamp when the current action started, if any. + pub current_action_started_unix: Option, + /// Current apply phase for the in-flight action, if any. + pub current_apply_phase: Option, + /// Total `tc` commands planned for the in-flight action. + pub current_apply_total_tc_commands: usize, + /// `tc` commands already completed for the in-flight action. + pub current_apply_completed_tc_commands: usize, + /// Total apply chunks planned for the in-flight action. + pub current_apply_total_chunks: usize, + /// Apply chunks already completed for the in-flight action. + pub current_apply_completed_chunks: usize, + /// Currently active circuit count. + pub active_circuits: usize, + /// Unix timestamp of the last successful apply, if any. + pub last_success_unix: Option, + /// Unix timestamp of the last successful full reload, if any. + pub last_full_reload_success_unix: Option, + /// Unix timestamp of the last failed apply, if any. + pub last_failure_unix: Option, + /// Summary of the last failure, if any. + pub last_failure_summary: Option, + /// Type of the last apply recorded by Bakery. + pub last_apply_type: BakeryApplyType, + /// Number of `tc` commands in the last apply. + pub last_total_tc_commands: usize, + /// Number of `class` commands in the last apply. + pub last_class_commands: usize, + /// Number of `qdisc` commands in the last apply. + pub last_qdisc_commands: usize, + /// Time spent expanding/building the last apply command list. + pub last_build_duration_ms: u64, + /// Time spent running the last apply through `tc`. + pub last_apply_duration_ms: u64, + /// Average interval between real unified Bakery `tc` reads/writes, in milliseconds. + pub avg_tc_io_interval_ms: Option, + /// Unix timestamp when Bakery last performed a real unified `tc` read/write. + pub last_tc_io_unix: Option, + /// Number of recent intervals contributing to the average TC I/O interval. + pub tc_io_interval_samples: usize, + /// Current runtime node-operation summary. + pub runtime_operations: BakeryRuntimeOperationsSnapshot, + /// Current queue-root distribution summary. + pub queue_distribution: Vec, + /// Current live per-interface qdisc usage snapshot. + pub live_capacity_interfaces: Vec, + /// Safe per-interface qdisc budget used for the live usage bar. + pub live_capacity_safe_budget: usize, + /// Unix timestamp when the live capacity snapshot was last refreshed. + pub live_capacity_updated_at_unix: Option, + /// Last qdisc preflight summary known to Bakery. + pub preflight: Option, + /// Whether Bakery has detected enough runtime drift to require a structural full reload. + pub reload_required: bool, + /// Operator-facing reason for why a full reload is now required, if any. + pub reload_required_reason: Option, + /// Number of runtime node operations currently marked dirty. + pub dirty_subtree_count: usize, +} + +/// Recent operator-facing Bakery activity event. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BakeryActivityEntry { + /// Unix timestamp in seconds. + pub ts: u64, + /// Stable short event code. + pub event: String, + /// `info`, `warning`, or `error`. + pub status: String, + /// Stable Bakery site hash associated with the event, if any. + pub site_hash: Option, + /// Human site name retained by Bakery when the event was created, if any. + pub site_name: Option, + /// Human-readable summary. + pub summary: String, +} + +#[derive(Clone, Debug)] +struct BakeryTelemetryState { + mode: BakeryMode, + current_action_started_unix: Option, + current_apply_phase: Option, + current_apply_total_tc_commands: usize, + current_apply_completed_tc_commands: usize, + current_apply_total_chunks: usize, + current_apply_completed_chunks: usize, + last_success_unix: Option, + last_full_reload_success_unix: Option, + last_failure_unix: Option, + last_failure_summary: Option, + last_apply_type: BakeryApplyType, + last_total_tc_commands: usize, + last_class_commands: usize, + last_qdisc_commands: usize, + last_build_duration_ms: u64, + last_apply_duration_ms: u64, + runtime_operations: BakeryRuntimeOperationsSnapshot, + queue_distribution: Vec, + live_capacity_interfaces: Vec, + live_capacity_updated_at_unix: Option, + preflight: Option, + reload_required: bool, + reload_required_reason: Option, + dirty_subtree_count: usize, + runtime_operations_by_site: HashMap, + runtime_branch_states_by_site: HashMap, + activity: VecDeque, +} + +struct GroupedBakeryEventState { + event: String, + status: String, + emitted: usize, + suppressed: usize, + suppression_summary: String, +} + +#[derive(Default)] +struct GroupedBakeryEventLimiter { + groups: BTreeMap, +} + +impl GroupedBakeryEventLimiter { + fn emit_with_site_name( + &mut self, + group_key: impl Into, + event: &str, + status: &str, + site: Option<(i64, Option)>, + summary: String, + suppression_summary: String, + ) { + let (site_hash, site_name) = site.map_or((None, None), |(site_hash, site_name)| { + (Some(site_hash), site_name) + }); + let state = + self.groups + .entry(group_key.into()) + .or_insert_with(|| GroupedBakeryEventState { + event: event.to_string(), + status: status.to_string(), + emitted: 0, + suppressed: 0, + suppression_summary, + }); + if state.emitted < BAKERY_GROUPED_EVENT_DETAIL_LIMIT { + push_bakery_event_with_site_name(event, status, site_hash, site_name, summary); + state.emitted += 1; + } else { + state.suppressed += 1; + } + } + + fn flush(self) { + for state in self.groups.into_values() { + if state.suppressed == 0 { + continue; + } + push_bakery_event( + &state.event, + &state.status, + format!( + "Suppressed {} additional {} this batch.", + state.suppressed, state.suppression_summary + ), + ); + } + } +} + +#[derive(Clone, Copy, Debug)] +struct BakeryApplyMetrics<'a> { + apply_type: BakeryApplyType, + summary: &'a str, + build_duration_ms: u64, + apply_duration_ms: u64, + total_tc_commands: usize, + class_commands: usize, + qdisc_commands: usize, + ok: bool, +} + +impl Default for BakeryTelemetryState { + fn default() -> Self { + Self { + mode: BakeryMode::Idle, + current_action_started_unix: None, + current_apply_phase: None, + current_apply_total_tc_commands: 0, + current_apply_completed_tc_commands: 0, + current_apply_total_chunks: 0, + current_apply_completed_chunks: 0, + last_success_unix: None, + last_full_reload_success_unix: None, + last_failure_unix: None, + last_failure_summary: None, + last_apply_type: BakeryApplyType::None, + last_total_tc_commands: 0, + last_class_commands: 0, + last_qdisc_commands: 0, + last_build_duration_ms: 0, + last_apply_duration_ms: 0, + runtime_operations: BakeryRuntimeOperationsSnapshot { + submitted_count: 0, + deferred_count: 0, + applying_count: 0, + awaiting_cleanup_count: 0, + failed_count: 0, + dirty_count: 0, + latest: None, + }, + queue_distribution: Vec::new(), + live_capacity_interfaces: Vec::new(), + live_capacity_updated_at_unix: None, + preflight: None, + reload_required: false, + reload_required_reason: None, + dirty_subtree_count: 0, + runtime_operations_by_site: HashMap::new(), + runtime_branch_states_by_site: HashMap::new(), + activity: VecDeque::with_capacity(BAKERY_EVENT_LIMIT), + } + } +} + +static BAKERY_TELEMETRY: OnceLock> = OnceLock::new(); /// Message Queue sender for the bakery pub static BAKERY_SENDER: OnceLock> = OnceLock::new(); static MQ_CREATED: AtomicBool = AtomicBool::new(false); +static SHAPING_TREE_ACTIVE: AtomicBool = AtomicBool::new(false); /// Indicates that at least one command batch has been processed and applied. /// Used to avoid racing live activation against initial class creation. static FIRST_COMMIT_APPLIED: AtomicBool = AtomicBool::new(false); +struct FullReloadScope; + +impl Drop for FullReloadScope { + fn drop(&mut self) { + FULL_RELOAD_IN_PROGRESS.store(false, Ordering::Relaxed); + let mut state = telemetry_state().write(); + clear_bakery_apply_progress(&mut state); + if state.mode == BakeryMode::ApplyingFullReload { + state.mode = BakeryMode::Idle; + state.current_action_started_unix = None; + } + } +} + +fn telemetry_state() -> &'static RwLock { + BAKERY_TELEMETRY.get_or_init(|| RwLock::new(BakeryTelemetryState::default())) +} + +fn count_tc_command_types(commands: &[Vec]) -> (usize, usize, usize) { + let total = commands.len(); + let mut class_count = 0usize; + let mut qdisc_count = 0usize; + for argv in commands { + if let Some(kind) = argv.first() { + match kind.as_str() { + "class" => class_count += 1, + "qdisc" => qdisc_count += 1, + _ => {} + } + } + } + (total, class_count, qdisc_count) +} + +fn push_bakery_event(event: &str, status: &str, summary: String) { + push_bakery_event_with_site(event, status, None, summary); +} + +fn push_bakery_event_with_site(event: &str, status: &str, site_hash: Option, summary: String) { + push_bakery_event_with_site_name(event, status, site_hash, None, summary); +} + +fn push_bakery_event_with_site_name( + event: &str, + status: &str, + site_hash: Option, + site_name: Option, + summary: String, +) { + let entry = BakeryActivityEntry { + ts: current_timestamp(), + event: event.to_string(), + status: status.to_string(), + site_hash, + site_name, + summary, + }; + let mut state = telemetry_state().write(); + state.activity.push_front(entry); + while state.activity.len() > BAKERY_EVENT_LIMIT { + state.activity.pop_back(); + } +} + +fn announce_full_reload(summary: &str) { + warn!("{summary}"); + push_bakery_event("full_reload_trigger", "warning", summary.to_string()); +} + +fn clear_bakery_apply_progress(state: &mut BakeryTelemetryState) { + state.current_apply_phase = None; + state.current_apply_total_tc_commands = 0; + state.current_apply_completed_tc_commands = 0; + state.current_apply_total_chunks = 0; + state.current_apply_completed_chunks = 0; +} + +fn mark_reload_required(summary: String) { + let mut should_emit = false; + { + let mut state = telemetry_state().write(); + if !state.reload_required + || state.reload_required_reason.as_deref() != Some(summary.as_str()) + { + state.reload_required = true; + state.reload_required_reason = Some(summary.clone()); + should_emit = true; + } + } + if should_emit { + push_bakery_event("reload_required", "error", summary); + } +} + +fn clear_reload_required(summary: &str) { + let mut should_emit = false; + { + let mut state = telemetry_state().write(); + if state.reload_required || state.reload_required_reason.is_some() { + state.reload_required = false; + state.reload_required_reason = None; + should_emit = true; + } + } + if should_emit { + push_bakery_event("reload_required_cleared", "info", summary.to_string()); + } +} + +fn mark_bakery_action_started(mode: BakeryMode, event: &str, summary: String) { + let ts = current_timestamp(); + { + let mut state = telemetry_state().write(); + state.mode = mode; + state.current_action_started_unix = Some(ts); + clear_bakery_apply_progress(&mut state); + } + push_bakery_event(event, "info", summary); +} + +fn update_bakery_apply_progress( + phase: Option<&str>, + total_tc_commands: usize, + completed_tc_commands: usize, + total_chunks: usize, + completed_chunks: usize, +) { + let mut state = telemetry_state().write(); + state.current_apply_phase = phase.map(str::to_string); + state.current_apply_total_tc_commands = total_tc_commands; + state.current_apply_completed_tc_commands = completed_tc_commands; + state.current_apply_total_chunks = total_chunks; + state.current_apply_completed_chunks = completed_chunks; +} + +fn mark_bakery_action_finished(metrics: BakeryApplyMetrics<'_>) { + let ts = current_timestamp(); + { + let mut state = telemetry_state().write(); + state.last_apply_type = metrics.apply_type; + state.last_total_tc_commands = metrics.total_tc_commands; + state.last_class_commands = metrics.class_commands; + state.last_qdisc_commands = metrics.qdisc_commands; + state.last_build_duration_ms = metrics.build_duration_ms; + state.last_apply_duration_ms = metrics.apply_duration_ms; + if metrics.ok { + state.last_success_unix = Some(ts); + if metrics.apply_type == BakeryApplyType::FullReload { + state.last_full_reload_success_unix = Some(ts); + } + } else { + state.last_failure_unix = Some(ts); + state.last_failure_summary = Some(metrics.summary.to_string()); + } + clear_bakery_apply_progress(&mut state); + if state.mode != BakeryMode::ApplyingFullReload { + state.mode = BakeryMode::Idle; + state.current_action_started_unix = None; + } + } + push_bakery_event( + if metrics.ok { + "apply_finished" + } else { + "apply_failed" + }, + if metrics.ok { "info" } else { "error" }, + metrics.summary.to_string(), + ); +} + +/// Returns the latest Bakery status snapshot for UI/status consumers. +pub fn bakery_status_snapshot() -> BakeryStatusSnapshot { + let state = telemetry_state().read().clone(); + let tc_io = tc_io_cadence_snapshot(); + BakeryStatusSnapshot { + mode: state.mode, + current_action_started_unix: state.current_action_started_unix, + current_apply_phase: state.current_apply_phase, + current_apply_total_tc_commands: state.current_apply_total_tc_commands, + current_apply_completed_tc_commands: state.current_apply_completed_tc_commands, + current_apply_total_chunks: state.current_apply_total_chunks, + current_apply_completed_chunks: state.current_apply_completed_chunks, + active_circuits: ACTIVE_CIRCUITS.load(Ordering::Relaxed), + last_success_unix: state.last_success_unix, + last_full_reload_success_unix: state.last_full_reload_success_unix, + last_failure_unix: state.last_failure_unix, + last_failure_summary: state.last_failure_summary, + last_apply_type: state.last_apply_type, + last_total_tc_commands: state.last_total_tc_commands, + last_class_commands: state.last_class_commands, + last_qdisc_commands: state.last_qdisc_commands, + last_build_duration_ms: state.last_build_duration_ms, + last_apply_duration_ms: state.last_apply_duration_ms, + avg_tc_io_interval_ms: tc_io.avg_interval_ms, + last_tc_io_unix: tc_io.last_event_unix, + tc_io_interval_samples: tc_io.sample_count, + runtime_operations: state.runtime_operations, + queue_distribution: state.queue_distribution, + live_capacity_interfaces: state.live_capacity_interfaces, + live_capacity_safe_budget: SAFE_QDISC_BUDGET_PER_INTERFACE, + live_capacity_updated_at_unix: state.live_capacity_updated_at_unix, + preflight: state.preflight, + reload_required: state.reload_required, + reload_required_reason: state.reload_required_reason, + dirty_subtree_count: state.dirty_subtree_count, + } +} + +/// Returns the latest Bakery-tracked runtime node-operation snapshot for a site, if any. +pub fn bakery_runtime_node_operation_snapshot( + site_hash: i64, +) -> Option { + telemetry_state() + .read() + .runtime_operations_by_site + .get(&site_hash) + .cloned() +} + +/// Returns the latest Bakery-tracked runtime branch-state snapshot for a site, if any. +pub fn bakery_runtime_node_branch_snapshot( + site_hash: i64, +) -> Option { + telemetry_state() + .read() + .runtime_branch_states_by_site + .get(&site_hash) + .cloned() +} + +/// Returns the current Bakery reload-required reason, if runtime drift has frozen incremental +/// topology mutation. +pub fn bakery_reload_required_reason() -> Option { + let state = telemetry_state().read(); + if state.reload_required { + return state.reload_required_reason.clone(); + } + None +} + +/// Returns recent Bakery activity entries for UI/status consumers. +pub fn bakery_activity_snapshot() -> Vec { + telemetry_state().read().activity.iter().cloned().collect() +} + +fn configured_live_capacity_interfaces(config: &Config) -> Vec { + let mut interfaces = BTreeSet::new(); + interfaces.insert(config.isp_interface()); + interfaces.insert(config.internet_interface()); + interfaces.into_iter().collect() +} + +fn refresh_live_capacity_snapshot(config: &Config, force: bool) { + let now_unix = current_timestamp(); + { + let state = telemetry_state().read(); + if !force + && state + .live_capacity_updated_at_unix + .is_some_and(|last| now_unix.saturating_sub(last) < LIVE_CAPACITY_REFRESH_SECONDS) + { + return; + } + } + + let interfaces = configured_live_capacity_interfaces(config); + let mut snapshot = Vec::with_capacity(interfaces.len()); + for interface in interfaces { + let live_qdiscs = read_live_qdisc_handle_majors(&interface) + .map(|handles| handles.len()) + .unwrap_or(0); + snapshot.push(BakeryLiveCapacityInterfaceSnapshot { + name: interface, + live_qdiscs, + }); + } + + let mut state = telemetry_state().write(); + state.live_capacity_interfaces = snapshot; + state.live_capacity_updated_at_unix = Some(now_unix); +} + +/// Stores the latest qdisc-budget preflight result for UI/status consumers. +/// +/// This function is not pure: it updates retained in-memory Bakery telemetry state. +pub fn record_qdisc_preflight_snapshot(snapshot: BakeryPreflightSnapshot) { + let ok = snapshot.ok; + let summary = snapshot.message.clone(); + { + let mut state = telemetry_state().write(); + state.preflight = Some(snapshot); + } + push_bakery_event( + if ok { + "preflight_ok" + } else { + "preflight_blocked" + }, + if ok { "info" } else { "warning" }, + summary, + ); +} + +fn summarize_apply_result(purpose: &str, result: &ExecuteResult) -> String { + match &result.failure_summary { + Some(summary) if !summary.is_empty() => format!("{purpose}: {summary}"), + _ if result.ok => format!("{purpose}: completed"), + _ => format!("{purpose}: failed"), + } +} + +fn consume_test_fault_once_from_path(path: &Path, purpose: &str) -> Option { + let Ok(raw) = std::fs::read_to_string(path) else { + return None; + }; + let needle = raw.trim(); + if needle.is_empty() || !(needle == "*" || purpose.contains(needle)) { + return None; + } + let _ = std::fs::remove_file(path); + Some(format!( + "synthetic Bakery test fault for purpose '{purpose}' matched selector '{needle}'" + )) +} + +fn consume_test_fault_once(purpose: &str) -> Option { + consume_test_fault_once_from_path(Path::new(TEST_FAULT_ONCE_PATH), purpose) +} + +fn execute_and_record_live_change(command_buffer: &[Vec], purpose: &str) -> ExecuteResult { + let (total, class_commands, qdisc_commands) = count_tc_command_types(command_buffer); + mark_bakery_action_started( + BakeryMode::ApplyingLiveChange, + "live_change_started", + format!("{purpose}: started"), + ); + let result = if let Some(fault_summary) = consume_test_fault_once(purpose) { + warn!("Bakery test fault injected: {fault_summary}"); + ExecuteResult { + ok: false, + duration_ms: 0, + failure_summary: Some(fault_summary), + } + } else { + execute_in_memory(command_buffer, purpose) + }; + let summary = summarize_apply_result(purpose, &result); + mark_bakery_action_finished(BakeryApplyMetrics { + apply_type: BakeryApplyType::LiveChange, + summary: &summary, + build_duration_ms: 0, + apply_duration_ms: result.duration_ms, + total_tc_commands: total, + class_commands, + qdisc_commands, + ok: result.ok, + }); + if result.ok + && let Ok(config) = lqos_config::load_config() + { + refresh_live_capacity_snapshot(&config, true); + } + result +} + +/// Estimated qdisc-handle usage for a planned full reload. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct QdiscInterfaceEstimate { + /// Total planned qdiscs for the interface. + pub planned_qdiscs: usize, + /// Planned infrastructure qdiscs (`mq`, `htb`, etc.) for the interface. + pub infra_qdiscs: usize, + /// Planned CAKE leaf qdiscs for the interface. + pub cake_qdiscs: usize, + /// Planned fq_codel leaf qdiscs for the interface. + pub fq_codel_qdiscs: usize, + /// Estimated kernel memory cost for the interface's planned qdiscs. + pub estimated_memory_bytes: u64, +} + +/// Estimated qdisc budget and conservative memory-safety forecast for a planned full reload. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct QdiscBudgetEstimate { + /// Estimated total qdisc count grouped by network interface name. + pub interfaces: BTreeMap, + /// Detailed per-interface qdisc breakdown. + pub interface_details: BTreeMap, + /// Conservative operational limit enforced before a full reload is committed. + pub safe_budget: usize, + /// Kernel hard limit for the per-device qdisc-handle namespace. + pub hard_limit: usize, + /// Estimated total kernel memory cost of the planned qdisc model. + pub estimated_total_memory_bytes: u64, + /// Current host memory snapshot used for preflight, if available. + pub memory_snapshot: Option, + /// Whether the memory preflight passed. + pub memory_ok: bool, +} + +impl QdiscBudgetEstimate { + /// Returns `true` when all planned per-interface counts fit within the safe budget. + pub fn ok(&self) -> bool { + self.interfaces + .values() + .all(|count| *count <= self.safe_budget) + && self.memory_ok + } +} + +fn find_arg_value<'a>(argv: &'a [String], key: &str) -> Option<&'a str> { + argv.windows(2) + .find_map(|pair| (pair[0] == key).then_some(pair[1].as_str())) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PlannedQdiscKind { + Infra, + Cake, + FqCodel, +} + +fn planned_qdisc_kind(argv: &[String]) -> Option { + if argv.len() < 2 || argv[0] != "qdisc" { + return None; + } + if !matches!(argv[1].as_str(), "add" | "replace") { + return None; + } + + if argv.iter().any(|arg| arg == "cake") { + return Some(if planned_qdisc_is_leaf(argv) { + PlannedQdiscKind::Cake + } else { + PlannedQdiscKind::Infra + }); + } + if argv.iter().any(|arg| arg == "fq_codel") { + return Some(if planned_qdisc_is_leaf(argv) { + PlannedQdiscKind::FqCodel + } else { + PlannedQdiscKind::Infra + }); + } + Some(PlannedQdiscKind::Infra) +} + +fn planned_qdisc_is_leaf(argv: &[String]) -> bool { + let Some(parent) = find_arg_value(argv, "parent") else { + return false; + }; + let Ok(parent_handle) = TcHandle::from_string(parent) else { + return false; + }; + let (_, minor) = parent_handle.get_major_minor(); + minor != 1 && minor != 2 +} + +fn qdisc_kind_estimated_memory_bytes(kind: PlannedQdiscKind) -> u64 { + match kind { + PlannedQdiscKind::Infra => INFRA_QDISC_ESTIMATED_MEMORY_BYTES, + PlannedQdiscKind::Cake => CAKE_QDISC_ESTIMATED_MEMORY_BYTES, + PlannedQdiscKind::FqCodel => FQ_CODEL_QDISC_ESTIMATED_MEMORY_BYTES, + } +} + +fn planned_qdisc_identity(argv: &[String]) -> Option<(String, String)> { + if argv.len() < 2 || argv[0] != "qdisc" { + return None; + } + if !matches!(argv[1].as_str(), "add" | "replace") { + return None; + } + + let dev = find_arg_value(argv, "dev")?.to_string(); + if let Some(handle) = find_arg_value(argv, "handle") { + return Some((dev, format!("handle:{handle}"))); + } + let parent = find_arg_value(argv, "parent")?.to_string(); + Some((dev, format!("parent:{parent}"))) +} + +/// Estimates total qdisc usage for the current full-reload builder queue. +/// +/// This expands the queued Bakery commands through the same builder-path `tc` +/// argv generation used by a structural full reload, then counts the resulting +/// qdiscs per device. Explicit qdisc handles are deduplicated by `handle`, +/// while any remaining auto-handled paths are conservatively deduplicated by +/// `parent`. +pub fn estimate_full_reload_auto_qdisc_budget( + config: &Arc, + queue: &[BakeryCommands], +) -> QdiscBudgetEstimate { + let mut interfaces = BTreeMap::new(); + let mut interface_details: BTreeMap = BTreeMap::new(); + let mut seen_qdiscs = HashSet::new(); + + for command in queue { + let Some(builder_commands) = command.to_commands(config, ExecutionMode::Builder) else { + continue; + }; + for argv in builder_commands { + let Some((dev, identity)) = planned_qdisc_identity(&argv) else { + continue; + }; + if seen_qdiscs.insert((dev.clone(), identity)) { + let kind = planned_qdisc_kind(&argv).unwrap_or(PlannedQdiscKind::Infra); + *interfaces.entry(dev.clone()).or_insert(0) += 1; + let detail = + interface_details + .entry(dev) + .or_insert_with(|| QdiscInterfaceEstimate { + planned_qdiscs: 0, + infra_qdiscs: 0, + cake_qdiscs: 0, + fq_codel_qdiscs: 0, + estimated_memory_bytes: 0, + }); + detail.planned_qdiscs += 1; + match kind { + PlannedQdiscKind::Infra => detail.infra_qdiscs += 1, + PlannedQdiscKind::Cake => detail.cake_qdiscs += 1, + PlannedQdiscKind::FqCodel => detail.fq_codel_qdiscs += 1, + } + detail.estimated_memory_bytes = detail + .estimated_memory_bytes + .saturating_add(qdisc_kind_estimated_memory_bytes(kind)); + } + } + } + + let estimated_total_memory_bytes = interface_details.values().fold(0u64, |acc, detail| { + acc.saturating_add(detail.estimated_memory_bytes) + }); + let memory_snapshot = read_memory_snapshot().ok(); + let memory_ok = memory_snapshot.as_ref().is_none_or(|snapshot| { + snapshot + .available_bytes + .saturating_sub(BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES) + >= estimated_total_memory_bytes + }); + + QdiscBudgetEstimate { + interfaces, + interface_details, + safe_budget: SAFE_QDISC_BUDGET_PER_INTERFACE, + hard_limit: HARD_QDISC_HANDLE_LIMIT_PER_INTERFACE, + estimated_total_memory_bytes, + memory_snapshot, + memory_ok, + } +} + +fn desired_shaping_tree_active(config: &Arc) -> bool { + !config.queues.queue_mode.is_observe() +} + +fn live_tree_mutation_blocker_for_config(config: &Arc) -> Option { + if FULL_RELOAD_IN_PROGRESS.load(Ordering::Relaxed) { + return Some("a full reload is currently in progress".to_string()); + } + if config.queues.queue_mode.is_observe() { + return Some( + "queue_mode is observe; root MQ is retained but the shaping tree is not live" + .to_string(), + ); + } + if !SHAPING_TREE_ACTIVE.load(Ordering::Relaxed) { + return Some("the shaping tree is not currently active".to_string()); + } + None +} + +const ROOT_MQ_MAJOR: u16 = 0x7fff; + +fn managed_interfaces_for_config(config: &Arc) -> Vec { + let mut interfaces = vec![config.isp_interface()]; + if !config.on_a_stick_mode() { + interfaces.push(config.internet_interface()); + } + interfaces +} + +fn retained_root_mq_entry(snapshot: &[LiveTcQdiscEntry]) -> Option<&LiveTcQdiscEntry> { + snapshot.iter().find(|entry| { + entry.is_root + && entry.kind == "mq" + && entry + .handle + .is_some_and(|handle| handle.get_major_minor().0 == ROOT_MQ_MAJOR) + }) +} + +fn managed_root_child_parent_handles(snapshot: &[LiveTcQdiscEntry]) -> HashSet { + snapshot + .iter() + .filter(|entry| { + entry + .parent + .is_some_and(|parent| parent.get_major_minor().0 == ROOT_MQ_MAJOR) + && entry + .handle + .is_some_and(|handle| handle.get_major_minor().0 != 0) + }) + .filter_map(|entry| entry.parent) + .collect() +} + +fn verify_root_mq_snapshot(snapshot: &[LiveTcQdiscEntry], interface: &str) -> Result<(), String> { + if retained_root_mq_entry(snapshot).is_none() { + return Err(format!( + "interface {interface} does not currently have root mq handle 7fff:" + )); + } + Ok(()) +} + +fn verify_clean_root_child_tree( + qdisc_snapshot: &[LiveTcQdiscEntry], + class_snapshot: &HashMap, + interface: &str, +) -> Result<(), String> { + verify_root_mq_snapshot(qdisc_snapshot, interface)?; + + let child_handles = managed_root_child_parent_handles(qdisc_snapshot); + if !child_handles.is_empty() { + return Err(format!( + "interface {interface} still has {} managed child qdisc(s) beneath root mq 7fff:", + child_handles.len() + )); + } + + let managed_class_count = class_snapshot + .values() + .filter(|entry| entry.class_id.get_major_minor().0 != ROOT_MQ_MAJOR) + .count(); + if managed_class_count != 0 { + return Err(format!( + "interface {interface} still has {} managed tc class(es) after root-child prune", + managed_class_count + )); + } + + Ok(()) +} + +fn root_replace_failure_is_fallbackable(summary: &str) -> bool { + let normalized = summary.to_ascii_lowercase(); + normalized.contains("exclusivity flag on, cannot modify") + || normalized.contains("rtnetlink answers: file exists") + || normalized.contains("file exists") +} + +fn run_root_preflight_commands(commands: &[Vec], purpose: &str) -> Result<(), String> { + let result = execute_in_memory_chunked(commands, purpose, 1, None, |_, _, _, _| {}); + if result.ok { + Ok(()) + } else { + Err(result + .failure_summary + .unwrap_or_else(|| format!("{purpose} failed"))) + } +} + +fn root_mq_replace_command(interface_name: &str) -> Vec { + vec![ + "qdisc".to_string(), + "replace".to_string(), + "dev".to_string(), + interface_name.to_string(), + "root".to_string(), + "handle".to_string(), + "7FFF:".to_string(), + "mq".to_string(), + ] +} + +fn root_mq_delete_command(interface_name: &str) -> Vec { + vec![ + "qdisc".to_string(), + "del".to_string(), + "dev".to_string(), + interface_name.to_string(), + "root".to_string(), + ] +} + +fn root_mq_add_command(interface_name: &str) -> Vec { + vec![ + "qdisc".to_string(), + "add".to_string(), + "dev".to_string(), + interface_name.to_string(), + "root".to_string(), + "handle".to_string(), + "7FFF:".to_string(), + "mq".to_string(), + ] +} + +fn prepare_root_mq_for_full_reload(config: &Arc) -> Result<(), String> { + for interface in managed_interfaces_for_config(config) { + let qdisc_snapshot = read_live_qdisc_snapshot(&interface)?; + if retained_root_mq_entry(&qdisc_snapshot).is_some() { + let prune_commands = managed_root_child_parent_handles(&qdisc_snapshot) + .into_iter() + .map(|parent| { + vec![ + "qdisc".to_string(), + "del".to_string(), + "dev".to_string(), + interface.clone(), + "parent".to_string(), + parent.as_tc_string(), + ] + }) + .collect::>(); + if !prune_commands.is_empty() { + run_root_preflight_commands( + &prune_commands, + &format!("full reload retained-root child prune on {interface}"), + )?; + invalidate_live_tc_snapshots(); + } + + let pruned_qdisc_snapshot = read_live_qdisc_snapshot(&interface)?; + let pruned_class_snapshot = read_live_class_snapshot(&interface)?; + if verify_clean_root_child_tree( + &pruned_qdisc_snapshot, + &pruned_class_snapshot, + &interface, + ) + .is_ok() + { + continue; + } + } + + let replace_summary = run_root_preflight_commands( + &[root_mq_replace_command(&interface)], + &format!("full reload root mq replace on {interface}"), + ) + .err(); + invalidate_live_tc_snapshots(); + + if let Some(summary) = replace_summary { + if !root_replace_failure_is_fallbackable(&summary) { + return Err(summary); + } + } else { + let replaced_qdisc_snapshot = read_live_qdisc_snapshot(&interface)?; + let replaced_class_snapshot = read_live_class_snapshot(&interface)?; + if verify_clean_root_child_tree( + &replaced_qdisc_snapshot, + &replaced_class_snapshot, + &interface, + ) + .is_ok() + { + continue; + } + } + + run_root_preflight_commands( + &[root_mq_delete_command(&interface)], + &format!("full reload root mq delete on {interface}"), + )?; + invalidate_live_tc_snapshots(); + run_root_preflight_commands( + &[root_mq_add_command(&interface)], + &format!("full reload root mq add on {interface}"), + )?; + invalidate_live_tc_snapshots(); + + let recovered_qdisc_snapshot = read_live_qdisc_snapshot(&interface)?; + let recovered_class_snapshot = read_live_class_snapshot(&interface)?; + verify_clean_root_child_tree( + &recovered_qdisc_snapshot, + &recovered_class_snapshot, + &interface, + )?; + } + + Ok(()) +} + +fn live_tree_mutations_allowed(config: &Arc) -> bool { + live_tree_mutation_blocker_for_config(config).is_none() +} + +/// Returns the reason live shaping-tree mutations are currently blocked, if any. +/// +/// This function is not pure: it reads runtime config and Bakery shaping-tree state. +pub fn bakery_live_tree_mutation_blocker() -> Option { + let Ok(config) = lqos_config::load_config() else { + return Some("configuration could not be loaded".to_string()); + }; + live_tree_mutation_blocker_for_config(&config) +} + +/// Overrides Bakery's shaping-tree-active flag for tests and restores callers' access to the +/// previous value. +/// +/// This function has side effects: it mutates process-global Bakery runtime state. +#[doc(hidden)] +pub fn set_shaping_tree_active_for_tests(active: bool) -> bool { + SHAPING_TREE_ACTIVE.swap(active, Ordering::Relaxed) +} + +fn current_mq_layout( + batch: &[Arc], + config: &Arc, + existing: &Option, +) -> Option { + let mut latest = existing.clone(); + for command in batch { + if let BakeryCommands::MqSetup { + queues_available, + stick_offset, + } = command.as_ref() + { + latest = Some(MqDeviceLayout::from_setup( + config, + *queues_available, + *stick_offset, + )); + } + } + latest +} + +#[cfg_attr(not(test), allow(dead_code))] +fn with_assigned_qdisc_handles( + command: &Arc, + config: &Arc, + mq_layout: &MqDeviceLayout, + qdisc_handles: &mut QdiscHandleState, +) -> Arc { + with_assigned_qdisc_handles_reserved(command, config, mq_layout, qdisc_handles, &HashMap::new()) +} + +fn with_assigned_qdisc_handles_reserved( + command: &Arc, + config: &Arc, + mq_layout: &MqDeviceLayout, + qdisc_handles: &mut QdiscHandleState, + extra_reserved_handles: &HashMap>, +) -> Arc { + let BakeryCommands::AddCircuit { circuit_hash, .. } = command.as_ref() else { + return Arc::clone(command); + }; + + if config.queues.queue_mode.is_observe() { + return Arc::clone(command); + } + + let mut enriched = command.as_ref().clone(); + let isp_interface = config.isp_interface(); + let internet_interface = config.internet_interface(); + let mut isp_reserved = mq_layout.reserved_handles(&isp_interface); + if let Some(extra) = extra_reserved_handles.get(&isp_interface) { + isp_reserved.extend(extra.iter().copied()); + } + let mut up_reserved = mq_layout.reserved_handles(&internet_interface); + if let Some(extra) = extra_reserved_handles.get(&internet_interface) { + up_reserved.extend(extra.iter().copied()); + } + + if let BakeryCommands::AddCircuit { + down_qdisc_handle, + up_qdisc_handle, + .. + } = &mut enriched + { + if down_qdisc_handle.is_none() { + *down_qdisc_handle = + qdisc_handles.assign_circuit_handle(&isp_interface, *circuit_hash, &isp_reserved); + } + if !config.on_a_stick_mode() && up_qdisc_handle.is_none() { + *up_qdisc_handle = qdisc_handles.assign_circuit_handle( + &internet_interface, + *circuit_hash, + &up_reserved, + ); + } + } + + Arc::new(enriched) +} + +fn assign_fresh_qdisc_handles_reserved( + command: &Arc, + config: &Arc, + mq_layout: &MqDeviceLayout, + qdisc_handles: &mut QdiscHandleState, + extra_reserved_handles: &HashMap>, +) -> Result, String> { + let BakeryCommands::AddCircuit { circuit_hash, .. } = command.as_ref() else { + return Ok(Arc::clone(command)); + }; + + if config.queues.queue_mode.is_observe() { + return Ok(Arc::clone(command)); + } + + let (down_parent, up_parent) = effective_directional_qdisc_parents(command.as_ref(), config); + let mut refreshed = command.as_ref().clone(); + let isp_interface = config.isp_interface(); + let internet_interface = config.internet_interface(); + let mut isp_reserved = mq_layout.reserved_handles(&isp_interface); + if let Some(extra) = extra_reserved_handles.get(&isp_interface) { + isp_reserved.extend(extra.iter().copied()); + } + let mut up_reserved = mq_layout.reserved_handles(&internet_interface); + if let Some(extra) = extra_reserved_handles.get(&internet_interface) { + up_reserved.extend(extra.iter().copied()); + } + + if let BakeryCommands::AddCircuit { + down_qdisc_handle, + up_qdisc_handle, + .. + } = &mut refreshed + { + *down_qdisc_handle = if down_parent.is_some() { + qdisc_handles.rotate_circuit_handle(&isp_interface, *circuit_hash, &isp_reserved) + } else { + None + }; + if down_parent.is_some() && down_qdisc_handle.is_none() { + return Err(format!( + "Bakery could not allocate a fresh downlink qdisc handle for restored circuit {}", + circuit_hash + )); + } + + *up_qdisc_handle = if up_parent.is_some() { + qdisc_handles.rotate_circuit_handle(&internet_interface, *circuit_hash, &up_reserved) + } else { + None + }; + if up_parent.is_some() && up_qdisc_handle.is_none() { + return Err(format!( + "Bakery could not allocate a fresh uplink qdisc handle for restored circuit {}", + circuit_hash + )); + } + } + + Ok(Arc::new(refreshed)) +} + +fn snapshot_live_qdisc_handle_majors( + config: &Arc, +) -> Result>, String> { + let mut reserved = HashMap::new(); + let isp_interface = config.isp_interface(); + reserved.insert( + isp_interface.clone(), + read_live_qdisc_handle_majors(&isp_interface)?, + ); + + if !config.on_a_stick_mode() { + let internet_interface = config.internet_interface(); + if internet_interface != isp_interface { + reserved.insert( + internet_interface.clone(), + read_live_qdisc_handle_majors(&internet_interface)?, + ); + } + } + + Ok(reserved) +} + +fn parse_directional_sqm_override( + sqm_override: &Option, +) -> (Option, Option) { + match sqm_override { + None => (None, None), + Some(s) => { + if s.contains('/') { + let mut it = s.splitn(2, '/'); + let down = it.next().unwrap_or("").trim(); + let up = it.next().unwrap_or("").trim(); + let map = |t: &str| -> Option { + if t.is_empty() { + None + } else { + Some(t.to_string()) + } + }; + (map(down), map(up)) + } else { + (Some(s.clone()), Some(s.clone())) + } + } + } +} + +fn effective_directional_sqm_kinds( + command: &BakeryCommands, + config: &Arc, +) -> (Option, Option) { + let BakeryCommands::AddCircuit { + download_bandwidth_max, + upload_bandwidth_max, + sqm_override, + .. + } = command + else { + return (None, None); + }; + + if config.queues.queue_mode.is_observe() { + return (None, None); + } + + let (down_override_opt, up_override_opt) = parse_directional_sqm_override(sqm_override); + + let down_kind = + (!matches!(down_override_opt.as_deref(), Some(s) if s.eq_ignore_ascii_case("none"))) + .then(|| effective_sqm_kind(*download_bandwidth_max, config, &down_override_opt)); + let up_kind = (!config.on_a_stick_mode() + && !matches!(up_override_opt.as_deref(), Some(s) if s.eq_ignore_ascii_case("none"))) + .then(|| effective_sqm_kind(*upload_bandwidth_max, config, &up_override_opt)); + + (down_kind, up_kind) +} + +fn effective_directional_qdisc_parents( + command: &BakeryCommands, + config: &Arc, +) -> (Option, Option) { + let BakeryCommands::AddCircuit { + class_minor, + class_major, + up_class_major, + sqm_override, + .. + } = command + else { + return (None, None); + }; + + if config.queues.queue_mode.is_observe() { + return (None, None); + } + + let (down_override_opt, up_override_opt) = parse_directional_sqm_override(sqm_override); + + let down_parent = + (!matches!(down_override_opt.as_deref(), Some(s) if s.eq_ignore_ascii_case("none"))) + .then(|| TcHandle::from_u32(((*class_major as u32) << 16) | (*class_minor as u32))); + let up_parent = (!config.on_a_stick_mode() + && !matches!(up_override_opt.as_deref(), Some(s) if s.eq_ignore_ascii_case("none"))) + .then(|| TcHandle::from_u32(((*up_class_major as u32) << 16) | (*class_minor as u32))); + + (down_parent, up_parent) +} + +#[cfg_attr(not(test), allow(dead_code))] +fn rotate_changed_qdisc_handles( + previous: &BakeryCommands, + command: &Arc, + config: &Arc, + mq_layout: &MqDeviceLayout, + qdisc_handles: &mut QdiscHandleState, +) -> Arc { + rotate_changed_qdisc_handles_reserved( + previous, + command, + config, + mq_layout, + qdisc_handles, + &HashMap::new(), + ) +} + +fn rotate_changed_qdisc_handles_reserved( + previous: &BakeryCommands, + command: &Arc, + config: &Arc, + mq_layout: &MqDeviceLayout, + qdisc_handles: &mut QdiscHandleState, + extra_reserved_handles: &HashMap>, +) -> Arc { + let (old_down_kind, old_up_kind) = effective_directional_sqm_kinds(previous, config); + let (new_down_kind, new_up_kind) = effective_directional_sqm_kinds(command.as_ref(), config); + let (old_down_parent, old_up_parent) = effective_directional_qdisc_parents(previous, config); + let (new_down_parent, new_up_parent) = + effective_directional_qdisc_parents(command.as_ref(), config); + + let mut rotated = command.as_ref().clone(); + let isp_interface = config.isp_interface(); + let internet_interface = config.internet_interface(); + let mut isp_reserved = mq_layout.reserved_handles(&isp_interface); + if let Some(extra) = extra_reserved_handles.get(&isp_interface) { + isp_reserved.extend(extra.iter().copied()); + } + let mut up_reserved = mq_layout.reserved_handles(&internet_interface); + if let Some(extra) = extra_reserved_handles.get(&internet_interface) { + up_reserved.extend(extra.iter().copied()); + } + + if let BakeryCommands::AddCircuit { + circuit_hash, + down_qdisc_handle, + up_qdisc_handle, + .. + } = &mut rotated + { + let down_kind_changed = + old_down_kind.is_some() && new_down_kind.is_some() && old_down_kind != new_down_kind; + let down_parent_changed = old_down_parent.is_some() + && new_down_parent.is_some() + && old_down_parent != new_down_parent; + let down_handle_conflicts_live = down_qdisc_handle.as_ref().is_some_and(|handle| { + isp_reserved.contains(handle) && Some(*handle) != previous_down_qdisc_handle(previous) + }); + if down_kind_changed || down_parent_changed || down_handle_conflicts_live { + *down_qdisc_handle = + qdisc_handles.rotate_circuit_handle(&isp_interface, *circuit_hash, &isp_reserved); + } + let up_kind_changed = + old_up_kind.is_some() && new_up_kind.is_some() && old_up_kind != new_up_kind; + let up_parent_changed = + old_up_parent.is_some() && new_up_parent.is_some() && old_up_parent != new_up_parent; + let up_handle_conflicts_live = up_qdisc_handle.as_ref().is_some_and(|handle| { + up_reserved.contains(handle) && Some(*handle) != previous_up_qdisc_handle(previous) + }); + if up_kind_changed || up_parent_changed || up_handle_conflicts_live { + *up_qdisc_handle = qdisc_handles.rotate_circuit_handle( + &internet_interface, + *circuit_hash, + &up_reserved, + ); + } + } + + Arc::new(rotated) +} + +fn previous_down_qdisc_handle(previous: &BakeryCommands) -> Option { + let BakeryCommands::AddCircuit { + down_qdisc_handle, .. + } = previous + else { + return None; + }; + *down_qdisc_handle +} + +fn previous_up_qdisc_handle(previous: &BakeryCommands) -> Option { + let BakeryCommands::AddCircuit { + up_qdisc_handle, .. + } = previous + else { + return None; + }; + *up_qdisc_handle +} + +fn snapshot_live_qdisc_handle_majors_or_empty( + config: &Arc, + purpose: &str, +) -> HashMap> { + match snapshot_live_qdisc_handle_majors(config) { + Ok(handles) => handles, + Err(error) => { + warn!( + "Bakery could not snapshot live qdisc handles before {purpose}; proceeding without extra live reservations: {error}" + ); + HashMap::new() + } + } +} + +fn runtime_site_label(site_hash: i64, site_name: Option<&str>) -> String { + match site_name { + Some(name) if !name.is_empty() => format!("{name} ({site_hash})"), + _ => site_hash.to_string(), + } +} + +fn runtime_site_display_name(site_hash: i64, site_name: Option<&str>) -> String { + match site_name { + Some(name) if !name.is_empty() => name.to_string(), + _ => site_hash.to_string(), + } +} + +fn runtime_circuit_label(circuit_hash: i64, circuit_name: Option<&str>) -> String { + match circuit_name { + Some(name) if !name.is_empty() => format!("{name} ({circuit_hash})"), + _ => circuit_hash.to_string(), + } +} + +fn migration_target_label(migration: &Migration) -> String { + let circuit_label = + runtime_circuit_label(migration.circuit_hash, migration.circuit_name.as_deref()); + match migration.site_name.as_deref() { + Some(site_name) if !site_name.is_empty() => { + format!("circuit {circuit_label} at site {site_name}") + } + _ => format!("circuit {circuit_label}"), + } +} + +fn runtime_network_json_path(config: &Config) -> std::path::PathBuf { + let base_path = Path::new(&config.lqos_directory); + if config + .long_term_stats + .enable_insight_topology + .unwrap_or(false) + { + let tmp_path = base_path.join("network.insight.json"); + if tmp_path.exists() { + return tmp_path; + } + } + base_path.join("network.json") +} + +fn runtime_hash_to_i64(text: &str) -> i64 { + use std::hash::{DefaultHasher, Hasher}; + + let mut hasher = DefaultHasher::new(); + hasher.write(text.as_bytes()); + hasher.finish() as i64 +} + +fn network_json_entry_looks_like_node(node: &Map) -> bool { + node.get("type") + .and_then(|value| value.as_str()) + .is_some_and(|value| { + value.eq_ignore_ascii_case("site") + || value.eq_ignore_ascii_case("ap") + || value.eq_ignore_ascii_case("client") + }) + || node.contains_key("downloadBandwidthMbps") + || node.contains_key("uploadBandwidthMbps") + || node.contains_key("children") +} + +fn collect_network_json_site_names(map: &Map, names: &mut HashMap) { + for (name, value) in map { + let Some(node) = value.as_object() else { + continue; + }; + if !network_json_entry_looks_like_node(node) { + continue; + } + names + .entry(runtime_hash_to_i64(name)) + .or_insert_with(|| name.clone()); + if let Some(Value::Object(children)) = node.get("children") { + collect_network_json_site_names(children, names); + } + } +} + +fn load_current_runtime_site_names(config: &Config) -> Option> { + let path = runtime_network_json_path(config); + let raw = std::fs::read_to_string(&path).ok()?; + let json: Value = serde_json::from_str(&raw).ok()?; + let root = json.as_object()?; + let mut names = HashMap::new(); + collect_network_json_site_names(root, &mut names); + Some(names) +} + +fn resolve_runtime_site_name( + site_hash: i64, + current_site_names: Option<&HashMap>, + virtualized_sites: &HashMap, +) -> Option { + current_site_names + .and_then(|names| names.get(&site_hash).cloned()) + .or_else(|| { + virtualized_sites + .get(&site_hash) + .map(|state| state.site_name.clone()) + }) +} + +fn stale_retained_runtime_branch_summary( + site_hash: i64, + saved_state: &VirtualizedSiteState, + current_site_names: &HashMap, +) -> Option { + if !current_site_names.contains_key(&site_hash) { + return Some(format!( + "Bakery dropped stale retained runtime branch for {} because the backing node is no longer present in current topology.", + runtime_site_label(site_hash, Some(saved_state.site_name.as_str())) + )); + } + + let missing_saved_site = saved_state + .saved_sites + .keys() + .find(|saved_hash| !current_site_names.contains_key(saved_hash))?; + Some(format!( + "Bakery dropped stale retained runtime branch for {} because retained child site hash {} is no longer present in current topology.", + runtime_site_label(site_hash, Some(saved_state.site_name.as_str())), + missing_saved_site + )) +} + +/// Returns `true` while Bakery is applying a full reload batch to Linux `tc`. +pub fn full_reload_in_progress() -> bool { + FULL_RELOAD_IN_PROGRESS.load(Ordering::Relaxed) +} + #[derive(Debug, Default, Clone, Copy)] struct MappedLimitStats { enforced_limit: Option, @@ -450,6 +2915,62 @@ fn maybe_emit_mapped_circuit_limit_urgent(stats: &MappedLimitStats) { }); } +fn maybe_emit_memory_guard_urgent(summary: &str) { + if !summary.contains("Bakery memory guard stopped") { + return; + } + + let message = summary.to_string(); + let context = read_memory_snapshot().ok().map(|snapshot| { + format!( + "{{\"available_bytes\":{},\"total_bytes\":{},\"memory_guard_floor_bytes\":{}}}", + snapshot.available_bytes, snapshot.total_bytes, BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES + ) + }); + + let rt = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(rt) => rt, + Err(e) => { + warn!("Bakery: failed to build runtime for urgent issue submission: {e:?}"); + return; + } + }; + rt.block_on(async { + if let Ok(mut bus) = LibreqosBusClient::new().await { + let _ = bus + .request(vec![BusRequest::SubmitUrgentIssue { + source: UrgentSource::System, + severity: UrgentSeverity::Error, + code: "BAKERY_MEMORY_GUARD".to_string(), + message, + context, + dedupe_key: Some("bakery_memory_guard".to_string()), + }]) + .await; + } + }); +} + +fn log_mapped_limit_decision( + context: &str, + mapped_limit: ResolvedMappedLimit, + stats: MappedLimitStats, +) { + warn!( + "Bakery mapped circuit decision ({}): requested={}, allowed={}, dropped={}, effective_limit={}, licensed={}, max_circuits={:?}", + context, + stats.requested_mapped, + stats.allowed_mapped, + stats.dropped_mapped, + format_mapped_limit(stats.enforced_limit), + mapped_limit.licensed, + mapped_limit.max_circuits + ); +} + /// Starts the Bakery system, returning a channel sender for sending commands to the Bakery. pub fn start_bakery() -> anyhow::Result> { let (tx, rx) = crossbeam_channel::bounded(CHANNEL_CAPACITY); @@ -472,8 +2993,16 @@ fn bakery_main(rx: Receiver, tx: Sender) { let mut sites: HashMap> = HashMap::new(); let mut circuits: HashMap> = HashMap::new(); let mut live_circuits: HashMap = HashMap::new(); + let mut mq_layout: Option = None; + let mut qdisc_handles = lqos_config::load_config() + .ok() + .map(|config| QdiscHandleState::load(&config)) + .unwrap_or_default(); // Persist latest StormGuard ceilings keyed by interface + class so we can replay after rebuilds. let mut stormguard_overrides: HashMap = HashMap::new(); + let mut virtualized_sites: HashMap = HashMap::new(); + let mut runtime_node_operations: HashMap = HashMap::new(); + let mut next_runtime_operation_id: u64 = 1; // Mapping state #[derive(Clone, Hash, PartialEq, Eq, Debug)] @@ -550,6 +3079,103 @@ fn bakery_main(rx: Receiver, tx: Sender) { mapping_current.remove(&key); } + fn mapping_key_cidr(key: &MappingKey) -> String { + format!("{}/{}", key.ip, key.prefix) + } + + fn rollback_migration_ip_remaps( + remapped: &[(MappingKey, Option)], + mapping_current: &mut HashMap, + ) -> Result<(), String> { + let mut failures = Vec::new(); + + for (key, previous) in remapped.iter().rev() { + let cidr = mapping_key_cidr(key); + let rollback_result = if let Some(previous) = previous { + lqos_sys::add_ip_to_tc(&cidr, previous.handle, previous.cpu, false, 0, 0).map( + |_| { + mapping_current.insert(key.clone(), previous.clone()); + }, + ) + } else { + lqos_sys::del_ip_from_tc(&cidr, false).map(|_| { + mapping_current.remove(key); + }) + }; + + if let Err(error) = rollback_result { + failures.push(format!("{cidr}: {error}")); + } + } + + if let Err(error) = lqos_sys::clear_hot_cache() { + failures.push(format!("clear hot cache after rollback: {error}")); + } + + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("; ")) + } + } + + fn remap_migration_ips( + mig: &Migration, + target_minor: u16, + mapping_current: &mut HashMap, + ) -> Result<(), String> { + let target_handle = tc_handle_from_major_minor(mig.class_major, target_minor); + let mut remapped: Vec<(MappingKey, Option)> = Vec::new(); + + for ip in &mig.ips { + let (ip_s, prefix) = parse_ip_and_prefix(ip); + let key = MappingKey { ip: ip_s, prefix }; + let cidr = mapping_key_cidr(&key); + let previous = mapping_current.get(&key).cloned(); + let cpu = previous.as_ref().map(|value| value.cpu).unwrap_or(0); + + if let Err(error) = lqos_sys::add_ip_to_tc(&cidr, target_handle, cpu, false, 0, 0) { + let rollback_summary = + rollback_migration_ip_remaps(&remapped, mapping_current).err(); + return Err(match rollback_summary { + Some(rollback_error) => format!( + "failed to remap {cidr} to {}: {error}; rollback also failed: {rollback_error}", + target_handle.as_tc_string() + ), + None => format!( + "failed to remap {cidr} to {}: {error}; prior remaps were rolled back", + target_handle.as_tc_string() + ), + }); + } + + mapping_current.insert( + key.clone(), + MappingVal { + handle: target_handle, + cpu, + }, + ); + remapped.push((key, previous)); + } + + if let Err(error) = lqos_sys::clear_hot_cache() { + let rollback_summary = rollback_migration_ip_remaps(&remapped, mapping_current).err(); + return Err(match rollback_summary { + Some(rollback_error) => format!( + "remap to {} succeeded but clearing the hot cache failed: {error}; rollback also failed: {rollback_error}", + target_handle.as_tc_string() + ), + None => format!( + "remap to {} succeeded but clearing the hot cache failed: {error}; remaps were rolled back", + target_handle.as_tc_string() + ), + }); + } + + Ok(()) + } + fn build_known_handle_set(circuits: &HashMap>) -> HashSet { let mut down = HashSet::new(); for (_k, v) in circuits.iter() { @@ -608,6 +3234,437 @@ fn bakery_main(rx: Receiver, tx: Sender) { }) } + fn verify_migration_direction( + interface_name: String, + snapshot: &HashMap, + expected_handle: TcHandle, + expected_parent: TcHandle, + ) -> MigrationDirectionVerification { + let observed = snapshot.get(&expected_handle); + MigrationDirectionVerification { + interface_name, + expected_handle, + expected_parent, + observed_parent: observed.and_then(|entry| entry.parent), + observed_leaf_qdisc_major: observed.and_then(|entry| entry.leaf_qdisc_major), + } + } + + fn migration_branch_verification( + config: &Arc, + down_handle: TcHandle, + down_parent: TcHandle, + up_handle: TcHandle, + up_parent: TcHandle, + ) -> Result { + let down_snapshot = read_live_class_snapshot(&config.isp_interface())?; + let down = verify_migration_direction( + config.isp_interface(), + &down_snapshot, + down_handle, + down_parent, + ); + + if config.on_a_stick_mode() { + return Ok(MigrationBranchVerification { down, up: None }); + } + + let up_snapshot = read_live_class_snapshot(&config.internet_interface())?; + let up = verify_migration_direction( + config.internet_interface(), + &up_snapshot, + up_handle, + up_parent, + ); + + Ok(MigrationBranchVerification { down, up: Some(up) }) + } + + fn migration_shadow_verification( + config: &Arc, + migration: &Migration, + ) -> Result { + let down_handle = TcHandle::from_u32( + (u32::from(migration.class_major) << 16) | u32::from(migration.shadow_minor), + ); + let up_handle = TcHandle::from_u32( + (u32::from(migration.up_class_major) << 16) | u32::from(migration.shadow_minor), + ); + migration_branch_verification( + config, + down_handle, + migration.parent_class_id, + up_handle, + migration.up_parent_class_id, + ) + } + + fn migration_final_verification( + config: &Arc, + migration: &Migration, + ) -> Result { + let down_handle = TcHandle::from_u32( + (u32::from(migration.class_major) << 16) | u32::from(migration.final_minor), + ); + let up_handle = TcHandle::from_u32( + (u32::from(migration.up_class_major) << 16) | u32::from(migration.final_minor), + ); + migration_branch_verification( + config, + down_handle, + migration.parent_class_id, + up_handle, + migration.up_parent_class_id, + ) + } + + fn process_pending_migrations( + config: &Arc, + circuits: &mut HashMap>, + sites: &HashMap>, + migrations: &mut HashMap, + mapping_current: &mut HashMap, + virtualized_sites: &mut HashMap, + runtime_node_operations: &mut HashMap, + ) { + let mut advanced = 0usize; + let mut to_remove = Vec::new(); + let mut effective_state_changed = false; + let persisted_qdisc_handles = QdiscHandleState::load(config); + + for (_hash, mig) in migrations.iter_mut() { + if advanced >= MIGRATIONS_PER_TICK { + break; + } + + match mig.stage { + MigrationStage::PrepareShadow => { + let live_reserved_handles = snapshot_live_qdisc_handle_majors_or_empty( + config, + "live-move: create shadow", + ); + if let Some(temp) = build_shadow_add_cmd( + mig, + config, + &persisted_qdisc_handles, + &live_reserved_handles, + ) { + let mut cmds = Vec::new(); + match config.queues.lazy_queues.as_ref() { + None | Some(LazyQueueMode::No) => { + if let Some(c) = + add_commands_for_circuit(&temp, config, ExecutionMode::Builder) + { + cmds.extend(c); + } + } + Some(LazyQueueMode::Htb) => { + if let Some(c) = + add_commands_for_circuit(&temp, config, ExecutionMode::Builder) + { + cmds.extend(c); + } + if let Some(c) = add_commands_for_circuit( + &temp, + config, + ExecutionMode::LiveUpdate, + ) { + cmds.extend(c); + } + } + Some(LazyQueueMode::Full) => { + if let Some(c) = add_commands_for_circuit( + &temp, + config, + ExecutionMode::LiveUpdate, + ) { + cmds.extend(c); + } + } + } + if !cmds.is_empty() { + let result = + execute_and_record_live_change(&cmds, "live-move: create shadow"); + let _ = migration_stage_apply_succeeded( + mig, + "live-move: create shadow", + &result, + MigrationStage::VerifyShadowReady, + ); + } else { + mig.stage = MigrationStage::VerifyShadowReady; + } + advanced += 1; + } else { + warn!( + "live-move: failed to build shadow add cmd for {}", + mig.circuit_hash + ); + mig.stage = MigrationStage::Done; + advanced += 1; + } + } + MigrationStage::VerifyShadowReady => { + let target_label = migration_target_label(mig); + let verification = match migration_shadow_verification(config, mig) { + Ok(verification) => verification, + Err(e) => { + mark_reload_required(format!( + "Bakery live-move shadow verification failed for {target_label}: {e}. A full reload is now required before further incremental topology mutations." + )); + mig.stage = MigrationStage::Done; + advanced += 1; + continue; + } + }; + if !verification.ready() { + if mig.shadow_verify_attempts < MIGRATION_VERIFICATION_MAX_RETRIES { + mig.shadow_verify_attempts = + mig.shadow_verify_attempts.saturating_add(1); + warn!( + "Bakery live-move shadow verification retry {}/{} for {}: {}", + mig.shadow_verify_attempts, + MIGRATION_VERIFICATION_MAX_RETRIES, + target_label, + verification.summary() + ); + advanced += 1; + continue; + } + mark_reload_required(format!( + "Bakery live-move shadow verification did not find the expected shadow class/qdisc for {target_label}: {}. A full reload is now required before further incremental topology mutations.", + verification.summary() + )); + mig.stage = MigrationStage::Done; + advanced += 1; + continue; + } + mig.stage = MigrationStage::SwapToShadow; + mig.shadow_verify_attempts = 0; + advanced += 1; + } + MigrationStage::SwapToShadow => { + let target_label = migration_target_label(mig); + if let Err(summary) = + remap_migration_ips(mig, mig.shadow_minor, mapping_current) + { + mark_reload_required(format!( + "Bakery live-move shadow remap failed for {target_label}: {summary}. A full reload is now required before further incremental topology mutations." + )); + mig.stage = MigrationStage::Done; + advanced += 1; + continue; + } + mig.stage = MigrationStage::BuildFinal; + advanced += 1; + } + MigrationStage::BuildFinal => { + let target_label = migration_target_label(mig); + let mut cmds = Vec::new(); + let live_qdisc_handles = snapshot_live_qdisc_handle_majors_or_empty( + config, + "live-move: build final", + ); + let down_live_qdisc_handles = live_qdisc_handles.get(&config.isp_interface()); + let up_live_qdisc_handles = + live_qdisc_handles.get(&config.internet_interface()); + if let Some(final_cmd) = build_temp_add_cmd( + mig.desired_cmd.as_ref(), + mig.final_minor, + mig.new_down_min, + mig.new_down_max, + mig.new_up_min, + mig.new_up_max, + true, + ) { + if let Some(summary) = + build_final_qdisc_handle_rotation_invariant_error_with_live_reservations( + mig, + &final_cmd, + down_live_qdisc_handles, + up_live_qdisc_handles, + ) + { + mark_reload_required(format!( + "Bakery live migration refused to build final state for {target_label} because the final qdisc handles were unsafe: {summary}. A full reload is now required before further incremental topology mutations." + )); + mig.stage = MigrationStage::Done; + advanced += 1; + continue; + } + match config.queues.lazy_queues.as_ref() { + None | Some(LazyQueueMode::No) => { + if let Some(c) = add_commands_for_circuit( + &final_cmd, + config, + ExecutionMode::Builder, + ) { + cmds.extend(c); + } + } + Some(LazyQueueMode::Htb) => { + if let Some(c) = add_commands_for_circuit( + &final_cmd, + config, + ExecutionMode::Builder, + ) { + cmds.extend(c); + } + if let Some(c) = add_commands_for_circuit( + &final_cmd, + config, + ExecutionMode::LiveUpdate, + ) { + cmds.extend(c); + } + } + Some(LazyQueueMode::Full) => { + if let Some(c) = add_commands_for_circuit( + &final_cmd, + config, + ExecutionMode::LiveUpdate, + ) { + cmds.extend(c); + } + } + } + } + if !cmds.is_empty() { + let result = + execute_and_record_live_change(&cmds, "live-move: build final"); + let _ = migration_stage_apply_succeeded( + mig, + "live-move: build final", + &result, + MigrationStage::VerifyFinalReady, + ); + } else { + mig.stage = MigrationStage::VerifyFinalReady; + } + advanced += 1; + } + MigrationStage::VerifyFinalReady => { + let target_label = migration_target_label(mig); + let verification = match migration_final_verification(config, mig) { + Ok(verification) => verification, + Err(e) => { + mark_reload_required(format!( + "Bakery live-move final verification failed for {target_label}: {e}. A full reload is now required before further incremental topology mutations." + )); + mig.stage = MigrationStage::Done; + advanced += 1; + continue; + } + }; + if !verification.ready() { + if mig.final_verify_attempts < MIGRATION_VERIFICATION_MAX_RETRIES { + mig.final_verify_attempts = mig.final_verify_attempts.saturating_add(1); + warn!( + "Bakery live-move final verification retry {}/{} for {}: {}", + mig.final_verify_attempts, + MIGRATION_VERIFICATION_MAX_RETRIES, + target_label, + verification.summary() + ); + advanced += 1; + continue; + } + mark_reload_required(format!( + "Bakery live-move final verification did not find the expected final class/qdisc for {target_label}: {}. A full reload is now required before further incremental topology mutations.", + verification.summary() + )); + mig.stage = MigrationStage::Done; + advanced += 1; + continue; + } + mig.stage = MigrationStage::SwapToFinal; + mig.final_verify_attempts = 0; + advanced += 1; + } + MigrationStage::SwapToFinal => { + let target_label = migration_target_label(mig); + if let Err(summary) = remap_migration_ips(mig, mig.final_minor, mapping_current) + { + mark_reload_required(format!( + "Bakery live-move final remap failed for {target_label}: {summary}. A full reload is now required before further incremental topology mutations." + )); + mig.stage = MigrationStage::Done; + advanced += 1; + continue; + } + circuits.insert(mig.circuit_hash, Arc::clone(&mig.desired_cmd)); + effective_state_changed = true; + mig.stage = MigrationStage::TeardownMigrationScaffold; + advanced += 1; + } + MigrationStage::TeardownMigrationScaffold => { + let mut prune = Vec::new(); + if let Some(shadow_cmd) = build_temp_add_cmd( + &BakeryCommands::AddCircuit { + circuit_hash: mig.circuit_hash, + circuit_name: mig.circuit_name.clone(), + site_name: mig.site_name.clone(), + parent_class_id: mig.parent_class_id, + up_parent_class_id: mig.up_parent_class_id, + class_minor: mig.shadow_minor, + download_bandwidth_min: mig.old_down_min, + upload_bandwidth_min: mig.old_up_min, + download_bandwidth_max: mig.old_down_max, + upload_bandwidth_max: mig.old_up_max, + class_major: mig.class_major, + up_class_major: mig.up_class_major, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "".to_string(), + sqm_override: mig.sqm_override.clone(), + }, + mig.shadow_minor, + mig.old_down_min, + mig.old_down_max, + mig.old_up_min, + mig.old_up_max, + false, + ) && let Some(shadow_prune) = shadow_cmd.to_prune(config, true) + { + prune.extend(shadow_prune); + } + if !prune.is_empty() { + let result = execute_and_record_live_change( + &prune, + "live-move: prune migration scaffold", + ); + let _ = migration_stage_apply_succeeded( + mig, + "live-move: prune migration scaffold", + &result, + MigrationStage::Done, + ); + } else { + mig.stage = MigrationStage::Done; + } + advanced += 1; + } + MigrationStage::Done => { + to_remove.push(mig.circuit_hash); + } + } + } + + for circuit_hash in to_remove { + migrations.remove(&circuit_hash); + } + + flush_deferred_runtime_site_prunes( + config, + virtualized_sites, + migrations, + runtime_node_operations, + ); + if effective_state_changed { + update_queue_distribution_snapshot(sites, circuits); + } + } + { let Ok(config) = lqos_config::load_config() else { error!("Failed to load configuration, exiting Bakery thread."); @@ -618,9 +3675,35 @@ fn bakery_main(rx: Receiver, tx: Sender) { config.queues.lazy_queues, config.queues.lazy_expire_seconds.unwrap_or(600) ); + push_bakery_event( + "baseline_rebuild_startup", + "info", + "Bakery started with empty runtime state; the first commit will rebuild the baseline queue tree.".to_string(), + ); } - while let Ok(command) = rx.recv() { + loop { + let command = match rx.recv_timeout(Duration::from_millis(BAKERY_BACKGROUND_INTERVAL_MS)) { + Ok(command) => command, + Err(RecvTimeoutError::Timeout) => { + let Ok(config) = lqos_config::load_config() else { + error!("Failed to load configuration while processing pending migrations."); + continue; + }; + process_pending_migrations( + &config, + &mut circuits, + &sites, + &mut migrations, + &mut mapping_current, + &mut virtualized_sites, + &mut runtime_node_operations, + ); + refresh_live_capacity_snapshot(&config, false); + continue; + } + Err(RecvTimeoutError::Disconnected) => break, + }; debug!("Bakery received command: {:?}", command); match command { @@ -742,14 +3825,36 @@ fn bakery_main(rx: Receiver, tx: Sender) { batch = Some(Vec::new()); } BakeryCommands::CommitBatch => { + push_bakery_event( + "commit_received", + "info", + "Bakery commit received.".to_string(), + ); handle_commit_batch( &mut batch, &mut sites, &mut circuits, &mut live_circuits, + &mut mq_layout, + &mut qdisc_handles, &tx, &mut migrations, &stormguard_overrides, + &mut virtualized_sites, + &mut runtime_node_operations, + ); + let Ok(config) = lqos_config::load_config() else { + error!("Failed to load configuration while processing pending migrations."); + continue; + }; + process_pending_migrations( + &config, + &mut circuits, + &sites, + &mut migrations, + &mut mapping_current, + &mut virtualized_sites, + &mut runtime_node_operations, ); } BakeryCommands::MqSetup { .. } => { @@ -773,255 +3878,19 @@ fn bakery_main(rx: Receiver, tx: Sender) { BakeryCommands::Tick => { // Reset per-cycle counters at the start of the tick handle_tick(&mut circuits, &mut live_circuits, &mut sites); - - // Advance live-move migrations let Ok(config) = lqos_config::load_config() else { - error!("Failed to load configuration, exiting Bakery thread."); + error!("Failed to load configuration while processing pending migrations."); continue; }; - let mut advanced = 0usize; - let mut to_remove = Vec::new(); - for (_hash, mig) in migrations.iter_mut() { - if advanced >= MIGRATIONS_PER_TICK { - break; - } - match mig.stage { - MigrationStage::PrepareShadow => { - // Create shadow HTB+CAKE with OLD rates - if let Some(temp) = build_temp_add_cmd( - &BakeryCommands::AddCircuit { - circuit_hash: mig.circuit_hash, - parent_class_id: mig.parent_class_id, - up_parent_class_id: mig.up_parent_class_id, - class_minor: mig.shadow_minor, - download_bandwidth_min: mig.old_down_min, - upload_bandwidth_min: mig.old_up_min, - download_bandwidth_max: mig.old_down_max, - upload_bandwidth_max: mig.old_up_max, - class_major: mig.class_major, - up_class_major: mig.up_class_major, - ip_addresses: "".to_string(), - sqm_override: mig.sqm_override.clone(), - }, - mig.shadow_minor, - mig.old_down_min, - mig.old_down_max, - mig.old_up_min, - mig.old_up_max, - ) { - let mut cmds = Vec::new(); - match config.queues.lazy_queues.as_ref() { - None | Some(LazyQueueMode::No) => { - if let Some(c) = add_commands_for_circuit( - &temp, - &config, - ExecutionMode::Builder, - ) { - cmds.extend(c); - } - } - Some(LazyQueueMode::Htb) => { - if let Some(c) = add_commands_for_circuit( - &temp, - &config, - ExecutionMode::Builder, - ) { - cmds.extend(c); - } - if let Some(c) = add_commands_for_circuit( - &temp, - &config, - ExecutionMode::LiveUpdate, - ) { - cmds.extend(c); - } - } - Some(LazyQueueMode::Full) => { - if let Some(c) = add_commands_for_circuit( - &temp, - &config, - ExecutionMode::LiveUpdate, - ) { - cmds.extend(c); - } - } - } - if !cmds.is_empty() { - execute_in_memory(&cmds, "live-move: create shadow"); - } - mig.stage = MigrationStage::SwapToShadow; - advanced += 1; - } else { - warn!( - "live-move: failed to build shadow add cmd for {}", - mig.circuit_hash - ); - mig.stage = MigrationStage::Done; - advanced += 1; - } - } - MigrationStage::SwapToShadow => { - // Remap all IPs to shadow handles using existing CPU. - // Upload mapping is derived in the dataplane for on-a-stick mode. - for ip in &mig.ips { - let (ip_s, prefix) = parse_ip_and_prefix(ip); - let key = MappingKey { - ip: ip_s.clone(), - prefix, - }; - let cpu = mapping_current.get(&key).map(|v| v.cpu).unwrap_or(0); - let handle = - tc_handle_from_major_minor(mig.class_major, mig.shadow_minor); - let _ = lqos_sys::add_ip_to_tc(&ip_s, handle, cpu, false, 0, 0); - // Update local mapping view - mapping_current.insert(key, MappingVal { handle, cpu }); - } - // Clear the hot cache directly - let _ = lqos_sys::clear_hot_cache(); - mig.stage = MigrationStage::BuildFinal; - advanced += 1; - } - MigrationStage::BuildFinal => { - // Delete old classes/qdiscs and create final with NEW rates at original minor - if let Some(old_cmd) = build_temp_add_cmd( - &BakeryCommands::AddCircuit { - circuit_hash: mig.circuit_hash, - parent_class_id: mig.parent_class_id, - up_parent_class_id: mig.up_parent_class_id, - class_minor: mig.old_minor, - download_bandwidth_min: mig.old_down_min, - upload_bandwidth_min: mig.old_up_min, - download_bandwidth_max: mig.old_down_max, - upload_bandwidth_max: mig.old_up_max, - class_major: mig.class_major, - up_class_major: mig.up_class_major, - ip_addresses: "".to_string(), - sqm_override: mig.sqm_override.clone(), - }, - mig.old_minor, - mig.old_down_min, - mig.old_down_max, - mig.old_up_min, - mig.old_up_max, - ) { - let mut cmds = Vec::new(); - if let Some(prune) = old_cmd.to_prune(&config, true) { - cmds.extend(prune); - } - // Final add (new rates) at final_minor - if let Some(final_cmd) = build_temp_add_cmd( - &old_cmd, - mig.final_minor, - mig.new_down_min, - mig.new_down_max, - mig.new_up_min, - mig.new_up_max, - ) { - match config.queues.lazy_queues.as_ref() { - None | Some(LazyQueueMode::No) => { - if let Some(c) = add_commands_for_circuit( - &final_cmd, - &config, - ExecutionMode::Builder, - ) { - cmds.extend(c); - } - } - Some(LazyQueueMode::Htb) => { - if let Some(c) = add_commands_for_circuit( - &final_cmd, - &config, - ExecutionMode::Builder, - ) { - cmds.extend(c); - } - if let Some(c) = add_commands_for_circuit( - &final_cmd, - &config, - ExecutionMode::LiveUpdate, - ) { - cmds.extend(c); - } - } - Some(LazyQueueMode::Full) => { - if let Some(c) = add_commands_for_circuit( - &final_cmd, - &config, - ExecutionMode::LiveUpdate, - ) { - cmds.extend(c); - } - } - } - } - if !cmds.is_empty() { - execute_in_memory(&cmds, "live-move: build final"); - } - mig.stage = MigrationStage::SwapToFinal; - advanced += 1; - } else { - warn!( - "live-move: failed to build old prune cmd for {}", - mig.circuit_hash - ); - mig.stage = MigrationStage::Done; - advanced += 1; - } - } - MigrationStage::SwapToFinal => { - for ip in &mig.ips { - let (ip_s, prefix) = parse_ip_and_prefix(ip); - let key = MappingKey { - ip: ip_s.clone(), - prefix, - }; - let cpu = mapping_current.get(&key).map(|v| v.cpu).unwrap_or(0); - let handle = - tc_handle_from_major_minor(mig.class_major, mig.final_minor); - let _ = lqos_sys::add_ip_to_tc(&ip_s, handle, cpu, false, 0, 0); - mapping_current.insert(key, MappingVal { handle, cpu }); - } - let _ = lqos_sys::clear_hot_cache(); - mig.stage = MigrationStage::TeardownShadow; - advanced += 1; - } - MigrationStage::TeardownShadow => { - // Remove shadow classes - if let Some(shadow_cmd) = build_temp_add_cmd( - &BakeryCommands::AddCircuit { - circuit_hash: mig.circuit_hash, - parent_class_id: mig.parent_class_id, - up_parent_class_id: mig.up_parent_class_id, - class_minor: mig.shadow_minor, - download_bandwidth_min: mig.old_down_min, - upload_bandwidth_min: mig.old_up_min, - download_bandwidth_max: mig.old_down_max, - upload_bandwidth_max: mig.old_up_max, - class_major: mig.class_major, - up_class_major: mig.up_class_major, - ip_addresses: "".to_string(), - sqm_override: mig.sqm_override.clone(), - }, - mig.shadow_minor, - mig.old_down_min, - mig.old_down_max, - mig.old_up_min, - mig.old_up_max, - ) && let Some(prune) = shadow_cmd.to_prune(&config, true) - { - execute_in_memory(&prune, "live-move: prune shadow"); - } - mig.stage = MigrationStage::Done; - advanced += 1; - } - MigrationStage::Done => { - to_remove.push(mig.circuit_hash); - } - } - } - for h in to_remove { - migrations.remove(&h); - } + process_pending_migrations( + &config, + &mut circuits, + &sites, + &mut migrations, + &mut mapping_current, + &mut virtualized_sites, + &mut runtime_node_operations, + ); } BakeryCommands::ChangeSiteSpeedLive { site_hash, @@ -1050,6 +3919,10 @@ fn bakery_main(rx: Receiver, tx: Sender) { debug!("StormGuardAdjustment received before MQ setup, skipping."); continue; } + let Ok(config) = lqos_config::load_config() else { + error!("Failed to load configuration, skipping StormGuardAdjustment."); + continue; + }; let Ok(tc_handle) = TcHandle::from_string(&class_id) else { warn!( "StormGuardAdjustment has invalid class_id [{}], skipping.", @@ -1064,6 +3937,13 @@ fn bakery_main(rx: Receiver, tx: Sender) { }; stormguard_overrides.insert(key, new_rate); } + if let Some(reason) = live_tree_mutation_blocker_for_config(&config) { + info!( + "Skipping StormGuard live class change for {} {} because {}.", + interface_name, class_id, reason + ); + continue; + } let normalized_class = tc_handle.as_tc_string(); // Build the HTB command let args = vec![ @@ -1103,38 +3983,125 @@ fn bakery_main(rx: Receiver, tx: Sender) { } } } + BakeryCommands::TreeGuardSetNodeVirtual { + site_hash, + virtualized, + reply, + } => { + let result = handle_treeguard_set_node_virtual_live( + site_hash, + virtualized, + &mut sites, + &mut circuits, + &live_circuits, + &mq_layout, + &mut qdisc_handles, + &mut migrations, + &mut virtualized_sites, + &mut runtime_node_operations, + &mut next_runtime_operation_id, + ); + update_queue_distribution_snapshot(&sites, &circuits); + if let Some(reply) = reply { + let _ = reply.send(result); + } + } } } error!("Bakery thread exited unexpectedly."); } +#[allow(clippy::too_many_arguments)] fn handle_commit_batch( batch: &mut Option>>, sites: &mut HashMap>, circuits: &mut HashMap>, live_circuits: &mut HashMap, + mq_layout: &mut Option, + qdisc_handles: &mut QdiscHandleState, tx: &Sender, migrations: &mut HashMap, stormguard_overrides: &HashMap, + virtualized_sites: &mut HashMap, + runtime_node_operations: &mut HashMap, ) { let Ok(config) = lqos_config::load_config() else { error!("Failed to load configuration, exiting Bakery thread."); return; }; + qdisc_handles.clear_retired_handles(); - let Some(new_batch) = batch.take() else { + let Some(raw_batch) = batch.take() else { debug!("CommitBatch received without a batch to commit."); return; }; + let (baseline_sites, baseline_circuits) = + reconstruct_structural_baseline_state(sites, circuits, virtualized_sites); + let effective_new_batch = + apply_runtime_virtualization_overlay(raw_batch.clone(), virtualized_sites); + let resolved_mq_layout = current_mq_layout(&raw_batch, &config, mq_layout); let mapped_limit = resolve_mapped_circuit_limit(); let effective_limit = mapped_limit.effective_limit; let limit_label = format_mapped_limit(effective_limit); + if let Some(reason) = bakery_reload_required_reason() { + let summary = format!( + "Bakery full reload triggered by reload-required state: {}", + reason + ); + let (new_batch, mapped_limit_stats) = filter_batch_by_mapped_circuit_limit( + raw_batch.clone(), + &baseline_circuits, + effective_limit, + ); + log_mapped_limit_decision("reload-required rebuild", mapped_limit, mapped_limit_stats); + if mapped_limit_stats.dropped_mapped > 0 { + warn!( + "Bakery mapped circuit cap enforced (reload-required rebuild): requested={}, allowed={}, dropped={}, limit={} (licensed={}, max_circuits={:?})", + mapped_limit_stats.requested_mapped, + mapped_limit_stats.allowed_mapped, + mapped_limit_stats.dropped_mapped, + limit_label, + mapped_limit.licensed, + mapped_limit.max_circuits + ); + maybe_emit_mapped_circuit_limit_urgent(&mapped_limit_stats); + } + announce_full_reload(&summary); + full_reload( + batch, + sites, + circuits, + live_circuits, + mq_layout, + qdisc_handles, + &config, + new_batch, + resolved_mq_layout, + stormguard_overrides, + virtualized_sites, + runtime_node_operations, + summary, + ); + return; + } + let has_mq_been_setup = MQ_CREATED.load(std::sync::atomic::Ordering::Relaxed); + let shaping_tree_active = SHAPING_TREE_ACTIVE.load(std::sync::atomic::Ordering::Relaxed); + let desired_tree_active = desired_shaping_tree_active(&config); if !has_mq_been_setup { - let (new_batch, mapped_limit_stats) = - filter_batch_by_mapped_circuit_limit(new_batch, circuits, effective_limit); + push_bakery_event( + "baseline_rebuild_required", + "warning", + "Bakery runtime state was reset by restart/cold start; performing explicit baseline full reload.".to_string(), + ); + let (new_batch, mapped_limit_stats) = filter_batch_by_mapped_circuit_limit( + raw_batch.clone(), + &baseline_circuits, + effective_limit, + ); + log_mapped_limit_decision("full reload", mapped_limit, mapped_limit_stats); if mapped_limit_stats.dropped_mapped > 0 { warn!( "Bakery mapped circuit cap enforced (full reload): requested={}, allowed={}, dropped={}, limit={} (licensed={}, max_circuits={:?})", @@ -1148,24 +4115,82 @@ fn handle_commit_batch( maybe_emit_mapped_circuit_limit_urgent(&mapped_limit_stats); } // If the MQ hasn't been created, we need to do this as a full, unadjusted run. - info!("MQ not created, performing full reload."); + let summary = "Bakery full reload triggered by baseline rebuild after restart/cold start." + .to_string(); + info!("Bakery baseline rebuild after restart/cold start: performing explicit full reload."); + announce_full_reload(&summary); full_reload( batch, sites, circuits, live_circuits, + mq_layout, + qdisc_handles, &config, new_batch, + resolved_mq_layout, stormguard_overrides, + virtualized_sites, + runtime_node_operations, + summary, ); - MQ_CREATED.store(true, std::sync::atomic::Ordering::Relaxed); return; } - let site_change_mode = diff_sites(&new_batch, sites); - if matches!(site_change_mode, SiteDiffResult::RebuildRequired) { - let (new_batch, mapped_limit_stats) = - filter_batch_by_mapped_circuit_limit(new_batch, circuits, effective_limit); + if shaping_tree_active != desired_tree_active { + let (new_batch, mapped_limit_stats) = filter_batch_by_mapped_circuit_limit( + raw_batch.clone(), + &baseline_circuits, + effective_limit, + ); + log_mapped_limit_decision("queue-mode rebuild", mapped_limit, mapped_limit_stats); + if mapped_limit_stats.dropped_mapped > 0 { + warn!( + "Bakery mapped circuit cap enforced (queue-mode rebuild): requested={}, allowed={}, dropped={}, limit={} (licensed={}, max_circuits={:?})", + mapped_limit_stats.requested_mapped, + mapped_limit_stats.allowed_mapped, + mapped_limit_stats.dropped_mapped, + limit_label, + mapped_limit.licensed, + mapped_limit.max_circuits + ); + maybe_emit_mapped_circuit_limit_urgent(&mapped_limit_stats); + } + let summary = format!( + "Bakery full reload triggered by queue mode transition to {}.", + if desired_tree_active { + "shape" + } else { + "observe" + } + ); + announce_full_reload(&summary); + full_reload( + batch, + sites, + circuits, + live_circuits, + mq_layout, + qdisc_handles, + &config, + new_batch, + resolved_mq_layout, + stormguard_overrides, + virtualized_sites, + runtime_node_operations, + summary, + ); + return; + } + + let structural_site_change_mode = diff_sites(&raw_batch, &baseline_sites); + if let SiteDiffResult::RebuildRequired { summary } = &structural_site_change_mode { + let (new_batch, mapped_limit_stats) = filter_batch_by_mapped_circuit_limit( + raw_batch.clone(), + &baseline_circuits, + effective_limit, + ); + log_mapped_limit_decision("site-structure rebuild", mapped_limit, mapped_limit_stats); if mapped_limit_stats.dropped_mapped > 0 { warn!( "Bakery mapped circuit cap enforced (site-structure rebuild): requested={}, allowed={}, dropped={}, limit={} (licensed={}, max_circuits={:?})", @@ -1180,20 +4205,31 @@ fn handle_commit_batch( } // If the site structure has changed, we need to rebuild everything. info!("Bakery full reload: site_struct=1, circuit_struct=0"); + announce_full_reload(summary); full_reload( batch, sites, circuits, live_circuits, + mq_layout, + qdisc_handles, &config, new_batch, + resolved_mq_layout, stormguard_overrides, + virtualized_sites, + runtime_node_operations, + summary.clone(), ); - MQ_CREATED.store(true, std::sync::atomic::Ordering::Relaxed); return; } - let circuit_change_mode = diff_circuits(&new_batch, circuits); + let baseline_circuits_for_diff = + circuits_with_pending_migration_targets(&baseline_circuits, migrations); + let structural_circuit_change_mode = diff_circuits(&raw_batch, &baseline_circuits_for_diff); + let site_change_mode = diff_sites(&effective_new_batch, sites); + let circuits_for_diff = circuits_with_pending_migration_targets(circuits, migrations); + let circuit_change_mode = diff_circuits(&effective_new_batch, &circuits_for_diff); // If neither has changed, there's nothing to do. if matches!(site_change_mode, SiteDiffResult::NoChange) @@ -1205,11 +4241,23 @@ fn handle_commit_batch( } // If any structural changes occurred, do a full reload - if let CircuitDiffResult::Categorized(categories) = &circuit_change_mode + if let CircuitDiffResult::Categorized(categories) = &structural_circuit_change_mode && !categories.structural_changed.is_empty() { - let (new_batch, mapped_limit_stats) = - filter_batch_by_mapped_circuit_limit(new_batch.clone(), circuits, effective_limit); + let summary = format!( + "Bakery full reload triggered by circuit structural diff: structural_changed_count={}", + categories.structural_changed.len() + ); + let (new_batch, mapped_limit_stats) = filter_batch_by_mapped_circuit_limit( + raw_batch.clone(), + &baseline_circuits, + effective_limit, + ); + log_mapped_limit_decision( + "circuit-structure rebuild", + mapped_limit, + mapped_limit_stats, + ); if mapped_limit_stats.dropped_mapped > 0 { warn!( "Bakery mapped circuit cap enforced (circuit-structure rebuild): requested={}, allowed={}, dropped={}, limit={} (licensed={}, max_circuits={:?})", @@ -1226,16 +4274,22 @@ fn handle_commit_batch( "Bakery full reload: site_struct=0, circuit_struct={}", categories.structural_changed.len() ); + announce_full_reload(&summary); full_reload( batch, sites, circuits, live_circuits, + mq_layout, + qdisc_handles, &config, new_batch, + resolved_mq_layout, stormguard_overrides, + virtualized_sites, + runtime_node_operations, + summary, ); - MQ_CREATED.store(true, std::sync::atomic::Ordering::Relaxed); return; } @@ -1269,14 +4323,25 @@ fn handle_commit_batch( }) { error!("Channel full, falling back to full rebuild: {}", e); info!("Bakery full reload: site_struct=0, circuit_struct=0"); + let summary = format!( + "Bakery full reload triggered because site speed live-change enqueue failed: {}", + e + ); + announce_full_reload(&summary); full_reload( batch, sites, circuits, live_circuits, + mq_layout, + qdisc_handles, &config, - new_batch.clone(), + raw_batch.clone(), + resolved_mq_layout.clone(), stormguard_overrides, + virtualized_sites, + runtime_node_operations, + summary, ); return; // Skip the rest of this CommitBatch processing } @@ -1287,11 +4352,12 @@ fn handle_commit_batch( if let CircuitDiffResult::Categorized(categories) = circuit_change_mode { // One-line summary of changes (info!) info!( - "Bakery changes: sites_speed={}, circuits_added={}, removed={}, speed={}, ip={}", + "Bakery changes: sites_speed={}, circuits_added={}, removed={}, speed={}, migrated={}, ip={}", site_speed_change_count, categories.newly_added.len(), categories.removed_circuits.len(), categories.speed_changed.len(), + categories.migrated.len(), categories.ip_changed.len() ); @@ -1318,9 +4384,13 @@ fn handle_commit_batch( } }; if let Some(cmd) = commands { - execute_in_memory(&cmd, "removing circuit"); + execute_and_record_live_change(&cmd, "removing circuit"); } live_circuits.remove(&circuit_hash); + qdisc_handles.release_circuit(&config.isp_interface(), circuit_hash); + if !config.on_a_stick_mode() { + qdisc_handles.release_circuit(&config.internet_interface(), circuit_hash); + } } else { debug!( "RemoveCircuit received for unknown circuit: {}", @@ -1332,105 +4402,127 @@ fn handle_commit_batch( // 2) Speed changes (avoid linux TC deadlock by removing qdisc first) if !categories.speed_changed.is_empty() { + let live_tree_allowed = live_tree_mutations_allowed(&config); + if !live_tree_allowed + && let Some(reason) = live_tree_mutation_blocker_for_config(&config) + { + info!( + "Skipping live circuit speed fallback updates because {}. Runtime state will be updated in memory only.", + reason + ); + push_bakery_event( + "live_circuit_speed_skipped", + "info", + format!( + "Skipping live circuit speed fallback updates because {}.", + reason + ), + ); + } + let live_reserved_handles = if live_tree_allowed { + snapshot_live_qdisc_handle_majors_or_empty(&config, "circuit speed updates") + } else { + HashMap::new() + }; let mut immediate_commands = Vec::new(); for cmd in &categories.speed_changed { - if let BakeryCommands::AddCircuit { - circuit_hash, - parent_class_id, - up_parent_class_id, - class_minor, - download_bandwidth_min, - upload_bandwidth_min, - download_bandwidth_max, - upload_bandwidth_max, - class_major, - up_class_major, - ip_addresses, - sqm_override, - } = cmd.as_ref() + let mut enriched_cmd = if live_tree_allowed { + let Some(layout) = resolved_mq_layout.as_ref() else { + warn!("Bakery: missing MQ layout during circuit speed updates"); + return; + }; + with_assigned_qdisc_handles_reserved( + cmd, + &config, + layout, + qdisc_handles, + &live_reserved_handles, + ) + } else { + Arc::clone(cmd) + }; + let old_cmd = if let BakeryCommands::AddCircuit { circuit_hash, .. } = + enriched_cmd.as_ref() { - let was_activated = live_circuits.contains_key(circuit_hash); - if was_activated { - // Attempt live-move - if let Some(shadow_minor) = - find_free_minor(circuits, parent_class_id, up_parent_class_id) - { - // Find old command for old rates - if let Some(old_cmd) = circuits.get(circuit_hash) - && let BakeryCommands::AddCircuit { - download_bandwidth_min: old_down_min, - upload_bandwidth_min: old_up_min, - download_bandwidth_max: old_down_max, - upload_bandwidth_max: old_up_max, - .. - } = old_cmd.as_ref() - { - let mig = Migration { - circuit_hash: *circuit_hash, - parent_class_id: *parent_class_id, - up_parent_class_id: *up_parent_class_id, - class_major: *class_major, - up_class_major: *up_class_major, - old_down_min: *old_down_min, - old_down_max: *old_down_max, - old_up_min: *old_up_min, - old_up_max: *old_up_max, - new_down_min: *download_bandwidth_min, - new_down_max: *download_bandwidth_max, - new_up_min: *upload_bandwidth_min, - new_up_max: *upload_bandwidth_max, - old_minor: *class_minor, - shadow_minor, - final_minor: *class_minor, - ips: parse_ip_list(ip_addresses), - sqm_override: sqm_override.clone(), - stage: MigrationStage::PrepareShadow, - }; - migrations.insert(*circuit_hash, mig); - // Update desired circuit definition now - circuits.insert(*circuit_hash, Arc::clone(cmd)); - continue; // skip immediate path - } - } + circuits.get(circuit_hash).cloned() + } else { + None + }; + if let Some(old_cmd) = old_cmd.as_ref() + && live_tree_allowed + { + let Some(layout) = resolved_mq_layout.as_ref() else { + warn!("Bakery: missing MQ layout during circuit speed updates"); + return; + }; + enriched_cmd = rotate_changed_qdisc_handles_reserved( + old_cmd.as_ref(), + &enriched_cmd, + &config, + layout, + qdisc_handles, + &live_reserved_handles, + ); + } + if live_tree_allowed + && let Some(old_cmd) = old_cmd.as_ref() + && queue_live_migration( + old_cmd.as_ref(), + &enriched_cmd, + sites, + circuits, + live_circuits, + migrations, + ) + { + continue; + } + if let BakeryCommands::AddCircuit { circuit_hash, .. } = enriched_cmd.as_ref() { + if !live_tree_allowed { + circuits.insert(*circuit_hash, enriched_cmd); + continue; } + let was_activated = live_circuits.contains_key(circuit_hash); // Fallback: immediate safe update match config.queues.lazy_queues.as_ref() { None | Some(LazyQueueMode::No) => { - if let Some(prune) = cmd.to_prune(&config, true) { + if let Some(prune) = enriched_cmd.to_prune(&config, true) { immediate_commands.extend(prune); } - if let Some(add) = cmd.to_commands(&config, ExecutionMode::Builder) { + if let Some(add) = + enriched_cmd.to_commands(&config, ExecutionMode::Builder) + { immediate_commands.extend(add); } } Some(LazyQueueMode::Htb) => { if was_activated { - if let Some(prune) = cmd.to_prune(&config, false) { + if let Some(prune) = enriched_cmd.to_prune(&config, false) { immediate_commands.extend(prune); } if let Some(add_htb) = - cmd.to_commands(&config, ExecutionMode::Builder) + enriched_cmd.to_commands(&config, ExecutionMode::Builder) { immediate_commands.extend(add_htb); } if let Some(add_qdisc) = - cmd.to_commands(&config, ExecutionMode::LiveUpdate) + enriched_cmd.to_commands(&config, ExecutionMode::LiveUpdate) { immediate_commands.extend(add_qdisc); } } else if let Some(add_htb) = - cmd.to_commands(&config, ExecutionMode::Builder) + enriched_cmd.to_commands(&config, ExecutionMode::Builder) { immediate_commands.extend(add_htb); } } Some(LazyQueueMode::Full) => { if was_activated { - if let Some(prune) = cmd.to_prune(&config, true) { + if let Some(prune) = enriched_cmd.to_prune(&config, true) { immediate_commands.extend(prune); } if let Some(add_all) = - cmd.to_commands(&config, ExecutionMode::LiveUpdate) + enriched_cmd.to_commands(&config, ExecutionMode::LiveUpdate) { immediate_commands.extend(add_all); } @@ -1439,11 +4531,172 @@ fn handle_commit_batch( } } } - circuits.insert(*circuit_hash, Arc::clone(cmd)); + circuits.insert(*circuit_hash, enriched_cmd); } } + if live_tree_allowed && !immediate_commands.is_empty() { + execute_and_record_live_change(&immediate_commands, "updating circuit speeds live"); + } + } + + // 2b) Parent/class migrations + if !categories.migrated.is_empty() { + let Some(layout) = resolved_mq_layout.as_ref() else { + warn!("Bakery: missing MQ layout during circuit migrations"); + return; + }; + let live_reserved_handles = + snapshot_live_qdisc_handle_majors_or_empty(&config, "circuit migrations"); + let down_live_snapshot = read_live_class_snapshot(&config.isp_interface()).ok(); + let up_live_snapshot = read_live_class_snapshot(&config.internet_interface()).ok(); + let mut immediate_commands = Vec::new(); + let mut migrated_updates = Vec::new(); + let mut prepared_migrations = Vec::new(); + let mut protected_down_classes = HashSet::new(); + let mut protected_up_classes = HashSet::new(); + for cmd in &categories.migrated { + let mut enriched_cmd = with_assigned_qdisc_handles_reserved( + cmd, + &config, + layout, + qdisc_handles, + &live_reserved_handles, + ); + let Some(old_cmd) = (if let BakeryCommands::AddCircuit { circuit_hash, .. } = + enriched_cmd.as_ref() + { + circuits.get(circuit_hash).cloned() + } else { + None + }) else { + continue; + }; + + enriched_cmd = rotate_changed_qdisc_handles_reserved( + old_cmd.as_ref(), + &enriched_cmd, + &config, + layout, + qdisc_handles, + &live_reserved_handles, + ); + let BakeryCommands::AddCircuit { circuit_hash, .. } = enriched_cmd.as_ref() else { + continue; + }; + if let Some((down_class, up_class)) = circuit_class_handles(enriched_cmd.as_ref()) { + protected_down_classes.insert(down_class); + protected_up_classes.insert(up_class); + } + prepared_migrations.push((*circuit_hash, old_cmd, enriched_cmd)); + } + + for (circuit_hash, old_cmd, enriched_cmd) in prepared_migrations { + if queue_live_migration( + old_cmd.as_ref(), + &enriched_cmd, + sites, + circuits, + live_circuits, + migrations, + ) { + continue; + } + + if live_circuits.contains_key(&circuit_hash) { + warn!( + "Bakery: falling back to full reload for active migrated circuit {} because live migration setup failed", + circuit_hash + ); + let (_new_batch, mapped_limit_stats) = filter_batch_by_mapped_circuit_limit( + raw_batch.clone(), + &baseline_circuits, + effective_limit, + ); + log_mapped_limit_decision( + "circuit-migration rebuild", + mapped_limit, + mapped_limit_stats, + ); + let summary = format!( + "Bakery full reload triggered because live migration setup failed for active migrated circuit {}.", + circuit_hash + ); + announce_full_reload(&summary); + full_reload( + batch, + sites, + circuits, + live_circuits, + mq_layout, + qdisc_handles, + &config, + raw_batch.clone(), + resolved_mq_layout.clone(), + stormguard_overrides, + virtualized_sites, + runtime_node_operations, + summary, + ); + return; + } + + match config.queues.lazy_queues.as_ref() { + None | Some(LazyQueueMode::No) => { + if let Some(prune) = observed_circuit_prune_commands( + &config, + old_cmd.as_ref(), + down_live_snapshot.as_ref(), + up_live_snapshot.as_ref(), + &protected_down_classes, + &protected_up_classes, + ) { + immediate_commands.extend(prune); + } + if let Some(add) = enriched_cmd.to_commands(&config, ExecutionMode::Builder) + { + immediate_commands.extend(add); + } + } + Some(LazyQueueMode::Htb) => { + if let Some(prune) = observed_circuit_prune_commands( + &config, + old_cmd.as_ref(), + down_live_snapshot.as_ref(), + up_live_snapshot.as_ref(), + &protected_down_classes, + &protected_up_classes, + ) { + immediate_commands.extend(prune); + } + if let Some(add_htb) = + enriched_cmd.to_commands(&config, ExecutionMode::Builder) + { + immediate_commands.extend(add_htb); + } + } + Some(LazyQueueMode::Full) => {} + } + migrated_updates.push((circuit_hash, enriched_cmd)); + } if !immediate_commands.is_empty() { - execute_in_memory(&immediate_commands, "updating circuit speeds (fallback)"); + let result = execute_and_record_live_change( + &immediate_commands, + "migrating circuits between parent nodes (fallback)", + ); + if !result.ok { + let summary = summarize_apply_result( + "migrating circuits between parent nodes (fallback)", + &result, + ); + mark_reload_required(format!( + "Bakery circuit parent-move fallback failed: {}. A full reload is now required before further incremental topology mutations.", + summary + )); + migrated_updates.clear(); + } + } + for (circuit_hash, enriched_cmd) in migrated_updates { + circuits.insert(circuit_hash, enriched_cmd); } } @@ -1491,18 +4744,49 @@ fn handle_commit_batch( ); maybe_emit_mapped_circuit_limit_urgent(&stats); } + log_mapped_limit_decision( + "incremental additions", + mapped_limit, + MappedLimitStats { + enforced_limit: effective_limit, + requested_mapped: requested_mapped_additions, + allowed_mapped: requested_mapped_additions + .saturating_sub(dropped_mapped_additions), + dropped_mapped: dropped_mapped_additions, + }, + ); - let commands: Vec> = accepted_additions + let Some(layout) = resolved_mq_layout.as_ref() else { + warn!( + "Bakery: missing MQ layout during incremental additions; skipping TC changes" + ); + return; + }; + let live_reserved_handles = + snapshot_live_qdisc_handle_majors_or_empty(&config, "incremental additions"); + let enriched_additions: Vec> = accepted_additions + .iter() + .map(|command| { + with_assigned_qdisc_handles_reserved( + command, + &config, + layout, + qdisc_handles, + &live_reserved_handles, + ) + }) + .collect(); + let commands: Vec> = enriched_additions .iter() .filter_map(|c| c.to_commands(&config, ExecutionMode::Builder)) .flatten() .collect(); if !commands.is_empty() { - execute_in_memory(&commands, "adding new circuits"); + execute_and_record_live_change(&commands, "adding new circuits"); } - for command in accepted_additions { + for command in enriched_additions { if let BakeryCommands::AddCircuit { circuit_hash, .. } = command.as_ref() { - circuits.insert(*circuit_hash, Arc::clone(command)); + circuits.insert(*circuit_hash, command); } } } @@ -1510,13 +4794,30 @@ fn handle_commit_batch( // 4) IP-only changes require no TC commands; mappings already handled by mapping engine // We still refresh the stored circuit snapshot for those entries if !categories.ip_changed.is_empty() { + let Some(layout) = resolved_mq_layout.as_ref() else { + warn!("Bakery: missing MQ layout during IP-only circuit updates"); + return; + }; + let live_reserved_handles = + snapshot_live_qdisc_handle_majors_or_empty(&config, "IP-only circuit updates"); for command in categories.ip_changed { - if let BakeryCommands::AddCircuit { circuit_hash, .. } = command.as_ref() { - circuits.insert(*circuit_hash, Arc::clone(command)); + let enriched = with_assigned_qdisc_handles_reserved( + command, + &config, + layout, + qdisc_handles, + &live_reserved_handles, + ); + if let BakeryCommands::AddCircuit { circuit_hash, .. } = enriched.as_ref() { + circuits.insert(*circuit_hash, enriched); } } } } + + *mq_layout = resolved_mq_layout; + qdisc_handles.save(&config); + update_queue_distribution_snapshot(sites, circuits); } fn handle_circuit_activity( @@ -1585,7 +4886,7 @@ fn handle_circuit_activity( if commands.is_empty() { return; // No commands to write } - execute_in_memory(&commands, "enabling live circuits"); + execute_and_record_live_change(&commands, "enabling live circuits"); } fn handle_tick( @@ -1677,7 +4978,7 @@ fn handle_tick( if commands.is_empty() { return; // No commands to write } - execute_in_memory(&commands, "pruning lazy queues"); + execute_and_record_live_change(&commands, "pruning lazy queues"); } fn handle_change_site_speed_live( @@ -1729,6 +5030,31 @@ fn handle_change_site_speed_live( } else { download_bandwidth_min }; + if let Some(reason) = live_tree_mutation_blocker_for_config(&config) { + let summary = format!( + "Skipping live site speed change for site {} because {}.", + site_hash, reason + ); + info!("{summary}"); + push_bakery_event_with_site( + "live_site_speed_skipped", + "info", + Some(site_hash), + summary, + ); + let new_site = Arc::new(BakeryCommands::AddSite { + site_hash, + parent_class_id: *parent_class_id, + up_parent_class_id: *up_parent_class_id, + class_minor: *class_minor, + download_bandwidth_min, + upload_bandwidth_min, + download_bandwidth_max, + upload_bandwidth_max, + }); + sites.insert(site_hash, new_site); + return; + } let commands = vec![ vec![ "class".to_string(), @@ -1742,6 +5068,13 @@ fn handle_change_site_speed_live( format_rate_for_tc_f32(upload_bandwidth_min), "ceil".to_string(), format_rate_for_tc_f32(upload_bandwidth_max), + "prio".to_string(), + "3".to_string(), + "quantum".to_string(), + quantum( + upload_bandwidth_max as u64, + r2q(config.queues.uplink_bandwidth_mbps), + ), ], vec![ "class".to_string(), @@ -1755,9 +5088,16 @@ fn handle_change_site_speed_live( format_rate_for_tc_f32(download_bandwidth_min), "ceil".to_string(), format_rate_for_tc_f32(download_bandwidth_max), + "prio".to_string(), + "3".to_string(), + "quantum".to_string(), + quantum( + download_bandwidth_max as u64, + r2q(config.queues.downlink_bandwidth_mbps), + ), ], ]; - execute_in_memory(&commands, "changing site speed live"); + execute_and_record_live_change(&commands, "changing site speed live"); // Update the site speeds in the site map - create a new Arc with updated values let new_site = Arc::new(BakeryCommands::AddSite { site_hash, @@ -1778,22 +5118,4508 @@ fn handle_change_site_speed_live( } } +fn site_class_handles(site: &BakeryCommands) -> Option<(TcHandle, TcHandle)> { + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + class_minor, + .. + } = site + else { + return None; + }; + Some(( + tc_handle_from_major_minor(parent_class_id.get_major_minor().0, *class_minor), + tc_handle_from_major_minor(up_parent_class_id.get_major_minor().0, *class_minor), + )) +} + +fn site_is_top_level(site: &BakeryCommands) -> bool { + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + .. + } = site + else { + return false; + }; + let (_, down_parent_minor) = parent_class_id.get_major_minor(); + let (_, up_parent_minor) = up_parent_class_id.get_major_minor(); + matches!(down_parent_minor, 0 | 3) && matches!(up_parent_minor, 0 | 3) +} + +fn site_runtime_virtualization_eligibility_error(site: &BakeryCommands) -> Option { + let BakeryCommands::AddSite { + site_hash, + parent_class_id, + up_parent_class_id, + .. + } = site + else { + return Some("TreeGuard runtime virtualization requires an AddSite command".to_string()); + }; + + if site_is_top_level(site) { + return Some(format!( + "Site {} is top-level in HTB and cannot be runtime-virtualized in v1", + site_hash + )); + } + + let Some((site_down_class, site_up_class)) = site_class_handles(site) else { + return Some(format!("Site {} is not a valid AddSite command", site_hash)); + }; + + let (parent_down_major, _) = parent_class_id.get_major_minor(); + let (parent_up_major, _) = up_parent_class_id.get_major_minor(); + let (site_down_major, _) = site_down_class.get_major_minor(); + let (site_up_major, _) = site_up_class.get_major_minor(); + + if parent_down_major != site_down_major || parent_up_major != site_up_major { + return Some(format!( + "Site {} crosses queue/major domains (parent down/up majors {:x}/{:x}, site down/up majors {:x}/{:x}) and cannot be runtime-virtualized in v1", + site_hash, parent_down_major, parent_up_major, site_down_major, site_up_major + )); + } + + None +} + +fn top_level_runtime_virtualization_eligibility_error( + site_hash: i64, + target_site: &BakeryCommands, + sites: &HashMap>, + circuits: &HashMap>, +) -> Option { + if !site_is_top_level(target_site) { + return Some(RuntimeNodeEligibilityError::plain(format!( + "Site {} is not top-level and cannot use the top-level runtime virtualization path", + site_hash + ))); + } + + if current_site_queue(target_site).is_none() { + return Some(RuntimeNodeEligibilityError::plain(format!( + "Site {} does not have a deterministic current queue assignment", + site_hash + ))); + } + + let children_by_parent = direct_child_sites_by_parent(sites); + let child_sites = children_by_parent + .get(&site_hash) + .map(|children| children.len()) + .unwrap_or(0); + let direct_circuits = site_class_handles(target_site) + .map(|(down, up)| collect_direct_circuit_hashes(circuits, down, up).len()) + .unwrap_or(0); + + if child_sites == 0 && direct_circuits == 0 { + return Some(RuntimeNodeEligibilityError::new( + format!( + "Site {} has no direct child sites or direct circuits to promote safely", + site_hash + ), + RuntimeNodeOperationFailureReason::StructuralIneligibleNoPromotableChildren, + )); + } + + if child_sites + direct_circuits < 2 { + return Some(RuntimeNodeEligibilityError::new( + format!( + "Site {} has only one promotable direct child, so top-level runtime virtualization would not produce a deterministic v1 split point", + site_hash + ), + RuntimeNodeOperationFailureReason::StructuralIneligibleSinglePromotableChild, + )); + } + + None +} + +#[derive(Clone, Debug)] +struct RuntimeNodeEligibilityError { + message: String, + failure_reason: Option, +} + +impl RuntimeNodeEligibilityError { + fn plain(message: String) -> Self { + Self { + message, + failure_reason: None, + } + } + + fn new(message: String, failure_reason: RuntimeNodeOperationFailureReason) -> Self { + Self { + message, + failure_reason: Some(failure_reason), + } + } +} + +fn site_prune_commands( + config: &Arc, + state: &VirtualizedSiteState, +) -> Option>> { + let mut commands = site_prune_qdisc_commands(config, &state.qdisc_handles).unwrap_or_default(); + commands.extend(site_prune_class_commands(config, state.site.as_ref())?); + Some(commands) +} + +fn site_prune_qdisc_commands( + config: &Arc, + handles: &VirtualizedSiteQdiscHandles, +) -> Option>> { + let mut commands = Vec::new(); + if let Some(handle) = handles.down { + commands.push(vec![ + "qdisc".to_string(), + "del".to_string(), + "dev".to_string(), + config.isp_interface(), + "handle".to_string(), + format!("0x{:x}:", handle), + ]); + } + if let Some(handle) = handles.up { + commands.push(vec![ + "qdisc".to_string(), + "del".to_string(), + "dev".to_string(), + config.internet_interface(), + "handle".to_string(), + format!("0x{:x}:", handle), + ]); + } + if commands.is_empty() { + None + } else { + Some(commands) + } +} + +fn site_prune_class_commands( + config: &Arc, + site: &BakeryCommands, +) -> Option>> { + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + class_minor, + .. + } = site + else { + return None; + }; + + Some(vec![ + vec![ + "class".to_string(), + "del".to_string(), + "dev".to_string(), + config.isp_interface(), + "parent".to_string(), + parent_class_id.as_tc_string(), + "classid".to_string(), + format!( + "0x{:x}:0x{:x}", + parent_class_id.get_major_minor().0, + class_minor + ), + ], + vec![ + "class".to_string(), + "del".to_string(), + "dev".to_string(), + config.internet_interface(), + "parent".to_string(), + up_parent_class_id.as_tc_string(), + "classid".to_string(), + format!( + "0x{:x}:0x{:x}", + up_parent_class_id.get_major_minor().0, + class_minor + ), + ], + ]) +} + +fn rebuild_site_command( + site: &Arc, + parent_class_id: TcHandle, + up_parent_class_id: TcHandle, + class_minor: u16, +) -> Option> { + let BakeryCommands::AddSite { + site_hash, + download_bandwidth_min, + upload_bandwidth_min, + download_bandwidth_max, + upload_bandwidth_max, + .. + } = site.as_ref() + else { + return None; + }; + + Some(Arc::new(BakeryCommands::AddSite { + site_hash: *site_hash, + parent_class_id, + up_parent_class_id, + class_minor, + download_bandwidth_min: *download_bandwidth_min, + upload_bandwidth_min: *upload_bandwidth_min, + download_bandwidth_max: *download_bandwidth_max, + upload_bandwidth_max: *upload_bandwidth_max, + })) +} + +fn reparent_circuit_command( + circuit: &Arc, + parent_class_id: TcHandle, + up_parent_class_id: TcHandle, +) -> Option> { + let BakeryCommands::AddCircuit { + circuit_hash, + circuit_name, + site_name, + class_minor, + download_bandwidth_min, + upload_bandwidth_min, + download_bandwidth_max, + upload_bandwidth_max, + class_major, + up_class_major, + ip_addresses, + sqm_override, + .. + } = circuit.as_ref() + else { + return None; + }; + + Some(Arc::new(BakeryCommands::AddCircuit { + circuit_hash: *circuit_hash, + circuit_name: circuit_name.clone(), + site_name: site_name.clone(), + parent_class_id, + up_parent_class_id, + class_minor: *class_minor, + download_bandwidth_min: *download_bandwidth_min, + upload_bandwidth_min: *upload_bandwidth_min, + download_bandwidth_max: *download_bandwidth_max, + upload_bandwidth_max: *upload_bandwidth_max, + class_major: *class_major, + up_class_major: *up_class_major, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: ip_addresses.clone(), + sqm_override: sqm_override.clone(), + })) +} + +fn collect_direct_circuit_hashes( + circuits: &HashMap>, + target_down: TcHandle, + target_up: TcHandle, +) -> Vec { + let mut hashes: Vec = circuits + .iter() + .filter_map(|(circuit_hash, circuit)| { + let BakeryCommands::AddCircuit { + parent_class_id, + up_parent_class_id, + .. + } = circuit.as_ref() + else { + return None; + }; + (*parent_class_id == target_down && *up_parent_class_id == target_up) + .then_some(*circuit_hash) + }) + .collect(); + hashes.sort_unstable(); + hashes +} + +#[derive(Clone, Debug)] +struct PlannedSiteUpdate { + queue: u32, + parent_site: Option, + stage_depth: usize, + command: Arc, +} + +#[derive(Clone, Debug)] +struct PlannedCircuitUpdate { + queue: u32, + parent_site: Option, + command: Arc, +} + +#[derive(Clone, Debug)] +struct TopLevelVirtualizationPlan { + saved_sites: HashMap>, + saved_circuits: HashMap>, + active_sites: HashMap, + active_circuits: HashMap, + site_stages: Vec>, +} + +fn site_hash_from_command(site: &BakeryCommands) -> Option { + if let BakeryCommands::AddSite { site_hash, .. } = site { + Some(*site_hash) + } else { + None + } +} + +fn circuit_hash_from_command(circuit: &BakeryCommands) -> Option { + if let BakeryCommands::AddCircuit { circuit_hash, .. } = circuit { + Some(*circuit_hash) + } else { + None + } +} + +fn site_max_weight(site: &BakeryCommands) -> f64 { + if let BakeryCommands::AddSite { + download_bandwidth_max, + upload_bandwidth_max, + .. + } = site + { + f64::from(*download_bandwidth_max + *upload_bandwidth_max) + } else { + 1.0 + } +} + +fn circuit_max_weight(circuit: &BakeryCommands) -> f64 { + if let BakeryCommands::AddCircuit { + download_bandwidth_max, + upload_bandwidth_max, + .. + } = circuit + { + f64::from(*download_bandwidth_max + *upload_bandwidth_max) + } else { + 1.0 + } +} + +fn current_site_queue(site: &BakeryCommands) -> Option { + let BakeryCommands::AddSite { + parent_class_id, + class_minor, + .. + } = site + else { + return None; + }; + let (major, _) = parent_class_id.get_major_minor(); + let class_handle = tc_handle_from_major_minor(major, *class_minor); + Some(u32::from(class_handle.get_major_minor().0)) +} + +fn current_circuit_queue(circuit: &BakeryCommands) -> Option { + let BakeryCommands::AddCircuit { class_major, .. } = circuit else { + return None; + }; + Some(u32::from(*class_major)) +} + +fn circuit_class_handles(circuit: &BakeryCommands) -> Option<(TcHandle, TcHandle)> { + let BakeryCommands::AddCircuit { + class_minor, + class_major, + up_class_major, + .. + } = circuit + else { + return None; + }; + + Some(( + tc_handle_from_major_minor(*class_major, *class_minor), + tc_handle_from_major_minor(*up_class_major, *class_minor), + )) +} + +fn rebuild_queue_distribution_snapshot( + sites: &HashMap>, + circuits: &HashMap>, +) -> Vec { + #[derive(Default)] + struct QueueBucket { + top_level_site_count: usize, + site_count: usize, + circuit_count: usize, + download_mbps: f64, + upload_mbps: f64, + } + + let mut buckets: BTreeMap = BTreeMap::new(); + + for site in sites.values() { + let Some(queue) = current_site_queue(site.as_ref()) else { + continue; + }; + let bucket = buckets.entry(queue).or_default(); + bucket.site_count += 1; + if site_is_top_level(site.as_ref()) { + bucket.top_level_site_count += 1; + } + } + + for circuit in circuits.values() { + let BakeryCommands::AddCircuit { + class_major, + download_bandwidth_max, + upload_bandwidth_max, + .. + } = circuit.as_ref() + else { + continue; + }; + let bucket = buckets.entry(u32::from(*class_major)).or_default(); + bucket.circuit_count += 1; + bucket.download_mbps += f64::from(*download_bandwidth_max); + bucket.upload_mbps += f64::from(*upload_bandwidth_max); + } + + buckets + .into_iter() + .map(|(queue, bucket)| BakeryQueueDistributionSnapshot { + queue, + top_level_site_count: bucket.top_level_site_count, + site_count: bucket.site_count, + circuit_count: bucket.circuit_count, + download_mbps: bucket.download_mbps.round().max(0.0) as u64, + upload_mbps: bucket.upload_mbps.round().max(0.0) as u64, + }) + .collect() +} + +fn update_queue_distribution_snapshot( + sites: &HashMap>, + circuits: &HashMap>, +) { + telemetry_state().write().queue_distribution = + rebuild_queue_distribution_snapshot(sites, circuits); +} + +fn rebuild_runtime_operations_snapshot( + runtime_node_operations: &HashMap, +) -> BakeryRuntimeOperationsSnapshot { + let mut submitted_count = 0usize; + let mut deferred_count = 0usize; + let mut applying_count = 0usize; + let mut awaiting_cleanup_count = 0usize; + let mut failed_count = 0usize; + let mut dirty_count = 0usize; + + let latest = runtime_node_operations + .values() + .inspect(|operation| match operation.status { + RuntimeNodeOperationStatus::Submitted => submitted_count += 1, + RuntimeNodeOperationStatus::Deferred => deferred_count += 1, + RuntimeNodeOperationStatus::Applying => applying_count += 1, + RuntimeNodeOperationStatus::AppliedAwaitingCleanup => awaiting_cleanup_count += 1, + RuntimeNodeOperationStatus::Failed => failed_count += 1, + RuntimeNodeOperationStatus::Dirty => dirty_count += 1, + RuntimeNodeOperationStatus::Completed => {} + }) + .max_by_key(|operation| operation.updated_at_unix) + .map(|operation| BakeryRuntimeOperationHeadlineSnapshot { + operation_id: operation.operation_id, + site_hash: operation.site_hash, + site_name: operation.site_name.clone(), + action: operation.action, + status: operation.status, + attempt_count: operation.attempt_count, + updated_at_unix: operation.updated_at_unix, + next_retry_at_unix: operation.next_retry_at_unix, + last_error: operation.last_error.clone(), + }); + + BakeryRuntimeOperationsSnapshot { + submitted_count, + deferred_count, + applying_count, + awaiting_cleanup_count, + failed_count, + dirty_count, + latest, + } +} + +fn observed_circuit_prune_commands( + config: &Arc, + old_cmd: &BakeryCommands, + down_snapshot: Option<&HashMap>, + up_snapshot: Option<&HashMap>, + protected_down_classes: &HashSet, + protected_up_classes: &HashSet, +) -> Option>> { + let BakeryCommands::AddCircuit { + parent_class_id, + up_parent_class_id, + class_minor, + class_major, + up_class_major, + sqm_override, + .. + } = old_cmd + else { + return old_cmd.to_prune(config, true); + }; + + let down_class = TcHandle::from_u32((u32::from(*class_major) << 16) | u32::from(*class_minor)); + let up_class = TcHandle::from_u32((u32::from(*up_class_major) << 16) | u32::from(*class_minor)); + + let (down_override_opt, up_override_opt) = match sqm_override.as_ref() { + None => (None, None), + Some(s) if s.contains('/') => { + let mut it = s.splitn(2, '/'); + let down = it.next().unwrap_or("").trim(); + let up = it.next().unwrap_or("").trim(); + let map = |t: &str| -> Option { + if t.is_empty() { + None + } else { + Some(t.to_string()) + } + }; + (map(down), map(up)) + } + Some(s) => (Some(s.clone()), Some(s.clone())), + }; + + let prune_down_qdisc = + !matches!(down_override_opt.as_deref(), Some(s) if s.eq_ignore_ascii_case("none")); + let prune_up_qdisc = + !matches!(up_override_opt.as_deref(), Some(s) if s.eq_ignore_ascii_case("none")); + + let mut result = Vec::new(); + + if !protected_up_classes.contains(&up_class) + && let Some(snapshot) = up_snapshot + && let Some(entry) = snapshot.get(&up_class) + { + if prune_up_qdisc && !config.on_a_stick_mode() && entry.leaf_qdisc_major.is_some() { + result.push(vec![ + "qdisc".to_string(), + "del".to_string(), + "dev".to_string(), + config.internet_interface(), + "parent".to_string(), + format!("0x{:x}:0x{:x}", up_class_major, class_minor), + ]); + } + result.push(vec![ + "class".to_string(), + "del".to_string(), + "dev".to_string(), + config.internet_interface(), + "parent".to_string(), + up_parent_class_id.as_tc_string(), + "classid".to_string(), + format!( + "0x{:x}:0x{:x}", + up_parent_class_id.get_major_minor().0, + class_minor + ), + ]); + } + + if !protected_down_classes.contains(&down_class) + && let Some(snapshot) = down_snapshot + && let Some(entry) = snapshot.get(&down_class) + { + if prune_down_qdisc && entry.leaf_qdisc_major.is_some() { + result.push(vec![ + "qdisc".to_string(), + "del".to_string(), + "dev".to_string(), + config.isp_interface(), + "parent".to_string(), + format!("0x{:x}:0x{:x}", class_major, class_minor), + ]); + } + result.push(vec![ + "class".to_string(), + "del".to_string(), + "dev".to_string(), + config.isp_interface(), + "parent".to_string(), + parent_class_id.as_tc_string(), + "classid".to_string(), + format!( + "0x{:x}:0x{:x}", + parent_class_id.get_major_minor().0, + class_minor + ), + ]); + } + + if result.is_empty() { + old_cmd.to_prune(config, true).map(|_| Vec::new()) + } else { + Some(result) + } +} + +fn site_parent_hash( + site_hash: i64, + sites: &HashMap>, + class_to_site: &HashMap<(TcHandle, TcHandle), i64>, +) -> Option { + let site = sites.get(&site_hash)?; + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + .. + } = site.as_ref() + else { + return None; + }; + class_to_site + .get(&(*parent_class_id, *up_parent_class_id)) + .copied() +} + +fn direct_child_sites_by_parent( + sites: &HashMap>, +) -> HashMap> { + let mut class_to_site = HashMap::new(); + for (site_hash, site) in sites { + if let Some(handles) = site_class_handles(site.as_ref()) { + class_to_site.insert(handles, *site_hash); + } + } + + let mut result: HashMap> = HashMap::new(); + for site_hash in sites.keys().copied() { + if let Some(parent_hash) = site_parent_hash(site_hash, sites, &class_to_site) { + result.entry(parent_hash).or_default().push(site_hash); + } + } + for children in result.values_mut() { + children.sort_unstable(); + } + result +} + +fn collect_site_subtree_hashes( + root_hash: i64, + children_by_parent: &HashMap>, +) -> Vec { + let mut ordered = Vec::new(); + let mut stack = vec![root_hash]; + while let Some(site_hash) = stack.pop() { + ordered.push(site_hash); + if let Some(children) = children_by_parent.get(&site_hash) { + for child in children.iter().rev() { + stack.push(*child); + } + } + } + ordered +} + +fn collect_circuits_attached_to_sites( + circuits: &HashMap>, + sites: &HashMap>, + site_hashes: &[i64], +) -> Vec { + let mut site_handles = HashSet::new(); + for site_hash in site_hashes { + if let Some(site) = sites.get(site_hash) + && let Some(handles) = site_class_handles(site.as_ref()) + { + site_handles.insert(handles); + } + } + + let mut result: Vec = circuits + .iter() + .filter_map(|(circuit_hash, circuit)| { + let BakeryCommands::AddCircuit { + parent_class_id, + up_parent_class_id, + .. + } = circuit.as_ref() + else { + return None; + }; + site_handles + .contains(&(*parent_class_id, *up_parent_class_id)) + .then_some(*circuit_hash) + }) + .collect(); + result.sort_unstable(); + result +} + +fn root_handle_for_queue(queue: u32) -> TcHandle { + TcHandle::from_u32(queue << 16) +} + +fn site_stick_offset(site: &BakeryCommands) -> u16 { + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + .. + } = site + else { + return 0; + }; + let (down_major, _) = parent_class_id.get_major_minor(); + let (up_major, _) = up_parent_class_id.get_major_minor(); + up_major.saturating_sub(down_major) +} + +fn top_level_bin_name(queue: u32) -> String { + format!("CpueQueue{}", queue.saturating_sub(1)) +} + +fn queue_from_bin_name(name: &str) -> Option { + name.strip_prefix("CpueQueue") + .and_then(|v| v.parse::().ok()) + .map(|cpu| cpu + 1) +} + +fn planner_site_key(site_hash: i64) -> String { + format!("site:{site_hash}") +} + +fn planner_circuit_key(circuit_hash: i64) -> String { + format!("circuit:{circuit_hash}") +} + +fn build_current_planner_state( + sites: &HashMap>, + circuits: &HashMap>, +) -> ( + BTreeMap, + BTreeMap, +) { + let mut class_to_site = HashMap::new(); + for (site_hash, site) in sites { + if let Some(handles) = site_class_handles(site.as_ref()) { + class_to_site.insert(handles, *site_hash); + } + } + + let mut previous_sites = BTreeMap::new(); + for (site_hash, site) in sites { + let Some(queue) = current_site_queue(site.as_ref()) else { + continue; + }; + let Some(class_minor) = (if let BakeryCommands::AddSite { class_minor, .. } = site.as_ref() + { + Some(*class_minor) + } else { + None + }) else { + continue; + }; + let parent_path = site_parent_hash(*site_hash, sites, &class_to_site) + .map(planner_site_key) + .unwrap_or_default(); + previous_sites.insert( + planner_site_key(*site_hash), + PlannerSiteIdentityState { + class_minor, + queue, + parent_path, + class_major: queue as u16, + up_class_major: 0, + }, + ); + } + + let mut previous_circuits = BTreeMap::new(); + for (circuit_hash, circuit) in circuits { + let BakeryCommands::AddCircuit { + class_minor, + class_major, + up_class_major, + parent_class_id, + up_parent_class_id, + .. + } = circuit.as_ref() + else { + continue; + }; + let parent_node = class_to_site + .get(&(*parent_class_id, *up_parent_class_id)) + .map(|site_hash| planner_site_key(*site_hash)) + .unwrap_or_else(|| format!("root:{}", class_major)); + previous_circuits.insert( + planner_circuit_key(*circuit_hash), + PlannerCircuitIdentityState { + class_minor: *class_minor, + queue: u32::from(*class_major), + parent_node, + class_major: *class_major, + up_class_major: *up_class_major, + }, + ); + } + + (previous_sites, previous_circuits) +} + +fn build_effective_runtime_state( + sites: &HashMap>, + circuits: &HashMap>, + virtualized_sites: &HashMap, +) -> ( + HashMap>, + HashMap>, +) { + if virtualized_sites.is_empty() { + return (sites.clone(), circuits.clone()); + } + + let mut effective_sites = sites.clone(); + let mut effective_circuits = circuits.clone(); + + for (site_hash, state) in virtualized_sites { + if state.active_branch_hides_original_site() { + effective_sites.remove(site_hash); + } + for (hash, command) in &state.active_sites { + effective_sites.insert(*hash, Arc::clone(command)); + } + for (hash, command) in &state.active_circuits { + effective_circuits.insert(*hash, Arc::clone(command)); + } + } + + (effective_sites, effective_circuits) +} + +fn reserve_live_snapshot_minors( + reservations: &mut PlannerMinorReservations, + snapshot: &HashMap, +) { + for handle in snapshot.keys() { + let (major, minor) = handle.get_major_minor(); + if major == 0 || minor == 0 { + continue; + } + reservations + .entry(u32::from(major)) + .or_default() + .insert(u32::from(minor)); + } +} + +fn top_level_site_hashes(sites: &HashMap>) -> Vec { + let mut hashes: Vec = sites + .iter() + .filter_map(|(site_hash, site)| site_is_top_level(site.as_ref()).then_some(*site_hash)) + .collect(); + hashes.sort_unstable(); + hashes +} + +fn site_descendant_depth(site_hash: i64, parents: &HashMap>) -> usize { + let mut depth = 0usize; + let mut current = parents.get(&site_hash).copied().flatten(); + while let Some(parent) = current { + depth += 1; + current = parents.get(&parent).copied().flatten(); + } + depth +} + +fn build_top_level_virtualization_plan( + target_site: Arc, + sites: &HashMap>, + circuits: &HashMap>, + config: Option<&Arc>, + virtualized_sites: &HashMap, + stick_offset: u16, +) -> Result { + let Some(target_site_hash) = site_hash_from_command(target_site.as_ref()) else { + return Err("TreeGuard top-level virtualization target is not a site".to_string()); + }; + if !site_is_top_level(target_site.as_ref()) { + return Err(format!( + "Site {} is not top-level and should use the v1 same-queue runtime virtualization path", + target_site_hash + )); + } + + let children_by_parent = direct_child_sites_by_parent(sites); + let Some((target_down_class, target_up_class)) = site_class_handles(target_site.as_ref()) + else { + return Err(format!( + "Site {} is not a valid AddSite command", + target_site_hash + )); + }; + + let promoted_site_roots = children_by_parent + .get(&target_site_hash) + .cloned() + .unwrap_or_default(); + let direct_circuits = + collect_direct_circuit_hashes(circuits, target_down_class, target_up_class); + let top_level_sites = top_level_site_hashes(sites); + + let bins: Vec = top_level_sites + .iter() + .filter_map(|site_hash| { + sites + .get(site_hash) + .and_then(|site| current_site_queue(site.as_ref())) + .map(top_level_bin_name) + }) + .collect::>() + .into_iter() + .collect(); + if bins.is_empty() { + return Err( + "No top-level queues available for runtime top-level virtualization".to_string(), + ); + } + + let target_queue = current_site_queue(target_site.as_ref()).unwrap_or(1); + let mut planner_items = Vec::new(); + let mut prev_assign = BTreeMap::new(); + for site_hash in &promoted_site_roots { + let Some(site) = sites.get(site_hash) else { + continue; + }; + let id = planner_site_key(*site_hash); + planner_items.push(TopLevelPlannerItem { + id: id.clone(), + weight: site_max_weight(site.as_ref()), + }); + prev_assign.insert(id, top_level_bin_name(target_queue)); + } + for circuit_hash in &direct_circuits { + let Some(circuit) = circuits.get(circuit_hash) else { + continue; + }; + let id = planner_circuit_key(*circuit_hash); + planner_items.push(TopLevelPlannerItem { + id: id.clone(), + weight: circuit_max_weight(circuit.as_ref()), + }); + prev_assign.insert(id, top_level_bin_name(target_queue)); + } + + let planner = plan_top_level_assignments( + &planner_items, + &bins, + &prev_assign, + &BTreeMap::new(), + current_timestamp() as f64, + &TopLevelPlannerParams { + mode: TopLevelPlannerMode::StableGreedy, + move_budget_per_run: planner_items.len().clamp(1, 32), + ..TopLevelPlannerParams::default() + }, + ); + + let mut future_top_level_queue = HashMap::new(); + for site_hash in &top_level_sites { + if *site_hash == target_site_hash { + continue; + } + let Some(site) = sites.get(site_hash) else { + continue; + }; + let Some(queue) = current_site_queue(site.as_ref()) else { + continue; + }; + future_top_level_queue.insert(*site_hash, queue); + } + for site_hash in &promoted_site_roots { + let key = planner_site_key(*site_hash); + let assigned = planner + .assignment + .get(&key) + .or_else(|| prev_assign.get(&key)) + .ok_or_else(|| { + format!( + "Planner did not assign promoted top-level site {}", + site_hash + ) + })?; + let queue = queue_from_bin_name(assigned) + .ok_or_else(|| format!("Invalid planner queue assignment {}", assigned))?; + future_top_level_queue.insert(*site_hash, queue); + } + + let mut moved_top_level_roots = promoted_site_roots.clone(); + moved_top_level_roots.sort_unstable(); + moved_top_level_roots.dedup(); + + let mut saved_sites = HashMap::new(); + let mut saved_circuits = HashMap::new(); + let mut affected_site_hashes = HashSet::new(); + for site_hash in &moved_top_level_roots { + for hash in collect_site_subtree_hashes(*site_hash, &children_by_parent) { + affected_site_hashes.insert(hash); + } + } + for site_hash in &affected_site_hashes { + if let Some(site) = sites.get(site_hash) { + saved_sites.insert(*site_hash, Arc::clone(site)); + } + } + let affected_site_hashes_vec: Vec = affected_site_hashes.iter().copied().collect(); + for circuit_hash in + collect_circuits_attached_to_sites(circuits, sites, &affected_site_hashes_vec) + { + if let Some(circuit) = circuits.get(&circuit_hash) { + saved_circuits.insert(circuit_hash, Arc::clone(circuit)); + } + } + for circuit_hash in &direct_circuits { + if let Some(circuit) = circuits.get(circuit_hash) { + saved_circuits.insert(*circuit_hash, Arc::clone(circuit)); + } + } + + let (effective_sites, effective_circuits) = + build_effective_runtime_state(sites, circuits, virtualized_sites); + let (previous_sites, previous_circuits) = + build_current_planner_state(&effective_sites, &effective_circuits); + + let mut future_parent_by_site: HashMap> = HashMap::new(); + for site_hash in future_top_level_queue.keys() { + future_parent_by_site.insert(*site_hash, None); + } + for site_hash in &affected_site_hashes { + if future_parent_by_site.contains_key(site_hash) { + continue; + } + let parent_hash = site_parent_hash(*site_hash, sites, &{ + let mut class_to_site = HashMap::new(); + for (hash, site) in sites { + if let Some(handles) = site_class_handles(site.as_ref()) { + class_to_site.insert(handles, *hash); + } + } + class_to_site + }); + future_parent_by_site.insert(*site_hash, parent_hash); + } + + let mut planner_site_inputs = Vec::new(); + let mut planner_site_hashes: Vec = affected_site_hashes + .iter() + .copied() + .filter(|hash| *hash != target_site_hash) + .collect(); + planner_site_hashes + .sort_by_key(|hash| (site_descendant_depth(*hash, &future_parent_by_site), *hash)); + for site_hash in &planner_site_hashes { + let Some(site) = sites.get(site_hash) else { + continue; + }; + let queue = if let Some(root_queue) = future_top_level_queue.get(site_hash) { + *root_queue + } else { + let mut cursor = *site_hash; + loop { + let Some(parent) = future_parent_by_site.get(&cursor).copied().flatten() else { + break future_top_level_queue.get(&cursor).copied().unwrap_or(1); + }; + cursor = parent; + } + }; + let parent_path = future_parent_by_site + .get(site_hash) + .copied() + .flatten() + .map(planner_site_key) + .unwrap_or_default(); + let has_children = children_by_parent + .get(site_hash) + .map(|children| !children.is_empty()) + .unwrap_or(false); + planner_site_inputs.push(SiteIdentityInput { + site_key: planner_site_key(*site_hash), + parent_path, + queue, + has_children, + }); + let _ = site; + } + + let class_to_site = { + let mut m = HashMap::new(); + for (hash, site) in sites { + if let Some(handles) = site_class_handles(site.as_ref()) { + m.insert(handles, *hash); + } + } + m + }; + + let mut circuit_ids_by_parent: BTreeMap> = BTreeMap::new(); + for (circuit_hash, circuit) in &saved_circuits { + let BakeryCommands::AddCircuit { + parent_class_id, + up_parent_class_id, + .. + } = circuit.as_ref() + else { + continue; + }; + if *parent_class_id == target_down_class && *up_parent_class_id == target_up_class { + continue; + } + let parent = class_to_site + .get(&(*parent_class_id, *up_parent_class_id)) + .map(|site_hash| planner_site_key(*site_hash)) + .unwrap_or_else(|| { + format!( + "root:{}", + current_circuit_queue(circuit.as_ref()).unwrap_or(1) + ) + }); + circuit_ids_by_parent + .entry(parent) + .or_default() + .push(planner_circuit_key(*circuit_hash)); + } + for circuit_hash in &direct_circuits { + let id = planner_circuit_key(*circuit_hash); + let assigned = planner + .assignment + .get(&id) + .or_else(|| prev_assign.get(&id)) + .ok_or_else(|| format!("Planner did not assign direct circuit {}", circuit_hash))?; + let queue = queue_from_bin_name(assigned) + .ok_or_else(|| format!("Invalid planner queue assignment {}", assigned))?; + circuit_ids_by_parent + .entry(format!("root:{queue}")) + .or_default() + .push(id); + } + + let mut planner_circuit_groups = Vec::new(); + for (parent_node, mut circuit_ids) in circuit_ids_by_parent { + circuit_ids.sort(); + let queue = if let Some(root) = parent_node.strip_prefix("root:") { + root.parse::().unwrap_or(1) + } else if let Some(site_hash) = parent_node + .strip_prefix("site:") + .and_then(|s| s.parse::().ok()) + { + planner_site_inputs + .iter() + .find(|site| site.site_key == planner_site_key(site_hash)) + .map(|site| site.queue) + .unwrap_or(1) + } else { + 1 + }; + planner_circuit_groups.push(CircuitIdentityGroupInput { + parent_node, + queue, + circuit_ids, + }); + } + + let (mut reserved_site_minors, mut reserved_circuit_minors) = build_class_identity_reservations( + &planner_site_inputs, + &planner_circuit_groups, + &previous_sites, + &previous_circuits, + ); + if let Some(config) = config { + let down_snapshot = read_live_class_snapshot(&config.isp_interface())?; + let up_snapshot = read_live_class_snapshot(&config.internet_interface())?; + reserve_live_snapshot_minors(&mut reserved_site_minors, &down_snapshot); + reserve_live_snapshot_minors(&mut reserved_site_minors, &up_snapshot); + reserve_live_snapshot_minors(&mut reserved_circuit_minors, &down_snapshot); + reserve_live_snapshot_minors(&mut reserved_circuit_minors, &up_snapshot); + } + let constraints = ClassIdentityPlannerConstraints { + reserved_site_minors, + reserved_circuit_minors, + site_minor_start: ACTIVE_RUNTIME_MINOR_START, + circuit_minor_start: ACTIVE_RUNTIME_MINOR_START, + stick_offset, + circuit_padding: 0, + }; + let identity = plan_class_identities_with_constraints( + &planner_site_inputs, + &planner_circuit_groups, + &previous_sites, + &previous_circuits, + &constraints, + ); + let site_identity_by_key: HashMap = identity + .sites + .iter() + .map(|entry| (entry.site_key.clone(), entry.clone())) + .collect(); + let circuit_identity_by_key: HashMap = identity + .circuits + .iter() + .map(|entry| (entry.circuit_id.clone(), entry.clone())) + .collect(); + + let mut active_sites = HashMap::new(); + for site_hash in &planner_site_hashes { + let Some(old_site) = sites.get(site_hash) else { + continue; + }; + let site_key = planner_site_key(*site_hash); + let Some(identity_entry) = site_identity_by_key.get(&site_key) else { + continue; + }; + let parent_handles = future_parent_by_site + .get(site_hash) + .copied() + .flatten() + .and_then(|parent_hash| { + site_identity_by_key + .get(&planner_site_key(parent_hash)) + .map(|parent_identity| { + ( + tc_handle_from_major_minor( + parent_identity.class_major, + parent_identity.class_minor, + ), + tc_handle_from_major_minor( + parent_identity.up_class_major, + parent_identity.class_minor, + ), + ) + }) + }) + .unwrap_or_else(|| { + ( + root_handle_for_queue(identity_entry.queue), + root_handle_for_queue(u32::from(identity_entry.up_class_major)), + ) + }); + let BakeryCommands::AddSite { + site_hash, + download_bandwidth_min, + upload_bandwidth_min, + download_bandwidth_max, + upload_bandwidth_max, + .. + } = old_site.as_ref() + else { + continue; + }; + active_sites.insert( + *site_hash, + PlannedSiteUpdate { + queue: identity_entry.queue, + parent_site: future_parent_by_site.get(site_hash).copied().flatten(), + stage_depth: site_descendant_depth(*site_hash, &future_parent_by_site), + command: Arc::new(BakeryCommands::AddSite { + site_hash: *site_hash, + parent_class_id: parent_handles.0, + up_parent_class_id: parent_handles.1, + class_minor: identity_entry.class_minor, + download_bandwidth_min: *download_bandwidth_min, + upload_bandwidth_min: *upload_bandwidth_min, + download_bandwidth_max: *download_bandwidth_max, + upload_bandwidth_max: *upload_bandwidth_max, + }), + }, + ); + } + + active_sites.retain(|hash, _| saved_sites.contains_key(hash)); + + let mut active_circuits = HashMap::new(); + for (circuit_hash, circuit) in &saved_circuits { + let circuit_key = planner_circuit_key(*circuit_hash); + let Some(identity_entry) = circuit_identity_by_key.get(&circuit_key) else { + continue; + }; + let parent_site = previous_circuits + .get(&circuit_key) + .and_then(|prev| prev.parent_node.strip_prefix("site:")) + .and_then(|s| s.parse::().ok()); + let future_parent_site = if direct_circuits.contains(circuit_hash) { + None + } else { + class_to_site + .get(&{ + let BakeryCommands::AddCircuit { + parent_class_id, + up_parent_class_id, + .. + } = circuit.as_ref() + else { + continue; + }; + (*parent_class_id, *up_parent_class_id) + }) + .copied() + }; + let parent_handles = if let Some(parent_site_hash) = future_parent_site { + if let Some(parent_identity) = + site_identity_by_key.get(&planner_site_key(parent_site_hash)) + { + ( + tc_handle_from_major_minor( + parent_identity.class_major, + parent_identity.class_minor, + ), + tc_handle_from_major_minor( + parent_identity.up_class_major, + parent_identity.class_minor, + ), + ) + } else { + ( + root_handle_for_queue(identity_entry.queue), + root_handle_for_queue(u32::from(identity_entry.up_class_major)), + ) + } + } else { + ( + root_handle_for_queue(identity_entry.queue), + root_handle_for_queue(u32::from(identity_entry.up_class_major)), + ) + }; + let BakeryCommands::AddCircuit { + circuit_hash, + circuit_name, + site_name, + download_bandwidth_min, + upload_bandwidth_min, + download_bandwidth_max, + upload_bandwidth_max, + ip_addresses, + sqm_override, + .. + } = circuit.as_ref() + else { + continue; + }; + active_circuits.insert( + *circuit_hash, + PlannedCircuitUpdate { + queue: identity_entry.queue, + parent_site: future_parent_site.or(parent_site), + command: Arc::new(BakeryCommands::AddCircuit { + circuit_hash: *circuit_hash, + circuit_name: circuit_name.clone(), + site_name: site_name.clone(), + parent_class_id: parent_handles.0, + up_parent_class_id: parent_handles.1, + class_minor: identity_entry.class_minor, + download_bandwidth_min: *download_bandwidth_min, + upload_bandwidth_min: *upload_bandwidth_min, + download_bandwidth_max: *download_bandwidth_max, + upload_bandwidth_max: *upload_bandwidth_max, + class_major: identity_entry.class_major, + up_class_major: identity_entry.up_class_major, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: ip_addresses.clone(), + sqm_override: sqm_override.clone(), + }), + }, + ); + } + let mut site_stages_map: BTreeMap> = BTreeMap::new(); + for site_hash in planner_site_hashes + .into_iter() + .filter(|hash| active_sites.contains_key(hash)) + { + let update = active_sites + .get(&site_hash) + .expect("active top-level site update"); + site_stages_map + .entry(update.stage_depth) + .or_default() + .push(site_hash); + } + let site_stages = site_stages_map.into_values().collect(); + + Ok(TopLevelVirtualizationPlan { + saved_sites, + saved_circuits, + active_sites, + active_circuits, + site_stages, + }) +} + +fn build_non_top_level_virtualization_plan( + target_site: Arc, + sites: &HashMap>, + circuits: &HashMap>, +) -> Result { + let site_hash = site_hash_from_command(target_site.as_ref()) + .ok_or_else(|| "TreeGuard runtime virtualization requires an AddSite target".to_string())?; + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + .. + } = target_site.as_ref() + else { + return Err("TreeGuard runtime virtualization requires an AddSite target".to_string()); + }; + + let children_by_parent = direct_child_sites_by_parent(sites); + let subtree_sites = collect_site_subtree_hashes(site_hash, &children_by_parent); + let descendant_sites: Vec = subtree_sites + .iter() + .copied() + .filter(|hash| *hash != site_hash) + .collect(); + let subtree_circuits = collect_circuits_attached_to_sites(circuits, sites, &subtree_sites); + + let saved_sites: HashMap> = descendant_sites + .iter() + .filter_map(|hash| sites.get(hash).cloned().map(|command| (*hash, command))) + .collect(); + let saved_circuits: HashMap> = subtree_circuits + .iter() + .filter_map(|hash| circuits.get(hash).cloned().map(|command| (*hash, command))) + .collect(); + + let class_to_site = { + let mut m = HashMap::new(); + for (hash, site) in sites { + if let Some(handles) = site_class_handles(site.as_ref()) { + m.insert(handles, *hash); + } + } + m + }; + + let mut site_parents = HashMap::new(); + for desc_hash in &descendant_sites { + site_parents.insert( + *desc_hash, + site_parent_hash(*desc_hash, sites, &class_to_site), + ); + } + + let mut ordered_sites = descendant_sites; + ordered_sites.sort_by_key(|hash| (site_descendant_depth(*hash, &site_parents), *hash)); + + let mut active_sites = HashMap::new(); + let mut stage_depth_by_site: HashMap = HashMap::new(); + let mut shadow_handles_by_site: HashMap = HashMap::new(); + for child_hash in &ordered_sites { + let Some(child_site) = sites.get(child_hash).cloned() else { + continue; + }; + let parent_hash = site_parents.get(child_hash).copied().flatten(); + let (new_parent_down, new_parent_up, planner_parent_site, stage_depth) = match parent_hash { + Some(parent) if parent == site_hash => { + (*parent_class_id, *up_parent_class_id, None, 0usize) + } + Some(parent) => { + let Some(handles) = shadow_handles_by_site.get(&parent).copied() else { + return Err(format!( + "Missing shadow parent handles while planning runtime virtualization for child site {}", + child_hash + )); + }; + let parent_depth = stage_depth_by_site.get(&parent).copied().ok_or_else(|| { + format!( + "Missing shadow stage depth while planning runtime virtualization for child site {}", + child_hash + ) + })?; + (handles.0, handles.1, Some(parent), parent_depth + 1) + } + None => (*parent_class_id, *up_parent_class_id, None, 0usize), + }; + let Some(shadow_minor) = find_free_site_shadow_minor( + sites, + circuits, + &HashMap::new(), + &active_sites, + &HashMap::new(), + &new_parent_down, + &new_parent_up, + ) else { + return Err(format!( + "Unable to allocate runtime shadow site class for child site {}", + child_hash + )); + }; + let Some(shadow_site) = + rebuild_site_command(&child_site, new_parent_down, new_parent_up, shadow_minor) + else { + continue; + }; + let Some(shadow_handles) = site_class_handles(shadow_site.as_ref()) else { + return Err(format!( + "Failed to derive shadow handles for runtime child site {}", + child_hash + )); + }; + stage_depth_by_site.insert(*child_hash, stage_depth); + shadow_handles_by_site.insert(*child_hash, shadow_handles); + active_sites.insert( + *child_hash, + PlannedSiteUpdate { + queue: current_site_queue(shadow_site.as_ref()).unwrap_or(1), + parent_site: planner_parent_site, + stage_depth, + command: shadow_site, + }, + ); + } + + let mut site_stages_map: BTreeMap> = BTreeMap::new(); + for (site_hash, update) in &active_sites { + site_stages_map + .entry(update.stage_depth) + .or_default() + .push(*site_hash); + } + let mut site_stages: Vec> = site_stages_map + .into_values() + .map(|mut hashes| { + hashes.sort_by_key(|hash| { + let update = active_sites.get(hash).expect("planned site update"); + (update.queue, update.parent_site.unwrap_or_default(), *hash) + }); + hashes + }) + .collect(); + if site_stages.is_empty() && !active_sites.is_empty() { + site_stages.push(active_sites.keys().copied().collect()); + } + + let mut active_circuits = HashMap::new(); + for circuit_hash in &subtree_circuits { + let Some(old_circuit) = circuits.get(circuit_hash).cloned() else { + continue; + }; + let BakeryCommands::AddCircuit { + parent_class_id: old_parent_down, + up_parent_class_id: old_parent_up, + .. + } = old_circuit.as_ref() + else { + continue; + }; + let Some(current_parent_site_hash) = class_to_site + .get(&(*old_parent_down, *old_parent_up)) + .copied() + else { + return Err(format!( + "Unable to identify current parent site for circuit {} during runtime virtualization", + circuit_hash + )); + }; + let (new_parent_down, new_parent_up, planner_parent_site) = if current_parent_site_hash + == site_hash + { + (*parent_class_id, *up_parent_class_id, None) + } else { + let Some(handles) = shadow_handles_by_site + .get(¤t_parent_site_hash) + .copied() + else { + return Err(format!( + "Missing shadow parent handles for circuit {} during runtime virtualization", + circuit_hash + )); + }; + (handles.0, handles.1, Some(current_parent_site_hash)) + }; + let Some(updated_circuit) = + reparent_circuit_command(&old_circuit, new_parent_down, new_parent_up) + else { + continue; + }; + active_circuits.insert( + *circuit_hash, + PlannedCircuitUpdate { + queue: current_circuit_queue(updated_circuit.as_ref()).unwrap_or(1), + parent_site: planner_parent_site, + command: updated_circuit, + }, + ); + } + + Ok(TopLevelVirtualizationPlan { + saved_sites, + saved_circuits, + active_sites, + active_circuits, + site_stages, + }) +} + +fn site_has_observed_child_classes( + site: &BakeryCommands, + down_snapshot: &HashMap, + up_snapshot: &HashMap, +) -> bool { + let Some((down_class, up_class)) = site_class_handles(site) else { + return false; + }; + + down_snapshot + .values() + .any(|entry| entry.parent == Some(down_class)) + || up_snapshot + .values() + .any(|entry| entry.parent == Some(up_class)) +} + +fn verify_site_updates_live( + config: &Arc, + updates: &HashMap, +) -> Result<(), String> { + if updates.is_empty() { + return Ok(()); + } + + let down_snapshot = read_live_class_snapshot(&config.isp_interface())?; + let up_snapshot = read_live_class_snapshot(&config.internet_interface())?; + + for (site_hash, update) in updates { + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + class_minor, + .. + } = update.command.as_ref() + else { + continue; + }; + let Some((down_class, up_class)) = site_class_handles(update.command.as_ref()) else { + continue; + }; + let Some(down_entry) = down_snapshot.get(&down_class) else { + return Err(format!( + "Runtime child site {} was not observed on {} after shadow create", + site_hash, + config.isp_interface() + )); + }; + let Some(up_entry) = up_snapshot.get(&up_class) else { + return Err(format!( + "Runtime child site {} was not observed on {} after shadow create", + site_hash, + config.internet_interface() + )); + }; + if !live_parent_matches(down_entry.parent, *parent_class_id) + || !live_parent_matches(up_entry.parent, *up_parent_class_id) + { + return Err(format!( + "Runtime child site {} shadow verify failed: expected parents {}/{} for class minor 0x{:x}, observed parents {:?}/{:?}", + site_hash, + parent_class_id.as_tc_string(), + up_parent_class_id.as_tc_string(), + class_minor, + down_entry.parent, + up_entry.parent + )); + } + } + + Ok(()) +} + +fn ordered_prune_site_hashes(prune_sites: &HashMap>) -> Vec { + let mut class_to_site = HashMap::new(); + for (site_hash, site) in prune_sites { + if let Some(handles) = site_class_handles(site.as_ref()) { + class_to_site.insert(handles, *site_hash); + } + } + + let mut parents = HashMap::new(); + for site_hash in prune_sites.keys().copied() { + parents.insert( + site_hash, + site_parent_hash(site_hash, prune_sites, &class_to_site), + ); + } + + let mut ordered: Vec = prune_sites.keys().copied().collect(); + ordered.sort_by_key(|site_hash| { + let depth = site_descendant_depth(*site_hash, &parents); + (std::cmp::Reverse(depth), *site_hash) + }); + ordered +} + +fn ordered_prune_circuit_hashes(prune_circuits: &HashMap>) -> Vec { + let mut ordered: Vec = prune_circuits.keys().copied().collect(); + ordered.sort_unstable(); + ordered +} + +fn apply_site_command_updates( + config: &Arc, + sites: &mut HashMap>, + updates: &HashMap, + action_label: &str, +) -> Result<(), String> { + let mut single_stage: Vec = updates.keys().copied().collect(); + single_stage.sort_unstable(); + apply_site_command_update_stages(config, sites, updates, &[single_stage], action_label, false) +} + +fn apply_site_command_update_stages( + config: &Arc, + sites: &mut HashMap>, + updates: &HashMap, + site_stages: &[Vec], + action_label: &str, + verify_each_stage: bool, +) -> Result<(), String> { + if updates.is_empty() { + return Ok(()); + } + + let mut remaining: BTreeSet = updates.keys().copied().collect(); + let mut effective_stages: Vec> = site_stages + .iter() + .filter_map(|stage| { + let filtered: Vec = stage + .iter() + .copied() + .filter(|site_hash| updates.contains_key(site_hash)) + .collect(); + (!filtered.is_empty()).then_some(filtered) + }) + .collect(); + if effective_stages.is_empty() { + effective_stages.push(remaining.iter().copied().collect()); + } + + for (stage_index, stage_hashes) in effective_stages.iter().enumerate() { + let mut ordered: Vec<_> = stage_hashes + .iter() + .filter_map(|site_hash| updates.get(site_hash).map(|update| (site_hash, update))) + .collect(); + ordered.sort_by_key(|(site_hash, update)| { + ( + update.parent_site.is_some(), + update.queue, + update.parent_site.unwrap_or_default(), + **site_hash, + ) + }); + + let mut commands = Vec::new(); + for (_, update) in &ordered { + if let Some(cmds) = update.command.to_commands(config, ExecutionMode::Builder) { + commands.extend(cmds); + } + } + + let stage_label = if verify_each_stage { + format!("{action_label} [stage {}]", stage_index + 1) + } else { + action_label.to_string() + }; + + if !commands.is_empty() { + let result = execute_and_record_live_change(&commands, &stage_label); + if !result.ok { + return Err(summarize_apply_result(&stage_label, &result)); + } + } + + let mut stage_updates = HashMap::new(); + for (site_hash, update) in ordered { + remaining.remove(site_hash); + if verify_each_stage { + stage_updates.insert(*site_hash, update.clone()); + } + sites.insert(*site_hash, Arc::clone(&update.command)); + } + + if verify_each_stage && !stage_updates.is_empty() { + verify_site_updates_live(config, &stage_updates)?; + } + } + + if !remaining.is_empty() { + let fallback_stage: Vec = remaining.into_iter().collect(); + apply_site_command_update_stages( + config, + sites, + updates, + &[fallback_stage], + action_label, + verify_each_stage, + )?; + } + + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn apply_circuit_command_updates( + config: &Arc, + sites: &HashMap>, + circuits: &mut HashMap>, + updates: &HashMap, + live_circuits: &HashMap, + mq_layout: &Option, + qdisc_handles: &mut QdiscHandleState, + migrations: &mut HashMap, + action_label: &str, +) -> Result<(), String> { + if updates.is_empty() { + return Ok(()); + } + let Some(layout) = mq_layout.as_ref() else { + return Err("Bakery runtime virtualization requires MQ layout to be available".to_string()); + }; + let live_reserved_handles = snapshot_live_qdisc_handle_majors_or_empty(config, action_label); + let mut immediate_commands = Vec::new(); + let mut updated_circuits = Vec::new(); + let mut ordered: Vec<_> = updates.iter().collect(); + ordered.sort_by_key(|(_, update)| { + ( + update.parent_site.is_none(), + update.queue, + update.parent_site.unwrap_or_default(), + circuit_hash_from_command(update.command.as_ref()).unwrap_or_default(), + ) + }); + + for (circuit_hash, update) in ordered { + let Some(old_cmd) = circuits.get(circuit_hash).cloned() else { + continue; + }; + let enriched_base = with_assigned_qdisc_handles_reserved( + &update.command, + config, + layout, + qdisc_handles, + &live_reserved_handles, + ); + let enriched_cmd = rotate_changed_qdisc_handles_reserved( + old_cmd.as_ref(), + &enriched_base, + config, + layout, + qdisc_handles, + &live_reserved_handles, + ); + if let Some(summary) = qdisc_handle_rotation_invariant_error_with_live_reservations( + old_cmd.as_ref(), + enriched_cmd.as_ref(), + config, + Some(&live_reserved_handles), + ) { + return Err(format!( + "Bakery runtime virtualization refusing live migration for active circuit {}: {}", + circuit_hash, summary + )); + } + if live_circuits.contains_key(circuit_hash) + && let BakeryCommands::AddCircuit { + class_major, + up_class_major, + .. + } = enriched_cmd.as_ref() + && find_free_circuit_shadow_minor( + sites, + circuits, + migrations, + *class_major, + *up_class_major, + ) + .is_none() + { + return Err(format!( + "Unable to queue live migration for active circuit {}: no shadow minor available", + circuit_hash + )); + } + if queue_live_migration( + old_cmd.as_ref(), + &enriched_cmd, + sites, + circuits, + live_circuits, + migrations, + ) { + continue; + } + if live_circuits.contains_key(circuit_hash) { + return Err(format!( + "Bakery runtime virtualization could not live-migrate active circuit {}", + circuit_hash + )); + } + match config.queues.lazy_queues.as_ref() { + None | Some(LazyQueueMode::No) | Some(LazyQueueMode::Htb) => { + if let Some(prune) = old_cmd.to_prune(config, true) { + immediate_commands.extend(prune); + } + if let Some(add) = enriched_cmd.to_commands(config, ExecutionMode::Builder) { + immediate_commands.extend(add); + } + } + Some(LazyQueueMode::Full) => {} + } + updated_circuits.push((*circuit_hash, enriched_cmd)); + } + + if !immediate_commands.is_empty() { + let result = execute_and_record_live_change(&immediate_commands, action_label); + if !result.ok { + return Err(summarize_apply_result(action_label, &result)); + } + } + for (circuit_hash, command) in updated_circuits { + circuits.insert(circuit_hash, command); + } + Ok(()) +} + +fn circuit_qdisc_parent_changed( + old_cmd: &BakeryCommands, + new_cmd: &BakeryCommands, + config: &Arc, +) -> bool { + let (old_down_parent, old_up_parent) = effective_directional_qdisc_parents(old_cmd, config); + let (new_down_parent, new_up_parent) = effective_directional_qdisc_parents(new_cmd, config); + old_down_parent != new_down_parent || old_up_parent != new_up_parent +} + +fn qdisc_handle_rotation_invariant_error_with_live_reservations( + old_cmd: &BakeryCommands, + new_cmd: &BakeryCommands, + config: &Arc, + live_reserved_handles: Option<&HashMap>>, +) -> Option { + let (old_down_parent, old_up_parent) = effective_directional_qdisc_parents(old_cmd, config); + let (new_down_parent, new_up_parent) = effective_directional_qdisc_parents(new_cmd, config); + let BakeryCommands::AddCircuit { + down_qdisc_handle: old_down_qdisc_handle, + up_qdisc_handle: old_up_qdisc_handle, + .. + } = old_cmd + else { + return None; + }; + let BakeryCommands::AddCircuit { + down_qdisc_handle: new_down_qdisc_handle, + up_qdisc_handle: new_up_qdisc_handle, + .. + } = new_cmd + else { + return None; + }; + + if old_down_parent != new_down_parent + && old_down_qdisc_handle.is_some() + && old_down_qdisc_handle == new_down_qdisc_handle + { + return Some(format!( + "downlink qdisc handle {:?} was preserved even though the qdisc parent changed from {:?} to {:?}", + old_down_qdisc_handle, old_down_parent, new_down_parent + )); + } + + if old_up_parent != new_up_parent + && old_up_qdisc_handle.is_some() + && old_up_qdisc_handle == new_up_qdisc_handle + { + return Some(format!( + "uplink qdisc handle {:?} was preserved even though the qdisc parent changed from {:?} to {:?}", + old_up_qdisc_handle, old_up_parent, new_up_parent + )); + } + + if let Some(live_reserved) = live_reserved_handles { + let isp_interface = config.isp_interface(); + if let Some(new_down_handle) = new_down_qdisc_handle + && live_reserved + .get(&isp_interface) + .is_some_and(|handles| handles.contains(new_down_handle)) + && Some(*new_down_handle) != *old_down_qdisc_handle + { + return Some(format!( + "downlink qdisc handle {:?} is already live on {} and cannot be reused for a different circuit/parent", + new_down_handle, isp_interface + )); + } + + let up_interface = config.internet_interface(); + if let Some(new_up_handle) = new_up_qdisc_handle + && live_reserved + .get(&up_interface) + .is_some_and(|handles| handles.contains(new_up_handle)) + && Some(*new_up_handle) != *old_up_qdisc_handle + { + return Some(format!( + "uplink qdisc handle {:?} is already live on {} and cannot be reused for a different circuit/parent", + new_up_handle, up_interface + )); + } + } + + None +} + +fn migration_qdisc_handle_rotation_invariant_error(mig: &Migration) -> Option { + let old_down_parent = tc_handle_from_major_minor(mig.old_class_major, mig.old_minor); + let new_down_parent = tc_handle_from_major_minor(mig.class_major, mig.final_minor); + if old_down_parent != new_down_parent + && mig.old_down_qdisc_handle.is_some() + && mig.old_down_qdisc_handle == mig.down_qdisc_handle + { + return Some(format!( + "downlink qdisc handle {:?} is still the old handle while the final parent changed from {} to {}", + mig.down_qdisc_handle, + old_down_parent.as_tc_string(), + new_down_parent.as_tc_string() + )); + } + + let old_up_parent = tc_handle_from_major_minor(mig.old_up_class_major, mig.old_minor); + let new_up_parent = tc_handle_from_major_minor(mig.up_class_major, mig.final_minor); + if old_up_parent != new_up_parent + && mig.old_up_qdisc_handle.is_some() + && mig.old_up_qdisc_handle == mig.up_qdisc_handle + { + return Some(format!( + "uplink qdisc handle {:?} is still the old handle while the final parent changed from {} to {}", + mig.up_qdisc_handle, + old_up_parent.as_tc_string(), + new_up_parent.as_tc_string() + )); + } + + None +} + +#[cfg_attr(not(test), allow(dead_code))] +fn build_final_qdisc_handle_rotation_invariant_error( + mig: &Migration, + final_cmd: &BakeryCommands, +) -> Option { + build_final_qdisc_handle_rotation_invariant_error_with_live_reservations( + mig, final_cmd, None, None, + ) +} + +fn build_final_qdisc_handle_rotation_invariant_error_with_live_reservations( + mig: &Migration, + final_cmd: &BakeryCommands, + live_down_reserved_handles: Option<&HashSet>, + live_up_reserved_handles: Option<&HashSet>, +) -> Option { + let BakeryCommands::AddCircuit { + class_minor, + down_qdisc_handle, + up_qdisc_handle, + .. + } = final_cmd + else { + return Some("build-final command was not an AddCircuit".to_string()); + }; + + if *class_minor != mig.final_minor { + return Some(format!( + "build-final command used unexpected class minor {} instead of {}", + class_minor, mig.final_minor + )); + } + + if *down_qdisc_handle != mig.down_qdisc_handle { + return Some(format!( + "build-final command used downlink qdisc handle {:?}, expected rotated handle {:?}", + down_qdisc_handle, mig.down_qdisc_handle + )); + } + + if *up_qdisc_handle != mig.up_qdisc_handle { + return Some(format!( + "build-final command used uplink qdisc handle {:?}, expected rotated handle {:?}", + up_qdisc_handle, mig.up_qdisc_handle + )); + } + + if let Some(summary) = migration_qdisc_handle_rotation_invariant_error(mig) { + return Some(summary); + } + + if let Some(live_down_reserved) = live_down_reserved_handles + && let Some(down_handle) = *down_qdisc_handle + && live_down_reserved.contains(&down_handle) + && Some(down_handle) != mig.old_down_qdisc_handle + { + return Some(format!( + "build-final command used downlink qdisc handle {:?}, but that handle is already live on the interface under another parent", + down_handle + )); + } + + if let Some(live_up_reserved) = live_up_reserved_handles + && let Some(up_handle) = *up_qdisc_handle + && live_up_reserved.contains(&up_handle) + && Some(up_handle) != mig.old_up_qdisc_handle + { + return Some(format!( + "build-final command used uplink qdisc handle {:?}, but that handle is already live on the interface under another parent", + up_handle + )); + } + + None +} + +#[allow(clippy::too_many_arguments)] +fn apply_top_level_circuit_command_updates( + config: &Arc, + sites: &HashMap>, + circuits: &mut HashMap>, + updates: &HashMap, + live_circuits: &HashMap, + mq_layout: &Option, + qdisc_handles: &mut QdiscHandleState, + migrations: &mut HashMap, + action_label: &str, +) -> Result<(), String> { + if updates.is_empty() { + return Ok(()); + } + let Some(layout) = mq_layout.as_ref() else { + return Err("Bakery runtime virtualization requires MQ layout to be available".to_string()); + }; + let live_reserved_handles = + snapshot_live_qdisc_handle_majors_or_empty(config, "top-level runtime virtualization"); + + let mut ordered: Vec<_> = updates.iter().collect(); + ordered.sort_by_key(|(_, update)| { + ( + update.parent_site.is_none(), + update.queue, + update.parent_site.unwrap_or_default(), + circuit_hash_from_command(update.command.as_ref()).unwrap_or_default(), + ) + }); + + for (circuit_hash, update) in &ordered { + let Some(old_cmd) = circuits.get(circuit_hash).cloned() else { + continue; + }; + let enriched_base = with_assigned_qdisc_handles_reserved( + &update.command, + config, + layout, + qdisc_handles, + &live_reserved_handles, + ); + let enriched_cmd = rotate_changed_qdisc_handles_reserved( + old_cmd.as_ref(), + &enriched_base, + config, + layout, + qdisc_handles, + &live_reserved_handles, + ); + if circuit_qdisc_parent_changed(old_cmd.as_ref(), enriched_cmd.as_ref(), config) + && let BakeryCommands::AddCircuit { + class_major, + up_class_major, + .. + } = enriched_cmd.as_ref() + && find_free_circuit_shadow_minor( + sites, + circuits, + migrations, + *class_major, + *up_class_major, + ) + .is_none() + { + return Err(format!( + "Unable to queue top-level runtime migration for circuit {}: no shadow minor available", + circuit_hash + )); + } + } + + let mut immediate_commands = Vec::new(); + let mut updated_circuits = Vec::new(); + + for (circuit_hash, update) in ordered { + let Some(old_cmd) = circuits.get(circuit_hash).cloned() else { + continue; + }; + let enriched_base = with_assigned_qdisc_handles_reserved( + &update.command, + config, + layout, + qdisc_handles, + &live_reserved_handles, + ); + let enriched_cmd = rotate_changed_qdisc_handles_reserved( + old_cmd.as_ref(), + &enriched_base, + config, + layout, + qdisc_handles, + &live_reserved_handles, + ); + if let Some(summary) = qdisc_handle_rotation_invariant_error_with_live_reservations( + old_cmd.as_ref(), + enriched_cmd.as_ref(), + config, + Some(&live_reserved_handles), + ) { + return Err(format!( + "Bakery runtime top-level virtualization refusing live migration for active circuit {}: {}", + circuit_hash, summary + )); + } + + if circuit_qdisc_parent_changed(old_cmd.as_ref(), enriched_cmd.as_ref(), config) { + if !queue_top_level_runtime_migration( + old_cmd.as_ref(), + &enriched_cmd, + sites, + circuits, + live_circuits, + migrations, + ) { + return Err(format!( + "Bakery runtime top-level virtualization could not queue migration for circuit {}", + circuit_hash + )); + } + continue; + } + + if queue_live_migration( + old_cmd.as_ref(), + &enriched_cmd, + sites, + circuits, + live_circuits, + migrations, + ) { + continue; + } + + if live_circuits.contains_key(circuit_hash) { + return Err(format!( + "Bakery runtime top-level virtualization could not live-migrate active circuit {}", + circuit_hash + )); + } + + match config.queues.lazy_queues.as_ref() { + None | Some(LazyQueueMode::No) | Some(LazyQueueMode::Htb) => { + if let Some(prune) = old_cmd.to_prune(config, true) { + immediate_commands.extend(prune); + } + if let Some(add) = enriched_cmd.to_commands(config, ExecutionMode::Builder) { + immediate_commands.extend(add); + } + } + Some(LazyQueueMode::Full) => {} + } + updated_circuits.push((*circuit_hash, enriched_cmd)); + } + + if !immediate_commands.is_empty() { + let result = execute_and_record_live_change(&immediate_commands, action_label); + if !result.ok { + return Err(summarize_apply_result(action_label, &result)); + } + } + + for (circuit_hash, command) in updated_circuits { + circuits.insert(circuit_hash, command); + } + + Ok(()) +} + +fn apply_runtime_virtualization_overlay( + batch: Vec>, + virtualized_sites: &HashMap, +) -> Vec> { + if virtualized_sites.is_empty() { + return batch; + } + + let mut hidden_site_hashes = HashSet::new(); + let mut active_site_overrides: HashMap> = HashMap::new(); + let mut active_circuit_overrides: HashMap> = HashMap::new(); + for (site_hash, state) in virtualized_sites { + if state.active_branch_hides_original_site() { + hidden_site_hashes.insert(*site_hash); + } + for (hash, command) in &state.active_sites { + active_site_overrides.insert(*hash, Arc::clone(command)); + } + for (hash, command) in &state.active_circuits { + active_circuit_overrides.insert(*hash, Arc::clone(command)); + } + } + + batch + .into_iter() + .filter_map(|command| match command.as_ref() { + BakeryCommands::AddSite { site_hash, .. } => { + if hidden_site_hashes.contains(site_hash) { + return None; + } + if let Some(override_cmd) = active_site_overrides.get(site_hash) { + return Some(Arc::clone(override_cmd)); + } + Some(command) + } + BakeryCommands::AddCircuit { circuit_hash, .. } => { + if let Some(override_cmd) = active_circuit_overrides.get(circuit_hash) { + return Some(Arc::clone(override_cmd)); + } + Some(command) + } + _ => Some(command), + }) + .collect() +} + +fn reconstruct_structural_baseline_state( + sites: &HashMap>, + circuits: &HashMap>, + virtualized_sites: &HashMap, +) -> ( + HashMap>, + HashMap>, +) { + if virtualized_sites.is_empty() { + return (sites.clone(), circuits.clone()); + } + + let mut baseline_sites = sites.clone(); + let mut baseline_circuits = circuits.clone(); + + for (site_hash, state) in virtualized_sites { + baseline_sites.insert(*site_hash, Arc::clone(&state.site)); + for (saved_hash, saved_site) in &state.saved_sites { + baseline_sites.insert(*saved_hash, Arc::clone(saved_site)); + } + for (saved_hash, saved_circuit) in &state.saved_circuits { + baseline_circuits.insert(*saved_hash, Arc::clone(saved_circuit)); + } + } + + (baseline_sites, baseline_circuits) +} + +fn runtime_virtualized_site_has_pending_migrations( + state: &VirtualizedSiteState, + migrations: &HashMap, +) -> bool { + state + .active_circuits + .keys() + .any(|circuit_hash| migrations.contains_key(circuit_hash)) +} + +#[cfg(test)] +fn runtime_virtualized_site_has_remaining_observed_child_classes( + state: &VirtualizedSiteState, + down_snapshot: &HashMap, + up_snapshot: &HashMap, +) -> bool { + let Some((target_down_class, target_up_class)) = site_class_handles(state.site.as_ref()) else { + return false; + }; + + down_snapshot + .values() + .any(|entry| entry.parent == Some(target_down_class)) + || up_snapshot + .values() + .any(|entry| entry.parent == Some(target_up_class)) +} + +fn sync_runtime_virtualized_site_qdisc_handles_from_live_snapshot( + state: &mut VirtualizedSiteState, + down_snapshot: &HashMap, + up_snapshot: &HashMap, +) { + let Some((target_down_class, target_up_class)) = site_class_handles(state.site.as_ref()) else { + return; + }; + + if state.qdisc_handles.down.is_none() + && let Some(entry) = down_snapshot.get(&target_down_class) + { + state.qdisc_handles.down = entry.leaf_qdisc_major; + } + + if state.qdisc_handles.up.is_none() + && let Some(entry) = up_snapshot.get(&target_up_class) + { + state.qdisc_handles.up = entry.leaf_qdisc_major; + } +} + +fn runtime_virtualized_site_prune_ready( + state: &VirtualizedSiteState, + migrations: &HashMap, + now_unix: u64, +) -> bool { + state.pending_prune + && state.next_prune_attempt_unix <= now_unix + && !runtime_virtualized_site_has_pending_migrations(state, migrations) +} + +fn runtime_site_prune_missing_qdisc_is_harmless(summary: &str) -> bool { + summary + .to_ascii_lowercase() + .contains("cannot find specified qdisc on specified device") +} + +fn remaining_inactive_branch_class_handles( + state: &VirtualizedSiteState, +) -> (HashSet, HashSet) { + let mut down = HashSet::new(); + let mut up = HashSet::new(); + + for site in state.prune_sites.values() { + if let Some((down_class, up_class)) = site_class_handles(site.as_ref()) { + down.insert(down_class); + up.insert(up_class); + } + } + + for circuit in state.prune_circuits.values() { + if let BakeryCommands::AddCircuit { + class_major, + up_class_major, + class_minor, + .. + } = circuit.as_ref() + { + down.insert(tc_handle_from_major_minor(*class_major, *class_minor)); + up.insert(tc_handle_from_major_minor(*up_class_major, *class_minor)); + } + } + + if state.active_branch_hides_original_site() + && let Some((down_class, up_class)) = site_class_handles(state.site.as_ref()) + { + down.insert(down_class); + up.insert(up_class); + } + + (down, up) +} + +fn summarize_handle_set(handles: &[TcHandle]) -> String { + handles + .iter() + .take(4) + .map(TcHandle::as_tc_string) + .collect::>() + .join(", ") +} + +fn classify_inactive_branch_observed_children( + site: &BakeryCommands, + down_snapshot: &HashMap, + up_snapshot: &HashMap, + expected_down: &HashSet, + expected_up: &HashSet, +) -> Result<(), String> { + let Some((parent_down_class, parent_up_class)) = site_class_handles(site) else { + return Ok(()); + }; + + let observed_down: Vec = down_snapshot + .values() + .filter_map(|entry| (entry.parent == Some(parent_down_class)).then_some(entry.class_id)) + .collect(); + let observed_up: Vec = up_snapshot + .values() + .filter_map(|entry| (entry.parent == Some(parent_up_class)).then_some(entry.class_id)) + .collect(); + + let unexpected_down: Vec = observed_down + .iter() + .copied() + .filter(|handle| !expected_down.contains(handle)) + .collect(); + let unexpected_up: Vec = observed_up + .iter() + .copied() + .filter(|handle| !expected_up.contains(handle)) + .collect(); + + if unexpected_down.is_empty() && unexpected_up.is_empty() { + return Ok(()); + } + + let mut details = Vec::new(); + if !unexpected_down.is_empty() { + details.push(format!("down [{}]", summarize_handle_set(&unexpected_down))); + } + if !unexpected_up.is_empty() { + details.push(format!("up [{}]", summarize_handle_set(&unexpected_up))); + } + Err(format!( + "Inactive branch cleanup encountered unexpected live child classes still attached: {}", + details.join(", ") + )) +} + +fn verify_pruned_handles_absent( + config: &Arc, + down_handles: &[TcHandle], + up_handles: &[TcHandle], + context: &str, +) -> Result<(), String> { + let down_snapshot = read_live_class_snapshot(&config.isp_interface())?; + let up_snapshot = read_live_class_snapshot(&config.internet_interface())?; + + let lingering_down: Vec = down_handles + .iter() + .filter(|handle| down_snapshot.contains_key(handle)) + .map(TcHandle::as_tc_string) + .collect(); + let lingering_up: Vec = up_handles + .iter() + .filter(|handle| up_snapshot.contains_key(handle)) + .map(TcHandle::as_tc_string) + .collect(); + + if lingering_down.is_empty() && lingering_up.is_empty() { + return Ok(()); + } + + let mut details = Vec::new(); + if !lingering_down.is_empty() { + details.push(format!( + "{} still present on {}", + lingering_down.join(", "), + config.isp_interface() + )); + } + if !lingering_up.is_empty() { + details.push(format!( + "{} still present on {}", + lingering_up.join(", "), + config.internet_interface() + )); + } + + Err(format!( + "{context}: targeted old-branch class prune did not take effect: {}", + details.join("; ") + )) +} + +fn site_commands_observed_live( + commands: &HashMap>, + down_snapshot: &HashMap, + up_snapshot: &HashMap, +) -> bool { + commands.values().all(|site| { + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + .. + } = site.as_ref() + else { + return false; + }; + let Some((down_class, up_class)) = site_class_handles(site.as_ref()) else { + return false; + }; + let Some(down_entry) = down_snapshot.get(&down_class) else { + return false; + }; + let Some(up_entry) = up_snapshot.get(&up_class) else { + return false; + }; + live_parent_matches(down_entry.parent, *parent_class_id) + && live_parent_matches(up_entry.parent, *up_parent_class_id) + }) +} + +fn live_parent_matches(observed: Option, expected: TcHandle) -> bool { + if observed == Some(expected) { + return true; + } + + let (_, expected_minor) = expected.get_major_minor(); + expected_minor == 0 && observed.is_none() +} + +fn verify_runtime_shadow_active( + state: &VirtualizedSiteState, + down_snapshot: &HashMap, + up_snapshot: &HashMap, +) -> Result<(), String> { + for (site_hash, site) in &state.active_sites { + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + .. + } = site.as_ref() + else { + continue; + }; + let Some((down_class, up_class)) = site_class_handles(site.as_ref()) else { + return Err(format!( + "Runtime cutover activation verification could not derive class handles for active site {}", + site_hash + )); + }; + let Some(down_entry) = down_snapshot.get(&down_class) else { + return Err(format!( + "Runtime cutover activation verification did not observe active shadow site {} on downlink", + site_hash + )); + }; + let Some(up_entry) = up_snapshot.get(&up_class) else { + return Err(format!( + "Runtime cutover activation verification did not observe active shadow site {} on uplink", + site_hash + )); + }; + if !live_parent_matches(down_entry.parent, *parent_class_id) + || !live_parent_matches(up_entry.parent, *up_parent_class_id) + { + warn!( + "Bakery: top-level cutover site {} planned down_class={} up_class={} expected parents {}/{} observed parents {:?}/{:?}", + site_hash, + down_class.as_tc_string(), + up_class.as_tc_string(), + parent_class_id.as_tc_string(), + up_parent_class_id.as_tc_string(), + down_entry.parent, + up_entry.parent + ); + return Err(format!( + "Runtime cutover activation verification observed active shadow site {} with wrong parents {:?}/{:?}, expected {}/{}", + site_hash, + down_entry.parent, + up_entry.parent, + parent_class_id.as_tc_string(), + up_parent_class_id.as_tc_string() + )); + } + } + + for (circuit_hash, active_circuit) in &state.active_circuits { + let BakeryCommands::AddCircuit { + parent_class_id, + up_parent_class_id, + .. + } = active_circuit.as_ref() + else { + continue; + }; + let Some((active_down_class, active_up_class)) = + circuit_class_handles(active_circuit.as_ref()) + else { + return Err(format!( + "Runtime cutover activation verification could not derive active circuit handles for {}", + circuit_hash + )); + }; + let Some(down_entry) = down_snapshot.get(&active_down_class) else { + return Err(format!( + "Runtime cutover activation verification did not observe active shadow circuit {} on downlink", + circuit_hash + )); + }; + let Some(up_entry) = up_snapshot.get(&active_up_class) else { + return Err(format!( + "Runtime cutover activation verification did not observe active shadow circuit {} on uplink", + circuit_hash + )); + }; + if !live_parent_matches(down_entry.parent, *parent_class_id) + || !live_parent_matches(up_entry.parent, *up_parent_class_id) + { + return Err(format!( + "Runtime cutover activation verification observed active shadow circuit {} with wrong parents {:?}/{:?}, expected {}/{}", + circuit_hash, + down_entry.parent, + up_entry.parent, + parent_class_id.as_tc_string(), + up_parent_class_id.as_tc_string() + )); + } + } + + Ok(()) +} + +fn site_prune_class_commands_for_observed_state( + config: &Arc, + site: &BakeryCommands, + down_exists: bool, + up_exists: bool, +) -> Option>> { + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + class_minor, + .. + } = site + else { + return None; + }; + + let mut commands = Vec::new(); + if down_exists { + commands.push(vec![ + "class".to_string(), + "del".to_string(), + "dev".to_string(), + config.isp_interface(), + "parent".to_string(), + parent_class_id.as_tc_string(), + "classid".to_string(), + format!( + "0x{:x}:0x{:x}", + parent_class_id.get_major_minor().0, + class_minor + ), + ]); + } + if up_exists { + commands.push(vec![ + "class".to_string(), + "del".to_string(), + "dev".to_string(), + config.internet_interface(), + "parent".to_string(), + up_parent_class_id.as_tc_string(), + "classid".to_string(), + format!( + "0x{:x}:0x{:x}", + up_parent_class_id.get_major_minor().0, + class_minor + ), + ]); + } + + (!commands.is_empty()).then_some(commands) +} + +fn execute_runtime_site_prune( + config: &Arc, + state: &mut VirtualizedSiteState, + down_snapshot: &HashMap, + up_snapshot: &HashMap, +) -> Result<(), String> { + sync_runtime_virtualized_site_qdisc_handles_from_live_snapshot( + state, + down_snapshot, + up_snapshot, + ); + + if let Some(qdisc_prune) = site_prune_qdisc_commands(config, &state.qdisc_handles) { + let qdisc_result = + execute_in_memory(&qdisc_prune, "TreeGuard runtime deferred site qdisc prune"); + if !qdisc_result.ok { + let summary = summarize_apply_result( + "TreeGuard runtime deferred site qdisc prune", + &qdisc_result, + ); + if !runtime_site_prune_missing_qdisc_is_harmless(&summary) { + return Err(summary); + } + } + } + + let Some((target_down_class, target_up_class)) = site_class_handles(state.site.as_ref()) else { + return Ok(()); + }; + let down_exists = down_snapshot.contains_key(&target_down_class); + let up_exists = up_snapshot.contains_key(&target_up_class); + let Some(class_prune) = site_prune_class_commands_for_observed_state( + config, + state.site.as_ref(), + down_exists, + up_exists, + ) else { + return Ok(()); + }; + let class_result = execute_in_memory(&class_prune, "TreeGuard runtime deferred site prune"); + if !class_result.ok { + return Err(summarize_apply_result( + "TreeGuard runtime deferred site prune", + &class_result, + )); + } + + Ok(()) +} + +#[derive(Debug)] +enum RuntimePrunePassResult { + Completed, + Progress(String), + Pending(String), + Failed(String), +} + +fn execute_runtime_virtualized_subtree_prune( + config: &Arc, + state: &mut VirtualizedSiteState, + down_snapshot: &HashMap, + up_snapshot: &HashMap, +) -> RuntimePrunePassResult { + let (expected_inactive_down_classes, expected_inactive_up_classes) = + remaining_inactive_branch_class_handles(state); + let mut protected_down_classes = HashSet::new(); + let mut protected_up_classes = HashSet::new(); + for command in state.active_circuits.values() { + if let BakeryCommands::AddCircuit { + class_major, + up_class_major, + class_minor, + .. + } = command.as_ref() + { + protected_down_classes.insert(tc_handle_from_major_minor(*class_major, *class_minor)); + protected_up_classes.insert(tc_handle_from_major_minor(*up_class_major, *class_minor)); + } + } + + let ordered_circuit_hashes = ordered_prune_circuit_hashes(&state.prune_circuits); + for circuit_hash in ordered_circuit_hashes { + let Some(prune_circuit) = state.prune_circuits.get(&circuit_hash).cloned() else { + continue; + }; + let Some(commands) = observed_circuit_prune_commands( + config, + prune_circuit.as_ref(), + Some(down_snapshot), + Some(up_snapshot), + &protected_down_classes, + &protected_up_classes, + ) else { + state.prune_circuits.remove(&circuit_hash); + continue; + }; + if commands.is_empty() { + state.prune_circuits.remove(&circuit_hash); + continue; + } + let result = execute_in_memory(&commands, "TreeGuard runtime deferred child circuit prune"); + if !result.ok { + return RuntimePrunePassResult::Failed(summarize_apply_result( + "TreeGuard runtime deferred child circuit prune", + &result, + )); + } + let BakeryCommands::AddCircuit { + class_major, + up_class_major, + class_minor, + .. + } = prune_circuit.as_ref() + else { + return RuntimePrunePassResult::Failed( + "TreeGuard runtime deferred child circuit prune targeted a non-circuit command" + .to_string(), + ); + }; + if let Err(summary) = verify_pruned_handles_absent( + config, + &[tc_handle_from_major_minor(*class_major, *class_minor)], + &[tc_handle_from_major_minor(*up_class_major, *class_minor)], + "TreeGuard runtime deferred child circuit prune", + ) { + return RuntimePrunePassResult::Failed(summary); + } + state.prune_circuits.remove(&circuit_hash); + return RuntimePrunePassResult::Progress( + "Inactive branch circuit cleanup applied; refreshing live verification".to_string(), + ); + } + + let ordered_hashes = ordered_prune_site_hashes(&state.prune_sites); + for site_hash in ordered_hashes { + let Some(prune_site) = state.prune_sites.get(&site_hash).cloned() else { + continue; + }; + if site_has_observed_child_classes(prune_site.as_ref(), down_snapshot, up_snapshot) { + if let Err(summary) = classify_inactive_branch_observed_children( + prune_site.as_ref(), + down_snapshot, + up_snapshot, + &expected_inactive_down_classes, + &expected_inactive_up_classes, + ) { + return RuntimePrunePassResult::Failed(summary); + } + return RuntimePrunePassResult::Pending( + "Observed live child classes still attached".to_string(), + ); + } + let Some((down_class, up_class)) = site_class_handles(prune_site.as_ref()) else { + state.prune_sites.remove(&site_hash); + continue; + }; + let down_exists = down_snapshot.contains_key(&down_class); + let up_exists = up_snapshot.contains_key(&up_class); + if let Some(class_prune) = site_prune_class_commands_for_observed_state( + config, + prune_site.as_ref(), + down_exists, + up_exists, + ) { + let result = + execute_in_memory(&class_prune, "TreeGuard runtime deferred child site prune"); + if !result.ok { + return RuntimePrunePassResult::Failed(summarize_apply_result( + "TreeGuard runtime deferred child site prune", + &result, + )); + } + if let Err(summary) = verify_pruned_handles_absent( + config, + &[down_class], + &[up_class], + "TreeGuard runtime deferred child site prune", + ) { + return RuntimePrunePassResult::Failed(summary); + } + state.prune_sites.remove(&site_hash); + return RuntimePrunePassResult::Progress( + "Inactive branch site cleanup applied; refreshing live verification".to_string(), + ); + } + state.prune_sites.remove(&site_hash); + } + + if state.active_branch_hides_original_site() { + if site_has_observed_child_classes(state.site.as_ref(), down_snapshot, up_snapshot) { + if let Err(summary) = classify_inactive_branch_observed_children( + state.site.as_ref(), + down_snapshot, + up_snapshot, + &expected_inactive_down_classes, + &expected_inactive_up_classes, + ) { + return RuntimePrunePassResult::Failed(summary); + } + return RuntimePrunePassResult::Pending( + "Observed live child classes still attached".to_string(), + ); + } + + if site_prune_commands(config, state).is_none() { + return RuntimePrunePassResult::Completed; + } + + return match execute_runtime_site_prune(config, state, down_snapshot, up_snapshot) { + Ok(()) => { + let Some((down_class, up_class)) = site_class_handles(state.site.as_ref()) else { + return RuntimePrunePassResult::Completed; + }; + match verify_pruned_handles_absent( + config, + &[down_class], + &[up_class], + "TreeGuard runtime deferred root site prune", + ) { + Ok(()) => RuntimePrunePassResult::Progress( + "Inactive branch root cleanup applied; refreshing live verification" + .to_string(), + ), + Err(summary) => RuntimePrunePassResult::Failed(summary), + } + } + Err(summary) => RuntimePrunePassResult::Failed(summary), + }; + } + + RuntimePrunePassResult::Completed +} + +fn unix_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or(0) +} + +fn build_post_restore_virtualized_state( + saved_state: VirtualizedSiteState, + restore_active_sites: HashMap>, + restore_active_circuits: HashMap>, + now_unix: u64, +) -> Option { + let prune_shadow_sites = saved_state.active_sites.clone(); + let prune_shadow_circuits = saved_state.active_circuits.clone(); + + if prune_shadow_sites.is_empty() && prune_shadow_circuits.is_empty() { + return None; + } + + Some(VirtualizedSiteState { + site_name: saved_state.site_name, + site: saved_state.site, + saved_sites: saved_state.saved_sites, + saved_circuits: saved_state.saved_circuits, + active_sites: restore_active_sites, + active_circuits: restore_active_circuits, + prune_sites: prune_shadow_sites, + prune_circuits: prune_shadow_circuits, + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Original, + lifecycle: RuntimeVirtualizedBranchLifecycle::PhysicalActiveCleanupPending, + pending_prune: true, + next_prune_attempt_unix: now_unix.saturating_add(RUNTIME_SITE_PRUNE_RETRY_SECONDS), + }) +} + +fn runtime_node_operation_action(virtualized: bool) -> RuntimeNodeOperationAction { + if virtualized { + RuntimeNodeOperationAction::Virtualize + } else { + RuntimeNodeOperationAction::Restore + } +} + +fn runtime_node_operation_is_active(status: RuntimeNodeOperationStatus) -> bool { + matches!( + status, + RuntimeNodeOperationStatus::Submitted + | RuntimeNodeOperationStatus::Applying + | RuntimeNodeOperationStatus::AppliedAwaitingCleanup + ) +} + +fn runtime_node_operation_consumes_capacity(status: RuntimeNodeOperationStatus) -> bool { + matches!( + status, + RuntimeNodeOperationStatus::Submitted + | RuntimeNodeOperationStatus::Applying + | RuntimeNodeOperationStatus::AppliedAwaitingCleanup + ) +} + +fn active_runtime_node_operation_count( + runtime_node_operations: &HashMap, +) -> usize { + runtime_node_operations + .values() + .filter(|operation| runtime_node_operation_consumes_capacity(operation.status)) + .count() +} + +fn runtime_virtualization_target_site<'a>( + site_hash: i64, + sites: &'a HashMap>, + virtualized_sites: &'a HashMap, +) -> Option<&'a Arc> { + sites + .get(&site_hash) + .or_else(|| virtualized_sites.get(&site_hash).map(|state| &state.site)) +} + +fn runtime_virtualization_target_is_top_level( + site_hash: i64, + sites: &HashMap>, + virtualized_sites: &HashMap, +) -> bool { + runtime_virtualization_target_site(site_hash, sites, virtualized_sites) + .is_some_and(|site| site_is_top_level(site.as_ref())) +} + +fn active_top_level_runtime_operation_conflict( + site_hash: i64, + runtime_node_operations: &HashMap, + sites: &HashMap>, + virtualized_sites: &HashMap, +) -> Option { + runtime_node_operations + .iter() + .find_map(|(other_site_hash, operation)| { + if *other_site_hash != site_hash + && runtime_node_operation_is_active(operation.status) + && runtime_virtualization_target_is_top_level( + *other_site_hash, + sites, + virtualized_sites, + ) + { + Some(*other_site_hash) + } else { + None + } + }) +} + +fn runtime_error_suggests_material_desync(summary: &str) -> bool { + let lower = summary.to_ascii_lowercase(); + lower.contains("rtnetlink") + || lower.contains("specified class not found") + || lower.contains("cannot find specified qdisc") + || lower.contains("cannot move an existing qdisc") + || lower.contains("device or resource busy") + || lower.contains("top-level runtime cutover failed") + || lower.contains("top-level runtime cutover verification failed") + || lower.contains("unexpected live child classes") +} + +fn rebuild_runtime_branch_snapshots( + virtualized_sites: &HashMap, +) -> HashMap { + virtualized_sites + .iter() + .map(|(site_hash, state)| { + let mut active_site_hashes: Vec = state.active_sites.keys().copied().collect(); + active_site_hashes.sort_unstable(); + let mut saved_site_hashes: Vec = state.saved_sites.keys().copied().collect(); + saved_site_hashes.sort_unstable(); + let mut prune_site_hashes: Vec = state.prune_sites.keys().copied().collect(); + prune_site_hashes.sort_unstable(); + ( + *site_hash, + BakeryRuntimeNodeBranchSnapshot { + site_hash: *site_hash, + site_name: state.site_name.clone(), + active_branch: runtime_active_branch_label(state.active_branch).to_string(), + lifecycle: runtime_lifecycle_label(state.lifecycle).to_string(), + pending_prune: state.pending_prune, + next_prune_attempt_unix: state + .pending_prune + .then_some(state.next_prune_attempt_unix), + active_site_hashes, + saved_site_hashes, + prune_site_hashes, + qdisc_down_major: state.qdisc_handles.down, + qdisc_up_major: state.qdisc_handles.up, + }, + ) + }) + .collect() +} + +fn update_desync_state_from_runtime_state( + runtime_node_operations: &HashMap, + virtualized_sites: &HashMap, +) { + let snapshot = rebuild_runtime_operations_snapshot(runtime_node_operations); + let dirty_count = snapshot.dirty_count; + let runtime_operations_by_site = runtime_node_operations + .iter() + .map(|(site_hash, operation)| (*site_hash, operation.snapshot())) + .collect(); + let runtime_branch_states_by_site = rebuild_runtime_branch_snapshots(virtualized_sites); + { + let mut state = telemetry_state().write(); + state.runtime_operations = snapshot; + state.dirty_subtree_count = dirty_count; + state.runtime_operations_by_site = runtime_operations_by_site; + state.runtime_branch_states_by_site = runtime_branch_states_by_site; + } + if dirty_count >= RUNTIME_DIRTY_SUBTREE_RELOAD_THRESHOLD + && bakery_reload_required_reason().is_none() + { + mark_reload_required(format!( + "Bakery detected {} dirty runtime subtree operations; a full reload is now required before further incremental topology mutations.", + dirty_count + )); + } +} + +fn flush_deferred_runtime_site_prunes( + config: &Arc, + virtualized_sites: &mut HashMap, + migrations: &HashMap, + runtime_node_operations: &mut HashMap, +) { + if bakery_reload_required_reason().is_some() { + return; + } + let now_unix = unix_now(); + let mut grouped_events = GroupedBakeryEventLimiter::default(); + let pending_site_hashes: Vec = virtualized_sites + .iter() + .filter_map(|(site_hash, state)| { + runtime_virtualized_site_prune_ready(state, migrations, now_unix).then_some(*site_hash) + }) + .collect(); + + if pending_site_hashes.is_empty() { + return; + } + + let down_snapshot = match read_live_class_snapshot(&config.isp_interface()) { + Ok(snapshot) => snapshot, + Err(summary) => { + warn!( + "Bakery: unable to snapshot live downlink classes for deferred prune pass: {summary}" + ); + for site_hash in pending_site_hashes { + if let Some(state) = virtualized_sites.get_mut(&site_hash) { + let site_name = state.site_name.clone(); + let site_display = + runtime_site_display_name(site_hash, Some(site_name.as_str())); + let retry_at = now_unix.saturating_add(RUNTIME_SITE_PRUNE_RETRY_SECONDS); + state.next_prune_attempt_unix = retry_at; + if let Some(operation) = runtime_node_operations.get_mut(&site_hash) { + operation.attempt_count = operation.attempt_count.saturating_add(1); + if operation.attempt_count >= RUNTIME_SITE_PRUNE_MAX_ATTEMPTS { + state.pending_prune = false; + state.next_prune_attempt_unix = 0; + state.lifecycle = RuntimeVirtualizedBranchLifecycle::Failed; + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + Some(summary.clone()), + None, + ); + grouped_events.emit_with_site_name( + format!("runtime_site_prune_dirty|down_snapshot|{summary}"), + "runtime_site_prune_dirty", + "error", + Some((site_hash, Some(site_name.clone()))), + format!( + "Deferred runtime site prune for site {} marked Dirty after snapshot failure: {}", + site_display, summary + ), + format!( + "runtime site prune Dirty events due to downlink snapshot failure: {}", + summary + ), + ); + } else { + state.lifecycle = + preserve_cutover_pending_or_lifecycle_from_active_branch(state); + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + Some(summary.clone()), + Some(retry_at), + ); + grouped_events.emit_with_site_name( + format!("runtime_site_prune_retry|down_snapshot|{summary}"), + "runtime_site_prune_retry", + "warning", + Some((site_hash, Some(site_name))), + format!( + "Deferred runtime site prune retry {}/{} for site {} postponed: {}", + operation.attempt_count, + RUNTIME_SITE_PRUNE_MAX_ATTEMPTS, + site_display, + summary + ), + format!( + "runtime site prune retry events due to downlink snapshot failure: {}", + summary + ), + ); + } + } + } + } + grouped_events.flush(); + return; + } + }; + let up_snapshot = match read_live_class_snapshot(&config.internet_interface()) { + Ok(snapshot) => snapshot, + Err(summary) => { + warn!( + "Bakery: unable to snapshot live uplink classes for deferred prune pass: {summary}" + ); + for site_hash in pending_site_hashes { + if let Some(state) = virtualized_sites.get_mut(&site_hash) { + let site_name = state.site_name.clone(); + let site_display = + runtime_site_display_name(site_hash, Some(site_name.as_str())); + let retry_at = now_unix.saturating_add(RUNTIME_SITE_PRUNE_RETRY_SECONDS); + state.next_prune_attempt_unix = retry_at; + if let Some(operation) = runtime_node_operations.get_mut(&site_hash) { + operation.attempt_count = operation.attempt_count.saturating_add(1); + if operation.attempt_count >= RUNTIME_SITE_PRUNE_MAX_ATTEMPTS { + state.pending_prune = false; + state.next_prune_attempt_unix = 0; + state.lifecycle = RuntimeVirtualizedBranchLifecycle::Failed; + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + Some(summary.clone()), + None, + ); + grouped_events.emit_with_site_name( + format!("runtime_site_prune_dirty|up_snapshot|{summary}"), + "runtime_site_prune_dirty", + "error", + Some((site_hash, Some(site_name.clone()))), + format!( + "Deferred runtime site prune for site {} marked Dirty after snapshot failure: {}", + site_display, summary + ), + format!( + "runtime site prune Dirty events due to uplink snapshot failure: {}", + summary + ), + ); + } else { + state.lifecycle = + preserve_cutover_pending_or_lifecycle_from_active_branch(state); + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + Some(summary.clone()), + Some(retry_at), + ); + grouped_events.emit_with_site_name( + format!("runtime_site_prune_retry|up_snapshot|{summary}"), + "runtime_site_prune_retry", + "warning", + Some((site_hash, Some(site_name))), + format!( + "Deferred runtime site prune retry {}/{} for site {} postponed: {}", + operation.attempt_count, + RUNTIME_SITE_PRUNE_MAX_ATTEMPTS, + site_display, + summary + ), + format!( + "runtime site prune retry events due to uplink snapshot failure: {}", + summary + ), + ); + } + } + } + } + grouped_events.flush(); + return; + } + }; + + flush_deferred_runtime_site_prunes_with_snapshots( + config, + virtualized_sites, + migrations, + runtime_node_operations, + &down_snapshot, + &up_snapshot, + ); +} + +fn flush_deferred_runtime_site_prunes_with_snapshots( + config: &Arc, + virtualized_sites: &mut HashMap, + migrations: &HashMap, + runtime_node_operations: &mut HashMap, + down_snapshot: &HashMap, + up_snapshot: &HashMap, +) { + if bakery_reload_required_reason().is_some() { + return; + } + let now_unix = unix_now(); + let mut grouped_events = GroupedBakeryEventLimiter::default(); + let pending_site_hashes: Vec = virtualized_sites + .iter() + .filter_map(|(site_hash, state)| { + runtime_virtualized_site_prune_ready(state, migrations, now_unix).then_some(*site_hash) + }) + .collect(); + + if pending_site_hashes.is_empty() { + return; + } + + let mut completed_restore_cleanup = Vec::new(); + for site_hash in pending_site_hashes { + let Some(state) = virtualized_sites.get_mut(&site_hash) else { + continue; + }; + if state.lifecycle == RuntimeVirtualizedBranchLifecycle::CutoverPending { + debug!( + "Bakery: deferred runtime handler inspecting CutoverPending site {} (pending_prune={}, active_branch={:?}, next_prune_attempt_unix={})", + site_hash, state.pending_prune, state.active_branch, state.next_prune_attempt_unix + ); + } + sync_runtime_virtualized_site_qdisc_handles_from_live_snapshot( + state, + down_snapshot, + up_snapshot, + ); + if state.lifecycle == RuntimeVirtualizedBranchLifecycle::CutoverPending + && state.active_branch_hides_original_site() + { + let site_name = state.site_name.clone(); + let site_display = runtime_site_display_name(site_hash, Some(site_name.as_str())); + debug!( + "Bakery: evaluating runtime cutover activation for site {}", + site_hash + ); + match verify_runtime_shadow_active(state, down_snapshot, up_snapshot) { + Ok(()) => { + state.pending_prune = false; + state.next_prune_attempt_unix = 0; + state.lifecycle = RuntimeVirtualizedBranchLifecycle::FlattenedActive; + if let Some(operation) = runtime_node_operations.get_mut(&site_hash) { + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + None, + None, + ); + } + grouped_events.emit_with_site_name( + "runtime_cutover_completed|completed", + "runtime_cutover_completed", + "info", + Some((site_hash, Some(site_name.clone()))), + format!( + "Runtime cutover completed for site {}; shadow branch is active and original branch is standby.", + site_display + ), + "runtime cutover completion events".to_string(), + ); + continue; + } + Err(summary) => { + if let Some(operation) = runtime_node_operations.get_mut(&site_hash) { + operation.attempt_count = operation.attempt_count.saturating_add(1); + if operation.attempt_count >= RUNTIME_SITE_PRUNE_MAX_ATTEMPTS { + warn!( + "Bakery: runtime cutover activation for site {} not ready on final attempt {}: {}", + site_hash, operation.attempt_count, summary + ); + state.pending_prune = false; + state.next_prune_attempt_unix = 0; + state.lifecycle = RuntimeVirtualizedBranchLifecycle::Failed; + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + Some(summary.clone()), + None, + ); + grouped_events.emit_with_site_name( + format!("runtime_cutover_dirty|activation_failed|{summary}"), + "runtime_cutover_dirty", + "error", + Some((site_hash, Some(site_name.clone()))), + format!( + "Runtime cutover for site {} marked Dirty after {} attempts: {}", + site_display, operation.attempt_count, summary + ), + format!( + "runtime cutover Dirty events due to activation verification failure: {}", + summary + ), + ); + } else { + debug!( + "Bakery: runtime cutover activation for site {} not ready on retry {}/{}: {}", + site_hash, + operation.attempt_count, + RUNTIME_SITE_PRUNE_MAX_ATTEMPTS, + summary + ); + let retry_at = now_unix.saturating_add(RUNTIME_CUTOVER_RETRY_SECONDS); + state.next_prune_attempt_unix = retry_at; + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + Some(summary.clone()), + Some(retry_at), + ); + grouped_events.emit_with_site_name( + format!("runtime_cutover_retry|activation_failed|{summary}"), + "runtime_cutover_retry", + "warning", + Some((site_hash, Some(site_name.clone()))), + format!( + "Runtime cutover retry {}/{} for site {} waiting for active/standby convergence: {}", + operation.attempt_count, + RUNTIME_SITE_PRUNE_MAX_ATTEMPTS, + site_display, + summary + ), + format!( + "runtime cutover retry events due to activation verification failure: {}", + summary + ), + ); + } + } else { + debug!( + "Bakery: runtime cutover activation for site {} not ready with no runtime operation state: {}", + site_hash, summary + ); + } + continue; + } + } + } + match execute_runtime_virtualized_subtree_prune(config, state, down_snapshot, up_snapshot) { + RuntimePrunePassResult::Completed => { + if state.lifecycle == RuntimeVirtualizedBranchLifecycle::CutoverPending { + debug!( + "Bakery: CutoverPending site {} fell through to generic prune Completed", + site_hash + ); + } + state.pending_prune = false; + state.next_prune_attempt_unix = 0; + state.lifecycle = preserve_cutover_pending_or_lifecycle_from_active_branch(state); + if let Some(operation) = runtime_node_operations.get_mut(&site_hash) { + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + None, + None, + ); + } + grouped_events.emit_with_site_name( + "runtime_site_prune_completed|completed", + "runtime_site_prune_completed", + "info", + Some((site_hash, Some(state.site_name.clone()))), + format!( + "Deferred runtime site prune completed for site {}.", + runtime_site_display_name(site_hash, Some(state.site_name.as_str())) + ), + "runtime site prune completion events".to_string(), + ); + if !state.active_branch_hides_original_site() { + completed_restore_cleanup.push(site_hash); + } + } + RuntimePrunePassResult::Progress(summary) => { + if state.lifecycle == RuntimeVirtualizedBranchLifecycle::CutoverPending { + debug!( + "Bakery: CutoverPending site {} fell through to generic prune Progress: {}", + site_hash, summary + ); + } + state.next_prune_attempt_unix = now_unix; + state.lifecycle = preserve_cutover_pending_or_lifecycle_from_active_branch(state); + if let Some(operation) = runtime_node_operations.get_mut(&site_hash) { + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + Some(summary.clone()), + Some(now_unix), + ); + } + grouped_events.emit_with_site_name( + format!("runtime_site_prune_progress|{summary}"), + "runtime_site_prune_progress", + "info", + Some((site_hash, Some(state.site_name.clone()))), + format!( + "Deferred runtime site prune for site {} made progress: {}", + runtime_site_display_name(site_hash, Some(state.site_name.as_str())), + summary + ), + format!("runtime site prune progress events: {}", summary), + ); + } + RuntimePrunePassResult::Pending(summary) => { + if state.lifecycle == RuntimeVirtualizedBranchLifecycle::CutoverPending { + debug!( + "Bakery: CutoverPending site {} fell through to generic prune Pending: {}", + site_hash, summary + ); + } + let retry_at = now_unix.saturating_add(RUNTIME_SITE_PRUNE_RETRY_SECONDS); + state.next_prune_attempt_unix = retry_at; + state.lifecycle = preserve_cutover_pending_or_lifecycle_from_active_branch(state); + if let Some(operation) = runtime_node_operations.get_mut(&site_hash) { + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + Some(summary), + Some(retry_at), + ); + } + } + RuntimePrunePassResult::Failed(summary) => { + if state.lifecycle == RuntimeVirtualizedBranchLifecycle::CutoverPending { + warn!( + "Bakery: CutoverPending site {} fell through to generic prune Failed: {}", + site_hash, summary + ); + } + if let Some(operation) = runtime_node_operations.get_mut(&site_hash) { + operation.attempt_count = operation.attempt_count.saturating_add(1); + if operation.attempt_count >= RUNTIME_SITE_PRUNE_MAX_ATTEMPTS { + state.pending_prune = false; + state.next_prune_attempt_unix = 0; + state.lifecycle = RuntimeVirtualizedBranchLifecycle::Failed; + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + Some(summary.clone()), + None, + ); + grouped_events.emit_with_site_name( + format!("runtime_site_prune_dirty|execute_failed|{summary}"), + "runtime_site_prune_dirty", + "error", + Some((site_hash, Some(state.site_name.clone()))), + format!( + "Deferred runtime site prune for site {} marked Dirty after {} attempts: {}", + runtime_site_display_name(site_hash, Some(state.site_name.as_str())), + operation.attempt_count, + summary + ), + format!( + "runtime site prune Dirty events due to execution failure: {}", + summary + ), + ); + } else { + let retry_at = now_unix.saturating_add(RUNTIME_SITE_PRUNE_RETRY_SECONDS); + state.next_prune_attempt_unix = retry_at; + state.lifecycle = + preserve_cutover_pending_or_lifecycle_from_active_branch(state); + operation.update_status( + runtime_status_for_virtualized_state(state), + now_unix, + Some(summary.clone()), + Some(retry_at), + ); + grouped_events.emit_with_site_name( + format!("runtime_site_prune_retry|execute_failed|{summary}"), + "runtime_site_prune_retry", + "warning", + Some((site_hash, Some(state.site_name.clone()))), + format!( + "Deferred runtime site prune retry {}/{} for site {} failed: {}", + operation.attempt_count, + RUNTIME_SITE_PRUNE_MAX_ATTEMPTS, + runtime_site_display_name( + site_hash, + Some(state.site_name.as_str()) + ), + summary + ), + format!( + "runtime site prune retry events due to execution failure: {}", + summary + ), + ); + } + } else { + state.next_prune_attempt_unix = + now_unix.saturating_add(RUNTIME_SITE_PRUNE_RETRY_SECONDS); + grouped_events.emit_with_site_name( + format!("runtime_site_prune_retry|outside_tracking|{summary}"), + "runtime_site_prune_retry", + "warning", + Some((site_hash, Some(state.site_name.clone()))), + format!( + "Deferred runtime site prune for site {} failed outside operation tracking: {}", + runtime_site_display_name(site_hash, Some(state.site_name.as_str())), + summary + ), + format!( + "runtime site prune retry events outside operation tracking: {}", + summary + ), + ); + } + debug!( + "Bakery: deferred runtime site prune for {} failed again: {}", + site_hash, summary + ); + } + } + } + for site_hash in completed_restore_cleanup { + virtualized_sites.remove(&site_hash); + } + grouped_events.flush(); + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); +} + +#[allow(clippy::too_many_arguments)] +fn handle_treeguard_set_node_virtual_live( + site_hash: i64, + virtualized: bool, + sites: &mut HashMap>, + circuits: &mut HashMap>, + live_circuits: &HashMap, + mq_layout: &Option, + qdisc_handles: &mut QdiscHandleState, + migrations: &mut HashMap, + virtualized_sites: &mut HashMap, + runtime_node_operations: &mut HashMap, + next_runtime_operation_id: &mut u64, +) -> RuntimeNodeOperationSnapshot { + let now_unix = unix_now(); + let action = runtime_node_operation_action(virtualized); + let retained_site_name = virtualized_sites + .get(&site_hash) + .map(|state| state.site_name.clone()); + if let Some(reason) = bakery_reload_required_reason() { + let snapshot = if let Some(existing) = runtime_node_operations.get(&site_hash) + && existing.action == action + { + existing.snapshot() + } else { + let operation_id = *next_runtime_operation_id; + *next_runtime_operation_id = next_runtime_operation_id.saturating_add(1); + let mut operation = RuntimeNodeOperation::new( + operation_id, + site_hash, + retained_site_name.clone(), + action, + now_unix, + ); + operation.attempt_count = 1; + operation.update_status( + RuntimeNodeOperationStatus::Dirty, + now_unix, + Some(reason.clone()), + None, + ); + runtime_node_operations.insert(site_hash, operation.clone()); + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); + operation.snapshot() + }; + return snapshot; + } + if let Some(existing) = runtime_node_operations.get(&site_hash) + && runtime_node_operation_is_active(existing.status) + { + return existing.snapshot(); + } + + if runtime_virtualization_target_is_top_level(site_hash, sites, virtualized_sites) + && let Some(conflicting_site_hash) = active_top_level_runtime_operation_conflict( + site_hash, + runtime_node_operations, + sites, + virtualized_sites, + ) + { + let operation_id = runtime_node_operations + .get(&site_hash) + .map(|operation| operation.operation_id) + .unwrap_or_else(|| { + let next = *next_runtime_operation_id; + *next_runtime_operation_id = next_runtime_operation_id.saturating_add(1); + next + }); + let retry_at = now_unix.saturating_add(RUNTIME_NODE_OPERATION_DEFERRED_RETRY_SECONDS); + let mut operation = RuntimeNodeOperation::new( + operation_id, + site_hash, + retained_site_name.clone(), + action, + now_unix, + ); + operation.attempt_count = runtime_node_operations + .get(&site_hash) + .map(|existing| existing.attempt_count.saturating_add(1)) + .unwrap_or(1); + let site_label = runtime_site_label(site_hash, operation.site_name.as_deref()); + let summary = format!( + "Bakery allows only one top-level TreeGuard runtime operation in flight; site {} is still active, so TreeGuard {} for {} is deferred.", + conflicting_site_hash, + if virtualized { + "virtualization" + } else { + "restore" + }, + site_label + ); + operation.update_status( + RuntimeNodeOperationStatus::Deferred, + now_unix, + Some(summary.clone()), + Some(retry_at), + ); + runtime_node_operations.insert(site_hash, operation.clone()); + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); + push_bakery_event_with_site( + "runtime_node_op_deferred", + "warning", + Some(site_hash), + summary, + ); + return operation.snapshot(); + } + + if active_runtime_node_operation_count(runtime_node_operations) + >= RUNTIME_NODE_OPERATION_CAPACITY + { + let operation_id = runtime_node_operations + .get(&site_hash) + .map(|operation| operation.operation_id) + .unwrap_or_else(|| { + let next = *next_runtime_operation_id; + *next_runtime_operation_id = next_runtime_operation_id.saturating_add(1); + next + }); + let retry_at = now_unix.saturating_add(RUNTIME_NODE_OPERATION_DEFERRED_RETRY_SECONDS); + let mut operation = RuntimeNodeOperation::new( + operation_id, + site_hash, + retained_site_name.clone(), + action, + now_unix, + ); + operation.attempt_count = runtime_node_operations + .get(&site_hash) + .map(|existing| existing.attempt_count.saturating_add(1)) + .unwrap_or(1); + let site_label = runtime_site_label(site_hash, operation.site_name.as_deref()); + let summary = format!( + "Bakery runtime node operation capacity ({}) is saturated; deferring TreeGuard {} for {}.", + RUNTIME_NODE_OPERATION_CAPACITY, + if virtualized { + "virtualization" + } else { + "restore" + }, + site_label + ); + operation.update_status( + RuntimeNodeOperationStatus::Deferred, + now_unix, + Some(summary.clone()), + Some(retry_at), + ); + runtime_node_operations.insert(site_hash, operation.clone()); + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); + push_bakery_event_with_site( + "runtime_node_op_deferred", + "warning", + Some(site_hash), + summary, + ); + return operation.snapshot(); + } + + let operation_id = *next_runtime_operation_id; + *next_runtime_operation_id = next_runtime_operation_id.saturating_add(1); + let runtime_ops_snapshot = runtime_node_operations.clone(); + let sites_snapshot = sites.clone(); + let circuits_snapshot = circuits.clone(); + let qdisc_handles_snapshot = qdisc_handles.clone(); + let migrations_snapshot = migrations.clone(); + let virtualized_sites_snapshot = virtualized_sites.clone(); + + let mut operation = RuntimeNodeOperation::new( + operation_id, + site_hash, + retained_site_name, + action, + now_unix, + ); + operation.attempt_count = 1; + operation.update_status(RuntimeNodeOperationStatus::Applying, now_unix, None, None); + runtime_node_operations.insert(site_hash, operation.clone()); + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); + + let Ok(config) = lqos_config::load_config() else { + operation.update_status( + RuntimeNodeOperationStatus::Failed, + now_unix, + Some("Failed to load configuration".to_string()), + None, + ); + runtime_node_operations.insert(site_hash, operation.clone()); + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); + return operation.snapshot(); + }; + let current_site_names = load_current_runtime_site_names(&config); + if operation.site_name.is_none() { + operation.site_name = + resolve_runtime_site_name(site_hash, current_site_names.as_ref(), virtualized_sites); + } + let site_label = runtime_site_label(site_hash, operation.site_name.as_deref()); + if let Some(reason) = live_tree_mutation_blocker_for_config(&config) { + let summary = format!( + "TreeGuard runtime {} for {} is blocked because {}.", + if virtualized { + "virtualization" + } else { + "restore" + }, + site_label, + reason + ); + operation.update_status( + RuntimeNodeOperationStatus::Deferred, + now_unix, + Some(summary.clone()), + None, + ); + runtime_node_operations.insert(site_hash, operation.clone()); + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); + push_bakery_event_with_site("runtime_node_op_deferred", "info", Some(site_hash), summary); + return operation.snapshot(); + } + + if !cfg!(test) && !virtualized { + let stale_runtime_summary = virtualized_sites.get(&site_hash).and_then(|saved_state| { + current_site_names.as_ref().and_then(|names| { + stale_retained_runtime_branch_summary(site_hash, saved_state, names) + .map(|summary| (saved_state.site_name.clone(), summary)) + }) + }); + if let Some((saved_site_name, summary)) = stale_runtime_summary { + operation.site_name = Some(saved_site_name.clone()); + let summary_for_status = summary.clone(); + virtualized_sites.remove(&site_hash); + operation.update_status( + RuntimeNodeOperationStatus::Completed, + now_unix, + Some(summary_for_status), + None, + ); + runtime_node_operations.insert(site_hash, operation.clone()); + push_bakery_event_with_site_name( + "runtime_state_invalidated", + "warning", + Some(site_hash), + Some(saved_site_name), + summary, + ); + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); + return operation.snapshot(); + } + } + + let mut failure_reason = None; + let result: Result<(), String> = (|| { + if virtualized { + if virtualized_sites.contains_key(&site_hash) { + return Ok(()); + } + + let Some(target_site) = sites.get(&site_hash).cloned() else { + return Err(format!("Unknown site {}", site_label)); + }; + + if site_is_top_level(target_site.as_ref()) { + if let Some(reason) = top_level_runtime_virtualization_eligibility_error( + site_hash, + target_site.as_ref(), + sites, + circuits, + ) { + failure_reason = reason.failure_reason; + return Err(reason.message); + } + let plan = build_top_level_virtualization_plan( + Arc::clone(&target_site), + sites, + circuits, + Some(&config), + virtualized_sites, + site_stick_offset(target_site.as_ref()), + )?; + apply_site_command_update_stages( + &config, + sites, + &plan.active_sites, + &plan.site_stages, + "TreeGuard runtime top-level site reparent", + true, + )?; + apply_top_level_circuit_command_updates( + &config, + sites, + circuits, + &plan.active_circuits, + live_circuits, + mq_layout, + qdisc_handles, + migrations, + "TreeGuard runtime top-level circuit reparent", + )?; + sites.remove(&site_hash); + let state = VirtualizedSiteState { + site_name: operation + .site_name + .clone() + .unwrap_or_else(|| site_hash.to_string()), + site: target_site, + saved_sites: plan.saved_sites.clone(), + saved_circuits: plan.saved_circuits.clone(), + active_sites: plan + .active_sites + .into_iter() + .map(|(hash, update)| (hash, update.command)) + .collect(), + active_circuits: plan + .active_circuits + .into_iter() + .map(|(hash, update)| (hash, update.command)) + .collect(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::CutoverPending, + pending_prune: true, + next_prune_attempt_unix: now_unix.saturating_add(RUNTIME_CUTOVER_RETRY_SECONDS), + }; + debug!( + "Bakery: queued top-level runtime cutover for site {} with {} active sites and {} active circuits", + site_hash, + state.active_sites.len(), + state.active_circuits.len() + ); + virtualized_sites.insert(site_hash, state); + if let Some(inserted) = virtualized_sites.get(&site_hash) { + debug!( + "Bakery: inserted top-level runtime state for site {} lifecycle={} pending_prune={} next_prune_attempt_unix={} active_branch={:?}", + site_hash, + runtime_lifecycle_label(inserted.lifecycle), + inserted.pending_prune, + inserted.next_prune_attempt_unix, + inserted.active_branch + ); + } + return Ok(()); + } + + if let Some(reason) = + site_runtime_virtualization_eligibility_error(target_site.as_ref()) + { + return Err(reason); + } + + let plan = + build_non_top_level_virtualization_plan(Arc::clone(&target_site), sites, circuits)?; + apply_site_command_update_stages( + &config, + sites, + &plan.active_sites, + &plan.site_stages, + "TreeGuard runtime child-site shadow create", + true, + )?; + apply_circuit_command_updates( + &config, + sites, + circuits, + &plan.active_circuits, + live_circuits, + mq_layout, + qdisc_handles, + migrations, + "TreeGuard runtime circuit reparent", + )?; + sites.remove(&site_hash); + virtualized_sites.insert( + site_hash, + VirtualizedSiteState { + site_name: operation + .site_name + .clone() + .unwrap_or_else(|| site_hash.to_string()), + site: target_site, + saved_sites: plan.saved_sites.clone(), + saved_circuits: plan.saved_circuits.clone(), + active_sites: plan + .active_sites + .into_iter() + .map(|(hash, update)| (hash, update.command)) + .collect(), + active_circuits: plan + .active_circuits + .into_iter() + .map(|(hash, update)| (hash, update.command)) + .collect(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::CutoverPending, + pending_prune: true, + next_prune_attempt_unix: now_unix.saturating_add(RUNTIME_CUTOVER_RETRY_SECONDS), + }, + ); + return Ok(()); + } + + let Some(saved_state) = virtualized_sites.get(&site_hash).cloned() else { + return Ok(()); + }; + operation.site_name = Some(saved_state.site_name.clone()); + + if !site_is_top_level(saved_state.site.as_ref()) + && let Some(reason) = + site_runtime_virtualization_eligibility_error(saved_state.site.as_ref()) + { + return Err(reason); + } + + let reversible_standby = saved_state.active_branch_hides_original_site(); + let live_restore_snapshots = if reversible_standby { + Some(( + read_live_class_snapshot(&config.isp_interface())?, + read_live_class_snapshot(&config.internet_interface())?, + )) + } else { + None + }; + + if !saved_state.pending_prune + && let Some(cmds) = saved_state + .site + .to_commands(&config, ExecutionMode::Builder) + { + let root_already_live = + live_restore_snapshots + .as_ref() + .is_some_and(|(down_snapshot, up_snapshot)| { + let mut only_root = HashMap::new(); + only_root.insert(site_hash, saved_state.site.clone()); + site_commands_observed_live(&only_root, down_snapshot, up_snapshot) + }); + if !root_already_live { + let result = + execute_and_record_live_change(&cmds, "TreeGuard runtime hidden site restore"); + if !result.ok { + return Err(summarize_apply_result( + "TreeGuard runtime hidden site restore", + &result, + )); + } + } + } + sites.insert(site_hash, saved_state.site.clone()); + + let restore_sites: HashMap = saved_state + .saved_sites + .iter() + .map(|(hash, command)| { + ( + *hash, + PlannedSiteUpdate { + queue: current_site_queue(command.as_ref()).unwrap_or(1), + parent_site: None, + stage_depth: 0, + command: Arc::clone(command), + }, + ) + }) + .collect(); + let originals_already_live = + live_restore_snapshots + .as_ref() + .is_some_and(|(down_snapshot, up_snapshot)| { + site_commands_observed_live( + &saved_state.saved_sites, + down_snapshot, + up_snapshot, + ) + }); + if reversible_standby && originals_already_live { + for (hash, command) in &saved_state.saved_sites { + sites.insert(*hash, Arc::clone(command)); + } + } else { + apply_site_command_updates( + &config, + sites, + &restore_sites, + "TreeGuard runtime site restore", + )?; + } + + let restore_circuits: HashMap = + if saved_state.saved_circuits.is_empty() { + HashMap::new() + } else { + let Some(layout) = mq_layout.as_ref() else { + return Err( + "Bakery runtime virtualization requires MQ layout to be available" + .to_string(), + ); + }; + let live_reserved_handles = snapshot_live_qdisc_handle_majors_or_empty( + &config, + "TreeGuard runtime circuit restore", + ); + let updates = saved_state + .saved_circuits + .iter() + .map(|(hash, command)| { + let refreshed = assign_fresh_qdisc_handles_reserved( + command, + &config, + layout, + qdisc_handles, + &live_reserved_handles, + )?; + Ok(( + *hash, + PlannedCircuitUpdate { + queue: current_circuit_queue(refreshed.as_ref()).unwrap_or(1), + parent_site: None, + command: refreshed, + }, + )) + }) + .collect::, String>>()?; + apply_circuit_command_updates( + &config, + sites, + circuits, + &updates, + live_circuits, + mq_layout, + qdisc_handles, + migrations, + "TreeGuard runtime circuit restore", + )?; + updates + }; + + let restore_active_sites: HashMap> = restore_sites + .iter() + .map(|(hash, update)| (*hash, Arc::clone(&update.command))) + .collect(); + let restore_active_circuits: HashMap> = restore_circuits + .iter() + .map(|(hash, update)| (*hash, Arc::clone(&update.command))) + .collect(); + if let Some(restored_state) = build_post_restore_virtualized_state( + saved_state, + restore_active_sites, + restore_active_circuits, + now_unix, + ) { + virtualized_sites.insert(site_hash, restored_state); + } else { + virtualized_sites.remove(&site_hash); + } + Ok(()) + })(); + + if let Err(error) = result { + *sites = sites_snapshot; + *circuits = circuits_snapshot; + *qdisc_handles = qdisc_handles_snapshot; + *migrations = migrations_snapshot; + *virtualized_sites = virtualized_sites_snapshot; + *runtime_node_operations = runtime_ops_snapshot; + operation.update_status_with_reason( + RuntimeNodeOperationStatus::Failed, + unix_now(), + Some(error.clone()), + failure_reason, + None, + ); + runtime_node_operations.insert(site_hash, operation.clone()); + if runtime_error_suggests_material_desync(&error) { + mark_reload_required(format!( + "Bakery detected material runtime drift while processing TreeGuard {} for {}: {}", + if virtualized { + "virtualization" + } else { + "restore" + }, + runtime_site_label(site_hash, operation.site_name.as_deref()), + error + )); + } + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); + return operation.snapshot(); + } + + let finished_at = unix_now(); + let next_retry = virtualized_sites + .get(&site_hash) + .and_then(|state| state.pending_prune.then_some(state.next_prune_attempt_unix)); + let status = virtualized_sites + .get(&site_hash) + .map(runtime_status_for_virtualized_state) + .unwrap_or_else(|| { + if next_retry.is_some() { + RuntimeNodeOperationStatus::AppliedAwaitingCleanup + } else { + RuntimeNodeOperationStatus::Completed + } + }); + if let Some(state) = virtualized_sites.get(&site_hash) { + debug!( + "Bakery: final runtime status for site {} action={:?} lifecycle={} pending_prune={} next_retry={:?} computed_status={:?}", + site_hash, + action, + runtime_lifecycle_label(state.lifecycle), + state.pending_prune, + next_retry, + status + ); + } else { + debug!( + "Bakery: final runtime status for site {} action={:?} with no retained state next_retry={:?} computed_status={:?}", + site_hash, action, next_retry, status + ); + } + operation.update_status(status, finished_at, None, next_retry); + runtime_node_operations.insert(site_hash, operation.clone()); + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); + operation.snapshot() +} + +#[allow(clippy::too_many_arguments)] fn full_reload( batch: &mut Option>>, sites: &mut HashMap>, circuits: &mut HashMap>, live_circuits: &mut HashMap, + mq_layout: &mut Option, + qdisc_handles: &mut QdiscHandleState, config: &Arc, new_batch: Vec>, + resolved_mq_layout: Option, stormguard_overrides: &HashMap, + virtualized_sites: &mut HashMap, + runtime_node_operations: &mut HashMap, + trigger_summary: String, ) { - warn!("Bakery: Full reload triggered due to site or circuit changes."); - sites.clear(); - circuits.clear(); - live_circuits.clear(); - process_batch(new_batch, config, sites, circuits); + FULL_RELOAD_IN_PROGRESS.store(true, Ordering::Relaxed); + mark_bakery_action_started( + BakeryMode::ApplyingFullReload, + "full_reload_started", + trigger_summary, + ); + let _reload_scope = FullReloadScope; + let previous_sites = sites.clone(); + let previous_circuits = circuits.clone(); + let previous_live_circuits = live_circuits.clone(); + let previous_mq_layout = mq_layout.clone(); + let previous_mq_created = MQ_CREATED.load(Ordering::Relaxed); + let previous_shaping_tree_active = SHAPING_TREE_ACTIVE.load(Ordering::Relaxed); + + if let Err(error) = prepare_root_mq_for_full_reload(config) { + let summary = format!("Failed to prepare root mq state before full reload: {error}"); + error!("{summary}"); + mark_bakery_action_finished(BakeryApplyMetrics { + apply_type: BakeryApplyType::FullReload, + summary: &summary, + build_duration_ms: 0, + apply_duration_ms: 0, + total_tc_commands: 0, + class_commands: 0, + qdisc_commands: 0, + ok: false, + }); + *batch = None; + return; + } + invalidate_live_tc_snapshots(); + MQ_CREATED.store(true, Ordering::Relaxed); + + let live_reserved_handles = match snapshot_live_qdisc_handle_majors(config) { + Ok(handles) => handles, + Err(error) => { + let summary = + format!("Failed to snapshot live qdisc handles before full reload: {error}"); + error!("{summary}"); + mark_bakery_action_finished(BakeryApplyMetrics { + apply_type: BakeryApplyType::FullReload, + summary: &summary, + build_duration_ms: 0, + apply_duration_ms: 0, + total_tc_commands: 0, + class_commands: 0, + qdisc_commands: 0, + ok: false, + }); + *batch = None; + return; + } + }; + + let mut working_sites = HashMap::new(); + let mut working_circuits = HashMap::new(); + let mut working_qdisc_handles = QdiscHandleState::default(); + let layout = resolved_mq_layout.clone().unwrap_or_default(); + if resolved_mq_layout.is_none() { + warn!("Bakery: full reload skipped MQ layout restore because layout is unknown"); + } + + let result = process_batch( + new_batch, + config, + &mut working_sites, + &mut working_circuits, + &layout, + &mut working_qdisc_handles, + &live_reserved_handles, + ); + + if result.ok { + *sites = working_sites; + *circuits = working_circuits; + live_circuits.clear(); + *qdisc_handles = working_qdisc_handles; + qdisc_handles.save(config); + MQ_CREATED.store(true, Ordering::Relaxed); + SHAPING_TREE_ACTIVE.store(desired_shaping_tree_active(config), Ordering::Relaxed); + if resolved_mq_layout.is_some() { + *mq_layout = resolved_mq_layout; + } + virtualized_sites.clear(); + runtime_node_operations.clear(); + update_desync_state_from_runtime_state(runtime_node_operations, virtualized_sites); + clear_reload_required( + "A successful Bakery full reload re-established baseline state; incremental topology mutations can resume.", + ); + FIRST_COMMIT_APPLIED.store(true, Ordering::Relaxed); + apply_stormguard_overrides(stormguard_overrides, config); + } else { + *sites = previous_sites; + *circuits = previous_circuits; + *live_circuits = previous_live_circuits; + *mq_layout = previous_mq_layout; + MQ_CREATED.store(previous_mq_created, Ordering::Relaxed); + SHAPING_TREE_ACTIVE.store(previous_shaping_tree_active, Ordering::Relaxed); + } + if result.ok { + refresh_live_capacity_snapshot(config, true); + } + update_queue_distribution_snapshot(sites, circuits); *batch = None; - apply_stormguard_overrides(stormguard_overrides, config); } fn process_batch( @@ -1801,11 +9627,25 @@ fn process_batch( config: &Arc, sites: &mut HashMap>, circuits: &mut HashMap>, -) { + mq_layout: &MqDeviceLayout, + qdisc_handles: &mut QdiscHandleState, + extra_reserved_handles: &HashMap>, +) -> ExecuteResult { info!("Bakery: Processing batch of {} commands", batch.len()); + update_bakery_apply_progress(Some("Building tc command batch"), 0, 0, 0, 0); + let build_started = std::time::Instant::now(); let mut circuit_count = 0u64; let commands = batch .into_iter() + .map(|b| { + with_assigned_qdisc_handles_reserved( + &b, + config, + mq_layout, + qdisc_handles, + extra_reserved_handles, + ) + }) .filter_map(|b| { // Ensure that our state map is up to date with the latest commands match b.as_ref() { @@ -1825,17 +9665,63 @@ fn process_batch( let path = Path::new(&config.lqos_directory).join("linux_tc_rust.txt"); write_command_file(&path, &commands); - execute_in_memory(&commands, "processing batch"); + let build_duration_ms = build_started.elapsed().as_millis() as u64; + let (total_tc_commands, class_commands, qdisc_commands) = count_tc_command_types(&commands); + let total_chunks = if total_tc_commands == 0 { + 0 + } else { + total_tc_commands.div_ceil(FULL_RELOAD_TC_CHUNK_SIZE) + }; + update_bakery_apply_progress( + Some("Applying tc command chunks"), + total_tc_commands, + 0, + total_chunks, + 0, + ); + let result = execute_in_memory_chunked( + &commands, + "processing batch", + FULL_RELOAD_TC_CHUNK_SIZE, + Some(BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES), + |completed_tc_commands, total_tc_commands, completed_chunks, total_chunks| { + update_bakery_apply_progress( + Some("Applying tc command chunks"), + total_tc_commands, + completed_tc_commands, + total_chunks, + completed_chunks, + ); + }, + ); + let summary = summarize_apply_result("processing batch", &result); + maybe_emit_memory_guard_urgent(&summary); + mark_bakery_action_finished(BakeryApplyMetrics { + apply_type: BakeryApplyType::FullReload, + summary: &summary, + build_duration_ms, + apply_duration_ms: result.duration_ms, + total_tc_commands, + class_commands, + qdisc_commands, + ok: result.ok, + }); - // Mark that at least one batch has been applied, unblocking live activation. - FIRST_COMMIT_APPLIED.store(true, Ordering::Relaxed); + result } fn apply_stormguard_overrides( overrides: &HashMap, config: &Arc, ) { - let _ = config; // currently unused but kept for future interface-specific logic + if config.queues.queue_mode.is_observe() { + push_bakery_event( + "stormguard_override_replay_skipped", + "info", + "Skipping StormGuard HTB override replay because queue_mode is observe.".to_string(), + ); + return; + } if overrides.is_empty() { return; } @@ -1855,16 +9741,73 @@ fn apply_stormguard_overrides( format!("{}mbit", rate), ]); } - execute_in_memory(&commands, "replaying StormGuard overrides"); + let result = execute_in_memory(&commands, "replaying StormGuard overrides"); + if !result.ok { + push_bakery_event( + "stormguard_override_replay_failed", + "error", + summarize_apply_result("replaying StormGuard overrides", &result), + ); + } } #[cfg(test)] mod tests { use super::*; + use lqos_config::Config; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn bakery_test_lock() -> &'static std::sync::Mutex<()> { + crate::test_state_lock() + } + + fn install_bakery_test_config() { + static TEST_CONFIG_PATH: std::sync::OnceLock = + std::sync::OnceLock::new(); + + let config_path = TEST_CONFIG_PATH.get_or_init(|| { + let runtime_dir = std::env::temp_dir().join("lqos_bakery_test_runtime"); + std::fs::create_dir_all(&runtime_dir) + .expect("create lqos_bakery test runtime directory"); + + let config = lqos_config::Config { + lqos_directory: runtime_dir.display().to_string(), + bridge: Some(lqos_config::BridgeConfig { + use_xdp_bridge: false, + to_internet: "lo".to_string(), + to_network: "lo".to_string(), + }), + ..lqos_config::Config::default() + }; + + let config_path = std::env::temp_dir().join("lqos_bakery_test_lqos.conf"); + let raw = toml::to_string_pretty(&config).expect("serialize lqos_bakery test config"); + std::fs::write(&config_path, raw).expect("write lqos_bakery test config"); + config_path + }); + + // SAFETY: these unit tests call this helper only while holding the bakery test lock, + // so the process environment is mutated in a serialized way within this test binary. + unsafe { + std::env::set_var("LQOS_CONFIG", config_path); + } + lqos_config::clear_cached_config(); + } + + fn reset_bakery_test_state() { + *telemetry_state().write() = BakeryTelemetryState::default(); + MQ_CREATED.store(false, Ordering::Relaxed); + SHAPING_TREE_ACTIVE.store(false, Ordering::Relaxed); + FIRST_COMMIT_APPLIED.store(false, Ordering::Relaxed); + FULL_RELOAD_IN_PROGRESS.store(false, Ordering::Relaxed); + install_bakery_test_config(); + } fn mk_add_circuit(hash: i64, ip_addresses: &str) -> Arc { Arc::new(BakeryCommands::AddCircuit { circuit_hash: hash, + circuit_name: None, + site_name: None, parent_class_id: TcHandle::from_u32(0x1), up_parent_class_id: TcHandle::from_u32(0x2), class_minor: 0x10, @@ -1874,11 +9817,109 @@ mod tests { upload_bandwidth_max: 100.0, class_major: 0x100, up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, ip_addresses: ip_addresses.to_string(), sqm_override: None, }) } + fn mk_add_site( + site_hash: i64, + parent_class_id: u32, + up_parent_class_id: u32, + class_minor: u16, + ) -> Arc { + Arc::new(BakeryCommands::AddSite { + site_hash, + parent_class_id: TcHandle::from_u32(parent_class_id), + up_parent_class_id: TcHandle::from_u32(up_parent_class_id), + class_minor, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + }) + } + + fn test_config_with_runtime_dir(name: &str) -> Arc { + let mut cfg = Config::default(); + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time") + .as_nanos(); + let dir = std::env::temp_dir().join(format!("lqos-bakery-{name}-{ts}")); + std::fs::create_dir_all(&dir).expect("temp runtime dir"); + cfg.lqos_directory = dir.display().to_string(); + Arc::new(cfg) + } + + fn mk_test_circuit( + circuit_hash: i64, + parent_class_id: u32, + up_parent_class_id: u32, + class_minor: u16, + class_major: u16, + up_class_major: u16, + ip_addresses: &str, + ) -> Arc { + Arc::new(BakeryCommands::AddCircuit { + circuit_hash, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(parent_class_id), + up_parent_class_id: TcHandle::from_u32(up_parent_class_id), + class_minor, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major, + up_class_major, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + ip_addresses: ip_addresses.to_string(), + sqm_override: None, + }) + } + + fn mk_runtime_operation( + operation_id: u64, + site_hash: i64, + action: RuntimeNodeOperationAction, + status: RuntimeNodeOperationStatus, + attempt_count: u32, + next_retry_at_unix: Option, + ) -> RuntimeNodeOperation { + let now = unix_now(); + let mut operation = RuntimeNodeOperation::new(operation_id, site_hash, None, action, now); + operation.attempt_count = attempt_count; + operation.update_status(status, now, None, next_retry_at_unix); + operation + } + + fn live_class_entry(class_id: u32, parent: Option) -> LiveTcClassEntry { + LiveTcClassEntry { + class_id: TcHandle::from_u32(class_id), + parent: parent.map(TcHandle::from_u32), + leaf_qdisc_major: None, + } + } + + fn live_qdisc_entry( + kind: &str, + handle: Option, + parent: Option, + is_root: bool, + ) -> LiveTcQdiscEntry { + LiveTcQdiscEntry { + kind: kind.to_string(), + handle: handle.map(TcHandle::from_u32), + parent: parent.map(TcHandle::from_u32), + is_root, + } + } + fn has_hash(batch: &[Arc], hash: i64) -> bool { batch.iter().any(|cmd| { matches!( @@ -1951,6 +9992,1700 @@ mod tests { assert_eq!(filtered.len(), 1001); } + #[test] + fn runtime_virtualization_overlay_hides_virtualized_site_and_reparents_children() { + let parent_site = mk_add_site(10, 0x10001, 0x20001, 0x20); + let virtualized_site = mk_add_site(20, 0x10020, 0x20020, 0x21); + let child_site = mk_add_site(30, 0x10021, 0x20021, 0x22); + let child_circuit = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 40, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10021), + up_parent_class_id: TcHandle::from_u32(0x20021), + class_minor: 0x30, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x110, + up_class_major: 0x210, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.40/32".to_string(), + sqm_override: None, + }); + + let mut virtualized_sites = HashMap::new(); + virtualized_sites.insert( + 20, + VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: virtualized_site.clone(), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::from([( + 30, + Arc::new(BakeryCommands::AddSite { + site_hash: 30, + parent_class_id: TcHandle::from_u32(0x10020), + up_parent_class_id: TcHandle::from_u32(0x20020), + class_minor: 0x22, + download_bandwidth_min: 50.0, + upload_bandwidth_min: 50.0, + download_bandwidth_max: 500.0, + upload_bandwidth_max: 500.0, + }), + )]), + active_circuits: HashMap::from([( + 40, + Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 40, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10020), + up_parent_class_id: TcHandle::from_u32(0x20020), + class_minor: 0x30, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x110, + up_class_major: 0x210, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.40/32".to_string(), + sqm_override: None, + }), + )]), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: false, + next_prune_attempt_unix: 0, + }, + ); + + let overlaid = apply_runtime_virtualization_overlay( + vec![parent_site, virtualized_site, child_site, child_circuit], + &virtualized_sites, + ); + + assert_eq!(overlaid.len(), 3); + assert!(!overlaid.iter().any(|cmd| matches!( + cmd.as_ref(), + BakeryCommands::AddSite { site_hash, .. } if *site_hash == 20 + ))); + + let child_site = overlaid + .iter() + .find_map(|cmd| match cmd.as_ref() { + BakeryCommands::AddSite { + site_hash, + parent_class_id, + up_parent_class_id, + .. + } if *site_hash == 30 => Some((*parent_class_id, *up_parent_class_id)), + _ => None, + }) + .expect("child site should remain in batch"); + assert_eq!(child_site.0, TcHandle::from_u32(0x10020)); + assert_eq!(child_site.1, TcHandle::from_u32(0x20020)); + + let child_circuit = overlaid + .iter() + .find_map(|cmd| match cmd.as_ref() { + BakeryCommands::AddCircuit { + circuit_hash, + parent_class_id, + up_parent_class_id, + .. + } if *circuit_hash == 40 => Some((*parent_class_id, *up_parent_class_id)), + _ => None, + }) + .expect("child circuit should remain in batch"); + assert_eq!(child_circuit.0, TcHandle::from_u32(0x10020)); + assert_eq!(child_circuit.1, TcHandle::from_u32(0x20020)); + } + + #[test] + fn structural_baseline_reconstruction_ignores_hidden_runtime_virtualized_sites() { + let parent_site = mk_add_site(10, 0x10001, 0x20001, 0x20); + let virtualized_site = mk_add_site(20, 0x10020, 0x20020, 0x21); + let child_site = mk_add_site(30, 0x10021, 0x20021, 0x22); + let child_site_runtime = Arc::new(BakeryCommands::AddSite { + site_hash: 30, + parent_class_id: TcHandle::from_u32(0x10020), + up_parent_class_id: TcHandle::from_u32(0x20020), + class_minor: 0x22, + download_bandwidth_min: 50.0, + upload_bandwidth_min: 50.0, + download_bandwidth_max: 500.0, + upload_bandwidth_max: 500.0, + }); + let child_circuit = + mk_test_circuit(40, 0x10021, 0x20021, 0x30, 0x110, 0x210, "192.0.2.40/32"); + let child_circuit_runtime = + mk_test_circuit(40, 0x10020, 0x20020, 0x30, 0x110, 0x210, "192.0.2.40/32"); + + let effective_sites = HashMap::from([ + (10, Arc::clone(&parent_site)), + (30, Arc::clone(&child_site_runtime)), + ]); + let effective_circuits = HashMap::from([(40, Arc::clone(&child_circuit_runtime))]); + let virtualized_sites = HashMap::from([( + 20, + VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: Arc::clone(&virtualized_site), + saved_sites: HashMap::from([(30, Arc::clone(&child_site))]), + saved_circuits: HashMap::from([(40, Arc::clone(&child_circuit))]), + active_sites: HashMap::from([(30, Arc::clone(&child_site_runtime))]), + active_circuits: HashMap::from([(40, Arc::clone(&child_circuit_runtime))]), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: true, + next_prune_attempt_unix: 0, + }, + )]); + + let raw_batch = vec![ + Arc::clone(&parent_site), + Arc::clone(&virtualized_site), + Arc::clone(&child_site), + Arc::clone(&child_circuit), + ]; + + let (baseline_sites, baseline_circuits) = reconstruct_structural_baseline_state( + &effective_sites, + &effective_circuits, + &virtualized_sites, + ); + + assert!(matches!( + diff_sites(&raw_batch, &effective_sites), + SiteDiffResult::RebuildRequired { .. } + )); + assert!(matches!( + diff_sites(&raw_batch, &baseline_sites), + SiteDiffResult::NoChange + )); + assert!(matches!( + baseline_circuits + .get(&40) + .expect("baseline circuit restored") + .as_ref(), + BakeryCommands::AddCircuit { + circuit_hash, + parent_class_id, + up_parent_class_id, + .. + } if *circuit_hash == 40 + && *parent_class_id == TcHandle::from_u32(0x10021) + && *up_parent_class_id == TcHandle::from_u32(0x20021) + )); + } + + #[test] + fn runtime_virtualized_site_prune_ready_respects_backoff_and_migrations() { + let state = VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(20, 0x10020, 0x20020, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::from([(40, mk_add_circuit(40, "192.0.2.40/32"))]), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: true, + next_prune_attempt_unix: 120, + }; + + let mut migrations = HashMap::new(); + migrations.insert( + 40, + Migration { + circuit_hash: 40, + circuit_name: None, + site_name: None, + old_class_major: 0x100, + old_up_class_major: 0x200, + old_down_qdisc_handle: None, + old_up_qdisc_handle: None, + parent_class_id: TcHandle::from_u32(0x1), + up_parent_class_id: TcHandle::from_u32(0x2), + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + old_down_min: 1.0, + old_down_max: 1.0, + old_up_min: 1.0, + old_up_max: 1.0, + new_down_min: 1.0, + new_down_max: 1.0, + new_up_min: 1.0, + new_up_max: 1.0, + old_minor: 0x10, + shadow_minor: 0x31, + final_minor: 0x10, + ips: vec!["192.0.2.40/32".to_string()], + sqm_override: None, + desired_cmd: mk_add_circuit(40, "192.0.2.40/32"), + stage: MigrationStage::PrepareShadow, + shadow_verify_attempts: 0, + final_verify_attempts: 0, + }, + ); + + assert!(!runtime_virtualized_site_prune_ready( + &state, + &migrations, + 119 + )); + assert!(!runtime_virtualized_site_prune_ready( + &state, + &migrations, + 120 + )); + + migrations.clear(); + assert!(!runtime_virtualized_site_prune_ready( + &state, + &migrations, + 119 + )); + assert!(runtime_virtualized_site_prune_ready( + &state, + &migrations, + 120 + )); + } + + #[test] + fn ordered_prune_site_hashes_runs_children_before_parents() { + let parent = mk_add_site(30, 0x10021, 0x20021, 0x22); + let child = mk_add_site(31, 0x10022, 0x20022, 0x23); + let ordered = ordered_prune_site_hashes(&HashMap::from([(30, parent), (31, child)])); + assert_eq!(ordered, vec![31, 30]); + } + + #[test] + fn site_prune_commands_delete_qdisc_by_handle_before_class() { + let config = Arc::new(lqos_config::Config::default()); + let state = VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(20, 0x10020, 0x20020, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: Some(0x9000), + up: Some(0x9001), + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: true, + next_prune_attempt_unix: 0, + }; + let commands = site_prune_commands(&config, &state).expect("site prune commands"); + + assert_eq!(commands.len(), 4); + assert_eq!(commands[0][0], "qdisc"); + assert_eq!(commands[0][1], "del"); + assert_eq!(commands[0][4], "handle"); + assert_eq!(commands[0][5], "0x9000:"); + assert_eq!(commands[1][0], "qdisc"); + assert_eq!(commands[1][1], "del"); + assert_eq!(commands[1][4], "handle"); + assert_eq!(commands[1][5], "0x9001:"); + assert_eq!(commands[2][0], "class"); + assert_eq!(commands[2][1], "del"); + assert_eq!(commands[3][0], "class"); + assert_eq!(commands[3][1], "del"); + } + + #[test] + fn site_prune_commands_skip_qdisc_delete_without_tracked_handles() { + let config = Arc::new(lqos_config::Config::default()); + let state = VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(20, 0x10020, 0x20020, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: true, + next_prune_attempt_unix: 0, + }; + + let commands = site_prune_commands(&config, &state).expect("site prune commands"); + assert_eq!(commands.len(), 2); + assert_eq!(commands[0][0], "class"); + assert_eq!(commands[1][0], "class"); + } + + #[test] + fn observed_circuit_prune_commands_only_delete_existing_old_state() { + let config = Arc::new(lqos_config::Config::default()); + let circuit = mk_test_circuit(9300, 0x10020, 0x20020, 0x21, 0x1, 0x2, "192.0.2.94/32"); + + let mut down_snapshot = HashMap::new(); + down_snapshot.insert( + TcHandle::from_u32(0x10021), + LiveTcClassEntry { + class_id: TcHandle::from_u32(0x10021), + parent: Some(TcHandle::from_u32(0x10020)), + leaf_qdisc_major: Some(0x9000), + }, + ); + let up_snapshot = HashMap::new(); + + let commands = observed_circuit_prune_commands( + &config, + circuit.as_ref(), + Some(&down_snapshot), + Some(&up_snapshot), + &HashSet::new(), + &HashSet::new(), + ) + .expect("observed circuit prune commands"); + + assert_eq!(commands.len(), 2); + assert_eq!(commands[0][0], "qdisc"); + assert_eq!(commands[0][3], config.isp_interface()); + assert_eq!(commands[1][0], "class"); + assert_eq!(commands[1][3], config.isp_interface()); + } + + #[test] + fn observed_circuit_prune_commands_skip_missing_old_classes() { + let config = Arc::new(lqos_config::Config::default()); + let circuit = mk_test_circuit(9301, 0x10020, 0x20020, 0x21, 0x1, 0x2, "192.0.2.95/32"); + let down_snapshot = HashMap::new(); + let up_snapshot = HashMap::new(); + + let commands = observed_circuit_prune_commands( + &config, + circuit.as_ref(), + Some(&down_snapshot), + Some(&up_snapshot), + &HashSet::new(), + &HashSet::new(), + ) + .expect("observed circuit prune commands"); + + assert!(commands.is_empty()); + } + + #[test] + fn observed_circuit_prune_commands_skip_handles_reserved_by_batch_targets() { + let config = Arc::new(lqos_config::Config::default()); + let circuit = mk_test_circuit(9302, 0x10020, 0x20020, 0x21, 0x1, 0x2, "192.0.2.96/32"); + + let mut down_snapshot = HashMap::new(); + down_snapshot.insert( + TcHandle::from_u32(0x10021), + LiveTcClassEntry { + class_id: TcHandle::from_u32(0x10021), + parent: Some(TcHandle::from_u32(0x10020)), + leaf_qdisc_major: Some(0x9000), + }, + ); + let mut up_snapshot = HashMap::new(); + up_snapshot.insert( + TcHandle::from_u32(0x20021), + LiveTcClassEntry { + class_id: TcHandle::from_u32(0x20021), + parent: Some(TcHandle::from_u32(0x20020)), + leaf_qdisc_major: Some(0x9001), + }, + ); + + let protected_down = HashSet::from([TcHandle::from_u32(0x10021)]); + let protected_up = HashSet::from([TcHandle::from_u32(0x20021)]); + + let commands = observed_circuit_prune_commands( + &config, + circuit.as_ref(), + Some(&down_snapshot), + Some(&up_snapshot), + &protected_down, + &protected_up, + ) + .expect("observed circuit prune commands"); + + assert!(commands.is_empty()); + } + + #[test] + fn sync_runtime_virtualized_site_qdisc_handles_from_live_snapshot_recovers_leaf_handles() { + let mut state = VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(20, 0x10020, 0x20020, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: true, + next_prune_attempt_unix: 0, + }; + + let mut down_snapshot = HashMap::new(); + down_snapshot.insert( + TcHandle::from_u32(0x10021), + LiveTcClassEntry { + class_id: TcHandle::from_u32(0x10021), + parent: Some(TcHandle::from_u32(0x10020)), + leaf_qdisc_major: Some(0x9000), + }, + ); + let mut up_snapshot = HashMap::new(); + up_snapshot.insert( + TcHandle::from_u32(0x20021), + LiveTcClassEntry { + class_id: TcHandle::from_u32(0x20021), + parent: Some(TcHandle::from_u32(0x20020)), + leaf_qdisc_major: Some(0x9001), + }, + ); + + sync_runtime_virtualized_site_qdisc_handles_from_live_snapshot( + &mut state, + &down_snapshot, + &up_snapshot, + ); + + assert_eq!(state.qdisc_handles.down, Some(0x9000)); + assert_eq!(state.qdisc_handles.up, Some(0x9001)); + } + + #[test] + fn runtime_virtualized_site_observed_child_gate_blocks_prune() { + let state = VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(20, 0x10020, 0x20020, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: true, + next_prune_attempt_unix: 0, + }; + + let mut down_snapshot = HashMap::new(); + down_snapshot.insert( + TcHandle::from_u32(0x10022), + LiveTcClassEntry { + class_id: TcHandle::from_u32(0x10022), + parent: Some(TcHandle::from_u32(0x10021)), + leaf_qdisc_major: Some(0x9000), + }, + ); + let mut up_snapshot = HashMap::new(); + up_snapshot.insert( + TcHandle::from_u32(0x20022), + LiveTcClassEntry { + class_id: TcHandle::from_u32(0x20022), + parent: Some(TcHandle::from_u32(0x20021)), + leaf_qdisc_major: Some(0x9001), + }, + ); + + assert!( + runtime_virtualized_site_has_remaining_observed_child_classes( + &state, + &down_snapshot, + &up_snapshot + ) + ); + } + + #[test] + fn site_prune_class_commands_for_observed_state_only_emits_existing_directions() { + let config = Arc::new(lqos_config::Config::default()); + let site = mk_add_site(20, 0x10020, 0x20020, 0x21); + + let down_only = + site_prune_class_commands_for_observed_state(&config, site.as_ref(), true, false) + .expect("down-only commands"); + assert_eq!(down_only.len(), 1); + assert_eq!(down_only[0][0], "class"); + assert_eq!(down_only[0][3], config.isp_interface()); + + let up_only = + site_prune_class_commands_for_observed_state(&config, site.as_ref(), false, true) + .expect("up-only commands"); + assert_eq!(up_only.len(), 1); + assert_eq!(up_only[0][0], "class"); + assert_eq!(up_only[0][3], config.internet_interface()); + + assert!( + site_prune_class_commands_for_observed_state(&config, site.as_ref(), false, false) + .is_none() + ); + } + + #[test] + fn cutover_pending_maps_to_applying_status() { + let state = VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(20, 0x10020, 0x20020, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::CutoverPending, + pending_prune: true, + next_prune_attempt_unix: 0, + }; + + assert_eq!( + runtime_status_for_virtualized_state(&state), + RuntimeNodeOperationStatus::Applying + ); + } + + #[test] + fn cutover_failure_counts_as_material_desync() { + assert!(runtime_error_suggests_material_desync( + "Top-level runtime cutover verification failed to converge: Observed live child classes still attached" + )); + assert!(runtime_error_suggests_material_desync( + "Inactive branch cleanup encountered unexpected live child classes still attached: down [0x4:0xf]" + )); + } + + #[test] + fn stale_retained_runtime_branch_summary_includes_site_name_for_missing_root() { + let saved_state = VirtualizedSiteState { + site_name: "7232 Rochester".to_string(), + site: mk_add_site(20, 0x10000, 0x20000, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: false, + next_prune_attempt_unix: 0, + }; + + let summary = stale_retained_runtime_branch_summary(20, &saved_state, &HashMap::new()) + .expect("stale root summary"); + + assert!(summary.contains("7232 Rochester")); + assert!(summary.contains("20")); + assert!(summary.contains("no longer present in current topology")); + } + + #[test] + fn top_level_virtualization_plan_promotes_children_and_direct_circuits() { + let target_site = mk_add_site(20, 0x10000, 0x20000, 0x21); + let sibling_site = mk_add_site(10, 0x10000, 0x20000, 0x20); + let child_site = mk_add_site(30, 0x10021, 0x20021, 0x22); + let grandchild_site = mk_add_site(31, 0x10022, 0x20022, 0x23); + let direct_circuit = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 40, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10021), + up_parent_class_id: TcHandle::from_u32(0x20021), + class_minor: 0x30, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x1, + up_class_major: 0x2, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.40/32".to_string(), + sqm_override: None, + }); + + let mut sites = HashMap::new(); + sites.insert(10, sibling_site); + sites.insert(20, Arc::clone(&target_site)); + sites.insert(30, child_site); + sites.insert(31, grandchild_site); + + let mut circuits = HashMap::new(); + circuits.insert(40, direct_circuit); + + let plan = build_top_level_virtualization_plan( + Arc::clone(&target_site), + &sites, + &circuits, + None, + &HashMap::new(), + 1, + ) + .expect("top-level plan should build"); + + assert!(plan.saved_sites.contains_key(&30)); + assert!(plan.saved_sites.contains_key(&31)); + assert!(plan.saved_circuits.contains_key(&40)); + assert!( + plan.active_sites + .keys() + .all(|hash| plan.saved_sites.contains_key(hash)) + ); + + let promoted_site = plan + .active_sites + .get(&30) + .expect("promoted child site should be rewritten"); + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + .. + } = promoted_site.command.as_ref() + else { + panic!("expected AddSite"); + }; + assert_eq!(parent_class_id.get_major_minor().1, 0); + assert_eq!(up_parent_class_id.get_major_minor().1, 0); + + let promoted_circuit = plan + .active_circuits + .get(&40) + .expect("direct circuit should be rewritten"); + let BakeryCommands::AddCircuit { + parent_class_id, + up_parent_class_id, + .. + } = promoted_circuit.command.as_ref() + else { + panic!("expected AddCircuit"); + }; + assert_eq!(parent_class_id.get_major_minor().1, 0); + assert_eq!(up_parent_class_id.get_major_minor().1, 0); + assert_eq!(plan.site_stages, vec![vec![30], vec![31]]); + assert_eq!( + plan.active_sites.get(&30).expect("child site").stage_depth, + 0 + ); + assert_eq!( + plan.active_sites + .get(&31) + .expect("grandchild site") + .stage_depth, + 1 + ); + } + + #[test] + fn non_top_level_virtualization_plan_uses_shadow_minor_unique_within_major_domain() { + let parent_site = mk_add_site(10, 0x10003, 0x20003, 0x2002); + let target_site = mk_add_site(20, 0x10003, 0x20003, 0x2003); + let child_site = mk_add_site(30, 0x12003, 0x22003, 0x2000); + + let mut sites = HashMap::new(); + sites.insert(10, parent_site); + sites.insert(20, Arc::clone(&target_site)); + sites.insert(30, Arc::clone(&child_site)); + + let circuits = HashMap::new(); + + let plan = + build_non_top_level_virtualization_plan(Arc::clone(&target_site), &sites, &circuits) + .expect("non-top-level plan should build"); + + let rewritten_child = plan + .active_sites + .get(&30) + .expect("child site should be shadowed"); + let BakeryCommands::AddSite { + parent_class_id, + up_parent_class_id, + class_minor, + .. + } = rewritten_child.command.as_ref() + else { + panic!("expected AddSite"); + }; + + assert_eq!(*parent_class_id, TcHandle::from_u32(0x10003)); + assert_eq!(*up_parent_class_id, TcHandle::from_u32(0x20003)); + assert_ne!( + *class_minor, 0x2000, + "shadow child site minor must not reuse an existing classid in the same major domain" + ); + assert_eq!(plan.site_stages, vec![vec![30]]); + } + + #[test] + fn non_top_level_virtualization_plan_stages_shadow_sites_by_depth() { + let parent_site = mk_add_site(10, 0x10003, 0x20003, 0x2002); + let target_site = mk_add_site(20, 0x10003, 0x20003, 0x2003); + let child_site = mk_add_site(30, 0x12003, 0x22003, 0x2000); + let grandchild_site = mk_add_site(40, 0x12000, 0x22000, 0x2001); + + let mut sites = HashMap::new(); + sites.insert(10, parent_site); + sites.insert(20, Arc::clone(&target_site)); + sites.insert(30, Arc::clone(&child_site)); + sites.insert(40, Arc::clone(&grandchild_site)); + + let circuits = HashMap::new(); + + let plan = + build_non_top_level_virtualization_plan(Arc::clone(&target_site), &sites, &circuits) + .expect("non-top-level plan should build"); + + assert_eq!(plan.site_stages, vec![vec![30], vec![40]]); + assert_eq!( + plan.active_sites.get(&30).expect("child site").stage_depth, + 0 + ); + assert_eq!( + plan.active_sites + .get(&40) + .expect("grandchild site") + .stage_depth, + 1 + ); + } + + #[test] + fn site_runtime_virtualization_rejects_top_level_sites() { + let site = mk_add_site(99, 0x10000, 0x20000, 0x21); + let reason = site_runtime_virtualization_eligibility_error(site.as_ref()) + .expect("top-level site should be rejected"); + assert!(reason.contains("top-level")); + } + + #[test] + fn site_runtime_virtualization_rejects_queue_root_child_sites_as_top_level() { + let site = mk_add_site(99, 0x10003, 0x20003, 0x21); + let reason = site_runtime_virtualization_eligibility_error(site.as_ref()) + .expect("queue-root child site should be rejected as top-level"); + assert!(reason.contains("top-level")); + } + + #[test] + fn site_runtime_virtualization_accepts_normal_same_queue_site() { + let site = mk_add_site(77, 0x10020, 0x20020, 0x22); + assert!(site_runtime_virtualization_eligibility_error(site.as_ref()).is_none()); + } + + #[test] + fn site_runtime_virtualization_rejects_non_site_commands() { + let circuit = mk_add_circuit(77, "192.0.2.77/32"); + let reason = site_runtime_virtualization_eligibility_error(circuit.as_ref()) + .expect("non-site commands should be rejected"); + assert!(reason.contains("AddSite command")); + } + + #[test] + fn top_level_runtime_virtualization_rejects_single_promotable_child() { + let target_site = mk_add_site(20, 0x10000, 0x20000, 0x21); + let only_child = mk_add_site(30, 0x10021, 0x20021, 0x22); + + let mut sites = HashMap::new(); + sites.insert(20, Arc::clone(&target_site)); + sites.insert(30, only_child); + + let reason = top_level_runtime_virtualization_eligibility_error( + 20, + target_site.as_ref(), + &sites, + &HashMap::new(), + ) + .expect("single-child top-level virtualization should be rejected"); + + assert!(reason.message.contains("only one promotable direct child")); + assert_eq!( + reason.failure_reason, + Some(RuntimeNodeOperationFailureReason::StructuralIneligibleSinglePromotableChild) + ); + } + + #[test] + fn runtime_node_capacity_returns_deferred_status() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + + let mut runtime_node_operations = HashMap::new(); + for index in 0..RUNTIME_NODE_OPERATION_CAPACITY { + let site_hash = index as i64 + 1; + let mut operation = RuntimeNodeOperation::new( + site_hash as u64, + site_hash, + None, + RuntimeNodeOperationAction::Virtualize, + unix_now(), + ); + operation.attempt_count = 1; + operation.update_status(RuntimeNodeOperationStatus::Applying, unix_now(), None, None); + runtime_node_operations.insert(site_hash, operation); + } + update_desync_state_from_runtime_state(&runtime_node_operations, &HashMap::new()); + + let snapshot = handle_treeguard_set_node_virtual_live( + 999, + true, + &mut HashMap::new(), + &mut HashMap::new(), + &HashMap::new(), + &None, + &mut QdiscHandleState::default(), + &mut HashMap::new(), + &mut HashMap::new(), + &mut runtime_node_operations, + &mut 10_000, + ); + + assert_eq!(snapshot.status, RuntimeNodeOperationStatus::Deferred); + assert!(snapshot.next_retry_at_unix.is_some()); + assert!( + snapshot + .last_error + .as_deref() + .unwrap_or_default() + .contains("capacity") + ); + + let telemetry = bakery_status_snapshot(); + assert_eq!(telemetry.runtime_operations.deferred_count, 1); + assert!(!telemetry.reload_required); + } + + #[test] + fn top_level_runtime_operations_are_serialized() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + + let site_a = mk_add_site(20, 0x10000, 0x20000, 0x21); + let site_b = mk_add_site(21, 0x110000, 0x120000, 0x22); + let mut sites = HashMap::from([(20, site_a), (21, site_b)]); + + let mut runtime_node_operations = HashMap::new(); + let mut operation = RuntimeNodeOperation::new( + 1, + 20, + None, + RuntimeNodeOperationAction::Virtualize, + unix_now(), + ); + operation.attempt_count = 1; + operation.update_status(RuntimeNodeOperationStatus::Applying, unix_now(), None, None); + runtime_node_operations.insert(20, operation); + update_desync_state_from_runtime_state(&runtime_node_operations, &HashMap::new()); + + let snapshot = handle_treeguard_set_node_virtual_live( + 21, + true, + &mut sites, + &mut HashMap::new(), + &HashMap::new(), + &None, + &mut QdiscHandleState::default(), + &mut HashMap::new(), + &mut HashMap::new(), + &mut runtime_node_operations, + &mut 10_000, + ); + + assert_eq!(snapshot.status, RuntimeNodeOperationStatus::Deferred); + assert!(snapshot.next_retry_at_unix.is_some()); + assert!( + snapshot + .last_error + .as_deref() + .unwrap_or_default() + .contains("one top-level TreeGuard runtime operation in flight") + ); + } + + #[test] + fn non_top_level_runtime_virtualization_enters_cutover_pending_without_pruning_standby() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + SHAPING_TREE_ACTIVE.store(true, Ordering::Relaxed); + + let parent_site = mk_add_site(10, 0x10003, 0x20003, 0x20); + let target_site = mk_add_site(20, 0x10020, 0x20020, 0x21); + let mut sites = HashMap::from([(10, parent_site), (20, target_site)]); + let mut circuits = HashMap::new(); + let mut qdisc_handles = QdiscHandleState::default(); + let mut migrations = HashMap::new(); + let mut virtualized_sites = HashMap::new(); + let mut runtime_node_operations = HashMap::new(); + let mut next_runtime_operation_id = 10_000; + + let snapshot = handle_treeguard_set_node_virtual_live( + 20, + true, + &mut sites, + &mut circuits, + &HashMap::new(), + &None, + &mut qdisc_handles, + &mut migrations, + &mut virtualized_sites, + &mut runtime_node_operations, + &mut next_runtime_operation_id, + ); + + assert_eq!(snapshot.status, RuntimeNodeOperationStatus::Applying); + let state = virtualized_sites + .get(&20) + .expect("non-top-level virtualized site state"); + assert_eq!( + state.lifecycle, + RuntimeVirtualizedBranchLifecycle::CutoverPending + ); + assert_eq!(state.active_branch, RuntimeVirtualizedActiveBranch::Shadow); + assert!(state.pending_prune); + assert!(state.prune_sites.is_empty()); + assert!(state.prune_circuits.is_empty()); + } + + #[test] + fn restore_with_pending_cleanup_is_not_reported_completed() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + SHAPING_TREE_ACTIVE.store(true, Ordering::Relaxed); + + let site_hash = 20; + let standby_site = mk_add_site(site_hash, 0x10000, 0x20000, 0x21); + let shadow_child = mk_add_site(30, 0x10020, 0x20020, 0x22); + let mut virtualized_sites = HashMap::from([( + site_hash, + VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: Arc::clone(&standby_site), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::from([(30, shadow_child)]), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: true, + next_prune_attempt_unix: unix_now(), + }, + )]); + + let snapshot = handle_treeguard_set_node_virtual_live( + site_hash, + false, + &mut HashMap::new(), + &mut HashMap::new(), + &HashMap::new(), + &None, + &mut QdiscHandleState::default(), + &mut HashMap::new(), + &mut virtualized_sites, + &mut HashMap::new(), + &mut 10_000, + ); + + assert_eq!( + snapshot.status, + RuntimeNodeOperationStatus::AppliedAwaitingCleanup + ); + let state = virtualized_sites + .get(&site_hash) + .expect("restored virtualized site state"); + assert_eq!( + state.active_branch, + RuntimeVirtualizedActiveBranch::Original + ); + assert!(!state.active_branch_hides_original_site()); + assert!(state.pending_prune); + } + + #[test] + fn deferred_restore_cleanup_completes_and_removes_retained_state() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + + let config = Arc::new(Config::default()); + let site_hash = 20; + let site = mk_add_site(site_hash, 0x10020, 0x20020, 0x21); + let mut virtualized_sites = HashMap::from([( + site_hash, + VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site, + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Original, + lifecycle: RuntimeVirtualizedBranchLifecycle::PhysicalActiveCleanupPending, + pending_prune: true, + next_prune_attempt_unix: 0, + }, + )]); + let mut runtime_node_operations = HashMap::from([( + site_hash, + mk_runtime_operation( + 1, + site_hash, + RuntimeNodeOperationAction::Restore, + RuntimeNodeOperationStatus::AppliedAwaitingCleanup, + 1, + Some(0), + ), + )]); + + flush_deferred_runtime_site_prunes_with_snapshots( + &config, + &mut virtualized_sites, + &HashMap::new(), + &mut runtime_node_operations, + &HashMap::new(), + &HashMap::new(), + ); + + assert!(!virtualized_sites.contains_key(&site_hash)); + let operation = runtime_node_operations + .get(&site_hash) + .expect("runtime operation"); + assert_eq!(operation.status, RuntimeNodeOperationStatus::Completed); + assert_eq!(operation.next_retry_at_unix, None); + + let telemetry = bakery_status_snapshot(); + assert_eq!(telemetry.runtime_operations.awaiting_cleanup_count, 0); + assert_eq!( + bakery_runtime_node_operation_snapshot(site_hash) + .expect("runtime op snapshot") + .status, + RuntimeNodeOperationStatus::Completed + ); + } + + #[test] + fn cutover_pending_converges_to_completed_when_shadow_is_verified_active() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + + let config = Arc::new(Config::default()); + let site_hash = 20; + let active_site = mk_add_site(30, 0x10020, 0x20020, 0x22); + let active_circuit = + mk_test_circuit(40, 0x10020, 0x20020, 0x30, 0x110, 0x210, "192.0.2.40/32"); + let mut virtualized_sites = HashMap::from([( + site_hash, + VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(site_hash, 0x10000, 0x20000, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::from([(30, Arc::clone(&active_site))]), + active_circuits: HashMap::from([(40, Arc::clone(&active_circuit))]), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::CutoverPending, + pending_prune: true, + next_prune_attempt_unix: 0, + }, + )]); + let mut runtime_node_operations = HashMap::from([( + site_hash, + mk_runtime_operation( + 2, + site_hash, + RuntimeNodeOperationAction::Virtualize, + RuntimeNodeOperationStatus::Applying, + 1, + Some(0), + ), + )]); + let down_snapshot = HashMap::from([ + ( + TcHandle::from_u32(0x10022), + live_class_entry(0x10022, Some(0x10020)), + ), + ( + tc_handle_from_major_minor(0x110, 0x30), + LiveTcClassEntry { + class_id: tc_handle_from_major_minor(0x110, 0x30), + parent: Some(TcHandle::from_u32(0x10020)), + leaf_qdisc_major: Some(0x9000), + }, + ), + ]); + let up_snapshot = HashMap::from([ + ( + TcHandle::from_u32(0x20022), + live_class_entry(0x20022, Some(0x20020)), + ), + ( + tc_handle_from_major_minor(0x210, 0x30), + LiveTcClassEntry { + class_id: tc_handle_from_major_minor(0x210, 0x30), + parent: Some(TcHandle::from_u32(0x20020)), + leaf_qdisc_major: Some(0x9001), + }, + ), + ]); + + flush_deferred_runtime_site_prunes_with_snapshots( + &config, + &mut virtualized_sites, + &HashMap::new(), + &mut runtime_node_operations, + &down_snapshot, + &up_snapshot, + ); + + let state = virtualized_sites + .get(&site_hash) + .expect("retained shadow runtime state"); + assert_eq!( + state.lifecycle, + RuntimeVirtualizedBranchLifecycle::FlattenedActive + ); + assert!(!state.pending_prune); + let operation = runtime_node_operations + .get(&site_hash) + .expect("runtime operation"); + assert_eq!(operation.status, RuntimeNodeOperationStatus::Completed); + } + + #[test] + fn restore_without_shadow_remainders_completes_immediately() { + let saved_state = VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(20, 0x10000, 0x20000, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: false, + next_prune_attempt_unix: 0, + }; + + assert!( + build_post_restore_virtualized_state(saved_state, HashMap::new(), HashMap::new(), 0) + .is_none() + ); + } + + #[test] + fn restore_cleanup_retry_keeps_operation_in_applied_awaiting_cleanup() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + + let config = Arc::new(Config::default()); + let site_hash = 20; + let prune_site = mk_add_site(site_hash, 0x10020, 0x20020, 0x21); + let mut virtualized_sites = HashMap::from([( + site_hash, + VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(site_hash, 0x10000, 0x20000, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::from([(site_hash, Arc::clone(&prune_site))]), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Original, + lifecycle: RuntimeVirtualizedBranchLifecycle::PhysicalActiveCleanupPending, + pending_prune: true, + next_prune_attempt_unix: 0, + }, + )]); + let mut runtime_node_operations = HashMap::from([( + site_hash, + mk_runtime_operation( + 3, + site_hash, + RuntimeNodeOperationAction::Restore, + RuntimeNodeOperationStatus::AppliedAwaitingCleanup, + 1, + Some(0), + ), + )]); + let down_snapshot = HashMap::from([( + TcHandle::from_u32(0x10022), + live_class_entry(0x10022, Some(0x10021)), + )]); + let up_snapshot = HashMap::from([( + TcHandle::from_u32(0x20022), + live_class_entry(0x20022, Some(0x20021)), + )]); + + flush_deferred_runtime_site_prunes_with_snapshots( + &config, + &mut virtualized_sites, + &HashMap::new(), + &mut runtime_node_operations, + &down_snapshot, + &up_snapshot, + ); + + let state = virtualized_sites + .get(&site_hash) + .expect("virtualized site state"); + assert!(state.pending_prune); + let operation = runtime_node_operations + .get(&site_hash) + .expect("runtime operation"); + assert_eq!( + operation.status, + RuntimeNodeOperationStatus::AppliedAwaitingCleanup + ); + assert!(operation.next_retry_at_unix.is_some()); + assert!(operation.last_error.is_some()); + } + + #[test] + fn restore_cleanup_failure_eventually_marks_dirty() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + + let config = Arc::new(Config::default()); + let site_hash = 20; + let prune_site = mk_add_site(site_hash, 0x10020, 0x20020, 0x21); + let mut virtualized_sites = HashMap::from([( + site_hash, + VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(site_hash, 0x10000, 0x20000, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::from([(site_hash, Arc::clone(&prune_site))]), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Original, + lifecycle: RuntimeVirtualizedBranchLifecycle::PhysicalActiveCleanupPending, + pending_prune: true, + next_prune_attempt_unix: 0, + }, + )]); + let mut runtime_node_operations = HashMap::from([( + site_hash, + mk_runtime_operation( + 4, + site_hash, + RuntimeNodeOperationAction::Restore, + RuntimeNodeOperationStatus::AppliedAwaitingCleanup, + RUNTIME_SITE_PRUNE_MAX_ATTEMPTS - 1, + Some(0), + ), + )]); + let down_snapshot = HashMap::from([( + TcHandle::from_u32(0x10022), + live_class_entry(0x10022, Some(0x10021)), + )]); + let up_snapshot = HashMap::from([( + TcHandle::from_u32(0x20022), + live_class_entry(0x20022, Some(0x20021)), + )]); + + flush_deferred_runtime_site_prunes_with_snapshots( + &config, + &mut virtualized_sites, + &HashMap::new(), + &mut runtime_node_operations, + &down_snapshot, + &up_snapshot, + ); + + let state = virtualized_sites + .get(&site_hash) + .expect("virtualized site state"); + assert!(!state.pending_prune); + assert_eq!(state.lifecycle, RuntimeVirtualizedBranchLifecycle::Failed); + let operation = runtime_node_operations + .get(&site_hash) + .expect("runtime operation"); + assert_eq!(operation.status, RuntimeNodeOperationStatus::Dirty); + assert!(operation.next_retry_at_unix.is_none()); + } + + #[test] + fn runtime_branch_snapshot_tracks_restore_completion() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + + let config = Arc::new(Config::default()); + let site_hash = 20; + let mut virtualized_sites = HashMap::from([( + site_hash, + VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: mk_add_site(site_hash, 0x10000, 0x20000, 0x21), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Original, + lifecycle: RuntimeVirtualizedBranchLifecycle::PhysicalActiveCleanupPending, + pending_prune: true, + next_prune_attempt_unix: 0, + }, + )]); + let mut runtime_node_operations = HashMap::from([( + site_hash, + mk_runtime_operation( + 5, + site_hash, + RuntimeNodeOperationAction::Restore, + RuntimeNodeOperationStatus::AppliedAwaitingCleanup, + 1, + Some(0), + ), + )]); + + update_desync_state_from_runtime_state(&runtime_node_operations, &virtualized_sites); + let before = bakery_runtime_node_branch_snapshot(site_hash).expect("branch snapshot"); + assert_eq!(before.active_branch, "Original"); + assert!(before.pending_prune); + + flush_deferred_runtime_site_prunes_with_snapshots( + &config, + &mut virtualized_sites, + &HashMap::new(), + &mut runtime_node_operations, + &HashMap::new(), + &HashMap::new(), + ); + + assert!(bakery_runtime_node_branch_snapshot(site_hash).is_none()); + assert_eq!( + bakery_runtime_node_operation_snapshot(site_hash) + .expect("runtime operation snapshot") + .status, + RuntimeNodeOperationStatus::Completed + ); + } + + #[test] + fn deferred_runtime_site_prune_advances_without_pending_migrations() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + + let cfg = Config { + bridge: Some(lqos_config::BridgeConfig { + use_xdp_bridge: false, + to_internet: "__bakery-missing-wan__".to_string(), + to_network: "__bakery-missing-lan__".to_string(), + }), + ..Config::default() + }; + let config = Arc::new(cfg); + + let site_hash = 20; + let site = mk_add_site(site_hash, 0x10020, 0x20020, 0x21); + let mut virtualized_sites = HashMap::from([( + site_hash, + VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site, + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::new(), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Shadow, + lifecycle: RuntimeVirtualizedBranchLifecycle::FlattenedActive, + pending_prune: true, + next_prune_attempt_unix: 0, + }, + )]); + let mut runtime_node_operations = HashMap::new(); + let submitted_at = unix_now().saturating_sub(5); + let mut operation = RuntimeNodeOperation::new( + 1, + site_hash, + None, + RuntimeNodeOperationAction::Virtualize, + submitted_at, + ); + operation.attempt_count = 1; + operation.update_status( + RuntimeNodeOperationStatus::AppliedAwaitingCleanup, + submitted_at, + None, + Some(0), + ); + runtime_node_operations.insert(site_hash, operation); + + flush_deferred_runtime_site_prunes( + &config, + &mut virtualized_sites, + &HashMap::new(), + &mut runtime_node_operations, + ); + + let state = virtualized_sites + .get(&site_hash) + .expect("virtualized site state"); + let operation = runtime_node_operations + .get(&site_hash) + .expect("runtime operation"); + + assert!(state.pending_prune); + assert_eq!( + operation.status, + RuntimeNodeOperationStatus::AppliedAwaitingCleanup + ); + assert!(operation.attempt_count >= 1); + assert!(operation.updated_at_unix >= submitted_at); + assert_eq!( + operation.next_retry_at_unix, + Some(state.next_prune_attempt_unix) + ); + assert!(operation.next_retry_at_unix.expect("retry time") >= operation.updated_at_unix); + assert!( + operation + .last_error + .as_deref() + .unwrap_or_default() + .contains("Failed to snapshot live classes on __bakery-missing-lan__") + ); + } + + #[test] + fn inactive_branch_prune_fails_fast_on_unexpected_live_child_classes() { + let config = Arc::new(lqos_config::Config::default()); + let prune_root = mk_add_site(20, 0x10020, 0x20020, 0x21); + + let mut state = VirtualizedSiteState { + site_name: "test-site-20".to_string(), + site: Arc::clone(&prune_root), + saved_sites: HashMap::new(), + saved_circuits: HashMap::new(), + active_sites: HashMap::new(), + active_circuits: HashMap::new(), + prune_sites: HashMap::from([(20, Arc::clone(&prune_root))]), + prune_circuits: HashMap::new(), + qdisc_handles: VirtualizedSiteQdiscHandles { + down: None, + up: None, + }, + active_branch: RuntimeVirtualizedActiveBranch::Original, + lifecycle: RuntimeVirtualizedBranchLifecycle::PhysicalActiveCleanupPending, + pending_prune: true, + next_prune_attempt_unix: 0, + }; + + let mut down_snapshot = HashMap::new(); + down_snapshot.insert( + TcHandle::from_u32(0x10022), + LiveTcClassEntry { + class_id: TcHandle::from_u32(0x10022), + parent: Some(TcHandle::from_u32(0x10021)), + leaf_qdisc_major: None, + }, + ); + let mut up_snapshot = HashMap::new(); + up_snapshot.insert( + TcHandle::from_u32(0x20022), + LiveTcClassEntry { + class_id: TcHandle::from_u32(0x20022), + parent: Some(TcHandle::from_u32(0x20021)), + leaf_qdisc_major: None, + }, + ); + + let result = execute_runtime_virtualized_subtree_prune( + &config, + &mut state, + &down_snapshot, + &up_snapshot, + ); + + match result { + RuntimePrunePassResult::Failed(summary) => { + assert!(summary.contains("unexpected live child classes")); + } + _ => panic!("expected Failed"), + } + } + + #[test] + fn dirty_runtime_subtrees_below_threshold_do_not_require_full_reload() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + + let mut runtime_node_operations = HashMap::new(); + for index in 0..(RUNTIME_DIRTY_SUBTREE_RELOAD_THRESHOLD - 1) { + let site_hash = index as i64 + 1; + let mut operation = RuntimeNodeOperation::new( + site_hash as u64, + site_hash, + None, + RuntimeNodeOperationAction::Virtualize, + unix_now(), + ); + operation.attempt_count = 2; + operation.update_status( + RuntimeNodeOperationStatus::Dirty, + unix_now(), + Some("test dirty subtree".to_string()), + None, + ); + runtime_node_operations.insert(site_hash, operation); + } + + update_desync_state_from_runtime_state(&runtime_node_operations, &HashMap::new()); + + let telemetry = bakery_status_snapshot(); + assert_eq!( + telemetry.dirty_subtree_count, + RUNTIME_DIRTY_SUBTREE_RELOAD_THRESHOLD - 1 + ); + assert!(!telemetry.reload_required); + assert!(bakery_reload_required_reason().is_none()); + } + + #[test] + fn dirty_runtime_subtrees_at_threshold_require_full_reload() { + let _guard = bakery_test_lock().lock().expect("test lock"); + reset_bakery_test_state(); + + let mut runtime_node_operations = HashMap::new(); + for index in 0..RUNTIME_DIRTY_SUBTREE_RELOAD_THRESHOLD { + let site_hash = index as i64 + 1; + let mut operation = RuntimeNodeOperation::new( + site_hash as u64, + site_hash, + None, + RuntimeNodeOperationAction::Virtualize, + unix_now(), + ); + operation.attempt_count = 3; + operation.update_status( + RuntimeNodeOperationStatus::Dirty, + unix_now(), + Some("test dirty subtree".to_string()), + None, + ); + runtime_node_operations.insert(site_hash, operation); + } + + update_desync_state_from_runtime_state(&runtime_node_operations, &HashMap::new()); + + let telemetry = bakery_status_snapshot(); + assert_eq!( + telemetry.dirty_subtree_count, + RUNTIME_DIRTY_SUBTREE_RELOAD_THRESHOLD + ); + assert!(telemetry.reload_required); + assert!( + telemetry + .reload_required_reason + .as_deref() + .unwrap_or_default() + .contains("dirty runtime subtree operations") + ); + assert!( + bakery_reload_required_reason() + .unwrap_or_default() + .contains("dirty runtime subtree operations") + ); + } + #[test] fn mapped_circuit_limit_uses_custom_limit() { let existing = HashMap::new(); @@ -1984,4 +11719,1561 @@ mod tests { assert_eq!(stats.dropped_mapped, 0); assert_eq!(filtered.len(), 3); } + + #[test] + fn qdisc_budget_estimate_counts_total_qdiscs_per_interface() { + let _guard = bakery_test_lock().lock().expect("lock"); + reset_bakery_test_state(); + let config = Arc::new(Config::default()); + let queue = vec![ + BakeryCommands::StartBatch, + BakeryCommands::MqSetup { + queues_available: 2, + stick_offset: 0, + }, + BakeryCommands::AddSite { + site_hash: 1, + parent_class_id: TcHandle::from_u32(0x1), + up_parent_class_id: TcHandle::from_u32(0x2), + class_minor: 0x10, + download_bandwidth_min: 50.0, + upload_bandwidth_min: 50.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + }, + BakeryCommands::AddCircuit { + circuit_hash: 2, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + ip_addresses: "192.0.2.1/32".to_string(), + sqm_override: None, + }, + BakeryCommands::CommitBatch, + ]; + + let estimate = estimate_full_reload_auto_qdisc_budget(&config, &queue); + assert_eq!(estimate.interfaces.get("eth1"), Some(&8)); + assert_eq!(estimate.interfaces.get("eth0"), Some(&8)); + assert_eq!( + estimate + .interface_details + .get("eth0") + .map(|d| d.cake_qdiscs), + Some(1) + ); + assert_eq!( + estimate + .interface_details + .get("eth1") + .map(|d| d.cake_qdiscs), + Some(1) + ); + assert_eq!( + estimate + .interface_details + .get("eth0") + .map(|d| d.infra_qdiscs), + Some(7) + ); + assert_eq!( + estimate + .interface_details + .get("eth1") + .map(|d| d.infra_qdiscs), + Some(7) + ); + assert!(estimate.ok()); + } + + #[test] + fn qdisc_budget_estimate_counts_only_root_mq_in_observe_mode() { + let _guard = bakery_test_lock().lock().expect("lock"); + reset_bakery_test_state(); + let mut cfg = Config::default(); + cfg.queues.set_queue_mode(lqos_config::QueueMode::Observe); + let config = Arc::new(cfg); + let queue = vec![ + BakeryCommands::MqSetup { + queues_available: 1, + stick_offset: 0, + }, + BakeryCommands::AddCircuit { + circuit_hash: 2, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + ip_addresses: "192.0.2.1/32".to_string(), + sqm_override: None, + }, + ]; + + let estimate = estimate_full_reload_auto_qdisc_budget(&config, &queue); + assert_eq!(estimate.interfaces.get("eth1"), Some(&1)); + assert_eq!(estimate.interfaces.get("eth0"), Some(&1)); + assert_eq!( + estimate + .interface_details + .get("eth0") + .map(|d| d.infra_qdiscs), + Some(1) + ); + assert_eq!( + estimate + .interface_details + .get("eth1") + .map(|d| d.infra_qdiscs), + Some(1) + ); + assert_eq!( + estimate + .interface_details + .get("eth0") + .map(|d| d.cake_qdiscs), + Some(0) + ); + assert_eq!( + estimate + .interface_details + .get("eth1") + .map(|d| d.cake_qdiscs), + Some(0) + ); + } + + #[test] + fn live_tree_mutation_blocker_reports_observe_mode_and_inactive_tree() { + let _guard = bakery_test_lock().lock().expect("lock"); + reset_bakery_test_state(); + + let mut observe_cfg = Config::default(); + observe_cfg + .queues + .set_queue_mode(lqos_config::QueueMode::Observe); + let observe_cfg = Arc::new(observe_cfg); + assert_eq!( + live_tree_mutation_blocker_for_config(&observe_cfg), + Some( + "queue_mode is observe; root MQ is retained but the shaping tree is not live" + .to_string() + ) + ); + + let active_cfg = Arc::new(Config::default()); + assert_eq!( + live_tree_mutation_blocker_for_config(&active_cfg), + Some("the shaping tree is not currently active".to_string()) + ); + + SHAPING_TREE_ACTIVE.store(true, Ordering::Relaxed); + assert_eq!(live_tree_mutation_blocker_for_config(&active_cfg), None); + } + + #[test] + fn live_tree_mutation_blocker_reports_full_reload_in_progress() { + let _guard = bakery_test_lock().lock().expect("lock"); + reset_bakery_test_state(); + + let active_cfg = Arc::new(Config::default()); + SHAPING_TREE_ACTIVE.store(true, Ordering::Relaxed); + FULL_RELOAD_IN_PROGRESS.store(true, Ordering::Relaxed); + + assert_eq!( + live_tree_mutation_blocker_for_config(&active_cfg), + Some("a full reload is currently in progress".to_string()) + ); + reset_bakery_test_state(); + } + + #[test] + fn verify_clean_root_child_tree_accepts_empty_retained_root() { + let qdisc_snapshot = vec![live_qdisc_entry("mq", Some(0x7fff0000), None, true)]; + let class_snapshot = HashMap::new(); + assert!(verify_clean_root_child_tree(&qdisc_snapshot, &class_snapshot, "eth0").is_ok()); + } + + #[test] + fn verify_clean_root_child_tree_accepts_kernel_default_root_children() { + let qdisc_snapshot = vec![ + live_qdisc_entry("mq", Some(0x7fff0000), None, true), + live_qdisc_entry("fq_codel", Some(0), Some(0x7fff0001), false), + ]; + let class_snapshot = HashMap::new(); + assert!(verify_clean_root_child_tree(&qdisc_snapshot, &class_snapshot, "eth0").is_ok()); + } + + #[test] + fn verify_clean_root_child_tree_accepts_kernel_default_mq_classes() { + let qdisc_snapshot = vec![live_qdisc_entry("mq", Some(0x7fff0000), None, true)]; + let class_snapshot = HashMap::from([( + TcHandle::from_u32(0x7fff0001), + live_class_entry(0x7fff0001, None), + )]); + assert!(verify_clean_root_child_tree(&qdisc_snapshot, &class_snapshot, "eth0").is_ok()); + } + + #[test] + fn verify_clean_root_child_tree_rejects_lingering_managed_root_children() { + let qdisc_snapshot = vec![ + live_qdisc_entry("mq", Some(0x7fff0000), None, true), + live_qdisc_entry("htb", Some(0x00010000), Some(0x7fff0001), false), + ]; + let class_snapshot = HashMap::new(); + let err = verify_clean_root_child_tree(&qdisc_snapshot, &class_snapshot, "eth0") + .expect_err("child qdisc should fail retained-root verification"); + assert!(err.contains("still has 1 managed child qdisc")); + } + + #[test] + fn verify_clean_root_child_tree_rejects_lingering_classes() { + let qdisc_snapshot = vec![live_qdisc_entry("mq", Some(0x7fff0000), None, true)]; + let class_snapshot = HashMap::from([( + TcHandle::from_u32(0x10001), + live_class_entry(0x10001, Some(0x10000)), + )]); + let err = verify_clean_root_child_tree(&qdisc_snapshot, &class_snapshot, "eth0") + .expect_err("lingering classes should fail retained-root verification"); + assert!(err.contains("still has 1 managed tc class")); + } + + #[test] + fn managed_root_child_parent_handles_ignore_kernel_default_qdiscs() { + let qdisc_snapshot = vec![ + live_qdisc_entry("mq", Some(0x7fff0000), None, true), + live_qdisc_entry("fq_codel", Some(0), Some(0x7fff0001), false), + ]; + assert!(managed_root_child_parent_handles(&qdisc_snapshot).is_empty()); + + let managed_snapshot = vec![ + live_qdisc_entry("mq", Some(0x7fff0000), None, true), + live_qdisc_entry("htb", Some(0x00010000), Some(0x7fff0001), false), + ]; + assert_eq!( + managed_root_child_parent_handles(&managed_snapshot), + HashSet::from([TcHandle::from_u32(0x7fff0001)]) + ); + } + + #[test] + fn grouped_runtime_site_events_retain_human_site_name() { + let _guard = bakery_test_lock().lock().expect("lock"); + reset_bakery_test_state(); + + let site_hash = 3571466403324592518i64; + let site_name = "JR_AP_TR_F".to_string(); + let mut grouped = GroupedBakeryEventLimiter::default(); + grouped.emit_with_site_name( + "runtime_cutover_completed|completed", + "runtime_cutover_completed", + "info", + Some((site_hash, Some(site_name.clone()))), + format!( + "Runtime cutover completed for site {}; shadow branch is active and original branch is standby.", + runtime_site_display_name(site_hash, Some(site_name.as_str())) + ), + "runtime cutover completion events".to_string(), + ); + grouped.flush(); + + let activity = bakery_activity_snapshot(); + assert_eq!(activity.len(), 1); + assert_eq!(activity[0].site_hash, Some(site_hash)); + assert_eq!(activity[0].site_name.as_deref(), Some(site_name.as_str())); + assert!(activity[0].summary.contains(site_name.as_str())); + assert!(!activity[0].summary.contains(&format!("site {site_hash}"))); + } + + #[test] + fn qdisc_budget_estimate_counts_fq_codel_leaf_qdiscs_separately() { + let config = Arc::new(Config::default()); + let queue = vec![ + BakeryCommands::MqSetup { + queues_available: 1, + stick_offset: 0, + }, + BakeryCommands::AddCircuit { + circuit_hash: 2, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 2_000.0, + upload_bandwidth_max: 2_000.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + ip_addresses: "192.0.2.1/32".to_string(), + sqm_override: Some("fq_codel".to_string()), + }, + ]; + + let estimate = estimate_full_reload_auto_qdisc_budget(&config, &queue); + assert_eq!( + estimate + .interface_details + .get("eth0") + .map(|d| d.fq_codel_qdiscs), + Some(1) + ); + assert_eq!( + estimate + .interface_details + .get("eth1") + .map(|d| d.fq_codel_qdiscs), + Some(1) + ); + assert_eq!( + estimate + .interface_details + .get("eth0") + .map(|d| d.cake_qdiscs), + Some(0) + ); + } + + #[test] + fn kind_switch_rotates_only_changed_direction_and_reuses_old_handle() { + let config = test_config_with_runtime_dir("kind-switch"); + let layout = MqDeviceLayout::from_setup(&config, 2, 0); + let mut handles = QdiscHandleState::default(); + + let original = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 101, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.1/32".to_string(), + sqm_override: None, + }); + + let original = with_assigned_qdisc_handles(&original, &config, &layout, &mut handles); + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(original_down), + up_qdisc_handle: Some(original_up), + .. + } = original.as_ref() + else { + panic!("expected assigned handles"); + }; + + handles.save(&config); + let mut reloaded = QdiscHandleState::load(&config); + + let switched = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 101, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.1/32".to_string(), + sqm_override: Some("fq_codel/cake".to_string()), + }); + + let switched = with_assigned_qdisc_handles(&switched, &config, &layout, &mut reloaded); + let rotated = rotate_changed_qdisc_handles( + original.as_ref(), + &switched, + &config, + &layout, + &mut reloaded, + ); + + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(rotated_down), + up_qdisc_handle: Some(rotated_up), + .. + } = rotated.as_ref() + else { + panic!("expected rotated handles"); + }; + + assert_ne!(*rotated_down, *original_down); + assert_eq!(*rotated_up, *original_up); + + reloaded.save(&config); + let mut persisted = QdiscHandleState::load(&config); + let new_circuit = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 202, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x21, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.2/32".to_string(), + sqm_override: None, + }); + let new_circuit = + with_assigned_qdisc_handles(&new_circuit, &config, &layout, &mut persisted); + + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(new_down), + .. + } = new_circuit.as_ref() + else { + panic!("expected new circuit handle"); + }; + + assert_eq!(*new_down, *original_down); + } + + #[test] + fn parent_change_rotates_only_changed_direction_and_reuses_old_handle() { + let config = test_config_with_runtime_dir("parent-change"); + let layout = MqDeviceLayout::from_setup(&config, 2, 0); + let mut handles = QdiscHandleState::default(); + + let original = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 303, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.3/32".to_string(), + sqm_override: None, + }); + + let original = with_assigned_qdisc_handles(&original, &config, &layout, &mut handles); + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(original_down), + up_qdisc_handle: Some(original_up), + .. + } = original.as_ref() + else { + panic!("expected assigned handles"); + }; + + handles.save(&config); + let mut reloaded = QdiscHandleState::load(&config); + + let moved = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 303, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10002), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x101, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.3/32".to_string(), + sqm_override: None, + }); + + let moved = with_assigned_qdisc_handles(&moved, &config, &layout, &mut reloaded); + let rotated = rotate_changed_qdisc_handles( + original.as_ref(), + &moved, + &config, + &layout, + &mut reloaded, + ); + + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(rotated_down), + up_qdisc_handle: Some(rotated_up), + .. + } = rotated.as_ref() + else { + panic!("expected rotated handles"); + }; + + assert_ne!(*rotated_down, *original_down); + assert_eq!(*rotated_up, *original_up); + + reloaded.save(&config); + let mut persisted = QdiscHandleState::load(&config); + let new_circuit = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 404, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x21, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.4/32".to_string(), + sqm_override: None, + }); + let new_circuit = + with_assigned_qdisc_handles(&new_circuit, &config, &layout, &mut persisted); + + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(new_down), + .. + } = new_circuit.as_ref() + else { + panic!("expected new circuit handle"); + }; + assert_eq!(*new_down, *original_down); + } + + #[test] + fn parent_change_rotation_avoids_live_reserved_handles() { + let config = test_config_with_runtime_dir("parent-change-live-reserved"); + let layout = MqDeviceLayout::from_setup(&config, 2, 0); + let mut handles = QdiscHandleState::default(); + + let original = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 313, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.13/32".to_string(), + sqm_override: None, + }); + + let original = with_assigned_qdisc_handles(&original, &config, &layout, &mut handles); + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(original_down), + up_qdisc_handle: Some(original_up), + .. + } = original.as_ref() + else { + panic!("expected assigned handles"); + }; + + handles.save(&config); + let mut reloaded = QdiscHandleState::load(&config); + + let moved = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 313, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10002), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x101, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.13/32".to_string(), + sqm_override: None, + }); + + let live_reserved = HashMap::from([ + (config.isp_interface(), HashSet::from([*original_down + 1])), + (config.internet_interface(), HashSet::from([*original_up])), + ]); + let moved = with_assigned_qdisc_handles_reserved( + &moved, + &config, + &layout, + &mut reloaded, + &live_reserved, + ); + let rotated = rotate_changed_qdisc_handles_reserved( + original.as_ref(), + &moved, + &config, + &layout, + &mut reloaded, + &live_reserved, + ); + + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(rotated_down), + .. + } = rotated.as_ref() + else { + panic!("expected rotated handles"); + }; + + assert_ne!(*rotated_down, *original_down); + assert_ne!(*rotated_down, *original_down + 1); + } + + #[test] + fn restore_rotation_avoids_live_reserved_prefilled_handle_without_parent_change() { + let config = test_config_with_runtime_dir("restore-live-reserved-prefilled-handle"); + let layout = MqDeviceLayout::from_setup(&config, 2, 0); + let mut handles = QdiscHandleState::default(); + + let original = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 414, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.14/32".to_string(), + sqm_override: None, + }); + + let original = with_assigned_qdisc_handles(&original, &config, &layout, &mut handles); + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(original_down), + up_qdisc_handle: Some(original_up), + .. + } = original.as_ref() + else { + panic!("expected assigned handles"); + }; + + handles.save(&config); + let mut reloaded = QdiscHandleState::load(&config); + let conflicting_down = *original_down + 1; + let conflicting_up = *original_up + 1; + + let restore = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 414, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: Some(conflicting_down), + up_qdisc_handle: Some(conflicting_up), + ip_addresses: "192.0.2.14/32".to_string(), + sqm_override: None, + }); + + let live_reserved = HashMap::from([ + (config.isp_interface(), HashSet::from([conflicting_down])), + (config.internet_interface(), HashSet::from([conflicting_up])), + ]); + + let rotated = rotate_changed_qdisc_handles_reserved( + original.as_ref(), + &restore, + &config, + &layout, + &mut reloaded, + &live_reserved, + ); + + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(rotated_down), + up_qdisc_handle: Some(rotated_up), + .. + } = rotated.as_ref() + else { + panic!("expected rotated handles"); + }; + + assert_eq!( + *original_down, + previous_down_qdisc_handle(original.as_ref()).expect("old down handle") + ); + assert_eq!( + *original_up, + previous_up_qdisc_handle(original.as_ref()).expect("old up handle") + ); + assert_ne!(*rotated_down, conflicting_down); + assert_ne!(*rotated_up, conflicting_up); + } + + #[test] + fn restore_assigns_fresh_handles_distinct_from_saved_and_live_shadow_handles() { + let config = test_config_with_runtime_dir("restore-fresh-cutover-handles"); + let layout = MqDeviceLayout::from_setup(&config, 2, 0); + let mut handles = QdiscHandleState::default(); + + let original = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 415, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.15/32".to_string(), + sqm_override: None, + }); + let original = with_assigned_qdisc_handles(&original, &config, &layout, &mut handles); + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(saved_down), + up_qdisc_handle: Some(saved_up), + .. + } = original.as_ref() + else { + panic!("expected saved/original handles"); + }; + + let shadow_down = handles + .rotate_circuit_handle(&config.isp_interface(), 415, &HashSet::new()) + .expect("shadow down handle"); + let shadow_up = handles + .rotate_circuit_handle(&config.internet_interface(), 415, &HashSet::new()) + .expect("shadow up handle"); + let live_reserved = HashMap::from([ + (config.isp_interface(), HashSet::from([shadow_down])), + (config.internet_interface(), HashSet::from([shadow_up])), + ]); + + let restored = assign_fresh_qdisc_handles_reserved( + &original, + &config, + &layout, + &mut handles, + &live_reserved, + ) + .expect("fresh restore handles"); + + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(restored_down), + up_qdisc_handle: Some(restored_up), + .. + } = restored.as_ref() + else { + panic!("expected restored handles"); + }; + + assert_ne!(*restored_down, *saved_down); + assert_ne!(*restored_up, *saved_up); + assert_ne!(*restored_down, shadow_down); + assert_ne!(*restored_up, shadow_up); + } + + #[test] + fn queue_live_migration_succeeds_for_active_parent_move() { + let old_cmd = mk_test_circuit(9001, 0x10020, 0x20020, 0x21, 0x1, 0x2, "192.0.2.90/32"); + let new_cmd = mk_test_circuit(9001, 0x10034, 0x20034, 0x35, 0x3, 0x4, "192.0.2.90/32"); + + let mut circuits = HashMap::from([(9001, Arc::clone(&old_cmd))]); + let sites = HashMap::new(); + let live_circuits = HashMap::from([(9001, 1u64)]); + let mut migrations = HashMap::new(); + + let queued = queue_live_migration( + old_cmd.as_ref(), + &new_cmd, + &sites, + &mut circuits, + &live_circuits, + &mut migrations, + ); + + assert!(queued); + let migration = migrations.get(&9001).expect("migration should be queued"); + assert_eq!(migration.stage, MigrationStage::PrepareShadow); + assert_eq!(migration.parent_class_id, TcHandle::from_u32(0x10034)); + assert_eq!(migration.final_minor, 0x35); + assert_ne!(migration.shadow_minor, migration.old_minor); + assert!(matches!( + circuits.get(&9001), + Some(cmd) if Arc::ptr_eq(cmd, &old_cmd) + )); + assert!(Arc::ptr_eq(&migration.desired_cmd, &new_cmd)); + } + + #[test] + fn queue_live_migration_fails_when_target_parent_has_no_shadow_minor() { + let old_cmd = mk_test_circuit(9100, 0x10020, 0x20020, 0x21, 0x1, 0x2, "192.0.2.91/32"); + let new_cmd = mk_test_circuit(9100, 0x10034, 0x20034, 0x35, 0x3, 0x4, "192.0.2.91/32"); + + let mut circuits = HashMap::from([(9100, Arc::clone(&old_cmd))]); + let sites = HashMap::new(); + for minor in 1..=0xFFFEu16 { + circuits.insert( + 100_000 + i64::from(minor), + mk_test_circuit( + 100_000 + i64::from(minor), + 0x10034, + 0x20034, + minor, + 0x3, + 0x4, + &format!("198.51.100.{}/32", (minor % 250) + 1), + ), + ); + } + + let live_circuits = HashMap::from([(9100, 1u64)]); + let mut migrations = HashMap::new(); + + let queued = queue_live_migration( + old_cmd.as_ref(), + &new_cmd, + &sites, + &mut circuits, + &live_circuits, + &mut migrations, + ); + + assert!(!queued); + assert!(!migrations.contains_key(&9100)); + assert!(matches!( + circuits.get(&9100), + Some(cmd) if Arc::ptr_eq(cmd, &old_cmd) + )); + } + + #[test] + fn find_free_circuit_shadow_minor_respects_major_domain_occupancy() { + let sites = HashMap::from([(7, mk_add_site(7, 0x10001, 0x20001, 0x2001))]); + let circuits = HashMap::from([( + 42, + mk_test_circuit(42, 0x10099, 0x20034, 0x2000, 0x1, 0x2, "192.0.2.142/32"), + )]); + + let chosen = find_free_circuit_shadow_minor(&sites, &circuits, &HashMap::new(), 0x1, 0x2) + .expect("free minor"); + + assert_ne!(chosen, 0x2000); + assert_ne!(chosen, 0x2001); + assert_eq!(chosen, 0x2002); + } + + #[test] + fn queue_runtime_migration_reserves_shadow_minor_across_pending_migrations() { + let old_cmd_a = mk_test_circuit(9300, 0x10020, 0x20020, 0x21, 0x1, 0x2, "192.0.2.94/32"); + let new_cmd_a = mk_test_circuit(9300, 0x80034, 0x80034, 0x35, 0x8, 0x8, "192.0.2.94/32"); + let old_cmd_b = mk_test_circuit(9301, 0x10022, 0x20022, 0x23, 0x1, 0x2, "192.0.2.95/32"); + let new_cmd_b = mk_test_circuit(9301, 0x80036, 0x80036, 0x37, 0x8, 0x8, "192.0.2.95/32"); + + let mut circuits = HashMap::from([ + (9300, Arc::clone(&old_cmd_a)), + (9301, Arc::clone(&old_cmd_b)), + ]); + let sites = HashMap::new(); + let live_circuits = HashMap::new(); + let mut migrations = HashMap::new(); + + assert!(queue_top_level_runtime_migration( + old_cmd_a.as_ref(), + &new_cmd_a, + &sites, + &mut circuits, + &live_circuits, + &mut migrations, + )); + assert!(queue_top_level_runtime_migration( + old_cmd_b.as_ref(), + &new_cmd_b, + &sites, + &mut circuits, + &live_circuits, + &mut migrations, + )); + + let migration_a = migrations.get(&9300).expect("first migration"); + let migration_b = migrations.get(&9301).expect("second migration"); + assert_eq!(migration_a.shadow_minor, 0x2000); + assert_eq!(migration_b.shadow_minor, 0x2001); + } + + #[test] + fn pending_migration_overlay_uses_desired_state_for_diff_only() { + let old_cmd = mk_test_circuit(9300, 0x10020, 0x20020, 0x21, 0x1, 0x2, "192.0.2.94/32"); + let new_cmd = mk_test_circuit(9300, 0x10034, 0x20034, 0x35, 0x3, 0x4, "192.0.2.94/32"); + + let mut circuits = HashMap::from([(9300, Arc::clone(&old_cmd))]); + let sites = HashMap::new(); + let live_circuits = HashMap::from([(9300, 1u64)]); + let mut migrations = HashMap::new(); + + assert!(queue_live_migration( + old_cmd.as_ref(), + &new_cmd, + &sites, + &mut circuits, + &live_circuits, + &mut migrations, + )); + + assert!(matches!( + circuits.get(&9300), + Some(cmd) if Arc::ptr_eq(cmd, &old_cmd) + )); + + let overlaid = circuits_with_pending_migration_targets(&circuits, &migrations); + assert!(matches!( + overlaid.get(&9300), + Some(cmd) if Arc::ptr_eq(cmd, &new_cmd) + )); + } + + #[test] + fn migration_stage_failure_marks_reload_required_and_stops_migration() { + let _guard = bakery_test_lock().lock().expect("test lock"); + clear_reload_required( + "reset before migration_stage_failure_marks_reload_required_and_stops_migration", + ); + let mut migration = Migration { + circuit_hash: 9002, + circuit_name: None, + site_name: None, + old_class_major: 0x1, + old_up_class_major: 0x2, + old_down_qdisc_handle: Some(0x9000), + old_up_qdisc_handle: Some(0x9001), + parent_class_id: TcHandle::from_u32(0x10034), + up_parent_class_id: TcHandle::from_u32(0x20034), + class_major: 0x1, + up_class_major: 0x2, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + old_down_min: 1.0, + old_down_max: 10.0, + old_up_min: 1.0, + old_up_max: 10.0, + new_down_min: 1.0, + new_down_max: 20.0, + new_up_min: 1.0, + new_up_max: 20.0, + old_minor: 0x21, + shadow_minor: 0x2000, + final_minor: 0x35, + ips: vec!["192.0.2.92/32".to_string()], + sqm_override: None, + desired_cmd: mk_test_circuit(9002, 0x10034, 0x20034, 0x35, 0x1, 0x2, "192.0.2.92/32"), + stage: MigrationStage::BuildFinal, + shadow_verify_attempts: 0, + final_verify_attempts: 0, + }; + let result = ExecuteResult { + ok: false, + duration_ms: 1, + failure_summary: Some("RTNETLINK answers: Invalid argument".to_string()), + }; + + let advanced = migration_stage_apply_succeeded( + &mut migration, + "live-move: build final", + &result, + MigrationStage::VerifyFinalReady, + ); + + assert!(!advanced); + assert_eq!(migration.stage, MigrationStage::Done); + let reason = bakery_reload_required_reason().expect("reload required reason should be set"); + assert!(reason.contains("9002")); + assert!(reason.contains("live-move: build final")); + clear_reload_required( + "reset after migration_stage_failure_marks_reload_required_and_stops_migration", + ); + } + + #[test] + fn migration_stage_success_advances_without_reload_required() { + let _guard = bakery_test_lock().lock().expect("test lock"); + clear_reload_required( + "reset before migration_stage_success_advances_without_reload_required", + ); + let mut migration = Migration { + circuit_hash: 9003, + circuit_name: None, + site_name: None, + old_class_major: 0x1, + old_up_class_major: 0x2, + old_down_qdisc_handle: Some(0x9000), + old_up_qdisc_handle: Some(0x9001), + parent_class_id: TcHandle::from_u32(0x10034), + up_parent_class_id: TcHandle::from_u32(0x20034), + class_major: 0x1, + up_class_major: 0x2, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + old_down_min: 1.0, + old_down_max: 10.0, + old_up_min: 1.0, + old_up_max: 10.0, + new_down_min: 1.0, + new_down_max: 20.0, + new_up_min: 1.0, + new_up_max: 20.0, + old_minor: 0x21, + shadow_minor: 0x2000, + final_minor: 0x35, + ips: vec!["192.0.2.93/32".to_string()], + sqm_override: None, + desired_cmd: mk_test_circuit(9003, 0x10034, 0x20034, 0x35, 0x1, 0x2, "192.0.2.93/32"), + stage: MigrationStage::BuildFinal, + shadow_verify_attempts: 0, + final_verify_attempts: 0, + }; + let result = ExecuteResult { + ok: true, + duration_ms: 1, + failure_summary: None, + }; + + let advanced = migration_stage_apply_succeeded( + &mut migration, + "live-move: build final", + &result, + MigrationStage::VerifyFinalReady, + ); + + assert!(advanced); + assert_eq!(migration.stage, MigrationStage::VerifyFinalReady); + assert!(bakery_reload_required_reason().is_none()); + } + + #[test] + fn migration_target_label_includes_circuit_and_site_names_when_known() { + let migration = Migration { + circuit_hash: 9006, + circuit_name: Some("Rochester-Roof-Switch".to_string()), + site_name: Some("7232 Rochester".to_string()), + old_class_major: 0x1, + old_up_class_major: 0x2, + old_down_qdisc_handle: Some(0x9000), + old_up_qdisc_handle: Some(0x9001), + parent_class_id: TcHandle::from_u32(0x10034), + up_parent_class_id: TcHandle::from_u32(0x20034), + class_major: 0x1, + up_class_major: 0x2, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + old_down_min: 1.0, + old_down_max: 10.0, + old_up_min: 1.0, + old_up_max: 10.0, + new_down_min: 1.0, + new_down_max: 20.0, + new_up_min: 1.0, + new_up_max: 20.0, + old_minor: 0x21, + shadow_minor: 0x2000, + final_minor: 0x35, + ips: vec!["192.0.2.96/32".to_string()], + sqm_override: None, + desired_cmd: mk_test_circuit(9006, 0x10034, 0x20034, 0x35, 0x1, 0x2, "192.0.2.96/32"), + stage: MigrationStage::BuildFinal, + shadow_verify_attempts: 0, + final_verify_attempts: 0, + }; + + let label = migration_target_label(&migration); + assert!(label.contains("Rochester-Roof-Switch")); + assert!(label.contains("9006")); + assert!(label.contains("7232 Rochester")); + } + + #[test] + fn migration_branch_verification_summary_describes_expected_and_observed_state() { + let verification = MigrationBranchVerification { + down: MigrationDirectionVerification { + interface_name: "if-down".to_string(), + expected_handle: TcHandle::from_u32(0x4002000), + expected_parent: TcHandle::from_u32(0x4000015), + observed_parent: Some(TcHandle::from_u32(0x4000013)), + observed_leaf_qdisc_major: None, + }, + up: Some(MigrationDirectionVerification { + interface_name: "if-up".to_string(), + expected_handle: TcHandle::from_u32(0x5002000), + expected_parent: TcHandle::from_u32(0x5000015), + observed_parent: None, + observed_leaf_qdisc_major: None, + }), + }; + + let summary = verification.summary(); + assert!(summary.contains("if-down")); + assert!(summary.contains("expected class")); + assert!(summary.contains("2000")); + assert!(summary.contains("parent")); + assert!(summary.contains("15")); + assert!(summary.contains("observed parent")); + assert!(summary.contains("13")); + assert!(summary.contains("with no leaf qdisc")); + assert!(summary.contains("if-up")); + assert!(summary.contains("500")); + assert!(summary.contains("observed missing")); + assert!(!verification.ready()); + } + + #[test] + fn migration_invariant_rejects_stale_qdisc_handles_when_parent_changes() { + let migration = Migration { + circuit_hash: 9004, + circuit_name: None, + site_name: None, + old_class_major: 0x1, + old_up_class_major: 0x2, + old_down_qdisc_handle: Some(0x9000), + old_up_qdisc_handle: Some(0x9001), + parent_class_id: TcHandle::from_u32(0x10034), + up_parent_class_id: TcHandle::from_u32(0x20034), + class_major: 0x1, + up_class_major: 0x2, + down_qdisc_handle: Some(0x9000), + up_qdisc_handle: Some(0x9001), + old_down_min: 1.0, + old_down_max: 10.0, + old_up_min: 1.0, + old_up_max: 10.0, + new_down_min: 1.0, + new_down_max: 20.0, + new_up_min: 1.0, + new_up_max: 20.0, + old_minor: 0x21, + shadow_minor: 0x2000, + final_minor: 0x35, + ips: vec!["192.0.2.94/32".to_string()], + sqm_override: None, + desired_cmd: mk_test_circuit(9004, 0x10034, 0x20034, 0x35, 0x1, 0x2, "192.0.2.94/32"), + stage: MigrationStage::BuildFinal, + shadow_verify_attempts: 0, + final_verify_attempts: 0, + }; + + let summary = migration_qdisc_handle_rotation_invariant_error(&migration) + .expect("stale qdisc handles should be rejected"); + assert!(summary.contains("old handle")); + + let final_cmd = build_temp_add_cmd( + migration.desired_cmd.as_ref(), + migration.final_minor, + migration.new_down_min, + migration.new_down_max, + migration.new_up_min, + migration.new_up_max, + true, + ) + .expect("final command"); + let final_summary = + build_final_qdisc_handle_rotation_invariant_error(&migration, &final_cmd) + .expect("final command should also be rejected"); + assert!(final_summary.contains("old handle")); + } + + #[test] + fn migration_invariant_rejects_live_reserved_handle_collisions() { + let migration = Migration { + circuit_hash: 9005, + circuit_name: None, + site_name: None, + old_class_major: 0x1, + old_up_class_major: 0x2, + old_down_qdisc_handle: Some(0x9000), + old_up_qdisc_handle: Some(0x9001), + parent_class_id: TcHandle::from_u32(0x10034), + up_parent_class_id: TcHandle::from_u32(0x20034), + class_major: 0x1, + up_class_major: 0x2, + down_qdisc_handle: Some(0x93c9), + up_qdisc_handle: Some(0x93ca), + old_down_min: 1.0, + old_down_max: 10.0, + old_up_min: 1.0, + old_up_max: 10.0, + new_down_min: 1.0, + new_down_max: 20.0, + new_up_min: 1.0, + new_up_max: 20.0, + old_minor: 0x21, + shadow_minor: 0x2000, + final_minor: 0x35, + ips: vec!["192.0.2.95/32".to_string()], + sqm_override: None, + desired_cmd: Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 9005, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10034), + up_parent_class_id: TcHandle::from_u32(0x20034), + class_minor: 0x35, + download_bandwidth_min: 1.0, + upload_bandwidth_min: 1.0, + download_bandwidth_max: 20.0, + upload_bandwidth_max: 20.0, + class_major: 0x1, + up_class_major: 0x2, + down_qdisc_handle: Some(0x93c9), + up_qdisc_handle: Some(0x93ca), + ip_addresses: "192.0.2.95/32".to_string(), + sqm_override: None, + }), + stage: MigrationStage::BuildFinal, + shadow_verify_attempts: 0, + final_verify_attempts: 0, + }; + + let final_cmd = build_temp_add_cmd( + migration.desired_cmd.as_ref(), + migration.final_minor, + migration.new_down_min, + migration.new_down_max, + migration.new_up_min, + migration.new_up_max, + true, + ) + .expect("final command"); + let down_reserved = HashSet::from([0x93c9]); + let up_reserved = HashSet::from([0x93ca]); + let final_summary = + build_final_qdisc_handle_rotation_invariant_error_with_live_reservations( + &migration, + &final_cmd, + Some(&down_reserved), + Some(&up_reserved), + ) + .expect("live reserved collision should be rejected"); + assert!(final_summary.contains("already live")); + } + + #[test] + fn test_fault_once_matches_selector_and_clears_file() { + let dir = std::env::temp_dir(); + let path = dir.join(format!( + "lqos-bakery-fault-{}.txt", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time") + .as_nanos() + )); + std::fs::write(&path, "migrating circuits between parent nodes").expect("fault file"); + + let summary = consume_test_fault_once_from_path( + &path, + "migrating circuits between parent nodes (fallback)", + ) + .expect("fault should trigger"); + assert!(summary.contains("synthetic Bakery test fault")); + assert!(!path.exists()); + } + + #[test] + fn test_fault_once_ignores_non_matching_selector() { + let dir = std::env::temp_dir(); + let path = dir.join(format!( + "lqos-bakery-fault-{}.txt", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time") + .as_nanos() + )); + std::fs::write(&path, "TreeGuard runtime circuit reparent").expect("fault file"); + + let summary = consume_test_fault_once_from_path( + &path, + "migrating circuits between parent nodes (fallback)", + ); + assert!(summary.is_none()); + assert!(path.exists()); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn queue_top_level_runtime_migration_succeeds_for_inactive_parent_move() { + let old_cmd = mk_test_circuit(9200, 0x10020, 0x20020, 0x21, 0x1, 0x2, "192.0.2.92/32"); + let new_cmd = mk_test_circuit(9200, 0x10034, 0x20034, 0x35, 0x3, 0x4, "192.0.2.92/32"); + + let mut circuits = HashMap::from([(9200, Arc::clone(&old_cmd))]); + let sites = HashMap::new(); + let live_circuits = HashMap::new(); + let mut migrations = HashMap::new(); + + let queued = queue_top_level_runtime_migration( + old_cmd.as_ref(), + &new_cmd, + &sites, + &mut circuits, + &live_circuits, + &mut migrations, + ); + + assert!(queued); + let migration = migrations.get(&9200).expect("migration should be queued"); + assert_eq!(migration.stage, MigrationStage::PrepareShadow); + assert_eq!(migration.parent_class_id, TcHandle::from_u32(0x10034)); + assert_eq!(migration.final_minor, 0x35); + assert!(migration.shadow_minor >= 0x2000); + assert_ne!(migration.shadow_minor, migration.old_minor); + assert!(matches!( + circuits.get(&9200), + Some(cmd) if Arc::ptr_eq(cmd, &old_cmd) + )); + assert!(Arc::ptr_eq(&migration.desired_cmd, &new_cmd)); + } + + #[test] + fn build_shadow_add_cmd_uses_ephemeral_qdisc_handles_distinct_from_old_and_final() { + let config = test_config_with_runtime_dir("shadow-create-handles"); + let layout = MqDeviceLayout::from_setup(&config, 2, 0); + let mut handles = QdiscHandleState::default(); + + let original = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 9400, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10020), + up_parent_class_id: TcHandle::from_u32(0x20020), + class_minor: 0x21, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x1, + up_class_major: 0x2, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.96/32".to_string(), + sqm_override: None, + }); + let original = with_assigned_qdisc_handles(&original, &config, &layout, &mut handles); + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(original_down), + up_qdisc_handle: Some(original_up), + .. + } = original.as_ref() + else { + panic!("expected original handles"); + }; + + handles.save(&config); + let mut reloaded = QdiscHandleState::load(&config); + let moved = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 9400, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10034), + up_parent_class_id: TcHandle::from_u32(0x20034), + class_minor: 0x35, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x3, + up_class_major: 0x4, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.96/32".to_string(), + sqm_override: None, + }); + let moved = with_assigned_qdisc_handles(&moved, &config, &layout, &mut reloaded); + let moved = rotate_changed_qdisc_handles( + original.as_ref(), + &moved, + &config, + &layout, + &mut reloaded, + ); + reloaded.save(&config); + + let mut circuits = HashMap::from([(9400, Arc::clone(&original))]); + let sites = HashMap::new(); + let live_circuits = HashMap::from([(9400, 1u64)]); + let mut migrations = HashMap::new(); + assert!(queue_live_migration( + original.as_ref(), + &moved, + &sites, + &mut circuits, + &live_circuits, + &mut migrations, + )); + let migration = migrations.get(&9400).expect("migration should exist"); + let persisted_handles = QdiscHandleState::load(&config); + let live_reserved = HashMap::from([ + (config.isp_interface(), HashSet::from([*original_down])), + (config.internet_interface(), HashSet::from([*original_up])), + ]); + let shadow = build_shadow_add_cmd(migration, &config, &persisted_handles, &live_reserved) + .expect("shadow command"); + + let BakeryCommands::AddCircuit { + class_minor, + down_qdisc_handle: Some(shadow_down), + up_qdisc_handle: Some(shadow_up), + .. + } = shadow + else { + panic!("expected shadow qdisc handles"); + }; + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(final_down), + up_qdisc_handle: Some(final_up), + .. + } = moved.as_ref() + else { + panic!("expected final qdisc handles"); + }; + + assert_eq!(class_minor, migration.shadow_minor); + assert_ne!(shadow_down, *original_down); + assert_ne!(shadow_up, *original_up); + assert_ne!(shadow_down, *final_down); + assert_ne!(shadow_up, *final_up); + } + + #[test] + fn full_reload_allocation_uses_fresh_handles_reserved_from_live_tree() { + let config = test_config_with_runtime_dir("fresh-full-reload"); + let layout = MqDeviceLayout::from_setup(&config, 2, 0); + let mut persisted = QdiscHandleState::default(); + + let original = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 505, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.5/32".to_string(), + sqm_override: None, + }); + + let original = with_assigned_qdisc_handles(&original, &config, &layout, &mut persisted); + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(original_down), + up_qdisc_handle: Some(original_up), + .. + } = original.as_ref() + else { + panic!("expected assigned handles"); + }; + + let mut full_reload_handles = QdiscHandleState::default(); + let live_reserved = HashMap::from([ + (config.isp_interface(), HashSet::from([*original_down])), + (config.internet_interface(), HashSet::from([*original_up])), + ]); + let rebuilt_cmd = Arc::new(BakeryCommands::AddCircuit { + circuit_hash: 505, + circuit_name: None, + site_name: None, + parent_class_id: TcHandle::from_u32(0x10001), + up_parent_class_id: TcHandle::from_u32(0x20001), + class_minor: 0x20, + download_bandwidth_min: 10.0, + upload_bandwidth_min: 10.0, + download_bandwidth_max: 100.0, + upload_bandwidth_max: 100.0, + class_major: 0x100, + up_class_major: 0x200, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: "192.0.2.5/32".to_string(), + sqm_override: None, + }); + let rebuilt = with_assigned_qdisc_handles_reserved( + &rebuilt_cmd, + &config, + &layout, + &mut full_reload_handles, + &live_reserved, + ); + + let BakeryCommands::AddCircuit { + down_qdisc_handle: Some(rebuilt_down), + up_qdisc_handle: Some(rebuilt_up), + .. + } = rebuilt.as_ref() + else { + panic!("expected rebuilt handles"); + }; + + assert_ne!(*rebuilt_down, *original_down); + assert_ne!(*rebuilt_up, *original_up); + assert_eq!(*rebuilt_down, *original_down + 1); + assert_eq!(*rebuilt_up, *original_up + 1); + } } diff --git a/src/rust/lqos_bakery/src/qdisc_handles.rs b/src/rust/lqos_bakery/src/qdisc_handles.rs new file mode 100644 index 00000000..7b7a93bc --- /dev/null +++ b/src/rust/lqos_bakery/src/qdisc_handles.rs @@ -0,0 +1,494 @@ +use lqos_config::Config; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use tracing::{debug, warn}; + +/// Root MQ handle major reserved by LibreQoS. +pub(crate) const MQ_ROOT_HANDLE_MAJOR: u16 = 0x7FFF; +/// Prefer allocating dynamic circuit qdiscs from the upper handle space first. +const PREFERRED_CIRCUIT_START: u16 = 0x9000; +/// Highest qdisc handle major we will allocate dynamically. +const MAX_DYNAMIC_HANDLE_MAJOR: u16 = 0xFFFE; +/// Base range used for deterministic per-queue infrastructure qdisc handles. +const INFRA_HANDLE_BASE: u16 = 0x2000; +/// JSON version for persisted Bakery qdisc-handle state. +const QDISC_HANDLE_FILE_VERSION: u32 = 1; +const QDISC_HANDLE_FILE: &str = "bakery_qdisc_handles.json"; + +/// Per-device MQ queue layout used to reserve non-circuit qdisc handles. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct MqDeviceLayout { + devices: BTreeMap>, +} + +impl MqDeviceLayout { + /// Builds the reserved queue-major layout from the current MQ setup command. + pub(crate) fn from_setup( + config: &Config, + queues_available: usize, + stick_offset: usize, + ) -> Self { + let mut devices = BTreeMap::>::new(); + + let isp = config.isp_interface().to_string(); + let isp_entry = devices.entry(isp).or_default(); + for queue in 0..queues_available { + let major = (queue + 1) as u16; + isp_entry.insert(major); + } + + let internet = config.internet_interface().to_string(); + let internet_entry = devices.entry(internet).or_default(); + for queue in 0..queues_available { + let major = (queue + stick_offset + 1) as u16; + internet_entry.insert(major); + } + + Self { devices } + } + + /// Returns the qdisc-handle majors reserved on a single interface. + pub(crate) fn reserved_handles(&self, interface: &str) -> HashSet { + let mut reserved = HashSet::new(); + reserved.insert(MQ_ROOT_HANDLE_MAJOR); + reserved.insert(0xFFFF); + + if let Some(majors) = self.devices.get(interface) { + for major in majors { + reserved.insert(*major); + reserved.insert(infra_qdisc_handle(*major, InfraQdiscSlot::Primary)); + reserved.insert(infra_qdisc_handle(*major, InfraQdiscSlot::Default)); + } + } + + reserved + } +} + +/// Slot identifier for fixed per-queue infrastructure qdiscs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InfraQdiscSlot { + Primary, + Default, +} + +/// Computes the deterministic explicit qdisc handle for a fixed infrastructure leaf. +pub(crate) fn infra_qdisc_handle(queue_major: u16, slot: InfraQdiscSlot) -> u16 { + let base = INFRA_HANDLE_BASE.saturating_add(queue_major.saturating_mul(2)); + match slot { + InfraQdiscSlot::Primary => base, + InfraQdiscSlot::Default => base.saturating_add(1), + } +} + +/// Persisted circuit-to-qdisc handle assignments, grouped by interface. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct QdiscHandleState { + interfaces: BTreeMap, + retired_handles: BTreeMap>, +} + +impl QdiscHandleState { + /// Loads qdisc-handle state from the LibreQoS runtime directory. + pub(crate) fn load(config: &Config) -> Self { + let path = state_path(config); + let Ok(raw) = std::fs::read_to_string(&path) else { + return Self::default(); + }; + let Ok(file) = serde_json::from_str::(&raw) else { + warn!( + "Bakery: ignoring unreadable qdisc-handle state at {:?}", + path + ); + return Self::default(); + }; + if file.version != QDISC_HANDLE_FILE_VERSION { + debug!( + "Bakery: ignoring qdisc-handle state version {} at {:?}", + file.version, path + ); + return Self::default(); + } + + let interfaces = file + .interfaces + .into_iter() + .map(|(iface, entry)| { + let circuits = entry + .circuits + .into_iter() + .filter_map(|(hash, handle)| hash.parse::().ok().map(|h| (h, handle))) + .collect(); + (iface, InterfaceHandleState::from_circuits(circuits)) + }) + .collect(); + + Self { + interfaces, + retired_handles: BTreeMap::new(), + } + } + + /// Clears transient retired-handle reservations from the previous apply cycle. + pub(crate) fn clear_retired_handles(&mut self) { + for (interface, retired) in self.retired_handles.iter() { + let Some(state) = self.interfaces.get_mut(interface) else { + continue; + }; + for handle in retired { + state.note_reusable_handle(*handle); + } + } + self.retired_handles.clear(); + } + + /// Saves qdisc-handle state to the LibreQoS runtime directory. + pub(crate) fn save(&self, config: &Config) { + let path = state_path(config); + if let Some(parent) = path.parent() + && let Err(e) = std::fs::create_dir_all(parent) + { + warn!( + "Bakery: unable to create qdisc-handle state directory {:?}: {}", + parent, e + ); + return; + } + + let persisted = PersistedQdiscHandleState { + version: QDISC_HANDLE_FILE_VERSION, + interfaces: self + .interfaces + .iter() + .map(|(iface, circuits)| { + ( + iface.clone(), + PersistedInterfaceHandleState { + circuits: circuits + .circuits + .iter() + .map(|(hash, handle)| (hash.to_string(), *handle)) + .collect(), + }, + ) + }) + .collect(), + }; + + let Ok(serialized) = serde_json::to_string_pretty(&persisted) else { + warn!("Bakery: unable to serialize qdisc-handle state"); + return; + }; + + let temp_path = path.with_extension("json.tmp"); + if let Err(e) = std::fs::write(&temp_path, serialized.as_bytes()) { + warn!( + "Bakery: unable to write temporary qdisc-handle state {:?}: {}", + temp_path, e + ); + return; + } + + if let Err(e) = std::fs::rename(&temp_path, &path) { + warn!( + "Bakery: unable to atomically replace qdisc-handle state {:?}: {}", + path, e + ); + let _ = std::fs::remove_file(&temp_path); + } + } + + /// Returns a stable explicit qdisc handle for the circuit on the given interface. + pub(crate) fn assign_circuit_handle( + &mut self, + interface: &str, + circuit_hash: i64, + reserved: &HashSet, + ) -> Option { + if let Some(existing) = self + .interfaces + .get(interface) + .and_then(|state| state.circuits.get(&circuit_hash)) + .copied() + && !reserved.contains(&existing) + && self + .interfaces + .get(interface) + .and_then(|state| state.owner_of(existing)) + == Some(circuit_hash) + { + return Some(existing); + } + + let retired = self.retired_handles.get(interface); + let state = self.interfaces.entry(interface.to_string()).or_default(); + let next = state.next_free_handle(reserved, retired)?; + state.insert(circuit_hash, next); + Some(next) + } + + /// Forces a circuit to rotate to a different explicit qdisc handle on an interface. + pub(crate) fn rotate_circuit_handle( + &mut self, + interface: &str, + circuit_hash: i64, + reserved: &HashSet, + ) -> Option { + let retired_handle = self + .interfaces + .get(interface) + .and_then(|state| state.circuits.get(&circuit_hash)) + .copied(); + + let retired = self.retired_handles.get(interface); + let state = self.interfaces.entry(interface.to_string()).or_default(); + let next = state.next_free_handle(reserved, retired)?; + if let Some(handle) = retired_handle { + self.retired_handles + .entry(interface.to_string()) + .or_default() + .insert(handle); + } + state.insert(circuit_hash, next); + Some(next) + } + + /// Releases a circuit handle on an interface so it can be reused deterministically. + pub(crate) fn release_circuit(&mut self, interface: &str, circuit_hash: i64) -> bool { + self.interfaces + .get_mut(interface) + .and_then(|state| state.remove(circuit_hash)) + .is_some() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct InterfaceHandleState { + circuits: BTreeMap, + owners: HashMap, + next_candidate: u16, +} + +impl Default for InterfaceHandleState { + fn default() -> Self { + Self { + circuits: BTreeMap::new(), + owners: HashMap::new(), + next_candidate: PREFERRED_CIRCUIT_START, + } + } +} + +impl InterfaceHandleState { + fn from_circuits(circuits: BTreeMap) -> Self { + let mut state = Self { + circuits, + owners: HashMap::new(), + next_candidate: PREFERRED_CIRCUIT_START, + }; + for (circuit_hash, handle) in state.circuits.iter() { + state.owners.insert(*handle, *circuit_hash); + } + state + } + + fn owner_of(&self, handle: u16) -> Option { + self.owners.get(&handle).copied() + } + + fn insert(&mut self, circuit_hash: i64, handle: u16) { + if let Some(previous) = self.circuits.insert(circuit_hash, handle) { + self.owners.remove(&previous); + self.note_reusable_handle(previous); + } + self.owners.insert(handle, circuit_hash); + if self.next_candidate == handle { + self.next_candidate = advance_handle_candidate(handle); + } + } + + fn remove(&mut self, circuit_hash: i64) -> Option { + let removed = self.circuits.remove(&circuit_hash)?; + self.owners.remove(&removed); + self.note_reusable_handle(removed); + Some(removed) + } + + fn note_reusable_handle(&mut self, handle: u16) { + if handle >= PREFERRED_CIRCUIT_START || self.next_candidate < PREFERRED_CIRCUIT_START { + self.next_candidate = self.next_candidate.min(handle); + } else if self.circuits.is_empty() { + self.next_candidate = PREFERRED_CIRCUIT_START; + } + } + + fn next_free_handle( + &mut self, + reserved: &HashSet, + retired: Option<&BTreeSet>, + ) -> Option { + if let Some(handle) = self.scan_available( + self.next_candidate, + MAX_DYNAMIC_HANDLE_MAJOR, + reserved, + retired, + ) { + self.next_candidate = advance_handle_candidate(handle); + return Some(handle); + } + + if self.next_candidate > PREFERRED_CIRCUIT_START + && let Some(handle) = self.scan_available( + PREFERRED_CIRCUIT_START, + self.next_candidate - 1, + reserved, + retired, + ) + { + self.next_candidate = advance_handle_candidate(handle); + return Some(handle); + } + + let low_end = PREFERRED_CIRCUIT_START.saturating_sub(1); + let low_start = if self.next_candidate < PREFERRED_CIRCUIT_START { + self.next_candidate.max(1) + } else { + 1 + }; + + if let Some(handle) = self.scan_available(low_start, low_end, reserved, retired) { + self.next_candidate = advance_handle_candidate(handle); + return Some(handle); + } + + if low_start > 1 + && let Some(handle) = self.scan_available(1, low_start - 1, reserved, retired) + { + self.next_candidate = advance_handle_candidate(handle); + return Some(handle); + } + + None + } + + fn scan_available( + &self, + start: u16, + end: u16, + reserved: &HashSet, + retired: Option<&BTreeSet>, + ) -> Option { + if start > end { + return None; + } + + for handle in start..=end { + if reserved.contains(&handle) { + continue; + } + if self.owners.contains_key(&handle) { + continue; + } + if retired.is_some_and(|set| set.contains(&handle)) { + continue; + } + return Some(handle); + } + None + } +} + +fn advance_handle_candidate(handle: u16) -> u16 { + if handle >= MAX_DYNAMIC_HANDLE_MAJOR { + 1 + } else { + handle + 1 + } +} + +fn state_path(config: &Config) -> PathBuf { + Path::new(&config.lqos_directory).join(QDISC_HANDLE_FILE) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct PersistedQdiscHandleState { + version: u32, + #[serde(default)] + interfaces: BTreeMap, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +struct PersistedInterfaceHandleState { + #[serde(default)] + circuits: BTreeMap, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deterministic_primary_default_handles_are_unique() { + let primary = infra_qdisc_handle(1, InfraQdiscSlot::Primary); + let default = infra_qdisc_handle(1, InfraQdiscSlot::Default); + assert_ne!(primary, default); + assert_eq!(primary, 0x2002); + assert_eq!(default, 0x2003); + } + + #[test] + fn allocator_prefers_high_range_and_reuses_existing() { + let mut state = QdiscHandleState::default(); + let reserved = HashSet::from([0x7FFF, 0xFFFF]); + let a = state + .assign_circuit_handle("eth0", 1, &reserved) + .expect("first handle"); + let b = state + .assign_circuit_handle("eth0", 2, &reserved) + .expect("second handle"); + assert_eq!(a, PREFERRED_CIRCUIT_START); + assert_eq!(b, PREFERRED_CIRCUIT_START + 1); + let a_again = state + .assign_circuit_handle("eth0", 1, &reserved) + .expect("reused handle"); + assert_eq!(a, a_again); + } + + #[test] + fn allocator_can_rotate_to_a_new_handle() { + let mut state = QdiscHandleState::default(); + let reserved = HashSet::from([0x7FFF, 0xFFFF]); + let first = state + .assign_circuit_handle("eth0", 1, &reserved) + .expect("first handle"); + let rotated = state + .rotate_circuit_handle("eth0", 1, &reserved) + .expect("rotated handle"); + assert_ne!(first, rotated); + assert_eq!(rotated, first + 1); + } + + #[test] + fn rotated_handles_are_not_reused_until_cleared() { + let mut state = QdiscHandleState::default(); + let reserved = HashSet::from([0x7FFF, 0xFFFF]); + let first = state + .assign_circuit_handle("eth0", 1, &reserved) + .expect("first handle"); + let rotated = state + .rotate_circuit_handle("eth0", 1, &reserved) + .expect("rotated handle"); + let second_circuit = state + .assign_circuit_handle("eth0", 2, &reserved) + .expect("second circuit handle"); + assert_ne!(second_circuit, first); + assert_eq!(second_circuit, rotated + 1); + + state.clear_retired_handles(); + let third_circuit = state + .assign_circuit_handle("eth0", 3, &reserved) + .expect("third circuit handle"); + assert_eq!(third_circuit, first); + } +} diff --git a/src/rust/lqos_bakery/src/queue_math.rs b/src/rust/lqos_bakery/src/queue_math.rs index 7c20cbb2..c58cea57 100644 --- a/src/rust/lqos_bakery/src/queue_math.rs +++ b/src/rust/lqos_bakery/src/queue_math.rs @@ -1,6 +1,12 @@ use std::sync::Arc; use tracing::debug; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SqmKind { + Cake, + FqCodel, +} + pub(crate) fn sqm_as_vec(config: &Arc) -> Vec { config .queues @@ -192,3 +198,17 @@ pub(crate) fn sqm_tokens_for( Some(_) => sqm_rate_fixup(rate, config), // defensive fallback } } + +pub(crate) fn effective_sqm_kind( + rate: f32, + config: &Arc, + override_opt: &Option, +) -> SqmKind { + match sqm_tokens_for(rate, config, override_opt) + .first() + .map(String::as_str) + { + Some("fq_codel") => SqmKind::FqCodel, + _ => SqmKind::Cake, + } +} diff --git a/src/rust/lqos_bakery/src/utils.rs b/src/rust/lqos_bakery/src/utils.rs index 41e9f68e..94eb96cb 100644 --- a/src/rust/lqos_bakery/src/utils.rs +++ b/src/rust/lqos_bakery/src/utils.rs @@ -1,8 +1,12 @@ +use lqos_bus::TcHandle; use parking_lot::Mutex; +use std::collections::{HashMap, HashSet, VecDeque}; use std::fs::File; use std::io::{BufWriter, Write}; use std::path::Path; -use tracing::{error, info}; +use std::sync::LazyLock; +use std::time::{Duration, Instant}; +use tracing::{debug, error, info, warn}; /// Get the current Unix timestamp in seconds pub(crate) fn current_timestamp() -> u64 { @@ -13,79 +17,709 @@ pub(crate) fn current_timestamp() -> u64 { } static FILE_LOCK: Mutex<()> = Mutex::new(()); +static LIVE_TC_SNAPSHOT_CACHE: LazyLock> = + LazyLock::new(|| Mutex::new(LiveTcSnapshotCache::new())); +static TC_IO_CADENCE_STATE: LazyLock> = + LazyLock::new(|| Mutex::new(TcIoCadenceState::new())); +const LIVE_TC_SNAPSHOT_MAX_AGE_MS: u64 = 250; +const TC_IO_INTERVAL_WINDOW: usize = 128; -pub(crate) fn execute_in_memory(command_buffer: &Vec>, purpose: &str) { - let lock = FILE_LOCK.lock(); +#[derive(Clone)] +struct TimedClassSnapshot { + captured_at: Instant, + snapshot: HashMap, +} + +#[derive(Clone)] +struct TimedQdiscSnapshot { + captured_at: Instant, + entries: Vec, +} + +struct LiveTcSnapshotCache { + class_snapshots: HashMap, + qdisc_snapshots: HashMap, +} + +impl LiveTcSnapshotCache { + fn new() -> Self { + Self { + class_snapshots: HashMap::new(), + qdisc_snapshots: HashMap::new(), + } + } + + fn invalidate(&mut self) { + self.class_snapshots.clear(); + self.qdisc_snapshots.clear(); + } +} + +struct TcIoCadenceState { + last_event_at: Option, + last_event_unix: Option, + intervals_ms: VecDeque, + interval_sum_ms: u128, +} + +impl TcIoCadenceState { + fn new() -> Self { + Self { + last_event_at: None, + last_event_unix: None, + intervals_ms: VecDeque::new(), + interval_sum_ms: 0, + } + } + + fn record_event(&mut self) { + let now = Instant::now(); + let now_unix = current_timestamp(); + if let Some(last) = self.last_event_at { + let interval_ms = now.duration_since(last).as_millis() as u64; + self.intervals_ms.push_back(interval_ms); + self.interval_sum_ms = self.interval_sum_ms.saturating_add(u128::from(interval_ms)); + while self.intervals_ms.len() > TC_IO_INTERVAL_WINDOW { + if let Some(removed) = self.intervals_ms.pop_front() { + self.interval_sum_ms = self.interval_sum_ms.saturating_sub(u128::from(removed)); + } + } + } + self.last_event_at = Some(now); + self.last_event_unix = Some(now_unix); + } + + fn snapshot(&self) -> TcIoCadenceSnapshot { + let sample_count = self.intervals_ms.len(); + let avg_interval_ms = if sample_count == 0 { + None + } else { + Some((self.interval_sum_ms / sample_count as u128) as u64) + }; + TcIoCadenceSnapshot { + avg_interval_ms, + last_event_unix: self.last_event_unix, + sample_count, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct TcIoCadenceSnapshot { + pub(crate) avg_interval_ms: Option, + pub(crate) last_event_unix: Option, + pub(crate) sample_count: usize, +} + +#[derive(Clone, Debug)] +pub(crate) struct ExecuteResult { + pub(crate) ok: bool, + pub(crate) duration_ms: u64, + pub(crate) failure_summary: Option, +} + +/// Lightweight host-memory snapshot used by Bakery safety checks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MemorySnapshot { + /// Total installed RAM in bytes. + pub total_bytes: u64, + /// Currently available RAM in bytes. + pub available_bytes: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct LiveTcClassEntry { + pub(crate) class_id: TcHandle, + pub(crate) parent: Option, + pub(crate) leaf_qdisc_major: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct LiveTcQdiscEntry { + pub(crate) kind: String, + pub(crate) handle: Option, + pub(crate) parent: Option, + pub(crate) is_root: bool, +} + +fn format_numbered_lines(lines: &str, starting_line_number: usize) -> String { + let mut numbered = String::new(); + for (i, line) in lines.lines().enumerate() { + numbered.push_str(&format!("{:>4}: {}\n", starting_line_number + i, line)); + } + numbered +} + +fn run_tc_batch(path: &Path, purpose: &str) -> Result { + record_tc_io_event(); + std::process::Command::new("/sbin/tc") + .args(["-f", "-batch", path.to_str().unwrap_or_default()]) + .output() + .map_err(|_| format!("Failed to execute tc batch command for {purpose}.")) +} + +fn summarize_tc_batch_failure(output: &std::process::Output) -> Option { + if output.status.success() { + return None; + } + + let stderr = String::from_utf8_lossy(&output.stderr); + let stderr = stderr.trim(); + + let status_summary = match output.status.code() { + Some(code) => format!("tc batch exited with status {code}"), + None => "tc batch terminated by signal".to_string(), + }; + + if stderr.is_empty() { + Some(status_summary) + } else { + Some(format!("{status_summary}: {stderr}")) + } +} + +fn tc_batch_command_is_delete_only(line: &str) -> bool { + let mut parts = line.split_whitespace(); + matches!( + (parts.next(), parts.next()), + (Some("qdisc"), Some("del")) | (Some("class"), Some("del")) + ) +} + +fn tc_batch_failure_is_ignorable_delete_absence( + output: &std::process::Output, + lines: &str, +) -> bool { + if output.status.success() { + return false; + } + + if lines.trim().is_empty() || !lines.lines().all(tc_batch_command_is_delete_only) { + return false; + } + + let stderr = String::from_utf8_lossy(&output.stderr); + if stderr.trim().is_empty() { + return false; + } + + stderr.lines().all(|line| { + let trimmed = line.trim().to_ascii_lowercase(); + trimmed.is_empty() + || trimmed.starts_with("command failed ") + || trimmed.starts_with("error: specified class not found") + || trimmed.starts_with("error: cannot find specified qdisc on specified device") + || trimmed.starts_with("rtnetlink answers: no such file or directory") + }) +} + +fn tc_success_stderr_is_harmless(stderr: &str) -> bool { + let trimmed = stderr.trim(); + if trimmed.is_empty() { + return false; + } + + trimmed.lines().all(|line| { + let normalized = line.trim(); + normalized.is_empty() || normalized.to_ascii_lowercase().starts_with("warning:") + }) +} + +pub(crate) fn read_memory_snapshot() -> Result { + let raw = std::fs::read_to_string("/proc/meminfo") + .map_err(|e| format!("Failed to read /proc/meminfo: {e}"))?; + parse_memory_snapshot(&raw) +} + +fn parse_memory_snapshot(raw: &str) -> Result { + let mut total_kib = None; + let mut available_kib = None; + + for line in raw.lines() { + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let kib = value + .split_whitespace() + .next() + .and_then(|n| n.parse::().ok()); + match key.trim() { + "MemTotal" => total_kib = kib, + "MemAvailable" => available_kib = kib, + _ => {} + } + } + + let total_kib = total_kib.ok_or_else(|| "MemTotal missing from /proc/meminfo".to_string())?; + let available_kib = + available_kib.ok_or_else(|| "MemAvailable missing from /proc/meminfo".to_string())?; + + Ok(MemorySnapshot { + total_bytes: total_kib.saturating_mul(1024), + available_bytes: available_kib.saturating_mul(1024), + }) +} + +fn memory_guard_failure_summary( + snapshot: MemorySnapshot, + min_available_bytes: u64, + purpose: &str, + chunk_number: usize, + total_chunks: usize, + phase: &str, +) -> String { + format!( + "Bakery memory guard stopped {purpose} {phase} chunk {}/{}: available memory {} bytes is below safety floor {} bytes (total RAM {} bytes).", + chunk_number, + total_chunks, + snapshot.available_bytes, + min_available_bytes, + snapshot.total_bytes + ) +} + +#[allow(dead_code)] +pub(crate) fn invalidate_live_tc_snapshots() { + let mut cache = LIVE_TC_SNAPSHOT_CACHE.lock(); + cache.invalidate(); +} + +fn record_tc_io_event() { + TC_IO_CADENCE_STATE.lock().record_event(); +} + +pub(crate) fn tc_io_cadence_snapshot() -> TcIoCadenceSnapshot { + TC_IO_CADENCE_STATE.lock().snapshot() +} + +fn read_live_qdisc_snapshot_raw(interface: &str) -> Result, String> { + record_tc_io_event(); + let output = std::process::Command::new("/sbin/tc") + .args(["-s", "-j", "qdisc", "show", "dev", interface]) + .output() + .map_err(|e| format!("Failed to snapshot live qdiscs on {interface}: {e}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!( + "Failed to snapshot live qdiscs on {interface}: {}", + stderr.trim() + )); + } + + let stdout = String::from_utf8(output.stdout) + .map_err(|e| format!("Live qdisc snapshot on {interface} was not UTF-8: {e}"))?; + + parse_live_qdisc_snapshot(&stdout) + .map_err(|e| format!("Failed to parse live qdisc snapshot on {interface}: {e}")) +} + +pub(crate) fn read_live_qdisc_snapshot(interface: &str) -> Result, String> { + let _lock = FILE_LOCK.lock(); + let mut cache = LIVE_TC_SNAPSHOT_CACHE.lock(); + let now = Instant::now(); + if let Some(entry) = cache.qdisc_snapshots.get(interface) + && now.duration_since(entry.captured_at) + <= Duration::from_millis(LIVE_TC_SNAPSHOT_MAX_AGE_MS) + { + return Ok(entry.entries.clone()); + } + + let entries = read_live_qdisc_snapshot_raw(interface)?; + cache.qdisc_snapshots.insert( + interface.to_string(), + TimedQdiscSnapshot { + captured_at: now, + entries: entries.clone(), + }, + ); + Ok(entries) +} + +pub(crate) fn read_live_qdisc_handle_majors(interface: &str) -> Result, String> { + let entries = read_live_qdisc_snapshot(interface)?; + Ok(entries + .into_iter() + .filter_map(|entry| entry.handle) + .filter_map(|handle| { + let (major, _) = handle.get_major_minor(); + (major != 0).then_some(major) + }) + .collect()) +} + +fn read_live_class_snapshot_raw( + interface: &str, +) -> Result, String> { + record_tc_io_event(); + let output = std::process::Command::new("/sbin/tc") + .args(["class", "show", "dev", interface]) + .output() + .map_err(|e| format!("Failed to snapshot live classes on {interface}: {e}"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!( + "Failed to snapshot live classes on {interface}: {}", + stderr.trim() + )); + } + + let stdout = String::from_utf8(output.stdout) + .map_err(|e| format!("Live class snapshot on {interface} was not UTF-8: {e}"))?; + + parse_live_class_snapshot(&stdout) + .map_err(|e| format!("Failed to parse live class snapshot on {interface}: {e}")) +} + +pub(crate) fn read_live_class_snapshot( + interface: &str, +) -> Result, String> { + let _lock = FILE_LOCK.lock(); + let mut cache = LIVE_TC_SNAPSHOT_CACHE.lock(); + let now = Instant::now(); + if let Some(entry) = cache.class_snapshots.get(interface) + && now.duration_since(entry.captured_at) + <= Duration::from_millis(LIVE_TC_SNAPSHOT_MAX_AGE_MS) + { + return Ok(entry.snapshot.clone()); + } + + let snapshot = read_live_class_snapshot_raw(interface)?; + cache.class_snapshots.insert( + interface.to_string(), + TimedClassSnapshot { + captured_at: now, + snapshot: snapshot.clone(), + }, + ); + Ok(snapshot) +} + +fn parse_live_qdisc_snapshot(raw_json: &str) -> Result, String> { + let parsed = serde_json::from_str::(raw_json) + .map_err(|e| format!("invalid JSON: {e}"))?; + let items = parsed + .as_array() + .ok_or_else(|| "expected JSON array from tc qdisc show -j".to_string())?; + + let mut entries = Vec::with_capacity(items.len()); + for item in items { + let kind = item + .get("kind") + .and_then(|value| value.as_str()) + .unwrap_or_default() + .to_string(); + let handle = item + .get("handle") + .and_then(|value| value.as_str()) + .and_then(|value| TcHandle::from_string(value).ok()); + let parent_raw = item.get("parent").and_then(|value| value.as_str()); + let parent = parent_raw.and_then(|value| { + if value.eq_ignore_ascii_case("root") { + None + } else { + TcHandle::from_string(value).ok() + } + }); + let is_root = item + .get("root") + .and_then(|value| value.as_bool()) + .unwrap_or(false) + || matches!(parent_raw, Some(value) if value.eq_ignore_ascii_case("root")); + entries.push(LiveTcQdiscEntry { + kind, + handle, + parent, + is_root, + }); + } + + Ok(entries) +} + +fn parse_live_class_snapshot(raw: &str) -> Result, String> { + let mut snapshot = HashMap::new(); + + for line in raw.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || !trimmed.starts_with("class ") { + continue; + } + + let tokens: Vec<&str> = trimmed.split_whitespace().collect(); + if tokens.len() < 3 { + return Err(format!("Malformed tc class line: {trimmed}")); + } + + let class_id = TcHandle::from_string(tokens[2]).map_err(|e| { + format!( + "Invalid tc class handle {:?} in line {:?}: {:?}", + tokens[2], trimmed, e + ) + })?; + + let mut parent = None; + let mut leaf_qdisc_major = None; + let mut idx = 3usize; + while idx < tokens.len() { + match tokens[idx] { + "parent" if idx + 1 < tokens.len() => { + if let Ok(handle) = TcHandle::from_string(tokens[idx + 1]) { + parent = Some(handle); + } + idx += 2; + } + "leaf" if idx + 1 < tokens.len() => { + if let Ok(handle) = TcHandle::from_string(tokens[idx + 1]) { + let (major, _) = handle.get_major_minor(); + if major != 0 { + leaf_qdisc_major = Some(major); + } + } + idx += 2; + } + _ => idx += 1, + } + } + + snapshot.insert( + class_id, + LiveTcClassEntry { + class_id, + parent, + leaf_qdisc_major, + }, + ); + } + + Ok(snapshot) +} + +pub(crate) fn execute_in_memory(command_buffer: &[Vec], purpose: &str) -> ExecuteResult { + execute_in_memory_chunked( + command_buffer, + purpose, + command_buffer.len().max(1), + None, + |_, _, _, _| {}, + ) +} + +pub(crate) fn execute_in_memory_chunked( + command_buffer: &[Vec], + purpose: &str, + chunk_size: usize, + memory_guard_min_available_bytes: Option, + mut on_progress: F, +) -> ExecuteResult +where + F: FnMut(usize, usize, usize, usize), +{ + let started = std::time::Instant::now(); + let _lock = FILE_LOCK.lock(); info!( "Bakery: Executing in-memory commands: {} lines, for {purpose}", command_buffer.len() ); - let path = Path::new("/tmp/lqos_bakery_commands.txt"); - let Some(lines) = write_command_file(path, command_buffer) else { + let full_path = Path::new("/tmp/lqos_bakery_commands.txt"); + let Some(_) = write_command_file(full_path, command_buffer) else { error!("Failed to write commands to file for {purpose}"); - return; + return ExecuteResult { + ok: false, + duration_ms: started.elapsed().as_millis() as u64, + failure_summary: Some(format!("Failed to write commands to file for {purpose}")), + }; }; - let Ok(output) = std::process::Command::new("/sbin/tc") - .args(["-f", "-batch", path.to_str().unwrap_or_default()]) - .output() - else { - let message = format!("Failed to execute tc batch command for {purpose}."); - error!(message); - return; - }; - - let output_str = String::from_utf8_lossy(&output.stdout) - .replace("Error: Exclusivity flag on, cannot modify.\n", ""); - if !output_str.is_empty() { - error!("Command output for ({purpose}): {:?}", output_str.trim()); + if command_buffer.is_empty() { + return ExecuteResult { + ok: true, + duration_ms: started.elapsed().as_millis() as u64, + failure_summary: None, + }; } - let error_str = String::from_utf8_lossy(&output.stderr); - if !error_str.is_empty() { - // Add line numbers to aid debugging with tc -batch's reported line numbers - let mut numbered = String::new(); - for (i, line) in lines.lines().enumerate() { - // tc -batch reports 1-based line numbers - numbered.push_str(&format!("{:>4}: {}\n", i + 1, line)); + let chunk_size = chunk_size.max(1); + let total_chunks = command_buffer.len().div_ceil(chunk_size); + let chunk_path = Path::new("/tmp/lqos_bakery_commands_chunk.txt"); + let mut completed_commands = 0usize; + let mut completed_chunks = 0usize; + + for chunk in command_buffer.chunks(chunk_size) { + if let Some(min_available_bytes) = memory_guard_min_available_bytes { + match read_memory_snapshot() { + Ok(snapshot) if snapshot.available_bytes < min_available_bytes => { + let summary = memory_guard_failure_summary( + snapshot, + min_available_bytes, + purpose, + completed_chunks + 1, + total_chunks, + "before", + ); + error!("{summary}"); + return ExecuteResult { + ok: false, + duration_ms: started.elapsed().as_millis() as u64, + failure_summary: Some(summary), + }; + } + Ok(_) => {} + Err(err) => { + let summary = format!( + "Bakery memory guard could not read host memory state before chunk {}/{} for {purpose}: {err}", + completed_chunks + 1, + total_chunks + ); + error!("{summary}"); + return ExecuteResult { + ok: false, + duration_ms: started.elapsed().as_millis() as u64, + failure_summary: Some(summary), + }; + } + } } - let detailed = format!( - "Command error for ({purpose}): {}\nCommands with line numbers:\n{}", - error_str.trim(), - numbered + + let global_line_start = completed_commands + 1; + let Some(lines) = write_command_file(chunk_path, chunk) else { + error!("Failed to write chunked commands to file for {purpose}"); + return ExecuteResult { + ok: false, + duration_ms: started.elapsed().as_millis() as u64, + failure_summary: Some(format!("Failed to write commands to file for {purpose}")), + }; + }; + + let output = match run_tc_batch(chunk_path, purpose) { + Ok(output) => output, + Err(message) => { + error!(message); + return ExecuteResult { + ok: false, + duration_ms: started.elapsed().as_millis() as u64, + failure_summary: Some(message), + }; + } + }; + + let output_str = String::from_utf8_lossy(&output.stdout) + .replace("Error: Exclusivity flag on, cannot modify.\n", ""); + if !output_str.is_empty() { + error!("Command output for ({purpose}): {:?}", output_str.trim()); + } + let stderr = String::from_utf8_lossy(&output.stderr); + if output.status.success() && !stderr.trim().is_empty() { + if tc_success_stderr_is_harmless(stderr.trim()) { + debug!("Command stderr for ({purpose}): {:?}", stderr.trim()); + } else { + warn!("Command stderr for ({purpose}): {:?}", stderr.trim()); + } + } + + if tc_batch_failure_is_ignorable_delete_absence(&output, &lines) { + debug!( + "Bakery tolerated delete-only tc batch absence during {purpose}; targets were already gone" + ); + } else if let Some(failure_summary) = summarize_tc_batch_failure(&output) { + let numbered = format_numbered_lines(&lines, global_line_start); + let chunk_line_end = global_line_start + chunk.len().saturating_sub(1); + let detailed = format!( + "Command error for ({purpose}): {}\nFailed chunk {}/{} (global lines {}-{})\nFull batch: {}\nChunk batch: {}\nChunk commands with global line numbers:\n{}", + failure_summary, + completed_chunks + 1, + total_chunks, + global_line_start, + chunk_line_end, + full_path.display(), + chunk_path.display(), + numbered + ); + error!(detailed); + + let ts = current_timestamp(); + let path_ts = Path::new("/tmp").join(format!("lqos_bakery_failed_{}.txt", ts)); + if let Ok(mut f) = File::create(&path_ts) { + let _ = f.write_all(detailed.as_bytes()); + let _ = f.flush(); + error!( + "Bakery wrote numbered command failure to {}", + path_ts.display() + ); + } else { + error!( + "Bakery failed to write numbered command failure file: {}", + path_ts.display() + ); + } + let path_last = Path::new("/tmp/lqos_bakery_last_error.txt"); + if let Ok(mut f) = File::create(path_last) { + let _ = f.write_all(detailed.as_bytes()); + let _ = f.flush(); + } + return ExecuteResult { + ok: false, + duration_ms: started.elapsed().as_millis() as u64, + failure_summary: Some(failure_summary), + }; + } + + completed_commands += chunk.len(); + completed_chunks += 1; + LIVE_TC_SNAPSHOT_CACHE.lock().invalidate(); + + if let Some(min_available_bytes) = memory_guard_min_available_bytes { + match read_memory_snapshot() { + Ok(snapshot) if snapshot.available_bytes < min_available_bytes => { + let summary = memory_guard_failure_summary( + snapshot, + min_available_bytes, + purpose, + completed_chunks, + total_chunks, + "after", + ); + error!("{summary}"); + return ExecuteResult { + ok: false, + duration_ms: started.elapsed().as_millis() as u64, + failure_summary: Some(summary), + }; + } + Ok(_) => {} + Err(err) => { + let summary = format!( + "Bakery memory guard could not read host memory state after chunk {}/{} for {purpose}: {err}", + completed_chunks, total_chunks + ); + error!("{summary}"); + return ExecuteResult { + ok: false, + duration_ms: started.elapsed().as_millis() as u64, + failure_summary: Some(summary), + }; + } + } + } + + on_progress( + completed_commands, + command_buffer.len(), + completed_chunks, + total_chunks, ); - // Log to console - error!(detailed); - - // Also persist to /tmp for easier debugging of large batches - let ts = current_timestamp(); - let path_ts = Path::new("/tmp").join(format!("lqos_bakery_failed_{}.txt", ts)); - if let Ok(mut f) = File::create(&path_ts) { - let _ = f.write_all(detailed.as_bytes()); - let _ = f.flush(); - error!( - "Bakery wrote numbered command failure to {}", - path_ts.display() - ); - } else { - error!( - "Bakery failed to write numbered command failure file: {}", - path_ts.display() - ); - } - // Convenience: keep a rolling 'last error' file - let path_last = Path::new("/tmp/lqos_bakery_last_error.txt"); - if let Ok(mut f) = File::create(path_last) { - let _ = f.write_all(detailed.as_bytes()); - let _ = f.flush(); - } } - drop(lock); // Explicitly drop the lock to release it. This happens automatically at the end of the scope, but it's good to be explicit. + ExecuteResult { + ok: true, + duration_ms: started.elapsed().as_millis() as u64, + failure_summary: None, + } } -pub(crate) fn write_command_file(path: &Path, commands: &Vec>) -> Option { +pub(crate) fn write_command_file(path: &Path, commands: &[Vec]) -> Option { let mut lines = String::new(); let Ok(file) = File::create(path) else { error!("Failed to create output file: {}", path.display()); @@ -120,3 +754,195 @@ pub(crate) fn write_command_file(path: &Path, commands: &Vec>) -> Op } Some(lines) } + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::process::ExitStatusExt; + + fn mock_tc_output(status: i32, stdout: &str, stderr: &str) -> std::process::Output { + std::process::Output { + status: std::process::ExitStatus::from_raw(status << 8), + stdout: stdout.as_bytes().to_vec(), + stderr: stderr.as_bytes().to_vec(), + } + } + + #[test] + fn parse_live_qdisc_snapshot_extracts_root_parent_and_handle_data() { + let raw = r#" +[ + { "kind": "mq", "handle": "7fff:", "parent": "root" }, + { "kind": "cake", "handle": "90f1:", "parent": "2:1039" }, + { "kind": "fq_codel", "handle": "50c0:", "parent": "8:24dd" }, + { "kind": "ingress", "handle": "ffff:", "parent": "ffff:fff1", "root": false }, + { "kind": "fq_codel", "handle": "0:", "parent": "3:20" } +] +"#; + let snapshot = parse_live_qdisc_snapshot(raw).expect("snapshot parsed"); + assert_eq!(snapshot.len(), 5); + + let root = &snapshot[0]; + assert_eq!(root.kind, "mq"); + assert_eq!( + root.handle, + Some(TcHandle::from_string("7fff:").expect("valid")) + ); + assert_eq!(root.parent, None); + assert!(root.is_root); + + let child = &snapshot[1]; + assert_eq!(child.kind, "cake"); + assert_eq!( + child.parent, + Some(TcHandle::from_string("2:1039").expect("valid")) + ); + assert!(!child.is_root); + + let zero_handle = &snapshot[4]; + assert_eq!( + zero_handle.handle, + Some(TcHandle::from_string("0:").expect("valid")) + ); + assert_eq!( + zero_handle.parent, + Some(TcHandle::from_string("3:20").expect("valid")) + ); + } + + #[test] + fn read_live_qdisc_handle_majors_collects_non_zero_handles_from_snapshot() { + let raw = r#" +[ + { "kind": "mq", "handle": "7fff:", "parent": "root" }, + { "kind": "cake", "handle": "90f1:", "parent": "2:1039" }, + { "kind": "fq_codel", "handle": "50c0:", "parent": "8:24dd" }, + { "kind": "ingress", "handle": "ffff:", "parent": "ffff:fff1" }, + { "kind": "fq_codel", "handle": "0:", "parent": "3:20" } +] +"#; + let snapshot = parse_live_qdisc_snapshot(raw).expect("snapshot parsed"); + let handles: HashSet = snapshot + .into_iter() + .filter_map(|entry| entry.handle) + .filter_map(|handle| { + let (major, _) = handle.get_major_minor(); + (major != 0).then_some(major) + }) + .collect(); + assert!(handles.contains(&0x7fff)); + assert!(handles.contains(&0x90f1)); + assert!(handles.contains(&0x50c0)); + assert!(handles.contains(&0xffff)); + assert!(!handles.contains(&0)); + } + + #[test] + fn parse_live_qdisc_snapshot_rejects_non_arrays() { + let err = + parse_live_qdisc_snapshot(r#"{"handle":"90f1:"}"#).expect_err("non-array should fail"); + assert!(err.contains("expected JSON array")); + } + + #[test] + fn parse_live_class_snapshot_extracts_parent_and_leaf() { + let raw = "\ +class htb 1:da parent 1:4 leaf ddad: prio 3 rate 20Mbit ceil 100Mbit burst 1600b cburst 1600b +class htb 1:4 root rate 949Mbit ceil 950Mbit burst 1423b cburst 1425b +"; + let snapshot = parse_live_class_snapshot(raw).expect("snapshot parsed"); + let class_da = snapshot + .get(&TcHandle::from_string("1:da").expect("valid class")) + .expect("class 1:da present"); + assert_eq!( + class_da.parent, + Some(TcHandle::from_string("1:4").expect("valid parent")) + ); + assert_eq!(class_da.leaf_qdisc_major, Some(0xddad)); + + let class_root = snapshot + .get(&TcHandle::from_string("1:4").expect("valid class")) + .expect("class 1:4 present"); + assert_eq!(class_root.parent, None); + assert_eq!(class_root.leaf_qdisc_major, None); + } + + #[test] + fn parse_memory_snapshot_extracts_total_and_available() { + let raw = "\ +MemTotal: 32768000 kB +MemFree: 1024000 kB +MemAvailable: 8192000 kB +"; + let snapshot = parse_memory_snapshot(raw).expect("meminfo parsed"); + assert_eq!(snapshot.total_bytes, 32_768_000_u64 * 1024); + assert_eq!(snapshot.available_bytes, 8_192_000_u64 * 1024); + } + + #[test] + fn parse_memory_snapshot_requires_memavailable() { + let raw = "\ +MemTotal: 32768000 kB +MemFree: 1024000 kB +"; + let err = parse_memory_snapshot(raw).expect_err("memavailable should be required"); + assert!(err.contains("MemAvailable")); + } + + #[test] + fn summarize_tc_batch_failure_rejects_nonzero_exit_without_stderr() { + let output = mock_tc_output(1, "", ""); + let summary = summarize_tc_batch_failure(&output).expect("nonzero exit should fail"); + assert!(summary.contains("status 1")); + } + + #[test] + fn summarize_tc_batch_failure_includes_stderr_with_exit_status() { + let output = mock_tc_output(2, "", "RTNETLINK answers: Invalid argument\n"); + let summary = summarize_tc_batch_failure(&output).expect("stderr failure should be kept"); + assert!(summary.contains("status 2")); + assert!(summary.contains("Invalid argument")); + } + + #[test] + fn summarize_tc_batch_failure_accepts_clean_success() { + let output = mock_tc_output(0, "", ""); + assert!(summarize_tc_batch_failure(&output).is_none()); + } + + #[test] + fn summarize_tc_batch_failure_accepts_success_with_stderr_warning() { + let output = mock_tc_output(0, "", "Warning: sch_htb: quantum of class 10134 is big.\n"); + assert!(summarize_tc_batch_failure(&output).is_none()); + assert!(tc_success_stderr_is_harmless( + "Warning: sch_htb: quantum of class 10134 is big.\n" + )); + } + + #[test] + fn harmless_success_stderr_rejects_non_warning_lines() { + assert!(!tc_success_stderr_is_harmless( + "RTNETLINK answers: No such file or directory\n" + )); + } + + #[test] + fn ignorable_delete_absence_accepts_missing_delete_targets() { + let output = mock_tc_output( + 1, + "", + "Error: Specified class not found.\nCommand failed /tmp/x:1\nRTNETLINK answers: No such file or directory\nCommand failed /tmp/x:2\n", + ); + let lines = "qdisc del dev if0 parent 0x1:0x2000\nclass del dev if0 parent 0x1:0x3 classid 0x1:0x2000\n"; + assert!(tc_batch_failure_is_ignorable_delete_absence(&output, lines)); + } + + #[test] + fn ignorable_delete_absence_rejects_non_delete_failures() { + let output = mock_tc_output(1, "", "Error: HTB class in use.\n"); + let lines = "class del dev if0 parent 0x1:0x35 classid 0x1:0x39\n"; + assert!(!tc_batch_failure_is_ignorable_delete_absence( + &output, lines + )); + } +} diff --git a/src/rust/lqos_bus/src/bus/mod.rs b/src/rust/lqos_bus/src/bus/mod.rs index fb042258..39826cf0 100644 --- a/src/rust/lqos_bus/src/bus/mod.rs +++ b/src/rust/lqos_bus/src/bus/mod.rs @@ -12,11 +12,15 @@ mod unix_socket_server; pub use client::{LibreqosBusClient, bus_request}; pub use queue_data::*; pub use reply::BusReply; -pub use request::{BlackboardSystem, BusRequest, TopFlowType, UrgentSeverity, UrgentSource}; +pub use request::{ + BakeryCapacityReportInterface, BlackboardSystem, BusRequest, TopFlowType, UrgentSeverity, + UrgentSource, +}; #[allow(unused_imports)] pub use response::{ AsnHeatmapData, BakeryStatsSnapshot, BusResponse, CircuitHeatmapData, SiteHeatmapData, - StormguardDebugDirection, StormguardDebugEntry, UrgentIssue, + StormguardDebugDirection, StormguardDebugEntry, TreeGuardRuntimeNodeBranchSnapshot, + TreeGuardRuntimeNodeOperationSnapshot, UrgentIssue, }; pub use session::BusSession; use thiserror::Error; @@ -30,20 +34,28 @@ pub const BUS_SOCKET_PATH: &str = "/run/lqos/bus"; /// that the directory exists. pub(crate) const BUS_SOCKET_DIRECTORY: &str = "/run/lqos"; +/// Errors returned by local bus clients while connecting to or communicating +/// with the `lqosd` Unix socket. #[derive(Error, Debug)] pub enum BusClientError { + /// The local bus socket does not yet exist or is not reachable. #[error( "Socket (typically /run/lqos/bus) not found. Check that lqosd is running, and you have permission to access the socket path." )] SocketNotFound, + /// The request could not be encoded for transport. #[error("Unable to encode request")] EncodingError, + /// The reply could not be decoded from the socket stream. #[error("Unable to decode request")] DecodingError, + /// Writing the request to the socket failed. #[error("Cannot write to socket")] StreamWriteError, + /// Reading the reply from the socket failed. #[error("Cannot read from socket")] StreamReadError, + /// The socket connection is no longer usable. #[error("Stream is no longer connected")] StreamNotConnected, } diff --git a/src/rust/lqos_bus/src/bus/request.rs b/src/rust/lqos_bus/src/bus/request.rs index af02e55d..912f374e 100644 --- a/src/rust/lqos_bus/src/bus/request.rs +++ b/src/rust/lqos_bus/src/bus/request.rs @@ -6,6 +6,23 @@ use allocative::Allocative; use lqos_config::Tunables; use serde::{Deserialize, Serialize}; +/// Per-interface Bakery qdisc-budget report entry. +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Allocative)] +pub struct BakeryCapacityReportInterface { + /// Interface name, e.g. `ens19`. + pub name: String, + /// Planned qdisc count for the interface. + pub planned_qdiscs: usize, + /// Planned infrastructure qdiscs (`mq`, `htb`, etc.) for the interface. + pub infra_qdiscs: usize, + /// Planned `cake` leaf qdiscs for the interface. + pub cake_qdiscs: usize, + /// Planned `fq_codel` leaf qdiscs for the interface. + pub fq_codel_qdiscs: usize, + /// Estimated kernel memory cost for the interface's planned qdiscs. + pub estimated_memory_bytes: u64, +} + /// Source system for an urgent issue #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Copy, Allocative)] pub enum UrgentSource { @@ -182,6 +199,9 @@ pub enum BusRequest { /// Requests that the configuration be updated UpdateLqosdConfig(Box), + /// Invalidate cached Node Manager authentication state. + InvalidateAuthCache, + /// Request that we start watching a circuit's queue WatchQueue(String), @@ -327,6 +347,10 @@ pub enum BusRequest { BakeryAddCircuit { /// The circuit hash, which is a unique identifier for the circuit circuit_hash: i64, + /// Human-readable circuit name when known at queue-build time. + circuit_name: Option, + /// Human-readable parent site/node name when known at queue-build time. + site_name: Option, /// The parent class ID for the circuit parent_class_id: TcHandle, /// The upload parent class ID for the circuit @@ -360,6 +384,57 @@ pub enum BusRequest { /// Get current Bakery statistics GetBakeryStats, + /// Report the most recent Bakery qdisc-budget preflight result. + BakeryReportPreflight { + /// Whether the planned full reload fits within the current safe budget. + ok: bool, + /// Operator-facing summary of the preflight result. + message: String, + /// Safe operational budget used for the preflight. + safe_budget: usize, + /// Hard kernel limit for reference. + hard_limit: usize, + /// Estimated total kernel memory cost of the planned qdisc model. + estimated_total_memory_bytes: u64, + /// Current host memory available during preflight, if known. + memory_available_bytes: Option, + /// Memory floor that must remain available before the apply proceeds. + memory_guard_min_available_bytes: u64, + /// Whether the memory preflight passed. + memory_ok: bool, + /// Per-interface planned qdisc counts. + interfaces: Vec, + }, + + /// Request a live Bakery runtime node virtualization or restore for a named node. + /// + /// This is intended for local daemon-side orchestration and tests that need a deterministic + /// runtime node operation without waiting for TreeGuard policy to fire. + TreeGuardSetNodeVirtual { + /// Exact node name from `network.json`. + node_name: String, + /// `true` to runtime-virtualize, `false` to restore. + virtualized: bool, + }, + + /// Request the latest Bakery runtime node-operation snapshot for a named node. + /// + /// This is intended for local daemon-side orchestration and tests that need to observe whether + /// a runtime node operation was deferred, is still applying, or failed after acceptance. + TreeGuardGetNodeVirtualStatus { + /// Exact node name from `network.json`. + node_name: String, + }, + + /// Request the latest Bakery runtime branch-state snapshot for a named node. + /// + /// This is intended for local daemon-side orchestration and tests that need to verify which + /// retained branch is active after a runtime virtualization or restore. + TreeGuardGetNodeVirtualBranchState { + /// Exact node name from `network.json`. + node_name: String, + }, + /// Announce that the API is ready ApiReady, diff --git a/src/rust/lqos_bus/src/bus/response.rs b/src/rust/lqos_bus/src/bus/response.rs index 192024ec..d87d5716 100644 --- a/src/rust/lqos_bus/src/bus/response.rs +++ b/src/rust/lqos_bus/src/bus/response.rs @@ -48,6 +48,56 @@ pub struct BakeryStatsSnapshot { pub active_circuits: u64, } +/// Serializable snapshot of a Bakery-tracked TreeGuard runtime node operation. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Allocative)] +pub struct TreeGuardRuntimeNodeOperationSnapshot { + /// Monotonic Bakery-local operation identifier. + pub operation_id: u64, + /// Stable Bakery site hash derived from the node name. + pub site_hash: i64, + /// Requested runtime action (`virtualize` or `restore`). + pub action: String, + /// Current operation status. + pub status: String, + /// Number of retry/apply attempts performed so far. + pub attempt_count: u32, + /// Unix timestamp when the operation was submitted. + pub submitted_at_unix: u64, + /// Unix timestamp when the operation last changed state. + pub updated_at_unix: u64, + /// Optional unix timestamp for the next retry, if waiting. + pub next_retry_at_unix: Option, + /// Last error observed by Bakery for this operation, if any. + pub last_error: Option, + /// Structured failure reason observed by Bakery for this operation, if any. + pub failure_reason: Option, +} + +/// Serializable snapshot of Bakery's retained runtime branch state for a TreeGuard node. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Allocative)] +pub struct TreeGuardRuntimeNodeBranchSnapshot { + /// Stable Bakery site hash derived from the node name. + pub site_hash: i64, + /// Which retained branch is currently active (`shadow` or `original`). + pub active_branch: String, + /// Current runtime lifecycle label for the node. + pub lifecycle: String, + /// Whether Bakery is still waiting to prune the inactive branch. + pub pending_prune: bool, + /// Optional unix timestamp for the next prune/cleanup retry, if one is pending. + pub next_prune_attempt_unix: Option, + /// Hashes of branch sites currently marked active. + pub active_site_hashes: Vec, + /// Hashes of original/saved sites retained for restore logic. + pub saved_site_hashes: Vec, + /// Hashes of inactive-branch sites queued for pruning. + pub prune_site_hashes: Vec, + /// Observed downlink qdisc major for the runtime root, if known. + pub qdisc_down_major: Option, + /// Observed uplink qdisc major for the runtime root, if known. + pub qdisc_up_major: Option, +} + /// Circuit-level TemporalHeatmap data for the executive summary. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Allocative)] pub struct CircuitHeatmapData { @@ -111,6 +161,14 @@ pub struct ExecutiveSummaryHeader { pub htb_queue_count: u64, /// Number of CAKE queues being tracked. pub cake_queue_count: u64, + /// Number of fq_codel queues being tracked. + pub fq_codel_queue_count: u64, + /// Whether the displayed queue counts are intentionally held at their last-known values. + #[serde(default)] + pub queue_stats_stale: bool, + /// Whether Bakery is currently applying a full reload batch. + #[serde(default)] + pub bakery_reload_in_progress: bool, /// Whether Insight is connected. pub insight_connected: bool, } @@ -625,4 +683,10 @@ pub enum BusResponse { /// Insight license summary (licensed + optional max circuits). InsightLicenseSummary(InsightLicenseSummary), + + /// Latest Bakery runtime node-operation snapshot for a named TreeGuard node, if any. + TreeGuardRuntimeNodeOperation(Option), + + /// Latest Bakery runtime branch-state snapshot for a named TreeGuard node, if any. + TreeGuardRuntimeNodeBranch(Option), } diff --git a/src/rust/lqos_bus/src/ip_stats.rs b/src/rust/lqos_bus/src/ip_stats.rs index 14b22899..bf68742a 100644 --- a/src/rust/lqos_bus/src/ip_stats.rs +++ b/src/rust/lqos_bus/src/ip_stats.rs @@ -3,7 +3,7 @@ use crate::TcHandle; use allocative::Allocative; -use lqos_utils::units::DownUpOrder; +use lqos_utils::units::{DownUpOrder, TcpRetransmitSample}; use serde::{Deserialize, Serialize}; use std::net::IpAddr; @@ -30,8 +30,8 @@ pub struct IpStats { /// Associated TC traffic control handle. pub tc_handle: TcHandle, - /// TCP Retransmits for this host at the current time. - pub tcp_retransmits: (f64, f64), + /// TCP retransmit samples for this host at the current time. + pub tcp_retransmit_sample: DownUpOrder, } /// Represents an IP Mapping in the XDP IP to TC/CPU mapping system. @@ -224,10 +224,8 @@ pub struct Circuit { /// QoO score (0..100), per direction. #[serde(default)] pub qoo: DownUpOrder>, - /// TCP Retransmits for this host at the current time. - pub tcp_retransmits: DownUpOrder, - /// The number of TCP packets per second. - pub tcp_packets: DownUpOrder, + /// TCP retransmit samples for this host at the current time. + pub tcp_retransmit_sample: DownUpOrder, /// The mapped circuit ID pub circuit_id: Option, /// The mapped device ID diff --git a/src/rust/lqos_bus/src/lib.rs b/src/rust/lqos_bus/src/lib.rs index 7d076a29..90f82972 100644 --- a/src/rust/lqos_bus/src/lib.rs +++ b/src/rust/lqos_bus/src/lib.rs @@ -29,12 +29,14 @@ pub use bus::response::{ CircuitHeatmapData, CountryListEntry, DeviceCounts, ExecutiveSummaryHeader, FlowMapPoint, FlowTimelineEntry, InsightLicenseSummary, NodeCapacity, ProtocolListEntry, QueueStatsTotal, RetransmitSummary, SchedulerDetails, SearchResultEntry, SiteHeatmapData, - StormguardDebugDirection, StormguardDebugEntry, UrgentIssue, WarningLevel, + StormguardDebugDirection, StormguardDebugEntry, TreeGuardRuntimeNodeBranchSnapshot, + TreeGuardRuntimeNodeOperationSnapshot, UrgentIssue, WarningLevel, }; pub use bus::{ - BUS_SOCKET_PATH, BlackboardSystem, BusReply, BusRequest, BusResponse, BusSession, - CakeDiffTinTransit, CakeDiffTransit, CakeTransit, LibreqosBusClient, QueueStoreTransit, - TopFlowType, UnixSocketServer, UrgentSeverity, UrgentSource, bus_request, + BUS_SOCKET_PATH, BakeryCapacityReportInterface, BlackboardSystem, BusClientError, BusReply, + BusRequest, BusResponse, BusSession, CakeDiffTinTransit, CakeDiffTransit, CakeTransit, + LibreqosBusClient, QueueStoreTransit, TopFlowType, UnixSocketServer, UrgentSeverity, + UrgentSource, bus_request, }; pub use tc_handle::TcHandle; diff --git a/src/rust/lqos_config/Cargo.toml b/src/rust/lqos_config/Cargo.toml index 170ac5ab..64de7383 100644 --- a/src/rust/lqos_config/Cargo.toml +++ b/src/rust/lqos_config/Cargo.toml @@ -13,6 +13,8 @@ csv = { workspace = true } ip_network_table = { workspace = true } ip_network = { workspace = true } sha2 = { workspace = true } +argon2 = { workspace = true } +rand_core = { workspace = true } uuid = { workspace = true } tracing = { workspace = true } toml = { workspace = true } diff --git a/src/rust/lqos_config/src/authentication.rs b/src/rust/lqos_config/src/authentication.rs index 154bc84d..3262030b 100644 --- a/src/rust/lqos_config/src/authentication.rs +++ b/src/rust/lqos_config/src/authentication.rs @@ -2,11 +2,13 @@ //! local web UI on LibreQoS boxes. It maps to `//lqusers.toml` use allocative::Allocative; +use argon2::{Argon2, PasswordHash, PasswordHasher, PasswordVerifier, password_hash::SaltString}; +use rand_core::OsRng; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ fmt::Display, - fs::{OpenOptions, read_to_string, remove_file}, + fs::{OpenOptions, read_to_string, remove_file, rename}, io::Write, path::{Path, PathBuf}, }; @@ -14,6 +16,21 @@ use thiserror::Error; use tracing::{error, warn}; use uuid::Uuid; +const AUTH_FILE_VERSION: u32 = 2; +const LEGACY_AUTH_FILE_VERSION: u32 = 1; +const INITIAL_AUTH_EPOCH: u64 = 1; +const LEGACY_AUTH_FILE_NAME: &str = "webusers.toml"; +const CURRENT_AUTH_FILE_NAME: &str = "lqusers.toml"; +const LEGACY_PASSWORD_PEPPER: &str = "_LibreQosLikesPasswordsForDinner"; + +fn default_auth_file_version() -> u32 { + LEGACY_AUTH_FILE_VERSION +} + +fn default_auth_epoch() -> u64 { + INITIAL_AUTH_EPOCH +} + /// Access rights of a user #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, Allocative)] pub enum UserRole { @@ -59,29 +76,91 @@ impl Display for UserRole { pub struct WebUser { /// The user's username. pub username: String, - /// The user's password hash. + /// The user's password hash. This may be a legacy SHA-256 hash or an + /// Argon2id PHC string until the user next logs in. pub password_hash: String, /// The user's role. pub role: UserRole, - /// The user's token. - pub token: String, +} + +/// Result of authenticating a single user. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuthenticatedUser { + /// The authenticated username. + pub username: String, + /// The user's role. + pub role: UserRole, + /// Current auth epoch after any migration-side rewrite. + pub auth_epoch: u64, + /// True when a legacy password hash was upgraded to Argon2id. + pub password_upgraded: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct PasswordVerification { + valid: bool, + needs_rehash: bool, } /// Container holding the authorized web users. -#[derive(Clone, Debug, Deserialize, Serialize, Default, Allocative)] +#[derive(Clone, Debug, Deserialize, Serialize, Allocative)] pub struct WebUsers { + #[serde(default = "default_auth_file_version")] + version: u32, + #[serde(default = "default_auth_epoch")] + auth_epoch: u64, + #[serde(default)] allow_unauthenticated_to_view: bool, + #[serde(default)] users: Vec, } +impl Default for WebUsers { + fn default() -> Self { + Self { + version: AUTH_FILE_VERSION, + auth_epoch: INITIAL_AUTH_EPOCH, + allow_unauthenticated_to_view: false, + users: Vec::new(), + } + } +} + impl WebUsers { - fn path() -> Result { + fn base_path() -> Result { let base_path = crate::load_config() .map_err(|_| AuthenticationError::UnableToLoadEtcLqos)? .lqos_directory .clone(); - let filename = Path::new(&base_path).join("lqusers.toml"); - Ok(filename) + Ok(PathBuf::from(base_path)) + } + + fn primary_path() -> Result { + Ok(Self::base_path()?.join(CURRENT_AUTH_FILE_NAME)) + } + + fn legacy_path() -> Result { + Ok(Self::base_path()?.join(LEGACY_AUTH_FILE_NAME)) + } + + /// Returns the current `lqusers.toml` path. + pub fn path() -> Result { + Self::primary_path() + } + + /// Returns the existing auth file path, checking the legacy filename as a fallback. + pub fn existing_path() -> Result, AuthenticationError> { + let current = Self::primary_path()?; + if current.exists() { + return Ok(Some(current)); + } + + let legacy = Self::legacy_path()?; + if legacy.exists() { + return Ok(Some(legacy)); + } + + Ok(None) } /// Is the list of users empty? @@ -89,66 +168,156 @@ impl WebUsers { self.users.is_empty() } + fn normalize_for_save(&self) -> Self { + let mut normalized = self.clone(); + normalized.version = AUTH_FILE_VERSION; + if normalized.auth_epoch == 0 { + normalized.auth_epoch = INITIAL_AUTH_EPOCH; + } + normalized + } + fn save_to_disk(&self) -> Result<(), AuthenticationError> { - let path = Self::path()?; - let new_contents = - toml_edit::ser::to_string(&self).map_err(AuthenticationError::SerializationError)?; - if path.exists() && remove_file(&path).is_err() { - error!("Unable to delete web users file"); - return Err(AuthenticationError::UnableToDelete); + let path = Self::primary_path()?; + let tmp_path = path.with_extension(format!("toml.tmp-{}", Uuid::new_v4())); + let normalized = self.normalize_for_save(); + let new_contents = toml_edit::ser::to_string(&normalized) + .map_err(AuthenticationError::SerializationError)?; + + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp_path) + .map_err(|e| { + error!( + "Unable to open temporary auth file {:?} for writing: {e}", + tmp_path + ); + AuthenticationError::UnableToWrite + })?; + file.write_all(new_contents.as_bytes()).map_err(|e| { + error!("Unable to write temporary auth file {:?}: {e}", tmp_path); + AuthenticationError::UnableToWrite + })?; + drop(file); + + rename(&tmp_path, &path).map_err(|e| { + error!( + "Unable to rename temporary auth file {:?} to {:?}: {e}", + tmp_path, path + ); + let _ = remove_file(&tmp_path); + AuthenticationError::UnableToWrite + })?; + + let legacy_path = Self::legacy_path()?; + if legacy_path != path + && legacy_path.exists() + && let Err(e) = remove_file(&legacy_path) + { + warn!("Unable to remove legacy auth file {:?}: {e}", legacy_path); } - if let Ok(mut file) = OpenOptions::new().write(true).create_new(true).open(path) { - if file.write_all(new_contents.as_bytes()).is_err() { - error!("Unable to write web users file to disk."); - return Err(AuthenticationError::UnableToWrite); - } - } else { - error!("Unable to open web users file for writing."); - return Err(AuthenticationError::UnableToWrite); + + Ok(()) + } + + fn migrate_if_needed(&mut self, loaded_from: &Path) -> Result<(), AuthenticationError> { + let current_path = Self::primary_path()?; + let loaded_from_legacy_path = loaded_from != current_path; + let mut needs_rewrite = false; + + if self.version != AUTH_FILE_VERSION { + self.version = AUTH_FILE_VERSION; + needs_rewrite = true; } + + if self.auth_epoch == 0 { + self.auth_epoch = INITIAL_AUTH_EPOCH; + needs_rewrite = true; + } + + if loaded_from_legacy_path { + needs_rewrite = true; + } + + if needs_rewrite { + self.bump_auth_epoch(); + self.save_to_disk()?; + } + Ok(()) } /// Does the user's file exist? True if it does, false otherwise. pub fn does_users_file_exist() -> Result { - Ok(Self::path()?.exists()) + Ok(Self::existing_path()?.is_some()) } - /// Try to load `lqusers.toml`. If it is unavailable, create a new--empty-- - /// file. + /// Try to load `lqusers.toml`, creating a new version 2 file if no auth file exists. pub fn load_or_create() -> Result { - let path = Self::path()?; - if !path.exists() { - // Create a new users file, save it and return the - // empty file + if let Some(path) = Self::existing_path()? { + let raw = read_to_string(&path).map_err(|e| { + error!("Unable to read auth file {:?}: {e}", path); + AuthenticationError::UnableToRead + })?; + let mut users: Self = toml_edit::de::from_str(&raw).map_err(|e| { + error!("Unable to deserialize auth file {:?}: {e}", path); + AuthenticationError::UnableToParse + })?; + users.migrate_if_needed(&path)?; + Ok(users) + } else { let new_users = Self::default(); new_users.save_to_disk()?; Ok(new_users) - } else { - // Load from disk - if let Ok(raw) = read_to_string(path) { - let parse_result = toml_edit::de::from_str(&raw); - if let Ok(users) = parse_result { - Ok(users) - } else { - error!("Unable to deserialize lqusers.toml. Error in next message."); - error!("{:?}", parse_result); - Err(AuthenticationError::UnableToParse) - } - } else { - error!("Unable to read lqusers.toml"); - Err(AuthenticationError::UnableToRead) - } } } - fn hash_password(password: &str) -> String { - let salted = format!("!x{password}_LibreQosLikesPasswordsForDinner"); + fn hash_password(password: &str) -> Result { + let salt = SaltString::generate(&mut OsRng); + Argon2::default() + .hash_password(password.as_bytes(), &salt) + .map(|hash| hash.to_string()) + .map_err(AuthenticationError::PasswordHashError) + } + + fn hash_password_legacy(password: &str) -> String { + let salted = format!("!x{password}{LEGACY_PASSWORD_PEPPER}"); let mut sha256 = Sha256::new(); sha256.update(salted); format!("{:X}", sha256.finalize()) } + fn verify_password( + password: &str, + password_hash: &str, + ) -> Result { + if password_hash.starts_with("$argon2") { + let parsed = + PasswordHash::new(password_hash).map_err(AuthenticationError::PasswordHashError)?; + Ok(PasswordVerification { + valid: Argon2::default() + .verify_password(password.as_bytes(), &parsed) + .is_ok(), + needs_rehash: false, + }) + } else { + Ok(PasswordVerification { + valid: Self::hash_password_legacy(password) == password_hash, + needs_rehash: true, + }) + } + } + + fn bump_auth_epoch(&mut self) { + self.auth_epoch = self.auth_epoch.saturating_add(1).max(INITIAL_AUTH_EPOCH); + } + + /// Returns the current auth epoch used to revoke signed sessions. + pub fn auth_epoch(&self) -> u64 { + self.auth_epoch + } + /// If a user exists with this username, update their details to the /// provided values. If the user does not exist, create them with the /// provided values. @@ -157,25 +326,23 @@ impl WebUsers { username: &str, password: &str, role: UserRole, - ) -> Result { - let token; // Assigned in a branch + ) -> Result<(), AuthenticationError> { + let password_hash = Self::hash_password(password)?; if let Some(user) = self.users.iter_mut().find(|u| u.username == username) { - user.password_hash = Self::hash_password(password); + user.password_hash = password_hash; user.role = role; - token = user.token.clone(); } else { - token = Uuid::new_v4().to_string(); let new_user = WebUser { username: username.to_string(), - password_hash: Self::hash_password(password), + password_hash, role, - token: token.clone(), }; self.users.push(new_user); } + self.bump_auth_epoch(); self.save_to_disk()?; - Ok(token) + Ok(()) } /// Update an existing user, optionally changing their password. @@ -189,20 +356,18 @@ impl WebUsers { username: &str, password: Option<&str>, role: UserRole, - ) -> Result { - let token; - if let Some(user) = self.users.iter_mut().find(|u| u.username == username) { - if let Some(password) = password { - user.password_hash = Self::hash_password(password); - } - user.role = role; - token = user.token.clone(); - } else { + ) -> Result<(), AuthenticationError> { + let Some(user) = self.users.iter_mut().find(|u| u.username == username) else { return Err(AuthenticationError::UserNotFound); - } + }; + if let Some(password) = password { + user.password_hash = Self::hash_password(password)?; + } + user.role = role; + self.bump_auth_epoch(); self.save_to_disk()?; - Ok(token) + Ok(()) } /// Delete a user from `lqusers.toml` @@ -213,48 +378,42 @@ impl WebUsers { error!("User {username} not found, hence not deleted."); return Err(AuthenticationError::UserNotFound); } + self.bump_auth_epoch(); self.save_to_disk()?; Ok(()) } - /// Attempt a login with the specified username and password. If - /// the login succeeds, returns the publically shareable token that - /// uniquely identifies the user a a string. If it fails, returns an - /// `Err`. - pub fn login(&self, username: &str, password: &str) -> Result { - let hash = Self::hash_password(password); - if let Some(user) = self - .users - .iter() - .find(|u| u.username == username && u.password_hash == hash) - { - Ok(user.token.clone()) - } else if self.allow_unauthenticated_to_view { - Ok("default".to_string()) - } else { - Err(AuthenticationError::InvalidLogin) - } - } + /// Attempt a login with the specified username and password. If the login + /// succeeds, returns the authenticated user details and transparently + /// upgrades legacy password hashes to Argon2id. + pub fn authenticate( + &mut self, + username: &str, + password: &str, + ) -> Result { + let Some(index) = self.users.iter().position(|u| u.username == username) else { + return Err(AuthenticationError::InvalidLogin); + }; - /// Given a token, lookup the matching user and return their role. - pub fn get_role_from_token(&self, token: &str) -> Result { - if let Some(user) = self.users.iter().find(|u| u.token == token) { - Ok(user.role) - } else if self.allow_unauthenticated_to_view { - Ok(UserRole::ReadOnly) - } else { - warn!("Token {token} not found, invalid data access attempt."); - Err(AuthenticationError::InvalidToken) + let verification = Self::verify_password(password, &self.users[index].password_hash)?; + if !verification.valid { + return Err(AuthenticationError::InvalidLogin); } - } - /// Given a token, lookup the matching user and return their username. - pub fn get_username(&self, token: &str) -> String { - if let Some(user) = self.users.iter().find(|u| u.token == token) { - user.username.clone() - } else { - "Anonymous".to_string() + let mut password_upgraded = false; + if verification.needs_rehash { + self.users[index].password_hash = Self::hash_password(password)?; + self.bump_auth_epoch(); + self.save_to_disk()?; + password_upgraded = true; } + + Ok(AuthenticatedUser { + username: self.users[index].username.clone(), + role: self.users[index].role, + auth_epoch: self.auth_epoch, + password_upgraded, + }) } /// Dump all users to the console. @@ -275,6 +434,7 @@ impl WebUsers { /// for demonstration purposes. pub fn allow_anonymous(&mut self, allow: bool) -> Result<(), AuthenticationError> { self.allow_unauthenticated_to_view = allow; + self.bump_auth_epoch(); self.save_to_disk()?; Ok(()) } @@ -286,16 +446,6 @@ impl WebUsers { } /// Errors that can occur while managing web-UI authentication. -/// -/// This enum groups failures encountered by helpers in this module while -/// interacting with `lqusers.toml` and related configuration, including: -/// - Resolving the LibreQoS config directory via `crate::load_config`. -/// - Serializing/deserializing user data with `toml_edit`. -/// - Reading, writing, or deleting the `lqusers.toml` file on disk. -/// - Looking up users, validating credentials, and resolving tokens. -/// -/// Each variant implements a concise display message via `thiserror::Error` -/// and is suitable for logging or bubbling up to higher layers. #[derive(Error, Debug)] pub enum AuthenticationError { /// Failed to load the main configuration (`/etc/lqos.conf`) to @@ -306,11 +456,8 @@ pub enum AuthenticationError { /// `lqusers.toml` to disk. #[error("Unable to serialize to TOML")] SerializationError(toml_edit::ser::Error), - /// Failed to remove an existing `lqusers.toml` prior to rewriting it. - #[error("Unable to remove existing web users file")] - UnableToDelete, - /// Failed to create or write `lqusers.toml` (e.g., permissions or IO). - #[error("Unable to open lqusers.toml for writing. Check permissions?")] + /// Failed to persist `lqusers.toml` (e.g., permissions or IO). + #[error("Unable to write lqusers.toml")] UnableToWrite, /// Failed to read `lqusers.toml` from disk. #[error("Unable to read lqusers.toml")] @@ -318,13 +465,13 @@ pub enum AuthenticationError { /// Failed to parse `lqusers.toml` contents as valid TOML. #[error("Unable to parse lqusers.toml")] UnableToParse, + /// Password hash creation or verification failed. + #[error("Unable to process password hash")] + PasswordHashError(argon2::password_hash::Error), /// Attempted to remove or reference a user that does not exist. #[error("User not found")] UserNotFound, - /// Username/password did not match (and anonymous read-only is disabled). + /// Username/password did not match. #[error("Invalid Login")] InvalidLogin, - /// Provided token did not match any user (and anonymous read-only is disabled). - #[error("Invalid User Token")] - InvalidToken, } diff --git a/src/rust/lqos_config/src/circuit_ethernet_metadata.rs b/src/rust/lqos_config/src/circuit_ethernet_metadata.rs new file mode 100644 index 00000000..784d6a5c --- /dev/null +++ b/src/rust/lqos_config/src/circuit_ethernet_metadata.rs @@ -0,0 +1,42 @@ +use serde::{Deserialize, Serialize}; + +/// Runtime metadata filename for circuit Ethernet advisories emitted by integrations. +pub const CIRCUIT_ETHERNET_METADATA_FILENAME: &str = "circuit_ethernet_metadata.json"; + +/// Collection of circuit Ethernet advisories keyed by circuit identity. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct CircuitEthernetMetadataFile { + /// Per-circuit Ethernet advisory entries. + pub circuits: Vec, +} + +/// Describes a detected negotiated Ethernet speed and any automatic shaping cap applied to a circuit. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct CircuitEthernetMetadata { + /// Circuit identifier as emitted to `ShapedDevices.csv`. + pub circuit_id: String, + /// Human-readable circuit name for UI display. + pub circuit_name: String, + /// Device IDs considered when determining the circuit Ethernet limit. + pub device_ids: Vec, + /// Integration/source that produced the advisory. + pub source: String, + /// Negotiated Ethernet speed in Mbps for the limiting device/interface. + pub negotiated_ethernet_mbps: u64, + /// Requested download max before any Ethernet-based cap was applied. + pub requested_download_mbps: f32, + /// Requested upload max before any Ethernet-based cap was applied. + pub requested_upload_mbps: f32, + /// Applied download max after Ethernet-based capping. + pub applied_download_mbps: f32, + /// Applied upload max after Ethernet-based capping. + pub applied_upload_mbps: f32, + /// Whether the Ethernet advisory reduced at least one shaping direction. + pub auto_capped: bool, + /// Device ID of the limiting device when known. + pub limiting_device_id: Option, + /// Device name of the limiting device when known. + pub limiting_device_name: Option, + /// Interface name that reported the limiting Ethernet speed when known. + pub limiting_interface_name: Option, +} diff --git a/src/rust/lqos_config/src/cpu_topology.rs b/src/rust/lqos_config/src/cpu_topology.rs index 6032d37e..5b3472a2 100644 --- a/src/rust/lqos_config/src/cpu_topology.rs +++ b/src/rust/lqos_config/src/cpu_topology.rs @@ -432,6 +432,11 @@ fn detect_from_cpuid(possible: &[u32]) -> Option { #[cfg(target_arch = "x86_64")] use std::arch::x86_64::__cpuid_count; + // Query leaf 0 first to determine the maximum supported CPUID leaf before using leaf 0x1A. + if __cpuid_count(0, 0).eax < 0x1a { + return None; + } + let original_affinity = sched_getaffinity(Pid::from_raw(0)).ok()?; let restore_affinity = original_affinity; let mut performance = Vec::new(); @@ -448,7 +453,7 @@ fn detect_from_cpuid(possible: &[u32]) -> Option { return None; } - let leaf = unsafe { __cpuid_count(0x1a, 0) }; + let leaf = __cpuid_count(0x1a, 0); let core_type = (leaf.eax >> 24) & 0xff; match core_type { CPUID_CORE_TYPE_CORE => performance.push(*cpu), diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs index fa272d54..4e04c636 100644 --- a/src/rust/lqos_config/src/etc/migration.rs +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -202,7 +202,13 @@ fn migrate_queues( new_config: &mut Config, ) -> Result<(), MigrationError> { new_config.queues.default_sqm = python_config.sqm.clone(); - new_config.queues.monitor_only = python_config.monitor_only_mode; + new_config + .queues + .set_queue_mode(if python_config.monitor_only_mode { + super::QueueMode::Observe + } else { + super::QueueMode::Shape + }); new_config.queues.uplink_bandwidth_mbps = python_config.upstream_bandwidth_capacity_upload_mbps as u64; new_config.queues.downlink_bandwidth_mbps = diff --git a/src/rust/lqos_config/src/etc/mod.rs b/src/rust/lqos_config/src/etc/mod.rs index 1e538123..244a3ef6 100644 --- a/src/rust/lqos_config/src/etc/mod.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -10,6 +10,7 @@ use once_cell::sync::Lazy; use std::path::Path; use std::sync::Arc; use thiserror::Error; +use toml_edit::{DocumentMut, value}; use tracing::{debug, error, info}; mod migration; @@ -18,12 +19,14 @@ mod python_migration; pub mod test_data; mod v15; pub use v15::{ - BridgeConfig, LazyQueueMode, RttThresholds, SingleInterfaceConfig, StormguardConfig, + BridgeConfig, LazyQueueMode, QueueMode, RttThresholds, SingleInterfaceConfig, StormguardConfig, StormguardStrategy, TreeguardCircuitsConfig, TreeguardConfig, TreeguardCpuConfig, TreeguardCpuMode, TreeguardLinksConfig, TreeguardQooConfig, Tunables, }; static CONFIG: Lazy>>> = Lazy::new(|| ArcSwap::from_pointee(None)); +static TREEGUARD_CPU_MODE_MIGRATION_NOTICE: Lazy>> = + Lazy::new(|| std::sync::Mutex::new(None)); /// Load the configuration from `/etc/lqos.conf`. pub fn load_config() -> Result, LibreQoSConfigError> { @@ -39,6 +42,91 @@ pub fn load_config() -> Result, LibreQoSConfigError> { Ok(config) } +/// Clears the in-process cached configuration so the next `load_config()` reads from disk again. +/// +/// This function has side effects: it mutates the process-global config cache. +#[doc(hidden)] +pub fn clear_cached_config() { + CONFIG.store(None.into()); +} + +/// Returns the current TreeGuard CPU-mode migration notice, if an automatic upgrade rewrite +/// occurred during config load. +pub fn treeguard_cpu_mode_migration_notice() -> Option { + TREEGUARD_CPU_MODE_MIGRATION_NOTICE + .lock() + .ok() + .and_then(|notice| notice.clone()) +} + +fn treeguard_cpu_mode_migration_stamp_path(config_path: &str) -> String { + format!("{config_path}.treeguard_cpu_mode_migrated") +} + +fn maybe_migrate_treeguard_cpu_mode( + config_location: &str, + raw: String, +) -> Result { + let mut doc: DocumentMut = raw.parse().map_err(|e| LibreQoSConfigError::ParseError { + path: config_location.to_string(), + details: format!("Error parsing config: {e}"), + })?; + + let mode_item = doc + .get("treeguard") + .and_then(|section| section.as_table_like()) + .and_then(|treeguard| treeguard.get("cpu")) + .and_then(|cpu| cpu.as_table_like()) + .and_then(|cpu| cpu.get("mode")) + .and_then(|mode| mode.as_str()); + if mode_item != Some("traffic_rtt_only") { + return Ok(raw); + } + + let stamp_path = treeguard_cpu_mode_migration_stamp_path(config_location); + if Path::new(&stamp_path).exists() { + return Ok(raw); + } + + let Some(cpu_table) = doc + .get_mut("treeguard") + .and_then(|section| section.as_table_like_mut()) + .and_then(|treeguard| treeguard.get_mut("cpu")) + .and_then(|cpu| cpu.as_table_like_mut()) + else { + return Ok(raw); + }; + + cpu_table.insert("mode", value("cpu_aware")); + let migrated = doc.to_string(); + + let config_path = Path::new(config_location); + if config_path.exists() { + let backup_path = format!("{config_location}.treeguard_cpu_mode_backup"); + std::fs::copy(config_path, &backup_path).map_err(|e| LibreQoSConfigError::CannotWrite { + path: backup_path, + source: e, + })?; + } + + std::fs::write(config_path, &migrated).map_err(|e| LibreQoSConfigError::CannotWrite { + path: config_location.to_string(), + source: e, + })?; + std::fs::write(&stamp_path, b"cpu_aware\n").map_err(|e| LibreQoSConfigError::CannotWrite { + path: stamp_path, + source: e, + })?; + + let notice = "TreeGuard CPU mode was automatically migrated from traffic_rtt_only to cpu_aware during upgrade. CPU-aware mode is now the default and recommended virtualization policy.".to_string(); + info!("{notice}"); + if let Ok(mut slot) = TREEGUARD_CPU_MODE_MIGRATION_NOTICE.lock() { + *slot = Some(notice); + } + + Ok(migrated) +} + fn actually_load_from_disk() -> Result, LibreQoSConfigError> { let config_location = if let Ok(lqos_config) = std::env::var("LQOS_CONFIG") { info!("Overriding lqos.conf location from environment variable."); @@ -79,6 +167,8 @@ fn actually_load_from_disk() -> Result, LibreQoSConfigError> { } })?; + let raw = maybe_migrate_treeguard_cpu_mode(&config_location, raw)?; + let mut final_config = Config::load_from_string(&raw).map_err(|e| { error!("Unable to parse {config_location}"); LibreQoSConfigError::ParseError { diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index 31df4a26..43b37393 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -32,6 +32,8 @@ to_network = "eth1" [queues] default_sqm = "cake diffserv4" +queue_mode = "shape" +# Legacy compatibility field preserved for older binaries during upgrades. monitor_only = false uplink_bandwidth_mbps = 1000 downlink_bandwidth_mbps = 1000 @@ -48,7 +50,7 @@ dry_run = false tick_seconds = 1 [treeguard.cpu] -mode = "traffic_rtt_only" +mode = "cpu_aware" cpu_high_pct = 75 cpu_low_pct = 55 @@ -82,7 +84,7 @@ persist_sqm_overrides = true [treeguard.qoo] enabled = true -min_score = 80.0 +min_score = 70.0 [long_term_stats] gather_stats = true @@ -153,6 +155,8 @@ snmp_community = "public" airmax_model_ids = [ "" ] ltu_model_ids = [ "" ] active_status_ids = [ "" ] +recurring_excluded_service_names = [] +recurring_service_rates = [] [influxdb] enable_influxdb = false diff --git a/src/rust/lqos_config/src/etc/v15/mod.rs b/src/rust/lqos_config/src/etc/v15/mod.rs index 316d7f3d..68b9b3c5 100644 --- a/src/rust/lqos_config/src/etc/v15/mod.rs +++ b/src/rust/lqos_config/src/etc/v15/mod.rs @@ -23,7 +23,7 @@ mod wispgate; pub use bridge::*; pub use long_term_stats::LongTermStats; -pub use queues::LazyQueueMode; +pub use queues::{LazyQueueMode, QueueMode}; pub use stormguard::{StormguardConfig, StormguardStrategy}; pub use treeguard::{ TreeguardCircuitsConfig, TreeguardConfig, TreeguardCpuConfig, TreeguardCpuMode, diff --git a/src/rust/lqos_config/src/etc/v15/queues.rs b/src/rust/lqos_config/src/etc/v15/queues.rs index 9acaac8a..0eee8348 100644 --- a/src/rust/lqos_config/src/etc/v15/queues.rs +++ b/src/rust/lqos_config/src/etc/v15/queues.rs @@ -1,15 +1,34 @@ //! Queue Generation definitions (originally from ispConfig.py) use allocative::Allocative; -use serde::{Deserialize, Serialize}; +use serde::ser::SerializeStruct; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; -#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Allocative)] +/// Queue application mode. +#[derive(Clone, Copy, Serialize, Deserialize, Debug, PartialEq, Eq, Default, Allocative)] +#[serde(rename_all = "snake_case")] +pub enum QueueMode { + /// Apply LibreQoS queueing and shaping. + #[default] + Shape, + /// Observe only; do not apply managed TC objects. + Observe, +} + +impl QueueMode { + /// Returns `true` when the queue mode is observe-only. + pub const fn is_observe(self) -> bool { + matches!(self, Self::Observe) + } +} + +#[derive(Clone, Debug, PartialEq, Allocative)] pub struct QueueConfig { /// Which SQM to use by default pub default_sqm: String, - /// Should we monitor only, and not shape traffic? - pub monitor_only: bool, + /// Queue application mode. + pub queue_mode: QueueMode, /// Upstream bandwidth total - download pub uplink_bandwidth_mbps: u64, @@ -49,6 +68,56 @@ pub struct QueueConfig { pub fast_queues_fq_codel: Option, } +impl Serialize for QueueConfig { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut state = serializer.serialize_struct("QueueConfig", 15)?; + state.serialize_field("default_sqm", &self.default_sqm)?; + state.serialize_field("queue_mode", &self.queue_mode)?; + // Preserve the legacy field during rewrites so older binaries that still + // require it continue to parse upgraded configs successfully. + state.serialize_field("monitor_only", &self.queue_mode.is_observe())?; + state.serialize_field("uplink_bandwidth_mbps", &self.uplink_bandwidth_mbps)?; + state.serialize_field("downlink_bandwidth_mbps", &self.downlink_bandwidth_mbps)?; + state.serialize_field( + "generated_pn_download_mbps", + &self.generated_pn_download_mbps, + )?; + state.serialize_field("generated_pn_upload_mbps", &self.generated_pn_upload_mbps)?; + state.serialize_field("dry_run", &self.dry_run)?; + state.serialize_field("sudo", &self.sudo)?; + state.serialize_field("override_available_queues", &self.override_available_queues)?; + state.serialize_field("use_binpacking", &self.use_binpacking)?; + state.serialize_field("lazy_queues", &self.lazy_queues)?; + state.serialize_field("lazy_expire_seconds", &self.lazy_expire_seconds)?; + state.serialize_field("lazy_threshold_bytes", &self.lazy_threshold_bytes)?; + state.serialize_field("fast_queues_fq_codel", &self.fast_queues_fq_codel)?; + state.end() + } +} + +#[derive(Deserialize)] +#[serde(default)] +struct QueueConfigCompat { + default_sqm: String, + queue_mode: Option, + monitor_only: bool, + uplink_bandwidth_mbps: u64, + downlink_bandwidth_mbps: u64, + generated_pn_download_mbps: u64, + generated_pn_upload_mbps: u64, + dry_run: bool, + sudo: bool, + override_available_queues: Option, + use_binpacking: bool, + lazy_queues: Option, + lazy_expire_seconds: Option, + lazy_threshold_bytes: Option, + fast_queues_fq_codel: Option, +} + /// Lazy queue creation modes #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Default, Allocative)] pub enum LazyQueueMode { @@ -65,7 +134,7 @@ impl Default for QueueConfig { fn default() -> Self { Self { default_sqm: "cake diffserv4".to_string(), - monitor_only: false, + queue_mode: QueueMode::Shape, uplink_bandwidth_mbps: 1_000, downlink_bandwidth_mbps: 1_000, generated_pn_download_mbps: 1_000, @@ -81,3 +150,121 @@ impl Default for QueueConfig { } } } + +impl QueueConfig { + /// Sets the queue application mode. + pub fn set_queue_mode(&mut self, queue_mode: QueueMode) { + self.queue_mode = queue_mode; + } +} + +impl Default for QueueConfigCompat { + fn default() -> Self { + let defaults = QueueConfig::default(); + Self { + default_sqm: defaults.default_sqm.clone(), + queue_mode: None, + monitor_only: false, + uplink_bandwidth_mbps: defaults.uplink_bandwidth_mbps, + downlink_bandwidth_mbps: defaults.downlink_bandwidth_mbps, + generated_pn_download_mbps: defaults.generated_pn_download_mbps, + generated_pn_upload_mbps: defaults.generated_pn_upload_mbps, + dry_run: defaults.dry_run, + sudo: defaults.sudo, + override_available_queues: defaults.override_available_queues, + use_binpacking: defaults.use_binpacking, + lazy_queues: defaults.lazy_queues.clone(), + lazy_expire_seconds: defaults.lazy_expire_seconds, + lazy_threshold_bytes: defaults.lazy_threshold_bytes, + fast_queues_fq_codel: defaults.fast_queues_fq_codel, + } + } +} + +impl<'de> Deserialize<'de> for QueueConfig { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let compat = QueueConfigCompat::deserialize(deserializer)?; + let queue_mode = compat.queue_mode.unwrap_or({ + if compat.monitor_only { + QueueMode::Observe + } else { + QueueMode::Shape + } + }); + + let mut cfg = Self { + default_sqm: compat.default_sqm, + queue_mode, + uplink_bandwidth_mbps: compat.uplink_bandwidth_mbps, + downlink_bandwidth_mbps: compat.downlink_bandwidth_mbps, + generated_pn_download_mbps: compat.generated_pn_download_mbps, + generated_pn_upload_mbps: compat.generated_pn_upload_mbps, + dry_run: compat.dry_run, + sudo: compat.sudo, + override_available_queues: compat.override_available_queues, + use_binpacking: compat.use_binpacking, + lazy_queues: compat.lazy_queues, + lazy_expire_seconds: compat.lazy_expire_seconds, + lazy_threshold_bytes: compat.lazy_threshold_bytes, + fast_queues_fq_codel: compat.fast_queues_fq_codel, + }; + cfg.set_queue_mode(queue_mode); + Ok(cfg) + } +} + +#[cfg(test)] +mod tests { + use super::{QueueConfig, QueueMode}; + + #[test] + fn deserialize_legacy_monitor_only_maps_to_observe() { + let parsed: QueueConfig = + toml::from_str("default_sqm = \"cake diffserv4\"\nmonitor_only = true\n") + .expect("legacy monitor_only config should deserialize"); + assert_eq!(parsed.queue_mode, QueueMode::Observe); + } + + #[test] + fn deserialize_without_queue_mode_defaults_to_shape() { + let parsed: QueueConfig = toml::from_str("default_sqm = \"cake diffserv4\"\n") + .expect("queue config without queue_mode should deserialize"); + assert_eq!(parsed.queue_mode, QueueMode::Shape); + } + + #[test] + fn deserialize_prefers_queue_mode_over_legacy_monitor_only() { + let parsed: QueueConfig = toml::from_str( + "default_sqm = \"cake diffserv4\"\nqueue_mode = \"shape\"\nmonitor_only = true\n", + ) + .expect("mixed config should deserialize"); + assert_eq!(parsed.queue_mode, QueueMode::Shape); + } + + #[test] + fn serialize_queue_config_preserves_legacy_monitor_only_field() { + let mut config = QueueConfig::default(); + config.set_queue_mode(QueueMode::Observe); + let serialized = toml::to_string(&config).expect("queue config should serialize"); + assert!(serialized.contains("queue_mode = \"observe\"")); + assert!( + serialized.contains("monitor_only = true"), + "serialized config should preserve legacy monitor_only for compatibility" + ); + } + + #[test] + fn serialize_shape_queue_config_sets_legacy_monitor_only_false() { + let mut config = QueueConfig::default(); + config.set_queue_mode(QueueMode::Shape); + let serialized = toml::to_string(&config).expect("queue config should serialize"); + assert!(serialized.contains("queue_mode = \"shape\"")); + assert!( + serialized.contains("monitor_only = false"), + "serialized config should preserve legacy monitor_only=false for compatibility" + ); + } +} diff --git a/src/rust/lqos_config/src/etc/v15/sonar_integration.rs b/src/rust/lqos_config/src/etc/v15/sonar_integration.rs index 9dafe114..2651f564 100644 --- a/src/rust/lqos_config/src/etc/v15/sonar_integration.rs +++ b/src/rust/lqos_config/src/etc/v15/sonar_integration.rs @@ -1,6 +1,14 @@ use allocative::Allocative; use serde::{Deserialize, Serialize}; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Allocative, Default)] +pub struct SonarRecurringServiceRate { + pub enabled: bool, + pub service_name: String, + pub download_mbps: f32, + pub upload_mbps: f32, +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Allocative)] pub struct SonarIntegration { pub enable_sonar: bool, @@ -10,6 +18,10 @@ pub struct SonarIntegration { pub airmax_model_ids: Vec, pub ltu_model_ids: Vec, pub active_status_ids: Vec, + #[serde(default)] + pub recurring_service_rates: Vec, + #[serde(default)] + pub recurring_excluded_service_names: Vec, } impl Default for SonarIntegration { @@ -22,6 +34,8 @@ impl Default for SonarIntegration { airmax_model_ids: vec![], ltu_model_ids: vec![], active_status_ids: vec![], + recurring_service_rates: vec![], + recurring_excluded_service_names: vec![], } } } diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index 2b6a423b..65b82160 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -511,7 +511,7 @@ mod test { assert!(!cfg.treeguard.dry_run); assert_eq!( cfg.treeguard.cpu.mode, - crate::etc::v15::treeguard::TreeguardCpuMode::TrafficRttOnly + crate::etc::v15::treeguard::TreeguardCpuMode::CpuAware ); assert!(cfg.treeguard.links.enabled); assert!(cfg.treeguard.links.all_nodes); diff --git a/src/rust/lqos_config/src/etc/v15/treeguard.rs b/src/rust/lqos_config/src/etc/v15/treeguard.rs index dd4f1493..7b6bcbde 100644 --- a/src/rust/lqos_config/src/etc/v15/treeguard.rs +++ b/src/rust/lqos_config/src/etc/v15/treeguard.rs @@ -68,7 +68,7 @@ fn default_max_switches_per_hour() -> u32 { } fn default_min_score() -> f32 { - 80.0 + 70.0 } /// CPU modes supported by TreeGuard. @@ -76,9 +76,9 @@ fn default_min_score() -> f32 { #[serde(rename_all = "snake_case")] pub enum TreeguardCpuMode { /// Consider CPU thresholds when making decisions. + #[default] CpuAware, /// Use only traffic and RTT signals. - #[default] TrafficRttOnly, } diff --git a/src/rust/lqos_config/src/lib.rs b/src/rust/lqos_config/src/lib.rs index 2233cd59..646d78af 100644 --- a/src/rust/lqos_config/src/lib.rs +++ b/src/rust/lqos_config/src/lib.rs @@ -8,24 +8,38 @@ #![deny(clippy::unwrap_used)] #![warn(missing_docs)] pub mod authentication; +mod circuit_ethernet_metadata; mod cpu_topology; mod etc; mod network_json; +mod planner; mod program_control; mod qoo_profiles; mod shaped_devices; -pub use authentication::{UserRole, WebUser, WebUsers}; +pub use authentication::{AuthenticatedUser, UserRole, WebUser, WebUsers}; +pub use circuit_ethernet_metadata::{ + CIRCUIT_ETHERNET_METADATA_FILENAME, CircuitEthernetMetadata, CircuitEthernetMetadataFile, +}; pub use cpu_topology::{ CpuListParseError, ShapingCpuDetection, ShapingCpuSource, detect_shaping_cpus, }; pub use etc::{ - BridgeConfig, Config, LazyQueueMode, RttThresholds, SingleInterfaceConfig, StormguardConfig, - StormguardStrategy, TreeguardCircuitsConfig, TreeguardConfig, TreeguardCpuConfig, - TreeguardCpuMode, TreeguardLinksConfig, TreeguardQooConfig, Tunables, disable_xdp_bridge, - enable_long_term_stats, load_config, update_config, + BridgeConfig, Config, LazyQueueMode, QueueMode, RttThresholds, SingleInterfaceConfig, + StormguardConfig, StormguardStrategy, TreeguardCircuitsConfig, TreeguardConfig, + TreeguardCpuConfig, TreeguardCpuMode, TreeguardLinksConfig, TreeguardQooConfig, Tunables, + clear_cached_config, disable_xdp_bridge, enable_long_term_stats, load_config, + treeguard_cpu_mode_migration_notice, update_config, }; pub use network_json::{NetworkJson, NetworkJsonNode, NetworkJsonTransport}; +pub use planner::{ + CircuitIdentityAssignment, CircuitIdentityGroupInput, ClassIdentityPlannerConstraints, + ClassIdentityPlannerOutput, PlannerCircuitIdentityState, PlannerMinorReservations, + PlannerSiteIdentityState, SiteIdentityAssignment, SiteIdentityInput, TopLevelPlannerItem, + TopLevelPlannerMode, TopLevelPlannerOutput, TopLevelPlannerParams, + build_class_identity_reservations, plan_class_identities, + plan_class_identities_with_constraints, plan_top_level_assignments, +}; pub use program_control::load_libreqos; pub use qoo_profiles::{ DEFAULT_QOO_PROFILE_ID, QooProfileInfo, QooProfilesError, active_qoo_profile, diff --git a/src/rust/lqos_config/src/network_json.rs b/src/rust/lqos_config/src/network_json.rs index 079c00fd..b9a95746 100644 --- a/src/rust/lqos_config/src/network_json.rs +++ b/src/rust/lqos_config/src/network_json.rs @@ -87,6 +87,7 @@ impl NetworkJson { current_udp_packets: DownUpOrder::zeroed(), current_icmp_packets: DownUpOrder::zeroed(), current_tcp_retransmits: DownUpOrder::zeroed(), + current_tcp_retransmit_packets: DownUpOrder::zeroed(), current_drops: DownUpOrder::zeroed(), current_marks: DownUpOrder::zeroed(), parents: Vec::new(), @@ -169,6 +170,7 @@ impl NetworkJson { n.current_udp_packets.set_to_zero(); n.current_icmp_packets.set_to_zero(); n.current_tcp_retransmits.set_to_zero(); + n.current_tcp_retransmit_packets.set_to_zero(); n.rtt_buffer.clear(); n.current_drops.set_to_zero(); n.current_marks.set_to_zero(); @@ -214,11 +216,17 @@ impl NetworkJson { } /// Record TCP Retransmits in the tree. - pub fn add_retransmit_cycle(&mut self, targets: &[usize], tcp_retransmits: DownUpOrder) { + pub fn add_retransmit_cycle( + &mut self, + targets: &[usize], + tcp_retransmits: DownUpOrder, + tcp_packets: DownUpOrder, + ) { for idx in targets { // Safety first; use "get" to ensure that the node exists if let Some(node) = self.nodes.get_mut(*idx) { node.current_tcp_retransmits.checked_add(tcp_retransmits); + node.current_tcp_retransmit_packets.checked_add(tcp_packets); } else { warn!("No network tree entry for index {idx}"); } @@ -280,10 +288,12 @@ impl NetworkJson { .map(|rtt| rtt.as_millis() as f32); let retransmit_down = retransmit_percent( node.current_tcp_retransmits.down, - node.current_tcp_packets.down, + node.current_tcp_retransmit_packets.down, + ); + let retransmit_up = retransmit_percent( + node.current_tcp_retransmits.up, + node.current_tcp_retransmit_packets.up, ); - let retransmit_up = - retransmit_percent(node.current_tcp_retransmits.up, node.current_tcp_packets.up); let heatmap = node.heatmap.get_or_insert_with(TemporalHeatmap::new); heatmap.add_sample( @@ -299,11 +309,11 @@ impl NetworkJson { let loss_download = tcp_retransmit_loss_proxy( node.current_tcp_retransmits.down, - node.current_tcp_packets.down, + node.current_tcp_retransmit_packets.down, ); let loss_upload = tcp_retransmit_loss_proxy( node.current_tcp_retransmits.up, - node.current_tcp_packets.up, + node.current_tcp_retransmit_packets.up, ); let scores = if let Some(profile) = qoo_profile.as_ref() { compute_qoq_scores( @@ -380,6 +390,7 @@ fn recurse_node( current_udp_packets: DownUpOrder::zeroed(), current_icmp_packets: DownUpOrder::zeroed(), current_tcp_retransmits: DownUpOrder::zeroed(), + current_tcp_retransmit_packets: DownUpOrder::zeroed(), current_drops: DownUpOrder::zeroed(), current_marks: DownUpOrder::zeroed(), name: name.to_string(), @@ -467,6 +478,7 @@ mod test { current_udp_packets: DownUpOrder::zeroed(), current_icmp_packets: DownUpOrder::zeroed(), current_tcp_retransmits: DownUpOrder::zeroed(), + current_tcp_retransmit_packets: DownUpOrder::zeroed(), current_drops: DownUpOrder::zeroed(), current_marks: DownUpOrder::zeroed(), parents: Vec::new(), @@ -570,8 +582,10 @@ mod test { assert!(tiny.max_throughput.1 > 0.0); let transport = tiny.clone_to_transit(); + assert!(!transport.runtime_virtualized); let encoded = serde_json::to_value(&transport).expect("transport must serialize"); assert_eq!(encoded["max_throughput"], serde_json::json!([1.5, 0.5])); + assert_eq!(encoded["runtime_virtualized"], serde_json::json!(false)); } #[test] @@ -606,14 +620,8 @@ mod test { assert_eq!(site.id.as_deref(), Some("uisp:site:123")); assert_eq!(ap.id.as_deref(), Some("uisp:device:456")); - assert_eq!( - site.clone_to_transit().id.as_deref(), - Some("uisp:site:123") - ); - assert_eq!( - ap.clone_to_transit().id.as_deref(), - Some("uisp:device:456") - ); + assert_eq!(site.clone_to_transit().id.as_deref(), Some("uisp:site:123")); + assert_eq!(ap.clone_to_transit().id.as_deref(), Some("uisp:device:456")); } #[test] @@ -647,14 +655,20 @@ mod test { .find(|n| n.name == "Bad Tower") .expect("Bad Tower must be present"); - assert!((good.latitude.unwrap() - 45.123).abs() < 0.001); - assert!((good.longitude.unwrap() + 111.75).abs() < 0.001); + let good_latitude = good.latitude.expect("Tower A latitude must parse"); + let good_longitude = good.longitude.expect("Tower A longitude must parse"); + assert!((good_latitude - 45.123).abs() < 0.001); + assert!((good_longitude + 111.75).abs() < 0.001); assert_eq!(bad.latitude, None); assert_eq!(bad.longitude, None); let encoded = serde_json::to_value(good.clone_to_transit()).expect("transport serializes"); - let encoded_lat = encoded["latitude"].as_f64().expect("latitude encodes as number"); - let encoded_lon = encoded["longitude"].as_f64().expect("longitude encodes as number"); + let encoded_lat = encoded["latitude"] + .as_f64() + .expect("latitude encodes as number"); + let encoded_lon = encoded["longitude"] + .as_f64() + .expect("longitude encodes as number"); assert!((encoded_lat - 45.123).abs() < 0.001); assert!((encoded_lon + 111.75).abs() < 0.001); } diff --git a/src/rust/lqos_config/src/network_json/network_json_node.rs b/src/rust/lqos_config/src/network_json/network_json_node.rs index 1fc075f2..59a51dfd 100644 --- a/src/rust/lqos_config/src/network_json/network_json_node.rs +++ b/src/rust/lqos_config/src/network_json/network_json_node.rs @@ -41,6 +41,9 @@ pub struct NetworkJsonNode { /// Current TCP Retransmits pub current_tcp_retransmits: DownUpOrder, // In retries + /// TCP packets paired to the retransmit counters for the current cycle. + pub current_tcp_retransmit_packets: DownUpOrder, + /// Current Cake Marks pub current_marks: DownUpOrder, @@ -107,6 +110,7 @@ impl NetworkJsonNode { name: self.name.clone(), id: self.id.clone(), is_virtual: self.virtual_node, + runtime_virtualized: false, max_throughput: self.max_throughput, configured_max_throughput: self.max_throughput, effective_max_throughput: None, @@ -134,6 +138,10 @@ impl NetworkJsonNode { self.current_tcp_retransmits.get_down(), self.current_tcp_retransmits.get_up(), ), + current_tcp_retransmit_packets: ( + self.current_tcp_retransmit_packets.get_down(), + self.current_tcp_retransmit_packets.get_up(), + ), current_marks: (self.current_marks.get_down(), self.current_marks.get_up()), current_drops: (self.current_drops.get_down(), self.current_drops.get_up()), rtts, @@ -143,6 +151,9 @@ impl NetworkJsonNode { node_type: self.node_type.clone(), latitude: self.latitude, longitude: self.longitude, + subtree_site_count: 0, + subtree_circuit_count: 0, + subtree_device_count: 0, } } } diff --git a/src/rust/lqos_config/src/network_json/network_json_transport.rs b/src/rust/lqos_config/src/network_json/network_json_transport.rs index 16c56738..ac42a217 100644 --- a/src/rust/lqos_config/src/network_json/network_json_transport.rs +++ b/src/rust/lqos_config/src/network_json/network_json_transport.rs @@ -14,6 +14,9 @@ pub struct NetworkJsonTransport { /// True if this node is a "virtual" (logical-only) node. #[serde(rename = "virtual", default)] pub is_virtual: bool, + /// True if TreeGuard has runtime-virtualized this node in the active Bakery tree. + #[serde(default)] + pub runtime_virtualized: bool, /// Max throughput for node (not clamped) pub max_throughput: (f64, f64), /// Configured max throughput from `network.json`. @@ -34,6 +37,9 @@ pub struct NetworkJsonTransport { pub current_icmp_packets: (u64, u64), /// Current count of TCP retransmits pub current_retransmits: (u64, u64), + /// TCP packets paired with current retransmit counts. + #[serde(default)] + pub current_tcp_retransmit_packets: (u64, u64), /// Cake marks pub current_marks: (u64, u64), /// Cake drops @@ -58,4 +64,16 @@ pub struct NetworkJsonTransport { /// Optional node longitude from network.json metadata. #[serde(default)] pub longitude: Option, + /// Total number of descendant site-tree nodes below this node. + /// + /// This excludes the node itself. For the synthetic root node, this is the + /// total number of nodes in the loaded site tree. + #[serde(default)] + pub subtree_site_count: u32, + /// Total number of unique circuits attached to this node or any descendant node. + #[serde(default)] + pub subtree_circuit_count: u32, + /// Total number of shaped devices attached to this node or any descendant node. + #[serde(default)] + pub subtree_device_count: u32, } diff --git a/src/rust/lqos_config/src/planner.rs b/src/rust/lqos_config/src/planner.rs new file mode 100644 index 00000000..9b2c87ac --- /dev/null +++ b/src/rust/lqos_config/src/planner.rs @@ -0,0 +1,877 @@ +//! Shared planning helpers for physical queue placement. +//! +//! This module provides a reusable top-level CPU/bin planner so both the Python shaper build path +//! and future Bakery/TreeGuard runtime replans can use the same assignment logic. + +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// A top-level item to be assigned to a shaping queue/bin. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TopLevelPlannerItem { + /// Stable logical identifier for the item being planned. + pub id: String, + /// Relative weight used when balancing items across bins. + pub weight: f64, +} + +/// Supported planner strategies. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub enum TopLevelPlannerMode { + /// Deterministic round-robin assignment by item id. + RoundRobin, + /// Deterministic greedy balancing with no hysteresis or cooldown. + Greedy, + /// Greedy balancing that prefers to keep prior assignments unless a move is worthwhile. + #[default] + StableGreedy, +} + +/// Tunables for top-level queue planning. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TopLevelPlannerParams { + /// Planning mode. + pub mode: TopLevelPlannerMode, + /// Fraction of total weight improvement required before a move is considered worthwhile. + pub hysteresis_threshold: f64, + /// Minimum seconds between moves for a previously assigned item. + pub cooldown_seconds: f64, + /// Maximum number of assignment changes allowed in a single planning run. + pub move_budget_per_run: usize, +} + +impl Default for TopLevelPlannerParams { + fn default() -> Self { + Self { + mode: TopLevelPlannerMode::StableGreedy, + hysteresis_threshold: 0.03, + cooldown_seconds: 3600.0, + move_budget_per_run: 1, + } + } +} + +/// Result of a top-level assignment pass. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TopLevelPlannerOutput { + /// Final per-item bin assignment. + pub assignment: BTreeMap, + /// Item ids whose assigned bin changed relative to the previous assignment map. + pub changed: Vec, + /// Whether the stable planner path, rather than a simple fallback, was used. + pub planner_used: bool, +} + +/// Input describing a site that needs class identity assignment. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct SiteIdentityInput { + /// Stable path key for the site within the logical hierarchy. + pub site_key: String, + /// Stable path key for the parent site, or empty for top-level nodes. + pub parent_path: String, + /// Target shaping queue number, 1-based. + pub queue: u32, + /// Whether this site has child sites in the physical tree. + pub has_children: bool, +} + +/// Assigned class identity for a site. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct SiteIdentityAssignment { + /// Stable path key for the site. + pub site_key: String, + /// Assigned queue number, 1-based. + pub queue: u32, + /// Assigned class minor. + pub class_minor: u16, + /// Assigned downlink class major. + pub class_major: u16, + /// Assigned uplink class major. + pub up_class_major: u16, + /// Parent path used when deciding minor reuse. + pub parent_path: String, +} + +/// Input describing a circuit group under a single parent site. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitIdentityGroupInput { + /// Parent node name for this group of circuits. + pub parent_node: String, + /// Queue number, 1-based. + pub queue: u32, + /// Circuit ids in deterministic order for this parent. + pub circuit_ids: Vec, +} + +/// Assigned class identity for a circuit. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitIdentityAssignment { + /// Stable circuit id. + pub circuit_id: String, + /// Parent node used when deciding minor reuse. + pub parent_node: String, + /// Assigned queue number, 1-based. + pub queue: u32, + /// Assigned class minor. + pub class_minor: u16, + /// Assigned downlink class major. + pub class_major: u16, + /// Assigned uplink class major. + pub up_class_major: u16, +} + +/// Persisted planner identity for a site. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct PlannerSiteIdentityState { + /// Assigned class minor. + pub class_minor: u16, + /// Assigned queue number, 1-based. + pub queue: u32, + /// Stable parent path for reuse checks. + pub parent_path: String, + /// Downlink class major. + pub class_major: u16, + /// Uplink class major. + pub up_class_major: u16, +} + +/// Persisted planner identity for a circuit. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct PlannerCircuitIdentityState { + /// Assigned class minor. + pub class_minor: u16, + /// Assigned queue number, 1-based. + pub queue: u32, + /// Stable parent node for reuse checks. + pub parent_node: String, + /// Downlink class major. + pub class_major: u16, + /// Uplink class major. + pub up_class_major: u16, +} + +/// Output of site/circuit identity assignment. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ClassIdentityPlannerOutput { + /// Assigned site identities keyed in traversal order. + pub sites: Vec, + /// Assigned circuit identities keyed in traversal order. + pub circuits: Vec, + /// Planner-state site map suitable for persistence. + pub site_state: BTreeMap, + /// Planner-state circuit map suitable for persistence. + pub circuit_state: BTreeMap, + /// Highest allocated minor counter per queue after assignment. + pub last_used_minor_by_queue: BTreeMap, +} + +/// Reserved minor numbers keyed by shaping queue. +pub type PlannerMinorReservations = BTreeMap>; + +/// Structural constraints and numbering offsets for class identity planning. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ClassIdentityPlannerConstraints { + /// Reserved site minors keyed by queue. + pub reserved_site_minors: PlannerMinorReservations, + /// Reserved circuit minors keyed by queue. + pub reserved_circuit_minors: PlannerMinorReservations, + /// Starting minor for planned site classes. + pub site_minor_start: u32, + /// Starting minor for planned circuit classes. + pub circuit_minor_start: u32, + /// Uplink major offset for on-a-stick layouts. + pub stick_offset: u16, + /// Padding to leave after each circuit group allocation. + pub circuit_padding: u32, +} + +fn sanitize_weight(weight: f64) -> f64 { + if !weight.is_finite() || weight <= 0.0 { + 1.0 + } else { + weight + } +} + +fn next_free_minor(start_minor: u32, reserved: &BTreeSet) -> u32 { + let mut candidate = start_minor.max(3); + while reserved.contains(&candidate) { + candidate += 1; + } + candidate +} + +/// Builds reservation maps for identities that are not part of the current planning scope. +pub fn build_class_identity_reservations( + sites: &[SiteIdentityInput], + circuit_groups: &[CircuitIdentityGroupInput], + previous_sites: &BTreeMap, + previous_circuits: &BTreeMap, +) -> (PlannerMinorReservations, PlannerMinorReservations) { + let planned_site_keys: BTreeSet<&str> = + sites.iter().map(|site| site.site_key.as_str()).collect(); + let planned_circuit_ids: BTreeSet<&str> = circuit_groups + .iter() + .flat_map(|group| group.circuit_ids.iter().map(|id| id.as_str())) + .collect(); + + let mut reserved_site_minors: PlannerMinorReservations = BTreeMap::new(); + for (site_key, stored) in previous_sites { + if planned_site_keys.contains(site_key.as_str()) { + continue; + } + reserved_site_minors + .entry(stored.queue) + .or_default() + .insert(stored.class_minor as u32); + } + + let mut reserved_circuit_minors: PlannerMinorReservations = BTreeMap::new(); + for (circuit_id, stored) in previous_circuits { + if planned_circuit_ids.contains(circuit_id.as_str()) { + continue; + } + reserved_circuit_minors + .entry(stored.queue) + .or_default() + .insert(stored.class_minor as u32); + } + + (reserved_site_minors, reserved_circuit_minors) +} + +fn round_robin_assign(items: &[TopLevelPlannerItem], bins: &[String]) -> BTreeMap { + let mut ids: Vec<&str> = items.iter().map(|item| item.id.as_str()).collect(); + ids.sort_unstable(); + + let mut assignment = BTreeMap::new(); + for (idx, id) in ids.into_iter().enumerate() { + if let Some(bin) = bins.get(idx % bins.len()) { + assignment.insert(id.to_string(), bin.clone()); + } + } + assignment +} + +fn greedy_assign(items: &[TopLevelPlannerItem], bins: &[String]) -> BTreeMap { + let mut pairs: Vec<(&str, f64)> = items + .iter() + .map(|item| (item.id.as_str(), sanitize_weight(item.weight))) + .collect(); + pairs.sort_by(|a, b| { + b.1.partial_cmp(&a.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.0.cmp(b.0)) + }); + + let mut loads: BTreeMap = bins.iter().cloned().map(|bin| (bin, 0.0)).collect(); + let mut assignment = BTreeMap::new(); + for (id, weight) in pairs { + let target = loads + .iter() + .min_by(|a, b| { + a.1.partial_cmp(b.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.0.cmp(b.0)) + }) + .map(|(bin, _)| bin.clone()) + .unwrap_or_else(|| bins[0].clone()); + assignment.insert(id.to_string(), target.clone()); + if let Some(load) = loads.get_mut(&target) { + *load += weight; + } + } + assignment +} + +fn load_by_bin( + assignment: &BTreeMap, + item_weights: &BTreeMap, + bins: &[String], +) -> BTreeMap { + let mut loads: BTreeMap = bins.iter().cloned().map(|bin| (bin, 0.0)).collect(); + for (id, bin) in assignment { + if let Some(weight) = item_weights.get(id) + && let Some(load) = loads.get_mut(bin) + { + *load += *weight; + } + } + loads +} + +/// Plans top-level queue assignments for the given logical items and bins. +/// +/// This function is pure: it computes a deterministic assignment and does not perform any I/O. +pub fn plan_top_level_assignments( + items: &[TopLevelPlannerItem], + bins: &[String], + prev_assign: &BTreeMap, + last_change_ts: &BTreeMap, + now_ts: f64, + params: &TopLevelPlannerParams, +) -> TopLevelPlannerOutput { + if bins.is_empty() || items.is_empty() { + return TopLevelPlannerOutput { + assignment: BTreeMap::new(), + changed: Vec::new(), + planner_used: !matches!(params.mode, TopLevelPlannerMode::RoundRobin), + }; + } + + let item_ids: Vec = items.iter().map(|item| item.id.clone()).collect(); + let item_weights: BTreeMap = items + .iter() + .map(|item| (item.id.clone(), sanitize_weight(item.weight))) + .collect(); + let valid_bins: std::collections::BTreeSet<&String> = bins.iter().collect(); + let prev_valid: BTreeMap = prev_assign + .iter() + .filter(|(id, bin)| item_weights.contains_key(*id) && valid_bins.contains(*bin)) + .map(|(id, bin)| (id.clone(), bin.clone())) + .collect(); + + let rr_assignment = round_robin_assign(items, bins); + let greedy_assignment = greedy_assign(items, bins); + + let (mut assignment, planner_used) = match params.mode { + TopLevelPlannerMode::RoundRobin => (rr_assignment.clone(), false), + TopLevelPlannerMode::Greedy => (greedy_assignment.clone(), false), + TopLevelPlannerMode::StableGreedy => { + let mut assignment = rr_assignment.clone(); + for id in &item_ids { + let chosen = prev_valid + .get(id) + .cloned() + .or_else(|| greedy_assignment.get(id).cloned()) + .or_else(|| rr_assignment.get(id).cloned()) + .unwrap_or_else(|| bins[0].clone()); + assignment.insert(id.clone(), chosen); + } + + let mut loads = load_by_bin(&assignment, &item_weights, bins); + let total_weight: f64 = item_weights.values().sum(); + let min_improvement = total_weight * params.hysteresis_threshold.max(0.0); + let mut weighted_ids = item_ids.clone(); + weighted_ids.sort_by(|a, b| { + item_weights[b] + .partial_cmp(&item_weights[a]) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.cmp(b)) + }); + + let mut moves_used = 0usize; + for id in weighted_ids { + let Some(current_bin) = assignment.get(&id).cloned() else { + continue; + }; + let Some(target_bin) = greedy_assignment.get(&id).cloned() else { + continue; + }; + if current_bin == target_bin { + continue; + } + if moves_used >= params.move_budget_per_run { + continue; + } + let last_changed = last_change_ts.get(&id).copied().unwrap_or(0.0); + if now_ts - last_changed < params.cooldown_seconds { + continue; + } + let current_load = loads.get(¤t_bin).copied().unwrap_or(0.0); + let target_load = loads.get(&target_bin).copied().unwrap_or(0.0); + if current_load - target_load <= min_improvement { + continue; + } + + let weight = item_weights.get(&id).copied().unwrap_or(1.0); + if let Some(load) = loads.get_mut(¤t_bin) { + *load -= weight; + } + if let Some(load) = loads.get_mut(&target_bin) { + *load += weight; + } + assignment.insert(id, target_bin); + moves_used += 1; + } + + (assignment, true) + } + }; + + let used_bins: std::collections::BTreeSet<&String> = assignment.values().collect(); + if bins.len() > 1 && assignment.len() > 1 && used_bins.len() <= 1 { + assignment = greedy_assignment.clone(); + } + + let mut changed: Vec = assignment + .iter() + .filter_map(|(id, bin)| { + prev_valid + .get(id) + .and_then(|prev| (prev != bin).then_some(id.clone())) + }) + .collect(); + changed.sort_unstable(); + + TopLevelPlannerOutput { + assignment, + changed, + planner_used, + } +} + +/// Plans site and circuit class identities for a physical queue tree. +/// +/// This function is pure: it computes deterministic minor/class-major assignments from the +/// provided planner state and traversal order without performing any I/O. +pub fn plan_class_identities_with_constraints( + sites: &[SiteIdentityInput], + circuit_groups: &[CircuitIdentityGroupInput], + previous_sites: &BTreeMap, + previous_circuits: &BTreeMap, + constraints: &ClassIdentityPlannerConstraints, +) -> ClassIdentityPlannerOutput { + let mut reserved_site_minors = constraints.reserved_site_minors.clone(); + let mut next_site_minor_by_queue: BTreeMap = BTreeMap::new(); + let mut site_assignments = Vec::with_capacity(sites.len()); + let mut site_state = BTreeMap::new(); + for (queue, reserved) in &reserved_site_minors { + let next_minor = next_site_minor_by_queue + .entry(*queue) + .or_insert(constraints.site_minor_start.max(3)); + if let Some(last_reserved) = reserved.iter().next_back().copied() { + *next_minor = (*next_minor).max(last_reserved.saturating_add(1)); + } + } + + for site in sites { + let reserved = reserved_site_minors.entry(site.queue).or_default(); + let next_minor = next_site_minor_by_queue + .entry(site.queue) + .or_insert(constraints.site_minor_start.max(3)); + let reuse_minor = previous_sites.get(&site.site_key).and_then(|stored| { + (stored.queue == site.queue + && stored.parent_path == site.parent_path + && !reserved.contains(&(stored.class_minor as u32))) + .then_some(stored.class_minor as u32) + }); + + let class_minor_u32 = reuse_minor.unwrap_or_else(|| next_free_minor(*next_minor, reserved)); + let class_minor = u16::try_from(class_minor_u32).unwrap_or(u16::MAX); + let class_major = u16::try_from(site.queue).unwrap_or(u16::MAX); + let up_class_major = class_major.saturating_add(constraints.stick_offset); + reserved.insert(class_minor_u32); + *next_minor = (*next_minor).max(class_minor_u32).saturating_add(1); + if site.has_children { + *next_minor = (*next_minor).saturating_add(1); + } + + let assigned = SiteIdentityAssignment { + site_key: site.site_key.clone(), + queue: site.queue, + class_minor, + class_major, + up_class_major, + parent_path: site.parent_path.clone(), + }; + site_state.insert( + site.site_key.clone(), + PlannerSiteIdentityState { + class_minor, + queue: site.queue, + parent_path: site.parent_path.clone(), + class_major, + up_class_major, + }, + ); + site_assignments.push(assigned); + } + + let mut reserved_circuit_minors = reserved_site_minors.clone(); + for (queue, extras) in &constraints.reserved_circuit_minors { + reserved_circuit_minors + .entry(*queue) + .or_default() + .extend(extras.iter().copied()); + } + let mut next_circuit_minor_by_queue = BTreeMap::new(); + for (queue, reserved) in &reserved_circuit_minors { + let start = next_site_minor_by_queue + .get(queue) + .copied() + .unwrap_or_else(|| next_free_minor(constraints.circuit_minor_start.max(3), reserved)); + next_circuit_minor_by_queue.insert( + *queue, + next_free_minor(start.max(constraints.circuit_minor_start.max(3)), reserved), + ); + } + + let mut circuit_assignments = Vec::new(); + let mut circuit_state = BTreeMap::new(); + for group in circuit_groups { + let reserved = reserved_circuit_minors.entry(group.queue).or_default(); + let next_minor = next_circuit_minor_by_queue + .entry(group.queue) + .or_insert_with(|| next_free_minor(constraints.circuit_minor_start.max(3), reserved)); + for circuit_id in &group.circuit_ids { + let reuse_minor = previous_circuits.get(circuit_id).and_then(|stored| { + (stored.queue == group.queue + && stored.parent_node == group.parent_node + && !reserved.contains(&(stored.class_minor as u32))) + .then_some(stored.class_minor as u32) + }); + let class_minor_u32 = + reuse_minor.unwrap_or_else(|| next_free_minor(*next_minor, reserved)); + let class_minor = u16::try_from(class_minor_u32).unwrap_or(u16::MAX); + let class_major = u16::try_from(group.queue).unwrap_or(u16::MAX); + let up_class_major = class_major.saturating_add(constraints.stick_offset); + reserved.insert(class_minor_u32); + *next_minor = (*next_minor).max(class_minor_u32).saturating_add(1); + + circuit_assignments.push(CircuitIdentityAssignment { + circuit_id: circuit_id.clone(), + parent_node: group.parent_node.clone(), + queue: group.queue, + class_minor, + class_major, + up_class_major, + }); + circuit_state.insert( + circuit_id.clone(), + PlannerCircuitIdentityState { + class_minor, + queue: group.queue, + parent_node: group.parent_node.clone(), + class_major, + up_class_major, + }, + ); + } + *next_minor = (*next_minor).saturating_add(constraints.circuit_padding); + } + + let mut last_used_minor_by_queue = BTreeMap::new(); + for queue in reserved_circuit_minors + .keys() + .chain(next_site_minor_by_queue.keys()) + .cloned() + .collect::>() + { + let site_next = next_site_minor_by_queue.get(&queue).copied().unwrap_or(3); + let circuit_next = next_circuit_minor_by_queue + .get(&queue) + .copied() + .unwrap_or(3); + last_used_minor_by_queue.insert(queue, site_next.max(circuit_next)); + } + + ClassIdentityPlannerOutput { + sites: site_assignments, + circuits: circuit_assignments, + site_state, + circuit_state, + last_used_minor_by_queue, + } +} + +/// Plans site and circuit class identities using the default structural allocator policy. +pub fn plan_class_identities( + sites: &[SiteIdentityInput], + circuit_groups: &[CircuitIdentityGroupInput], + previous_sites: &BTreeMap, + previous_circuits: &BTreeMap, + stick_offset: u16, + circuit_padding: u32, +) -> ClassIdentityPlannerOutput { + let (reserved_site_minors, reserved_circuit_minors) = + build_class_identity_reservations(sites, circuit_groups, previous_sites, previous_circuits); + let constraints = ClassIdentityPlannerConstraints { + reserved_site_minors, + reserved_circuit_minors, + site_minor_start: 3, + circuit_minor_start: 3, + stick_offset, + circuit_padding, + }; + plan_class_identities_with_constraints( + sites, + circuit_groups, + previous_sites, + previous_circuits, + &constraints, + ) +} + +#[cfg(test)] +mod tests { + use super::{ + CircuitIdentityGroupInput, ClassIdentityPlannerConstraints, PlannerCircuitIdentityState, + PlannerSiteIdentityState, SiteIdentityInput, TopLevelPlannerItem, TopLevelPlannerMode, + TopLevelPlannerParams, build_class_identity_reservations, plan_class_identities, + plan_class_identities_with_constraints, plan_top_level_assignments, + }; + use std::collections::BTreeMap; + + fn bins() -> Vec { + vec!["CpueQueue0".to_string(), "CpueQueue1".to_string()] + } + + #[test] + fn round_robin_is_deterministic() { + let items = vec![ + TopLevelPlannerItem { + id: "b".to_string(), + weight: 1.0, + }, + TopLevelPlannerItem { + id: "a".to_string(), + weight: 1.0, + }, + ]; + let result = plan_top_level_assignments( + &items, + &bins(), + &BTreeMap::new(), + &BTreeMap::new(), + 0.0, + &TopLevelPlannerParams { + mode: TopLevelPlannerMode::RoundRobin, + ..Default::default() + }, + ); + assert_eq!( + result.assignment.get("a").map(String::as_str), + Some("CpueQueue0") + ); + assert_eq!( + result.assignment.get("b").map(String::as_str), + Some("CpueQueue1") + ); + assert!(!result.planner_used); + } + + #[test] + fn stable_greedy_breaks_degenerate_single_bin_assignments() { + let items = vec![ + TopLevelPlannerItem { + id: "heavy".to_string(), + weight: 100.0, + }, + TopLevelPlannerItem { + id: "light".to_string(), + weight: 1.0, + }, + ]; + let prev_assign = BTreeMap::from([ + ("heavy".to_string(), "CpueQueue0".to_string()), + ("light".to_string(), "CpueQueue0".to_string()), + ]); + let last_change_ts = + BTreeMap::from([("heavy".to_string(), 990.0), ("light".to_string(), 990.0)]); + let result = plan_top_level_assignments( + &items, + &bins(), + &prev_assign, + &last_change_ts, + 1000.0, + &TopLevelPlannerParams { + mode: TopLevelPlannerMode::StableGreedy, + cooldown_seconds: 3600.0, + move_budget_per_run: 1, + hysteresis_threshold: 0.0, + }, + ); + assert_eq!( + result.assignment.get("light").map(String::as_str), + Some("CpueQueue1") + ); + assert_eq!(result.changed, vec!["light".to_string()]); + } + + #[test] + fn stable_greedy_can_move_when_cooldown_expired() { + let items = vec![ + TopLevelPlannerItem { + id: "heavy".to_string(), + weight: 100.0, + }, + TopLevelPlannerItem { + id: "light".to_string(), + weight: 1.0, + }, + ]; + let prev_assign = BTreeMap::from([ + ("heavy".to_string(), "CpueQueue0".to_string()), + ("light".to_string(), "CpueQueue0".to_string()), + ]); + let result = plan_top_level_assignments( + &items, + &bins(), + &prev_assign, + &BTreeMap::new(), + 10_000.0, + &TopLevelPlannerParams { + mode: TopLevelPlannerMode::StableGreedy, + cooldown_seconds: 0.0, + move_budget_per_run: 1, + hysteresis_threshold: 0.0, + }, + ); + assert_eq!( + result.assignment.get("light").map(String::as_str), + Some("CpueQueue1") + ); + assert_eq!(result.changed, vec!["light".to_string()]); + assert!(result.planner_used); + } + + #[test] + fn class_identity_planner_reuses_site_and_circuit_minors() { + let site_inputs = vec![ + SiteIdentityInput { + site_key: "CpueQueue0/site-a".to_string(), + parent_path: "CpueQueue0".to_string(), + queue: 1, + has_children: true, + }, + SiteIdentityInput { + site_key: "CpueQueue0/site-a/child".to_string(), + parent_path: "CpueQueue0/site-a".to_string(), + queue: 1, + has_children: false, + }, + ]; + let circuit_groups = vec![CircuitIdentityGroupInput { + parent_node: "child".to_string(), + queue: 1, + circuit_ids: vec!["c1".to_string(), "c2".to_string()], + }]; + let prev_sites = BTreeMap::from([ + ( + "CpueQueue0/site-a".to_string(), + PlannerSiteIdentityState { + class_minor: 0x20, + queue: 1, + parent_path: "CpueQueue0".to_string(), + class_major: 1, + up_class_major: 0x41, + }, + ), + ( + "CpueQueue0/site-a/child".to_string(), + PlannerSiteIdentityState { + class_minor: 0x22, + queue: 1, + parent_path: "CpueQueue0/site-a".to_string(), + class_major: 1, + up_class_major: 0x41, + }, + ), + ]); + let prev_circuits = BTreeMap::from([( + "c1".to_string(), + PlannerCircuitIdentityState { + class_minor: 0x30, + queue: 1, + parent_node: "child".to_string(), + class_major: 1, + up_class_major: 0x41, + }, + )]); + + let result = plan_class_identities( + &site_inputs, + &circuit_groups, + &prev_sites, + &prev_circuits, + 0x40, + 8, + ); + + assert_eq!(result.sites[0].class_minor, 0x20); + assert_eq!(result.sites[1].class_minor, 0x22); + assert_eq!(result.circuits[0].class_minor, 0x30); + assert_eq!(result.circuits[1].queue, 1); + assert_eq!(result.circuits[1].class_major, 1); + } + + #[test] + fn class_identity_planner_reserves_untouched_previous_minors() { + let site_inputs = vec![SiteIdentityInput { + site_key: "CpueQueue6/site-moved".to_string(), + parent_path: "CpueQueue6/site-parent".to_string(), + queue: 7, + has_children: false, + }]; + let prev_sites = BTreeMap::from([ + ( + "CpueQueue6/site-untouched".to_string(), + PlannerSiteIdentityState { + class_minor: 0x29, + queue: 7, + parent_path: "".to_string(), + class_major: 7, + up_class_major: 7, + }, + ), + ( + "CpueQueue0/site-moved".to_string(), + PlannerSiteIdentityState { + class_minor: 0x13, + queue: 1, + parent_path: "CpueQueue0/site-parent".to_string(), + class_major: 1, + up_class_major: 1, + }, + ), + ]); + + let result = plan_class_identities(&site_inputs, &[], &prev_sites, &BTreeMap::new(), 0, 0); + + assert_eq!(result.sites.len(), 1); + assert_eq!(result.sites[0].queue, 7); + assert_ne!(result.sites[0].class_minor, 0x29); + } + + #[test] + fn class_identity_planner_runtime_constraints_prefer_runtime_band() { + let site_inputs = vec![SiteIdentityInput { + site_key: "CpueQueue6/site-runtime".to_string(), + parent_path: "CpueQueue6/site-parent".to_string(), + queue: 7, + has_children: false, + }]; + let prev_sites = BTreeMap::from([( + "CpueQueue6/site-untouched".to_string(), + PlannerSiteIdentityState { + class_minor: 0x29, + queue: 7, + parent_path: "".to_string(), + class_major: 7, + up_class_major: 7, + }, + )]); + let (reserved_site_minors, reserved_circuit_minors) = + build_class_identity_reservations(&site_inputs, &[], &prev_sites, &BTreeMap::new()); + let constraints = ClassIdentityPlannerConstraints { + reserved_site_minors, + reserved_circuit_minors, + site_minor_start: 0x1000, + circuit_minor_start: 0x1000, + stick_offset: 0, + circuit_padding: 0, + }; + + let result = plan_class_identities_with_constraints( + &site_inputs, + &[], + &BTreeMap::new(), + &BTreeMap::new(), + &constraints, + ); + + assert_eq!(result.sites.len(), 1); + assert!(result.sites[0].class_minor >= 0x1000); + } +} diff --git a/src/rust/lqos_overrides/src/main.rs b/src/rust/lqos_overrides/src/main.rs index b0c18309..d0cd3e2c 100644 --- a/src/rust/lqos_overrides/src/main.rs +++ b/src/rust/lqos_overrides/src/main.rs @@ -303,12 +303,14 @@ struct AddDeviceSpeedArgs { #[derive(Args, Debug, Default)] struct AddSiteSpeedArgs { + #[arg(long)] + node_id: Option, #[arg(long)] site_name: String, #[arg(long)] - download_bandwidth_mbps: Option, + download_bandwidth_mbps: Option, #[arg(long)] - upload_bandwidth_mbps: Option, + upload_bandwidth_mbps: Option, } fn main() -> Result<()> { @@ -419,6 +421,7 @@ fn main() -> Result<()> { Commands::NetworkAdjustments { command: cmd } => match cmd { NetworkAdjustmentsCommand::AddSiteSpeed(args) => { let adj = NetworkAdjustment::AdjustSiteSpeed { + node_id: args.node_id, site_name: args.site_name, download_bandwidth_mbps: args.download_bandwidth_mbps, upload_bandwidth_mbps: args.upload_bandwidth_mbps, diff --git a/src/rust/lqos_overrides/src/overrides_file.rs b/src/rust/lqos_overrides/src/overrides_file.rs index 0cc448b5..7a5c1abd 100644 --- a/src/rust/lqos_overrides/src/overrides_file.rs +++ b/src/rust/lqos_overrides/src/overrides_file.rs @@ -87,17 +87,20 @@ pub enum CircuitAdjustment { } /// A network-level override applied while generating `network.json`. -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] #[serde(tag = "type", rename_all = "snake_case")] pub enum NetworkAdjustment { /// Replaces site bandwidth values for a named site. AdjustSiteSpeed { + /// Optional stable node identifier to update. + #[serde(default, skip_serializing_if = "Option::is_none")] + node_id: Option, /// Site name to update. site_name: String, /// Replacement download bandwidth in Mbps. - download_bandwidth_mbps: Option, + download_bandwidth_mbps: Option, /// Replacement upload bandwidth in Mbps. - upload_bandwidth_mbps: Option, + upload_bandwidth_mbps: Option, }, /// Marks a named node as virtual or non-virtual. SetNodeVirtual { @@ -368,52 +371,42 @@ fn merge_circuit_adjustments_owned( fn merge_network_adjustments_owned( operator_adjustments: &[NetworkAdjustment], stormguard_adjustments: &[NetworkAdjustment], - treeguard_adjustments: &[NetworkAdjustment], + _treeguard_adjustments: &[NetworkAdjustment], ) -> Vec { use std::collections::{HashMap, HashSet}; - let mut stormguard_site_speeds: HashMap<&str, (Option, Option)> = HashMap::new(); - let mut stormguard_site_order: Vec<&str> = Vec::new(); - let mut stormguard_site_seen: HashSet<&str> = HashSet::new(); + let mut stormguard_site_speeds: HashMap = HashMap::new(); + let mut stormguard_site_order: Vec = Vec::new(); + let mut stormguard_site_seen: HashSet = HashSet::new(); for adj in stormguard_adjustments { if let NetworkAdjustment::AdjustSiteSpeed { + node_id, site_name, download_bandwidth_mbps, upload_bandwidth_mbps, } = adj { - let name = site_name.as_str(); - stormguard_site_speeds.insert(name, (*download_bandwidth_mbps, *upload_bandwidth_mbps)); - if !stormguard_site_seen.contains(name) { - stormguard_site_order.push(name); - stormguard_site_seen.insert(name); - } - } - } - - let mut treeguard_virtual: HashMap<&str, bool> = HashMap::new(); - let mut treeguard_virtual_order: Vec<&str> = Vec::new(); - let mut treeguard_virtual_seen: HashSet<&str> = HashSet::new(); - - for adj in treeguard_adjustments { - if let NetworkAdjustment::SetNodeVirtual { - node_name, - virtual_node, - } = adj - { - let name = node_name.as_str(); - treeguard_virtual.insert(name, *virtual_node); - if !treeguard_virtual_seen.contains(name) { - treeguard_virtual_order.push(name); - treeguard_virtual_seen.insert(name); + let key = site_speed_key(node_id.as_deref(), site_name); + stormguard_site_speeds.insert( + key.clone(), + NetworkAdjustment::AdjustSiteSpeed { + node_id: node_id.clone(), + site_name: site_name.clone(), + download_bandwidth_mbps: *download_bandwidth_mbps, + upload_bandwidth_mbps: *upload_bandwidth_mbps, + }, + ); + if !stormguard_site_seen.contains(&key) { + stormguard_site_order.push(key.clone()); + stormguard_site_seen.insert(key); } } } let mut out = Vec::new(); - let mut used_treeguard_virtual: HashSet<&str> = HashSet::new(); let mut operator_virtual_seen: HashSet<&str> = HashSet::new(); - let mut operator_site_speed_seen: HashSet<&str> = HashSet::new(); + let mut operator_site_speed_seen: HashSet = HashSet::new(); + let mut operator_site_name_seen: HashSet = HashSet::new(); for adj in operator_adjustments { match adj { @@ -426,31 +419,25 @@ fn merge_network_adjustments_owned( continue; } operator_virtual_seen.insert(name); - - if let Some(v) = treeguard_virtual.get(name) { - out.push(NetworkAdjustment::SetNodeVirtual { - node_name: node_name.clone(), - virtual_node: *v, - }); - used_treeguard_virtual.insert(name); - } else { - out.push(NetworkAdjustment::SetNodeVirtual { - node_name: node_name.clone(), - virtual_node: *virtual_node, - }); - } + out.push(NetworkAdjustment::SetNodeVirtual { + node_name: node_name.clone(), + virtual_node: *virtual_node, + }); } NetworkAdjustment::AdjustSiteSpeed { + node_id, site_name, download_bandwidth_mbps, upload_bandwidth_mbps, } => { - let name = site_name.as_str(); - if operator_site_speed_seen.contains(name) { + let key = site_speed_key(node_id.as_deref(), site_name); + if operator_site_speed_seen.contains(&key) { continue; } - operator_site_speed_seen.insert(name); + operator_site_speed_seen.insert(key); + operator_site_name_seen.insert(site_name.clone()); out.push(NetworkAdjustment::AdjustSiteSpeed { + node_id: node_id.clone(), site_name: site_name.clone(), download_bandwidth_mbps: *download_bandwidth_mbps, upload_bandwidth_mbps: *upload_bandwidth_mbps, @@ -459,33 +446,27 @@ fn merge_network_adjustments_owned( } } - for name in stormguard_site_order { - if operator_site_speed_seen.contains(name) { + for key in stormguard_site_order { + if operator_site_speed_seen.contains(&key) { continue; } - let Some((download_bandwidth_mbps, upload_bandwidth_mbps)) = - stormguard_site_speeds.get(name) + let Some(NetworkAdjustment::AdjustSiteSpeed { + node_id, + site_name, + download_bandwidth_mbps, + upload_bandwidth_mbps, + }) = stormguard_site_speeds.get(&key) else { continue; }; - out.push(NetworkAdjustment::AdjustSiteSpeed { - site_name: name.to_string(), - download_bandwidth_mbps: *download_bandwidth_mbps, - upload_bandwidth_mbps: *upload_bandwidth_mbps, - }); - } - - // Append TreeGuard-only virtual-node entries (ignore other network adjustments). - for name in treeguard_virtual_order { - if used_treeguard_virtual.contains(name) { + if operator_site_name_seen.contains(site_name) { continue; } - let Some(v) = treeguard_virtual.get(name) else { - continue; - }; - out.push(NetworkAdjustment::SetNodeVirtual { - node_name: name.to_string(), - virtual_node: *v, + out.push(NetworkAdjustment::AdjustSiteSpeed { + node_id: node_id.clone(), + site_name: site_name.clone(), + download_bandwidth_mbps: *download_bandwidth_mbps, + upload_bandwidth_mbps: *upload_bandwidth_mbps, }); } @@ -679,10 +660,11 @@ impl OverrideFile { } if current == &device_id ) }); - self.circuit_adjustments.push(CircuitAdjustment::DeviceAdjustSqm { - device_id, - sqm_override: normalized, - }); + self.circuit_adjustments + .push(CircuitAdjustment::DeviceAdjustSqm { + device_id, + sqm_override: normalized, + }); true } @@ -714,34 +696,95 @@ impl OverrideFile { self.network_adjustments.push(adj); } + /// Returns the stored site bandwidth override that best matches this node. + /// + /// Preference order: + /// 1. Exact `node_id` match when one is supplied. + /// 2. Legacy name-only match for the same `site_name`. + pub fn find_site_bandwidth_override( + &self, + node_id: Option<&str>, + site_name: &str, + ) -> Option<&NetworkAdjustment> { + if let Some(node_id) = node_id + && let Some(found) = self.network_adjustments.iter().find(|adj| { + matches!( + adj, + NetworkAdjustment::AdjustSiteSpeed { + node_id: Some(current_node_id), + .. + } if current_node_id == node_id + ) + }) + { + return Some(found); + } + + self.network_adjustments.iter().find(|adj| { + matches!( + adj, + NetworkAdjustment::AdjustSiteSpeed { + node_id: None, + site_name: current_site_name, + .. + } if current_site_name == site_name + ) + }) + } + /// Add or replace a site bandwidth override for `site_name`. pub fn set_site_bandwidth_override( &mut self, + node_id: Option, site_name: String, - download_bandwidth_mbps: Option, - upload_bandwidth_mbps: Option, - ) { + download_bandwidth_mbps: Option, + upload_bandwidth_mbps: Option, + ) -> bool { + let desired = NetworkAdjustment::AdjustSiteSpeed { + node_id: node_id.clone(), + site_name: site_name.clone(), + download_bandwidth_mbps, + upload_bandwidth_mbps, + }; + if self.find_site_bandwidth_override(node_id.as_deref(), &site_name) == Some(&desired) { + return false; + } + self.network_adjustments.retain(|adj| match adj { NetworkAdjustment::AdjustSiteSpeed { - site_name: current, .. - } => current != &site_name, + node_id: current_node_id, + site_name: current_site_name, + .. + } => !site_speed_override_matches( + current_node_id.as_deref(), + current_site_name, + node_id.as_deref(), + &site_name, + ), _ => true, }); - self.network_adjustments - .push(NetworkAdjustment::AdjustSiteSpeed { - site_name, - download_bandwidth_mbps, - upload_bandwidth_mbps, - }); + self.network_adjustments.push(desired); + true } - /// Remove any site bandwidth overrides for `site_name`. Returns number removed. - pub fn remove_site_bandwidth_override_by_name_count(&mut self, site_name: &str) -> usize { + /// Remove any site bandwidth overrides for `site_name` or `node_id`. Returns number removed. + pub fn remove_site_bandwidth_override_count( + &mut self, + node_id: Option<&str>, + site_name: &str, + ) -> usize { let before = self.network_adjustments.len(); self.network_adjustments.retain(|adj| match adj { NetworkAdjustment::AdjustSiteSpeed { - site_name: current, .. - } => current != site_name, + node_id: current_node_id, + site_name: current_site_name, + .. + } => !site_speed_override_matches( + current_node_id.as_deref(), + current_site_name, + node_id, + site_name, + ), _ => true, }); before.saturating_sub(self.network_adjustments.len()) @@ -819,6 +862,29 @@ impl OverrideFile { } } +fn site_speed_key(node_id: Option<&str>, site_name: &str) -> String { + match node_id { + Some(node_id) if !node_id.trim().is_empty() => format!("id:{node_id}"), + _ => format!("name:{site_name}"), + } +} + +fn site_speed_override_matches( + current_node_id: Option<&str>, + current_site_name: &str, + requested_node_id: Option<&str>, + requested_site_name: &str, +) -> bool { + if let Some(requested_node_id) = requested_node_id { + if current_node_id == Some(requested_node_id) { + return true; + } + return current_node_id.is_none() && current_site_name == requested_site_name; + } + + current_site_name == requested_site_name +} + impl OverrideStore { /// Loads a single overrides layer. /// @@ -930,18 +996,22 @@ mod tests { let mut stormguard = OverrideFile::default(); stormguard.add_persistent_shaped_device_return_changed(shaped_device_with_comment( - "dev1", "stormguard", + "dev1", + "stormguard", )); stormguard.add_persistent_shaped_device_return_changed(shaped_device_with_comment( - "dev3", "stormguard", + "dev3", + "stormguard", )); let mut treeguard = OverrideFile::default(); treeguard.add_persistent_shaped_device_return_changed(shaped_device_with_comment( - "dev1", "treeguard", + "dev1", + "treeguard", )); treeguard.add_persistent_shaped_device_return_changed(shaped_device_with_comment( - "dev4", "treeguard", + "dev4", + "treeguard", )); let merged = merge_owned_sections(operator, stormguard, treeguard); @@ -1009,34 +1079,35 @@ mod tests { "dev1".to_string(), Some("fq_codel".to_string()) )); - assert!(of.set_device_sqm_override_return_changed( - "dev1".to_string(), - Some("cake".to_string()) - )); + assert!( + of.set_device_sqm_override_return_changed("dev1".to_string(), Some("cake".to_string())) + ); assert_eq!(of.remove_device_sqm_override_by_device_count("dev1"), 1); assert_eq!(of.remove_device_sqm_override_by_device_count("dev1"), 0); } #[test] - fn effective_merge_treeguard_wins_for_node_virtual_only() { + fn effective_merge_keeps_operator_node_virtual_and_ignores_treeguard_runtime_virtualization() { let mut operator = OverrideFile::default(); operator.set_network_node_virtual("NodeA".to_string(), false); operator.add_network_adjustment(NetworkAdjustment::AdjustSiteSpeed { + node_id: Some("node-site-1".to_string()), site_name: "Site1".to_string(), - download_bandwidth_mbps: Some(100), - upload_bandwidth_mbps: Some(50), + download_bandwidth_mbps: Some(100.0), + upload_bandwidth_mbps: Some(50.0), }); let mut stormguard = OverrideFile::default(); - stormguard.set_site_bandwidth_override("Site1".to_string(), Some(80), Some(40)); - stormguard.set_site_bandwidth_override("Site2".to_string(), Some(150), Some(75)); + stormguard.set_site_bandwidth_override(None, "Site1".to_string(), Some(80.0), Some(40.0)); + stormguard.set_site_bandwidth_override(None, "Site2".to_string(), Some(150.0), Some(75.0)); let mut treeguard = OverrideFile::default(); treeguard.set_network_node_virtual("NodeA".to_string(), true); treeguard.add_network_adjustment(NetworkAdjustment::AdjustSiteSpeed { + node_id: Some("node-site-3".to_string()), site_name: "Site3".to_string(), - download_bandwidth_mbps: Some(200), - upload_bandwidth_mbps: Some(100), + download_bandwidth_mbps: Some(200.0), + upload_bandwidth_mbps: Some(100.0), }); let merged = merge_owned_sections(operator, stormguard, treeguard); @@ -1051,7 +1122,19 @@ mod tests { } if node_name == "NodeA" => Some(*virtual_node), _ => None, }); - assert_eq!(node_a_virtual, Some(true)); + assert_eq!(node_a_virtual, Some(false)); + + let node_three_virtual = merged + .network_adjustments() + .iter() + .find_map(|adj| match adj { + NetworkAdjustment::SetNodeVirtual { + node_name, + virtual_node, + } if node_name == "Node3" => Some(*virtual_node), + _ => None, + }); + assert_eq!(node_three_virtual, None); // Operator site speed should beat StormGuard; TreeGuard site speed should be ignored. let site_speed_names: Vec<&str> = merged @@ -1065,6 +1148,29 @@ mod tests { assert_eq!(site_speed_names, vec!["Site1", "Site2"]); } + #[test] + fn find_site_bandwidth_override_prefers_node_id_match() { + let mut of = OverrideFile::default(); + of.set_site_bandwidth_override(None, "AP27".to_string(), Some(80.0), Some(40.0)); + of.set_site_bandwidth_override( + Some("node-ap27".to_string()), + "AP27".to_string(), + Some(120.5), + Some(60.25), + ); + + let found = of.find_site_bandwidth_override(Some("node-ap27"), "AP27"); + assert!(matches!( + found, + Some(NetworkAdjustment::AdjustSiteSpeed { + node_id: Some(node_id), + download_bandwidth_mbps: Some(download), + upload_bandwidth_mbps: Some(upload), + .. + }) if node_id == "node-ap27" && *download == 120.5 && *upload == 60.25 + )); + } + #[test] fn override_file_defaults_accept_empty_json() { let of: OverrideFile = serde_json::from_str("{}").expect("empty JSON should deserialize"); diff --git a/src/rust/lqos_python/Cargo.toml b/src/rust/lqos_python/Cargo.toml index 928cec88..aab57e9d 100644 --- a/src/rust/lqos_python/Cargo.toml +++ b/src/rust/lqos_python/Cargo.toml @@ -10,7 +10,9 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { workspace = true, features = ["extension-module"] } +lqos_bakery = { path = "../lqos_bakery" } lqos_bus = { path = "../lqos_bus" } +lqos_sys = { path = "../lqos_sys" } lqos_utils = { path = "../lqos_utils" } lqos_config = { path = "../lqos_config" } lqos_overrides = { path = "../lqos_overrides" } diff --git a/src/rust/lqos_python/src/blocking.rs b/src/rust/lqos_python/src/blocking.rs index 58361351..0b9c99c6 100644 --- a/src/rust/lqos_python/src/blocking.rs +++ b/src/rust/lqos_python/src/blocking.rs @@ -1,5 +1,9 @@ -use anyhow::Result; -use lqos_bus::{BusRequest, BusResponse, bus_request}; +use anyhow::{Error, Result}; +use lqos_bus::{BusClientError, BusRequest, BusResponse, bus_request}; +use std::{ + thread::sleep, + time::{Duration, Instant}, +}; pub fn run_query(requests: Vec) -> Result> { let mut replies = Vec::with_capacity(8); @@ -11,3 +15,35 @@ pub fn run_query(requests: Vec) -> Result> { Ok(replies) }) } + +fn should_retry_bus_error(error: &Error) -> bool { + let Some(bus_error) = error.downcast_ref::() else { + return false; + }; + matches!( + bus_error, + BusClientError::SocketNotFound + | BusClientError::StreamReadError + | BusClientError::StreamWriteError + ) +} + +/// Runs a bus query, retrying briefly while the local bus socket is still starting up. +pub fn run_query_wait_for_bus( + requests: Vec, + timeout: Duration, + retry_interval: Duration, +) -> Result> { + let deadline = Instant::now() + timeout; + loop { + match run_query(requests.clone()) { + Ok(replies) => return Ok(replies), + Err(error) => { + if !should_retry_bus_error(&error) || Instant::now() >= deadline { + return Err(error); + } + sleep(retry_interval); + } + } + } +} diff --git a/src/rust/lqos_python/src/device_weights.rs b/src/rust/lqos_python/src/device_weights.rs index 956c462d..4650f485 100644 --- a/src/rust/lqos_python/src/device_weights.rs +++ b/src/rust/lqos_python/src/device_weights.rs @@ -133,7 +133,7 @@ pub(crate) fn get_weights_rust() -> Result> { let mut shaped_devices_weights = get_weights_from_shaped_devices()?; if use_lts_weights() { // This allows us to use Python printing - println!("Using LTS weights"); + println!("Using Insight weights"); let config = load_config().unwrap(); let org_key = config.long_term_stats.license_key.clone().unwrap(); @@ -147,10 +147,10 @@ pub(crate) fn get_weights_rust() -> Result> { let duration_seconds = 60 * 60 * 24; // 1 day let percentile = 0.95; - eprintln!("Getting weights from LTS"); + eprintln!("Getting weights from Insight"); let weights = get_weights_from_lts(&org_key, &node_id, start, duration_seconds, percentile); if let Ok(weights) = weights { - eprintln!("Retrieved {} weights from LTS", weights.len()); + eprintln!("Retrieved {} weights from Insight", weights.len()); // Merge them for weight in weights.iter() { if let Some(existing) = shaped_devices_weights @@ -161,7 +161,7 @@ pub(crate) fn get_weights_rust() -> Result> { } } } else { - eprintln!("Failed to get weights from LTS: {:?}", weights); + eprintln!("Failed to get weights from Insight: {:?}", weights); } } diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index d7bdb8c4..6d9eeba0 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -3,13 +3,17 @@ #![allow(non_local_definitions)] // Temporary: rewrite required for much of this, for newer PyO3. #![allow(unsafe_op_in_unsafe_fn)] #![warn(missing_docs)] -use lqos_bus::{BlackboardSystem, BusRequest, BusResponse, TcHandle, UrgentSeverity, UrgentSource}; +use lqos_bus::{ + BakeryCapacityReportInterface, BlackboardSystem, BusRequest, BusResponse, TcHandle, + UrgentSeverity, UrgentSource, +}; use lqos_utils::hex_string::read_hex_string; use lqos_utils::rustls::ensure_rustls_crypto_provider; use nix::libc::getpid; use pyo3::exceptions::PyOSError; use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::types::{PyDict, PyList}; +use std::fmt::Write as _; use std::{ fs::{File, read_to_string, remove_file}, io::Write, @@ -17,7 +21,8 @@ use std::{ }; mod blocking; use anyhow::{Error, Result}; -use blocking::run_query; +use blocking::{run_query, run_query_wait_for_bus}; +use lqos_bakery::estimate_full_reload_auto_qdisc_budget; use sysinfo::System; mod device_weights; use base64::Engine as _; @@ -110,6 +115,274 @@ fn get_bool(d: &pyo3::Bound<'_, pyo3::types::PyDict>, key: &str, default: bool) } } +#[pyfunction( + signature = ( + items, + queues_available, + prev_assign=None, + last_change_ts=None, + now_ts=None, + mode=None, + move_budget_per_run=None, + cooldown_seconds=None, + hysteresis_threshold=None + ) +)] +#[allow(clippy::too_many_arguments)] +fn plan_top_level_cpu_bins( + py: Python, + items: PyObject, + queues_available: usize, + prev_assign: Option, + last_change_ts: Option, + now_ts: Option, + mode: Option, + move_budget_per_run: Option, + cooldown_seconds: Option, + hysteresis_threshold: Option, +) -> PyResult { + let items_any = items.bind(py); + let items_list = items_any.downcast::()?; + let planner_items: Vec = items_list + .iter() + .map(|item| -> PyResult { + let dict = item.downcast::()?; + let id = get_string(dict, "id", String::new()); + let weight = get_f64(dict, "weight", 1.0); + Ok(lqos_config::TopLevelPlannerItem { id, weight }) + }) + .collect::>>()?; + + let prev_assign_map = match prev_assign { + Some(obj) => { + let any = obj.bind(py); + let dict = any.downcast::()?; + dict.iter() + .filter_map(|(k, v)| { + Some((k.extract::().ok()?, v.extract::().ok()?)) + }) + .collect::>() + } + None => BTreeMap::new(), + }; + let last_change_map = match last_change_ts { + Some(obj) => { + let any = obj.bind(py); + let dict = any.downcast::()?; + dict.iter() + .filter_map(|(k, v)| Some((k.extract::().ok()?, v.extract::().ok()?))) + .collect::>() + } + None => BTreeMap::new(), + }; + + let bins: Vec = (0..queues_available) + .map(|idx| format!("CpueQueue{idx}")) + .collect(); + let planner_mode = match mode.as_deref() { + Some("round_robin") => lqos_config::TopLevelPlannerMode::RoundRobin, + Some("greedy") => lqos_config::TopLevelPlannerMode::Greedy, + _ => lqos_config::TopLevelPlannerMode::StableGreedy, + }; + let params = lqos_config::TopLevelPlannerParams { + mode: planner_mode, + hysteresis_threshold: hysteresis_threshold.unwrap_or(0.03), + cooldown_seconds: cooldown_seconds.unwrap_or(3600.0), + move_budget_per_run: move_budget_per_run.unwrap_or(1), + }; + let result = lqos_config::plan_top_level_assignments( + &planner_items, + &bins, + &prev_assign_map, + &last_change_map, + now_ts.unwrap_or_else(|| { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0) + }), + ¶ms, + ); + + let out = PyDict::new(py); + let assignment = PyDict::new(py); + for (item, bin) in result.assignment { + assignment.set_item(item, bin)?; + } + out.set_item("assignment", assignment)?; + out.set_item("changed", result.changed)?; + out.set_item("planner_used", result.planner_used)?; + Ok(out.into()) +} + +#[pyfunction( + signature = ( + sites, + circuit_groups, + site_state=None, + circuit_state=None, + stick_offset=0, + circuit_padding=8 + ) +)] +fn plan_class_identities( + py: Python, + sites: PyObject, + circuit_groups: PyObject, + site_state: Option, + circuit_state: Option, + stick_offset: u16, + circuit_padding: u32, +) -> PyResult { + let sites_list = sites.bind(py).downcast::()?; + let site_inputs: Vec = sites_list + .iter() + .map(|item| -> PyResult { + let dict = item.downcast::()?; + Ok(lqos_config::SiteIdentityInput { + site_key: get_string(dict, "site_key", String::new()), + parent_path: get_string(dict, "parent_path", String::new()), + queue: get_i64(dict, "queue", 0).max(0) as u32, + has_children: get_bool(dict, "has_children", false), + }) + }) + .collect::>>()?; + + let groups_list = circuit_groups.bind(py).downcast::()?; + let circuit_group_inputs: Vec = groups_list + .iter() + .map(|item| -> PyResult { + let dict = item.downcast::()?; + let ids = match dict.get_item("circuit_ids")? { + Some(value) => value + .downcast::()? + .iter() + .filter_map(|entry| entry.extract::().ok()) + .collect::>(), + None => Vec::new(), + }; + Ok(lqos_config::CircuitIdentityGroupInput { + parent_node: get_string(dict, "parent_node", String::new()), + queue: get_i64(dict, "queue", 0).max(0) as u32, + circuit_ids: ids, + }) + }) + .collect::>>()?; + + let previous_sites = match site_state { + Some(obj) => { + let dict = obj.bind(py).downcast::()?; + dict.iter() + .filter_map(|(k, v)| { + let key = k.extract::().ok()?; + let entry = v.downcast::().ok()?; + Some(( + key, + lqos_config::PlannerSiteIdentityState { + class_minor: get_i64(entry, "class_minor", 0).max(0) as u16, + queue: get_i64(entry, "queue", 0).max(0) as u32, + parent_path: get_string(entry, "parent_path", String::new()), + class_major: get_i64(entry, "class_major", 0).max(0) as u16, + up_class_major: get_i64(entry, "up_class_major", 0).max(0) as u16, + }, + )) + }) + .collect::>() + } + None => BTreeMap::new(), + }; + + let previous_circuits = match circuit_state { + Some(obj) => { + let dict = obj.bind(py).downcast::()?; + dict.iter() + .filter_map(|(k, v)| { + let key = k.extract::().ok()?; + let entry = v.downcast::().ok()?; + Some(( + key, + lqos_config::PlannerCircuitIdentityState { + class_minor: get_i64(entry, "class_minor", 0).max(0) as u16, + queue: get_i64(entry, "queue", 0).max(0) as u32, + parent_node: get_string(entry, "parent_node", String::new()), + class_major: get_i64(entry, "class_major", 0).max(0) as u16, + up_class_major: get_i64(entry, "up_class_major", 0).max(0) as u16, + }, + )) + }) + .collect::>() + } + None => BTreeMap::new(), + }; + + let result = lqos_config::plan_class_identities( + &site_inputs, + &circuit_group_inputs, + &previous_sites, + &previous_circuits, + stick_offset, + circuit_padding, + ); + + let out = PyDict::new(py); + let sites_out = PyList::empty(py); + for site in result.sites { + let d = PyDict::new(py); + d.set_item("site_key", site.site_key)?; + d.set_item("queue", site.queue)?; + d.set_item("class_minor", site.class_minor)?; + d.set_item("class_major", site.class_major)?; + d.set_item("up_class_major", site.up_class_major)?; + d.set_item("parent_path", site.parent_path)?; + sites_out.append(d)?; + } + out.set_item("sites", sites_out)?; + + let circuits_out = PyList::empty(py); + for circuit in result.circuits { + let d = PyDict::new(py); + d.set_item("circuit_id", circuit.circuit_id)?; + d.set_item("parent_node", circuit.parent_node)?; + d.set_item("queue", circuit.queue)?; + d.set_item("class_minor", circuit.class_minor)?; + d.set_item("class_major", circuit.class_major)?; + d.set_item("up_class_major", circuit.up_class_major)?; + circuits_out.append(d)?; + } + out.set_item("circuits", circuits_out)?; + + let site_state_out = PyDict::new(py); + for (key, state) in result.site_state { + let d = PyDict::new(py); + d.set_item("class_minor", state.class_minor)?; + d.set_item("queue", state.queue)?; + d.set_item("parent_path", state.parent_path)?; + d.set_item("class_major", state.class_major)?; + d.set_item("up_class_major", state.up_class_major)?; + site_state_out.set_item(key, d)?; + } + out.set_item("site_state", site_state_out)?; + + let circuit_state_out = PyDict::new(py); + for (key, state) in result.circuit_state { + let d = PyDict::new(py); + d.set_item("class_minor", state.class_minor)?; + d.set_item("queue", state.queue)?; + d.set_item("parent_node", state.parent_node)?; + d.set_item("class_major", state.class_major)?; + d.set_item("up_class_major", state.up_class_major)?; + circuit_state_out.set_item(key, d)?; + } + out.set_item("circuit_state", circuit_state_out)?; + + let last_used = PyDict::new(py); + for (queue, minor) in result.last_used_minor_by_queue { + last_used.set_item(queue, minor)?; + } + out.set_item("last_used_minor_by_queue", last_used)?; + Ok(out.into()) +} + #[pyfunction] fn write_planner_cbor(py: Python, path: String, state: PyObject) -> PyResult { use std::fs; @@ -580,9 +853,11 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(is_lqosd_alive, m)?)?; m.add_function(wrap_pyfunction!(list_ip_mappings, m)?)?; m.add_function(wrap_pyfunction!(clear_ip_mappings, m)?)?; + m.add_function(wrap_pyfunction!(sync_lqosd_config_from_disk, m)?)?; m.add_function(wrap_pyfunction!(delete_ip_mapping, m)?)?; m.add_function(wrap_pyfunction!(add_ip_mapping, m)?)?; m.add_function(wrap_pyfunction!(validate_shaped_devices, m)?)?; + m.add_function(wrap_pyfunction!(wait_for_bus_ready, m)?)?; m.add_function(wrap_pyfunction!(is_libre_already_running, m)?)?; m.add_function(wrap_pyfunction!(create_lock_file, m)?)?; m.add_function(wrap_pyfunction!(free_lock_file, m)?)?; @@ -602,7 +877,7 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(interface_b, m)?)?; m.add_function(wrap_pyfunction!(enable_actual_shell_commands, m)?)?; m.add_function(wrap_pyfunction!(use_bin_packing_to_balance_cpu, m)?)?; - m.add_function(wrap_pyfunction!(monitor_mode_only, m)?)?; + m.add_function(wrap_pyfunction!(queue_mode, m)?)?; m.add_function(wrap_pyfunction!(shaping_cpu_count, m)?)?; m.add_function(wrap_pyfunction!(efficiency_core_ids, m)?)?; m.add_function(wrap_pyfunction!(run_shell_commands_as_sudo, m)?)?; @@ -657,6 +932,8 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(sonar_airmax_ap_model_ids, m)?)?; m.add_function(wrap_pyfunction!(sonar_ltu_ap_model_ids, m)?)?; m.add_function(wrap_pyfunction!(sonar_active_status_ids, m)?)?; + m.add_function(wrap_pyfunction!(sonar_recurring_service_rates, m)?)?; + m.add_function(wrap_pyfunction!(sonar_recurring_excluded_service_names, m)?)?; m.add_function(wrap_pyfunction!(influx_db_enabled, m)?)?; m.add_function(wrap_pyfunction!(influx_db_bucket, m)?)?; m.add_function(wrap_pyfunction!(influx_db_org, m)?)?; @@ -667,16 +944,28 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(get_libreqos_directory, m)?)?; m.add_function(wrap_pyfunction!(overrides_persistent_devices, m)?)?; m.add_function(wrap_pyfunction!(overrides_persistent_devices_effective, m)?)?; + m.add_function(wrap_pyfunction!( + overrides_persistent_devices_materialized, + m + )?)?; m.add_function(wrap_pyfunction!(overrides_circuit_adjustments, m)?)?; m.add_function(wrap_pyfunction!( overrides_circuit_adjustments_effective, m )?)?; + m.add_function(wrap_pyfunction!( + overrides_circuit_adjustments_materialized, + m + )?)?; m.add_function(wrap_pyfunction!(overrides_network_adjustments, m)?)?; m.add_function(wrap_pyfunction!( overrides_network_adjustments_effective, m )?)?; + m.add_function(wrap_pyfunction!( + overrides_network_adjustments_materialized, + m + )?)?; m.add_function(wrap_pyfunction!(is_network_flat, m)?)?; m.add_function(wrap_pyfunction!(blackboard_finish, m)?)?; m.add_function(wrap_pyfunction!(blackboard_submit, m)?)?; @@ -688,18 +977,28 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(promote_to_root_list, m)?)?; m.add_function(wrap_pyfunction!(client_bandwidth_multiplier, m)?)?; m.add_function(wrap_pyfunction!(calculate_hash, m)?)?; + m.add_function(wrap_pyfunction!(calculate_shaping_runtime_hash, m)?)?; m.add_function(wrap_pyfunction!(scheduler_alive, m)?)?; m.add_function(wrap_pyfunction!(scheduler_error, m)?)?; m.add_function(wrap_pyfunction!(scheduler_output, m)?)?; m.add_function(wrap_pyfunction!(submit_urgent_issue, m)?)?; + m.add_function(wrap_pyfunction!(xdp_ip_mapping_capacity, m)?)?; m.add_function(wrap_pyfunction!(is_insight_enabled, m)?)?; m.add_function(wrap_pyfunction!(log_info, m)?)?; + m.add_function(wrap_pyfunction!(treeguard_set_node_virtual_live, m)?)?; + m.add_function(wrap_pyfunction!(treeguard_get_node_virtual_status, m)?)?; + m.add_function(wrap_pyfunction!( + treeguard_get_node_virtual_branch_state, + m + )?)?; m.add_function(wrap_pyfunction!(hash_to_i64, m)?)?; // Planner remote fetch/store for Insight integration m.add_function(wrap_pyfunction!(fetch_planner_remote, m)?)?; m.add_function(wrap_pyfunction!(store_planner_remote, m)?)?; m.add_function(wrap_pyfunction!(write_planner_cbor, m)?)?; m.add_function(wrap_pyfunction!(read_planner_cbor, m)?)?; + m.add_function(wrap_pyfunction!(plan_top_level_cpu_bins, m)?)?; + m.add_function(wrap_pyfunction!(plan_class_identities, m)?)?; m.add_class::()?; Ok(()) @@ -766,6 +1065,33 @@ fn clear_ip_mappings(_py: Python) -> PyResult<()> { Ok(()) } +/// Reloads `/etc/lqos.conf` from disk in the current process and pushes the same +/// config into the running `lqosd` process. +/// +/// This is intended for local admin workflows and test harnesses that update the +/// config file out-of-band and then need `lqosd` to observe the new values before +/// issuing runtime actions. +#[pyfunction] +fn sync_lqosd_config_from_disk(_py: Python) -> PyResult<()> { + lqos_config::clear_cached_config(); + let config = lqos_config::load_config() + .map_err(|e| PyOSError::new_err(format!("Unable to load /etc/lqos.conf: {e}")))?; + let responses = run_query(vec![BusRequest::UpdateLqosdConfig(Box::new( + (*config).clone(), + ))]) + .map_err(|e| PyOSError::new_err(format!("Unable to push config into lqosd: {e}")))?; + if !responses + .iter() + .any(|response| matches!(response, BusResponse::Ack)) + { + return Err(PyOSError::new_err( + "lqosd did not acknowledge the config update request", + )); + } + lqos_config::clear_cached_config(); + Ok(()) +} + /// Deletes an IP to CPU/TC mapping. /// /// ## Arguments @@ -833,13 +1159,34 @@ fn add_ip_mapping( ) -> PyResult<()> { let request = parse_add_ip(&ip, &classid, &cpu, upload, &circuit_id, &device_id); if let Ok(request) = request { - run_query(vec![request]).unwrap(); + let responses = run_query(vec![request]).map_err(|e| PyOSError::new_err(e.to_string()))?; + for response in responses { + if let BusResponse::Fail(message) = response { + return Err(PyOSError::new_err(message)); + } + } Ok(()) } else { Err(PyOSError::new_err(request.err().unwrap().to_string())) } } +fn summarize_failure_examples(failures: &BTreeMap) -> String { + const MAX_EXAMPLES: usize = 3; + failures + .iter() + .take(MAX_EXAMPLES) + .map(|(message, count)| { + if *count > 1 { + format!("{message} (x{count})") + } else { + message.clone() + } + }) + .collect::>() + .join("; ") +} + #[pyclass] /// Collects IP mapping commands so they can be submitted to `lqosd` in batches. pub struct BatchedCommands { @@ -895,22 +1242,50 @@ impl BatchedCommands { /// Sends queued requests to `lqosd` in chunks and returns how many were submitted. pub fn submit(&mut self) -> PyResult { const MAX_BATH_SIZE: usize = 512; - // We're draining the request list out, which is a move that - // *should* be elided by the optimizing compiler. let len = self.batch.len(); + let mut failed_requests = 0usize; + let mut failure_examples: BTreeMap = BTreeMap::new(); while !self.batch.is_empty() { let batch_size = usize::min(MAX_BATH_SIZE, self.batch.len()); let batch: Vec = self.batch.drain(0..batch_size).collect(); - run_query(batch).unwrap(); + let responses = run_query(batch).map_err(|e| { + PyOSError::new_err(format!("IP mapping batch transport failed: {e}")) + })?; + responses.into_iter().for_each(|response| { + if let BusResponse::Fail(message) = response { + failed_requests += 1; + *failure_examples.entry(message).or_insert(0) += 1; + } + }); + } + if failed_requests > 0 { + let example_text = summarize_failure_examples(&failure_examples); + return Err(PyOSError::new_err(format!( + "IP mapping apply failed for {failed_requests} of {len} queued requests. Examples: {example_text}" + ))); } Ok(len) } } +/// Returns the current XDP IP-mapping capacity. +/// +/// If the live pinned map exists, this reflects its `max_entries`; otherwise it +/// falls back to the compiled default capacity. +#[pyfunction] +fn xdp_ip_mapping_capacity() -> PyResult { + Ok(lqos_sys::ip_mapping_capacity()) +} + /// Requests Rust-side validation of `ShapedDevices.csv` #[pyfunction] fn validate_shaped_devices() -> PyResult { - let result = run_query(vec![BusRequest::ValidateShapedDevicesCsv]).unwrap(); + let result = run_query_wait_for_bus( + vec![BusRequest::ValidateShapedDevicesCsv], + Duration::from_secs(5), + Duration::from_millis(100), + ) + .map_err(|e| PyOSError::new_err(format!("Unable to validate shaped devices: {e}")))?; for response in result.iter() { match response { BusResponse::Ack => return Ok("OK".to_string()), @@ -921,6 +1296,29 @@ fn validate_shaped_devices() -> PyResult { Ok("".to_string()) } +/// Waits until the local `lqosd` bus is ready to answer requests. +/// +/// This is intended for scheduler startup sequencing. It retries only for +/// short-lived socket or handshake errors while `lqosd` is still binding the +/// bus socket. +#[pyfunction(signature = (timeout_ms = 5000))] +fn wait_for_bus_ready(timeout_ms: u64) -> PyResult { + let replies = run_query_wait_for_bus( + vec![BusRequest::Ping], + Duration::from_millis(timeout_ms), + Duration::from_millis(100), + ) + .map_err(|e| PyOSError::new_err(format!("Timed out waiting for lqosd bus readiness: {e}")))?; + for response in replies { + if let BusResponse::Ack = response { + return Ok(true); + } + } + Err(PyOSError::new_err( + "Timed out waiting for lqosd bus readiness: bus did not acknowledge ping", + )) +} + /// Returns a Python list of dictionaries representing persistent devices for ShapedDevices.csv /// The dictionary keys mirror the normalized loader used in LibreQoS.py: /// circuitID, circuitName, deviceID, deviceName, ParentNode, mac, @@ -1032,6 +1430,59 @@ fn overrides_persistent_devices_effective(py: Python<'_>) -> PyResult) -> PyResult> { + let overrides = match lqos_overrides::OverrideStore::load_effective(false, false) { + Ok(o) => o, + Err(e) => return Err(PyOSError::new_err(e.to_string())), + }; + + let mut out: Vec = Vec::new(); + for dev in overrides.persistent_devices().iter() { + let ipv4s: Vec = dev + .ipv4 + .iter() + .map(|(ip, prefix)| format!("{}/{}", ip, prefix)) + .collect(); + let ipv6s: Vec = dev + .ipv6 + .iter() + .map(|(ip, prefix)| format!("{}/{}", ip, prefix)) + .collect(); + + let d = PyDict::new(py); + d.set_item("circuitID", dev.circuit_id.clone())?; + d.set_item("circuitName", dev.circuit_name.clone())?; + d.set_item("deviceID", dev.device_id.clone())?; + d.set_item("deviceName", dev.device_name.clone())?; + d.set_item("ParentNode", dev.parent_node.clone())?; + d.set_item("mac", dev.mac.clone())?; + d.set_item("ipv4s", ipv4s)?; + d.set_item("ipv6s", ipv6s)?; + d.set_item("minDownload", dev.download_min_mbps)?; + d.set_item("minUpload", dev.upload_min_mbps)?; + d.set_item("maxDownload", dev.download_max_mbps)?; + d.set_item("maxUpload", dev.upload_max_mbps)?; + d.set_item("comment", dev.comment.clone())?; + d.set_item( + "sqm", + dev.sqm_override + .as_ref() + .map(|s| s.to_string()) + .unwrap_or_default(), + )?; + let obj: PyObject = d.unbind().into(); + out.push(obj); + } + + Ok(out) +} + /// Returns the list of circuit adjustments as Python dicts. #[pyfunction] fn overrides_circuit_adjustments(py: Python<'_>) -> PyResult> { @@ -1221,6 +1672,99 @@ fn overrides_circuit_adjustments_effective(py: Python<'_>) -> PyResult) -> PyResult> { + let overrides = match lqos_overrides::OverrideStore::load_effective(false, false) { + Ok(o) => o, + Err(e) => return Err(PyOSError::new_err(e.to_string())), + }; + + let mut out: Vec = Vec::new(); + for adj in overrides.circuit_adjustments().iter() { + let d = PyDict::new(py); + match adj { + lqos_overrides::CircuitAdjustment::CircuitAdjustSpeed { + circuit_id, + min_download_bandwidth, + max_download_bandwidth, + min_upload_bandwidth, + max_upload_bandwidth, + } => { + d.set_item("type", "circuit_adjust_speed")?; + d.set_item("circuit_id", circuit_id.clone())?; + if let Some(v) = min_download_bandwidth { + d.set_item("min_download_bandwidth", *v)?; + } + if let Some(v) = max_download_bandwidth { + d.set_item("max_download_bandwidth", *v)?; + } + if let Some(v) = min_upload_bandwidth { + d.set_item("min_upload_bandwidth", *v)?; + } + if let Some(v) = max_upload_bandwidth { + d.set_item("max_upload_bandwidth", *v)?; + } + } + lqos_overrides::CircuitAdjustment::DeviceAdjustSpeed { + device_id, + min_download_bandwidth, + max_download_bandwidth, + min_upload_bandwidth, + max_upload_bandwidth, + } => { + d.set_item("type", "device_adjust_speed")?; + d.set_item("device_id", device_id.clone())?; + if let Some(v) = min_download_bandwidth { + d.set_item("min_download_bandwidth", *v)?; + } + if let Some(v) = max_download_bandwidth { + d.set_item("max_download_bandwidth", *v)?; + } + if let Some(v) = min_upload_bandwidth { + d.set_item("min_upload_bandwidth", *v)?; + } + if let Some(v) = max_upload_bandwidth { + d.set_item("max_upload_bandwidth", *v)?; + } + } + lqos_overrides::CircuitAdjustment::DeviceAdjustSqm { + device_id, + sqm_override, + } => { + d.set_item("type", "device_adjust_sqm")?; + d.set_item("device_id", device_id.clone())?; + if let Some(value) = sqm_override { + d.set_item("sqm_override", value.clone())?; + } + } + lqos_overrides::CircuitAdjustment::RemoveCircuit { circuit_id } => { + d.set_item("type", "remove_circuit")?; + d.set_item("circuit_id", circuit_id.clone())?; + } + lqos_overrides::CircuitAdjustment::RemoveDevice { device_id } => { + d.set_item("type", "remove_device")?; + d.set_item("device_id", device_id.clone())?; + } + lqos_overrides::CircuitAdjustment::ReparentCircuit { + circuit_id, + parent_node, + } => { + d.set_item("type", "reparent_circuit")?; + d.set_item("circuit_id", circuit_id.clone())?; + d.set_item("parent_node", parent_node.clone())?; + } + } + let obj: PyObject = d.unbind().into(); + out.push(obj); + } + + Ok(out) +} + /// Returns the list of network adjustments as Python dicts. #[pyfunction] fn overrides_network_adjustments(py: Python<'_>) -> PyResult> { @@ -1229,38 +1773,7 @@ fn overrides_network_adjustments(py: Python<'_>) -> PyResult> { Err(e) => return Err(PyOSError::new_err(e.to_string())), }; - let mut out: Vec = Vec::new(); - for adj in overrides.network_adjustments().iter() { - let d = PyDict::new(py); - match adj { - lqos_overrides::NetworkAdjustment::AdjustSiteSpeed { - site_name, - download_bandwidth_mbps, - upload_bandwidth_mbps, - } => { - d.set_item("type", "adjust_site_speed")?; - d.set_item("site_name", site_name.clone())?; - if let Some(v) = download_bandwidth_mbps { - d.set_item("download_bandwidth_mbps", *v)?; - } - if let Some(v) = upload_bandwidth_mbps { - d.set_item("upload_bandwidth_mbps", *v)?; - } - } - lqos_overrides::NetworkAdjustment::SetNodeVirtual { - node_name, - virtual_node, - } => { - d.set_item("type", "set_node_virtual")?; - d.set_item("node_name", node_name.clone())?; - d.set_item("virtual", *virtual_node)?; - } - } - let obj: PyObject = d.unbind().into(); - out.push(obj); - } - - Ok(out) + network_adjustments_to_py(py, overrides.network_adjustments()) } /// Returns the list of network adjustments as Python dicts, using the effective overrides view @@ -1280,16 +1793,44 @@ fn overrides_network_adjustments_effective(py: Python<'_>) -> PyResult return Err(PyOSError::new_err(e.to_string())), }; + network_adjustments_to_py(py, overrides.network_adjustments()) +} + +/// Returns the list of network adjustments that should be materialized into `network.json`. +/// +/// This includes only operator-owned network adjustments. +/// +/// TreeGuard virtual-node decisions and StormGuard adaptive site-speed decisions are intentionally +/// excluded so runtime automation does not overwrite the operator-authored topology/source-of-truth +/// file. +#[pyfunction] +fn overrides_network_adjustments_materialized(py: Python<'_>) -> PyResult> { + let overrides = match lqos_overrides::OverrideStore::load_effective(false, false) { + Ok(o) => o, + Err(e) => return Err(PyOSError::new_err(e.to_string())), + }; + + network_adjustments_to_py(py, overrides.network_adjustments()) +} + +fn network_adjustments_to_py( + py: Python<'_>, + adjustments: &[lqos_overrides::NetworkAdjustment], +) -> PyResult> { let mut out: Vec = Vec::new(); - for adj in overrides.network_adjustments().iter() { + for adj in adjustments.iter() { let d = PyDict::new(py); match adj { lqos_overrides::NetworkAdjustment::AdjustSiteSpeed { + node_id, site_name, download_bandwidth_mbps, upload_bandwidth_mbps, } => { d.set_item("type", "adjust_site_speed")?; + if let Some(node_id) = node_id { + d.set_item("node_id", node_id.clone())?; + } d.set_item("site_name", site_name.clone())?; if let Some(v) = download_bandwidth_mbps { d.set_item("download_bandwidth_mbps", *v)?; @@ -1420,9 +1961,12 @@ fn use_bin_packing_to_balance_cpu() -> PyResult { } #[pyfunction] -fn monitor_mode_only() -> PyResult { +fn queue_mode() -> PyResult { let config = lqos_config::load_config().unwrap(); - Ok(config.queues.monitor_only) + Ok(match config.queues.queue_mode { + lqos_config::QueueMode::Shape => "shape".to_string(), + lqos_config::QueueMode::Observe => "observe".to_string(), + }) } /// Returns the number of CPUs that should be used for shaping / binning. @@ -1843,6 +2387,34 @@ fn sonar_active_status_ids() -> PyResult> { Ok(key) } +#[pyfunction] +fn sonar_recurring_service_rates() -> PyResult> { + let config = lqos_config::load_config().unwrap(); + let rules = config + .sonar_integration + .recurring_service_rates + .iter() + .map(|rule| { + ( + rule.enabled, + rule.service_name.clone(), + rule.download_mbps, + rule.upload_mbps, + ) + }) + .collect(); + Ok(rules) +} + +#[pyfunction] +fn sonar_recurring_excluded_service_names() -> PyResult> { + let config = lqos_config::load_config().unwrap(); + Ok(config + .sonar_integration + .recurring_excluded_service_names + .clone()) +} + #[pyfunction] fn influx_db_enabled() -> PyResult { let config = lqos_config::load_config().unwrap(); @@ -2035,6 +2607,73 @@ fn calculate_hash() -> PyResult { Ok(hash) } +fn shaping_runtime_sqm_fingerprint() -> Result { + let config = lqos_config::load_config()?; + let apply_stormguard = config + .stormguard + .as_ref() + .is_some_and(|sg| sg.enabled && !sg.dry_run); + let apply_treeguard = config.treeguard.enabled; + let overrides = + lqos_overrides::OverrideStore::load_effective(apply_stormguard, apply_treeguard)?; + + let mut sqm_entries: Vec<(String, String)> = overrides + .circuit_adjustments() + .iter() + .filter_map(|adj| match adj { + lqos_overrides::CircuitAdjustment::DeviceAdjustSqm { + device_id, + sqm_override, + } => { + let token = sqm_override + .as_deref() + .map(str::trim) + .map(str::to_lowercase) + .unwrap_or_default(); + if token.is_empty() { + None + } else { + Some((device_id.clone(), token)) + } + } + _ => None, + }) + .collect(); + sqm_entries.sort(); + + let mut fingerprint = String::new(); + for (device_id, token) in sqm_entries { + let _ = writeln!(&mut fingerprint, "{device_id}={token}"); + } + + Ok(fingerprint) +} + +#[pyfunction] +fn calculate_shaping_runtime_hash() -> PyResult { + let Ok(config) = lqos_config::load_config() else { + return Ok(0); + }; + let nj_path = Path::new(&config.lqos_directory).join("network.json"); + let sd_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); + + let Ok(nj_as_string) = read_to_string(nj_path) else { + return Ok(0); + }; + let Ok(sd_as_string) = read_to_string(sd_path) else { + return Ok(0); + }; + let Ok(runtime_sqm_fingerprint) = shaping_runtime_sqm_fingerprint() else { + return Ok(0); + }; + + let combined = format!( + "{}\n{}\n{}", + nj_as_string, sd_as_string, runtime_sqm_fingerprint + ); + Ok(lqos_utils::hash_to_i64(&combined)) +} + ////////////////////////////// The Bakery class ////////////////////////////// #[derive(Clone)] @@ -2057,6 +2696,8 @@ enum BakeryCommands { }, AddCircuit { circuit_hash: i64, + circuit_name: Option, + site_name: Option, parent_class_id: TcHandle, up_parent_class_id: TcHandle, class_minor: u16, @@ -2071,6 +2712,74 @@ enum BakeryCommands { }, } +impl BakeryCommands { + fn as_runtime_command(&self) -> lqos_bakery::BakeryCommands { + match self { + BakeryCommands::StartBatch => lqos_bakery::BakeryCommands::StartBatch, + BakeryCommands::Commit => lqos_bakery::BakeryCommands::CommitBatch, + BakeryCommands::MqSetup { + queues_available, + stick_offset, + } => lqos_bakery::BakeryCommands::MqSetup { + queues_available: *queues_available, + stick_offset: *stick_offset, + }, + BakeryCommands::AddSite { + site_hash, + parent_class_id, + up_parent_class_id, + class_minor, + download_bandwidth_min, + upload_bandwidth_min, + download_bandwidth_max, + upload_bandwidth_max, + } => lqos_bakery::BakeryCommands::AddSite { + site_hash: *site_hash, + parent_class_id: *parent_class_id, + up_parent_class_id: *up_parent_class_id, + class_minor: *class_minor, + download_bandwidth_min: *download_bandwidth_min, + upload_bandwidth_min: *upload_bandwidth_min, + download_bandwidth_max: *download_bandwidth_max, + upload_bandwidth_max: *upload_bandwidth_max, + }, + BakeryCommands::AddCircuit { + circuit_hash, + circuit_name, + site_name, + parent_class_id, + up_parent_class_id, + class_minor, + download_bandwidth_min, + upload_bandwidth_min, + download_bandwidth_max, + upload_bandwidth_max, + class_major, + up_class_major, + ip_addresses, + sqm_override, + } => lqos_bakery::BakeryCommands::AddCircuit { + circuit_hash: *circuit_hash, + circuit_name: circuit_name.clone(), + site_name: site_name.clone(), + parent_class_id: *parent_class_id, + up_parent_class_id: *up_parent_class_id, + class_minor: *class_minor, + download_bandwidth_min: *download_bandwidth_min, + upload_bandwidth_min: *upload_bandwidth_min, + download_bandwidth_max: *download_bandwidth_max, + upload_bandwidth_max: *upload_bandwidth_max, + class_major: *class_major, + up_class_major: *up_class_major, + down_qdisc_handle: None, + up_qdisc_handle: None, + ip_addresses: ip_addresses.clone(), + sqm_override: sqm_override.clone(), + }, + } + } +} + #[pyclass] /// Queues Bakery operations for batched submission to the LibreQoS daemon. pub struct Bakery { @@ -2103,7 +2812,10 @@ impl Bakery { .build() .unwrap() .block_on(async { - let mut bus = lqos_bus::LibreqosBusClient::new().await.unwrap(); + let Ok(mut bus) = lqos_bus::LibreqosBusClient::new().await else { + eprintln!("Failed to connect to lqosd bus for Bakery commit"); + return; + }; let chunks = queue.chunks(1024); for chunk in chunks { let mut requests = Vec::new(); @@ -2146,6 +2858,8 @@ impl Bakery { } BakeryCommands::AddCircuit { circuit_hash, + circuit_name, + site_name, parent_class_id, up_parent_class_id, class_minor, @@ -2160,6 +2874,8 @@ impl Bakery { } => { let command = BusRequest::BakeryAddCircuit { circuit_hash: *circuit_hash, + circuit_name: circuit_name.clone(), + site_name: site_name.clone(), parent_class_id: *parent_class_id, up_parent_class_id: *up_parent_class_id, class_minor: *class_minor, @@ -2189,6 +2905,129 @@ impl Bakery { Ok(()) } + /// Estimates whether the queued full-reload batch fits within the per-interface qdisc budget. + pub fn estimate_qdisc_budget(&self, py: Python) -> PyResult { + let config = lqos_config::load_config().map_err(|e| PyOSError::new_err(e.to_string()))?; + let queue: Vec = self + .queue + .iter() + .map(BakeryCommands::as_runtime_command) + .collect(); + let estimate = estimate_full_reload_auto_qdisc_budget(&config, &queue); + let is_ok = estimate.ok(); + let interface_reports = estimate + .interface_details + .iter() + .map(|(name, detail)| BakeryCapacityReportInterface { + name: name.clone(), + planned_qdiscs: detail.planned_qdiscs, + infra_qdiscs: detail.infra_qdiscs, + cake_qdiscs: detail.cake_qdiscs, + fq_codel_qdiscs: detail.fq_codel_qdiscs, + estimated_memory_bytes: detail.estimated_memory_bytes, + }) + .collect::>(); + let memory_summary = if let Some(snapshot) = estimate.memory_snapshot.as_ref() { + format!( + "estimated qdisc memory {} bytes with {} bytes currently available and safety floor {} bytes", + estimate.estimated_total_memory_bytes, + snapshot.available_bytes, + lqos_bakery::BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES + ) + } else { + format!( + "estimated qdisc memory {} bytes; host memory snapshot unavailable", + estimate.estimated_total_memory_bytes + ) + }; + let summary = if interface_reports.is_empty() { + format!( + "Planned queue model {} preflight. No shaping interfaces were queued; {memory_summary}.", + if is_ok { "fits" } else { "exceeds" }, + ) + } else { + let interface_summary = interface_reports + .iter() + .map(|entry| { + let detail = estimate + .interface_details + .get(&entry.name) + .expect("detail should exist for interface"); + format!( + "{} estimated {} qdiscs (infra {}, cake {}, fq_codel {})", + entry.name, + entry.planned_qdiscs, + detail.infra_qdiscs, + detail.cake_qdiscs, + detail.fq_codel_qdiscs + ) + }) + .collect::>() + .join(", "); + format!( + "Planned queue model {} preflight. {interface_summary}; safe budget {}, kernel limit {}; {memory_summary}.", + if is_ok { "fits" } else { "exceeds" }, + estimate.safe_budget, + estimate.hard_limit, + ) + }; + let _ = run_query(vec![BusRequest::BakeryReportPreflight { + ok: is_ok, + message: summary.clone(), + safe_budget: estimate.safe_budget, + hard_limit: estimate.hard_limit, + estimated_total_memory_bytes: estimate.estimated_total_memory_bytes, + memory_available_bytes: estimate + .memory_snapshot + .as_ref() + .map(|snapshot| snapshot.available_bytes), + memory_guard_min_available_bytes: lqos_bakery::BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES, + memory_ok: estimate.memory_ok, + interfaces: interface_reports, + }]); + + let result = PyDict::new(py); + let interfaces = PyDict::new(py); + let interface_details = PyDict::new(py); + + for (interface, count) in estimate.interfaces { + interfaces.set_item(interface, count)?; + } + for (interface, detail) in estimate.interface_details { + let detail_dict = PyDict::new(py); + detail_dict.set_item("planned_qdiscs", detail.planned_qdiscs)?; + detail_dict.set_item("infra_qdiscs", detail.infra_qdiscs)?; + detail_dict.set_item("cake_qdiscs", detail.cake_qdiscs)?; + detail_dict.set_item("fq_codel_qdiscs", detail.fq_codel_qdiscs)?; + detail_dict.set_item("estimated_memory_bytes", detail.estimated_memory_bytes)?; + interface_details.set_item(interface, detail_dict)?; + } + + result.set_item("interfaces", interfaces)?; + result.set_item("interface_details", interface_details)?; + result.set_item("safe_budget", estimate.safe_budget)?; + result.set_item("hard_limit", estimate.hard_limit)?; + result.set_item( + "estimated_total_memory_bytes", + estimate.estimated_total_memory_bytes, + )?; + result.set_item("memory_ok", estimate.memory_ok)?; + result.set_item( + "memory_guard_min_available_bytes", + lqos_bakery::BAKERY_MEMORY_GUARD_MIN_AVAILABLE_BYTES, + )?; + if let Some(snapshot) = estimate.memory_snapshot { + result.set_item("memory_total_bytes", snapshot.total_bytes)?; + result.set_item("memory_available_bytes", snapshot.available_bytes)?; + } else { + result.set_item("memory_total_bytes", py.None())?; + result.set_item("memory_available_bytes", py.None())?; + } + result.set_item("ok", is_ok)?; + result.set_item("summary", summary)?; + Ok(result.into()) + } + /// Queues multi-queue setup for the target shaper. pub fn setup_mq(&mut self, queues_available: usize, stick_offset: usize) -> PyResult<()> { self.queue.push(BakeryCommands::MqSetup { @@ -2232,6 +3071,7 @@ impl Bakery { pub fn add_circuit( &mut self, circuit_name: String, + site_name: Option, parent_class_id: String, up_parent_class_id: String, class_minor: u16, @@ -2248,6 +3088,8 @@ impl Bakery { //println!("Name: {circuit_name}, hash: {circuit_hash}"); let command = BakeryCommands::AddCircuit { circuit_hash, + circuit_name: Some(circuit_name), + site_name, parent_class_id: TcHandle::from_string(&parent_class_id).unwrap(), up_parent_class_id: TcHandle::from_string(&up_parent_class_id).unwrap(), class_minor, @@ -2362,6 +3204,93 @@ fn log_info(_py: Python, message: String) -> PyResult { Ok(false) } +/// Submit a live Bakery runtime node virtualization or restore intent for a named node. +#[pyfunction] +fn treeguard_set_node_virtual_live( + _py: Python, + node_name: String, + virtualized: bool, +) -> PyResult { + if let Ok(reply) = run_query(vec![BusRequest::TreeGuardSetNodeVirtual { + node_name, + virtualized, + }]) { + for resp in reply.iter() { + if let BusResponse::Ack = resp { + return Ok(true); + } + } + } + Ok(false) +} + +/// Fetch the latest Bakery runtime node-operation snapshot for a named node, if any. +#[pyfunction] +fn treeguard_get_node_virtual_status( + py: Python, + node_name: String, +) -> PyResult>> { + let Ok(reply) = run_query(vec![BusRequest::TreeGuardGetNodeVirtualStatus { + node_name, + }]) else { + return Ok(None); + }; + for resp in reply { + if let BusResponse::TreeGuardRuntimeNodeOperation(snapshot) = resp { + let Some(snapshot) = snapshot else { + return Ok(None); + }; + let failure_reason = snapshot.failure_reason.clone(); + let d = PyDict::new(py); + d.set_item("operation_id", snapshot.operation_id)?; + d.set_item("site_hash", snapshot.site_hash)?; + d.set_item("action", snapshot.action)?; + d.set_item("status", snapshot.status)?; + d.set_item("attempt_count", snapshot.attempt_count)?; + d.set_item("submitted_at_unix", snapshot.submitted_at_unix)?; + d.set_item("updated_at_unix", snapshot.updated_at_unix)?; + d.set_item("next_retry_at_unix", snapshot.next_retry_at_unix)?; + d.set_item("last_error", snapshot.last_error)?; + d.set_item("failure_reason", failure_reason)?; + return Ok(Some(d.unbind())); + } + } + Ok(None) +} + +/// Fetch the latest Bakery runtime branch-state snapshot for a named node, if one is retained. +#[pyfunction] +fn treeguard_get_node_virtual_branch_state( + py: Python, + node_name: String, +) -> PyResult>> { + let Ok(reply) = run_query(vec![BusRequest::TreeGuardGetNodeVirtualBranchState { + node_name, + }]) else { + return Ok(None); + }; + for resp in reply { + if let BusResponse::TreeGuardRuntimeNodeBranch(snapshot) = resp { + let Some(snapshot) = snapshot else { + return Ok(None); + }; + let d = PyDict::new(py); + d.set_item("site_hash", snapshot.site_hash)?; + d.set_item("active_branch", snapshot.active_branch)?; + d.set_item("lifecycle", snapshot.lifecycle)?; + d.set_item("pending_prune", snapshot.pending_prune)?; + d.set_item("next_prune_attempt_unix", snapshot.next_prune_attempt_unix)?; + d.set_item("active_site_hashes", snapshot.active_site_hashes)?; + d.set_item("saved_site_hashes", snapshot.saved_site_hashes)?; + d.set_item("prune_site_hashes", snapshot.prune_site_hashes)?; + d.set_item("qdisc_down_major", snapshot.qdisc_down_major)?; + d.set_item("qdisc_up_major", snapshot.qdisc_up_major)?; + return Ok(Some(d.unbind())); + } + } + Ok(None) +} + #[pyfunction] /// Returns whether Insight features are currently enabled in `lqosd`. pub fn is_insight_enabled() -> PyResult { @@ -2381,3 +3310,23 @@ pub fn is_insight_enabled() -> PyResult { pub fn hash_to_i64(text: String) -> PyResult { Ok(lqos_utils::hash_to_i64(&text)) } + +#[cfg(test)] +mod tests { + use super::summarize_failure_examples; + use std::collections::BTreeMap; + + #[test] + fn summarize_failure_examples_limits_output() { + let mut failures = BTreeMap::new(); + failures.insert("alpha".to_string(), 2); + failures.insert("beta".to_string(), 1); + failures.insert("delta".to_string(), 1); + failures.insert("gamma".to_string(), 1); + + assert_eq!( + summarize_failure_examples(&failures), + "alpha (x2); beta; delta" + ); + } +} diff --git a/src/rust/lqos_queue_tracker/Cargo.toml b/src/rust/lqos_queue_tracker/Cargo.toml index d6dcd712..64333321 100644 --- a/src/rust/lqos_queue_tracker/Cargo.toml +++ b/src/rust/lqos_queue_tracker/Cargo.toml @@ -9,6 +9,7 @@ thiserror = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } lqos_bus = { path = "../lqos_bus" } +lqos_bakery = { path = "../lqos_bakery" } lqos_config = { path = "../lqos_config" } lqos_sys = { path = "../lqos_sys" } lqos_utils = { path = "../lqos_utils" } diff --git a/src/rust/lqos_queue_tracker/src/lib.rs b/src/rust/lqos_queue_tracker/src/lib.rs index 64291d51..cebcae6a 100644 --- a/src/rust/lqos_queue_tracker/src/lib.rs +++ b/src/rust/lqos_queue_tracker/src/lib.rs @@ -20,11 +20,12 @@ const NUM_QUEUE_HISTORY: usize = 600; pub use bus::get_raw_circuit_data; pub use interval::set_queue_refresh_interval; +#[doc(hidden)] pub use queue_structure::{ - EFFECTIVE_NODE_RATES, QUEUE_STRUCTURE, QUEUE_STRUCTURE_CHANGED_STORMGUARD, - spawn_queue_structure_monitor, + EFFECTIVE_NODE_RATES, QUEUE_STRUCTURE, QUEUE_STRUCTURE_CHANGED_STORMGUARD, QueueNode, + QueueStructure, spawn_queue_structure_monitor, }; pub use queue_types::deserialize_tc_tree; // Exported for the benchmarker pub use tracking::spawn_queue_monitor; -pub use tracking::{ALL_QUEUE_SUMMARY, QueueCounts, TOTAL_QUEUE_STATS}; +pub use tracking::{ALL_QUEUE_SUMMARY, QueueCounts, TOTAL_QUEUE_STATS, queue_stats_stale}; pub use tracking::{add_watched_queue, still_watching}; diff --git a/src/rust/lqos_queue_tracker/src/queue_structure/mod.rs b/src/rust/lqos_queue_tracker/src/queue_structure/mod.rs index 409ce614..6d33280c 100644 --- a/src/rust/lqos_queue_tracker/src/queue_structure/mod.rs +++ b/src/rust/lqos_queue_tracker/src/queue_structure/mod.rs @@ -2,11 +2,13 @@ mod queing_structure_json_monitor; mod queue_network; mod queue_node; pub use queing_structure_json_monitor::spawn_queue_structure_monitor; +#[doc(hidden)] pub use queing_structure_json_monitor::{ - EFFECTIVE_NODE_RATES, QUEUE_STRUCTURE, QUEUE_STRUCTURE_CHANGED_STORMGUARD, + EFFECTIVE_NODE_RATES, QUEUE_STRUCTURE, QUEUE_STRUCTURE_CHANGED_STORMGUARD, QueueStructure, }; use queue_network::QueueNetwork; -pub(crate) use queue_node::QueueNode; +#[doc(hidden)] +pub use queue_node::QueueNode; use thiserror::Error; pub(crate) fn read_queueing_structure() -> Result, QueueStructureError> { diff --git a/src/rust/lqos_queue_tracker/src/queue_structure/queing_structure_json_monitor.rs b/src/rust/lqos_queue_tracker/src/queue_structure/queing_structure_json_monitor.rs index 58e89c3f..7d1783b4 100644 --- a/src/rust/lqos_queue_tracker/src/queue_structure/queing_structure_json_monitor.rs +++ b/src/rust/lqos_queue_tracker/src/queue_structure/queing_structure_json_monitor.rs @@ -1,7 +1,6 @@ use crate::queue_structure::{ queue_network::QueueNetwork, queue_node::QueueNode, read_queueing_structure, }; -use crate::tracking::ALL_QUEUE_SUMMARY; use arc_swap::ArcSwap; use lqos_utils::file_watcher::FileWatcher; use once_cell::sync::Lazy; @@ -31,7 +30,9 @@ pub static EFFECTIVE_NODE_RATES: Lazy>> = La /// to avoid circular dependencies. pub static QUEUE_STRUCTURE_CHANGED_STORMGUARD: AtomicBool = AtomicBool::new(false); +#[allow(missing_docs)] #[derive(Clone)] +/// Snapshot of the current flattened queue tree loaded from `queuingStructure.json`. pub struct QueueStructure { pub maybe_queues: Option>, } @@ -94,7 +95,6 @@ fn update_queue_structure() { maybe_queues: Some(queues.clone()), }; let effective_node_rates = build_effective_node_rates(&queues); - ALL_QUEUE_SUMMARY.clear(); QUEUE_STRUCTURE.store(Arc::new(new_queue_structure)); EFFECTIVE_NODE_RATES.store(Arc::new(effective_node_rates)); QUEUE_STRUCTURE_CHANGED_STORMGUARD.store(true, std::sync::atomic::Ordering::Relaxed); @@ -108,7 +108,6 @@ fn update_queue_structure() { warn!( "Failed to load queuingStructure.json ({err:?}); leaving queue structure unavailable" ); - ALL_QUEUE_SUMMARY.clear(); QUEUE_STRUCTURE.store(Arc::new(QueueStructure { maybe_queues: None })); EFFECTIVE_NODE_RATES.store(Arc::new(HashMap::new())); } diff --git a/src/rust/lqos_queue_tracker/src/queue_structure/queue_node.rs b/src/rust/lqos_queue_tracker/src/queue_structure/queue_node.rs index fcbf8e89..2d4a7ab9 100644 --- a/src/rust/lqos_queue_tracker/src/queue_structure/queue_node.rs +++ b/src/rust/lqos_queue_tracker/src/queue_structure/queue_node.rs @@ -5,6 +5,7 @@ use lqos_utils::hex_string::read_hex_string; use serde_json::Value; use tracing::{debug, error, warn}; +#[allow(missing_docs)] #[derive(Default, Clone, Debug)] pub struct QueueNode { pub name: Option, @@ -247,7 +248,7 @@ impl QueueNode { } return Err(QueueStructureError::Children); }; - result.circuits.push(n); + result.children.push(n); } } else { warn!("Children was not an object"); @@ -263,6 +264,9 @@ impl QueueNode { } else { warn!("Unable to parse node structure for [{key}]"); } + if result.name.is_none() && result.circuit_id.is_none() && result.device_id.is_none() { + result.name = Some(key.to_string()); + } Ok(result) } @@ -290,13 +294,14 @@ impl QueueNode { #[cfg(test)] mod test { use super::*; + use std::collections::HashSet; const EXAMPLE_QUEUE_STRUCTURE_WITH_CHILDREN: &str = include_str!("./example_queue_with_children.test.json"); const EXAMPLE_QUEUE_STRUCTURE_NO_CHILDREN: &str = include_str!("./example_queue_flat.test.json"); - fn try_load_queue_structure(raw_string: &str) { + fn try_load_queue_structure(raw_string: &str) -> super::super::QueueNetwork { let mut result = super::super::QueueNetwork { cpu_node: Vec::new(), }; @@ -318,6 +323,8 @@ mod test { } else { panic!("Unable to parse queueStructure.json"); } + + result } #[test] @@ -329,4 +336,62 @@ mod test { fn load_queue_structure_with_children() { try_load_queue_structure(EXAMPLE_QUEUE_STRUCTURE_WITH_CHILDREN); } + + #[test] + fn object_key_nodes_preserve_names_when_name_field_is_absent() { + let network = try_load_queue_structure(EXAMPLE_QUEUE_STRUCTURE_WITH_CHILDREN); + + let flat = network.to_flat(); + let names: HashSet = flat.iter().filter_map(|node| node.name.clone()).collect(); + + assert!(names.contains("Site_1")); + assert!(names.contains("Site_3")); + assert!(names.contains("PoP_5")); + assert!(names.contains("PoP_6")); + assert!(names.contains("AP_11")); + } + + #[test] + fn children_are_tracked_as_children_not_circuits() { + let network = try_load_queue_structure(EXAMPLE_QUEUE_STRUCTURE_WITH_CHILDREN); + + let site_1 = network + .cpu_node + .iter() + .find(|node| node.name.as_deref() == Some("Site_1")) + .expect("Site_1 root should exist"); + + assert_eq!(site_1.children.len(), 2); + assert!(site_1.circuits.is_empty()); + assert!( + site_1 + .children + .iter() + .any(|child| child.name.as_deref() == Some("AP_A")) + ); + assert!( + site_1 + .children + .iter() + .any(|child| child.name.as_deref() == Some("Site_3")) + ); + } + + #[test] + fn flattened_snapshot_supports_stormguard_style_name_lookup() { + let network = try_load_queue_structure(EXAMPLE_QUEUE_STRUCTURE_WITH_CHILDREN); + let flat = network.to_flat(); + + let target = flat + .iter() + .find(|node| node.name.as_deref() == Some("PoP_6")) + .expect("PoP_6 should be resolvable by name"); + + assert_eq!(target.download_bandwidth_mbps, 60); + assert_eq!(target.upload_bandwidth_mbps, 60); + assert_eq!( + target.class_id, + TcHandle::from_string("0x1:0xb").expect("valid tc handle") + ); + } } diff --git a/src/rust/lqos_queue_tracker/src/tracking/all_queue_data.rs b/src/rust/lqos_queue_tracker/src/tracking/all_queue_data.rs index da612304..585be9b5 100644 --- a/src/rust/lqos_queue_tracker/src/tracking/all_queue_data.rs +++ b/src/rust/lqos_queue_tracker/src/tracking/all_queue_data.rs @@ -9,6 +9,8 @@ use std::collections::HashMap; pub struct QueueCounts { /// Number of CAKE queues. pub cake: usize, + /// Number of fq_codel queues. + pub fq_codel: usize, /// Number of HTB queues. pub htb: usize, } diff --git a/src/rust/lqos_queue_tracker/src/tracking/mod.rs b/src/rust/lqos_queue_tracker/src/tracking/mod.rs index 2915ca1e..8904bb31 100644 --- a/src/rust/lqos_queue_tracker/src/tracking/mod.rs +++ b/src/rust/lqos_queue_tracker/src/tracking/mod.rs @@ -2,7 +2,9 @@ use crate::{ circuit_to_queue::CIRCUIT_TO_QUEUE, interval::QUEUE_MONITOR_INTERVAL, queue_store::QueueStore, tracking::reader::read_named_queue_from_interface, }; +use lqos_bakery::full_reload_in_progress; use lqos_utils::fdtimer::periodic; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use timerfd::{SetTimeFlags, TimerFd, TimerState}; use tracing::{debug, error, warn}; @@ -18,7 +20,25 @@ use crate::tracking::reader::read_all_queues_from_interface; use watched_queues::WATCHED_QUEUES; pub use watched_queues::{add_watched_queue, still_watching}; +static QUEUE_STATS_STALE: AtomicBool = AtomicBool::new(false); +const ALL_QUEUE_MONITOR_INTERVAL_SECONDS: u64 = 2; +const RELOAD_QUEUE_MONITOR_INTERVAL_SECONDS: u64 = 10; + +/// Returns `true` when queue counts are intentionally held at their last-known values. +/// +/// This is currently used during Bakery full reloads, where shelling out to `tc` +/// for full-queue snapshots is too expensive to provide reliable live counts. +pub fn queue_stats_stale() -> bool { + QUEUE_STATS_STALE.load(Ordering::Relaxed) +} + fn track_queues() { + if full_reload_in_progress() { + QUEUE_STATS_STALE.store(true, Ordering::Relaxed); + debug!("(TC monitor) Skipping watched queue reads during Bakery full reload"); + expire_watched_queues(); + return; + } if WATCHED_QUEUES.is_empty() { //info!("No queues marked for read."); return; // There's nothing to do - bail out fast @@ -83,6 +103,7 @@ fn count_queue_types(queues: &[QueueType]) -> QueueCounts { for queue in queues.iter() { match queue { QueueType::Cake(_) => counts.cake += 1, + QueueType::FqCodel(_) => counts.fq_codel += 1, QueueType::Htb(_) => counts.htb += 1, _ => {} } @@ -153,6 +174,12 @@ fn connect_queues_to_circuit_up( fn all_queue_reader() { let start = Instant::now(); + if full_reload_in_progress() { + QUEUE_STATS_STALE.store(true, Ordering::Relaxed); + debug!("(TC monitor) Skipping full queue read during Bakery full reload"); + return; + } + let structure = QUEUE_STRUCTURE.load(); if let Some(structure) = &structure.maybe_queues { if let Ok(config) = lqos_config::load_config() { @@ -191,6 +218,7 @@ fn all_queue_reader() { .unwrap_or_default(); let counts = QueueCounts { cake: counts_down.cake + counts_up.cake, + fq_codel: counts_down.fq_codel + counts_up.fq_codel, htb: counts_down.htb + counts_up.htb, }; (download, upload, counts) @@ -198,10 +226,13 @@ fn all_queue_reader() { //println!("{}", download.len() + upload.len()); ALL_QUEUE_SUMMARY.ingest_batch(download, upload, queue_counts); + QUEUE_STATS_STALE.store(false, Ordering::Relaxed); } else { + QUEUE_STATS_STALE.store(true, Ordering::Relaxed); warn!("(TC monitor) Unable to read configuration"); } } else { + QUEUE_STATS_STALE.store(true, Ordering::Relaxed); warn!("(TC monitor) Not reading queues due to structure not yet ready"); } let elapsed = start.elapsed(); @@ -238,7 +269,7 @@ pub fn spawn_queue_monitor() -> anyhow::Result<()> { std::thread::Builder::new() .name("All Queue Monitor".to_string()) .spawn(|| { - let mut interval_seconds = 2; + let mut interval_seconds = ALL_QUEUE_MONITOR_INTERVAL_SECONDS; let Ok(mut tfd) = TimerFd::new() else { error!("Unable to start timer file descriptor. All queue monitor cannot run."); return; @@ -246,7 +277,7 @@ pub fn spawn_queue_monitor() -> anyhow::Result<()> { assert_eq!(tfd.get_state(), TimerState::Disarmed); tfd.set_state( TimerState::Periodic { - current: Duration::new(2, 0), + current: Duration::new(interval_seconds, 0), interval: Duration::new(interval_seconds, 0), }, SetTimeFlags::Default, @@ -254,13 +285,34 @@ pub fn spawn_queue_monitor() -> anyhow::Result<()> { let _ = tfd.read(); // Initial pause loop { - all_queue_reader(); + let desired_interval_seconds = if full_reload_in_progress() { + RELOAD_QUEUE_MONITOR_INTERVAL_SECONDS + } else { + ALL_QUEUE_MONITOR_INTERVAL_SECONDS + }; + if interval_seconds != desired_interval_seconds { + interval_seconds = desired_interval_seconds; + tfd.set_state( + TimerState::Periodic { + current: Duration::new(interval_seconds, 0), + interval: Duration::new(interval_seconds, 0), + }, + SetTimeFlags::Default, + ); + } + + if full_reload_in_progress() { + QUEUE_STATS_STALE.store(true, Ordering::Relaxed); + debug!("(TC monitor) Backing off full queue reads during Bakery full reload"); + } else { + all_queue_reader(); + } // Sleep until the next second let missed_ticks = tfd.read(); if missed_ticks > 1 { warn!("All Queue Reader: Missed {} ticks", missed_ticks - 1); - interval_seconds = 2 + (missed_ticks - 1); + interval_seconds = desired_interval_seconds + (missed_ticks - 1); tfd.set_state( TimerState::Periodic { current: Duration::new(interval_seconds, 0), @@ -274,3 +326,63 @@ pub fn spawn_queue_monitor() -> anyhow::Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::connect_queues_to_circuit; + use crate::{QueueNode, deserialize_tc_tree}; + use lqos_bus::TcHandle; + use lqos_utils::hash_to_i64; + + fn test_structure() -> Vec { + vec![QueueNode { + class_id: TcHandle::from_string("3:14af").expect("valid class id"), + up_class_id: TcHandle::from_string("43:14af").expect("valid up class id"), + parent_class_id: TcHandle::from_string("3:20").expect("valid parent"), + up_parent_class_id: TcHandle::from_string("43:20").expect("valid up parent"), + class_major: 0x0003, + up_class_major: 0x0043, + class_minor: 0x14af, + circuit_id: Some("circuit-1".to_string()), + circuit_hash: Some(hash_to_i64("circuit-1")), + ..QueueNode::default() + }] + } + + #[test] + fn cake_queue_matching_uses_parent_not_leaf_handle() { + let structure = test_structure(); + let old_handle_queues = deserialize_tc_tree( + r#"[{ + "kind":"cake", + "handle":"9000:", + "parent":"3:14af", + "options":{}, + "tins":[], + "drops":7 + }]"#, + ) + .expect("old-handle cake should parse"); + let rotated_handle_queues = deserialize_tc_tree( + r#"[{ + "kind":"cake", + "handle":"9002:", + "parent":"3:14af", + "options":{}, + "tins":[], + "drops":11 + }]"#, + ) + .expect("rotated-handle cake should parse"); + + let old_matches = connect_queues_to_circuit(&structure, &old_handle_queues); + let rotated_matches = connect_queues_to_circuit(&structure, &rotated_handle_queues); + + assert_eq!(old_matches.len(), 1); + assert_eq!(rotated_matches.len(), 1); + assert_eq!(old_matches[0].circuit_hash, hash_to_i64("circuit-1")); + assert_eq!(rotated_matches[0].circuit_hash, hash_to_i64("circuit-1")); + assert_eq!(old_matches[0].drops, 7); + assert_eq!(rotated_matches[0].drops, 11); + } +} diff --git a/src/rust/lqos_stormguard/src/adaptive_actions.rs b/src/rust/lqos_stormguard/src/adaptive_actions.rs index 62ef6369..b50ba262 100644 --- a/src/rust/lqos_stormguard/src/adaptive_actions.rs +++ b/src/rust/lqos_stormguard/src/adaptive_actions.rs @@ -15,8 +15,8 @@ pub enum CircuitFallbackOutcome { pub struct SiteOverrideUpdate { pub site_name: String, - pub download_bandwidth_mbps: Option, - pub upload_bandwidth_mbps: Option, + pub download_bandwidth_mbps: Option, + pub upload_bandwidth_mbps: Option, } #[derive(Clone)] @@ -36,7 +36,7 @@ pub fn apply_site_override_updates(updates: &[SiteOverrideUpdate]) -> Result 0 { changed = true; } @@ -48,6 +48,7 @@ pub fn apply_site_override_updates(updates: &[SiteOverrideUpdate]) -> Result Result Option<(Option, Option)> { +) -> Option<(Option, Option)> { overrides .network_adjustments() .iter() .find_map(|adj| match adj { lqos_overrides::NetworkAdjustment::AdjustSiteSpeed { + node_id: _, site_name: current, download_bandwidth_mbps, upload_bandwidth_mbps, @@ -298,6 +300,11 @@ fn apply_circuit_sqm_override_live( bakery_sender.send(BakeryCommands::AddCircuit { circuit_hash: hash_to_i64(circuit_id), + circuit_name: node + .circuit_name + .clone() + .or_else(|| Some(circuit_id.to_string())), + site_name: node.parent_node.clone(), parent_class_id: node.parent_class_id, up_parent_class_id: node.up_parent_class_id, class_minor, @@ -307,6 +314,8 @@ fn apply_circuit_sqm_override_live( upload_bandwidth_max: node.upload_bandwidth_mbps as f32, class_major, up_class_major, + down_qdisc_handle: None, + up_qdisc_handle: None, ip_addresses: ip_list(devices), sqm_override: sqm_override.map(|value| value.to_string()), })?; @@ -357,12 +366,13 @@ fn group_circuit_fallbacks( continue; } - let entry = grouped - .entry(device.circuit_id.clone()) - .or_insert_with(|| PersistedCircuitFallback { - sqm_override: token.to_string(), - devices: Vec::new(), - }); + let entry = + grouped + .entry(device.circuit_id.clone()) + .or_insert_with(|| PersistedCircuitFallback { + sqm_override: token.to_string(), + devices: Vec::new(), + }); if entry.sqm_override != token { continue; } @@ -434,10 +444,12 @@ mod tests { "dev2".to_string(), Some("fq_codel".to_string()) )); - assert!(overrides.set_device_sqm_override_return_changed( - "dev3".to_string(), - Some("cake".to_string()) - )); + assert!( + overrides.set_device_sqm_override_return_changed( + "dev3".to_string(), + Some("cake".to_string()) + ) + ); let grouped = group_circuit_fallbacks(&overrides, &[d1, d2, d3]); assert_eq!(grouped.len(), 2); @@ -479,14 +491,16 @@ mod tests { "dev1".to_string(), Some("fq_codel".to_string()) )); - assert!(overrides.set_device_sqm_override_return_changed( - "dev2".to_string(), - Some("cake".to_string()) - )); - assert!(overrides.set_device_sqm_override_return_changed( - "dev3".to_string(), - Some(" ".to_string()) - )); + assert!( + overrides.set_device_sqm_override_return_changed( + "dev2".to_string(), + Some("cake".to_string()) + ) + ); + assert!( + overrides + .set_device_sqm_override_return_changed("dev3".to_string(), Some(" ".to_string())) + ); let grouped = group_circuit_fallbacks(&overrides, &[d1, d2, d3]); assert_eq!( diff --git a/src/rust/lqos_stormguard/src/config.rs b/src/rust/lqos_stormguard/src/config.rs index c12b70d4..6e897728 100644 --- a/src/rust/lqos_stormguard/src/config.rs +++ b/src/rust/lqos_stormguard/src/config.rs @@ -130,7 +130,7 @@ pub fn configure() -> anyhow::Result { Ok(result) } -fn load_stormguard_site_overrides() -> HashMap, Option)> { +fn load_stormguard_site_overrides() -> HashMap, Option)> { let Ok(overrides) = OverrideStore::load_layer(OverrideLayer::Stormguard) else { warn!("Unable to load StormGuard override layer; starting from planned rates."); return HashMap::new(); @@ -141,6 +141,7 @@ fn load_stormguard_site_overrides() -> HashMap, Option .iter() .filter_map(|adj| match adj { NetworkAdjustment::AdjustSiteSpeed { + node_id: _, site_name, download_bandwidth_mbps, upload_bandwidth_mbps, @@ -155,7 +156,7 @@ fn load_stormguard_site_overrides() -> HashMap, Option fn get_sites_from_queueing_structure( sg_config: &lqos_config::StormguardConfig, - persisted_site_overrides: &HashMap, Option)>, + persisted_site_overrides: &HashMap, Option)>, ) -> HashMap { let mut selected: Vec = if sg_config.all_sites { all_candidate_site_names() @@ -196,12 +197,12 @@ fn get_sites_from_queueing_structure( .unwrap_or((None, None)); let current_download_mbps = persisted .0 - .map(u64::from) + .map(|mbps| mbps.max(0.0) as u64) .unwrap_or(max_down) .clamp(min_down, max_down); let current_upload_mbps = persisted .1 - .map(u64::from) + .map(|mbps| mbps.max(0.0) as u64) .unwrap_or(max_up) .clamp(min_up, max_up); diff --git a/src/rust/lqos_stormguard/src/site_state.rs b/src/rust/lqos_stormguard/src/site_state.rs index 7ffd9664..c48c1915 100644 --- a/src/rust/lqos_stormguard/src/site_state.rs +++ b/src/rust/lqos_stormguard/src/site_state.rs @@ -836,9 +836,9 @@ impl SiteStateTracker { SiteOverrideUpdate { site_name: site.config.name.clone(), download_bandwidth_mbps: (site.queue_download_mbps != site.config.max_download_mbps) - .then_some(site.queue_download_mbps as u32), + .then_some(site.queue_download_mbps as f32), upload_bandwidth_mbps: (site.queue_upload_mbps != site.config.max_upload_mbps) - .then_some(site.queue_upload_mbps as u32), + .then_some(site.queue_upload_mbps as f32), } } @@ -1019,7 +1019,7 @@ mod tests { fn site_override_update_keeps_only_changed_directions() { let site = site_state(75, 50, 100, 50); let update = SiteStateTracker::site_override_update_from_state(&site); - assert_eq!(update.download_bandwidth_mbps, Some(75)); + assert_eq!(update.download_bandwidth_mbps, Some(75.0)); assert_eq!(update.upload_bandwidth_mbps, None); } diff --git a/src/rust/lqos_sys/build.rs b/src/rust/lqos_sys/build.rs index a02b60a9..15d79dae 100644 --- a/src/rust/lqos_sys/build.rs +++ b/src/rust/lqos_sys/build.rs @@ -32,6 +32,37 @@ fn command_warnings_errors_only(section: &str, command_result: &std::io::Result< } } +fn require_successful_output(section: &str, command_result: std::io::Result) -> Output { + match command_result { + Ok(output) => { + if !output.stderr.is_empty() { + println!( + "cargo:warning=[{section}] {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + if !output.status.success() { + panic!( + "{section} failed with status {}", + output + .status + .code() + .map_or_else(|| "signal".to_string(), |code| code.to_string()) + ); + } + + output + } + Err(error) => { + panic!( + "{section} is required to build lqos_sys but was not available: {error}. \ +Install bpftool (or provide it in PATH) before building." + ); + } + } +} + fn main() { let out_dir = env::var_os("OUT_DIR").unwrap(); @@ -84,7 +115,8 @@ fn main() { .args(["gen", "skeleton", &link_target]) .output(); command_warnings_errors_only("bpf skel", &skel_result); - let header_file = String::from_utf8(skel_result.unwrap().stdout).unwrap(); + let skel_output = require_successful_output("bpftool gen skeleton", skel_result); + let header_file = String::from_utf8(skel_output.stdout).unwrap(); std::fs::write(skel_target, header_file).unwrap(); // 4: Copy the wrapper to our out dir diff --git a/src/rust/lqos_sys/src/lib.rs b/src/rust/lqos_sys/src/lib.rs index 94ffb597..d7f7a462 100644 --- a/src/rust/lqos_sys/src/lib.rs +++ b/src/rust/lqos_sys/src/lib.rs @@ -32,6 +32,7 @@ pub use ip_mapping::{ pub use kernel_wrapper::LibreQoSKernels; pub use linux::num_possible_cpus; pub use lqos_kernel::interface_name_to_index; +pub use lqos_kernel::ip_mapping_capacity; pub use lqos_kernel::max_tracked_ips; pub use lqos_kernel::unload_xdp_from_interface; pub use throughput::{HostCounter, throughput_for_each}; diff --git a/src/rust/lqos_sys/src/lqos_kernel.rs b/src/rust/lqos_sys/src/lqos_kernel.rs index 5d57b3ac..7094d44c 100644 --- a/src/rust/lqos_sys/src/lqos_kernel.rs +++ b/src/rust/lqos_sys/src/lqos_kernel.rs @@ -37,6 +37,49 @@ pub fn max_tracked_ips() -> usize { (unsafe { bpf::max_tracker_ips() }) as usize } +fn pinned_map_max_entries(path: &str) -> Result> { + if !Path::new(path).exists() { + return Ok(None); + } + + let path_c = CString::new(path)?; + let fd = unsafe { bpf_obj_get(path_c.as_ptr()) }; + if fd < 0 { + warn!("Unable to open pinned BPF map '{path}' for capacity check (fd={fd})"); + return Ok(None); + } + + let mut info = MaybeUninit::::zeroed(); + let mut len: u32 = std::mem::size_of::() as u32; + let err = unsafe { bpf_obj_get_info_by_fd(fd, info.as_mut_ptr() as *mut c_void, &mut len) }; + unsafe { + close(fd); + } + if err != 0 { + return Err(Error::msg(format!( + "Unable to query pinned BPF map '{path}' capacity (err={err})." + ))); + } + + let info = unsafe { info.assume_init() }; + Ok(Some(info.max_entries)) +} + +/// Returns the currently available IP-mapping capacity. +/// +/// If the live pinned map exists, this returns its `max_entries` value. +/// Otherwise, it falls back to the compiled `MAX_TRACKED_IPS` constant. +pub fn ip_mapping_capacity() -> usize { + match pinned_map_max_entries("/sys/fs/bpf/map_ip_to_cpu_and_tc") { + Ok(Some(entries)) => entries as usize, + Ok(None) => max_tracked_ips(), + Err(e) => { + warn!("Unable to determine live IP-mapping capacity: {e:?}"); + max_tracked_ips() + } + } +} + pub fn check_root() -> Result<()> { unsafe { if geteuid() == 0 { diff --git a/src/rust/lqos_utils/src/units/down_up.rs b/src/rust/lqos_utils/src/units/down_up.rs index f30e7e9e..bad39c12 100644 --- a/src/rust/lqos_utils/src/units/down_up.rs +++ b/src/rust/lqos_utils/src/units/down_up.rs @@ -8,6 +8,120 @@ use serde::{Deserialize, Serialize}; use std::ops::AddAssign; use zerocopy::FromBytes; +/// Strongly-typed count of TCP retransmit events. +#[repr(transparent)] +#[derive( + Copy, + Clone, + Debug, + Default, + Eq, + PartialEq, + Serialize, + Deserialize, + Ord, + PartialOrd, + Hash, + Allocative, +)] +#[serde(transparent)] +pub struct RetransmitCount(pub u64); + +impl RetransmitCount { + /// Returns the underlying retransmit count. + pub const fn get(self) -> u64 { + self.0 + } +} + +/// Strongly-typed count of TCP packets used as the retransmit denominator. +#[repr(transparent)] +#[derive( + Copy, + Clone, + Debug, + Default, + Eq, + PartialEq, + Serialize, + Deserialize, + Ord, + PartialOrd, + Hash, + Allocative, +)] +#[serde(transparent)] +pub struct TcpPacketCount(pub u64); + +impl TcpPacketCount { + /// Returns the underlying packet count. + pub const fn get(self) -> u64 { + self.0 + } +} + +/// Strongly-typed retransmit fraction in the inclusive range `0.0..=1.0`. +#[repr(transparent)] +#[derive( + Copy, Clone, Debug, Default, PartialEq, PartialOrd, Serialize, Deserialize, Allocative, +)] +#[serde(transparent)] +pub struct RetransmitFraction(f64); + +impl RetransmitFraction { + /// Constructs a retransmit fraction if the provided value is within `0.0..=1.0`. + pub fn new(value: f64) -> Option { + if (0.0..=1.0).contains(&value) { + Some(Self(value)) + } else { + None + } + } + + /// Returns the underlying fraction value. + pub const fn get(self) -> f64 { + self.0 + } + + /// Converts the fraction to a `0.0..=100.0` percentage. + pub fn percent_0_to_100(self) -> f64 { + self.0 * 100.0 + } +} + +/// Keeps TCP retransmit numerator and denominator together so callers cannot mix units. +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize, Allocative)] +pub struct TcpRetransmitSample { + /// The retransmit count for the sample window. + pub retransmits: RetransmitCount, + /// The TCP packet count for the same sample window. + pub packets: TcpPacketCount, +} + +impl TcpRetransmitSample { + /// Creates a new retransmit sample from raw counts. + pub const fn new(retransmits: u64, packets: u64) -> Self { + Self { + retransmits: RetransmitCount(retransmits), + packets: TcpPacketCount(packets), + } + } + + /// Computes the retransmit fraction for the sample window. + pub fn fraction(self) -> Option { + let packets = self.packets.get(); + if packets == 0 { + return None; + } + RetransmitFraction::new(self.retransmits.get() as f64 / packets as f64) + } + + /// Computes the retransmit percentage for the sample window. + pub fn percent_0_to_100(self) -> Option { + self.fraction().map(RetransmitFraction::percent_0_to_100) + } +} + /// Provides strong download/upload separation for /// stored statistics to eliminate confusion. This is a generic /// type: you can control the type stored inside. @@ -33,6 +147,44 @@ pub struct DownUpOrder { pub up: T, } +impl DownUpOrder { + /// Create a new DownUpOrder with the given down and up values. + pub const fn new(down: T, up: T) -> Self { + Self { down, up } + } + + /// In the C code, it's common to refer to a "direction" byte: + /// + /// * 0: down + /// * 1: up + /// * >1: error + /// + /// This is a helper function to translate that byte into the + /// appropriate value. + pub fn dir(&self, direction: usize) -> T + where + T: Copy, + { + if direction == 0 { self.down } else { self.up } + } + + /// Get the `down` value. + pub fn get_down(&self) -> T + where + T: Copy, + { + self.down + } + + /// Get the `up` value. + pub fn get_up(&self) -> T + where + T: Copy, + { + self.up + } +} + impl DownUpOrder where T: std::cmp::Ord @@ -46,23 +198,6 @@ where + num_traits::SaturatingAdd + Default, { - /// Create a new DownUpOrder with the given down and up values. - pub fn new(down: T, up: T) -> Self { - Self { down, up } - } - - /// In the C code, it's common to refer to a "direction" byte: - /// - /// * 0: down - /// * 1: up - /// * >1: error - /// - /// This is a helper function to translate that byte into the - /// appropriate value. - pub fn dir(&self, direction: usize) -> T { - if direction == 0 { self.down } else { self.up } - } - /// Return a new DownUpOrder with both down and up set to zero. pub fn zeroed() -> Self { Self { @@ -127,16 +262,6 @@ where } } - /// Get the `down` value. - pub fn get_down(&self) -> T { - self.down - } - - /// Get the `up` value. - pub fn get_up(&self) -> T { - self.up - } - /// Set both the `down` and `up` values to zero. pub fn set_to_zero(&mut self) { self.down = T::zero(); @@ -177,6 +302,17 @@ pub fn down_up_divide(left: DownUpOrder, right: DownUpOrder) -> (f64, (safe_div(left.down, right.down), safe_div(left.up, right.up)) } +/// Pairs retransmit counts with TCP packet counts in the same directional ordering. +pub fn down_up_retransmit_sample( + retransmits: DownUpOrder, + packets: DownUpOrder, +) -> DownUpOrder { + DownUpOrder::new( + TcpRetransmitSample::new(retransmits.down, packets.down), + TcpRetransmitSample::new(retransmits.up, packets.up), + ) +} + #[cfg(test)] mod tests { use super::*; @@ -226,4 +362,23 @@ mod tests { assert_eq!(a.down, 2); assert_eq!(a.up, 3); } + + #[test] + fn retransmit_fraction_rejects_invalid_values() { + assert!(RetransmitFraction::new(-0.1).is_none()); + assert!(RetransmitFraction::new(1.1).is_none()); + assert_eq!(RetransmitFraction::new(0.25).map(|f| f.get()), Some(0.25)); + } + + #[test] + fn retransmit_sample_computes_fraction_and_percent() { + let sample = TcpRetransmitSample::new(5, 200); + let fraction = sample.fraction().expect("fraction should exist"); + assert_eq!(fraction.get(), 0.025); + assert_eq!(sample.percent_0_to_100(), Some(2.5)); + + let empty = TcpRetransmitSample::new(5, 0); + assert!(empty.fraction().is_none()); + assert!(empty.percent_0_to_100().is_none()); + } } diff --git a/src/rust/lqosd/Cargo.toml b/src/rust/lqosd/Cargo.toml index 5ee808a6..7f710d2a 100644 --- a/src/rust/lqosd/Cargo.toml +++ b/src/rust/lqosd/Cargo.toml @@ -31,6 +31,9 @@ nix = { workspace = true } sysinfo = { workspace = true } itertools = "0.13.0" reqwest = { workspace = true } +hmac = { workspace = true } +base64 = { workspace = true } +sha2 = { workspace = true } flate2 = "1" bincode = { workspace = true } ip_network_table = { workspace = true } diff --git a/src/rust/lqosd/src/lts2_sys/control_channel.rs b/src/rust/lqosd/src/lts2_sys/control_channel.rs index 88b589d1..774b42ad 100644 --- a/src/rust/lqosd/src/lts2_sys/control_channel.rs +++ b/src/rust/lqosd/src/lts2_sys/control_channel.rs @@ -20,6 +20,13 @@ pub use messages::{ SupportTicketSummary, WsMessage, }; +const SHAPER_VERSION_STRING: &str = include_str!("../../../../VERSION_STRING"); + +fn current_shaper_version() -> Option { + let version = SHAPER_VERSION_STRING.trim(); + (!version.is_empty()).then(|| version.to_string()) +} + #[derive(Debug)] pub struct HistoryQueryResultPayload { pub tag: String, @@ -1064,6 +1071,7 @@ async fn persistent_connection( license, node_id: config.node_id.clone(), node_name: config.node_name.clone(), + shaper_version: current_shaper_version(), }; let Ok((_, _, bytes)) = message.to_bytes() else { error!("Failed to serialize license message"); @@ -1204,6 +1212,7 @@ async fn send_license( license, node_id: config.node_id.clone(), node_name: config.node_name.clone(), + shaper_version: current_shaper_version(), }; let (_, _, bytes) = message.to_bytes()?; timeout( diff --git a/src/rust/lqosd/src/lts2_sys/control_channel/messages.rs b/src/rust/lqosd/src/lts2_sys/control_channel/messages.rs index bfe445a9..cb531504 100644 --- a/src/rust/lqosd/src/lts2_sys/control_channel/messages.rs +++ b/src/rust/lqosd/src/lts2_sys/control_channel/messages.rs @@ -64,7 +64,7 @@ pub struct SupportTicket { pub const MAX_DECOMPRESSED_WS_MSG_BYTES: usize = 16 * 1024 * 1024; // 16 MiB -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub enum WsMessage { // Messages FROM the Shaper Hello { @@ -74,6 +74,8 @@ pub enum WsMessage { license: Uuid, node_id: String, node_name: String, + #[serde(default)] + shaper_version: Option, }, LicenseGrantRequest { public_key: Vec, @@ -254,7 +256,7 @@ pub enum WsMessage { }, } -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub enum ApiRequestType { Get, Post, @@ -277,3 +279,69 @@ impl WsMessage { Ok(serde_cbor::from_slice(&decompressed_bytes)?) } } + +#[cfg(test)] +mod tests { + use super::WsMessage; + use serde::Serialize; + use uuid::Uuid; + + #[derive(Serialize)] + enum LegacyWsMessage { + License { + license: Uuid, + node_id: String, + node_name: String, + }, + } + + #[test] + fn license_round_trips_with_version() { + let message = WsMessage::License { + license: Uuid::nil(), + node_id: "node-1".to_string(), + node_name: "Node 1".to_string(), + shaper_version: Some("2.1.0".to_string()), + }; + let (_, _, bytes) = message.to_bytes().unwrap(); + + match WsMessage::from_bytes(&bytes).unwrap() { + WsMessage::License { + shaper_version, + node_id, + node_name, + .. + } => { + assert_eq!(node_id, "node-1"); + assert_eq!(node_name, "Node 1"); + assert_eq!(shaper_version.as_deref(), Some("2.1.0")); + } + other => panic!("unexpected decoded message: {other:?}"), + } + } + + #[test] + fn legacy_license_payload_without_version_still_decodes() { + let legacy = LegacyWsMessage::License { + license: Uuid::nil(), + node_id: "legacy-node".to_string(), + node_name: "Legacy Node".to_string(), + }; + let raw_bytes = serde_cbor::to_vec(&legacy).unwrap(); + let bytes = miniz_oxide::deflate::compress_to_vec(&raw_bytes, 1); + + match WsMessage::from_bytes(&bytes).unwrap() { + WsMessage::License { + shaper_version, + node_id, + node_name, + .. + } => { + assert_eq!(node_id, "legacy-node"); + assert_eq!(node_name, "Legacy Node"); + assert_eq!(shaper_version, None); + } + other => panic!("unexpected decoded message: {other:?}"), + } + } +} diff --git a/src/rust/lqosd/src/main.rs b/src/rust/lqosd/src/main.rs index 7630d238..0966dda0 100644 --- a/src/rust/lqosd/src/main.rs +++ b/src/rust/lqosd/src/main.rs @@ -12,7 +12,6 @@ pub mod lts2_sys; mod node_manager; mod preflight_checks; mod program_control; -mod reload_lock; mod remote_commands; mod rtt_exclusions; mod scheduler_control; @@ -38,10 +37,11 @@ use crate::{ ip_mapping::{clear_ip_flows, del_ip_flow, list_mapped_ips, map_ip_to_flow}, throughput_tracker::flow_data::{FlowActor, flowbee_handle_events, setup_netflow_tracker}, }; -#[cfg(feature = "flamegraphs")] -use allocative::Allocative; use anyhow::Result; -use lqos_bus::{BusRequest, BusResponse, InsightLicenseSummary, UnixSocketServer}; +use lqos_bus::{ + BusRequest, BusResponse, InsightLicenseSummary, TreeGuardRuntimeNodeBranchSnapshot, + TreeGuardRuntimeNodeOperationSnapshot, UnixSocketServer, +}; use lqos_heimdall::{n_second_packet_dump, perf_interface::heimdall_handle_events, start_heimdall}; use lqos_queue_tracker::{ add_watched_queue, get_raw_circuit_data, spawn_queue_monitor, spawn_queue_structure_monitor, @@ -66,12 +66,6 @@ use crate::blackboard::{BLACKBOARD_SENDER, BlackboardCommand}; use crate::lts2_sys::get_lts_license_status; use crate::lts2_sys::shared_types::LtsStatus; use crate::remote_commands::start_remote_commands; -#[cfg(feature = "flamegraphs")] -use crate::shaped_devices_tracker::NETWORK_JSON; -#[cfg(feature = "flamegraphs")] -use crate::throughput_tracker::THROUGHPUT_TRACKER; -#[cfg(feature = "flamegraphs")] -use crate::throughput_tracker::flow_data::{ALL_FLOWS, RECENT_FLOWS}; use lqos_stormguard::{STORMGUARD_DEBUG, STORMGUARD_STATS}; use tracing::level_filters::LevelFilter; // Use MiMalloc only on supported platforms @@ -515,6 +509,10 @@ fn handle_bus_requests(requests: &[BusRequest], responses: &mut Vec } BusResponse::Ack } + BusRequest::InvalidateAuthCache => { + crate::node_manager::invalidate_auth_cache(); + BusResponse::Ack + } #[cfg(feature = "equinix_tests")] BusRequest::RequestLqosEquinixTest => lqos_daht_test::lqos_daht_test(), BusRequest::ValidateShapedDevicesCsv => validation::validate_shaped_devices_csv(), @@ -676,6 +674,8 @@ fn handle_bus_requests(requests: &[BusRequest], responses: &mut Vec } BusRequest::BakeryAddCircuit { circuit_hash, + circuit_name, + site_name, parent_class_id, up_parent_class_id, class_minor, @@ -702,6 +702,8 @@ fn handle_bus_requests(requests: &[BusRequest], responses: &mut Vec let sender = sender.clone(); let _ = sender.send(lqos_bakery::BakeryCommands::AddCircuit { circuit_hash: *circuit_hash, + circuit_name: circuit_name.clone(), + site_name: site_name.clone(), parent_class_id: *parent_class_id, up_parent_class_id: *up_parent_class_id, class_minor: *class_minor, @@ -711,6 +713,8 @@ fn handle_bus_requests(requests: &[BusRequest], responses: &mut Vec upload_bandwidth_max: *upload_bandwidth_max, class_major: *class_major, up_class_major: *up_class_major, + down_qdisc_handle: None, + up_qdisc_handle: None, ip_addresses: ip_addresses.clone(), sqm_override: sqm_override.clone(), }); @@ -736,6 +740,126 @@ fn handle_bus_requests(requests: &[BusRequest], responses: &mut Vec BusRequest::GetBakeryStats => BusResponse::BakeryActiveCircuits( lqos_bakery::ACTIVE_CIRCUITS.load(std::sync::atomic::Ordering::Relaxed), ), + BusRequest::BakeryReportPreflight { + ok, + message, + safe_budget, + hard_limit, + estimated_total_memory_bytes, + memory_available_bytes, + memory_guard_min_available_bytes, + memory_ok, + interfaces, + } => { + lqos_bakery::record_qdisc_preflight_snapshot( + lqos_bakery::BakeryPreflightSnapshot { + ok: *ok, + message: message.clone(), + safe_budget: *safe_budget, + hard_limit: *hard_limit, + estimated_total_memory_bytes: *estimated_total_memory_bytes, + memory_available_bytes: *memory_available_bytes, + memory_guard_min_available_bytes: *memory_guard_min_available_bytes, + memory_ok: *memory_ok, + interfaces: interfaces + .iter() + .map(|entry| lqos_bakery::BakeryCapacityInterfaceSnapshot { + name: entry.name.clone(), + planned_qdiscs: entry.planned_qdiscs, + infra_qdiscs: entry.infra_qdiscs, + cake_qdiscs: entry.cake_qdiscs, + fq_codel_qdiscs: entry.fq_codel_qdiscs, + estimated_memory_bytes: entry.estimated_memory_bytes, + }) + .collect(), + }, + ); + BusResponse::Ack + }, + BusRequest::TreeGuardSetNodeVirtual { + node_name, + virtualized, + } => match crate::treeguard::bakery::submit_node_virtualization_live( + node_name, + *virtualized, + ) { + Ok(()) => BusResponse::Ack, + Err(err) => BusResponse::Fail(err.to_string()), + }, + BusRequest::TreeGuardGetNodeVirtualStatus { node_name } => { + let snapshot = crate::treeguard::bakery::node_virtualization_operation_status( + node_name, + ) + .map(|snapshot| { + let failure_reason = snapshot.failure_reason.map(|reason| match reason { + lqos_bakery::BakeryRuntimeNodeOperationFailureReason::StructuralIneligibleNoPromotableChildren => { + "structural_ineligible_no_promotable_children".to_string() + } + lqos_bakery::BakeryRuntimeNodeOperationFailureReason::StructuralIneligibleSinglePromotableChild => { + "structural_ineligible_single_promotable_child".to_string() + } + }); + TreeGuardRuntimeNodeOperationSnapshot { + operation_id: snapshot.operation_id, + site_hash: snapshot.site_hash, + action: match snapshot.action { + lqos_bakery::BakeryRuntimeNodeOperationAction::Virtualize => { + "virtualize".to_string() + } + lqos_bakery::BakeryRuntimeNodeOperationAction::Restore => { + "restore".to_string() + } + }, + status: match snapshot.status { + lqos_bakery::BakeryRuntimeNodeOperationStatus::Submitted => { + "submitted".to_string() + } + lqos_bakery::BakeryRuntimeNodeOperationStatus::Deferred => { + "deferred".to_string() + } + lqos_bakery::BakeryRuntimeNodeOperationStatus::Applying => { + "applying".to_string() + } + lqos_bakery::BakeryRuntimeNodeOperationStatus::AppliedAwaitingCleanup => { + "applied_awaiting_cleanup".to_string() + } + lqos_bakery::BakeryRuntimeNodeOperationStatus::Completed => { + "completed".to_string() + } + lqos_bakery::BakeryRuntimeNodeOperationStatus::Failed => { + "failed".to_string() + } + lqos_bakery::BakeryRuntimeNodeOperationStatus::Dirty => { + "dirty".to_string() + } + }, + attempt_count: snapshot.attempt_count, + submitted_at_unix: snapshot.submitted_at_unix, + updated_at_unix: snapshot.updated_at_unix, + next_retry_at_unix: snapshot.next_retry_at_unix, + last_error: snapshot.last_error, + failure_reason, + }}); + BusResponse::TreeGuardRuntimeNodeOperation(snapshot) + } + BusRequest::TreeGuardGetNodeVirtualBranchState { node_name } => { + let snapshot = crate::treeguard::bakery::node_virtualization_branch_state( + node_name, + ) + .map(|snapshot| TreeGuardRuntimeNodeBranchSnapshot { + site_hash: snapshot.site_hash, + active_branch: snapshot.active_branch.to_ascii_lowercase(), + lifecycle: snapshot.lifecycle, + pending_prune: snapshot.pending_prune, + next_prune_attempt_unix: snapshot.next_prune_attempt_unix, + active_site_hashes: snapshot.active_site_hashes, + saved_site_hashes: snapshot.saved_site_hashes, + prune_site_hashes: snapshot.prune_site_hashes, + qdisc_down_major: snapshot.qdisc_down_major, + qdisc_up_major: snapshot.qdisc_up_major, + }); + BusResponse::TreeGuardRuntimeNodeBranch(snapshot) + } BusRequest::ApiReady => { tool_status::api_seen(); BusResponse::Ack diff --git a/src/rust/lqosd/src/node_manager.rs b/src/rust/lqosd/src/node_manager.rs index 72077653..daf80a84 100644 --- a/src/rust/lqosd/src/node_manager.rs +++ b/src/rust/lqosd/src/node_manager.rs @@ -1,5 +1,5 @@ mod auth; -mod local_api; +pub(crate) mod local_api; mod run; mod shaper_queries_actor; mod static_pages; @@ -7,8 +7,10 @@ mod template; mod warnings; pub(crate) mod ws; +pub(crate) use auth::invalidate_auth_cache; pub use local_api::circuit_count::circuit_count_data; pub use local_api::device_counts::device_count; +pub(crate) use local_api::executive_cache::invalidate_executive_cache_snapshot; pub use local_api::flow_explorer::{ FlowTimeline, asn_list_data, country_list_data, country_timeline_data, flow_timeline_data, protocol_list_data, protocol_timeline_data, diff --git a/src/rust/lqosd/src/node_manager/auth.rs b/src/rust/lqosd/src/node_manager/auth.rs index d4edbb1e..d0faac37 100644 --- a/src/rust/lqosd/src/node_manager/auth.rs +++ b/src/rust/lqosd/src/node_manager/auth.rs @@ -1,26 +1,89 @@ //! Provides authentication for the Node Manager. -//! This is designed to be broadly compatible with the original -//! cookie-based system but now uses an Axum layer to be largely -//! invisible. use axum::Json; use axum::http::StatusCode; -use axum::response::{Html, Response}; +use axum::response::{Html, IntoResponse, Redirect, Response}; use axum_extra::extract::CookieJar; -use axum_extra::extract::cookie::Cookie; -use lqos_config::{UserRole, WebUsers, load_config}; +use axum_extra::extract::cookie::{Cookie, SameSite}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use hmac::{Hmac, Mac}; +use lqos_config::{AuthenticatedUser, UserRole, WebUsers, load_config}; use once_cell::sync::Lazy; +use parking_lot::Mutex; +use rand::RngCore; use serde::{Deserialize, Serialize}; -use std::path::Path; +use sha2::Sha256; +use std::fs::OpenOptions; +use std::io::Write; +use std::path::{Path, PathBuf}; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering::Relaxed; use std::time::{SystemTime, UNIX_EPOCH}; -use tokio::sync::Mutex; -use tracing::warn; +use tracing::{error, warn}; -const COOKIE_PATH: &str = "User-Token"; +const COOKIE_NAME: &str = "User-Token"; +const SESSION_TOKEN_VERSION: &str = "v1"; +const SESSION_DURATION_SECS: u64 = 60 * 60 * 24 * 30; +const SESSION_KEY_FILE_NAME: &str = "lqusers.session.key"; -static WEB_USERS: Lazy>> = Lazy::new(|| Mutex::new(None)); +type HmacSha256 = Hmac; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct AuthFileFingerprint { + path: PathBuf, + modified_unix_secs: Option, + modified_subsec_nanos: Option, + len: u64, +} + +#[derive(Clone, Debug)] +struct CachedAuthSnapshot { + fingerprint: Option, + snapshot: AuthSnapshot, +} + +#[derive(Clone, Debug)] +struct AuthSnapshot { + bootstrap_state: AuthBootstrapState, + auth_epoch: u64, + allow_anonymous: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum AuthBootstrapState { + MissingUsersFile, + NoUsersConfigured, + Ready, + CorruptUsersFile, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct SessionClaims { + sub: String, + role: UserRole, + auth_epoch: u64, + iat: u64, + exp: u64, +} + +#[derive(Clone, Debug, Serialize)] +pub struct LoginResponse { + ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + message: Option, +} + +#[derive(Clone, Debug)] +struct SessionUser { + username: String, + role: UserRole, +} + +static AUTH_SNAPSHOT: Lazy>> = Lazy::new(|| Mutex::new(None)); +static SESSION_KEY: Lazy>>> = Lazy::new(|| Mutex::new(None)); pub static FIRST_LOAD: AtomicU64 = AtomicU64::new(0); fn record_first_login_timestamp_if_needed() { @@ -60,15 +123,291 @@ fn record_first_login_timestamp_if_needed() { } } -pub async fn get_username(jar: &CookieJar) -> String { - let lock = WEB_USERS.lock().await; - if let Some(users) = &*lock - && let Some(token) = jar.get(COOKIE_PATH) +fn now_unix_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +fn auth_file_fingerprint(path: &Path) -> Result { + let metadata = std::fs::metadata(path)?; + let modified = metadata + .modified() + .ok() + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()); + + Ok(AuthFileFingerprint { + path: path.to_path_buf(), + modified_unix_secs: modified.map(|d| d.as_secs()), + modified_subsec_nanos: modified.map(|d| d.subsec_nanos()), + len: metadata.len(), + }) +} + +fn auth_snapshot() -> AuthSnapshot { + let current_fingerprint = match WebUsers::existing_path() { + Ok(Some(path)) => match auth_file_fingerprint(&path) { + Ok(fingerprint) => Some(fingerprint), + Err(e) => { + warn!("Unable to stat auth file {:?}: {e}", path); + None + } + }, + Ok(None) => None, + Err(e) => { + warn!("Unable to resolve auth file path: {e}"); + return AuthSnapshot { + bootstrap_state: AuthBootstrapState::CorruptUsersFile, + auth_epoch: 0, + allow_anonymous: false, + }; + } + }; + + let mut cache = AUTH_SNAPSHOT.lock(); + if let Some(cached) = &*cache + && cached.fingerprint == current_fingerprint { - return users.get_username(token.value()); + return cached.snapshot.clone(); } - "Anonymous".to_string() + let snapshot = match current_fingerprint { + None => AuthSnapshot { + bootstrap_state: AuthBootstrapState::MissingUsersFile, + auth_epoch: 0, + allow_anonymous: false, + }, + Some(_) => match WebUsers::load_or_create() { + Ok(users) => AuthSnapshot { + bootstrap_state: if users.is_empty() { + AuthBootstrapState::NoUsersConfigured + } else { + AuthBootstrapState::Ready + }, + auth_epoch: users.auth_epoch(), + allow_anonymous: users.do_we_allow_anonymous(), + }, + Err(e) => { + warn!("Unable to load auth state: {e}"); + AuthSnapshot { + bootstrap_state: AuthBootstrapState::CorruptUsersFile, + auth_epoch: 0, + allow_anonymous: false, + } + } + }, + }; + + let refreshed_fingerprint = match WebUsers::existing_path() { + Ok(Some(path)) => auth_file_fingerprint(&path).ok(), + Ok(None) => None, + Err(e) => { + warn!("Unable to refresh auth file path after load: {e}"); + None + } + }; + + *cache = Some(CachedAuthSnapshot { + fingerprint: refreshed_fingerprint, + snapshot: snapshot.clone(), + }); + snapshot +} + +fn session_key_path() -> Result { + let config = load_config().map_err(|_| { + std::io::Error::other("Unable to load /etc/lqos.conf while locating session key") + })?; + Ok(Path::new(&config.lqos_directory).join(SESSION_KEY_FILE_NAME)) +} + +fn session_key() -> Result, std::io::Error> { + let mut cache = SESSION_KEY.lock(); + if let Some(key) = &*cache { + return Ok(key.clone()); + } + + let path = session_key_path()?; + let key = if path.exists() { + let bytes = std::fs::read(&path)?; + if bytes.is_empty() { + return Err(std::io::Error::other(format!( + "Session key file {:?} is empty", + path + ))); + } + bytes + } else { + let mut new_key = vec![0u8; 32]; + rand::thread_rng().fill_bytes(&mut new_key); + + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&path)?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + } + + file.write_all(&new_key)?; + drop(file); + new_key + }; + + *cache = Some(key.clone()); + Ok(key) +} + +fn build_session_cookie(token: String) -> Cookie<'static> { + let mut cookie = Cookie::new(COOKIE_NAME, token); + cookie.set_path("/"); + cookie.set_same_site(SameSite::Lax); + cookie +} + +fn build_signed_session(key: &[u8], user: &AuthenticatedUser) -> Result { + let now = now_unix_secs(); + let claims = SessionClaims { + sub: user.username.clone(), + role: user.role, + auth_epoch: user.auth_epoch, + iat: now, + exp: now.saturating_add(SESSION_DURATION_SECS), + }; + let payload = serde_json::to_vec(&claims).map_err(|e| { + error!("Unable to serialize session claims: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + let payload_b64 = URL_SAFE_NO_PAD.encode(payload); + + let mut mac = HmacSha256::new_from_slice(key).map_err(|e| { + error!("Unable to initialize session signer: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + mac.update(payload_b64.as_bytes()); + let signature = URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()); + + Ok(format!("{SESSION_TOKEN_VERSION}.{payload_b64}.{signature}")) +} + +fn verify_signed_session( + key: &[u8], + token: &str, + snapshot: &AuthSnapshot, +) -> Result, StatusCode> { + let Some((version, remainder)) = token.split_once('.') else { + return Ok(None); + }; + if version != SESSION_TOKEN_VERSION { + return Ok(None); + } + let Some((payload_b64, signature_b64)) = remainder.rsplit_once('.') else { + return Ok(None); + }; + + let mut mac = HmacSha256::new_from_slice(key).map_err(|e| { + error!("Unable to initialize session verifier: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + mac.update(payload_b64.as_bytes()); + let signature = URL_SAFE_NO_PAD + .decode(signature_b64) + .map_err(|_| StatusCode::UNAUTHORIZED)?; + if mac.verify_slice(&signature).is_err() { + return Ok(None); + } + + let payload = URL_SAFE_NO_PAD + .decode(payload_b64) + .map_err(|_| StatusCode::UNAUTHORIZED)?; + let claims: SessionClaims = + serde_json::from_slice(&payload).map_err(|_| StatusCode::UNAUTHORIZED)?; + + let now = now_unix_secs(); + if claims.exp <= now || claims.auth_epoch != snapshot.auth_epoch { + return Ok(None); + } + + Ok(Some(SessionUser { + username: claims.sub, + role: claims.role, + })) +} + +fn session_from_cookie( + jar: &CookieJar, + snapshot: &AuthSnapshot, +) -> Result, StatusCode> { + let Some(token) = jar.get(COOKIE_NAME) else { + return Ok(None); + }; + let key = session_key().map_err(|e| { + error!("Unable to load session key: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + verify_signed_session(&key, token.value(), snapshot) +} + +fn standalone_page_path(page: &str) -> Result { + let config = load_config().map_err(|e| { + error!("Unable to load config for standalone page {page}: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + Ok(Path::new(&config.lqos_directory) + .join("bin") + .join("static2") + .join(page)) +} + +fn serve_standalone_page(page: &str) -> Result { + let path = standalone_page_path(page)?; + let body = std::fs::read_to_string(&path).map_err(|e| { + error!("Unable to read standalone page {:?}: {e}", path); + StatusCode::INTERNAL_SERVER_ERROR + })?; + Ok(Html(body).into_response()) +} + +pub async fn login_page(jar: CookieJar) -> Response { + let snapshot = auth_snapshot(); + match snapshot.bootstrap_state { + AuthBootstrapState::MissingUsersFile | AuthBootstrapState::NoUsersConfigured => { + Redirect::temporary("/first-run.html").into_response() + } + AuthBootstrapState::Ready => match session_from_cookie(&jar, &snapshot) { + Ok(Some(_)) => Redirect::temporary("/index.html").into_response(), + Ok(None) => serve_standalone_page("login.html") + .unwrap_or_else(|status| (status, "Unable to serve login page").into_response()), + Err(status) => (status, "Unable to validate login session").into_response(), + }, + AuthBootstrapState::CorruptUsersFile => serve_standalone_page("login.html") + .unwrap_or_else(|status| (status, "Unable to serve login page").into_response()), + } +} + +pub async fn first_run_page() -> Response { + let snapshot = auth_snapshot(); + match snapshot.bootstrap_state { + AuthBootstrapState::MissingUsersFile | AuthBootstrapState::NoUsersConfigured => { + serve_standalone_page("first-run.html") + .unwrap_or_else(|status| (status, "Unable to serve first-run page").into_response()) + } + AuthBootstrapState::Ready | AuthBootstrapState::CorruptUsersFile => { + Redirect::temporary("/index.html").into_response() + } + } +} + +pub async fn get_username(jar: &CookieJar) -> String { + let snapshot = auth_snapshot(); + match session_from_cookie(jar, &snapshot) { + Ok(Some(user)) => user.username, + Ok(None) | Err(_) => "Anonymous".to_string(), + } } #[derive(Copy, Clone, Eq, PartialEq)] @@ -78,100 +417,76 @@ pub enum LoginResult { Denied, } -async fn check_login(jar: &CookieJar, users: &WebUsers) -> LoginResult { - if let Some(token) = jar.get(COOKIE_PATH) { - // Validate the token - return match users.get_role_from_token(token.value()) { - Ok(UserRole::ReadOnly) => LoginResult::ReadOnly, - Ok(UserRole::Admin) => LoginResult::Admin, - Err(_e) => LoginResult::Denied, - }; +fn login_result_for_session(user: Option, allow_anonymous: bool) -> LoginResult { + match user { + Some(SessionUser { + role: UserRole::Admin, + .. + }) => LoginResult::Admin, + Some(SessionUser { + role: UserRole::ReadOnly, + .. + }) => LoginResult::ReadOnly, + None if allow_anonymous => LoginResult::ReadOnly, + None => LoginResult::Denied, } - LoginResult::Denied } -/// Checks an incoming request for a User-Token cookie. If found, -/// it validates the request against the web users file. If the -/// web users file isn't found, it redirects to 'first run'. If -/// it is found, the token is checked (and redirected to login if -/// it isn't good). Finally, the user's role is injected into -/// the middleware. +/// Checks an incoming request for a `User-Token` cookie. If found, +/// it validates the request against the signed session and current auth epoch. +/// Missing or empty auth state redirects to first-run; invalid sessions redirect +/// to login unless anonymous read-only is enabled. pub async fn auth_layer( jar: CookieJar, mut req: axum::extract::Request, next: axum::middleware::Next, -) -> Result> { - const BOUNCE: &str = - ""; - const FIRST_RUN: &str = - ""; - - let mut lock = WEB_USERS.lock().await; - if lock.is_none() { - // No lock - let's see if there's a file to use? - if WebUsers::does_users_file_exist().expect("Error checking if the users file exists") { - // It exists - we load it - let users = WebUsers::load_or_create().expect("Unable to load users file"); - *lock = Some(users); - } else { - // No users file - redirect to first run - return Err(Html(FIRST_RUN)); +) -> Response { + let snapshot = auth_snapshot(); + match snapshot.bootstrap_state { + AuthBootstrapState::MissingUsersFile | AuthBootstrapState::NoUsersConfigured => { + return Redirect::temporary("/first-run.html").into_response(); } + AuthBootstrapState::CorruptUsersFile => { + return Redirect::temporary("/login.html").into_response(); + } + AuthBootstrapState::Ready => {} } - if let Some(users) = &*lock { - let login_result = check_login(&jar, users).await; - return match login_result { - LoginResult::Admin | LoginResult::ReadOnly => { - record_first_login_timestamp_if_needed(); - req.extensions_mut().insert(login_result); - Ok(next.run(req).await) - } - LoginResult::Denied => { - let users = WebUsers::load_or_create().expect("Could not load users file"); - if users.do_we_allow_anonymous() { - req.extensions_mut().insert(LoginResult::ReadOnly); - Ok(next.run(req).await) - } else { - Err(Html(BOUNCE)) - } - } - }; + let login_result = match session_from_cookie(&jar, &snapshot) { + Ok(user) => login_result_for_session(user, snapshot.allow_anonymous), + Err(status) => return (status, "Unable to validate session").into_response(), + }; + + match login_result { + LoginResult::Admin | LoginResult::ReadOnly => { + record_first_login_timestamp_if_needed(); + req.extensions_mut().insert(login_result); + next.run(req).await + } + LoginResult::Denied => Redirect::temporary("/login.html").into_response(), } - Err(Html(BOUNCE)) } pub async fn login_from_token(token: &str) -> LoginResult { - let mut lock = WEB_USERS.lock().await; - if lock.is_none() { - match WebUsers::does_users_file_exist() { - Ok(true) => match WebUsers::load_or_create() { - Ok(users) => { - *lock = Some(users); - } - Err(e) => { - warn!("Unable to load users file for websocket auth: {e}"); - return LoginResult::Denied; - } - }, - Ok(false) => { - return LoginResult::Denied; - } - Err(e) => { - warn!("Unable to check users file for websocket auth: {e}"); - return LoginResult::Denied; - } - } + let snapshot = auth_snapshot(); + if snapshot.bootstrap_state != AuthBootstrapState::Ready { + return LoginResult::Denied; } - let Some(users) = &*lock else { - return LoginResult::Denied; + let key = match session_key() { + Ok(key) => key, + Err(e) => { + warn!("Unable to load session key for websocket auth: {e}"); + return LoginResult::Denied; + } }; - let login_result = match users.get_role_from_token(token) { - Ok(UserRole::ReadOnly) => LoginResult::ReadOnly, - Ok(UserRole::Admin) => LoginResult::Admin, - Err(_) => LoginResult::Denied, + let login_result = match verify_signed_session(&key, token, &snapshot) { + Ok(user) => login_result_for_session(user, snapshot.allow_anonymous), + Err(e) => { + warn!("Unable to verify websocket session token: {e}"); + LoginResult::Denied + } }; if login_result != LoginResult::Denied { @@ -181,25 +496,15 @@ pub async fn login_from_token(token: &str) -> LoginResult { login_result } -/// Reload the cached users from disk after user-management changes. +/// Invalidate the cached auth snapshot after user-management changes. +pub fn invalidate_auth_cache() { + let mut lock = AUTH_SNAPSHOT.lock(); + *lock = None; +} + +/// Reload the cached auth snapshot after user-management changes. pub async fn refresh_cached_users() { - let mut lock = WEB_USERS.lock().await; - match WebUsers::does_users_file_exist() { - Ok(true) => match WebUsers::load_or_create() { - Ok(users) => { - *lock = Some(users); - } - Err(e) => { - warn!("Unable to refresh users cache: {e}"); - } - }, - Ok(false) => { - *lock = None; - } - Err(e) => { - warn!("Unable to check users file while refreshing cache: {e}"); - } - } + invalidate_auth_cache(); } #[derive(Serialize, Deserialize)] @@ -211,25 +516,88 @@ pub struct LoginAttempt { pub async fn try_login( jar: CookieJar, Json(login): Json, -) -> Result<(CookieJar, StatusCode), StatusCode> { - let users = WEB_USERS.lock().await; - if let Some(users) = &*users { - return match users.login(&login.username, &login.password) { - Ok(token) => { - record_first_login_timestamp_if_needed(); - Ok((jar.add(Cookie::new(COOKIE_PATH, token)), StatusCode::OK)) - } - Err(..) => { - if users.do_we_allow_anonymous() { - record_first_login_timestamp_if_needed(); - Ok((jar, StatusCode::OK)) - } else { - Err(StatusCode::UNAUTHORIZED) - } - } - }; +) -> Result<(CookieJar, Json), (StatusCode, Json)> { + let snapshot = auth_snapshot(); + match snapshot.bootstrap_state { + AuthBootstrapState::MissingUsersFile | AuthBootstrapState::NoUsersConfigured => { + return Err(( + StatusCode::CONFLICT, + Json(LoginResponse { + ok: false, + reason: Some("first_run_required"), + message: Some("No users are configured yet.".to_string()), + }), + )); + } + AuthBootstrapState::CorruptUsersFile => { + return Err(( + StatusCode::CONFLICT, + Json(LoginResponse { + ok: false, + reason: Some("auth_corrupt"), + message: Some("The auth file is corrupt and must be repaired.".to_string()), + }), + )); + } + AuthBootstrapState::Ready => {} } - Err(StatusCode::UNAUTHORIZED) + + let mut users = WebUsers::load_or_create().map_err(|e| { + warn!("Unable to load users during login: {e}"); + ( + StatusCode::CONFLICT, + Json(LoginResponse { + ok: false, + reason: Some("auth_corrupt"), + message: Some("The auth file is corrupt and must be repaired.".to_string()), + }), + ) + })?; + let authenticated = users + .authenticate(&login.username, &login.password) + .map_err(|_| { + ( + StatusCode::UNAUTHORIZED, + Json(LoginResponse { + ok: false, + reason: Some("invalid_credentials"), + message: Some("Invalid username or password.".to_string()), + }), + ) + })?; + + invalidate_auth_cache(); + let key = session_key().map_err(|e| { + error!("Unable to load session key during login: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(LoginResponse { + ok: false, + reason: Some("session_error"), + message: Some("Unable to create session token.".to_string()), + }), + ) + })?; + let token = build_signed_session(&key, &authenticated).map_err(|status| { + ( + status, + Json(LoginResponse { + ok: false, + reason: Some("session_error"), + message: Some("Unable to create session token.".to_string()), + }), + ) + })?; + + record_first_login_timestamp_if_needed(); + Ok(( + jar.add(build_session_cookie(token)), + Json(LoginResponse { + ok: true, + reason: None, + message: None, + }), + )) } #[derive(Serialize, Deserialize)] @@ -242,16 +610,106 @@ pub struct FirstUser { pub async fn first_user( jar: CookieJar, Json(new_user): Json, -) -> (CookieJar, StatusCode) { - let mut users = WebUsers::load_or_create().expect("Could not load users file"); +) -> Result<(CookieJar, Json), (StatusCode, Json)> { + let snapshot = auth_snapshot(); + match snapshot.bootstrap_state { + AuthBootstrapState::Ready => { + return Err(( + StatusCode::CONFLICT, + Json(LoginResponse { + ok: false, + reason: Some("already_configured"), + message: Some("Web authentication is already configured.".to_string()), + }), + )); + } + AuthBootstrapState::CorruptUsersFile => { + return Err(( + StatusCode::CONFLICT, + Json(LoginResponse { + ok: false, + reason: Some("auth_corrupt"), + message: Some("The auth file is corrupt and must be repaired.".to_string()), + }), + )); + } + AuthBootstrapState::MissingUsersFile | AuthBootstrapState::NoUsersConfigured => {} + } + + let mut users = WebUsers::load_or_create().map_err(|e| { + warn!("Unable to load users during first-run setup: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(LoginResponse { + ok: false, + reason: Some("auth_corrupt"), + message: Some("Unable to initialize auth storage.".to_string()), + }), + ) + })?; users .allow_anonymous(new_user.allow_anonymous) - .expect("Unable to set property"); - let token = users + .map_err(|e| { + warn!("Unable to set anonymous auth policy: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(LoginResponse { + ok: false, + reason: Some("auth_corrupt"), + message: Some("Unable to update auth settings.".to_string()), + }), + ) + })?; + users .add_or_update_user(&new_user.username, &new_user.password, UserRole::Admin) - .expect("Unable to add or update user"); - let mut lock = WEB_USERS.lock().await; - *lock = Some(users); + .map_err(|e| { + warn!("Unable to create first user: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(LoginResponse { + ok: false, + reason: Some("auth_corrupt"), + message: Some("Unable to create the first user.".to_string()), + }), + ) + })?; + + invalidate_auth_cache(); + let authenticated = AuthenticatedUser { + username: new_user.username, + role: UserRole::Admin, + auth_epoch: users.auth_epoch(), + password_upgraded: false, + }; + let key = session_key().map_err(|e| { + error!("Unable to load session key during first-run setup: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(LoginResponse { + ok: false, + reason: Some("session_error"), + message: Some("Unable to create session token.".to_string()), + }), + ) + })?; + let token = build_signed_session(&key, &authenticated).map_err(|status| { + ( + status, + Json(LoginResponse { + ok: false, + reason: Some("session_error"), + message: Some("Unable to create session token.".to_string()), + }), + ) + })?; + record_first_login_timestamp_if_needed(); - (jar.add(Cookie::new(COOKIE_PATH, token)), StatusCode::OK) + Ok(( + jar.add(build_session_cookie(token)), + Json(LoginResponse { + ok: true, + reason: None, + message: None, + }), + )) } diff --git a/src/rust/lqosd/src/node_manager/js_build/entrypoints.txt b/src/rust/lqosd/src/node_manager/js_build/entrypoints.txt index b8d41373..2cb88538 100644 --- a/src/rust/lqosd/src/node_manager/js_build/entrypoints.txt +++ b/src/rust/lqosd/src/node_manager/js_build/entrypoints.txt @@ -8,11 +8,13 @@ help.js unknown-ips.js configuration.js circuit.js +ethernet_caps.js site_map.js flow_map.js all_tree_sankey.js asn_analysis.js lts_trial.js +lts_trial_setup.js config_general.js config_rtt.js config_tuning.js diff --git a/src/rust/lqosd/src/node_manager/js_build/src/all_tree_sankey.js b/src/rust/lqosd/src/node_manager/js_build/src/all_tree_sankey.js index 3834221a..fe53910f 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/all_tree_sankey.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/all_tree_sankey.js @@ -38,6 +38,15 @@ const MIN_LINK_BITS_PER_SEC = 1_000_000; let sankeyOverlay = null; +function setRootNodeLabel(name) { + const target = document.getElementById("rootNode"); + if (!target) { + return; + } + target.textContent = name || "Root"; + target.classList.add("redactable"); +} + function listenOnceWithTimeout(eventName, timeoutMs, handler, onTimeout) { let done = false; const wrapped = (msg) => { @@ -126,6 +135,157 @@ class AllTreeSankeyGraph extends GenericRingBuffer { this.pending = false; } + render(graph) { + const head = this.getHead(); + if (!Array.isArray(head) || head.length === 0) { + setRootNodeLabel("Root"); + if (sankeyOverlay) { + sankeyOverlay.show("Limited throughput", "Nothing to render yet."); + } + graph.update([], []); + return; + } + + if (!head[rootId] || !head[rootId][1]) { + rootId = 0; + } + + const rootEntry = head[rootId]; + if (!rootEntry || !rootEntry[1]) { + setRootNodeLabel("Root"); + if (sankeyOverlay) { + sankeyOverlay.show("Limited throughput", "Nothing to render yet."); + } + graph.update([], []); + return; + } + + allNodes = head; + setRootNodeLabel(rootEntry[1].name); + + let redact = isRedacted(); + let nodes = []; + let links = []; + const linkMap = new Map(); + const rootParents = rootEntry[1].parents || []; + let startDepth = Math.max(0, rootParents.length - 1); + + for (let i=0; i maxDepth) { + continue; + } + if (rootId !== 0 && i !== rootId && !parents.includes(rootId)) { + continue; + } + + const downBytesPerSec = toNumber(head[i][1].current_throughput?.[0], 0); + const downBitsPerSec = downBytesPerSec * 8; + const maxBitsPerSec = toNumber(head[i][1].max_throughput?.[0], 0) * 1_000_000; + const percent = Math.min(100, maxBitsPerSec > 0 ? (downBitsPerSec / maxBitsPerSec) * 100 : 0); + + let capacityColor = isColorBlindMode() + ? lerpViridis(percent / 100) + : lerpGreenToRedViaOrange(100 - percent, 100); + + let label = { + fontSize: 10, + color: "#999", + formatter: (params) => { + return trimStringWithElipsis(params.name.replace("(Generated Site) ", ""), 14); + } + }; + if (redact) { + label.fontFamily = "Illegible"; + } + + nodes.push({ + name: head[i][1].name, + label: label, + itemStyle: { + color: capacityColor + }, + n: 1, + }); + + if (i > 0) { + let immediateParent = head[i][1].immediate_parent; + if (immediateParent === null || immediateParent === undefined) continue; + if (!head[immediateParent] || !head[immediateParent][1]) continue; + links.push({ + source: head[immediateParent][1].name, + target: head[i][1].name, + value: downBitsPerSec, + lineStyle: { + color: capacityColor, + }, + maxBitsPerSec: maxBitsPerSec, + n: 1, + }); + linkMap.set( + `${head[immediateParent][1].name}\u0000${head[i][1].name}`, + links[links.length - 1], + ); + } + } + + this.iterate((data) => { + for (let i=0; i 0) { + let immediateParent = data[i][1].immediate_parent; + if (immediateParent === null || immediateParent === undefined) continue; + if (!data[immediateParent] || !data[immediateParent][1]) continue; + + const link = linkMap.get( + `${data[immediateParent][1].name}\u0000${data[i][1].name}`, + ); + if (link !== undefined) { + link.value += toNumber(data[i][1].current_throughput?.[0], 0) * 8; + link.n++; + } + } + } + }); + + for (let i=0; i 0 ? (links[i].value / maxBits) * 100 : 0); + let capacityColor = isColorBlindMode() + ? lerpViridis(percent / 100) + : lerpGreenToRedViaOrange(100 - percent, 100); + links[i].lineStyle.color = capacityColor; + } + + links = links.filter(link => link.value >= MIN_LINK_BITS_PER_SEC); + + if (links.length === 0) { + if (sankeyOverlay) { + sankeyOverlay.show("Limited throughput", "Nothing to render yet."); + } + graph.update([], []); + return; + } + + if (sankeyOverlay) { + sankeyOverlay.hide(); + } + + const referenced = new Set(); + links.forEach(link => { + referenced.add(link.source); + referenced.add(link.target); + }); + + referenced.add(rootEntry[1].name); + nodes = nodes.filter(node => referenced.has(node.name)); + graph.update(nodes, links); + } + onTick(graph) { if (this.pending) { return; @@ -133,7 +293,7 @@ class AllTreeSankeyGraph extends GenericRingBuffer { this.pending = true; const self = this; - this.pendingCancel = listenOnceWithTimeout("NetworkTree", REQUEST_TIMEOUT_MS, (msg) => { + this.pendingCancel = listenOnceWithTimeout("NetworkTreeLite", REQUEST_TIMEOUT_MS, (msg) => { self.pending = false; self.pendingCancel = null; @@ -142,162 +302,8 @@ class AllTreeSankeyGraph extends GenericRingBuffer { } const data = msg && msg.data ? msg.data : []; - // Maintain a 10-second ringbuffer of recent data self.push(data); - - let redact = isRedacted(); - let nodes = []; - let links = []; - const linkMap = new Map(); - - // Build the basic tree from the current head, to ensure - // that we're displaying the most recent nodes. - let head = self.getHead(); - if (!Array.isArray(head) || head.length === 0 || !head[rootId] || !head[rootId][1]) { - if (sankeyOverlay) { - sankeyOverlay.show("Limited throughput", "Nothing to render yet."); - } - graph.update([], []); - return; - } - - allNodes = head; - - const rootParents = head[rootId][1].parents || []; - let startDepth = rootParents.length - 1; - - for (let i=0; i maxDepth) { - continue; - } - // If head[i][1].parents does not contain rootId, skip - if (rootId !== 0 && !parents.includes(rootId)) { - continue; - } - - const downBytesPerSec = toNumber(head[i][1].current_throughput?.[0], 0); - const downBitsPerSec = downBytesPerSec * 8; - const maxBitsPerSec = toNumber(head[i][1].max_throughput?.[0], 0) * 1_000_000; - const percent = Math.min(100, maxBitsPerSec > 0 ? (downBitsPerSec / maxBitsPerSec) * 100 : 0); - - // Use appropriate color scale based on color blind mode - let capacityColor = isColorBlindMode() - ? lerpViridis(percent / 100) - : lerpGreenToRedViaOrange(100 - percent, 100); - - // Use appropriate color scale for node - let color = capacityColor; - - let label = { - fontSize: 10, - color: "#999", - formatter: (params) => { - // Trim to 10 chars with elipsis - return trimStringWithElipsis(params.name.replace("(Generated Site) ", ""), 14); - } - }; - if (redact) { - label.fontFamily = "Illegible"; - } - - nodes.push({ - name: head[i][1].name, - label: label, - itemStyle: { - color: color - }, - n: 1, - }); - - if (i > 0) { - let immediateParent = head[i][1].immediate_parent; - if (immediateParent === null || immediateParent === undefined) continue; - if (!head[immediateParent] || !head[immediateParent][1]) continue; - links.push({ - source: head[immediateParent][1].name, - target: head[i][1].name, - value: downBitsPerSec, - lineStyle: { - color: capacityColor, - }, - maxBitsPerSec: maxBitsPerSec, - n: 1, - }); - linkMap.set( - `${head[immediateParent][1].name}\u0000${head[i][1].name}`, - links[links.length - 1], - ); - } - } - - // Now we iterate over the entire ringbuffer to accumulate data over the period - // of the ringbuffer. - self.iterate((data) => { - for (let i=0; i 0) { - let immediateParent = data[i][1].immediate_parent; - if (immediateParent === null || immediateParent === undefined) continue; - if (!data[immediateParent] || !data[immediateParent][1]) continue; - - const link = linkMap.get( - `${data[immediateParent][1].name}\u0000${data[i][1].name}`, - ); - if (link !== undefined) { - link.value += toNumber(data[i][1].current_throughput?.[0], 0) * 8; - link.n++; - } - } - } - }); - - // Now go through the links and average the values, recalculating the color - for (let i=0; i 0 ? (links[i].value / maxBits) * 100 : 0); - let capacityColor = isColorBlindMode() - ? lerpViridis(percent / 100) - : lerpGreenToRedViaOrange(100 - percent, 100); - links[i].lineStyle.color = capacityColor; - } - - // Filter links with <1 Mbps average throughput - links = links.filter(link => link.value >= MIN_LINK_BITS_PER_SEC); - - if (links.length === 0) { - if (sankeyOverlay) { - sankeyOverlay.show("Limited throughput", "Nothing to render yet."); - } - graph.update([], []); - return; - } - - if (sankeyOverlay) { - sankeyOverlay.hide(); - } - - // Collect node names that are still referenced by links - const referenced = new Set(); - links.forEach(link => { - referenced.add(link.source); - referenced.add(link.target); - }); - - // Always keep the root node - let rootName = nodes.length > 0 ? nodes[0].name : null; - if (rootName) referenced.add(rootName); - - // Filter nodes to only those referenced - nodes = nodes.filter(node => referenced.has(node.name)); - - // Update the graph - graph.update(nodes, links); + self.render(graph); }, () => { self.pending = false; self.pendingCancel = null; @@ -306,12 +312,16 @@ class AllTreeSankeyGraph extends GenericRingBuffer { return; } if (sankeyOverlay) { - sankeyOverlay.show("Waiting for data", "No NetworkTree websocket response received yet."); + sankeyOverlay.show("Waiting for data", "No NetworkTreeLite websocket response received yet."); } graph.update([], []); }); - wsClient.send({ NetworkTree: {} }); + wsClient.send({ NetworkTreeLite: {} }); + } + + rerender(graph) { + this.render(graph); } } @@ -349,17 +359,18 @@ class AllTreeSankey extends DashboardGraph { this.option && this.chart.setOption(this.option); this.chart.showLoading(); this.chart.on('click', (params) => { - console.log(params.name); - console.log(this.nodeMap); let name = params.name; - // If it contains a >, it's a link if (name.indexOf(" > ") === -1) { rootId = idOfNode(name); } else { rootId = idOfNode(params.data.source); } + this.model.rerender(this); + }); + $("#btnRoot").click(() => { + rootId = 0; + this.model.rerender(this); }); - $("#btnRoot").click(() => { rootId = 0; }); } update(data, links) { @@ -394,15 +405,16 @@ function getMaxDepth() { function bindMaxDepth() { let d = document.getElementById("maxDepth"); d.value = maxDepth; - d.onclick = () => { + d.addEventListener("change", () => { maxDepth = parseInt(d.value); localStorage.setItem("atsDepth", maxDepth.toString()); - }; + graph.model.rerender(graph); + }); } let maxDepth = getMaxDepth(); -bindMaxDepth(); let graph = new AllTreeSankey("sankey"); +bindMaxDepth(); sankeyOverlay = makeOverlay(graph.dom, "allTreeSankeyOverlay"); sankeyOverlay.show("Waiting for data", "Requesting network tree..."); let loopTimer = null; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/asn_analysis.js b/src/rust/lqosd/src/node_manager/js_build/src/asn_analysis.js index 24de678b..4a83d42e 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/asn_analysis.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/asn_analysis.js @@ -3,7 +3,12 @@ import {colorByQoqScore, colorByRetransmitPct, colorByRttMs} from "./helpers/col import {isDarkMode} from "./helpers/dark_mode"; import {openFlowRttExcludeWizard} from "./lq_js_common/helpers/flow_rtt_exclude_wizard"; import {scaleNanos, scaleNumber, toNumber} from "./lq_js_common/helpers/scaling"; -import {listenExecutiveHeatmaps, medianFromBlocks, sumBlocks} from "./executive_utils"; +import { + medianFromBlocks, + pollExecutiveHeatmapPage, + pollExecutiveLeaderboardPage, + sumBlocks, +} from "./executive_utils"; import {get_ws_client} from "./pubsub/ws"; const wsClient = get_ws_client(); @@ -11,7 +16,8 @@ const EVIDENCE_ITEMS_PER_PAGE = 10; const TOP_LABEL_COUNT = 4; const TOP_ROW_LIMIT = 20; const HEATMAP_BLOCK_SECONDS = 60; -const LIVE_INTERVAL_TEXT = "Heatmaps auto-refresh about every second"; +const EXECUTIVE_PAGE_SIZE = 250; +const LIVE_INTERVAL_TEXT = "Executive data auto-refreshes about every 5 seconds"; const NEUTRAL_CHART_COLOR = "#94a3b8"; const RTT_THRESHOLDS = (() => { const configured = window.rttThresholds || window.rtt_thresholds || window.config?.rtt_thresholds || {}; @@ -22,7 +28,10 @@ const RTT_THRESHOLDS = (() => { })(); const state = { - executiveHeatmaps: null, + executiveTopAsns: null, + executiveDownloadHeatmap: null, + executiveRttHeatmap: null, + executiveRetransmitHeatmap: null, executiveRows: [], selectedAsn: null, asnList: [], @@ -43,11 +52,12 @@ const state = { retransmitChartTheme: null, themeObserver: null, activeEvidenceRequestToken: 0, + executivePollHandles: [], }; const SORT_OPTIONS = { - start: (a, b) => b.start - a.start, - duration: (a, b) => b.duration_nanos - a.duration_nanos, + start: (a, b) => toNumber(b?.start, 0) - toNumber(a?.start, 0), + duration: (a, b) => toNumber(b?.duration_nanos, 0) - toNumber(a?.duration_nanos, 0), bytes: (a, b) => totalFlowBytes(b) - totalFlowBytes(a), }; @@ -183,26 +193,60 @@ function arrayMedian(values) { } function buildExecutiveRows() { - const heatmaps = state.executiveHeatmaps?.asns || []; + const topAsns = (state.executiveTopAsns?.rows || []) + .filter((row) => row?.kind === "TopAsnByTraffic"); + const downloadRows = (state.executiveDownloadHeatmap?.rows || []); + const rttRows = (state.executiveRttHeatmap?.rows || []); + const retransmitRows = (state.executiveRetransmitHeatmap?.rows || []); + const downloadByAsn = new Map(downloadRows + .map((row) => [toNumber(row.asn, null), row])); + const rttByAsn = new Map(rttRows + .map((row) => [toNumber(row.asn, null), row])); + const retransmitByAsn = new Map(retransmitRows + .map((row) => [toNumber(row.asn, null), row])); const countMap = new Map( (state.asnList || []).map((entry) => [entry.asn, toNumber(entry.count, 0)]), ); + const allAsns = new Set(); + downloadRows.forEach((row) => allAsns.add(toNumber(row?.asn, null))); + rttRows.forEach((row) => allAsns.add(toNumber(row?.asn, null))); + retransmitRows.forEach((row) => allAsns.add(toNumber(row?.asn, null))); + topAsns.forEach((row) => allAsns.add(toNumber(row?.asn, null))); - const baseRows = heatmaps.map((asn) => { - const downloadBlocks = asn?.blocks?.download || []; - const uploadBlocks = asn?.blocks?.upload || []; - const rttBlocks = asn?.blocks?.rtt || []; - const retransmitBlocks = asn?.blocks?.retransmit || []; + const baseRows = [...allAsns] + .filter((asnNumber) => Number.isFinite(asnNumber) && asnNumber > 0) + .map((asnNumber) => { + const topAsn = topAsns.find((row) => toNumber(row?.asn, 0) === asnNumber) || null; + const downloadRow = downloadByAsn.get(asnNumber); + const rttRow = rttByAsn.get(asnNumber); + const retransmitRow = retransmitByAsn.get(asnNumber); + const downloadBlocks = downloadRow?.split_blocks?.download || []; + const uploadBlocks = downloadRow?.split_blocks?.upload || []; + const rttBlocks = rttRow?.rtt_blocks?.rtt || []; + const retransmitBlocks = retransmitRow?.scalar_blocks?.values || []; const totalMbpsMinutes = sumBlocks(downloadBlocks) + sumBlocks(uploadBlocks); - const avgMbps = totalMbpsMinutes / Math.max(1, Math.max(downloadBlocks.length, uploadBlocks.length, 15)); - const totalBytes = ((totalMbpsMinutes * 1_000_000) / 8) * HEATMAP_BLOCK_SECONDS; + const totalBytes = toNumber(topAsn?.total_bytes_15m, 0) + || ((totalMbpsMinutes * 1_000_000) / 8) * HEATMAP_BLOCK_SECONDS; + const avgMbps = totalMbpsMinutes > 0 + ? totalMbpsMinutes / Math.max(1, Math.max(downloadBlocks.length, uploadBlocks.length, 15)) + : (totalBytes * 8) / (HEATMAP_BLOCK_SECONDS * 15 * 1_000_000); const medianRtt = medianFromBlocks(rttBlocks); const medianRetrans = medianFromBlocks(retransmitBlocks); - const flowCount = countMap.get(asn.asn) || 0; + const flowCount = countMap.get(asnNumber) || 0; + const asnName = topAsn?.asn_name + || downloadRow?.label + || rttRow?.label + || retransmitRow?.label + || `ASN ${asnNumber}`; return { - asn: asn.asn, - asnName: asn.asn_name || `ASN ${asn.asn}`, - heatmap: asn, + asn: asnNumber, + asnName, + heatmap: { + download: downloadBlocks, + upload: uploadBlocks, + rtt: rttBlocks, + retransmit: retransmitBlocks, + }, totalBytes, avgMbps, flowCount, @@ -655,7 +699,7 @@ function requestLists() { function requestAsnList() { listenOnce("AsnList", (msg) => { - state.asnList = (msg?.data || []).slice().sort((a, b) => b.count - a.count); + state.asnList = (msg?.data || []).slice().sort((a, b) => toNumber(b?.count, 0) - toNumber(a?.count, 0)); state.listCountsByAsn = new Map(state.asnList.map((row) => [row.asn, toNumber(row.count, 0)])); buildExecutiveRows(); renderAll(); @@ -700,7 +744,7 @@ function renderDropdown({ targetId, buttonText, items, emptyText, itemRenderer } function requestCountryList() { listenOnce("CountryList", (msg) => { - state.countryList = (msg?.data || []).slice().sort((a, b) => b.count - a.count); + state.countryList = (msg?.data || []).slice().sort((a, b) => toNumber(b?.count, 0) - toNumber(a?.count, 0)); renderDropdown({ targetId: "asnAnalysisCountryControl", buttonText: "Country", @@ -723,7 +767,7 @@ function requestCountryList() { function requestProtocolList() { listenOnce("ProtocolList", (msg) => { - state.protocolList = (msg?.data || []).slice().sort((a, b) => b.count - a.count); + state.protocolList = (msg?.data || []).slice().sort((a, b) => toNumber(b?.count, 0) - toNumber(a?.count, 0)); renderDropdown({ targetId: "asnAnalysisProtocolControl", buttonText: "Protocol", @@ -1034,6 +1078,7 @@ function wireControls() { refreshButton?.addEventListener("click", () => { requestLists(); + state.executivePollHandles.forEach((handle) => handle?.refresh?.()); const top = sortedLeaderboardRows()[0]; if (top) { setSelectedAsn(top.asn); @@ -1083,14 +1128,46 @@ function observeThemeAndResize() { } function initExecutiveFeed() { - listenExecutiveHeatmaps((data) => { - state.executiveHeatmaps = data || {}; - state.lastExecutiveUpdate = new Date(); + const syncExecutiveState = (key, data) => { + state[key] = data || null; + state.lastExecutiveUpdate = new Date(toNumber(data?.generated_at_unix_ms, Date.now())); buildExecutiveRows(); chooseDefaultSelection(); renderAll(); updateLiveClock(); - }); + }; + + state.executivePollHandles = [ + pollExecutiveLeaderboardPage({ + kind: "TopAsnsByTraffic", + page: 0, + page_size: EXECUTIVE_PAGE_SIZE, + }, (data) => syncExecutiveState("executiveTopAsns", data)), + pollExecutiveHeatmapPage({ + metric: "Download", + entity_kinds: ["Asn"], + page: 0, + page_size: EXECUTIVE_PAGE_SIZE, + sort: "Label", + descending: false, + }, (data) => syncExecutiveState("executiveDownloadHeatmap", data)), + pollExecutiveHeatmapPage({ + metric: "Rtt", + entity_kinds: ["Asn"], + page: 0, + page_size: EXECUTIVE_PAGE_SIZE, + sort: "Label", + descending: false, + }, (data) => syncExecutiveState("executiveRttHeatmap", data)), + pollExecutiveHeatmapPage({ + metric: "Retransmit", + entity_kinds: ["Asn"], + page: 0, + page_size: EXECUTIVE_PAGE_SIZE, + sort: "Label", + descending: false, + }, (data) => syncExecutiveState("executiveRetransmitHeatmap", data)), + ]; } function init() { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/circuit.js b/src/rust/lqosd/src/node_manager/js_build/src/circuit.js index 3cbbf1bf..6163b718 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/circuit.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/circuit.js @@ -1,7 +1,14 @@ // Obtain URL parameters import {DirectChannel} from "./pubsub/direct_channels"; import {clearDiv, formatLastSeen, simpleRow, simpleRowHtml, theading} from "./helpers/builders"; -import {formatRetransmit, formatRtt, formatThroughput, lerpGreenToRedViaOrange, formatMbps} from "./helpers/scaling"; +import { + formatRetransmitFraction, + formatRtt, + formatThroughput, + lerpGreenToRedViaOrange, + formatMbps, + retransmitFractionFromSample, +} from "./helpers/scaling"; import {colorByQoqScore, colorByRttMs} from "./helpers/color_scales"; import {BitsPerSecondGauge} from "./graphs/bits_gauge"; import {QooScoreGauge} from "./graphs/qoo_score_gauge"; @@ -32,7 +39,6 @@ let plan = null; let channelLink = null; let cakeChannel = null; let pinger = null; -let flowChannel = null; let funnelSubscription = null; let speedometer = null; let qooGauge = null; @@ -49,17 +55,29 @@ let funnelParentNodeName = null; let excludeRttToggle = null; let excludeRttLastValue = false; let excludeRttBusy = false; -let latestFlowMsg = null; let latestCakeMsg = null; let cakeGraphs = null; let cakeQueueUnavailable = false; +let circuitSqmOverride = ""; let queuingActivityGraph = null; let latestCircuitDevices = []; +let circuitConfigDevices = []; +let latestCircuitSummary = null; let latestCircuitQooScore = null; +let latestSankeyFlowMsg = { flows: [] }; +let latestTopAsnData = { total_asns: 0, rows: [] }; +let latestTrafficPage = null; let queuingActivityDirection = "down"; let deviceGraphSpecs = []; let deviceGraphsInitialized = false; -const QUEUING_ACTIVITY_RTT_FLOOR_BPS = 200_000; +let devicePollTimer = null; +let deviceRequestInFlight = false; +let sankeyPollTimer = null; +let sankeyRequestInFlight = false; +let topAsnPollTimer = null; +let topAsnRequestInFlight = false; +let trafficPollTimer = null; +let trafficRequestInFlight = false; const DEFAULT_RTT_THRESHOLDS = { green_ms: 0, yellow_ms: 100, red_ms: 200 }; let currentRttThresholds = { ...DEFAULT_RTT_THRESHOLDS }; const wsClient = get_ws_client(); @@ -67,6 +85,79 @@ const RECENT_TRAFFIC_FLOW_WINDOW_NANOS = 30_000_000_000; const TRAFFIC_FLOW_HIDE_THRESHOLD_BPS = 1024 * 1024; const DEFAULT_TRAFFIC_PAGE_SIZE = 100; +function formatEthernetPortLabel(mbps) { + const value = toNumber(mbps, 0); + if (value >= 1000 && value % 1000 === 0) { + return `${value / 1000}G`; + } + if (value >= 1000) { + return `${(value / 1000).toFixed(1)}G`; + } + return `${Math.round(value)}M`; +} + +function ethernetCapsPageHref(advisory) { + const portLabel = encodeURIComponent(formatEthernetPortLabel(advisory?.negotiated_ethernet_mbps)); + return `/ethernet_caps.html?tier=${portLabel}`; +} + +function ethernetTooltipHtml(advisory) { + return ( + `Requested plan ${formatMbps(advisory.requested_download_mbps)} / ${formatMbps(advisory.requested_upload_mbps)} exceeded detected Ethernet speed. ` + + `Shaping auto-capped to ${formatMbps(advisory.applied_download_mbps)} / ${formatMbps(advisory.applied_upload_mbps)}.` + ); +} + +function formatPlanSpeedPair(downloadMbps, uploadMbps) { + const down = toNumber(downloadMbps, 0).toFixed(1); + const up = toNumber(uploadMbps, 0).toFixed(1); + return `${down} / ${up} Mbps`; +} + +function renderEthernetAdvisory(advisory) { + const badge = document.getElementById("ethernetAdvisoryBadge"); + const badgeText = document.getElementById("ethernetAdvisoryBadgeText"); + if (!badge || !badgeText) { + return; + } + + const disposeTooltip = () => { + if (typeof bootstrap === "undefined" || !bootstrap.Tooltip) { + return; + } + const existing = bootstrap.Tooltip.getInstance?.(badge); + existing?.dispose(); + }; + + if (!advisory?.auto_capped) { + badge.classList.add("d-none"); + badgeText.textContent = ""; + badge.removeAttribute("title"); + badge.removeAttribute("data-bs-original-title"); + disposeTooltip(); + return; + } + + const portLabel = formatEthernetPortLabel(advisory.negotiated_ethernet_mbps); + const note = ethernetTooltipHtml(advisory); + + badgeText.textContent = portLabel; + badge.classList.remove("d-none"); + badge.href = ethernetCapsPageHref(advisory); + badge.setAttribute("title", note); + badge.setAttribute("data-bs-original-title", note); + badge.setAttribute("aria-label", `Review ${portLabel} Ethernet-limited circuits`); + disposeTooltip(); + initTooltipsWithin(badge.parentElement || document); +} + +function retransmitPacketsForNode(node, direction) { + return toNumber( + node.current_tcp_retransmit_packets?.[direction] ?? node.current_tcp_packets?.[direction], + 0, + ); +} + const listenOnce = (eventName, handler) => { const wrapped = (msg) => { wsClient.off(eventName, wrapped); @@ -103,6 +194,13 @@ function runWhenRenderable(el, callback, attempts = 10) { }, 50); } +function clearPollingTimer(timerId) { + if (timerId !== null) { + window.clearInterval(timerId); + } + return null; +} + function loadingBlockHtml(label, sizeClass = "") { const size = sizeClass ? ` ${sizeClass}` : ""; return `
${label}
`; @@ -166,71 +264,13 @@ function formatBitsPerSecondLabel(bitsPerSecond) { return `${scaleNumber(bitsPerSecond, 1)}bps`; } -function medianOfSorted(values) { - if (!Array.isArray(values) || values.length === 0) { - return null; - } - const middle = Math.floor(values.length / 2); - if (values.length % 2 === 1) { - return values[middle]; - } - return (values[middle - 1] + values[middle]) / 2; -} - -function weightedMedian(entries) { - if (!Array.isArray(entries) || entries.length === 0) { - return null; - } - - const totalWeight = entries.reduce((sum, entry) => sum + Math.max(0, toNumber(entry.weight, 0)), 0); - if (!(totalWeight > 0)) { - return null; - } - - const threshold = totalWeight / 2; - let running = 0; - for (const entry of entries) { - running += Math.max(0, toNumber(entry.weight, 0)); - if (running >= threshold) { - return toNumber(entry.value, null); - } - } - - return toNumber(entries[entries.length - 1].value, null); -} - function currentCircuitRttP50Ms(direction) { const directional = direction === "up" ? "up" : "down"; - const weightedEntries = []; - const fallbackValues = []; - - latestCircuitDevices.forEach((device) => { - const currentP50Nanos = toNumber(device?.rtt_current_p50_nanos?.[directional], 0); - if (!(currentP50Nanos > 0)) { - return; - } - - const throughputBps = toNumber(device?.bytes_per_second?.[directional], 0) * 8; - const currentP50Ms = currentP50Nanos / 1_000_000.0; - fallbackValues.push(currentP50Ms); - - if (throughputBps > QUEUING_ACTIVITY_RTT_FLOOR_BPS) { - weightedEntries.push({ - value: currentP50Ms, - weight: throughputBps, - }); - } - }); - - weightedEntries.sort((a, b) => a.value - b.value); - const weighted = weightedMedian(weightedEntries); - if (Number.isFinite(weighted)) { - return weighted; + const nanos = toNumber(latestCircuitSummary?.rtt_current_p50_nanos?.[directional], 0); + if (!(nanos > 0)) { + return null; } - - fallbackValues.sort((a, b) => a - b); - const fallbackMedian = medianOfSorted(fallbackValues); - return Number.isFinite(fallbackMedian) ? fallbackMedian : null; + return nanos / 1_000_000.0; } function formatCircuitRttLabel(rttMs) { @@ -272,7 +312,7 @@ function loadRttThresholds() { } function currentActiveFlowCount() { - return latestTrafficRows.length; + return toNumber(latestCircuitSummary?.active_flow_count, 0); } function currentDirectionValue(pair, direction, fallback = 0) { @@ -280,9 +320,7 @@ function currentDirectionValue(pair, direction, fallback = 0) { } function currentQueuingActivitySnapshot() { - const throughputBps = latestCircuitDevices.reduce((sum, device) => { - return sum + (currentDirectionValue(device?.bytes_per_second, queuingActivityDirection, 0) * 8); - }, 0); + const throughputBps = currentDirectionValue(latestCircuitSummary?.bytes_per_second, queuingActivityDirection, 0) * 8; const ceilingMbps = currentDirectionValue(plan, queuingActivityDirection, 0); const ceilingBps = ceilingMbps * 1_000_000.0; const atCeiling = ceilingBps > 0 && throughputBps >= (ceilingBps * 0.95); @@ -322,23 +360,16 @@ function updateQueuingActivityCards() { } function pushQueuingActivitySample() { - if (!queuingActivityGraph || !latestCircuitDevices.length || !plan) { + if (!queuingActivityGraph || !latestCircuitSummary || !plan) { updateQueuingActivityCards(); return; } - const downThroughputBps = latestCircuitDevices.reduce((sum, device) => { - return sum + (currentDirectionValue(device?.bytes_per_second, "down", 0) * 8); - }, 0); - const upThroughputBps = latestCircuitDevices.reduce((sum, device) => { - return sum + (currentDirectionValue(device?.bytes_per_second, "up", 0) * 8); - }, 0); - queuingActivityGraph.pushSample({ timestamp: Date.now(), throughputBps: { - down: downThroughputBps, - up: upThroughputBps, + down: currentDirectionValue(latestCircuitSummary?.bytes_per_second, "down", 0) * 8, + up: currentDirectionValue(latestCircuitSummary?.bytes_per_second, "up", 0) * 8, }, ceilingBps: { down: currentDirectionValue(plan, "down", 0) * 1_000_000.0, @@ -400,6 +431,14 @@ function isTrafficTabActive() { return document.getElementById("traffic-tab")?.classList.contains("active") ?? false; } +function isDevicesTabActive() { + return document.getElementById("devs-tab")?.classList.contains("active") ?? false; +} + +function isTopAsnTabActive() { + return document.getElementById("top-asns-tab")?.classList.contains("active") ?? false; +} + function isSankeyTabActive() { return document.getElementById("sankey-tab")?.classList.contains("active") ?? false; } @@ -423,7 +462,7 @@ function ensureFlowSankey() { return; } flowSankey = new FlowsSankey("flowSankey"); - applyFlowSankeyMessage(latestFlowMsg); + applyFlowSankeyMessage(latestSankeyFlowMsg); }); } @@ -446,13 +485,24 @@ function initializeDeviceGraphs() { }); deviceGraphsInitialized = true; if (latestCircuitDevices.length > 0) { - fillLiveDevices(latestCircuitDevices); - if (speedometer && totalThroughput && totalRetransmits) { - updateSpeedometer(latestCircuitDevices); - } + applyDeviceLiveData(latestCircuitDevices); } } +function ensureDeviceGraphs() { + const target = document.getElementById("devs"); + if (!target || !isElementVisible(target)) { + return; + } + runWhenRenderable(target, () => { + if (!hasRenderableSize(target)) { + return; + } + initializeDeviceGraphs(); + resizeDeviceGraphs(); + }); +} + function resizeDeviceGraphs() { Object.values(deviceGraphs).forEach((graph) => resizeGraphIfVisible(graph)); } @@ -494,6 +544,82 @@ function resolveFunnelState(msg, parentNode) { }; } +function buildFunnelPathCard(state, displayParents) { + const pathCard = document.createElement("div"); + pathCard.classList.add("lqos-funnel-path-card"); + + const topRow = document.createElement("div"); + topRow.classList.add("lqos-funnel-path-top"); + + const title = document.createElement("div"); + title.classList.add("lqos-funnel-path-title"); + title.textContent = "Queue Path"; + topRow.appendChild(title); + + const meta = document.createElement("div"); + meta.classList.add("lqos-funnel-path-meta"); + + const parentPill = document.createElement("span"); + parentPill.classList.add("lqos-funnel-path-pill"); + parentPill.textContent = "Circuit parent"; + meta.appendChild(parentPill); + + const countPill = document.createElement("span"); + countPill.classList.add("lqos-funnel-path-pill"); + countPill.textContent = `${displayParents.length} upstream node${displayParents.length === 1 ? "" : "s"}`; + meta.appendChild(countPill); + + topRow.appendChild(meta); + pathCard.appendChild(topRow); + + const chain = document.createElement("div"); + chain.classList.add("lqos-funnel-path-chain"); + + const origin = document.createElement("span"); + origin.classList.add("lqos-funnel-path-node", "is-origin", "redactable"); + origin.textContent = state.immediateParent.name || "Unknown"; + chain.appendChild(origin); + + displayParents.forEach(({ node }) => { + const separator = document.createElement("span"); + separator.classList.add("lqos-funnel-path-separator"); + separator.textContent = "→"; + chain.appendChild(separator); + + const ancestor = document.createElement("span"); + ancestor.classList.add("lqos-funnel-path-node", "redactable"); + ancestor.textContent = node.name || "Unknown"; + chain.appendChild(ancestor); + }); + + pathCard.appendChild(chain); + + const note = document.createElement("p"); + note.classList.add("text-muted", "small", "mb-0"); + note.textContent = + displayParents.length > 0 + ? "Live upstream queue ancestors for this circuit, shown with the same order used below." + : "This circuit parent does not currently have additional upstream queue ancestors."; + pathCard.appendChild(note); + + return pathCard; +} + +function buildFunnelEmptyState(message) { + const empty = document.createElement("div"); + empty.classList.add("lqos-funnel-empty-state"); + + const icon = document.createElement("i"); + icon.classList.add("fa", "fa-circle-info"); + empty.appendChild(icon); + + const text = document.createElement("span"); + text.textContent = message; + empty.appendChild(text); + + return empty; +} + function renderFunnel(state) { const target = document.getElementById("theFunnel"); if (!target) { @@ -505,51 +631,106 @@ function renderFunnel(state) { funnelParents = []; funnelParentSignature = []; clearDiv(target); - target.appendChild(document.createTextNode("No parent node found")); + target.appendChild(buildFunnelEmptyState("No parent node found for this circuit.")); return; } - const parentIndexes = [...state.parentIndexes].reverse(); - let parentDiv = document.createElement("div"); - parentIndexes.forEach((parent) => { - const node = state.data[parent] && state.data[parent][1] ? state.data[parent][1] : null; - if (!node) { - return; + const displayParents = [...state.parentIndexes] + .reverse() + .map((parent, index) => { + const node = state.data[parent] && state.data[parent][1] ? state.data[parent][1] : null; + if (!node) { + return null; + } + return { + parent, + node, + position: index + 1, + }; + }) + .filter(Boolean); + + const parentDiv = document.createElement("div"); + parentDiv.classList.add("lqos-funnel-stack"); + parentDiv.appendChild(buildFunnelPathCard(state, displayParents)); + + if (displayParents.length === 0) { + parentDiv.appendChild(buildFunnelEmptyState("No upstream queue ancestors are currently available beyond the circuit parent.")); + } + + displayParents.forEach(({ parent, node, position }) => { + const card = document.createElement("section"); + card.classList.add("lqos-funnel-node-card"); + + const header = document.createElement("div"); + header.classList.add("lqos-funnel-node-header"); + + const titleWrap = document.createElement("div"); + titleWrap.classList.add("lqos-funnel-node-title"); + + const heading = document.createElement("h5"); + const icon = document.createElement("i"); + icon.classList.add("fa", "fa-sitemap"); + heading.appendChild(icon); + + const name = document.createElement("span"); + name.classList.add("redactable"); + name.textContent = node.name || "Unknown"; + heading.appendChild(name); + titleWrap.appendChild(heading); + + const subtitle = document.createElement("p"); + subtitle.classList.add("text-muted", "small", "mb-0"); + subtitle.textContent = "Live queue telemetry for this upstream node."; + titleWrap.appendChild(subtitle); + + header.appendChild(titleWrap); + + const badges = document.createElement("div"); + badges.classList.add("lqos-funnel-node-badges"); + + const typeBadge = document.createElement("span"); + typeBadge.classList.add("lqos-funnel-node-type"); + if (node.is_virtual === true) { + typeBadge.classList.add("is-virtual"); } + typeBadge.textContent = node.is_virtual === true ? "Virtual" : "Physical"; + badges.appendChild(typeBadge); - let row = document.createElement("div"); - row.classList.add("row"); + const stepBadge = document.createElement("span"); + stepBadge.classList.add("lqos-funnel-node-step"); + stepBadge.textContent = `Ancestor ${position} / ${displayParents.length}`; + badges.appendChild(stepBadge); - let col = document.createElement("div"); - col.classList.add("col-12"); - let heading = document.createElement("h5"); - heading.classList.add("redactable"); - const virtualLabel = node.is_virtual === true ? " Virtual" : ""; - heading.innerHTML = " " + node.name + virtualLabel; - col.appendChild(heading); - row.appendChild(col); - parentDiv.appendChild(row); + header.appendChild(badges); + card.appendChild(header); - row = document.createElement("div"); - row.classList.add("row"); + const row = document.createElement("div"); + row.classList.add("row", "g-3"); - let col_tp = document.createElement("div"); - col_tp.classList.add("col-4"); - col_tp.id = "funnel_tp_" + parent; - col_tp.style.height = "250px"; - row.appendChild(col_tp); + const chartSpecs = [ + { id: "funnel_tp_" + parent, height: "250px" }, + { id: "funnel_rxmit_" + parent, height: "250px" }, + { id: "funnel_rtt_" + parent, height: "250px" }, + ]; - let col_rxmit = document.createElement("div"); - col_rxmit.classList.add("col-4"); - col_rxmit.id = "funnel_rxmit_" + parent; - row.appendChild(col_rxmit); + chartSpecs.forEach((chart) => { + const col = document.createElement("div"); + col.classList.add("col-12", "col-xl-4"); - let col_rtt = document.createElement("div"); - col_rtt.classList.add("col-4"); - col_rtt.id = "funnel_rtt_" + parent; - row.appendChild(col_rtt); + const panel = document.createElement("div"); + panel.classList.add("lqos-funnel-chart-panel"); - parentDiv.appendChild(row); + const graph = document.createElement("div"); + graph.id = chart.id; + graph.style.height = chart.height; + panel.appendChild(graph); + col.appendChild(panel); + row.appendChild(col); + }); + + card.appendChild(row); + parentDiv.appendChild(card); }); funnelGraphs = {}; @@ -558,7 +739,7 @@ function renderFunnel(state) { requestAnimationFrame(() => { setTimeout(() => { - parentIndexes.forEach((parent) => { + displayParents.forEach(({ parent }) => { if (!document.getElementById("funnel_tp_" + parent)) { return; } @@ -615,36 +796,77 @@ function renderCakeGraphShell() { return; } cakeTab.innerHTML = ` -
-
-
+
+
+
+
Queue Stats
+

Raw 1s scatter samples over the last 3 minutes. Hover any queue chart to inspect the same timestamp across all six charts.

+
+
+ Last 3 minutes + 1s samples + Synchronized hover +
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Queue Memory: ? -
Queue Type: ?
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Queue Memory
+
?
+
+
+
Live Queue Type
+
?
+
+
+
Interpretation
+
Download plots above zero, upload plots below zero. Tooltips always show absolute values with direction labels.
+
+
+
`; - setQueueTypeDisplay(""); + setQueueTypeDisplayFromKinds(latestCakeMsg?.kind_down, latestCakeMsg?.kind_up); } function applyCakeMessage(msg) { - if (!cakeGraphs || !msg) { + if (!msg) { + return; + } + setQueueTypeDisplayFromKinds(msg.kind_down, msg.kind_up); + if (!cakeGraphs) { return; } $("#cakeQueueMemory").text(scaleNumber(msg.current_download.memory_used) + " / " + scaleNumber(msg.current_upload.memory_used)); @@ -680,6 +902,13 @@ function ensureCakeGraphs() { marks: new CakeMarks("cakeMarks"), drops: new CakeDrops("cakeDrops"), }; + const cakeChartInstances = Object.values(cakeGraphs) + .map((graph) => graph?.chart) + .filter(Boolean); + cakeChartInstances.forEach((chart) => { + chart.group = "circuitCakeQueueStats"; + }); + echarts.connect("circuitCakeQueueStats"); applyCakeMessage(latestCakeMsg); }); } @@ -693,23 +922,32 @@ function initTabLifecycle(parentNode) { if (target === "#queuing") { ensureQueuingActivityGraph(); updateQueuingActivityCards(); + syncCircuitDetailSubscriptions(); return; } if (target === "#devs") { - initializeDeviceGraphs(); - resizeDeviceGraphs(); + ensureDeviceGraphs(); + syncCircuitDetailSubscriptions(); return; } if (target === "#sankey") { ensureFlowSankey(); - applyFlowSankeyMessage(latestFlowMsg); + applyFlowSankeyMessage(latestSankeyFlowMsg); + syncCircuitDetailSubscriptions(); + return; + } + if (target === "#top-asns") { + renderTopAsnTab(); + syncCircuitDetailSubscriptions(); return; } if (target === "#traffic") { renderTrafficTab(); + syncCircuitDetailSubscriptions(); return; } if (target === "#funnel") { + syncCircuitDetailSubscriptions(); if (!funnelInitialized) { funnelInitialized = true; initialFunnel(parentNode); @@ -721,6 +959,7 @@ function initTabLifecycle(parentNode) { if (target === "#cake") { ensureCakeGraphs(); applyCakeMessage(latestCakeMsg); + syncCircuitDetailSubscriptions(); } }); }); @@ -729,6 +968,7 @@ function initTabLifecycle(parentNode) { window.requestAnimationFrame(() => { ensureQueuingActivityGraph(); updateQueuingActivityCards(); + syncCircuitDetailSubscriptions(); }); } @@ -775,6 +1015,12 @@ function parseDirectionalSqmToken(token) { function formatQueueTypeDisplay(sqmToken) { const { down, up } = parseDirectionalSqmToken(sqmToken); + return formatDirectionalQueueTypeDisplay(down, up); +} + +function formatDirectionalQueueTypeDisplay(downToken, upToken) { + const down = (downToken ?? "").toString().trim().toLowerCase(); + const up = (upToken ?? "").toString().trim().toLowerCase(); const downLabel = down || "Unknown"; const upLabel = up || down || "Unknown"; return `${downLabel} / ${upLabel}`; @@ -788,6 +1034,16 @@ function setQueueTypeDisplay(sqmToken) { queueTypeEl.textContent = formatQueueTypeDisplay(sqmToken); } +function setQueueTypeDisplayFromKinds(kindDown, kindUp) { + const queueTypeEl = document.getElementById("cakeQueueType"); + if (!queueTypeEl) { + return; + } + const down = (kindDown ?? "").toString().trim().toLowerCase(); + const up = (kindUp ?? "").toString().trim().toLowerCase(); + queueTypeEl.textContent = formatDirectionalQueueTypeDisplay(down, up); +} + function requestCircuitById(onSuccess, onError) { listenOnce("CircuitByIdResult", (msg) => { if (!msg || !msg.ok) { @@ -798,11 +1054,73 @@ function requestCircuitById(onSuccess, onError) { if (onError) onError(); return; } - onSuccess(msg.devices || []); + const payload = msg.data || {}; + onSuccess(payload); }); wsClient.send({ CircuitById: { id: circuit_id } }); } +function applyCircuitSummary(summary) { + latestCircuitSummary = summary || null; + latestCircuitQooScore = toNumber(summary?.qoo_score, NaN); + if (!Number.isFinite(latestCircuitQooScore)) { + latestCircuitQooScore = null; + } + if (excludeRttToggle && summary?.rtt_excluded !== undefined) { + excludeRttLastValue = !!summary.rtt_excluded; + excludeRttToggle.checked = excludeRttLastValue; + } + if (speedometer) { + speedometer.update( + currentDirectionValue(summary?.bytes_per_second, "down", 0) * 8, + currentDirectionValue(summary?.bytes_per_second, "up", 0) * 8, + currentDirectionValue(plan, "down", 0), + currentDirectionValue(plan, "up", 0) + ); + } + if (totalThroughput) { + totalThroughput.update( + currentDirectionValue(summary?.bytes_per_second, "down", 0) * 8, + currentDirectionValue(summary?.bytes_per_second, "up", 0) * 8 + ); + } + if (totalRetransmits) { + totalRetransmits.update( + retransmitFractionFromSample(summary?.tcp_retransmit_sample?.down) * 100.0, + retransmitFractionFromSample(summary?.tcp_retransmit_sample?.up) * 100.0 + ); + } + if (qooGauge !== null) { + qooGauge.update(summary?.qoo_score); + } + updateTopAsnCountBadge(); + updateTrafficCountBadge(); + pushQueuingActivitySample(); +} + +function applyDeviceLiveData(devices) { + latestCircuitDevices = devices || []; + fillLiveDevices(latestCircuitDevices); + + latestCircuitDevices.forEach((device) => { + const throughputGraph = deviceGraphs["throughputGraph_" + device.device_id]; + if (throughputGraph !== undefined) { + throughputGraph.update( + toNumber(device.bytes_per_second?.down, 0) * 8, + toNumber(device.bytes_per_second?.up, 0) * 8 + ); + } + + const retransmitGraph = deviceGraphs["tcpRetransmitsGraph_" + device.device_id]; + if (retransmitGraph !== undefined) { + retransmitGraph.update( + retransmitFractionFromSample(device.tcp_retransmit_sample?.down) * 100.0, + retransmitFractionFromSample(device.tcp_retransmit_sample?.up) * 100.0 + ); + } + }); +} + function initExcludeRttToggle() { excludeRttToggle = document.getElementById("excludeRttToggle"); if (!excludeRttToggle) return; @@ -837,30 +1155,15 @@ function initExcludeRttToggle() { }); } -function connectPrivateChannel() { +function connectCircuitSummaryChannel() { channelLink = new DirectChannel({ CircuitWatcher: { circuit: circuit_id } }, (msg) => { - latestCircuitQooScore = toNumber(msg.qoo_score, NaN); - if (!Number.isFinite(latestCircuitQooScore)) { - latestCircuitQooScore = null; + if (msg?.data) { + applyCircuitSummary(msg.data); } - if (msg.devices !== null) { - latestCircuitDevices = msg.devices || []; - fillLiveDevices(msg.devices); - updateSpeedometer(msg.devices); - pushQueuingActivitySample(); - if (excludeRttToggle && msg.rtt_excluded !== undefined) { - excludeRttLastValue = !!msg.rtt_excluded; - excludeRttToggle.checked = excludeRttLastValue; - } - } - if (qooGauge !== null) { - qooGauge.update(msg.qoo_score); - } - updateQueuingActivityCards(); }); } @@ -877,7 +1180,7 @@ function fullIpList(circuits) { return ipList; } -function connectPingers(circuits) { +function startPingMonitor(circuits) { let ipList = fullIpList(circuits); pinger = new DirectChannel({ @@ -943,28 +1246,149 @@ function connectPingers(circuits) { }); } -function connectFlowChannel() { - flowChannel = new DirectChannel({ - FlowsByCircuit: { - circuit: circuit_id +function stopPingMonitor() { + if (pinger) { + wsClient.send({ Private: { StopPingMonitorWatch: null } }); + pinger.close(); + pinger = null; + } +} + +function requestCircuitDevicesSnapshot() { + if (deviceRequestInFlight) { + return; + } + deviceRequestInFlight = true; + listenOnce("CircuitDevicesResult", (msg) => { + deviceRequestInFlight = false; + if (!msg?.data?.ok || msg.data.circuit_id !== circuit_id) { + return; } - }, (msg) => { - latestFlowMsg = msg; - ingestTrafficRows(msg); + applyDeviceLiveData(msg.data.devices || []); + }); + wsClient.send({ CircuitDevices: { circuit: circuit_id } }); +} + +function requestCircuitFlowSankey() { + if (sankeyRequestInFlight) { + return; + } + sankeyRequestInFlight = true; + listenOnce("CircuitFlowSankeyResult", (msg) => { + sankeyRequestInFlight = false; + if (msg?.circuit_id !== circuit_id) { + return; + } + latestSankeyFlowMsg = { flows: Array.isArray(msg.flows) ? msg.flows : [] }; if (isSankeyTabActive()) { ensureFlowSankey(); - applyFlowSankeyMessage(msg); + applyFlowSankeyMessage(latestSankeyFlowMsg); } else { - $("#activeFlowCount").text(getRenderableSankeyFlowCount(msg)); + $("#activeFlowCount").text(getRenderableSankeyFlowCount(latestSankeyFlowMsg)); } + }); + wsClient.send({ CircuitFlowSankey: { circuit: circuit_id } }); +} + +function requestCircuitTopAsns() { + if (topAsnRequestInFlight) { + return; + } + topAsnRequestInFlight = true; + listenOnce("CircuitTopAsnsResult", (msg) => { + topAsnRequestInFlight = false; + if (msg?.circuit_id !== circuit_id) { + return; + } + latestTopAsnData = msg.data || { total_asns: 0, rows: [] }; + if (isTopAsnTabActive()) { + renderTopAsnTab(); + } else { + updateTopAsnCountBadge(); + } + }); + wsClient.send({ + CircuitTopAsns: { + query: { + circuit: circuit_id, + hide_small: hideSmallFlowsEnabled(), + }, + }, + }); +} + +function requestTrafficFlowsPage() { + if (trafficRequestInFlight) { + return; + } + trafficRequestInFlight = true; + const query = { + circuit: circuit_id, + page: trafficCurrentPage, + page_size: trafficPageSize, + hide_small: hideSmallFlowsEnabled(), + sort_column: trafficSortColumn, + sort_direction: trafficSortDirection, + }; + listenOnce("CircuitTrafficFlowsPageResult", (msg) => { + trafficRequestInFlight = false; + if (msg?.circuit_id !== circuit_id) { + return; + } + latestTrafficPage = msg.data || null; + trafficCurrentPage = Math.max(1, toNumber(latestTrafficPage?.query?.page, query.page)); + trafficPageSize = Math.max(1, toNumber(latestTrafficPage?.query?.page_size, query.page_size)); if (isTrafficTabActive()) { renderTrafficTab(); } else { updateTrafficCountBadge(); updateTrafficPaginationControls(); } - updateQueuingActivityCards(); }); + wsClient.send({ CircuitTrafficFlowsPage: { query } }); +} + +function syncCircuitDetailSubscriptions(circuits = null) { + const pingSourceCircuits = Array.isArray(circuits) && circuits.length > 0 ? circuits : circuitConfigDevices; + if (isDevicesTabActive()) { + if (!pinger && Array.isArray(pingSourceCircuits) && pingSourceCircuits.length > 0) { + startPingMonitor(pingSourceCircuits); + } + requestCircuitDevicesSnapshot(); + if (devicePollTimer === null) { + devicePollTimer = window.setInterval(requestCircuitDevicesSnapshot, 1000); + } + } else { + devicePollTimer = clearPollingTimer(devicePollTimer); + stopPingMonitor(); + } + + if (isSankeyTabActive()) { + requestCircuitFlowSankey(); + if (sankeyPollTimer === null) { + sankeyPollTimer = window.setInterval(requestCircuitFlowSankey, 1000); + } + } else { + sankeyPollTimer = clearPollingTimer(sankeyPollTimer); + } + + if (isTopAsnTabActive()) { + requestCircuitTopAsns(); + if (topAsnPollTimer === null) { + topAsnPollTimer = window.setInterval(requestCircuitTopAsns, 1000); + } + } else { + topAsnPollTimer = clearPollingTimer(topAsnPollTimer); + } + + if (isTrafficTabActive()) { + requestTrafficFlowsPage(); + if (trafficPollTimer === null) { + trafficPollTimer = window.setInterval(requestTrafficFlowsPage, 1000); + } + } else { + trafficPollTimer = clearPollingTimer(trafficPollTimer); + } } function initFlowFilters() { @@ -977,7 +1401,20 @@ function initFlowFilters() { if (hideSmallFlows) { hideSmallFlows.addEventListener("change", () => { trafficCurrentPage = 1; - renderTrafficTab(); + if (isTrafficTabActive()) { + requestTrafficFlowsPage(); + } else { + updateTrafficCountBadge(); + updateTrafficPaginationControls(); + } + if (isTopAsnTabActive()) { + requestCircuitTopAsns(); + } else { + updateTopAsnCountBadge(); + } + if (isSankeyTabActive()) { + requestCircuitFlowSankey(); + } }); } if (pageSize) { @@ -986,14 +1423,14 @@ function initFlowFilters() { const parsed = parseInt(pageSize.value, 10); trafficPageSize = Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TRAFFIC_PAGE_SIZE; trafficCurrentPage = 1; - renderTrafficTab(); + requestTrafficFlowsPage(); }); } if (prev) { prev.addEventListener("click", () => { if (trafficCurrentPage > 1) { trafficCurrentPage--; - renderTrafficTab(); + requestTrafficFlowsPage(); } }); } @@ -1002,7 +1439,7 @@ function initFlowFilters() { const totalPages = getTrafficTotalPages(); if (trafficCurrentPage < totalPages) { trafficCurrentPage++; - renderTrafficTab(); + requestTrafficFlowsPage(); } }); } @@ -1023,22 +1460,13 @@ function initFlowFilters() { } } -let movingAverages = new Map(); -let prevFlowBytes = new Map(); -let tickCount = 0; let trafficSortColumn = 'rate'; // Default sort by rate let trafficSortDirection = 'desc'; // 'asc' or 'desc' -let latestTrafficRows = []; +let topAsnSortColumn = 'rate'; +let topAsnSortDirection = 'desc'; let trafficCurrentPage = 1; let trafficPageSize = DEFAULT_TRAFFIC_PAGE_SIZE; -function diffToNumber(current, previous, fallback = 0) { - if (typeof current === "bigint" && typeof previous === "bigint") { - return toNumber(current - previous, fallback); - } - return toNumber(current, fallback) - toNumber(previous, fallback); -} - function formatQooScore(score0to100, fallback = "-") { if (score0to100 === null || score0to100 === undefined) { return fallback; @@ -1072,34 +1500,33 @@ function formatRttPair(p50Nanos, p95Nanos) { return formatRttNanos(p50) + " / " + scaleNanos(p95); } -function visibleTrafficRows() { - if (!hideSmallFlowsEnabled()) { - return latestTrafficRows; +function truncatedTrafficCell(text, cellClass) { + const td = document.createElement("td"); + if (cellClass) { + td.classList.add(cellClass); } - return latestTrafficRows.filter((row) => row.downBps > TRAFFIC_FLOW_HIDE_THRESHOLD_BPS || row.upBps > TRAFFIC_FLOW_HIDE_THRESHOLD_BPS); + + const value = String(text || ""); + td.title = value; + + const span = document.createElement("span"); + span.classList.add("lqos-table-cell-ellipsis"); + span.textContent = value; + td.appendChild(span); + return td; +} + +function visibleTrafficRows() { + return Array.isArray(latestTrafficPage?.rows) ? latestTrafficPage.rows : []; } function hideSmallFlowsEnabled() { return document.getElementById("hideSmallFlows")?.checked ?? false; } -function sortTrafficRows(rows) { - rows.sort((a, b) => { - let aVal = a.sortKeys[trafficSortColumn]; - let bVal = b.sortKeys[trafficSortColumn]; - if (typeof aVal === "string" && typeof bVal === "string") { - aVal = aVal.toLowerCase(); - bVal = bVal.toLowerCase(); - } - if (trafficSortDirection === "asc") { - return aVal < bVal ? -1 : aVal > bVal ? 1 : 0; - } - return aVal > bVal ? -1 : aVal < bVal ? 1 : 0; - }); -} - function getTrafficTotalPages() { - return Math.max(1, Math.ceil(visibleTrafficRows().length / trafficPageSize)); + const totalRows = toNumber(latestTrafficPage?.total_rows, 0); + return Math.max(1, Math.ceil(totalRows / trafficPageSize)); } function updateTrafficPaginationControls() { @@ -1121,111 +1548,136 @@ function updateTrafficPaginationControls() { } function updateTrafficCountBadge() { - $("#trafficFlowCount").text(visibleTrafficRows().length); + $("#trafficFlowCount").text(toNumber(latestTrafficPage?.total_rows, latestCircuitSummary?.active_flow_count ?? 0)); } -function ingestTrafficRows(msg) { - tickCount++; - const rows = []; - const seenFlowKeys = new Set(); - const flowList = Array.isArray(msg?.flows) ? msg.flows : []; +function updateTopAsnCountBadge() { + $("#topAsnCount").text(toNumber(latestTopAsnData?.total_asns, latestCircuitSummary?.active_asn_count ?? 0)); +} - flowList.forEach((flow) => { - const flowKey = `${flow?.[0]?.protocol_name || ""}${flow?.[0]?.row_id || ""}`; - seenFlowKeys.add(flowKey); +function sortTopAsnRows(rows) { + rows.sort((a, b) => { + const asc = topAsnSortDirection === "asc"; + const normalize = (value) => typeof value === "string" ? value.toLowerCase() : value; + let aVal; + let bVal; + switch (topAsnSortColumn) { + case "asn": + aVal = normalize(a.asn_name); + bVal = normalize(b.asn_name); + break; + case "country": + aVal = normalize(a.asn_country); + bVal = normalize(b.asn_country); + break; + case "bytes": + aVal = a.bytes_sent_down + a.bytes_sent_up; + bVal = b.bytes_sent_down + b.bytes_sent_up; + break; + case "packets": + aVal = a.packets_sent_down + a.packets_sent_up; + bVal = b.packets_sent_down + b.packets_sent_up; + break; + case "retransmits": + aVal = a.retransmit_down_pct + a.retransmit_up_pct; + bVal = b.retransmit_down_pct + b.retransmit_up_pct; + break; + case "flows": + aVal = a.flow_count; + bVal = b.flow_count; + break; + case "rate": + default: + aVal = a.down_bps + a.up_bps; + bVal = b.down_bps + b.up_bps; + break; + } + if (aVal === bVal) { + return 0; + } + return asc ? (aVal < bVal ? -1 : 1) : (aVal > bVal ? -1 : 1); + }); +} - let down = toNumber(flow?.[1]?.rate_estimate_bps?.down, 0); - let up = toNumber(flow?.[1]?.rate_estimate_bps?.up, 0); - const prev = prevFlowBytes.get(flowKey); - if (prev) { - const ticks = tickCount - prev.tick; - if (ticks === 1) { - down = diffToNumber(flow[1].bytes_sent.down, prev.downBytes, 0) * 8; - up = diffToNumber(flow[1].bytes_sent.up, prev.upBytes, 0) * 8; - } else if (ticks > 1) { - down = diffToNumber(flow[1].bytes_sent.down, prev.downBytes, 0) * 8 / ticks; - up = diffToNumber(flow[1].bytes_sent.up, prev.upBytes, 0) * 8 / ticks; +function renderTopAsnTab() { + const target = document.getElementById("topAsnsTable"); + if (!target) { + return; + } + + const displayRows = Array.isArray(latestTopAsnData?.rows) ? latestTopAsnData.rows.slice() : []; + sortTopAsnRows(displayRows); + + const tableWrap = document.createElement("div"); + tableWrap.classList.add("lqos-table-wrap"); + + const table = document.createElement("table"); + table.classList.add("lqos-table", "lqos-table-tight", "lqos-circuit-traffic-table"); + const thead = document.createElement("thead", "small"); + thead.style.fontSize = "0.8em"; + + const createSortableHeader = (text, sortKey, colspan = 1) => { + const th = theading(text, colspan); + th.style.cursor = "pointer"; + th.onclick = () => { + if (topAsnSortColumn === sortKey) { + topAsnSortDirection = topAsnSortDirection === "asc" ? "desc" : "asc"; + } else { + topAsnSortColumn = sortKey; + topAsnSortDirection = "desc"; } + renderTopAsnTab(); + }; + if (topAsnSortColumn === sortKey) { + th.innerHTML += topAsnSortDirection === "asc" ? " ▲" : " ▼"; } - if (down < 0) down = 0; - if (up < 0) up = 0; + return th; + }; - prevFlowBytes.set(flowKey, { - downBytes: flow?.[1]?.bytes_sent?.down, - upBytes: flow?.[1]?.bytes_sent?.up, - tick: tickCount, + thead.appendChild(createSortableHeader("ASN", "asn")); + thead.appendChild(createSortableHeader("Country", "country")); + thead.appendChild(createSortableHeader("Current Rate (d/u)", "rate", 2)); + thead.appendChild(createSortableHeader("Total Bytes (d/u)", "bytes", 2)); + thead.appendChild(createSortableHeader("Total Packets (d/u)", "packets", 2)); + thead.appendChild(createSortableHeader("TCP rxmit (d/u)", "retransmits", 2)); + thead.appendChild(createSortableHeader("Flows", "flows")); + table.appendChild(thead); + + const tbody = document.createElement("tbody"); + if (displayRows.length === 0) { + const empty = document.createElement("tr"); + const td = document.createElement("td"); + td.colSpan = 11; + td.classList.add("text-center", "text-muted", "small"); + td.textContent = "No recent ASN activity available for this circuit."; + empty.appendChild(td); + tbody.appendChild(empty); + } else { + displayRows.forEach((rowData) => { + const row = document.createElement("tr"); + row.classList.add("small"); + + row.appendChild(truncatedTrafficCell(rowData.asn_name, "lqos-circuit-traffic-asn-cell")); + row.appendChild(truncatedTrafficCell(rowData.asn_country, "lqos-circuit-traffic-country-cell")); + row.appendChild(simpleRowHtml(formatThroughput(rowData.down_bps, plan.down))); + row.appendChild(simpleRowHtml(formatThroughput(rowData.up_bps, plan.up))); + row.appendChild(simpleRow(scaleNumber(rowData.bytes_sent_down))); + row.appendChild(simpleRow(scaleNumber(rowData.bytes_sent_up))); + row.appendChild(simpleRow(scaleNumber(rowData.packets_sent_down))); + row.appendChild(simpleRow(scaleNumber(rowData.packets_sent_up))); + row.appendChild(simpleRowHtml(rowData.retransmit_down_pct > 0 ? formatRetransmitFraction(rowData.retransmit_down_pct) : "-")); + row.appendChild(simpleRowHtml(rowData.retransmit_up_pct > 0 ? formatRetransmitFraction(rowData.retransmit_up_pct) : "-")); + row.appendChild(simpleRow(scaleNumber(rowData.flow_count))); + + tbody.appendChild(row); }); + } - const currentRate = down + up; - const average = movingAverages.get(flowKey) || { values: [], total: 0 }; - average.values.push(currentRate); - average.total += currentRate; - if (average.values.length > 10) { - average.total -= average.values.shift(); - } - movingAverages.set(flowKey, average); - - const lastSeenNanos = toNumber(flow?.[0]?.last_seen_nanos, 0); - if (lastSeenNanos > RECENT_TRAFFIC_FLOW_WINDOW_NANOS) { - return; - } - - const tcpRetransmitsDown = toNumber(flow?.[1]?.tcp_retransmits?.down, 0); - const tcpRetransmitsUp = toNumber(flow?.[1]?.tcp_retransmits?.up, 0); - const packetsSentDown = toNumber(flow?.[1]?.packets_sent?.down, 0); - const packetsSentUp = toNumber(flow?.[1]?.packets_sent?.up, 0); - const retransmitDownPct = tcpRetransmitsDown > 0 && packetsSentDown > 0 ? tcpRetransmitsDown / packetsSentDown : 0; - const retransmitUpPct = tcpRetransmitsUp > 0 && packetsSentUp > 0 ? tcpRetransmitsUp / packetsSentUp : 0; - const bytesSentDown = toNumber(flow?.[1]?.bytes_sent?.down, 0); - const bytesSentUp = toNumber(flow?.[1]?.bytes_sent?.up, 0); - const rttDownNanos = toNumber(flow?.[1]?.rtt?.[0]?.nanoseconds, 0); - const rttUpNanos = toNumber(flow?.[1]?.rtt?.[1]?.nanoseconds, 0); - const qoq = flow?.[1]?.qoq || null; - const qooDown = qoq ? qoq.download_total : null; - const qooUp = qoq ? qoq.upload_total : null; - const remoteIp = String(flow?.[0]?.remote_ip || "").trim(); - - rows.push({ - protocolName: flow?.[0]?.protocol_name || "", - downBps: down, - upBps: up, - bytesSentDown, - bytesSentUp, - packetsSentDown, - packetsSentUp, - retransmitDownPct, - retransmitUpPct, - rttDownNanos, - rttUpNanos, - qooDown, - qooUp, - asnName: flow?.[0]?.asn_name || "", - asnCountry: flow?.[0]?.asn_country || "", - remoteIp, - opacity: 1.0 - Math.min(1, lastSeenNanos / RECENT_TRAFFIC_FLOW_WINDOW_NANOS), - sortKeys: { - protocol: flow?.[0]?.protocol_name || "", - rate: average.values.length > 0 ? average.total / average.values.length : currentRate, - bytes: bytesSentDown + bytesSentUp, - packets: packetsSentDown + packetsSentUp, - retransmits: retransmitDownPct + retransmitUpPct, - rtt: rttDownNanos + rttUpNanos, - qoo: (typeof qooDown === "number" ? qooDown : 0) + (typeof qooUp === "number" ? qooUp : 0), - asn: flow?.[0]?.asn_name || "", - country: flow?.[0]?.asn_country || "", - ip: remoteIp, - }, - }); - }); - - Array.from(prevFlowBytes.keys()).forEach((key) => { - if (!seenFlowKeys.has(key)) { - prevFlowBytes.delete(key); - movingAverages.delete(key); - } - }); - - latestTrafficRows = rows; + table.appendChild(tbody); + tableWrap.appendChild(table); + clearDiv(target); + target.appendChild(tableWrap); + updateTopAsnCountBadge(); } function renderTrafficTab() { @@ -1235,17 +1687,15 @@ function renderTrafficTab() { } const visibleRows = visibleTrafficRows().slice(); - sortTrafficRows(visibleRows); - const totalPages = Math.max(1, Math.ceil(visibleRows.length / trafficPageSize)); + const totalPages = Math.max(1, Math.ceil(toNumber(latestTrafficPage?.total_rows, 0) / trafficPageSize)); trafficCurrentPage = Math.min(Math.max(1, trafficCurrentPage), totalPages); - const startIndex = (trafficCurrentPage - 1) * trafficPageSize; - const pagedRows = visibleRows.slice(startIndex, startIndex + trafficPageSize); + const pagedRows = visibleRows; let tableWrap = document.createElement("div"); tableWrap.classList.add("lqos-table-wrap"); let table = document.createElement("table"); - table.classList.add("lqos-table", "lqos-table-tight"); + table.classList.add("lqos-table", "lqos-table-tight", "lqos-circuit-traffic-table"); let thead = document.createElement("thead", "small"); thead.style.fontSize = "0.8em"; @@ -1260,7 +1710,7 @@ function renderTrafficTab() { trafficSortDirection = "desc"; } trafficCurrentPage = 1; - renderTrafficTab(); + requestTrafficFlowsPage(); }; if (trafficSortColumn === sortKey) { th.innerHTML += trafficSortDirection === "asc" ? " ▲" : " ▼"; @@ -1294,24 +1744,24 @@ function renderTrafficTab() { pagedRows.forEach((rowData) => { let row = document.createElement("tr"); row.classList.add("small"); - row.style.opacity = rowData.opacity; + row.style.opacity = toNumber(rowData.opacity, 1); - row.appendChild(simpleRow(rowData.protocolName)); - row.appendChild(simpleRowHtml(formatThroughput(rowData.downBps, plan.down))); - row.appendChild(simpleRowHtml(formatThroughput(rowData.upBps, plan.up))); - row.appendChild(simpleRow(scaleNumber(rowData.bytesSentDown))); - row.appendChild(simpleRow(scaleNumber(rowData.bytesSentUp))); - row.appendChild(simpleRow(scaleNumber(rowData.packetsSentDown))); - row.appendChild(simpleRow(scaleNumber(rowData.packetsSentUp))); - row.appendChild(simpleRowHtml(rowData.retransmitDownPct > 0 ? formatRetransmit(rowData.retransmitDownPct) : "-")); - row.appendChild(simpleRowHtml(rowData.retransmitUpPct > 0 ? formatRetransmit(rowData.retransmitUpPct) : "-")); - row.appendChild(simpleRowHtml(formatRttNanos(rowData.rttDownNanos))); - row.appendChild(simpleRowHtml(formatRttNanos(rowData.rttUpNanos))); - row.appendChild(simpleRowHtml(formatQooScore(rowData.qooDown))); - row.appendChild(simpleRowHtml(formatQooScore(rowData.qooUp))); - row.appendChild(simpleRow(rowData.asnName)); - row.appendChild(simpleRow(rowData.asnCountry)); - row.appendChild(simpleRow(rowData.remoteIp)); + row.appendChild(truncatedTrafficCell(rowData.protocol_name, "lqos-circuit-traffic-protocol-cell")); + row.appendChild(simpleRowHtml(formatThroughput(rowData.down_bps, plan.down))); + row.appendChild(simpleRowHtml(formatThroughput(rowData.up_bps, plan.up))); + row.appendChild(simpleRow(scaleNumber(rowData.bytes_sent_down))); + row.appendChild(simpleRow(scaleNumber(rowData.bytes_sent_up))); + row.appendChild(simpleRow(scaleNumber(rowData.packets_sent_down))); + row.appendChild(simpleRow(scaleNumber(rowData.packets_sent_up))); + row.appendChild(simpleRowHtml(rowData.retransmit_down_pct > 0 ? formatRetransmitFraction(rowData.retransmit_down_pct) : "-")); + row.appendChild(simpleRowHtml(rowData.retransmit_up_pct > 0 ? formatRetransmitFraction(rowData.retransmit_up_pct) : "-")); + row.appendChild(simpleRowHtml(formatRttNanos(rowData.rtt_down_nanos))); + row.appendChild(simpleRowHtml(formatRttNanos(rowData.rtt_up_nanos))); + row.appendChild(simpleRowHtml(formatQooScore(rowData.qoo_down))); + row.appendChild(simpleRowHtml(formatQooScore(rowData.qoo_up))); + row.appendChild(truncatedTrafficCell(rowData.asn_name, "lqos-circuit-traffic-asn-cell")); + row.appendChild(truncatedTrafficCell(rowData.asn_country, "lqos-circuit-traffic-country-cell")); + row.appendChild(simpleRow(rowData.remote_ip)); const td = document.createElement("td"); td.classList.add("text-center"); @@ -1319,9 +1769,9 @@ function renderTrafficTab() { button.type = "button"; button.className = "btn btn-outline-secondary btn-sm flow-rtt-exclude-btn"; button.textContent = "Exclude"; - button.disabled = !rowData.remoteIp; + button.disabled = !rowData.remote_ip; button.title = "Open a wizard to exclude RTT samples for this remote IP/CIDR (requires saving in Flow Tracking config)."; - button.dataset.remoteIp = rowData.remoteIp; + button.dataset.remoteIp = rowData.remote_ip; td.appendChild(button); row.appendChild(td); @@ -1337,41 +1787,6 @@ function renderTrafficTab() { updateTrafficPaginationControls(); } -function updateSpeedometer(devices) { - let totalDown = 0; - let totalUp = 0; - let planDown = 0; - let planUp = 0; - let retransmitsDown = 0; - let retransmitsUp = 0; - devices.forEach((device) => { - const deviceDown = toNumber(device.bytes_per_second.down, 0); - const deviceUp = toNumber(device.bytes_per_second.up, 0); - totalDown += deviceDown; - totalUp += deviceUp; - planDown = Math.max(planDown, toNumber(device.plan.down, 0)); - planUp = Math.max(planUp, toNumber(device.plan.up, 0)); - retransmitsDown += toNumber(device.tcp_retransmits.down, 0); - retransmitsUp += toNumber(device.tcp_retransmits.up, 0); - - let throughputGraph = deviceGraphs["throughputGraph_" + device.device_id]; - if (throughputGraph !== undefined) { - throughputGraph.update(deviceDown * 8, deviceUp * 8); - } - - let retransmitGraph = deviceGraphs["tcpRetransmitsGraph_" + device.device_id]; - if (retransmitGraph !== undefined) { - retransmitGraph.update( - toNumber(device.tcp_retransmits.down, 0), - toNumber(device.tcp_retransmits.up, 0) - ); - } - }); - speedometer.update(totalDown * 8, totalUp * 8, planDown, planUp); - totalThroughput.update(totalDown * 8, totalUp * 8); - totalRetransmits.update(retransmitsDown, retransmitsUp); -} - function fillLiveDevices(devices) { devices.forEach((device) => { let last_seen = document.getElementById("last_seen_" + device.device_id); @@ -1388,15 +1803,15 @@ function fillLiveDevices(devices) { if (throughputDown !== null) { throughputDown.innerHTML = formatThroughput( - toNumber(device.bytes_per_second.down, 0) * 8, - toNumber(device.plan.down, 0) + toNumber(device.bytes_per_second?.down, 0) * 8, + toNumber(device.plan?.down, 0) ); } if (throughputUp !== null) { throughputUp.innerHTML = formatThroughput( - toNumber(device.bytes_per_second.up, 0) * 8, - toNumber(device.plan.up, 0) + toNumber(device.bytes_per_second?.up, 0) * 8, + toNumber(device.plan?.up, 0) ); } @@ -1423,11 +1838,15 @@ function fillLiveDevices(devices) { } if (tcp_retransmitsDown !== null) { - tcp_retransmitsDown.innerHTML = formatRetransmit(device.tcp_retransmits.down); + tcp_retransmitsDown.innerHTML = formatRetransmitFraction( + retransmitFractionFromSample(device.tcp_retransmit_sample?.down) + ); } if (tcp_retransmitsUp !== null) { - tcp_retransmitsUp.innerHTML = formatRetransmit(device.tcp_retransmits.up); + tcp_retransmitsUp.innerHTML = formatRetransmitFraction( + retransmitFractionFromSample(device.tcp_retransmit_sample?.up) + ); } // Local RTT histogram (5-minute window, p50 samples) @@ -1469,12 +1888,16 @@ function initialDevices(circuits) { outer.classList.add("col-12", "mb-3"); target.appendChild(outer); + let card = document.createElement("div"); + card.classList.add("lqos-circuit-device-card"); + outer.appendChild(card); + let row = document.createElement("div"); row.classList.add("row", "g-2"); - outer.appendChild(row); + card.appendChild(row); let d = document.createElement("div"); - d.classList.add("col-3"); + d.classList.add("col-12", "col-xl-5", "col-xxl-4", "lqos-circuit-device-summary"); row.appendChild(d); // Device Information Section @@ -1677,18 +2100,19 @@ function initialDevices(circuits) { // Graph container (2x2) let graphCol = document.createElement("div"); - graphCol.classList.add("col-9"); + graphCol.classList.add("col-12", "col-xl-7", "col-xxl-8", "lqos-circuit-device-graphs"); row.appendChild(graphCol); let graphRow = document.createElement("div"); - graphRow.classList.add("row", "g-2"); + graphRow.classList.add("row", "g-2", "lqos-circuit-device-graphs-row"); graphCol.appendChild(graphRow); function addGraph(divId, graphFactory) { let col = document.createElement("div"); - col.classList.add("col-6"); + col.classList.add("col-12", "col-md-6"); let div = document.createElement("div"); div.id = divId; + div.classList.add("lqos-circuit-device-graph"); div.style.height = "250px"; div.innerHTML = loadingBlockHtml("Loading chart…", "lqos-loading-block-sm"); col.appendChild(div); @@ -1709,18 +2133,18 @@ function initialDevices(circuits) { function initialFunnel(parentNode) { funnelParentNodeName = parentNode; - listenOnce("NetworkTree", (msg) => { + listenOnce("NetworkTreeLite", (msg) => { renderFunnel(resolveFunnelState(msg, parentNode)); if (funnelSubscription) { funnelSubscription.dispose(); } - funnelSubscription = subscribeWS(["NetworkTree"], onTreeEvent); + funnelSubscription = subscribeWS(["NetworkTreeLite"], onTreeEvent); }); - wsClient.send({ NetworkTree: {} }); + wsClient.send({ NetworkTreeLite: {} }); } function onTreeEvent(msg) { - if (msg.event !== "NetworkTree" || !funnelParentNodeName) { + if (msg.event !== "NetworkTreeLite" || !funnelParentNodeName) { return; } @@ -1748,8 +2172,8 @@ function onTreeEvent(msg) { toNumber(myMessage.current_throughput[1], 0) * 8 ); let rxmit = [0, 0]; - const packetsDown = toNumber(myMessage.current_tcp_packets[0], 0); - const packetsUp = toNumber(myMessage.current_tcp_packets[1], 0); + const packetsDown = retransmitPacketsForNode(myMessage, 0); + const packetsUp = retransmitPacketsForNode(myMessage, 1); const retransmitsDown = toNumber(myMessage.current_retransmits[0], 0); const retransmitsUp = toNumber(myMessage.current_retransmits[1], 0); if (retransmitsDown > 0 && packetsDown > 0) { @@ -1785,6 +2209,7 @@ function subscribeToCake() { }, (msg) => { //console.log(msg); latestCakeMsg = msg; + setQueueTypeDisplayFromKinds(msg?.kind_down, msg?.kind_up); // Clear the timeout and set flag that we've received data if (noDataTimeout) { @@ -1852,8 +2277,8 @@ function wireupAnalysis(circuits) { //console.log(url); // Restore the buttons - requestCircuitById((circuits) => { - wireupAnalysis(circuits); + requestCircuitById((payload) => { + wireupAnalysis(payload.devices || []); }); } return; @@ -1885,19 +2310,24 @@ function loadInitial() { initFlowFilters(); initQueuingActivityControls(); loadRttThresholds(); - requestCircuitById((circuits) => { + requestCircuitById((payload) => { + const circuits = payload.devices || []; + const advisory = payload.ethernet_advisory || null; let circuit = circuits[0]; + circuitConfigDevices = circuits; $("#circuitName").text(circuit.circuit_name); $("#circuitName").attr("title", circuit.circuit_name || ""); applyParentNodeLink(circuit.parent_node); - $("#bwMax").text(formatMbps(circuit.download_max_mbps) + " / " + formatMbps(circuit.upload_max_mbps)); - $("#bwMin").text(formatMbps(circuit.download_min_mbps) + " / " + formatMbps(circuit.upload_min_mbps)); + $("#bwMax").text(formatPlanSpeedPair(circuit.download_max_mbps, circuit.upload_max_mbps)); + $("#bwMin").text(formatPlanSpeedPair(circuit.download_min_mbps, circuit.upload_min_mbps)); + renderEthernetAdvisory(advisory); plan = { down: toNumber(circuit.download_max_mbps, 0), up: toNumber(circuit.upload_max_mbps, 0), }; latestCircuitDevices = circuits; - setQueueTypeDisplay(circuit.sqm_override || ""); + circuitSqmOverride = circuit.sqm_override || ""; + setQueueTypeDisplayFromKinds(latestCakeMsg?.kind_down, latestCakeMsg?.kind_up); initialDevices(circuits); speedometer = new BitsPerSecondGauge("bitsGauge", "Plan"); qooGauge = new QooScoreGauge("qooGauge"); @@ -1906,9 +2336,7 @@ function loadInitial() { initTabLifecycle(circuit.parent_node); updateQueuingActivityCards(); - connectPrivateChannel(); - connectPingers(circuits); - connectFlowChannel(); + connectCircuitSummaryChannel(); subscribeToCake(); wireupAnalysis(circuits); }, () => { @@ -1918,6 +2346,7 @@ function loadInitial() { function cleanupCircuitPage() { if (channelLink) { + wsClient.send({ Private: { StopCircuitWatcher: null } }); channelLink.close(); channelLink = null; } @@ -1926,13 +2355,12 @@ function cleanupCircuitPage() { cakeChannel = null; } if (pinger) { - pinger.close(); - pinger = null; - } - if (flowChannel) { - flowChannel.close(); - flowChannel = null; + stopPingMonitor(); } + devicePollTimer = clearPollingTimer(devicePollTimer); + sankeyPollTimer = clearPollingTimer(sankeyPollTimer); + topAsnPollTimer = clearPollingTimer(topAsnPollTimer); + trafficPollTimer = clearPollingTimer(trafficPollTimer); if (funnelSubscription) { funnelSubscription.dispose(); funnelSubscription = null; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config/config_helper.js b/src/rust/lqosd/src/node_manager/js_build/src/config/config_helper.js index 30e15573..17959146 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config/config_helper.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config/config_helper.js @@ -34,6 +34,25 @@ function ensureOptionalConfigSections(config) { config.splynx_integration = {}; } + if (!config.sonar_integration || typeof config.sonar_integration !== "object") { + config.sonar_integration = {}; + } + + const sonar = config.sonar_integration; + if (typeof sonar.enable_sonar !== "boolean") sonar.enable_sonar = false; + if (typeof sonar.sonar_api_url !== "string") sonar.sonar_api_url = ""; + if (typeof sonar.sonar_api_key !== "string") sonar.sonar_api_key = ""; + if (typeof sonar.snmp_community !== "string") sonar.snmp_community = "public"; + if (!Array.isArray(sonar.airmax_model_ids)) sonar.airmax_model_ids = []; + if (!Array.isArray(sonar.ltu_model_ids)) sonar.ltu_model_ids = []; + if (!Array.isArray(sonar.active_status_ids)) sonar.active_status_ids = []; + if (!Array.isArray(sonar.recurring_excluded_service_names)) { + sonar.recurring_excluded_service_names = []; + } + if (!Array.isArray(sonar.recurring_service_rates)) { + sonar.recurring_service_rates = []; + } + const splynx = config.splynx_integration; if (typeof splynx.enable_splynx !== "boolean") splynx.enable_splynx = false; if (typeof splynx.api_key !== "string") splynx.api_key = ""; @@ -46,6 +65,34 @@ function ensureOptionalConfigSections(config) { return config; } +const TOPOLOGY_SOURCE_INTEGRATIONS = [ + { name: "UISP", enabled: (config) => !!config?.uisp_integration?.enable_uisp }, + { name: "Splynx", enabled: (config) => !!config?.splynx_integration?.enable_splynx }, + { name: "Powercode", enabled: (config) => !!config?.powercode_integration?.enable_powercode }, + { name: "Sonar", enabled: (config) => !!config?.sonar_integration?.enable_sonar }, + { name: "Netzur", enabled: (config) => !!config?.netzur_integration?.enable_netzur }, + { name: "VISP", enabled: (config) => !!config?.visp_integration?.enable_visp }, + { name: "WispGate", enabled: (config) => !!config?.wispgate_integration?.enable_wispgate }, +]; + +export function activeTopologySourceIntegrations(config) { + return TOPOLOGY_SOURCE_INTEGRATIONS.filter((entry) => entry.enabled(config)).map( + (entry) => entry.name, + ); +} + +export function topologyEditorsLocked(config) { + return activeTopologySourceIntegrations(config).length > 0; +} + +export function topologyEditorsLockMessage(config) { + const active = activeTopologySourceIntegrations(config); + if (active.length === 0) { + return ""; + } + return `Editing is disabled because these integrations are the source of truth: ${active.join(", ")}.`; +} + export function loadConfig(onComplete, onError) { sendWsRequest( "GetConfig", @@ -156,6 +203,30 @@ export function saveNetworkAndDevices(network_json, shaped_devices, onComplete, ); } +export function saveNetworkJsonOnly(network_json, onComplete, onError) { + if (!network_json || typeof network_json !== "object") { + alert("Invalid network configuration"); + return; + } + + sendWsRequest( + "UpdateNetworkJsonOnlyResult", + { UpdateNetworkJsonOnly: { network_json } }, + (msg) => { + if (onComplete) onComplete(!!msg.ok, msg.message); + }, + (msg) => { + const errorMsg = (msg && msg.message) ? msg.message : "Request failed"; + if (onComplete) onComplete(false, errorMsg); + if (onError) { + onError(msg); + } else { + alert("Error saving network configuration: " + errorMsg); + } + }, + ); +} + export function adminCheck(onComplete, onError) { sendWsRequest( "AdminCheck", @@ -201,9 +272,129 @@ export function loadNetworkJson(onComplete, onError) { } export function loadAllShapedDevices(onComplete, onError) { + const pageSize = 250; + const rows = []; + + const loadPage = (page) => { + sendWsRequest( + "ShapedDevicesPage", + { ShapedDevicesPage: { query: { page, page_size: pageSize } } }, + (msg) => { + const data = msg && msg.data ? msg.data : {}; + const pageRows = Array.isArray(data.rows) ? data.rows : []; + rows.push(...pageRows); + const totalRows = Number.isFinite(Number(data.total_rows)) + ? Math.max(0, Math.trunc(Number(data.total_rows))) + : rows.length; + if (rows.length >= totalRows || pageRows.length < pageSize) { + if (onComplete) onComplete(rows); + return; + } + loadPage(page + 1); + }, + onError, + ); + }; + + loadPage(0); +} + +export function loadShapedDevicesPage(query, onComplete, onError) { sendWsRequest( - "AllShapedDevices", - { AllShapedDevices: {} }, + "ShapedDevicesPage", + { ShapedDevicesPage: { query } }, + (msg) => { + if (onComplete) onComplete(msg.data || null); + }, + onError, + ); +} + +export function getShapedDevice(deviceId, onComplete, onError) { + sendWsRequest( + "GetShapedDeviceResult", + { GetShapedDevice: { device_id: deviceId } }, + (msg) => { + if (onComplete) onComplete(msg); + }, + onError, + ); +} + +export function createShapedDevice(device, onComplete, onError) { + sendWsRequest( + "CreateShapedDeviceResult", + { CreateShapedDevice: { device } }, + (msg) => { + if (onComplete) onComplete(msg); + }, + onError, + ); +} + +export function updateShapedDevice(originalDeviceId, device, onComplete, onError) { + sendWsRequest( + "UpdateShapedDeviceResult", + { UpdateShapedDevice: { original_device_id: originalDeviceId, device } }, + (msg) => { + if (onComplete) onComplete(msg); + }, + onError, + ); +} + +export function deleteShapedDevice(deviceId, onComplete, onError) { + sendWsRequest( + "DeleteShapedDeviceResult", + { DeleteShapedDevice: { device_id: deviceId } }, + (msg) => { + if (onComplete) onComplete(msg); + }, + onError, + ); +} + +export function loadCircuitDirectoryPage(query, onComplete, onError) { + sendWsRequest( + "CircuitDirectoryPage", + { CircuitDirectoryPage: { query } }, + (msg) => { + if (onComplete) onComplete(msg.data || null); + }, + onError, + ); +} + +export function loadAllCircuitDirectoryRows(onComplete, onError) { + const pageSize = 250; + const rows = []; + + const loadPage = (page) => { + loadCircuitDirectoryPage( + { page, page_size: pageSize }, + (data) => { + const pageRows = Array.isArray(data?.rows) ? data.rows : []; + rows.push(...pageRows); + const totalRows = Number.isFinite(Number(data?.total_rows)) + ? Math.max(0, Math.trunc(Number(data.total_rows))) + : rows.length; + if (rows.length >= totalRows || pageRows.length < pageSize) { + if (onComplete) onComplete(rows); + return; + } + loadPage(page + 1); + }, + onError, + ); + }; + + loadPage(0); +} + +export function loadNodeDirectory(onComplete, onError) { + sendWsRequest( + "NodeDirectory", + { NodeDirectory: {} }, (msg) => { if (onComplete) onComplete(msg.data || []); }, @@ -211,6 +402,17 @@ export function loadAllShapedDevices(onComplete, onError) { ); } +export function loadTreeGuardMetadataSummary(onComplete, onError) { + sendWsRequest( + "TreeGuardMetadataSummary", + { TreeGuardMetadataSummary: {} }, + (msg) => { + if (onComplete) onComplete(msg.data || null); + }, + onError, + ); +} + export function getUsers(onComplete, onError) { sendWsRequest( "GetUsers", @@ -301,8 +503,8 @@ export function renderConfigMenu(currentPage) {
    ${menuItems.map(item => ` -
  • - +
  • + ${item.text}
    • diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config/treeguard_defaults.js b/src/rust/lqosd/src/node_manager/js_build/src/config/treeguard_defaults.js index 7ca9cf89..e2f41bfd 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config/treeguard_defaults.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config/treeguard_defaults.js @@ -4,7 +4,7 @@ export function defaultTreeguardConfig() { dry_run: false, tick_seconds: 1, cpu: { - mode: "traffic_rtt_only", + mode: "cpu_aware", cpu_high_pct: 75, cpu_low_pct: 55, }, @@ -38,7 +38,7 @@ export function defaultTreeguardConfig() { }, qoo: { enabled: true, - min_score: 80.0, + min_score: 70.0, }, }; } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_devices.js b/src/rust/lqosd/src/node_manager/js_build/src/config_devices.js index c9c22a3a..38f740bf 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_devices.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_devices.js @@ -1,21 +1,52 @@ import { - loadAllShapedDevices, + loadConfig, + createShapedDevice, + deleteShapedDevice, loadNetworkJson, + loadShapedDevicesPage, renderConfigMenu, - saveNetworkAndDevices, + topologyEditorsLockMessage, + topologyEditorsLocked, + updateShapedDevice, validNodeList, } from "./config/config_helper"; -let shaped_devices = []; +let current_rows = []; let network_json = null; -let filtered_indices = []; +let total_rows = 0; +let total_circuits = 0; let page = 0; let page_size = 25; let search_term = ""; -let invalid_indices = new Set(); -let edit_index = null; +let edit_device_id = null; let creating_new = false; let search_timer = null; +let modal_busy = false; +let topology_editor_locked = false; +let topology_editor_lock_message = ""; + +function setModalReadOnly(readOnly) { + $("#sdEditForm input, #sdEditForm textarea, #sdEditForm select").prop("disabled", readOnly); + $("#sdModalSave").prop("disabled", readOnly || modal_busy); +} + +function applyEditorLockState() { + $("#btnAddDevice").prop("disabled", topology_editor_locked); + $("#sdTableContainer").toggleClass("opacity-75", topology_editor_locked); + + const banner = $("#devicesEditorLock"); + if (topology_editor_locked && topology_editor_lock_message) { + banner.removeClass("d-none").text(topology_editor_lock_message); + } else { + banner.addClass("d-none").text(""); + } + + setModalReadOnly(topology_editor_locked); +} + +function actionButtonAttrs() { + return topology_editor_locked ? " disabled aria-disabled='true'" : ""; +} function defaultDevice() { return { @@ -231,94 +262,93 @@ function buildParentNodeOptions(selectedNode) { } function totalPages() { - return Math.max(1, Math.ceil(filtered_indices.length / page_size)); + return Math.max(1, Math.ceil(total_rows / page_size)); } -function applySearch(resetPage = true) { - const term = (search_term || "").toLowerCase().trim(); - filtered_indices = []; +function currentQuery() { + return { + page, + page_size, + search: search_term.trim().length > 0 ? search_term.trim() : undefined, + }; +} - if (!term) { - for (let i = 0; i < shaped_devices.length; i++) { - filtered_indices.push(i); - } - } else { - for (let i = 0; i < shaped_devices.length; i++) { - const device = shaped_devices[i] || {}; - const parts = [ - device.circuit_id, - device.circuit_name, - device.device_id, - device.device_name, - device.parent_node, - device.mac, - ]; +function renderLoading() { + $("#sdTableContainer").html( + "
    ", + ); +} - if (Array.isArray(device.ipv4)) { - device.ipv4.forEach((tuple) => parts.push(formatIpTuple(tuple, 32))); - } - if (Array.isArray(device.ipv6)) { - device.ipv6.forEach((tuple) => parts.push(formatIpTuple(tuple, 128))); +function requestPage() { + renderLoading(); + $("#btnAddDevice").prop("disabled", true); + $("#btnSaveDevices").prop("disabled", true); + loadShapedDevicesPage( + currentQuery(), + (data) => { + current_rows = Array.isArray(data?.rows) ? data.rows : []; + total_rows = Number.isFinite(Number(data?.total_rows)) + ? Math.max(0, Math.trunc(Number(data.total_rows))) + : current_rows.length; + total_circuits = Number.isFinite(Number(data?.total_circuits)) + ? Math.max(0, Math.trunc(Number(data.total_circuits))) + : 0; + + const totalPagesForResult = Math.max(1, Math.ceil(total_rows / page_size)); + if (page >= totalPagesForResult && total_rows > 0) { + page = totalPagesForResult - 1; + requestPage(); + return; } - const haystack = parts - .filter((val) => val !== undefined && val !== null) - .join(" ") - .toLowerCase(); - - if (haystack.includes(term)) { - filtered_indices.push(i); - } - } - } - - if (resetPage) { - page = 0; - } else { - page = Math.min(page, totalPages() - 1); - } - - render(); + render(); + $("#btnAddDevice").prop("disabled", topology_editor_locked); + $("#btnSaveDevices").prop("disabled", false); + applyEditorLockState(); + }, + () => { + $("#sdTableContainer").html( + "
    Failed to load shaped devices.
    ", + ); + $("#btnAddDevice").prop("disabled", topology_editor_locked); + $("#btnSaveDevices").prop("disabled", false); + }, + ); } function setPage(newPage) { const total = totalPages(); page = Math.min(Math.max(newPage, 0), total - 1); - render(); + requestPage(); } function setPageSize(newSize) { page_size = newSize; page = 0; - render(); + requestPage(); } function renderSummary() { - const total = shaped_devices.length; - const filtered = filtered_indices.length; - const start = filtered === 0 ? 0 : page * page_size + 1; - const end = filtered === 0 ? 0 : Math.min(page * page_size + page_size, filtered); + const start = total_rows === 0 ? 0 : page * page_size + 1; + const end = total_rows === 0 ? 0 : page * page_size + current_rows.length; let text = ""; - if (filtered === 0) { + if (total_rows === 0) { text = "No devices to display"; - } else if (filtered === total) { - text = `Showing ${start}-${end} of ${total} devices`; + } else if (search_term.trim().length > 0) { + text = `Showing ${start}-${end} of ${total_rows} matching devices`; } else { - text = `Showing ${start}-${end} of ${filtered} devices (filtered from ${total})`; + text = `Showing ${start}-${end} of ${total_rows} devices across ${total_circuits} circuits`; } $("#sdSummary").text(text); } function renderTable() { const container = $("#sdTableContainer"); - if (filtered_indices.length === 0) { + if (current_rows.length === 0) { container.html("
    No devices match the current search.
    "); return; } - const start = page * page_size; - const end = Math.min(start + page_size, filtered_indices.length); - let html = ""; html += ""; @@ -338,21 +368,20 @@ function renderTable() { html += ""; html += ""; - for (let i = start; i < end; i++) { - const idx = filtered_indices[i]; - const row = shaped_devices[idx] || {}; - const invalidClass = invalid_indices.has(idx) ? " table-danger" : ""; + current_rows.forEach((row) => { const comment = row.comment ? String(row.comment) : ""; + const deviceId = escapeHtml(row.device_id || ""); + const actionAttrs = actionButtonAttrs(); - html += ""; + html += ""; html += ""; html += ""; html += @@ -386,14 +415,14 @@ function renderTable() { } html += ""; html += ""; - } + }); html += "
    " + - "" + - "" + + "" + + "" + "" + formatIdCell(row.circuit_id, 5) + "
    "; container.html(html); } function buildPaginationModel(total, current) { - let pages = []; + const pages = []; const add = (value) => { if (value < 0 || value >= total) return; if (!pages.includes(value)) pages.push(value); @@ -407,7 +436,7 @@ function buildPaginationModel(total, current) { add(current + 2); pages.sort((a, b) => a - b); - let result = []; + const result = []; let last = null; pages.forEach((p) => { if (last !== null && p - last > 1) { @@ -423,7 +452,7 @@ function renderPagination() { const total = totalPages(); const listBottom = $("#sdPagination"); const listTop = $("#sdPaginationTop"); - if (filtered_indices.length === 0) { + if (total_rows === 0) { listBottom.html(""); listTop.html(""); return; @@ -531,19 +560,22 @@ function collectModalDevice() { const sqmDown = $("#sdModalSqmDown").val().trim().toLowerCase(); const sqmUp = $("#sdModalSqmUp").val().trim().toLowerCase(); - if (!sqmDown && !sqmUp) { - delete device.sqm_override; - } else { + if (sqmDown || sqmUp) { device.sqm_override = `${sqmDown}/${sqmUp}`; } return device; } -function openEditModal(index) { - const device = shaped_devices[index]; +function findCurrentRow(deviceId) { + return current_rows.find((row) => row.device_id === deviceId); +} + +function openEditModal(deviceId) { + if (topology_editor_locked) return; + const device = findCurrentRow(deviceId); if (!device) return; - edit_index = index; + edit_device_id = device.device_id; creating_new = false; $("#sdEditModalLabel").text("Edit Device"); populateModal(device); @@ -551,39 +583,18 @@ function openEditModal(index) { } function openNewModal() { - edit_index = null; + if (topology_editor_locked) return; + edit_device_id = null; creating_new = true; $("#sdEditModalLabel").text("Add Device"); populateModal(defaultDevice()); showModal(); } -function saveModalChanges() { - const device = collectModalDevice(); - if (creating_new) { - shaped_devices.unshift(device); - } else if (edit_index !== null) { - shaped_devices[edit_index] = device; - } - invalid_indices = new Set(); - hideModal(); - applySearch(creating_new); -} - -function deleteDevice(index) { - const device = shaped_devices[index]; - if (!device) return; - const label = device.device_name || device.device_id || "this device"; - if (!confirm(`Delete ${label}?`)) return; - shaped_devices.splice(index, 1); - invalid_indices = new Set(); - applySearch(false); -} - function checkIpv4(ip) { - const ipv4Pattern = /^(\d{1,3}\.){3}\d{1,3}$/; + const ipv4Pattern = /^(\d{1,3}\.){3}\d{1,3}(\/\d{1,2})?$/; + if (!ipv4Pattern.test(ip)) return false; const parts = ip.split("/"); - if (!ipv4Pattern.test(parts[0])) return false; if (parts.length > 1 && parts[1].length > 0) { const prefix = parseInt(parts[1], 10); if (Number.isNaN(prefix) || prefix < 0 || prefix > 32) return false; @@ -596,148 +607,134 @@ function checkIpv6(ip) { return regex.test(ip); } -function normalizeIp(tuple, family) { - const defaultPrefix = family === 6 ? 128 : 32; - if (!tuple || tuple.length === 0) return ""; - const addr = tuple[0] ? formatIpAddr(tuple[0], family).trim() : ""; - if (!addr) return ""; - const prefix = Number.isFinite(tuple[1]) ? tuple[1] : defaultPrefix; - const normalizedAddr = family === 6 ? addr.toLowerCase() : addr; - return normalizedAddr + "/" + prefix; -} - -function validateDevices() { - let valid = true; - let errors = []; - invalid_indices = new Set(); - +function validateModalDevice(device) { + const errors = []; const nodes = network_json ? validNodeList(network_json) : []; - const deviceIds = new Map(); - const ipv4s = new Map(); - const ipv6s = new Map(); - const markInvalid = (index, message) => { - valid = false; - errors.push(message); - invalid_indices.add(index); - }; + if (!device.circuit_id) errors.push("Circuit ID is required"); + if (!device.circuit_name) errors.push("Circuit Name is required"); + if (!device.device_id) errors.push("Device ID is required"); + if (!device.device_name) errors.push("Device Name is required"); - shaped_devices.forEach((device, index) => { - if (!device.circuit_id || device.circuit_id.trim().length === 0) { - markInvalid(index, "Circuits must have a Circuit ID"); - } - if (!device.circuit_name || device.circuit_name.trim().length === 0) { - markInvalid(index, "Circuits must have a Circuit Name"); - } - if (!device.device_id || device.device_id.trim().length === 0) { - markInvalid(index, "Circuits must have a Device ID"); - } - if (!device.device_name || device.device_name.trim().length === 0) { - markInvalid(index, "Circuits must have a Device Name"); + const parentNode = device.parent_node || ""; + if (nodes.length === 0) { + if (parentNode.length > 0) { + errors.push("You have a flat network, so you can't specify a parent node."); } + } else if (parentNode.length > 0 && nodes.indexOf(parentNode) === -1) { + errors.push("Parent node does not exist."); + } - if (device.device_id) { - const existing = deviceIds.get(device.device_id); - if (existing !== undefined && existing !== index) { - markInvalid(index, `Devices with duplicate ID [${device.device_id}] detected`); - invalid_indices.add(existing); - } else { - deviceIds.set(device.device_id, index); - } - } + const ipv4List = Array.isArray(device.ipv4) ? device.ipv4 : []; + const ipv6List = Array.isArray(device.ipv6) ? device.ipv6 : []; + if (ipv4List.length === 0 && ipv6List.length === 0) { + errors.push("You must specify either an IPv4 or IPv6 (or both) address."); + } - const parentNode = device.parent_node || ""; - if (nodes.length === 0) { - if (parentNode.length > 0) { - markInvalid(index, "You have a flat network, so you can't specify a parent node."); - } - } else if (parentNode.length > 0 && nodes.indexOf(parentNode) === -1) { - markInvalid(index, "Parent node: " + parentNode + " does not exist"); - } - - const ipv4List = Array.isArray(device.ipv4) ? device.ipv4 : []; - const ipv6List = Array.isArray(device.ipv6) ? device.ipv6 : []; - if (ipv4List.length === 0 && ipv6List.length === 0) { - markInvalid(index, "You must specify either an IPv4 or IPv6 (or both) address"); - } - - ipv4List.forEach((tuple) => { - const formatted = normalizeIp(tuple, 4); - if (!formatted) return; - if (!checkIpv4(formatted)) { - markInvalid(index, formatted + " is not a valid IPv4 address"); - } - const dupe = ipv4s.get(formatted); - if (dupe !== undefined && dupe !== index) { - markInvalid(index, formatted + " is a duplicate IP"); - invalid_indices.add(dupe); - } else { - ipv4s.set(formatted, index); - } - }); - - ipv6List.forEach((tuple) => { - const formatted = normalizeIp(tuple, 6); - if (!formatted) return; - if (!checkIpv6(formatted)) { - markInvalid(index, formatted + " is not a valid IPv6 address"); - } - const dupe = ipv6s.get(formatted); - if (dupe !== undefined && dupe !== index) { - markInvalid(index, formatted + " is a duplicate IP"); - invalid_indices.add(dupe); - } else { - ipv6s.set(formatted, index); - } - }); - - const downloadMin = parseFloat(device.download_min_mbps); - const uploadMin = parseFloat(device.upload_min_mbps); - const downloadMax = parseFloat(device.download_max_mbps); - const uploadMax = parseFloat(device.upload_max_mbps); - - if (Number.isNaN(downloadMin)) { - markInvalid(index, "Download min is not a valid number"); - } else if (downloadMin < 0.1) { - markInvalid(index, "Download min must be 0.1 or more"); - } - - if (Number.isNaN(uploadMin)) { - markInvalid(index, "Upload min is not a valid number"); - } else if (uploadMin < 0.1) { - markInvalid(index, "Upload min must be 0.1 or more"); - } - - if (Number.isNaN(downloadMax)) { - markInvalid(index, "Download max is not a valid number"); - } else if (downloadMax < 0.2) { - markInvalid(index, "Download max must be 0.2 or more"); - } - - if (Number.isNaN(uploadMax)) { - markInvalid(index, "Upload max is not a valid number"); - } else if (uploadMax < 0.2) { - markInvalid(index, "Upload max must be 0.2 or more"); + ipv4List.forEach((tuple) => { + const formatted = formatIpTuple(tuple, 32); + if (!formatted || !checkIpv4(formatted)) { + errors.push(`${formatted || "IPv4 entry"} is not a valid IPv4 address.`); } }); - if (!valid) { - let errorMessage = "Invalid ShapedDevices Entries:\n"; - errors.forEach((message) => { - errorMessage += message + "\n"; - }); - alert(errorMessage); + ipv6List.forEach((tuple) => { + const formatted = formatIpTuple(tuple, 128); + if (!formatted || !checkIpv6(formatted)) { + errors.push(`${formatted || "IPv6 entry"} is not a valid IPv6 address.`); + } + }); + + if (!Number.isFinite(device.download_min_mbps) || device.download_min_mbps < 0.1) { + errors.push("Download min must be 0.1 or more."); + } + if (!Number.isFinite(device.upload_min_mbps) || device.upload_min_mbps < 0.1) { + errors.push("Upload min must be 0.1 or more."); + } + if (!Number.isFinite(device.download_max_mbps) || device.download_max_mbps < 0.2) { + errors.push("Download max must be 0.2 or more."); + } + if (!Number.isFinite(device.upload_max_mbps) || device.upload_max_mbps < 0.2) { + errors.push("Upload max must be 0.2 or more."); } - render(); - return { valid, errors }; + const sqm = device.sqm_override ? parseSqmOverride(device.sqm_override) : { down: "", up: "" }; + const validSqm = (token) => token === "" || token === "cake" || token === "fq_codel" || token === "none"; + if (!validSqm(sqm.down) || !validSqm(sqm.up)) { + errors.push("SQM overrides must be blank, cake, fq_codel, or none."); + } + + return { valid: errors.length === 0, errors }; +} + +function setModalBusy(busy) { + modal_busy = busy; + $("#sdModalSave").prop("disabled", busy || topology_editor_locked); +} + +function saveModalChanges() { + if (topology_editor_locked) return; + if (modal_busy) return; + const device = collectModalDevice(); + const validation = validateModalDevice(device); + if (!validation.valid) { + alert(validation.errors.join("\n")); + return; + } + + setModalBusy(true); + const onComplete = (msg) => { + setModalBusy(false); + if (msg && msg.ok) { + hideModal(); + if (creating_new) { + page = 0; + } + requestPage(); + return; + } + alert((msg && msg.message) ? msg.message : "Unable to save shaped device."); + }; + + if (creating_new) { + createShapedDevice(device, onComplete, () => { + setModalBusy(false); + alert("Unable to create shaped device."); + }); + return; + } + + updateShapedDevice(edit_device_id, device, onComplete, () => { + setModalBusy(false); + alert("Unable to update shaped device."); + }); +} + +function deleteDevice(deviceId) { + if (topology_editor_locked) return; + const device = findCurrentRow(deviceId); + const label = device?.device_name || deviceId || "this device"; + if (!confirm(`Delete ${label}?`)) return; + deleteShapedDevice( + deviceId, + (msg) => { + if (msg && msg.ok) { + requestPage(); + return; + } + alert((msg && msg.message) ? msg.message : "Unable to delete shaped device."); + }, + () => { + alert("Unable to delete shaped device."); + }, + ); } function start() { renderConfigMenu("devices"); - $("#btnSaveDevices").prop("disabled", true); $("#btnAddDevice").prop("disabled", true); + $("#btnSaveDevices").prop("disabled", true); const initialPageSize = parseInt($("#sdPageSize").val(), 10); if (!Number.isNaN(initialPageSize)) { page_size = initialPageSize; @@ -748,7 +745,10 @@ function start() { if (search_timer) { clearTimeout(search_timer); } - search_timer = setTimeout(() => applySearch(true), 200); + search_timer = setTimeout(() => { + page = 0; + requestPage(); + }, 200); }); $("#sdPageSize").on("change", (event) => { @@ -756,7 +756,7 @@ function start() { if (!Number.isNaN(value)) setPageSize(value); }); - $("#sdPagination").on("click", "a.page-link", (event) => { + $("#sdPagination, #sdPaginationTop").on("click", "a.page-link", (event) => { event.preventDefault(); const target = parseInt($(event.currentTarget).data("page"), 10); if (!Number.isNaN(target)) setPage(target); @@ -764,14 +764,18 @@ function start() { $("#sdTableContainer").on("click", ".sd-edit", (event) => { event.preventDefault(); - const idx = parseInt($(event.currentTarget).data("index"), 10); - if (!Number.isNaN(idx)) openEditModal(idx); + const deviceId = $(event.currentTarget).data("device-id"); + if (typeof deviceId === "string" && deviceId.length > 0) { + openEditModal(deviceId); + } }); $("#sdTableContainer").on("click", ".sd-delete", (event) => { event.preventDefault(); - const idx = parseInt($(event.currentTarget).data("index"), 10); - if (!Number.isNaN(idx)) deleteDevice(idx); + const deviceId = $(event.currentTarget).data("device-id"); + if (typeof deviceId === "string" && deviceId.length > 0) { + deleteDevice(deviceId); + } }); $("#btnAddDevice").on("click", (event) => { @@ -784,41 +788,34 @@ function start() { saveModalChanges(); }); - $("#btnSaveDevices").on("click", () => { - const validation = validateDevices(); - if (!validation.valid) { - return; - } - saveNetworkAndDevices(network_json, shaped_devices, (success, message) => { - if (success) { - alert("Configuration saved successfully!"); - } else { - alert("Failed to save configuration: " + message); - } - }); + $("#btnSaveDevices").on("click", (event) => { + event.preventDefault(); + requestPage(); }); - loadNetworkJson( - (njs) => { - network_json = njs; - loadAllShapedDevices( - (data) => { - shaped_devices = Array.isArray(data) ? data : []; - invalid_indices = new Set(); - applySearch(true); - $("#btnSaveDevices").prop("disabled", false); - $("#btnAddDevice").prop("disabled", false); + loadConfig( + (msg) => { + const config = msg?.data || window.config || {}; + topology_editor_locked = topologyEditorsLocked(config); + topology_editor_lock_message = topologyEditorsLockMessage(config); + applyEditorLockState(); + + loadNetworkJson( + (njs) => { + network_json = njs; + requestPage(); + applyEditorLockState(); }, () => { $("#sdTableContainer").html( - "
    Failed to load shaped devices.
    ", + "
    Failed to load network configuration.
    ", ); }, ); }, () => { $("#sdTableContainer").html( - "
    Failed to load network configuration.
    ", + "
    Failed to load configuration.
    ", ); }, ); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_iprange.js b/src/rust/lqosd/src/node_manager/js_build/src/config_iprange.js index 439eb40e..5aed0dd2 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_iprange.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_iprange.js @@ -1,30 +1,43 @@ import {saveConfig, loadConfig, renderConfigMenu} from "./config/config_helper"; +function isValidIPv4(ip) { + if (!/^(\d{1,3}\.){3}\d{1,3}$/.test(ip)) return false; + const parts = ip.split('.').map((p) => parseInt(p, 10)); + return parts.length === 4 && !parts.some((p) => Number.isNaN(p) || p < 0 || p > 255); +} + +function isValidIPv6(ip) { + return ip.includes(':') && /^[0-9a-fA-F:]+$/.test(ip); +} + +function normalizeSubnetInput(value) { + const raw = String(value || "").trim(); + if (!raw) return ""; + if (raw.includes('/')) return raw; + if (isValidIPv4(raw)) return `${raw}/32`; + if (isValidIPv6(raw)) return `${raw}/128`; + return raw; +} + function isValidCIDR(cidr) { try { - const [ip, mask] = cidr.split('/'); - if (!ip || !mask) return false; - - // Validate IP address + const [ip, mask, extra] = String(cidr).trim().split('/'); + if (!ip || !mask || extra !== undefined) return false; + if (ip.includes(':')) { - // IPv6 - if (!/^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$/.test(ip)) return false; - } else { - // IPv4 - if (!/^(\d{1,3}\.){3}\d{1,3}$/.test(ip)) return false; + if (!isValidIPv6(ip)) return false; + } else if (!isValidIPv4(ip)) { + return false; } - - // Validate mask - const maskNum = parseInt(mask); - if (isNaN(maskNum)) return false; + + const maskNum = parseInt(mask, 10); + if (Number.isNaN(maskNum)) return false; if (ip.includes(':')) { - // IPv6 if (maskNum < 0 || maskNum > 128) return false; } else { - // IPv4 if (maskNum < 0 || maskNum > 32) return false; } - + return true; } catch { return false; @@ -44,17 +57,16 @@ function populateSubnetList(selectId, subnets) { function addSubnet(listId, inputId) { const input = document.getElementById(inputId); - const cidr = input.value.trim(); - + const cidr = normalizeSubnetInput(input.value); + if (!isValidCIDR(cidr)) { - alert('Please enter a valid CIDR notation (e.g. 192.168.1.0/24 or 2001:db8::/32)'); + alert('Please enter a valid IP or CIDR notation (e.g. 192.168.1.0/24, 2803:16d0:40::/48, or 2001:db8::1)'); return; } - + const select = document.getElementById(listId); - // Check for duplicates for (let i = 0; i < select.options.length; i++) { - if (select.options[i].value === cidr) { + if (select.options[i].value.toLowerCase() === cidr.toLowerCase()) { alert('This CIDR is already in the list'); return; } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_lts.js b/src/rust/lqosd/src/node_manager/js_build/src/config_lts.js index 3aad7af3..9178cb96 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_lts.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_lts.js @@ -20,7 +20,7 @@ function validateConfig() { try { new URL(ltsUrl); } catch { - alert("LTS Server URL must be a valid URL"); + alert("Insight Server URL must be a valid URL"); return false; } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_network.js b/src/rust/lqosd/src/node_manager/js_build/src/config_network.js index 38a54565..a4da1ac8 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_network.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_network.js @@ -1,48 +1,73 @@ import { + activeTopologySourceIntegrations, loadAllShapedDevices, + loadConfig, loadNetworkJson, renderConfigMenu, saveNetworkAndDevices, + saveNetworkJsonOnly, + topologyEditorsLockMessage, + topologyEditorsLocked, } from "./config/config_helper"; let network_json = null; let shaped_devices = null; +let shaped_devices_dirty = false; const MIN_NODE_MBPS = 0.1; +let topology_editor_locked = false; +let topology_editor_lock_message = ""; + +function editorButtonAttrs() { + return topology_editor_locked ? " disabled aria-disabled='true'" : ""; +} + +function applyEditorLockState() { + $("#btnSaveNetwork").prop("disabled", topology_editor_locked); + $("#netjson").toggleClass("opacity-75", topology_editor_locked); + + const banner = $("#networkEditorLock"); + if (topology_editor_locked && topology_editor_lock_message) { + banner.removeClass("d-none").text(topology_editor_lock_message); + } else { + banner.addClass("d-none").text(""); + } +} function renderNetworkNode(level, depth) { - let html = `
    `; + let html = `
    `; html += `
    `; for (const [key, value] of Object.entries(level)) { const isVirtual = value && value.virtual === true; + const actionAttrs = editorButtonAttrs(); // Node header with actions - html += `
    `; - html += `
    ${key}${isVirtual ? ' Virtual' : ''}
    `; - html += `
    `; + html += `
    `; + html += `
    ${key}${isVirtual ? ' Virtual' : ''}
    `; + html += `
    `; if (depth > 0) { - html += ``; } - html += ``; - html += ``; - html += ``; html += `
    `; // Node details - html += `
    `; - html += `Download: ${value.downloadBandwidthMbps} Mbps`; - html += ``; - html += `Upload: ${value.uploadBandwidthMbps} Mbps`; - html += ``; html += `
    `; @@ -59,13 +84,14 @@ function renderNetworkNode(level, depth) { function renderNetwork() { if (!network_json || Object.keys(network_json).length === 0) { - $("#netjson").html(`
    No network nodes found. Add one to get started!
    `); + $("#netjson").html(`
    No network nodes found. Add one to get started.
    `); return; } $("#netjson").html(renderNetworkNode(network_json, 0)); } function promoteNode(nodeId) { + if (topology_editor_locked) return; console.log("Promoting ", nodeId); let previousParent = null; @@ -89,6 +115,7 @@ function promoteNode(nodeId) { } function nodeSpeedChange(nodeId, direction) { + if (topology_editor_locked) return; let newVal = prompt(`New ${direction === 'd' ? 'download' : 'upload'} value in Mbps`); newVal = parseFloat(newVal); if (isNaN(newVal)) { @@ -121,6 +148,7 @@ function nodeSpeedChange(nodeId, direction) { } function toggleVirtualNode(nodeId) { + if (topology_editor_locked) return; function iterate(tree) { for (const [key, value] of Object.entries(tree)) { if (key === nodeId) { @@ -138,6 +166,7 @@ function toggleVirtualNode(nodeId) { } function deleteNode(nodeId) { + if (topology_editor_locked) return; if (!confirm(`Are you sure you want to delete ${nodeId} and all its children?`)) { return; } @@ -188,6 +217,7 @@ function deleteNode(nodeId) { let sd = shaped_devices[i]; if (deleteList.indexOf(sd.parent_node) > -1) { sd.parent_node = ""; + shaped_devices_dirty = true; } } alert("Because there was no obvious parent, you may have to fix some parenting in your Shaped Devices list."); @@ -197,6 +227,7 @@ function deleteNode(nodeId) { let sd = shaped_devices[i]; if (deleteList.indexOf(sd.parent_node) > -1) { sd.parent_node = deleteParent; + shaped_devices_dirty = true; } } } @@ -207,6 +238,7 @@ function deleteNode(nodeId) { } function renameNode(nodeId) { + if (topology_editor_locked) return; let newName = prompt("New node name?"); if (!newName || newName.trim() === "") { alert("Please enter a valid name"); @@ -254,6 +286,7 @@ function renameNode(nodeId) { let sd = shaped_devices[i]; if (sd.parent_node === nodeId) { sd.parent_node = newName; + shaped_devices_dirty = true; } } @@ -275,34 +308,71 @@ function start() { // Add save button handler // Add network save button handler $("#btnSaveNetwork").on('click', () => { + if (topology_editor_locked) { + alert(topology_editor_lock_message); + return; + } // Validate network structure if (!network_json || Object.keys(network_json).length === 0) { alert("Network configuration is empty"); return; } - // Save with empty shaped_devices since we're only saving network - saveNetworkAndDevices(network_json, shaped_devices, (success, message) => { + const onComplete = (success, message) => { if (success) { + shaped_devices_dirty = false; alert(message); } else { alert("Failed to save network configuration: " + message); } - }); + }; + + if (shaped_devices_dirty) { + saveNetworkAndDevices(network_json, shaped_devices, onComplete); + return; + } + + saveNetworkJsonOnly(network_json, onComplete); }); - // Load network data - loadAllShapedDevices((data) => { - shaped_devices = data; - loadNetworkJson((njs) => { - network_json = njs; - renderNetwork(); - }, () => { - alert("Failed to load network configuration"); - }); - }, () => { - alert("Failed to load shaped devices"); - }); + loadConfig( + (msg) => { + const config = msg?.data || window.config || {}; + topology_editor_locked = topologyEditorsLocked(config); + topology_editor_lock_message = topologyEditorsLockMessage(config); + const activeIntegrations = activeTopologySourceIntegrations(config); + if (!topology_editor_lock_message && activeIntegrations.length > 0) { + topology_editor_lock_message = `Editing is disabled because these integrations are the source of truth: ${activeIntegrations.join(", ")}.`; + } + applyEditorLockState(); + + const finishLoad = () => { + loadNetworkJson((njs) => { + network_json = njs; + renderNetwork(); + applyEditorLockState(); + }, () => { + alert("Failed to load network configuration"); + }); + }; + + if (topology_editor_locked) { + shaped_devices = []; + finishLoad(); + return; + } + + loadAllShapedDevices((data) => { + shaped_devices = data; + finishLoad(); + }, () => { + alert("Failed to load shaped devices"); + }); + }, + () => { + alert("Failed to load configuration"); + }, + ); } $(document).ready(start); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_queues.js b/src/rust/lqosd/src/node_manager/js_build/src/config_queues.js index df3080c6..70ebf62a 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_queues.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_queues.js @@ -51,9 +51,10 @@ function validateConfig() { function updateConfig() { // Update only the queues section + const queueMode = document.getElementById("queueMode").value; window.config.queues = { default_sqm: document.getElementById("defaultSqm").value, - monitor_only: document.getElementById("monitorOnly").checked, + queue_mode: queueMode, uplink_bandwidth_mbps: parseInt(document.getElementById("uplinkBandwidth").value), downlink_bandwidth_mbps: parseInt(document.getElementById("downlinkBandwidth").value), generated_pn_download_mbps: parseInt(document.getElementById("generatedPnDownload").value), @@ -82,9 +83,9 @@ loadConfig(() => { // Select field document.getElementById("defaultSqm").value = queues.default_sqm; + document.getElementById("queueMode").value = queues.queue_mode ?? ((queues.monitor_only ?? false) ? "observe" : "shape"); // Boolean fields - document.getElementById("monitorOnly").checked = queues.monitor_only ?? false; document.getElementById("dryRun").checked = queues.dry_run ?? false; document.getElementById("sudo").checked = queues.sudo ?? false; document.getElementById("useBinpacking").checked = queues.use_binpacking ?? false; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_sonar.js b/src/rust/lqosd/src/node_manager/js_build/src/config_sonar.js index 465372c4..c87c4d28 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_sonar.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_sonar.js @@ -8,6 +8,50 @@ function stringToArray(str) { return str ? str.split(',').map(s => s.trim()).filter(s => s.length > 0) : []; } +function sanitizeRecurringRate(rule = {}) { + return { + enabled: rule.enabled !== false, + service_name: (rule.service_name ?? "").trim(), + download_mbps: Number(rule.download_mbps ?? 0), + upload_mbps: Number(rule.upload_mbps ?? 0), + }; +} + +function buildRecurringRateRow(rule = {}) { + const row = document.createElement("tr"); + const sanitized = sanitizeRecurringRate(rule); + row.innerHTML = ` + + + + + + `; + row.querySelector(".recurring-enabled").checked = sanitized.enabled; + row.querySelector(".recurring-service-name").value = sanitized.service_name; + row.querySelector(".recurring-download").value = Number.isFinite(sanitized.download_mbps) ? sanitized.download_mbps : 0; + row.querySelector(".recurring-upload").value = Number.isFinite(sanitized.upload_mbps) ? sanitized.upload_mbps : 0; + row.querySelector(".recurring-remove").addEventListener("click", () => row.remove()); + return row; +} + +function renderRecurringRateRows(rules = []) { + const tbody = document.getElementById("recurringServiceRatesBody"); + tbody.innerHTML = ""; + rules.forEach((rule) => tbody.appendChild(buildRecurringRateRow(rule))); +} + +function collectRecurringRateRows() { + return Array.from(document.querySelectorAll("#recurringServiceRatesBody tr")) + .map((row) => ({ + enabled: row.querySelector(".recurring-enabled").checked, + service_name: row.querySelector(".recurring-service-name").value.trim(), + download_mbps: Number(row.querySelector(".recurring-download").value), + upload_mbps: Number(row.querySelector(".recurring-upload").value), + })) + .filter((rule) => rule.service_name.length > 0); +} + function validateConfig() { // Validate required fields when enabled if (document.getElementById("enableSonar").checked) { @@ -35,6 +79,17 @@ function validateConfig() { return false; } } + + for (const rule of collectRecurringRateRows()) { + if (!Number.isFinite(rule.download_mbps) || rule.download_mbps <= 0) { + alert(`Recurring service "${rule.service_name}" must have a valid download rate`); + return false; + } + if (!Number.isFinite(rule.upload_mbps) || rule.upload_mbps <= 0) { + alert(`Recurring service "${rule.service_name}" must have a valid upload rate`); + return false; + } + } return true; } @@ -47,7 +102,9 @@ function updateConfig() { snmp_community: document.getElementById("snmpCommunity").value.trim(), airmax_model_ids: stringToArray(document.getElementById("airmaxModelIds").value), ltu_model_ids: stringToArray(document.getElementById("ltuModelIds").value), - active_status_ids: stringToArray(document.getElementById("activeStatusIds").value) + active_status_ids: stringToArray(document.getElementById("activeStatusIds").value), + recurring_excluded_service_names: stringToArray(document.getElementById("recurringExcludedServiceNames").value), + recurring_service_rates: collectRecurringRateRows(), }; } @@ -79,6 +136,12 @@ loadConfig(() => { arrayToString(sonar.ltu_model_ids); document.getElementById("activeStatusIds").value = arrayToString(sonar.active_status_ids); + document.getElementById("recurringExcludedServiceNames").value = + arrayToString(sonar.recurring_excluded_service_names); + renderRecurringRateRows(sonar.recurring_service_rates ?? []); + document.getElementById("addRecurringRateRow").addEventListener("click", () => { + document.getElementById("recurringServiceRatesBody").appendChild(buildRecurringRateRow()); + }); // Add save button click handler document.getElementById('saveButton').addEventListener('click', () => { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_treeguard.js b/src/rust/lqosd/src/node_manager/js_build/src/config_treeguard.js index 25e0fe28..61a48e27 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_treeguard.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_treeguard.js @@ -1,5 +1,5 @@ import { - loadAllShapedDevices, + loadAllCircuitDirectoryRows, loadConfig, loadNetworkJson, renderConfigMenu, @@ -8,16 +8,83 @@ import { import {defaultTreeguardConfig, ensureTreeguardConfig} from "./config/treeguard_defaults"; let networkData = null; -let shapedDevices = null; +let circuitRows = null; let selectedNodes = []; let selectedCircuits = []; +function setSummaryValue(id, value) { + const el = document.getElementById(id); + if (el) { + el.textContent = value; + } +} + +function updateOperationalSummary() { + const enabled = document.getElementById("enabled")?.checked ?? false; + const dryRun = document.getElementById("dryRun")?.checked ?? true; + const linksEnabled = document.getElementById("linksEnabled")?.checked ?? false; + const linksAllNodes = document.getElementById("linksAllNodes")?.checked ?? true; + const circuitsEnabled = document.getElementById("circuitsEnabled")?.checked ?? false; + const circuitsAllCircuits = document.getElementById("circuitsAllCircuits")?.checked ?? true; + const persistSqmOverrides = document.getElementById("persistSqmOverrides")?.checked ?? false; + + const mode = !enabled + ? "Disabled" + : dryRun + ? "Dry Run" + : "Live"; + const links = !linksEnabled + ? "Disabled" + : linksAllNodes + ? "All links" + : `Allowlist (${selectedNodes.length})`; + const circuits = !circuitsEnabled + ? "Disabled" + : circuitsAllCircuits + ? "All circuits" + : `Allowlist (${selectedCircuits.length})`; + const persistence = persistSqmOverrides ? "Persist overrides" : "In memory only"; + + setSummaryValue("treeguardSummaryMode", mode); + setSummaryValue("treeguardSummaryLinks", links); + setSummaryValue("treeguardSummaryCircuits", circuits); + setSummaryValue("treeguardSummaryPersistence", persistence); + + const note = document.getElementById("treeguardContextNote"); + if (!note) { + return; + } + + const warnings = []; + if (enabled && !dryRun) { + if (linksEnabled && linksAllNodes) { + warnings.push("live changes across all links"); + } + if (circuitsEnabled && circuitsAllCircuits) { + warnings.push("live changes across all circuits"); + } + if (persistSqmOverrides) { + warnings.push("persistent SQM overrides"); + } + } + + if (warnings.length === 0) { + note.classList.add("d-none"); + note.textContent = ""; + return; + } + + note.classList.remove("d-none"); + note.innerHTML = `Live scope: TreeGuard is currently set to apply ${warnings.join(", ")}.`; +} + function updateLinksEnrollmentUi() { const allNodes = document.getElementById("linksAllNodes")?.checked ?? true; const section = document.getElementById("nodesAllowlistSection"); if (section) { section.style.display = allNodes ? "none" : ""; } + updateOperationalSummary(); } function updateCircuitsEnrollmentUi() { @@ -26,6 +93,7 @@ function updateCircuitsEnrollmentUi() { if (section) { section.style.display = allCircuits ? "none" : ""; } + updateOperationalSummary(); } function loadNetworkData() { @@ -53,19 +121,19 @@ function loadNetworkData() { function loadCircuitsData() { return new Promise((resolve, reject) => { - loadAllShapedDevices( + loadAllCircuitDirectoryRows( (data) => { if (!Array.isArray(data)) { - console.warn("Shaped devices response was not an array:", data); - shapedDevices = []; + console.warn("Circuit directory response was not an array:", data); + circuitRows = []; } else { - shapedDevices = data; + circuitRows = data; } populateCircuitSelector(); resolve(); }, (err) => { - console.error("Error loading shaped devices:", err); + console.error("Error loading circuit directory:", err); alert("Failed to load circuit list. You can still add circuits manually."); reject(err); }, @@ -108,23 +176,24 @@ function populateCircuitSelector() { const selector = document.getElementById("circuitSelector"); selector.innerHTML = ''; - const circuits = new Set(); - if (Array.isArray(shapedDevices)) { - shapedDevices.forEach((device) => { - if (device && typeof device.circuit_id === "string" && device.circuit_id.trim() !== "") { - circuits.add(device.circuit_id.trim()); - } - }); + if (!Array.isArray(circuitRows)) { + return; } - Array.from(circuits) - .sort((a, b) => a.localeCompare(b)) - .forEach((circuitId) => { - const option = document.createElement("option"); - option.value = circuitId; - option.textContent = circuitId; - selector.appendChild(option); - }); + circuitRows.forEach((row) => { + const circuitId = (row?.circuit_id ?? "").trim(); + if (!circuitId) { + return; + } + const circuitName = (row?.circuit_name ?? "").trim(); + const display = circuitName && circuitName !== circuitId + ? `${circuitName} (${circuitId})` + : circuitId; + const option = document.createElement("option"); + option.value = circuitId; + option.textContent = display; + selector.appendChild(option); + }); } function addNode() { @@ -158,6 +227,7 @@ function updateNodesList() { if (selectedNodes.length === 0) { listContainer.innerHTML = '
    No nodes selected
    '; + updateOperationalSummary(); return; } @@ -177,6 +247,7 @@ function updateNodesList() { listItem.appendChild(removeBtn); listContainer.appendChild(listItem); }); + updateOperationalSummary(); } function addCircuitFromSelector() { @@ -225,6 +296,7 @@ function updateCircuitsList() { if (selectedCircuits.length === 0) { listContainer.innerHTML = '
    No circuits selected
    '; + updateOperationalSummary(); return; } @@ -244,6 +316,7 @@ function updateCircuitsList() { listItem.appendChild(removeBtn); listContainer.appendChild(listItem); }); + updateOperationalSummary(); } function validatePercent(name, value, min = 0, max = 100) { @@ -479,6 +552,18 @@ Promise.all([ document.getElementById("qooEnabled").checked = qoo.enabled; document.getElementById("minScore").value = qoo.min_score; + [ + "enabled", + "dryRun", + "linksEnabled", + "linksAllNodes", + "circuitsEnabled", + "circuitsAllCircuits", + "persistSqmOverrides", + ].forEach((id) => { + document.getElementById(id)?.addEventListener("change", updateOperationalSummary); + }); + document.getElementById("addNodeBtn").addEventListener("click", addNode); document.getElementById("linksAllNodes").addEventListener("change", updateLinksEnrollmentUi); document.getElementById("nodeSelector").addEventListener("keypress", (e) => { @@ -514,4 +599,6 @@ Promise.all([ alert("TreeGuard configuration saved successfully!"); }); }); + + updateOperationalSummary(); }); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/configuration.js b/src/rust/lqosd/src/node_manager/js_build/src/configuration.js index 8cb95635..53993d5e 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/configuration.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/configuration.js @@ -1,12 +1,23 @@ // Mostly ported from the original node manager import { get_ws_client } from "./pubsub/ws"; +import { + activeTopologySourceIntegrations, + loadAllShapedDevices, + topologyEditorsLockMessage, + topologyEditorsLocked, +} from "./config/config_helper"; let nics = null; let lqosd_config = null; let shaped_devices = null; let network_json = null; let qoo_profiles = null; +let shaped_devices_loaded = false; +let shaped_devices_loading = false; +let shaped_devices_dirty = false; const MIN_NODE_MBPS = 0.1; +let topology_editor_locked = false; +let topology_editor_lock_message = ""; const wsClient = get_ws_client(); @@ -67,7 +78,7 @@ const bindings = [ // Queues { field: "bindSqm", path: ".queues.default_sqm", data: "select-premade", editable: true }, - { field: "bindMonitorMode", path: ".queues.monitor_only", data: "bool", editable: true }, + { field: "bindQueueMode", path: ".queues.queue_mode", data: "select-premade", editable: true }, { field: "bindUplinkMbps", path: ".queues.uplink_bandwidth_mbps", data: "integer", editable: true }, { field: "bindDownlinkMbps", path: ".queues.downlink_bandwidth_mbps", data: "integer", editable: true }, { field: "bindGeneratedDownlinkMbps", path: ".queues.generated_pn_download_mbps", data: "integer", editable: true }, @@ -193,6 +204,33 @@ function ensureSplynxSection(config) { return config; } +function topologyEditDisabledAttr() { + return topology_editor_locked ? " disabled aria-disabled='true'" : ""; +} + +function applyTopologyEditorLockState() { + $("#btnSaveNetDevices").prop("disabled", topology_editor_locked); + $("#btnFlattenNetwork").prop("disabled", topology_editor_locked); + $("#btnAddNetworkNode").prop("disabled", topology_editor_locked); + $("#btnAddShapedDeviceRow").prop("disabled", topology_editor_locked || !shaped_devices_loaded); + $("#njsNewNodeName, #njsNewNodeDown, #njsNewNodeUp").prop("disabled", topology_editor_locked); + $("#netjson, #shapedDeviceTable").toggleClass("opacity-75", topology_editor_locked); + + const targets = [ + "#legacyTopologyEditorLockSummary", + "#legacyNetworkEditorLock", + "#legacyDevicesEditorLock", + ]; + for (const selector of targets) { + const element = $(selector); + if (topology_editor_locked && topology_editor_lock_message) { + element.removeClass("d-none").text(topology_editor_lock_message); + } else { + element.addClass("d-none").text(""); + } + } +} + function doBindings() { let active = true; @@ -439,11 +477,12 @@ function getFinalValue(target) { function updateSavedConfig(changes) { for (let i=0; i"; + const devices = Array.isArray(shaped_devices) ? shaped_devices : []; + const disabledAttr = topologyEditDisabledAttr(); for (const [key, value] of Object.entries(level)) { const isVirtual = value && value.virtual === true; html += "
    "; html += "" + key + ""; if (depth > 0) { - html += " "; + html += " "; } - html += " "; - html += " "; + html += " "; + html += " "; html += "
    "; - html += "Download: " + value.downloadBandwidthMbps + " Mbps
    "; - html += "Upload: " + value.uploadBandwidthMbps + " Mbps
    "; + html += "Download: " + value.downloadBandwidthMbps + " Mbps
    "; + html += "Upload: " + value.uploadBandwidthMbps + " Mbps
    "; html += "Virtual: " + (isVirtual ? " Yes" : "No") + - "
    "; + "
    "; let num_children = 0; - for (let i=0; i" + html = "" } else { - html = "" + html = "" } return html; } function makeSheetNumberBox(rowId, boxId, value) { - let html = "" + let html = "" return html; } @@ -753,14 +819,14 @@ function separatedIpArray(rowId, boxId, value) { if (val.length > 0) { val = val.substring(0, val.length-2); } - html += ""; + html += ""; html += ""; return html; } function nodeDropDown(rowId, boxId, selectedNode) { let html = ""; - html += ""; function iterate(data, level) { let html = ""; @@ -800,7 +866,70 @@ function validNodeList() { return nodes; } +function renderShapedDevicesPlaceholder(message, tone="secondary") { + $("#shapedDeviceTable").html( + "
    " + message + "
    ", + ); +} + +function setShapedDevicesControlsEnabled(enabled) { + $("#btnAddShapedDeviceRow").prop("disabled", !enabled || topology_editor_locked); +} + +function ensureShapedDevicesLoaded(onReady) { + if (shaped_devices_loaded) { + if (onReady) onReady(); + return; + } + if (shaped_devices_loading) { + if (onReady) { + const waitForLoad = () => { + if (shaped_devices_loaded) { + onReady(); + } else if (shaped_devices_loading) { + window.setTimeout(waitForLoad, 100); + } + }; + waitForLoad(); + } + return; + } + + shaped_devices_loading = true; + setShapedDevicesControlsEnabled(false); + renderShapedDevicesPlaceholder( + "
    Loading shaped devices...
    ", + "info", + ); + loadAllShapedDevices( + (data) => { + shaped_devices = data; + shaped_devices_loaded = true; + shaped_devices_loading = false; + shaped_devices_dirty = false; + setShapedDevicesControlsEnabled(true); + shapedDevices(); + RenderNetworkJson(); + applyTopologyEditorLockState(); + if (onReady) onReady(); + }, + () => { + shaped_devices_loading = false; + setShapedDevicesControlsEnabled(false); + renderShapedDevicesPlaceholder("Unable to load shaped devices.", "danger"); + applyTopologyEditorLockState(); + }, + ); +} + function newSdRow() { + if (topology_editor_locked) return; + if (!shaped_devices_loaded) { + ensureShapedDevicesLoaded(() => { + newSdRow(); + }); + return; + } shaped_devices.unshift({ circuit_id: "new_circuit", circuit_name: "new circuit", @@ -815,11 +944,17 @@ function newSdRow() { upload_max_mbps: 100, comment: "", }); + shaped_devices_dirty = true; shapedDevices(); } function deleteSdRow(id) { + if (topology_editor_locked) return; + if (!shaped_devices_loaded) { + return; + } shaped_devices.splice(id, 1); + shaped_devices_dirty = true; shapedDevices(); } @@ -1127,6 +1262,29 @@ function ipAddressesToTuple(ip) { } function saveNetAndDevices() { + if (topology_editor_locked) { + alert(topology_editor_lock_message); + return; + } + if (!shaped_devices_loaded || !shaped_devices_dirty) { + sendWsRequest( + "UpdateNetworkJsonOnlyResult", + { UpdateNetworkJsonOnly: { network_json: network_json } }, + (msg) => { + if (msg && msg.ok) { + alert("Configuration saved"); + } else { + const message = msg && msg.message ? msg.message : "Error"; + alert("Configuration not saved: " + message); + } + }, + () => { + alert("Configuration not saved: Error"); + }, + ); + return; + } + let isValid = validateSd().valid; if (!isValid) { alert("Validation errors in ShapedDevices. Not Saving"); @@ -1162,6 +1320,7 @@ function saveNetAndDevices() { { UpdateNetworkAndDevices: submission }, (msg) => { if (msg && msg.ok) { + shaped_devices_dirty = false; alert("Configuration saved"); } else { const message = msg && msg.message ? msg.message : "Error"; @@ -1175,6 +1334,10 @@ function saveNetAndDevices() { } function shapedDevices() { + if (!shaped_devices_loaded) { + renderShapedDevicesPlaceholder("Open the Shaped Devices tab to load and edit device rows."); + return; + } let html = ""; html += ""; html += ""; @@ -1195,7 +1358,7 @@ function shapedDevices() { html += makeSheetNumberBox(i, "download_max_mbps", row.download_max_mbps); html += makeSheetNumberBox(i, "upload_max_mbps", row.upload_max_mbps); html += makeSheetBox(i, "comment", row.comment, true); - html += ""; + html += ""; html += ""; } @@ -1301,6 +1464,15 @@ function start() { window.nodeSpeedChange = nodeSpeedChange; window.toggleVirtualNode = toggleVirtualNode; window.deleteSdRow = deleteSdRow; + renderShapedDevicesPlaceholder("Open the Shaped Devices tab to load and edit device rows."); + $("#v-pills-shapeddevs-tab").on("shown.bs.tab", () => { + ensureShapedDevicesLoaded(); + }); + $("#shapedDeviceTable").on("input change", "input, select, textarea", () => { + if (shaped_devices_loaded) { + shaped_devices_dirty = true; + } + }); // Old display(); @@ -1327,6 +1499,13 @@ function start() { { GetConfig: {} }, (cfgMsg) => { lqosd_config = ensureSplynxSection(cfgMsg.data); + topology_editor_locked = topologyEditorsLocked(lqosd_config); + topology_editor_lock_message = topologyEditorsLockMessage(lqosd_config); + const activeIntegrations = activeTopologySourceIntegrations(lqosd_config); + if (!topology_editor_lock_message && activeIntegrations.length > 0) { + topology_editor_lock_message = `Editing is disabled because these integrations are the source of truth: ${activeIntegrations.join(", ")}.`; + } + applyTopologyEditorLockState(); console.log("Bindings Done"); sendWsRequest( "ListNics", @@ -1352,18 +1531,9 @@ function start() { { NetworkJson: {} }, (netMsg) => { network_json = netMsg.data; - sendWsRequest( - "AllShapedDevices", - { AllShapedDevices: {} }, - (sdMsg) => { - shaped_devices = sdMsg.data; - shapedDevices(); - RenderNetworkJson(); - }, - () => { - alert("Unable to load shaped devices"); - }, - ); + shapedDevices(); + RenderNetworkJson(); + applyTopologyEditorLockState(); }, () => { alert("Unable to load network.json"); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/cpu_weights.js b/src/rust/lqosd/src/node_manager/js_build/src/cpu_weights.js index 10763d34..010f38eb 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/cpu_weights.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/cpu_weights.js @@ -1,17 +1,23 @@ -import { clearDiv, theading, simpleRow, simpleRowHtml, enableTooltips } from "./helpers/builders"; -import { DashboardGraph } from "./graphs/dashboard_graph"; +import { clearDiv, enableTooltips, simpleRow, theading } from "./helpers/builders"; +import { scaleNumber, toNumber } from "./lq_js_common/helpers/scaling"; import { get_ws_client } from "./pubsub/ws"; +const wsClient = get_ws_client(); + let state = { selectedCpu: null, + detailTab: "nodes", direction: "down", page: 1, - pageSize: 50, + pageSize: 10, + showExcludedCores: false, plannerEnabled: null, + excludeEfficiencyCores: null, + runtimeSnapshot: null, + liveCpuUsage: [], + overviewScrollTop: 0, }; -let dlMaxPie = null; // dashboard-graph wrapped chart for DL Max distribution -const wsClient = get_ws_client(); const listenOnce = (eventName, handler) => { const wrapped = (msg) => { wsClient.off(eventName, wrapped); @@ -28,9 +34,7 @@ function requestConfig() { done = true; resolve(cfg); }; - const timeout = setTimeout(() => { - finish(null); - }, 5000); + const timeout = setTimeout(() => finish(null), 5000); listenOnce("GetConfig", (msg) => { clearTimeout(timeout); finish(msg && msg.data ? msg.data : null); @@ -39,163 +43,464 @@ function requestConfig() { }); } -function fmtMbps(x) { - if (x === null || x === undefined) return "-"; - // keep up to 2 decimals, trim trailing zeros - let s = Number(x).toFixed(2); - s = s.replace(/\.00$/, ""); - return s; +function requestRuntimeSnapshot() { + return new Promise((resolve) => { + const timeout = setTimeout(() => resolve(null), 10000); + listenOnce("CpuAffinityRuntimeSnapshot", (msg) => { + clearTimeout(timeout); + resolve(msg && msg.data ? msg.data : null); + }); + wsClient.send({ CpuAffinityRuntimeSnapshot: {} }); + }); } -function renderSummary(data) { - const target = document.getElementById("cpuSummary"); +function fmtMbps(x) { + if (x === null || x === undefined) return "-"; + let s = Number(x).toFixed(2); + s = s.replace(/\.00$/, "").replace(/(\.\d*?[1-9])0+$/, "$1"); + return `${s} Mbps`; +} + +function fmtBps(bytesPerSecond) { + const bitsPerSecond = Math.max(0, toNumber(bytesPerSecond, 0) * 8); + return `${scaleNumber(bitsPerSecond, bitsPerSecond >= 1_000_000 ? 1 : 0)}bps`; +} + +function nodeUtilizationPercent(node) { + const downMaxMbps = Math.max(0, toNumber(node?.effective_max_mbps?.[0], 0)); + const upMaxMbps = Math.max(0, toNumber(node?.effective_max_mbps?.[1], 0)); + const downBps = Math.max(0, toNumber(node?.current_throughput_bps?.[0], 0) * 8); + const upBps = Math.max(0, toNumber(node?.current_throughput_bps?.[1], 0) * 8); + + const downUtilization = downMaxMbps > 0 ? (downBps / (downMaxMbps * 1_000_000)) * 100 : null; + const upUtilization = upMaxMbps > 0 ? (upBps / (upMaxMbps * 1_000_000)) * 100 : null; + if (!Number.isFinite(downUtilization) && !Number.isFinite(upUtilization)) { + return null; + } + return Math.max(0, Math.max(downUtilization ?? 0, upUtilization ?? 0)); +} + +function fmtUtilizationPercent(node) { + const utilization = nodeUtilizationPercent(node); + if (!Number.isFinite(utilization)) { + return "-"; + } + return `${Math.round(utilization)}%`; +} + +function cpuUsageValue(core) { + const live = state.liveCpuUsage?.[core.cpu]; + if (Number.isFinite(live)) { + return live; + } + const fallback = toNumber(core.live_usage_pct, NaN); + return Number.isFinite(fallback) ? fallback : null; +} + +function usageToneClass(usage) { + if (!Number.isFinite(usage)) return "bg-secondary"; + if (usage >= 90) return "bg-danger"; + if (usage >= 75) return "bg-warning"; + return "bg-success"; +} + +function assignmentBadge(node) { + const reason = (node?.assignment_reason || "").toString(); + if (reason === "runtime_virtualized_hidden") { + return { label: "Runtime Virtualized", cls: "bg-warning text-dark" }; + } + if (reason === "inherited_from_virtualized_ancestor") { + return { label: "Inherited", cls: "bg-info text-dark" }; + } + if (reason === "planned") { + return { label: "Planned", cls: "bg-success" }; + } + return { label: "Unknown", cls: "bg-secondary" }; +} + +function topLevelNodesForCore(core) { + const nodes = Array.isArray(core?.nodes) ? core.nodes : []; + return [...nodes] + .filter((node) => !!node?.is_cpu_root) + .sort((a, b) => { + const circuitDelta = toNumber(b?.subtree_circuit_count, 0) - toNumber(a?.subtree_circuit_count, 0); + if (circuitDelta !== 0) { + return circuitDelta; + } + const throughputA = toNumber(a?.current_throughput_bps?.[0], 0) + toNumber(a?.current_throughput_bps?.[1], 0); + const throughputB = toNumber(b?.current_throughput_bps?.[0], 0) + toNumber(b?.current_throughput_bps?.[1], 0); + if (throughputB !== throughputA) { + return throughputB - throughputA; + } + return String(a?.name || "").localeCompare(String(b?.name || "")); + }); +} + +function getCores() { + const cores = Array.isArray(state.runtimeSnapshot?.cores) ? [...state.runtimeSnapshot.cores] : []; + cores.sort((a, b) => toNumber(a.cpu, 0) - toNumber(b.cpu, 0)); + return cores; +} + +function getShapingCpuSet() { + const shaping = Array.isArray(state.runtimeSnapshot?.shaping_cpus) + ? state.runtimeSnapshot.shaping_cpus + : []; + return new Set(shaping.map((cpu) => toNumber(cpu, -1)).filter((cpu) => cpu >= 0)); +} + +function getExcludedCpuSet() { + const excluded = Array.isArray(state.runtimeSnapshot?.excluded_cpus) + ? state.runtimeSnapshot.excluded_cpus + : []; + return new Set(excluded.map((cpu) => toNumber(cpu, -1)).filter((cpu) => cpu >= 0)); +} + +function hasExcludedCpuFilter() { + return getExcludedCpuSet().size > 0; +} + +function hasUsableShapingCpuFilter() { + const shaping = getShapingCpuSet(); + return shaping.size > 0 && shaping.size < getCores().length; +} + +function getInferredIncludedCpuSet() { + if (!state.excludeEfficiencyCores) { + return new Set(); + } + const included = getCores() + .filter((core) => + toNumber(core.planned_circuit_count, 0) > 0 + || toNumber(core.planned_max_mbps, 0) > 0 + || toNumber(core.effective_node_count, 0) > 0 + || toNumber(core.effective_circuit_count, 0) > 0 + ) + .map((core) => toNumber(core.cpu, -1)) + .filter((cpu) => cpu >= 0); + return new Set(included); +} + +function hasFallbackIncludedCpuFilter() { + const included = getInferredIncludedCpuSet(); + return included.size > 0 && included.size < getCores().length; +} + +function getIncludedCpuSet() { + if (hasExcludedCpuFilter()) { + const excluded = getExcludedCpuSet(); + return new Set( + getCores() + .map((core) => toNumber(core.cpu, -1)) + .filter((cpu) => cpu >= 0 && !excluded.has(cpu)) + ); + } + if (hasUsableShapingCpuFilter()) { + return getShapingCpuSet(); + } + if (hasFallbackIncludedCpuFilter()) { + return getInferredIncludedCpuSet(); + } + return new Set(); +} + +function getVisibleCores() { + const cores = getCores(); + if (state.showExcludedCores) { + return cores; + } + const included = getIncludedCpuSet(); + if (included.size > 0) { + return cores.filter((core) => included.has(toNumber(core.cpu, -1))); + } + return cores; +} + +function getSelectedCore() { + return getVisibleCores().find((core) => toNumber(core.cpu, -1) === toNumber(state.selectedCpu, -2)) || null; +} + +function selectDefaultCpu() { + const cores = getVisibleCores(); + if (cores.length === 0) { + state.selectedCpu = null; + return; + } + const stillExists = cores.some((core) => toNumber(core.cpu, -1) === toNumber(state.selectedCpu, -2)); + if (stillExists) { + return; + } + state.selectedCpu = cores[0].cpu; +} + +function renderBinpackBadge() { + const badge = document.getElementById("binpackBadge"); + if (!badge) return; + const enabled = !!state.plannerEnabled; + badge.className = `cpu-affinity-chip ${enabled ? "is-positive" : "is-muted"}`; + badge.textContent = enabled ? "Binpacking: Enabled" : "Binpacking: Disabled"; +} + +function applyOverviewRowLiveMetrics(row, core) { + const usage = cpuUsageValue(core); + const chip = row.querySelector(".cpu-affinity-overview-usage"); + const bar = row.querySelector(".cpu-affinity-overview-progress-bar"); + if (chip) { + chip.className = `cpu-affinity-overview-usage ${usageToneClass(usage)}`; + chip.textContent = Number.isFinite(usage) ? `${Math.round(usage)}%` : "N/A"; + } + if (bar) { + bar.className = `cpu-affinity-overview-progress-bar ${usageToneClass(usage)}`; + bar.style.width = `${Math.max(0, Math.min(100, toNumber(usage, 0)))}%`; + } +} + +function updateOverviewLiveMetrics() { + const overview = document.getElementById("cpuOverview"); + if (!overview) return; + const coreMap = new Map(getVisibleCores().map((core) => [String(toNumber(core.cpu, -1)), core])); + overview.querySelectorAll(".cpu-affinity-overview-row[data-cpu]").forEach((row) => { + const core = coreMap.get(row.dataset.cpu); + if (!core) return; + applyOverviewRowLiveMetrics(row, core); + }); +} + +function renderOverview() { + const target = document.getElementById("cpuOverview"); + const existingList = target?.querySelector(".cpu-affinity-overview-list"); + if (existingList) { + state.overviewScrollTop = existingList.scrollTop; + } clearDiv(target); - if (!data || !Array.isArray(data) || data.length === 0) { - target.innerHTML = '

    No queue structure found. Run LibreQoS to generate queuingStructure.json.

    '; + renderBinpackBadge(); + + const cores = getVisibleCores(); + if (cores.length === 0) { + target.innerHTML = (hasExcludedCpuFilter() || hasUsableShapingCpuFilter() || hasFallbackIncludedCpuFilter()) + ? '

    No shaping CPUs are visible with the current filter.

    ' + : '

    No runtime CPU assignment data found. Run LibreQoS to generate queuingStructure.json and live tree state.

    '; return; } - // Update binpacking badge in header bar - const badge = document.getElementById('binpackBadge'); - const enabled = !!state.plannerEnabled; - if (badge) { - badge.className = `badge ${enabled ? 'bg-success' : 'bg-secondary'}`; - badge.textContent = enabled ? "Binpacking: Enabled" : "Binpacking: Disabled"; - } - // Optional Insight promo below summary + const panel = document.createElement("div"); + panel.className = "cpu-affinity-card h-100"; + + const body = document.createElement("div"); + body.className = "cpu-affinity-card-body"; + + const header = document.createElement("div"); + header.className = "cpu-affinity-card-header"; + + const headerText = document.createElement("div"); + const shapingCount = Array.isArray(state.runtimeSnapshot?.shaping_cpus) ? state.runtimeSnapshot.shaping_cpus.length : 0; + const excludedCount = Array.isArray(state.runtimeSnapshot?.excluded_cpus) ? state.runtimeSnapshot.excluded_cpus.length : 0; + const hasHybridSplit = !!state.runtimeSnapshot?.has_hybrid_split; + const inferredIncludedCount = getInferredIncludedCpuSet().size; + const overviewSubtitle = (hasExcludedCpuFilter() || hasUsableShapingCpuFilter() || hasFallbackIncludedCpuFilter()) && !state.showExcludedCores + ? `Showing shaping CPUs only${hasHybridSplit && excludedCount > 0 ? ` (${shapingCount} shaping cores, ${excludedCount} excluded host cores hidden)` : ""}.` + : hasHybridSplit && excludedCount > 0 + ? `Showing all host cores, including ${excludedCount} excluded non-shaping core${excludedCount === 1 ? "" : "s"}.` + : state.showExcludedCores && hasFallbackIncludedCpuFilter() + ? `Showing all host cores (${inferredIncludedCount} included cores inferred from current queue ownership).` + : hasFallbackIncludedCpuFilter() + ? `Showing included cores only (${inferredIncludedCount} included cores inferred from current queue ownership).` + : "Choose a core to inspect live ownership, promoted runtime branches, and planner drift."; + headerText.innerHTML = ` +

    CPU Core List

    +

    ${overviewSubtitle}

    + `; + header.appendChild(headerText); + + const generatedAt = toNumber(state.runtimeSnapshot?.generated_at_unix_ms, Date.now()); + const status = document.createElement("div"); + status.className = "cpu-affinity-card-meta"; + status.textContent = `Updated ${new Date(generatedAt).toLocaleTimeString()}`; + header.appendChild(status); + body.appendChild(header); + if (!window.hasInsight) { const promo = document.createElement("div"); - promo.className = "text-muted small mb-2"; - promo.innerHTML = `Enable Insight to make binpacking smarter with historical and recent data.`; - target.appendChild(promo); + promo.className = "cpu-affinity-inline-note mb-3"; + promo.innerHTML = 'Enable Insight to improve CPU planning with historical data.'; + body.appendChild(promo); } - // Build a row with table (left) and chart (right) - const row = document.createElement('div'); - row.className = 'row'; - const left = document.createElement('div'); - left.className = 'col-12 col-lg-8'; - const right = document.createElement('div'); - right.className = 'col-12 col-lg-4'; + const list = document.createElement("div"); + list.className = "cpu-affinity-overview-list"; + list.addEventListener("scroll", () => { + state.overviewScrollTop = list.scrollTop; + }); - const tableWrap = document.createElement("div"); - tableWrap.classList.add("lqos-table-wrap"); - const table = document.createElement("table"); - table.classList.add("lqos-table", "lqos-table-compact"); - const thead = document.createElement("thead"); - thead.appendChild(theading("CPU")); - thead.appendChild(theading("DL Circuits")); - thead.appendChild(theading("Weight")); - thead.appendChild(theading("DL Max (Mbps)")); - thead.appendChild(theading("Actions")); - table.appendChild(thead); - const tbody = document.createElement("tbody"); - - // helper to build a centered cell - const centerCell = (text) => { - const td = simpleRow(String(text)); - td.classList.add('text-center'); - return td; - }; - - // simple weight formatter - const fmtWeight = (x) => { - const n = Number(x || 0); - return n > 0 ? n.toLocaleString() : '-'; - }; - - data.forEach((row) => { - const tr = document.createElement("tr"); - tr.appendChild(centerCell(row.cpu)); - tr.appendChild(centerCell(row.download.circuits)); - tr.appendChild(centerCell(fmtWeight(row.download && row.download.weight_sum))); - tr.appendChild(centerCell(fmtMbps(row.download.max_sum_mbps))); - - const actions = document.createElement("td"); - actions.classList.add('text-center'); - const downBtn = document.createElement("button"); - downBtn.className = "btn btn-sm btn-outline-primary me-1"; - downBtn.innerHTML = " View"; - downBtn.onclick = () => { - state.selectedCpu = row.cpu; - state.direction = "down"; + cores.forEach((core) => { + const row = document.createElement("button"); + row.type = "button"; + row.className = `cpu-affinity-overview-row ${toNumber(state.selectedCpu, -1) === toNumber(core.cpu, -2) ? "is-selected" : ""}`; + row.dataset.cpu = String(toNumber(core.cpu, -1)); + row.onclick = () => { + state.selectedCpu = core.cpu; state.page = 1; - renderDetailsHeader(); - fetchCircuits(); + renderOverview(); + renderSelectedCore(); + if (state.detailTab === "circuits") { + fetchCircuits(); + } }; - actions.appendChild(downBtn); - tr.appendChild(actions); + const topRow = document.createElement("div"); + topRow.className = "d-flex justify-content-between align-items-start gap-3 mb-2"; + + const titleWrap = document.createElement("div"); + const title = document.createElement("div"); + title.className = "fw-semibold"; + title.textContent = `CPU ${core.cpu}`; + titleWrap.appendChild(title); + const cpuId = toNumber(core.cpu, -1); + const included = getIncludedCpuSet(); + const isExcluded = included.size > 0 && !included.has(cpuId); + topRow.appendChild(titleWrap); + + const summary = document.createElement("div"); + summary.className = "cpu-affinity-overview-summary"; + summary.textContent = `${core.effective_node_count.toLocaleString()} nodes · ${core.effective_circuit_count.toLocaleString()} circuits`; + topRow.appendChild(summary); + + const usage = cpuUsageValue(core); + row.appendChild(topRow); + + if (isExcluded) { + const subtitle = document.createElement("div"); + subtitle.className = "text-muted small mb-2"; + subtitle.textContent = "Excluded from shaping"; + row.appendChild(subtitle); + } + + const progress = document.createElement("div"); + progress.className = "cpu-affinity-overview-progress"; + const bar = document.createElement("div"); + bar.className = `cpu-affinity-overview-progress-bar ${usageToneClass(usage)}`; + bar.style.width = `${Math.max(0, Math.min(100, toNumber(usage, 0)))}%`; + progress.appendChild(bar); + row.appendChild(progress); + list.appendChild(row); + }); + + body.appendChild(list); + panel.appendChild(body); + target.appendChild(panel); + list.scrollTop = state.overviewScrollTop; +} + +function renderDetailsHeader() { + const label = document.getElementById("currentCpuLabel"); + if (state.selectedCpu === null || state.selectedCpu === undefined) { + label.textContent = ""; + return; + } + label.textContent = ` – CPU ${state.selectedCpu}`; +} + +function renderSelectedSummary() { + const target = document.getElementById("cpuSelectedSummary"); + clearDiv(target); + const core = getSelectedCore(); + if (!core) { + target.innerHTML = 'Select a core from the list.'; + return; + } + + const row = document.createElement("div"); + row.className = "cpu-affinity-kpi-grid"; + const cards = [ + { + title: "Live Load", + value: Number.isFinite(cpuUsageValue(core)) ? `${Math.round(cpuUsageValue(core))}%` : "N/A", + }, + { + title: "Assigned Nodes", + value: toNumber(core.effective_node_count, 0).toLocaleString(), + }, + { + title: "Assigned Circuits", + value: toNumber(core.effective_circuit_count, 0).toLocaleString(), + }, + ]; + + cards.forEach((entry) => { + const card = document.createElement("div"); + card.className = "cpu-affinity-kpi"; + const label = document.createElement("div"); + label.className = "cpu-affinity-kpi-label"; + label.textContent = entry.title; + const value = document.createElement("div"); + value.className = "cpu-affinity-kpi-value"; + value.textContent = entry.value; + card.appendChild(label); + card.appendChild(value); + row.appendChild(card); + }); + target.appendChild(row); +} + +function renderNodesTable(core) { + const target = document.getElementById("cpuDetailsContent"); + clearDiv(target); + + const nodes = topLevelNodesForCore(core); + if (nodes.length === 0) { + target.innerHTML = '

    No top-level owned branches are associated with this CPU.

    '; + return; + } + + const note = document.createElement("div"); + note.className = "cpu-affinity-inline-note mb-2"; + note.textContent = "Showing the branches directly attached to this CPU's effective HTB subtree after runtime virtualization and promotion."; + target.appendChild(note); + + const tableWrap = document.createElement("div"); + tableWrap.className = "lqos-table-wrap"; + const table = document.createElement("table"); + table.className = "lqos-table lqos-table-compact"; + const thead = document.createElement("thead"); + thead.appendChild(theading("Node")); + thead.appendChild(theading("Assignment")); + thead.appendChild(theading("Utilization")); + thead.appendChild(theading("Sub-Nodes")); + thead.appendChild(theading("Circuits")); + table.appendChild(thead); + + const tbody = document.createElement("tbody"); + nodes.forEach((node) => { + const tr = document.createElement("tr"); + if (node.runtime_virtualized || node.assignment_reason !== "planned") { + tr.classList.add("cpu-affinity-row-change"); + } + + const nameCell = document.createElement("td"); + nameCell.textContent = node.name || ""; + tr.appendChild(nameCell); + + const assignmentCell = document.createElement("td"); + const badgeMeta = assignmentBadge(node); + const badge = document.createElement("span"); + badge.className = `badge ${badgeMeta.cls}`; + badge.textContent = badgeMeta.label; + assignmentCell.appendChild(badge); + tr.appendChild(assignmentCell); + + tr.appendChild(simpleRow(fmtUtilizationPercent(node))); + tr.appendChild(simpleRow(toNumber(node.subtree_node_count, 0).toLocaleString())); + tr.appendChild(simpleRow(toNumber(node.subtree_circuit_count, 0).toLocaleString())); tbody.appendChild(tr); }); table.appendChild(tbody); tableWrap.appendChild(table); - left.appendChild(tableWrap); - - // Right column pie chart (DL Max per CPU) - const chartDiv = document.createElement('div'); - chartDiv.id = 'cpuDlMaxPie'; - chartDiv.style.width = '100%'; - chartDiv.style.height = '260px'; - right.appendChild(chartDiv); - - row.appendChild(left); - row.appendChild(right); - target.appendChild(row); - - // Build donut (ring) chart with theme support - try { - // Recreate the graph each render to bind to the fresh DOM - if (dlMaxPie && dlMaxPie.chart) { - try { dlMaxPie.chart.dispose(); } catch (_) {} - dlMaxPie = null; - } - dlMaxPie = new DashboardGraph('cpuDlMaxPie'); - const seriesData = data.map((row) => { - const w = Number((row.download && row.download.weight_sum) ? row.download.weight_sum : 0); - const v = Number(row.download.max_sum_mbps || 0); - return { - name: `CPU ${row.cpu}`, - value: (w > 0 ? w : v) - }; - }); - dlMaxPie.option = { - tooltip: { trigger: 'item' }, - legend: { show: false }, - series: [ - { - name: 'DL Max by CPU', - type: 'pie', - radius: ['55%','80%'], - center: ['50%', '50%'], - avoidLabelOverlap: true, - data: seriesData, - label: { - show: true, - formatter: '{b}: {d}%' - } - } - ] - }; - dlMaxPie.chart.setOption(dlMaxPie.option); - dlMaxPie.chart.hideLoading(); - } catch (e) { - // If echarts is unavailable, leave chart blank - } -} - -function renderDetailsHeader() { - const label = document.getElementById("currentCpuLabel"); - if (state.selectedCpu === null) { - label.textContent = ""; - } else { - label.textContent = ` – CPU ${state.selectedCpu}`; - } + target.appendChild(tableWrap); } function renderCircuits(page) { - const target = document.getElementById("circuitsTable"); + const target = document.getElementById("cpuDetailsContent"); clearDiv(target); if (!page || !Array.isArray(page.items)) { @@ -205,7 +510,7 @@ function renderCircuits(page) { const smallNote = document.createElement("div"); smallNote.className = "text-muted small mb-1"; - smallNote.innerText = `Total: ${page.total.toLocaleString()} circuits`; + smallNote.innerText = `Total: ${page.total.toLocaleString()} planned downlink circuits`; target.appendChild(smallNote); const tableWrap = document.createElement("div"); @@ -213,7 +518,6 @@ function renderCircuits(page) { const table = document.createElement("table"); table.classList.add("lqos-table", "lqos-table-compact"); const thead = document.createElement("thead"); - thead.appendChild(theading("Circuit ID")); thead.appendChild(theading("Circuit Name")); thead.appendChild(theading("Parent")); thead.appendChild(theading("ClassID")); @@ -226,26 +530,28 @@ function renderCircuits(page) { page.items.forEach((c) => { const tr = document.createElement("tr"); if (c.ignored || (c.weight !== undefined && c.weight <= 0)) { - tr.classList.add('text-muted'); + tr.classList.add("text-muted"); } - const idCell = document.createElement("td"); - if (c.circuit_id) { + const nameCell = document.createElement("td"); + if (c.circuit_id && c.circuit_name) { const a = document.createElement("a"); a.href = `circuit.html?id=${encodeURIComponent(c.circuit_id)}`; - a.textContent = c.circuit_id; - idCell.appendChild(a); + a.textContent = c.circuit_name; + nameCell.appendChild(a); } else { - idCell.textContent = ""; + nameCell.textContent = c.circuit_name || ""; } - tr.appendChild(idCell); - tr.appendChild(simpleRow(c.circuit_name || "", true)); + tr.appendChild(nameCell); tr.appendChild(simpleRow(c.parent_node || "", true)); tr.appendChild(simpleRow(c.classid || "")); - const weightCell = document.createElement('td'); + const weightCell = document.createElement("td"); if (c.ignored || (c.weight !== undefined && c.weight <= 0)) { - weightCell.innerHTML = "ignored"; + const badge = document.createElement("span"); + badge.className = "cpu-affinity-chip is-muted"; + badge.textContent = "ignored"; + weightCell.appendChild(badge); } else { - weightCell.innerText = (c.weight && c.weight > 0) ? c.weight.toLocaleString() : '-'; + weightCell.textContent = (c.weight && c.weight > 0) ? c.weight.toLocaleString() : "-"; } tr.appendChild(weightCell); tr.appendChild(simpleRow(fmtMbps(c.max_mbps))); @@ -258,41 +564,50 @@ function renderCircuits(page) { target.appendChild(tableWrap); } -function fetchSummary() { - const target = document.getElementById("cpuSummary"); - if (target) { - target.innerHTML = ' Loading summary...'; - } - const configPromise = requestConfig().then((cfg) => { - state.plannerEnabled = !!(cfg && cfg.queues && cfg.queues.use_binpacking); - }); +function renderPagerVisibility() { + const pager = document.getElementById("cpuDetailsPager"); + pager.style.display = state.detailTab === "circuits" && state.selectedCpu !== null ? "flex" : "none"; +} - return new Promise((resolve) => { - const timeout = setTimeout(() => { - if (target) { - target.innerHTML = '
    Failed to load summary: timeout
    '; - } - resolve(); - }, 10000); - - listenOnce("CpuAffinitySummary", async (msg) => { - clearTimeout(timeout); - await configPromise; - const data = msg && msg.data ? msg.data : []; - renderSummary(data); - resolve(); - }); - wsClient.send({ CpuAffinitySummary: {} }); +function renderActiveTab() { + const tabButtons = [ + ["btnTabNodes", "nodes"], + ["btnTabCircuits", "circuits"], + ]; + tabButtons.forEach(([id, tab]) => { + const btn = document.getElementById(id); + if (!btn) return; + btn.className = `cpu-affinity-tab ${state.detailTab === tab ? "is-active" : ""}`; }); } +function renderSelectedCore() { + renderDetailsHeader(); + renderSelectedSummary(); + renderActiveTab(); + renderPagerVisibility(); + + const core = getSelectedCore(); + if (!core) { + const target = document.getElementById("cpuDetailsContent"); + target.innerHTML = 'Select a CPU from the list.'; + return; + } + + if (state.detailTab === "circuits") { + fetchCircuits(); + return; + } + renderNodesTable(core); + enableTooltips(); +} + function fetchCircuits() { - const target = document.getElementById("circuitsTable"); + const target = document.getElementById("cpuDetailsContent"); target.innerHTML = ' Loading circuits...'; - const dir = state.direction; const cpu = state.selectedCpu; if (cpu === null || cpu === undefined) { - target.innerHTML = 'Select a CPU from the table above.'; + target.innerHTML = 'Select a CPU from the list.'; return; } const timeout = setTimeout(() => { @@ -308,48 +623,112 @@ function fetchCircuits() { }); wsClient.send({ CpuAffinityCircuits: { - cpu: cpu, - direction: dir, + cpu, + direction: state.direction, page: state.page, page_size: state.pageSize, - search: null, - } + }, }); } -// Wire up controls +async function refreshAll() { + const overview = document.getElementById("cpuOverview"); + const showExcludedCores = document.getElementById("showExcludedCores"); + if (overview && !state.runtimeSnapshot) { + overview.innerHTML = ' Loading core assignment...'; + } + + const configPromise = requestConfig().then((cfg) => { + state.plannerEnabled = !!(cfg && cfg.queues && cfg.queues.use_binpacking); + state.excludeEfficiencyCores = !!(cfg && cfg.exclude_efficiency_cores); + }); + const snapshot = await requestRuntimeSnapshot(); + await configPromise; + state.runtimeSnapshot = snapshot; + if (showExcludedCores) { + showExcludedCores.disabled = !hasExcludedCpuFilter() && !hasUsableShapingCpuFilter() && !hasFallbackIncludedCpuFilter(); + if (showExcludedCores.disabled) { + state.showExcludedCores = false; + showExcludedCores.checked = false; + } else { + showExcludedCores.checked = !!state.showExcludedCores; + } + } + selectDefaultCpu(); + renderOverview(); + renderSelectedCore(); + enableTooltips(); +} + function initControls() { const pageSizeInput = document.getElementById("pageSize"); const btnPrev = document.getElementById("btnPrev"); const btnNext = document.getElementById("btnNext"); const btnRefresh = document.getElementById("btnRefresh"); + const btnTabNodes = document.getElementById("btnTabNodes"); + const btnTabCircuits = document.getElementById("btnTabCircuits"); + const showExcludedCores = document.getElementById("showExcludedCores"); + pageSizeInput.onchange = () => { - let v = parseInt(pageSizeInput.value || "50", 10); + let v = parseInt(pageSizeInput.value || "10", 10); if (!Number.isFinite(v) || v < 10) v = 10; if (v > 1000) v = 1000; state.pageSize = v; state.page = 1; - fetchCircuits(); + if (state.detailTab === "circuits") { + fetchCircuits(); + } }; btnPrev.onclick = () => { if (state.page > 1) { state.page -= 1; - fetchCircuits(); + if (state.detailTab === "circuits") fetchCircuits(); } }; btnNext.onclick = () => { state.page += 1; - fetchCircuits(); + if (state.detailTab === "circuits") fetchCircuits(); }; btnRefresh.onclick = () => { - fetchSummary(); - if (state.selectedCpu !== null) fetchCircuits(); + refreshAll(); + }; + if (showExcludedCores) { + showExcludedCores.checked = !!state.showExcludedCores; + showExcludedCores.disabled = !hasExcludedCpuFilter() && !hasUsableShapingCpuFilter() && !hasFallbackIncludedCpuFilter(); + showExcludedCores.onchange = () => { + state.showExcludedCores = !!showExcludedCores.checked; + state.overviewScrollTop = 0; + selectDefaultCpu(); + renderOverview(); + renderSelectedCore(); + if (state.detailTab === "circuits") { + fetchCircuits(); + } + }; + } + btnTabNodes.onclick = () => { + state.detailTab = "nodes"; + renderSelectedCore(); + }; + btnTabCircuits.onclick = () => { + state.detailTab = "circuits"; + state.page = 1; + renderSelectedCore(); }; } -// Initialize +function initLiveCpuSubscription() { + wsClient.subscribe(["Cpu"]); + wsClient.on("Cpu", (msg) => { + state.liveCpuUsage = Array.isArray(msg?.data) ? msg.data.map((value) => toNumber(value, 0)) : []; + updateOverviewLiveMetrics(); + renderSelectedSummary(); + }); +} + initControls(); renderDetailsHeader(); -fetchSummary().then(() => enableTooltips()); - -// No preview section (binpacking) in this view +renderActiveTab(); +renderPagerVisibility(); +initLiveCpuSubscription(); +refreshAll(); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_activity.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_activity.js new file mode 100644 index 00000000..ba26fb87 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_activity.js @@ -0,0 +1,627 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {formatUnixSecondsToLocalDateTime, mkBadge} from "./bakery_shared"; +import {renderOperationCards} from "./operation_cards"; + +const BAKERY_ACTIVITY_PAGE_SIZE = 10; +const BAKERY_ACTIVITY_SUMMARY_MAX_CHARS = 220; +const BAKERY_OPERATION_SUMMARY_COUNT = 6; +const BAKERY_OPERATION_MERGE_WINDOW_SECONDS = 180; +const BAKERY_ACTIVITY_VIEW_STORAGE_KEY = "lqos_bakery_activity_view"; +const FULL_RELOAD_SUPPORT_EVENTS = new Set([ + "baseline_rebuild_startup", + "baseline_rebuild_required", + "commit_received", +]); +const LIVE_CHANGE_PREFIXES = [ + "updating circuit speeds live", + "migrating circuits between parent nodes (fallback)", + "treeguard runtime top-level site reparent", + "treeguard runtime top-level circuit reparent", + "treeguard runtime child-site shadow create", + "treeguard runtime circuit reparent", + "treeguard runtime hidden site restore", + "treeguard runtime site restore", + "treeguard runtime circuit restore", +]; + +function truncateSummary(summary) { + const normalized = (summary ?? "").toString().trim(); + if (normalized.length <= BAKERY_ACTIVITY_SUMMARY_MAX_CHARS) { + return normalized; + } + return normalized.slice(0, BAKERY_ACTIVITY_SUMMARY_MAX_CHARS - 3).trimEnd() + "..."; +} + +function summaryPrefix(summary) { + return ((summary ?? "").toString().split(":")[0] || "").trim(); +} + +function isKnownLiveChangeSummary(summary) { + const prefix = summaryPrefix(summary).toLowerCase(); + return LIVE_CHANGE_PREFIXES.some((candidate) => prefix === candidate); +} + +function classifyEvent(entry) { + const event = (entry?.event ?? "").toString().trim().toLowerCase(); + const status = (entry?.status ?? "").toString().trim().toLowerCase(); + const summary = (entry?.summary ?? "").toString().trim(); + const summaryLower = summary.toLowerCase(); + + let stage = "Pipeline"; + let scope = "Bakery"; + let outcome = "Info"; + let outcomeClass = "bg-light text-secondary border"; + + if (event.startsWith("preflight_")) { + stage = "Preflight"; + scope = "Safety"; + } else if (event.includes("prune")) { + stage = "Cleanup"; + scope = "TreeGuard"; + } else if (event.includes("started")) { + stage = "Apply"; + } else if (event.includes("finished") || event.includes("failed")) { + stage = "Verify"; + } + + if (event === "full_reload_started" || summaryLower.includes("full reload")) { + scope = "Full Reload"; + } else if ( + event === "live_change_started" + || (!summaryLower.includes("processing batch") && summaryLower.includes("live")) + || isKnownLiveChangeSummary(summary) + ) { + scope = "Live Change"; + } else if (event.startsWith("runtime_site_prune_")) { + scope = "TreeGuard Cleanup"; + } else if (event.startsWith("preflight_")) { + scope = "Preflight"; + } else if (event.startsWith("stormguard_")) { + scope = "StormGuard"; + } else if (summaryLower.startsWith("processing batch")) { + scope = "Full Reload"; + } + + if (event.endsWith("_started")) { + outcome = "Running"; + outcomeClass = "bg-primary-subtle text-primary border border-primary-subtle"; + } else if (event === "reload_required") { + stage = "Verify"; + scope = "Full Reload"; + outcome = "Reload Required"; + outcomeClass = "bg-danger-subtle text-danger border border-danger-subtle"; + } else if (event === "reload_required_cleared") { + stage = "Verify"; + scope = "Full Reload"; + outcome = "Cleared"; + outcomeClass = "bg-success-subtle text-success border border-success-subtle"; + } else if (event === "preflight_ok") { + outcome = "Passed"; + outcomeClass = "bg-success-subtle text-success border border-success-subtle"; + } else if (event === "preflight_blocked") { + outcome = "Blocked"; + outcomeClass = "bg-warning-subtle text-warning border border-warning-subtle"; + } else if (event === "runtime_site_prune_retry") { + outcome = "Retrying"; + outcomeClass = "bg-warning-subtle text-warning border border-warning-subtle"; + } else if (event === "runtime_site_prune_dirty") { + outcome = "Dirty"; + outcomeClass = "bg-danger-subtle text-danger border border-danger-subtle"; + } else if (event === "runtime_site_prune_completed" || event === "apply_finished") { + outcome = "Completed"; + outcomeClass = "bg-success-subtle text-success border border-success-subtle"; + } else if (event === "apply_failed" || status === "error") { + outcome = "Failed"; + outcomeClass = "bg-danger-subtle text-danger border border-danger-subtle"; + } else if (status === "warning") { + outcome = "Warning"; + outcomeClass = "bg-warning-subtle text-warning border border-warning-subtle"; + } + + return { + stage, + scope, + outcome, + outcomeClass, + event, + }; +} + +function humanizeLabel(raw, fallback = "Bakery update") { + const normalized = (raw ?? "").toString().trim(); + if (!normalized) { + return fallback; + } + + const spaced = normalized + .replace(/[|:_-]+/g, " ") + .replace(/\s+/g, " ") + .trim(); + + return spaced.charAt(0).toUpperCase() + spaced.slice(1); +} + +function extractSiteToken(summary) { + const match = (summary ?? "").toString().match(/\bsite\s+(-?\d+)\b/i); + return match ? match[1] : null; +} + +function resolvedSiteName(entry) { + return (entry?.siteName ?? "").toString().trim(); +} + +function displaySummary(entry) { + const summary = (entry?.summary ?? "—").toString(); + const siteName = resolvedSiteName(entry); + if (!siteName) { + return summary; + } + return summary.replace(/\bsite\s+-?\d+\b/i, `site ${siteName}`); +} + +function describeOperation(entry, meta) { + const summary = (entry?.summary ?? "").toString().trim(); + const summaryLower = summary.toLowerCase(); + const prefix = summaryPrefix(summary); + const siteToken = extractSiteToken(summary); + const siteName = resolvedSiteName(entry); + const event = meta.event; + + if (event.startsWith("preflight_")) { + return { + kind: "preflight", + key: "preflight", + label: "Reload safety check", + scopeLabel: "Preflight", + }; + } + + if ( + event === "full_reload_trigger" + || event === "full_reload_started" + || event === "reload_required" + || event === "reload_required_cleared" + || meta.scope === "Full Reload" + ) { + return { + kind: "full_reload", + key: "full_reload", + label: "Full reload", + scopeLabel: "Full Reload", + }; + } + + if (event.startsWith("runtime_site_prune_")) { + const label = siteName ? `Cleanup ${siteName}` : (siteToken ? `Cleanup site ${siteToken}` : "Deferred cleanup"); + return { + kind: "runtime_cleanup", + key: `runtime_cleanup:${siteName || siteToken || label}`, + label, + scopeLabel: "Cleanup", + }; + } + + if (summaryLower.includes("treeguard runtime hidden site restore")) { + const label = siteName ? `Restore ${siteName}` : (siteToken ? `Restore site ${siteToken}` : "Restore hidden site"); + return { + kind: "runtime_restore", + key: `runtime_restore:${siteName || siteToken || prefix}`, + label, + scopeLabel: "Live Change", + }; + } + + if (meta.scope === "Live Change" || summaryLower.includes("treeguard runtime")) { + const label = humanizeLabel(prefix || "Live topology change", "Live topology change"); + return { + kind: "live_change", + key: `live_change:${siteToken ?? label}`, + label, + scopeLabel: "Live Change", + }; + } + + if (meta.scope === "StormGuard") { + return { + kind: "stormguard", + key: `stormguard:${prefix || event}`, + label: humanizeLabel(prefix || "StormGuard update", "StormGuard update"), + scopeLabel: "StormGuard", + }; + } + + return { + kind: "bakery", + key: `bakery:${prefix || event || meta.scope}`, + label: humanizeLabel(prefix || meta.scope || "Bakery update", "Bakery update"), + scopeLabel: meta.scope, + }; +} + +function shouldFoldIntoFullReload(entry, descriptor, meta) { + if (descriptor.kind === "preflight") { + return meta.outcome !== "Blocked"; + } + + return descriptor.kind === "bakery" && FULL_RELOAD_SUPPORT_EVENTS.has(meta.event); +} + +function findMergeTarget(groups, entry, descriptor, meta) { + const entryTs = entry?.ts ?? 0; + + if (shouldFoldIntoFullReload(entry, descriptor, meta)) { + return groups.find((group) => + group.kind === "full_reload" + && Math.abs((group.oldestTs ?? 0) - entryTs) <= BAKERY_OPERATION_MERGE_WINDOW_SECONDS); + } + + return groups.find((group) => + group.key === descriptor.key + && Math.abs((group.oldestTs ?? 0) - entryTs) <= BAKERY_OPERATION_MERGE_WINDOW_SECONDS); +} + +function operationStageLabels(group) { + switch (group.kind) { + case "preflight": + return ["Queued", "Checking", "Done"]; + default: + return ["Queued", "Applying", "Verifying", "Cleanup", "Done"]; + } +} + +function operationShowsProgress(group) { + return group.kind === "full_reload" + || group.kind === "preflight" + || group.kind === "live_change" + || group.kind === "runtime_cleanup" + || group.kind === "runtime_restore"; +} + +function operationProgressStep(group) { + const outcome = group.latestMeta?.outcome; + const stage = group.latestMeta?.stage; + if (group.kind === "preflight") { + if (outcome === "Passed" || outcome === "Blocked") { + return 2; + } + return 1; + } + + if (outcome === "Completed" || outcome === "Cleared") { + return 4; + } + if (stage === "Cleanup") { + return 3; + } + if (stage === "Verify") { + return 2; + } + if (stage === "Apply") { + return 1; + } + return 0; +} + +function operationProgressPercent(group) { + const labels = operationStageLabels(group); + const divisor = Math.max(1, labels.length - 1); + return (operationProgressStep(group) / divisor) * 100; +} + +function operationProgressBarClass(group) { + const outcome = group.latestMeta?.outcome; + switch (outcome) { + case "Completed": + case "Passed": + case "Cleared": + return "bg-success"; + case "Failed": + case "Dirty": + case "Reload Required": + return "bg-danger"; + case "Blocked": + case "Retrying": + case "Warning": + return "bg-warning"; + default: + return "bg-info"; + } +} + +function buildOperationSummaries(entries) { + const groups = []; + + entries.forEach((entry) => { + const meta = classifyEvent(entry); + const descriptor = describeOperation(entry, meta); + const targetGroup = findMergeTarget(groups, entry, descriptor, meta); + + if (targetGroup) { + targetGroup.events.push(entry); + targetGroup.oldestTs = Math.min(targetGroup.oldestTs, entry?.ts ?? targetGroup.oldestTs); + return; + } + + groups.push({ + kind: descriptor.kind, + key: descriptor.key, + label: descriptor.label, + scopeLabel: descriptor.scopeLabel, + latestEntry: entry, + latestMeta: meta, + oldestTs: entry?.ts ?? 0, + events: [entry], + }); + }); + + return groups.slice(0, BAKERY_OPERATION_SUMMARY_COUNT); +} + +function loadViewMode() { + try { + const saved = window?.localStorage?.getItem(BAKERY_ACTIVITY_VIEW_STORAGE_KEY); + return saved === "events" ? "events" : "operations"; + } catch (_) { + return "operations"; + } +} + +export class BakeryActivityDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 12; + this.entries = []; + this.currentPage = 0; + this.viewMode = loadViewMode(); + } + + title() { + return "Recent Operations"; + } + + tooltip() { + return "
    Recent Operations

    Summarizes recent Bakery work as human-readable operations with simple progress, while keeping the detailed event log available underneath for operators who need the raw feed.

    "; + } + + subscribeTo() { + return ["BakeryActivity"]; + } + + buildContainer() { + const base = super.buildContainer(); + base.classList.add("dashbox-body-scroll"); + + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + const viewControls = document.createElement("div"); + viewControls.classList.add("d-flex", "justify-content-between", "align-items-center", "gap-2", "flex-wrap", "mb-3"); + + const operationsHeader = document.createElement("div"); + operationsHeader.classList.add("small", "fw-semibold", "text-uppercase", "text-body-secondary", "mb-2"); + operationsHeader.textContent = "View"; + operationsHeader.classList.remove("mb-2"); + operationsHeader.classList.add("mb-0"); + viewControls.appendChild(operationsHeader); + + const toggleGroup = document.createElement("div"); + toggleGroup.classList.add("btn-group", "btn-group-sm"); + + this.operationsButton = document.createElement("button"); + this.operationsButton.type = "button"; + this.operationsButton.textContent = "Operations"; + this.operationsButton.addEventListener("click", () => this.setViewMode("operations")); + + this.eventsButton = document.createElement("button"); + this.eventsButton.type = "button"; + this.eventsButton.textContent = "Event Log"; + this.eventsButton.addEventListener("click", () => this.setViewMode("events")); + + toggleGroup.appendChild(this.operationsButton); + toggleGroup.appendChild(this.eventsButton); + viewControls.appendChild(toggleGroup); + wrap.appendChild(viewControls); + + this.operationsSection = document.createElement("div"); + const operationsSectionHeader = document.createElement("div"); + operationsSectionHeader.classList.add("small", "fw-semibold", "text-uppercase", "text-body-secondary", "mb-2"); + operationsSectionHeader.textContent = "Recent Operations"; + this.operationsSection.appendChild(operationsSectionHeader); + + this.operationsList = document.createElement("div"); + this.operationsList.classList.add("d-flex", "flex-column", "gap-2", "mb-3"); + this.operationsSection.appendChild(this.operationsList); + wrap.appendChild(this.operationsSection); + + this.eventsSection = document.createElement("div"); + const detailHeader = document.createElement("div"); + detailHeader.classList.add("small", "fw-semibold", "text-uppercase", "text-body-secondary", "mb-2"); + detailHeader.textContent = "Detailed Events"; + this.eventsSection.appendChild(detailHeader); + + const tableWrap = document.createElement("div"); + tableWrap.classList.add("lqos-table-wrap"); + + const table = document.createElement("table"); + table.classList.add("lqos-table", "lqos-table-compact", "mb-0", "small"); + + const thead = document.createElement("thead"); + const hr = document.createElement("tr"); + ["Local Time", "Stage", "Outcome", "Scope", "Summary"].forEach((label) => { + const th = document.createElement("th"); + th.textContent = label; + hr.appendChild(th); + }); + thead.appendChild(hr); + + this.tbody = document.createElement("tbody"); + table.appendChild(thead); + table.appendChild(this.tbody); + tableWrap.appendChild(table); + this.eventsSection.appendChild(tableWrap); + + const footer = document.createElement("div"); + footer.classList.add("d-flex", "justify-content-between", "align-items-center", "gap-2", "pt-2", "flex-wrap"); + + this.pageSummary = document.createElement("div"); + this.pageSummary.classList.add("small", "text-secondary"); + footer.appendChild(this.pageSummary); + + const pager = document.createElement("div"); + pager.classList.add("btn-group", "btn-group-sm"); + + this.prevButton = document.createElement("button"); + this.prevButton.type = "button"; + this.prevButton.classList.add("btn", "btn-outline-secondary"); + this.prevButton.innerHTML = " Newer"; + this.prevButton.setAttribute("aria-label", "Show newer Bakery events"); + this.prevButton.addEventListener("click", () => { + if (this.currentPage > 0) { + this.currentPage -= 1; + this.renderPage(); + } + }); + + this.nextButton = document.createElement("button"); + this.nextButton.type = "button"; + this.nextButton.classList.add("btn", "btn-outline-secondary"); + this.nextButton.innerHTML = "Older "; + this.nextButton.setAttribute("aria-label", "Show older Bakery events"); + this.nextButton.addEventListener("click", () => { + if (this.currentPage + 1 < this.totalPages()) { + this.currentPage += 1; + this.renderPage(); + } + }); + + pager.appendChild(this.prevButton); + pager.appendChild(this.nextButton); + footer.appendChild(pager); + this.eventsSection.appendChild(footer); + wrap.appendChild(this.eventsSection); + base.appendChild(wrap); + this.renderViewMode(); + return base; + } + + setViewMode(mode) { + this.viewMode = mode === "events" ? "events" : "operations"; + try { + window?.localStorage?.setItem(BAKERY_ACTIVITY_VIEW_STORAGE_KEY, this.viewMode); + } catch (_) {} + this.renderViewMode(); + } + + renderViewMode() { + if (this.operationsSection) { + this.operationsSection.classList.toggle("d-none", this.viewMode !== "operations"); + } + if (this.eventsSection) { + this.eventsSection.classList.toggle("d-none", this.viewMode !== "events"); + } + if (this.operationsButton) { + this.operationsButton.className = this.viewMode === "operations" ? "btn btn-primary" : "btn btn-outline-secondary"; + } + if (this.eventsButton) { + this.eventsButton.className = this.viewMode === "events" ? "btn btn-primary" : "btn btn-outline-secondary"; + } + } + + totalPages() { + return Math.max(1, Math.ceil(this.entries.length / BAKERY_ACTIVITY_PAGE_SIZE)); + } + + renderPage() { + const entries = this.entries; + const totalRows = entries.length; + const totalPages = this.totalPages(); + this.currentPage = Math.min(this.currentPage, totalPages - 1); + + this.tbody.innerHTML = ""; + if (totalRows === 0) { + const tr = document.createElement("tr"); + const td = document.createElement("td"); + td.colSpan = 5; + td.textContent = "No recent activity"; + tr.appendChild(td); + this.tbody.appendChild(tr); + this.pageSummary.textContent = "0 events"; + this.prevButton.disabled = true; + this.nextButton.disabled = true; + this.renderOperationSummaries([]); + this.renderViewMode(); + return; + } + + const start = this.currentPage * BAKERY_ACTIVITY_PAGE_SIZE; + const end = Math.min(start + BAKERY_ACTIVITY_PAGE_SIZE, totalRows); + const visibleEntries = entries.slice(start, end); + + visibleEntries.forEach((entry) => { + const meta = classifyEvent(entry); + const tr = document.createElement("tr"); + const tdTime = document.createElement("td"); + const tdStage = document.createElement("td"); + const tdOutcome = document.createElement("td"); + const tdScope = document.createElement("td"); + const tdSummary = document.createElement("td"); + + tdTime.textContent = formatUnixSecondsToLocalDateTime(entry?.ts); + tdStage.appendChild( + mkBadge(meta.stage, "bg-body-tertiary text-body-secondary border", meta.event || ""), + ); + tdOutcome.appendChild( + mkBadge(meta.outcome, meta.outcomeClass, entry?.summary || meta.event || ""), + ); + tdScope.appendChild( + mkBadge(meta.scope, "bg-info-subtle text-info border border-info-subtle", entry?.event || ""), + ); + const fullSummary = displaySummary(entry); + tdSummary.textContent = truncateSummary(fullSummary); + tdSummary.title = fullSummary; + + tr.appendChild(tdTime); + tr.appendChild(tdStage); + tr.appendChild(tdOutcome); + tr.appendChild(tdScope); + tr.appendChild(tdSummary); + this.tbody.appendChild(tr); + }); + + this.pageSummary.textContent = `${start + 1}-${end} of ${totalRows} events`; + this.prevButton.disabled = this.currentPage === 0; + this.nextButton.disabled = this.currentPage + 1 >= totalPages; + this.renderOperationSummaries(entries); + this.renderViewMode(); + } + + renderOperationSummaries(entries) { + const groups = buildOperationSummaries(entries); + const cards = groups.map((group) => { + const eventCount = group.events.length; + return { + label: group.label, + outcomeLabel: group.latestMeta?.outcome || "Info", + outcomeClass: group.latestMeta?.outcomeClass || "bg-light text-secondary border", + outcomeTitle: group.latestEntry?.summary || "", + summary: truncateSummary(displaySummary(group.latestEntry)), + summaryTitle: displaySummary(group.latestEntry), + footerLeft: `${group.scopeLabel} • ${eventCount} event${eventCount === 1 ? "" : "s"}`, + footerRight: formatUnixSecondsToLocalDateTime(group.latestEntry?.ts), + stages: operationShowsProgress(group) ? operationStageLabels(group) : [], + progressPercent: operationShowsProgress(group) ? operationProgressPercent(group) : 0, + progressBarClass: operationShowsProgress(group) ? operationProgressBarClass(group) : "bg-info", + }; + }); + renderOperationCards(this.operationsList, cards, { emptyText: "No recent operations" }); + } + + onMessage(msg) { + if (msg.event !== "BakeryActivity") { + return; + } + + this.entries = Array.isArray(msg.data) ? msg.data : []; + this.renderPage(); + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_apply_summary.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_apply_summary.js new file mode 100644 index 00000000..e52ea14d --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_apply_summary.js @@ -0,0 +1,100 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {formatDurationMs, mkBadge} from "./bakery_shared"; + +function formatApplyType(kindRaw) { + switch ((kindRaw ?? "").toString()) { + case "FullReload": + return mkBadge("Full Reload", "bg-warning-subtle text-warning border border-warning-subtle"); + case "LiveChange": + return mkBadge("Live Change", "bg-info-subtle text-info border border-info-subtle"); + case "None": + default: + return mkBadge("None", "bg-light text-secondary border"); + } +} + +export class BakeryApplySummaryDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 4; + } + + title() { + return "Apply Summary"; + } + + tooltip() { + return "
    Apply Summary

    Shows the most recent Bakery apply type, command counts, and recorded build/apply durations.

    "; + } + + subscribeTo() { + return ["BakeryStatus"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + const tableWrap = document.createElement("div"); + tableWrap.classList.add("lqos-table-wrap"); + + const table = document.createElement("table"); + table.classList.add("lqos-table", "lqos-table-compact", "mb-0", "small"); + const tbody = document.createElement("tbody"); + + const mkRow = (label, valueEl) => { + const tr = document.createElement("tr"); + const tdL = document.createElement("td"); + tdL.classList.add("table-label-cell"); + tdL.style.width = "42%"; + tdL.textContent = label; + const tdV = document.createElement("td"); + tdV.classList.add("table-value-cell"); + tdV.appendChild(valueEl); + tr.appendChild(tdL); + tr.appendChild(tdV); + return tr; + }; + + this.applyTypeEl = document.createElement("div"); + this.totalCommandsEl = document.createElement("span"); + this.classCommandsEl = document.createElement("span"); + this.qdiscCommandsEl = document.createElement("span"); + this.buildDurationEl = document.createElement("span"); + this.applyDurationEl = document.createElement("span"); + + tbody.appendChild(mkRow("Last Apply Type", this.applyTypeEl)); + tbody.appendChild(mkRow("Total tc Commands", this.totalCommandsEl)); + tbody.appendChild(mkRow("Class Commands", this.classCommandsEl)); + tbody.appendChild(mkRow("Qdisc Commands", this.qdiscCommandsEl)); + tbody.appendChild(mkRow("Build Time", this.buildDurationEl)); + tbody.appendChild(mkRow("Apply Time", this.applyDurationEl)); + + table.appendChild(tbody); + tableWrap.appendChild(table); + wrap.appendChild(tableWrap); + base.appendChild(wrap); + return base; + } + + onMessage(msg) { + if (msg.event !== "BakeryStatus") { + return; + } + const status = msg?.data?.currentState || {}; + this.applyTypeEl.innerHTML = ""; + this.applyTypeEl.appendChild(formatApplyType(status.lastApplyType)); + this.totalCommandsEl.textContent = Number.isFinite(status.lastTotalTcCommands) + ? status.lastTotalTcCommands.toLocaleString() + : "—"; + this.classCommandsEl.textContent = Number.isFinite(status.lastClassCommands) + ? status.lastClassCommands.toLocaleString() + : "—"; + this.qdiscCommandsEl.textContent = Number.isFinite(status.lastQdiscCommands) + ? status.lastQdiscCommands.toLocaleString() + : "—"; + this.buildDurationEl.textContent = formatDurationMs(status.lastBuildDurationMs); + this.applyDurationEl.textContent = formatDurationMs(status.lastApplyDurationMs); + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_capacity.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_capacity.js new file mode 100644 index 00000000..1e342d0a --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_capacity.js @@ -0,0 +1,184 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {bakeryPreflightBadge, mkBadge} from "./bakery_shared"; + +function clamp(n, min, max) { + return Math.min(Math.max(n, min), max); +} + +function formatPercent(value, max) { + if (!Number.isFinite(value) || !Number.isFinite(max) || max <= 0) { + return "—"; + } + const pct = (value / max) * 100.0; + return `${pct >= 10 ? pct.toFixed(0) : pct.toFixed(1)}%`; +} + +function usageBarClass(plannedQdiscs, safeBudget, hardLimit) { + if (!Number.isFinite(plannedQdiscs) || !Number.isFinite(safeBudget) || safeBudget <= 0) { + return "bg-secondary"; + } + if ((Number.isFinite(hardLimit) && plannedQdiscs > hardLimit) || plannedQdiscs > safeBudget) { + return "bg-danger"; + } + const pct = (plannedQdiscs / safeBudget) * 100.0; + if (pct >= 85) { + return "bg-warning"; + } + if (pct >= 60) { + return "bg-info"; + } + return "bg-success"; +} + +function bakeryMemoryBadge(preflight) { + if (!preflight) { + return mkBadge("Memory Unknown", "bg-light text-secondary border"); + } + if (preflight.memoryOk) { + return mkBadge("Memory OK", "bg-success-subtle text-success border border-success-subtle", preflight.message || ""); + } + return mkBadge("Memory Guard", "bg-danger-subtle text-danger border border-danger-subtle", preflight.message || ""); +} + +export class BakeryCapacityDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 4; + this.lastPreflight = null; + this.liveCapacityInterfaces = []; + this.liveCapacitySafeBudget = null; + } + + title() { + return "Capacity / Safety"; + } + + tooltip() { + return "
    Capacity / Safety

    Shows current live TC handle usage by interface against Bakery's safe budget. The badges summarize the most recent full-reload preflight.

    "; + } + + subscribeTo() { + return ["BakeryStatus"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + this.badgeWrap = document.createElement("div"); + this.badgeWrap.classList.add("d-flex", "flex-wrap", "gap-2", "mb-3"); + wrap.appendChild(this.badgeWrap); + + const tableWrap = document.createElement("div"); + tableWrap.classList.add("lqos-table-wrap", "mb-3"); + + const table = document.createElement("table"); + table.classList.add("lqos-table", "lqos-table-compact", "mb-0", "small"); + + const thead = document.createElement("thead"); + const hr = document.createElement("tr"); + ["Interface", "Usage"].forEach((label) => { + const th = document.createElement("th"); + th.textContent = label; + hr.appendChild(th); + }); + thead.appendChild(hr); + + this.interfacesTbody = document.createElement("tbody"); + + table.appendChild(thead); + table.appendChild(this.interfacesTbody); + tableWrap.appendChild(table); + wrap.appendChild(tableWrap); + base.appendChild(wrap); + return base; + } + + onMessage(msg) { + if (msg.event !== "BakeryStatus") { + return; + } + this.lastPreflight = msg?.data?.currentState?.preflight || null; + this.liveCapacityInterfaces = Array.isArray(msg?.data?.currentState?.liveCapacityInterfaces) + ? msg.data.currentState.liveCapacityInterfaces + : []; + this.liveCapacitySafeBudget = Number.isFinite(msg?.data?.currentState?.liveCapacitySafeBudget) + ? msg.data.currentState.liveCapacitySafeBudget + : null; + this.renderCapacity(); + } + + renderCapacity() { + this.badgeWrap.innerHTML = ""; + this.badgeWrap.appendChild(bakeryPreflightBadge(this.lastPreflight)); + this.badgeWrap.appendChild(bakeryMemoryBadge(this.lastPreflight)); + + this.interfacesTbody.innerHTML = ""; + + const interfaces = Array.isArray(this.liveCapacityInterfaces) + ? [...this.liveCapacityInterfaces].sort((left, right) => { + const liveDiff = (right?.liveQdiscs || 0) - (left?.liveQdiscs || 0); + if (liveDiff !== 0) { + return liveDiff; + } + return (left?.name || "").localeCompare(right?.name || ""); + }) + : []; + + if (interfaces.length === 0) { + const tr = document.createElement("tr"); + const td = document.createElement("td"); + td.colSpan = 2; + td.textContent = "No live interface usage data"; + tr.appendChild(td); + this.interfacesTbody.appendChild(tr); + return; + } + + interfaces.forEach((entry) => { + const tr = document.createElement("tr"); + const tdName = document.createElement("td"); + const tdUsage = document.createElement("td"); + + tdName.textContent = entry?.name || "—"; + + const safeBudget = this.liveCapacitySafeBudget; + const usageText = document.createElement("div"); + usageText.classList.add("d-flex", "justify-content-between", "small", "mb-1"); + const usageLabel = document.createElement("span"); + usageLabel.textContent = Number.isFinite(entry?.liveQdiscs) && safeBudget !== null + ? formatPercent(entry.liveQdiscs, safeBudget) + : "—"; + const usageCount = document.createElement("span"); + usageCount.classList.add("text-body-secondary"); + usageCount.textContent = Number.isFinite(entry?.liveQdiscs) + ? `${entry.liveQdiscs.toLocaleString()} handles` + : "—"; + usageText.appendChild(usageLabel); + usageText.appendChild(usageCount); + + const progress = document.createElement("div"); + progress.classList.add("progress"); + progress.style.height = "0.55rem"; + const bar = document.createElement("div"); + bar.classList.add("progress-bar", usageBarClass(entry?.liveQdiscs, safeBudget, null)); + bar.setAttribute("role", "progressbar"); + const pct = Number.isFinite(entry?.liveQdiscs) && safeBudget !== null && safeBudget > 0 + ? clamp((entry.liveQdiscs / safeBudget) * 100.0, 0, 100) + : 0; + bar.style.width = `${pct.toFixed(1)}%`; + bar.setAttribute("aria-valuemin", "0"); + bar.setAttribute("aria-valuemax", safeBudget !== null ? safeBudget.toString() : "0"); + bar.setAttribute("aria-valuenow", Number.isFinite(entry?.liveQdiscs) ? entry.liveQdiscs.toString() : "0"); + progress.appendChild(bar); + + tdUsage.appendChild(usageText); + tdUsage.appendChild(progress); + + tr.appendChild(tdName); + tr.appendChild(tdUsage); + this.interfacesTbody.appendChild(tr); + }); + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_change_mix.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_change_mix.js new file mode 100644 index 00000000..5d5fb03a --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_change_mix.js @@ -0,0 +1,276 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {formatElapsedSince, mkBadge} from "./bakery_shared"; + +function compactCount(value) { + return Number.isFinite(value) ? value.toLocaleString() : "0"; +} + +function formatSeconds(deltaSeconds) { + const delta = Math.max(0, Math.floor(deltaSeconds)); + if (delta < 60) { + return `${delta}s`; + } + if (delta < 3600) { + return `${Math.floor(delta / 60)}m ${delta % 60}s`; + } + return `${Math.floor(delta / 3600)}h ${Math.floor((delta % 3600) / 60)}m`; +} + +function formatUntil(unixSeconds) { + const n = typeof unixSeconds === "number" ? unixSeconds : parseInt(unixSeconds, 10); + if (!Number.isFinite(n) || n <= 0) { + return "—"; + } + return formatSeconds(n - (Date.now() / 1000)); +} + +function statusTone(status) { + switch (status) { + case "Submitted": + return "text-primary"; + case "Applying": + return "text-info"; + case "Deferred": + return "text-warning"; + case "AppliedAwaitingCleanup": + return "text-warning"; + case "Dirty": + return "text-danger"; + case "Failed": + return "text-danger"; + case "Completed": + return "text-success"; + default: + return "text-primary"; + } +} + +function statusBadgeClass(status) { + switch (status) { + case "Submitted": + return "bg-primary-subtle text-primary border border-primary-subtle"; + case "Applying": + return "bg-info-subtle text-info border border-info-subtle"; + case "Deferred": + case "AppliedAwaitingCleanup": + return "bg-warning-subtle text-warning border border-warning-subtle"; + case "Dirty": + case "Failed": + return "bg-danger-subtle text-danger border border-danger-subtle"; + case "Completed": + return "bg-success-subtle text-success border border-success-subtle"; + default: + return "bg-light text-secondary border"; + } +} + +function siteLabelFor(latest) { + const name = (latest?.siteName ?? "").toString().trim(); + return name || `site ${latest?.siteHash ?? "?"}`; +} + +function actionWords(action) { + if ((action ?? "").toString().trim() === "Restore") { + return { + base: "Restore", + progressive: "Restoring", + completed: "Restored", + }; + } + + return { + base: "Virtualize", + progressive: "Virtualizing", + completed: "Virtualized", + }; +} + +function latestActionLabel(latest) { + const words = actionWords(latest?.action); + const site = siteLabelFor(latest); + switch (latest?.status) { + case "Completed": + return `${words.completed} ${site}`; + case "Applying": + case "AppliedAwaitingCleanup": + return `${words.progressive} ${site}`; + case "Deferred": + return `${words.base} deferred for ${site}`; + case "Submitted": + return `${words.base} requested for ${site}`; + case "Failed": + return `${words.base} failed for ${site}`; + case "Dirty": + return `${words.base} needs reload for ${site}`; + default: + return `${words.base} ${site}`; + } +} + +function statCard(label, value, tone = "text-primary") { + const wrap = document.createElement("div"); + wrap.classList.add( + "d-flex", + "align-items-center", + "justify-content-between", + "gap-2", + "border", + "rounded", + "px-2", + "py-1", + "bg-body-tertiary", + ); + + const top = document.createElement("div"); + top.classList.add("small", "text-body-secondary", "text-truncate"); + top.textContent = label; + + const bottom = document.createElement("div"); + bottom.classList.add("fw-semibold", tone, "small"); + bottom.textContent = value; + + wrap.appendChild(top); + wrap.appendChild(bottom); + return wrap; +} + +export class BakeryChangeMixDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 4; + } + + title() { + return "Live Topology Changes"; + } + + tooltip() { + return "
    Live Topology Changes

    Shows human-readable TreeGuard and Bakery runtime changes, including queued requests, active virtualization work, deferred cleanup, failures, and whether incremental topology changes are currently frozen.

    "; + } + + subscribeTo() { + return ["BakeryStatus"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + this.summaryGrid = document.createElement("div"); + this.summaryGrid.classList.add("d-flex", "flex-wrap", "gap-2", "mb-2"); + + this.latestWrap = document.createElement("div"); + this.latestWrap.classList.add("border", "rounded", "px-2", "py-2", "mb-2"); + + this.badgeWrap = document.createElement("div"); + this.badgeWrap.classList.add("d-flex", "flex-wrap", "gap-2", "mb-2"); + + this.latestMain = document.createElement("div"); + this.latestMain.classList.add("fw-semibold", "mb-1"); + + this.latestMeta = document.createElement("div"); + this.latestMeta.classList.add("small", "text-body-secondary"); + + this.latestWrap.appendChild(this.badgeWrap); + this.latestWrap.appendChild(this.latestMain); + this.latestWrap.appendChild(this.latestMeta); + + this.footerEl = document.createElement("div"); + this.footerEl.classList.add("small", "text-body-secondary"); + + wrap.appendChild(this.summaryGrid); + wrap.appendChild(this.latestWrap); + wrap.appendChild(this.footerEl); + base.appendChild(wrap); + return base; + } + + onMessage(msg) { + if (msg.event !== "BakeryStatus") return; + const status = msg?.data?.currentState || {}; + const runtime = status.runtimeOperations || {}; + this.render(status, runtime); + } + + render(status, runtime) { + const submitted = runtime?.submittedCount || 0; + const deferred = runtime?.deferredCount || 0; + const applying = runtime?.applyingCount || 0; + const cleanup = runtime?.awaitingCleanupCount || 0; + const failed = runtime?.failedCount || 0; + const dirty = runtime?.dirtyCount || 0; + const latest = runtime?.latest || null; + + this.summaryGrid.innerHTML = ""; + [ + statCard("Submitted", compactCount(submitted), submitted > 0 ? "text-primary" : "text-body"), + statCard("Applying", compactCount(applying), applying > 0 ? "text-info" : "text-body"), + statCard("Deferred", compactCount(deferred), deferred > 0 ? "text-warning" : "text-body"), + statCard("Cleanup", compactCount(cleanup), cleanup > 0 ? "text-warning" : "text-body"), + statCard("Failed", compactCount(failed), failed > 0 ? "text-danger" : "text-body"), + statCard("Dirty", compactCount(dirty), dirty > 0 ? "text-danger" : "text-body"), + ].forEach((card) => { + card.style.minWidth = "calc(50% - 0.25rem)"; + this.summaryGrid.appendChild(card); + }); + + this.badgeWrap.innerHTML = ""; + if (status?.reloadRequired) { + this.badgeWrap.appendChild( + mkBadge("Reload Required", "bg-danger-subtle text-danger border border-danger-subtle", status?.reloadRequiredReason || ""), + ); + } + + if (latest?.status) { + this.badgeWrap.appendChild( + mkBadge(latest.status, statusBadgeClass(latest.status), latest?.lastError || latestActionLabel(latest)), + ); + } else if (!status?.reloadRequired && applying + cleanup + failed + dirty + deferred + submitted === 0) { + this.badgeWrap.appendChild(mkBadge("Idle", "bg-light text-secondary border")); + } + + if (latest) { + this.latestMain.className = `fw-semibold mb-1 ${statusTone(latest.status)}`; + this.latestMain.textContent = latestActionLabel(latest); + + const meta = []; + meta.push(`Op ${latest.operationId}`); + if (!latest?.siteName) { + meta.push(`site hash ${latest.siteHash}`); + } + meta.push(`updated ${formatElapsedSince(latest.updatedAtUnix)} ago`); + if (Number.isFinite(latest.attemptCount) && latest.attemptCount > 1) { + meta.push(`${latest.attemptCount} attempts`); + } + if (latest.nextRetryAtUnix) { + meta.push(`retry in ${formatUntil(latest.nextRetryAtUnix)}`); + } + if (latest.lastError) { + meta.push(latest.lastError); + } + this.latestMeta.textContent = meta.join(" • "); + } else { + this.latestMain.className = "fw-semibold mb-1"; + this.latestMain.textContent = "No runtime Bakery operations"; + this.latestMeta.textContent = "TreeGuard has no live virtualization or restore work in flight."; + } + + if (status?.reloadRequiredReason) { + this.footerEl.textContent = status.reloadRequiredReason; + } else if (dirty > 0) { + this.footerEl.textContent = `${compactCount(dirty)} runtime subtree operations are marked dirty.`; + } else if (failed > 0) { + this.footerEl.textContent = `${compactCount(failed)} runtime operations failed and may need operator attention or a full reload.`; + } else if (deferred > 0) { + this.footerEl.textContent = `${compactCount(deferred)} runtime operations were deferred and will retry later.`; + } else if (cleanup > 0) { + this.footerEl.textContent = `${compactCount(cleanup)} runtime operations are waiting for deferred Bakery cleanup.`; + } else if (applying > 0 || submitted > 0) { + this.footerEl.textContent = "Bakery is processing live topology mutations from TreeGuard."; + } else { + this.footerEl.textContent = ""; + } + this.footerEl.classList.toggle("d-none", !this.footerEl.textContent); + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_pipeline.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_pipeline.js new file mode 100644 index 00000000..b7d97d8a --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_pipeline.js @@ -0,0 +1,238 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {bakeryModeBadge, bakeryPreflightBadge, formatDurationMs, formatElapsedSince, mkBadge} from "./bakery_shared"; + +function stageClasses(kind) { + switch (kind) { + case "active": + return ["border-primary-subtle", "bg-primary-subtle", "text-primary"]; + case "ok": + return ["border-success-subtle", "bg-success-subtle", "text-success"]; + case "warning": + return ["border-warning-subtle", "bg-warning-subtle", "text-warning"]; + case "danger": + return ["border-danger-subtle", "bg-danger-subtle", "text-danger"]; + default: + return ["border-secondary-subtle", "bg-body-tertiary", "text-body-secondary"]; + } +} + +function renderStage(host, iconClass, title, statusText, tone, footerNode = null, titleNode = null) { + host.innerHTML = ""; + host.className = ""; + host.classList.add( + "bakery-pipeline-stage", + "border", + "rounded", + "d-flex", + "flex-column", + "h-100", + "small", + ); + stageClasses(tone).forEach((cls) => host.classList.add(cls)); + + const top = document.createElement("div"); + top.classList.add("bakery-pipeline-stage-top", "d-flex", "align-items-center", "gap-2"); + + const icon = document.createElement("i"); + icon.classList.add("fa", "fa-fw", iconClass, "bakery-pipeline-stage-icon"); + top.appendChild(icon); + + const titleWrap = document.createElement("div"); + titleWrap.classList.add("bakery-pipeline-stage-title", "fw-semibold"); + if (titleNode) { + titleWrap.appendChild(titleNode); + } else { + titleWrap.textContent = title; + } + top.appendChild(titleWrap); + host.appendChild(top); + + const status = document.createElement("div"); + status.classList.add("bakery-pipeline-stage-status", "fw-semibold"); + status.textContent = statusText; + status.title = statusText; + host.appendChild(status); + + if (footerNode) { + const footer = document.createElement("div"); + footer.classList.add("bakery-pipeline-stage-footer", "text-body-secondary"); + footer.appendChild(footerNode); + footer.title = footer.textContent || ""; + host.appendChild(footer); + } +} + +export class BakeryPipelineDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 4; + this.lastStatus = null; + } + + title() { + return "Pipeline"; + } + + tooltip() { + return "
    Bakery Pipeline

    Visual overview of Bakery's queue-control flow: plan, preflight, build, apply, and verify.

    "; + } + + subscribeTo() { + return ["BakeryStatus"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + const layout = document.createElement("div"); + layout.classList.add("bakery-pipeline-layout"); + const topRow = document.createElement("div"); + topRow.classList.add("bakery-pipeline-row", "bakery-pipeline-row-top"); + const bottomRow = document.createElement("div"); + bottomRow.classList.add("bakery-pipeline-row", "bakery-pipeline-row-bottom"); + this.stageEls = []; + for (let i = 0; i < 3; i++) { + const col = document.createElement("div"); + col.classList.add("bakery-pipeline-cell"); + const stage = document.createElement("div"); + col.appendChild(stage); + topRow.appendChild(col); + this.stageEls.push(stage); + } + for (let i = 0; i < 3; i++) { + const col = document.createElement("div"); + col.classList.add("bakery-pipeline-cell"); + const stage = document.createElement("div"); + col.appendChild(stage); + bottomRow.appendChild(col); + this.stageEls.push(stage); + } + layout.appendChild(topRow); + layout.appendChild(bottomRow); + + this.alertEl = document.createElement("div"); + this.alertEl.classList.add("alert", "alert-danger", "small", "py-2", "px-3", "mt-3", "d-none"); + + wrap.appendChild(layout); + wrap.appendChild(this.alertEl); + base.appendChild(wrap); + return base; + } + + onMessage(msg) { + if (msg.event !== "BakeryStatus") return; + this.lastStatus = msg?.data?.currentState || null; + this.renderStatus(this.lastStatus); + } + + renderStatus(status = {}) { + const preflight = status.preflight || null; + const applying = status.mode === "ApplyingFullReload" || status.mode === "ApplyingLiveChange"; + const lastFailureUnix = Number.isFinite(status.lastFailureUnix) ? status.lastFailureUnix : 0; + const lastSuccessUnix = Number.isFinite(status.lastSuccessUnix) ? status.lastSuccessUnix : 0; + const lastOutcomeFailed = lastFailureUnix > lastSuccessUnix; + const reloadRequired = !!status.reloadRequired; + const reloadRequiredReason = (status.reloadRequiredReason || "").toString().trim(); + const totalCommands = Number.isFinite(status.currentApplyTotalTcCommands) ? status.currentApplyTotalTcCommands : 0; + const completedCommands = Number.isFinite(status.currentApplyCompletedTcCommands) ? status.currentApplyCompletedTcCommands : 0; + const totalChunks = Number.isFinite(status.currentApplyTotalChunks) ? status.currentApplyTotalChunks : 0; + const completedChunks = Number.isFinite(status.currentApplyCompletedChunks) ? status.currentApplyCompletedChunks : 0; + const progressPct = totalCommands > 0 ? Math.max(0, Math.min(100, (completedCommands / totalCommands) * 100)) : 0; + + const planFooter = document.createElement("span"); + planFooter.textContent = Number.isFinite(status.activeCircuits) + ? `${status.activeCircuits.toLocaleString()} active circuits` + : "Waiting for plan"; + renderStage( + this.stageEls[0], + "fa-sitemap", + "Plan", + applying ? "Queued model ready" : "Ready", + applying ? "active" : "ok", + planFooter, + ); + + renderStage( + this.stageEls[1], + "fa-shield-halved", + "Preflight", + preflight ? (preflight.ok ? "Within budget" : "Blocked") : "Unknown", + !preflight ? "idle" : (preflight.ok ? "ok" : "danger"), + preflight ? bakeryPreflightBadge(preflight) : mkBadge("No snapshot", "bg-light text-secondary border"), + ); + + const buildFooter = document.createElement("span"); + buildFooter.textContent = formatElapsedSince(status.lastFullReloadSuccessUnix); + renderStage( + this.stageEls[2], + "fa-cubes", + "Build", + applying ? (status.currentApplyPhase || "Preparing commands") : "Last build", + applying ? "active" : (status.lastFullReloadSuccessUnix ? "ok" : "idle"), + buildFooter, + ); + + const applyFooter = document.createElement("span"); + applyFooter.textContent = applying + ? `${completedCommands.toLocaleString()} / ${totalCommands.toLocaleString()} tc` + : formatDurationMs(status.lastApplyDurationMs); + renderStage( + this.stageEls[3], + "fa-bolt", + "Apply", + applying ? `${progressPct.toFixed(0)}%` : (status.lastApplyType === "None" ? "Idle" : status.lastApplyType), + applying ? "active" : (status.lastApplyType === "None" ? "idle" : "ok"), + applyFooter, + ); + + const verifyTitleNode = applying ? bakeryModeBadge(status.mode) : null; + const verifyFooter = document.createElement("span"); + if (applying && status.currentActionStartedUnix) { + verifyFooter.textContent = `Running ${formatElapsedSince(status.currentActionStartedUnix)}`; + } else if (lastFailureUnix || lastSuccessUnix) { + verifyFooter.textContent = lastOutcomeFailed + ? (status.lastFailureSummary || "Last run failed") + : "Last run verified"; + } else { + verifyFooter.textContent = "No completed apply yet"; + } + renderStage( + this.stageEls[4], + reloadRequired || lastOutcomeFailed ? "fa-circle-exclamation" : "fa-circle-check", + "Verify", + reloadRequired + ? "Reload Required" + : (applying ? "In progress" : (lastOutcomeFailed ? "Failed" : (lastSuccessUnix ? "Success" : "Pending"))), + reloadRequired + ? "danger" + : (applying ? "warning" : (lastOutcomeFailed ? "danger" : (lastSuccessUnix ? "ok" : "idle"))), + verifyFooter, + verifyTitleNode, + ); + + const tcIoFooter = document.createElement("span"); + if (Number.isFinite(status.lastTcIoUnix) && status.lastTcIoUnix > 0) { + tcIoFooter.textContent = `Last I/O ${formatElapsedSince(status.lastTcIoUnix)}`; + } else { + tcIoFooter.textContent = "Tracks real tc reads/writes"; + } + const tcIoSamples = Number.isFinite(status.tcIoIntervalSamples) ? status.tcIoIntervalSamples : 0; + renderStage( + this.stageEls[5], + "fa-clock", + "TC Interval", + Number.isFinite(status.avgTcIoIntervalMs) + ? formatDurationMs(status.avgTcIoIntervalMs) + : (Number.isFinite(status.lastTcIoUnix) && status.lastTcIoUnix > 0 ? "Collecting" : "No samples"), + tcIoSamples > 0 ? "ok" : "idle", + tcIoFooter, + ); + + this.alertEl.classList.toggle("d-none", !reloadRequired); + this.alertEl.textContent = reloadRequired + ? `Incremental topology mutations are frozen until Bakery performs a structural full reload. ${reloadRequiredReason}`.trim() + : ""; + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_shared.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_shared.js new file mode 100644 index 00000000..92ea012f --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_shared.js @@ -0,0 +1,90 @@ +export function formatUnixSecondsToLocalDateTime(unixSeconds) { + const n = typeof unixSeconds === "number" ? unixSeconds : parseInt(unixSeconds, 10); + if (!Number.isFinite(n) || n <= 0) { + return "—"; + } + return new Date(n * 1000).toLocaleString(undefined, { + year: "numeric", + month: "short", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); +} + +export function formatDurationMs(durationMs) { + const n = typeof durationMs === "number" ? durationMs : parseInt(durationMs, 10); + if (!Number.isFinite(n) || n < 0) { + return "—"; + } + if (n < 1000) { + return `${n} ms`; + } + if (n < 60_000) { + return `${(n / 1000).toFixed(1)} s`; + } + return `${(n / 60_000).toFixed(1)} min`; +} + +export function formatElapsedSince(unixSeconds) { + const n = typeof unixSeconds === "number" ? unixSeconds : parseInt(unixSeconds, 10); + if (!Number.isFinite(n) || n <= 0) { + return "—"; + } + const delta = Math.max(0, Math.floor(Date.now() / 1000) - n); + if (delta < 60) { + return `${delta}s`; + } + if (delta < 3600) { + return `${Math.floor(delta / 60)}m ${delta % 60}s`; + } + if (delta < 86_400) { + const hours = Math.floor(delta / 3600); + const mins = Math.floor((delta % 3600) / 60); + return `${hours}h ${mins}m`; + } + const days = Math.floor(delta / 86_400); + const hours = Math.floor((delta % 86_400) / 3600); + return `${days}d ${hours}h`; +} + +export function mkBadge(text, className, title = "") { + const span = document.createElement("span"); + span.className = `badge ${className}`; + span.textContent = text; + if (title) { + span.title = title; + } + return span; +} + +export function bakeryModeBadge(modeRaw) { + const mode = (modeRaw ?? "").toString(); + switch (mode) { + case "ApplyingFullReload": + return mkBadge("Applying Full Reload", "bg-warning-subtle text-warning border border-warning-subtle"); + case "ApplyingLiveChange": + return mkBadge("Applying Live Change", "bg-info-subtle text-info border border-info-subtle"); + case "Idle": + default: + return mkBadge("Idle", "bg-light text-secondary border"); + } +} + +export function bakeryPreflightBadge(preflight) { + if (!preflight) { + return mkBadge("Unknown", "bg-light text-secondary border"); + } + if (preflight.ok) { + return mkBadge("Within Budget", "bg-success-subtle text-success border border-success-subtle", preflight.message || ""); + } + const interfaces = Array.isArray(preflight.interfaces) ? preflight.interfaces : []; + const safeBudget = Number.isFinite(preflight.safeBudget) ? preflight.safeBudget : null; + const overBudget = safeBudget !== null + && interfaces.some((entry) => Number.isFinite(entry?.plannedQdiscs) && entry.plannedQdiscs > safeBudget); + if (!overBudget && preflight.memoryOk === false) { + return mkBadge("Memory Blocked", "bg-danger-subtle text-danger border border-danger-subtle", preflight.message || ""); + } + return mkBadge("Over Budget", "bg-danger-subtle text-danger border border-danger-subtle", preflight.message || ""); +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_status.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_status.js index 69124e82..a3393ad4 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_status.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_status.js @@ -1,11 +1,80 @@ -import {BakeryCircuitsGraph} from "../graphs/bakery_circuits_graph"; import {DashletBaseInsight} from "./insight_dashlet_base"; +import {formatUnixSecondsToLocalDateTime, mkBadge} from "./bakery_shared"; + +const CIRCUIT_ACTIVITY_MAX_ITEMS = 1; + +function truncateSummary(summary, maxChars = 180) { + const normalized = (summary ?? "").toString().trim(); + if (normalized.length <= maxChars) { + return normalized; + } + return `${normalized.slice(0, maxChars - 3).trimEnd()}...`; +} + +function classifyOutcome(entry) { + const event = (entry?.event ?? "").toString().trim().toLowerCase(); + const status = (entry?.status ?? "").toString().trim().toLowerCase(); + + if (event === "reload_required") { + return { + label: "Reload Required", + className: "bg-danger-subtle text-danger border border-danger-subtle", + }; + } + if (event === "apply_failed" || status === "error") { + return { + label: "Failed", + className: "bg-danger-subtle text-danger border border-danger-subtle", + }; + } + if (event.endsWith("_started")) { + return { + label: "Running", + className: "bg-primary-subtle text-primary border border-primary-subtle", + }; + } + if (event === "apply_finished") { + return { + label: "Completed", + className: "bg-success-subtle text-success border border-success-subtle", + }; + } + if (status === "warning") { + return { + label: "Warning", + className: "bg-warning-subtle text-warning border border-warning-subtle", + }; + } + return { + label: "Info", + className: "bg-light text-secondary border", + }; +} + +function isCircuitActivityEntry(entry) { + const event = (entry?.event ?? "").toString().toLowerCase(); + const summary = (entry?.summary ?? "").toString().toLowerCase(); + if (!summary && !event) { + return false; + } + if (summary.includes("circuit")) { + return true; + } + return event.includes("circuit"); +} + +function recentCircuitEntries(entries) { + return (Array.isArray(entries) ? entries : []) + .filter((entry) => isCircuitActivityEntry(entry)) + .slice(0, CIRCUIT_ACTIVITY_MAX_ITEMS); +} export class BakeryStatusDashlet extends DashletBaseInsight { constructor(slot) { super(slot); - this.lastUpdate = {}; - this.graphDivs = []; + this.size = 6; + this.lastStatus = null; + this.entries = []; } title() { @@ -13,57 +82,146 @@ export class BakeryStatusDashlet extends DashletBaseInsight { } tooltip() { - return "
    Bakery Circuit Activity

    Real-time visualization of active and lazy circuits over the last 5 minutes. Shows how the queue management system is activating and deactivating circuits based on traffic.

    "; + return "
    Bakery Circuit Activity

    Shows live Bakery circuit-change progress while a live change is applying, then keeps the single most recent circuit-scoped Bakery operation underneath.

    "; } subscribeTo() { - return ["BakeryStatus"]; + return ["BakeryStatus", "BakeryActivity"]; } buildContainer() { - let base = super.buildContainer(); - let graphs = this.graphDiv(); - - base.appendChild(graphs); - this.graphDivs.forEach((g) => { - base.appendChild(g); - }); + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + const progressWrap = document.createElement("div"); + progressWrap.classList.add("mb-3"); + + const progressHeader = document.createElement("div"); + progressHeader.classList.add("d-flex", "justify-content-between", "align-items-center", "small", "mb-1", "gap-2", "flex-wrap"); + this.progressSummaryEl = document.createElement("span"); + this.progressSummaryEl.classList.add("text-body-secondary"); + this.progressPercentEl = document.createElement("span"); + this.progressPercentEl.classList.add("fw-semibold"); + progressHeader.appendChild(this.progressSummaryEl); + progressHeader.appendChild(this.progressPercentEl); + + this.progressBarWrapEl = document.createElement("div"); + this.progressBarWrapEl.classList.add("progress"); + this.progressBarWrapEl.style.height = "0.85rem"; + this.progressBarEl = document.createElement("div"); + this.progressBarEl.classList.add("progress-bar"); + this.progressBarEl.setAttribute("role", "progressbar"); + this.progressBarEl.style.width = "0%"; + this.progressBarEl.textContent = "0%"; + this.progressBarWrapEl.appendChild(this.progressBarEl); + + progressWrap.appendChild(progressHeader); + progressWrap.appendChild(this.progressBarWrapEl); + wrap.appendChild(progressWrap); + + const operationsHeader = document.createElement("div"); + operationsHeader.classList.add("small", "fw-semibold", "text-uppercase", "text-body-secondary", "mb-2"); + operationsHeader.textContent = "Recent Circuit Operations"; + wrap.appendChild(operationsHeader); + + this.operationsList = document.createElement("div"); + this.operationsList.classList.add("d-flex", "flex-column", "gap-2"); + wrap.appendChild(this.operationsList); + + base.appendChild(wrap); return base; } - setup() { - super.setup(); - this.graph = new BakeryCircuitsGraph(this.graphDivId()); + renderProgress() { + const status = this.lastStatus || {}; + const totalCommands = Number.isFinite(status.currentApplyTotalTcCommands) + ? status.currentApplyTotalTcCommands + : 0; + const completedCommands = Number.isFinite(status.currentApplyCompletedTcCommands) + ? status.currentApplyCompletedTcCommands + : 0; + const activeLiveChange = status.mode === "ApplyingLiveChange" && totalCommands > 0; + const percent = totalCommands > 0 + ? Math.max(0, Math.min(100, (completedCommands / totalCommands) * 100)) + : 0; + + this.progressSummaryEl.textContent = activeLiveChange + ? `${status.currentApplyPhase || "Applying live change"} • ${completedCommands.toLocaleString()} / ${totalCommands.toLocaleString()} tc` + : "No live change currently applying"; + this.progressPercentEl.textContent = activeLiveChange ? `${percent.toFixed(1)}%` : "Idle"; + this.progressBarEl.style.width = `${percent}%`; + this.progressBarEl.textContent = activeLiveChange ? `${percent.toFixed(1)}%` : "0%"; + this.progressBarEl.setAttribute("aria-valuenow", percent.toFixed(1)); + this.progressBarEl.setAttribute("aria-valuemin", "0"); + this.progressBarEl.setAttribute("aria-valuemax", "100"); + this.progressBarEl.className = activeLiveChange + ? "progress-bar progress-bar-striped progress-bar-animated bg-info" + : "progress-bar bg-secondary"; } - setupZoomed() { - this.zoomGraph = new BakeryCircuitsGraph(this.zoomGraphDivId()); + renderOperations() { + this.operationsList.innerHTML = ""; + const entries = recentCircuitEntries(this.entries); + if (entries.length === 0) { + const empty = document.createElement("div"); + empty.classList.add("border", "rounded", "p-2", "text-muted", "small"); + empty.textContent = "No recent circuit-scoped Bakery activity"; + this.operationsList.appendChild(empty); + return; + } + + entries.forEach((entry) => { + const outcome = classifyOutcome(entry); + const card = document.createElement("div"); + card.classList.add("border", "rounded", "p-2", "bg-body-tertiary"); + + const top = document.createElement("div"); + top.classList.add("d-flex", "justify-content-between", "align-items-start", "gap-2", "flex-wrap", "mb-2"); + + const summary = document.createElement("div"); + summary.classList.add("small", "fw-semibold"); + const fullSummary = (entry?.summary ?? "Bakery update").toString(); + summary.textContent = truncateSummary(fullSummary); + summary.title = fullSummary; + top.appendChild(summary); + top.appendChild(mkBadge(outcome.label, outcome.className, fullSummary)); + card.appendChild(top); + + const footer = document.createElement("div"); + footer.classList.add("d-flex", "justify-content-between", "align-items-center", "gap-2", "small", "text-body-secondary", "flex-wrap"); + + const left = document.createElement("div"); + left.textContent = (entry?.event ?? "bakery_update").toString().replaceAll("_", " "); + footer.appendChild(left); + + const right = document.createElement("div"); + right.textContent = formatUnixSecondsToLocalDateTime(entry?.ts); + footer.appendChild(right); + + card.appendChild(footer); + this.operationsList.appendChild(card); + }); } - teardownZoomed() { - super.teardownZoomed(); - this.zoomGraph = null; + render() { + this.renderProgress(); + this.renderOperations(); } onMessage(msg) { if (msg.event === "BakeryStatus") { - this.lastUpdate = msg.data; - if (msg.data && msg.data.currentState) { - this.graph.update( - msg.data.currentState.activeCircuits, - ); - if (this.zoomGraph) { - this.zoomGraph.update(msg.data.currentState.activeCircuits); - } - } + this.lastStatus = msg?.data?.currentState || null; + this.render(); + return; + } + if (msg.event === "BakeryActivity") { + this.entries = Array.isArray(msg.data) ? msg.data : []; + this.render(); } } canBeSlowedDown() { - return false; // Always update in real-time - } - - supportsZoom() { - return true; + return false; } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_status_summary.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_status_summary.js new file mode 100644 index 00000000..8ed1d8c6 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/bakery_status_summary.js @@ -0,0 +1,109 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import { + bakeryModeBadge, + formatElapsedSince, + formatUnixSecondsToLocalDateTime, + mkBadge, +} from "./bakery_shared"; + +export class BakeryStatusSummaryDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 4; + this.lastStatus = null; + } + + title() { + return "Bakery Status"; + } + + tooltip() { + return "
    Bakery Status

    Current Bakery mode, how long the current action has been running, and the last recorded success/failure timestamps.

    "; + } + + subscribeTo() { + return ["BakeryStatus"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + const tableWrap = document.createElement("div"); + tableWrap.classList.add("lqos-table-wrap"); + + const table = document.createElement("table"); + table.classList.add("lqos-table", "lqos-table-compact", "mb-0", "small"); + const tbody = document.createElement("tbody"); + + const mkRow = (label, valueEl) => { + const tr = document.createElement("tr"); + const tdL = document.createElement("td"); + tdL.classList.add("table-label-cell"); + tdL.style.width = "42%"; + tdL.textContent = label; + const tdV = document.createElement("td"); + tdV.classList.add("table-value-cell"); + tdV.appendChild(valueEl); + tr.appendChild(tdL); + tr.appendChild(tdV); + return tr; + }; + + this.modeEl = document.createElement("div"); + this.durationEl = document.createElement("span"); + this.lastSuccessEl = document.createElement("span"); + this.lastFailureEl = document.createElement("span"); + this.lastFailureSummaryEl = document.createElement("span"); + this.lastFailureSummaryEl.style.whiteSpace = "normal"; + this.lastFailureSummaryEl.style.wordBreak = "break-word"; + this.reloadRequiredEl = document.createElement("div"); + this.reloadReasonEl = document.createElement("span"); + this.reloadReasonEl.style.whiteSpace = "normal"; + this.reloadReasonEl.style.wordBreak = "break-word"; + + tbody.appendChild(mkRow("Current Mode", this.modeEl)); + tbody.appendChild(mkRow("Current Duration", this.durationEl)); + tbody.appendChild(mkRow("Runtime Drift", this.reloadRequiredEl)); + tbody.appendChild(mkRow("Last Success", this.lastSuccessEl)); + tbody.appendChild(mkRow("Last Failure", this.lastFailureEl)); + tbody.appendChild(mkRow("Failure Summary", this.lastFailureSummaryEl)); + tbody.appendChild(mkRow("Reload Reason", this.reloadReasonEl)); + + table.appendChild(tbody); + tableWrap.appendChild(table); + wrap.appendChild(tableWrap); + base.appendChild(wrap); + return base; + } + + onMessage(msg) { + if (msg.event !== "BakeryStatus") { + return; + } + this.lastStatus = msg?.data?.currentState || null; + this.renderStatus(); + } + + renderStatus() { + const status = this.lastStatus || {}; + + this.modeEl.innerHTML = ""; + this.modeEl.appendChild(bakeryModeBadge(status.mode)); + this.reloadRequiredEl.innerHTML = ""; + this.reloadRequiredEl.appendChild( + status.reloadRequired + ? mkBadge("Full Reload Required", "bg-danger-subtle text-danger border border-danger-subtle") + : mkBadge("No Drift", "bg-success-subtle text-success border border-success-subtle"), + ); + + this.durationEl.textContent = status.currentActionStartedUnix + ? formatElapsedSince(status.currentActionStartedUnix) + : "—"; + this.lastSuccessEl.textContent = formatUnixSecondsToLocalDateTime(status.lastSuccessUnix); + this.lastFailureEl.textContent = formatUnixSecondsToLocalDateTime(status.lastFailureUnix); + this.lastFailureSummaryEl.textContent = status.lastFailureSummary || "—"; + this.reloadReasonEl.textContent = status.reloadRequiredReason || "—"; + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/children_sankey_down.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/children_sankey_down.js index 8a8f6ee0..04a2f970 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/children_sankey_down.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/children_sankey_down.js @@ -3,6 +3,14 @@ import {DashboardGraph} from "../graphs/dashboard_graph"; import {colorByRetransmitPct} from "../helpers/color_scales"; import {isRedacted} from "../helpers/redact"; +function retransmitPacketsForNode(node, direction) { + return Number( + node?.current_tcp_retransmit_packets?.[direction] + ?? node?.current_tcp_packets?.[direction] + ?? 0, + ); +} + class ChildrenSankeyGraph extends DashboardGraph { constructor(id, direction) { super(id); @@ -61,7 +69,12 @@ export class ShaperChildrenDown extends BaseDashlet { tooltip(){ return "
    Child Throughput

    Top child nodes by download throughput."; } subscribeTo(){ return ["TreeSummary", "TreeSummaryL2"]; } buildContainer(){ let b = super.buildContainer(); b.appendChild(this.graphDiv()); return b; } - setup(){ this.graph = new ChildrenSankeyGraph(this.graphDivId(), 'down'); if (this.last) this.graph.update(this.last); } + setup(){ + this.traceRender("setup-start"); + this.graph = new ChildrenSankeyGraph(this.graphDivId(), 'down'); + this.traceRender("setup-complete", { graphId: this.graphDivId() }); + if (this.last) this.graph.update(this.last); + } _showEmpty(show, msg = "No recent data"){ const card = document.getElementById(this.id); if (!card) return; @@ -119,8 +132,9 @@ export class ShaperChildrenDown extends BaseDashlet { const pName = p.name || String(parentId); // Parent color by rxmit (download) let pRx = 0; - if ((p.current_tcp_packets?.[0]||0) > 0) { - pRx = ((p.current_retransmits?.[0]||0) / Math.max(1, p.current_tcp_packets?.[0]||0)) * 100.0; + const pPackets = retransmitPacketsForNode(p, 0); + if (pPackets > 0) { + pRx = ((p.current_retransmits?.[0]||0) / Math.max(1, pPackets)) * 100.0; } nodes.push({ name: pName, itemStyle: { color: colorByRetransmitPct(pRx) } }); @@ -131,8 +145,9 @@ export class ShaperChildrenDown extends BaseDashlet { parentSum += v; hasData = hasData || v > 0.5; let cRx = 0; - if ((c.current_tcp_packets?.[0]||0) > 0) { - cRx = ((c.current_retransmits?.[0]||0) / Math.max(1, c.current_tcp_packets?.[0]||0)) * 100.0; + const cPackets = retransmitPacketsForNode(c, 0); + if (cPackets > 0) { + cRx = ((c.current_retransmits?.[0]||0) / Math.max(1, cPackets)) * 100.0; } nodes.push({ name: c.name, itemStyle: { color: colorByRetransmitPct(cRx) } }); links.push({ source: c.name, target: pName, value: v }); @@ -144,8 +159,15 @@ export class ShaperChildrenDown extends BaseDashlet { this._showEmpty(!hasData); if (hasData) { - this.graph.update({ nodes, links }); - return true; + this.traceRender("onMessage", { eventName: "TreeSummaryL2", nodeCount: nodes.length, linkCount: links.length }); + try { + this.graph.update({ nodes, links }); + this.traceRender("update-ok", { eventName: "TreeSummaryL2", nodeCount: nodes.length, linkCount: links.length }); + return true; + } catch (err) { + this.traceRender("update-error", { eventName: "TreeSummaryL2", error: err && err.message ? err.message : String(err) }); + throw err; + } } return false; } @@ -167,8 +189,9 @@ export class ShaperChildrenDown extends BaseDashlet { const m = r[1] || {}; const name = m.name || String(r[0]); const down = Number((m.current_throughput||[0,0])[0]||0); - const rxmit = (m.current_tcp_packets && m.current_tcp_packets[0] > 0) - ? ( (m.current_retransmits?.[0]||0) / Math.max(1, m.current_tcp_packets?.[0]||0) ) * 100.0 + const rxmitPackets = retransmitPacketsForNode(m, 0); + const rxmit = rxmitPackets > 0 + ? ( (m.current_retransmits?.[0]||0) / Math.max(1, rxmitPackets) ) * 100.0 : 0; return { name, value: down, rxmit }; }); @@ -177,7 +200,14 @@ export class ShaperChildrenDown extends BaseDashlet { this._showEmpty(!hasData); if (hasData) { this.last = rows; - this.graph.update(rows); + this.traceRender("onMessage", { eventName: msg.event, rowCount: rows.length }); + try { + this.graph.update(rows); + this.traceRender("update-ok", { eventName: msg.event, rowCount: rows.length }); + } catch (err) { + this.traceRender("update-error", { eventName: msg.event, error: err && err.message ? err.message : String(err) }); + throw err; + } } } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/children_sankey_up.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/children_sankey_up.js index 9b909704..26fd61b0 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/children_sankey_up.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/children_sankey_up.js @@ -3,6 +3,14 @@ import {DashboardGraph} from "../graphs/dashboard_graph"; import {colorByRetransmitPct} from "../helpers/color_scales"; import {isRedacted} from "../helpers/redact"; +function retransmitPacketsForNode(node, direction) { + return Number( + node?.current_tcp_retransmit_packets?.[direction] + ?? node?.current_tcp_packets?.[direction] + ?? 0, + ); +} + class ChildrenSankeyGraphUp extends DashboardGraph { constructor(id) { super(id); @@ -52,7 +60,12 @@ export class ShaperChildrenUp extends BaseDashlet { tooltip(){ return "

    Child Throughput

    Top child nodes by upload throughput."; } subscribeTo(){ return ["TreeSummary", "TreeSummaryL2"]; } buildContainer(){ let b=super.buildContainer(); b.appendChild(this.graphDiv()); return b; } - setup(){ this.graph = new ChildrenSankeyGraphUp(this.graphDivId()); if (this.last) this.graph.update(this.last); } + setup(){ + this.traceRender("setup-start"); + this.graph = new ChildrenSankeyGraphUp(this.graphDivId()); + this.traceRender("setup-complete", { graphId: this.graphDivId() }); + if (this.last) this.graph.update(this.last); + } _showEmpty(show, msg = "No recent data"){ const card = document.getElementById(this.id); if (!card) return; @@ -109,8 +122,9 @@ export class ShaperChildrenUp extends BaseDashlet { const pName = p.name || String(parentId); // Parent color by rxmit (upload) let pRx = 0; - if ((p.current_tcp_packets?.[1]||0) > 0) { - pRx = ((p.current_retransmits?.[1]||0) / Math.max(1, p.current_tcp_packets?.[1]||0)) * 100.0; + const pPackets = retransmitPacketsForNode(p, 1); + if (pPackets > 0) { + pRx = ((p.current_retransmits?.[1]||0) / Math.max(1, pPackets)) * 100.0; } nodes.push({ name: pName, itemStyle: { color: colorByRetransmitPct(pRx) } }); @@ -121,8 +135,9 @@ export class ShaperChildrenUp extends BaseDashlet { parentSum += v; hasData = hasData || v > 0.5; let cRx = 0; - if ((c.current_tcp_packets?.[1]||0) > 0) { - cRx = ((c.current_retransmits?.[1]||0) / Math.max(1, c.current_tcp_packets?.[1]||0)) * 100.0; + const cPackets = retransmitPacketsForNode(c, 1); + if (cPackets > 0) { + cRx = ((c.current_retransmits?.[1]||0) / Math.max(1, cPackets)) * 100.0; } nodes.push({ name: c.name, itemStyle: { color: colorByRetransmitPct(cRx) } }); links.push({ source: pName, target: c.name, value: v }); @@ -134,8 +149,15 @@ export class ShaperChildrenUp extends BaseDashlet { this._showEmpty(!hasData); if (hasData) { - this.graph.update({ nodes, links }); - return true; + this.traceRender("onMessage", { eventName: "TreeSummaryL2", nodeCount: nodes.length, linkCount: links.length }); + try { + this.graph.update({ nodes, links }); + this.traceRender("update-ok", { eventName: "TreeSummaryL2", nodeCount: nodes.length, linkCount: links.length }); + return true; + } catch (err) { + this.traceRender("update-error", { eventName: "TreeSummaryL2", error: err && err.message ? err.message : String(err) }); + throw err; + } } return false; } @@ -154,8 +176,9 @@ export class ShaperChildrenUp extends BaseDashlet { const m = r[1] || {}; const name = m.name || String(r[0]); const up = Number((m.current_throughput||[0,0])[1]||0); - const rxmit = (m.current_tcp_packets && m.current_tcp_packets[1] > 0) - ? ( (m.current_retransmits?.[1]||0) / Math.max(1, m.current_tcp_packets?.[1]||0) ) * 100.0 + const rxmitPackets = retransmitPacketsForNode(m, 1); + const rxmit = rxmitPackets > 0 + ? ( (m.current_retransmits?.[1]||0) / Math.max(1, rxmitPackets) ) * 100.0 : 0; return { name, value: up, rxmit }; }); @@ -164,7 +187,14 @@ export class ShaperChildrenUp extends BaseDashlet { this._showEmpty(!hasData); if (hasData) { this.last = rows; - this.graph.update(rows); + this.traceRender("onMessage", { eventName: msg.event, rowCount: rows.length }); + try { + this.graph.update(rows); + this.traceRender("update-ok", { eventName: msg.event, rowCount: rows.length }); + } catch (err) { + this.traceRender("update-error", { eventName: msg.event, error: err && err.message ? err.message : String(err) }); + throw err; + } } } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/cpu_dash.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/cpu_dash.js index 4dccfab8..76a90ace 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/cpu_dash.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/cpu_dash.js @@ -30,7 +30,11 @@ export class CpuDash extends BaseDashlet{ setup() { super.setup(); + this.traceRender("setup-start"); this.graph = new CpuHistogram(this.graphDivId()); + this.traceRender("setup-complete", { + graphId: this.graphDivId(), + }); } setupZoomed() { @@ -44,9 +48,24 @@ export class CpuDash extends BaseDashlet{ onMessage(msg) { if (msg.event === "Cpu") { - this.graph.update(msg.data); - if (this.zoomGraph) { - this.zoomGraph.update(msg.data); + this.traceRender("onMessage", { + eventName: msg.event, + cpuRows: Array.isArray(msg.data) ? msg.data.length : 0, + }); + try { + this.graph.update(msg.data); + if (this.zoomGraph) { + this.zoomGraph.update(msg.data); + } + this.traceRender("update-ok", { + eventName: msg.event, + }); + } catch (err) { + this.traceRender("update-error", { + eventName: msg.event, + error: err && err.message ? err.message : String(err), + }); + throw err; } } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/dashlet_index.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/dashlet_index.js index 68211401..20f8e7c4 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/dashlet_index.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/dashlet_index.js @@ -28,11 +28,24 @@ import {LtsLast24Hours} from "./ltsLast24Hours"; import {TcpRetransmitsDash} from "./total_retransmits"; import {StormguardStatusDashlet} from "./stormguard_status"; import {BakeryStatusDashlet} from "./bakery_status"; +import {BakeryStatusSummaryDashlet} from "./bakery_status_summary"; +import {BakeryApplySummaryDashlet} from "./bakery_apply_summary"; +import {BakeryCapacityDashlet} from "./bakery_capacity"; +import {BakeryActivityDashlet} from "./bakery_activity"; +import {BakeryPipelineDashlet} from "./bakery_pipeline"; +import {BakeryChangeMixDashlet} from "./bakery_change_mix"; import {TreeGuardStatusDashlet} from "./treeguard_status"; import {TreeGuardActivityDashlet} from "./treeguard_activity"; import {TreeguardControlsDashlet} from "./treeguard_controls"; +import {TreeGuardControlLoopDashlet} from "./treeguard_control_loop"; +import {TreeGuardStateMixDashlet} from "./treeguard_state_mix"; +import {TreeGuardDecisionImpactDashlet} from "./treeguard_decision_impact"; import {Top10UploadersVisual} from "./top10_uploads_graphic"; import {Top10Uploaders} from "./top10_uploaders"; +import {StormguardSummaryDashlet} from "./stormguard_summary"; +import {StormguardSiteListDashlet} from "./stormguard_site_list"; +import {StormguardSiteDetailDashlet} from "./stormguard_site_detail"; +import {StormguardRecentActivityDashlet} from "./stormguard_recent_activity"; // New Traffic Overview dashlets import {ShaperTopAsnDownload} from "./top_asn_download"; import {ShaperTopAsnUpload} from "./top_asn_upload"; @@ -82,9 +95,22 @@ export const DashletMenu = [ { name: "(Insight) Shaper Status", tag: "ltsShaperStatus", size: 3, category: "Insight" }, { name: "(Insight) Last 24 Hours", tag: "ltsLast24", size: 3, category: "Insight" }, { name: "Stormguard Bandwidth Adjustments", tag: "stormguardStatus", size: 6, category: "Queue Management" }, + { name: "StormGuard Summary", tag: "stormguardSummary", size: 12, category: "Queue Management" }, + { name: "StormGuard Sites", tag: "stormguardSiteList", size: 4, category: "Queue Management" }, + { name: "StormGuard Site Detail", tag: "stormguardSiteDetail", size: 8, category: "Queue Management" }, + { name: "StormGuard Recent Activity", tag: "stormguardRecentActivity", size: 12, category: "Queue Management" }, { name: "Bakery Circuit Activity", tag: "bakeryStatus", size: 6, category: "Queue Management" }, + { name: "Bakery Status", tag: "bakeryStatusSummary", size: 4, category: "Bakery" }, + { name: "Bakery Apply Summary", tag: "bakeryApplySummary", size: 4, category: "Bakery" }, + { name: "Bakery Pipeline", tag: "bakeryPipeline", size: 4, category: "Bakery" }, + { name: "Bakery Runtime Operations", tag: "bakeryChangeMix", size: 4, category: "Bakery" }, + { name: "Bakery Capacity / Safety", tag: "bakeryCapacity", size: 4, category: "Bakery" }, + { name: "Bakery Recent Activity", tag: "bakeryActivity", size: 12, category: "Bakery" }, { name: "TreeGuard Controls", tag: "treeguardControls", size: 6, category: "TreeGuard" }, { name: "TreeGuard Status", tag: "treeguardStatus", size: 6, category: "TreeGuard" }, + { name: "TreeGuard Control Loop", tag: "treeguardControlLoop", size: 4, category: "TreeGuard" }, + { name: "TreeGuard State Mix", tag: "treeguardStateMix", size: 4, category: "TreeGuard" }, + { name: "TreeGuard Decision Impact", tag: "treeguardDecisionImpact", size: 4, category: "TreeGuard" }, { name: "TreeGuard Activity", tag: "treeguardActivity", size: 12, category: "TreeGuard" }, // Traffic Overview (Insight-like) { name: "Shaper Top ASN (Download)", tag: "shaperTopAsnDown", size: 6, category: "Traffic" }, @@ -136,9 +162,22 @@ export function widgetFactory(widgetName, count) { case "ltsShaperStatus" : widget = new LtsShaperStatus(count); break; case "ltsLast24" : widget = new LtsLast24Hours(count); break; case "stormguardStatus" : widget = new StormguardStatusDashlet(count); break; + case "stormguardSummary": widget = new StormguardSummaryDashlet(count); break; + case "stormguardSiteList": widget = new StormguardSiteListDashlet(count); break; + case "stormguardSiteDetail": widget = new StormguardSiteDetailDashlet(count); break; + case "stormguardRecentActivity": widget = new StormguardRecentActivityDashlet(count); break; case "bakeryStatus" : widget = new BakeryStatusDashlet(count); break; + case "bakeryStatusSummary": widget = new BakeryStatusSummaryDashlet(count); break; + case "bakeryApplySummary": widget = new BakeryApplySummaryDashlet(count); break; + case "bakeryPipeline": widget = new BakeryPipelineDashlet(count); break; + case "bakeryChangeMix": widget = new BakeryChangeMixDashlet(count); break; + case "bakeryCapacity": widget = new BakeryCapacityDashlet(count); break; + case "bakeryActivity": widget = new BakeryActivityDashlet(count); break; case "treeguardControls": widget = new TreeguardControlsDashlet(count); break; case "treeguardStatus" : widget = new TreeGuardStatusDashlet(count); break; + case "treeguardControlLoop": widget = new TreeGuardControlLoopDashlet(count); break; + case "treeguardStateMix": widget = new TreeGuardStateMixDashlet(count); break; + case "treeguardDecisionImpact": widget = new TreeGuardDecisionImpactDashlet(count); break; case "treeguardActivity": widget = new TreeGuardActivityDashlet(count); break; // Traffic Overview case "shaperTopAsnDown" : widget = new ShaperTopAsnDownload(count); break; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/endpoints_by_country.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/endpoints_by_country.js index a470c994..9305680a 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/endpoints_by_country.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/endpoints_by_country.js @@ -39,7 +39,7 @@ export class Top10EndpointsByCountry extends DashletBaseInsight { let target = document.getElementById(this.id); let t = document.createElement("table"); - t.classList.add("dash-table", "table-sm", "small"); + t.classList.add("lqos-table", "lqos-table-compact", "small", "lqos-topn-plain"); let th = document.createElement("thead"); th.classList.add("small"); @@ -89,14 +89,17 @@ export class Top10EndpointsByCountry extends DashletBaseInsight { rttu.innerHTML = rttNanosAsSpan(r[2][1]); row.appendChild(rttu); - t.appendChild(row); + tbody.appendChild(row); count++; }); t.appendChild(tbody); // Display it clearDashDiv(this.id, target); - target.appendChild(t); + const tableWrap = document.createElement("div"); + tableWrap.classList.add("lqos-table-wrap"); + tableWrap.appendChild(t); + target.appendChild(tableWrap); } } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/executive_heatmap_panels.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/executive_heatmap_panels.js index d62a527c..49e6ba1a 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/executive_heatmap_panels.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/executive_heatmap_panels.js @@ -1,12 +1,11 @@ import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {disposeTooltipsWithin, enableTooltipsWithin} from "../lq_js_common/helpers/tooltips"; import {colorByQoqScore, colorByRetransmitPct, colorByRttMs} from "../helpers/color_scales"; -import {getSiteIdMap, linkToCircuit, linkToSite} from "../executive_utils"; +import {badgeForEntityKind, linkToExecutiveMetricRow} from "../executive_utils"; import { - buildHeatmapRows, colorByCapacity, formatLatest, latestValue, - nonNullCount, heatRow, rttHeatRow, retransmitHeatRow, @@ -41,19 +40,6 @@ function qooInfoIconHtml() { return ``; } -function initTooltipsWithin(rootEl) { - if (!rootEl) return; - if (typeof bootstrap === "undefined" || !bootstrap.Tooltip) return; - const elements = rootEl.querySelectorAll('[data-bs-toggle="tooltip"]'); - elements.forEach((element) => { - if (bootstrap.Tooltip.getOrCreateInstance) { - bootstrap.Tooltip.getOrCreateInstance(element); - } else { - new bootstrap.Tooltip(element); - } - }); -} - function qoqHeatmapRow(blocks, colorFn) { const length = Array.isArray(blocks?.upload_total) && blocks.upload_total.length @@ -153,6 +139,69 @@ function qoqHeatRow(label, badge, blocks, link = null) { `; } +function executiveBadge(row) { + return badgeForEntityKind(row?.entity_kind); +} + +function executiveSplitBlocks(row) { + return { + download: row?.split_blocks?.download || [], + upload: row?.split_blocks?.upload || [], + }; +} + +function executiveQoqBlocks(row) { + return { + download_total: row?.split_blocks?.download || [], + upload_total: row?.split_blocks?.upload || [], + }; +} + +function executiveRttBlocks(row) { + return { + rtt: row?.rtt_blocks?.rtt || [], + rtt_p50_down: row?.rtt_blocks?.dl_p50 || [], + rtt_p90_down: row?.rtt_blocks?.dl_p90 || [], + rtt_p50_up: row?.rtt_blocks?.ul_p50 || [], + rtt_p90_up: row?.rtt_blocks?.ul_p90 || [], + }; +} + +function executiveRetransmitBlocks(row) { + return { + retransmit: row?.scalar_blocks?.values || [], + retransmit_down: row?.split_blocks?.download || [], + retransmit_up: row?.split_blocks?.upload || [], + }; +} + +function latestUtilization(row) { + const split = executiveSplitBlocks(row); + return Math.max( + latestValue(split.download) ?? Number.NEGATIVE_INFINITY, + latestValue(split.upload) ?? Number.NEGATIVE_INFINITY, + ); +} + +function mergedUtilizationRows(summary) { + const byKey = new Map(); + [...(summary?.top_download || []), ...(summary?.top_upload || [])].forEach((row) => { + if (!row?.row_key || byKey.has(row.row_key)) { + return; + } + byKey.set(row.row_key, row); + }); + return [...byKey.values()] + .sort((left, right) => { + const latestDiff = latestUtilization(right) - latestUtilization(left); + if (Number.isFinite(latestDiff) && latestDiff !== 0) { + return latestDiff; + } + return String(left?.label || "").localeCompare(String(right?.label || "")); + }) + .slice(0, MAX_HEATMAP_ROWS); +} + class ExecutiveHeatmapBase extends BaseDashlet { constructor(slot) { super(slot); @@ -167,12 +216,12 @@ class ExecutiveHeatmapBase extends BaseDashlet { } canBeSlowedDown() { return true; } - subscribeTo() { return ["ExecutiveHeatmaps"]; } + subscribeTo() { return ["ExecutiveDashboardSummary"]; } onMessage(msg) { - if (msg.event === "ExecutiveHeatmaps") { + if (msg.event === "ExecutiveDashboardSummary") { this.lastData = msg.data; - window.executiveHeatmapData = msg.data; + window.executiveDashboardSummary = msg.data; this.render(); } } @@ -182,8 +231,8 @@ class ExecutiveHeatmapBase extends BaseDashlet { window.addEventListener("colorBlindModeChanged", this._colorBlindListener); this._colorBlindBound = true; } - if (window.executiveHeatmapData) { - this.lastData = window.executiveHeatmapData; + if (window.executiveDashboardSummary) { + this.lastData = window.executiveDashboardSummary; this.render(); } } @@ -260,6 +309,7 @@ export class ExecutiveGlobalHeatmapDashlet extends ExecutiveHeatmapBase { return heatRow(row.label, row.badge, row.values, row.color, row.format); }) .join(""); + disposeTooltipsWithin(target); target.innerHTML = `

    @@ -271,7 +321,7 @@ export class ExecutiveGlobalHeatmapDashlet extends ExecutiveHeatmapBase {
    `; - initTooltipsWithin(target); + enableTooltipsWithin(target); } } @@ -293,166 +343,101 @@ class ExecutiveMetricHeatmapBase extends ExecutiveHeatmapBase { render() { const target = document.getElementById(this._contentId); if (!target) return; - const rows = buildHeatmapRows(this.lastData || {}); - const filteredRows = this.config.hideAsns ? rows.filter(row => row.badge !== "ASN") : rows; - if (!filteredRows.length) { + const summary = this.lastData || {}; + const rows = this.summaryRows(summary); + if (!rows.length) { target.innerHTML = this.emptyCard(); return; } - const sorted = filteredRows.slice().sort((a, b) => this.metricSort(a, b)); - const limited = sorted.slice(0, MAX_HEATMAP_ROWS); - getSiteIdMap().then((siteIdMap) => { - const activeTarget = document.getElementById(this._contentId); - if (!activeTarget) return; - const metricRows = limited.map(row => { - const link = row.badge === "Circuit" - ? linkToCircuit(row.circuit_id) - : row.badge === "Site" - ? linkToSite(row.site_name || row.label, siteIdMap) - : null; - if (this.config.metricKey === "rtt") { - return rttHeatRow( - row.label, - row.badge, - row.blocks, - this.config.colorFn, - this.config.formatFn, - link - ); - } - if (this.config.metricKey === "retransmit") { - return retransmitHeatRow( - row.label, - row.badge, - row.blocks, - this.config.colorFn, - this.config.formatFn, - link - ); - } - if (this.config.metricKey === "utilization") { - return utilizationHeatRow( - row.label, - row.badge, - row.blocks, - this.config.colorFn, - this.config.formatFn, - link - ); - } - if (this.config.metricKey === "qoo") { - return qoqHeatRow( - row.label, - row.badge, - row.qoq_blocks, - link, - ); - } - return heatRow( + const activeTarget = document.getElementById(this._contentId); + if (!activeTarget) return; + const metricRows = rows.map((row) => { + const badge = executiveBadge(row); + const link = linkToExecutiveMetricRow(row); + if (this.config.metricKey === "rtt") { + return rttHeatRow( row.label, - row.badge, - row.blocks[this.config.metricKey] || [], + badge, + executiveRttBlocks(row), this.config.colorFn, this.config.formatFn, - link + link, ); - }).join(""); - const linkIcon = this.config.link - ? `` - : ""; - const isQooMetric = this.config.metricKey === "qoo"; - const titleLabel = isQooMetric - ? `QoO${qooInfoIconHtml()} Heatmap` - : this.config.title; - const titleHtml = this.config.link - ? `${titleLabel}${linkIcon}` - : `${titleLabel}`; - activeTarget.innerHTML = ` -
    -
    -
    -
    ${titleHtml}
    -
    -
    ${metricRows}
    + } + if (this.config.metricKey === "retransmit") { + return retransmitHeatRow( + row.label, + badge, + executiveRetransmitBlocks(row), + this.config.colorFn, + this.config.formatFn, + link, + ); + } + if (this.config.metricKey === "utilization") { + return utilizationHeatRow( + row.label, + badge, + executiveSplitBlocks(row), + this.config.colorFn, + this.config.formatFn, + link, + ); + } + if (this.config.metricKey === "qoo") { + return qoqHeatRow( + row.label, + badge, + executiveQoqBlocks(row), + link, + ); + } + return heatRow( + row.label, + badge, + row?.scalar_blocks?.values || [], + this.config.colorFn, + this.config.formatFn, + link, + ); + }).join(""); + const linkIcon = this.config.link + ? `` + : ""; + const isQooMetric = this.config.metricKey === "qoo"; + const titleLabel = isQooMetric + ? `QoO${qooInfoIconHtml()} Heatmap` + : this.config.title; + const titleHtml = this.config.link + ? `${titleLabel}${linkIcon}` + : `${titleLabel}`; + disposeTooltipsWithin(activeTarget); + activeTarget.innerHTML = ` +
    +
    +
    +
    ${titleHtml}
    +
    ${metricRows}
    - `; - initTooltipsWithin(activeTarget); - }); +
    + `; + enableTooltipsWithin(activeTarget); } - metricSort(a, b) { - const metricKey = this.config.metricKey; - const isUtilization = metricKey === "utilization"; - const isQoo = metricKey === "qoo"; - const aDownVals = a.blocks.download || []; - const aUpVals = a.blocks.upload || []; - const bDownVals = b.blocks.download || []; - const bUpVals = b.blocks.upload || []; - - const aVals = isUtilization ? [] : (a.blocks[metricKey] || []); - const bVals = isUtilization ? [] : (b.blocks[metricKey] || []); - - const latestQoo = (blocks) => { - if (!blocks) return null; - const vals = [ - latestValue(blocks.download_total), - latestValue(blocks.upload_total), - ].filter((v) => v !== null && v !== undefined); - if (!vals.length) return null; - const sum = vals.reduce((x, y) => x + y, 0); - return sum / vals.length; - }; - const countQoo = (blocks) => { - if (!blocks) return 0; - return Math.max( - nonNullCount(blocks.download_total), - nonNullCount(blocks.upload_total), - ); - }; - - const aLatest = isQoo - ? latestQoo(a.qoq_blocks) - : isUtilization - ? Math.max(latestValue(aDownVals) ?? -Infinity, latestValue(aUpVals) ?? -Infinity) - : latestValue(aVals); - const bLatest = isQoo - ? latestQoo(b.qoq_blocks) - : isUtilization - ? Math.max(latestValue(bDownVals) ?? -Infinity, latestValue(bUpVals) ?? -Infinity) - : latestValue(bVals); - const aCount = isQoo - ? countQoo(a.qoq_blocks) - : isUtilization - ? Math.max(nonNullCount(aDownVals), nonNullCount(aUpVals)) - : nonNullCount(aVals); - const bCount = isQoo - ? countQoo(b.qoq_blocks) - : isUtilization - ? Math.max(nonNullCount(bDownVals), nonNullCount(bUpVals)) - : nonNullCount(bVals); - const countWeight = this.config.countWeight || 0; - const minSamples = this.config.minSamples || 0; - - const score = (latest, count) => { - if (latest === null || latest === undefined) return -Infinity; - let base = latest; - if (isQoo) { - // QoO: higher is better; show worst first by inverting for sort score. - base = -latest; - } - let s = base + countWeight * (count / 15); - if (count < minSamples) { - s -= 1000; // Heavily de-prioritize sparse data - } - return s; - }; - - const aScore = score(aLatest, aCount); - const bScore = score(bLatest, bCount); - if (bScore !== aScore) return bScore - aScore; - if (bCount !== aCount) return bCount - aCount; - return (a.label || "").localeCompare(b.label || ""); + summaryRows(summary) { + switch (this.config.metricKey) { + case "rtt": + return (summary?.top_rtt || []).slice(0, MAX_HEATMAP_ROWS); + case "retransmit": + return (summary?.top_retransmit || []).slice(0, MAX_HEATMAP_ROWS); + case "utilization": + return mergedUtilizationRows(summary); + case "qoo": + return (summary?.top_qoo || []).slice(0, MAX_HEATMAP_ROWS); + default: + return []; + } } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/executive_snapshot.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/executive_snapshot.js index e41474f1..fe94faad 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/executive_snapshot.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/executive_snapshot.js @@ -18,7 +18,7 @@ export class ExecutiveSnapshotDashlet extends BaseDashlet { canBeSlowedDown() { return true; } title() { return "Network Snapshot"; } tooltip() { return "Headline health metrics for the executive view."; } - subscribeTo() { return ["ExecutiveHeatmaps", "Throughput"]; } + subscribeTo() { return ["ExecutiveDashboardSummary", "Throughput"]; } buildContainer() { const container = super.buildContainer(); @@ -36,14 +36,14 @@ export class ExecutiveSnapshotDashlet extends BaseDashlet { } setup() { - if (window.executiveHeatmapData) { + if (window.executiveDashboardSummary) { this.render(); } } onMessage(msg) { - if (msg.event === "ExecutiveHeatmaps") { - window.executiveHeatmapData = msg.data; + if (msg.event === "ExecutiveDashboardSummary") { + window.executiveDashboardSummary = msg.data; this.render(); return; } @@ -54,7 +54,7 @@ export class ExecutiveSnapshotDashlet extends BaseDashlet { } render() { - const header = window.executiveHeatmapData?.header; + const header = window.executiveDashboardSummary?.header; const target = document.getElementById(this._contentId); if (!target) return; if (!header) { @@ -63,15 +63,9 @@ export class ExecutiveSnapshotDashlet extends BaseDashlet { } target.innerHTML = ` -
    -
    Network Snapshot
    -
    - ${this.inventoryCard(header)} - ${this.queuesCard(header)} ${this.throughputCard()} - ${this.simpleCard("Mapped IPs", "fa-map-marker-alt", "text-primary", formatCount(header.mapped_ip_count))} - ${this.simpleCard("Unknown IPs", "fa-question-circle", "text-warning", formatCount(header.unmapped_ip_count))} + ${this.inventoryCard(header)} ${this.insightCard(header)}
    `; @@ -86,14 +80,6 @@ export class ExecutiveSnapshotDashlet extends BaseDashlet { return this.groupCard("Inventory", "fa-layer-group", "text-primary", items); } - queuesCard(header) { - const items = [ - { label: "HTB", value: formatCount(header.htb_queue_count) }, - { label: "CAKE", value: formatCount(header.cake_queue_count) }, - ]; - return this.groupCard("Queues", "fa-stream", "text-secondary", items, false, true); - } - throughputCard() { const bps = this.latestThroughput?.bps; const items = [ @@ -103,22 +89,6 @@ export class ExecutiveSnapshotDashlet extends BaseDashlet { return this.groupCard("Throughput", "fa-tachometer-alt", "text-info", items, true); } - simpleCard(label, icon, accent, value) { - return ` -
    -
    -
    - -
    -
    ${label}
    -
    ${value}
    -
    -
    -
    -
    - `; - } - insightCard(header) { const value = header.insight_connected; const badge = value === undefined @@ -141,7 +111,7 @@ export class ExecutiveSnapshotDashlet extends BaseDashlet { `; } - groupCard(title, icon, accent, items, allowHtmlLabel = false, allowAlerts = false) { + groupCard(title, icon, accent, items, allowHtmlLabel = false, allowAlerts = false, statusBadge = "", footer = "") { const hasZero = allowAlerts && items.some(item => this.isZero(item.value)); const rows = items.map((item, idx) => `
    @@ -155,8 +125,12 @@ export class ExecutiveSnapshotDashlet extends BaseDashlet {
    -
    ${title}
    +
    +
    ${title}
    + ${statusBadge} +
    ${rows}
    + ${footer}
    @@ -166,7 +140,7 @@ export class ExecutiveSnapshotDashlet extends BaseDashlet { renderValueWithAlert(value, showAlert) { const isZero = this.isZero(value); - const valueClass = isZero ? "text-danger" : "text-secondary"; + const valueClass = isZero && showAlert ? "text-danger" : "text-secondary"; if (!showAlert || !isZero) { return `${value}`; } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/operation_cards.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/operation_cards.js new file mode 100644 index 00000000..ef7c6bc6 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/operation_cards.js @@ -0,0 +1,87 @@ +import {mkBadge} from "./bakery_shared"; + +export function renderOperationCards(host, groups, options = {}) { + const emptyText = options.emptyText || "No recent operations"; + host.innerHTML = ""; + + if (!Array.isArray(groups) || groups.length === 0) { + const empty = document.createElement("div"); + empty.classList.add("border", "rounded", "p-2", "text-muted", "small"); + empty.textContent = emptyText; + host.appendChild(empty); + return; + } + + groups.forEach((group) => { + const card = document.createElement("div"); + card.classList.add("border", "rounded", "p-2", "bg-body-tertiary"); + + const top = document.createElement("div"); + top.classList.add("d-flex", "justify-content-between", "align-items-start", "gap-2", "flex-wrap", "mb-2"); + + const titleWrap = document.createElement("div"); + const title = document.createElement("div"); + title.classList.add("fw-semibold"); + title.textContent = group.label || "Operation"; + titleWrap.appendChild(title); + + const right = document.createElement("div"); + right.classList.add("d-flex", "flex-wrap", "gap-2", "align-items-center"); + right.appendChild( + mkBadge( + group.outcomeLabel || "Info", + group.outcomeClass || "bg-light text-secondary border", + group.outcomeTitle || "", + ), + ); + + top.appendChild(titleWrap); + top.appendChild(right); + card.appendChild(top); + + if ((group.summary || "").toString().trim()) { + const summary = document.createElement("div"); + summary.classList.add("small"); + summary.textContent = group.summary; + if (group.summaryTitle) { + summary.title = group.summaryTitle; + } + card.appendChild(summary); + } + + const footer = document.createElement("div"); + footer.classList.add("d-flex", "justify-content-between", "align-items-center", "gap-2", "mt-2", "small", "text-body-secondary", "flex-wrap"); + + const footerLeft = document.createElement("div"); + footerLeft.textContent = group.footerLeft || ""; + footer.appendChild(footerLeft); + + const footerRight = document.createElement("div"); + footerRight.textContent = group.footerRight || ""; + footer.appendChild(footerRight); + card.appendChild(footer); + + if (Array.isArray(group.stages) && group.stages.length > 0) { + const progress = document.createElement("div"); + progress.classList.add("progress", "mt-2", "mb-2"); + progress.style.height = "0.45rem"; + + const progressBar = document.createElement("div"); + progressBar.classList.add("progress-bar", group.progressBarClass || "bg-info"); + progressBar.style.width = `${group.progressPercent || 0}%`; + progressBar.setAttribute("role", "progressbar"); + progressBar.setAttribute("aria-valuemin", "0"); + progressBar.setAttribute("aria-valuemax", "100"); + progressBar.setAttribute("aria-valuenow", `${Math.round(group.progressPercent || 0)}`); + progress.appendChild(progressBar); + card.appendChild(progress); + + const stages = document.createElement("div"); + stages.classList.add("small", "text-body-secondary"); + stages.textContent = group.stages.join(" -> "); + card.appendChild(stages); + } + + host.appendChild(card); + }); +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/queue_stats_total.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/queue_stats_total.js index 1b1d56d5..e0d68c12 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/queue_stats_total.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/queue_stats_total.js @@ -72,7 +72,7 @@ export class QueueStatsTotalDash extends DashletBaseInsight { onMessage(msg) { if (msg.event === "QueueStatsTotal" && window.timePeriods.activePeriod === "Live") { - this.graph.update(msg.marks, msg.drops); + this.#applyLiveSample(msg.marks, msg.drops); } } @@ -141,4 +141,8 @@ export class QueueStatsTotalDash extends DashletBaseInsight { ); wsClient.send({ LtsCake: { seconds } }); } + + #applyLiveSample(marks, drops) { + this.graph.update(marks, drops); + } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/ram_dash.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/ram_dash.js index 2891330c..eb6337a4 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/ram_dash.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/ram_dash.js @@ -31,7 +31,11 @@ export class RamDash extends BaseDashlet{ setup() { super.setup(); + this.traceRender("setup-start"); this.graph = new RamPie(this.graphDivId()); + this.traceRender("setup-complete", { + graphId: this.graphDivId(), + }); } setupZoomed() { @@ -48,9 +52,26 @@ export class RamDash extends BaseDashlet{ let total = toNumber(msg.data.total, 0); let used = toNumber(msg.data.used, 0); let free = Math.max(0, total - used); - this.graph.update(free, used); - if (this.zoomGraph) { - this.zoomGraph.update(free, used); + this.traceRender("onMessage", { + eventName: msg.event, + total, + used, + free, + }); + try { + this.graph.update(free, used); + if (this.zoomGraph) { + this.zoomGraph.update(free, used); + } + this.traceRender("update-ok", { + eventName: msg.event, + }); + } catch (err) { + this.traceRender("update-error", { + eventName: msg.event, + error: err && err.message ? err.message : String(err), + }); + throw err; } } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/rtt_histo_dash.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/rtt_histo_dash.js index 09826c13..a4313a08 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/rtt_histo_dash.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/rtt_histo_dash.js @@ -31,8 +31,12 @@ export class RttHistoDash extends DashletBaseInsight { setup() { super.setup(); + this.traceRender("setup-start"); this.graph = new RttHistogram(this.graphDivId()); window.timeGraphs.push(this); + this.traceRender("setup-complete", { + graphId: this.graphDivId(), + }); } setupZoomed() { @@ -51,9 +55,24 @@ export class RttHistoDash extends DashletBaseInsight { onMessage(msg) { if (msg.event === "RttHistogram" && window.timePeriods.activePeriod === "Live") { - this.graph.update(msg.data); - if (this.zoomGraph) { - this.zoomGraph.update(msg.data); + this.traceRender("onMessage", { + eventName: msg.event, + bins: Array.isArray(msg.data) ? msg.data.length : 0, + }); + try { + this.graph.update(msg.data); + if (this.zoomGraph) { + this.zoomGraph.update(msg.data); + } + this.traceRender("update-ok", { + eventName: msg.event, + }); + } catch (err) { + this.traceRender("update-error", { + eventName: msg.event, + error: err && err.message ? err.message : String(err), + }); + throw err; } } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_recent_activity.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_recent_activity.js new file mode 100644 index 00000000..906b8bbd --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_recent_activity.js @@ -0,0 +1,99 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {mkBadge} from "./bakery_shared"; +import { + formatStormguardAgeSeconds, + stormguardActivityRows, + subscribeStormguardState, + updateStormguardDebug, + updateStormguardStatus, +} from "./stormguard_shared"; + +function activityBadge(row) { + return mkBadge(row.summary.label, row.summary.className, row.summary.reason || ""); +} + +export class StormguardRecentActivityDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 12; + this.unsubscribe = null; + } + + title() { + return "StormGuard Recent Activity"; + } + + tooltip() { + return "
    StormGuard Recent Activity

    Shows the freshest StormGuard actions and holding states so operators can quickly answer what the control loop is doing right now.

    "; + } + + subscribeTo() { + return ["StormguardStatus", "StormguardDebug"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + const tableWrap = document.createElement("div"); + tableWrap.classList.add("table-responsive", "lqos-table-wrap"); + const table = document.createElement("table"); + table.classList.add("lqos-table", "lqos-table-compact", "align-middle", "mb-0"); + table.innerHTML = ` +
    + + + + + + + + + + `; + this.tbody = document.createElement("tbody"); + table.appendChild(this.tbody); + tableWrap.appendChild(table); + wrap.appendChild(tableWrap); + + base.appendChild(wrap); + return base; + } + + setup() { + this.unsubscribe = subscribeStormguardState((snapshot) => this.renderSnapshot(snapshot)); + } + + onMessage(msg) { + if (msg.event === "StormguardStatus") { + updateStormguardStatus(msg.data || []); + } + if (msg.event === "StormguardDebug") { + updateStormguardDebug(msg.data || []); + } + } + + renderSnapshot(snapshot) { + const rows = stormguardActivityRows(snapshot, 12); + if (rows.length === 0) { + this.tbody.innerHTML = ""; + return; + } + + this.tbody.innerHTML = ""; + rows.forEach((row) => { + const tr = document.createElement("tr"); + tr.innerHTML = ` + + + + + + + `; + tr.children[2].appendChild(activityBadge(row)); + this.tbody.appendChild(tr); + }); + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_shared.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_shared.js new file mode 100644 index 00000000..59d69abb --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_shared.js @@ -0,0 +1,423 @@ +import {loadConfig} from "../config/config_helper"; + +const CHANGE_RECENT_WINDOW_SECONDS = 300; + +const listeners = new Set(); +const store = { + configRequested: false, + config: null, + configError: false, + lastStatusPayload: null, + lastDebugPayload: null, + statusRows: [], + debugEntries: [], + selectedSite: null, +}; + +function num(value) { + const n = Number(value); + return Number.isFinite(n) ? n : null; +} + +function text(value) { + return (value ?? "").toString(); +} + +function titleCase(value) { + const s = text(value).trim(); + if (!s) return ""; + return s.charAt(0).toUpperCase() + s.slice(1); +} + +export function formatStormguardMbps(value) { + const n = num(value); + if (n === null) return "—"; + return n.toLocaleString(undefined, {maximumFractionDigits: 2}); +} + +export function formatStormguardPercent(value) { + const n = num(value); + if (n === null) return "—"; + return `${(n * 100).toFixed(2)}%`; +} + +export function formatStormguardMs(value) { + const n = num(value); + if (n === null) return "—"; + return `${n.toFixed(2)} ms`; +} + +export function formatStormguardAgeSeconds(value) { + const n = num(value); + if (n === null) return "—"; + if (n < 60) return `${Math.round(n)}s`; + if (n < 3600) return `${Math.floor(n / 60)}m ${Math.round(n % 60)}s`; + return `${Math.floor(n / 3600)}h ${Math.floor((n % 3600) / 60)}m`; +} + +export function directionSummary(direction) { + if (!direction) { + return { + label: "No Data", + className: "bg-light text-secondary border", + reason: "No live metrics yet", + }; + } + + const cooldown = num(direction.cooldown_remaining_secs) ?? 0; + const queue = num(direction.queue_mbps); + const min = num(direction.min_mbps); + const max = num(direction.max_mbps); + const action = text(direction.last_action).trim().toLowerCase(); + const state = text(direction.state).trim().toLowerCase(); + const delay = num(direction.delay_ms); + const retrans = num(direction.retrans_ma) ?? num(direction.retrans); + + if (cooldown > 0.05) { + return { + label: "Cooling Down", + className: "bg-warning-subtle text-warning border border-warning-subtle", + reason: `${cooldown.toFixed(1)}s remaining`, + }; + } + + if (action.includes("decrease")) { + return { + label: "Reducing", + className: "bg-danger-subtle text-danger border border-danger-subtle", + reason: "StormGuard most recently lowered this queue", + }; + } + + if (action.includes("increase")) { + return { + label: "Recovering", + className: "bg-success-subtle text-success border border-success-subtle", + reason: "StormGuard most recently raised this queue", + }; + } + + if (queue !== null && min !== null && queue <= min && direction.can_decrease === false) { + return { + label: "At Floor", + className: "bg-secondary-subtle text-secondary border border-secondary-subtle", + reason: "Queue is already at its configured minimum", + }; + } + + if (queue !== null && max !== null && queue >= max && direction.can_increase === false) { + return { + label: "At Ceiling", + className: "bg-primary-subtle text-primary border border-primary-subtle", + reason: "Queue is already at its configured maximum", + }; + } + + if (delay !== null && delay > 5) { + return { + label: "Congestion Signal", + className: "bg-danger-subtle text-danger border border-danger-subtle", + reason: `Standing delay is ${delay.toFixed(2)} ms above baseline`, + }; + } + + if (retrans !== null && retrans >= 0.01) { + return { + label: "Loss Signal", + className: "bg-warning-subtle text-warning border border-warning-subtle", + reason: `Retransmits are at ${formatStormguardPercent(retrans)}`, + }; + } + + if (state === "warmup") { + return { + label: "Warming Up", + className: "bg-info-subtle text-info border border-info-subtle", + reason: "StormGuard is collecting baseline samples", + }; + } + + return { + label: "Holding", + className: "bg-light text-secondary border", + reason: "Monitoring conditions without changing the queue", + }; +} + +export function directionReason(direction) { + if (!direction) return "No live metrics yet."; + const cooldown = num(direction.cooldown_remaining_secs) ?? 0; + if (cooldown > 0.05) { + return `Cooldown active for ${cooldown.toFixed(1)}s.`; + } + + const delay = num(direction.delay_ms); + if (delay !== null && delay > 0.5) { + return `Standing delay is ${delay.toFixed(2)} ms above the learned baseline.`; + } + + const retrans = num(direction.retrans_ma) ?? num(direction.retrans); + if (retrans !== null && retrans >= 0.005) { + return `Retransmits are ${formatStormguardPercent(retrans)}.`; + } + + const queue = num(direction.queue_mbps); + const min = num(direction.min_mbps); + const max = num(direction.max_mbps); + if (queue !== null && min !== null && queue <= min && direction.can_decrease === false) { + return "The queue is already at the configured minimum."; + } + if (queue !== null && max !== null && queue >= max && direction.can_increase === false) { + return "The queue is already at the configured maximum."; + } + + const throughput = num(direction.throughput_ma_mbps) ?? num(direction.throughput_mbps); + if (throughput !== null && max !== null && throughput < max * 0.2) { + return "Load is low, so StormGuard is holding steady."; + } + + return "StormGuard is monitoring conditions and waiting for a clearer signal."; +} + +function computeAttentionScore(site) { + let score = 0; + if (site.changedRecently) score += 100; + if (site.inCooldown) score += 80; + if (site.downloadSummary?.label === "Congestion Signal" || site.uploadSummary?.label === "Congestion Signal") { + score += 60; + } + if (site.downloadSummary?.label === "Reducing" || site.uploadSummary?.label === "Reducing") { + score += 50; + } + if (site.downloadSummary?.label === "Recovering" || site.uploadSummary?.label === "Recovering") { + score += 40; + } + if (site.downloadSummary?.label === "At Floor" || site.uploadSummary?.label === "At Floor") { + score += 20; + } + if (site.downloadSummary?.label === "At Ceiling" || site.uploadSummary?.label === "At Ceiling") { + score += 10; + } + return score; +} + +function changedRecently(direction) { + const age = num(direction?.last_action_age_secs); + return age !== null && age <= CHANGE_RECENT_WINDOW_SECONDS; +} + +function normalizeStatusRows(rows) { + if (!Array.isArray(rows)) return []; + return rows.map((row) => ({ + site: text(row?.[0]).trim(), + currentDownMbps: num(row?.[1]), + currentUpMbps: num(row?.[2]), + })).filter((row) => row.site.length > 0); +} + +function normalizeDebugEntries(entries) { + if (!Array.isArray(entries)) return []; + return entries + .map((entry) => ({ + site: text(entry?.site).trim(), + download: entry?.download ?? null, + upload: entry?.upload ?? null, + })) + .filter((entry) => entry.site.length > 0); +} + +function buildSites() { + const bySite = new Map(); + + store.statusRows.forEach((row) => { + bySite.set(row.site, { + site: row.site, + currentDownMbps: row.currentDownMbps, + currentUpMbps: row.currentUpMbps, + download: null, + upload: null, + }); + }); + + store.debugEntries.forEach((entry) => { + const existing = bySite.get(entry.site) || { + site: entry.site, + currentDownMbps: null, + currentUpMbps: null, + download: null, + upload: null, + }; + existing.download = entry.download; + existing.upload = entry.upload; + if (existing.currentDownMbps === null) { + existing.currentDownMbps = num(entry.download?.queue_mbps); + } + if (existing.currentUpMbps === null) { + existing.currentUpMbps = num(entry.upload?.queue_mbps); + } + bySite.set(entry.site, existing); + }); + + return Array.from(bySite.values()) + .map((site) => { + const downloadSummary = directionSummary(site.download); + const uploadSummary = directionSummary(site.upload); + const ages = [num(site.download?.last_action_age_secs), num(site.upload?.last_action_age_secs)] + .filter((value) => value !== null); + const lastActionAgeSeconds = ages.length > 0 ? Math.min(...ages) : null; + const actions = [text(site.download?.last_action).trim(), text(site.upload?.last_action).trim()].filter(Boolean); + const cooldowns = [num(site.download?.cooldown_remaining_secs), num(site.upload?.cooldown_remaining_secs)] + .filter((value) => value !== null && value > 0); + const delayValues = [num(site.download?.delay_ms), num(site.upload?.delay_ms)] + .filter((value) => value !== null); + const maxDelayMs = delayValues.length > 0 ? Math.max(...delayValues) : null; + const recentlyChanged = changedRecently(site.download) || changedRecently(site.upload); + const inCooldown = cooldowns.length > 0; + const siteVm = { + ...site, + downloadSummary, + uploadSummary, + lastActionAgeSeconds, + lastActionLabel: actions[0] || "", + changedRecently: recentlyChanged, + inCooldown, + maxDelayMs, + attentionScore: 0, + }; + siteVm.attentionScore = computeAttentionScore(siteVm); + return siteVm; + }) + .sort((a, b) => { + if (b.attentionScore !== a.attentionScore) return b.attentionScore - a.attentionScore; + return a.site.localeCompare(b.site); + }); +} + +function ensureSelectedSite(sites) { + if (sites.length === 0) { + store.selectedSite = null; + return null; + } + const existing = sites.find((site) => site.site === store.selectedSite); + if (existing) return existing.site; + store.selectedSite = sites[0].site; + return store.selectedSite; +} + +function buildSnapshot() { + const sites = buildSites(); + const selectedSite = ensureSelectedSite(sites); + const recentChanges = sites.filter((site) => site.changedRecently).length; + const cooldownSites = sites.filter((site) => site.inCooldown).length; + return { + config: store.config, + configError: store.configError, + sites, + selectedSite, + singleSite: sites.length === 1, + empty: sites.length === 0, + watchedSiteCount: sites.length, + cooldownSiteCount: cooldownSites, + recentChangeCount: recentChanges, + }; +} + +function notify() { + const snapshot = buildSnapshot(); + listeners.forEach((listener) => listener(snapshot)); +} + +export function subscribeStormguardState(listener) { + listeners.add(listener); + listener(buildSnapshot()); + return () => listeners.delete(listener); +} + +export function requestStormguardConfig() { + if (store.configRequested) return; + store.configRequested = true; + loadConfig( + (msg) => { + store.config = msg?.data?.stormguard ?? window.config?.stormguard ?? null; + notify(); + }, + () => { + store.configError = true; + notify(); + }, + ); +} + +export function updateStormguardStatus(rows) { + if (rows === store.lastStatusPayload) return; + store.lastStatusPayload = rows; + store.statusRows = normalizeStatusRows(rows); + notify(); +} + +export function updateStormguardDebug(entries) { + if (entries === store.lastDebugPayload) return; + store.lastDebugPayload = entries; + store.debugEntries = normalizeDebugEntries(entries); + notify(); +} + +export function selectStormguardSite(siteName) { + const site = text(siteName).trim(); + store.selectedSite = site || null; + notify(); +} + +export function stormguardActivityRows(snapshot, limit = 12) { + const rows = []; + (snapshot?.sites ?? []).forEach((site) => { + [ + ["download", site.download, site.downloadSummary], + ["upload", site.upload, site.uploadSummary], + ].forEach(([directionName, direction, summary]) => { + if (!direction) return; + const age = num(direction.last_action_age_secs); + const cooldown = num(direction.cooldown_remaining_secs); + const sortAge = age !== null ? age : (cooldown !== null ? cooldown : 1e9); + rows.push({ + site: site.site, + direction: directionName, + summary, + action: text(direction.last_action).trim() || summary.label, + ageSeconds: age, + cooldownSeconds: cooldown, + reason: directionReason(direction), + sortAge, + }); + }); + }); + + return rows + .sort((a, b) => a.sortAge - b.sortAge || a.site.localeCompare(b.site)) + .slice(0, limit); +} + +export function stormguardSelectedSite(snapshot) { + if (!snapshot?.selectedSite) return null; + return (snapshot.sites ?? []).find((site) => site.site === snapshot.selectedSite) || null; +} + +export function stormguardConfigBadgeData(config) { + if (!config) { + return { + enabledLabel: "Unknown", + enabledClass: "bg-light text-secondary border", + strategy: "—", + dryRunLabel: "—", + }; + } + + return { + enabledLabel: config.enabled ? "Enabled" : "Disabled", + enabledClass: config.enabled + ? "bg-success-subtle text-success border border-success-subtle" + : "bg-light text-secondary border", + strategy: titleCase(config.strategy || "unknown"), + dryRunLabel: config.dry_run ? "Dry Run" : "Live", + }; +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_site_detail.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_site_detail.js new file mode 100644 index 00000000..a3d6f97f --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_site_detail.js @@ -0,0 +1,208 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {StormguardAdjustmentsGraph} from "../graphs/stormguard_adjustments_graph"; +import {mkBadge} from "./bakery_shared"; +import { + directionReason, + formatStormguardAgeSeconds, + formatStormguardMbps, + formatStormguardMs, + formatStormguardPercent, + stormguardSelectedSite, + subscribeStormguardState, + updateStormguardDebug, + updateStormguardStatus, +} from "./stormguard_shared"; + +function metricRow(label, value) { + return ` + + + + + `; +} + +function summaryBadge(summary) { + return mkBadge(summary.label, summary.className, summary.reason || ""); +} + +function renderDirectionTable(direction, fallbackRatesLabel) { + if (!direction) { + return ` +
    + No live debug metrics yet. Current enforced rate: ${fallbackRatesLabel}. +
    + `; + } + + return ` +
    +
    Circuit IDCircuit NameDevice IDDevice NameParent NodeMACIPv4IPv6Download MinUpload MinDownload MaxUpload MaxComment
    SiteDirStatusActionAgeWhy
    No recent StormGuard activity yet.
    ${row.site}${row.direction === "download" ? " Down" : " Up"}${row.action || "Holding"}${row.ageSeconds != null ? formatStormguardAgeSeconds(row.ageSeconds) : "—"}${row.reason}
    ${label}${value}
    + + ${metricRow("Queue", `${formatStormguardMbps(direction.queue_mbps)} Mbps`)} + ${metricRow("Min / Max", `${formatStormguardMbps(direction.min_mbps)} / ${formatStormguardMbps(direction.max_mbps)} Mbps`)} + ${metricRow("State", direction.state || "—")} + ${metricRow("Last Action", direction.last_action || "—")} + ${metricRow("Action Age", formatStormguardAgeSeconds(direction.last_action_age_secs))} + ${metricRow("Cooldown", direction.cooldown_remaining_secs != null ? formatStormguardAgeSeconds(direction.cooldown_remaining_secs) : "—")} + ${metricRow("Throughput", `${formatStormguardMbps(direction.throughput_mbps)} / ${formatStormguardMbps(direction.throughput_ma_mbps)} Mbps`)} + ${metricRow("Retrans", `${formatStormguardPercent(direction.retrans)} / ${formatStormguardPercent(direction.retrans_ma)}`)} + ${metricRow("RTT", `${formatStormguardMs(direction.rtt)} / ${formatStormguardMs(direction.rtt_ma)}`)} + ${metricRow("Baseline / Delay", `${formatStormguardMs(direction.baseline_rtt_ms)} / ${formatStormguardMs(direction.delay_ms)}`)} + ${metricRow("Saturation", `${direction.saturation_current || "—"} / ${direction.saturation_max || "—"}`)} + ${metricRow("Can +/-", `${direction.can_increase ? "Yes" : "No"} / ${direction.can_decrease ? "Yes" : "No"}`)} + +
    +
    + `; +} + +export class StormguardSiteDetailDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 8; + this.unsubscribe = null; + this.currentSiteName = null; + } + + title() { + return "StormGuard Site Detail"; + } + + tooltip() { + return "
    StormGuard Site Detail

    Explains the selected site’s current enforced limits, last actions, cooldown state, and the signals StormGuard is evaluating in each direction.

    "; + } + + subscribeTo() { + return ["StormguardStatus", "StormguardDebug"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + this.header = document.createElement("div"); + this.header.classList.add("d-flex", "justify-content-between", "align-items-start", "flex-wrap", "gap-2", "mb-3"); + + this.titleEl = document.createElement("div"); + this.titleEl.classList.add("fw-semibold"); + + this.badgesEl = document.createElement("div"); + this.badgesEl.classList.add("d-flex", "gap-2", "flex-wrap"); + + this.header.appendChild(this.titleEl); + this.header.appendChild(this.badgesEl); + wrap.appendChild(this.header); + + const ratesRow = document.createElement("div"); + ratesRow.classList.add("row", "g-3", "mb-3"); + + this.downCard = document.createElement("div"); + this.downCard.classList.add("col-md-6"); + this.upCard = document.createElement("div"); + this.upCard.classList.add("col-md-6"); + + ratesRow.appendChild(this.downCard); + ratesRow.appendChild(this.upCard); + wrap.appendChild(ratesRow); + + this.graphEl = document.createElement("div"); + this.graphEl.id = this.graphDivId(); + this.graphEl.classList.add("dashgraph"); + wrap.appendChild(this.graphEl); + + const detailRow = document.createElement("div"); + detailRow.classList.add("row", "g-3", "mt-1"); + this.downDetail = document.createElement("div"); + this.downDetail.classList.add("col-lg-6"); + this.upDetail = document.createElement("div"); + this.upDetail.classList.add("col-lg-6"); + detailRow.appendChild(this.downDetail); + detailRow.appendChild(this.upDetail); + wrap.appendChild(detailRow); + + this.emptyEl = document.createElement("div"); + this.emptyEl.classList.add("small", "text-muted", "mt-3"); + wrap.appendChild(this.emptyEl); + + base.appendChild(wrap); + return base; + } + + setup() { + this.graph = new StormguardAdjustmentsGraph(this.graphDivId()); + this.unsubscribe = subscribeStormguardState((snapshot) => this.renderSnapshot(snapshot)); + } + + onMessage(msg) { + if (msg.event === "StormguardStatus") { + updateStormguardStatus(msg.data || []); + } + if (msg.event === "StormguardDebug") { + updateStormguardDebug(msg.data || []); + } + } + + renderSnapshot(snapshot) { + const site = stormguardSelectedSite(snapshot); + if (!site) { + this.titleEl.textContent = "No StormGuard site selected"; + this.badgesEl.replaceChildren(); + this.downCard.innerHTML = ""; + this.upCard.innerHTML = ""; + this.downDetail.innerHTML = ""; + this.upDetail.innerHTML = ""; + this.graph.update([]); + this.emptyEl.textContent = "When StormGuard begins watching sites, this panel will explain the selected site’s live decisions."; + return; + } + + this.currentSiteName = site.site; + this.titleEl.textContent = site.site; + this.badgesEl.replaceChildren( + summaryBadge(site.downloadSummary), + summaryBadge(site.uploadSummary), + mkBadge(site.inCooldown ? "Cooldown Active" : "No Cooldown", site.inCooldown ? "bg-warning-subtle text-warning border border-warning-subtle" : "bg-light text-secondary border"), + ); + + this.downCard.innerHTML = ` +
    +
    Download
    +
    ${formatStormguardMbps(site.currentDownMbps)} Mbps
    +
    ${directionReason(site.download)}
    +
    + `; + this.upCard.innerHTML = ` +
    +
    Upload
    +
    ${formatStormguardMbps(site.currentUpMbps)} Mbps
    +
    ${directionReason(site.upload)}
    +
    + `; + + this.downDetail.innerHTML = ` +
    +
    +
    Download evaluation
    + ${summaryBadge(site.downloadSummary).outerHTML} +
    + ${renderDirectionTable(site.download, `${formatStormguardMbps(site.currentDownMbps)} Mbps`)} +
    + `; + this.upDetail.innerHTML = ` +
    +
    +
    Upload evaluation
    + ${summaryBadge(site.uploadSummary).outerHTML} +
    + ${renderDirectionTable(site.upload, `${formatStormguardMbps(site.currentUpMbps)} Mbps`)} +
    + `; + + this.graph.update([[site.site, site.currentDownMbps ?? 0, site.currentUpMbps ?? 0]]); + this.emptyEl.textContent = snapshot.singleSite + ? "Single-site deployment: this panel is the primary StormGuard view." + : "Use the site list to switch focus. The graph here follows the selected site only."; + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_site_list.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_site_list.js new file mode 100644 index 00000000..1a1f7f53 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_site_list.js @@ -0,0 +1,128 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {mkBadge} from "./bakery_shared"; +import { + directionSummary, + formatStormguardMbps, + selectStormguardSite, + subscribeStormguardState, + updateStormguardDebug, + updateStormguardStatus, +} from "./stormguard_shared"; + +function directionBadge(summary) { + return mkBadge(summary.label, summary.className, summary.reason || ""); +} + +export class StormguardSiteListDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 4; + this.unsubscribe = null; + } + + title() { + return "StormGuard Sites"; + } + + tooltip() { + return "
    StormGuard Sites

    Ranks watched sites so operators can quickly see where StormGuard is acting, cooling down, or simply holding steady.

    "; + } + + subscribeTo() { + return ["StormguardStatus", "StormguardDebug"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + this.caption = document.createElement("div"); + this.caption.classList.add("text-muted", "small", "mb-2"); + wrap.appendChild(this.caption); + + const tableWrap = document.createElement("div"); + tableWrap.classList.add("table-responsive", "lqos-table-wrap"); + const table = document.createElement("table"); + table.classList.add("lqos-table", "lqos-table-compact", "align-middle", "mb-0"); + table.innerHTML = ` + + + Site + Rates + Status + + + `; + this.tbody = document.createElement("tbody"); + table.appendChild(this.tbody); + tableWrap.appendChild(table); + wrap.appendChild(tableWrap); + + base.appendChild(wrap); + return base; + } + + setup() { + this.unsubscribe = subscribeStormguardState((snapshot) => this.renderSnapshot(snapshot)); + } + + onMessage(msg) { + if (msg.event === "StormguardStatus") { + updateStormguardStatus(msg.data || []); + } + if (msg.event === "StormguardDebug") { + updateStormguardDebug(msg.data || []); + } + } + + renderSnapshot(snapshot) { + if (snapshot.empty) { + this.caption.textContent = "No watched sites are currently available."; + this.tbody.innerHTML = "StormGuard has no live site data to rank yet."; + return; + } + + this.caption.textContent = snapshot.singleSite + ? "Single watched site. The detail panel on the right is the main view." + : "Sorted by recent change activity, cooldowns, and attention score."; + + this.tbody.innerHTML = ""; + snapshot.sites.forEach((site) => { + const tr = document.createElement("tr"); + if (site.site === snapshot.selectedSite) { + tr.classList.add("table-active"); + } + tr.style.cursor = "pointer"; + tr.onclick = () => selectStormguardSite(site.site); + + const nameTd = document.createElement("td"); + const name = document.createElement("div"); + name.classList.add("fw-semibold"); + name.textContent = site.site; + const action = document.createElement("div"); + action.classList.add("small", "text-muted"); + action.textContent = site.lastActionLabel || "No recent action"; + nameTd.appendChild(name); + nameTd.appendChild(action); + + const rateTd = document.createElement("td"); + rateTd.innerHTML = ` +
    ${formatStormguardMbps(site.currentDownMbps)}
    +
    ${formatStormguardMbps(site.currentUpMbps)}
    + `; + + const statusTd = document.createElement("td"); + const badges = document.createElement("div"); + badges.classList.add("d-flex", "flex-column", "gap-1"); + badges.appendChild(directionBadge(site.downloadSummary || directionSummary(site.download))); + badges.appendChild(directionBadge(site.uploadSummary || directionSummary(site.upload))); + statusTd.appendChild(badges); + + tr.appendChild(nameTd); + tr.appendChild(rateTd); + tr.appendChild(statusTd); + this.tbody.appendChild(tr); + }); + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_summary.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_summary.js new file mode 100644 index 00000000..11702494 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/stormguard_summary.js @@ -0,0 +1,169 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {mkBadge} from "./bakery_shared"; +import { + requestStormguardConfig, + stormguardConfigBadgeData, + subscribeStormguardState, + updateStormguardDebug, + updateStormguardStatus, +} from "./stormguard_shared"; + +function summaryCard(label, icon, toneClass) { + const col = document.createElement("div"); + col.classList.add("col-6", "col-lg-3"); + + const card = document.createElement("div"); + card.classList.add("border", "rounded", "p-3", "h-100"); + + const top = document.createElement("div"); + top.classList.add("d-flex", "align-items-center", "justify-content-between", "mb-2"); + + const title = document.createElement("div"); + title.classList.add("text-muted", "small", "text-uppercase"); + title.textContent = label; + + const iconEl = document.createElement("i"); + iconEl.className = `fa ${icon} ${toneClass}`; + + const value = document.createElement("div"); + value.classList.add("fs-4", "fw-semibold"); + value.textContent = "—"; + + const sub = document.createElement("div"); + sub.classList.add("small", "text-muted"); + sub.textContent = ""; + + top.appendChild(title); + top.appendChild(iconEl); + card.appendChild(top); + card.appendChild(value); + card.appendChild(sub); + col.appendChild(card); + + return {col, card, value, sub}; +} + +export class StormguardSummaryDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 12; + this.unsubscribe = null; + } + + title() { + return "StormGuard Summary"; + } + + tooltip() { + return "
    StormGuard Summary

    Shows whether StormGuard is enabled, how many watched sites are active, and where recent adjustments or cooldowns are concentrated.

    "; + } + + subscribeTo() { + return ["StormguardStatus", "StormguardDebug"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + const header = document.createElement("div"); + header.classList.add("d-flex", "justify-content-between", "align-items-center", "flex-wrap", "gap-2", "mb-3"); + + this.statusWrap = document.createElement("div"); + this.statusWrap.classList.add("d-flex", "align-items-center", "gap-2", "flex-wrap"); + + const links = document.createElement("div"); + links.classList.add("d-flex", "gap-2"); + + const configLink = document.createElement("a"); + configLink.href = "config_stormguard.html"; + configLink.className = "btn btn-sm btn-outline-primary"; + configLink.innerHTML = " Config"; + + const debugLink = document.createElement("a"); + debugLink.href = "stormguard_debug.html"; + debugLink.className = "btn btn-sm btn-outline-secondary"; + debugLink.innerHTML = " Debug"; + + links.appendChild(configLink); + links.appendChild(debugLink); + header.appendChild(this.statusWrap); + header.appendChild(links); + wrap.appendChild(header); + + const row = document.createElement("div"); + row.classList.add("row", "g-3"); + + this.watchedCard = summaryCard("Watched Sites", "fa-sitemap", "text-primary"); + this.activeCard = summaryCard("Active Sites", "fa-wave-square", "text-success"); + this.cooldownCard = summaryCard("Cooling Down", "fa-hourglass-half", "text-warning"); + this.changedCard = summaryCard("Changed Recently", "fa-bolt", "text-danger"); + + [ + this.watchedCard, + this.activeCard, + this.cooldownCard, + this.changedCard, + ].forEach((card) => row.appendChild(card.col)); + + this.emptyEl = document.createElement("div"); + this.emptyEl.classList.add("small", "text-muted", "mt-3"); + + wrap.appendChild(row); + wrap.appendChild(this.emptyEl); + base.appendChild(wrap); + return base; + } + + setup() { + requestStormguardConfig(); + this.unsubscribe = subscribeStormguardState((snapshot) => this.renderSnapshot(snapshot)); + } + + onMessage(msg) { + if (msg.event === "StormguardStatus") { + updateStormguardStatus(msg.data || []); + } + if (msg.event === "StormguardDebug") { + updateStormguardDebug(msg.data || []); + } + } + + renderSnapshot(snapshot) { + const configData = stormguardConfigBadgeData(snapshot.config); + this.statusWrap.replaceChildren( + mkBadge(configData.enabledLabel, configData.enabledClass), + mkBadge(configData.dryRunLabel, "bg-light text-secondary border"), + mkBadge(`Strategy: ${configData.strategy}`, "bg-info-subtle text-info border border-info-subtle"), + ); + + this.watchedCard.value.textContent = snapshot.watchedSiteCount.toString(); + this.watchedCard.sub.textContent = "Sites currently emitting StormGuard data"; + + this.activeCard.value.textContent = snapshot.sites.filter((site) => site.download || site.upload).length.toString(); + this.activeCard.sub.textContent = "Sites with live debug metrics"; + + this.cooldownCard.value.textContent = snapshot.cooldownSiteCount.toString(); + this.cooldownCard.sub.textContent = snapshot.cooldownSiteCount > 0 + ? "StormGuard is pacing follow-up actions" + : "No sites are waiting out cooldowns"; + + this.changedCard.value.textContent = snapshot.recentChangeCount.toString(); + this.changedCard.sub.textContent = snapshot.recentChangeCount > 0 + ? "Sites changed in the last 5 minutes" + : "No recent StormGuard adjustments"; + + if (snapshot.empty) { + if (snapshot.config && snapshot.config.enabled) { + this.emptyEl.textContent = "StormGuard is enabled, but no watched sites are currently publishing live data."; + } else { + this.emptyEl.textContent = "StormGuard is disabled or idle. Use the config page to enable it and choose watched sites."; + } + } else { + this.emptyEl.textContent = snapshot.singleSite + ? "Single-site deployment detected. The detail panel expands that site’s current state and recent decisions." + : "Select a site from the list to inspect why StormGuard is holding, cooling down, or changing rates."; + } + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/throughput_pps_dash.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/throughput_pps_dash.js index 57a90ac8..00939d5c 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/throughput_pps_dash.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/throughput_pps_dash.js @@ -2,7 +2,14 @@ import {PacketsPerSecondBar} from "../graphs/packets_bar"; import {PacketsPerSecondTimescale} from "../graphs/packets_bar_insight"; import {DashletBaseInsight} from "./insight_dashlet_base"; +const LIVE_SAMPLE_LIMIT = 300; + export class ThroughputPpsDash extends DashletBaseInsight{ + constructor(slot) { + super(slot); + this.backgroundSamples = []; + } + title() { return "Throughput PPS"; } @@ -19,6 +26,14 @@ export class ThroughputPpsDash extends DashletBaseInsight{ return [ "Throughput" ]; } + keepAliveWhenHidden() { + return true; + } + + keepSubscribedWhenHidden() { + return true; + } + buildContainer() { let base = super.buildContainer(); base.appendChild(this.graphDiv()); @@ -47,15 +62,42 @@ export class ThroughputPpsDash extends DashletBaseInsight{ onMessage(msg) { if (msg.event === "Throughput" && window.timePeriods.activePeriod === "Live") { - this.graph.update(msg.data.pps.down, msg.data.pps.up, msg.data.tcp_pps, msg.data.udp_pps, msg.data.icmp_pps); - if (this.zoomGraph) { - this.zoomGraph.update(msg.data.pps.down, msg.data.pps.up, msg.data.tcp_pps, msg.data.udp_pps, msg.data.icmp_pps); - } + this.#applyLiveSample(msg.data.pps.down, msg.data.pps.up, msg.data.tcp_pps, msg.data.udp_pps, msg.data.icmp_pps); } } + onBackgroundMessage(msg) { + if (msg.event !== "Throughput" || window.timePeriods.activePeriod !== "Live") { + return; + } + this.backgroundSamples.push({ + down: msg.data.pps.down, + up: msg.data.pps.up, + tcp: msg.data.tcp_pps, + udp: msg.data.udp_pps, + icmp: msg.data.icmp_pps, + }); + if (this.backgroundSamples.length > LIVE_SAMPLE_LIMIT) { + this.backgroundSamples.shift(); + } + } + + flushBackgroundMessages() { + if (window.timePeriods.activePeriod !== "Live" || this.backgroundSamples.length === 0) { + this.backgroundSamples = []; + return false; + } + for (let i = 0; i < this.backgroundSamples.length; i++) { + const sample = this.backgroundSamples[i]; + this.#applyLiveSample(sample.down, sample.up, sample.tcp, sample.udp, sample.icmp); + } + this.backgroundSamples = []; + return true; + } + onTimeChange() { super.onTimeChange(); + this.backgroundSamples = []; this.graph.chart.clear(); this.graph.chart.showLoading(); if (window.timePeriods.activePeriod === "Live") { @@ -68,4 +110,11 @@ export class ThroughputPpsDash extends DashletBaseInsight{ this.setupZoomed(); } } + + #applyLiveSample(down, up, tcp, udp, icmp) { + this.graph.update(down, up, tcp, udp, icmp); + if (this.zoomGraph) { + this.zoomGraph.update(down, up, tcp, udp, icmp); + } + } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/throughput_ring_dash.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/throughput_ring_dash.js index f1ad35ac..d9bb6bb7 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/throughput_ring_dash.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/throughput_ring_dash.js @@ -2,10 +2,13 @@ import {ThroughputRingBufferGraph} from "../graphs/throughput_ring_graph"; import {ThroughputRingBufferGraphTimescale} from "../graphs/throughput_ring_graph_timescale"; import {DashletBaseInsight} from "./insight_dashlet_base"; +const LIVE_SAMPLE_LIMIT = 60 * 5; + export class ThroughputRingDash extends DashletBaseInsight{ constructor(slot) { super(slot); this.counter = 0; + this.backgroundSamples = []; } currentPeriod() { @@ -49,6 +52,14 @@ export class ThroughputRingDash extends DashletBaseInsight{ return [ "Throughput" ]; } + keepAliveWhenHidden() { + return true; + } + + keepSubscribedWhenHidden() { + return true; + } + buildContainer() { let base = super.buildContainer(); let graphs = this.graphDiv(); @@ -88,10 +99,7 @@ export class ThroughputRingDash extends DashletBaseInsight{ onMessage(msg) { if (msg.event === "Throughput" && window.timePeriods.activePeriod === "Live") { - this.graph.update(msg.data.shaped_bps, msg.data.bps); - if (this.zoomGraph) { - this.zoomGraph.update(msg.data.shaped_bps, msg.data.bps); - } + this.#applyLiveSample(msg.data.shaped_bps, msg.data.bps); this.counter++; if (this.counter > 120) { @@ -102,6 +110,32 @@ export class ThroughputRingDash extends DashletBaseInsight{ } } + onBackgroundMessage(msg) { + if (msg.event !== "Throughput" || window.timePeriods.activePeriod !== "Live") { + return; + } + this.backgroundSamples.push({ + shaped: msg.data.shaped_bps, + unshaped: msg.data.bps, + }); + if (this.backgroundSamples.length > LIVE_SAMPLE_LIMIT) { + this.backgroundSamples.shift(); + } + } + + flushBackgroundMessages() { + if (window.timePeriods.activePeriod !== "Live" || this.backgroundSamples.length === 0) { + this.backgroundSamples = []; + return false; + } + for (let i = 0; i < this.backgroundSamples.length; i++) { + const sample = this.backgroundSamples[i]; + this.#applyLiveSample(sample.shaped, sample.unshaped); + } + this.backgroundSamples = []; + return true; + } + supportsZoom() { return true; } @@ -155,6 +189,7 @@ export class ThroughputRingDash extends DashletBaseInsight{ onTimeChange() { super.onTimeChange(); + this.backgroundSamples = []; this.updateTitlesAndTooltip(); this.graph.chart.clear(); this.graph.chart.showLoading(); @@ -168,4 +203,11 @@ export class ThroughputRingDash extends DashletBaseInsight{ this.setupZoomed(); } } + + #applyLiveSample(shaped, unshaped) { + this.graph.update(shaped, unshaped); + if (this.zoomGraph) { + this.zoomGraph.update(shaped, unshaped); + } + } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10_downloaders.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10_downloaders.js index b682233b..52a4f875 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10_downloaders.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10_downloaders.js @@ -81,7 +81,7 @@ export class Top10Downloaders extends DashletBaseInsight { let target = document.getElementById(this.id); let table = document.createElement("table"); - table.classList.add("lqos-table", "lqos-table-compact", "small"); + table.classList.add("lqos-table", "lqos-table-compact", "small", "lqos-topn-plain"); let thead = document.createElement("thead"); thead.appendChild(theading("Circuit")); thead.appendChild(theading("Bytes Downloaded")); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10flows_bytes.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10flows_bytes.js index a81b3600..c5c10d39 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10flows_bytes.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10flows_bytes.js @@ -1,14 +1,26 @@ import {clearDashDiv, simpleRow, simpleRowHtml, theading} from "../helpers/builders"; import {scaleNumber, toNumber} from "../lq_js_common/helpers/scaling"; -import {RttCache} from "../helpers/rtt_cache"; -import {formatRetransmit, formatRtt, rttNanosAsSpan} from "../helpers/scaling"; +import {formatRtt} from "../helpers/scaling"; import {TrimToFit} from "../lq_js_common/helpers/text_utils"; import {periodNameToSeconds} from "../helpers/time_periods"; import {DashletBaseInsight} from "./insight_dashlet_base"; import {get_ws_client} from "../pubsub/ws"; const wsClient = get_ws_client(); -const flowCacheKey = (row) => `${row.remote_ip}|${row.analysis}`; +const MAX_VISIBLE_ASN_CHARS = 12; +const MAX_VISIBLE_TOPFLOW_KEY_CHARS = 12; +const compactAsnLabel = (label) => { + if (!label) return ""; + return label.length > MAX_VISIBLE_ASN_CHARS + ? `${label.slice(0, MAX_VISIBLE_ASN_CHARS)}…` + : label; +}; +const compactTopflowKeyLabel = (label) => { + if (!label) return ""; + return label.length > MAX_VISIBLE_TOPFLOW_KEY_CHARS + ? `${label.slice(0, MAX_VISIBLE_TOPFLOW_KEY_CHARS)}…` + : label; +}; const listenOnceForSeconds = (eventName, seconds, handler) => { const wrapped = (msg) => { if (!msg || msg.seconds !== seconds) return; @@ -21,7 +33,6 @@ const listenOnceForSeconds = (eventName, seconds, handler) => { export class Top10FlowsBytes extends DashletBaseInsight { constructor(slot) { super(slot); - this.rttCache = new RttCache(); } canBeSlowedDown() { @@ -33,7 +44,7 @@ export class Top10FlowsBytes extends DashletBaseInsight { } tooltip() { - return "
    Top 10 Flows (by total bytes)

    Top 10 Flows by total bytes, including protocol, local and remote IP addresses, download and upload rates, total bytes, round-trip time, TCP retransmits, remote ASN, and country.

    "; + return "
    Top 10 Flows (by total bytes)

    Top 10 flows by total bytes, including protocol, local and remote IP addresses, download and upload rates, total bytes, and remote ASN.

    "; } subscribeTo() { @@ -56,17 +67,25 @@ export class Top10FlowsBytes extends DashletBaseInsight { let target = document.getElementById(this.id); let t = document.createElement("table"); - t.classList.add("dash-table", "lqos-table", "lqos-table-compact", "small"); + t.classList.add("lqos-table", "lqos-table-compact", "small", "lqos-topflow-table", "lqos-topn-plain"); let th = document.createElement("thead"); th.classList.add("small"); - th.appendChild(theading("IP/Circuit")); - th.appendChild(theading("Protocol")); - th.appendChild(theading("DL ⬇️")); - th.appendChild(theading("UL ⬆️")); - th.appendChild(theading("Total")); - th.appendChild(theading("RTT", 2)); - th.appendChild(theading("TCP Retransmits", 2)); + const keyHeading = theading("IP/Circuit"); + keyHeading.classList.add("lqos-topflow-key-cell"); + th.appendChild(keyHeading); + const protocolHeading = theading("Protocol"); + protocolHeading.classList.add("lqos-topflow-protocol-cell"); + th.appendChild(protocolHeading); + const dlHeading = theading("DL ⬇️"); + dlHeading.classList.add("lqos-topflow-rate-cell"); + th.appendChild(dlHeading); + const ulHeading = theading("UL ⬆️"); + ulHeading.classList.add("lqos-topflow-rate-cell"); + th.appendChild(ulHeading); + const totalHeading = theading("Total"); + totalHeading.classList.add("lqos-topflow-total-cell"); + th.appendChild(totalHeading); const asnHeading = theading("Remote ASN"); asnHeading.classList.add("lqos-asn-cell"); th.appendChild(asnHeading); @@ -80,84 +99,72 @@ export class Top10FlowsBytes extends DashletBaseInsight { if (r.circuit_id !== "") { let circuit = document.createElement("td"); + circuit.classList.add("lqos-topflow-key-cell"); let link = document.createElement("a"); link.href = "circuit.html?id=" + encodeURI(r.circuit_id); - link.innerText = TrimToFit(r.circuit_name); - link.classList.add("redactable"); + link.innerText = compactTopflowKeyLabel(TrimToFit(r.circuit_name, MAX_VISIBLE_TOPFLOW_KEY_CHARS + 1)); + link.title = r.circuit_name || ""; + link.classList.add("redactable", "lqos-table-cell-ellipsis"); circuit.appendChild(link); row.appendChild(circuit); } else { let localIp = document.createElement("td"); - localIp.innerText = r.local_ip; - localIp.classList.add("redactable"); + localIp.classList.add("lqos-topflow-key-cell"); + const localIpText = document.createElement("span"); + localIpText.innerText = compactTopflowKeyLabel(r.local_ip); + localIpText.title = r.local_ip || ""; + localIpText.classList.add("redactable", "lqos-table-cell-ellipsis"); + localIp.appendChild(localIpText); row.appendChild(localIp); } let proto = document.createElement("td"); - proto.innerText = r.analysis; + proto.classList.add("lqos-topflow-protocol-cell"); + const protoText = document.createElement("span"); + protoText.innerText = r.analysis; + protoText.title = r.analysis || ""; + protoText.classList.add("lqos-table-cell-ellipsis"); + proto.appendChild(protoText); row.appendChild(proto); let dl = document.createElement("td"); + dl.classList.add("lqos-topflow-rate-cell"); dl.innerText = scaleNumber(r.rate_estimate_bps.down, 0); row.appendChild(dl); let ul = document.createElement("td"); + ul.classList.add("lqos-topflow-rate-cell"); ul.innerText = scaleNumber(r.rate_estimate_bps.up, 0); row.appendChild(ul); let total = document.createElement("td"); + total.classList.add("lqos-topflow-total-cell"); total.innerText = scaleNumber(r.bytes_sent.down, 0) + " / " + scaleNumber(r.bytes_sent.up, 0); row.appendChild(total); - const cacheKey = flowCacheKey(r); - if (r.rtt_nanos['down'] !== undefined) { - this.rttCache.set(cacheKey, r.rtt_nanos); - } - let rtt = this.rttCache.get(cacheKey); - if (rtt === 0) { - rtt = { down: 0, up: 0 }; - } - - let rttD = document.createElement("td"); - rttD.innerHTML = rttNanosAsSpan(rtt.down); - row.appendChild(rttD); - - let rttU = document.createElement("td"); - rttU.innerHTML = rttNanosAsSpan(rtt.up); - row.appendChild(rttU); - - let tcp1 = document.createElement("td"); - const packetsDown = toNumber(r.packets_sent.down, 0); - const retransmitsDown = toNumber(r.tcp_retransmits.down, 0); - tcp1.innerHTML = formatRetransmit(packetsDown > 0 ? retransmitsDown / packetsDown : 0); - row.appendChild(tcp1); - - let tcp2 = document.createElement("td"); - const packetsUp = toNumber(r.packets_sent.up, 0); - const retransmitsUp = toNumber(r.tcp_retransmits.up, 0); - tcp2.innerHTML = formatRetransmit(packetsUp > 0 ? retransmitsUp / packetsUp : 0); - row.appendChild(tcp2); - let asn = document.createElement("td"); asn.classList.add("lqos-asn-cell"); const asnLabel = (r.remote_asn_name && r.remote_asn_name.length > 0) ? r.remote_asn_name : r.remote_ip; const asnText = document.createElement("span"); asnText.classList.add("lqos-table-cell-ellipsis"); - if (asnLabel && asnLabel.length > 13) { + if (asnLabel && asnLabel.length > MAX_VISIBLE_ASN_CHARS) { asnText.classList.add("tiny"); } - asnText.textContent = asnLabel || ""; + asnText.textContent = compactAsnLabel(asnLabel || ""); asnText.title = asnLabel || ""; asn.appendChild(asnText); row.appendChild(asn); - t.appendChild(row); + tbody.appendChild(row); }); t.appendChild(tbody); // Display it clearDashDiv(this.id, target); - target.appendChild(t); + const tableWrap = document.createElement("div"); + tableWrap.classList.add("lqos-table-wrap"); + tableWrap.appendChild(t); + target.appendChild(tableWrap); } } @@ -174,7 +181,7 @@ export class Top10FlowsBytes extends DashletBaseInsight { let target = document.getElementById(this.id); let table = document.createElement("table"); - table.classList.add("lqos-table", "lqos-table-compact", "small"); + table.classList.add("lqos-table", "lqos-table-compact", "small", "lqos-topn-plain"); let thead = document.createElement("thead"); thead.appendChild(theading("Circuit")); thead.appendChild(theading("Protocol")); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10flows_rate.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10flows_rate.js index 55508c01..1ce07cb9 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10flows_rate.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top10flows_rate.js @@ -1,18 +1,24 @@ import {clearDashDiv, theading} from "../helpers/builders"; -import {formatRetransmit, rttNanosAsSpan} from "../helpers/scaling"; -import {RttCache} from "../helpers/rtt_cache"; -import {scaleNumber, toNumber} from "../lq_js_common/helpers/scaling"; +import {scaleNumber} from "../lq_js_common/helpers/scaling"; import {TrimToFit} from "../lq_js_common/helpers/text_utils"; import {DashletBaseInsight} from "./insight_dashlet_base"; -const flowCacheKey = (row) => `${row.remote_ip}|${row.analysis}`; +const MAX_VISIBLE_ASN_CHARS = 12; +const MAX_VISIBLE_TOPFLOW_KEY_CHARS = 12; +const compactAsnLabel = (label) => { + if (!label) return ""; + return label.length > MAX_VISIBLE_ASN_CHARS + ? `${label.slice(0, MAX_VISIBLE_ASN_CHARS)}…` + : label; +}; +const compactTopflowKeyLabel = (label) => { + if (!label) return ""; + return label.length > MAX_VISIBLE_TOPFLOW_KEY_CHARS + ? `${label.slice(0, MAX_VISIBLE_TOPFLOW_KEY_CHARS)}…` + : label; +}; export class Top10FlowsRate extends DashletBaseInsight { - constructor(slot) { - super(slot); - this.rttCache = new RttCache(); - } - canBeSlowedDown() { return true; } @@ -22,7 +28,7 @@ export class Top10FlowsRate extends DashletBaseInsight { } tooltip() { - return "
    Top 10 Flows (by rate)

    Top 10 Flows by rate, including protocol, local and remote IP addresses, download and upload rates, total bytes, round-trip time, TCP retransmits, remote ASN, and country.

    "; + return "
    Top 10 Flows (by rate)

    Top 10 flows by rate, including protocol, local and remote IP addresses, download and upload rates, total bytes, and remote ASN.

    "; } subscribeTo() { @@ -44,17 +50,25 @@ export class Top10FlowsRate extends DashletBaseInsight { let target = document.getElementById(this.id); let t = document.createElement("table"); - t.classList.add("dash-table", "table-sm", "small"); + t.classList.add("lqos-table", "lqos-table-compact", "small", "lqos-topflow-table", "lqos-topn-plain"); let th = document.createElement("thead"); th.classList.add("small"); - th.appendChild(theading("IP/Circuit")); - th.appendChild(theading("Protocol")); - th.appendChild(theading("DL ⬇️")); - th.appendChild(theading("UL ⬆️")); - th.appendChild(theading("Total")); - th.appendChild(theading("RTT", 2)); - th.appendChild(theading("TCP Retransmits", 2)); + const keyHeading = theading("IP/Circuit"); + keyHeading.classList.add("lqos-topflow-key-cell"); + th.appendChild(keyHeading); + const protocolHeading = theading("Protocol"); + protocolHeading.classList.add("lqos-topflow-protocol-cell"); + th.appendChild(protocolHeading); + const dlHeading = theading("DL ⬇️"); + dlHeading.classList.add("lqos-topflow-rate-cell"); + th.appendChild(dlHeading); + const ulHeading = theading("UL ⬆️"); + ulHeading.classList.add("lqos-topflow-rate-cell"); + th.appendChild(ulHeading); + const totalHeading = theading("Total"); + totalHeading.classList.add("lqos-topflow-total-cell"); + th.appendChild(totalHeading); const asnHeading = theading("Remote ASN"); asnHeading.classList.add("lqos-asn-cell"); th.appendChild(asnHeading); @@ -68,84 +82,72 @@ export class Top10FlowsRate extends DashletBaseInsight { if (r.circuit_id !== "") { let circuit = document.createElement("td"); + circuit.classList.add("lqos-topflow-key-cell"); let link = document.createElement("a"); link.href = "circuit.html?id=" + encodeURI(r.circuit_id); - link.innerText = TrimToFit(r.circuit_name); - link.classList.add("redactable"); + link.innerText = compactTopflowKeyLabel(TrimToFit(r.circuit_name, MAX_VISIBLE_TOPFLOW_KEY_CHARS + 1)); + link.title = r.circuit_name || ""; + link.classList.add("redactable", "lqos-table-cell-ellipsis"); circuit.appendChild(link); row.appendChild(circuit); } else { let localIp = document.createElement("td"); - localIp.innerText = r.local_ip; - localIp.classList.add("redactable"); + localIp.classList.add("lqos-topflow-key-cell"); + const localIpText = document.createElement("span"); + localIpText.innerText = compactTopflowKeyLabel(r.local_ip); + localIpText.title = r.local_ip || ""; + localIpText.classList.add("redactable", "lqos-table-cell-ellipsis"); + localIp.appendChild(localIpText); row.appendChild(localIp); } let proto = document.createElement("td"); - proto.innerText = r.analysis; + proto.classList.add("lqos-topflow-protocol-cell"); + const protoText = document.createElement("span"); + protoText.innerText = r.analysis; + protoText.title = r.analysis || ""; + protoText.classList.add("lqos-table-cell-ellipsis"); + proto.appendChild(protoText); row.appendChild(proto); let dl = document.createElement("td"); + dl.classList.add("lqos-topflow-rate-cell"); dl.innerText = scaleNumber(r.rate_estimate_bps.down, 0); row.appendChild(dl); let ul = document.createElement("td"); + ul.classList.add("lqos-topflow-rate-cell"); ul.innerText = scaleNumber(r.rate_estimate_bps.up, 0); row.appendChild(ul); let total = document.createElement("td"); + total.classList.add("lqos-topflow-total-cell"); total.innerText = scaleNumber(r.bytes_sent.down, 0) + " / " + scaleNumber(r.bytes_sent.up, 0); row.appendChild(total); - const cacheKey = flowCacheKey(r); - if (r.rtt_nanos['down'] !== undefined) { - this.rttCache.set(cacheKey, r.rtt_nanos); - } - let rtt = this.rttCache.get(cacheKey); - if (rtt === 0) { - rtt = { down: 0, up: 0 }; - } - - let rttD = document.createElement("td"); - rttD.innerHTML = rttNanosAsSpan(rtt.down); - row.appendChild(rttD); - - let rttU = document.createElement("td"); - rttU.innerHTML = rttNanosAsSpan(rtt.up); - row.appendChild(rttU); - - let tcp1 = document.createElement("td"); - const packetsDown = toNumber(r.packets_sent.down, 0); - const retransmitsDown = toNumber(r.tcp_retransmits.down, 0); - tcp1.innerHTML = formatRetransmit(packetsDown > 0 ? retransmitsDown / packetsDown : 0); - row.appendChild(tcp1); - - let tcp2 = document.createElement("td"); - const packetsUp = toNumber(r.packets_sent.up, 0); - const retransmitsUp = toNumber(r.tcp_retransmits.up, 0); - tcp2.innerHTML = formatRetransmit(packetsUp > 0 ? retransmitsUp / packetsUp : 0); - row.appendChild(tcp2); - let asn = document.createElement("td"); asn.classList.add("lqos-asn-cell"); const asnLabel = (r.remote_asn_name && r.remote_asn_name.length > 0) ? r.remote_asn_name : r.remote_ip; const asnText = document.createElement("span"); asnText.classList.add("lqos-table-cell-ellipsis"); - if (asnLabel && asnLabel.length > 13) { + if (asnLabel && asnLabel.length > MAX_VISIBLE_ASN_CHARS) { asnText.classList.add("tiny"); } - asnText.textContent = asnLabel || ""; + asnText.textContent = compactAsnLabel(asnLabel || ""); asnText.title = asnLabel || ""; asn.appendChild(asnText); row.appendChild(asn); - t.appendChild(row); + tbody.appendChild(row); }); t.appendChild(tbody); // Display it clearDashDiv(this.id, target); - target.appendChild(t); + const tableWrap = document.createElement("div"); + tableWrap.classList.add("lqos-table-wrap"); + tableWrap.appendChild(t); + target.appendChild(tableWrap); } } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_asn_download.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_asn_download.js index f2f00ab9..3840e303 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_asn_download.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_asn_download.js @@ -61,7 +61,12 @@ export class ShaperTopAsnDownload extends BaseDashlet { tooltip() { return "
    Top Download ASNs

    Aggregated from live top flows by rate (bps)."; } subscribeTo() { return ["AsnTopDownload", "TopFlowsRate"]; } buildContainer() { let b = super.buildContainer(); b.appendChild(this.graphDiv()); return b; } - setup() { this.graph = new TopAsnSankeyGraph(this.graphDivId(), 'down'); if (this.last) this.graph.update(this.last); } + setup() { + this.traceRender("setup-start"); + this.graph = new TopAsnSankeyGraph(this.graphDivId(), 'down'); + this.traceRender("setup-complete", { graphId: this.graphDivId() }); + if (this.last) this.graph.update(this.last); + } _showEmpty(show, msg = "No recent data") { const card = document.getElementById(this.id); if (!card) return; @@ -118,7 +123,14 @@ export class ShaperTopAsnDownload extends BaseDashlet { this._showEmpty(!hasData); if (hasData) { this.last = items; - this.graph.update(items); + this.traceRender("onMessage", { eventName: msg.event, rowCount: items.length }); + try { + this.graph.update(items); + this.traceRender("update-ok", { eventName: msg.event, rowCount: items.length }); + } catch (err) { + this.traceRender("update-error", { eventName: msg.event, error: err && err.message ? err.message : String(err) }); + throw err; + } } return; } @@ -142,7 +154,14 @@ export class ShaperTopAsnDownload extends BaseDashlet { this._showEmpty(!hasData); if (hasData) { this.last = items; - this.graph.update(items); + this.traceRender("onMessage", { eventName: msg.event, rowCount: items.length }); + try { + this.graph.update(items); + this.traceRender("update-ok", { eventName: msg.event, rowCount: items.length }); + } catch (err) { + this.traceRender("update-error", { eventName: msg.event, error: err && err.message ? err.message : String(err) }); + throw err; + } } } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_asn_upload.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_asn_upload.js index 5ba0abfc..67e1b207 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_asn_upload.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_asn_upload.js @@ -51,7 +51,12 @@ export class ShaperTopAsnUpload extends BaseDashlet { tooltip() { return "

    Top Upload ASNs

    Aggregated from live top flows by rate (bps)."; } subscribeTo() { return ["AsnTopUpload", "TopFlowsRate"]; } buildContainer() { let b = super.buildContainer(); b.appendChild(this.graphDiv()); return b; } - setup() { this.graph = new TopAsnSankeyGraphUp(this.graphDivId()); if (this.last) this.graph.update(this.last); } + setup() { + this.traceRender("setup-start"); + this.graph = new TopAsnSankeyGraphUp(this.graphDivId()); + this.traceRender("setup-complete", { graphId: this.graphDivId() }); + if (this.last) this.graph.update(this.last); + } _showEmpty(show, msg = "No recent data") { const card = document.getElementById(this.id); if (!card) return; @@ -105,7 +110,14 @@ export class ShaperTopAsnUpload extends BaseDashlet { this._showEmpty(!hasData); if (hasData) { this.last = items; - this.graph.update(items); + this.traceRender("onMessage", { eventName: msg.event, rowCount: items.length }); + try { + this.graph.update(items); + this.traceRender("update-ok", { eventName: msg.event, rowCount: items.length }); + } catch (err) { + this.traceRender("update-error", { eventName: msg.event, error: err && err.message ? err.message : String(err) }); + throw err; + } } return; } @@ -129,7 +141,14 @@ export class ShaperTopAsnUpload extends BaseDashlet { this._showEmpty(!hasData); if (hasData) { this.last = items; - this.graph.update(items); + this.traceRender("onMessage", { eventName: msg.event, rowCount: items.length }); + try { + this.graph.update(items); + this.traceRender("update-ok", { eventName: msg.event, rowCount: items.length }); + } catch (err) { + this.traceRender("update-error", { eventName: msg.event, error: err && err.message ? err.message : String(err) }); + throw err; + } } } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_tree_summary.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_tree_summary.js index 6d71b0f5..7eb3e231 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_tree_summary.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/top_tree_summary.js @@ -7,6 +7,13 @@ function effectiveMax(node) { return node.effective_max_throughput || node.max_throughput || [0, 0]; } +function retransmitPacketsForNode(node, direction) { + return toNumber( + node.current_tcp_retransmit_packets?.[direction] ?? node.current_tcp_packets?.[direction], + 0, + ); +} + export class TopTreeSummary extends DashletBaseInsight { constructor(slot) { super(slot); @@ -85,8 +92,8 @@ export class TopTreeSummary extends DashletBaseInsight { row.appendChild(simpleRowHtml(formatThroughput(tpDown, maxThroughput[0]))); row.appendChild(simpleRowHtml(formatThroughput(tpUp, maxThroughput[1]))); - const tcpPacketsDown = toNumber(r[1].current_tcp_packets[0], 0); - const tcpPacketsUp = toNumber(r[1].current_tcp_packets[1], 0); + const tcpPacketsDown = retransmitPacketsForNode(r[1], 0); + const tcpPacketsUp = retransmitPacketsForNode(r[1], 1); const retransmitsDown = toNumber(r[1].current_retransmits[0], 0); const retransmitsUp = toNumber(r[1].current_retransmits[1], 0); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/tracked_flow_count_dash.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/tracked_flow_count_dash.js index e1c76bb1..2f27fac2 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/tracked_flow_count_dash.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/tracked_flow_count_dash.js @@ -2,7 +2,14 @@ import {FlowCountGraph} from "../graphs/flows_graph"; import {FlowCountGraphTimescale} from "../graphs/flows_graph_timeseries"; import {DashletBaseInsight} from "./insight_dashlet_base"; +const LIVE_SAMPLE_LIMIT = 300; + export class TrackedFlowsCount extends DashletBaseInsight{ + constructor(slot) { + super(slot); + this.backgroundSamples = []; + } + title() { return "Tracked Flows"; } @@ -19,6 +26,14 @@ export class TrackedFlowsCount extends DashletBaseInsight{ return [ "FlowCount" ]; } + keepAliveWhenHidden() { + return true; + } + + keepSubscribedWhenHidden() { + return true; + } + buildContainer() { let base = super.buildContainer(); base.appendChild(this.graphDiv()); @@ -47,15 +62,39 @@ export class TrackedFlowsCount extends DashletBaseInsight{ onMessage(msg) { if (msg.event === "FlowCount" && window.timePeriods.activePeriod === "Live") { - this.graph.update(msg.active, msg.recent); - if (this.zoomGraph) { - this.zoomGraph.update(msg.active, msg.recent); - } + this.#applyLiveSample(msg.active, msg.recent); } } + onBackgroundMessage(msg) { + if (msg.event !== "FlowCount" || window.timePeriods.activePeriod !== "Live") { + return; + } + this.backgroundSamples.push({ + active: msg.active, + recent: msg.recent, + }); + if (this.backgroundSamples.length > LIVE_SAMPLE_LIMIT) { + this.backgroundSamples.shift(); + } + } + + flushBackgroundMessages() { + if (window.timePeriods.activePeriod !== "Live" || this.backgroundSamples.length === 0) { + this.backgroundSamples = []; + return false; + } + for (let i = 0; i < this.backgroundSamples.length; i++) { + const sample = this.backgroundSamples[i]; + this.#applyLiveSample(sample.active, sample.recent); + } + this.backgroundSamples = []; + return true; + } + onTimeChange() { super.onTimeChange(); + this.backgroundSamples = []; this.graph.chart.clear(); this.graph.chart.showLoading(); if (window.timePeriods.activePeriod === "Live") { @@ -68,4 +107,11 @@ export class TrackedFlowsCount extends DashletBaseInsight{ this.setupZoomed(); } } + + #applyLiveSample(active, recent) { + this.graph.update(active, recent); + if (this.zoomGraph) { + this.zoomGraph.update(active, recent); + } + } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_activity.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_activity.js index 856e9f84..4f7b75d7 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_activity.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_activity.js @@ -1,5 +1,12 @@ import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; +import {toNumber} from "../lq_js_common/helpers/scaling"; import {get_ws_client} from "../pubsub/ws"; +import {mkBadge} from "./bakery_shared"; +import {renderOperationCards} from "./operation_cards"; + +const TREEGUARD_OPERATION_MERGE_WINDOW_SECONDS = 180; +const TREEGUARD_ACTIVITY_VIEW_STORAGE_KEY = "lqos_treeguard_activity_view"; +const TREEGUARD_ACTIVITY_PAGE_SIZE = 5; function formatUnixSecondsToLocalTime(unixSeconds) { const n = typeof unixSeconds === "number" ? unixSeconds : parseInt(unixSeconds, 10); @@ -80,6 +87,85 @@ function formatReason(reasonRaw) { return { label: raw.replace(m[0], `next allowed ${next}`), title: raw }; } +function loadViewMode() { + try { + const saved = window?.localStorage?.getItem(TREEGUARD_ACTIVITY_VIEW_STORAGE_KEY); + return saved === "events" ? "events" : "operations"; + } catch (_) { + return "operations"; + } +} + +function classifyOutcome(entry, action) { + const rawAction = (entry?.action ?? "").toString().trim().toLowerCase(); + const reasonRaw = (entry?.reason ?? "").toString().trim(); + const reasonLower = reasonRaw.toLowerCase(); + + if (rawAction.startsWith("would_")) { + return { + label: "Dry Run", + className: "bg-light text-secondary border", + detail: null, + }; + } + + if (rawAction.endsWith("_requested")) { + return { + label: "Queued", + className: "bg-primary-subtle text-primary border border-primary-subtle", + detail: mkBadge("Bakery", "bg-info-subtle text-info border border-info-subtle"), + }; + } + + if (rawAction === "reload_skipped") { + return { + label: "Skipped", + className: "bg-light text-secondary border", + detail: null, + }; + } + + if (rawAction.endsWith("_failed") || rawAction.includes("failed")) { + return { + label: "Failed", + className: "bg-danger-subtle text-danger border border-danger-subtle", + detail: null, + }; + } + + if (reasonLower.includes("cleanup pending") || reasonLower.includes("awaiting cleanup")) { + return { + label: "Cleanup Pending", + className: "bg-warning-subtle text-warning border border-warning-subtle", + detail: mkBadge("Live", "bg-info-subtle text-info border border-info-subtle"), + }; + } + + if (rawAction === "dry_run_toggled") { + return { + label: "Updated", + className: "bg-primary-subtle text-primary border border-primary-subtle", + detail: null, + }; + } + + const actionLower = (action?.label ?? "").toLowerCase(); + const isLiveIntent = actionLower.includes("virtualize") + || actionLower.includes("sqm live") + || actionLower.includes("reload"); + const detail = isLiveIntent + ? mkBadge("Live", "bg-info-subtle text-info border border-info-subtle") + : (entry?.persisted + ? mkBadge("Stored", "bg-primary-subtle text-primary border border-primary-subtle") + : null); + + return { + label: "Applied", + className: "bg-success-subtle text-success border border-success-subtle", + detail, + }; +} + function renderAction(actionRaw) { const raw = (actionRaw ?? "").toString(); const [verbRaw, payloadRaw] = splitOnce(raw, ":"); @@ -97,10 +183,18 @@ function renderAction(actionRaw) { iconClass = "fa-compress"; iconExtra = []; label = "Virtualize"; + } else if (lowerVerb === "virtualize_requested") { + iconClass = "fa-hourglass-half"; + iconExtra = ["text-primary"]; + label = "Queued virtualization"; } else if (lowerVerb === "unvirtualize") { iconClass = "fa-expand"; iconExtra = []; label = "Unvirtualize"; + } else if (lowerVerb === "unvirtualize_requested") { + iconClass = "fa-hourglass-half"; + iconExtra = ["text-primary"]; + label = "Queued restore"; } else if (lowerVerb === "dry_run_toggled") { iconClass = "fa-toggle-on"; iconExtra = ["text-muted"]; @@ -167,12 +261,238 @@ function renderAction(actionRaw) { return { raw, label, iconClass, iconExtra }; } +function normalizedActionFamily(actionRaw) { + const raw = (actionRaw ?? "").toString().trim().toLowerCase(); + const [verbRaw] = splitOnce(raw, ":"); + let verb = (verbRaw ?? "").trim(); + if (!verb) return "activity"; + + if (verb.startsWith("would_")) { + verb = verb.slice("would_".length); + } + + if ( + verb.startsWith("set_sqm_") + || verb.startsWith("clear_sqm_") + || verb.startsWith("apply_sqm_") + ) { + return "sqm"; + } + + if (verb.startsWith("reload")) { + return "reload"; + } + + return verb + .replace(/_requested$/, "") + .replace(/_failed$/, "") + .replace(/_conflict$/, ""); +} + +function entityLabel(entry) { + const entityType = (entry?.entity_type ?? "").toString().trim().toLowerCase(); + const entityId = (entry?.entity_id ?? "").toString().trim(); + if (!entityId) { + return ""; + } + if (entityType === "circuit") { + return parseCircuitEntityId(entityId).display || entityId; + } + return entityId; +} + +function isSqmBatchEntry(entry) { + return (entry?.batchKind ?? "").toString().trim().toLowerCase() === "sqm" + && !!(entry?.batchId ?? "").toString().trim(); +} + +function describeTreeGuardOperation(entry, action) { + const family = normalizedActionFamily(entry?.action); + const target = entityLabel(entry); + const scopedTarget = target ? ` ${target}` : ""; + const rawAction = (entry?.action ?? "").toString().trim().toLowerCase(); + + if (isSqmBatchEntry(entry)) { + return { + kind: "sqm_batch", + key: `sqm_batch:${entry.batchId}`, + label: "SQM change batch", + scopeLabel: "Circuits", + stages: rawAction.startsWith("would_") + ? ["Observed", "Would Apply"] + : ["Queued", "Applied", "Cleanup", "Done"], + }; + } + + if (rawAction.startsWith("would_")) { + return { + kind: "dry_run", + key: `${family}:${(entry?.entity_type ?? "").toString()}:${(entry?.entity_id ?? "").toString()}`, + label: target ? `${action.label} ${target}` : action.label, + scopeLabel: (entry?.entity_type ?? "TreeGuard").toString(), + stages: ["Observed", "Would Apply"], + }; + } + + switch (family) { + case "virtualize": + return { + kind: "mutation", + key: `${family}:${(entry?.entity_type ?? "").toString()}:${(entry?.entity_id ?? "").toString()}`, + label: `Virtualize${scopedTarget}`, + scopeLabel: (entry?.entity_type ?? "Node").toString(), + stages: ["Queued", "Applied", "Cleanup", "Done"], + }; + case "unvirtualize": + return { + kind: "mutation", + key: `${family}:${(entry?.entity_type ?? "").toString()}:${(entry?.entity_id ?? "").toString()}`, + label: `Restore${scopedTarget}`, + scopeLabel: (entry?.entity_type ?? "Node").toString(), + stages: ["Queued", "Applied", "Cleanup", "Done"], + }; + case "sqm": + return { + kind: "sqm", + key: `${family}:${(entry?.entity_type ?? "").toString()}:${(entry?.entity_id ?? "").toString()}`, + label: `SQM change${scopedTarget}`, + scopeLabel: (entry?.entity_type ?? "Circuit").toString(), + stages: ["Queued", "Applied", "Cleanup", "Done"], + }; + case "reload": + return { + kind: "reload", + key: `${family}:${(entry?.entity_type ?? "").toString()}:${(entry?.entity_id ?? "").toString()}`, + label: target ? `Reload ${target}` : "Reload", + scopeLabel: "Reload", + stages: ["Requested", "Applied", "Done"], + }; + case "dry_run_toggled": + return { + kind: "config", + key: family, + label: "Dry-run toggled", + scopeLabel: "Config", + stages: [], + }; + default: + return { + kind: "activity", + key: `${family}:${(entry?.entity_type ?? "").toString()}:${(entry?.entity_id ?? "").toString()}`, + label: target ? `${action.label} ${target}` : action.label, + scopeLabel: (entry?.entity_type ?? "TreeGuard").toString(), + stages: ["Queued", "Applied", "Cleanup", "Done"], + }; + } +} + +function summarizeSqmBatch(group) { + const uniqueCircuits = new Set( + group.events + .map((entry) => (entry?.entity_id ?? "").toString().trim()) + .filter(Boolean), + ).size; + const failed = group.events.filter((entry) => { + const action = renderAction(entry.action); + return classifyOutcome(entry, action).label === "Failed"; + }).length; + const dryRun = group.events.filter((entry) => { + const action = renderAction(entry.action); + return classifyOutcome(entry, action).label === "Dry Run"; + }).length; + const applied = group.events.filter((entry) => { + const action = renderAction(entry.action); + return classifyOutcome(entry, action).label === "Applied"; + }).length; + const queued = group.events.filter((entry) => { + const action = renderAction(entry.action); + return classifyOutcome(entry, action).label === "Queued"; + }).length; + + const parts = [`${uniqueCircuits} circuit${uniqueCircuits === 1 ? "" : "s"}`]; + if (applied > 0) parts.push(`${applied} applied`); + if (failed > 0) parts.push(`${failed} failed`); + if (dryRun > 0) parts.push(`${dryRun} dry-run`); + if (queued > 0) parts.push(`${queued} queued`); + return parts.join(" • "); +} + +function treeguardProgressPercent(group) { + const outcome = group.outcome.label; + if (group.kind === "dry_run") return 100; + if (outcome === "Queued") return 8; + if (outcome === "Cleanup Pending") return 75; + if (outcome === "Failed") return 50; + if (outcome === "Dry Run") return 100; + if (outcome === "Applied" || outcome === "Updated" || outcome === "Skipped") return 100; + return 55; +} + +function treeguardProgressClass(group) { + switch (group.outcome.label) { + case "Applied": + case "Updated": + case "Skipped": + return "bg-success"; + case "Dry Run": + return "bg-secondary"; + case "Failed": + return "bg-danger"; + case "Cleanup Pending": + return "bg-warning"; + case "Queued": + return "bg-primary"; + default: + return "bg-info"; + } +} + +function buildTreeGuardOperationGroups(entries) { + const groups = []; + const recent = Array.isArray(entries) ? entries.slice(0, 50) : []; + + recent.forEach((entry) => { + const action = renderAction(entry.action); + const descriptor = describeTreeGuardOperation(entry, action); + const outcome = classifyOutcome(entry, action); + const reason = formatReason(entry.reason); + const ts = toNumber(entry.time, 0); + const match = groups.find((group) => + group.key === descriptor.key + && Math.abs((group.oldestTs ?? 0) - ts) <= TREEGUARD_OPERATION_MERGE_WINDOW_SECONDS); + + if (match) { + match.events.push(entry); + match.latestEntry = entry; + match.latestAction = action; + match.outcome = outcome; + match.reason = reason; + match.oldestTs = Math.min(match.oldestTs, ts); + return; + } + + groups.push({ + ...descriptor, + latestEntry: entry, + latestAction: action, + outcome, + reason, + oldestTs: ts, + events: [entry], + }); + }); + + return groups; +} + export class TreeGuardActivityDashlet extends BaseDashlet { constructor(slot) { super(slot); this.size = 12; - this.circuitNameById = new Map(); this.nodeIdByName = new Map(); + this.lastEntries = []; + this.currentPage = 0; + this.viewMode = loadViewMode(); } title() { @@ -180,7 +500,7 @@ export class TreeGuardActivityDashlet extends BaseDashlet { } tooltip() { - return "

    TreeGuard Activity

    Recent TreeGuard actions, including dry-run entries and persisted changes.

    "; + return "
    TreeGuard Activity

    Recent TreeGuard intents with explicit outcomes so operators can distinguish queued requests, dry-runs, successful applies, cleanup-pending actions, skips, and failures.

    "; } subscribeTo() { @@ -189,37 +509,21 @@ export class TreeGuardActivityDashlet extends BaseDashlet { setup() { const wsClient = get_ws_client(); - const shapedWrapped = (msg) => { - wsClient.off("AllShapedDevices", shapedWrapped); - const devices = msg && Array.isArray(msg.data) ? msg.data : []; - devices.forEach((d) => { - const id = (d && d.circuit_id ? String(d.circuit_id) : "").trim(); - const name = (d && d.circuit_name ? String(d.circuit_name) : "").trim(); - if (!id || !name) return; - if (!this.circuitNameById.has(id)) { - this.circuitNameById.set(id, name); - } - }); - }; - wsClient.on("AllShapedDevices", shapedWrapped); - wsClient.send({ AllShapedDevices: {} }); - - const treeWrapped = (msg) => { - wsClient.off("NetworkTree", treeWrapped); + const nodeWrapped = (msg) => { + wsClient.off("NodeDirectory", nodeWrapped); const data = msg && Array.isArray(msg.data) ? msg.data : []; data.forEach((entry) => { - if (!Array.isArray(entry) || entry.length < 2) return; - const id = entry[0]; - const node = entry[1]; - const name = (node && node.name ? String(node.name) : "").trim(); + const id = entry?.tree_index; + const name = (entry?.node_name ?? "").trim(); if (!name) return; if (!this.nodeIdByName.has(name)) { this.nodeIdByName.set(name, id); } }); + this.rerenderWithMetadata(); }; - wsClient.on("NetworkTree", treeWrapped); - wsClient.send({ NetworkTree: {} }); + wsClient.on("NodeDirectory", nodeWrapped); + wsClient.send({ NodeDirectory: {} }); } buildContainer() { @@ -227,6 +531,46 @@ export class TreeGuardActivityDashlet extends BaseDashlet { const wrap = document.createElement("div"); wrap.classList.add("p-2"); + const viewControls = document.createElement("div"); + viewControls.classList.add("d-flex", "justify-content-between", "align-items-center", "gap-2", "flex-wrap", "mb-3"); + + const viewLabel = document.createElement("div"); + viewLabel.classList.add("small", "fw-semibold", "text-uppercase", "text-body-secondary", "mb-0"); + viewLabel.textContent = "View"; + viewControls.appendChild(viewLabel); + + const toggleGroup = document.createElement("div"); + toggleGroup.classList.add("btn-group", "btn-group-sm"); + this.operationsButton = document.createElement("button"); + this.operationsButton.type = "button"; + this.operationsButton.textContent = "Operations"; + this.operationsButton.addEventListener("click", () => this.setViewMode("operations")); + this.eventsButton = document.createElement("button"); + this.eventsButton.type = "button"; + this.eventsButton.textContent = "Event Log"; + this.eventsButton.addEventListener("click", () => this.setViewMode("events")); + toggleGroup.appendChild(this.operationsButton); + toggleGroup.appendChild(this.eventsButton); + viewControls.appendChild(toggleGroup); + wrap.appendChild(viewControls); + + this.operationsSection = document.createElement("div"); + const operationsHeader = document.createElement("div"); + operationsHeader.classList.add("small", "fw-semibold", "text-uppercase", "text-body-secondary", "mb-2"); + operationsHeader.textContent = "Recent Operations"; + this.operationsSection.appendChild(operationsHeader); + + this.operationsList = document.createElement("div"); + this.operationsList.classList.add("d-flex", "flex-column", "gap-2"); + this.operationsSection.appendChild(this.operationsList); + wrap.appendChild(this.operationsSection); + + this.eventsSection = document.createElement("div"); + const eventHeader = document.createElement("div"); + eventHeader.classList.add("small", "fw-semibold", "text-uppercase", "text-body-secondary", "mb-2"); + eventHeader.textContent = "Detailed Events"; + this.eventsSection.appendChild(eventHeader); + const tableWrap = document.createElement("div"); tableWrap.classList.add("lqos-table-wrap"); @@ -236,7 +580,7 @@ export class TreeGuardActivityDashlet extends BaseDashlet { const thead = document.createElement("thead"); thead.classList.add("small"); const headRow = document.createElement("tr"); - ["Local Time", "Entity", "Action", "Persisted", "Reason"].forEach((header) => { + ["Local Time", "Target", "Intent", "Outcome", "Why"].forEach((header) => { const th = document.createElement("th"); th.textContent = header; headRow.appendChild(th); @@ -247,17 +591,136 @@ export class TreeGuardActivityDashlet extends BaseDashlet { table.appendChild(thead); table.appendChild(this.tbody); tableWrap.appendChild(table); - wrap.appendChild(tableWrap); + this.eventsSection.appendChild(tableWrap); + wrap.appendChild(this.eventsSection); + + const footer = document.createElement("div"); + footer.classList.add("d-flex", "justify-content-between", "align-items-center", "gap-2", "pt-2", "flex-wrap"); + + this.pageSummary = document.createElement("div"); + this.pageSummary.classList.add("small", "text-secondary"); + footer.appendChild(this.pageSummary); + + const pager = document.createElement("div"); + pager.classList.add("btn-group", "btn-group-sm"); + + this.prevButton = document.createElement("button"); + this.prevButton.type = "button"; + this.prevButton.classList.add("btn", "btn-outline-secondary"); + this.prevButton.innerHTML = " Newer"; + this.prevButton.setAttribute("aria-label", "Show newer TreeGuard activity"); + this.prevButton.addEventListener("click", () => { + if (this.currentPage > 0) { + this.currentPage -= 1; + this.renderEntries(this.lastEntries); + } + }); + + this.nextButton = document.createElement("button"); + this.nextButton.type = "button"; + this.nextButton.classList.add("btn", "btn-outline-secondary"); + this.nextButton.innerHTML = "Older "; + this.nextButton.setAttribute("aria-label", "Show older TreeGuard activity"); + this.nextButton.addEventListener("click", () => { + if (this.currentPage + 1 < this.totalPages()) { + this.currentPage += 1; + this.renderEntries(this.lastEntries); + } + }); + + pager.appendChild(this.prevButton); + pager.appendChild(this.nextButton); + footer.appendChild(pager); + wrap.appendChild(footer); + base.appendChild(wrap); + this.renderViewMode(); return base; } + setViewMode(mode) { + this.viewMode = mode === "events" ? "events" : "operations"; + this.currentPage = 0; + try { + window?.localStorage?.setItem(TREEGUARD_ACTIVITY_VIEW_STORAGE_KEY, this.viewMode); + } catch (_) {} + this.renderViewMode(); + this.renderEntries(this.lastEntries); + } + + renderViewMode() { + if (this.operationsSection) { + this.operationsSection.classList.toggle("d-none", this.viewMode !== "operations"); + } + if (this.eventsSection) { + this.eventsSection.classList.toggle("d-none", this.viewMode !== "events"); + } + if (this.operationsButton) { + this.operationsButton.className = this.viewMode === "operations" ? "btn btn-primary" : "btn btn-outline-secondary"; + } + if (this.eventsButton) { + this.eventsButton.className = this.viewMode === "events" ? "btn btn-primary" : "btn btn-outline-secondary"; + } + } + + totalPagesForCount(count) { + return Math.max(1, Math.ceil(count / TREEGUARD_ACTIVITY_PAGE_SIZE)); + } + + totalPages() { + const operationCount = buildTreeGuardOperationGroups(this.lastEntries).length; + const activeCount = this.viewMode === "events" ? this.lastEntries.length : operationCount; + return this.totalPagesForCount(activeCount); + } + onMessage(msg) { if (msg.event !== "TreeGuardActivity") { return; } const entries = Array.isArray(msg.data) ? msg.data : []; + this.lastEntries = entries; + this.renderEntries(entries); + } + + rerenderWithMetadata() { + if (this.lastEntries.length > 0) { + this.renderEntries(this.lastEntries); + } + } + + renderEntries(entries) { + const operationGroups = buildTreeGuardOperationGroups(entries); + const totalRows = this.viewMode === "events" ? entries.length : operationGroups.length; + const totalPages = this.totalPagesForCount(totalRows); + this.currentPage = Math.min(this.currentPage, totalPages - 1); + const start = this.currentPage * TREEGUARD_ACTIVITY_PAGE_SIZE; + const end = Math.min(start + TREEGUARD_ACTIVITY_PAGE_SIZE, totalRows); + + const pagedGroups = operationGroups.slice(start, end).map((group) => { + const fullReason = group.kind === "sqm_batch" + ? summarizeSqmBatch(group) + : (group.reason.label || ""); + const eventCount = group.events.length; + const footerLeft = group.kind === "sqm_batch" + ? `Circuits • ${eventCount} event${eventCount === 1 ? "" : "s"}` + : `${group.scopeLabel} • ${eventCount} event${eventCount === 1 ? "" : "s"}`; + return { + label: group.label, + outcomeLabel: group.outcome.label, + outcomeClass: group.outcome.className, + outcomeTitle: group.latestEntry?.reason || "", + summary: fullReason, + summaryTitle: group.reason.title || fullReason, + footerLeft, + footerRight: formatUnixSecondsToLocalTime(group.latestEntry?.time), + stages: group.stages, + progressPercent: group.stages.length > 0 ? treeguardProgressPercent(group) : 0, + progressBarClass: treeguardProgressClass(group), + }; + }); + renderOperationCards(this.operationsList, pagedGroups, { emptyText: "No recent operations" }); + this.tbody.innerHTML = ""; if (entries.length === 0) { @@ -268,10 +731,14 @@ export class TreeGuardActivityDashlet extends BaseDashlet { td.textContent = "No recent activity"; tr.appendChild(td); this.tbody.appendChild(tr); + this.pageSummary.textContent = this.viewMode === "events" ? "0 events" : "0 operations"; + this.prevButton.disabled = true; + this.nextButton.disabled = true; + this.renderViewMode(); return; } - entries.slice(0, 50).forEach((entry) => { + entries.slice(start, end).forEach((entry) => { const tr = document.createElement("tr"); tr.classList.add("small"); @@ -303,13 +770,6 @@ export class TreeGuardActivityDashlet extends BaseDashlet { const circuitId = parsed.circuitId; let display = parsed.display; let title = parsed.hasName ? circuitId : ""; - if (!parsed.hasName && circuitId) { - const name = this.circuitNameById.get(circuitId); - if (name) { - display = name; - title = circuitId; - } - } tdEntity.appendChild( mkLink(`circuit.html?id=${encodeURIComponent(circuitId)}`, display, title), ); @@ -344,15 +804,13 @@ export class TreeGuardActivityDashlet extends BaseDashlet { actionText.textContent = ` ${action.label}`; tdAction.appendChild(actionText); - const tdPersisted = document.createElement("td"); - tdPersisted.classList.add("text-center"); - const persisted = !!entry.persisted; - const persistedIcon = persisted - ? mkIcon("fa-check", ["text-success"]) - : mkIcon("fa-times", ["text-muted"]); - persistedIcon.setAttribute("aria-label", persisted ? "Persisted" : "Not persisted"); - persistedIcon.title = persisted ? "Persisted" : "Not persisted"; - tdPersisted.appendChild(persistedIcon); + const tdOutcome = document.createElement("td"); + const outcome = classifyOutcome(entry, action); + tdOutcome.appendChild(mkBadge(outcome.label, outcome.className, entry.reason || "")); + if (outcome.detail) { + tdOutcome.appendChild(document.createTextNode(" ")); + tdOutcome.appendChild(outcome.detail); + } const tdReason = document.createElement("td"); const reason = formatReason(entry.reason); @@ -362,9 +820,15 @@ export class TreeGuardActivityDashlet extends BaseDashlet { tr.appendChild(tdTime); tr.appendChild(tdEntity); tr.appendChild(tdAction); - tr.appendChild(tdPersisted); + tr.appendChild(tdOutcome); tr.appendChild(tdReason); this.tbody.appendChild(tr); }); + + const label = this.viewMode === "events" ? "events" : "operations"; + this.pageSummary.textContent = `${start + 1}-${end} of ${totalRows} ${label}`; + this.prevButton.disabled = this.currentPage === 0; + this.nextButton.disabled = this.currentPage + 1 >= totalPages; + this.renderViewMode(); } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_control_loop.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_control_loop.js new file mode 100644 index 00000000..0e0c82a1 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_control_loop.js @@ -0,0 +1,132 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; + +function toneClasses(kind) { + switch (kind) { + case "active": + return ["border-primary-subtle", "bg-primary-subtle", "text-primary"]; + case "ok": + return ["border-success-subtle", "bg-success-subtle", "text-success"]; + case "warning": + return ["border-warning-subtle", "bg-warning-subtle", "text-warning"]; + case "danger": + return ["border-danger-subtle", "bg-danger-subtle", "text-danger"]; + default: + return ["border-secondary-subtle", "bg-body-tertiary", "text-body-secondary"]; + } +} + +function renderLoopCard(host, iconClass, title, statusText, tone, detailText) { + host.innerHTML = ""; + host.className = ""; + host.classList.add("border", "rounded", "p-2", "h-100", "d-flex", "flex-column", "gap-1", "small"); + toneClasses(tone).forEach((cls) => host.classList.add(cls)); + + const top = document.createElement("div"); + top.classList.add("d-flex", "align-items-center", "gap-2"); + const icon = document.createElement("i"); + icon.classList.add("fa", "fa-fw", iconClass); + const titleEl = document.createElement("span"); + titleEl.classList.add("fw-semibold"); + titleEl.textContent = title; + top.appendChild(icon); + top.appendChild(titleEl); + + const status = document.createElement("div"); + status.classList.add("fw-semibold"); + status.textContent = statusText; + + const detail = document.createElement("div"); + detail.classList.add("text-body-secondary"); + detail.textContent = detailText; + + host.appendChild(top); + host.appendChild(status); + host.appendChild(detail); +} + +export class TreeGuardControlLoopDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 4; + this.lastStatus = null; + } + + title() { + return "Control Loop"; + } + + tooltip() { + return "
    TreeGuard Control Loop

    Visualizes TreeGuard as a control loop: observe, evaluate, and act.

    "; + } + + subscribeTo() { + return ["TreeGuardStatus"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + const grid = document.createElement("div"); + grid.classList.add("row", "g-2"); + this.cards = []; + for (let i = 0; i < 3; i++) { + const col = document.createElement("div"); + col.classList.add("col-12", "col-md-4"); + const card = document.createElement("div"); + col.appendChild(card); + grid.appendChild(col); + this.cards.push(card); + } + + wrap.appendChild(grid); + base.appendChild(wrap); + return base; + } + + onMessage(msg) { + if (msg.event !== "TreeGuardStatus") return; + this.lastStatus = msg.data || null; + this.renderStatus(); + } + + renderStatus() { + const status = this.lastStatus || {}; + const totalNodes = Number.isFinite(status.total_nodes) ? status.total_nodes : 0; + const totalCircuits = Number.isFinite(status.total_circuits) ? status.total_circuits : 0; + const virtualizedNodes = Number.isFinite(status.virtualized_nodes) ? status.virtualized_nodes : 0; + const fqCodelCircuits = Number.isFinite(status.fq_codel_circuits) ? status.fq_codel_circuits : 0; + const cpuMax = Number.isFinite(Number(status.cpu_max_pct)) ? Math.max(0, Math.trunc(Number(status.cpu_max_pct))) : null; + const enabled = !!status.enabled; + const dryRun = !!status.dry_run; + const paused = !!status.paused_for_bakery_reload; + + renderLoopCard( + this.cards[0], + "fa-eye", + "Observe", + enabled ? "Watching" : "Disabled", + enabled ? "ok" : "idle", + `${totalNodes.toLocaleString()} nodes • ${totalCircuits.toLocaleString()} circuits`, + ); + + renderLoopCard( + this.cards[1], + "fa-brain", + "Evaluate", + paused ? "Paused" : (dryRun ? "Dry Run" : "Live"), + paused ? "warning" : (enabled ? "active" : "idle"), + cpuMax === null ? "CPU trigger idle" : `CPU max ${cpuMax}%`, + ); + + renderLoopCard( + this.cards[2], + paused ? "fa-pause-circle" : "fa-bolt", + "Act", + paused ? "Paused" : (dryRun ? "Dry Run" : "Live"), + paused ? "warning" : (dryRun ? "warning" : (enabled ? "ok" : "idle")), + `${virtualizedNodes.toLocaleString()} virtualized • ${fqCodelCircuits.toLocaleString()} fq_codel`, + ); + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_controls.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_controls.js index c4d672d1..e774e58d 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_controls.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_controls.js @@ -8,6 +8,8 @@ export class TreeguardControlsDashlet extends BaseDashlet { this.size = 6; this.treeguard = defaultTreeguardConfig(); this.isSaving = false; + this.liveStatus = null; + this.configLoaded = false; } title() { @@ -19,7 +21,7 @@ export class TreeguardControlsDashlet extends BaseDashlet { } subscribeTo() { - return []; + return ["TreeGuardStatus"]; } statusLineId() { @@ -76,7 +78,8 @@ export class TreeguardControlsDashlet extends BaseDashlet { setup() { this.bindControls(); - this.refreshConfig(); + this.syncControls(); + this.refreshConfig(false); } bindControls() { @@ -86,20 +89,33 @@ export class TreeguardControlsDashlet extends BaseDashlet { dryRun.addEventListener("change", () => this.persistToggle("dry_run", dryRun.checked)); } - refreshConfig() { - this.setSaveState("Loading config...", false); + refreshConfig(showLoading = true, onLoaded = null, onFailed = null) { + if (showLoading) { + this.setSaveState("Loading config...", false); + } loadConfig( () => { this.treeguard = ensureTreeguardConfig(window.config); + this.configLoaded = true; this.syncControls(); this.renderStatusLine(); - this.setSaveState("", false); + if (showLoading) { + this.setSaveState("", false); + } + if (onLoaded) { + onLoaded(); + } }, () => { this.treeguard = ensureTreeguardConfig(window.config || {}); this.syncControls(); this.renderStatusLine(); - this.setSaveState("Unable to refresh TreeGuard config.", true); + if (showLoading) { + this.setSaveState("Unable to refresh TreeGuard config.", true); + } + if (onFailed) { + onFailed(); + } }, ); } @@ -107,26 +123,92 @@ export class TreeguardControlsDashlet extends BaseDashlet { syncControls() { const enabled = document.getElementById(this.enabledId()); const dryRun = document.getElementById(this.dryRunId()); - enabled.checked = !!this.treeguard.enabled; - dryRun.checked = !!this.treeguard.dry_run; - enabled.disabled = this.isSaving; - dryRun.disabled = this.isSaving; + const controls = document.getElementById(this.controlsId()); + const statusEnabled = this.liveStatus?.enabled; + const statusDryRun = this.liveStatus?.dry_run; + const paused = !!this.liveStatus?.paused_for_bakery_reload; + enabled.checked = this.configLoaded ? !!this.treeguard.enabled : !!(statusEnabled ?? this.treeguard.enabled); + dryRun.checked = this.configLoaded ? !!this.treeguard.dry_run : !!(statusDryRun ?? this.treeguard.dry_run); + enabled.disabled = this.isSaving || paused; + dryRun.disabled = this.isSaving || paused; + if (controls) { + controls.classList.toggle("opacity-50", paused); + } } renderStatusLine() { const statusLine = document.getElementById(this.statusLineId()); - const liveState = this.treeguard.dry_run ? "dry run" : "live"; + const liveStatus = this.liveStatus || {}; + const enabled = (liveStatus.enabled ?? this.treeguard.enabled) ? "Enabled" : "Disabled"; + const dryRun = !!(liveStatus.dry_run ?? this.treeguard.dry_run); + const paused = !!liveStatus.paused_for_bakery_reload; + const pauseReason = (liveStatus.pause_reason || "Bakery full reload in progress").trim(); + const liveState = dryRun ? "dry run" : "live"; const mode = this.treeguard.cpu?.mode === "cpu_aware" ? "CPU-aware" : "Traffic/RTT only"; const linkScope = this.treeguard.links?.all_nodes ? "all links" : "allowlisted links"; const circuitScope = this.treeguard.circuits?.all_circuits ? "all circuits" : "allowlisted circuits"; - const enabled = this.treeguard.enabled ? "Enabled" : "Disabled"; - statusLine.innerHTML = ` - Status: ${enabled}, ${liveState}, ${mode}, ${linkScope}, ${circuitScope}. - Open configuration - `; + const managedNodes = Number.isFinite(Number(liveStatus.managed_nodes)) + ? Math.max(0, Math.trunc(Number(liveStatus.managed_nodes))) + : null; + const managedCircuits = Number.isFinite(Number(liveStatus.managed_circuits)) + ? Math.max(0, Math.trunc(Number(liveStatus.managed_circuits))) + : null; + const warningCount = Array.isArray(liveStatus.warnings) ? liveStatus.warnings.length : 0; + const summaryBits = [ + `Status: ${enabled}`, + liveState, + mode, + linkScope, + circuitScope, + ]; + if (paused) { + summaryBits.push(`paused: ${pauseReason}`); + } + if (managedNodes !== null || managedCircuits !== null) { + const counts = []; + if (managedNodes !== null) counts.push(`${managedNodes} node${managedNodes === 1 ? "" : "s"}`); + if (managedCircuits !== null) counts.push(`${managedCircuits} circuit${managedCircuits === 1 ? "" : "s"}`); + if (counts.length > 0) { + summaryBits.push(`managing ${counts.join(", ")}`); + } + } + if (warningCount > 0) { + summaryBits.push(`${warningCount} warning${warningCount === 1 ? "" : "s"}`); + } + statusLine.innerHTML = `${summaryBits.join(", ")}. Open configuration`; + } + + onMessage(msg) { + if (msg.event !== "TreeGuardStatus") { + return; + } + this.liveStatus = msg.data || null; + if (!this.configLoaded) { + this.syncControls(); + if (!this.isSaving) { + this.setSaveState("", false); + } + } + this.renderStatusLine(); } persistToggle(field, nextValue) { + if (!this.configLoaded) { + this.isSaving = true; + this.syncControls(); + this.refreshConfig( + true, + () => { + this.isSaving = false; + this.persistToggle(field, nextValue); + }, + () => { + this.isSaving = false; + this.syncControls(); + }, + ); + return; + } const previousValue = this.treeguard[field]; this.treeguard[field] = nextValue; window.config = window.config || {}; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_decision_impact.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_decision_impact.js new file mode 100644 index 00000000..4d6e450b --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_decision_impact.js @@ -0,0 +1,250 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; + +function chip(text, className) { + const span = document.createElement("span"); + span.className = `badge ${className}`; + span.textContent = text; + return span; +} + +function normalizeMessage(message) { + return (message ?? "").toString().trim(); +} + +function isErrorMessage(message) { + const normalized = normalizeMessage(message).toLowerCase(); + return normalized.includes("failed") + || normalized.includes("dirty") + || normalized.includes("reload required") + || normalized.includes("requires full reload") + || normalized.includes("unable to load"); +} + +function splitAlerts(messages) { + const errors = []; + const warnings = []; + (Array.isArray(messages) ? messages : []) + .map(normalizeMessage) + .filter(Boolean) + .forEach((message) => { + if (isErrorMessage(message)) { + errors.push(message); + } else { + warnings.push(message); + } + }); + return { errors, warnings }; +} + +function buildAlertTooltip(label, messages) { + if (!messages.length) { + return label; + } + return `${label}\n\n${messages.join("\n")}`; +} + +function compactAlertMessage(message) { + const normalized = normalizeMessage(message); + if (!normalized) { + return ""; + } + + let match = normalized.match(/^TreeGuard links: deferred (\d+) lower-value or over-budget node virtualization changes this tick\.?$/i); + if (match) { + const count = Number.parseInt(match[1], 10) || 0; + return `${count} deferred link change${count === 1 ? "" : "s"}`; + } + + match = normalized.match(/^TreeGuard links: skipped (\d+) low-value automatic node virtualization candidates this tick because the subtree was too small for its current throughput\.?$/i); + if (match) { + const count = Number.parseInt(match[1], 10) || 0; + return `${count} low-value candidate${count === 1 ? "" : "s"} skipped`; + } + + match = normalized.match(/^TreeGuard links: deferred runtime ([^ ]+) for node '([^']+)': .*$/i); + if (match) { + return `Deferred runtime ${match[1]} for ${match[2]}`; + } + + return normalized; +} + +function summarizeAlerts(errors, warnings) { + const lines = []; + if (errors[0]) { + lines.push({ + icon: "fa-circle-exclamation", + tone: "text-danger", + text: compactAlertMessage(errors[0]), + fullText: errors[0], + }); + } + if (warnings[0]) { + lines.push({ + icon: "fa-triangle-exclamation", + tone: "text-warning", + text: compactAlertMessage(warnings[0]), + fullText: warnings[0], + }); + } + if (!errors[0] && warnings[1]) { + lines.push({ + icon: "fa-triangle-exclamation", + tone: "text-warning", + text: compactAlertMessage(warnings[1]), + fullText: warnings[1], + }); + } + return lines; +} + +function statCard(label, value, toneClass = "text-primary") { + const wrap = document.createElement("div"); + wrap.classList.add( + "d-flex", + "align-items-center", + "justify-content-between", + "gap-2", + "border", + "rounded", + "px-2", + "py-1", + "bg-body-tertiary", + "small", + ); + + const left = document.createElement("div"); + left.classList.add("text-body-secondary"); + left.textContent = label; + + const right = document.createElement("div"); + right.classList.add("fw-semibold", toneClass); + right.textContent = value; + + wrap.appendChild(left); + wrap.appendChild(right); + return wrap; +} + +export class TreeGuardDecisionImpactDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 4; + this.status = null; + } + + title() { + return "Decision Impact"; + } + + tooltip() { + return "
    TreeGuard Decision Impact

    Summarizes TreeGuard's current live impact and current warnings or errors without repeating the recent activity feed.

    "; + } + + subscribeTo() { + return ["TreeGuardStatus"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + this.chipsEl = document.createElement("div"); + this.chipsEl.classList.add("d-flex", "flex-wrap", "gap-2", "mb-3"); + + this.statsGrid = document.createElement("div"); + this.statsGrid.classList.add("d-flex", "flex-wrap", "gap-2", "mb-3"); + + this.alertSummaryEl = document.createElement("div"); + this.alertSummaryEl.classList.add("d-flex", "flex-column", "gap-1", "small"); + + wrap.appendChild(this.chipsEl); + wrap.appendChild(this.statsGrid); + wrap.appendChild(this.alertSummaryEl); + base.appendChild(wrap); + return base; + } + + onMessage(msg) { + if (msg.event !== "TreeGuardStatus") { + return; + } + this.status = msg.data || null; + this.renderDecision(); + } + + renderDecision() { + this.chipsEl.innerHTML = ""; + this.alertSummaryEl.innerHTML = ""; + this.chipsEl.appendChild( + chip(this.status?.dry_run ? "Dry Run" : "Live", this.status?.dry_run ? "bg-warning-subtle text-warning border border-warning-subtle" : "bg-success-subtle text-success border border-success-subtle"), + ); + if (this.status?.paused_for_bakery_reload) { + const reason = (this.status?.pause_reason || "").toString().toLowerCase(); + this.chipsEl.appendChild( + chip( + reason.includes("requires full reload") ? "Full Reload Required" : "Paused for Bakery", + reason.includes("requires full reload") + ? "bg-danger-subtle text-danger border border-danger-subtle" + : "bg-warning-subtle text-warning border border-warning-subtle", + ), + ); + } + + this.statsGrid.innerHTML = ""; + const virtualized = Number.isFinite(this.status?.virtualized_nodes) ? this.status.virtualized_nodes : 0; + const fqCodel = Number.isFinite(this.status?.fq_codel_circuits) ? this.status.fq_codel_circuits : 0; + const cpuMax = Number.isFinite(Number(this.status?.cpu_max_pct)) + ? `${Math.max(0, Math.trunc(Number(this.status.cpu_max_pct)))}%` + : "N/A"; + [ + statCard("Virtualized", virtualized.toLocaleString(), virtualized > 0 ? "text-warning" : "text-body"), + statCard("fq_codel", fqCodel.toLocaleString(), fqCodel > 0 ? "text-info" : "text-body"), + statCard("CPU Max", cpuMax, "text-body"), + ].forEach((card) => { + card.style.minWidth = "calc(50% - 0.25rem)"; + this.statsGrid.appendChild(card); + }); + + const { errors, warnings } = splitAlerts(this.status?.warnings); + if (warnings.length > 0) { + const warningChip = chip( + `${warnings.length} warning${warnings.length === 1 ? "" : "s"}`, + "bg-warning-subtle text-warning border border-warning-subtle", + ); + warningChip.title = buildAlertTooltip("TreeGuard warnings", warnings); + this.chipsEl.appendChild(warningChip); + } + if (errors.length > 0) { + const errorChip = chip( + `${errors.length} error${errors.length === 1 ? "" : "s"}`, + "bg-danger-subtle text-danger border border-danger-subtle", + ); + errorChip.title = buildAlertTooltip("TreeGuard errors", errors); + this.chipsEl.appendChild(errorChip); + } + + const alertSummaryLines = summarizeAlerts(errors, warnings); + alertSummaryLines.forEach((item) => { + const row = document.createElement("div"); + row.classList.add("d-flex", "align-items-start", "gap-2", "text-body-secondary"); + row.title = item.fullText || item.text; + + const icon = document.createElement("i"); + icon.className = `fa fa-fw ${item.icon} ${item.tone}`; + row.appendChild(icon); + + const text = document.createElement("div"); + text.textContent = item.text; + row.appendChild(text); + + this.alertSummaryEl.appendChild(row); + }); + if (alertSummaryLines.length === 0) { + this.alertSummaryEl.classList.add("d-none"); + } else { + this.alertSummaryEl.classList.remove("d-none"); + } + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_state_mix.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_state_mix.js new file mode 100644 index 00000000..d75f76d9 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_state_mix.js @@ -0,0 +1,186 @@ +import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; + +function progressMetric(label, badgeClass) { + const wrap = document.createElement("div"); + wrap.classList.add("d-flex", "flex-column", "gap-1"); + + const top = document.createElement("div"); + top.classList.add("d-flex", "justify-content-between", "align-items-center", "gap-2", "small"); + const tag = document.createElement("span"); + tag.className = `badge ${badgeClass}`; + tag.textContent = label; + const count = document.createElement("span"); + count.classList.add("fw-semibold"); + top.appendChild(tag); + top.appendChild(count); + + const barWrap = document.createElement("div"); + barWrap.classList.add("progress"); + barWrap.style.height = "0.7rem"; + const bar = document.createElement("div"); + bar.classList.add("progress-bar"); + barWrap.appendChild(bar); + + wrap.appendChild(top); + wrap.appendChild(barWrap); + return { wrap, count, bar }; +} + +function stackedMetric(label) { + const wrap = document.createElement("div"); + wrap.classList.add("d-flex", "flex-column", "gap-1"); + + const top = document.createElement("div"); + top.classList.add("d-flex", "justify-content-between", "align-items-center", "gap-2", "small"); + const title = document.createElement("span"); + title.classList.add("fw-semibold"); + title.textContent = label; + const count = document.createElement("span"); + count.classList.add("fw-semibold"); + top.appendChild(title); + top.appendChild(count); + + const barWrap = document.createElement("div"); + barWrap.classList.add("progress"); + barWrap.style.height = "0.7rem"; + + const cakeBar = document.createElement("div"); + cakeBar.classList.add("progress-bar", "bg-success"); + const mixedBar = document.createElement("div"); + mixedBar.classList.add("progress-bar", "bg-warning"); + const fqBar = document.createElement("div"); + fqBar.classList.add("progress-bar", "bg-info"); + [cakeBar, mixedBar, fqBar].forEach((bar) => { + barWrap.appendChild(bar); + }); + + const legend = document.createElement("div"); + legend.classList.add("d-flex", "flex-wrap", "gap-2", "small", "text-body-secondary"); + + const mkLegend = (labelText, badgeClass) => { + const span = document.createElement("span"); + span.className = `badge ${badgeClass}`; + span.textContent = labelText; + return span; + }; + legend.appendChild(mkLegend("cake", "bg-success")); + legend.appendChild(mkLegend("mixed", "bg-warning text-dark")); + legend.appendChild(mkLegend("fq_codel", "bg-info text-dark")); + + wrap.appendChild(top); + wrap.appendChild(barWrap); + wrap.appendChild(legend); + return { wrap, count, cakeBar, mixedBar, fqBar }; +} + +function updateMetric(metric, value, max, colorClass, title) { + const hasMax = Number.isFinite(max) && max > 0; + const safeMax = hasMax ? max : Math.max(1, Number.isFinite(value) ? value : 1); + const safeValue = Number.isFinite(value) ? value : 0; + const pct = Math.max(0, Math.min(100, (safeValue / safeMax) * 100)); + metric.count.textContent = hasMax + ? `${safeValue.toLocaleString()} / ${max.toLocaleString()}` + : safeValue.toLocaleString(); + metric.bar.className = `progress-bar ${colorClass}`; + metric.bar.style.width = `${pct}%`; + metric.bar.title = title || ""; + metric.bar.setAttribute("aria-valuenow", safeValue.toString()); + metric.bar.setAttribute("aria-valuemin", "0"); + metric.bar.setAttribute("aria-valuemax", safeMax.toString()); +} + +export class TreeGuardStateMixDashlet extends BaseDashlet { + constructor(slot) { + super(slot); + this.size = 4; + this.status = null; + } + + title() { + return "State Mix"; + } + + tooltip() { + return "
    TreeGuard State Mix

    Shows how much of the current topology TreeGuard is managing and how much is currently virtualized or on fq_codel.

    "; + } + + subscribeTo() { + return ["TreeGuardStatus"]; + } + + buildContainer() { + const base = super.buildContainer(); + const wrap = document.createElement("div"); + wrap.classList.add("p-2"); + + this.grid = document.createElement("div"); + this.grid.classList.add("d-flex", "flex-column", "gap-3"); + + this.nodesManaged = progressMetric("Managed Nodes", "bg-primary"); + this.nodesVirtual = progressMetric("Virtualized", "bg-warning text-dark"); + this.circuitsManaged = progressMetric("Managed Circuits", "bg-success"); + this.circuitSqmMix = stackedMetric("Circuit SQM Mix"); + + [ + this.nodesManaged.wrap, + this.nodesVirtual.wrap, + this.circuitsManaged.wrap, + this.circuitSqmMix.wrap, + ].forEach((el) => this.grid.appendChild(el)); + + this.footerEl = document.createElement("div"); + this.footerEl.classList.add("small", "text-body-secondary", "mt-3"); + + wrap.appendChild(this.grid); + wrap.appendChild(this.footerEl); + base.appendChild(wrap); + return base; + } + + onMessage(msg) { + if (msg.event !== "TreeGuardStatus") return; + this.status = msg.data || null; + this.renderState(); + } + + renderState() { + const totalNodes = Number.isFinite(this.status?.total_nodes) ? this.status.total_nodes : 0; + const totalCircuits = Number.isFinite(this.status?.total_circuits) ? this.status.total_circuits : 0; + const managedNodes = Number.isFinite(this.status?.managed_nodes) ? this.status.managed_nodes : 0; + const managedCircuits = Number.isFinite(this.status?.managed_circuits) ? this.status.managed_circuits : 0; + const virtualizedNodes = Number.isFinite(this.status?.virtualized_nodes) + ? this.status.virtualized_nodes + : 0; + const cakeCircuits = Number.isFinite(this.status?.cake_circuits) + ? this.status.cake_circuits + : 0; + const mixedCircuits = Number.isFinite(this.status?.mixed_sqm_circuits) + ? this.status.mixed_sqm_circuits + : 0; + const fqCodelCircuits = Number.isFinite(this.status?.fq_codel_circuits) + ? this.status.fq_codel_circuits + : 0; + + updateMetric(this.nodesManaged, managedNodes, totalNodes, "bg-primary", "Nodes currently managed by TreeGuard"); + updateMetric(this.nodesVirtual, virtualizedNodes, totalNodes, "bg-warning", "Nodes currently runtime-virtualized"); + updateMetric(this.circuitsManaged, managedCircuits, totalCircuits, "bg-success", "Circuits under TreeGuard management"); + const sqmTotal = Math.max(1, cakeCircuits + mixedCircuits + fqCodelCircuits); + this.circuitSqmMix.count.textContent = `${cakeCircuits.toLocaleString()} / ${mixedCircuits.toLocaleString()} / ${fqCodelCircuits.toLocaleString()}`; + [ + [this.circuitSqmMix.cakeBar, cakeCircuits, "cake/cake circuits"], + [this.circuitSqmMix.mixedBar, mixedCircuits, "mixed cake/fq_codel circuits"], + [this.circuitSqmMix.fqBar, fqCodelCircuits, "fq_codel/fq_codel circuits"], + ].forEach(([bar, value, label]) => { + const pct = Math.max(0, Math.min(100, (value / sqmTotal) * 100)); + bar.style.width = `${pct}%`; + bar.title = `${value.toLocaleString()} ${label}`; + bar.setAttribute("aria-valuenow", value.toString()); + bar.setAttribute("aria-valuemin", "0"); + bar.setAttribute("aria-valuemax", sqmTotal.toString()); + }); + + this.footerEl.textContent = totalNodes > 0 || totalCircuits > 0 + ? "This view shows how much of the discovered topology is currently under TreeGuard influence." + : "Waiting for TreeGuard topology totals."; + } +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_status.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_status.js index 534ec798..933dd988 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_status.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/treeguard_status.js @@ -1,5 +1,4 @@ import {BaseDashlet} from "../lq_js_common/dashboard/base_dashlet"; -import {get_ws_client} from "../pubsub/ws"; function clamp(n, min, max) { return Math.min(Math.max(n, min), max); @@ -123,6 +122,13 @@ function renderLastAction(summaryRaw) { label = `${m[1]} node: ${m[2]}`; } + m = raw.match(/^Queued (virtualization|restore) for node '(.+)'$/); + if (m) { + iconClass = "fa-hourglass-half"; + iconExtra = ["text-primary"]; + label = `Queued ${m[1]}: ${m[2]}`; + } + m = raw.match(/^Would set SQM override for circuit '(.+)' -> (.+)$/); if (m) { iconClass = "fa-eye"; @@ -162,10 +168,6 @@ export class TreeGuardStatusDashlet extends BaseDashlet { constructor(slot) { super(slot); this.size = 6; - this.totalNodes = null; - this.totalCircuits = null; - this.estimatedVirtualizedNodes = null; - this.estimatedFqCodelCircuits = null; this.lastStatusData = null; } @@ -181,63 +183,6 @@ export class TreeGuardStatusDashlet extends BaseDashlet { return ["TreeGuardStatus"]; } - setup() { - const wsClient = get_ws_client(); - - const treeWrapped = (msg) => { - wsClient.off("NetworkTree", treeWrapped); - const data = msg && Array.isArray(msg.data) ? msg.data : []; - let count = 0; - let virtualCount = 0; - data.forEach((entry) => { - if (!Array.isArray(entry) || entry.length < 2) return; - const node = entry[1]; - const name = (node && node.name ? String(node.name) : "").trim(); - if (!name || name === "Root") return; - count += 1; - if (node && (node.virtual === true || node.is_virtual === true)) { - virtualCount += 1; - } - }); - this.totalNodes = count; - this.estimatedVirtualizedNodes = virtualCount; - this.rerenderWithEstimates(); - }; - wsClient.on("NetworkTree", treeWrapped); - wsClient.send({ NetworkTree: {} }); - - const shapedWrapped = (msg) => { - wsClient.off("AllShapedDevices", shapedWrapped); - const devices = msg && Array.isArray(msg.data) ? msg.data : []; - const circuits = new Set(); - const fqCodelCircuits = new Set(); - devices.forEach((d) => { - const id = (d && d.circuit_id ? String(d.circuit_id) : "").trim(); - if (id) circuits.add(id); - - const override = (d && d.sqm_override !== undefined ? String(d.sqm_override) : "").trim(); - if (!override || !id) return; - const { down, up } = parseDirectionalSqmToken(override); - const downLower = (down ?? "").toString().trim().toLowerCase(); - const upLower = (up ?? "").toString().trim().toLowerCase(); - if (downLower === "fq_codel" || upLower === "fq_codel") { - fqCodelCircuits.add(id); - } - }); - this.totalCircuits = circuits.size; - this.estimatedFqCodelCircuits = fqCodelCircuits.size; - this.rerenderWithEstimates(); - }; - wsClient.on("AllShapedDevices", shapedWrapped); - wsClient.send({ AllShapedDevices: {} }); - } - - rerenderWithEstimates() { - if (this.lastStatusData) { - this.renderStatus(this.lastStatusData); - } - } - buildContainer() { const base = super.buildContainer(); const wrap = document.createElement("div"); @@ -339,68 +284,54 @@ export class TreeGuardStatusDashlet extends BaseDashlet { const managedCircuits = Math.max(0, Math.trunc(Number(data.managed_circuits ?? 0) || 0)); const virtualizedNodesExact = Math.max(0, Math.trunc(Number(data.virtualized_nodes ?? 0) || 0)); const fqCodelCircuitsExact = Math.max(0, Math.trunc(Number(data.fq_codel_circuits ?? 0) || 0)); - - const virtualizedEstimate = Math.max(0, Math.trunc(Number(this.estimatedVirtualizedNodes ?? 0) || 0)); - const fqCodelEstimate = Math.max(0, Math.trunc(Number(this.estimatedFqCodelCircuits ?? 0) || 0)); - - const usingVirtualEstimate = virtualizedNodesExact === 0 && virtualizedEstimate > 0; - const usingFqCodelEstimate = fqCodelCircuitsExact === 0 && fqCodelEstimate > 0; - - const virtualizedNodes = usingVirtualEstimate ? virtualizedEstimate : virtualizedNodesExact; - const fqCodelCircuits = usingFqCodelEstimate ? fqCodelEstimate : fqCodelCircuitsExact; - - const totalNodes = Number(this.totalNodes ?? 0); + const totalNodes = Math.max(0, Math.trunc(Number(data.total_nodes ?? 0) || 0)); const nodesMax = totalNodes > 0 ? totalNodes : Math.max(1, managedNodes); const nodesPct = nodesMax ? clamp((managedNodes / nodesMax) * 100, 0, 100) : 0; updateProgressMetric(this.nodesMetric, { value: managedNodes, max: nodesMax, - text: managedNodes.toString(), + text: totalNodes > 0 ? `${managedNodes} / ${totalNodes}` : managedNodes.toString(), bgClass: "bg-info", title: totalNodes > 0 ? `${managedNodes} / ${totalNodes} (${nodesPct.toFixed(0)}%)` : `${managedNodes}`, }); - const totalCircuits = Number(this.totalCircuits ?? 0); + const totalCircuits = Math.max(0, Math.trunc(Number(data.total_circuits ?? 0) || 0)); const circuitsMax = totalCircuits > 0 ? totalCircuits : Math.max(1, managedCircuits); const circuitsPct = circuitsMax ? clamp((managedCircuits / circuitsMax) * 100, 0, 100) : 0; updateProgressMetric(this.circuitsMetric, { value: managedCircuits, max: circuitsMax, - text: managedCircuits.toString(), + text: totalCircuits > 0 ? `${managedCircuits} / ${totalCircuits}` : managedCircuits.toString(), bgClass: "bg-info", title: totalCircuits > 0 ? `${managedCircuits} / ${totalCircuits} (${circuitsPct.toFixed(0)}%)` : `${managedCircuits}`, }); + const virtualizedNodes = virtualizedNodesExact; const virtMax = Math.max(1, managedNodes); const virtPct = managedNodes > 0 ? clamp((virtualizedNodes / virtMax) * 100, 0, 100) : 0; updateProgressMetric(this.virtualizedMetric, { value: virtualizedNodes, max: virtMax, - text: usingVirtualEstimate ? `≈${virtualizedNodes}` : virtualizedNodes.toString(), + text: managedNodes > 0 ? `${virtualizedNodes} / ${managedNodes}` : virtualizedNodes.toString(), bgClass: "bg-primary", - title: (() => { - const core = managedNodes > 0 - ? `${virtualizedNodes} / ${managedNodes} (${virtPct.toFixed(0)}%)` - : `${virtualizedNodes}`; - return usingVirtualEstimate ? `Estimated from NetworkTree snapshot: ${core}` : core; - })(), + title: managedNodes > 0 + ? `${virtualizedNodes} / ${managedNodes} (${virtPct.toFixed(0)}%)` + : `${virtualizedNodes}`, }); + const fqCodelCircuits = fqCodelCircuitsExact; const fqMax = Math.max(1, managedCircuits); const fqPct = managedCircuits > 0 ? clamp((fqCodelCircuits / fqMax) * 100, 0, 100) : 0; updateProgressMetric(this.fqCodelMetric, { value: fqCodelCircuits, max: fqMax, - text: usingFqCodelEstimate ? `≈${fqCodelCircuits}` : fqCodelCircuits.toString(), + text: managedCircuits > 0 ? `${fqCodelCircuits} / ${managedCircuits}` : fqCodelCircuits.toString(), bgClass: "bg-warning", - title: (() => { - const core = managedCircuits > 0 - ? `${fqCodelCircuits} / ${managedCircuits} (${fqPct.toFixed(0)}%)` - : `${fqCodelCircuits}`; - return usingFqCodelEstimate ? `Estimated from ShapedDevices SQM overrides: ${core}` : core; - })(), + title: managedCircuits > 0 + ? `${fqCodelCircuits} / ${managedCircuits} (${fqPct.toFixed(0)}%)` + : `${fqCodelCircuits}`, }); this.lastActionEl.innerHTML = ""; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/world_map_down.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/world_map_down.js index d8796620..fd9aeb54 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/world_map_down.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/world_map_down.js @@ -108,7 +108,12 @@ export class ShaperWorldMapDown extends BaseDashlet { tooltip(){ return "
    World Map

    Endpoint locations sized by traffic and colored by RTT."; } subscribeTo(){ return ["EndpointLatLon"]; } buildContainer(){ let b=super.buildContainer(); b.appendChild(this.graphDiv()); return b; } - setup(){ this.graph = new WorldMap3DGraph(this.graphDivId()); if (this.last) this.graph.update(this.last); } + setup(){ + this.traceRender("setup-start"); + this.graph = new WorldMap3DGraph(this.graphDivId()); + this.traceRender("setup-complete", { graphId: this.graphDivId() }); + if (this.last) this.graph.update(this.last); + } _showEmpty(show, msg = "No recent data"){ const card = document.getElementById(this.id); if (!card) return; @@ -158,11 +163,24 @@ export class ShaperWorldMapDown extends BaseDashlet { this._showEmpty(!hasData); if (hasData) { this.last = out; + this.traceRender("onMessage", { eventName: msg.event, pointCount: out.length }); ensureWorldMap().then(() => { - this.graph.update(out); + try { + this.graph.update(out); + this.traceRender("update-ok", { eventName: msg.event, pointCount: out.length }); + } catch (err) { + this.traceRender("update-error", { eventName: msg.event, error: err && err.message ? err.message : String(err) }); + throw err; + } }).catch(() => { // If the world map fails to load, still attempt to render points - this.graph.update(out); + try { + this.graph.update(out); + this.traceRender("update-ok", { eventName: msg.event, pointCount: out.length, fallback: true }); + } catch (err) { + this.traceRender("update-error", { eventName: msg.event, error: err && err.message ? err.message : String(err), fallback: true }); + throw err; + } }); } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/world_map_up.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/world_map_up.js index 3de83a6f..e49aa713 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/world_map_up.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/world_map_up.js @@ -35,7 +35,12 @@ export class ShaperWorldMapUp extends BaseDashlet { tooltip(){ return "

    World Map

    Endpoint locations sized by traffic and colored by RTT."; } subscribeTo(){ return ["EndpointLatLon"]; } buildContainer(){ let b=super.buildContainer(); b.appendChild(this.graphDiv()); return b; } - setup(){ this.graph = new WorldMap3DGraph(this.graphDivId()); if (this.last) this.graph.update(this.last); } + setup(){ + this.traceRender("setup-start"); + this.graph = new WorldMap3DGraph(this.graphDivId()); + this.traceRender("setup-complete", { graphId: this.graphDivId() }); + if (this.last) this.graph.update(this.last); + } _showEmpty(show, msg = "No recent data"){ const card = document.getElementById(this.id); if (!card) return; @@ -85,10 +90,23 @@ export class ShaperWorldMapUp extends BaseDashlet { this._showEmpty(!hasData); if (hasData) { this.last = out; + this.traceRender("onMessage", { eventName: msg.event, pointCount: out.length }); ensureWorldMap().then(() => { - this.graph.update(out); + try { + this.graph.update(out); + this.traceRender("update-ok", { eventName: msg.event, pointCount: out.length }); + } catch (err) { + this.traceRender("update-error", { eventName: msg.event, error: err && err.message ? err.message : String(err) }); + throw err; + } }).catch(() => { - this.graph.update(out); + try { + this.graph.update(out); + this.traceRender("update-ok", { eventName: msg.event, pointCount: out.length, fallback: true }); + } catch (err) { + this.traceRender("update-error", { eventName: msg.event, error: err && err.message ? err.message : String(err), fallback: true }); + throw err; + } }); } } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_downloaders.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_downloaders.js index 3debf3b8..85c46f07 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_downloaders.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_downloaders.js @@ -81,7 +81,7 @@ export class Worst10Downloaders extends DashletBaseInsight { let target = document.getElementById(this.id); let table = document.createElement("table"); - table.classList.add("lqos-table", "lqos-table-compact", "small"); + table.classList.add("lqos-table", "lqos-table-compact", "small", "lqos-topn-plain"); let thead = document.createElement("thead"); thead.appendChild(theading("Circuit")); thead.appendChild(theading("Bytes Downloaded")); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_retransmits.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_retransmits.js index 1ea35bb8..7d9d8730 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_retransmits.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_retransmits.js @@ -1,7 +1,7 @@ import {clearDashDiv, simpleRow, simpleRowHtml, theading, TopNTableFromMsgData} from "../helpers/builders"; import {TimedCache} from "../lq_js_common/helpers/timed_cache"; import {periodNameToSeconds} from "../helpers/time_periods"; -import {formatRetransmit, formatRtt} from "../helpers/scaling"; +import {formatRetransmit, formatRtt, retransmitFractionFromSample} from "../helpers/scaling"; import {scaleNumber, toNumber} from "../lq_js_common/helpers/scaling"; import {DashletBaseInsight} from "./insight_dashlet_base"; import {get_ws_client} from "../pubsub/ws"; @@ -55,7 +55,11 @@ export class Worst10Retransmits extends DashletBaseInsight { msg.data.forEach((r) => { let key = r.circuit_id; - this.timeCache.addOrUpdate(key, r, r.tcp_retransmits[0]); + this.timeCache.addOrUpdate( + key, + r, + retransmitFractionFromSample(r.tcp_retransmit_sample?.down) + ); }); this.timeCache.tick(); @@ -81,7 +85,7 @@ export class Worst10Retransmits extends DashletBaseInsight { let target = document.getElementById(this.id); let table = document.createElement("table"); - table.classList.add("lqos-table", "lqos-table-compact", "small"); + table.classList.add("lqos-table", "lqos-table-compact", "small", "lqos-topn-plain"); let thead = document.createElement("thead"); thead.appendChild(theading("Circuit")); thead.appendChild(theading("Bytes Downloaded")); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_retransmits_graphic.js b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_retransmits_graphic.js index e4721de7..d72d8af8 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_retransmits_graphic.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/dashlets/worst10_retransmits_graphic.js @@ -1,5 +1,6 @@ import {TopNSankey} from "../graphs/top_n_sankey"; import {TimedCache} from "../lq_js_common/helpers/timed_cache"; +import {retransmitFractionFromSample} from "../helpers/scaling"; import {DashletBaseInsight} from "./insight_dashlet_base"; export class Worst10RetransmitsVisual extends DashletBaseInsight { @@ -39,7 +40,11 @@ export class Worst10RetransmitsVisual extends DashletBaseInsight { if (msg.event === "WorstRetransmits") { msg.data.forEach((r) => { let key = r.circuit_id; - this.timeCache.addOrUpdate(key, r, r.tcp_retransmits[0]); + this.timeCache.addOrUpdate( + key, + r, + retransmitFractionFromSample(r.tcp_retransmit_sample?.down) + ); }); this.timeCache.tick(); @@ -47,4 +52,4 @@ export class Worst10RetransmitsVisual extends DashletBaseInsight { this.graph.processMessage({ data: items }); } } -} \ No newline at end of file +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/ethernet_caps.js b/src/rust/lqosd/src/node_manager/js_build/src/ethernet_caps.js new file mode 100644 index 00000000..45999914 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/ethernet_caps.js @@ -0,0 +1,220 @@ +import {clearDiv, simpleRow, theading} from "./helpers/builders"; +import {formatMbps} from "./helpers/scaling"; +import {get_ws_client} from "./pubsub/ws"; + +const wsClient = get_ws_client(); +const params = new Proxy(new URLSearchParams(window.location.search), { + get: (searchParams, prop) => searchParams.get(prop), +}); + +let page = 0; +let pageSize = 100; +let search = ""; +let tier = ""; +let totalRows = 0; + +function sendWsRequest(responseEvent, request) { + return new Promise((resolve, reject) => { + let done = false; + const responseHandler = (msg) => { + if (done) return; + done = true; + wsClient.off(responseEvent, responseHandler); + wsClient.off("Error", errorHandler); + resolve(msg); + }; + const errorHandler = (msg) => { + if (done) return; + done = true; + wsClient.off(responseEvent, responseHandler); + wsClient.off("Error", errorHandler); + reject(msg); + }; + wsClient.on(responseEvent, responseHandler); + wsClient.on("Error", errorHandler); + wsClient.send(request); + }); +} + +function formatPlanPair(down, up) { + return `${formatMbps(down)} / ${formatMbps(up)} Mbps`; +} + +function tierBadgeClass(tierLabel) { + if (tierLabel === "10M") return "text-bg-danger"; + if (tierLabel === "100M") return "text-bg-warning"; + return "text-bg-info"; +} + +function buildTierBadge(row) { + const badge = document.createElement("span"); + badge.className = `badge rounded-pill ${tierBadgeClass(row?.badge?.tier_label || "")}`; + badge.textContent = row?.badge?.tier_label || "?"; + return badge; +} + +function tableRow(row) { + const tr = document.createElement("tr"); + tr.classList.add("small"); + + const circuitCell = document.createElement("td"); + const link = document.createElement("a"); + link.href = `/circuit.html?id=${encodeURIComponent(row.circuit_id)}`; + link.classList.add("redactable"); + link.textContent = row.circuit_name || row.circuit_id; + circuitCell.appendChild(link); + tr.appendChild(circuitCell); + + tr.appendChild(simpleRow(row.parent_node || "-", true)); + + const ethernetCell = document.createElement("td"); + ethernetCell.appendChild(buildTierBadge(row)); + tr.appendChild(ethernetCell); + + tr.appendChild(simpleRow(formatPlanPair(row.badge.requested_download_mbps, row.badge.requested_upload_mbps))); + tr.appendChild(simpleRow(formatPlanPair(row.badge.applied_download_mbps, row.badge.applied_upload_mbps))); + tr.appendChild(simpleRow(row.limiting_device_name || "-")); + tr.appendChild(simpleRow(row.limiting_interface_name || "-")); + + return tr; +} + +function renderTable(rows) { + const target = document.getElementById("ethernetCapsTableWrap"); + if (!target) return; + clearDiv(target); + + if (!Array.isArray(rows) || rows.length === 0) { + const empty = document.createElement("div"); + empty.className = "text-muted small"; + empty.textContent = "No Ethernet-limited circuits matched this filter."; + target.appendChild(empty); + return; + } + + const wrap = document.createElement("div"); + wrap.classList.add("lqos-table-wrap"); + const table = document.createElement("table"); + table.classList.add("lqos-table", "lqos-table-tight"); + + const thead = document.createElement("thead"); + thead.appendChild(theading("Circuit")); + thead.appendChild(theading("Parent")); + thead.appendChild(theading("Ethernet")); + thead.appendChild(theading("Requested")); + thead.appendChild(theading("Applied")); + thead.appendChild(theading("Limiting Device")); + thead.appendChild(theading("Interface")); + table.appendChild(thead); + + const tbody = document.createElement("tbody"); + rows.forEach((row) => tbody.appendChild(tableRow(row))); + table.appendChild(tbody); + wrap.appendChild(table); + target.appendChild(wrap); +} + +function updateSummary(query, rows) { + const summary = document.getElementById("ethernetCapsSummary"); + const pager = document.getElementById("ethernetCapsPager"); + const prev = document.getElementById("ethernetCapsPrev"); + const next = document.getElementById("ethernetCapsNext"); + if (!summary || !pager || !prev || !next) return; + + const totalPages = Math.max(1, Math.ceil(totalRows / pageSize)); + const currentPage = (query?.page ?? 0) + 1; + const start = totalRows === 0 ? 0 : (currentPage - 1) * pageSize + 1; + const end = totalRows === 0 ? 0 : start + Math.max(0, rows.length - 1); + + summary.textContent = totalRows === 0 + ? "No active Ethernet caps" + : `${totalRows} Ethernet-limited circuits`; + pager.textContent = totalRows === 0 + ? "Page 1 / 1" + : `Showing ${start}–${end} of ${totalRows} • Page ${currentPage} / ${totalPages}`; + prev.disabled = currentPage <= 1; + next.disabled = currentPage >= totalPages; +} + +async function requestPage() { + try { + const query = { + page, + page_size: pageSize, + }; + if (search.trim()) { + query.search = search.trim(); + } + if (tier) { + query.tier = tier === "10M" ? "TenM" : tier === "100M" ? "HundredM" : "GigPlus"; + } + const msg = await sendWsRequest("EthernetCapsPage", { + EthernetCapsPage: { query }, + }); + const data = msg?.data || { rows: [], total_rows: 0, query }; + totalRows = Number.isFinite(Number(data.total_rows)) ? Number(data.total_rows) : 0; + renderTable(data.rows || []); + updateSummary(data.query, data.rows || []); + } catch (_error) { + totalRows = 0; + renderTable([]); + updateSummary({ page: 0 }, []); + } +} + +function bindControls() { + const searchInput = document.getElementById("ethernetCapsSearch"); + const tierSelect = document.getElementById("ethernetCapsTier"); + const pageSizeSelect = document.getElementById("ethernetCapsPageSize"); + const prev = document.getElementById("ethernetCapsPrev"); + const next = document.getElementById("ethernetCapsNext"); + + if (searchInput) { + searchInput.value = search; + searchInput.addEventListener("input", () => { + search = searchInput.value || ""; + page = 0; + requestPage(); + }); + } + if (tierSelect) { + tierSelect.value = tier; + tierSelect.addEventListener("change", () => { + tier = tierSelect.value || ""; + page = 0; + requestPage(); + }); + } + if (pageSizeSelect) { + pageSizeSelect.value = String(pageSize); + pageSizeSelect.addEventListener("change", () => { + const nextSize = parseInt(pageSizeSelect.value, 10); + pageSize = Number.isFinite(nextSize) && nextSize > 0 ? nextSize : 100; + page = 0; + requestPage(); + }); + } + prev?.addEventListener("click", () => { + page = Math.max(0, page - 1); + requestPage(); + }); + next?.addEventListener("click", () => { + const totalPages = Math.max(1, Math.ceil(totalRows / pageSize)); + page = Math.min(totalPages - 1, page + 1); + requestPage(); + }); +} + +function initFromQueryString() { + const rawTier = typeof params.tier === "string" ? params.tier.trim().toUpperCase() : ""; + tier = rawTier === "10M" || rawTier === "100M" || rawTier === "1G" ? rawTier : ""; + search = typeof params.search === "string" ? params.search : ""; +} + +wsClient.on("join", () => { + requestPage(); +}); + +initFromQueryString(); +bindControls(); +requestPage(); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/executive_circuits_due_upgrade.js b/src/rust/lqosd/src/node_manager/js_build/src/executive_circuits_due_upgrade.js index 7d97a4ee..142df326 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/executive_circuits_due_upgrade.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/executive_circuits_due_upgrade.js @@ -1,36 +1,3 @@ -import {listenExecutiveHeatmaps, averageWithCount, renderTable, renderCircuitLink} from "./executive_utils"; +import {renderExecutiveLeaderboardPage} from "./executive_leaderboard_page_template"; -const THRESHOLD = 80; -const MIN_SAMPLES = 10; - -function buildRows(data) { - const rows = (data.circuits || []).map(circuit => { - const down = averageWithCount(circuit.blocks?.download || []); - const up = averageWithCount(circuit.blocks?.upload || []); - return { - name: circuit.circuit_name || circuit.circuit_id || `Circuit ${circuit.circuit_hash}`, - circuit_id: circuit.circuit_id, - down, - up, - }; - }).filter(row => - row.down.count >= MIN_SAMPLES && - row.up.count >= MIN_SAMPLES && - (row.down.avg || 0) >= THRESHOLD && - (row.up.avg || 0) >= THRESHOLD - ); - - rows.sort((a, b) => ((b.down.avg || 0) + (b.up.avg || 0)) - ((a.down.avg || 0) + (a.up.avg || 0))); - return rows; -} - -function render(data) { - const rows = buildRows(data); - renderTable("executiveCircuitsDueTable", [ - { header: "Circuit", render: (r) => renderCircuitLink(r.name, r.circuit_id) }, - { header: "Avg Down Util (%)", render: (r) => r.down.avg !== null ? r.down.avg.toFixed(1) : "—" }, - { header: "Avg Up Util (%)", render: (r) => r.up.avg !== null ? r.up.avg.toFixed(1) : "—" }, - ], rows, "No circuits meet the 80%+ utilization threshold yet."); -} - -listenExecutiveHeatmaps(render); +renderExecutiveLeaderboardPage("executiveCircuitsDueTable", "circuits_due_upgrade"); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/executive_heatmap_page_template.js b/src/rust/lqosd/src/node_manager/js_build/src/executive_heatmap_page_template.js index fa909067..81e4531c 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/executive_heatmap_page_template.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/executive_heatmap_page_template.js @@ -1,30 +1,30 @@ -import {listenExecutiveHeatmaps, renderTable, getSiteIdMap, linkToCircuit, linkToSite} from "./executive_utils"; import { - buildHeatmapRows, + badgeForEntityKind, + linkToExecutiveMetricRow, + pollExecutiveHeatmapPage, +} from "./executive_utils"; +import { heatRow, rttHeatRow, - latestValue, - nonNullCount, + retransmitHeatRow, + utilizationHeatRow, formatLatest, colorByCapacity, } from "./dashlets/executive_heatmap_shared"; import {colorByRttMs, colorByRetransmitPct} from "./helpers/color_scales"; +import {toNumber} from "./lq_js_common/helpers/scaling"; + +const DEFAULT_PAGE_SIZE = 50; +const PAGE_REFRESH_MS = 5000; const metricConfigs = { rtt: { title: "RTT (p50/p90) Heatmap", - metricKey: "rtt", + metric: "Rtt", colorFn: (v) => colorByRttMs(v), formatFn: (v) => formatLatest(v, "ms"), - describeFn: (v) => { - if (v === null || v === undefined || Number.isNaN(v)) return ""; - if (v < 20) return "Good"; - if (v < 60) return "Moderate"; - if (v < 120) return "Elevated"; - return "Severe"; - }, icon: "fa-stopwatch", - description: "Full list of sites, circuits, and ASNs sorted by latest RTT (p50/p90 quadrants).", + description: "Paged list of sites, circuits, and ASNs ranked server-side by latest RTT (p50/p90 quadrants).", legend: [ { label: "Good", sample: 10 }, { label: "Moderate", sample: 40 }, @@ -34,18 +34,11 @@ const metricConfigs = { }, retransmit: { title: "TCP Retransmits Heatmap", - metricKey: "retransmit", + metric: "Retransmit", colorFn: (v) => colorByRetransmitPct(Math.min(10, Math.max(0, v || 0))), formatFn: (v) => formatLatest(v, "%", 1), - describeFn: (v) => { - if (v === null || v === undefined || Number.isNaN(v)) return ""; - if (v < 1) return "Low"; - if (v < 2.5) return "Watch"; - if (v < 5) return "Elevated"; - return "Severe"; - }, icon: "fa-undo-alt", - description: "Full list sorted by latest retransmit percentage.", + description: "Paged list ranked server-side by latest retransmit percentage.", legend: [ { label: "Low", sample: 0.5 }, { label: "Watch", sample: 1.5 }, @@ -55,18 +48,11 @@ const metricConfigs = { }, download: { title: "Download Utilization Heatmap", - metricKey: "download", + metric: "Download", colorFn: colorByCapacity, formatFn: (v) => formatLatest(v, "%"), - describeFn: (v) => { - if (v === null || v === undefined || Number.isNaN(v)) return ""; - if (v < 60) return "Comfortable"; - if (v < 80) return "Busy"; - if (v < 95) return "Near capacity"; - return "Saturated"; - }, icon: "fa-arrow-down", - description: "Full list sorted by latest download utilization.", + description: "Paged list ranked server-side by latest download utilization.", legend: [ { label: "Comfortable", sample: 30 }, { label: "Busy", sample: 70 }, @@ -76,18 +62,11 @@ const metricConfigs = { }, upload: { title: "Upload Utilization Heatmap", - metricKey: "upload", + metric: "Upload", colorFn: colorByCapacity, formatFn: (v) => formatLatest(v, "%"), - describeFn: (v) => { - if (v === null || v === undefined || Number.isNaN(v)) return ""; - if (v < 60) return "Comfortable"; - if (v < 80) return "Busy"; - if (v < 95) return "Near capacity"; - return "Saturated"; - }, icon: "fa-arrow-up", - description: "Full list sorted by latest upload utilization.", + description: "Paged list ranked server-side by latest upload utilization.", legend: [ { label: "Comfortable", sample: 30 }, { label: "Busy", sample: 70 }, @@ -108,89 +87,140 @@ function renderLegend(cfg) {

    Legend: ${items} - Latest value column includes text labels so the page is readable without relying on color alone. + Detail pages now page server-ranked rows instead of streaming the full executive universe every second.
    `; } -function metricSort(metricKey) { - return (a, b) => { - const aVals = a.blocks[metricKey] || []; - const bVals = b.blocks[metricKey] || []; - const aLatest = latestValue(aVals); - const bLatest = latestValue(bVals); - if (bLatest !== aLatest) { - if (aLatest === null) return 1; - if (bLatest === null) return -1; - return bLatest - aLatest; - } - const aCount = nonNullCount(aVals); - const bCount = nonNullCount(bVals); - if (bCount !== aCount) return bCount - aCount; - return (a.label || "").localeCompare(b.label || ""); +function rowSplitBlocks(row) { + return { + download: row?.split_blocks?.download || [], + upload: row?.split_blocks?.upload || [], }; } +function rowRetransmitBlocks(row) { + return { + retransmit: row?.scalar_blocks?.values || [], + retransmit_down: row?.split_blocks?.download || [], + retransmit_up: row?.split_blocks?.upload || [], + }; +} + +function rowRttBlocks(row) { + return { + rtt: row?.rtt_blocks?.rtt || [], + rtt_p50_down: row?.rtt_blocks?.dl_p50 || [], + rtt_p90_down: row?.rtt_blocks?.dl_p90 || [], + rtt_p50_up: row?.rtt_blocks?.ul_p50 || [], + rtt_p90_up: row?.rtt_blocks?.ul_p90 || [], + }; +} + +function renderMetricRow(cfg, metricKey, row) { + const badge = badgeForEntityKind(row?.entity_kind); + const link = linkToExecutiveMetricRow(row); + if (metricKey === "rtt") { + return rttHeatRow( + row.label, + badge, + rowRttBlocks(row), + cfg.colorFn, + cfg.formatFn, + link, + ); + } + if (metricKey === "retransmit") { + return retransmitHeatRow( + row.label, + badge, + rowRetransmitBlocks(row), + cfg.colorFn, + cfg.formatFn, + link, + ); + } + return utilizationHeatRow( + row.label, + badge, + rowSplitBlocks(row), + cfg.colorFn, + cfg.formatFn, + link, + ); +} + +function attachPaginationHandlers(target, state) { + target.querySelectorAll("[data-exec-page]").forEach((button) => { + button.addEventListener("click", () => { + const nextPage = Number(button.dataset.execPage); + if (!Number.isFinite(nextPage) || nextPage < 0 || nextPage === Number(state.query.page || 0)) { + return; + } + state.query.page = nextPage; + state.pollHandle.refresh(); + }); + }); +} + function renderHeatmapTable(targetId, metricKey) { const cfg = metricConfigs[metricKey]; if (!cfg) return; const target = document.getElementById(targetId); if (!target) return; - let lastData = null; - const renderRows = (data) => { - lastData = data; - getSiteIdMap().then((siteIdMap) => { - const activeTarget = document.getElementById(targetId); - if (!activeTarget) return; - const rows = buildHeatmapRows(data).sort(metricSort(metricKey)); - const body = rows.map(row => { - const link = row.badge === "Circuit" - ? linkToCircuit(row.circuit_id) - : row.badge === "Site" - ? linkToSite(row.site_name || row.label, siteIdMap) - : null; - if (metricKey === "rtt") { - return rttHeatRow( - row.label, - row.badge, - row.blocks, - cfg.colorFn, - cfg.formatFn, - link, - cfg.describeFn, - ); - } - return heatRow( - row.label, - row.badge, - row.blocks[metricKey] || [], - cfg.colorFn, - cfg.formatFn, - link, - cfg.describeFn, - ); - }).join(""); - activeTarget.innerHTML = ` -
    -
    -
    -
    ${cfg.title}
    -
    -

    ${cfg.description}

    - ${renderLegend(cfg)} -
    ${body}
    -
    -
    - `; - }); + + const state = { + query: { + metric: cfg.metric, + entity_kinds: ["Site", "Circuit", "Asn"], + page: 0, + page_size: DEFAULT_PAGE_SIZE, + sort: "LatestValue", + descending: true, + }, + lastData: null, + pollHandle: null, }; + + const renderRows = (data) => { + state.lastData = data; + const pageSize = Number(data?.query?.page_size ?? state.query.page_size ?? DEFAULT_PAGE_SIZE); + const page = Number(data?.query?.page ?? state.query.page ?? 0); + const totalRows = Number(data?.total_rows ?? 0); + const totalPages = Math.max(1, Math.ceil(totalRows / Math.max(1, pageSize))); + const hasPrev = page > 0; + const hasNext = page + 1 < totalPages; + const generatedAt = toNumber(data?.generated_at_unix_ms, Date.now()); + const body = (data?.rows || []).map((row) => renderMetricRow(cfg, metricKey, row)).join(""); + + target.innerHTML = ` +
    +
    +
    +
    ${cfg.title}
    +
    + Updated ${new Date(generatedAt).toLocaleTimeString()} + + Page ${page + 1} / ${totalPages} + +
    +
    +

    ${cfg.description}

    + ${renderLegend(cfg)} +
    ${body || `
    No heatmap rows available.
    `}
    +
    +
    + `; + attachPaginationHandlers(target, state); + }; + window.addEventListener("colorBlindModeChanged", () => { - if (lastData) { - renderRows(lastData); + if (state.lastData) { + renderRows(state.lastData); } }); - listenExecutiveHeatmaps(renderRows); - // initial placeholder + + state.pollHandle = pollExecutiveHeatmapPage(state.query, renderRows, PAGE_REFRESH_MS); target.innerHTML = `
    Waiting for heatmap data…
    `; } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/executive_leaderboard_page_template.js b/src/rust/lqosd/src/node_manager/js_build/src/executive_leaderboard_page_template.js new file mode 100644 index 00000000..f169cde7 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/executive_leaderboard_page_template.js @@ -0,0 +1,227 @@ +import { + colorSwatch, + linkToExecutiveLeaderboardRow, + pollExecutiveLeaderboardPage, + renderCircuitLink, +} from "./executive_utils"; +import {colorByCapacity, formatLatest} from "./dashlets/executive_heatmap_shared"; +import {colorByRetransmitPct, colorByRttMs} from "./helpers/color_scales"; +import {toNumber} from "./lq_js_common/helpers/scaling"; + +const DEFAULT_PAGE_SIZE = 50; +const PAGE_REFRESH_MS = 5000; +const SECONDS_PER_BLOCK = 60; +const BITS_PER_MEGABIT = 1_000_000; + +function formatBytes(bytes) { + let value = toNumber(bytes, NaN); + if (!Number.isFinite(value) || value <= 0) return "—"; + const units = ["B", "KB", "MB", "GB", "TB"]; + let idx = 0; + while (value >= 1024 && idx < units.length - 1) { + value /= 1024; + idx += 1; + } + return `${value.toFixed(value >= 10 ? 1 : 2)} ${units[idx]}`; +} + +function renderLinkedLabel(label, link) { + if (!link) return `${label}`; + return `${label}`; +} + +const configs = { + worst_sites: { + kind: "WorstSitesByRtt", + empty: "No site RTT heatmap data yet.", + columns: [ + { + header: "Site", + render: (row) => renderLinkedLabel(row.site_name, linkToExecutiveLeaderboardRow(row)), + }, + { + header: "Median RTT (ms)", + render: (row) => `${colorSwatch(colorByRttMs(row.median_rtt_ms))}${row.median_rtt_ms.toFixed(1)}`, + }, + { + header: "Avg Down Util (%)", + render: (row) => row.avg_down_util === null || row.avg_down_util === undefined + ? "—" + : `${colorSwatch(colorByCapacity(row.avg_down_util))}${row.avg_down_util.toFixed(1)}`, + }, + { + header: "Avg Up Util (%)", + render: (row) => row.avg_up_util === null || row.avg_up_util === undefined + ? "—" + : `${colorSwatch(colorByCapacity(row.avg_up_util))}${row.avg_up_util.toFixed(1)}`, + }, + ], + }, + oversubscribed_sites: { + kind: "OversubscribedSites", + empty: "No oversubscription data available yet.", + columns: [ + { + header: "Site", + render: (row) => renderLinkedLabel(row.site_name, linkToExecutiveLeaderboardRow(row)), + }, + { header: "Oversub Down", render: (row) => row.ratio_down === null || row.ratio_down === undefined ? "—" : `${row.ratio_down.toFixed(2)}x` }, + { header: "Oversub Up", render: (row) => row.ratio_up === null || row.ratio_up === undefined ? "—" : `${row.ratio_up.toFixed(2)}x` }, + { + header: "Cap (D/U)", + render: (row) => `${formatLatest(row.cap_down, "Mbps", row.cap_down >= 10 ? 1 : 2)} / ${formatLatest(row.cap_up, "Mbps", row.cap_up >= 10 ? 1 : 2)}`, + }, + { + header: "Subscribed (D/U)", + render: (row) => `${formatLatest(row.sub_down, "Mbps", row.sub_down >= 10 ? 1 : 2)} / ${formatLatest(row.sub_up, "Mbps", row.sub_up >= 10 ? 1 : 2)}`, + }, + { + header: "Median RTT (ms)", + render: (row) => row.median_rtt_ms === null || row.median_rtt_ms === undefined + ? "—" + : `${colorSwatch(colorByRttMs(row.median_rtt_ms))}${formatLatest(row.median_rtt_ms, "ms", 1)}`, + }, + { + header: "Avg Down Util (%)", + render: (row) => row.avg_down_util === null || row.avg_down_util === undefined + ? "—" + : `${colorSwatch(colorByCapacity(row.avg_down_util))}${row.avg_down_util.toFixed(1)}`, + }, + { + header: "Avg Up Util (%)", + render: (row) => row.avg_up_util === null || row.avg_up_util === undefined + ? "—" + : `${colorSwatch(colorByCapacity(row.avg_up_util))}${row.avg_up_util.toFixed(1)}`, + }, + ], + }, + sites_due_upgrade: { + kind: "SitesDueUpgrade", + empty: "No sites meet the 80%+ utilization threshold yet.", + columns: [ + { + header: "Site", + render: (row) => renderLinkedLabel(row.site_name, linkToExecutiveLeaderboardRow(row)), + }, + { + header: "Avg Down Util (%)", + render: (row) => row.avg_down_util.toFixed(1), + }, + { + header: "Avg Up Util (%)", + render: (row) => row.avg_up_util.toFixed(1), + }, + ], + }, + circuits_due_upgrade: { + kind: "CircuitsDueUpgrade", + empty: "No circuits meet the 80%+ utilization threshold yet.", + columns: [ + { + header: "Circuit", + render: (row) => renderCircuitLink(row.circuit_name, row.circuit_id), + }, + { + header: "Avg Down Util (%)", + render: (row) => row.avg_down_util.toFixed(1), + }, + { + header: "Avg Up Util (%)", + render: (row) => row.avg_up_util.toFixed(1), + }, + ], + }, + top_asns: { + kind: "TopAsnsByTraffic", + empty: "No ASN heatmap data yet.", + columns: [ + { + header: "ASN", + render: (row) => row.asn_name ? `${row.asn_name} (ASN ${row.asn})` : `ASN ${row.asn}`, + }, + { + header: "Total Traffic (15m)", + render: (row) => formatBytes(row.total_bytes_15m || 0), + }, + { + header: "Median RTT (ms)", + render: (row) => row.median_rtt_ms === null || row.median_rtt_ms === undefined + ? "—" + : `${colorSwatch(colorByRttMs(row.median_rtt_ms))}${row.median_rtt_ms.toFixed(1)}`, + }, + { + header: "Median Retrans (%)", + render: (row) => row.median_retransmit_pct === null || row.median_retransmit_pct === undefined + ? "—" + : `${colorSwatch(colorByRetransmitPct(Math.min(10, Math.max(0, row.median_retransmit_pct))))}${row.median_retransmit_pct.toFixed(2)}`, + }, + ], + }, +}; + +function attachPaginationHandlers(target, state) { + target.querySelectorAll("[data-exec-page]").forEach((button) => { + button.addEventListener("click", () => { + const nextPage = Number(button.dataset.execPage); + if (!Number.isFinite(nextPage) || nextPage < 0 || nextPage === Number(state.query.page || 0)) { + return; + } + state.query.page = nextPage; + state.pollHandle.refresh(); + }); + }); +} + +function tableMarkup(columns, rows, emptyMessage) { + if (!rows.length) { + return `
    ${emptyMessage}
    `; + } + const thead = `${columns.map((column) => `${column.header}`).join("")}`; + const tbody = rows.map((row) => `${columns.map((column) => `${column.render(row)}`).join("")}`).join(""); + return ` +
    + + ${thead} + ${tbody} +
    +
    + `; +} + +export function renderExecutiveLeaderboardPage(targetId, configKey) { + const cfg = configs[configKey]; + const target = document.getElementById(targetId); + if (!cfg || !target) return; + + const state = { + query: { + kind: cfg.kind, + page: 0, + page_size: DEFAULT_PAGE_SIZE, + }, + pollHandle: null, + }; + + const renderRows = (data) => { + const pageSize = Number(data?.query?.page_size ?? state.query.page_size ?? DEFAULT_PAGE_SIZE); + const page = Number(data?.query?.page ?? state.query.page ?? 0); + const totalRows = Number(data?.total_rows ?? 0); + const totalPages = Math.max(1, Math.ceil(totalRows / Math.max(1, pageSize))); + const hasPrev = page > 0; + const hasNext = page + 1 < totalPages; + const generatedAt = toNumber(data?.generated_at_unix_ms, Date.now()); + target.innerHTML = ` +
    + Updated ${new Date(generatedAt).toLocaleTimeString()} + + Page ${page + 1} / ${totalPages} + +
    + ${tableMarkup(cfg.columns, data?.rows || [], cfg.empty)} + `; + attachPaginationHandlers(target, state); + }; + + state.pollHandle = pollExecutiveLeaderboardPage(state.query, renderRows, PAGE_REFRESH_MS); + target.innerHTML = `
    Waiting for executive data…
    `; +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/executive_oversubscribed_sites.js b/src/rust/lqosd/src/node_manager/js_build/src/executive_oversubscribed_sites.js index 0e64c199..271ca7b2 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/executive_oversubscribed_sites.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/executive_oversubscribed_sites.js @@ -1,94 +1,3 @@ -import { - listenExecutiveHeatmaps, - averageWithCount, - renderTable, - medianFromBlocks, - getSiteIdMap, - renderSiteLink -} from "./executive_utils"; -import {colorSwatch} from "./executive_utils"; -import {colorByCapacity, formatLatest} from "./dashlets/executive_heatmap_shared"; -import {colorByRttMs} from "./helpers/color_scales"; +import {renderExecutiveLeaderboardPage} from "./executive_leaderboard_page_template"; -const MIN_SAMPLES = 6; -const MAX_ROWS = 20; - -function buildRows(data) { - const oversub = data.oversubscribed_sites || []; - if (!oversub.length) return []; - - // Build a lookup for heatmap stats by site - const siteStats = new Map(); - (data.sites || []).forEach(site => { - const down = averageWithCount(site.blocks?.download || []); - const up = averageWithCount(site.blocks?.upload || []); - const rtt = medianFromBlocks(site.blocks?.rtt || []); - siteStats.set(site.site_name || site.name || "", { - down, - up, - rtt, - }); - }); - - const rows = oversub.map(item => { - const name = item.site_name || "Site"; - const stats = siteStats.get(name) || {}; - const ratio = item.ratio_max ?? Math.max(item.ratio_down || 0, item.ratio_up || 0); - return { - name, - ratio_down: item.ratio_down, - ratio_up: item.ratio_up, - ratio, - cap_down: item.cap_down, - cap_up: item.cap_up, - sub_down: item.sub_down, - sub_up: item.sub_up, - perf_down: stats.down || { avg: null, count: 0 }, - perf_up: stats.up || { avg: null, count: 0 }, - rtt: stats.rtt ?? null, - }; - }); - - rows.sort((a, b) => (b.ratio || 0) - (a.ratio || 0)); - return rows.slice(0, MAX_ROWS); -} - -function fmtRatio(r) { - if (r === undefined || r === null || Number.isNaN(r)) return "—"; - return `${r.toFixed(2)}x`; -} - -function fmtMbps(val) { - if (val === undefined || val === null || Number.isNaN(val)) return "—"; - return `${val.toFixed(val >= 10 ? 1 : 2)} Mbps`; -} - -function render(data) { - const rows = buildRows(data); - getSiteIdMap().then((siteIdMap) => { - renderTable("executiveOversubscribedTable", [ - { header: "Site", render: (r) => renderSiteLink(r.name, siteIdMap) }, - { header: "Oversub Down", render: (r) => fmtRatio(r.ratio_down) }, - { header: "Oversub Up", render: (r) => fmtRatio(r.ratio_up) }, - { header: "Cap (D/U)", render: (r) => `${fmtMbps(r.cap_down)} / ${fmtMbps(r.cap_up)}` }, - { header: "Subscribed (D/U)", render: (r) => `${fmtMbps(r.sub_down)} / ${fmtMbps(r.sub_up)}` }, - { header: "Median RTT (ms)", render: (r) => { - if (r.rtt === null || r.rtt === undefined) return "—"; - const color = colorByRttMs(r.rtt); - return `${colorSwatch(color)}${formatLatest(r.rtt, "ms", 1)}`; - }}, - { header: "Avg Down Util (%)", render: (r) => { - if (r.perf_down.avg === null || r.perf_down.avg === undefined) return "—"; - const color = colorByCapacity(r.perf_down.avg); - return `${colorSwatch(color)}${r.perf_down.avg.toFixed(1)}`; - }}, - { header: "Avg Up Util (%)", render: (r) => { - if (r.perf_up.avg === null || r.perf_up.avg === undefined) return "—"; - const color = colorByCapacity(r.perf_up.avg); - return `${colorSwatch(color)}${r.perf_up.avg.toFixed(1)}`; - }}, - ], rows, "No oversubscription data available yet."); - }); -} - -listenExecutiveHeatmaps(render); +renderExecutiveLeaderboardPage("executiveOversubscribedTable", "oversubscribed_sites"); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/executive_sites_due_upgrade.js b/src/rust/lqosd/src/node_manager/js_build/src/executive_sites_due_upgrade.js index 5fdd0fa0..d779ecea 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/executive_sites_due_upgrade.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/executive_sites_due_upgrade.js @@ -1,37 +1,3 @@ -import {listenExecutiveHeatmaps, averageWithCount, renderTable, getSiteIdMap, renderSiteLink} from "./executive_utils"; +import {renderExecutiveLeaderboardPage} from "./executive_leaderboard_page_template"; -const THRESHOLD = 80; -const MIN_SAMPLES = 10; - -function buildRows(data) { - const rows = (data.sites || []).map(site => { - const down = averageWithCount(site.blocks?.download || []); - const up = averageWithCount(site.blocks?.upload || []); - return { - name: site.site_name || "Site", - down, - up, - }; - }).filter(row => - row.down.count >= MIN_SAMPLES && - row.up.count >= MIN_SAMPLES && - (row.down.avg || 0) >= THRESHOLD && - (row.up.avg || 0) >= THRESHOLD - ); - - rows.sort((a, b) => ((b.down.avg || 0) + (b.up.avg || 0)) - ((a.down.avg || 0) + (a.up.avg || 0))); - return rows; -} - -function render(data) { - const rows = buildRows(data); - getSiteIdMap().then((siteIdMap) => { - renderTable("executiveSitesDueTable", [ - { header: "Site", render: (r) => renderSiteLink(r.name, siteIdMap) }, - { header: "Avg Down Util (%)", render: (r) => r.down.avg !== null ? r.down.avg.toFixed(1) : "—" }, - { header: "Avg Up Util (%)", render: (r) => r.up.avg !== null ? r.up.avg.toFixed(1) : "—" }, - ], rows, "No sites meet the 80%+ utilization threshold yet."); - }); -} - -listenExecutiveHeatmaps(render); +renderExecutiveLeaderboardPage("executiveSitesDueTable", "sites_due_upgrade"); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/executive_top_asns.js b/src/rust/lqosd/src/node_manager/js_build/src/executive_top_asns.js index 399098ba..094cf747 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/executive_top_asns.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/executive_top_asns.js @@ -1,58 +1,3 @@ -import {listenExecutiveHeatmaps, medianFromBlocks, sumBlocks, renderTable, colorSwatch} from "./executive_utils"; -import {colorByRttMs, colorByRetransmitPct} from "./helpers/color_scales"; +import {renderExecutiveLeaderboardPage} from "./executive_leaderboard_page_template"; -const SECONDS_PER_BLOCK = 60; -const BITS_PER_MEGABIT = 1_000_000; - -function buildRows(data) { - const rows = (data.asns || []).map(asn => { - const downSum = sumBlocks(asn.blocks?.download || []); - const upSum = sumBlocks(asn.blocks?.upload || []); - const totalMbps = downSum + upSum; - const totalBytes = (totalMbps * BITS_PER_MEGABIT / 8) * SECONDS_PER_BLOCK; - const medianRtt = medianFromBlocks(asn.blocks?.rtt || []); - const medianRetrans = medianFromBlocks(asn.blocks?.retransmit || []); - return { - asn: asn.asn, - asn_name: asn.asn_name, - totalBytes, - medianRtt, - medianRetrans, - }; - }); - rows.sort((a, b) => b.totalBytes - a.totalBytes); - return rows.slice(0, 10); -} - -function formatBytes(bytes) { - if (!Number.isFinite(bytes) || bytes <= 0) return "—"; - const units = ["B", "KB", "MB", "GB", "TB"]; - let i = 0; - let val = bytes; - while (val >= 1024 && i < units.length - 1) { - val /= 1024; - i++; - } - return `${val.toFixed(val >= 10 ? 1 : 2)} ${units[i]}`; -} - -function render(data) { - const rows = buildRows(data); - renderTable("executiveTopAsnsTable", [ - { header: "ASN", render: (r) => r.asn_name ? `${r.asn_name} (ASN ${r.asn})` : r.asn }, - { header: "Total Traffic (15m)", render: (r) => formatBytes(r.totalBytes) }, - { header: "Median RTT (ms)", render: (r) => { - if (r.medianRtt === null) return "—"; - const color = colorByRttMs(r.medianRtt); - return `${colorSwatch(color)}${r.medianRtt.toFixed(1)}`; - }}, - { header: "Median Retrans (%)", render: (r) => { - if (r.medianRetrans === null) return "—"; - const capped = Math.min(10, Math.max(0, r.medianRetrans)); - const color = colorByRetransmitPct(capped); - return `${colorSwatch(color)}${r.medianRetrans.toFixed(2)}`; - }}, - ], rows, "No ASN heatmap data yet."); -} - -listenExecutiveHeatmaps(render); +renderExecutiveLeaderboardPage("executiveTopAsnsTable", "top_asns"); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/executive_utils.js b/src/rust/lqosd/src/node_manager/js_build/src/executive_utils.js index 13537567..287808de 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/executive_utils.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/executive_utils.js @@ -1,6 +1,9 @@ import {get_ws_client, subscribeWS} from "./pubsub/ws"; const wsClient = get_ws_client(); +const DEFAULT_EXECUTIVE_REFRESH_MS = 5000; +let executiveRequestCounter = 0; + const listenOnce = (eventName, handler) => { const wrapped = (msg) => { wsClient.off(eventName, wrapped); @@ -9,14 +12,158 @@ const listenOnce = (eventName, handler) => { wsClient.on(eventName, wrapped); }; -export function listenExecutiveHeatmaps(onData) { - subscribeWS(["ExecutiveHeatmaps"], (msg) => { - if (msg.event === "ExecutiveHeatmaps") { +function normalizeEntityKinds(entityKinds) { + return [...new Set((entityKinds || []).map((kind) => String(kind)))].sort(); +} + +function nextExecutiveRequestId(prefix) { + executiveRequestCounter += 1; + return `${prefix}-${Date.now()}-${executiveRequestCounter}`; +} + +function matchesExecutiveHeatmapQuery(responseQuery, requestQuery) { + if (!responseQuery || !requestQuery) return false; + const responseRequestId = String(responseQuery.client_request_id || ""); + const requestRequestId = String(requestQuery.client_request_id || ""); + if (responseRequestId || requestRequestId) { + return responseRequestId !== "" && responseRequestId === requestRequestId; + } + return String(responseQuery.metric || "") === String(requestQuery.metric || "") + && Number(responseQuery.page ?? 0) === Number(requestQuery.page ?? 0) + && Number(responseQuery.page_size ?? 0) === Number(requestQuery.page_size ?? 0) + && String(responseQuery.sort || "LatestValue") === String(requestQuery.sort || "LatestValue") + && Boolean(responseQuery.descending ?? true) === Boolean(requestQuery.descending ?? true) + && String(responseQuery.search || "") === String(requestQuery.search || "") + && JSON.stringify(normalizeEntityKinds(responseQuery.entity_kinds)) === JSON.stringify(normalizeEntityKinds(requestQuery.entity_kinds)); +} + +function matchesExecutiveLeaderboardQuery(responseQuery, requestQuery) { + if (!responseQuery || !requestQuery) return false; + const responseRequestId = String(responseQuery.client_request_id || ""); + const requestRequestId = String(requestQuery.client_request_id || ""); + if (responseRequestId || requestRequestId) { + return responseRequestId !== "" && responseRequestId === requestRequestId; + } + return String(responseQuery.kind || "") === String(requestQuery.kind || "") + && Number(responseQuery.page ?? 0) === Number(requestQuery.page ?? 0) + && Number(responseQuery.page_size ?? 0) === Number(requestQuery.page_size ?? 0) + && String(responseQuery.search || "") === String(requestQuery.search || ""); +} + +function requestMatchingResponse(eventName, request, matcher, onData, onError = null) { + let disposed = false; + const cleanup = wsClient.on(eventName, (msg) => { + if (disposed) return; + const data = msg?.data || {}; + if (!matcher(data)) return; + disposed = true; + cleanup(); + onData(data); + }); + try { + wsClient.send(request); + } catch (err) { + disposed = true; + cleanup(); + if (onError) { + onError(err); + } else { + console.error(err); + } + } + return () => { + if (disposed) return; + disposed = true; + cleanup(); + }; +} + +function pollMatchingResponse(eventName, buildRequest, matcher, onData, intervalMs = DEFAULT_EXECUTIVE_REFRESH_MS) { + const handlerDisposer = wsClient.on(eventName, (msg) => { + const data = msg?.data || {}; + if (!matcher(data)) return; + onData(data); + }); + const sendRequest = () => { + wsClient.send(buildRequest()); + }; + sendRequest(); + const timer = window.setInterval(sendRequest, intervalMs); + return { + dispose() { + window.clearInterval(timer); + handlerDisposer(); + }, + refresh() { + sendRequest(); + }, + }; +} + +export function listenExecutiveDashboardSummary(onData) { + return subscribeWS(["ExecutiveDashboardSummary"], (msg) => { + if (msg.event === "ExecutiveDashboardSummary") { onData(msg.data || {}); } }); } +export function requestExecutiveHeatmapPage(query, onData, onError = null) { + const requestQuery = { + ...query, + client_request_id: nextExecutiveRequestId("exec-heatmap"), + }; + return requestMatchingResponse( + "ExecutiveHeatmapPage", + { ExecutiveHeatmapPage: { query: requestQuery } }, + (data) => matchesExecutiveHeatmapQuery(data.query, requestQuery), + onData, + onError, + ); +} + +export function pollExecutiveHeatmapPage(query, onData, intervalMs = DEFAULT_EXECUTIVE_REFRESH_MS) { + const requestQuery = { ...query }; + return pollMatchingResponse( + "ExecutiveHeatmapPage", + () => { + requestQuery.client_request_id = nextExecutiveRequestId("exec-heatmap"); + return { ExecutiveHeatmapPage: { query: requestQuery } }; + }, + (data) => matchesExecutiveHeatmapQuery(data.query, requestQuery), + onData, + intervalMs, + ); +} + +export function requestExecutiveLeaderboardPage(query, onData, onError = null) { + const requestQuery = { + ...query, + client_request_id: nextExecutiveRequestId("exec-leaderboard"), + }; + return requestMatchingResponse( + "ExecutiveLeaderboardPage", + { ExecutiveLeaderboardPage: { query: requestQuery } }, + (data) => matchesExecutiveLeaderboardQuery(data.query, requestQuery), + onData, + onError, + ); +} + +export function pollExecutiveLeaderboardPage(query, onData, intervalMs = DEFAULT_EXECUTIVE_REFRESH_MS) { + const requestQuery = { ...query }; + return pollMatchingResponse( + "ExecutiveLeaderboardPage", + () => { + requestQuery.client_request_id = nextExecutiveRequestId("exec-leaderboard"); + return { ExecutiveLeaderboardPage: { query: requestQuery } }; + }, + (data) => matchesExecutiveLeaderboardQuery(data.query, requestQuery), + onData, + intervalMs, + ); +} + let siteIdMap = null; let siteIdMapPromise = null; @@ -31,7 +178,7 @@ export function getNodeIdMap() { siteIdMap = map; resolve(map); }; - listenOnce("NetworkTree", (msg) => { + listenOnce("NetworkTreeLite", (msg) => { const data = msg && msg.data ? msg.data : []; const map = new Map(); (data || []).forEach((entry) => { @@ -44,7 +191,7 @@ export function getNodeIdMap() { }); resolveOnce(map); }); - wsClient.send({ NetworkTree: {} }); + wsClient.send({ NetworkTreeLite: {} }); setTimeout(() => { resolveOnce(new Map()); }, 5000); @@ -61,6 +208,22 @@ export function linkToCircuit(circuitId) { return `circuit.html?id=${encodeURIComponent(circuitId)}`; } +export function linkToTreeLocator(locator) { + if (!locator) return null; + const params = new URLSearchParams(); + if (locator.parent_index !== undefined && locator.parent_index !== null) { + params.set("parent", String(locator.parent_index)); + } + if (locator.node_id) { + params.set("nodeId", locator.node_id); + } + if (Array.isArray(locator.node_path) && locator.node_path.length > 0) { + params.set("nodePath", JSON.stringify(locator.node_path)); + } + const query = params.toString(); + return query ? `tree.html?${query}` : null; +} + export function linkToSite(siteName, siteIdLookup) { if (!siteName || !siteIdLookup) return null; const siteId = siteIdLookup.get(siteName); @@ -75,6 +238,41 @@ export function linkToTreeNode(nodeName, nodeIdLookup) { return `tree.html?parent=${encodeURIComponent(nodeId)}`; } +export function badgeForEntityKind(entityKind) { + switch (String(entityKind || "")) { + case "Site": + return "Site"; + case "Circuit": + return "Circuit"; + case "Asn": + return "ASN"; + default: + return ""; + } +} + +export function linkToExecutiveMetricRow(row) { + if (!row) return null; + if (row.circuit_id) { + return linkToCircuit(row.circuit_id); + } + return linkToTreeLocator(row.tree); +} + +export function linkToExecutiveLeaderboardRow(row) { + if (!row || !row.kind) return null; + switch (row.kind) { + case "WorstSiteByRtt": + case "OversubscribedSite": + case "SiteDueUpgrade": + return linkToTreeLocator(row.tree); + case "CircuitDueUpgrade": + return linkToCircuit(row.circuit_id); + default: + return null; + } +} + export function renderCircuitLink(name, circuitId) { const link = linkToCircuit(circuitId); if (!link) return `${name}`; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/executive_worst_sites.js b/src/rust/lqosd/src/node_manager/js_build/src/executive_worst_sites.js index 572f46a2..74ced0b5 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/executive_worst_sites.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/executive_worst_sites.js @@ -1,54 +1,3 @@ -import { - listenExecutiveHeatmaps, - medianFromBlocks, - averageFromBlocks, - renderTable, - colorSwatch, - getSiteIdMap, - renderSiteLink -} from "./executive_utils"; -import {colorByRttMs} from "./helpers/color_scales"; -import {colorByCapacity} from "./dashlets/executive_heatmap_shared"; +import {renderExecutiveLeaderboardPage} from "./executive_leaderboard_page_template"; -function buildRows(data) { - const sites = (data.sites || []).map(site => { - const rtt = medianFromBlocks(site.blocks?.rtt || []); - const avgDown = averageFromBlocks(site.blocks?.download || []); - const avgUp = averageFromBlocks(site.blocks?.upload || []); - return { - name: site.site_name || "Site", - rtt, - avgDown, - avgUp, - }; - }).filter(row => row.rtt !== null); - - sites.sort((a, b) => (b.rtt || 0) - (a.rtt || 0)); - return sites.slice(0, 20); -} - -function render(data) { - const rows = buildRows(data); - getSiteIdMap().then((siteIdMap) => { - renderTable("executiveWorstSitesTable", [ - { header: "Site", render: (r) => renderSiteLink(r.name, siteIdMap) }, - { header: "Median RTT (ms)", render: (r) => { - if (r.rtt === null) return "—"; - const color = colorByRttMs(r.rtt); - return `${colorSwatch(color)}${r.rtt.toFixed(1)}`; - }}, - { header: "Avg Down Util (%)", render: (r) => { - if (r.avgDown === null) return "—"; - const color = colorByCapacity(r.avgDown); - return `${colorSwatch(color)}${r.avgDown.toFixed(1)}`; - }}, - { header: "Avg Up Util (%)", render: (r) => { - if (r.avgUp === null) return "—"; - const color = colorByCapacity(r.avgUp); - return `${colorSwatch(color)}${r.avgUp.toFixed(1)}`; - }}, - ], rows, "No site heatmap data yet."); - }); -} - -listenExecutiveHeatmaps(render); +renderExecutiveLeaderboardPage("executiveWorstSitesTable", "worst_sites"); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/first-run.js b/src/rust/lqosd/src/node_manager/js_build/src/first-run.js index c4a024cb..a605adb1 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/first-run.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/first-run.js @@ -1,5 +1,5 @@ $("#btnCreateUser").on('click', () => { - let username = $("#username").val(); + let username = ($("#username").val() || "").trim(); let password = $("#password").val(); let anon = document.getElementById("allowAnonymous").checked; if (username === "") { @@ -21,11 +21,22 @@ $("#btnCreateUser").on('click', () => { url: "/firstLogin", data: JSON.stringify(login), contentType: 'application/json', + beforeSend: () => { + $("#firstRunError").removeClass("show").addClass("d-none"); + }, success: () => { window.location.href = "/index.html"; }, - error: () => { - alert("Something went wrong"); + error: (xhr) => { + const response = xhr && xhr.responseJSON ? xhr.responseJSON : {}; + const reason = response.reason || ""; + if (reason === "already_configured") { + window.location.href = "/login.html"; + return; + } + + $("#firstRunErrorText").text(response.message || "Something went wrong"); + $("#firstRunError").removeClass("d-none").addClass("show"); } }) -}); \ No newline at end of file +}); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/flow_map.js b/src/rust/lqosd/src/node_manager/js_build/src/flow_map.js index bdf3d2f4..d54b6f93 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/flow_map.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/flow_map.js @@ -17,6 +17,8 @@ const DEFAULT_POINT_OF_VIEW = { lng: -32, altitude: 2.15, }; +const MIN_INITIAL_ALTITUDE = 1.05; +const MAX_INITIAL_ALTITUDE = 2.05; const MIN_ZOOM_SPEED = 1.8; const MAX_ZOOM_SPEED = 3.4; const FLOW_MAP_DEBUG = (() => { @@ -253,10 +255,24 @@ function normalizeVector(x, y, z) { }; } +function clamp(value, min, max) { + return Math.max(min, Math.min(max, value)); +} + +function unitVectorToLatLng(vector) { + const lat = Math.asin(clamp(vector.y, -1, 1)) * (180 / Math.PI); + const lng = Math.atan2(vector.x, vector.z) * (180 / Math.PI); + return { lat, lng }; +} + function dotProduct(a, b) { return (a.x * b.x) + (a.y * b.y) + (a.z * b.z); } +function angularDistanceDegrees(a, b) { + return Math.acos(clamp(dotProduct(a, b), -1, 1)) * (180 / Math.PI); +} + function formatRtt(rttNanos) { const rttMs = toNumber(rttNanos, 0) / 1_000_000; if (rttMs <= 0) { @@ -374,7 +390,7 @@ class FlowMapGlobe { maxBytes: 0, lastUpdatedLabel: "Waiting for data", }; - this.autoRotate = false; + this.autoRotate = true; this.autoRefresh = true; this.colorMode = "rtt"; this.maxPoints = 1000; @@ -385,6 +401,8 @@ class FlowMapGlobe { this.lastKnownAltitude = DEFAULT_POINT_OF_VIEW.altitude; this.activeClusterRadius = clusterRadiusPx(DEFAULT_POINT_OF_VIEW.altitude); this.selectedMeta = null; + this.initialPointOfView = { ...DEFAULT_POINT_OF_VIEW }; + this.hasAutoCenteredOnce = false; this.initGlobe(); this.observeThemeChanges(); @@ -428,7 +446,7 @@ class FlowMapGlobe { this.controls.autoRotate = this.autoRotate; this.controls.autoRotateSpeed = 0.45; - this.globe.pointOfView(DEFAULT_POINT_OF_VIEW, 0); + this.globe.pointOfView(this.initialPointOfView, 0); this.captureViewState(); this.syncControlSpeeds(); this.applyTheme(); @@ -816,13 +834,88 @@ class FlowMapGlobe { } resetView() { - this.globe.pointOfView(DEFAULT_POINT_OF_VIEW, 700); - this.lastKnownAltitude = DEFAULT_POINT_OF_VIEW.altitude; + this.globe.pointOfView(this.initialPointOfView, 700); + this.lastKnownAltitude = this.initialPointOfView.altitude; + this.activeClusterRadius = clusterRadiusPx(this.initialPointOfView.altitude); if (this.rawPayload.length > 0) { this.renderData(this.rawPayload); } } + computeInitialPointOfView(payload) { + const source = Array.isArray(payload) ? payload : []; + const valid = source + .map((row) => ({ + lat: toNumber(row?.[0], NaN), + lng: toNumber(row?.[1], NaN), + bytes: Math.max(1, toNumber(row?.[3], 0)), + })) + .filter((point) => Number.isFinite(point.lat) && Number.isFinite(point.lng)); + + if (valid.length === 0) { + return { ...DEFAULT_POINT_OF_VIEW }; + } + + let sumX = 0; + let sumY = 0; + let sumZ = 0; + let totalWeight = 0; + const vectors = valid.map((point) => { + const vector = latLngToUnitVector(point.lat, point.lng); + const weight = Math.max(1, Math.sqrt(point.bytes)); + sumX += vector.x * weight; + sumY += vector.y * weight; + sumZ += vector.z * weight; + totalWeight += weight; + return { ...point, vector, weight }; + }); + + if (totalWeight <= 0) { + return { ...DEFAULT_POINT_OF_VIEW }; + } + + const centerVector = normalizeVector(sumX, sumY, sumZ); + const center = unitVectorToLatLng(centerVector); + + let weightedSpread = 0; + let maxSpread = 0; + vectors.forEach((point) => { + const distance = angularDistanceDegrees(centerVector, point.vector); + weightedSpread += distance * point.weight; + maxSpread = Math.max(maxSpread, distance); + }); + + const averageSpread = weightedSpread / totalWeight; + const spreadReference = Math.max(averageSpread * 1.35, maxSpread * 0.78); + const width = Math.max(1, this.dom.clientWidth || 960); + const height = Math.max(1, this.dom.clientHeight || 640); + const aspect = width / height; + const aspectAdjustment = aspect > 1.3 ? Math.min(0.22, (aspect - 1.3) * 0.22) : 0; + const altitude = clamp( + 1.02 + clamp((spreadReference - 6) / 85, 0, 1) * 1.08 - aspectAdjustment, + MIN_INITIAL_ALTITUDE, + MAX_INITIAL_ALTITUDE, + ); + + return { + lat: center.lat, + lng: center.lng, + altitude, + }; + } + + updateInitialPointOfViewFromPayload(payload) { + if (this.hasAutoCenteredOnce) { + return; + } + this.initialPointOfView = this.computeInitialPointOfView(payload); + this.hasAutoCenteredOnce = true; + this.globe.pointOfView(this.initialPointOfView, 0); + this.lastKnownAltitude = this.initialPointOfView.altitude; + this.activeClusterRadius = clusterRadiusPx(this.initialPointOfView.altitude); + this.syncControlSpeeds(); + } + scheduleInteractionRelease() { if (this.interactionTimer) { clearTimeout(this.interactionTimer); @@ -1021,6 +1114,7 @@ function refreshNow() { map.renderData([]); map.updateSummary(0, totalBytes, lastUpdateAt); } else { + map.updateInitialPointOfViewFromPayload(data); overlay.hide(); map.renderData(data); map.updateSummary(map.renderedSummary.pointCount, totalBytes, lastUpdateAt); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/cake_backlog.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/cake_backlog.js index a6e0ee8e..1bd64112 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/cake_backlog.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/cake_backlog.js @@ -1,19 +1,22 @@ import {DashboardGraph} from "./dashboard_graph"; -import {scaleNumber} from "../lq_js_common/helpers/scaling"; +import { + CAKE_CHART_WINDOW_SECONDS, + cakeChartTitle, + cakeCommonGrid, + cakeCommonXAxis, + cakeHistoryWindow, + cakeScatterSeries, + cakeTooltip, + formatCakeBytes, +} from "./cake_history"; export class CakeBacklog extends DashboardGraph { constructor(id) { super(id); - let xaxis = []; - for (let i=0; i<600; i++) { - xaxis.push(i); - } - this.option = { - title: { - text: "Backlog", - }, + title: cakeChartTitle("Backlog", "Bytes"), + grid: cakeCommonGrid(), legend: { orient: "horizontal", right: 10, @@ -50,87 +53,24 @@ export class CakeBacklog extends DashboardGraph { } ] }, - xAxis: { - type: 'category', - data: xaxis, - }, + xAxis: cakeCommonXAxis(CAKE_CHART_WINDOW_SECONDS), yAxis: { type: 'value', axisLabel: { - formatter: (val) => { - return scaleNumber(Math.abs(val), 0); - }, + formatter: (val) => formatCakeBytes(val), } }, series: [ - { - name: 'Bulk', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[0] } - }, - { - name: 'Best Effort', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[1] } - }, - { - name: 'RT Video', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[2] } - }, - { - name: 'Voice', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[3] } - }, - { - name: 'Bulk Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[0] }, - }, - { - name: 'Best Effort Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[1] } - }, - { - name: 'RT Video Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[2] } - }, - { - name: 'RT Voice Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[3] } - }, + cakeScatterSeries("Bulk", window.graphPalette[0]), + cakeScatterSeries("Best Effort", window.graphPalette[1]), + cakeScatterSeries("RT Video", window.graphPalette[2]), + cakeScatterSeries("Voice", window.graphPalette[3]), + cakeScatterSeries("Bulk Up", window.graphPalette[0]), + cakeScatterSeries("Best Effort Up", window.graphPalette[1]), + cakeScatterSeries("RT Video Up", window.graphPalette[2]), + cakeScatterSeries("RT Voice Up", window.graphPalette[3]), ], - tooltip: { - trigger: 'item', - }, + tooltip: cakeTooltip(formatCakeBytes), animation: false, } this.option && this.chart.setOption(this.option); @@ -160,19 +100,15 @@ export class CakeBacklog extends DashboardGraph { for (let i=0; i<8; i++) { this.option.series[i].data = []; } - //console.log(msg); - for (let i=msg.history_head; i<600; i++) { + for (const sample of cakeHistoryWindow(msg, CAKE_CHART_WINDOW_SECONDS)) { for (let j=0; j<4; j++) { - if (msg.history[i][0].tins[0] === undefined) continue; - this.option.series[j].data.push(msg.history[i][0].tins[j].backlog_bytes); - this.option.series[j+4].data.push(0 - msg.history[i][1].tins[j].backlog_bytes); - } - } - for (let i=0; i formatCakeMillisecondsFromUs(val), + }, }, series: [ - { - name: 'Bulk', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[0] } - }, - { - name: 'Best Effort', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[1] } - }, - { - name: 'RT Video', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[2] } - }, - { - name: 'Voice', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[3] } - }, - { - name: 'Bulk Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[0] }, - }, - { - name: 'Best Effort Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[1] } - }, - { - name: 'RT Video Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[2] } - }, - { - name: 'RT Voice Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[3] } - }, + cakeScatterSeries("Bulk", window.graphPalette[0]), + cakeScatterSeries("Best Effort", window.graphPalette[1]), + cakeScatterSeries("RT Video", window.graphPalette[2]), + cakeScatterSeries("Voice", window.graphPalette[3]), + cakeScatterSeries("Bulk Up", window.graphPalette[0]), + cakeScatterSeries("Best Effort Up", window.graphPalette[1]), + cakeScatterSeries("RT Video Up", window.graphPalette[2]), + cakeScatterSeries("RT Voice Up", window.graphPalette[3]), ], - tooltip: { - trigger: 'item', - }, + tooltip: cakeTooltip(formatCakeMillisecondsFromUs), animation: false, } this.option && this.chart.setOption(this.option); @@ -153,19 +99,15 @@ export class CakeDelays extends DashboardGraph { for (let i=0; i<8; i++) { this.option.series[i].data = []; } - //console.log(msg); - for (let i=msg.history_head; i<600; i++) { + for (const sample of cakeHistoryWindow(msg, CAKE_CHART_WINDOW_SECONDS)) { for (let j=0; j<4; j++) { - if (msg.history[i][0].tins[0] === undefined) continue; - this.option.series[j].data.push(msg.history[i][0].tins[j].base_delay_us); - this.option.series[j+4].data.push(0 - msg.history[i][1].tins[j].base_delay_us); - } - } - for (let i=0; i { - return scaleNumber(Math.abs(val), 0); - }, + formatter: (val) => formatCakePackets(val), } }, series: [ - { - name: 'Bulk', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[0] } - }, - { - name: 'Best Effort', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[1] } - }, - { - name: 'RT Video', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[2] } - }, - { - name: 'Voice', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[3] } - }, - { - name: 'Bulk Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[0] } - }, - { - name: 'Best Effort Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[1] } - }, - { - name: 'RT Video Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[2] } - }, - { - name: 'RT Voice Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[3] } - }, + cakeScatterSeries("Bulk", window.graphPalette[0]), + cakeScatterSeries("Best Effort", window.graphPalette[1]), + cakeScatterSeries("RT Video", window.graphPalette[2]), + cakeScatterSeries("Voice", window.graphPalette[3]), + cakeScatterSeries("Bulk Up", window.graphPalette[0]), + cakeScatterSeries("Best Effort Up", window.graphPalette[1]), + cakeScatterSeries("RT Video Up", window.graphPalette[2]), + cakeScatterSeries("RT Voice Up", window.graphPalette[3]), ], - tooltip: { - trigger: 'item', - }, + tooltip: cakeTooltip(formatCakePackets), animation: false, } this.option && this.chart.setOption(this.option); @@ -159,19 +99,15 @@ export class CakeDrops extends DashboardGraph { for (let i=0; i<8; i++) { this.option.series[i].data = []; } - //console.log(msg); - for (let i=msg.history_head; i<600; i++) { + for (const sample of cakeHistoryWindow(msg, CAKE_CHART_WINDOW_SECONDS)) { for (let j=0; j<4; j++) { - if (msg.history[i][0].tins[0] === undefined) continue; - this.option.series[j].data.push(msg.history[i][0].tins[j].drops); - this.option.series[j+4].data.push(0 - msg.history[i][1].tins[j].drops); - } - } - for (let i=0; i= 1; minute--) { + const index = windowSize - (minute * 60); + if (index >= 0 && index < windowSize) { + labels[index] = `-${minute}m`; + } + } + + if (windowSize > 0) { + if (labels[0] === "") { + labels[0] = `-${Math.ceil(windowSize / 60)}m`; + } + labels[windowSize - 1] = "Now"; + } + + return labels; +} + +export function cakeChartTitle(text, unitsLabel) { + return { + text, + subtext: `${CAKE_CHART_WINDOW_LABEL} · ${unitsLabel}`, + left: 10, + top: 10, + textStyle: { + fontSize: 14, + fontWeight: 700, + }, + subtextStyle: { + fontSize: 11, + color: "#8c939d", + }, + }; +} + +export function cakeCommonGrid() { + return { + left: 58, + right: 16, + top: 62, + bottom: 46, + containLabel: true, + }; +} + +export function cakeCommonXAxis(windowSize = CAKE_CHART_WINDOW_SECONDS) { + return { + type: "category", + data: cakeChartXAxis(windowSize), + boundaryGap: false, + axisTick: { + show: false, + }, + axisLabel: { + color: "#8c939d", + fontSize: 10, + interval: 0, + }, + axisLine: { + lineStyle: { + color: "rgba(148, 163, 184, 0.28)", + }, + }, + splitLine: { + show: false, + }, + axisPointer: { + show: true, + label: { + show: false, + }, + }, + }; +} + +export function cakeScatterSeries(name, color) { + return { + name, + data: [], + type: "scatter", + symbol: "circle", + symbolSize: 4, + itemStyle: { + color, + opacity: 0.72, + }, + emphasis: { + scale: 1.8, + itemStyle: { + color, + opacity: 0.98, + borderColor: "rgba(255, 255, 255, 0.92)", + borderWidth: 1, + }, + }, + }; +} + +export function cakeTooltip(unitFormatter) { + return { + trigger: "axis", + axisPointer: { + type: "line", + snap: false, + }, + confine: true, + formatter: (params) => { + const rows = Array.isArray(params) ? params : [params]; + if (rows.length === 0) { + return ""; + } + const heading = rows[0]?.axisValueLabel || "Selected sample"; + const lines = [ + `
    ${heading}
    `, + ]; + + rows.forEach((entry) => { + const raw = Array.isArray(entry.value) ? entry.value[1] : entry.value; + if (raw === null || raw === undefined || Number.isNaN(Number(raw))) { + return; + } + const isUpload = entry.seriesName.endsWith(" Up"); + const label = isUpload + ? entry.seriesName.slice(0, -3) + : entry.seriesName; + const direction = isUpload ? "Upload" : "Download"; + lines.push( + `${entry.marker}${label} (${direction}): ${unitFormatter(Math.abs(Number(raw)))}`, + ); + }); + + return lines.join("
    "); + }, + }; +} + +export function formatCakeBytes(value) { + return `${scaleNumber(Math.abs(toNumber(value, 0)), 1)}B`; +} + +export function formatCakeBitsPerSecond(value) { + return `${scaleNumber(Math.abs(toNumber(value, 0)), 1)}bps`; +} + +export function formatCakePackets(value) { + return `${scaleNumber(Math.abs(toNumber(value, 0)), 0)} pkts`; +} + +export function formatCakeMillisecondsFromUs(value) { + const ms = Math.abs(toNumber(value, 0)) / 1000; + if (ms >= 100) { + return `${Math.round(ms)}ms`; + } + if (ms >= 10) { + return `${trimTrailingZeros(ms.toFixed(1))}ms`; + } + return `${trimTrailingZeros(ms.toFixed(2))}ms`; +} + +export function cakeHistoryWindow(msg, windowSize = CAKE_CHART_WINDOW_SECONDS) { + if (!msg || !Array.isArray(msg.history) || msg.history.length === 0) { + return new Array(windowSize).fill(null); + } + + const historySize = msg.history.length; + const historyHead = Number.isFinite(msg.history_head) ? msg.history_head : 0; + const ordered = []; + + for (let i = historyHead; i < historySize; i++) { + ordered.push(msg.history[i]); + } + for (let i = 0; i < historyHead; i++) { + ordered.push(msg.history[i]); + } + + const available = ordered.slice(-Math.min(windowSize, ordered.length)); + if (available.length >= windowSize) { + return available; + } + + return new Array(windowSize - available.length).fill(null).concat(available); +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/cake_marks.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/cake_marks.js index 9926c925..773ba18f 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/cake_marks.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/cake_marks.js @@ -1,19 +1,22 @@ import {DashboardGraph} from "./dashboard_graph"; -import {scaleNumber} from "../lq_js_common/helpers/scaling"; +import { + CAKE_CHART_WINDOW_SECONDS, + cakeChartTitle, + cakeCommonGrid, + cakeCommonXAxis, + cakeHistoryWindow, + cakeScatterSeries, + cakeTooltip, + formatCakePackets, +} from "./cake_history"; export class CakeMarks extends DashboardGraph { constructor(id) { super(id); - let xaxis = []; - for (let i=0; i<600; i++) { - xaxis.push(i); - } - this.option = { - title: { - text: "ECN Marks", - }, + title: cakeChartTitle("ECN Marks", "Packets per sample"), + grid: cakeCommonGrid(), legend: { orient: "horizontal", right: 10, @@ -50,87 +53,24 @@ export class CakeMarks extends DashboardGraph { } ] }, - xAxis: { - type: 'category', - data: xaxis, - }, + xAxis: cakeCommonXAxis(CAKE_CHART_WINDOW_SECONDS), yAxis: { type: 'value', axisLabel: { - formatter: (val) => { - return scaleNumber(Math.abs(val), 0); - }, + formatter: (val) => formatCakePackets(val), } }, series: [ - { - name: 'Bulk', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[0] } - }, - { - name: 'Best Effort', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[1] } - }, - { - name: 'RT Video', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[2] } - }, - { - name: 'Voice', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[3] } - }, - { - name: 'Bulk Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[0] }, - }, - { - name: 'Best Effort Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[1] } - }, - { - name: 'RT Video Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[2] } - }, - { - name: 'RT Voice Up', - data: [], - type: 'scatter', - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[3] } - }, + cakeScatterSeries("Bulk", window.graphPalette[0]), + cakeScatterSeries("Best Effort", window.graphPalette[1]), + cakeScatterSeries("RT Video", window.graphPalette[2]), + cakeScatterSeries("Voice", window.graphPalette[3]), + cakeScatterSeries("Bulk Up", window.graphPalette[0]), + cakeScatterSeries("Best Effort Up", window.graphPalette[1]), + cakeScatterSeries("RT Video Up", window.graphPalette[2]), + cakeScatterSeries("RT Voice Up", window.graphPalette[3]), ], - tooltip: { - trigger: 'item', - }, + tooltip: cakeTooltip(formatCakePackets), animation: false, } this.option && this.chart.setOption(this.option); @@ -159,19 +99,15 @@ export class CakeMarks extends DashboardGraph { for (let i=0; i<8; i++) { this.option.series[i].data = []; } - //console.log(msg); - for (let i=msg.history_head; i<600; i++) { + for (const sample of cakeHistoryWindow(msg, CAKE_CHART_WINDOW_SECONDS)) { for (let j=0; j<4; j++) { - if (msg.history[i][0].tins[0] === undefined) continue; - this.option.series[j].data.push(msg.history[i][0].tins[j].marks); - this.option.series[j+4].data.push(0 - msg.history[i][1].tins[j].marks); - } - } - for (let i=0; i formatCakePackets(val), + }, }, series: [ - { - name: 'Queue Length', - data: [], - type: 'scatter', - lineStyle: { - color: window.graphPalette[0] - }, - symbol: 'circle', - symbolSize: 2, - itemStyle: window.graphPalette[0] - }, - { - name: 'Queue Length Up', - data: [], - type: 'scatter', - lineStyle: { - color: window.graphPalette[0] - }, - symbol: 'circle', - symbolSize: 2, - itemStyle: window.graphPalette[0] - }, + cakeScatterSeries("Queue Length", window.graphPalette[0]), + cakeScatterSeries("Queue Length Up", window.graphPalette[4] || window.graphPalette[0]), ], - tooltip: { - trigger: 'item', - }, + tooltip: cakeTooltip(formatCakePackets), animation: false, } this.option && this.chart.setOption(this.option); @@ -73,10 +55,8 @@ export class CakeQueueLength extends DashboardGraph { onThemeChange() { super.onThemeChange(); this.option.legend.data[0].itemStyle.color = window.graphPalette[0]; - this.option.series[0].lineStyle.color = window.graphPalette[0]; - this.option.series[0].itemStyle = window.graphPalette[0]; - this.option.series[1].lineStyle.color = window.graphPalette[0]; - this.option.series[1].itemStyle = window.graphPalette + this.option.series[0].itemStyle.color = window.graphPalette[0]; + this.option.series[1].itemStyle.color = window.graphPalette[4] || window.graphPalette[0]; this.chart.setOption(this.option); } @@ -86,14 +66,14 @@ export class CakeQueueLength extends DashboardGraph { this.option.series[0].data = []; this.option.series[1].data = []; - //console.log(msg); - for (let i=msg.history_head; i<600; i++) { - this.option.series[0].data.push(msg.history[i][0].qlen); - this.option.series[1].data.push(0 - msg.history[i][1].qlen); - } - for (let i=0; i { - return scaleNumber(Math.abs(val), 0); - }, + formatter: (val) => formatCakeBitsPerSecond(val), } }, series: [ - { - name: 'Bulk', - data: [], - type: 'scatter', - lineStyle: { - color: window.graphPalette[0], - }, - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[0] } - }, - { - name: 'Best Effort', - data: [], - type: 'scatter', - lineStyle: { - color: window.graphPalette[1] - }, - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[1] } - }, - { - name: 'RT Video', - data: [], - type: 'scatter', - lineStyle: { - color: window.graphPalette[2] - }, - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[2] } - }, - { - name: 'Voice', - data: [], - type: 'scatter', - lineStyle: { - color: window.graphPalette[3] - }, - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[3] } - }, - { - name: 'Bulk Up', - data: [], - type: 'scatter', - lineStyle: { - color: window.graphPalette[0], - }, - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[0] } - }, - { - name: 'Best Effort Up', - data: [], - type: 'scatter', - lineStyle: { - color: window.graphPalette[1] - }, - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[1] } - }, - { - name: 'RT Video Up', - data: [], - type: 'scatter', - lineStyle: { - color: window.graphPalette[2] - }, - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[2] } - }, - { - name: 'RT Voice Up', - data: [], - type: 'scatter', - lineStyle: { - color: window.graphPalette[3] - }, - symbol: 'circle', - symbolSize: 2, - itemStyle: { color: window.graphPalette[3] } - }, + cakeScatterSeries("Bulk", window.graphPalette[0]), + cakeScatterSeries("Best Effort", window.graphPalette[1]), + cakeScatterSeries("RT Video", window.graphPalette[2]), + cakeScatterSeries("Voice", window.graphPalette[3]), + cakeScatterSeries("Bulk Up", window.graphPalette[0]), + cakeScatterSeries("Best Effort Up", window.graphPalette[1]), + cakeScatterSeries("RT Video Up", window.graphPalette[2]), + cakeScatterSeries("RT Voice Up", window.graphPalette[3]), ], - tooltip: { - trigger: 'item', - }, + tooltip: cakeTooltip(formatCakeBitsPerSecond), animation: false, } this.option && this.chart.setOption(this.option); @@ -183,21 +100,15 @@ export class CakeTraffic extends DashboardGraph { for (let i=0; i<8; i++) { this.option.series[i].data = []; } - //console.log(msg); - for (let i=msg.history_head; i<600; i++) { + for (const sample of cakeHistoryWindow(msg, CAKE_CHART_WINDOW_SECONDS)) { for (let j=0; j<4; j++) { - if (msg.history[i][0].tins[0] === undefined) continue; - const downBits = toNumber(msg.history[i][0].tins[j].sent_bytes, 0) * 8; - const upBits = toNumber(msg.history[i][1].tins[j].sent_bytes, 0) * 8; - this.option.series[j].data.push(downBits); - this.option.series[j+4].data.push(0 - upBits); - } - } - for (let i=0; i applyAxisTheme(axis, theme)); + } else { + applyAxisTheme(option.xAxis, theme); + } + + if (Array.isArray(option.yAxis)) { + option.yAxis.forEach((axis) => applyAxisTheme(axis, theme)); + } else { + applyAxisTheme(option.yAxis, theme); + } + + if (hasLegend && option.legend) { + option.legend = { + ...option.legend, + textStyle: { + ...((option.legend && option.legend.textStyle) || {}), + color: theme.legendText, + }, + }; + } + + if (option.tooltip) { + option.tooltip = { + ...option.tooltip, + backgroundColor: theme.tooltipBackground, + borderColor: theme.tooltipBorder, + borderWidth: 1, + textStyle: { + ...((option.tooltip && option.tooltip.textStyle) || {}), + color: theme.tooltipText, + }, + axisPointer: option.tooltip.axisPointer + ? { + ...option.tooltip.axisPointer, + label: { + ...((option.tooltip.axisPointer && option.tooltip.axisPointer.label) || {}), + backgroundColor: theme.tooltipAxisLabelBackground, + color: theme.tooltipText, + }, + } + : option.tooltip.axisPointer, + }; + } + + return option; +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/circuit_retransmit_graph.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/circuit_retransmit_graph.js index b5d7db48..6d32a5e5 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/circuit_retransmit_graph.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/circuit_retransmit_graph.js @@ -1,5 +1,6 @@ import {DashboardGraph} from "./dashboard_graph"; import {scaleNumber} from "../lq_js_common/helpers/scaling"; +import {applyCircuitDeviceChartTheme} from "./circuit_device_chart_theme"; // Simple time formatter (HH:MM:SS) function formatTime(ts) { @@ -13,6 +14,9 @@ const RING_SIZE = 60 * 5; // 5 Minutes export class CircuitRetransmitGraph extends DashboardGraph { constructor(id, title) { super(id); + if (this.dom && this.dom.classList) { + this.dom.classList.remove("muted"); + } this.title = title; this.ringbuffer = new RingBuffer(RING_SIZE); @@ -46,9 +50,6 @@ export class CircuitRetransmitGraph extends DashboardGraph { } } ], - textStyle: { - color: '#aaa' - }, }, xAxis: { type: 'category', @@ -89,9 +90,6 @@ export class CircuitRetransmitGraph extends DashboardGraph { trigger: 'axis', axisPointer: { type: 'cross', - label: { - backgroundColor: '#6a7985' - } }, formatter: (params) => { if (!params || params.length === 0) return ''; @@ -105,6 +103,7 @@ export class CircuitRetransmitGraph extends DashboardGraph { } }, } + applyCircuitDeviceChartTheme(this.option, { hasLegend: true }); this.option && this.chart.setOption(this.option); } @@ -114,6 +113,7 @@ export class CircuitRetransmitGraph extends DashboardGraph { this.option.legend.data[1].itemStyle.color = window.graphPalette[1]; this.option.series[0].lineStyle.color = window.graphPalette[0]; this.option.series[1].lineStyle.color = window.graphPalette[1]; + applyCircuitDeviceChartTheme(this.option, { hasLegend: true }); this.chart.setOption(this.option); } @@ -171,4 +171,4 @@ class RingBuffer { const bufferIdx = (this.head + idx) % this.size; return this.data[bufferIdx][2]; } -} \ No newline at end of file +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/circuit_throughput_graph.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/circuit_throughput_graph.js index e570cf01..5d4ada12 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/circuit_throughput_graph.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/circuit_throughput_graph.js @@ -1,5 +1,6 @@ import {DashboardGraph} from "./dashboard_graph"; import {scaleNumber} from "../lq_js_common/helpers/scaling"; +import {applyCircuitDeviceChartTheme} from "./circuit_device_chart_theme"; // Helper to format time as HH:MM:SS function formatTime(ts) { @@ -13,6 +14,9 @@ const RING_SIZE = 60 * 5; // 5 Minutes export class CircuitTotalGraph extends DashboardGraph { constructor(id, title) { super(id); + if (this.dom && this.dom.classList) { + this.dom.classList.remove("muted"); + } this.title = title; this.ringbuffer = new RingBuffer(RING_SIZE); @@ -46,9 +50,6 @@ export class CircuitTotalGraph extends DashboardGraph { } } ], - textStyle: { - color: '#aaa' - }, }, xAxis: { type: 'category', @@ -86,9 +87,6 @@ export class CircuitTotalGraph extends DashboardGraph { trigger: 'axis', axisPointer: { type: 'cross', - label: { - backgroundColor: '#6a7985' - } }, formatter: function(params) { console.log(params); @@ -103,6 +101,7 @@ export class CircuitTotalGraph extends DashboardGraph { } }, } + applyCircuitDeviceChartTheme(this.option, { hasLegend: true }); this.option && this.chart.setOption(this.option); } @@ -112,6 +111,7 @@ export class CircuitTotalGraph extends DashboardGraph { this.option.legend.data[1].itemStyle.color = window.graphPalette[1]; this.option.series[0].lineStyle.color = window.graphPalette[0]; this.option.series[1].lineStyle.color = window.graphPalette[1]; + applyCircuitDeviceChartTheme(this.option, { hasLegend: true }); this.chart.setOption(this.option); } @@ -185,4 +185,4 @@ class RingBuffer { let logicalIdx = (this.head + idx) % this.size; return this.data[logicalIdx][2]; } -} \ No newline at end of file +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/dashboard_graph.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/dashboard_graph.js index 1db9be26..0d9d42b0 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/dashboard_graph.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/dashboard_graph.js @@ -1,5 +1,41 @@ import {isDarkMode} from "../helpers/dark_mode"; +function emitDashboardRenderHook(entry) { + if (typeof window !== "undefined" && typeof window.__lqosDashboardRenderHook === "function") { + window.__lqosDashboardRenderHook(entry); + } +} + +function domStateSnapshot(dom) { + if (!dom) { + return { + hasDom: false, + }; + } + const rect = typeof dom.getBoundingClientRect === "function" + ? dom.getBoundingClientRect() + : { width: 0, height: 0 }; + const computed = typeof window !== "undefined" && window.getComputedStyle + ? window.getComputedStyle(dom) + : null; + return { + hasDom: true, + clientWidth: dom.clientWidth, + clientHeight: dom.clientHeight, + rectWidth: rect.width, + rectHeight: rect.height, + isConnected: typeof dom.isConnected === "boolean" ? dom.isConnected : true, + offsetParentNull: dom.offsetParent === null, + display: computed ? computed.display : "", + visibility: computed ? computed.visibility : "", + }; +} + +function domCanPaint(dom) { + const state = domStateSnapshot(dom); + return state.hasDom && state.clientWidth > 0 && state.clientHeight > 0 && state.display !== "none"; +} + function parsePixels(value) { if (!value || typeof value !== "string" || !value.endsWith("px")) { return 0; @@ -55,9 +91,17 @@ export class DashboardGraph { constructor(id) { this.id = id; this.dom = document.getElementById(id); + this.pendingSetOptionArgs = null; + this.pendingSetOptionRaf = null; + this.pendingSetOptionAttempts = 0; if (!this.dom) { throw new Error(`DashboardGraph: missing DOM element '${id}'`); } + emitDashboardRenderHook({ + graphId: this.id, + stage: "graph-constructor-start", + ...domStateSnapshot(this.dom), + }); this.dom.classList.add("muted"); // Some charts are created while Bootstrap/tab layout is still settling. // Ensure ECharts never initializes against a 0x0 container. @@ -100,6 +144,10 @@ export class DashboardGraph { ]; this.chart = echarts.init(this.dom, 'vintage'); } + this.#wrapChartMethod("setOption"); + this.#wrapChartMethod("resize"); + this.#wrapChartMethod("showLoading"); + this.#wrapChartMethod("hideLoading"); this.chart.showLoading(); this.option = {}; window.requestAnimationFrame(() => { @@ -108,6 +156,11 @@ export class DashboardGraph { this.chart.resize(); } }); + emitDashboardRenderHook({ + graphId: this.id, + stage: "graph-constructor-complete", + ...domStateSnapshot(this.dom), + }); // Apply to the global list of graphs if (window.graphList === undefined) { @@ -126,6 +179,97 @@ export class DashboardGraph { window.graphList.push(this); } + #scheduleDeferredSetOptionRetry() { + if (this.pendingSetOptionRaf !== null || this.pendingSetOptionArgs === null) { + return; + } + this.pendingSetOptionRaf = window.requestAnimationFrame(() => { + this.pendingSetOptionRaf = null; + if (this.pendingSetOptionArgs === null) { + return; + } + if (domCanPaint(this.dom)) { + this.#flushPendingSetOption("raf"); + return; + } + this.pendingSetOptionAttempts += 1; + if (this.pendingSetOptionAttempts >= 10) { + emitDashboardRenderHook({ + graphId: this.id, + stage: "chart-setOption-drop-deferred", + attempts: this.pendingSetOptionAttempts, + ...domStateSnapshot(this.dom), + }); + this.pendingSetOptionArgs = null; + this.pendingSetOptionAttempts = 0; + return; + } + this.#scheduleDeferredSetOptionRetry(); + }); + } + + #flushPendingSetOption(reason) { + if (this.pendingSetOptionArgs === null) { + return; + } + const args = this.pendingSetOptionArgs; + this.pendingSetOptionArgs = null; + this.pendingSetOptionAttempts = 0; + emitDashboardRenderHook({ + graphId: this.id, + stage: "chart-setOption-flush-deferred", + reason, + ...domStateSnapshot(this.dom), + }); + this.chart.setOption(...args); + } + + #wrapChartMethod(methodName) { + if (!this.chart || typeof this.chart[methodName] !== "function") { + return; + } + const original = this.chart[methodName].bind(this.chart); + this.chart[methodName] = (...args) => { + emitDashboardRenderHook({ + graphId: this.id, + stage: `chart-${methodName}-start`, + argsCount: args.length, + ...domStateSnapshot(this.dom), + }); + try { + if (methodName === "setOption" && !domCanPaint(this.dom)) { + this.pendingSetOptionArgs = args; + this.pendingSetOptionAttempts = 0; + emitDashboardRenderHook({ + graphId: this.id, + stage: "chart-setOption-deferred", + ...domStateSnapshot(this.dom), + }); + this.#scheduleDeferredSetOptionRetry(); + return undefined; + } + const result = original(...args); + emitDashboardRenderHook({ + graphId: this.id, + stage: `chart-${methodName}-ok`, + ...domStateSnapshot(this.dom), + }); + if (methodName === "resize" && domCanPaint(this.dom)) { + this.#flushPendingSetOption("resize"); + } + return result; + } catch (err) { + emitDashboardRenderHook({ + graphId: this.id, + stage: `chart-${methodName}-error`, + error: err && err.message ? err.message : String(err), + ...domStateSnapshot(this.dom), + }); + throw err; + } + }; + } + onThemeChange() { // Override this if you have to } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/device_ping_graph.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/device_ping_graph.js index f021e498..bd1ee936 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/device_ping_graph.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/device_ping_graph.js @@ -1,10 +1,14 @@ import {DashboardGraph} from "./dashboard_graph"; import {lerpGreenToRedViaOrange} from "../helpers/scaling"; import {toNumber} from "../lq_js_common/helpers/scaling"; +import {applyCircuitDeviceChartTheme} from "./circuit_device_chart_theme"; export class DevicePingHistogram extends DashboardGraph { constructor(id) { super(id); + if (this.dom && this.dom.classList) { + this.dom.classList.remove("muted"); + } let d = []; let axis = []; for (let i=0; i<20; i++) { @@ -32,10 +36,17 @@ export class DevicePingHistogram extends DashboardGraph { type: 'bar', } }; + applyCircuitDeviceChartTheme(this.option); this.option && this.chart.setOption(this.option); this.chart.hideLoading(); } + onThemeChange() { + super.onThemeChange(); + applyCircuitDeviceChartTheme(this.option); + this.chart.setOption(this.option); + } + update(ping_time_nanos) { this.chart.hideLoading(); let ping_time_ms = toNumber(ping_time_nanos, 0) / 1000000; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/flow_sankey.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/flow_sankey.js index 255317c5..ca7f4a07 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/flow_sankey.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/flow_sankey.js @@ -3,21 +3,16 @@ import {toNumber} from "../lq_js_common/helpers/scaling"; import {isRedacted} from "../helpers/redact"; const SANKEY_RECENT_FLOW_WINDOW_NANOS = 10_000_000_000; -const SANKEY_TOP_FLOW_LIMIT = 20; function totalFlowRate(flow) { - return ( - toNumber(flow?.[1]?.rate_estimate_bps?.down, 0) + - toNumber(flow?.[1]?.rate_estimate_bps?.up, 0) - ); + return toNumber(flow?.down_bps, 0) + toNumber(flow?.up_bps, 0); } function renderableSankeyFlows(flowMsg) { const flows = Array.isArray(flowMsg?.flows) ? flowMsg.flows : []; return flows - .filter((flow) => toNumber(flow?.[0]?.last_seen_nanos, 0) <= SANKEY_RECENT_FLOW_WINDOW_NANOS) - .sort((a, b) => totalFlowRate(b) - totalFlowRate(a)) - .slice(0, SANKEY_TOP_FLOW_LIMIT); + .filter((flow) => toNumber(flow?.last_seen_nanos, 0) <= SANKEY_RECENT_FLOW_WINDOW_NANOS) + .sort((a, b) => totalFlowRate(b) - totalFlowRate(a)); } export function getRenderableSankeyFlowCount(flowMsg) { @@ -53,11 +48,11 @@ export class FlowsSankey extends DashboardGraph { renderableSankeyFlows(flows).forEach((flow) => { flowCount++; - let localDevice = flow[0].device_name; - let proto = flow[0].protocol_name; - let asn = "ASN: " + flow[2].asn_id; - if (flow[0].asn_name !== "") asn += " " + flow[0].asn_name; - let remoteDevice = flow[0].remote_ip; + let localDevice = flow.device_name; + let proto = flow.protocol_name; + let asn = "ASN: " + flow.asn_id; + if (flow.asn_name !== "") asn += " " + flow.asn_name; + let remoteDevice = flow.remote_ip; // Ensure all members are present. The arrays hold links to subsequent // columns. @@ -75,9 +70,7 @@ export class FlowsSankey extends DashboardGraph { } // Accumulate traffic. - let currentRate = - toNumber(flow[1].rate_estimate_bps.down, 0) + - toNumber(flow[1].rate_estimate_bps.up, 0); + let currentRate = toNumber(flow.down_bps, 0) + toNumber(flow.up_bps, 0); if (localDevices[localDevice][asn] === undefined) { localDevices[localDevice][asn] = currentRate; } else { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/queue_stats_total_graph.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/queue_stats_total_graph.js index 552ca20d..d5071cb8 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/queue_stats_total_graph.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/queue_stats_total_graph.js @@ -120,7 +120,27 @@ export class QueueStatsTotalGraph extends DashboardGraph { update(marks, drops) { this.chart.hideLoading(); this.ringbuffer.push(marks, drops); - + this.#syncChart(); + } + + updateMany(samples) { + this.chart.hideLoading(); + if (!Array.isArray(samples) || samples.length === 0) { + return; + } + + for (let i = 0; i < samples.length; i++) { + const sample = samples[i]; + if (!sample) { + continue; + } + this.ringbuffer.push(sample.marks, sample.drops); + } + + this.#syncChart(); + } + + #syncChart() { let { series, times } = this.ringbuffer.seriesWithTimestamps(); for (let i=0; i= 10 ? 0 : 1)} ms`; } -function getWaveformTheme() { +function hexToRgba(hex, alpha) { + if (typeof hex !== "string" || !hex.startsWith("#")) { + return hex; + } + const normalized = hex.length === 4 + ? `#${hex[1]}${hex[1]}${hex[2]}${hex[2]}${hex[3]}${hex[3]}` + : hex; + const red = parseInt(normalized.slice(1, 3), 16); + const green = parseInt(normalized.slice(3, 5), 16); + const blue = parseInt(normalized.slice(5, 7), 16); + return `rgba(${red}, ${green}, ${blue}, ${alpha})`; +} + +function throughputPaletteColor(direction, fallback) { + const paletteIndex = direction === "up" ? 1 : 0; + return window.graphPalette?.[paletteIndex] || fallback; +} + +function getWaveformTheme(direction = "down") { const isDark = document.documentElement.getAttribute("data-bs-theme") !== "light"; + const throughputColor = throughputPaletteColor(direction, isDark ? "#4992ff" : "#d87c7c"); + if (isDark) { return { axisText: "rgba(216, 226, 244, 0.62)", @@ -33,14 +53,14 @@ function getWaveformTheme() { axisLine: "rgba(216, 226, 244, 0.28)", axisTick: "rgba(216, 226, 244, 0.2)", splitLine: "rgba(216, 226, 244, 0.045)", - throughputLine: "#32d3bd", - throughputGlow: "rgba(50, 211, 189, 0.34)", - throughputAreaTop: "rgba(50, 211, 189, 0.48)", - throughputAreaMid: "rgba(42, 156, 145, 0.3)", + throughputLine: throughputColor, + throughputGlow: hexToRgba(throughputColor, 0.34), + throughputAreaTop: hexToRgba(throughputColor, 0.48), + throughputAreaMid: hexToRgba(throughputColor, 0.28), throughputAreaBottom: "rgba(9, 20, 31, 0.03)", - rttLine: "#8fb0ff", - rttGlow: "rgba(143, 176, 255, 0.3)", - rttAreaTop: "rgba(143, 176, 255, 0.18)", + rttLine: "#b7a5ff", + rttGlow: "rgba(183, 165, 255, 0.32)", + rttAreaTop: "rgba(183, 165, 255, 0.18)", rttAreaBottom: "rgba(16, 22, 34, 0.01)", rttBandGood: "rgba(42, 168, 108, 0.08)", rttBandWarn: "rgba(228, 154, 44, 0.07)", @@ -58,14 +78,14 @@ function getWaveformTheme() { axisLine: "rgba(77, 94, 118, 0.28)", axisTick: "rgba(77, 94, 118, 0.18)", splitLine: "rgba(77, 94, 118, 0.08)", - throughputLine: "#129a87", - throughputGlow: "rgba(18, 154, 135, 0.18)", - throughputAreaTop: "rgba(18, 154, 135, 0.26)", - throughputAreaMid: "rgba(18, 154, 135, 0.14)", + throughputLine: throughputColor, + throughputGlow: hexToRgba(throughputColor, 0.2), + throughputAreaTop: hexToRgba(throughputColor, 0.24), + throughputAreaMid: hexToRgba(throughputColor, 0.13), throughputAreaBottom: "rgba(255, 255, 255, 0.02)", - rttLine: "#4f79d9", - rttGlow: "rgba(79, 121, 217, 0.18)", - rttAreaTop: "rgba(79, 121, 217, 0.1)", + rttLine: "#6f63cf", + rttGlow: "rgba(111, 99, 207, 0.2)", + rttAreaTop: "rgba(111, 99, 207, 0.1)", rttAreaBottom: "rgba(255, 255, 255, 0.01)", rttBandGood: "rgba(46, 163, 111, 0.06)", rttBandWarn: "rgba(214, 148, 40, 0.06)", @@ -191,15 +211,17 @@ function ceilingSeriesData(samples, direction, windowStart, windowEnd) { const ceilingBps = toNumber(ceilingData[i][1], 0); const throughputBps = toNumber(throughputData[i][1], 0); const atCeiling = ceilingBps > 0 && throughputBps >= (ceilingBps * 0.95); - const activeTimestamp = i > 0 ? timestamp : windowStart; + const activeValue = atCeiling ? ceilingBps : null; if (base.length === 0) { base.push([timestamp, ceilingBps]); - active.push([activeTimestamp, atCeiling ? ceilingBps : null]); + active.push([timestamp, activeValue]); + } else { + active.push([timestamp, activeValue]); } base.push([nextTimestamp, ceilingBps]); - active.push([nextTimestamp, atCeiling ? ceilingBps : null]); + active.push([nextTimestamp, activeValue]); } return { base, active }; } @@ -249,8 +271,8 @@ export class QueuingActivityWaveform extends DashboardGraph { if (this.dom && this.dom.classList) { this.dom.classList.remove("muted"); } - this.colors = getWaveformTheme(); this.direction = "down"; + this.colors = getWaveformTheme(this.direction); this.samples = []; this.sampleClockBase = null; this.lastSampleTimestamp = null; @@ -518,6 +540,46 @@ export class QueuingActivityWaveform extends DashboardGraph { this.scheduleRenderLoop(); } + applyThemeColors() { + this.option.xAxis[0].axisLine.lineStyle.color = this.colors.axisLine; + this.option.xAxis[1].axisLine.lineStyle.color = this.colors.axisLine; + this.option.xAxis[1].axisTick.lineStyle.color = this.colors.axisTick; + this.option.xAxis[1].axisLabel.color = this.colors.axisText; + this.option.yAxis[0].nameTextStyle.color = this.colors.axisName; + this.option.yAxis[0].axisLine.lineStyle.color = this.colors.axisLine; + this.option.yAxis[0].splitLine.lineStyle.color = this.colors.splitLine; + this.option.yAxis[0].axisLabel.color = this.colors.axisText; + this.option.yAxis[1].nameTextStyle.color = this.colors.axisName; + this.option.yAxis[1].axisLine.lineStyle.color = this.colors.axisLine; + this.option.yAxis[1].axisTick.lineStyle.color = this.colors.axisTick; + this.option.yAxis[1].splitLine.lineStyle.color = this.colors.splitLine; + this.option.yAxis[1].axisLabel.color = this.colors.axisText; + this.option.series[0].lineStyle.color = this.colors.throughputLine; + this.option.series[0].lineStyle.shadowColor = this.colors.throughputGlow; + this.option.series[0].areaStyle.color = new echarts.graphic.LinearGradient(0, 0, 0, 1, [ + { offset: 0, color: this.colors.throughputAreaTop }, + { offset: 0.45, color: this.colors.throughputAreaMid }, + { offset: 1, color: this.colors.throughputAreaBottom }, + ]); + this.option.series[1].lineStyle.color = this.colors.ceilingInactive; + this.option.series[1].lineStyle.shadowColor = this.colors.ceilingInactiveGlow; + this.option.series[2].lineStyle.color = this.colors.ceilingActive; + this.option.series[2].lineStyle.shadowColor = this.colors.ceilingActiveGlow; + this.option.series[3].lineStyle.color = this.colors.ceilingActive; + this.option.series[3].lineStyle.shadowColor = this.colors.ceilingActiveGlow; + this.option.series[4].lineStyle.color = this.colors.rttLine; + this.option.series[4].lineStyle.shadowColor = this.colors.rttGlow; + this.option.series[4].areaStyle.color = new echarts.graphic.LinearGradient(0, 0, 0, 1, [ + { offset: 0, color: this.colors.rttAreaTop }, + { offset: 1, color: this.colors.rttAreaBottom }, + ]); + this.option.series[4].markArea.data = rttMarkAreas( + this.colors, + this.rttThresholds, + this.option.yAxis[1].max || this.rttThresholds.red_ms, + ); + } + scheduleRenderLoop() { const step = (timestamp) => { if (!this.chart) { @@ -552,8 +614,11 @@ export class QueuingActivityWaveform extends DashboardGraph { setDirection(direction) { this.direction = direction === "up" ? "up" : "down"; + this.colors = getWaveformTheme(this.direction); this.cachedSeries.direction = this.direction; this.rebuildCachedSeries(); + this.applyThemeColors(); + this.chart.setOption(this.option); this.render(); } @@ -737,44 +802,8 @@ export class QueuingActivityWaveform extends DashboardGraph { } onThemeChange() { - this.colors = getWaveformTheme(); - this.option.xAxis[0].axisLine.lineStyle.color = this.colors.axisLine; - this.option.xAxis[1].axisLine.lineStyle.color = this.colors.axisLine; - this.option.xAxis[1].axisTick.lineStyle.color = this.colors.axisTick; - this.option.xAxis[1].axisLabel.color = this.colors.axisText; - this.option.yAxis[0].nameTextStyle.color = this.colors.axisName; - this.option.yAxis[0].axisLine.lineStyle.color = this.colors.axisLine; - this.option.yAxis[0].splitLine.lineStyle.color = this.colors.splitLine; - this.option.yAxis[0].axisLabel.color = this.colors.axisText; - this.option.yAxis[1].nameTextStyle.color = this.colors.axisName; - this.option.yAxis[1].axisLine.lineStyle.color = this.colors.axisLine; - this.option.yAxis[1].axisTick.lineStyle.color = this.colors.axisTick; - this.option.yAxis[1].splitLine.lineStyle.color = this.colors.splitLine; - this.option.yAxis[1].axisLabel.color = this.colors.axisText; - this.option.series[0].lineStyle.color = this.colors.throughputLine; - this.option.series[0].lineStyle.shadowColor = this.colors.throughputGlow; - this.option.series[0].areaStyle.color = new echarts.graphic.LinearGradient(0, 0, 0, 1, [ - { offset: 0, color: this.colors.throughputAreaTop }, - { offset: 0.45, color: this.colors.throughputAreaMid }, - { offset: 1, color: this.colors.throughputAreaBottom }, - ]); - this.option.series[1].lineStyle.color = this.colors.ceilingInactive; - this.option.series[1].lineStyle.shadowColor = this.colors.ceilingInactiveGlow; - this.option.series[2].lineStyle.color = this.colors.ceilingActive; - this.option.series[2].lineStyle.shadowColor = this.colors.ceilingActiveGlow; - this.option.series[3].lineStyle.color = this.colors.ceilingActive; - this.option.series[3].lineStyle.shadowColor = this.colors.ceilingActiveGlow; - this.option.series[4].lineStyle.color = this.colors.rttLine; - this.option.series[4].lineStyle.shadowColor = this.colors.rttGlow; - this.option.series[4].areaStyle.color = new echarts.graphic.LinearGradient(0, 0, 0, 1, [ - { offset: 0, color: this.colors.rttAreaTop }, - { offset: 1, color: this.colors.rttAreaBottom }, - ]); - this.option.series[4].markArea.data = rttMarkAreas( - this.colors, - this.rttThresholds, - this.option.yAxis[1].max || this.rttThresholds.red_ms, - ); + this.colors = getWaveformTheme(this.direction); + this.applyThemeColors(); this.chart.setOption(this.option); this.render(); } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/rtt_histo.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/rtt_histo.js index 8bb90993..17dd5bf7 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/rtt_histo.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/rtt_histo.js @@ -3,6 +3,27 @@ import {lerpGreenToRedViaOrange} from "../helpers/scaling"; export const N_ITEMS = 50; +function computeYAxisMax(maxPercent) { + if (!Number.isFinite(maxPercent) || maxPercent <= 0) { + return 1; + } + + const padded = maxPercent * 1.1; + if (padded <= 5) { + return 5; + } + if (padded <= 10) { + return 10; + } + if (padded <= 25) { + return 25; + } + if (padded <= 50) { + return 50; + } + return Math.min(100, Math.ceil(padded / 10) * 10); +} + export class RttHistogram extends DashboardGraph { constructor(id) { super(id); @@ -66,15 +87,17 @@ export class RttHistogram extends DashboardGraph { let sum = 0; this.rawCounts = [...rtt]; // Store raw counts rtt.forEach((v) => { - if (v > this.max) { - this.max = v; - } sum += v; - }) + }); + let maxPercent = 0; for (let i=0; i 0 ? (rtt[i] / sum) * 100 : 0; + if (pct > maxPercent) { + maxPercent = pct; + } + this.option.series.data[i].value = pct; } + this.max = computeYAxisMax(maxPercent); this.chart.setOption(this.option); } } - diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/top_n_sankey.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/top_n_sankey.js index 7568555c..44618f93 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/top_n_sankey.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/top_n_sankey.js @@ -1,5 +1,9 @@ import {DashboardGraph} from "./dashboard_graph"; -import {lerpColor, lerpGreenToRedViaOrange} from "../helpers/scaling"; +import { + lerpColor, + lerpGreenToRedViaOrange, + retransmitFractionFromSample, +} from "../helpers/scaling"; import {isColorBlindMode} from "../helpers/colorblind"; import {colorByRttMs} from "../helpers/color_scales"; import {toNumber} from "../lq_js_common/helpers/scaling"; @@ -92,7 +96,11 @@ export class TopNSankey extends DashboardGraph { let rtt = toNumber(r.median_tcp_rtt, 0); let rttColor = colorByRttMs(rtt); - let percentRxmit = Math.min(100, toNumber(r.tcp_retransmits[0], 0) + toNumber(r.tcp_retransmits[1], 0)) / 100; + const percentRxmit = Math.min( + 1, + retransmitFractionFromSample(r.tcp_retransmit_sample?.down) + + retransmitFractionFromSample(r.tcp_retransmit_sample?.up) + ); let rxmitColor = isColorBlindMode() ? lerpViridis(percentRxmit) : lerpColor([0, 255, 0], [255, 0, 0], percentRxmit); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/graphs/windowed_latency_histogram.js b/src/rust/lqosd/src/node_manager/js_build/src/graphs/windowed_latency_histogram.js index 4ae3ce3b..2fe07e19 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/graphs/windowed_latency_histogram.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/graphs/windowed_latency_histogram.js @@ -1,10 +1,14 @@ import {DashboardGraph} from "./dashboard_graph"; import {lerpGreenToRedViaOrange} from "../helpers/scaling"; import {toNumber} from "../lq_js_common/helpers/scaling"; +import {applyCircuitDeviceChartTheme} from "./circuit_device_chart_theme"; export class WindowedLatencyHistogram extends DashboardGraph { constructor(id, title = "Latency Histogram", windowMs = 300000) { super(id); + if (this.dom && this.dom.classList) { + this.dom.classList.remove("muted"); + } this.windowMs = windowMs; this.samples = []; this.sampleHead = 0; @@ -44,10 +48,17 @@ export class WindowedLatencyHistogram extends DashboardGraph { }, }; + applyCircuitDeviceChartTheme(this.option); this.option && this.chart.setOption(this.option); this.chart.hideLoading(); } + onThemeChange() { + super.onThemeChange(); + applyCircuitDeviceChartTheme(this.option); + this.chart.setOption(this.option); + } + updateMs(pingOrRttMs) { const now = Date.now(); const ms = toNumber(pingOrRttMs, 0); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/helpers/builders.js b/src/rust/lqosd/src/node_manager/js_build/src/helpers/builders.js index 602c1366..b73bb3cb 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/helpers/builders.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/helpers/builders.js @@ -1,6 +1,12 @@ -import {formatRetransmit, formatRtt, formatThroughput} from "./scaling"; +import { + formatRetransmitFraction, + formatRtt, + formatThroughput, + retransmitFractionFromSample, +} from "./scaling"; import {scaleNanos} from "../lq_js_common/helpers/scaling"; import {redactCell} from "./redact"; +import {disposeTooltipsWithin, enableTooltipsWithin} from "../lq_js_common/helpers/tooltips"; export function heading5Icon(icon, text) { let h5 = document.createElement("h5"); @@ -51,17 +57,14 @@ export function clearDashDiv(id, target) { } export function clearDiv(target, targetLength=0) { + disposeTooltipsWithin(target); while (target.children.length > targetLength) { target.removeChild(target.lastChild); } } export function enableTooltips() { - // Tooltips everywhere! - let tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')) - let tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { - return new bootstrap.Tooltip(tooltipTriggerEl) - }) + enableTooltipsWithin(document); } let pendingTooltips = []; @@ -73,7 +76,7 @@ export function tooltipsNextFrame(id) { pendingTooltips.forEach((id) => { let tooltipTriggerEl = document.getElementById(id); if (tooltipTriggerEl !== null) { - new bootstrap.Tooltip(tooltipTriggerEl); + enableTooltipsWithin(tooltipTriggerEl); } }); pendingTooltips = []; @@ -85,7 +88,9 @@ export function clientTableHeader() { let thead = document.createElement("thead"); thead.appendChild(theading("Circuit")); thead.appendChild(theading("Device")); - thead.appendChild(theading("Plan (Mbps)")); + const planHeading = theading("Plan (Mbps)"); + planHeading.classList.add("lqos-tree-plan-column"); + thead.appendChild(planHeading); thead.appendChild(theading("Parent")); thead.appendChild(theading("IP")); thead.appendChild(theading("Last Seen")); @@ -165,11 +170,15 @@ export function topNTableRow(r) { row.append(rtt); let tcp_xmit_down = document.createElement("td"); - tcp_xmit_down.innerHTML = formatRetransmit(r.tcp_retransmits[0]); + tcp_xmit_down.innerHTML = formatRetransmitFraction( + retransmitFractionFromSample(r.tcp_retransmit_sample?.down) + ); row.append(tcp_xmit_down); let tcp_xmit_up = document.createElement("td"); - tcp_xmit_up.innerHTML = formatRetransmit(r.tcp_retransmits[1]); + tcp_xmit_up.innerHTML = formatRetransmitFraction( + retransmitFractionFromSample(r.tcp_retransmit_sample?.up) + ); row.append(tcp_xmit_up); return row; @@ -177,13 +186,13 @@ export function topNTableRow(r) { export function TopNTableFromMsgData(circuits) { let t = document.createElement("table"); - t.classList.add("dash-table", "lqos-table", "lqos-table-compact"); + t.classList.add("dash-table", "lqos-table", "lqos-table-compact", "lqos-topn-plain"); t.appendChild(topNTableHeader()); let tbody = document.createElement("tbody"); circuits.forEach((r) => { - t.appendChild(topNTableRow(r)); + tbody.appendChild(topNTableRow(r)); }); t.appendChild(tbody); return t; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/helpers/scaling.js b/src/rust/lqosd/src/node_manager/js_build/src/helpers/scaling.js index ab210830..b3f95f86 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/helpers/scaling.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/helpers/scaling.js @@ -119,20 +119,44 @@ export function formatRtt(rtt) { return blob; } -export function formatRetransmit(retransmits) { +export function retransmitFractionFromSample(sample) { + const retransmits = toNumber(sample?.retransmits, 0); + const packets = toNumber(sample?.packets, 0); + if (packets <= 0) { + return 0; + } + return retransmits / packets; +} + +export function retransmitPercentFromSample(sample) { + return retransmitFractionFromSample(sample) * 100.0; +} + +export function formatRetransmitFraction(fraction) { + const percent = toNumber(fraction, 0) * 100.0; + const clampedPercent = Math.min(100, Math.max(0, percent)); + const color = lerpColor([0, 255, 0], [255, 0, 0], clampedPercent / 100.0); + return "■" + percent.toFixed(1) + "%"; +} + +export function formatRetransmitPercent(percent) { + percent = toNumber(percent, 0); + const clampedPercent = Math.min(100, Math.max(0, percent)); + const color = lerpColor([0, 255, 0], [255, 0, 0], clampedPercent / 100.0); + return "■" + percent.toFixed(1) + "%"; +} + +export function formatRetransmitCount(retransmits) { retransmits = toNumber(retransmits, 0); - retransmits *= 100; - let percent = Math.min(100, retransmits); - let color = lerpColor([0, 255, 0], [255, 0, 0], percent); - return "■" + retransmits.toFixed(1) + "%"; + return "" + retransmits.toFixed(0) + ""; +} + +export function formatRetransmit(retransmits) { + return formatRetransmitFraction(retransmits); } export function formatRetransmitRaw(retransmits) { - retransmits = toNumber(retransmits, 0); - retransmits *= 100; - let percent = Math.min(100, retransmits); - let color = lerpColor([0, 255, 0], [255, 0, 0], percent); - return "■" + retransmits.toFixed(0) + ""; + return formatRetransmitCount(retransmits); } export function formatCakeStat(n) { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/index.js b/src/rust/lqosd/src/node_manager/js_build/src/index.js index 03e58ea6..8672d76c 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/index.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/index.js @@ -1,5 +1,4 @@ import {Dashboard} from "./lq_js_common/dashboard/dashboard"; -import {checkForUpgrades} from "./toasts/version_check"; import {sponsorTag} from "./toasts/sponsor_us"; import {globalWarningToasts} from "./toasts/global_warnings"; import {showTimeControls} from "./components/timescale"; @@ -10,18 +9,20 @@ const defaultLayout = { activeTab: 0, tabs: [ { + id: "executive-summary", name: "Executive Summary", dashlets: [ { tag: "executiveSnapshot", size: 12 }, - { tag: "executiveHelpers", size: 12 }, { tag: "executiveGlobalHeatmap", size: 12 }, { tag: "executiveHeatmapDownload", size: 6 }, { tag: "executiveHeatmapUpload", size: 6 }, { tag: "executiveHeatmapRtt", size: 6 }, - { tag: "executiveHeatmapRetrans", size: 6 } + { tag: "executiveHeatmapRetrans", size: 6 }, + { tag: "executiveHelpers", size: 12 } ] }, { + id: "overview", name: "Overview", dashlets: [ { tag: "throughputRing", size: 12 }, @@ -34,18 +35,20 @@ const defaultLayout = { ] }, { + id: "system-overview", name: "System Overview", dashlets: [ { tag: "throughputBps", size: 2 }, { tag: "shapedUnshaped", size: 2 }, - { tag: "throughputPps", size: 2 }, - { tag: "trackedFlowsCount", size: 2 }, - { tag: "rttHistogram", size: 2 }, - { tag: "cpu", size: 3 }, - { tag: "ram", size: 3 } + { tag: "ram", size: 2 }, + { tag: "cpu", size: 5 }, + { tag: "rttHistogram", size: 3 }, + { tag: "throughputPps", size: 4 }, + { tag: "trackedFlowsCount", size: 4 } ] }, { + id: "network", name: "Network", dashlets: [ { tag: "networkTreeSankey", size: 6 }, @@ -53,6 +56,7 @@ const defaultLayout = { ] }, { + id: "top-10", name: "Top 10", dashlets: [ { tag: "top10downloaders", size: 6 }, @@ -65,6 +69,7 @@ const defaultLayout = { ] }, { + id: "protocols-cake", name: "Protocols & Cake", dashlets: [ { tag: "etherProtocols", size: 6 }, @@ -73,19 +78,41 @@ const defaultLayout = { ] }, { + id: "treeguard", name: "TreeGuard", dashlets: [ - { tag: "treeguardControls", size: 6 }, - { tag: "treeguardStatus", size: 6 }, + { tag: "treeguardControlLoop", size: 4 }, + { tag: "treeguardStateMix", size: 4 }, + { tag: "treeguardDecisionImpact", size: 4 }, { tag: "treeguardActivity", size: 12 } ] + }, + { + id: "stormguard", + name: "StormGuard", + dashlets: [ + { tag: "stormguardSummary", size: 12 }, + { tag: "stormguardSiteList", size: 4 }, + { tag: "stormguardSiteDetail", size: 8 }, + { tag: "stormguardRecentActivity", size: 12 } + ] + }, + { + id: "bakery", + name: "Bakery", + dashlets: [ + { tag: "bakeryPipeline", size: 3 }, + { tag: "bakeryChangeMix", size: 3 }, + { tag: "bakeryStatus", size: 3 }, + { tag: "bakeryCapacity", size: 3 }, + { tag: "bakeryActivity", size: 12 } + ] } ] }; window.timeGraphs = []; showTimeControls("timescale"); -checkForUpgrades(); sponsorTag("toasts"); globalWarningToasts(); const dashboard = new Dashboard("dashboard", "mainDashboard", defaultLayout, widgetFactory, DashletMenu, false, ""); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/login.js b/src/rust/lqosd/src/node_manager/js_build/src/login.js index 3a523734..f1f35873 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/login.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/login.js @@ -1,5 +1,5 @@ $("#btnLogin").on('click', () => { - let username = $("#username").val(); + let username = ($("#username").val() || "").trim(); let password = $("#password").val(); if (username === "") { alert("You must enter a username"); @@ -21,12 +21,27 @@ $("#btnLogin").on('click', () => { data: JSON.stringify(login), contentType: 'application/json', beforeSend: () => { + $("#loginErrorText").html("Login failed. You can manage users via the lqusers CLI tool on the LibreQoS server."); $("#loginError").removeClass("show").addClass("d-none"); }, success: () => { window.location.href = "/index.html"; }, - error: () => { + error: (xhr) => { + const response = xhr && xhr.responseJSON ? xhr.responseJSON : {}; + const reason = response.reason || ""; + if (reason === "first_run_required") { + window.location.href = "/first-run.html"; + return; + } + + if (reason === "auth_corrupt") { + $("#loginErrorText").text(response.message || "The auth file is corrupt and must be repaired before anyone can log in."); + } else if (reason === "invalid_credentials") { + $("#loginErrorText").text(response.message || "Invalid username or password."); + } else { + $("#loginErrorText").html("Login failed. You can manage users via the lqusers CLI tool on the LibreQoS server."); + } $("#loginError").removeClass("d-none").addClass("show"); } }) diff --git a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/base_dashlet.js b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/base_dashlet.js index 5fe7e127..950e623d 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/base_dashlet.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/base_dashlet.js @@ -48,9 +48,39 @@ export class BaseDashlet { return []; } + keepAliveWhenHidden() { + return false; + } + + keepSubscribedWhenHidden() { + return this.keepAliveWhenHidden(); + } + + renderWhenHidden() { + return false; + } + onMessage(msg) { } + onBackgroundMessage(_msg) { + } + + flushBackgroundMessages() { + return false; + } + + traceRender(stage, details = {}) { + if (typeof window !== "undefined" && typeof window.__lqosDashboardRenderHook === "function") { + window.__lqosDashboardRenderHook({ + dashletId: this.id, + title: this.title(), + stage, + ...details, + }); + } + } + setupOnce(msg) { if (this.disposed) { return; @@ -59,11 +89,23 @@ export class BaseDashlet { this.setup(msg); } this.setupDone = true; + if (typeof window !== "undefined" && typeof window.__lqosDashboardDashletHook === "function") { + window.__lqosDashboardDashletHook({ + type: "setup", + dashletId: this.id, + title: this.title(), + setupDone: this.setupDone, + }); + } enableTooltips(); } setup() {} + onTabActivated() {} + + onTabDeactivated() {} + graphDivId() { return this.id + "_graph"; } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/dashboard.js b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/dashboard.js index 3b151c73..f7448c8b 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/dashboard.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/dashboard.js @@ -1,10 +1,11 @@ // Provides a generic dashboard system for use in LibreQoS and Insight import {DashboardLayout} from "./layout"; -import {subscribeWS} from "./ws"; import {get_ws_client} from "../../pubsub/ws"; import {heading5Icon} from "../helpers/content_builders"; import {openDashboardEditor} from "./dashboard_editor"; +const DIAGNOSTIC_CHANNELS = new Set(["Cpu", "Ram", "RttHistogram"]); + export class Dashboard { // Takes the target DIV in which to build the dashboard, // and the name of the dashboard for cookie and builder @@ -16,6 +17,7 @@ export class Dashboard { this.widgetFactory = widgetFactory; this.dashletMenu = dashletMenu; this.savedDashUrl = savedDashUrl; + this.hasCadence = hasCadence; // Set up the div this.parentDiv = document.getElementById(this.divName); @@ -43,34 +45,227 @@ export class Dashboard { // Data Wrangling this.channels = []; - this.subscription = null; + this.tabChannels = {}; + this.pendingImmediateDashlets = new Set(); + this.wsClient = get_ws_client(); + this.channelDisposers = new Map(); + this.subscribedChannels = new Set(); + this.lastMessages = new Map(); // Auto Refresh Handling this.paused = false; - let cadence = localStorage.getItem("dashCadence"); - if (cadence === null) { + if (!this.hasCadence) { this.cadence = 1; - localStorage.setItem("dashCadence", this.cadence.toString()); } else { - this.cadence = parseInt(cadence); + let cadence = localStorage.getItem("dashCadence"); + if (cadence === null) { + this.cadence = 1; + localStorage.setItem("dashCadence", this.cadence.toString()); + } else { + const parsedCadence = parseInt(cadence); + this.cadence = Number.isFinite(parsedCadence) && parsedCadence > 0 ? parsedCadence : 1; + } } this.tickCounter = 0; } + #urlDebugEnabled() { + try { + const params = new URLSearchParams(window.location.search || ""); + if (!params.has("debug")) { + return false; + } + const value = (params.get("debug") || "").trim().toLowerCase(); + return value === "" || value === "1" || value === "true" || value === "dashboard"; + } catch (_) { + return false; + } + } + + #debugEnabled() { + return localStorage.getItem("debugDashboard") === "1" || this.#urlDebugEnabled(); + } + + #debugDumpEnabled() { + return this.#urlDebugEnabled(); + } + + #debug(event, details = {}) { + if (!this.#debugEnabled()) { + return; + } + if (!window.__lqosDashboardTrace) { + window.__lqosDashboardTrace = []; + } + const entry = { + ts: new Date().toISOString(), + event, + ...details, + }; + window.__lqosDashboardTrace.push(entry); + if (window.__lqosDashboardTrace.length > 1000) { + window.__lqosDashboardTrace.shift(); + } + console.debug("[dashboard]", entry); + } + + #markDashletState(dashlet, patch = {}) { + if (!dashlet) { + return; + } + if (!dashlet.__dashboardDebugState) { + dashlet.__dashboardDebugState = {}; + } + Object.assign(dashlet.__dashboardDebugState, patch); + const el = document.getElementById(dashlet.id); + if (!el) { + return; + } + if (dashlet.__dashboardDebugState.lastSetupAt) { + el.dataset.debugLastSetupAt = dashlet.__dashboardDebugState.lastSetupAt; + } + if (dashlet.__dashboardDebugState.lastMessageAt) { + el.dataset.debugLastMessageAt = dashlet.__dashboardDebugState.lastMessageAt; + } + if (dashlet.__dashboardDebugState.lastEvent) { + el.dataset.debugLastEvent = dashlet.__dashboardDebugState.lastEvent; + } + if (dashlet.__dashboardDebugState.lastReplayAt) { + el.dataset.debugLastReplayAt = dashlet.__dashboardDebugState.lastReplayAt; + } + if (dashlet.__dashboardDebugState.lastReplayEvent) { + el.dataset.debugLastReplayEvent = dashlet.__dashboardDebugState.lastReplayEvent; + } + } + + #paneStateSnapshot(pane) { + if (!pane) { + return { + hasPane: false, + }; + } + const rect = typeof pane.getBoundingClientRect === "function" + ? pane.getBoundingClientRect() + : { width: 0, height: 0 }; + const computed = window.getComputedStyle ? window.getComputedStyle(pane) : null; + return { + hasPane: true, + paneId: pane.id, + paneClientWidth: pane.clientWidth, + paneClientHeight: pane.clientHeight, + paneRectWidth: rect.width, + paneRectHeight: rect.height, + paneDisplay: computed ? computed.display : "", + paneVisibility: computed ? computed.visibility : "", + }; + } + + #dashletDebugSnapshot(dashlet) { + let channels = []; + if (dashlet && typeof dashlet.subscribeTo === "function") { + const subscribed = dashlet.subscribeTo(); + channels = Array.isArray(subscribed) ? [...subscribed] : [subscribed]; + } + return { + id: dashlet?.id || "", + title: dashlet?.title || "", + tag: dashlet?.tag || "", + tabIndex: dashlet?.tabIndex, + keepSubscribedWhenHidden: !!dashlet?.keepSubscribedWhenHidden?.(), + renderWhenHidden: !!dashlet?.renderWhenHidden?.(), + subscribed: this.#dashletShouldStayActive(dashlet), + renderable: dashlet?.tabIndex === this.layout.activeTab || !!dashlet?.renderWhenHidden?.(), + channels, + debugState: dashlet?.__dashboardDebugState || {}, + }; + } + + #downloadDebugDump() { + const tabs = Array.from(document.querySelectorAll(`#${this.divName}_tabs .nav-link`)).map((tab, index) => ({ + index, + title: tab.textContent ? tab.textContent.trim() : "", + active: tab.classList.contains("active"), + })); + const panes = Array.from(document.querySelectorAll(`#${this.divName}_tab_content .tab-pane`)).map((pane, index) => ({ + index, + classes: Array.from(pane.classList), + ...this.#paneStateSnapshot(pane), + })); + const desiredChannels = this.wsClient?.desiredChannels instanceof Map + ? Array.from(this.wsClient.desiredChannels.keys()) + : []; + const subscribedChannels = Array.from(this.subscribedChannels); + const dump = { + generatedAt: new Date().toISOString(), + href: window.location.href, + userAgent: navigator.userAgent, + activeTab: this.layout.activeTab, + cadence: this.cadence, + paused: this.paused, + desiredChannels, + subscribedChannels, + lastMessageChannels: Array.from(this.lastMessages.keys()), + tabs, + panes, + dashlets: this.dashlets.map((dashlet) => this.#dashletDebugSnapshot(dashlet)), + dashboardTrace: window.__lqosDashboardTrace || [], + wsTrace: window.__lqosDashboardWsTrace || [], + }; + const stamp = dump.generatedAt.replace(/[:.]/g, "-"); + const fileName = `${this.cookieName || "dashboard"}-debug-${stamp}.json`; + const blob = new Blob([JSON.stringify(dump, null, 2)], {type: "application/json"}); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = fileName; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + setTimeout(() => URL.revokeObjectURL(url), 1000); + this.#debug("debug-dump-download", { + fileName, + subscribedChannels, + desiredChannels, + }); + } + build() { this.destroy(); + window.__lqosDashboardDashletHook = (entry) => { + this.#debug("dashlet-hook", entry); + }; + window.__lqosDashboardRenderHook = (entry) => { + this.#debug("render-hook", entry); + }; this.#filterWidgetList(); this.#buildTabUI(); this.#buildTabContents(); this.#buildChannelList(this.dashlets); - this.#webSocketSubscription(); + const renderableDashlets = this.#renderableDashlets(); + this.#debug("build", { + activeTab: this.layout.activeTab, + channels: this.channels, + renderableDashlets: renderableDashlets.map((d) => d.id), + }); + for (let i = 0; i < renderableDashlets.length; i++) { + renderableDashlets[i].setupOnce({}); + this.#markDashletState(renderableDashlets[i], { + lastSetupAt: new Date().toISOString(), + }); + } + this.#queueImmediateDashlets(renderableDashlets); + this.#syncWebSocketChannels(); + this.#replayCachedMessages(renderableDashlets); } destroy() { - if (this.subscription) { - this.subscription.dispose(); - this.subscription = null; + this.#disposeWebSocketChannels(); + if (window.__lqosDashboardDashletHook) { + delete window.__lqosDashboardDashletHook; + } + if (window.__lqosDashboardRenderHook) { + delete window.__lqosDashboardRenderHook; } for (let i = 0; i < this.dashlets.length; i++) { if (this.dashlets[i] && this.dashlets[i].dispose) { @@ -80,7 +275,10 @@ export class Dashboard { this.dashlets = []; this.tabDashlets = {}; this.channels = []; + this.tabChannels = {}; this.childIds = []; + this.pendingImmediateDashlets.clear(); + this.lastMessages.clear(); this.#clearRenderedDashboard(); } @@ -149,7 +347,12 @@ export class Dashboard { switchTab(tabIndex) { if (tabIndex === this.layout.activeTab) return; - + const previousTabIndex = this.layout.activeTab; + this.#debug("switch-tab", { + from: previousTabIndex, + to: tabIndex, + }); + // Update active tab in layout this.layout.activeTab = tabIndex; this.layout.save(this.layout); @@ -170,8 +373,26 @@ export class Dashboard { if (index === tabIndex) { pane.classList.add('active'); pane.style.display = 'flex'; - // Notify dashlets in this tab that they're now visible + this.#debug("tab-pane-visible", { + tabIndex, + ...this.#paneStateSnapshot(pane), + }); let tabDashlets = this.tabDashlets[index] || []; + const skipReplayDashletIds = new Set(); + tabDashlets.forEach((dashlet) => { + dashlet.setupOnce({}); + this.#markDashletState(dashlet, { + lastSetupAt: new Date().toISOString(), + }); + }); + tabDashlets.forEach((dashlet) => { + if (dashlet.flushBackgroundMessages && dashlet.flushBackgroundMessages()) { + skipReplayDashletIds.add(dashlet.id); + } + }); + this.#queueImmediateDashlets(tabDashlets); + this.#replayCachedMessages(tabDashlets, skipReplayDashletIds); + // Notify dashlets in this tab that they're now visible tabDashlets.forEach(dashlet => { if (dashlet.onTabActivated) { dashlet.onTabActivated(); @@ -180,6 +401,10 @@ export class Dashboard { // Resize all ECharts instances in the newly visible tab setTimeout(() => { + this.#debug("tab-pane-resize-pass", { + tabIndex, + ...this.#paneStateSnapshot(pane), + }); const charts = pane.querySelectorAll('.dashgraph, .dashgraphZoomed'); charts.forEach(chartDiv => { if (typeof echarts !== 'undefined') { @@ -195,6 +420,16 @@ export class Dashboard { pane.style.display = 'none'; } }); + + const previousDashlets = this.tabDashlets[previousTabIndex] || []; + previousDashlets.forEach((dashlet) => { + if (dashlet.onTabDeactivated) { + dashlet.onTabDeactivated(); + } + }); + + this.#buildChannelList(); + this.#syncWebSocketChannels(); } #filterWidgetList() { @@ -229,11 +464,133 @@ export class Dashboard { } } + #dashletShouldStayActive(dashlet) { + return dashlet.keepSubscribedWhenHidden() || dashlet.tabIndex === this.layout.activeTab; + } + + #subscribedDashlets() { + return this.dashlets.filter((dashlet) => this.#dashletShouldStayActive(dashlet)); + } + + #renderableDashlets() { + return this.dashlets.filter( + (dashlet) => dashlet.tabIndex === this.layout.activeTab || dashlet.renderWhenHidden(), + ); + } + + #backgroundDashlets() { + return this.#subscribedDashlets().filter( + (dashlet) => + dashlet.tabIndex !== this.layout.activeTab + && dashlet.keepSubscribedWhenHidden() + && !dashlet.renderWhenHidden(), + ); + } + + #queueImmediateDashlets(dashlets) { + if (!Array.isArray(dashlets) || this.cadence <= 1) { + return; + } + for (let i = 0; i < dashlets.length; i++) { + const dashlet = dashlets[i]; + if (dashlet && dashlet.canBeSlowedDown()) { + this.pendingImmediateDashlets.add(dashlet.id); + this.#debug("queue-immediate", { + dashletId: dashlet.id, + }); + } + } + } + + #debugInterestingChannels(context, channels) { + if (!Array.isArray(channels)) { + return; + } + const interesting = channels.filter((channel) => DIAGNOSTIC_CHANNELS.has(channel)); + if (interesting.length === 0) { + return; + } + this.#debug("interesting-channels", { + context, + channels: interesting, + activeTab: this.layout.activeTab, + renderableDashlets: this.#renderableDashlets().map((d) => d.id), + subscribedDashlets: this.#subscribedDashlets().map((d) => d.id), + }); + } + + #dashletSubscribedToMessage(dashlet, eventName) { + const channels = dashlet.subscribeTo(); + if (Array.isArray(channels)) { + return channels.includes(eventName); + } + return channels === eventName; + } + + #replayCachedMessages(dashlets, skipDashletIds = new Set()) { + if (!Array.isArray(dashlets)) { + return; + } + for (let i = 0; i < dashlets.length; i++) { + const dashlet = dashlets[i]; + if (skipDashletIds.has(dashlet.id)) { + continue; + } + const channels = dashlet.subscribeTo(); + if (!Array.isArray(channels)) { + if (channels && this.lastMessages.has(channels)) { + dashlet.onMessage(this.lastMessages.get(channels)); + this.#markDashletState(dashlet, { + lastReplayAt: new Date().toISOString(), + lastReplayEvent: channels, + }); + this.#debug("replay-cached", { + dashletId: dashlet.id, + eventName: channels, + }); + this.pendingImmediateDashlets.delete(dashlet.id); + } + continue; + } + for (let j = 0; j < channels.length; j++) { + const channel = channels[j]; + if (this.lastMessages.has(channel)) { + dashlet.onMessage(this.lastMessages.get(channel)); + this.#markDashletState(dashlet, { + lastReplayAt: new Date().toISOString(), + lastReplayEvent: channel, + }); + this.#debug("replay-cached", { + dashletId: dashlet.id, + eventName: channel, + }); + this.pendingImmediateDashlets.delete(dashlet.id); + } + } + } + } + #buildChannelList() { this.channels = []; - for (let i=0; i { + this.tabChannels[tabIndex] = []; + }); + + for (let i = 0; i < this.dashlets.length; i++) { + const dashlet = this.dashlets[i]; + const channels = dashlet.subscribeTo(); + const tabChannels = this.tabChannels[dashlet.tabIndex] || []; + for (let j = 0; j < channels.length; j++) { + if (!tabChannels.includes(channels[j])) { + tabChannels.push(channels[j]); + } + } + this.tabChannels[dashlet.tabIndex] = tabChannels; + if (!this.#dashletShouldStayActive(dashlet)) { + continue; + } + for (let j = 0; j < channels.length; j++) { if (!this.#alreadySubscribed(channels[j])) { this.channels.push(channels[j]); } @@ -250,37 +607,154 @@ export class Dashboard { return false; } - #webSocketSubscription() { - if (this.channels.length === 0) { - // We're not in WS mode - for (let i=0; i 0) { + this.wsClient.unsubscribe(Array.from(this.subscribedChannels)); + } + for (const disposer of this.channelDisposers.values()) { + disposer(); + } + this.channelDisposers.clear(); + this.subscribedChannels.clear(); + } + + #syncWebSocketChannels() { + const hadEstablishedSubscription = + !!this.wsClient.ws && this.wsClient.handshake_done && this.subscribedChannels.size > 0; + const nextChannels = new Set(this.channels); + const addedChannels = []; + const removedChannels = []; + + for (const channel of nextChannels) { + if (!this.subscribedChannels.has(channel)) { + addedChannels.push(channel); } + } + for (const channel of this.subscribedChannels) { + if (!nextChannels.has(channel)) { + removedChannels.push(channel); + } + } + + for (let i = 0; i < removedChannels.length; i++) { + const channel = removedChannels[i]; + const disposer = this.channelDisposers.get(channel); + if (disposer) { + disposer(); + this.channelDisposers.delete(channel); + } + this.subscribedChannels.delete(channel); + } + if (removedChannels.length > 0) { + this.wsClient.unsubscribe(removedChannels); + } + if (removedChannels.length > 0) { + this.#debug("channels-removed", { + channels: removedChannels, + }); + } + this.#debugInterestingChannels("removed", removedChannels); + + for (let i = 0; i < addedChannels.length; i++) { + const channel = addedChannels[i]; + const disposer = this.wsClient.on(channel, (msg) => { + this.#handleWebSocketMessage(msg); + }); + this.channelDisposers.set(channel, disposer); + this.subscribedChannels.add(channel); + } + if (addedChannels.length > 0) { + this.wsClient.subscribe(addedChannels); + } + if (hadEstablishedSubscription && addedChannels.length > 0) { + this.wsClient.refreshSubscriptions("dashboard-channel-add"); + } + this.#debugInterestingChannels("added", addedChannels); + this.#debug("channels-sync", { + added: addedChannels, + removed: removedChannels, + subscribed: Array.from(this.subscribedChannels), + }); + } + + #handleWebSocketMessage(msg) { + if (!msg || !msg.event || this.paused) { return; } - this.subscription = subscribeWS(this.channels, (msg) => { - if (msg.event === "join") { - // The DOM will be present now, setup events - for (let i=0; i d.id), + backgroundDashlets: backgroundDashlets.map((d) => d.id), + subscribedDashlets: this.#subscribedDashlets().map((d) => d.id), + }); + if (DIAGNOSTIC_CHANNELS.has(msg.event)) { + this.#debug("interesting-message", { + eventName: msg.event, + activeTab: this.layout.activeTab, + renderableDashlets: renderableDashlets.map((d) => d.id), + backgroundDashlets: backgroundDashlets.map((d) => d.id), + subscribedDashlets: this.#subscribedDashlets().map((d) => d.id), + }); + } + + for (let i = 0; i < renderableDashlets.length; i++) { + const dashlet = renderableDashlets[i]; + if (!this.#dashletSubscribedToMessage(dashlet, msg.event)) { + continue; + } + const shouldDeliverImmediately = + this.pendingImmediateDashlets.has(dashlet.id); + if (dashlet.canBeSlowedDown()) { + if (shouldDeliverImmediately || this.tickCounter === 0) { + dashlet.onMessage(msg); + this.#markDashletState(dashlet, { + lastMessageAt: new Date().toISOString(), + lastEvent: msg.event, + }); + this.#debug("deliver", { + dashletId: dashlet.id, + eventName: msg.event, + immediate: shouldDeliverImmediately, + slowed: true, + }); + this.pendingImmediateDashlets.delete(dashlet.id); } } else { - // Propagate the message - if (!this.paused) { - this.tickCounter++; - this.tickCounter %= this.cadence; - for (let i = 0; i < this.dashlets.length; i++) { - if (this.dashlets[i].canBeSlowedDown()) { - if (this.tickCounter === 0) { - this.dashlets[i].onMessage(msg); - } - } else { - this.dashlets[i].onMessage(msg); - } - } - } + dashlet.onMessage(msg); + this.#markDashletState(dashlet, { + lastMessageAt: new Date().toISOString(), + lastEvent: msg.event, + }); + this.#debug("deliver", { + dashletId: dashlet.id, + eventName: msg.event, + immediate: false, + slowed: false, + }); } - }); + } + + for (let i = 0; i < backgroundDashlets.length; i++) { + const dashlet = backgroundDashlets[i]; + if (!this.#dashletSubscribedToMessage(dashlet, msg.event)) { + continue; + } + dashlet.onBackgroundMessage(msg); + this.#debug("background-deliver", { + dashletId: dashlet.id, + eventName: msg.event, + }); + } } #editButton(hasCadence) { @@ -323,7 +797,8 @@ export class Dashboard { cadencePicker.max = "60"; cadencePicker.value = this.cadence; cadencePicker.onchange = () => { - this.cadence = parseInt(cadencePicker.value); + const parsedCadence = parseInt(cadencePicker.value); + this.cadence = Number.isFinite(parsedCadence) && parsedCadence > 0 ? parsedCadence : 1; localStorage.setItem("dashCadence", this.cadence.toString()); } cadenceDiv.appendChild(cadenceLabel); @@ -343,7 +818,20 @@ export class Dashboard { } }; + let debugDumpDiv = null; + if (this.#debugDumpEnabled()) { + debugDumpDiv = document.createElement("span"); + debugDumpDiv.id = this.divName + "_debug_dump"; + debugDumpDiv.innerHTML = ""; + debugDumpDiv.onclick = () => { + this.#downloadDebugDump(); + }; + } + parent.appendChild(editDiv); + if (debugDumpDiv !== null) { + parent.appendChild(debugDumpDiv); + } if (hasCadence) { parent.appendChild(pauseDiv); parent.appendChild(cadenceDiv); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/dashboard_editor.js b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/dashboard_editor.js index 242548b4..6a83209f 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/dashboard_editor.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/dashboard_editor.js @@ -1,3 +1,5 @@ +import {DashboardLayout} from "./layout"; + /** * Opens a full screen modal that lets the user edit dashboard items with tab support. * @param {Object} layout - The current dashboard layout object with tabs @@ -8,7 +10,13 @@ export function openDashboardEditor(layout, availableElements, callback, cookieName) { // Keep track of current tab and layout state let currentLayout = JSON.parse(JSON.stringify(layout)); // Deep copy - let activeTabIndex = currentLayout.activeTab || 0; + currentLayout.tabs = DashboardLayout.normalizeTabs(currentLayout.tabs || []); + let activeTabIndex = DashboardLayout.resolveActiveTabIndex( + currentLayout.tabs, + currentLayout.activeTab, + currentLayout.tabs[currentLayout.activeTab || 0]?.id || null + ); + currentLayout.activeTab = activeTabIndex; const sortableInstances = []; let committedLayout = null; @@ -301,10 +309,12 @@ export function openDashboardEditor(layout, availableElements, callback, cookieN $('#addTabButton').on('click', function() { let newTabName = `Tab ${currentLayout.tabs.length + 1}`; currentLayout.tabs.push({ + id: DashboardLayout.makeTabId(newTabName), name: newTabName, dashlets: [] }); activeTabIndex = currentLayout.tabs.length - 1; + currentLayout.activeTab = activeTabIndex; renderTabs(); renderDashboard(); }); @@ -516,8 +526,12 @@ export function openDashboardEditor(layout, availableElements, callback, cookieN // Apply the layout currentLayout = { version: imported.version || 2, - tabs: imported.tabs, - activeTab: imported.activeTab || 0 + tabs: DashboardLayout.normalizeTabs(imported.tabs), + activeTab: DashboardLayout.resolveActiveTabIndex( + DashboardLayout.normalizeTabs(imported.tabs), + imported.activeTab, + imported.tabs?.[imported.activeTab || 0]?.id || null + ) }; // Re-render diff --git a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/layout.js b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/layout.js index 157dd9c3..8f071b78 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/layout.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/dashboard/layout.js @@ -3,30 +3,44 @@ const CURRENT_VERSION = 3; export class DashboardLayout { constructor(cookieName, defaultLayout) { this.cookieName = cookieName; + this.activeTabStorageKey = `${cookieName}.activeTabId`; // Keep a normalized copy of the default (tabs + dashlets only) this._defaultTabs = DashboardLayout.normalizeToTabs(defaultLayout); + const persistedActiveTabId = this.loadPersistedActiveTabId(); let template = localStorage.getItem(cookieName); if (template !== null) { let parsed = JSON.parse(template); // Check if it's the new format with version if (parsed.version >= 2) { this.version = parsed.version; - this.tabs = Array.isArray(parsed.tabs) ? parsed.tabs : []; - this.activeTab = parsed.activeTab || 0; + this.tabs = DashboardLayout.mergeMissingDefaultTabs( + Array.isArray(parsed.tabs) ? parsed.tabs : [], + this._defaultTabs + ); + this.activeTab = DashboardLayout.resolveActiveTabIndex( + this.tabs, + parsed.activeTab, + persistedActiveTabId + ); } else { // Old format saved in localStorage (pre-tabs) // Treat as a signal to reset to the new default layout, // so users see the new Overview tab instead of a single-tab conversion. this.version = CURRENT_VERSION; if (defaultLayout && defaultLayout.version >= 2) { - this.tabs = defaultLayout.tabs || []; - this.activeTab = defaultLayout.activeTab || 0; + this.tabs = DashboardLayout.normalizeToTabs(defaultLayout); + this.activeTab = DashboardLayout.resolveActiveTabIndex( + this.tabs, + defaultLayout.activeTab, + persistedActiveTabId + ); } else { - this.tabs = [{ - name: "Dashboard", - dashlets: defaultLayout || [] - }]; - this.activeTab = 0; + this.tabs = DashboardLayout.normalizeToTabs(defaultLayout); + this.activeTab = DashboardLayout.resolveActiveTabIndex( + this.tabs, + 0, + persistedActiveTabId + ); } try { localStorage.removeItem(this.cookieName); } catch (e) {} } @@ -35,18 +49,24 @@ export class DashboardLayout { if (defaultLayout && defaultLayout.version >= 2) { // New format default this.version = defaultLayout.version; - this.tabs = defaultLayout.tabs || []; - this.activeTab = defaultLayout.activeTab || 0; + this.tabs = DashboardLayout.normalizeToTabs(defaultLayout); + this.activeTab = DashboardLayout.resolveActiveTabIndex( + this.tabs, + defaultLayout.activeTab, + persistedActiveTabId + ); } else { // Old format default - convert to new format this.version = CURRENT_VERSION; - this.tabs = [{ - name: "Dashboard", - dashlets: defaultLayout || [] - }]; - this.activeTab = 0; + this.tabs = DashboardLayout.normalizeToTabs(defaultLayout); + this.activeTab = DashboardLayout.resolveActiveTabIndex( + this.tabs, + 0, + persistedActiveTabId + ); } } + this.persistActiveTabId(); } save(layoutData) { @@ -57,9 +77,14 @@ export class DashboardLayout { } else { // New style - full layout object this.version = layoutData.version || CURRENT_VERSION; - this.tabs = layoutData.tabs || []; - this.activeTab = layoutData.activeTab || 0; + this.tabs = DashboardLayout.normalizeTabs(layoutData.tabs || []); + this.activeTab = DashboardLayout.resolveActiveTabIndex( + this.tabs, + layoutData.activeTab, + this.getActiveTabId() + ); } + this.persistActiveTabId(); // Only persist if layout differs from default tabs/dashlets const isDefault = DashboardLayout.tabsEqual(this.tabs, this._defaultTabs); if (isDefault) { @@ -89,24 +114,97 @@ export class DashboardLayout { getCurrentTabDashlets() { return this.tabs[this.activeTab]?.dashlets || []; } + + getActiveTabId() { + return this.tabs[this.activeTab]?.id || null; + } + + loadPersistedActiveTabId() { + try { + return localStorage.getItem(this.activeTabStorageKey); + } catch (e) { + return null; + } + } + + persistActiveTabId() { + try { + const activeTabId = this.getActiveTabId(); + if (activeTabId) { + localStorage.setItem(this.activeTabStorageKey, activeTabId); + } else { + localStorage.removeItem(this.activeTabStorageKey); + } + } catch (e) {} + } } // Static helpers DashboardLayout.normalizeToTabs = function(defaultLayout) { if (!defaultLayout) return []; if (defaultLayout.version >= 2 && Array.isArray(defaultLayout.tabs)) { - return defaultLayout.tabs.map(t => ({ - name: t.name || "Dashboard", - dashlets: Array.isArray(t.dashlets) ? t.dashlets.map(d => ({ tag: d.tag, size: d.size })) : [] - })); + return DashboardLayout.normalizeTabs(defaultLayout.tabs); } // Old format: list of dashlets if (Array.isArray(defaultLayout)) { - return [{ name: "Dashboard", dashlets: defaultLayout.map(d => ({ tag: d.tag, size: d.size })) }]; + return [DashboardLayout.normalizeTab({ + id: "dashboard", + name: "Dashboard", + dashlets: defaultLayout + })]; } return []; }; +DashboardLayout.makeTabId = function(name = "tab") { + const base = String(name || "tab") + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") || "tab"; + return `${base}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; +}; + +DashboardLayout.normalizeTab = function(tab, fallbackName = "Dashboard", fallbackId = null) { + const name = tab?.name || fallbackName; + return { + id: tab?.id || fallbackId || DashboardLayout.makeTabId(name), + name, + dashlets: Array.isArray(tab?.dashlets) + ? tab.dashlets.map((dashlet) => ({ tag: dashlet.tag, size: dashlet.size })) + : [] + }; +}; + +DashboardLayout.normalizeTabs = function(tabs) { + if (!Array.isArray(tabs) || tabs.length === 0) { + return []; + } + return tabs.map((tab, index) => + DashboardLayout.normalizeTab(tab, `Tab ${index + 1}`) + ); +}; + +DashboardLayout.resolveActiveTabIndex = function(tabs, requestedIndex, requestedTabId) { + if (!Array.isArray(tabs) || tabs.length === 0) { + return 0; + } + + if (requestedTabId) { + const indexById = tabs.findIndex((tab) => tab?.id === requestedTabId); + if (indexById >= 0) { + return indexById; + } + } + + const numericIndex = Number.parseInt(requestedIndex, 10); + if (Number.isFinite(numericIndex) && numericIndex >= 0 && numericIndex < tabs.length) { + return numericIndex; + } + + return 0; +}; + DashboardLayout.tabsEqual = function(tabsA, tabsB) { if (!Array.isArray(tabsA) || !Array.isArray(tabsB)) return false; if (tabsA.length !== tabsB.length) return false; @@ -126,3 +224,32 @@ DashboardLayout.tabsEqual = function(tabsA, tabsB) { } return true; }; + +DashboardLayout.mergeMissingDefaultTabs = function(existingTabs, defaultTabs) { + if (!Array.isArray(existingTabs) || existingTabs.length === 0) { + return Array.isArray(defaultTabs) ? DashboardLayout.normalizeTabs(defaultTabs) : []; + } + + const normalizedExisting = DashboardLayout.normalizeTabs(existingTabs); + if (!Array.isArray(defaultTabs) || defaultTabs.length === 0) { + return normalizedExisting; + } + + const merged = normalizedExisting.slice(); + const knownIds = new Set(normalizedExisting.map((tab) => tab?.id || "")); + const knownNames = new Set(normalizedExisting.map((tab) => tab?.name || "")); + defaultTabs.forEach((tab, index) => { + const normalizedTab = DashboardLayout.normalizeTab(tab, `Tab ${index + 1}`); + if ( + !normalizedTab.name + || knownIds.has(normalizedTab.id) + || knownNames.has(normalizedTab.name) + ) { + return; + } + knownIds.add(normalizedTab.id); + knownNames.add(normalizedTab.name); + merged.push(normalizedTab); + }); + return merged; +}; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/helpers/content_builders.js b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/helpers/content_builders.js index f2da7a95..380ce926 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/helpers/content_builders.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/helpers/content_builders.js @@ -1,3 +1,5 @@ +import {enableTooltipsWithin} from "./tooltips"; + export function heading5Icon(icon, text) { let h5 = document.createElement("h5"); h5.innerHTML = " " + text; @@ -5,27 +7,5 @@ export function heading5Icon(icon, text) { } export function enableTooltips() { - // Tooltips everywhere! Make this idempotent to avoid leaking tooltip instances - // on websocket reconnects or repeated dashboard setup cycles. - if (typeof bootstrap === "undefined" || !bootstrap.Tooltip) { - return; - } - const Tooltip = bootstrap.Tooltip; - const tooltipTriggerList = Array.prototype.slice.call( - document.querySelectorAll('[data-bs-toggle="tooltip"]'), - ); - tooltipTriggerList.forEach((el) => { - if (!el) return; - if (Tooltip.getOrCreateInstance) { - Tooltip.getOrCreateInstance(el); - return; - } - if (Tooltip.getInstance) { - const existing = Tooltip.getInstance(el); - if (existing && existing.dispose) { - existing.dispose(); - } - } - new Tooltip(el); - }); + enableTooltipsWithin(document); } diff --git a/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/helpers/tooltips.js b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/helpers/tooltips.js new file mode 100644 index 00000000..89da0a7c --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/lq_js_common/helpers/tooltips.js @@ -0,0 +1,53 @@ +function tooltipElementsWithin(rootEl = document) { + if (!rootEl) { + return []; + } + + const elements = []; + if (rootEl.matches && rootEl.matches('[data-bs-toggle="tooltip"]')) { + elements.push(rootEl); + } + + if (rootEl.querySelectorAll) { + elements.push(...rootEl.querySelectorAll('[data-bs-toggle="tooltip"]')); + } + + return elements; +} + +export function enableTooltipsWithin(rootEl = document) { + if (typeof bootstrap === "undefined" || !bootstrap.Tooltip) { + return; + } + + const Tooltip = bootstrap.Tooltip; + tooltipElementsWithin(rootEl).forEach((el) => { + if (!el) { + return; + } + if (Tooltip.getOrCreateInstance) { + Tooltip.getOrCreateInstance(el); + return; + } + if (Tooltip.getInstance) { + const existing = Tooltip.getInstance(el); + if (existing && existing.dispose) { + existing.dispose(); + } + } + new Tooltip(el); + }); +} + +export function disposeTooltipsWithin(rootEl = document) { + if (typeof bootstrap === "undefined" || !bootstrap.Tooltip || !bootstrap.Tooltip.getInstance) { + return; + } + + tooltipElementsWithin(rootEl).forEach((el) => { + const existing = bootstrap.Tooltip.getInstance(el); + if (existing && existing.dispose) { + existing.dispose(); + } + }); +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/lts_teasers_shared.js b/src/rust/lqosd/src/node_manager/js_build/src/lts_teasers_shared.js index 0fc5914f..9739ff63 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/lts_teasers_shared.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/lts_teasers_shared.js @@ -2,55 +2,56 @@ // Used by both the standalone Insight trial page and the // in-app Insight nag modal so the benefits only need to be // maintained in one place. +// +// Draft teaser copy and image mapping for in-place review. export const PLACEHOLDER_TEASERS = [ { - id: "ai", - title: "AI Reports", + id: "executive_summary", + title: "Executive Summary", description: - "Diagnose problems quickly using AI-powered analyses of aggregated data.", - image: "01_ai_report.png", + "See network health, capacity, and the biggest problem areas at a glance so you know where to look next.", + image: "01_executive_summary.png", order: 1, }, { - id: "mapper", - title: "Insight Network Mapper", + id: "site_overview", + title: "Site Overview", description: - "Configure and manage complex networks and multiple shaper boxes with ease.", - image: "02_mapper.png", + "Open a site and compare performance, utilization, and quality over time without jumping between tools.", + image: "02_site_overview.png", order: 2, }, { - id: "long_term", - title: "Long Term Data", + id: "site_rankings", + title: "Site Rankings", description: - "Flexible time windows ranging from 15 minutes to 28 days, giving you comprehensive network performance insights.", - image: "03_long_term.png", + "Rank sites by quality over the selected period so the worst performers surface immediately.", + image: "03_site_rankings.png", order: 3, }, { - id: "heatmap", - title: "Heatmaps", + id: "subscriber_view", + title: "Subscriber View", description: - "Don’t waste time going through charts – instantly spot performance trends on APs, OLTs, and sites.", - image: "04_heatmap.png", + "Inspect subscriber behavior, usage, and performance from one place when a specific customer needs attention.", + image: "04_subscriber_view.png", order: 4, }, { - id: "alerts", - title: "Alerts", + id: "issue_triage", + title: "Issue Triage", description: - "Get notified before issues impact your customers with intelligent alert monitoring.", - image: "05_alerts.png", + "Turn symptoms into a focused troubleshooting summary with likely causes and recommended next steps.", + image: "05_issue_triage.png", order: 5, }, { - id: "endpoints", - title: "Endpoints by Circuit", + id: "api_only", + title: "API-Only Option", description: - "Observe important endpoint information, even for individual circuits.", - image: "06_endpoints.png", + "Choose the lower-cost API-only option when you only need provisioning and automation integrations. Hosted dashboards and remote Insight history are not included.", + image: "06_api_only.png", order: 6, }, ]; - diff --git a/src/rust/lqosd/src/node_manager/js_build/src/lts_trial.js b/src/rust/lqosd/src/node_manager/js_build/src/lts_trial.js index 51abdb4c..5723154e 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/lts_trial.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/lts_trial.js @@ -1,37 +1,5 @@ import {PLACEHOLDER_TEASERS} from "./lts_teasers_shared"; -import {get_ws_client} from "./pubsub/ws"; - -const wsClient = get_ws_client(); -const listenOnce = (eventName, handler) => { - const wrapped = (msg) => { - wsClient.off(eventName, wrapped); - handler(msg); - }; - wsClient.on(eventName, wrapped); -}; - -function sendWsRequest(responseEvent, request) { - return new Promise((resolve, reject) => { - let done = false; - const onResponse = (msg) => { - if (done) return; - done = true; - wsClient.off(responseEvent, onResponse); - wsClient.off("Error", onError); - resolve(msg); - }; - const onError = (msg) => { - if (done) return; - done = true; - wsClient.off(responseEvent, onResponse); - wsClient.off("Error", onError); - reject(msg); - }; - wsClient.on(responseEvent, onResponse); - wsClient.on("Error", onError); - wsClient.send(request); - }); -} +import {buildSignupApiUrl, sendWsRequest, wsClient} from "./lts_trial_shared"; // Paddle-compatible countries (excluding embargoed nations) const ALLOWED_COUNTRIES = [ @@ -288,6 +256,7 @@ $(document).ready(async function() { loadTeasers(); // Then load teasers with the correct URL fetchCircuitCount(); attachEventHandlers(); + openInitialViewFromQuery(); // Apply dark mode if needed if (isDarkMode()) { @@ -356,7 +325,7 @@ async function loadTeasers() { const response = await $.get(getLtsUrl('teasers')); if (response.teasers != null) { response.teasers.forEach(teaser => { - teaser.image = getLtsUrl(teaser.image.replace("signup-api/", "")); + teaser.image = getLtsUrl(teaser.image.replace(/^\/?signup-api\//, "")); }); } currentTeasers = response.teasers || PLACEHOLDER_TEASERS; @@ -434,7 +403,7 @@ function displayTeasers() { // Add click handlers to CTA buttons $('.btn-teaser-cta').on('click', function() { - showSection('signupSection'); + redirectToTrialSetup(); }); // Reinitialize carousel if needed @@ -448,19 +417,7 @@ function displayTeasers() { // Helper function to construct full LTS URLs function getLtsUrl(endpoint) { - // Ensure ltsBaseUrl starts with http:// or https://, otherwise prefix with https:// - let baseUrl = ltsBaseUrl; - if (!/^https?:\/\//i.test(baseUrl)) { - baseUrl = 'https://' + baseUrl; - } - // Ensure baseUrl ends with a single slash - let base = baseUrl.endsWith('/') ? baseUrl : baseUrl + '/'; - // Ensure 'signup-api/' is appended exactly once - base += base.endsWith('signup-api/') ? '' : 'signup-api/'; - // Remove any leading slash from endpoint - endpoint = endpoint.replace(/^\/+/, ''); - //console.log("Base: ", base, "Endpoint:", endpoint); - return base + endpoint; + return buildSignupApiUrl(ltsBaseUrl, endpoint); } // Fetch node ID and LTS URL from configuration @@ -517,6 +474,29 @@ function showSection(sectionId) { $(`#${sectionId}`).fadeIn(); } +function redirectToTrialSetup() { + window.location.href = 'lts_trial_setup.html'; +} + +function openInitialViewFromQuery() { + const params = new URLSearchParams(window.location.search); + const view = params.get('view'); + + if (view === 'signup') { + showSection('signupSection'); + return; + } + + if (view === 'license') { + showSection('licenseKeySection'); + return; + } + + if (view === 'recover') { + showSection('licenseRecoverySection'); + } +} + // Show loading modal function showLoading(message = 'Processing...') { $('#loadingMessage').text(message); @@ -579,7 +559,7 @@ function attachEventHandlers() { // Analytics tracking for new customer path const trackingImg = new Image(); trackingImg.src = getLtsUrl('signupPing') + '?t=' + Date.now() + '&type=new'; - showSection('signupSection'); + redirectToTrialSetup(); }); $('#btnBackFromLicense').on('click', () => showSection('teaserSection')); $('#btnBackFromSignup').on('click', () => showSection('teaserSection')); @@ -721,13 +701,13 @@ function attachEventHandlers() { showSection('successSection'); $('#successMessage').html(` License Key: ${response.licenseKey}
    - An email with your license key and portal access has been sent to your email address. + Your trial is almost active. Watch for an email from Insight with your account details. No charge occurs until the trial ends, and you can cancel any time. `); $('#configStatus').text('Saving configuration...'); wsClient.send({ LtsSignUp: { license_key: response.licenseKey } }); // Show success message now, then swap spinner to a check and enable dashboard after 5s - $('#configStatus').html(`Account created! Your configuration has been updated - data will start going to Insight shortly.`); + $('#configStatus').html(`Your Insight trial has been created. Check your email for the next steps while configuration is being saved.`); setTimeout(() => { $('#configSpinner').hide(); $('#configStatus').html(` Configuration saved.`); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/lts_trial_setup.js b/src/rust/lqosd/src/node_manager/js_build/src/lts_trial_setup.js new file mode 100644 index 00000000..f6adb028 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/lts_trial_setup.js @@ -0,0 +1,97 @@ +import {buildInsightUrl, sendWsRequest} from "./lts_trial_shared"; + +const setupStatusClasses = [ + 'alert-primary', + 'alert-success', + 'alert-warning', + 'alert-danger', + 'alert-secondary', +]; +const defaultInsightBaseUrl = 'https://insight.libreqos.com/'; +let insightBaseUrl = defaultInsightBaseUrl; + +function setSetupStatus(message, tone = 'primary', spinning = true) { + const status = $('#signupSetupStatus'); + const statusText = $('#signupSetupStatusText'); + const spinner = status.find('.spinner-border'); + + if (!status.length || !statusText.length) { + return; + } + + status.removeClass(setupStatusClasses.join(' ')).addClass(`alert-${tone}`); + statusText.text(message); + + if (spinning) { + spinner.show(); + } else { + spinner.hide(); + } +} + +function setSetupHelp(message) { + $('#signupSetupHelp').text(message); +} + +function getErrorMessage(error, fallback) { + if (error && typeof error.message === 'string' && error.message.trim().length > 0) { + return error.message.trim(); + } + return fallback; +} + +async function fetchTrialConfig() { + try { + const response = await sendWsRequest("LtsTrialConfigResult", { LtsTrialConfig: {} }); + const data = response && response.data ? response.data : {}; + if (typeof data.lts_url === 'string' && data.lts_url.trim().length > 0) { + insightBaseUrl = data.lts_url; + } + } catch (error) { + console.warn('Unable to load LTS trial config, using default Insight URL.', error); + insightBaseUrl = defaultInsightBaseUrl; + } +} + +function redirectToInsightSignup(claimId) { + window.location.href = buildInsightUrl( + insightBaseUrl, + `su/signup1.html?${encodeURIComponent(claimId)}`, + ); +} + +async function startSignupSession() { + setSetupStatus('Creating your Insight signup session…'); + setSetupHelp('LibreQoS is contacting Insight and preparing your claim.'); + await fetchTrialConfig(); + + try { + const response = await sendWsRequest("LtsStartSignupResult", { LtsStartSignup: {} }); + const claimId = response && typeof response.claim_id === 'string' + ? response.claim_id.trim() + : ''; + + if (!claimId) { + throw new Error('Missing claim ID from signup response.'); + } + + setSetupStatus('Signup session ready. Redirecting to Insight…', 'success', false); + setSetupHelp('Your signup session is ready. Continuing to Insight now.'); + redirectToInsightSignup(claimId); + } catch (error) { + console.error('Failed to start Insight signup session.', error); + setSetupStatus( + getErrorMessage(error, 'Unable to start the Insight signup session right now.'), + 'danger', + false, + ); + setSetupHelp( + 'Retry this page, use the direct signup form, or validate an existing license key below.', + ); + } +} + +$(document).ready(() => { + setSetupStatus('Preparing the Insight signup handoff…'); + void startSignupSession(); +}); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/lts_trial_shared.js b/src/rust/lqosd/src/node_manager/js_build/src/lts_trial_shared.js new file mode 100644 index 00000000..c2386e31 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/src/lts_trial_shared.js @@ -0,0 +1,58 @@ +import {get_ws_client} from "./pubsub/ws"; + +const DEFAULT_INSIGHT_BASE_URL = 'https://insight.libreqos.com/'; +const wsClient = get_ws_client(); + +function normalizeInsightBaseUrl(rawBaseUrl) { + let baseUrl = (rawBaseUrl || DEFAULT_INSIGHT_BASE_URL).trim(); + + if (!baseUrl) { + baseUrl = DEFAULT_INSIGHT_BASE_URL; + } + + if (!/^https?:\/\//i.test(baseUrl)) { + baseUrl = `https://${baseUrl}`; + } + + baseUrl = baseUrl.replace(/\/+$/, ''); + if (baseUrl.endsWith('/signup-api')) { + baseUrl = baseUrl.slice(0, -'/signup-api'.length); + } + + return `${baseUrl}/`; +} + +function buildInsightUrl(rawBaseUrl, endpoint) { + const path = String(endpoint || '').replace(/^\/+/, ''); + return `${normalizeInsightBaseUrl(rawBaseUrl)}${path}`; +} + +function buildSignupApiUrl(rawBaseUrl, endpoint) { + const path = String(endpoint || '').replace(/^\/+/, ''); + return buildInsightUrl(rawBaseUrl, `signup-api/${path}`); +} + +function sendWsRequest(responseEvent, request) { + return new Promise((resolve, reject) => { + let done = false; + const onResponse = (msg) => { + if (done) return; + done = true; + wsClient.off(responseEvent, onResponse); + wsClient.off("Error", onError); + resolve(msg); + }; + const onError = (msg) => { + if (done) return; + done = true; + wsClient.off(responseEvent, onResponse); + wsClient.off("Error", onError); + reject(msg); + }; + wsClient.on(responseEvent, onResponse); + wsClient.on("Error", onError); + wsClient.send(request); + }); +} + +export {buildInsightUrl, buildSignupApiUrl, normalizeInsightBaseUrl, sendWsRequest, wsClient}; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/pubsub/ws.js b/src/rust/lqosd/src/node_manager/js_build/src/pubsub/ws.js index 004b019a..b528b77c 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/pubsub/ws.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/pubsub/ws.js @@ -4,9 +4,64 @@ const ACK_TEXT = "I accept that this is an unstable, internal API and is unsuppo const EXPECTED_UI_VERSION = (window.LQOS_UI_VERSION || "").trim() || null; const USER_TOKEN_COOKIE = "User-Token"; const encoder = new Encoder({ useRecords: false, variableMapSize: true }); +const DIAGNOSTIC_CHANNELS = new Set(["Cpu", "Ram", "RttHistogram"]); let shared_client = null; +function dashboardDebugEnabled() { + if (typeof window === "undefined") { + return false; + } + try { + const params = new URLSearchParams(window.location.search || ""); + if (params.has("debug")) { + const value = (params.get("debug") || "").trim().toLowerCase(); + if (value === "" || value === "1" || value === "true" || value === "dashboard") { + return true; + } + } + } catch (_) { + // Ignore malformed URL search state and fall back to localStorage. + } + return !!window.localStorage && window.localStorage.getItem("debugDashboard") === "1"; +} + +function pushDashboardWsTrace(entry) { + if (typeof window === "undefined") { + return; + } + if (!window.__lqosDashboardWsTrace) { + window.__lqosDashboardWsTrace = []; + } + window.__lqosDashboardWsTrace.push({ + ts: new Date().toISOString(), + ...entry, + }); + if (window.__lqosDashboardWsTrace.length > 500) { + window.__lqosDashboardWsTrace.shift(); + } +} + +function dashboardWsDebug(event, details = {}) { + if (!dashboardDebugEnabled()) { + return; + } + const entry = { event, ...details }; + pushDashboardWsTrace(entry); + console.debug("[dashboard-ws]", entry); +} + +function dashboardWsInteresting(event, channels = [], details = {}) { + const interesting = channels.filter((channel) => DIAGNOSTIC_CHANNELS.has(channel)); + if (interesting.length === 0) { + return; + } + dashboardWsDebug(event, { + channels: interesting, + ...details, + }); +} + function get_cookie_value(name) { const cookies = document.cookie ? document.cookie.split(";") : []; const prefix = `${name}=`; @@ -51,10 +106,18 @@ export class WsClient { this.reconnectTimer = null; } this.manualClose = false; - this.ws = new WebSocket(ws_proto() + window.location.host + "/websocket/ws"); - this.ws.binaryType = "arraybuffer"; + const socket = new WebSocket(ws_proto() + window.location.host + "/websocket/ws"); + this.ws = socket; + socket.binaryType = "arraybuffer"; + dashboardWsDebug("connect", { + url: ws_proto() + window.location.host + "/websocket/ws", + desiredChannels: Array.from(this.desiredChannels.keys()), + }); - this.ws.onmessage = async (event) => { + socket.onmessage = async (event) => { + if (this.ws !== socket) { + return; + } let msg = null; try { if (event.data instanceof ArrayBuffer) { @@ -69,19 +132,43 @@ export class WsClient { } if (msg && msg.event === "Hello") { + dashboardWsDebug("hello", { + version: msg.version, + }); this._acknowledge_handshake(msg); return; } + dashboardWsDebug("message", { + eventName: msg && msg.event ? msg.event : "unknown", + }); + if (msg && msg.event && DIAGNOSTIC_CHANNELS.has(msg.event)) { + dashboardWsDebug("message-interesting", { + eventName: msg.event, + desiredChannels: Array.from(this.desiredChannels.keys()), + }); + } this._dispatch(msg); }; - this.ws.onclose = () => { + socket.onclose = () => { + if (this.ws !== socket) { + return; + } + dashboardWsDebug("close", { + desiredChannels: Array.from(this.desiredChannels.keys()), + }); this.ws = null; this.handshake_done = false; this._scheduleReconnect(); }; - this.ws.onerror = () => { + socket.onerror = () => { + if (this.ws !== socket) { + return; + } + dashboardWsDebug("error", { + desiredChannels: Array.from(this.desiredChannels.keys()), + }); this.ws = null; this.handshake_done = false; this._scheduleReconnect(); @@ -89,6 +176,9 @@ export class WsClient { } close() { + dashboardWsDebug("manual-close", { + desiredChannels: Array.from(this.desiredChannels.keys()), + }); this.manualClose = true; if (this.reconnectTimer) { clearTimeout(this.reconnectTimer); @@ -104,12 +194,48 @@ export class WsClient { this.handlers.clear(); } + refreshSubscriptions(reason = "unspecified") { + dashboardWsDebug("refresh-subscriptions", { + reason, + desiredChannels: Array.from(this.desiredChannels.keys()), + }); + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + const oldSocket = this.ws; + this.ws = null; + this.handshake_done = false; + if (oldSocket) { + try { + oldSocket.close(); + } catch (_) { + // Ignore close errors; a new socket is opened below if needed. + } + } + if (this._shouldMaintainConnection()) { + this.connect(); + } + } + subscribe(channels) { + dashboardWsDebug("subscribe-request", { + channels, + }); + dashboardWsInteresting("subscribe-request-interesting", channels, { + desiredChannels: Array.from(this.desiredChannels.keys()), + }); for (let i = 0; i < channels.length; i++) { const channel = channels[i]; const current = this.desiredChannels.get(channel) || 0; this.desiredChannels.set(channel, current + 1); if (current === 0 && this.handshake_done && this.ws) { + dashboardWsDebug("subscribe-wire", { + channel, + }); + dashboardWsInteresting("subscribe-wire-interesting", [channel], { + desiredChannels: Array.from(this.desiredChannels.keys()), + }); this.ws.send(encoder.encode({ Subscribe: { channel } })); } } @@ -119,12 +245,24 @@ export class WsClient { } unsubscribe(channels) { + dashboardWsDebug("unsubscribe-request", { + channels, + }); + dashboardWsInteresting("unsubscribe-request-interesting", channels, { + desiredChannels: Array.from(this.desiredChannels.keys()), + }); for (let i = 0; i < channels.length; i++) { const channel = channels[i]; const current = this.desiredChannels.get(channel) || 0; if (current <= 1) { this.desiredChannels.delete(channel); if (this.handshake_done && this.ws) { + dashboardWsDebug("unsubscribe-wire", { + channel, + }); + dashboardWsInteresting("unsubscribe-wire-interesting", [channel], { + desiredChannels: Array.from(this.desiredChannels.keys()), + }); this.ws.send(encoder.encode({ Unsubscribe: { channel } })); } } else { @@ -135,6 +273,9 @@ export class WsClient { send(request_obj) { const normalized = normalizeRequest(request_obj); + dashboardWsDebug("send", { + request: normalized && typeof normalized === "object" ? Object.keys(normalized)[0] : "unknown", + }); if (!this.ws) { this.connect(); } @@ -181,6 +322,9 @@ export class WsClient { return; } this.handshake_done = true; + dashboardWsDebug("handshake-complete", { + desiredChannels: Array.from(this.desiredChannels.keys()), + }); this.ws.send( encoder.encode({ HelloReply: { @@ -194,6 +338,10 @@ export class WsClient { } this.pending = []; for (const channel of this.desiredChannels.keys()) { + dashboardWsInteresting("subscribe-wire-interesting", [channel], { + desiredChannels: Array.from(this.desiredChannels.keys()), + fromHandshake: true, + }); this.ws.send(encoder.encode({ Subscribe: { channel } })); } this.reconnectDelayMs = 1000; @@ -220,6 +368,10 @@ export class WsClient { return; } const delay = this.reconnectDelayMs; + dashboardWsDebug("schedule-reconnect", { + delayMs: delay, + desiredChannels: Array.from(this.desiredChannels.keys()), + }); this.reconnectTimer = setTimeout(() => { this.reconnectTimer = null; this.connect(); @@ -240,12 +392,32 @@ function normalizeRequest(request_obj) { if (!request_obj || typeof request_obj !== "object") { return request_obj; } + const stripUndefined = (value) => { + if (Array.isArray(value)) { + return value.map((entry) => + entry === undefined ? null : stripUndefined(entry), + ); + } + if (!value || typeof value !== "object") { + return value; + } + const result = {}; + Object.keys(value).forEach((key) => { + const entry = value[key]; + if (entry === undefined) { + return; + } + result[key] = stripUndefined(entry); + }); + return result; + }; + const keys = Object.keys(request_obj); if (keys.length !== 1) { - return request_obj; + return stripUndefined(request_obj); } const key = keys[0]; - const value = request_obj[key]; + const value = stripUndefined(request_obj[key]); if ( value && typeof value === "object" && diff --git a/src/rust/lqosd/src/node_manager/js_build/src/shaped-devices.js b/src/rust/lqosd/src/node_manager/js_build/src/shaped-devices.js index 590c5651..049ca1aa 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/shaped-devices.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/shaped-devices.js @@ -1,16 +1,24 @@ import {clearDiv, formatLastSeen} from "./helpers/builders"; -import {get_ws_client, subscribeWS} from "./pubsub/ws"; -import {formatRetransmit, formatRtt, formatThroughput} from "./helpers/scaling"; +import {get_ws_client} from "./pubsub/ws"; +import { + formatRetransmit, + formatRtt, + formatThroughput, + retransmitFractionFromSample, +} from "./helpers/scaling"; import {colorByQoqScore} from "./helpers/color_scales"; import {toNumber} from "./lq_js_common/helpers/scaling"; let shapedDevices = []; -let displayDevices = []; let devicesPerPage = 24; let page = 0; let searchTerm = ""; let metricElsByCircuitId = new Map(); const latestByCircuitId = new Map(); +const planByCircuitId = new Map(); +let totalRows = 0; +let totalCircuits = 0; +let circuitMetricsWatchSignature = null; const wsClient = get_ws_client(); const QOO_TOOLTIP_HTML = @@ -19,13 +27,32 @@ const QOO_TOOLTIP_HTML = "https://datatracker.ietf.org/doc/draft-ietf-ippm-qoo/
    " + "LibreQoS implements a latency and loss-based model to estimate quality of outcome.

    "; -const listenOnce = (eventName, handler) => { - const wrapped = (msg) => { - wsClient.off(eventName, wrapped); - handler(msg); - }; - wsClient.on(eventName, wrapped); -}; +function sendWsRequest(responseEvent, request) { + return new Promise((resolve, reject) => { + let done = false; + const responseHandler = (msg) => { + if (done) return; + done = true; + wsClient.off(responseEvent, responseHandler); + wsClient.off("Error", errorHandler); + resolve(msg); + }; + const errorHandler = (msg) => { + if (done) return; + done = true; + wsClient.off(responseEvent, responseHandler); + wsClient.off("Error", errorHandler); + reject(msg); + }; + wsClient.on(responseEvent, responseHandler); + wsClient.on("Error", errorHandler); + wsClient.send(request); + }); +} + +function sendPrivateRequest(command) { + wsClient.send({Private: command}); +} function initTooltipsWithin(rootEl) { if (!rootEl) return; @@ -48,37 +75,12 @@ function formatPlanValue(value) { } function countCircuits() { - let entries = {}; - shapedDevices.forEach((d) => { - if (!entries.hasOwnProperty(d.circuit_id)) { - entries[d.circuit_id] = 1; - } - }); - let count = 0; - for (const _ in entries) { - count++; - } - return count; + return totalCircuits; } function filterDevices() { - const term = (searchTerm || "").toLowerCase().trim(); - if (term === "") { - displayDevices = shapedDevices; - } else { - displayDevices = shapedDevices.filter((d) => { - const deviceName = (d.device_name || "").toLowerCase(); - const circuitName = (d.circuit_name || "").toLowerCase(); - const parentNode = (d.parent_node || "").toLowerCase(); - return ( - deviceName.indexOf(term) > -1 || - circuitName.indexOf(term) > -1 || - parentNode.indexOf(term) > -1 - ); - }); - } page = 0; - renderCards(); + requestShapedDevicesPage(); } function formatQooScore(score0to100, fallback = "-") { @@ -105,6 +107,42 @@ function formatRttFromNanosOpt(nanosOpt, fallback = "-") { return formatRtt(nanos / 1_000_000.0); } +function formatIpAddress(address, family) { + if (address === null || address === undefined) return ""; + if (Array.isArray(address)) { + if (family === 4 && address.length === 4) { + return address.map((part) => String(part)).join("."); + } + if (family === 6) { + if (address.length === 16) { + const groups = []; + for (let i = 0; i < 16; i += 2) { + const high = Number(address[i]) || 0; + const low = Number(address[i + 1]) || 0; + const value = ((high << 8) | low) >>> 0; + groups.push(value.toString(16)); + } + return groups.join(":"); + } + if (address.length === 8) { + return address.map((part) => Number(part).toString(16)).join(":"); + } + return address.map((part) => String(part)).join(":"); + } + return address.map((part) => String(part)).join("."); + } + return String(address); +} + +function formatIpTuple(tuple, defaultPrefix) { + if (!tuple || tuple.length === 0) return ""; + const family = defaultPrefix === 128 ? 6 : 4; + const addr = formatIpAddress(tuple[0], family); + const prefix = Number.isFinite(tuple[1]) ? tuple[1] : defaultPrefix; + if (!addr) return ""; + return `${addr}/${prefix}`; +} + function registerMetricEl(circuitId, metricName, el) { if (!circuitId || circuitId === "") return; let metrics = metricElsByCircuitId.get(circuitId); @@ -138,17 +176,18 @@ function applyCircuitUpdate(device) { if (!device || !device.circuit_id) return; const circuitId = device.circuit_id; latestByCircuitId.set(circuitId, device); + const plan = planByCircuitId.get(circuitId) || {down: 0, up: 0}; updateMetricText(circuitId, "lastSeen", formatLastSeen(device.last_seen_nanos)); updateMetricHtml( circuitId, "tpDown", - formatThroughput(toNumber(device.bytes_per_second.down, 0) * 8, device.plan.down), + formatThroughput(toNumber(device.bytes_per_second.down, 0) * 8, toNumber(plan.down, 0)), ); updateMetricHtml( circuitId, "tpUp", - formatThroughput(toNumber(device.bytes_per_second.up, 0) * 8, device.plan.up), + formatThroughput(toNumber(device.bytes_per_second.up, 0) * 8, toNumber(plan.up, 0)), ); updateMetricHtml( circuitId, @@ -163,12 +202,8 @@ function applyCircuitUpdate(device) { updateMetricHtml(circuitId, "qooDown", formatQooScore(device.qoo ? device.qoo.down : null)); updateMetricHtml(circuitId, "qooUp", formatQooScore(device.qoo ? device.qoo.up : null)); - const packetsDown = toNumber(device.tcp_packets.down, 0); - const packetsUp = toNumber(device.tcp_packets.up, 0); - const retransDown = toNumber(device.tcp_retransmits.down, 0); - const retransUp = toNumber(device.tcp_retransmits.up, 0); - const fractionDown = packetsDown > 0 ? retransDown / packetsDown : 0; - const fractionUp = packetsUp > 0 ? retransUp / packetsUp : 0; + const fractionDown = retransmitFractionFromSample(device.tcp_retransmit_sample?.down); + const fractionUp = retransmitFractionFromSample(device.tcp_retransmit_sample?.up); updateMetricHtml(circuitId, "reXmitDown", formatRetransmit(fractionDown)); updateMetricHtml(circuitId, "reXmitUp", formatRetransmit(fractionUp)); } @@ -183,12 +218,14 @@ function buildIpListEl(device) { }; if (Array.isArray(device.ipv4)) { device.ipv4.forEach((ip) => { - if (ip && ip.length >= 2) addLine(ip[0] + "/" + ip[1]); + const formatted = formatIpTuple(ip, 32); + if (formatted) addLine(formatted); }); } if (Array.isArray(device.ipv6)) { device.ipv6.forEach((ip) => { - if (ip && ip.length >= 2) addLine(ip[0] + "/" + ip[1]); + const formatted = formatIpTuple(ip, 128); + if (formatted) addLine(formatted); }); } if (wrapper.children.length === 0) { @@ -439,7 +476,7 @@ function ensureLayout() { devicesPerPage = 24; } page = 0; - renderCards(); + requestShapedDevicesPage(); }; perPageWrap.appendChild(perPageLabel); perPageWrap.appendChild(perPageSelect); @@ -459,7 +496,7 @@ function ensureLayout() { prev.title = "Previous page"; prev.onclick = () => { page = Math.max(0, page - 1); - renderCards(); + requestShapedDevicesPage(); }; const next = document.createElement("button"); next.id = "sdNextPage"; @@ -469,9 +506,9 @@ function ensureLayout() { next.setAttribute("aria-label", "Next devices page"); next.title = "Next page"; next.onclick = () => { - const totalPages = Math.max(1, Math.ceil(displayDevices.length / devicesPerPage)); + const totalPages = Math.max(1, Math.ceil(totalRows / devicesPerPage)); page = Math.min(totalPages - 1, page + 1); - renderCards(); + requestShapedDevicesPage(); }; pager.appendChild(prev); pager.appendChild(next); @@ -509,11 +546,85 @@ function ensureLayout() { }; } +function currentShapedDevicesPageQuery() { + const query = { + page, + page_size: devicesPerPage, + }; + if (searchTerm && searchTerm.trim() !== "") { + query.search = searchTerm; + } + return query; +} + +function visibleCircuitIds() { + const seen = new Set(); + const ids = []; + shapedDevices.forEach((device) => { + const id = device && device.circuit_id ? String(device.circuit_id).trim() : ""; + if (!id || seen.has(id)) return; + seen.add(id); + ids.push(id); + }); + return ids; +} + +function requestCircuitMetricsWatch(force = false) { + const circuitIds = visibleCircuitIds(); + const signature = JSON.stringify(circuitIds); + if (!force && circuitMetricsWatchSignature === signature) { + return; + } + circuitMetricsWatchSignature = signature; + if (circuitIds.length === 0) { + sendPrivateRequest({StopCircuitMetricsWatch: null}); + return; + } + sendPrivateRequest({ + WatchCircuitMetrics: { + query: { + circuit_ids: circuitIds, + }, + }, + }); +} + +async function requestShapedDevicesPage() { + try { + const msg = await sendWsRequest("ShapedDevicesPage", { + ShapedDevicesPage: { + query: currentShapedDevicesPageQuery(), + }, + }); + const data = msg && msg.data ? msg.data : {}; + shapedDevices = Array.isArray(data.rows) ? data.rows : []; + totalRows = Number.isFinite(Number(data.total_rows)) ? Number(data.total_rows) : shapedDevices.length; + totalCircuits = Number.isFinite(Number(data.total_circuits)) ? Number(data.total_circuits) : countCircuits(); + planByCircuitId.clear(); + shapedDevices.forEach((device) => { + if (!device || !device.circuit_id) return; + const current = planByCircuitId.get(device.circuit_id) || {down: 0, up: 0}; + current.down = Math.max(toNumber(current.down, 0), toNumber(device.download_max_mbps, 0)); + current.up = Math.max(toNumber(current.up, 0), toNumber(device.upload_max_mbps, 0)); + planByCircuitId.set(device.circuit_id, current); + }); + renderCards(); + requestCircuitMetricsWatch(true); + $("#count").text(totalRows + " devices"); + $("#countCircuit").text(totalCircuits + " circuits"); + } catch (_error) { + shapedDevices = []; + totalRows = 0; + totalCircuits = 0; + renderCards(); + } +} + function renderCards() { const layout = ensureLayout(); if (!layout) return; - const totalPages = Math.max(1, Math.ceil(displayDevices.length / devicesPerPage)); + const totalPages = Math.max(1, Math.ceil(totalRows / devicesPerPage)); if (page >= totalPages) page = totalPages - 1; if (page < 0) page = 0; @@ -524,27 +635,24 @@ function renderCards() { if (layout.nextButton) layout.nextButton.disabled = page >= totalPages - 1; if (layout.pageCounter) layout.pageCounter.innerText = "Page " + (page + 1) + " / " + totalPages; if (layout.summary) { - if (displayDevices.length === 0) { + if (totalRows === 0) { layout.summary.innerText = "No matches"; } else { const start = page * devicesPerPage + 1; - const end = Math.min((page + 1) * devicesPerPage, displayDevices.length); - layout.summary.innerText = "Showing " + start + "–" + end + " of " + displayDevices.length; + const end = Math.min((page + 1) * devicesPerPage, totalRows); + layout.summary.innerText = "Showing " + start + "–" + end + " of " + totalRows; } } metricElsByCircuitId = new Map(); clearDiv(layout.grid); - const start = page * devicesPerPage; - const end = Math.min(start + devicesPerPage, displayDevices.length); - for (let i = start; i < end; i++) { - const device = displayDevices[i]; + shapedDevices.forEach((device) => { const col = document.createElement("div"); col.classList.add("col"); col.appendChild(buildDeviceCard(device)); layout.grid.appendChild(col); - } + }); // Fill visible cards from cached live data immediately. metricElsByCircuitId.forEach((_metrics, circuitId) => { @@ -557,24 +665,18 @@ function renderCards() { initTooltipsWithin(layout.target); } -function loadDevices() { - listenOnce("DevicesAll", (msg) => { - const data = msg && msg.data ? msg.data : []; - shapedDevices = data; - displayDevices = data; - renderCards(); - $("#count").text(shapedDevices.length + " devices"); - $("#countCircuit").text(countCircuits() + " circuits"); +function handleCircuitMetrics(msg) { + const metrics = msg && Array.isArray(msg.data) ? msg.data : []; + metrics.forEach((metric) => { + if (!metric || !metric.circuit_id) return; + applyCircuitUpdate(metric); }); - wsClient.send({ DevicesAll: {} }); } -loadDevices(); -subscribeWS(["NetworkTreeClients"], (msg) => { - if (msg.event !== "NetworkTreeClients") return; - const data = msg && msg.data ? msg.data : []; - data.forEach((d) => { - if (!d || !d.circuit_id) return; - applyCircuitUpdate(d); - }); +wsClient.on("CircuitMetricsSnapshot", handleCircuitMetrics); +wsClient.on("CircuitMetricsUpdate", handleCircuitMetrics); +wsClient.on("join", () => { + requestShapedDevicesPage(); }); + +requestShapedDevicesPage(); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/site_map.js b/src/rust/lqosd/src/node_manager/js_build/src/site_map.js index 535676b5..7e17c236 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/site_map.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/site_map.js @@ -1,6 +1,7 @@ import { colorByQoqScore, colorByRttMs } from "./helpers/color_scales"; import { isColorBlindMode } from "./helpers/colorblind"; import { isDarkMode } from "./helpers/dark_mode"; +import { isRedacted } from "./helpers/redact"; import { scaleNumber, toNumber } from "./lq_js_common/helpers/scaling"; import { get_ws_client, subscribeWS } from "./pubsub/ws"; @@ -8,44 +9,158 @@ const wsClient = get_ws_client(); const INITIAL_REQUEST_TIMEOUT_MS = 2500; const HISTORY_WINDOW_MS = 30_000; -const COUNTRIES_GEOJSON_PATH = "vendor/countries.geojson"; -const ADMIN1_BOUNDARIES_GEOJSON_PATH = "vendor/site_map_admin1_boundaries.geojson"; -const COASTLINES_GEOJSON_PATH = "vendor/site_map_coastlines.geojson"; -const LAKES_GEOJSON_PATH = "vendor/site_map_lakes.geojson"; -const RIVERS_GEOJSON_PATH = "vendor/site_map_rivers.geojson"; -const PHYSICAL_REGIONS_GEOJSON_PATH = "vendor/site_map_physical_regions.geojson"; -const PHYSICAL_REGIONS_10M_GEOJSON_PATH = "vendor/site_map_physical_regions_10m.geojson"; -const MARINE_AREAS_GEOJSON_PATH = "vendor/site_map_marine_areas.geojson"; -const MAJOR_ROADS_10M_GEOJSON_PATH = "vendor/site_map_major_roads_10m.geojson"; -const INITIAL_CENTER = [-101.5, 39.8]; -const INITIAL_ZOOM = 3.15; + +const TILE_BBOX_URL = "https://insight.libreqos.com/tiles/api/bbox"; +// NOTE: This key is intentionally non-secret. The remote OSM cache uses it as a lightweight gate. +// It must match both the bbox Authorization token and the tile `key=` query param. +const OSM_CACHE_KEY = "LibreQoSRocks"; +const WEB_MERCATOR_MAX_LAT = 85.05112878; +const INSIGHT_TILE_PROTOCOL = "insight"; +// Insight OSM cache tiles: `z/y/x.png` (Cloudflare cache-friendly). See `src/lts2/rust/osm_cache/src/http.rs`. +// The Insight tile server returns `503 + Retry-After` while it fetches missing tiles, so we route +// requests through a custom MapLibre protocol handler that retries before surfacing errors. +const TILE_URL_TEMPLATE = `${INSIGHT_TILE_PROTOCOL}://insight.libreqos.com/tiles/{z}/{y}/{x}.png?key=${encodeURIComponent(OSM_CACHE_KEY)}`; +const TILE_ATTRIBUTION = "© OpenStreetMap contributors"; +const TILE_MAX_ZOOM = 17; + +const INITIAL_FIT_PADDING = 70; +const INITIAL_FIT_MAX_ZOOM = 11.5; +const SINGLE_POINT_INITIAL_ZOOM = 11.5; +const SITE_LABEL_MIN_ZOOM = 6; +const SELECTED_SITE_LABEL_MIN_ZOOM = 4; +const MAX_SITE_LABELS = 24; + +const urlParams = new URLSearchParams(window.location.search); +const FIXTURE_MODE = urlParams.get("fixture") === "1"; +const FIXTURE_NETWORK_TREE = [ + [0, { + id: "site-central", + name: "CENTRAL", + type: "Site", + immediate_parent: null, + latitude: 39.0997, + longitude: -92.2196, + configured_max_throughput: [2000, 800], + current_throughput: [20000000, 4000000], + qoo: [96.5, 97.2], + rtts: [18.4, 22.1], + }], + [1, { + id: "site-north", + name: "NORTH", + type: "Site", + immediate_parent: 0, + latitude: 39.2564, + longitude: -92.1842, + configured_max_throughput: [1500, 400], + current_throughput: [80000000, 12000000], + qoo: [88.0, 84.0], + rtts: [35.0, 41.0], + }], + [2, { + id: "site-east", + name: "EAST", + type: "Site", + immediate_parent: 0, + latitude: 39.0482, + longitude: -91.9631, + configured_max_throughput: [500, 100], + current_throughput: [6000000, 1000000], + qoo: [62.0, 70.0], + rtts: [120.0, 165.0], + }], + [3, { + id: "site-rural", + name: "RURAL", + type: "Site", + immediate_parent: 1, + latitude: 39.3689, + longitude: -92.4509, + configured_max_throughput: [100, 20], + current_throughput: [1200000, 240000], + qoo: [41.0, 52.0], + rtts: [55.0, 60.0], + }], + [4, { + id: "site-unmapped", + name: "UNMAPPED", + type: "Site", + immediate_parent: 0, + configured_max_throughput: [50, 10], + current_throughput: [0, 0], + qoo: [null, null], + rtts: [], + }], + [5, { + id: "ap-north-1", + name: "NORTH AP 1", + type: "AP", + immediate_parent: 1, + latitude: 39.2753, + longitude: -92.2116, + current_throughput: [15000000, 2000000], + qoo: [92.0, 90.0], + rtts: [28.0, 33.0], + }], + [6, { + id: "ap-north-2", + name: "NORTH AP 2", + type: "AP", + immediate_parent: 1, + current_throughput: [10000000, 1000000], + qoo: [null, 80.0], + rtts: [25.0, 29.0], + }], + [7, { + id: "ap-east-1", + name: "EAST AP", + type: "AP", + immediate_parent: 2, + latitude: 39.0349, + longitude: -91.9387, + current_throughput: [3000000, 400000], + qoo: [55.0, 60.0], + rtts: [180.0, 220.0], + }], +]; + +const OSM_RASTER_SOURCE_ID = "site-map-osm"; +const OSM_RASTER_LAYER_ID = "site-map-osm-tiles"; const SITE_SOURCE_ID = "site-map-sites"; const AP_SOURCE_ID = "site-map-aps"; -const COUNTRIES_SOURCE_ID = "site-map-countries"; -const ADMIN1_SOURCE_ID = "site-map-admin1"; -const COASTLINE_SOURCE_ID = "site-map-coastlines"; -const LAKES_SOURCE_ID = "site-map-lakes"; -const RIVERS_SOURCE_ID = "site-map-rivers"; -const PHYSICAL_REGIONS_SOURCE_ID = "site-map-physical-regions"; -const PHYSICAL_REGIONS_10M_SOURCE_ID = "site-map-physical-regions-10m"; -const MARINE_AREAS_SOURCE_ID = "site-map-marine-areas"; -const MAJOR_ROADS_10M_SOURCE_ID = "site-map-major-roads-10m"; +const SITE_LINK_SOURCE_ID = "site-map-site-links"; +const FANOUT_LINE_SOURCE_ID = "site-map-fanout-lines"; +const FANOUT_POINT_SOURCE_ID = "site-map-fanout-points"; const SITE_CLUSTER_LAYER_ID = "site-map-site-clusters"; const SITE_POINTS_LAYER_ID = "site-map-site-points"; -const AP_CLUSTER_LAYER_ID = "site-map-ap-clusters"; const AP_POINTS_LAYER_ID = "site-map-ap-points"; -const COUNTRY_FILL_LAYER_ID = "site-map-country-fill"; -const COUNTRY_LINE_LAYER_ID = "site-map-country-line"; -const ADMIN1_LINE_LAYER_ID = "site-map-admin1-line"; -const COASTLINE_LAYER_ID = "site-map-coastline-line"; -const LAKES_FILL_LAYER_ID = "site-map-lakes-fill"; -const RIVERS_LINE_LAYER_ID = "site-map-rivers-line"; -const PHYSICAL_REGIONS_FILL_LAYER_ID = "site-map-physical-regions-fill"; -const PHYSICAL_REGIONS_LINE_LAYER_ID = "site-map-physical-regions-line"; -const PHYSICAL_REGIONS_10M_FILL_LAYER_ID = "site-map-physical-regions-10m-fill"; -const PHYSICAL_REGIONS_10M_LINE_LAYER_ID = "site-map-physical-regions-10m-line"; -const MARINE_AREAS_FILL_LAYER_ID = "site-map-marine-areas-fill"; -const MAJOR_ROADS_10M_LAYER_ID = "site-map-major-roads-10m-line"; +const FANOUT_LINE_LAYER_ID = "site-map-fanout-lines-layer"; +const FANOUT_POINT_LAYER_ID = "site-map-fanout-points-layer"; +const SITE_LINK_LAYER_ID = "site-map-site-links-line"; + +const INSIGHT_TILE_MAX_PARALLEL_FETCHES = 2; +let insightTileFetchActive = 0; +const insightTileFetchWaiters = []; +let insightTileRateLimitUntil = 0; + +async function withInsightTileFetchSlot(fn) { + while (Date.now() < insightTileRateLimitUntil) { + await sleepMs(Math.min(30_000, insightTileRateLimitUntil - Date.now())); + } + if (insightTileFetchActive >= INSIGHT_TILE_MAX_PARALLEL_FETCHES) { + await new Promise((resolve) => insightTileFetchWaiters.push(resolve)); + } + insightTileFetchActive += 1; + try { + return await fn(); + } finally { + insightTileFetchActive = Math.max(0, insightTileFetchActive - 1); + const next = insightTileFetchWaiters.shift(); + if (next) { + next(); + } + } +} function listenOnceWithTimeout(eventName, timeoutMs, handler, onTimeout) { let done = false; @@ -74,6 +189,25 @@ function escapeHtml(value) { .replace(/'/g, "'"); } +function parseRetryAfterMs(headerValue) { + if (headerValue === null || headerValue === undefined) { + return null; + } + const raw = String(headerValue).trim(); + if (!raw) { + return null; + } + const seconds = Number(raw); + if (Number.isFinite(seconds)) { + return Math.max(0, Math.round(seconds * 1000)); + } + const parsed = Date.parse(raw); + if (!Number.isFinite(parsed)) { + return null; + } + return Math.max(0, parsed - Date.now()); +} + function formatBitsPerSecond(bitsPerSecond) { return scaleNumber(Math.max(0, toNumber(bitsPerSecond, 0)), 1); } @@ -94,6 +228,13 @@ function formatMs(value, digits = 1) { return `${numeric.toFixed(digits)} ms`; } +function rectsOverlap(left, right) { + return !(left.right <= right.left + || left.left >= right.right + || left.bottom <= right.top + || left.top >= right.bottom); +} + function nowText(ts) { if (!ts) return "Not updated yet"; const elapsed = Math.max(0, Math.round((Date.now() - ts) / 1000)); @@ -121,10 +262,49 @@ function worstRtt(rttDownMs, rttUpMs) { return Math.max(...values); } +function metricColorForMode(mode, metricValue) { + if (mode === "qoo") { + return colorByQoqScore(metricValue); + } + if (metricValue === null || metricValue === undefined) { + return "#8893a5"; + } + const numeric = Number(metricValue); + if (!Number.isFinite(numeric)) { + return "#8893a5"; + } + return colorByRttMs(numeric); +} + +function configuredMaxThroughput(node) { + return node?.configured_max_throughput || node?.max_throughput || [0, 0]; +} + +function effectiveMaxThroughput(node) { + return node?.effective_max_throughput || configuredMaxThroughput(node); +} + function averageOrNull(sum, count) { return count > 0 ? (sum / count) : null; } +function hasMeaningfulLatLon(lat, lon) { + if (!Number.isFinite(lat) || !Number.isFinite(lon)) { + return false; + } + // Many integrations default to (0,0) when coordinates are unknown. + if (lat === 0 && lon === 0) { + return false; + } + if (lat < -WEB_MERCATOR_MAX_LAT || lat > WEB_MERCATOR_MAX_LAT) { + return false; + } + if (lon < -180 || lon > 180) { + return false; + } + return true; +} + function stableNodeKey(index, node) { return node.id || `${index}:${node.name}`; } @@ -133,103 +313,350 @@ function asNodeType(node) { return String(node?.type || node?.node_type || "").toLowerCase(); } -function countryPalette() { +function immediateParentIndex(node) { + const raw = node?.immediate_parent ?? node?.immediateParent; + if (raw === null || raw === undefined) { + return null; + } + const numeric = Number(raw); + return Number.isFinite(numeric) ? numeric : null; +} + +function displayNodeName(name, nodeType) { + if (!name) { + return ""; + } + return isRedacted() && String(nodeType || "").toLowerCase() === "site" + ? "[redacted]" + : name; +} + +function displayParentName(name, parentType) { + if (!name) { + return ""; + } + return isRedacted() && String(parentType || "").toLowerCase() === "site" + ? "[redacted]" + : name; +} + +function findNearestAncestorSiteIndex(indexMap, startIndex) { + let currentIndex = startIndex; + const visited = new Set(); + while (currentIndex !== null && currentIndex !== undefined) { + const numeric = Number(currentIndex); + if (!Number.isFinite(numeric)) { + return null; + } + if (visited.has(numeric)) { + return null; + } + visited.add(numeric); + const node = indexMap.get(numeric); + if (!node) { + return null; + } + if (asNodeType(node) === "site") { + return numeric; + } + currentIndex = immediateParentIndex(node); + } + return null; +} + +function markerPalette() { if (isDarkMode()) { return { - background: "#09111d", - land: "#132339", - water: "#102945", - marine: "rgba(16, 34, 56, 0.42)", - terrain: "rgba(94, 118, 148, 0.12)", - roads: "rgba(147, 180, 226, 0.34)", - coast: "rgba(170, 203, 255, 0.56)", - admin1: "rgba(125, 155, 198, 0.22)", - rivers: "rgba(122, 174, 240, 0.36)", - borders: "rgba(160, 194, 255, 0.82)", siteStroke: "rgba(244, 248, 255, 0.68)", apStroke: "rgba(244, 248, 255, 0.54)", - popupBg: "rgba(11, 18, 30, 0.96)", - popupBorder: "rgba(148, 163, 184, 0.3)", + link: "rgba(15, 23, 42, 0.35)", }; } return { - background: "#d8e5f1", - land: "#eef4fb", - water: "#ccdeee", - marine: "rgba(194, 214, 234, 0.46)", - terrain: "rgba(118, 136, 158, 0.08)", - roads: "rgba(109, 135, 171, 0.28)", - coast: "rgba(71, 101, 141, 0.44)", - admin1: "rgba(77, 102, 135, 0.18)", - rivers: "rgba(83, 128, 185, 0.30)", - borders: "rgba(61, 90, 130, 0.72)", siteStroke: "rgba(15, 23, 42, 0.36)", apStroke: "rgba(15, 23, 42, 0.28)", - popupBg: "rgba(255, 255, 255, 0.96)", - popupBorder: "rgba(15, 23, 42, 0.12)", + link: "rgba(15, 23, 42, 0.35)", }; } -function buildMapStyle() { - const palette = countryPalette(); +function linkColorExpression() { + const stops = isDarkMode() + ? [ + 0.0, "rgba(76, 164, 255, 0.40)", + 0.55, "rgba(245, 184, 79, 0.52)", + 1.0, "rgba(217, 75, 91, 0.62)", + ] + : [ + 0.0, "rgba(76, 164, 255, 0.32)", + 0.55, "rgba(245, 184, 79, 0.44)", + 1.0, "rgba(217, 75, 91, 0.54)", + ]; + return [ + "interpolate", + ["linear"], + ["coalesce", ["get", "utilizationRatio"], 0], + ...stops, + ]; +} + +function linkWidthExpression() { + const ratio = ["min", 1, ["max", 0, ["coalesce", ["get", "trafficRatio"], 0]]]; + const trafficFactor = ["interpolate", ["exponential", 1.6], ratio, 0, 1.0, 1, 2.0]; + // MapLibre only permits zoom expressions at the top level of a step/interpolate. Keep it there. + const w2 = ["*", 1.8, trafficFactor]; + const w6 = ["*", 3.2, trafficFactor]; + const w10 = ["*", 4.6, trafficFactor]; + return ["interpolate", ["linear"], ["zoom"], 2, w2, 6, w6, 10, w10]; +} + +function rasterThemePaint() { + if (isDarkMode()) { + return { + "raster-opacity": 0.9, + "raster-brightness-min": 0.02, + "raster-brightness-max": 0.24, + "raster-contrast": 0.38, + "raster-saturation": -0.5, + "raster-hue-rotate": 208, + }; + } + return { + "raster-opacity": 1, + "raster-brightness-min": 0, + "raster-brightness-max": 1, + "raster-contrast": 0, + "raster-saturation": 0, + "raster-hue-rotate": 0, + }; +} + +function buildOsmRasterStyle() { + const rasterPaint = rasterThemePaint(); return { version: 8, - sources: {}, + sources: { + [OSM_RASTER_SOURCE_ID]: { + type: "raster", + tiles: [TILE_URL_TEMPLATE], + tileSize: 256, + attribution: TILE_ATTRIBUTION, + maxzoom: TILE_MAX_ZOOM, + }, + }, layers: [ { id: "site-map-background", type: "background", paint: { - "background-color": palette.background, + "background-color": isDarkMode() ? "#0b1220" : "#ffffff", }, }, + { + id: OSM_RASTER_LAYER_ID, + type: "raster", + source: OSM_RASTER_SOURCE_ID, + minzoom: 0, + maxzoom: TILE_MAX_ZOOM, + paint: rasterPaint, + }, ], }; } -function qooClusterColorExpression() { - return [ - "case", - [">", ["get", "qooCount"], 0], - [ - "step", - ["/", ["get", "qooSum"], ["get", "qooCount"]], - "#d94b5b", - 45, "#f5b84f", - 75, "#6ecc84", - 90, "#4ca4ff", - ], - "#8893a5", - ]; +function sleepMs(ms) { + return new Promise((resolve) => window.setTimeout(resolve, ms)); } -function rttClusterColorExpression() { - return [ - "case", - [">", ["get", "rttCount"], 0], - [ - "step", - ["/", ["get", "rttSum"], ["get", "rttCount"]], - "#61d26f", - 40, "#d7d86b", - 90, "#f0a44d", - 160, "#d94b5b", - ], - "#8893a5", - ]; +function installInsightTileProtocolOnce() { + if (!window.maplibregl?.addProtocol) { + return; + } + if (window.__lqosInsightTileProtocolInstalled) { + return; + } + window.__lqosInsightTileProtocolInstalled = true; + + window.maplibregl.addProtocol(INSIGHT_TILE_PROTOCOL, (params, abortControllerOrCallback) => { + const url = String(params?.url ?? "").replace(`${INSIGHT_TILE_PROTOCOL}://`, "https://"); + const responseType = String(params?.type || "arrayBuffer"); + const method = params?.method || "GET"; + const body = params?.body; + + const makeHeaders = () => { + try { + return new Headers(params?.headers ?? {}); + } catch (_) { + return new Headers(); + } + }; + + const fetchWithRetries = async (signal) => { + const deadlineMs = Date.now() + (5 * 60_000); + let consecutive429 = 0; + while (Date.now() < deadlineMs) { + if (signal?.aborted) { + throw new Error("AbortError"); + } + const headers = makeHeaders(); + if (responseType === "json" && !headers.has("Accept")) { + headers.set("Accept", "application/json"); + } + const resp = await withInsightTileFetchSlot(() => fetch(url, { + method, + body, + headers, + credentials: "omit", + cache: params?.cache, + signal, + })); + + if (resp.status === 429) { + consecutive429 += 1; + const retryAfterMs = parseRetryAfterMs(resp.headers.get("retry-after")); + const fallbackDelayMs = 10_000 * Math.pow(2, Math.min(consecutive429 - 1, 4)); + const baseDelayMs = Math.max(retryAfterMs ?? 0, fallbackDelayMs); + const delayMs = Math.min(Math.max(baseDelayMs, 500), 180_000) + Math.round(Math.random() * 1000); + insightTileRateLimitUntil = Math.max(insightTileRateLimitUntil, Date.now() + delayMs); + await sleepMs(Math.max(0, insightTileRateLimitUntil - Date.now())); + if (signal?.aborted) { + throw new Error("AbortError"); + } + continue; + } + + if (resp.status === 503) { + consecutive429 = 0; + const retryAfterMs = parseRetryAfterMs(resp.headers.get("retry-after")); + const baseDelayMs = retryAfterMs ?? 1000; + const delayMs = Math.min(Math.max(baseDelayMs, 250), 15_000) + Math.round(Math.random() * 250); + await sleepMs(delayMs); + if (signal?.aborted) { + throw new Error("AbortError"); + } + continue; + } + + consecutive429 = 0; + + if (!resp.ok) { + throw new Error(`tile request failed: ${resp.status}`); + } + + let data; + if (responseType === "json") { + data = await resp.json(); + } else if (responseType === "text") { + data = await resp.text(); + } else { + data = await resp.arrayBuffer(); + } + + if (signal?.aborted) { + throw new Error("AbortError"); + } + + return { + data, + cacheControl: resp.headers.get("cache-control") ?? undefined, + expires: resp.headers.get("expires") ?? undefined, + }; + } + throw new Error("tile request retries exhausted"); + }; + + // MapLibre's protocol handler signature differs by version: + // - newer: (params, abortController) => Promise<{data, cacheControl, expires}> + // - older: (params, callback) => { cancel() } + if (typeof abortControllerOrCallback === "function") { + const callback = abortControllerOrCallback; + const controller = new AbortController(); + fetchWithRetries(controller.signal) + .then((result) => callback(null, result.data, result.cacheControl, result.expires)) + .catch((err) => { + if (controller.signal.aborted) return; + callback(err); + }); + return { cancel: () => controller.abort() }; + } + + const signal = abortControllerOrCallback?.signal; + return fetchWithRetries(signal); + }); } -function clusterRadiusExpression() { - return [ - "step", - ["get", "throughputSum"], - 18, - 5_000_000, 22, - 25_000_000, 28, - 100_000_000, 36, - 500_000_000, 46, - 1_000_000_000, 56, - ]; +function normalizeBboxResponse(data) { + // Newer/expected shape (see osm_cache): { center: { lat, lon }, zoom } + if (data && typeof data === "object" && data.center && typeof data.center === "object") { + const lat = Number(data.center.lat ?? data.center.latitude); + const lon = Number(data.center.lon ?? data.center.lng ?? data.center.longitude); + const zoom = Number(data.zoom); + return { lat, lon, zoom }; + } + if (Array.isArray(data) && data.length >= 3) { + const [lat, lon, zoom] = data; + return { lat: Number(lat), lon: Number(lon), zoom: Number(zoom) }; + } + if (data && typeof data === "object") { + const lat = Number(data.lat ?? data.latitude); + const lon = Number(data.lon ?? data.lng ?? data.longitude); + const zoom = Number(data.zoom); + return { lat, lon, zoom }; + } + return null; +} + +async function requestOsmCenterFromBbox(siteLatLonPairs, timeoutMs = 2500) { + if (!Array.isArray(siteLatLonPairs) || siteLatLonPairs.length === 0) { + throw new Error("No site coordinates supplied"); + } + + const points = siteLatLonPairs + .map((pair) => { + if (!Array.isArray(pair) || pair.length < 2) { + return null; + } + const [lat, lon] = pair; + const latN = Number(lat); + const lonN = Number(lon); + if (!hasMeaningfulLatLon(latN, lonN)) { + return null; + } + return { lat: latN, lon: lonN }; + }) + .filter((item) => item !== null); + + if (points.length === 0) { + throw new Error("No valid site coordinates supplied"); + } + + const controller = new AbortController(); + const timeoutId = window.setTimeout(() => controller.abort(), timeoutMs); + + try { + const resp = await fetch(TILE_BBOX_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${OSM_CACHE_KEY}`, + }, + body: JSON.stringify(points), + signal: controller.signal, + }); + if (!resp.ok) { + throw new Error(`bbox request failed: ${resp.status}`); + } + const json = await resp.json(); + const normalized = normalizeBboxResponse(json); + if (!normalized + || !Number.isFinite(normalized.lat) + || !Number.isFinite(normalized.lon) + || !Number.isFinite(normalized.zoom)) { + throw new Error("bbox returned invalid center"); + } + return normalized; + } finally { + window.clearTimeout(timeoutId); + } } class SiteMapPage { @@ -245,9 +672,15 @@ class SiteMapPage { this.lastUpdateAt = 0; this.latestRender = null; this.unmappedOpen = false; - this.heavyLayersInstalled = false; - this.heavyLayersInstallQueued = false; - this.heavyLayersFallbackTimer = null; + this.mapInitPromise = null; + this.mapBootstrapped = false; + this.attachedInheritedApsBySite = new Map(); + this.fanoutSiteKey = null; + this.lastBboxAttemptAt = 0; + this.fixtureMode = FIXTURE_MODE; + this.siteLabelMarkers = new Map(); + this.clusterCountMarkers = new Map(); + this.pendingTextOverlayFrame = null; this.canvas = document.getElementById("siteMapCanvas"); this.statusChip = document.getElementById("siteMapStatusChip"); @@ -259,17 +692,24 @@ class SiteMapPage { this.detailsPanel = document.getElementById("siteMapDetails"); this.detailsTitle = document.getElementById("siteMapDetailsTitle"); this.detailsSubtitle = document.getElementById("siteMapDetailsSubtitle"); + this.attachedList = document.getElementById("siteMapAttachedList"); this.detailsGrid = document.getElementById("siteMapDetailsGrid"); this.legendGradient = document.getElementById("siteMapLegendGradient"); this.legendLow = document.getElementById("siteMapLegendLow"); this.legendHigh = document.getElementById("siteMapLegendHigh"); + this.sizeDotSmall = document.getElementById("siteMapSizeDotSmall"); + this.sizeDotMedium = document.getElementById("siteMapSizeDotMedium"); + this.sizeDotLarge = document.getElementById("siteMapSizeDotLarge"); } init() { this.bindControls(); this.refreshLegend(); - this.initMap(); - this.requestInitialTree(); + if (this.fixtureMode) { + this.processTreeMessage({ data: FIXTURE_NETWORK_TREE }, { fixture: true }); + } else { + this.requestInitialTree(); + } this.startUpdatedClock(); this.observeThemeChanges(); } @@ -296,22 +736,42 @@ class SiteMapPage { }); document.getElementById("siteMapDetailsClose")?.addEventListener("click", () => { this.selectedFeature = null; + this.fanoutSiteKey = null; + this.syncFanoutOverlays(); + this.scheduleTextOverlaySync(); this.renderDetails(null); }); window.addEventListener("colorBlindModeChanged", () => { this.refreshLegend(); this.renderFromHistory(); }); + window.addEventListener("redact-change", () => { + this.popup?.remove(); + this.renderFromHistory(); + }); } - initMap() { + initMap(center, zoom) { + if (!Array.isArray(center) + || center.length < 2 + || !Number.isFinite(center[0]) + || !Number.isFinite(center[1]) + || !Number.isFinite(zoom)) { + throw new Error("Invalid map center/zoom"); + } + + installInsightTileProtocolOnce(); + if (typeof window.maplibregl?.setMaxParallelImageRequests === "function") { + window.maplibregl.setMaxParallelImageRequests(INSIGHT_TILE_MAX_PARALLEL_FETCHES); + } this.map = new window.maplibregl.Map({ container: this.canvas, - style: buildMapStyle(), - center: INITIAL_CENTER, - zoom: INITIAL_ZOOM, + style: buildOsmRasterStyle(), + center, + zoom, + maxZoom: TILE_MAX_ZOOM, attributionControl: false, - customAttribution: "Natural Earth", + customAttribution: "LibreQoS Insight tile cache", }); this.map.addControl(new window.maplibregl.NavigationControl({ visualizePitch: false }), "bottom-left"); this.map.addControl(new window.maplibregl.AttributionControl({ compact: true }), "bottom-left"); @@ -324,13 +784,30 @@ class SiteMapPage { className: "site-map-popup", }); - this.map.on("load", () => { - this.installPointSourcesAndLayers(); - this.installInteractions(); + const bootstrapOverlays = () => { + if (!this.map) { + return; + } + try { + this.installSourcesAndLayers(); + } catch (err) { + return; + } + if (!this.map.getSource(SITE_SOURCE_ID) || !this.map.getLayer(SITE_POINTS_LAYER_ID)) { + return; + } + if (!this.mapBootstrapped) { + this.installInteractions(); + this.mapBootstrapped = true; + } this.applyTheme(); this.renderFromHistory(); - this.queueHeavyLayersInstall(); - }); + }; + + // `load` can be delayed while raster tiles are still coming in; `styledata` fires earlier. + this.map.on("load", bootstrapOverlays); + this.map.on("styledata", bootstrapOverlays); + bootstrapOverlays(); } observeThemeChanges() { @@ -348,43 +825,16 @@ class SiteMapPage { if (!this.map || !this.map.isStyleLoaded()) { return; } - const palette = countryPalette(); - this.map.setPaintProperty("site-map-background", "background-color", palette.background); - if (this.map.getLayer(COUNTRY_FILL_LAYER_ID)) { - this.map.setPaintProperty(COUNTRY_FILL_LAYER_ID, "fill-color", palette.land); + const palette = markerPalette(); + this.map.setPaintProperty("site-map-background", "background-color", isDarkMode() ? "#0b1220" : "#ffffff"); + if (this.map.getLayer(OSM_RASTER_LAYER_ID)) { + const rasterPaint = rasterThemePaint(); + Object.entries(rasterPaint).forEach(([key, value]) => { + this.map.setPaintProperty(OSM_RASTER_LAYER_ID, key, value); + }); } - if (this.map.getLayer(MARINE_AREAS_FILL_LAYER_ID)) { - this.map.setPaintProperty(MARINE_AREAS_FILL_LAYER_ID, "fill-color", palette.marine); - } - if (this.map.getLayer(PHYSICAL_REGIONS_FILL_LAYER_ID)) { - this.map.setPaintProperty(PHYSICAL_REGIONS_FILL_LAYER_ID, "fill-color", palette.terrain); - } - if (this.map.getLayer(PHYSICAL_REGIONS_LINE_LAYER_ID)) { - this.map.setPaintProperty(PHYSICAL_REGIONS_LINE_LAYER_ID, "line-color", palette.terrain); - } - if (this.map.getLayer(PHYSICAL_REGIONS_10M_FILL_LAYER_ID)) { - this.map.setPaintProperty(PHYSICAL_REGIONS_10M_FILL_LAYER_ID, "fill-color", palette.terrain); - } - if (this.map.getLayer(PHYSICAL_REGIONS_10M_LINE_LAYER_ID)) { - this.map.setPaintProperty(PHYSICAL_REGIONS_10M_LINE_LAYER_ID, "line-color", palette.terrain); - } - if (this.map.getLayer(MAJOR_ROADS_10M_LAYER_ID)) { - this.map.setPaintProperty(MAJOR_ROADS_10M_LAYER_ID, "line-color", palette.roads); - } - if (this.map.getLayer(LAKES_FILL_LAYER_ID)) { - this.map.setPaintProperty(LAKES_FILL_LAYER_ID, "fill-color", palette.water); - } - if (this.map.getLayer(RIVERS_LINE_LAYER_ID)) { - this.map.setPaintProperty(RIVERS_LINE_LAYER_ID, "line-color", palette.rivers); - } - if (this.map.getLayer(ADMIN1_LINE_LAYER_ID)) { - this.map.setPaintProperty(ADMIN1_LINE_LAYER_ID, "line-color", palette.admin1); - } - if (this.map.getLayer(COUNTRY_LINE_LAYER_ID)) { - this.map.setPaintProperty(COUNTRY_LINE_LAYER_ID, "line-color", palette.borders); - } - if (this.map.getLayer(COASTLINE_LAYER_ID)) { - this.map.setPaintProperty(COASTLINE_LAYER_ID, "line-color", palette.coast); + if (this.map.getLayer(SITE_LINK_LAYER_ID)) { + this.map.setPaintProperty(SITE_LINK_LAYER_ID, "line-color", linkColorExpression()); } if (this.map.getLayer(SITE_POINTS_LAYER_ID)) { this.map.setPaintProperty(SITE_POINTS_LAYER_ID, "circle-stroke-color", palette.siteStroke); @@ -394,430 +844,171 @@ class SiteMapPage { } } - installPointSourcesAndLayers() { - this.map.addSource(SITE_SOURCE_ID, { - type: "geojson", - data: { type: "FeatureCollection", features: [] }, - cluster: true, - clusterRadius: 42, - clusterMaxZoom: 7, - clusterProperties: { - throughputSum: ["+", ["get", "throughputCombined"]], - qooSum: ["+", ["coalesce", ["get", "qooWorst"], 0]], - qooCount: ["+", ["case", ["has", "qooWorst"], 1, 0]], - rttSum: ["+", ["coalesce", ["get", "rttWorst"], 0]], - rttCount: ["+", ["case", ["has", "rttWorst"], 1, 0]], - }, - }); - this.map.addSource(AP_SOURCE_ID, { - type: "geojson", - data: { type: "FeatureCollection", features: [] }, - cluster: true, - clusterRadius: 40, - clusterMaxZoom: 11, - clusterProperties: { - throughputSum: ["+", ["get", "throughputCombined"]], - qooSum: ["+", ["coalesce", ["get", "qooWorst"], 0]], - qooCount: ["+", ["case", ["has", "qooWorst"], 1, 0]], - rttSum: ["+", ["coalesce", ["get", "rttWorst"], 0]], - rttCount: ["+", ["case", ["has", "rttWorst"], 1, 0]], - }, - }); - - this.map.addLayer({ - id: SITE_CLUSTER_LAYER_ID, - type: "circle", - source: SITE_SOURCE_ID, - filter: ["has", "point_count"], - paint: { - "circle-color": qooClusterColorExpression(), - "circle-radius": clusterRadiusExpression(), - "circle-opacity": 0.88, - "circle-stroke-color": countryPalette().siteStroke, - "circle-stroke-width": 1.1, - }, - }); - this.map.addLayer({ - id: SITE_POINTS_LAYER_ID, - type: "circle", - source: SITE_SOURCE_ID, - filter: ["!", ["has", "point_count"]], - paint: { - "circle-color": ["get", "metricColor"], - "circle-radius": ["get", "markerRadius"], - "circle-opacity": [ - "interpolate", ["linear"], ["zoom"], - 0, 0.92, - 5, 0.62, - 8, 0.18, - ], - "circle-stroke-color": countryPalette().siteStroke, - "circle-stroke-width": 1.0, - "circle-blur": 0.08, - }, - }); - - this.map.addLayer({ - id: AP_CLUSTER_LAYER_ID, - type: "circle", - source: AP_SOURCE_ID, - filter: ["has", "point_count"], - paint: { - "circle-color": qooClusterColorExpression(), - "circle-radius": clusterRadiusExpression(), - "circle-opacity": [ - "interpolate", ["linear"], ["zoom"], - 0, 0.18, - 4, 0.44, - 6, 0.82, - 8, 0.9, - ], - "circle-stroke-color": countryPalette().apStroke, - "circle-stroke-width": 1.0, - }, - }); - this.map.addLayer({ - id: AP_POINTS_LAYER_ID, - type: "circle", - source: AP_SOURCE_ID, - filter: ["!", ["has", "point_count"]], - paint: { - "circle-color": ["get", "metricColor"], - "circle-radius": ["get", "markerRadius"], - "circle-opacity": [ - "interpolate", ["linear"], ["zoom"], - 0, 0.06, - 4, 0.18, - 6, 0.76, - 8, 0.96, - ], - "circle-stroke-color": countryPalette().apStroke, - "circle-stroke-width": 1.0, - "circle-blur": 0.05, - }, - }); - } - - installHeavyGeographyLayers() { - if (this.heavyLayersInstalled || !this.map?.isStyleLoaded()) { - return; + installSourcesAndLayers() { + if (!this.map.getSource(SITE_LINK_SOURCE_ID)) { + this.map.addSource(SITE_LINK_SOURCE_ID, { + type: "geojson", + data: { type: "FeatureCollection", features: [] }, + }); + } + if (!this.map.getSource(SITE_SOURCE_ID)) { + this.map.addSource(SITE_SOURCE_ID, { + type: "geojson", + data: { type: "FeatureCollection", features: [] }, + cluster: true, + clusterRadius: 50, + clusterMaxZoom: 12, + }); + } + if (!this.map.getSource(AP_SOURCE_ID)) { + this.map.addSource(AP_SOURCE_ID, { + type: "geojson", + data: { type: "FeatureCollection", features: [] }, + }); + } + if (!this.map.getSource(FANOUT_LINE_SOURCE_ID)) { + this.map.addSource(FANOUT_LINE_SOURCE_ID, { + type: "geojson", + data: { type: "FeatureCollection", features: [] }, + }); + } + if (!this.map.getSource(FANOUT_POINT_SOURCE_ID)) { + this.map.addSource(FANOUT_POINT_SOURCE_ID, { + type: "geojson", + data: { type: "FeatureCollection", features: [] }, + }); } - const addLayerBelowPoints = (layer) => this.map.addLayer(layer, SITE_CLUSTER_LAYER_ID); - - this.map.addSource(COUNTRIES_SOURCE_ID, { - type: "geojson", - data: COUNTRIES_GEOJSON_PATH, - }); - this.map.addSource(MARINE_AREAS_SOURCE_ID, { - type: "geojson", - data: MARINE_AREAS_GEOJSON_PATH, - }); - this.map.addSource(PHYSICAL_REGIONS_SOURCE_ID, { - type: "geojson", - data: PHYSICAL_REGIONS_GEOJSON_PATH, - }); - this.map.addSource(PHYSICAL_REGIONS_10M_SOURCE_ID, { - type: "geojson", - data: PHYSICAL_REGIONS_10M_GEOJSON_PATH, - }); - this.map.addSource(MAJOR_ROADS_10M_SOURCE_ID, { - type: "geojson", - data: MAJOR_ROADS_10M_GEOJSON_PATH, - }); - this.map.addSource(LAKES_SOURCE_ID, { - type: "geojson", - data: LAKES_GEOJSON_PATH, - }); - this.map.addSource(RIVERS_SOURCE_ID, { - type: "geojson", - data: RIVERS_GEOJSON_PATH, - }); - this.map.addSource(ADMIN1_SOURCE_ID, { - type: "geojson", - data: ADMIN1_BOUNDARIES_GEOJSON_PATH, - }); - this.map.addSource(COASTLINE_SOURCE_ID, { - type: "geojson", - data: COASTLINES_GEOJSON_PATH, - }); - - addLayerBelowPoints({ - id: MARINE_AREAS_FILL_LAYER_ID, - type: "fill", - source: MARINE_AREAS_SOURCE_ID, - paint: { - "fill-color": countryPalette().marine, - "fill-opacity": 0.9, - }, - }); - addLayerBelowPoints({ - id: COUNTRY_FILL_LAYER_ID, - type: "fill", - source: COUNTRIES_SOURCE_ID, - paint: { - "fill-color": countryPalette().land, - "fill-opacity": isDarkMode() ? 0.8 : 0.76, - }, - }); - addLayerBelowPoints({ - id: PHYSICAL_REGIONS_FILL_LAYER_ID, - type: "fill", - source: PHYSICAL_REGIONS_SOURCE_ID, - paint: { - "fill-color": countryPalette().terrain, - "fill-opacity": [ - "interpolate", ["linear"], ["zoom"], - 0, - [ - "match", ["get", "featurecla"], - "mountain range", isDarkMode() ? 0.22 : 0.15, - "mountain", isDarkMode() ? 0.18 : 0.12, - "plateau", isDarkMode() ? 0.12 : 0.08, - "basin", isDarkMode() ? 0.09 : 0.06, - "plain", isDarkMode() ? 0.06 : 0.04, - "desert", isDarkMode() ? 0.08 : 0.05, - isDarkMode() ? 0.08 : 0.05, + if (!this.map.getLayer(SITE_LINK_LAYER_ID)) { + this.map.addLayer({ + id: SITE_LINK_LAYER_ID, + type: "line", + source: SITE_LINK_SOURCE_ID, + layout: { + "line-join": "round", + "line-cap": "round", + }, + paint: { + "line-color": linkColorExpression(), + "line-width": linkWidthExpression(), + "line-opacity": [ + "interpolate", ["linear"], ["zoom"], + 2, 0.35, + 6, 0.55, + 10, 0.72, ], - 4.5, - [ - "match", ["get", "featurecla"], - "mountain range", isDarkMode() ? 0.18 : 0.11, - "mountain", isDarkMode() ? 0.15 : 0.09, - "plateau", isDarkMode() ? 0.08 : 0.05, - "basin", isDarkMode() ? 0.06 : 0.04, - "plain", isDarkMode() ? 0.04 : 0.02, - "desert", isDarkMode() ? 0.05 : 0.03, - isDarkMode() ? 0.05 : 0.03, - ], - 6, - 0, - ], - }, - }); - addLayerBelowPoints({ - id: PHYSICAL_REGIONS_LINE_LAYER_ID, - type: "line", - source: PHYSICAL_REGIONS_SOURCE_ID, - paint: { - "line-color": countryPalette().terrain, - "line-width": [ - "interpolate", ["linear"], ["zoom"], - 2, 0.18, - 5, 0.3, - 8, 0.46, - ], - "line-opacity": [ - "interpolate", ["linear"], ["zoom"], - 0, 0.05, - 4.5, 0.08, - 6, 0.03, - 7, 0, - ], - }, - }); - addLayerBelowPoints({ - id: PHYSICAL_REGIONS_10M_FILL_LAYER_ID, - type: "fill", - source: PHYSICAL_REGIONS_10M_SOURCE_ID, - minzoom: 4.5, - paint: { - "fill-color": countryPalette().terrain, - "fill-opacity": [ - "interpolate", ["linear"], ["zoom"], - 4.5, - 0, - 5.5, - [ - "match", ["get", "featurecla"], - "mountain range", isDarkMode() ? 0.24 : 0.16, - "mountain", isDarkMode() ? 0.2 : 0.13, - "plateau", isDarkMode() ? 0.12 : 0.08, - "basin", isDarkMode() ? 0.09 : 0.06, - "plain", isDarkMode() ? 0.05 : 0.03, - "desert", isDarkMode() ? 0.07 : 0.05, - isDarkMode() ? 0.07 : 0.05, - ], - 8, - [ - "match", ["get", "featurecla"], - "mountain range", isDarkMode() ? 0.28 : 0.19, - "mountain", isDarkMode() ? 0.24 : 0.16, - "plateau", isDarkMode() ? 0.15 : 0.1, - "basin", isDarkMode() ? 0.11 : 0.08, - "plain", isDarkMode() ? 0.07 : 0.05, - "desert", isDarkMode() ? 0.09 : 0.06, - isDarkMode() ? 0.09 : 0.06, - ], - ], - }, - }); - addLayerBelowPoints({ - id: PHYSICAL_REGIONS_10M_LINE_LAYER_ID, - type: "line", - source: PHYSICAL_REGIONS_10M_SOURCE_ID, - minzoom: 4.5, - paint: { - "line-color": countryPalette().terrain, - "line-width": [ - "interpolate", ["linear"], ["zoom"], - 4.5, 0.18, - 6, 0.34, - 8, 0.5, - ], - "line-opacity": [ - "interpolate", ["linear"], ["zoom"], - 4.5, 0, - 5.5, 0.05, - 8, 0.1, - ], - }, - }); - addLayerBelowPoints({ - id: MAJOR_ROADS_10M_LAYER_ID, - type: "line", - source: MAJOR_ROADS_10M_SOURCE_ID, - minzoom: 5.5, - paint: { - "line-color": countryPalette().roads, - "line-width": [ - "interpolate", ["linear"], ["zoom"], - 5.5, 0.3, - 7, 0.55, - 9, 0.95, - ], - "line-opacity": [ - "interpolate", ["linear"], ["zoom"], - 5.5, 0, - 6.5, isDarkMode() ? 0.18 : 0.14, - 9, isDarkMode() ? 0.28 : 0.22, - ], - }, - }); - this.map.setPaintProperty(MAJOR_ROADS_10M_LAYER_ID, "line-opacity", [ - "interpolate", ["linear"], ["zoom"], - 0, 0, - 4.5, isDarkMode() ? 0.1 : 0.08, - 5.5, isDarkMode() ? 0.14 : 0.1, - 6.25, 0, - ]); - addLayerBelowPoints({ - id: LAKES_FILL_LAYER_ID, - type: "fill", - source: LAKES_SOURCE_ID, - paint: { - "fill-color": countryPalette().water, - "fill-opacity": 0.92, - }, - }); - addLayerBelowPoints({ - id: RIVERS_LINE_LAYER_ID, - type: "line", - source: RIVERS_SOURCE_ID, - paint: { - "line-color": countryPalette().rivers, - "line-width": [ - "interpolate", ["linear"], ["zoom"], - 2, 0.22, - 5, 0.4, - 8, 0.62, - ], - "line-opacity": [ - "interpolate", ["linear"], ["zoom"], - 2, 0.42, - 5, 0.34, - 8, 0.28, - ], - }, - }); - addLayerBelowPoints({ - id: ADMIN1_LINE_LAYER_ID, - type: "line", - source: ADMIN1_SOURCE_ID, - paint: { - "line-color": countryPalette().admin1, - "line-width": [ - "interpolate", ["linear"], ["zoom"], - 2, 0.34, - 5, 0.58, - 8, 0.95, - ], - "line-opacity": [ - "interpolate", ["linear"], ["zoom"], - 0, 0.3, - 3, 0.46, - 5.5, 0.6, - 7, 0.52, - 9, 0.42, - ], - }, - }); - addLayerBelowPoints({ - id: COUNTRY_LINE_LAYER_ID, - type: "line", - source: COUNTRIES_SOURCE_ID, - paint: { - "line-color": countryPalette().borders, - "line-width": [ - "interpolate", ["linear"], ["zoom"], - 2, 0.5, - 5, 0.8, - 8, 1.2, - ], - "line-opacity": 0.9, - }, - }); - addLayerBelowPoints({ - id: COASTLINE_LAYER_ID, - type: "line", - source: COASTLINE_SOURCE_ID, - paint: { - "line-color": countryPalette().coast, - "line-width": [ - "interpolate", ["linear"], ["zoom"], - 2, 0.45, - 5, 0.8, - 8, 1.25, - ], - "line-opacity": 0.82, - }, - }); - - this.heavyLayersInstalled = true; - this.applyTheme(); - } - - queueHeavyLayersInstall() { - if (this.heavyLayersInstalled || this.heavyLayersInstallQueued) { - return; + }, + }); } - this.heavyLayersInstallQueued = true; - const install = () => { - if (this.heavyLayersFallbackTimer !== null) { - window.clearTimeout(this.heavyLayersFallbackTimer); - this.heavyLayersFallbackTimer = null; - } - this.heavyLayersInstallQueued = false; - this.installHeavyGeographyLayers(); - }; - const installWhenReady = () => { - if (this.heavyLayersInstalled) { - return; - } - if (this.map?.isStyleLoaded()) { - install(); - return; - } - this.map?.once("idle", install); - }; - this.map?.once("idle", installWhenReady); - this.heavyLayersFallbackTimer = window.setTimeout(installWhenReady, 1200); + if (!this.map.getLayer(SITE_POINTS_LAYER_ID)) { + this.map.addLayer({ + id: SITE_POINTS_LAYER_ID, + type: "circle", + source: SITE_SOURCE_ID, + filter: ["!", ["has", "point_count"]], + paint: { + "circle-color": ["get", "metricColor"], + "circle-radius": ["get", "markerRadius"], + "circle-opacity": [ + "interpolate", ["linear"], ["zoom"], + 0, 0.86, + 6, 0.76, + 10, 0.62, + ], + "circle-stroke-color": markerPalette().siteStroke, + "circle-stroke-width": 1.15, + "circle-blur": 0.06, + }, + }); + } + + if (!this.map.getLayer(SITE_CLUSTER_LAYER_ID)) { + this.map.addLayer({ + id: SITE_CLUSTER_LAYER_ID, + type: "circle", + source: SITE_SOURCE_ID, + filter: ["has", "point_count"], + paint: { + "circle-color": [ + "step", + ["get", "point_count"], + "#4ca4ff", + 8, "#f5b84f", + 20, "#d94b5b", + ], + "circle-radius": [ + "step", + ["get", "point_count"], + 18, + 8, 24, + 20, 30, + ], + "circle-opacity": 0.9, + "circle-stroke-color": markerPalette().siteStroke, + "circle-stroke-width": 1.3, + }, + }); + } + + if (!this.map.getLayer(AP_POINTS_LAYER_ID)) { + this.map.addLayer({ + id: AP_POINTS_LAYER_ID, + type: "circle", + source: AP_SOURCE_ID, + paint: { + "circle-color": ["get", "metricColor"], + "circle-radius": ["get", "markerRadius"], + "circle-opacity": [ + "interpolate", ["linear"], ["zoom"], + 0, 0.0, + 5, 0.08, + 7, 0.55, + 9, 0.86, + ], + "circle-stroke-color": markerPalette().apStroke, + "circle-stroke-width": 1.0, + "circle-blur": 0.05, + }, + }); + } + + if (!this.map.getLayer(FANOUT_LINE_LAYER_ID)) { + this.map.addLayer({ + id: FANOUT_LINE_LAYER_ID, + type: "line", + source: FANOUT_LINE_SOURCE_ID, + layout: { + "line-cap": "round", + "line-join": "round", + }, + paint: { + "line-color": isDarkMode() ? "rgba(248, 250, 252, 0.46)" : "rgba(15, 23, 42, 0.34)", + "line-width": 1.4, + "line-dasharray": [1.2, 1.2], + "line-opacity": 0.72, + }, + }); + } + + if (!this.map.getLayer(FANOUT_POINT_LAYER_ID)) { + this.map.addLayer({ + id: FANOUT_POINT_LAYER_ID, + type: "circle", + source: FANOUT_POINT_SOURCE_ID, + paint: { + "circle-color": ["get", "metricColor"], + "circle-radius": ["get", "markerRadius"], + "circle-opacity": 0.95, + "circle-stroke-color": markerPalette().apStroke, + "circle-stroke-width": 1.0, + }, + }); + } } installInteractions() { - const pointLayers = [SITE_POINTS_LAYER_ID, AP_POINTS_LAYER_ID]; - const clusterLayers = [SITE_CLUSTER_LAYER_ID, AP_CLUSTER_LAYER_ID]; + const pointLayers = [SITE_POINTS_LAYER_ID, AP_POINTS_LAYER_ID, FANOUT_POINT_LAYER_ID]; pointLayers.forEach((layerId) => { this.map.on("mouseenter", layerId, () => { @@ -838,81 +1029,127 @@ class SiteMapPage { this.map.on("click", layerId, (event) => { const feature = event.features?.[0]; if (!feature) return; - this.selectedFeature = feature.properties; - this.renderDetails(feature.properties); + this.selectFeature(feature.properties); }); }); - clusterLayers.forEach((layerId) => { + [SITE_CLUSTER_LAYER_ID].forEach((layerId) => { this.map.on("mouseenter", layerId, () => { this.map.getCanvas().style.cursor = "pointer"; }); this.map.on("mouseleave", layerId, () => { this.map.getCanvas().style.cursor = ""; - this.popup.remove(); - }); - this.map.on("mousemove", layerId, (event) => { - const feature = event.features?.[0]; - if (!feature) return; - const props = feature.properties || {}; - const count = toNumber(props.point_count, 0); - const typeLabel = layerId === SITE_CLUSTER_LAYER_ID ? "Site" : "AP"; - this.popup - .setLngLat(event.lngLat) - .setHTML(` -
    -
    ${typeLabel} cluster
    -
    ${count} grouped nodes at this zoom level.
    -
    `) - .addTo(this.map); }); this.map.on("click", layerId, async (event) => { const feature = event.features?.[0]; - if (!feature) return; - const sourceId = layerId === SITE_CLUSTER_LAYER_ID ? SITE_SOURCE_ID : AP_SOURCE_ID; - const clusterId = feature.properties?.cluster_id; - const source = this.map.getSource(sourceId); - if (!source || clusterId === undefined || clusterId === null) return; - source.getClusterExpansionZoom(clusterId, (err, zoom) => { - if (err) return; - this.map.easeTo({ - center: feature.geometry.coordinates, - zoom, - duration: 400, - }); + const clusterId = feature?.properties?.cluster_id; + if (!Number.isFinite(clusterId)) { + return; + } + const source = this.map.getSource(SITE_SOURCE_ID); + if (!source?.getClusterExpansionZoom) { + return; + } + const expansionZoom = await source.getClusterExpansionZoom(clusterId); + this.map.easeTo({ + center: feature.geometry.coordinates, + zoom: expansionZoom, + duration: 350, }); }); }); + + this.map.on("zoomend", () => { + if (this.fanoutSiteKey) { + this.fanoutSiteKey = null; + if (this.selectedFeature) { + this.renderDetails(this.selectedFeature); + } + } + this.syncFanoutOverlays(); + this.scheduleTextOverlaySync(); + }); + this.map.on("moveend", () => this.scheduleTextOverlaySync()); } requestInitialTree() { this.setStatus("Waiting for data", "spinner"); if (!this.subscription) { - this.subscription = subscribeWS(["NetworkTree"], (liveMsg) => { - if (liveMsg.event === "NetworkTree") { + this.subscription = subscribeWS(["NetworkTreeLite"], (liveMsg) => { + if (liveMsg.event === "NetworkTreeLite") { this.processTreeMessage(liveMsg); } }); } - listenOnceWithTimeout("NetworkTree", INITIAL_REQUEST_TIMEOUT_MS, (msg) => { + listenOnceWithTimeout("NetworkTreeLite", INITIAL_REQUEST_TIMEOUT_MS, (msg) => { this.processTreeMessage(msg); }, () => { this.setStatus("No data received yet", "warning"); }); - wsClient.send({ NetworkTree: {} }); + wsClient.send({ NetworkTreeLite: {} }); } - processTreeMessage(msg) { + processTreeMessage(msg, options = {}) { + const fixture = options?.fixture === true; const data = Array.isArray(msg?.data) ? msg.data : []; this.history.push({ timestamp: Date.now(), data }); const cutoff = Date.now() - HISTORY_WINDOW_MS; this.history = this.history.filter((entry) => entry.timestamp >= cutoff); this.latestSnapshot = data; this.lastUpdateAt = Date.now(); - this.setStatus("Live", "success"); + this.setStatus(fixture ? "Fixture" : "Live", "success"); + this.ensureMapInitialized(); this.renderFromHistory(); } + ensureMapInitialized() { + if (this.map || this.mapInitPromise) { + return; + } + const now = Date.now(); + if (this.lastBboxAttemptAt && (now - this.lastBboxAttemptAt) < 15_000) { + return; + } + this.lastBboxAttemptAt = now; + const siteLatLonPairs = this.latestSnapshot + .filter((entry) => Array.isArray(entry) && entry.length >= 2) + .map(([, node]) => node) + .filter((node) => asNodeType(node) === "site") + .map((node) => { + const lat = Number(node.latitude); + const lon = Number(node.longitude); + return hasMeaningfulLatLon(lat, lon) ? [lat, lon] : null; + }) + .filter((pair) => Array.isArray(pair)); + + if (siteLatLonPairs.length === 0) { + this.setStatus("No mapped sites", "warning"); + return; + } + + if (this.fixtureMode) { + const [lat, lon] = siteLatLonPairs[0]; + this.initMap([lon, lat], 9); + return; + } + + this.mapInitPromise = (async () => { + try { + if (siteLatLonPairs.length === 1) { + const [lat, lon] = siteLatLonPairs[0]; + siteLatLonPairs.push([lat + 0.0001, lon + 0.0001]); + } + const center = await requestOsmCenterFromBbox(siteLatLonPairs, 4000); + this.initMap([center.lon, center.lat], center.zoom); + } catch (err) { + console.error("Site map bbox request failed; map not initialized.", err); + this.setStatus("Waiting for Insight map location", "warning"); + } finally { + this.mapInitPromise = null; + } + })(); + } + buildAggregates() { if (!this.history.length) { return null; @@ -920,6 +1157,7 @@ class SiteMapPage { const aggregate = new Map(); let latestIndexMap = new Map(); + const attachedInheritedApsBySite = new Map(); this.history.forEach((entry) => { const snapshotIndexMap = new Map(); @@ -987,6 +1225,7 @@ class SiteMapPage { const unmappedSites = []; const unmappedAps = []; let maxBitsPerSecond = 0; + let maxSiteBitsPerSecond = 0; const byIndex = new Map(); aggregate.forEach((value) => { @@ -996,11 +1235,17 @@ class SiteMapPage { const avgUp = value.throughputSamples > 0 ? (value.throughputUp / value.throughputSamples) : 0; const throughputCombined = avgDown + avgUp; maxBitsPerSecond = Math.max(maxBitsPerSecond, throughputCombined); + if (nodeType === "site") { + maxSiteBitsPerSecond = Math.max(maxSiteBitsPerSecond, throughputCombined); + } const qooDown = averageOrNull(value.qooDownSum, value.qooDownCount); const qooUp = averageOrNull(value.qooUpSum, value.qooUpCount); const rttDownMs = averageOrNull(value.rttDownSum, value.rttDownCount); const rttUpMs = averageOrNull(value.rttUpSum, value.rttUpCount); + const maxMbps = effectiveMaxThroughput(node); + const limitDownMbps = toNumber(maxMbps?.[0], 0); + const limitUpMbps = toNumber(maxMbps?.[1], 0); const normalized = { key: value.key, @@ -1008,9 +1253,9 @@ class SiteMapPage { name: node.name, id: node.id || null, type: nodeType, - immediateParent: node.immediate_parent, - latitude: Number.isFinite(node.latitude) ? node.latitude : null, - longitude: Number.isFinite(node.longitude) ? node.longitude : null, + immediateParent: immediateParentIndex(node), + latitude: null, + longitude: null, throughputDown: avgDown, throughputUp: avgUp, throughputCombined, @@ -1021,8 +1266,17 @@ class SiteMapPage { rttUpMs, rttWorst: worstRtt(rttDownMs, rttUpMs), parentName: null, + parentType: null, inheritedCoords: false, + limitDownMbps, + limitUpMbps, }; + const lat = Number(node.latitude); + const lon = Number(node.longitude); + if (hasMeaningfulLatLon(lat, lon)) { + normalized.latitude = lat; + normalized.longitude = lon; + } byIndex.set(value.latestIndex, normalized); }); @@ -1033,16 +1287,31 @@ class SiteMapPage { node.latitude = parent.latitude; node.longitude = parent.longitude; node.parentName = parent.name; + node.parentType = parent.type; node.inheritedCoords = true; } } else if (node.immediateParent !== null && node.immediateParent !== undefined) { const parent = byIndex.get(node.immediateParent); if (parent) { node.parentName = parent.name; + node.parentType = parent.type; } } }); + byIndex.forEach((node) => { + if (node.type !== "ap" || !node.inheritedCoords) { + return; + } + const parent = byIndex.get(node.immediateParent); + if (!parent || parent.type !== "site") { + return; + } + const bucket = attachedInheritedApsBySite.get(parent.key) || []; + bucket.push(node); + attachedInheritedApsBySite.set(parent.key, bucket); + }); + byIndex.forEach((node) => { if (node.latitude === null || node.longitude === null) { const listTarget = node.type === "site" ? unmappedSites : unmappedAps; @@ -1050,10 +1319,12 @@ class SiteMapPage { return; } + if (node.type === "ap" && node.inheritedCoords) { + return; + } + const metricValue = this.mode === "qoo" ? node.qooWorst : node.rttWorst; - const metricColor = this.mode === "qoo" - ? colorByQoqScore(metricValue) - : colorByRttMs(metricValue); + const metricColor = metricColorForMode(this.mode, metricValue); const feature = { type: "Feature", geometry: { @@ -1064,9 +1335,12 @@ class SiteMapPage { key: node.key, nodeId: node.id || "", name: node.name, + displayName: displayNodeName(node.name, node.type), nodeType: node.type, parentName: node.parentName || "", + parentType: node.parentType || "", inheritedCoords: node.inheritedCoords ? 1 : 0, + attachedInheritedApCount: attachedInheritedApsBySite.get(node.key)?.length || 0, throughputDown: node.throughputDown, throughputUp: node.throughputUp, throughputCombined: node.throughputCombined, @@ -1087,19 +1361,80 @@ class SiteMapPage { features.push(feature); }); + const siteLinkFeatures = []; + const emittedLinks = new Set(); + byIndex.forEach((node) => { + if (node.type !== "site") { + return; + } + if (node.latitude === null || node.longitude === null) { + return; + } + const parentSiteIndex = findNearestAncestorSiteIndex(latestIndexMap, node.immediateParent); + if (parentSiteIndex === null) { + return; + } + const parent = byIndex.get(parentSiteIndex); + if (!parent || parent.type !== "site" || parent.latitude === null || parent.longitude === null) { + return; + } + const key = `${node.key}->${parent.key}`; + if (emittedLinks.has(key)) { + return; + } + emittedLinks.add(key); + const downLimitBits = node.limitDownMbps > 0 ? node.limitDownMbps * 1000 * 1000 : 0; + const upLimitBits = node.limitUpMbps > 0 ? node.limitUpMbps * 1000 * 1000 : 0; + const downRatio = downLimitBits > 0 ? (node.throughputDown / downLimitBits) : null; + const upRatio = upLimitBits > 0 ? (node.throughputUp / upLimitBits) : null; + const utilizationRatio = (downRatio === null && upRatio === null) + ? null + : Math.max(0, Math.min(1, Math.max(downRatio ?? 0, upRatio ?? 0))); + const trafficRatio = maxSiteBitsPerSecond > 0 + ? Math.max(0, Math.min(1, node.throughputCombined / maxSiteBitsPerSecond)) + : 0; + siteLinkFeatures.push({ + type: "Feature", + geometry: { + type: "LineString", + coordinates: [ + [node.longitude, node.latitude], + [parent.longitude, parent.latitude], + ], + }, + properties: { + key, + fromName: node.name, + toName: parent.name, + utilizationRatio, + trafficRatio, + }, + }); + }); + const siteFeatures = features.filter((feature) => feature.properties.nodeType === "site"); const apFeatures = features.filter((feature) => feature.properties.nodeType === "ap"); return { siteFeatures, apFeatures, + siteLinkFeatures, unmappedSites, unmappedAps, + attachedInheritedApsBySite, maxBitsPerSecond, + maxSiteBitsPerSecond, }; } renderFromHistory() { - if (!this.map || !this.map.isStyleLoaded()) { + if (!this.map) { + return; + } + // MapLibre may report the style as not-yet-loaded while raster tiles are still fetching + // (503/429). The overlays only require the GeoJSON sources/layers to exist. + if (!this.map.getSource(SITE_SOURCE_ID) + || !this.map.getSource(AP_SOURCE_ID) + || !this.map.getSource(SITE_LINK_SOURCE_ID)) { return; } const aggregate = this.buildAggregates(); @@ -1107,18 +1442,20 @@ class SiteMapPage { return; } this.latestRender = aggregate; + this.attachedInheritedApsBySite = aggregate.attachedInheritedApsBySite || new Map(); this.updateSources(aggregate); this.updateSelection(); this.renderUnmapped(aggregate.unmappedSites, aggregate.unmappedAps); if (!this.hasFitOnce) { - this.fitToData(aggregate.siteFeatures, aggregate.apFeatures); + this.fitToDataIfNeeded(aggregate.siteFeatures, aggregate.apFeatures); } } updateSources(aggregate) { const siteSource = this.map.getSource(SITE_SOURCE_ID); const apSource = this.map.getSource(AP_SOURCE_ID); - if (!siteSource || !apSource) { + const linkSource = this.map.getSource(SITE_LINK_SOURCE_ID); + if (!siteSource || !apSource || !linkSource) { return; } siteSource.setData({ @@ -1129,49 +1466,429 @@ class SiteMapPage { type: "FeatureCollection", features: aggregate.apFeatures, }); + linkSource.setData({ + type: "FeatureCollection", + features: aggregate.siteLinkFeatures ?? [], + }); const sparseApCoverage = aggregate.apFeatures.length < 24; const siteOpacity = sparseApCoverage - ? ["interpolate", ["linear"], ["zoom"], 0, 0.92, 5, 0.7, 8, 0.42] - : ["interpolate", ["linear"], ["zoom"], 0, 0.92, 5, 0.54, 7, 0.12, 8, 0.02]; - const siteClusterOpacity = sparseApCoverage - ? ["interpolate", ["linear"], ["zoom"], 0, 0.9, 5, 0.76, 8, 0.38] - : ["interpolate", ["linear"], ["zoom"], 0, 0.9, 5, 0.6, 7, 0.2, 8, 0.06]; + ? ["interpolate", ["linear"], ["zoom"], 0, 0.86, 6, 0.76, 10, 0.62] + : ["interpolate", ["linear"], ["zoom"], 0, 0.86, 6, 0.72, 8, 0.28, 10, 0.12]; + const apOpacity = sparseApCoverage + ? ["interpolate", ["linear"], ["zoom"], 0, 0.02, 6, 0.22, 8, 0.78, 10, 0.92] + : ["interpolate", ["linear"], ["zoom"], 0, 0.0, 6, 0.18, 8, 0.82, 10, 0.96]; this.map.setPaintProperty(SITE_POINTS_LAYER_ID, "circle-opacity", siteOpacity); - this.map.setPaintProperty(SITE_CLUSTER_LAYER_ID, "circle-opacity", siteClusterOpacity); + this.map.setPaintProperty(AP_POINTS_LAYER_ID, "circle-opacity", apOpacity); - const clusterColor = this.mode === "qoo" ? qooClusterColorExpression() : rttClusterColorExpression(); - this.map.setPaintProperty(SITE_CLUSTER_LAYER_ID, "circle-color", clusterColor); - this.map.setPaintProperty(AP_CLUSTER_LAYER_ID, "circle-color", clusterColor); + this.syncFanoutOverlays(); + this.scheduleTextOverlaySync(); + this.syncMarkerSizeLegend(aggregate.maxBitsPerSecond); } - fitToData(siteFeatures, apFeatures) { - const features = [...siteFeatures, ...apFeatures]; - if (!features.length) { + syncMarkerSizeLegend(maxBitsPerSecond) { + const dots = [this.sizeDotSmall, this.sizeDotMedium, this.sizeDotLarge]; + if (dots.some((dot) => !dot)) { return; } + + const max = Math.max(1, toNumber(maxBitsPerSecond, 1)); + // Match the actual marker sizing function by sampling at a few ratios. + const ratios = [0, 0.1, 1.0]; + const diameters = ratios.map((ratio) => throughputRadiusPx(max * ratio, max) * 2); + + const apply = (el, diameter) => { + const sizePx = Math.max(10, Math.min(64, Math.round(toNumber(diameter, 12)))); + el.style.width = `${sizePx}px`; + el.style.height = `${sizePx}px`; + }; + + apply(this.sizeDotSmall, diameters[0]); + apply(this.sizeDotMedium, diameters[1]); + apply(this.sizeDotLarge, diameters[2]); + } + + attachedApsForSite(siteKey) { + const aps = this.attachedInheritedApsBySite.get(siteKey) || []; + return [...aps].sort((left, right) => left.name.localeCompare(right.name)); + } + + buildFanoutFeatures(siteProps, attachedAps) { + if (!this.map || !siteProps || !Array.isArray(attachedAps) || attachedAps.length === 0) { + return { lines: [], points: [] }; + } + + const siteCoord = this.latestRender?.siteFeatures + ?.find((feature) => feature.properties?.key === siteProps.key) + ?.geometry?.coordinates; + if (!Array.isArray(siteCoord) || siteCoord.length < 2) { + return { lines: [], points: [] }; + } + + const center = this.map.project(siteCoord); + const radiusPx = Math.max(42, Math.min(94, 34 + (attachedAps.length * 4))); + const baseAngle = -Math.PI / 2; + const lines = []; + const points = []; + + attachedAps.forEach((node, index) => { + const angle = attachedAps.length === 1 + ? baseAngle + : baseAngle + ((Math.PI * 2 * index) / attachedAps.length); + const projected = [ + center.x + (Math.cos(angle) * radiusPx), + center.y + (Math.sin(angle) * radiusPx), + ]; + const lngLat = this.map.unproject(projected); + const metricValue = this.mode === "qoo" ? node.qooWorst : node.rttWorst; + const metricColor = metricColorForMode(this.mode, metricValue); + + lines.push({ + type: "Feature", + geometry: { + type: "LineString", + coordinates: [siteCoord, [lngLat.lng, lngLat.lat]], + }, + properties: { + key: `${siteProps.key}->${node.key}`, + }, + }); + + const point = { + type: "Feature", + geometry: { + type: "Point", + coordinates: [lngLat.lng, lngLat.lat], + }, + properties: { + key: node.key, + nodeId: node.id || "", + name: node.name, + nodeType: node.type, + parentName: node.parentName || siteProps.name || "", + parentType: node.parentType || "site", + inheritedCoords: 1, + throughputDown: node.throughputDown, + throughputUp: node.throughputUp, + throughputCombined: node.throughputCombined, + qooDown: node.qooDown, + qooUp: node.qooUp, + rttDownMs: node.rttDownMs, + rttUpMs: node.rttUpMs, + markerRadius: throughputRadiusPx( + node.throughputCombined, + Math.max(this.latestRender?.maxBitsPerSecond || 1, 1), + ), + metricColor, + isFanoutAp: 1, + }, + }; + if (Number.isFinite(node.qooWorst)) { + point.properties.qooWorst = node.qooWorst; + } + if (Number.isFinite(node.rttWorst)) { + point.properties.rttWorst = node.rttWorst; + } + points.push(point); + }); + + return { lines, points }; + } + + syncFanoutOverlays() { + if (!this.map) { + return; + } + + const lineSource = this.map.getSource(FANOUT_LINE_SOURCE_ID); + const pointSource = this.map.getSource(FANOUT_POINT_SOURCE_ID); + if (!lineSource || !pointSource) { + return; + } + + if (!this.fanoutSiteKey || !this.selectedFeature || this.selectedFeature.nodeType !== "site") { + lineSource.setData({ type: "FeatureCollection", features: [] }); + pointSource.setData({ type: "FeatureCollection", features: [] }); + return; + } + + const attachedAps = this.attachedApsForSite(this.fanoutSiteKey); + const fanout = this.buildFanoutFeatures(this.selectedFeature, attachedAps); + lineSource.setData({ type: "FeatureCollection", features: fanout.lines }); + pointSource.setData({ type: "FeatureCollection", features: fanout.points }); + } + + clearMarkerSet(markerSet) { + markerSet.forEach((marker) => marker.remove()); + markerSet.clear(); + } + + scheduleTextOverlaySync() { + if (!this.map) { + return; + } + if (this.pendingTextOverlayFrame !== null) { + window.cancelAnimationFrame(this.pendingTextOverlayFrame); + } + this.pendingTextOverlayFrame = window.requestAnimationFrame(() => { + this.pendingTextOverlayFrame = null; + this.syncClusterCountMarkers(); + this.syncSiteLabelMarkers(); + }); + } + + destroyTextOverlays() { + if (this.pendingTextOverlayFrame !== null) { + window.cancelAnimationFrame(this.pendingTextOverlayFrame); + this.pendingTextOverlayFrame = null; + } + this.clearMarkerSet(this.siteLabelMarkers); + this.clearMarkerSet(this.clusterCountMarkers); + } + + siteLabelPriority(feature, selectedKey) { + const props = feature?.properties || {}; + const throughputCombined = toNumber(props.throughputCombined, 0); + const attachedCount = toNumber(props.attachedInheritedApCount, 0); + const isSelected = props.key === selectedKey; + return (isSelected ? 1e18 : 0) + throughputCombined + (attachedCount * 5_000_000); + } + + estimateSiteLabelRect(feature, displayName, isSelected) { + if (!this.map || !feature?.geometry?.coordinates) { + return null; + } + const projected = this.map.project(feature.geometry.coordinates); + const textLength = Math.max(4, String(displayName || "").length); + const width = Math.min(220, Math.max(76, (textLength * 7.1) + 26)); + const height = isSelected ? 28 : 24; + const bottom = projected.y - 16; + const top = bottom - height; + return { + left: projected.x - (width / 2), + right: projected.x + (width / 2), + top, + bottom, + }; + } + + createTextMarker(className, text, lngLat, offset) { + const el = document.createElement("div"); + el.className = className; + el.textContent = text; + return new window.maplibregl.Marker({ + element: el, + anchor: "bottom", + offset, + }).setLngLat(lngLat); + } + + syncClusterCountMarkers() { + if (!this.map || this.map.getZoom() < 0) { + this.clearMarkerSet(this.clusterCountMarkers); + return; + } + + const clusterFeatures = this.map.queryRenderedFeatures({ layers: [SITE_CLUSTER_LAYER_ID] }) || []; + const nextKeys = new Set(); + clusterFeatures.forEach((feature) => { + const clusterId = feature?.properties?.cluster_id; + if (!Number.isFinite(clusterId)) { + return; + } + const key = `cluster:${clusterId}`; + nextKeys.add(key); + if (this.clusterCountMarkers.has(key)) { + return; + } + const text = String(feature.properties?.point_count_abbreviated ?? feature.properties?.point_count ?? ""); + const marker = this.createTextMarker( + "site-map-cluster-count-marker", + text, + feature.geometry.coordinates, + [0, 0], + ); + marker.addTo(this.map); + this.clusterCountMarkers.set(key, marker); + }); + + this.clusterCountMarkers.forEach((marker, key) => { + if (!nextKeys.has(key)) { + marker.remove(); + this.clusterCountMarkers.delete(key); + } + }); + } + + syncSiteLabelMarkers() { + if (!this.map || !this.latestRender) { + this.clearMarkerSet(this.siteLabelMarkers); + return; + } + + const zoom = this.map.getZoom(); + const selectedSiteFeature = this.selectedFeature?.nodeType === "site" + ? this.latestRender.siteFeatures?.find((feature) => feature.properties?.key === this.selectedFeature.key) + : null; + + const candidates = []; + if (zoom >= SITE_LABEL_MIN_ZOOM) { + const visibleFeatures = this.map.queryRenderedFeatures({ layers: [SITE_POINTS_LAYER_ID] }) || []; + visibleFeatures.forEach((feature) => { + const props = feature?.properties || {}; + if (!props.key) { + return; + } + const displayName = props.displayName || displayNodeName(props.name, props.nodeType); + candidates.push({ + key: props.key, + feature, + displayName, + selected: props.key === this.selectedFeature?.key, + priority: this.siteLabelPriority(feature, this.selectedFeature?.key), + }); + }); + } + + if (selectedSiteFeature && zoom >= SELECTED_SITE_LABEL_MIN_ZOOM) { + const alreadyPresent = candidates.some((candidate) => candidate.key === selectedSiteFeature.properties?.key); + if (!alreadyPresent) { + candidates.push({ + key: selectedSiteFeature.properties?.key, + feature: selectedSiteFeature, + displayName: selectedSiteFeature.properties?.displayName + || displayNodeName(selectedSiteFeature.properties?.name, "site"), + selected: true, + priority: this.siteLabelPriority(selectedSiteFeature, this.selectedFeature?.key), + }); + } + } + + candidates.sort((left, right) => right.priority - left.priority); + + const accepted = []; + const acceptedRects = []; + for (const candidate of candidates) { + if (!candidate.selected && accepted.length >= MAX_SITE_LABELS) { + continue; + } + const rect = this.estimateSiteLabelRect(candidate.feature, candidate.displayName, candidate.selected); + if (!rect) { + continue; + } + if (!candidate.selected && acceptedRects.some((existing) => rectsOverlap(existing, rect))) { + continue; + } + accepted.push(candidate); + acceptedRects.push(rect); + } + + this.clearMarkerSet(this.siteLabelMarkers); + accepted.forEach((candidate) => { + const marker = this.createTextMarker( + candidate.selected ? "site-map-site-label-marker is-selected" : "site-map-site-label-marker", + candidate.displayName, + candidate.feature.geometry.coordinates, + [0, -18], + ); + marker.addTo(this.map); + this.siteLabelMarkers.set(candidate.key, marker); + }); + } + + fitToDataIfNeeded(siteFeatures, apFeatures) { + if (!this.map) { + return; + } + + const features = [ + ...siteFeatures, + // Include only APs with their own coordinates so a bad parent/duplicate doesn't pull the view out. + ...apFeatures.filter((feature) => feature?.properties?.inheritedCoords !== 1), + ]; + if (!features.length) { + this.hasFitOnce = true; + return; + } + + this.map.resize(); + + if (features.length === 1) { + this.map.easeTo({ + center: features[0].geometry.coordinates, + zoom: SINGLE_POINT_INITIAL_ZOOM, + duration: 0, + }); + this.hasFitOnce = true; + return; + } + + let viewBounds; + try { + viewBounds = this.map.getBounds(); + } catch (_) { + viewBounds = null; + } + + if (viewBounds) { + const allInside = features.every((feature) => viewBounds.contains(feature.geometry.coordinates)); + if (allInside) { + this.hasFitOnce = true; + return; + } + } + const bounds = new window.maplibregl.LngLatBounds(); features.forEach((feature) => bounds.extend(feature.geometry.coordinates)); + const canvas = this.map.getCanvas(); + const minDimension = Math.max( + 320, + Math.min(canvas?.clientWidth || 0, canvas?.clientHeight || 0), + ); + const adaptivePadding = Math.max( + INITIAL_FIT_PADDING, + Math.round(minDimension * 0.08), + ); this.map.fitBounds(bounds, { - padding: 70, - maxZoom: 7.5, + padding: adaptivePadding, + maxZoom: INITIAL_FIT_MAX_ZOOM, duration: 0, }); this.hasFitOnce = true; } pointPopupHtml(props) { + const attachedApCount = toNumber(props.attachedInheritedApCount, 0); + const displayName = displayNodeName(props.name, props.nodeType); return `
    -
    ${escapeHtml(props.name)}
    +
    ${escapeHtml(displayName)}
    ${escapeHtml(String(props.nodeType || "").toUpperCase())}
    Throughput: ${escapeHtml(formatBitsPerSecond(props.throughputCombined))}
    + ${attachedApCount > 0 ? `
    Attached APs: ${escapeHtml(String(attachedApCount))}
    ` : ""}
    ${this.mode === "qoo" ? "QoO" : "RTT"}: ${escapeHtml(this.mode === "qoo" ? formatPercent(Math.min(toNumber(props.qooDown, NaN), toNumber(props.qooUp, NaN))) : formatMs(Math.max(toNumber(props.rttDownMs, NaN), toNumber(props.rttUpMs, NaN))))}
    - ${props.inheritedCoords ? `
    Using parent site coordinates.
    ` : ""}
    `; } + selectFeature(props) { + if (!props) { + return; + } + const attachedAps = props.nodeType === "site" ? this.attachedApsForSite(props.key) : []; + const sameSelection = this.selectedFeature?.key === props.key; + + this.selectedFeature = props; + if (props.nodeType === "site" && attachedAps.length > 0) { + this.fanoutSiteKey = sameSelection && this.fanoutSiteKey === props.key ? null : props.key; + } else { + this.fanoutSiteKey = null; + } + this.syncFanoutOverlays(); + this.scheduleTextOverlaySync(); + this.renderDetails(props); + } + updateSelection() { if (!this.selectedFeature || !this.latestRender) { return; @@ -1180,30 +1897,62 @@ class SiteMapPage { .find((feature) => feature.properties.key === this.selectedFeature.key); if (!current) { this.selectedFeature = null; + this.fanoutSiteKey = null; + this.syncFanoutOverlays(); + this.scheduleTextOverlaySync(); this.renderDetails(null); return; } this.selectedFeature = current.properties; + if (this.selectedFeature.nodeType !== "site" || this.selectedFeature.key !== this.fanoutSiteKey) { + this.fanoutSiteKey = null; + } + this.syncFanoutOverlays(); + this.scheduleTextOverlaySync(); this.renderDetails(current.properties); } renderDetails(props) { if (!props) { this.detailsPanel.style.display = "none"; + if (this.attachedList) { + this.attachedList.innerHTML = ""; + } return; } this.detailsPanel.style.display = "block"; - this.detailsTitle.textContent = props.name; - this.detailsSubtitle.textContent = `${String(props.nodeType || "").toUpperCase()}${props.parentName ? ` · parent ${props.parentName}` : ""}${props.inheritedCoords ? " · using parent site coordinates" : ""}`; + this.detailsTitle.textContent = displayNodeName(props.name, props.nodeType); + const parentName = displayParentName(props.parentName, props.parentType); + this.detailsSubtitle.textContent = `${String(props.nodeType || "").toUpperCase()}${parentName ? ` · parent ${parentName}` : ""}${props.inheritedCoords ? " · using parent site coordinates" : ""}`; + const attachedAps = props.nodeType === "site" ? this.attachedApsForSite(props.key) : []; + if (this.attachedList) { + if (attachedAps.length === 0) { + this.attachedList.innerHTML = ""; + } else { + const preview = attachedAps + .slice(0, 6) + .map((node) => `
    ${escapeHtml(displayNodeName(node.name, node.type))}
    `) + .join(""); + const more = attachedAps.length > 6 + ? `
    +${attachedAps.length - 6} more APs attached to this site
    ` + : ""; + const fanoutHint = this.fanoutSiteKey === props.key + ? "Click the selected site again or zoom the map to collapse them." + : "Click the selected site marker again to fan them out on the map."; + this.attachedList.innerHTML = `
    ${attachedAps.length} AP${attachedAps.length === 1 ? "" : "s"} inherit this site's coordinates. ${fanoutHint}
    ${preview}${more}`; + } + } this.detailsGrid.innerHTML = [ this.metricCard("Combined throughput", formatBitsPerSecond(props.throughputCombined)), this.metricCard("Download throughput", formatBitsPerSecond(props.throughputDown)), this.metricCard("Upload throughput", formatBitsPerSecond(props.throughputUp)), + props.nodeType === "site" + ? this.metricCard("Attached APs", String(attachedAps.length)) + : this.metricCard("Coordinate source", props.inheritedCoords ? "Inherited from site" : "Explicit"), this.metricCard("QoO download", formatPercent(props.qooDown)), this.metricCard("QoO upload", formatPercent(props.qooUp)), this.metricCard("RTT download", formatMs(props.rttDownMs)), this.metricCard("RTT upload", formatMs(props.rttUpMs)), - this.metricCard("Coordinate source", props.inheritedCoords ? "Inherited from site" : "Explicit"), ].join(""); } @@ -1235,7 +1984,7 @@ class SiteMapPage { renderUnmappedGroup(label, nodes) { const items = nodes .sort((a, b) => a.name.localeCompare(b.name)) - .map((node) => `
    ${escapeHtml(node.name)}${node.parentName ? `
    ${escapeHtml(node.parentName)}
    ` : ""}
    `) + .map((node) => `
    ${escapeHtml(displayNodeName(node.name, node.type))}${node.parentName ? `
    ${escapeHtml(displayParentName(node.parentName, node.parentType))}
    ` : ""}
    `) .join(""); return `
    ${escapeHtml(label)}
    ${items}
    `; } @@ -1263,7 +2012,7 @@ class SiteMapPage { if (this.mode === "qoo") { this.legendGradient.style.background = isColorBlindMode() ? "linear-gradient(90deg, #440154 0%, #3b528b 30%, #21918c 55%, #5ec962 78%, #fde725 100%)" - : "linear-gradient(90deg, #d94b5b 0%, #f5b84f 42%, #6ecc84 75%, #4ca4ff 100%)"; + : "linear-gradient(90deg, #ff0000 0%, #bf4000 25%, #808000 50%, #40bf00 75%, #00ff00 100%)"; this.legendLow.textContent = "Poor QoO"; this.legendHigh.textContent = "Healthy QoO"; } else { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/template.js b/src/rust/lqosd/src/node_manager/js_build/src/template.js index 7b921da1..7e82b7e0 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/template.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/template.js @@ -30,6 +30,7 @@ let schedulerStatusPollTimer = null; let schedulerStatusRequestInFlight = false; let schedulerStatusFirstRequestedAt = null; let schedulerStatusHealthy = false; +let cachedQueueModeConfig = null; function listenOnceWithTimeout(eventName, timeoutMs, handler, onTimeout) { let done = false; @@ -55,23 +56,20 @@ function renderSchedulerStatus(container, state) { let color = "text-secondary"; let icon = "fa-spinner fa-spin"; let label = "Scheduler status is loading"; - let buttonText = "Scheduler starting..."; + const buttonText = "Scheduler"; if (state === "healthy") { color = "text-success"; icon = "fa-check-circle"; label = "Scheduler is available"; - buttonText = "Scheduler"; } else if (state === "unavailable") { color = "text-warning"; icon = "fa-clock"; label = "Scheduler is still unavailable"; - buttonText = "Scheduler unavailable"; } else if (state === "error") { color = "text-danger"; icon = "fa-triangle-exclamation"; label = "Scheduler has an internal error"; - buttonText = "Scheduler error"; } container.innerHTML = ` @@ -359,18 +357,131 @@ function setupSearch() { function setupReload() { let link = document.getElementById("lnkReloadLqos"); link.onclick = () => { - const myModal = new bootstrap.Modal(document.getElementById('reloadModal'), { focus: true }); - myModal.show(); - $("#reloadLibreResult").html(" Reloading LibreQoS..."); - listenOnce("ReloadResult", (msg) => { - if (!msg) { - $("#reloadLibreResult").text("Failed to reload LibreQoS"); + triggerReloadLibreQoS("Reloading LibreQoS..."); + } +} + +function inferQueueMode(config) { + const queues = config && config.queues ? config.queues : {}; + if (queues.queue_mode === "observe" || queues.queue_mode === "shape") { + return queues.queue_mode; + } + return queues.monitor_only ? "observe" : "shape"; +} + +function renderQueueModeState(mode, busy = false) { + const nav = document.getElementById("observeShapeNav"); + const observeBtn = document.getElementById("btnQueueModeObserve"); + const shapeBtn = document.getElementById("btnQueueModeShape"); + if (!nav || !observeBtn || !shapeBtn) return; + + observeBtn.disabled = busy || mode === "observe"; + shapeBtn.disabled = busy || mode === "shape"; + observeBtn.className = `btn btn-sm ${mode === "observe" ? "btn-warning" : "btn-outline-secondary"}`; + shapeBtn.className = `btn btn-sm ${mode === "shape" ? "btn-success" : "btn-outline-secondary"}`; +} + +function triggerReloadLibreQoS(loadingMessage) { + const myModal = new bootstrap.Modal(document.getElementById('reloadModal'), { focus: true }); + myModal.show(); + $("#reloadLibreResult").html(` ${loadingMessage}`); + listenOnce("ReloadResult", (msg) => { + if (!msg) { + $("#reloadLibreResult").text("Failed to reload LibreQoS"); + return; + } + $("#reloadLibreResult").text(msg.message || ""); + }); + wsClient.send({ ReloadLibreQoS: {} }); +} + +function loadObserveShapeControl() { + const nav = document.getElementById("observeShapeNav"); + if (!nav) return; + + listenOnce("AdminCheck", (msg) => { + if (!msg || !msg.ok) { + nav.classList.add("d-none"); + return; + } + + listenOnce("GetConfig", (cfgMsg) => { + if (!cfgMsg || !cfgMsg.data) { + nav.classList.add("d-none"); return; } - $("#reloadLibreResult").text(msg.message || ""); + cachedQueueModeConfig = cfgMsg.data; + const mode = inferQueueMode(cachedQueueModeConfig); + nav.classList.remove("d-none"); + renderQueueModeState(mode, false); }); - wsClient.send({ ReloadLibreQoS: {} }); + wsClient.send({ GetConfig: {} }); + }); + wsClient.send({ AdminCheck: {} }); +} + +function saveQueueModeAndReload(nextMode) { + if (!cachedQueueModeConfig || !cachedQueueModeConfig.queues) { + alert("Unable to load current configuration"); + return; } + + const currentMode = inferQueueMode(cachedQueueModeConfig); + if (currentMode === nextMode) { + return; + } + + const confirmed = window.confirm( + nextMode === "observe" + ? "Switch LibreQoS to Observe mode? This reload will remove the active LibreQoS shaping tree so you can gather a true baseline. It may briefly interrupt traffic." + : "Switch LibreQoS to Shape mode? This reload will apply the LibreQoS shaping tree and begin active shaping. It may briefly interrupt traffic." + ); + if (!confirmed) { + return; + } + + renderQueueModeState(currentMode, true); + const updated = JSON.parse(JSON.stringify(cachedQueueModeConfig)); + updated.queues.queue_mode = nextMode; + + let done = false; + const cleanup = () => { + wsClient.off("UpdateConfigResult", onSuccess); + wsClient.off("Error", onError); + }; + const onSuccess = (msg) => { + if (done) return; + done = true; + cleanup(); + if (!msg || !msg.ok) { + renderQueueModeState(currentMode, false); + alert((msg && msg.message) ? msg.message : "Failed to update configuration"); + return; + } + cachedQueueModeConfig = updated; + renderQueueModeState(nextMode, false); + triggerReloadLibreQoS(`Saving queue mode '${nextMode}' and reloading LibreQoS...`); + }; + const onError = (msg) => { + if (done) return; + done = true; + cleanup(); + renderQueueModeState(currentMode, false); + alert((msg && msg.message) ? msg.message : "Failed to update configuration"); + }; + wsClient.on("UpdateConfigResult", onSuccess); + wsClient.on("Error", onError); + wsClient.send({ UpdateConfig: { config: updated } }); +} + +function initObserveShapeToggle() { + $(document).off("click", "#btnQueueModeObserve").on("click", "#btnQueueModeObserve", () => { + saveQueueModeAndReload("observe"); + }); + $(document).off("click", "#btnQueueModeShape").on("click", "#btnQueueModeShape", () => { + saveQueueModeAndReload("shape"); + }); + loadObserveShapeControl(); } function setupDynamicUrls() { @@ -541,6 +652,7 @@ initColorBlind(); getDeviceCounts(); setupSearch(); setupReload(); +initObserveShapeToggle(); setupDynamicUrls(); window.lqosInitUrgentIssues = initUrgentIssues; initSchedulerTooltips(); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/toasts/global_warnings.js b/src/rust/lqosd/src/node_manager/js_build/src/toasts/global_warnings.js index 145609b1..9a01c569 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/toasts/global_warnings.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/toasts/global_warnings.js @@ -75,7 +75,7 @@ async function loadModalTeasers() { const copy = { ...teaser }; if (copy.image) { const img = typeof copy.image === "string" ? copy.image : ""; - const cleaned = img.replace(/^signup-api\//, ""); + const cleaned = img.replace(/^\/?signup-api\//, ""); copy.image = getModalLtsUrl(cleaned); } return copy; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/toasts/version_check.js b/src/rust/lqosd/src/node_manager/js_build/src/toasts/version_check.js index 56c670e8..f346c288 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/toasts/version_check.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/toasts/version_check.js @@ -1,7 +1,6 @@ import {createBootstrapToast} from "../lq_js_common/helpers/toasts"; export function checkForUpgrades() { - // Wait 1 second to ensure the newVersion variable has been set setTimeout(() => { if (window.newVersion) { let headerSpan = document.createElement("span"); @@ -13,4 +12,4 @@ export function checkForUpgrades() { createBootstrapToast("versionToast", headerSpan, bodyDiv); } }, 100); -} \ No newline at end of file +} diff --git a/src/rust/lqosd/src/node_manager/js_build/src/tree.js b/src/rust/lqosd/src/node_manager/js_build/src/tree.js index 71443eec..e9302410 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/tree.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/tree.js @@ -1,11 +1,14 @@ import {clearDiv, clientTableHeader, formatLastSeen, simpleRow, simpleRowHtml, theading} from "./helpers/builders"; import { formatCakeStat, formatCakeStatPercent, - formatRetransmit, formatRetransmitRaw, + formatRetransmit, formatRetransmitFraction, retransmitFractionFromSample, formatRtt, formatThroughput, } from "./helpers/scaling"; import {colorByQoqScore} from "./helpers/color_scales"; +import {BitsPerSecondGauge} from "./graphs/bits_gauge"; +import {QooScoreGauge} from "./graphs/qoo_score_gauge"; +import {enableTooltipsWithin} from "./lq_js_common/helpers/tooltips"; import {scaleNumber, toNumber} from "./lq_js_common/helpers/scaling"; import {get_ws_client, subscribeWS} from "./pubsub/ws"; @@ -15,12 +18,41 @@ var upParent = 0; var subscribed = false; var expandedNodes = new Set(); var childrenByParentId = new Map(); +var stormguardNodes = new Set(); +var treeBitsGauge = null; +var treeQooGauge = null; +var lastAttachedCircuitsPage = null; +var attachedCircuitsWatchSignature = null; +var currentSelectionIdentity = null; +var rootGaugeConfigMax = null; +var selectionLocator = { + nodeId: null, + nodePath: null, + lastKnownIndex: 0, +}; +var nodeRateOverrideState = { + loading: false, + saving: false, + data: null, + error: null, + flash: null, +}; +var nodeOverrideInputsDirty = false; +var nodeOverrideLastSeedSignature = null; const wsClient = get_ws_client(); const QOO_TOOLTIP_HTML = "
    Quality of Outcome (QoO)
    " + "

    Quality of Outcome (QoO) is IETF IPPM “Internet Quality” (draft-ietf-ippm-qoo).
    " + "https://datatracker.ietf.org/doc/draft-ietf-ippm-qoo/
    " + "LibreQoS implements a latency and loss-based model to estimate quality of outcome.

    "; const THROUGHPUT_COMPARE_EPSILON_MBPS = 0.01; +const NODE_OVERRIDE_PENDING_TOOLTIP = "Stored as an operator override. Will be applied to generated network.json on the next scheduler run."; + +function retransmitPacketsForNode(node, direction) { + return toNumber( + node.current_tcp_retransmit_packets?.[direction] ?? node.current_tcp_packets?.[direction], + 0, + ); +} const listenOnce = (eventName, handler) => { const wrapped = (msg) => { @@ -30,6 +62,53 @@ const listenOnce = (eventName, handler) => { wsClient.on(eventName, wrapped); }; +function sendWsRequest(responseEvent, request) { + return new Promise((resolve, reject) => { + let done = false; + const responseHandler = (msg) => { + if (done) return; + done = true; + wsClient.off(responseEvent, responseHandler); + wsClient.off("Error", errorHandler); + resolve(msg); + }; + const errorHandler = (msg) => { + if (done) return; + done = true; + wsClient.off(responseEvent, responseHandler); + wsClient.off("Error", errorHandler); + reject(msg); + }; + wsClient.on(responseEvent, responseHandler); + wsClient.on("Error", errorHandler); + wsClient.send(request); + }); +} + +function sendPrivateRequest(command) { + wsClient.send({Private: command}); +} + +async function loadRootGaugeConfigMax() { + try { + const msg = await sendWsRequest("GetConfig", {GetConfig: {}}); + const queues = msg?.data?.queues; + if (!queues) { + return; + } + rootGaugeConfigMax = [ + toNumber(queues.downlink_bandwidth_mbps, 0), + toNumber(queues.uplink_bandwidth_mbps, 0), + ]; + const node = currentNode(); + if (node && isSyntheticRootNode(node)) { + updateTreeGauges(node); + } + } catch (_error) { + // If config load fails, keep the existing node-derived gauge behavior. + } +} + function formatDeviceIp(ip) { if (typeof ip === "string") { return ip; @@ -79,6 +158,58 @@ function formatIpBytes(bytes) { return list.join("."); } +function abbreviateIpForTable(ipText) { + if (typeof ipText !== "string") { + return String(ipText ?? ""); + } + if (!ipText.includes(":")) { + return ipText; + } + const groups = ipText.split(":").filter((group) => group.length > 0); + if (groups.length <= 3) { + return ipText; + } + return `${groups.slice(0, 3).join(":")}:...`; +} + +function summarizeIpListForTable(ipList) { + const displayList = ipList.map((entry) => abbreviateIpForTable(entry)); + if (displayList.length > 1) { + return `${displayList[0]} +${displayList.length - 1}`; + } + return displayList.join(", "); +} + +function formatEthernetPortLabel(mbps) { + const value = toNumber(mbps, 0); + if (value >= 1000 && value % 1000 === 0) { + return `${value / 1000}G`; + } + if (value >= 1000) { + return `${(value / 1000).toFixed(1)}G`; + } + return `${Math.round(value)}M`; +} + +function ethernetCapsPageHref(badge) { + const tier = encodeURIComponent(formatEthernetPortLabel(badge?.negotiated_ethernet_mbps)); + return `/ethernet_caps.html?tier=${tier}`; +} + +function formatEthernetTooltip(badge) { + if (!badge) { + return ""; + } + return `Requested plan ${toNumber(badge.requested_download_mbps, 0)} / ${toNumber(badge.requested_upload_mbps, 0)} Mbps exceeded detected Ethernet speed. ` + + `Shaping auto-capped to ${toNumber(badge.applied_download_mbps, 0)} / ${toNumber(badge.applied_upload_mbps, 0)} Mbps.`; +} + +function ethernetBadgeClass(tierLabel) { + if (tierLabel === "10M") return "text-bg-danger"; + if (tierLabel === "100M") return "text-bg-warning"; + return "text-bg-info"; +} + function configuredMax(node) { return node.configured_max_throughput || node.max_throughput || [0, 0]; } @@ -87,6 +218,23 @@ function effectiveMax(node) { return node.effective_max_throughput || configuredMax(node); } +function rootGaugeMaxAvailable() { + return Array.isArray(rootGaugeConfigMax) + && rootGaugeConfigMax.length === 2 + && Number.isFinite(Number(rootGaugeConfigMax[0])) + && Number.isFinite(Number(rootGaugeConfigMax[1])); +} + +function nodeSnapshotGaugeMax(node) { + if (isSyntheticRootNode(node) && rootGaugeMaxAvailable()) { + return [ + toNumber(rootGaugeConfigMax[0], 0), + toNumber(rootGaugeConfigMax[1], 0), + ]; + } + return effectiveMax(node); +} + function ratesMatch(a, b) { return Math.abs(toNumber(a?.[0], 0) - toNumber(b?.[0], 0)) <= THROUGHPUT_COMPARE_EPSILON_MBPS && Math.abs(toNumber(a?.[1], 0) - toNumber(b?.[1], 0)) <= THROUGHPUT_COMPARE_EPSILON_MBPS; @@ -99,6 +247,50 @@ function formatLimitValue(mbps) { return scaleNumber(toNumber(mbps, 0) * 1000 * 1000, 1); } +function formatMbpsInputValue(mbps) { + if (mbps === null || mbps === undefined) { + return ""; + } + const numeric = Number(mbps); + if (!Number.isFinite(numeric)) { + return ""; + } + return numeric.toFixed(2).replace(/\.?0+$/, ""); +} + +function ratesApproximatelyEqual(left, right) { + return Math.abs(toNumber(left, 0) - toNumber(right, 0)) <= THROUGHPUT_COMPARE_EPSILON_MBPS; +} + +function formatRatePair(downMbps, upMbps) { + return `${formatLimitValue(downMbps)} / ${formatLimitValue(upMbps)}`; +} + +function isSyntheticRootNode(node) { + return !!node + && (node.id === null || node.id === undefined) + && node.immediate_parent === null + && node.name === "Root"; +} + +function currentNode() { + return tree && tree[parent] ? tree[parent][1] : null; +} + +function currentNodeRateQuery() { + const node = currentNode(); + if (!node) { + return null; + } + const query = { + node_name: node.name, + }; + if (node.id) { + query.node_id = node.id; + } + return query; +} + function buildEffectiveLimitCellHtml(node) { const effective = effectiveMax(node); const effectiveValue = `${formatLimitValue(effective[0])} / ${formatLimitValue(effective[1])}`; @@ -127,19 +319,764 @@ function buildDirectionalLimitHtml(primaryMbps, configuredMbps, showConfigured) return `
    ${primary}
    Cfg ${formatLimitValue(configuredMbps)}
    `; } -function initTooltipsWithin(rootEl) { - if (!rootEl) return; - if (typeof bootstrap === "undefined" || !bootstrap.Tooltip) return; - const elements = rootEl.querySelectorAll('[data-bs-toggle="tooltip"]'); - elements.forEach((element) => { - if (bootstrap.Tooltip.getOrCreateInstance) { - bootstrap.Tooltip.getOrCreateInstance(element); - } else { - new bootstrap.Tooltip(element); +function subtreeSiteCount(node) { + return toNumber(node.subtree_site_count, 0); +} + +function subtreeCircuitCount(node) { + return toNumber(node.subtree_circuit_count, 0); +} + +function branchSizeParts(node) { + const parts = []; + const circuitCount = subtreeCircuitCount(node); + if (circuitCount > 0) { + parts.push(`${circuitCount} ${circuitCount === 1 ? "circuit" : "circuits"}`); + } + const siteCount = subtreeSiteCount(node); + if (siteCount > 0) { + parts.push(`${siteCount} ${siteCount === 1 ? "site" : "sites"}`); + } + return parts; +} + +function formatBranchSize(node) { + const parts = branchSizeParts(node); + if (parts.length === 0) { + return "No descendant sites or circuits"; + } + return parts.join(" / "); +} + +function buildStatusIcon(iconName, textClass, title) { + let icon = document.createElement("i"); + icon.classList.add("fa", "fa-fw", iconName, textClass); + icon.setAttribute("data-bs-toggle", "tooltip"); + icon.setAttribute("data-bs-placement", "top"); + icon.setAttribute("title", title); + return icon; +} + +function isStormguardNode(node) { + return node && typeof node.name === "string" && stormguardNodes.has(node.name); +} + +function renderHeaderStatusIcons(node) { + const target = document.getElementById("nodeStateBadges"); + if (!target) { + return; + } + clearDiv(target); + if (!node) { + return; + } + if (node.virtual === true) { + target.appendChild(buildStatusIcon( + "fa-ghost", + "text-secondary", + "Virtual node (logical only; not shaped in HTB)." + )); + } + if (node.runtime_virtualized === true) { + target.appendChild(buildStatusIcon( + "fa-layer-group", + "text-warning", + "TreeGuard runtime-virtualized node (physically bypassed in Bakery while preserved in the logical hierarchy)." + )); + } + if (isStormguardNode(node)) { + target.appendChild(buildStatusIcon( + "fa-cloud-bolt", + "text-primary", + "StormGuard-managed node (dynamic queue limits active)." + )); + } + enableTooltipsWithin(target); +} + +function sameStringSet(a, b) { + if (a.size !== b.size) { + return false; + } + for (const entry of a) { + if (!b.has(entry)) { + return false; + } + } + return true; +} + +function normalizeNodePath(path) { + if (!Array.isArray(path)) { + return null; + } + const normalized = path + .map((entry) => typeof entry === "string" ? entry : String(entry ?? "")) + .filter((entry) => entry.length > 0); + return normalized.length > 0 ? normalized : null; +} + +function parseNodePathParam(rawPath) { + if (typeof rawPath !== "string" || rawPath.length === 0) { + return null; + } + try { + return normalizeNodePath(JSON.parse(rawPath)); + } catch (_error) { + return null; + } +} + +function pathEquals(left, right) { + if (!Array.isArray(left) || !Array.isArray(right) || left.length !== right.length) { + return false; + } + for (let i = 0; i < left.length; i++) { + if (left[i] !== right[i]) { + return false; + } + } + return true; +} + +function getNodePathNames(nodeId, node = tree?.[nodeId]?.[1]) { + if (!node) { + return null; + } + if (isSyntheticRootNode(node)) { + return ["Root"]; + } + const parentIndexes = Array.isArray(node.parents) && node.parents.length > 0 + ? node.parents + : [nodeId]; + const names = []; + for (const idx of parentIndexes) { + const entry = tree?.[idx]; + if (!entry || !entry[1] || typeof entry[1].name !== "string" || entry[1].name.length === 0) { + return normalizeNodePath([node.name]); + } + names.push(entry[1].name); + } + if (names.length === 0 || names[names.length - 1] !== node.name) { + names.push(node.name); + } + return normalizeNodePath(names); +} + +function selectionIdentityForNode(nodeId, node = tree?.[nodeId]?.[1]) { + if (!node) { + return null; + } + if (typeof node.id === "string" && node.id.length > 0) { + return `id:${node.id}`; + } + const nodePath = getNodePathNames(nodeId, node); + if (nodePath) { + return `path:${JSON.stringify(nodePath)}`; + } + return `index:${nodeId}`; +} + +function syncBrowserUrlToSelection() { + if (typeof window === "undefined" || typeof history === "undefined") { + return; + } + const url = new URL(window.location.href); + url.searchParams.set("parent", String(parent)); + if (selectionLocator.nodeId) { + url.searchParams.set("nodeId", selectionLocator.nodeId); + } else { + url.searchParams.delete("nodeId"); + } + if (selectionLocator.nodePath) { + url.searchParams.set("nodePath", JSON.stringify(selectionLocator.nodePath)); + } else { + url.searchParams.delete("nodePath"); + } + const nextUrl = `${url.pathname}?${url.searchParams.toString()}${url.hash}`; + const currentUrl = `${window.location.pathname}${window.location.search}${window.location.hash}`; + if (nextUrl !== currentUrl) { + history.replaceState(null, "", nextUrl); + } +} + +function rememberSelection(nodeId, node = tree?.[nodeId]?.[1]) { + if (!node) { + return false; + } + const nextIdentity = selectionIdentityForNode(nodeId, node); + const identityChanged = currentSelectionIdentity !== nextIdentity; + currentSelectionIdentity = nextIdentity; + selectionLocator.nodeId = typeof node.id === "string" && node.id.length > 0 ? node.id : null; + selectionLocator.nodePath = getNodePathNames(nodeId, node); + selectionLocator.lastKnownIndex = nodeId; + syncBrowserUrlToSelection(); + return identityChanged; +} + +function findNodeIndexByStableId(nodeId) { + if (!tree || typeof nodeId !== "string" || nodeId.length === 0) { + return null; + } + for (let i = 0; i < tree.length; i++) { + const node = tree[i]?.[1]; + if (node?.id === nodeId) { + return i; + } + } + return null; +} + +function findNodeIndexByPath(nodePath) { + if (!tree || !Array.isArray(nodePath) || nodePath.length === 0) { + return null; + } + for (let i = 0; i < tree.length; i++) { + const node = tree[i]?.[1]; + if (!node) { + continue; + } + if (pathEquals(getNodePathNames(i, node), nodePath)) { + return i; + } + } + return null; +} + +function resolveSelectedParentIndex() { + const byStableId = findNodeIndexByStableId(selectionLocator.nodeId); + if (byStableId !== null) { + return byStableId; + } + const byPath = findNodeIndexByPath(selectionLocator.nodePath); + if (byPath !== null) { + return byPath; + } + if (Number.isInteger(selectionLocator.lastKnownIndex) && tree?.[selectionLocator.lastKnownIndex] !== undefined) { + return selectionLocator.lastKnownIndex; + } + if (tree?.[parent] !== undefined) { + return parent; + } + if (tree?.[0] !== undefined) { + return 0; + } + return null; +} + +function reconcileSelection() { + const resolvedParent = resolveSelectedParentIndex(); + if (resolvedParent === null) { + return {parentChanged: false, identityChanged: false}; + } + const previousParent = parent; + parent = resolvedParent; + const identityChanged = rememberSelection(parent); + return { + parentChanged: previousParent !== parent, + identityChanged, + }; +} + +function treeHref(nodeId, node = tree?.[nodeId]?.[1]) { + const params = new URLSearchParams(); + params.set("parent", String(nodeId)); + if (node?.id) { + params.set("nodeId", node.id); + } + const nodePath = getNodePathNames(nodeId, node); + if (nodePath) { + params.set("nodePath", JSON.stringify(nodePath)); + } + return `/tree.html?${params.toString()}`; +} + +function formatTreeCount(value) { + const numeric = toNumber(value, 0); + return numeric > 0 ? String(numeric) : "-"; +} + +function buildTreeRowStatusIcon(iconName, isActive, activeTitle, inactiveTitle) { + const icon = document.createElement("i"); + icon.classList.add("fa", "fa-fw", iconName, "lqos-tree-row-status-icon"); + if (!isActive) { + icon.classList.add("is-inactive"); + } + icon.setAttribute("data-bs-toggle", "tooltip"); + icon.setAttribute("data-bs-placement", "top"); + icon.setAttribute("title", isActive ? activeTitle : inactiveTitle); + return icon; +} + +function renderBreadcrumb() { + const target = document.getElementById("treeBreadcrumb"); + if (!target) { + return; + } + clearDiv(target); + if (!tree || tree[parent] === undefined) { + return; + } + + const current = tree[parent][1]; + const path = Array.isArray(current.parents) && current.parents.length > 0 + ? current.parents + : [parent]; + const ancestors = path.slice(0, -1); + const navWrap = document.createElement("div"); + navWrap.classList.add("lqos-tree-nav"); + const trail = document.createElement("div"); + trail.classList.add("lqos-tree-breadcrumb"); + + ancestors.forEach((nodeId, index) => { + const entry = tree[nodeId]; + if (!entry || !entry[1]) { + return; + } + if (index > 0) { + const separator = document.createElement("span"); + separator.classList.add("lqos-tree-breadcrumb-separator"); + separator.innerHTML = ""; + trail.appendChild(separator); + } + const link = document.createElement("a"); + link.href = treeHref(nodeId, entry[1]); + link.classList.add("lqos-tree-breadcrumb-link", "redactable"); + link.textContent = entry[1].name; + trail.appendChild(link); + }); + + if (ancestors.length > 0) { + const separator = document.createElement("span"); + separator.classList.add("lqos-tree-breadcrumb-separator"); + separator.innerHTML = ""; + trail.appendChild(separator); + } + + const currentNode = document.createElement("span"); + currentNode.classList.add("lqos-tree-breadcrumb-current", "redactable"); + currentNode.textContent = current.name; + trail.appendChild(currentNode); + + navWrap.appendChild(trail); + target.appendChild(navWrap); +} + +function renderContextMeta(node) { + const target = document.getElementById("treeContextMeta"); + if (!target) { + return; + } + clearDiv(target); + if (!node) { + return; + } + + const metaItems = []; + if (node.type) { + metaItems.push(node.type); + } + const branchSize = formatBranchSize(node); + if (branchSize) { + metaItems.push(branchSize); + } + metaItems.push(`Max ${formatRatePair(...effectiveMax(node))}`); + + metaItems.forEach((text) => { + const pill = document.createElement("span"); + pill.classList.add("lqos-tree-context-pill"); + pill.textContent = text; + target.appendChild(pill); + }); +} + +function setNodeOverrideFlash(message, variant = "success") { + nodeRateOverrideState.flash = message ? {message, variant} : null; + renderNodeSettings(); +} + +function renderAlertMessages(targetId, messages, variant) { + const target = document.getElementById(targetId); + if (!target) { + return; + } + clearDiv(target); + messages.forEach((message) => { + if (!message) { + return; + } + const alert = document.createElement("div"); + alert.classList.add("alert", `alert-${variant}`, "py-2", "px-3", "small"); + alert.textContent = message; + target.appendChild(alert); + }); +} + +function escapeHtml(text) { + return String(text) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function renderCompatibilityWarningsIndicator(messages) { + const target = document.getElementById("nodeOverrideCompatibilityIndicator"); + if (!target) { + return; + } + const visibleMessages = Array.isArray(messages) ? messages.filter(Boolean) : []; + const nextSignature = JSON.stringify(visibleMessages); + if (target.dataset.warningSignature === nextSignature) { + return; + } + target.dataset.warningSignature = nextSignature; + clearDiv(target); + if (visibleMessages.length === 0) { + return; + } + const tooltipHtml = visibleMessages + .map((message) => `
    ${escapeHtml(message)}
    `) + .join(""); + const icon = document.createElement("span"); + icon.classList.add("lqos-tree-compat-indicator"); + icon.setAttribute("data-bs-toggle", "tooltip"); + icon.setAttribute("data-bs-placement", "top"); + icon.setAttribute("data-bs-html", "true"); + icon.setAttribute("title", tooltipHtml); + icon.setAttribute("aria-label", `Compatibility warnings: ${visibleMessages.length}`); + icon.innerHTML = `${visibleMessages.length}`; + target.appendChild(icon); +} + +function currentOverridePair(node, overrideData) { + const base = configuredMax(node); + return [ + overrideData && overrideData.has_override && overrideData.override_download_bandwidth_mbps !== null && overrideData.override_download_bandwidth_mbps !== undefined + ? overrideData.override_download_bandwidth_mbps + : base[0], + overrideData && overrideData.has_override && overrideData.override_upload_bandwidth_mbps !== null && overrideData.override_upload_bandwidth_mbps !== undefined + ? overrideData.override_upload_bandwidth_mbps + : base[1], + ]; +} + +function isPendingApply(node, overrideData) { + if (!node || !overrideData || !overrideData.has_override) { + return false; + } + const configured = configuredMax(node); + if (overrideData.override_download_bandwidth_mbps !== null + && overrideData.override_download_bandwidth_mbps !== undefined + && !ratesApproximatelyEqual(overrideData.override_download_bandwidth_mbps, configured[0])) { + return true; + } + if (overrideData.override_upload_bandwidth_mbps !== null + && overrideData.override_upload_bandwidth_mbps !== undefined + && !ratesApproximatelyEqual(overrideData.override_upload_bandwidth_mbps, configured[1])) { + return true; + } + return false; +} + +function setNodeOverrideInputsDisabled(disabled) { + ["nodeOverrideDownload", "nodeOverrideUpload", "nodeOverrideSave", "nodeOverrideClear"].forEach((id) => { + const el = document.getElementById(id); + if (el) { + el.disabled = disabled; } }); } +function maybeSeedOverrideInputs(node, overrideData, force = false) { + const downloadInput = document.getElementById("nodeOverrideDownload"); + const uploadInput = document.getElementById("nodeOverrideUpload"); + if (!downloadInput || !uploadInput || !node) { + return; + } + + const [down, up] = currentOverridePair(node, overrideData); + const signature = JSON.stringify({ + nodeId: node.id ?? null, + nodeName: node.name, + download: down, + upload: up, + }); + if (!force && nodeOverrideInputsDirty) { + return; + } + if (!force && nodeOverrideLastSeedSignature === signature) { + return; + } + + downloadInput.value = formatMbpsInputValue(down); + uploadInput.value = formatMbpsInputValue(up); + nodeOverrideLastSeedSignature = signature; + nodeOverrideInputsDirty = false; +} + +function renderOverrideValue(node, overrideData) { + const target = document.getElementById("nodeSettingsOverride"); + if (!target) { + return; + } + target.textContent = ""; + clearDiv(target); + const wrap = document.createElement("span"); + wrap.classList.add("lqos-tree-detail-value"); + + const value = document.createElement("span"); + if (!overrideData || !overrideData.has_override) { + value.textContent = "- / -"; + wrap.appendChild(value); + target.appendChild(wrap); + return; + } + + const [overrideDown, overrideUp] = currentOverridePair(node, overrideData); + value.textContent = formatRatePair(overrideDown, overrideUp); + wrap.appendChild(value); + + if (isPendingApply(node, overrideData)) { + const pending = document.createElement("span"); + pending.classList.add("lqos-tree-pending"); + pending.setAttribute("data-bs-toggle", "tooltip"); + pending.setAttribute("data-bs-placement", "top"); + pending.setAttribute("title", NODE_OVERRIDE_PENDING_TOOLTIP); + + const symbol = document.createElement("span"); + symbol.classList.add("lqos-tree-pending-symbol"); + symbol.textContent = "⟳"; + pending.appendChild(symbol); + + const label = document.createElement("span"); + label.textContent = "Pending"; + pending.appendChild(label); + wrap.appendChild(pending); + } + + target.appendChild(wrap); +} + +function renderNodeSettings() { + const node = currentNode(); + if (!node) { + return; + } + const syntheticRoot = isSyntheticRootNode(node); + + const statusTarget = document.getElementById("nodeOverrideStatus"); + if (statusTarget) { + if (nodeRateOverrideState.loading) { + statusTarget.textContent = "Loading operator override..."; + } else if (nodeRateOverrideState.saving) { + statusTarget.textContent = "Saving operator override..."; + } else if (nodeRateOverrideState.error) { + statusTarget.textContent = nodeRateOverrideState.error; + } else { + statusTarget.textContent = ""; + } + } + + const typeTarget = document.getElementById("nodeSettingsType"); + if (typeTarget) { + typeTarget.textContent = node.type ?? "-"; + } + const branchSizeTarget = document.getElementById("nodeSettingsBranchSize"); + if (branchSizeTarget) { + branchSizeTarget.textContent = formatBranchSize(node); + } + const baseConfiguredTarget = document.getElementById("nodeSettingsBaseConfigured"); + if (baseConfiguredTarget) { + const configured = configuredMax(node); + baseConfiguredTarget.textContent = formatRatePair(configured[0], configured[1]); + } + const effectiveTarget = document.getElementById("nodeSettingsEffectiveNow"); + if (effectiveTarget) { + const effective = effectiveMax(node); + effectiveTarget.textContent = formatRatePair(effective[0], effective[1]); + } + + renderOverrideValue(node, nodeRateOverrideState.data); + + renderAlertMessages( + "nodeOverrideFlash", + nodeRateOverrideState.flash ? [nodeRateOverrideState.flash.message] : [], + nodeRateOverrideState.flash ? nodeRateOverrideState.flash.variant : "success", + ); + renderCompatibilityWarningsIndicator(nodeRateOverrideState.data?.legacy_warnings || []); + const disabledReason = syntheticRoot ? null : nodeRateOverrideState.data?.disabled_reason; + renderAlertMessages( + "nodeOverrideDisabledReason", + disabledReason ? [disabledReason] : [], + "secondary", + ); + + maybeSeedOverrideInputs(node, nodeRateOverrideState.data); + + const canEdit = !!nodeRateOverrideState.data?.can_edit && !nodeRateOverrideState.loading && !nodeRateOverrideState.saving; + setNodeOverrideInputsDisabled(!canEdit); + const editorSection = document.getElementById("nodeOverrideEditorSection"); + if (editorSection) { + editorSection.hidden = !canEdit; + } + const clearButton = document.getElementById("nodeOverrideClear"); + if (clearButton) { + clearButton.disabled = !canEdit || !nodeRateOverrideState.data?.has_override; + } + + const detailsPanel = document.querySelector(".lqos-tree-details-panel"); + if (detailsPanel) { + enableTooltipsWithin(detailsPanel); + } +} + +function currentTreeAttachedCircuitsQuery() { + const node = currentNode(); + if (!node) { + return null; + } + const query = { + page: 0, + page_size: 100, + sort: "CircuitName", + descending: false, + }; + const nodeId = selectionLocator.nodeId || node.id; + const nodePath = selectionLocator.nodePath || getNodePathNames(parent, node); + if (nodeId) { + query.node_id = nodeId; + } + if (nodePath) { + query.node_path = nodePath; + } + return query; +} + +function requestTreeAttachedCircuitsWatch(force = false) { + const query = currentTreeAttachedCircuitsQuery(); + if (!query) { + return; + } + const signature = JSON.stringify(query); + if (!force && attachedCircuitsWatchSignature === signature) { + return; + } + attachedCircuitsWatchSignature = signature; + sendPrivateRequest({ + WatchTreeAttachedCircuits: { + query, + }, + }); +} + +async function loadNodeRateOverrideState() { + const query = currentNodeRateQuery(); + if (!query) { + return; + } + nodeRateOverrideState.loading = true; + nodeRateOverrideState.error = null; + nodeRateOverrideState.data = null; + renderNodeSettings(); + try { + const response = await sendWsRequest("GetNodeRateOverride", { + GetNodeRateOverride: {query}, + }); + nodeRateOverrideState.data = response.data || null; + nodeRateOverrideState.error = null; + } catch (errorMsg) { + nodeRateOverrideState.data = null; + nodeRateOverrideState.error = errorMsg?.message || "Unable to load override state"; + } finally { + nodeRateOverrideState.loading = false; + renderNodeSettings(); + } +} + +async function saveNodeRateOverride() { + const node = currentNode(); + if (!node || !nodeRateOverrideState.data?.can_edit || nodeRateOverrideState.saving) { + return; + } + + const downloadRaw = document.getElementById("nodeOverrideDownload")?.value ?? ""; + const uploadRaw = document.getElementById("nodeOverrideUpload")?.value ?? ""; + const download = downloadRaw === "" ? null : Number.parseFloat(downloadRaw); + const upload = uploadRaw === "" ? null : Number.parseFloat(uploadRaw); + if ((download !== null && (!Number.isFinite(download) || download < 0)) + || (upload !== null && (!Number.isFinite(upload) || upload < 0)) + || (download === null && upload === null)) { + setNodeOverrideFlash("Enter valid non-negative download and upload rates before saving.", "danger"); + return; + } + + nodeRateOverrideState.saving = true; + nodeRateOverrideState.error = null; + renderNodeSettings(); + try { + const response = await sendWsRequest("SetNodeRateOverrideResult", { + SetNodeRateOverride: { + update: { + node_id: node.id, + node_name: node.name, + download_bandwidth_mbps: download, + upload_bandwidth_mbps: upload, + }, + }, + }); + if (!response.ok) { + setNodeOverrideFlash(response.message || "Unable to save override.", "danger"); + nodeRateOverrideState.data = response.data || nodeRateOverrideState.data; + } else { + nodeRateOverrideState.data = response.data || nodeRateOverrideState.data; + setNodeOverrideFlash(response.message || "Override saved.", "success"); + nodeOverrideInputsDirty = false; + nodeOverrideLastSeedSignature = null; + maybeSeedOverrideInputs(node, nodeRateOverrideState.data, true); + } + } catch (errorMsg) { + setNodeOverrideFlash(errorMsg?.message || "Unable to save override.", "danger"); + } finally { + nodeRateOverrideState.saving = false; + renderNodeSettings(); + } +} + +async function clearNodeRateOverride() { + const node = currentNode(); + if (!node || !nodeRateOverrideState.data?.can_edit || nodeRateOverrideState.saving) { + return; + } + + nodeRateOverrideState.saving = true; + nodeRateOverrideState.error = null; + renderNodeSettings(); + try { + const response = await sendWsRequest("ClearNodeRateOverrideResult", { + ClearNodeRateOverride: { + query: currentNodeRateQuery(), + }, + }); + if (!response.ok) { + setNodeOverrideFlash(response.message || "Unable to clear override.", "danger"); + nodeRateOverrideState.data = response.data || nodeRateOverrideState.data; + } else { + nodeRateOverrideState.data = response.data || nodeRateOverrideState.data; + nodeOverrideInputsDirty = false; + nodeOverrideLastSeedSignature = null; + maybeSeedOverrideInputs(node, nodeRateOverrideState.data, true); + setNodeOverrideFlash(response.message || "Override cleared.", "success"); + } + } catch (errorMsg) { + setNodeOverrideFlash(errorMsg?.message || "Unable to clear override.", "danger"); + } finally { + nodeRateOverrideState.saving = false; + renderNodeSettings(); + } +} + function formatQooScore(score0to100, fallback = "-") { if (score0to100 === null || score0to100 === undefined) { return fallback; @@ -153,9 +1090,49 @@ function formatQooScore(score0to100, fallback = "-") { return "■" + clamped; } +function representativeNodeQoo(node) { + if (!node || !node.qoo) { + return null; + } + const values = [node.qoo[0], node.qoo[1]] + .map((entry) => Number(entry)) + .filter((entry) => Number.isFinite(entry) && entry !== 255); + if (values.length === 0) { + return null; + } + return values.reduce((sum, entry) => sum + entry, 0) / values.length; +} + +function ensureTreeGauges() { + if (!treeBitsGauge) { + treeBitsGauge = new BitsPerSecondGauge("treeBitsGauge", "Max"); + } + if (!treeQooGauge) { + treeQooGauge = new QooScoreGauge("treeQooGauge"); + } +} + +function updateTreeGauges(node) { + if (!node) { + return; + } + ensureTreeGauges(); + const gaugeMax = nodeSnapshotGaugeMax(node); + treeBitsGauge.update( + toNumber(node.current_throughput?.[0], 0) * 8, + toNumber(node.current_throughput?.[1], 0) * 8, + gaugeMax[0], + gaugeMax[1], + ); + treeQooGauge.update(representativeNodeQoo(node)); +} + function buildChildrenMap() { childrenByParentId = new Map(); for (let i=0; i" + - " Up One Level" + - "" + tree[upParent][1].name + "" + - ""; - treeStack.appendChild(navWrap); - } - treeTable.appendChild(tbody); // Clear and apply @@ -236,7 +1202,7 @@ function renderTree() { tableWrap.appendChild(treeTable); treeStack.appendChild(tableWrap); target.appendChild(treeStack); - initTooltipsWithin(treeTable); + enableTooltipsWithin(treeTable); } // This runs first and builds the initial structure on the page @@ -245,13 +1211,16 @@ function getInitialTree() { const data = msg && msg.data ? msg.data : []; tree = data; buildChildrenMap(); + const selectionState = reconcileSelection(); if (tree[parent] !== undefined) { fillHeader(tree[parent][1]); + loadNodeRateOverrideState(); } renderTree(); + requestTreeAttachedCircuitsWatch(true); if (!subscribed) { - subscribeWS(["NetworkTree", "NetworkTreeClients"], onMessage); + subscribeWS(["NetworkTree", "StormguardStatus"], onMessage); subscribed = true; } }); @@ -259,8 +1228,10 @@ function getInitialTree() { } function fillHeader(node) { - //console.log("Header"); - $("#nodeName").text(node.name); + renderHeaderStatusIcons(node); + renderBreadcrumb(); + renderContextMeta(node); + updateTreeGauges(node); const configured = configuredMax(node); const effective = effectiveMax(node); const configuredDown = formatLimitValue(configured[0]); @@ -279,23 +1250,23 @@ function fillHeader(node) { $("#parentConfiguredU").text(configuredUp).removeClass("lqos-limit-secondary is-match").addClass(matchClassUp); $("#parentTpD").html(formatThroughput(toNumber(node.current_throughput[0], 0) * 8, effective[0])); $("#parentTpU").html(formatThroughput(toNumber(node.current_throughput[1], 0) * 8, effective[1])); - //console.log(node); $("#parentRttD").html(formatRtt(node.rtts[0])); $("#parentRttU").html(formatRtt(node.rtts[1])); $("#parentQooD").html(formatQooScore(node.qoo ? node.qoo[0] : null)); $("#parentQooU").html(formatQooScore(node.qoo ? node.qoo[1] : null)); let retr = 0; - const packetsDown = toNumber(node.current_tcp_packets[0], 0); + const packetsDown = retransmitPacketsForNode(node, 0); if (packetsDown > 0) { retr = toNumber(node.current_retransmits[0], 0) / packetsDown; } $("#parentRxmitD").html(formatRetransmit(retr)); retr = 0; - const packetsUp = toNumber(node.current_tcp_packets[1], 0); + const packetsUp = retransmitPacketsForNode(node, 1); if (packetsUp > 0) { retr = toNumber(node.current_retransmits[1], 0) / packetsUp; } $("#parentRxmitU").html(formatRetransmit(retr)); + renderNodeSettings(); } function iterateChildren(idx, tBody, depth) { @@ -317,18 +1288,20 @@ function buildRow(i, depth=0) { let row = document.createElement("tr"); row.classList.add("small"); let col = document.createElement("td"); - col.style.textOverflow = "ellipsis"; - col.classList.add("small"); - if (depth > 0) { - col.style.paddingLeft = (depth * 1.25) + "rem"; - } + col.classList.add("small", "lqos-tree-node-cell"); let nameWrap = document.createElement("div"); - nameWrap.classList.add("d-flex", "align-items-center", "gap-1"); + nameWrap.classList.add("lqos-tree-node-wrap"); + if (depth > 0) { + nameWrap.style.paddingLeft = (depth * 1.25) + "rem"; + } + const lead = document.createElement("div"); + lead.classList.add("lqos-tree-node-lead"); + const toggleSlot = document.createElement("div"); + toggleSlot.classList.add("lqos-tree-toggle-slot"); if (hasChildren(nodeId)) { let toggle = document.createElement("button"); toggle.type = "button"; - toggle.classList.add("btn", "btn-link", "btn-sm", "p-0", "text-decoration-none"); - toggle.style.lineHeight = "1"; + toggle.classList.add("btn", "btn-link", "btn-sm", "p-0", "text-decoration-none", "lqos-tree-toggle"); let icon = document.createElement("i"); icon.classList.add("fa", "fa-fw", expandedNodes.has(nodeId) ? "fa-minus" : "fa-plus"); toggle.appendChild(icon); @@ -339,34 +1312,49 @@ function buildRow(i, depth=0) { event.stopPropagation(); toggleNode(nodeId); }); - nameWrap.appendChild(toggle); - } else { - let spacer = document.createElement("i"); - spacer.classList.add("fa", "fa-fw", "fa-plus"); - spacer.style.visibility = "hidden"; - nameWrap.appendChild(spacer); - } - if (node.virtual === true) { - let virtualIcon = document.createElement("i"); - virtualIcon.classList.add("fa", "fa-fw", "fa-ghost", "text-secondary"); - virtualIcon.setAttribute("data-bs-toggle", "tooltip"); - virtualIcon.setAttribute("data-bs-placement", "top"); - virtualIcon.setAttribute("title", "Virtual node (logical only; not shaped in HTB)."); - nameWrap.appendChild(virtualIcon); + toggleSlot.appendChild(toggle); } + lead.appendChild(toggleSlot); let link = document.createElement("a"); - link.href = "/tree.html?parent=" + nodeId + "&upParent=" + parent; - link.classList.add("redactable"); + link.href = treeHref(nodeId, node); + link.classList.add("redactable", "lqos-tree-node-link"); link.textContent = node.name; - nameWrap.appendChild(link); - if (node.type !== null) { - let typeText = document.createElement("span"); - typeText.textContent = " (" + node.type + ")"; - nameWrap.appendChild(typeText); - } + lead.appendChild(link); + nameWrap.appendChild(lead); + const statusWrap = document.createElement("div"); + statusWrap.classList.add("lqos-tree-row-statuses"); + statusWrap.appendChild(buildTreeRowStatusIcon( + "fa-ghost", + node.virtual === true, + "Virtual node: Active", + "Virtual node: Inactive", + )); + statusWrap.appendChild(buildTreeRowStatusIcon( + "fa-layer-group", + node.runtime_virtualized === true, + "TreeGuard runtime virtualization: Active", + "TreeGuard runtime virtualization: Inactive", + )); + statusWrap.appendChild(buildTreeRowStatusIcon( + "fa-cloud-bolt", + isStormguardNode(node), + "StormGuard: Active", + "StormGuard: Inactive", + )); + nameWrap.appendChild(statusWrap); col.appendChild(nameWrap); row.appendChild(col); + col = document.createElement("td"); + col.classList.add("small"); + col.textContent = formatTreeCount(subtreeCircuitCount(node)); + row.appendChild(col); + + col = document.createElement("td"); + col.classList.add("small"); + col.textContent = formatTreeCount(subtreeSiteCount(node)); + row.appendChild(col); + col = document.createElement("td"); col.id = "limit-" + nodeId; col.classList.add("small"); @@ -423,7 +1411,12 @@ function buildRow(i, depth=0) { col.id = "re-xmit-down-" + nodeId; col.style.width = "6%"; if (node.current_retransmits[0] !== undefined) { - col.innerHTML = formatRetransmitRaw(node.current_retransmits[0]); + let retr = 0; + const packetsDown = retransmitPacketsForNode(node, 0); + if (packetsDown > 0) { + retr = toNumber(node.current_retransmits[0], 0) / packetsDown; + } + col.innerHTML = formatRetransmitFraction(retr); } else { col.textContent = "-"; } @@ -433,7 +1426,12 @@ function buildRow(i, depth=0) { col.id = "re-xmit-up-" + nodeId; col.style.width = "6%"; if (node.current_retransmits[1] !== undefined) { - col.innerHTML = formatRetransmitRaw(node.current_retransmits[1]); + let retr = 0; + const packetsUp = retransmitPacketsForNode(node, 1); + if (packetsUp > 0) { + retr = toNumber(node.current_retransmits[1], 0) / packetsUp; + } + col.innerHTML = formatRetransmitFraction(retr); } else { col.textContent = "-"; } @@ -485,24 +1483,28 @@ function buildRow(i, depth=0) { function treeUpdate(msg) { //console.log(msg); let needsRebuild = false; + const seenNodeIds = new Set(); msg.data.forEach((n) => { let nodeId = n[0]; let node = n[1]; + seenNodeIds.add(nodeId); if (tree[nodeId] === undefined) { tree[nodeId] = [nodeId, node]; needsRebuild = true; } else { - if (tree[nodeId][1].immediate_parent !== node.immediate_parent) { + if (tree[nodeId][1].immediate_parent !== node.immediate_parent + || tree[nodeId][1].subtree_site_count !== node.subtree_site_count + || tree[nodeId][1].subtree_circuit_count !== node.subtree_circuit_count + || tree[nodeId][1].type !== node.type + || tree[nodeId][1].virtual !== node.virtual + || tree[nodeId][1].runtime_virtualized !== node.runtime_virtualized + || tree[nodeId][1].name !== node.name) { needsRebuild = true; } tree[nodeId][1] = node; } - if (nodeId === parent) { - fillHeader(node); - } - let col = document.getElementById("limit-" + nodeId); if (col !== null) { col.innerHTML = buildEffectiveLimitCellHtml(node); @@ -540,7 +1542,7 @@ function treeUpdate(msg) { if (col !== null) { if (node.current_retransmits[0] !== undefined) { let retr = 0; - const packetsDown = toNumber(node.current_tcp_packets[0], 0); + const packetsDown = retransmitPacketsForNode(node, 0); if (packetsDown > 0) { retr = toNumber(node.current_retransmits[0], 0) / packetsDown; } @@ -553,7 +1555,7 @@ function treeUpdate(msg) { if (col !== null) { if (node.current_retransmits[1] !== undefined) { let retr = 0; - const packetsUp = toNumber(node.current_tcp_packets[1], 0); + const packetsUp = retransmitPacketsForNode(node, 1); if (packetsUp > 0) { retr = toNumber(node.current_retransmits[1], 0) / packetsUp; } @@ -595,14 +1597,35 @@ function treeUpdate(msg) { } } }); + for (let i = 0; i < tree.length; i++) { + if (tree[i] === undefined || seenNodeIds.has(i)) { + continue; + } + delete tree[i]; + needsRebuild = true; + } if (needsRebuild) { buildChildrenMap(); + } + const selectionState = reconcileSelection(); + if (selectionState.identityChanged) { + loadNodeRateOverrideState(); + } + if (tree[parent] !== undefined) { + fillHeader(tree[parent][1]); + } + if (needsRebuild || selectionState.parentChanged) { renderTree(); } + if (selectionState.parentChanged || selectionState.identityChanged) { + requestTreeAttachedCircuitsWatch(true); + } } -function clientsUpdate(msg) { - let myName = tree[parent][1].name; +function renderAttachedCircuitsRows(rows) { + if (!Array.isArray(rows)) { + return; + } let target = document.getElementById("clients"); let table = document.createElement("table"); @@ -610,63 +1633,7 @@ function clientsUpdate(msg) { table.appendChild(clientTableHeader()); let tbody = document.createElement("tbody"); clearDiv(target); - - const circuits = new Map(); - msg.data.forEach((device) => { - if (device.parent_node !== myName) { - return; - } - - const circuitId = device.circuit_id || `${device.parent_node || ""}:${device.circuit_name || ""}`; - if (!circuits.has(circuitId)) { - circuits.set(circuitId, { - circuit_id: circuitId, - circuit_name: device.circuit_name || "(Unknown circuit)", - parent_node: device.parent_node || "", - plan: { - down: toNumber(device.plan?.down, 0), - up: toNumber(device.plan?.up, 0), - }, - device_names: new Set(), - ips: new Set(), - last_seen_nanos: toNumber(device.last_seen_nanos, 0), - bytes_per_second: {down: 0, up: 0}, - median_latency: {down: null, up: null}, - tcp_packets: {down: 0, up: 0}, - tcp_retransmits: {down: 0, up: 0}, - }); - } - - const circuit = circuits.get(circuitId); - if (device.device_name) { - circuit.device_names.add(device.device_name); - } - const ipText = formatDeviceIp(device.ip); - if (ipText && ipText !== "-") { - circuit.ips.add(ipText); - } - - circuit.last_seen_nanos = Math.min(circuit.last_seen_nanos, toNumber(device.last_seen_nanos, 0)); - circuit.bytes_per_second.down += toNumber(device.bytes_per_second?.down, 0); - circuit.bytes_per_second.up += toNumber(device.bytes_per_second?.up, 0); - circuit.tcp_packets.down += toNumber(device.tcp_packets?.down, 0); - circuit.tcp_packets.up += toNumber(device.tcp_packets?.up, 0); - circuit.tcp_retransmits.down += toNumber(device.tcp_retransmits?.down, 0); - circuit.tcp_retransmits.up += toNumber(device.tcp_retransmits?.up, 0); - - const downLatency = toNumber(device.median_latency?.down, 0); - if (downLatency > 0 && (circuit.median_latency.down === null || downLatency > circuit.median_latency.down)) { - circuit.median_latency.down = downLatency; - } - const upLatency = toNumber(device.median_latency?.up, 0); - if (upLatency > 0 && (circuit.median_latency.up === null || upLatency > circuit.median_latency.up)) { - circuit.median_latency.up = upLatency; - } - }); - - Array.from(circuits.values()) - .sort((a, b) => a.circuit_name.localeCompare(b.circuit_name)) - .forEach((circuit) => { + rows.forEach((circuit) => { let tr = document.createElement("tr"); tr.classList.add("small"); @@ -675,10 +1642,12 @@ function clientsUpdate(msg) { circuitLink.href = "/circuit.html?id=" + circuit.circuit_id; circuitLink.innerText = circuit.circuit_name; circuitLink.classList.add("redactable"); + circuitLink.classList.add("lqos-tree-circuit-name"); + circuitLink.title = circuit.circuit_name; linkTd.appendChild(circuitLink); tr.appendChild(linkTd); - const deviceNames = Array.from(circuit.device_names); + const deviceNames = Array.isArray(circuit.device_names) ? circuit.device_names : []; const deviceCell = simpleRow( deviceNames.length > 2 ? `${deviceNames[0]}, ${deviceNames[1]} +${deviceNames.length - 2}` : deviceNames.join(", "), true @@ -688,48 +1657,52 @@ function clientsUpdate(msg) { } tr.appendChild(deviceCell); - tr.appendChild(simpleRow(circuit.plan.down + " / " + circuit.plan.up)); + const planCell = simpleRow( + `${toNumber(circuit.plan_mbps?.down, 0)} / ${toNumber(circuit.plan_mbps?.up, 0)}` + ); + planCell.classList.add("lqos-tree-plan-cell"); + if (circuit.ethernet_cap_badge) { + const badge = document.createElement("a"); + badge.className = `badge rounded-pill ms-2 text-decoration-none ${ethernetBadgeClass(circuit.ethernet_cap_badge.tier_label)}`; + badge.href = ethernetCapsPageHref(circuit.ethernet_cap_badge); + badge.setAttribute("aria-label", `Review ${circuit.ethernet_cap_badge.tier_label} Ethernet-limited circuits`); + badge.setAttribute("data-bs-toggle", "tooltip"); + badge.setAttribute("data-bs-placement", "top"); + badge.setAttribute("title", formatEthernetTooltip(circuit.ethernet_cap_badge)); + badge.textContent = circuit.ethernet_cap_badge.tier_label; + planCell.appendChild(badge); + } + tr.appendChild(planCell); tr.appendChild(simpleRow(circuit.parent_node, true)); - const ipList = Array.from(circuit.ips); - const ipCell = simpleRow( - ipList.length > 2 ? `${ipList[0]}, ${ipList[1]} +${ipList.length - 2}` : ipList.join(", "), - true - ); + const ipList = Array.isArray(circuit.ip_addrs) ? circuit.ip_addrs : []; + const ipCell = simpleRow(summarizeIpListForTable(ipList), true); + ipCell.classList.add("lqos-tree-ip-cell"); if (ipList.length > 0) { ipCell.title = ipList.join(", "); } tr.appendChild(ipCell); - tr.appendChild(simpleRow(formatLastSeen(circuit.last_seen_nanos))); - tr.appendChild(simpleRowHtml(formatThroughput(circuit.bytes_per_second.down * 8, circuit.plan.down))); - tr.appendChild(simpleRowHtml(formatThroughput(circuit.bytes_per_second.up * 8, circuit.plan.up))); + tr.appendChild(simpleRow(formatLastSeen(toNumber(circuit.last_seen_nanos, 0)))); + tr.appendChild(simpleRowHtml(formatThroughput(toNumber(circuit.bytes_per_second?.down, 0) * 8, toNumber(circuit.plan_mbps?.down, 0)))); + tr.appendChild(simpleRowHtml(formatThroughput(toNumber(circuit.bytes_per_second?.up, 0) * 8, toNumber(circuit.plan_mbps?.up, 0)))); - if (circuit.median_latency.down !== null) { - tr.appendChild(simpleRowHtml(formatRtt(circuit.median_latency.down))); + if (toNumber(circuit.rtt_current_p50_nanos?.down, 0) > 0) { + tr.appendChild(simpleRowHtml(formatRtt(toNumber(circuit.rtt_current_p50_nanos?.down, 0) / 1_000_000))); } else { tr.appendChild(simpleRow("-")); } - if (circuit.median_latency.up !== null) { - tr.appendChild(simpleRowHtml(formatRtt(circuit.median_latency.up))); + if (toNumber(circuit.rtt_current_p50_nanos?.up, 0) > 0) { + tr.appendChild(simpleRowHtml(formatRtt(toNumber(circuit.rtt_current_p50_nanos?.up, 0) / 1_000_000))); } else { tr.appendChild(simpleRow("-")); } - let retr = 0; - if (circuit.tcp_packets.down > 0) { - retr = circuit.tcp_retransmits.down / circuit.tcp_packets.down; - } - tr.appendChild(simpleRowHtml(formatRetransmit(retr))); - - retr = 0; - if (circuit.tcp_packets.up > 0) { - retr = circuit.tcp_retransmits.up / circuit.tcp_packets.up; - } - tr.appendChild(simpleRowHtml(formatRetransmit(retr))); + tr.appendChild(simpleRowHtml(formatRetransmit(retransmitFractionFromSample(circuit.tcp_retransmit_sample?.down)))); + tr.appendChild(simpleRowHtml(formatRetransmit(retransmitFractionFromSample(circuit.tcp_retransmit_sample?.up)))); tbody.appendChild(tr); - }); + }); table.appendChild(tbody); const sectionLabel = document.createElement("div"); sectionLabel.classList.add("lqos-tree-section-label"); @@ -739,13 +1712,41 @@ function clientsUpdate(msg) { tableWrap.appendChild(table); target.appendChild(sectionLabel); target.appendChild(tableWrap); + enableTooltipsWithin(target); +} + +function attachedCircuitsUpdate(msg) { + lastAttachedCircuitsPage = msg?.data || null; + renderAttachedCircuitsRows(lastAttachedCircuitsPage?.rows || []); +} + +function stormguardUpdate(msg) { + const nextNodes = new Set(); + msg.data.forEach((entry) => { + if (!Array.isArray(entry) || typeof entry[0] !== "string" || entry[0].length === 0) { + return; + } + nextNodes.add(entry[0]); + }); + if (sameStringSet(stormguardNodes, nextNodes)) { + return; + } + stormguardNodes = nextNodes; + const selectionState = reconcileSelection(); + if (selectionState.parentChanged || selectionState.identityChanged) { + requestTreeAttachedCircuitsWatch(true); + } + if (tree && tree[parent] !== undefined) { + fillHeader(tree[parent][1]); + renderTree(); + } } function onMessage(msg) { if (msg.event === "NetworkTree") { treeUpdate(msg); - } else if (msg.event === "NetworkTreeClients") { - clientsUpdate(msg); + } else if (msg.event === "StormguardStatus") { + stormguardUpdate(msg); } } @@ -754,13 +1755,42 @@ const params = new Proxy(new URLSearchParams(window.location.search), { }); if (params.parent !== null) { - parent = parseInt(params.parent); + const parsedParent = parseInt(params.parent, 10); + parent = Number.isFinite(parsedParent) ? parsedParent : 0; } else { parent = 0; } if (params.upParent !== null) { - upParent = parseInt(params.upParent); + const parsedUpParent = parseInt(params.upParent, 10); + upParent = Number.isFinite(parsedUpParent) ? parsedUpParent : 0; } +selectionLocator.nodeId = typeof params.nodeId === "string" && params.nodeId.length > 0 ? params.nodeId : null; +selectionLocator.nodePath = parseNodePathParam(params.nodePath); +selectionLocator.lastKnownIndex = parent; + +document.getElementById("nodeOverrideDownload")?.addEventListener("input", () => { + nodeOverrideInputsDirty = true; +}); +document.getElementById("nodeOverrideUpload")?.addEventListener("input", () => { + nodeOverrideInputsDirty = true; +}); +document.getElementById("nodeOverrideSave")?.addEventListener("click", () => { + saveNodeRateOverride(); +}); +document.getElementById("nodeOverrideClear")?.addEventListener("click", () => { + clearNodeRateOverride(); +}); +wsClient.on("TreeAttachedCircuitsSnapshot", attachedCircuitsUpdate); +wsClient.on("TreeAttachedCircuitsUpdate", attachedCircuitsUpdate); +wsClient.on("join", () => { + if (!tree) { + return; + } + requestTreeAttachedCircuitsWatch(true); + loadNodeRateOverrideState(); +}); + +loadRootGaugeConfigMax(); getInitialTree(); diff --git a/src/rust/lqosd/src/node_manager/local_api.rs b/src/rust/lqosd/src/node_manager/local_api.rs index d5f971d2..88040112 100644 --- a/src/rust/lqosd/src/node_manager/local_api.rs +++ b/src/rust/lqosd/src/node_manager/local_api.rs @@ -1,18 +1,28 @@ pub(crate) mod circuit; +pub(crate) mod circuit_activity; pub(crate) mod circuit_count; +pub(crate) mod circuit_live; pub(crate) mod config; pub(crate) mod cpu_affinity; pub(crate) mod dashboard_themes; pub(crate) mod device_counts; +pub(crate) mod directories; +pub(crate) mod ethernet_caps; +pub(crate) mod executive; +pub(crate) mod executive_cache; pub(crate) mod flow_explorer; pub(crate) mod flow_map; pub mod lts; pub(crate) mod network_tree; +pub(crate) mod network_tree_lite; +pub(crate) mod node_rate_overrides; pub(crate) mod packet_analysis; pub(crate) mod reload_libreqos; pub(crate) mod scheduler; pub(crate) mod search; pub(crate) mod shaped_device_api; +pub(crate) mod shaped_devices_page; +pub(crate) mod tree_attached_circuits; pub(crate) mod unknown_ips; pub(crate) mod urgent; pub(crate) mod warnings; diff --git a/src/rust/lqosd/src/node_manager/local_api/circuit.rs b/src/rust/lqosd/src/node_manager/local_api/circuit.rs index 8991bfe8..0e191a87 100644 --- a/src/rust/lqosd/src/node_manager/local_api/circuit.rs +++ b/src/rust/lqosd/src/node_manager/local_api/circuit.rs @@ -1,7 +1,29 @@ +use crate::node_manager::local_api::ethernet_caps::ethernet_advisory_for_circuit; use crate::shaped_devices_tracker::SHAPED_DEVICES; -use lqos_config::ShapedDevice; +use lqos_config::{CircuitEthernetMetadata, ShapedDevice}; +use serde::{Deserialize, Serialize}; -pub fn circuit_by_id_data(id: &str) -> Option> { +/// Circuit-page payload containing shaped devices plus optional Ethernet advisory metadata. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CircuitByIdData { + /// Shaped-device rows for the requested circuit. + pub devices: Vec, + /// Optional negotiated-Ethernet advisory derived from integration metadata. + pub ethernet_advisory: Option, +} + +fn load_ethernet_advisory( + circuit_id: &str, + devices: &[ShapedDevice], +) -> Option { + let device_ids: std::collections::HashSet<&str> = devices + .iter() + .map(|device| device.device_id.as_str()) + .collect(); + ethernet_advisory_for_circuit(circuit_id, &device_ids) +} + +pub fn circuit_by_id_data(id: &str) -> Option { let safe_id = id.to_lowercase().trim().to_string(); let reader = SHAPED_DEVICES.load(); let devices: Vec = reader @@ -14,6 +36,10 @@ pub fn circuit_by_id_data(id: &str) -> Option> { if devices.is_empty() { None } else { - Some(devices) + let ethernet_advisory = load_ethernet_advisory(&safe_id, &devices); + Some(CircuitByIdData { + devices, + ethernet_advisory, + }) } } diff --git a/src/rust/lqosd/src/node_manager/local_api/circuit_activity.rs b/src/rust/lqosd/src/node_manager/local_api/circuit_activity.rs new file mode 100644 index 00000000..1a779524 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/circuit_activity.rs @@ -0,0 +1,518 @@ +use crate::shaped_devices_tracker::{SHAPED_DEVICE_HASH_CACHE, SHAPED_DEVICES}; +use crate::throughput_tracker::flow_data::{ + ALL_FLOWS, FlowbeeLocalData, get_asn_name_and_country, +}; +use lqos_utils::hash_to_i64; +use lqos_utils::units::{DownUpOrder, TcpRetransmitSample}; +use lqos_utils::unix_time::time_since_boot; +use serde::{Deserialize, Serialize}; +use std::cmp::Ordering; +use std::time::Duration; + +const RECENT_CIRCUIT_FLOWS_WINDOW_NANOS: u64 = 30 * 1_000_000_000; +const SANKEY_RECENT_FLOW_WINDOW_NANOS: u64 = 10 * 1_000_000_000; +const SANKEY_TOP_FLOW_LIMIT: usize = 20; +const TOP_ASN_LIMIT: usize = 10; +const FLOW_RATE_SANITY_MULTIPLIER: f64 = 2.0; +const FLOW_RATE_SANITY_FLOOR_BPS: u64 = 25_000_000; +const TRAFFIC_FLOW_HIDE_THRESHOLD_BPS: u32 = 1_048_576; + +/// Lightweight live summary for the circuit page header and Queue Dynamics. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitSummaryData { + pub circuit_id: String, + pub bytes_per_second: DownUpOrder, + pub rtt_current_p50_nanos: DownUpOrder>, + pub tcp_retransmit_sample: DownUpOrder, + pub qoo_score: Option, + pub rtt_excluded: bool, + pub active_flow_count: usize, + pub active_asn_count: usize, +} + +/// Server-side query for the live `Traffic Flows` table. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitTrafficFlowsQuery { + pub circuit: String, + pub page: usize, + pub page_size: usize, + pub hide_small: bool, + pub sort_column: String, + pub sort_direction: String, +} + +/// Compact server-side row for the live `Traffic Flows` table. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitTrafficFlowRow { + pub protocol_name: String, + pub down_bps: u32, + pub up_bps: u32, + pub bytes_sent_down: u64, + pub bytes_sent_up: u64, + pub packets_sent_down: u64, + pub packets_sent_up: u64, + pub tcp_retransmits_down: u16, + pub tcp_retransmits_up: u16, + pub retransmit_down_pct: f64, + pub retransmit_up_pct: f64, + pub rtt_down_nanos: u64, + pub rtt_up_nanos: u64, + pub qoo_down: Option, + pub qoo_up: Option, + pub asn_name: String, + pub asn_country: String, + pub remote_ip: String, + pub opacity: f64, + pub sort_rate_bps: f64, +} + +/// Page of server-side circuit traffic-flow rows. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitTrafficFlowsPage { + pub query: CircuitTrafficFlowsQuery, + pub total_rows: usize, + pub rows: Vec, +} + +/// Server-side query for the circuit `Top ASNs` table. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitTopAsnsQuery { + pub circuit: String, + pub hide_small: bool, +} + +/// Aggregated row for the circuit `Top ASNs` table. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitTopAsnRow { + pub asn_name: String, + pub asn_country: String, + pub down_bps: u64, + pub up_bps: u64, + pub bytes_sent_down: u64, + pub bytes_sent_up: u64, + pub packets_sent_down: u64, + pub packets_sent_up: u64, + pub retransmit_down_pct: f64, + pub retransmit_up_pct: f64, + pub flow_count: usize, +} + +/// Server-side payload for the circuit `Top ASNs` table. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitTopAsnsData { + pub total_asns: usize, + pub rows: Vec, +} + +/// Compact flow row for the circuit `Flow Sankey` tab. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitFlowSankeyRow { + pub device_name: String, + pub asn_id: u32, + pub asn_name: String, + pub protocol_name: String, + pub remote_ip: String, + pub down_bps: u32, + pub up_bps: u32, + pub last_seen_nanos: u64, +} + +#[derive(Clone, Debug)] +struct CircuitFlowSnapshotRow { + device_name: String, + asn_id: u32, + asn_name: String, + asn_country: String, + protocol_name: String, + remote_ip: String, + down_bps: u32, + up_bps: u32, + bytes_sent_down: u64, + bytes_sent_up: u64, + packets_sent_down: u64, + packets_sent_up: u64, + tcp_retransmits_down: u16, + tcp_retransmits_up: u16, + retransmit_down_pct: f64, + retransmit_up_pct: f64, + rtt_down_nanos: u64, + rtt_up_nanos: u64, + qoo_down: Option, + qoo_up: Option, + last_seen_nanos: u64, + opacity: f64, + sort_rate_bps: f64, +} + +const fn clamp_u64_to_u32(value: u64) -> u32 { + if value > u32::MAX as u64 { + u32::MAX + } else { + value as u32 + } +} + +fn sanitized_plan_ceiling_bps(plan_mbps: f32) -> u32 { + if !plan_mbps.is_finite() || plan_mbps <= 0.0 { + return clamp_u64_to_u32(FLOW_RATE_SANITY_FLOOR_BPS); + } + + let scaled = (plan_mbps as f64 * 1_000_000.0 * FLOW_RATE_SANITY_MULTIPLIER).round(); + let scaled = scaled.max(FLOW_RATE_SANITY_FLOOR_BPS as f64) as u64; + clamp_u64_to_u32(scaled) +} + +fn circuit_display_rate_ceiling_bps( + shaped: &lqos_config::ConfigShapedDevices, + circuit_hash: i64, +) -> Option> { + let mut max_down_mbps = 0.0_f32; + let mut max_up_mbps = 0.0_f32; + + for device in &shaped.devices { + if device.circuit_hash != circuit_hash { + continue; + } + max_down_mbps = max_down_mbps.max(device.download_max_mbps); + max_up_mbps = max_up_mbps.max(device.upload_max_mbps); + } + + if max_down_mbps <= 0.0 && max_up_mbps <= 0.0 { + return None; + } + + Some(DownUpOrder { + down: sanitized_plan_ceiling_bps(max_down_mbps), + up: sanitized_plan_ceiling_bps(max_up_mbps), + }) +} + +fn display_rate_bps( + local: &FlowbeeLocalData, + ceiling_bps: Option>, +) -> DownUpOrder { + let Some(ceiling_bps) = ceiling_bps else { + return local.rate_estimate_bps; + }; + + DownUpOrder { + down: local.rate_estimate_bps.down.min(ceiling_bps.down), + up: local.rate_estimate_bps.up.min(ceiling_bps.up), + } +} + +fn flow_rtt_nanos(local: &FlowbeeLocalData) -> DownUpOrder { + let rtt = local.get_rtt_array(); + DownUpOrder { + down: rtt[0].as_nanos(), + up: rtt[1].as_nanos(), + } +} + +fn flow_qoo(local: &FlowbeeLocalData) -> DownUpOrder> { + let qoq = local.get_qoq_scores(); + DownUpOrder { + down: qoq.download_total_f32(), + up: qoq.upload_total_f32(), + } +} + +fn flow_snapshot_rows(circuit_id: &str) -> Vec { + let circuit_hash = hash_to_i64(circuit_id); + let shaped = SHAPED_DEVICES.load(); + let cache = SHAPED_DEVICE_HASH_CACHE.load(); + let display_rate_ceiling = circuit_display_rate_ceiling_bps(&shaped, circuit_hash); + let Ok(now) = time_since_boot() else { + return Vec::new(); + }; + let now_as_nanos = Duration::from(now).as_nanos() as u64; + let recent_cutoff = now_as_nanos.saturating_sub(RECENT_CIRCUIT_FLOWS_WINDOW_NANOS); + + let all_flows = ALL_FLOWS.lock(); + all_flows + .flow_data + .iter() + .filter_map(|(key, (local, analysis))| { + if local.last_seen < recent_cutoff { + return None; + } + if local.circuit_hash != Some(circuit_hash) { + return None; + } + + let device_name = local + .device_hash + .and_then(|hash| cache.index_by_device_hash(&shaped, hash)) + .and_then(|idx| shaped.devices.get(idx)) + .map(|d| d.device_name.clone()) + .unwrap_or_else(|| "Unknown".to_string()); + + let geo = get_asn_name_and_country(key.remote_ip.as_ip()); + let display_rate = display_rate_bps(local, display_rate_ceiling); + let current_rate = display_rate.down as u64 + display_rate.up as u64; + let packets_sent_down = local.packets_sent.down; + let packets_sent_up = local.packets_sent.up; + let tcp_retransmits_down = local.tcp_retransmits.down; + let tcp_retransmits_up = local.tcp_retransmits.up; + let retransmit_down_pct = if tcp_retransmits_down > 0 && packets_sent_down > 0 { + tcp_retransmits_down as f64 / packets_sent_down as f64 + } else { + 0.0 + }; + let retransmit_up_pct = if tcp_retransmits_up > 0 && packets_sent_up > 0 { + tcp_retransmits_up as f64 / packets_sent_up as f64 + } else { + 0.0 + }; + let rtt = flow_rtt_nanos(local); + let qoo = flow_qoo(local); + let last_seen_nanos = now_as_nanos.saturating_sub(local.last_seen); + + Some(CircuitFlowSnapshotRow { + device_name, + asn_id: analysis.asn_id.0, + asn_name: geo.name, + asn_country: geo.country, + protocol_name: analysis.protocol_analysis.to_string(), + remote_ip: key.remote_ip.to_string(), + down_bps: display_rate.down, + up_bps: display_rate.up, + bytes_sent_down: local.bytes_sent.down, + bytes_sent_up: local.bytes_sent.up, + packets_sent_down, + packets_sent_up, + tcp_retransmits_down, + tcp_retransmits_up, + retransmit_down_pct, + retransmit_up_pct, + rtt_down_nanos: rtt.down, + rtt_up_nanos: rtt.up, + qoo_down: qoo.down, + qoo_up: qoo.up, + last_seen_nanos, + opacity: 1.0 - f64::min(1.0, last_seen_nanos as f64 / RECENT_CIRCUIT_FLOWS_WINDOW_NANOS as f64), + sort_rate_bps: current_rate as f64, + }) + }) + .collect() +} + +fn sort_direction_is_asc(sort_direction: &str) -> bool { + sort_direction.eq_ignore_ascii_case("asc") +} + +fn compare_f64(left: f64, right: f64, asc: bool) -> Ordering { + let order = left.partial_cmp(&right).unwrap_or(Ordering::Equal); + if asc { order } else { order.reverse() } +} + +fn compare_u64(left: u64, right: u64, asc: bool) -> Ordering { + if asc { left.cmp(&right) } else { right.cmp(&left) } +} + +fn compare_strings(left: &str, right: &str, asc: bool) -> Ordering { + if asc { + left.cmp(right) + } else { + right.cmp(left) + } +} + +fn sort_traffic_rows(rows: &mut [CircuitFlowSnapshotRow], sort_column: &str, sort_direction: &str) { + let asc = sort_direction_is_asc(sort_direction); + rows.sort_by(|a, b| { + let primary = match sort_column { + "protocol" => compare_strings(&a.protocol_name, &b.protocol_name, asc), + "bytes" => compare_u64(a.bytes_sent_down + a.bytes_sent_up, b.bytes_sent_down + b.bytes_sent_up, asc), + "packets" => compare_u64(a.packets_sent_down + a.packets_sent_up, b.packets_sent_down + b.packets_sent_up, asc), + "retransmits" => compare_f64(a.retransmit_down_pct + a.retransmit_up_pct, b.retransmit_down_pct + b.retransmit_up_pct, asc), + "rtt" => compare_u64(a.rtt_down_nanos + a.rtt_up_nanos, b.rtt_down_nanos + b.rtt_up_nanos, asc), + "qoo" => compare_f64( + a.qoo_down.unwrap_or(0.0) as f64 + a.qoo_up.unwrap_or(0.0) as f64, + b.qoo_down.unwrap_or(0.0) as f64 + b.qoo_up.unwrap_or(0.0) as f64, + asc, + ), + "asn" => compare_strings(&a.asn_name, &b.asn_name, asc), + "country" => compare_strings(&a.asn_country, &b.asn_country, asc), + "ip" => compare_strings(&a.remote_ip, &b.remote_ip, asc), + _ => compare_f64(a.sort_rate_bps, b.sort_rate_bps, asc), + }; + if primary == Ordering::Equal { + compare_f64(a.sort_rate_bps, b.sort_rate_bps, false) + } else { + primary + } + }); +} + +pub fn circuit_flow_counts(circuit_id: &str) -> (usize, usize) { + let rows = flow_snapshot_rows(circuit_id); + let asn_count = rows + .iter() + .map(|row| (row.asn_name.clone(), row.asn_country.clone())) + .collect::>() + .len(); + (rows.len(), asn_count) +} + +pub fn circuit_traffic_flows_page(query: &CircuitTrafficFlowsQuery) -> CircuitTrafficFlowsPage { + let mut rows = flow_snapshot_rows(&query.circuit); + if query.hide_small { + rows.retain(|row| row.down_bps > TRAFFIC_FLOW_HIDE_THRESHOLD_BPS || row.up_bps > TRAFFIC_FLOW_HIDE_THRESHOLD_BPS); + } + sort_traffic_rows(&mut rows, &query.sort_column, &query.sort_direction); + + let total_rows = rows.len(); + let page_size = query.page_size.max(1); + let page = query.page.max(1); + let start = page_size.saturating_mul(page.saturating_sub(1)); + let paged = rows + .into_iter() + .skip(start) + .take(page_size) + .map(|row| CircuitTrafficFlowRow { + protocol_name: row.protocol_name, + down_bps: row.down_bps, + up_bps: row.up_bps, + bytes_sent_down: row.bytes_sent_down, + bytes_sent_up: row.bytes_sent_up, + packets_sent_down: row.packets_sent_down, + packets_sent_up: row.packets_sent_up, + tcp_retransmits_down: row.tcp_retransmits_down, + tcp_retransmits_up: row.tcp_retransmits_up, + retransmit_down_pct: row.retransmit_down_pct, + retransmit_up_pct: row.retransmit_up_pct, + rtt_down_nanos: row.rtt_down_nanos, + rtt_up_nanos: row.rtt_up_nanos, + qoo_down: row.qoo_down, + qoo_up: row.qoo_up, + asn_name: row.asn_name, + asn_country: row.asn_country, + remote_ip: row.remote_ip, + opacity: row.opacity, + sort_rate_bps: row.sort_rate_bps, + }) + .collect(); + + CircuitTrafficFlowsPage { + query: query.clone(), + total_rows, + rows: paged, + } +} + +pub fn circuit_top_asns_data(query: &CircuitTopAsnsQuery) -> CircuitTopAsnsData { + #[derive(Default)] + struct AsnBucket { + asn_name: String, + asn_country: String, + down_bps: u64, + up_bps: u64, + bytes_sent_down: u64, + bytes_sent_up: u64, + packets_sent_down: u64, + packets_sent_up: u64, + tcp_retransmits_down: u64, + tcp_retransmits_up: u64, + flow_count: usize, + } + + let mut rows = flow_snapshot_rows(&query.circuit); + if query.hide_small { + rows.retain(|row| row.down_bps > TRAFFIC_FLOW_HIDE_THRESHOLD_BPS || row.up_bps > TRAFFIC_FLOW_HIDE_THRESHOLD_BPS); + } + + let mut buckets: fxhash::FxHashMap<(String, String), AsnBucket> = fxhash::FxHashMap::default(); + for row in rows { + let key = ( + if row.asn_name.trim().is_empty() { + "Unknown ASN".to_string() + } else { + row.asn_name.clone() + }, + row.asn_country.clone(), + ); + let bucket = buckets.entry(key.clone()).or_insert_with(|| AsnBucket { + asn_name: key.0.clone(), + asn_country: key.1.clone(), + down_bps: 0, + up_bps: 0, + bytes_sent_down: 0, + bytes_sent_up: 0, + packets_sent_down: 0, + packets_sent_up: 0, + tcp_retransmits_down: 0, + tcp_retransmits_up: 0, + flow_count: 0, + }); + bucket.down_bps += row.down_bps as u64; + bucket.up_bps += row.up_bps as u64; + bucket.bytes_sent_down += row.bytes_sent_down; + bucket.bytes_sent_up += row.bytes_sent_up; + bucket.packets_sent_down += row.packets_sent_down; + bucket.packets_sent_up += row.packets_sent_up; + bucket.tcp_retransmits_down += row.tcp_retransmits_down as u64; + bucket.tcp_retransmits_up += row.tcp_retransmits_up as u64; + bucket.flow_count += 1; + } + + let total_asns = buckets.len(); + let mut bucket_rows: Vec = buckets + .into_values() + .map(|row| CircuitTopAsnRow { + asn_name: row.asn_name, + asn_country: row.asn_country, + down_bps: row.down_bps, + up_bps: row.up_bps, + bytes_sent_down: row.bytes_sent_down, + bytes_sent_up: row.bytes_sent_up, + packets_sent_down: row.packets_sent_down, + packets_sent_up: row.packets_sent_up, + retransmit_down_pct: if row.packets_sent_down > 0 { + row.tcp_retransmits_down as f64 / row.packets_sent_down as f64 + } else { + 0.0 + }, + retransmit_up_pct: if row.packets_sent_up > 0 { + row.tcp_retransmits_up as f64 / row.packets_sent_up as f64 + } else { + 0.0 + }, + flow_count: row.flow_count, + }) + .collect(); + + bucket_rows.sort_by(|a, b| { + let a_rate = a.down_bps + a.up_bps; + let b_rate = b.down_bps + b.up_bps; + b_rate + .cmp(&a_rate) + .then_with(|| b.flow_count.cmp(&a.flow_count)) + .then_with(|| a.asn_name.cmp(&b.asn_name)) + }); + bucket_rows.truncate(TOP_ASN_LIMIT); + + CircuitTopAsnsData { + total_asns, + rows: bucket_rows, + } +} + +pub fn circuit_flow_sankey_rows(circuit_id: &str) -> Vec { + let mut rows = flow_snapshot_rows(circuit_id); + rows.retain(|row| row.last_seen_nanos <= SANKEY_RECENT_FLOW_WINDOW_NANOS); + rows.sort_by(|a, b| compare_f64(a.sort_rate_bps, b.sort_rate_bps, false)); + rows.into_iter() + .take(SANKEY_TOP_FLOW_LIMIT) + .map(|row| CircuitFlowSankeyRow { + device_name: row.device_name, + asn_id: row.asn_id, + asn_name: row.asn_name, + protocol_name: row.protocol_name, + remote_ip: row.remote_ip, + down_bps: row.down_bps, + up_bps: row.up_bps, + last_seen_nanos: row.last_seen_nanos, + }) + .collect() +} diff --git a/src/rust/lqosd/src/node_manager/local_api/circuit_live.rs b/src/rust/lqosd/src/node_manager/local_api/circuit_live.rs new file mode 100644 index 00000000..010a25cd --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/circuit_live.rs @@ -0,0 +1,51 @@ +use crate::shaped_devices_tracker::circuit_live::fresh_circuit_live_snapshot; +use lqos_utils::units::{DownUpOrder, TcpRetransmitSample}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; + +const MAX_CIRCUIT_METRICS_IDS: usize = 250; + +/// Query for the compact live metrics needed by the shaped-devices page. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitMetricsQuery { + pub circuit_ids: Vec, +} + +/// Live metrics for a single visible circuit card. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitLiveMetrics { + pub circuit_id: String, + pub bytes_per_second: DownUpOrder, + pub rtt_current_p50_nanos: DownUpOrder>, + pub qoo: DownUpOrder>, + pub tcp_retransmit_sample: DownUpOrder, + pub last_seen_nanos: u64, +} + +/// Resolves the current live metrics for a bounded set of circuit ids. +pub fn circuit_live_metrics(query: &CircuitMetricsQuery) -> Vec { + let snapshot = fresh_circuit_live_snapshot(); + let mut seen = HashSet::new(); + query + .circuit_ids + .iter() + .filter_map(|id| { + let trimmed = id.trim(); + if trimmed.is_empty() || !seen.insert(trimmed.to_string()) { + return None; + } + snapshot + .by_circuit_id + .get(trimmed) + .map(|row| CircuitLiveMetrics { + circuit_id: row.circuit_id.clone(), + bytes_per_second: row.bytes_per_second, + rtt_current_p50_nanos: row.rtt_current_p50_nanos, + qoo: row.qoo, + tcp_retransmit_sample: row.tcp_retransmit_sample, + last_seen_nanos: row.last_seen_nanos, + }) + }) + .take(MAX_CIRCUIT_METRICS_IDS) + .collect() +} diff --git a/src/rust/lqosd/src/node_manager/local_api/config.rs b/src/rust/lqosd/src/node_manager/local_api/config.rs index 87634d9b..8838b79c 100644 --- a/src/rust/lqosd/src/node_manager/local_api/config.rs +++ b/src/rust/lqosd/src/node_manager/local_api/config.rs @@ -4,11 +4,41 @@ use axum::http::StatusCode; use default_net::get_interfaces; use lqos_bus::{BusRequest, bus_request}; use lqos_config::{Config, ConfigShapedDevices, ShapedDevice, UserRole, WebUser, WebUsers}; +use lqos_utils::hash_to_i64; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::sync::Arc; +type TopologySourceIntegration = (&'static str, fn(&Config) -> bool); + +const TOPOLOGY_SOURCE_INTEGRATIONS: [TopologySourceIntegration; 7] = [ + ("UISP", |config| config.uisp_integration.enable_uisp), + ("Splynx", |config| config.splynx_integration.enable_splynx), + ("Powercode", |config| { + config.powercode_integration.enable_powercode + }), + ("Sonar", |config| config.sonar_integration.enable_sonar), + ("Netzur", |config| { + config + .netzur_integration + .as_ref() + .is_some_and(|integration| integration.enable_netzur) + }), + ("VISP", |config| { + config + .visp_integration + .as_ref() + .is_some_and(|integration| integration.enable_visp) + }), + ("WispGate", |config| { + config + .wispgate_integration + .as_ref() + .is_some_and(|integration| integration.enable_wispgate) + }), +]; + pub fn admin_check_data(login: LoginResult) -> bool { matches!(login, LoginResult::Admin) } @@ -74,6 +104,252 @@ pub fn all_shaped_devices_data() -> Vec { SHAPED_DEVICES.load().devices.clone() } +/// Returns the enabled integration names that act as the source of truth for +/// `network.json` and `ShapedDevices.csv`. +pub fn active_topology_source_integrations(config: &Config) -> Vec<&'static str> { + TOPOLOGY_SOURCE_INTEGRATIONS + .iter() + .filter_map(|(name, enabled)| enabled(config).then_some(*name)) + .collect() +} + +fn topology_editor_lock_message() -> Result, String> { + let config = + lqos_config::load_config().map_err(|e| format!("Unable to load LibreQoS config: {e}"))?; + let integrations = active_topology_source_integrations(config.as_ref()); + if integrations.is_empty() { + return Ok(None); + } + Ok(Some(format!( + "Editing is disabled because these integrations are the source of truth: {}.", + integrations.join(", ") + ))) +} + +fn ensure_topology_editor_unlocked() -> Result<(), String> { + if let Some(message) = topology_editor_lock_message()? { + return Err(message); + } + Ok(()) +} + +fn validate_network_json(value: &Value) -> Result<(), String> { + let Some(map) = value.as_object() else { + return Err("network.json must be a JSON object".to_string()); + }; + + let mut seen = HashSet::new(); + let mut duplicates = BTreeSet::new(); + + fn walk( + map: &serde_json::Map, + seen: &mut HashSet, + duplicates: &mut BTreeSet, + ) { + for (name, node) in map { + if !seen.insert(name.clone()) { + duplicates.insert(name.clone()); + } + let Some(node_map) = node.as_object() else { + continue; + }; + let Some(children) = node_map.get("children").and_then(Value::as_object) else { + continue; + }; + walk(children, seen, duplicates); + } + } + + walk(map, &mut seen, &mut duplicates); + + if duplicates.is_empty() { + Ok(()) + } else { + Err(format!( + "network.json contains duplicate node names: {}. Node names must be globally unique.", + duplicates.into_iter().collect::>().join(", ") + )) + } +} + +fn persist_network_json(network_json: &Value) -> Result<(), String> { + validate_network_json(network_json)?; + let config = + lqos_config::load_config().map_err(|e| format!("Unable to load LibreQoS config: {e}"))?; + let serialized_string = serde_json::to_string_pretty(network_json) + .map_err(|e| format!("Unable to serialize network.json payload: {e}"))?; + let net_json_path = std::path::Path::new(&config.lqos_directory).join("network.json"); + let net_json_backup_path = + std::path::Path::new(&config.lqos_directory).join("network.json.backup"); + if net_json_path.exists() { + std::fs::copy(&net_json_path, net_json_backup_path) + .map_err(|e| format!("Unable to create network.json backup: {e}"))?; + } + std::fs::write(net_json_path, serialized_string) + .map_err(|e| format!("Unable to write network.json: {e}"))?; + Ok(()) +} + +fn normalize_sqm_override(raw: &Option) -> Option { + let token = raw.as_deref().unwrap_or("").trim().to_lowercase(); + if token.is_empty() { None } else { Some(token) } +} + +fn normalize_shaped_device(device: &mut ShapedDevice) { + device.circuit_id = device.circuit_id.trim().to_string(); + device.circuit_name = device.circuit_name.trim().to_string(); + device.device_id = device.device_id.trim().to_string(); + device.device_name = device.device_name.trim().to_string(); + device.parent_node = device.parent_node.trim().to_string(); + device.mac = device.mac.trim().to_string(); + device.comment = device.comment.trim().to_string(); + device.sqm_override = normalize_sqm_override(&device.sqm_override); +} + +fn populate_shaped_device_hashes(devices: &mut [ShapedDevice]) { + for device in devices { + device.circuit_hash = hash_to_i64(&device.circuit_id); + device.device_hash = hash_to_i64(&device.device_id); + device.parent_hash = hash_to_i64(&device.parent_node); + } +} + +fn collect_network_nodes(value: &Value, out: &mut HashSet) { + let Some(map) = value.as_object() else { + return; + }; + for (name, node) in map { + out.insert(name.to_string()); + if let Some(children) = node.get("children") { + collect_network_nodes(children, out); + } + } +} + +fn validate_sqm_override(raw: &Option) -> Result<(), String> { + let Some(token) = normalize_sqm_override(raw) else { + return Ok(()); + }; + let valid = |value: &str| matches!(value, "" | "cake" | "fq_codel" | "none"); + if token.contains('/') { + let mut parts = token.splitn(2, '/'); + let down = parts.next().unwrap_or("").trim(); + let up = parts.next().unwrap_or("").trim(); + if valid(down) && valid(up) { + return Ok(()); + } + } else if valid(&token) { + return Ok(()); + } + Err(format!( + "Invalid SQM override '{token}'. Allowed values: cake, fq_codel, none, or directional down/up tokens." + )) +} + +fn validate_shaped_devices(devices: &[ShapedDevice]) -> Result<(), String> { + let network_json = network_json_data(); + let mut nodes = HashSet::new(); + collect_network_nodes(&network_json, &mut nodes); + + let mut device_ids = HashSet::new(); + let mut ipv4s = HashSet::new(); + let mut ipv6s = HashSet::new(); + + for (index, device) in devices.iter().enumerate() { + let label = if device.device_id.is_empty() { + format!("row {}", index + 1) + } else { + format!("device '{}'", device.device_id) + }; + + if device.circuit_id.is_empty() { + return Err(format!("{label}: Circuit ID is required")); + } + if device.circuit_name.is_empty() { + return Err(format!("{label}: Circuit Name is required")); + } + if device.device_id.is_empty() { + return Err(format!("{label}: Device ID is required")); + } + if device.device_name.is_empty() { + return Err(format!("{label}: Device Name is required")); + } + if !device_ids.insert(device.device_id.clone()) { + return Err(format!("Duplicate device ID '{}'", device.device_id)); + } + if nodes.is_empty() { + if !device.parent_node.is_empty() { + return Err(format!( + "{label}: Parent node '{}' is invalid for a flat network", + device.parent_node + )); + } + } else if !device.parent_node.is_empty() && !nodes.contains(&device.parent_node) { + return Err(format!( + "{label}: Parent node '{}' does not exist", + device.parent_node + )); + } + if device.ipv4.is_empty() && device.ipv6.is_empty() { + return Err(format!( + "{label}: At least one IPv4 or IPv6 address is required" + )); + } + for (addr, prefix) in &device.ipv4 { + let key = format!("{addr}/{prefix}"); + if !ipv4s.insert(key.clone()) { + return Err(format!("Duplicate IPv4 entry '{key}'")); + } + } + for (addr, prefix) in &device.ipv6 { + let key = format!("{addr}/{prefix}"); + if !ipv6s.insert(key.clone()) { + return Err(format!("Duplicate IPv6 entry '{key}'")); + } + } + if !device.download_min_mbps.is_finite() || device.download_min_mbps < 0.1 { + return Err(format!("{label}: Download min must be >= 0.1 Mbps")); + } + if !device.upload_min_mbps.is_finite() || device.upload_min_mbps < 0.1 { + return Err(format!("{label}: Upload min must be >= 0.1 Mbps")); + } + if !device.download_max_mbps.is_finite() || device.download_max_mbps < 0.2 { + return Err(format!("{label}: Download max must be >= 0.2 Mbps")); + } + if !device.upload_max_mbps.is_finite() || device.upload_max_mbps < 0.2 { + return Err(format!("{label}: Upload max must be >= 0.2 Mbps")); + } + validate_sqm_override(&device.sqm_override)?; + } + + Ok(()) +} + +fn persist_shaped_devices(mut devices: Vec) -> Result<(), String> { + for device in &mut devices { + normalize_shaped_device(device); + } + validate_shaped_devices(&devices)?; + populate_shaped_device_hashes(&mut devices); + + let config = lqos_config::load_config().expect("Unable to load LibreQoS config"); + let sd_path = std::path::Path::new(&config.lqos_directory).join("ShapedDevices.csv"); + let sd_backup_path = + std::path::Path::new(&config.lqos_directory).join("ShapedDevices.csv.backup"); + if sd_path.exists() { + std::fs::copy(&sd_path, sd_backup_path) + .map_err(|e| format!("Unable to create ShapedDevices.csv backup: {e}"))?; + } + let mut copied = ConfigShapedDevices::default(); + copied.replace_with_new_data(devices); + copied + .write_csv("ShapedDevices.csv") + .map_err(|e| format!("Unable to write ShapedDevices.csv: {e}"))?; + SHAPED_DEVICES.store(Arc::new(copied)); + + Ok(()) +} + pub async fn update_lqosd_config_data( login: LoginResult, config: Config, @@ -87,44 +363,129 @@ pub async fn update_lqosd_config_data( Ok(()) } +/// Persists both `network.json` and `ShapedDevices.csv` for administrative +/// edits. +/// +/// Returns an error string when the caller is unauthorized, when integration- +/// managed topology editing is locked, or when validation/persistence fails. pub fn update_network_and_devices_data( login: LoginResult, network_json: Value, shaped_devices: Vec, -) -> Result<(), StatusCode> { +) -> Result<(), String> { + if login != LoginResult::Admin { + return Err("Unauthorized".to_string()); + } + ensure_topology_editor_unlocked()?; + persist_network_json(&network_json)?; + persist_shaped_devices(shaped_devices)?; + + Ok(()) +} + +/// Persists `network.json` for administrative edits without modifying shaped +/// devices. +/// +/// Returns an error string when the caller is unauthorized, when integration- +/// managed topology editing is locked, or when persistence fails. +pub fn update_network_json_only_data( + login: LoginResult, + network_json: Value, +) -> Result<(), String> { + if login != LoginResult::Admin { + return Err("Unauthorized".to_string()); + } + ensure_topology_editor_unlocked()?; + + persist_network_json(&network_json)?; + + Ok(()) +} + +/// Returns one shaped device row by device identifier for administrative +/// callers. +pub fn get_shaped_device_data( + login: LoginResult, + device_id: String, +) -> Result, StatusCode> { if login != LoginResult::Admin { return Err(StatusCode::FORBIDDEN); } + let wanted = device_id.trim(); + Ok(SHAPED_DEVICES + .load() + .devices + .iter() + .find(|device| device.device_id == wanted) + .cloned()) +} - let config = lqos_config::load_config().expect("Unable to load LibreQoS config"); - - // Save network.json - let serialized_string = serde_json::to_string_pretty(&network_json) - .expect("Unable to serialize network.json payload"); - let net_json_path = std::path::Path::new(&config.lqos_directory).join("network.json"); - let net_json_backup_path = - std::path::Path::new(&config.lqos_directory).join("network.json.backup"); - if net_json_path.exists() { - // Make a backup - std::fs::copy(&net_json_path, net_json_backup_path) - .expect("Unable to create network.json backup"); +/// Creates one shaped device row for administrative callers. +/// +/// Returns an error string when the caller is unauthorized, when integration- +/// managed topology editing is locked, or when validation/persistence fails. +pub fn create_shaped_device_data( + login: LoginResult, + device: ShapedDevice, +) -> Result { + if login != LoginResult::Admin { + return Err("Unauthorized".to_string()); } - std::fs::write(net_json_path, serialized_string).expect("Unable to write network.json"); + ensure_topology_editor_unlocked()?; + let mut devices = SHAPED_DEVICES.load().devices.clone(); + devices.push(device.clone()); + persist_shaped_devices(devices)?; + let created = get_shaped_device_data(login, device.device_id.clone()) + .map_err(|_| "Unable to reload shaped device".to_string())? + .ok_or_else(|| "Unable to reload shaped device".to_string())?; + Ok(created) +} - // Save the Shaped Devices - let sd_path = std::path::Path::new(&config.lqos_directory).join("ShapedDevices.csv"); - let sd_backup_path = - std::path::Path::new(&config.lqos_directory).join("ShapedDevices.csv.backup"); - if sd_path.exists() { - std::fs::copy(&sd_path, sd_backup_path).expect("Unable to create ShapedDevices.csv backup"); +/// Updates one shaped device row for administrative callers. +/// +/// Returns an error string when the caller is unauthorized, when integration- +/// managed topology editing is locked, when the row is not found, or when +/// validation/persistence fails. +pub fn update_shaped_device_data( + login: LoginResult, + original_device_id: String, + device: ShapedDevice, +) -> Result { + if login != LoginResult::Admin { + return Err("Unauthorized".to_string()); } - let mut copied = ConfigShapedDevices::default(); - copied.replace_with_new_data(shaped_devices); - copied - .write_csv(&format!("{}/ShapedDevices.csv", config.lqos_directory)) - .expect("Unable to write ShapedDevices.csv"); - SHAPED_DEVICES.store(Arc::new(copied)); + ensure_topology_editor_unlocked()?; + let mut devices = SHAPED_DEVICES.load().devices.clone(); + let wanted = original_device_id.trim(); + let Some(index) = devices.iter().position(|row| row.device_id == wanted) else { + return Err("Not found".to_string()); + }; + devices[index] = device.clone(); + persist_shaped_devices(devices)?; + let updated = get_shaped_device_data(login, device.device_id.clone()) + .map_err(|_| "Unable to reload shaped device".to_string())? + .ok_or_else(|| "Unable to reload shaped device".to_string())?; + Ok(updated) +} +/// Deletes one shaped device row for administrative callers. +/// +/// Returns an error string when the caller is unauthorized, when integration- +/// managed topology editing is locked, when the row is not found, or when +/// persistence fails. +pub fn delete_shaped_device_data(login: LoginResult, device_id: String) -> Result<(), String> { + if login != LoginResult::Admin { + return Err("Unauthorized".to_string()); + } + ensure_topology_editor_unlocked()?; + let wanted = device_id.trim(); + let mut devices = SHAPED_DEVICES.load().devices.clone(); + let before = devices.len(); + devices.retain(|device| device.device_id != wanted); + if devices.len() == before { + return Err("Not found".to_string()); + } + persist_shaped_devices(devices)?; Ok(()) } @@ -214,3 +575,60 @@ pub struct UserRequest { pub password: Option, pub role: String, } + +#[cfg(test)] +mod tests { + use super::validate_network_json; + use serde_json::json; + + #[test] + fn accepts_unique_node_names() { + let network = json!({ + "Site A": { + "downloadBandwidthMbps": 1000, + "uploadBandwidthMbps": 1000, + "children": { + "Site B": { + "downloadBandwidthMbps": 500, + "uploadBandwidthMbps": 500 + } + } + }, + "Site C": { + "downloadBandwidthMbps": 750, + "uploadBandwidthMbps": 750 + } + }); + + assert!(validate_network_json(&network).is_ok()); + } + + #[test] + fn rejects_duplicate_node_names_anywhere_in_tree() { + let network = json!({ + "Site A": { + "downloadBandwidthMbps": 1000, + "uploadBandwidthMbps": 1000, + "children": { + "Duplicate": { + "downloadBandwidthMbps": 500, + "uploadBandwidthMbps": 500 + } + } + }, + "Site B": { + "downloadBandwidthMbps": 750, + "uploadBandwidthMbps": 750, + "children": { + "Duplicate": { + "downloadBandwidthMbps": 300, + "uploadBandwidthMbps": 300 + } + } + } + }); + + let err = validate_network_json(&network).expect_err("duplicate names must fail"); + assert!(err.contains("Duplicate")); + } +} diff --git a/src/rust/lqosd/src/node_manager/local_api/cpu_affinity.rs b/src/rust/lqosd/src/node_manager/local_api/cpu_affinity.rs index 9e8b0804..747cad17 100644 --- a/src/rust/lqosd/src/node_manager/local_api/cpu_affinity.rs +++ b/src/rust/lqosd/src/node_manager/local_api/cpu_affinity.rs @@ -1,7 +1,14 @@ +use crate::node_manager::local_api::network_tree_lite::{ + NetworkTreeLiteNode, network_tree_lite_data, +}; +use crate::shaped_devices_tracker::SHAPED_DEVICES; +use crate::system_stats::{CPU_USAGE, NUM_CPUS}; +use lqos_config::{ShapingCpuDetection, detect_shaping_cpus, load_config}; use serde::Serialize; use serde_json::Value; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; #[derive(Serialize, Clone, Debug)] pub struct CpuSideSummary { @@ -47,6 +54,76 @@ pub struct CpuAffinitySiteTreeNode { pub children: Vec, } +/// Runtime-aware CPU affinity snapshot for the CPU assignment page. +#[derive(Serialize, Clone, Debug)] +pub struct CpuAffinityRuntimeSnapshot { + /// Snapshot generation time in milliseconds since the UNIX epoch. + pub generated_at_unix_ms: u64, + /// CPU IDs currently intended for shaping / binning. + pub shaping_cpus: Vec, + /// CPU IDs present on the host but excluded from shaping. + pub excluded_cpus: Vec, + /// True when hybrid-core detection found a trustworthy split. + pub has_hybrid_split: bool, + /// Per-core runtime assignment data. + pub cores: Vec, +} + +/// Runtime-aware data for a single CPU core. +#[derive(Serialize, Clone, Debug)] +pub struct CpuAffinityRuntimeCore { + /// Zero-based CPU index. + pub cpu: u32, + /// Live CPU usage for this core, if sampled. + pub live_usage_pct: Option, + /// Number of planned download-side circuits on this CPU. + pub planned_circuit_count: u32, + /// Sum of planner weights for download-side circuits on this CPU. + pub planned_weight_sum: f64, + /// Sum of planned download-side max rates for this CPU. + pub planned_max_mbps: f64, + /// Number of nodes effectively associated with this CPU. + pub effective_node_count: u32, + /// Number of effective site nodes associated with this CPU. + pub effective_site_count: u32, + /// Number of effective AP nodes associated with this CPU. + pub effective_ap_count: u32, + /// Number of effective circuits associated with this CPU. + pub effective_circuit_count: u32, + /// Number of runtime-changed nodes associated with this CPU. + pub runtime_changed_count: u32, + /// Detailed node rows associated with this CPU. + pub nodes: Vec, +} + +/// Runtime-aware node assignment details for a single site/AP branch. +#[derive(Serialize, Clone, Debug)] +pub struct CpuAffinityRuntimeNode { + /// Node name from the live tree. + pub name: String, + /// Optional node type metadata. + pub node_type: Option, + /// Planned CPU from queue generation, if known. + pub planned_cpu: Option, + /// Effective CPU after runtime virtualization, if known. + pub effective_cpu: Option, + /// True if TreeGuard has runtime-virtualized this node. + pub runtime_virtualized: bool, + /// Assignment reason used by the UI to explain runtime changes. + pub assignment_reason: String, + /// True when this node is a top-level owned branch directly attached to the + /// selected CPU's HTB subtree, rather than a descendant of another owned branch. + pub is_cpu_root: bool, + /// Number of descendant nodes beneath this node. + pub subtree_node_count: u32, + /// Number of descendant circuits beneath this node. + pub subtree_circuit_count: u32, + /// Effective max throughput in Mbps for down/up from the live tree. + pub effective_max_mbps: (f64, f64), + /// Current throughput in bytes per second, down/up. + pub current_throughput_bps: (u64, u64), +} + #[derive(Debug, Default, Clone)] struct CircuitRecord { cpu_down: Option, @@ -65,6 +142,21 @@ struct CircuitRecord { ip_count: usize, } +#[derive(Debug, Default, Clone, Copy)] +struct PlannedCoreMetrics { + circuit_count: u32, + weight_sum: f64, + max_mbps: f64, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +struct RuntimeNodePlacement { + owner_index: Option, + planned_cpu: Option, + effective_cpu: Option, + assignment_reason: &'static str, +} + // Query params for the circuits endpoint #[derive(Debug, Default, Clone, serde::Deserialize)] pub struct CircuitsQuery { @@ -270,6 +362,407 @@ fn trailing_u32(s: &str) -> Option { } } +fn node_cpu_from_queue_map( + node: &serde_json::Map, + inherited_cpu: Option, +) -> Option { + node.get("cpuNum") + .and_then(|v| v.as_str()) + .and_then(parse_hex_u32) + .or_else(|| { + node.get("classMajor") + .and_then(|v| v.as_str()) + .and_then(parse_hex_u32) + .map(|v| v.saturating_sub(1)) + }) + .or(inherited_cpu) +} + +fn collect_planned_node_cpus( + name: &str, + node: &serde_json::Map, + inherited_cpu: Option, + out: &mut HashMap, +) { + if !is_network_node(node) { + return; + } + + let cpu = node_cpu_from_queue_map(node, inherited_cpu); + if let Some(cpu) = cpu { + out.insert(name.to_string(), cpu); + } + + if let Some(Value::Object(children)) = node.get("children") { + for (child_name, child_value) in children { + if let Value::Object(child_map) = child_value { + collect_planned_node_cpus(child_name, child_map, cpu, out); + } + } + } +} + +fn load_planned_node_cpu_map() -> HashMap { + let Some(path) = queuing_structure_path() else { + return HashMap::new(); + }; + let raw = match std::fs::read_to_string(path) { + Ok(s) => s, + Err(_) => return HashMap::new(), + }; + let json: Value = match serde_json::from_str(&raw) { + Ok(v) => v, + Err(_) => return HashMap::new(), + }; + let Some(net) = json.get("Network").and_then(|v| v.as_object()) else { + return HashMap::new(); + }; + + let looks_binned = net.keys().any(|k| k.starts_with("CpueQueue")); + let mut out = HashMap::new(); + + if looks_binned { + for (cpu_key, cpu_value) in net { + let Some(cpu_node) = cpu_value.as_object() else { + continue; + }; + let cpu = trailing_u32(cpu_key) + .or_else(|| node_cpu_from_queue_map(cpu_node, None)) + .unwrap_or_default(); + if let Some(Value::Object(children)) = cpu_node.get("children") { + for (child_name, child_value) in children { + if let Value::Object(child_map) = child_value { + collect_planned_node_cpus(child_name, child_map, Some(cpu), &mut out); + } + } + } + } + } else { + for (child_name, child_value) in net { + let Some(child_map) = child_value.as_object() else { + continue; + }; + collect_planned_node_cpus(child_name, child_map, None, &mut out); + } + } + + out +} + +fn owner_index_for_node(nodes: &[(usize, NetworkTreeLiteNode)], idx: usize) -> Option { + if idx == 0 || idx >= nodes.len() { + return None; + } + + let node = &nodes[idx].1; + if node.runtime_virtualized { + let mut current = node.immediate_parent; + while let Some(parent_idx) = current { + if parent_idx >= nodes.len() { + return None; + } + let parent = &nodes[parent_idx].1; + if !parent.runtime_virtualized { + return Some(parent_idx); + } + current = parent.immediate_parent; + } + return None; + } + + let mut current = node.immediate_parent; + let mut saw_runtime_virtualized_ancestor = false; + while let Some(parent_idx) = current { + if parent_idx >= nodes.len() { + return None; + } + let parent = &nodes[parent_idx].1; + if parent.runtime_virtualized { + saw_runtime_virtualized_ancestor = true; + current = parent.immediate_parent; + continue; + } + if saw_runtime_virtualized_ancestor { + return Some(parent_idx); + } + current = parent.immediate_parent; + } + + Some(idx) +} + +fn fallback_effective_cpu( + nodes: &[(usize, NetworkTreeLiteNode)], + planned_cpu_by_name: &HashMap, + idx: usize, +) -> Option { + let mut current = Some(idx); + while let Some(node_idx) = current { + if node_idx >= nodes.len() { + return None; + } + if let Some(cpu) = planned_cpu_by_name + .get(nodes[node_idx].1.name.as_str()) + .copied() + { + return Some(cpu); + } + current = nodes[node_idx].1.immediate_parent; + } + None +} + +fn derive_runtime_node_placements( + nodes: &[(usize, NetworkTreeLiteNode)], + planned_cpu_by_name: &HashMap, +) -> Vec { + let mut placements = Vec::with_capacity(nodes.len()); + + for idx in 0..nodes.len() { + if idx == 0 { + placements.push(RuntimeNodePlacement::default()); + continue; + } + + let planned_cpu = planned_cpu_by_name.get(nodes[idx].1.name.as_str()).copied(); + let owner_index = owner_index_for_node(nodes, idx); + let effective_cpu = owner_index + .and_then(|owner| { + if owner == 0 { + None + } else { + planned_cpu_by_name + .get(nodes[owner].1.name.as_str()) + .copied() + } + }) + .or_else(|| fallback_effective_cpu(nodes, planned_cpu_by_name, idx)); + + let assignment_reason = if nodes[idx].1.runtime_virtualized { + "runtime_virtualized_hidden" + } else if owner_index.is_some_and(|owner| owner != idx) { + "inherited_from_virtualized_ancestor" + } else if effective_cpu.is_some() { + "planned" + } else { + "unknown" + }; + + placements.push(RuntimeNodePlacement { + owner_index, + planned_cpu, + effective_cpu, + assignment_reason, + }); + } + + placements +} + +fn direct_circuit_counts_by_node() -> HashMap { + let shaped = SHAPED_DEVICES.load(); + let mut circuits_by_node: HashMap> = HashMap::new(); + + for device in &shaped.devices { + let node_name = device.parent_node.trim(); + if node_name.is_empty() { + continue; + } + circuits_by_node + .entry(node_name.to_string()) + .or_default() + .insert(device.circuit_hash); + } + + circuits_by_node + .into_iter() + .map(|(name, circuits)| (name, circuits.len() as u32)) + .collect() +} + +fn build_subtree_counts( + nodes: &[(usize, NetworkTreeLiteNode)], + direct_circuit_counts: &HashMap, +) -> (Vec, Vec) { + let mut subtree_node_counts = vec![0u32; nodes.len()]; + let mut subtree_circuit_counts = vec![0u32; nodes.len()]; + + for (idx, (_, node)) in nodes.iter().enumerate() { + subtree_circuit_counts[idx] = direct_circuit_counts + .get(node.name.as_str()) + .copied() + .unwrap_or(0); + } + + for idx in (1..nodes.len()).rev() { + let Some(parent_idx) = nodes[idx].1.immediate_parent else { + continue; + }; + if parent_idx >= nodes.len() { + continue; + } + subtree_node_counts[parent_idx] = subtree_node_counts[parent_idx] + .saturating_add(1) + .saturating_add(subtree_node_counts[idx]); + subtree_circuit_counts[parent_idx] = + subtree_circuit_counts[parent_idx].saturating_add(subtree_circuit_counts[idx]); + } + + (subtree_node_counts, subtree_circuit_counts) +} + +fn is_cpu_root_node( + nodes: &[(usize, NetworkTreeLiteNode)], + placements: &[RuntimeNodePlacement], + idx: usize, +) -> bool { + if idx == 0 || idx >= nodes.len() || idx >= placements.len() { + return false; + } + + let placement = &placements[idx]; + let Some(effective_cpu) = placement.effective_cpu else { + return false; + }; + if nodes[idx].1.runtime_virtualized { + return false; + } + + let mut current = nodes[idx].1.immediate_parent; + let mut saw_runtime_virtualized_ancestor = false; + while let Some(parent_idx) = current { + if parent_idx == 0 || parent_idx >= nodes.len() { + break; + } + let parent = &nodes[parent_idx].1; + if parent.runtime_virtualized { + saw_runtime_virtualized_ancestor = true; + current = parent.immediate_parent; + continue; + } + break; + } + + if saw_runtime_virtualized_ancestor { + return true; + } + + if placement.owner_index != Some(idx) { + return false; + } + + let Some(parent_idx) = nodes[idx].1.immediate_parent else { + return true; + }; + if parent_idx == 0 || parent_idx >= placements.len() { + return true; + } + + placements[parent_idx].effective_cpu != Some(effective_cpu) +} + +fn aggregate_planned_core_metrics(circuits: &[CircuitRecord]) -> HashMap { + let mut planned = HashMap::new(); + + for circuit in circuits { + let ignored = circuit.has_planner_weight && circuit.planner_weight <= 0.0; + if ignored { + continue; + } + let Some(cpu) = circuit.cpu_down else { + continue; + }; + let entry = planned + .entry(cpu) + .or_insert_with(PlannedCoreMetrics::default); + entry.circuit_count = entry.circuit_count.saturating_add(1); + entry.weight_sum += circuit.planner_weight; + entry.max_mbps += circuit.max_down; + } + + planned +} + +fn live_cpu_usage_by_index() -> HashMap { + let count = NUM_CPUS.load(std::sync::atomic::Ordering::Relaxed); + let mut usage = HashMap::new(); + for cpu in 0..count { + usage.insert( + cpu as u32, + CPU_USAGE[cpu].load(std::sync::atomic::Ordering::Relaxed), + ); + } + usage +} + +fn inferred_included_cpus( + planned_core_metrics: &HashMap, + placements: &[RuntimeNodePlacement], +) -> BTreeSet { + let mut included = BTreeSet::new(); + + for cpu in planned_core_metrics.keys().copied() { + included.insert(cpu); + } + + for placement in placements { + if let Some(cpu) = placement.planned_cpu { + included.insert(cpu); + } + if let Some(cpu) = placement.effective_cpu { + included.insert(cpu); + } + } + + included +} + +fn resolve_snapshot_cpu_sets( + detection: Option, + all_cpus: &BTreeSet, + planned_core_metrics: &HashMap, + placements: &[RuntimeNodePlacement], +) -> (Vec, Vec, bool) { + let Some(detection) = detection else { + return (Vec::new(), Vec::new(), false); + }; + + let possible = if detection.possible.is_empty() { + all_cpus.clone() + } else { + detection.possible.iter().copied().collect::>() + }; + + if detection.has_hybrid_split { + let shaping = detection.shaping.into_iter().collect::>(); + let excluded = possible.difference(&shaping).copied().collect::>(); + return (shaping.into_iter().collect(), excluded, true); + } + + if detection.exclude_efficiency_cores { + let inferred = inferred_included_cpus(planned_core_metrics, placements); + if !inferred.is_empty() && inferred.len() < possible.len() { + let excluded = possible.difference(&inferred).copied().collect::>(); + return (inferred.into_iter().collect(), excluded, false); + } + } + + let shaping = detection.shaping; + let excluded = possible + .difference(&shaping.iter().copied().collect::>()) + .copied() + .collect::>(); + (shaping, excluded, detection.has_hybrid_split) +} + +fn is_site_type(node_type: Option<&str>) -> bool { + node_type.is_some_and(|t| t.eq_ignore_ascii_case("site")) +} + +fn is_ap_type(node_type: Option<&str>) -> bool { + node_type.is_some_and(|t| t.eq_ignore_ascii_case("ap")) +} + fn is_network_node(node: &serde_json::Map) -> bool { // Prefer explicit type checks where present if node @@ -402,6 +895,168 @@ pub fn cpu_affinity_site_tree_data() -> Option { }) } +/// Build the runtime-aware CPU affinity snapshot used by the CPU assignment page. +pub fn cpu_affinity_runtime_snapshot_data() -> CpuAffinityRuntimeSnapshot { + let live_nodes = network_tree_lite_data(); + let planned_cpu_by_name = load_planned_node_cpu_map(); + let placements = derive_runtime_node_placements(&live_nodes, &planned_cpu_by_name); + let direct_circuit_counts = direct_circuit_counts_by_node(); + let (subtree_node_counts, subtree_circuit_counts) = + build_subtree_counts(&live_nodes, &direct_circuit_counts); + let circuits = load_all_circuits(); + let planned_core_metrics = aggregate_planned_core_metrics(&circuits); + let live_cpu_usage = live_cpu_usage_by_index(); + + let mut node_index_by_name = HashMap::new(); + for (idx, (_, node)) in live_nodes.iter().enumerate() { + node_index_by_name.insert(node.name.clone(), idx); + } + + let mut cores_by_cpu: HashMap = HashMap::new(); + let mut all_cpus = BTreeSet::new(); + + for cpu in live_cpu_usage.keys().copied() { + all_cpus.insert(cpu); + } + for cpu in planned_core_metrics.keys().copied() { + all_cpus.insert(cpu); + } + + for idx in 1..live_nodes.len() { + let node = &live_nodes[idx].1; + let Some(cpu) = placements[idx].effective_cpu else { + continue; + }; + all_cpus.insert(cpu); + let core = cores_by_cpu + .entry(cpu) + .or_insert_with(|| CpuAffinityRuntimeCore { + cpu, + live_usage_pct: live_cpu_usage.get(&cpu).copied(), + planned_circuit_count: 0, + planned_weight_sum: 0.0, + planned_max_mbps: 0.0, + effective_node_count: 0, + effective_site_count: 0, + effective_ap_count: 0, + effective_circuit_count: 0, + runtime_changed_count: 0, + nodes: Vec::new(), + }); + + core.effective_node_count = core.effective_node_count.saturating_add(1); + if is_site_type(node.node_type.as_deref()) { + core.effective_site_count = core.effective_site_count.saturating_add(1); + } + if is_ap_type(node.node_type.as_deref()) { + core.effective_ap_count = core.effective_ap_count.saturating_add(1); + } + if placements[idx].assignment_reason != "planned" + || placements[idx].planned_cpu != placements[idx].effective_cpu + { + core.runtime_changed_count = core.runtime_changed_count.saturating_add(1); + } + + core.nodes.push(CpuAffinityRuntimeNode { + name: node.name.clone(), + node_type: node.node_type.clone(), + planned_cpu: placements[idx].planned_cpu, + effective_cpu: placements[idx].effective_cpu, + runtime_virtualized: node.runtime_virtualized, + assignment_reason: placements[idx].assignment_reason.to_string(), + is_cpu_root: is_cpu_root_node(&live_nodes, &placements, idx), + subtree_node_count: subtree_node_counts.get(idx).copied().unwrap_or_default(), + subtree_circuit_count: subtree_circuit_counts.get(idx).copied().unwrap_or_default(), + effective_max_mbps: node.max_throughput, + current_throughput_bps: node.current_throughput, + }); + } + + for (node_name, circuit_count) in direct_circuit_counts { + let Some(node_idx) = node_index_by_name.get(node_name.as_str()).copied() else { + continue; + }; + let Some(cpu) = placements + .get(node_idx) + .and_then(|placement| placement.effective_cpu) + else { + continue; + }; + all_cpus.insert(cpu); + let core = cores_by_cpu + .entry(cpu) + .or_insert_with(|| CpuAffinityRuntimeCore { + cpu, + live_usage_pct: live_cpu_usage.get(&cpu).copied(), + planned_circuit_count: 0, + planned_weight_sum: 0.0, + planned_max_mbps: 0.0, + effective_node_count: 0, + effective_site_count: 0, + effective_ap_count: 0, + effective_circuit_count: 0, + runtime_changed_count: 0, + nodes: Vec::new(), + }); + core.effective_circuit_count = core.effective_circuit_count.saturating_add(circuit_count); + } + + let generated_at_unix_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0); + + let shaping_detection = load_config() + .ok() + .map(|cfg| detect_shaping_cpus(cfg.as_ref())); + let (shaping_cpus, excluded_cpus, has_hybrid_split) = resolve_snapshot_cpu_sets( + shaping_detection, + &all_cpus, + &planned_core_metrics, + &placements, + ); + + let mut cores = Vec::with_capacity(all_cpus.len()); + for cpu in all_cpus { + let planned = planned_core_metrics.get(&cpu).copied().unwrap_or_default(); + let mut core = cores_by_cpu + .remove(&cpu) + .unwrap_or_else(|| CpuAffinityRuntimeCore { + cpu, + live_usage_pct: live_cpu_usage.get(&cpu).copied(), + planned_circuit_count: 0, + planned_weight_sum: 0.0, + planned_max_mbps: 0.0, + effective_node_count: 0, + effective_site_count: 0, + effective_ap_count: 0, + effective_circuit_count: 0, + runtime_changed_count: 0, + nodes: Vec::new(), + }); + core.live_usage_pct = live_cpu_usage.get(&cpu).copied().or(core.live_usage_pct); + core.planned_circuit_count = planned.circuit_count; + core.planned_weight_sum = planned.weight_sum; + core.planned_max_mbps = planned.max_mbps; + core.nodes.sort_by(|left, right| { + let left_tp = left.current_throughput_bps.0 + left.current_throughput_bps.1; + let right_tp = right.current_throughput_bps.0 + right.current_throughput_bps.1; + right_tp + .cmp(&left_tp) + .then_with(|| left.name.cmp(&right.name)) + }); + cores.push(core); + } + + CpuAffinityRuntimeSnapshot { + generated_at_unix_ms, + shaping_cpus, + excluded_cpus, + has_hybrid_split, + cores, + } +} + pub fn cpu_affinity_summary_data() -> Vec { let circuits = load_all_circuits(); let mut down: HashMap = HashMap::new(); @@ -672,3 +1327,201 @@ pub fn cpu_affinity_preview_weights_data(q: CircuitsQuery) -> Vec, + runtime_virtualized: bool, + ) -> NetworkTreeLiteNode { + NetworkTreeLiteNode { + name: name.to_string(), + id: None, + is_virtual: false, + runtime_virtualized, + max_throughput: (0.0, 0.0), + current_throughput: (0, 0), + current_tcp_packets: (0, 0), + current_tcp_retransmit_packets: (0, 0), + current_retransmits: (0, 0), + rtts: Vec::new(), + qoo: (None, None), + parents: Vec::new(), + immediate_parent, + node_type: None, + latitude: None, + longitude: None, + } + } + + #[test] + fn runtime_placements_collapse_virtualized_branch_into_parent_cpu() { + let nodes = vec![ + (0, test_node("Root", None, false)), + (1, test_node("Site A", Some(0), false)), + (2, test_node("Site B", Some(1), true)), + (3, test_node("Site C", Some(2), false)), + ]; + let planned = HashMap::from([ + ("Site A".to_string(), 2u32), + ("Site B".to_string(), 2u32), + ("Site C".to_string(), 2u32), + ]); + + let placements = derive_runtime_node_placements(&nodes, &planned); + + assert_eq!( + placements[1], + RuntimeNodePlacement { + owner_index: Some(1), + planned_cpu: Some(2), + effective_cpu: Some(2), + assignment_reason: "planned", + } + ); + assert_eq!( + placements[2], + RuntimeNodePlacement { + owner_index: Some(1), + planned_cpu: Some(2), + effective_cpu: Some(2), + assignment_reason: "runtime_virtualized_hidden", + } + ); + assert_eq!( + placements[3], + RuntimeNodePlacement { + owner_index: Some(1), + planned_cpu: Some(2), + effective_cpu: Some(2), + assignment_reason: "inherited_from_virtualized_ancestor", + } + ); + } + + #[test] + fn runtime_placements_fall_back_to_top_level_virtualized_cpu() { + let nodes = vec![ + (0, test_node("Root", None, false)), + (1, test_node("Top Level", Some(0), true)), + ]; + let planned = HashMap::from([("Top Level".to_string(), 5u32)]); + + let placements = derive_runtime_node_placements(&nodes, &planned); + + assert_eq!( + placements[1], + RuntimeNodePlacement { + owner_index: Some(0), + planned_cpu: Some(5), + effective_cpu: Some(5), + assignment_reason: "runtime_virtualized_hidden", + } + ); + } + + #[test] + fn cpu_root_selection_keeps_only_top_level_owned_branch() { + let nodes = vec![ + (0, test_node("Root", None, false)), + (1, test_node("Site A", Some(0), false)), + (2, test_node("AP A1", Some(1), false)), + (3, test_node("AP A2", Some(2), false)), + ]; + let planned = HashMap::from([ + ("Site A".to_string(), 2u32), + ("AP A1".to_string(), 2u32), + ("AP A2".to_string(), 2u32), + ]); + + let placements = derive_runtime_node_placements(&nodes, &planned); + + assert!(is_cpu_root_node(&nodes, &placements, 1)); + assert!(!is_cpu_root_node(&nodes, &placements, 2)); + assert!(!is_cpu_root_node(&nodes, &placements, 3)); + } + + #[test] + fn cpu_root_selection_ignores_virtualized_hidden_nodes() { + let nodes = vec![ + (0, test_node("Root", None, false)), + (1, test_node("Top Level", Some(0), true)), + (2, test_node("Child", Some(1), false)), + ]; + let planned = HashMap::from([("Top Level".to_string(), 5u32), ("Child".to_string(), 5u32)]); + + let placements = derive_runtime_node_placements(&nodes, &planned); + + assert!(!is_cpu_root_node(&nodes, &placements, 1)); + assert!(is_cpu_root_node(&nodes, &placements, 2)); + } + + #[test] + fn snapshot_cpu_sets_fall_back_to_planned_subset_when_hybrid_split_missing() { + let all_cpus = BTreeSet::from([0_u32, 1, 2, 3]); + let planned_core_metrics = HashMap::from([ + ( + 0_u32, + PlannedCoreMetrics { + circuit_count: 12, + weight_sum: 1.0, + max_mbps: 1000.0, + }, + ), + ( + 1_u32, + PlannedCoreMetrics { + circuit_count: 9, + weight_sum: 1.0, + max_mbps: 800.0, + }, + ), + ]); + let placements = vec![ + RuntimeNodePlacement { + owner_index: Some(0), + planned_cpu: Some(0), + effective_cpu: Some(0), + assignment_reason: "planned", + }, + RuntimeNodePlacement { + owner_index: Some(1), + planned_cpu: Some(1), + effective_cpu: Some(1), + assignment_reason: "planned", + }, + ]; + let detection = ShapingCpuDetection { + exclude_efficiency_cores: true, + source: ShapingCpuSource::FallbackAllPossible, + from_cache: false, + has_hybrid_split: false, + detail: "No trustworthy hybrid CPU split detected; using all possible CPUs".to_string(), + possible: vec![0, 1, 2, 3], + performance: vec![0, 1, 2, 3], + efficiency: Vec::new(), + shaping: vec![0, 1, 2, 3], + }; + + let (shaping, excluded, has_hybrid_split) = resolve_snapshot_cpu_sets( + Some(detection), + &all_cpus, + &planned_core_metrics, + &placements, + ); + + assert_eq!(shaping, vec![0, 1]); + assert_eq!(excluded, vec![2, 3]); + assert!(!has_hybrid_split); + } +} diff --git a/src/rust/lqosd/src/node_manager/local_api/directories.rs b/src/rust/lqosd/src/node_manager/local_api/directories.rs new file mode 100644 index 00000000..63a5e525 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/directories.rs @@ -0,0 +1,233 @@ +use crate::node_manager::local_api::network_tree; +use crate::shaped_devices_tracker::SHAPED_DEVICES; +use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +const DEFAULT_CIRCUIT_DIRECTORY_PAGE_SIZE: usize = 100; +const MAX_CIRCUIT_DIRECTORY_PAGE_SIZE: usize = 250; + +fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum MaybeString { + String(String), + Null, + } + + Ok(match Option::::deserialize(deserializer)? { + Some(MaybeString::String(value)) => Some(value), + Some(MaybeString::Null) | None => None, + }) +} + +/// Query parameters for the paged circuit directory used by configuration UIs. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitDirectoryQuery { + pub page: Option, + pub page_size: Option, + #[serde(default, deserialize_with = "deserialize_optional_string")] + pub search: Option, +} + +/// Compact circuit metadata row for selectors and lookups. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitDirectoryRow { + pub circuit_id: String, + pub circuit_name: String, + pub parent_node: String, + pub sqm_override: Option, +} + +/// A server-paged slice of the circuit directory. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitDirectoryPage { + pub query: CircuitDirectoryQuery, + pub total_rows: usize, + pub rows: Vec, +} + +/// Lightweight node directory entry for UI link resolution and selectors. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct NodeDirectoryEntry { + pub tree_index: usize, + pub node_id: Option, + pub node_name: String, + pub node_type: Option, +} + +/// Small TreeGuard summary used by dashboard/status panels. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TreeGuardMetadataSummary { + pub total_nodes: usize, + pub total_circuits: usize, + pub virtualized_nodes: usize, + pub fq_codel_circuits: usize, +} + +fn normalized_page_size(query: &CircuitDirectoryQuery) -> usize { + query + .page_size + .unwrap_or(DEFAULT_CIRCUIT_DIRECTORY_PAGE_SIZE) + .clamp(1, MAX_CIRCUIT_DIRECTORY_PAGE_SIZE) +} + +fn has_fq_codel_override(raw: &str) -> bool { + raw.split('/') + .map(|part| part.trim().to_ascii_lowercase()) + .any(|part| part == "fq_codel") +} + +/// Returns one filtered, sorted page of circuit directory rows. +pub fn circuit_directory_page(query: CircuitDirectoryQuery) -> CircuitDirectoryPage { + let page = query.page.unwrap_or(0); + let page_size = normalized_page_size(&query); + let search = query.search.as_deref().unwrap_or("").trim().to_lowercase(); + let devices = SHAPED_DEVICES.load(); + + let mut circuits: BTreeMap = BTreeMap::new(); + for device in &devices.devices { + let circuit_id = device.circuit_id.trim().to_string(); + if circuit_id.is_empty() { + continue; + } + let row = circuits + .entry(circuit_id.clone()) + .or_insert_with(|| CircuitDirectoryRow { + circuit_id: circuit_id.clone(), + circuit_name: device.circuit_name.trim().to_string(), + parent_node: device.parent_node.trim().to_string(), + sqm_override: { + let override_value = device.sqm_override.as_deref().unwrap_or("").trim(); + if override_value.is_empty() { + None + } else { + Some(override_value.to_string()) + } + }, + }); + if row.circuit_name.is_empty() && !device.circuit_name.trim().is_empty() { + row.circuit_name = device.circuit_name.trim().to_string(); + } + if row.parent_node.is_empty() && !device.parent_node.trim().is_empty() { + row.parent_node = device.parent_node.trim().to_string(); + } + let sqm_override = device.sqm_override.as_deref().unwrap_or("").trim(); + if row.sqm_override.is_none() && !sqm_override.is_empty() { + row.sqm_override = Some(sqm_override.to_string()); + } + } + + let mut filtered: Vec = circuits + .into_values() + .filter(|row| { + if search.is_empty() { + return true; + } + row.circuit_id.to_lowercase().contains(&search) + || row.circuit_name.to_lowercase().contains(&search) + || row.parent_node.to_lowercase().contains(&search) + || row + .sqm_override + .as_deref() + .unwrap_or("") + .to_lowercase() + .contains(&search) + }) + .collect(); + filtered.sort_by(|left, right| { + left.circuit_name + .cmp(&right.circuit_name) + .then_with(|| left.circuit_id.cmp(&right.circuit_id)) + }); + + let total_rows = filtered.len(); + let start = page.saturating_mul(page_size); + let rows = if start >= total_rows { + Vec::new() + } else { + let end = (start + page_size).min(total_rows); + filtered[start..end].to_vec() + }; + + CircuitDirectoryPage { + query: CircuitDirectoryQuery { + page: Some(page), + page_size: Some(page_size), + search: if search.is_empty() { + None + } else { + query.search + }, + }, + total_rows, + rows, + } +} + +/// Returns a compact flattened node directory for UI link resolution. +pub fn node_directory_data() -> Vec { + let mut nodes = network_tree::network_tree_data() + .into_iter() + .filter_map(|(tree_index, node)| { + let node_name = node.name.trim().to_string(); + if node_name.is_empty() || node_name == "Root" { + return None; + } + Some(NodeDirectoryEntry { + tree_index, + node_id: node.id.clone(), + node_name, + node_type: node.node_type.clone(), + }) + }) + .collect::>(); + nodes.sort_by(|left, right| { + left.node_name + .cmp(&right.node_name) + .then_with(|| left.tree_index.cmp(&right.tree_index)) + }); + nodes +} + +/// Returns a compact TreeGuard metadata summary without loading full device/node payloads into the browser. +pub fn treeguard_metadata_summary() -> TreeGuardMetadataSummary { + let network = network_tree::network_tree_data(); + let total_nodes = network + .iter() + .filter(|(_, node)| { + let name = node.name.trim(); + !name.is_empty() && name != "Root" + }) + .count(); + let virtualized_nodes = network + .iter() + .filter(|(_, node)| { + let name = node.name.trim(); + !name.is_empty() && name != "Root" && node.runtime_virtualized + }) + .count(); + + let devices = SHAPED_DEVICES.load(); + let mut circuit_ids = BTreeSet::new(); + let mut fq_codel_circuit_ids = BTreeSet::new(); + for device in &devices.devices { + let circuit_id = device.circuit_id.trim().to_string(); + if circuit_id.is_empty() { + continue; + } + circuit_ids.insert(circuit_id.clone()); + if has_fq_codel_override(device.sqm_override.as_deref().unwrap_or("").trim()) { + fq_codel_circuit_ids.insert(circuit_id); + } + } + + TreeGuardMetadataSummary { + total_nodes, + total_circuits: circuit_ids.len(), + virtualized_nodes, + fq_codel_circuits: fq_codel_circuit_ids.len(), + } +} diff --git a/src/rust/lqosd/src/node_manager/local_api/ethernet_caps.rs b/src/rust/lqosd/src/node_manager/local_api/ethernet_caps.rs new file mode 100644 index 00000000..94b72dc0 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/ethernet_caps.rs @@ -0,0 +1,294 @@ +use crate::shaped_devices_tracker::SHAPED_DEVICES; +use lqos_config::{ + CIRCUIT_ETHERNET_METADATA_FILENAME, CircuitEthernetMetadata, CircuitEthernetMetadataFile, + load_config, +}; +use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::HashMap; +use std::path::Path; + +const DEFAULT_ETHERNET_CAPS_PAGE_SIZE: usize = 100; +const MAX_ETHERNET_CAPS_PAGE_SIZE: usize = 250; + +fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum MaybeString { + String(String), + Null, + } + + Ok(match Option::::deserialize(deserializer)? { + Some(MaybeString::String(value)) => Some(value), + Some(MaybeString::Null) | None => None, + }) +} + +/// Normalized Ethernet-cap tier used by UI badges and filters. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum EthernetCapTier { + TenM, + HundredM, + GigPlus, +} + +impl EthernetCapTier { + /// Returns the short operator-facing label for this Ethernet tier. + pub const fn label(&self) -> &'static str { + match self { + Self::TenM => "10M", + Self::HundredM => "100M", + Self::GigPlus => "1G", + } + } +} + +/// Compact Ethernet-cap badge payload for circuit tables and detail views. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct EthernetCapBadge { + /// Normalized tier used for styling and filtering. + pub tier: EthernetCapTier, + /// Short display label such as `10M`, `100M`, or `1G`. + pub tier_label: String, + /// Negotiated Ethernet speed in Mbps reported by the limiting interface. + pub negotiated_ethernet_mbps: u64, + /// Requested download max before the cap was applied. + pub requested_download_mbps: f32, + /// Requested upload max before the cap was applied. + pub requested_upload_mbps: f32, + /// Applied download max after the cap. + pub applied_download_mbps: f32, + /// Applied upload max after the cap. + pub applied_upload_mbps: f32, +} + +/// Server-side paging/filter query for the Ethernet caps review page. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct EthernetCapsPageQuery { + pub page: Option, + pub page_size: Option, + #[serde(default, deserialize_with = "deserialize_optional_string")] + pub search: Option, + pub tier: Option, +} + +/// One Ethernet-limited circuit row for the review page. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct EthernetCapsPageRow { + /// Stable circuit identifier used for deep-linking to the circuit page. + pub circuit_id: String, + /// Human-facing circuit name. + pub circuit_name: String, + /// Parent node from shaped devices when available. + pub parent_node: String, + /// Compact warning badge metadata. + pub badge: EthernetCapBadge, + /// Device name that supplied the limiting Ethernet speed when known. + pub limiting_device_name: Option, + /// Interface name that supplied the limiting Ethernet speed when known. + pub limiting_interface_name: Option, +} + +/// A server-paged slice of Ethernet-capped circuits. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct EthernetCapsPage { + /// The normalized query that produced this page. + pub query: EthernetCapsPageQuery, + /// Total rows matching the query before paging. + pub total_rows: usize, + /// Page rows after filtering and paging. + pub rows: Vec, +} + +fn normalized_page_size(query: &EthernetCapsPageQuery) -> usize { + query + .page_size + .unwrap_or(DEFAULT_ETHERNET_CAPS_PAGE_SIZE) + .clamp(1, MAX_ETHERNET_CAPS_PAGE_SIZE) +} + +fn tier_for_speed(negotiated_ethernet_mbps: u64) -> EthernetCapTier { + match negotiated_ethernet_mbps { + 0..=10 => EthernetCapTier::TenM, + 11..=100 => EthernetCapTier::HundredM, + _ => EthernetCapTier::GigPlus, + } +} + +fn advisory_to_badge(advisory: &CircuitEthernetMetadata) -> Option { + if !advisory.auto_capped { + return None; + } + + let tier = tier_for_speed(advisory.negotiated_ethernet_mbps); + Some(EthernetCapBadge { + tier: tier.clone(), + tier_label: tier.label().to_string(), + negotiated_ethernet_mbps: advisory.negotiated_ethernet_mbps, + requested_download_mbps: advisory.requested_download_mbps, + requested_upload_mbps: advisory.requested_upload_mbps, + applied_download_mbps: advisory.applied_download_mbps, + applied_upload_mbps: advisory.applied_upload_mbps, + }) +} + +fn load_advisory_file() -> Option { + let cfg = load_config().ok()?; + let path = Path::new(&cfg.lqos_directory).join(CIRCUIT_ETHERNET_METADATA_FILENAME); + let payload = std::fs::read(path).ok()?; + serde_json::from_slice(&payload).ok() +} + +fn parent_node_by_circuit_id() -> HashMap { + let devices = SHAPED_DEVICES.load(); + let mut parent_nodes = HashMap::new(); + for device in &devices.devices { + let circuit_id = device.circuit_id.trim(); + if circuit_id.is_empty() { + continue; + } + parent_nodes + .entry(circuit_id.to_string()) + .or_insert_with(|| device.parent_node.trim().to_string()); + } + parent_nodes +} + +fn sort_rank(tier: &EthernetCapTier) -> u8 { + match tier { + EthernetCapTier::TenM => 0, + EthernetCapTier::HundredM => 1, + EthernetCapTier::GigPlus => 2, + } +} + +/// Returns a compact badge map keyed by circuit ID for Ethernet auto-capped circuits. +pub(crate) fn ethernet_cap_badge_map() -> HashMap { + let mut badges = HashMap::new(); + let Some(file) = load_advisory_file() else { + return badges; + }; + for advisory in file.circuits { + let Some(badge) = advisory_to_badge(&advisory) else { + continue; + }; + badges.insert(advisory.circuit_id.to_ascii_lowercase(), badge); + } + badges +} + +/// Finds one Ethernet advisory for the requested circuit and matching shaped-device IDs. +pub(crate) fn ethernet_advisory_for_circuit( + circuit_id: &str, + device_ids: &std::collections::HashSet<&str>, +) -> Option { + let file = load_advisory_file()?; + file.circuits.into_iter().find(|entry| { + entry.auto_capped + && entry.circuit_id.eq_ignore_ascii_case(circuit_id) + && entry + .device_ids + .iter() + .any(|device_id| device_ids.contains(device_id.as_str())) + }) +} + +/// Returns one filtered, sorted page of Ethernet-limited circuits for operator review. +pub fn ethernet_caps_page(query: EthernetCapsPageQuery) -> EthernetCapsPage { + let page = query.page.unwrap_or(0); + let page_size = normalized_page_size(&query); + let search = query.search.as_deref().unwrap_or("").trim().to_lowercase(); + let parent_nodes = parent_node_by_circuit_id(); + + let mut rows = Vec::new(); + if let Some(file) = load_advisory_file() { + for advisory in file.circuits { + let Some(badge) = advisory_to_badge(&advisory) else { + continue; + }; + if let Some(filter_tier) = query.tier.as_ref() + && &badge.tier != filter_tier + { + continue; + } + let parent_node = parent_nodes + .get(&advisory.circuit_id) + .cloned() + .unwrap_or_default(); + let row = EthernetCapsPageRow { + circuit_id: advisory.circuit_id, + circuit_name: advisory.circuit_name, + parent_node, + badge, + limiting_device_name: advisory.limiting_device_name, + limiting_interface_name: advisory.limiting_interface_name, + }; + if !search.is_empty() { + let limiting_device = row.limiting_device_name.as_deref().unwrap_or(""); + let limiting_interface = row.limiting_interface_name.as_deref().unwrap_or(""); + if !row.circuit_id.to_lowercase().contains(&search) + && !row.circuit_name.to_lowercase().contains(&search) + && !row.parent_node.to_lowercase().contains(&search) + && !row.badge.tier_label.to_lowercase().contains(&search) + && !limiting_device.to_lowercase().contains(&search) + && !limiting_interface.to_lowercase().contains(&search) + { + continue; + } + } + rows.push(row); + } + } + + rows.sort_by(|left, right| { + sort_rank(&left.badge.tier) + .cmp(&sort_rank(&right.badge.tier)) + .then_with(|| left.circuit_name.cmp(&right.circuit_name)) + .then_with(|| left.circuit_id.cmp(&right.circuit_id)) + }); + + let total_rows = rows.len(); + let start = page.saturating_mul(page_size); + let rows = if start >= total_rows { + Vec::new() + } else { + let end = (start + page_size).min(total_rows); + rows[start..end].to_vec() + }; + + EthernetCapsPage { + query: EthernetCapsPageQuery { + page: Some(page), + page_size: Some(page_size), + search: if search.is_empty() { + None + } else { + query.search + }, + tier: query.tier, + }, + total_rows, + rows, + } +} + +#[cfg(test)] +mod tests { + use super::{EthernetCapTier, sort_rank, tier_for_speed}; + + #[test] + fn ethernet_cap_tier_classifies_expected_speeds() { + assert_eq!(tier_for_speed(10), EthernetCapTier::TenM); + assert_eq!(tier_for_speed(100), EthernetCapTier::HundredM); + assert_eq!(tier_for_speed(1000), EthernetCapTier::GigPlus); + } + + #[test] + fn ethernet_cap_sort_rank_prioritizes_low_speed_tiers() { + assert!(sort_rank(&EthernetCapTier::TenM) < sort_rank(&EthernetCapTier::HundredM)); + assert!(sort_rank(&EthernetCapTier::HundredM) < sort_rank(&EthernetCapTier::GigPlus)); + } +} diff --git a/src/rust/lqosd/src/node_manager/local_api/executive.rs b/src/rust/lqosd/src/node_manager/local_api/executive.rs new file mode 100644 index 00000000..915b1b7e --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/executive.rs @@ -0,0 +1,512 @@ +use crate::node_manager::local_api::executive_cache::fresh_executive_cache_snapshot; +use lqos_bus::ExecutiveSummaryHeader; +use lqos_utils::{HeatmapBlocks, qoq_heatmap::QoqHeatmapBlocks}; +use serde::{Deserialize, Serialize}; + +const DEFAULT_EXECUTIVE_HEATMAP_PAGE_SIZE: usize = 50; +const DEFAULT_EXECUTIVE_LEADERBOARD_PAGE_SIZE: usize = 50; +const MAX_EXECUTIVE_PAGE_SIZE: usize = 250; + +/// Entity kinds shown in executive dashboard and detail views. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum ExecutiveEntityKind { + Site, + Circuit, + Asn, +} + +/// Executive metrics supported by summary and detail heatmap views. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum ExecutiveMetric { + Download, + Upload, + Retransmit, + Rtt, + Qoo, +} + +/// Server-side sort modes for executive heatmap detail pages. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum ExecutiveHeatmapSort { + LatestValue, + Label, + SampleCount, +} + +/// Named executive leaderboard views. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum ExecutiveLeaderboardKind { + WorstSitesByRtt, + OversubscribedSites, + SitesDueUpgrade, + CircuitsDueUpgrade, + TopAsnsByTraffic, +} + +/// Stable locator for linking executive rows back to the tree view. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)] +pub struct ExecutiveTreeLocator { + pub node_id: Option, + pub node_path: Option>, + pub parent_index: Option, +} + +/// One scalar historical series, usually used for a single metric. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)] +pub struct ExecutiveScalarBlocks { + pub values: Vec>, +} + +/// One split directional historical series. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)] +pub struct ExecutiveSplitBlocks { + pub download: Vec>, + pub upload: Vec>, +} + +/// RTT history including combined and directional percentile data. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)] +pub struct ExecutiveRttBlocks { + pub rtt: Vec>, + pub dl_p50: Vec>, + pub dl_p90: Vec>, + pub ul_p50: Vec>, + pub ul_p90: Vec>, +} + +/// One server-ranked metric row for the executive dashboard. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ExecutiveDashboardMetricRow { + pub row_key: String, + pub entity_kind: ExecutiveEntityKind, + pub metric: ExecutiveMetric, + pub label: String, + pub circuit_id: Option, + pub asn: Option, + pub tree: Option, + pub scalar_blocks: Option, + pub split_blocks: Option, + pub rtt_blocks: Option, +} + +/// Oversubscription leaderboard row for the executive dashboard. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ExecutiveOversubscribedSiteRow { + pub row_key: String, + pub site_name: String, + pub tree: Option, + pub ratio_down: Option, + pub ratio_up: Option, + pub ratio_max: Option, + pub cap_down: f32, + pub cap_up: f32, + pub sub_down: f32, + pub sub_up: f32, + pub avg_down_util: Option, + pub avg_up_util: Option, + pub median_rtt_ms: Option, +} + +/// Top-ASN row for executive dashboard summary. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ExecutiveTopAsnRow { + pub row_key: String, + pub asn: u32, + pub asn_name: Option, + pub total_bytes_15m: u64, + pub median_rtt_ms: Option, + pub median_retransmit_pct: Option, +} + +/// Compact executive summary payload for the dashboard. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ExecutiveDashboardSummary { + pub generated_at_unix_ms: u64, + pub header: ExecutiveSummaryHeader, + pub global: HeatmapBlocks, + pub global_qoq: QoqHeatmapBlocks, + pub top_download: Vec, + pub top_upload: Vec, + pub top_retransmit: Vec, + pub top_rtt: Vec, + pub top_qoo: Vec, + pub oversubscribed_sites: Vec, + pub top_asns: Vec, +} + +/// Query for a paged executive heatmap detail page. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ExecutiveHeatmapPageQuery { + pub metric: ExecutiveMetric, + pub entity_kinds: Vec, + pub page: Option, + pub page_size: Option, + pub search: Option, + pub sort: Option, + pub descending: Option, + pub client_request_id: Option, +} + +/// One paged executive heatmap detail row. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ExecutiveHeatmapPageRow { + pub row_key: String, + pub entity_kind: ExecutiveEntityKind, + pub label: String, + pub circuit_id: Option, + pub asn: Option, + pub tree: Option, + pub scalar_blocks: Option, + pub split_blocks: Option, + pub rtt_blocks: Option, +} + +/// One paged executive heatmap detail response. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ExecutiveHeatmapPage { + pub generated_at_unix_ms: u64, + pub query: ExecutiveHeatmapPageQuery, + pub total_rows: usize, + pub rows: Vec, +} + +/// Query for a paged executive leaderboard detail view. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ExecutiveLeaderboardPageQuery { + pub kind: ExecutiveLeaderboardKind, + pub page: Option, + pub page_size: Option, + pub search: Option, + pub client_request_id: Option, +} + +/// One paged executive leaderboard response. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ExecutiveLeaderboardPage { + pub generated_at_unix_ms: u64, + pub query: ExecutiveLeaderboardPageQuery, + pub total_rows: usize, + pub rows: Vec, +} + +/// Rows for the standalone executive leaderboard pages. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind")] +pub enum ExecutiveLeaderboardRow { + WorstSiteByRtt { + row_key: String, + site_name: String, + tree: Option, + median_rtt_ms: f32, + avg_down_util: Option, + avg_up_util: Option, + }, + OversubscribedSite { + row_key: String, + site_name: String, + tree: Option, + ratio_down: Option, + ratio_up: Option, + ratio_max: Option, + cap_down: f32, + cap_up: f32, + sub_down: f32, + sub_up: f32, + avg_down_util: Option, + avg_up_util: Option, + median_rtt_ms: Option, + }, + SiteDueUpgrade { + row_key: String, + site_name: String, + tree: Option, + avg_down_util: f32, + avg_up_util: f32, + }, + CircuitDueUpgrade { + row_key: String, + circuit_id: String, + circuit_name: String, + avg_down_util: f32, + avg_up_util: f32, + }, + TopAsnByTraffic { + row_key: String, + asn: u32, + asn_name: Option, + total_bytes_15m: u64, + median_rtt_ms: Option, + median_retransmit_pct: Option, + }, +} + +fn normalize_page_size(requested: Option, default_size: usize) -> usize { + requested + .unwrap_or(default_size) + .clamp(1, MAX_EXECUTIVE_PAGE_SIZE) +} + +fn normalized_search(search: &Option) -> Option { + let value = search.as_deref().unwrap_or("").trim().to_lowercase(); + if value.is_empty() { None } else { Some(value) } +} + +fn normalized_entity_kinds(mut kinds: Vec) -> Vec { + if kinds.is_empty() { + return vec![ + ExecutiveEntityKind::Site, + ExecutiveEntityKind::Circuit, + ExecutiveEntityKind::Asn, + ]; + } + kinds.sort_by_key(|kind| match kind { + ExecutiveEntityKind::Site => 0_u8, + ExecutiveEntityKind::Circuit => 1_u8, + ExecutiveEntityKind::Asn => 2_u8, + }); + kinds.dedup(); + kinds +} + +fn latest_value(row: &ExecutiveHeatmapPageRow, metric: &ExecutiveMetric) -> Option { + match metric { + ExecutiveMetric::Download | ExecutiveMetric::Upload | ExecutiveMetric::Retransmit => row + .scalar_blocks + .as_ref() + .and_then(|blocks| blocks.values.iter().rev().flatten().copied().next()), + ExecutiveMetric::Qoo => row.split_blocks.as_ref().and_then(|blocks| { + let values = [ + blocks.download.iter().rev().flatten().copied().next(), + blocks.upload.iter().rev().flatten().copied().next(), + ]; + let present: Vec = values.into_iter().flatten().collect(); + if present.is_empty() { + None + } else { + Some(present.iter().sum::() / present.len() as f32) + } + }), + ExecutiveMetric::Rtt => row + .rtt_blocks + .as_ref() + .and_then(|blocks| blocks.rtt.iter().rev().flatten().copied().next()), + } +} + +fn sample_count(row: &ExecutiveHeatmapPageRow, metric: &ExecutiveMetric) -> usize { + match metric { + ExecutiveMetric::Download | ExecutiveMetric::Upload | ExecutiveMetric::Retransmit => row + .scalar_blocks + .as_ref() + .map(|blocks| blocks.values.iter().filter(|value| value.is_some()).count()) + .unwrap_or(0), + ExecutiveMetric::Qoo => row + .split_blocks + .as_ref() + .map(|blocks| { + blocks + .download + .iter() + .chain(blocks.upload.iter()) + .filter(|value| value.is_some()) + .count() + }) + .unwrap_or(0), + ExecutiveMetric::Rtt => row + .rtt_blocks + .as_ref() + .map(|blocks| blocks.rtt.iter().filter(|value| value.is_some()).count()) + .unwrap_or(0), + } +} + +fn leaderboard_matches_search(row: &ExecutiveLeaderboardRow, search: &Option) -> bool { + let Some(search) = search else { + return true; + }; + match row { + ExecutiveLeaderboardRow::WorstSiteByRtt { site_name, .. } + | ExecutiveLeaderboardRow::OversubscribedSite { site_name, .. } + | ExecutiveLeaderboardRow::SiteDueUpgrade { site_name, .. } => { + site_name.to_lowercase().contains(search) + } + ExecutiveLeaderboardRow::CircuitDueUpgrade { + circuit_id, + circuit_name, + .. + } => { + circuit_id.to_lowercase().contains(search) + || circuit_name.to_lowercase().contains(search) + } + ExecutiveLeaderboardRow::TopAsnByTraffic { asn, asn_name, .. } => { + asn.to_string().contains(search) + || asn_name + .as_ref() + .map(|name| name.to_lowercase().contains(search)) + .unwrap_or(false) + } + } +} + +/// Returns the compact executive dashboard summary for published websocket updates. +pub fn executive_dashboard_summary() -> ExecutiveDashboardSummary { + fresh_executive_cache_snapshot().dashboard.clone() +} + +/// Returns one filtered, sorted executive heatmap detail page. +pub fn executive_heatmap_page(query: ExecutiveHeatmapPageQuery) -> ExecutiveHeatmapPage { + let snapshot = fresh_executive_cache_snapshot(); + let page = query.page.unwrap_or(0); + let page_size = normalize_page_size(query.page_size, DEFAULT_EXECUTIVE_HEATMAP_PAGE_SIZE); + let search = normalized_search(&query.search); + let sort = query + .sort + .clone() + .unwrap_or(ExecutiveHeatmapSort::LatestValue); + let descending = query.descending.unwrap_or(true); + let entity_kinds = normalized_entity_kinds(query.entity_kinds.clone()); + + let mut rows = snapshot + .heatmap_rows_for_metric(&query.metric) + .into_iter() + .filter(|row| entity_kinds.contains(&row.entity_kind)) + .filter(|row| { + let Some(search) = &search else { + return true; + }; + row.label.to_lowercase().contains(search) + || row + .circuit_id + .as_ref() + .map(|id| id.to_lowercase().contains(search)) + .unwrap_or(false) + || row + .asn + .map(|asn| asn.to_string().contains(search)) + .unwrap_or(false) + }) + .collect::>(); + + rows.sort_by(|left, right| match sort { + ExecutiveHeatmapSort::Label => left.label.cmp(&right.label), + ExecutiveHeatmapSort::SampleCount => sample_count(left, &query.metric) + .cmp(&sample_count(right, &query.metric)) + .then_with(|| left.label.cmp(&right.label)), + ExecutiveHeatmapSort::LatestValue => { + let left_latest = latest_value(left, &query.metric); + let right_latest = latest_value(right, &query.metric); + right_latest + .partial_cmp(&left_latest) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| right.label.cmp(&left.label)) + } + }); + + if !descending { + rows.reverse(); + } + + let total_rows = rows.len(); + let start = page.saturating_mul(page_size); + let rows = if start >= total_rows { + Vec::new() + } else { + rows.into_iter().skip(start).take(page_size).collect() + }; + + ExecutiveHeatmapPage { + generated_at_unix_ms: snapshot.generated_at_unix_ms, + query: ExecutiveHeatmapPageQuery { + metric: query.metric, + entity_kinds, + page: Some(page), + page_size: Some(page_size), + search, + sort: Some(sort), + descending: Some(descending), + client_request_id: query.client_request_id, + }, + total_rows, + rows, + } +} + +/// Returns one filtered executive leaderboard page. +pub fn executive_leaderboard_page( + query: ExecutiveLeaderboardPageQuery, +) -> ExecutiveLeaderboardPage { + let snapshot = fresh_executive_cache_snapshot(); + let page = query.page.unwrap_or(0); + let page_size = normalize_page_size(query.page_size, DEFAULT_EXECUTIVE_LEADERBOARD_PAGE_SIZE); + let search = normalized_search(&query.search); + let mut rows = snapshot + .leaderboard_rows + .get(&query.kind) + .cloned() + .unwrap_or_default(); + rows.retain(|row| leaderboard_matches_search(row, &search)); + let total_rows = rows.len(); + let start = page.saturating_mul(page_size); + let rows = if start >= total_rows { + Vec::new() + } else { + rows.into_iter().skip(start).take(page_size).collect() + }; + + ExecutiveLeaderboardPage { + generated_at_unix_ms: snapshot.generated_at_unix_ms, + query: ExecutiveLeaderboardPageQuery { + kind: query.kind, + page: Some(page), + page_size: Some(page_size), + search, + client_request_id: query.client_request_id, + }, + total_rows, + rows, + } +} + +#[cfg(test)] +mod tests { + use super::{ + ExecutiveEntityKind, ExecutiveHeatmapPageQuery, ExecutiveHeatmapSort, + ExecutiveLeaderboardKind, ExecutiveLeaderboardPageQuery, ExecutiveMetric, + executive_heatmap_page, executive_leaderboard_page, + }; + + #[test] + fn executive_heatmap_page_preserves_client_request_id() { + let page = executive_heatmap_page(ExecutiveHeatmapPageQuery { + metric: ExecutiveMetric::Rtt, + entity_kinds: vec![ExecutiveEntityKind::Site], + page: Some(0), + page_size: Some(10), + search: Some("WestRedd".to_string()), + sort: Some(ExecutiveHeatmapSort::LatestValue), + descending: Some(true), + client_request_id: Some("heatmap-req-1".to_string()), + }); + + assert_eq!( + page.query.client_request_id.as_deref(), + Some("heatmap-req-1") + ); + } + + #[test] + fn executive_leaderboard_page_preserves_client_request_id() { + let page = executive_leaderboard_page(ExecutiveLeaderboardPageQuery { + kind: ExecutiveLeaderboardKind::WorstSitesByRtt, + page: Some(0), + page_size: Some(10), + search: Some("WestRedd".to_string()), + client_request_id: Some("leaderboard-req-1".to_string()), + }); + + assert_eq!( + page.query.client_request_id.as_deref(), + Some("leaderboard-req-1") + ); + } +} diff --git a/src/rust/lqosd/src/node_manager/local_api/executive_cache.rs b/src/rust/lqosd/src/node_manager/local_api/executive_cache.rs new file mode 100644 index 00000000..7ccca272 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/executive_cache.rs @@ -0,0 +1,862 @@ +use crate::node_manager::local_api::executive::{ + ExecutiveDashboardMetricRow, ExecutiveDashboardSummary, ExecutiveEntityKind, + ExecutiveHeatmapPageRow, ExecutiveLeaderboardKind, ExecutiveLeaderboardRow, ExecutiveMetric, + ExecutiveOversubscribedSiteRow, ExecutiveRttBlocks, ExecutiveScalarBlocks, + ExecutiveSplitBlocks, ExecutiveTopAsnRow, ExecutiveTreeLocator, +}; +use crate::shaped_devices_tracker::{NETWORK_JSON, SHAPED_DEVICES}; +use crate::throughput_tracker::{ + THROUGHPUT_TRACKER, asn_heatmaps, circuit_heatmaps, executive_summary_header, global_heatmap, + site_heatmaps, +}; +use arc_swap::ArcSwap; +use fxhash::{FxHashMap, FxHashSet}; +use lqos_bus::{ + AsnHeatmapData, BusResponse, CircuitHeatmapData, ExecutiveSummaryHeader, SiteHeatmapData, +}; +use lqos_utils::{ + HeatmapBlocks, + qoq_heatmap::{QoqHeatmapBlocks, TemporalQoqHeatmap}, + temporal_heatmap::TemporalHeatmap, +}; +use once_cell::sync::Lazy; +use parking_lot::Mutex; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const EXECUTIVE_TOP_LIMIT: usize = 10; +const EXECUTIVE_OVERSUBSCRIBED_LIMIT: usize = 20; +const MIN_SAMPLES_FOR_ALERTS: usize = 10; +const DUE_UPGRADE_THRESHOLD: f32 = 80.0; +const COUNT_WEIGHT: f32 = 100.0; +const MIN_HEATMAP_SAMPLES: usize = 3; +const HEATMAP_BLOCK_SECONDS: u64 = 60; +const BITS_PER_MEGABIT: f32 = 1_000_000.0; + +#[derive(Clone, Debug)] +pub(crate) struct ExecutiveEntitySnapshot { + pub row_key: String, + pub entity_kind: ExecutiveEntityKind, + pub label: String, + pub circuit_id: Option, + pub asn: Option, + pub tree: Option, + pub heatmap: HeatmapBlocks, + pub qoq_blocks: Option, +} + +/// Shared once-per-second executive cache used by dashboard summary and detail pages. +#[derive(Clone, Debug)] +pub(crate) struct ExecutiveCacheSnapshot { + pub generated_at_unix_ms: u64, + pub dashboard: ExecutiveDashboardSummary, + pub entities: Vec, + pub leaderboard_rows: FxHashMap>, +} + +impl Default for ExecutiveCacheSnapshot { + fn default() -> Self { + Self { + generated_at_unix_ms: 0, + dashboard: ExecutiveDashboardSummary { + generated_at_unix_ms: 0, + header: ExecutiveSummaryHeader::default(), + global: TemporalHeatmap::new().blocks(), + global_qoq: TemporalQoqHeatmap::new().blocks(), + top_download: Vec::new(), + top_upload: Vec::new(), + top_retransmit: Vec::new(), + top_rtt: Vec::new(), + top_qoo: Vec::new(), + oversubscribed_sites: Vec::new(), + top_asns: Vec::new(), + }, + entities: Vec::new(), + leaderboard_rows: FxHashMap::default(), + } + } +} + +impl ExecutiveCacheSnapshot { + /// Builds transport rows for one requested metric from cached raw executive entities. + pub(crate) fn heatmap_rows_for_metric( + &self, + metric: &ExecutiveMetric, + ) -> Vec { + self.entities + .iter() + .map(|entity| ExecutiveHeatmapPageRow { + row_key: entity.row_key.clone(), + entity_kind: entity.entity_kind.clone(), + label: entity.label.clone(), + circuit_id: entity.circuit_id.clone(), + asn: entity.asn, + tree: entity.tree.clone(), + scalar_blocks: heatmap_scalar_blocks(metric, &entity.heatmap), + split_blocks: match metric { + ExecutiveMetric::Retransmit => Some(ExecutiveSplitBlocks { + download: array_to_vec(&entity.heatmap.retransmit_down), + upload: array_to_vec(&entity.heatmap.retransmit_up), + }), + ExecutiveMetric::Qoo => entity.qoq_blocks.as_ref().map(qoq_to_split_blocks), + ExecutiveMetric::Download | ExecutiveMetric::Upload => { + Some(ExecutiveSplitBlocks { + download: array_to_vec(&entity.heatmap.download), + upload: array_to_vec(&entity.heatmap.upload), + }) + } + ExecutiveMetric::Rtt => None, + }, + rtt_blocks: match metric { + ExecutiveMetric::Rtt => Some(heatmap_to_rtt_blocks(&entity.heatmap)), + _ => None, + }, + }) + .collect() + } +} + +static EXECUTIVE_CACHE_SNAPSHOT: Lazy> = + Lazy::new(|| ArcSwap::new(Arc::new(ExecutiveCacheSnapshot::default()))); +static EXECUTIVE_CACHE_LAST_REFRESH_SECS: AtomicU64 = AtomicU64::new(0); +static EXECUTIVE_CACHE_REFRESH_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); + +fn current_unix_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0) +} + +fn current_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or(0) +} + +fn array_to_vec(values: &[Option; N]) -> Vec> { + values.to_vec() +} + +fn heatmap_scalar_blocks( + metric: &ExecutiveMetric, + heatmap: &HeatmapBlocks, +) -> Option { + let values = match metric { + ExecutiveMetric::Download => Some(array_to_vec(&heatmap.download)), + ExecutiveMetric::Upload => Some(array_to_vec(&heatmap.upload)), + ExecutiveMetric::Retransmit => Some(array_to_vec(&heatmap.retransmit)), + ExecutiveMetric::Rtt => Some(array_to_vec(&heatmap.rtt)), + ExecutiveMetric::Qoo => None, + }?; + Some(ExecutiveScalarBlocks { values }) +} + +fn qoq_to_split_blocks(blocks: &QoqHeatmapBlocks) -> ExecutiveSplitBlocks { + ExecutiveSplitBlocks { + download: array_to_vec(&blocks.download_total), + upload: array_to_vec(&blocks.upload_total), + } +} + +fn heatmap_to_rtt_blocks(blocks: &HeatmapBlocks) -> ExecutiveRttBlocks { + ExecutiveRttBlocks { + rtt: array_to_vec(&blocks.rtt), + dl_p50: array_to_vec(&blocks.rtt_p50_down), + dl_p90: array_to_vec(&blocks.rtt_p90_down), + ul_p50: array_to_vec(&blocks.rtt_p50_up), + ul_p90: array_to_vec(&blocks.rtt_p90_up), + } +} + +fn latest_value(values: &[Option]) -> Option { + values.iter().rev().flatten().copied().next() +} + +fn non_null_count(values: &[Option]) -> usize { + values.iter().filter(|value| value.is_some()).count() +} + +fn latest_qoo(blocks: Option<&QoqHeatmapBlocks>) -> Option { + let blocks = blocks?; + let mut present = Vec::new(); + if let Some(download) = latest_value(&blocks.download_total) { + present.push(download); + } + if let Some(upload) = latest_value(&blocks.upload_total) { + present.push(upload); + } + if present.is_empty() { + None + } else { + Some(present.iter().sum::() / present.len() as f32) + } +} + +fn qoo_sample_count(blocks: Option<&QoqHeatmapBlocks>) -> usize { + let Some(blocks) = blocks else { + return 0; + }; + non_null_count(&blocks.download_total).max(non_null_count(&blocks.upload_total)) +} + +fn sort_score_for_metric(entity: &ExecutiveEntitySnapshot, metric: &ExecutiveMetric) -> f32 { + match metric { + ExecutiveMetric::Download => { + latest_value(&entity.heatmap.download).unwrap_or(f32::NEG_INFINITY) + } + ExecutiveMetric::Upload => { + latest_value(&entity.heatmap.upload).unwrap_or(f32::NEG_INFINITY) + } + ExecutiveMetric::Retransmit => { + let latest = latest_value(&entity.heatmap.retransmit).unwrap_or(f32::NEG_INFINITY); + let count = non_null_count(&entity.heatmap.retransmit); + let penalty = if count < MIN_HEATMAP_SAMPLES { + 1000.0 + } else { + 0.0 + }; + latest + COUNT_WEIGHT * (count as f32 / 15.0) - penalty + } + ExecutiveMetric::Rtt => { + let latest = latest_value(&entity.heatmap.rtt).unwrap_or(f32::NEG_INFINITY); + let count = non_null_count(&entity.heatmap.rtt); + let penalty = if count < MIN_HEATMAP_SAMPLES { + 1000.0 + } else { + 0.0 + }; + latest + COUNT_WEIGHT * (count as f32 / 15.0) - penalty + } + ExecutiveMetric::Qoo => { + let latest = latest_qoo(entity.qoq_blocks.as_ref()).unwrap_or(f32::INFINITY); + let count = qoo_sample_count(entity.qoq_blocks.as_ref()); + let penalty = if count < MIN_HEATMAP_SAMPLES { + 1000.0 + } else { + 0.0 + }; + -latest + COUNT_WEIGHT * (count as f32 / 15.0) - penalty + } + } +} + +fn metric_row_from_entity( + entity: &ExecutiveEntitySnapshot, + metric: ExecutiveMetric, +) -> ExecutiveDashboardMetricRow { + ExecutiveDashboardMetricRow { + row_key: entity.row_key.clone(), + entity_kind: entity.entity_kind.clone(), + metric: metric.clone(), + label: entity.label.clone(), + circuit_id: entity.circuit_id.clone(), + asn: entity.asn, + tree: entity.tree.clone(), + scalar_blocks: heatmap_scalar_blocks(&metric, &entity.heatmap), + split_blocks: match metric { + ExecutiveMetric::Download | ExecutiveMetric::Upload => Some(ExecutiveSplitBlocks { + download: array_to_vec(&entity.heatmap.download), + upload: array_to_vec(&entity.heatmap.upload), + }), + ExecutiveMetric::Retransmit => Some(ExecutiveSplitBlocks { + download: array_to_vec(&entity.heatmap.retransmit_down), + upload: array_to_vec(&entity.heatmap.retransmit_up), + }), + ExecutiveMetric::Qoo => entity.qoq_blocks.as_ref().map(qoq_to_split_blocks), + ExecutiveMetric::Rtt => None, + }, + rtt_blocks: match metric { + ExecutiveMetric::Rtt => Some(heatmap_to_rtt_blocks(&entity.heatmap)), + _ => None, + }, + } +} + +fn top_metric_rows( + entities: &[ExecutiveEntitySnapshot], + metric: ExecutiveMetric, + limit: usize, +) -> Vec { + let mut ranked = entities + .iter() + .filter(|entity| { + matches!( + entity.entity_kind, + ExecutiveEntityKind::Site | ExecutiveEntityKind::Circuit + ) + }) + .cloned() + .collect::>(); + ranked.sort_by(|left, right| { + sort_score_for_metric(right, &metric) + .partial_cmp(&sort_score_for_metric(left, &metric)) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| left.label.cmp(&right.label)) + }); + ranked + .into_iter() + .take(limit) + .map(|entity| metric_row_from_entity(&entity, metric.clone())) + .collect() +} + +fn total_bytes_from_heatmap(blocks: &HeatmapBlocks) -> u64 { + let total_mbps_minutes: f32 = blocks + .download + .iter() + .chain(blocks.upload.iter()) + .flatten() + .copied() + .sum(); + ((total_mbps_minutes * BITS_PER_MEGABIT / 8.0) * HEATMAP_BLOCK_SECONDS as f32) as u64 +} + +fn median_value(values: &[Option]) -> Option { + let mut numeric = values.iter().flatten().copied().collect::>(); + if numeric.is_empty() { + return None; + } + numeric.sort_by(|left, right| left.total_cmp(right)); + let midpoint = numeric.len() / 2; + if numeric.len() % 2 == 1 { + Some(numeric[midpoint]) + } else { + Some((numeric[midpoint - 1] + numeric[midpoint]) / 2.0) + } +} + +fn average_with_count(values: &[Option]) -> (Option, usize) { + let numeric = values.iter().flatten().copied().collect::>(); + if numeric.is_empty() { + return (None, 0); + } + let sum = numeric.iter().sum::(); + (Some(sum / numeric.len() as f32), numeric.len()) +} + +fn build_top_asn_rows(entities: &[ExecutiveEntitySnapshot]) -> Vec { + let mut rows = entities + .iter() + .filter(|entity| entity.entity_kind == ExecutiveEntityKind::Asn) + .map(|entity| ExecutiveTopAsnRow { + row_key: entity.row_key.clone(), + asn: entity.asn.unwrap_or_default(), + asn_name: match entity.label.as_str() { + label if label.starts_with("ASN ") => None, + label => Some(label.to_string()), + }, + total_bytes_15m: total_bytes_from_heatmap(&entity.heatmap), + median_rtt_ms: median_value(&entity.heatmap.rtt), + median_retransmit_pct: median_value(&entity.heatmap.retransmit), + }) + .collect::>(); + rows.sort_by(|left, right| right.total_bytes_15m.cmp(&left.total_bytes_15m)); + rows.truncate(EXECUTIVE_TOP_LIMIT); + rows +} + +fn build_leaderboard_rows( + entities: &[ExecutiveEntitySnapshot], + oversubscribed_sites: &[ExecutiveOversubscribedSiteRow], +) -> FxHashMap> { + let mut rows = FxHashMap::default(); + + let mut worst_sites = entities + .iter() + .filter(|entity| entity.entity_kind == ExecutiveEntityKind::Site) + .filter_map(|entity| { + let median_rtt_ms = median_value(&entity.heatmap.rtt)?; + let (avg_down_util, _) = average_with_count(&entity.heatmap.download); + let (avg_up_util, _) = average_with_count(&entity.heatmap.upload); + Some(ExecutiveLeaderboardRow::WorstSiteByRtt { + row_key: entity.row_key.clone(), + site_name: entity.label.clone(), + tree: entity.tree.clone(), + median_rtt_ms, + avg_down_util, + avg_up_util, + }) + }) + .collect::>(); + worst_sites.sort_by(|left, right| match (left, right) { + ( + ExecutiveLeaderboardRow::WorstSiteByRtt { + median_rtt_ms: left, + .. + }, + ExecutiveLeaderboardRow::WorstSiteByRtt { + median_rtt_ms: right, + .. + }, + ) => right.total_cmp(left), + _ => std::cmp::Ordering::Equal, + }); + rows.insert(ExecutiveLeaderboardKind::WorstSitesByRtt, worst_sites); + + rows.insert( + ExecutiveLeaderboardKind::OversubscribedSites, + oversubscribed_sites + .iter() + .cloned() + .map(|row| ExecutiveLeaderboardRow::OversubscribedSite { + row_key: row.row_key, + site_name: row.site_name, + tree: row.tree, + ratio_down: row.ratio_down, + ratio_up: row.ratio_up, + ratio_max: row.ratio_max, + cap_down: row.cap_down, + cap_up: row.cap_up, + sub_down: row.sub_down, + sub_up: row.sub_up, + avg_down_util: row.avg_down_util, + avg_up_util: row.avg_up_util, + median_rtt_ms: row.median_rtt_ms, + }) + .collect(), + ); + + let mut sites_due_upgrade = entities + .iter() + .filter(|entity| entity.entity_kind == ExecutiveEntityKind::Site) + .filter_map(|entity| { + let (avg_down_util, down_count) = average_with_count(&entity.heatmap.download); + let (avg_up_util, up_count) = average_with_count(&entity.heatmap.upload); + let (Some(avg_down_util), Some(avg_up_util)) = (avg_down_util, avg_up_util) else { + return None; + }; + if down_count < MIN_SAMPLES_FOR_ALERTS + || up_count < MIN_SAMPLES_FOR_ALERTS + || avg_down_util < DUE_UPGRADE_THRESHOLD + || avg_up_util < DUE_UPGRADE_THRESHOLD + { + return None; + } + Some(ExecutiveLeaderboardRow::SiteDueUpgrade { + row_key: entity.row_key.clone(), + site_name: entity.label.clone(), + tree: entity.tree.clone(), + avg_down_util, + avg_up_util, + }) + }) + .collect::>(); + sites_due_upgrade.sort_by(|left, right| match (left, right) { + ( + ExecutiveLeaderboardRow::SiteDueUpgrade { + avg_down_util: left_down, + avg_up_util: left_up, + .. + }, + ExecutiveLeaderboardRow::SiteDueUpgrade { + avg_down_util: right_down, + avg_up_util: right_up, + .. + }, + ) => (right_down + right_up) + .partial_cmp(&(left_down + left_up)) + .unwrap_or(std::cmp::Ordering::Equal), + _ => std::cmp::Ordering::Equal, + }); + rows.insert(ExecutiveLeaderboardKind::SitesDueUpgrade, sites_due_upgrade); + + let mut circuits_due_upgrade = entities + .iter() + .filter(|entity| entity.entity_kind == ExecutiveEntityKind::Circuit) + .filter_map(|entity| { + let (avg_down_util, down_count) = average_with_count(&entity.heatmap.download); + let (avg_up_util, up_count) = average_with_count(&entity.heatmap.upload); + let (Some(avg_down_util), Some(avg_up_util)) = (avg_down_util, avg_up_util) else { + return None; + }; + if down_count < MIN_SAMPLES_FOR_ALERTS + || up_count < MIN_SAMPLES_FOR_ALERTS + || avg_down_util < DUE_UPGRADE_THRESHOLD + || avg_up_util < DUE_UPGRADE_THRESHOLD + { + return None; + } + Some(ExecutiveLeaderboardRow::CircuitDueUpgrade { + row_key: entity.row_key.clone(), + circuit_id: entity.circuit_id.clone().unwrap_or_default(), + circuit_name: entity.label.clone(), + avg_down_util, + avg_up_util, + }) + }) + .collect::>(); + circuits_due_upgrade.sort_by(|left, right| match (left, right) { + ( + ExecutiveLeaderboardRow::CircuitDueUpgrade { + avg_down_util: left_down, + avg_up_util: left_up, + .. + }, + ExecutiveLeaderboardRow::CircuitDueUpgrade { + avg_down_util: right_down, + avg_up_util: right_up, + .. + }, + ) => (right_down + right_up) + .partial_cmp(&(left_down + left_up)) + .unwrap_or(std::cmp::Ordering::Equal), + _ => std::cmp::Ordering::Equal, + }); + rows.insert( + ExecutiveLeaderboardKind::CircuitsDueUpgrade, + circuits_due_upgrade, + ); + + let mut top_asns = build_top_asn_rows(entities) + .into_iter() + .map(|row| ExecutiveLeaderboardRow::TopAsnByTraffic { + row_key: row.row_key, + asn: row.asn, + asn_name: row.asn_name, + total_bytes_15m: row.total_bytes_15m, + median_rtt_ms: row.median_rtt_ms, + median_retransmit_pct: row.median_retransmit_pct, + }) + .collect::>(); + top_asns.sort_by(|left, right| match (left, right) { + ( + ExecutiveLeaderboardRow::TopAsnByTraffic { + total_bytes_15m: left, + .. + }, + ExecutiveLeaderboardRow::TopAsnByTraffic { + total_bytes_15m: right, + .. + }, + ) => right.cmp(left), + _ => std::cmp::Ordering::Equal, + }); + rows.insert(ExecutiveLeaderboardKind::TopAsnsByTraffic, top_asns); + + rows +} + +fn heatmap_blocks_from_response(response: BusResponse) -> HeatmapBlocks { + match response { + BusResponse::GlobalHeatmap(blocks) => blocks, + _ => TemporalHeatmap::new().blocks(), + } +} + +fn header_from_response(response: BusResponse) -> ExecutiveSummaryHeader { + match response { + BusResponse::ExecutiveSummaryHeader(header) => header, + _ => ExecutiveSummaryHeader::default(), + } +} + +fn circuit_heatmap_rows() -> Vec { + match circuit_heatmaps() { + BusResponse::CircuitHeatmaps(rows) => rows, + _ => Vec::new(), + } +} + +fn site_heatmap_rows() -> Vec { + match site_heatmaps() { + BusResponse::SiteHeatmaps(rows) => rows, + _ => Vec::new(), + } +} + +fn asn_heatmap_rows() -> Vec { + match asn_heatmaps() { + BusResponse::AsnHeatmaps(rows) => rows, + _ => Vec::new(), + } +} + +fn node_path_names(idx: usize, nodes: &[lqos_config::NetworkJsonNode]) -> Option> { + let node = nodes.get(idx)?; + if node.name == "Root" { + return Some(vec!["Root".to_string()]); + } + let parent_indexes = if node.parents.is_empty() { + vec![node.immediate_parent.unwrap_or(0)] + } else { + node.parents.clone() + }; + let mut names = Vec::new(); + for parent_idx in parent_indexes { + names.push(nodes.get(parent_idx)?.name.clone()); + } + if names.last() != Some(&node.name) { + names.push(node.name.clone()); + } + Some(names) +} + +fn node_locator_map() -> FxHashMap { + let reader = NETWORK_JSON.read(); + let nodes = reader.get_nodes_when_ready(); + let mut locators = FxHashMap::default(); + for (idx, node) in nodes.iter().enumerate() { + if node.name == "Root" || locators.contains_key(&node.name) { + continue; + } + locators.insert( + node.name.clone(), + ExecutiveTreeLocator { + node_id: node.id.clone(), + node_path: node_path_names(idx, nodes), + parent_index: Some(idx), + }, + ); + } + locators +} + +fn entity_snapshots( + circuits: Vec, + sites: Vec, + asns: Vec, +) -> Vec { + let locators = node_locator_map(); + let mut rows = Vec::new(); + for site in sites { + let row_key = locators + .get(&site.site_name) + .and_then(|locator| { + locator + .node_id + .as_ref() + .map(|node_id| format!("site:{node_id}")) + .or_else(|| { + locator + .node_path + .as_ref() + .map(|path| format!("site:{:?}", path)) + }) + }) + .unwrap_or_else(|| format!("site:{}", site.site_name)); + rows.push(ExecutiveEntitySnapshot { + row_key, + entity_kind: ExecutiveEntityKind::Site, + label: site.site_name.clone(), + circuit_id: None, + asn: None, + tree: locators.get(&site.site_name).cloned(), + heatmap: site.blocks, + qoq_blocks: site.qoq_blocks, + }); + } + for circuit in circuits { + let label = if circuit.circuit_name.trim().is_empty() { + circuit.circuit_id.clone() + } else { + circuit.circuit_name.clone() + }; + rows.push(ExecutiveEntitySnapshot { + row_key: format!("circuit:{}", circuit.circuit_id), + entity_kind: ExecutiveEntityKind::Circuit, + label, + circuit_id: Some(circuit.circuit_id), + asn: None, + tree: None, + heatmap: circuit.blocks, + qoq_blocks: circuit.qoq_blocks, + }); + } + for asn in asns { + rows.push(ExecutiveEntitySnapshot { + row_key: format!("asn:{}", asn.asn), + entity_kind: ExecutiveEntityKind::Asn, + label: asn + .asn_name + .clone() + .unwrap_or_else(|| format!("ASN {}", asn.asn)), + circuit_id: None, + asn: Some(asn.asn), + tree: None, + heatmap: asn.blocks, + qoq_blocks: None, + }); + } + rows +} + +fn build_oversubscribed_sites( + entities: &[ExecutiveEntitySnapshot], +) -> Vec { + let devices = SHAPED_DEVICES.load(); + let mut circuit_map: std::collections::HashMap = + std::collections::HashMap::new(); + for device in &devices.devices { + let entry = circuit_map + .entry(device.circuit_id.clone()) + .or_insert_with(|| (device.parent_node.clone(), 0.0_f32, 0.0_f32)); + entry.1 = entry.1.max(device.download_max_mbps); + entry.2 = entry.2.max(device.upload_max_mbps); + } + + let reader = NETWORK_JSON.read(); + let nodes = reader.get_nodes_when_ready(); + if nodes.is_empty() { + return Vec::new(); + } + + let mut children: Vec> = vec![Vec::new(); nodes.len()]; + for (idx, node) in nodes.iter().enumerate() { + if let Some(parent) = node.immediate_parent + && parent < children.len() + { + children[parent].push(idx); + } + } + + let entity_map = entities + .iter() + .filter(|entity| entity.entity_kind == ExecutiveEntityKind::Site) + .map(|entity| (entity.label.clone(), entity)) + .collect::>(); + + let mut results = Vec::new(); + for (idx, node) in nodes.iter().enumerate() { + if node.name == "Root" { + continue; + } + let node_type = node.node_type.as_deref().unwrap_or("").to_ascii_lowercase(); + if node_type != "site" && !node_type.is_empty() && node_type != "ap" { + continue; + } + + let mut stack = vec![idx]; + let mut visited = FxHashSet::default(); + let mut descendant_names = FxHashSet::default(); + while let Some(next_idx) = stack.pop() { + if !visited.insert(next_idx) { + continue; + } + if let Some(name) = nodes.get(next_idx).map(|entry| entry.name.clone()) { + descendant_names.insert(name); + } + if let Some(child_nodes) = children.get(next_idx) { + for child in child_nodes { + stack.push(*child); + } + } + } + + let cap_down = node.max_throughput.0 as f32; + let cap_up = node.max_throughput.1 as f32; + + let mut sub_down = 0.0_f32; + let mut sub_up = 0.0_f32; + for (parent_name, circuit_down, circuit_up) in circuit_map.values() { + if descendant_names.contains(parent_name) { + sub_down += *circuit_down; + sub_up += *circuit_up; + } + } + + let Some(stats) = entity_map.get(&node.name) else { + continue; + }; + let (avg_down_util, _) = average_with_count(&stats.heatmap.download); + let (avg_up_util, _) = average_with_count(&stats.heatmap.upload); + let median_rtt_ms = median_value(&stats.heatmap.rtt); + let ratio_down = (cap_down > 0.0).then_some(sub_down / cap_down); + let ratio_up = (cap_up > 0.0).then_some(sub_up / cap_up); + let ratio_max = match (ratio_down, ratio_up) { + (Some(down), Some(up)) => Some(down.max(up)), + (Some(down), None) => Some(down), + (None, Some(up)) => Some(up), + (None, None) => None, + }; + + results.push(ExecutiveOversubscribedSiteRow { + row_key: stats.row_key.clone(), + site_name: node.name.clone(), + tree: stats.tree.clone(), + ratio_down, + ratio_up, + ratio_max, + cap_down, + cap_up, + sub_down, + sub_up, + avg_down_util, + avg_up_util, + median_rtt_ms, + }); + } + + results.sort_by(|left, right| { + right + .ratio_max + .unwrap_or(0.0) + .total_cmp(&left.ratio_max.unwrap_or(0.0)) + }); + results.truncate(EXECUTIVE_OVERSUBSCRIBED_LIMIT); + results +} + +/// Invalidates the shared executive cache so the next read rebuilds immediately. +pub(crate) fn invalidate_executive_cache_snapshot() { + EXECUTIVE_CACHE_LAST_REFRESH_SECS.store(0, Ordering::Release); +} + +/// Rebuilds the once-per-second executive cache snapshot. +pub(crate) fn rebuild_executive_cache_snapshot() -> Arc { + let generated_at_unix_ms = current_unix_ms(); + let header = header_from_response(executive_summary_header()); + let global = heatmap_blocks_from_response(global_heatmap()); + let global_qoq = THROUGHPUT_TRACKER.global_qoq_heatmap.lock().blocks(); + let entities = entity_snapshots( + circuit_heatmap_rows(), + site_heatmap_rows(), + asn_heatmap_rows(), + ); + let oversubscribed_sites = build_oversubscribed_sites(&entities); + let top_download = top_metric_rows(&entities, ExecutiveMetric::Download, EXECUTIVE_TOP_LIMIT); + let top_upload = top_metric_rows(&entities, ExecutiveMetric::Upload, EXECUTIVE_TOP_LIMIT); + let top_retransmit = + top_metric_rows(&entities, ExecutiveMetric::Retransmit, EXECUTIVE_TOP_LIMIT); + let top_rtt = top_metric_rows(&entities, ExecutiveMetric::Rtt, EXECUTIVE_TOP_LIMIT); + let top_qoo = top_metric_rows(&entities, ExecutiveMetric::Qoo, EXECUTIVE_TOP_LIMIT); + let top_asns = build_top_asn_rows(&entities); + let leaderboard_rows = build_leaderboard_rows(&entities, &oversubscribed_sites); + + let snapshot = Arc::new(ExecutiveCacheSnapshot { + generated_at_unix_ms, + dashboard: ExecutiveDashboardSummary { + generated_at_unix_ms, + header, + global, + global_qoq, + top_download, + top_upload, + top_retransmit, + top_rtt, + top_qoo, + oversubscribed_sites, + top_asns, + }, + entities, + leaderboard_rows, + }); + EXECUTIVE_CACHE_SNAPSHOT.store(snapshot.clone()); + EXECUTIVE_CACHE_LAST_REFRESH_SECS.store(current_epoch_secs(), Ordering::Release); + snapshot +} + +/// Returns the current executive cache snapshot, rebuilding it if stale. +pub(crate) fn fresh_executive_cache_snapshot() -> Arc { + let now_secs = current_epoch_secs(); + if EXECUTIVE_CACHE_LAST_REFRESH_SECS.load(Ordering::Acquire) == now_secs { + return EXECUTIVE_CACHE_SNAPSHOT.load_full(); + } + let _guard = EXECUTIVE_CACHE_REFRESH_LOCK.lock(); + if EXECUTIVE_CACHE_LAST_REFRESH_SECS.load(Ordering::Acquire) == now_secs { + return EXECUTIVE_CACHE_SNAPSHOT.load_full(); + } + rebuild_executive_cache_snapshot() +} diff --git a/src/rust/lqosd/src/node_manager/local_api/lts.rs b/src/rust/lqosd/src/node_manager/local_api/lts.rs index 9d12686f..89c690bf 100644 --- a/src/rust/lqosd/src/node_manager/local_api/lts.rs +++ b/src/rust/lqosd/src/node_manager/local_api/lts.rs @@ -1,8 +1,10 @@ mod last_24_hours; mod shaper_status; +use crate::lts2_sys::lts2_client::{LicenseStatus, set_license_status}; use crate::lts2_sys::shared_types::LtsStatus; use crate::node_manager::auth::LoginResult; +use crate::node_manager::local_api::circuit_count; use axum::http::StatusCode; pub use last_24_hours::*; use lqos_bus::{BusRequest, bus_request}; @@ -10,9 +12,14 @@ use lqos_config::load_config; use serde::{Deserialize, Serialize}; pub use shaper_status::ShaperStatus; pub use shaper_status::shaper_status_data; -use std::ops::Deref; use std::process::Command; -use tracing::{info, warn}; +use std::time::Duration; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +const SIGNUP_POLL_INTERVAL: Duration = Duration::from_secs(10); +const INSIGHT_FREE_TRIAL_STATUS_CODE: i32 = 2; +const INSIGHT_TRIAL_DAYS_HINT: i32 = 30; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct LtsTrialConfig { @@ -20,6 +27,37 @@ pub struct LtsTrialConfig { pub lts_url: Option, } +#[derive(Debug, Serialize)] +struct SignupStartRequest { + node_id: String, + node_name: String, + active_circuits: usize, +} + +#[derive(Debug, Deserialize)] +struct SignupStartResponse { + claim_id: Uuid, +} + +#[derive(Debug, Serialize)] +struct SignupCheckRequest { + claim_id: String, +} + +#[derive(Debug, Deserialize)] +struct SignupCheckResponse { + status: String, + #[serde(default)] + account_key: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SignupCheckState { + Pending, + Provisioned { account_key: Uuid }, + Expired, +} + pub fn lts_trial_config_data(login: LoginResult) -> Result { if login != LoginResult::Admin { return Err(StatusCode::FORBIDDEN); @@ -31,6 +69,221 @@ pub fn lts_trial_config_data(login: LoginResult) -> Result) -> String { + let mut base_url = lts_url + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or("https://insight.libreqos.com/") + .to_string(); + + if !base_url.starts_with("http://") && !base_url.starts_with("https://") { + base_url = format!("https://{base_url}"); + } + + while base_url.ends_with('/') { + base_url.pop(); + } + + if let Some(stripped) = base_url.strip_suffix("/signup-api") { + base_url = stripped.to_string(); + } + + format!("{base_url}/") +} + +fn build_insight_url(lts_url: Option<&str>, path: &str) -> String { + let path = path.trim_start_matches('/'); + format!("{}{}", normalize_insight_base_url(lts_url), path) +} + +fn insight_http_client() -> Result { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(15)) + .danger_accept_invalid_certs(true) + .build() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) +} + +fn classify_signup_check_response(response: SignupCheckResponse) -> Result { + match response.status.as_str() { + "pending" => Ok(SignupCheckState::Pending), + "expired" => Ok(SignupCheckState::Expired), + "provisioned" => response + .account_key + .map(|account_key| SignupCheckState::Provisioned { account_key }) + .ok_or(()), + _ => Err(()), + } +} + +async fn apply_insight_license(license_key: String, restart_lqos_api: bool) -> Result<(), StatusCode> { + let license_key = license_key.trim().to_string(); + if license_key.is_empty() { + return Err(StatusCode::BAD_GATEWAY); + } + + let mut cfg = load_config() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? + .as_ref() + .clone(); + cfg.long_term_stats.gather_stats = true; + cfg.long_term_stats.license_key = Some(license_key.clone()); + bus_request(vec![BusRequest::UpdateLqosdConfig(Box::new(cfg))]) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + set_license_status(LicenseStatus { + license_type: INSIGHT_FREE_TRIAL_STATUS_CODE, + trial_expires: INSIGHT_TRIAL_DAYS_HINT, + }); + + info!("LQOSD configuration updated with new Insight license key."); + + if restart_lqos_api { + let _ = Command::new("/bin/systemctl") + .args(["restart", "lqos_api"]) + .output(); + } + + Ok(()) +} + +pub async fn lts_trial_start_signup_data() -> Result { + let config = load_config().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let start_url = build_insight_url(config.long_term_stats.lts_url.as_deref(), "/su/start"); + let request = SignupStartRequest { + node_id: config.node_id.clone(), + node_name: config.node_name.clone(), + active_circuits: circuit_count::circuit_count_data().count, + }; + debug!( + node_id = %request.node_id, + active_circuits = request.active_circuits, + url = %start_url, + "starting Insight signup session" + ); + + let client = insight_http_client()?; + + let response = client + .post(&start_url) + .json(&request) + .send() + .await + .map_err(|_| StatusCode::BAD_GATEWAY)?; + + if !response.status().is_success() { + warn!( + status = %response.status(), + url = %start_url, + "Insight signup start request failed" + ); + return Err(StatusCode::BAD_GATEWAY); + } + + let response = response + .json::() + .await + .map_err(|_| StatusCode::BAD_GATEWAY)?; + + Ok(response.claim_id.to_string()) +} + +pub fn spawn_signup_poll_loop(claim_id: String) { + tokio::spawn(async move { + if let Err(status) = poll_signup_until_complete(claim_id.clone()).await { + warn!( + claim_id = %claim_id, + status = %status, + "Insight signup poll loop stopped with an error" + ); + } + }); +} + +async fn poll_signup_until_complete(claim_id: String) -> Result<(), StatusCode> { + let config = load_config().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let check_url = build_insight_url(config.long_term_stats.lts_url.as_deref(), "/su/check"); + let client = insight_http_client()?; + let request = SignupCheckRequest { + claim_id: claim_id.clone(), + }; + + loop { + tokio::time::sleep(SIGNUP_POLL_INTERVAL).await; + + let response = match client.post(&check_url).json(&request).send().await { + Ok(response) => response, + Err(error) => { + warn!( + claim_id = %claim_id, + error = ?error, + "Insight signup status poll failed; retrying" + ); + continue; + } + }; + + if !response.status().is_success() { + warn!( + claim_id = %claim_id, + status = %response.status(), + "Insight signup status poll returned non-success; retrying" + ); + continue; + } + + let response = match response.json::().await { + Ok(response) => response, + Err(error) => { + warn!( + claim_id = %claim_id, + error = ?error, + "Insight signup status poll returned invalid JSON; retrying" + ); + continue; + } + }; + + match classify_signup_check_response(response) { + Ok(SignupCheckState::Pending) => {} + Ok(SignupCheckState::Expired) => { + debug!(claim_id = %claim_id, "Insight signup claim expired"); + return Ok(()); + } + Ok(SignupCheckState::Provisioned { account_key }) => { + match apply_insight_license(account_key.to_string(), false).await { + Ok(()) => { + info!( + claim_id = %claim_id, + account_key = %account_key, + "Applied Insight license from signup flow" + ); + return Ok(()); + } + Err(status) => { + warn!( + claim_id = %claim_id, + account_key = %account_key, + status = %status, + "Failed to apply Insight license; stopping signup poll loop" + ); + return Err(status); + } + } + } + Err(()) => { + warn!( + claim_id = %claim_id, + "Insight signup status poll returned an unexpected payload" + ); + return Err(StatusCode::BAD_GATEWAY); + } + } + } +} + pub(crate) async fn insight_gate() -> Result<(), StatusCode> { let (status, _) = crate::lts2_sys::get_lts_license_status_async().await; match status { @@ -56,20 +309,83 @@ pub async fn lts_trial_signup_data(license_key: String) -> Result<(), StatusCode return Err(StatusCode::INTERNAL_SERVER_ERROR); } info!("Free trial request succeeded, license key: {}", license_key); - let mut cfg = load_config() - .expect("Unable to load LibreQoS config") - .deref() - .clone(); - cfg.long_term_stats.gather_stats = true; - cfg.long_term_stats.license_key = Some(license_key); - bus_request(vec![BusRequest::UpdateLqosdConfig(Box::new(cfg))]) - .await - .expect("Unable to update lqosd config"); - info!("LQOSD configuration updated with new license key."); - // Best-effort: ensure the bundled lqos_api also reloads to pick up the new license - // Ignore errors if systemctl isn't present or permission is denied. - let _ = Command::new("/bin/systemctl") - .args(["restart", "lqos_api"]) - .output(); - Ok(()) + apply_insight_license(license_key, true).await +} + +#[cfg(test)] +mod tests { + use super::{ + SignupCheckResponse, SignupCheckState, build_insight_url, classify_signup_check_response, + normalize_insight_base_url, + }; + use uuid::Uuid; + + #[test] + fn normalize_insight_base_url_defaults_to_hosted_insight() { + assert_eq!( + normalize_insight_base_url(None), + "https://insight.libreqos.com/" + ); + } + + #[test] + fn normalize_insight_base_url_adds_scheme_and_trailing_slash() { + assert_eq!( + normalize_insight_base_url(Some("insight.example.com")), + "https://insight.example.com/" + ); + } + + #[test] + fn normalize_insight_base_url_strips_signup_api_suffix() { + assert_eq!( + normalize_insight_base_url(Some("https://insight.example.com/signup-api/")), + "https://insight.example.com/" + ); + } + + #[test] + fn build_insight_url_joins_paths_cleanly() { + assert_eq!( + build_insight_url(Some("insight.example.com"), "/su/start"), + "https://insight.example.com/su/start" + ); + } + + #[test] + fn classify_signup_check_response_handles_pending() { + let response = SignupCheckResponse { + status: "pending".to_string(), + account_key: None, + }; + + assert_eq!( + classify_signup_check_response(response), + Ok(SignupCheckState::Pending) + ); + } + + #[test] + fn classify_signup_check_response_requires_account_key_for_provisioned() { + let response = SignupCheckResponse { + status: "provisioned".to_string(), + account_key: None, + }; + + assert!(classify_signup_check_response(response).is_err()); + } + + #[test] + fn classify_signup_check_response_returns_provisioned_account_key() { + let account_key = Uuid::new_v4(); + let response = SignupCheckResponse { + status: "provisioned".to_string(), + account_key: Some(account_key), + }; + + assert_eq!( + classify_signup_check_response(response), + Ok(SignupCheckState::Provisioned { account_key }) + ); + } } diff --git a/src/rust/lqosd/src/node_manager/local_api/network_tree.rs b/src/rust/lqosd/src/node_manager/local_api/network_tree.rs index 38d8a4ea..bab10c27 100644 --- a/src/rust/lqosd/src/node_manager/local_api/network_tree.rs +++ b/src/rust/lqosd/src/node_manager/local_api/network_tree.rs @@ -1,12 +1,7 @@ -use crate::shaped_devices_tracker::{NETWORK_JSON, node_to_transport}; +use crate::shaped_devices_tracker::full_network_map_snapshot; use lqos_config::NetworkJsonTransport; +/// Returns the current full network tree snapshot for websocket/API consumers. pub fn network_tree_data() -> Vec<(usize, NetworkJsonTransport)> { - let net_json = NETWORK_JSON.read(); - net_json - .get_nodes_when_ready() - .iter() - .enumerate() - .map(|(i, n)| (i, node_to_transport(n))) - .collect() + full_network_map_snapshot() } diff --git a/src/rust/lqosd/src/node_manager/local_api/network_tree_lite.rs b/src/rust/lqosd/src/node_manager/local_api/network_tree_lite.rs new file mode 100644 index 00000000..075b6aaf --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/network_tree_lite.rs @@ -0,0 +1,52 @@ +use crate::shaped_devices_tracker::full_network_map_lite_snapshot; +use serde::{Deserialize, Serialize}; + +/// Minimal live tree payload for pages that do not need the full `NetworkJsonTransport`. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NetworkTreeLiteNode { + /// Display name for the node. + pub name: String, + /// Optional stable node identifier from `network.json` metadata. + #[serde(default)] + pub id: Option, + /// True if the node is virtual/logical-only. + #[serde(rename = "virtual", default)] + pub is_virtual: bool, + /// True if TreeGuard has runtime-virtualized this node in Bakery. + #[serde(default)] + pub runtime_virtualized: bool, + /// Configured maximum throughput in Mbps. + pub max_throughput: (f64, f64), + /// Current throughput in bytes per second. + pub current_throughput: (u64, u64), + /// Current TCP packets. + pub current_tcp_packets: (u64, u64), + /// TCP packets paired with the current retransmit counts. + #[serde(default)] + pub current_tcp_retransmit_packets: (u64, u64), + /// Current TCP retransmits. + pub current_retransmits: (u64, u64), + /// Approximate current RTT medians for down/up, in milliseconds. + pub rtts: Vec, + /// QoO score for download/upload directions. + #[serde(default)] + pub qoo: (Option, Option), + /// Parent node indexes. + pub parents: Vec, + /// Immediate parent node index. + pub immediate_parent: Option, + /// Optional node type metadata from `network.json`. + #[serde(rename = "type")] + pub node_type: Option, + /// Optional geographic latitude. + #[serde(default)] + pub latitude: Option, + /// Optional geographic longitude. + #[serde(default)] + pub longitude: Option, +} + +/// Returns the current lightweight network tree snapshot for websocket/API consumers. +pub fn network_tree_lite_data() -> Vec<(usize, NetworkTreeLiteNode)> { + full_network_map_lite_snapshot() +} diff --git a/src/rust/lqosd/src/node_manager/local_api/node_rate_overrides.rs b/src/rust/lqosd/src/node_manager/local_api/node_rate_overrides.rs new file mode 100644 index 00000000..370f0b13 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/node_rate_overrides.rs @@ -0,0 +1,292 @@ +use crate::node_manager::auth::LoginResult; +use axum::http::StatusCode; +use lqos_config::load_config; +use lqos_overrides::{NetworkAdjustment, OverrideLayer, OverrideStore}; +use serde::{Deserialize, Serialize}; +use std::path::Path; + +const GENERATED_NODE_ID_PREFIX: &str = "libreqos:generated:"; +const GENERATED_NODE_NAME_PREFIX: &str = "(Generated Site) "; +const LEGACY_UISP_FILE: &str = "integrationUISPbandwidths.csv"; +const LEGACY_SPLYNX_FILE: &str = "integrationSplynxBandwidths.csv"; + +/// Query payload for inspecting the operator rate override state of a tree node. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NodeRateOverrideQuery { + /// Stable node identifier from `network.json`, when available. + pub node_id: Option, + /// Display name of the selected node. + pub node_name: String, +} + +/// Update payload for persisting an operator-owned tree node rate override. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NodeRateOverrideUpdate { + /// Stable node identifier from `network.json`. + pub node_id: String, + /// Display name of the selected node. + pub node_name: String, + /// Override download bandwidth in Mbps. + pub download_bandwidth_mbps: Option, + /// Override upload bandwidth in Mbps. + pub upload_bandwidth_mbps: Option, +} + +/// Inspector/view-model data for the tree node rate override panel. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NodeRateOverrideData { + /// Whether the current session is allowed to persist overrides. + pub writable: bool, + /// Whether editing is allowed for this specific node and session. + pub can_edit: bool, + /// Human-readable reason editing is disabled, when applicable. + pub disabled_reason: Option, + /// Whether an operator-owned override currently exists. + pub has_override: bool, + /// Stable node identifier stored on the matching override, when present. + pub override_node_id: Option, + /// Stored operator override download bandwidth in Mbps. + pub override_download_bandwidth_mbps: Option, + /// Stored operator override upload bandwidth in Mbps. + pub override_upload_bandwidth_mbps: Option, + /// Warnings about active legacy integration bandwidth override files. + pub legacy_warnings: Vec, +} + +/// Load the current tree node rate override inspector data. +pub fn get_node_rate_override_data( + login: LoginResult, + query: NodeRateOverrideQuery, +) -> Result { + build_node_rate_override_data(login, query) +} + +/// Save or replace the operator-owned rate override for a tree node. +pub fn set_node_rate_override_data( + login: LoginResult, + update: NodeRateOverrideUpdate, +) -> Result { + if login != LoginResult::Admin { + return Err(StatusCode::FORBIDDEN); + } + validate_update_payload(&update)?; + + let query = NodeRateOverrideQuery { + node_id: Some(update.node_id.clone()), + node_name: update.node_name.clone(), + }; + if let Some(reason) = edit_disabled_reason(login, &query) { + tracing::warn!( + node_name = %update.node_name, + node_id = %update.node_id, + "Rejected tree node override save: {reason}" + ); + return Err(StatusCode::BAD_REQUEST); + } + + let mut overrides = OverrideStore::load_layer(OverrideLayer::Operator) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let changed = overrides.set_site_bandwidth_override( + Some(update.node_id), + update.node_name, + update.download_bandwidth_mbps, + update.upload_bandwidth_mbps, + ); + if changed { + OverrideStore::save_layer(OverrideLayer::Operator, &overrides) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + } + + build_node_rate_override_data(login, query) +} + +/// Remove the operator-owned rate override for a tree node. +pub fn clear_node_rate_override_data( + login: LoginResult, + query: NodeRateOverrideQuery, +) -> Result { + if login != LoginResult::Admin { + return Err(StatusCode::FORBIDDEN); + } + if let Some(reason) = edit_disabled_reason(login, &query) { + tracing::warn!( + node_name = %query.node_name, + node_id = %query.node_id.clone().unwrap_or_default(), + "Rejected tree node override clear: {reason}" + ); + return Err(StatusCode::BAD_REQUEST); + } + + let Some(node_id) = query.node_id.as_deref() else { + return Err(StatusCode::BAD_REQUEST); + }; + + let mut overrides = OverrideStore::load_layer(OverrideLayer::Operator) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let removed = overrides.remove_site_bandwidth_override_count(Some(node_id), &query.node_name); + if removed > 0 { + OverrideStore::save_layer(OverrideLayer::Operator, &overrides) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + } + + build_node_rate_override_data(login, query) +} + +fn build_node_rate_override_data( + login: LoginResult, + query: NodeRateOverrideQuery, +) -> Result { + let overrides = OverrideStore::load_layer(OverrideLayer::Operator) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let matched_override = + overrides.find_site_bandwidth_override(query.node_id.as_deref(), &query.node_name); + let (has_override, override_node_id, override_down, override_up) = match matched_override { + Some(NetworkAdjustment::AdjustSiteSpeed { + node_id, + download_bandwidth_mbps, + upload_bandwidth_mbps, + .. + }) => ( + true, + node_id.clone(), + *download_bandwidth_mbps, + *upload_bandwidth_mbps, + ), + _ => (false, None, None, None), + }; + + let disabled_reason = edit_disabled_reason(login, &query); + Ok(NodeRateOverrideData { + writable: login == LoginResult::Admin, + can_edit: disabled_reason.is_none(), + disabled_reason, + has_override, + override_node_id, + override_download_bandwidth_mbps: override_down, + override_upload_bandwidth_mbps: override_up, + legacy_warnings: legacy_warning_messages()?, + }) +} + +fn edit_disabled_reason(login: LoginResult, query: &NodeRateOverrideQuery) -> Option { + if login != LoginResult::Admin { + return Some("Only administrators can edit node rate overrides.".to_string()); + } + let trimmed_name = query.node_name.trim(); + if trimmed_name.is_empty() { + return Some("This node cannot be edited because it does not expose a stable name.".into()); + } + let Some(node_id) = query.node_id.as_deref() else { + return Some( + "This node cannot be edited from the tree because it does not expose a stable node ID." + .to_string(), + ); + }; + if node_id.starts_with(GENERATED_NODE_ID_PREFIX) + || trimmed_name.starts_with(GENERATED_NODE_NAME_PREFIX) + { + return Some("Generated nodes cannot be edited from the tree.".to_string()); + } + None +} + +fn validate_update_payload(update: &NodeRateOverrideUpdate) -> Result<(), StatusCode> { + if update.node_id.trim().is_empty() || update.node_name.trim().is_empty() { + return Err(StatusCode::BAD_REQUEST); + } + if update.download_bandwidth_mbps.is_none() && update.upload_bandwidth_mbps.is_none() { + return Err(StatusCode::BAD_REQUEST); + } + + for value in [update.download_bandwidth_mbps, update.upload_bandwidth_mbps] + .into_iter() + .flatten() + { + if !value.is_finite() || value < 0.0 { + return Err(StatusCode::BAD_REQUEST); + } + } + Ok(()) +} + +fn legacy_warning_messages() -> Result, StatusCode> { + let config = load_config().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let base = Path::new(&config.lqos_directory); + let mut warnings = Vec::new(); + + if config.uisp_integration.enable_uisp && base.join(LEGACY_UISP_FILE).exists() { + warnings.push(format!( + "Legacy UISP bandwidth overrides are present in `{LEGACY_UISP_FILE}`. Tree edits write operator overrides to `lqos_overrides.json` and do not modify that file." + )); + } + if config.splynx_integration.enable_splynx && base.join(LEGACY_SPLYNX_FILE).exists() { + warnings.push(format!( + "Legacy Splynx bandwidth overrides are present in `{LEGACY_SPLYNX_FILE}`. Tree edits write operator overrides to `lqos_overrides.json` and do not modify that file." + )); + } + + Ok(warnings) +} + +#[cfg(test)] +mod tests { + use super::{ + GENERATED_NODE_ID_PREFIX, GENERATED_NODE_NAME_PREFIX, NodeRateOverrideQuery, + NodeRateOverrideUpdate, edit_disabled_reason, validate_update_payload, + }; + use crate::node_manager::auth::LoginResult; + use axum::http::StatusCode; + + #[test] + fn read_only_sessions_cannot_edit_even_for_real_nodes() { + let query = NodeRateOverrideQuery { + node_id: Some("node-ap27".to_string()), + node_name: "AP27".to_string(), + }; + assert_eq!( + edit_disabled_reason(LoginResult::ReadOnly, &query), + Some("Only administrators can edit node rate overrides.".to_string()) + ); + } + + #[test] + fn generated_nodes_are_blocked_for_admins() { + let query = NodeRateOverrideQuery { + node_id: Some(format!("{GENERATED_NODE_ID_PREFIX}site:ap27")), + node_name: format!("{GENERATED_NODE_NAME_PREFIX}AP27"), + }; + assert_eq!( + edit_disabled_reason(LoginResult::Admin, &query), + Some("Generated nodes cannot be edited from the tree.".to_string()) + ); + } + + #[test] + fn missing_node_id_is_rejected_for_writes() { + let query = NodeRateOverrideQuery { + node_id: None, + node_name: "AP27".to_string(), + }; + assert_eq!( + edit_disabled_reason(LoginResult::Admin, &query), + Some( + "This node cannot be edited from the tree because it does not expose a stable node ID." + .to_string() + ) + ); + } + + #[test] + fn update_payload_requires_non_negative_finite_rates() { + let update = NodeRateOverrideUpdate { + node_id: "node-ap27".to_string(), + node_name: "AP27".to_string(), + download_bandwidth_mbps: Some(-1.0), + upload_bandwidth_mbps: Some(10.0), + }; + assert_eq!( + validate_update_payload(&update), + Err(StatusCode::BAD_REQUEST) + ); + } +} diff --git a/src/rust/lqosd/src/node_manager/local_api/scheduler.rs b/src/rust/lqosd/src/node_manager/local_api/scheduler.rs index 4448d6e9..d290d409 100644 --- a/src/rust/lqosd/src/node_manager/local_api/scheduler.rs +++ b/src/rust/lqosd/src/node_manager/local_api/scheduler.rs @@ -1,6 +1,8 @@ use serde::Serialize; -use crate::tool_status::{is_scheduler_available, scheduler_error_message, scheduler_output_message}; +use crate::tool_status::{ + is_scheduler_available, scheduler_error_message, scheduler_output_message, +}; // Remove ANSI escape sequences (basic CSI/OSC handling) for browser display fn strip_ansi(input: &str) -> String { diff --git a/src/rust/lqosd/src/node_manager/local_api/shaped_devices_page.rs b/src/rust/lqosd/src/node_manager/local_api/shaped_devices_page.rs new file mode 100644 index 00000000..684159b8 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/shaped_devices_page.rs @@ -0,0 +1,122 @@ +use crate::shaped_devices_tracker::SHAPED_DEVICES; +use lqos_config::ShapedDevice; +use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::HashSet; + +const DEFAULT_SHAPED_DEVICES_PAGE_SIZE: usize = 24; +const MAX_SHAPED_DEVICES_PAGE_SIZE: usize = 250; + +fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum MaybeString { + String(String), + Null, + } + + Ok(match Option::::deserialize(deserializer)? { + Some(MaybeString::String(value)) => Some(value), + Some(MaybeString::Null) | None => None, + }) +} + +/// Server-side paging and search query for the shaped-devices inventory page. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ShapedDevicesPageQuery { + pub page: Option, + pub page_size: Option, + #[serde(default, deserialize_with = "deserialize_optional_string")] + pub search: Option, +} + +/// A server-paged slice of shaped devices plus total result counts. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ShapedDevicesPage { + pub query: ShapedDevicesPageQuery, + pub total_rows: usize, + pub total_circuits: usize, + pub rows: Vec, +} + +fn normalized_page_size(query: &ShapedDevicesPageQuery) -> usize { + query + .page_size + .unwrap_or(DEFAULT_SHAPED_DEVICES_PAGE_SIZE) + .clamp(1, MAX_SHAPED_DEVICES_PAGE_SIZE) +} + +/// Returns one filtered, sorted page of shaped-device rows. +pub fn shaped_devices_page(query: ShapedDevicesPageQuery) -> ShapedDevicesPage { + let page = query.page.unwrap_or(0); + let page_size = normalized_page_size(&query); + let search = query.search.as_deref().unwrap_or("").trim().to_lowercase(); + let devices = SHAPED_DEVICES.load(); + + let mut filtered: Vec = devices + .devices + .iter() + .filter(|device| { + if search.is_empty() { + return true; + } + device.device_name.to_lowercase().contains(&search) + || device.circuit_name.to_lowercase().contains(&search) + || device.parent_node.to_lowercase().contains(&search) + || device.circuit_id.to_lowercase().contains(&search) + || device.device_id.to_lowercase().contains(&search) + || device.mac.to_lowercase().contains(&search) + || device.comment.to_lowercase().contains(&search) + || device + .sqm_override + .as_deref() + .unwrap_or("") + .to_lowercase() + .contains(&search) + || device.ipv4.iter().any(|(addr, prefix)| { + format!("{addr}/{prefix}").to_lowercase().contains(&search) + }) + || device.ipv6.iter().any(|(addr, prefix)| { + format!("{addr}/{prefix}").to_lowercase().contains(&search) + }) + }) + .cloned() + .collect(); + filtered.sort_by(|left, right| { + left.circuit_name + .cmp(&right.circuit_name) + .then_with(|| left.device_name.cmp(&right.device_name)) + .then_with(|| left.device_id.cmp(&right.device_id)) + }); + + let total_rows = filtered.len(); + let total_circuits = filtered + .iter() + .map(|device| device.circuit_id.clone()) + .collect::>() + .len(); + let start = page.saturating_mul(page_size); + let rows = if start >= total_rows { + Vec::new() + } else { + let end = (start + page_size).min(total_rows); + filtered[start..end].to_vec() + }; + + ShapedDevicesPage { + query: ShapedDevicesPageQuery { + page: Some(page), + page_size: Some(page_size), + search: if search.is_empty() { + None + } else { + query.search + }, + }, + total_rows, + total_circuits, + rows, + } +} diff --git a/src/rust/lqosd/src/node_manager/local_api/tree_attached_circuits.rs b/src/rust/lqosd/src/node_manager/local_api/tree_attached_circuits.rs new file mode 100644 index 00000000..c0937af4 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/local_api/tree_attached_circuits.rs @@ -0,0 +1,233 @@ +use crate::node_manager::local_api::ethernet_caps::{EthernetCapBadge, ethernet_cap_badge_map}; +use crate::shaped_devices_tracker::NETWORK_JSON; +use crate::shaped_devices_tracker::circuit_live::fresh_circuit_live_snapshot; +use lqos_utils::units::{DownUpOrder, TcpRetransmitSample}; +use serde::{Deserialize, Deserializer, Serialize}; + +const DEFAULT_ATTACHED_CIRCUITS_PAGE_SIZE: usize = 100; +const MAX_ATTACHED_CIRCUITS_PAGE_SIZE: usize = 250; + +fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum MaybeString { + String(String), + Null, + } + + Ok(match Option::::deserialize(deserializer)? { + Some(MaybeString::String(value)) => Some(value), + Some(MaybeString::Null) | None => None, + }) +} + +/// Sort options for the tree page's attached-circuits table. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum TreeAttachedCircuitsSort { + CircuitName, + LastSeen, + ThroughputDown, + ThroughputUp, +} + +/// Query for the selected node's attached-circuits table. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TreeAttachedCircuitsQuery { + #[serde(default, deserialize_with = "deserialize_optional_string")] + pub node_id: Option, + pub node_path: Option>, + pub page: Option, + pub page_size: Option, + #[serde(default, deserialize_with = "deserialize_optional_string")] + pub search: Option, + pub sort: Option, + pub descending: Option, +} + +/// One aggregated circuit row for the tree page's attached-circuits table. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TreeAttachedCircuitRow { + pub circuit_id: String, + pub circuit_name: String, + /// Optional Ethernet speed-cap advisory badge when this circuit was auto-capped. + pub ethernet_cap_badge: Option, + pub parent_node: String, + pub device_names: Vec, + pub ip_addrs: Vec, + pub plan_mbps: DownUpOrder, + pub bytes_per_second: DownUpOrder, + pub rtt_current_p50_nanos: DownUpOrder>, + pub qoo: DownUpOrder>, + pub tcp_retransmit_sample: DownUpOrder, + pub last_seen_nanos: u64, +} + +/// One server-paged response for the selected node's attached circuits. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TreeAttachedCircuitsPage { + pub query: TreeAttachedCircuitsQuery, + pub total_rows: usize, + pub rows: Vec, +} + +fn normalized_page_size(query: &TreeAttachedCircuitsQuery) -> usize { + query + .page_size + .unwrap_or(DEFAULT_ATTACHED_CIRCUITS_PAGE_SIZE) + .clamp(1, MAX_ATTACHED_CIRCUITS_PAGE_SIZE) +} + +fn resolve_node_name(query: &TreeAttachedCircuitsQuery) -> Option { + let reader = NETWORK_JSON.read(); + let nodes = reader.get_nodes_when_ready(); + + if let Some(node_id) = query.node_id.as_deref() + && let Some(node) = nodes + .iter() + .find(|node| node.id.as_deref() == Some(node_id)) + { + return Some(node.name.clone()); + } + + let node_path = query.node_path.as_ref()?; + if node_path.is_empty() { + return None; + } + nodes.iter().find_map(|node| { + let path = if node.name == "Root" { + vec!["Root".to_string()] + } else { + let parent_indexes = if node.parents.is_empty() { + vec![node.immediate_parent.unwrap_or(0)] + } else { + node.parents.clone() + }; + let mut names = Vec::new(); + for idx in parent_indexes { + let parent = nodes.get(idx)?; + names.push(parent.name.clone()); + } + if names.last() != Some(&node.name) { + names.push(node.name.clone()); + } + names + }; + if &path == node_path { + Some(node.name.clone()) + } else { + None + } + }) +} + +/// Returns one filtered, sorted page of circuits attached to the selected tree node. +pub fn tree_attached_circuits(query: TreeAttachedCircuitsQuery) -> TreeAttachedCircuitsPage { + let page = query.page.unwrap_or(0); + let page_size = normalized_page_size(&query); + let search = query.search.as_deref().unwrap_or("").trim().to_lowercase(); + let sort = query + .sort + .clone() + .unwrap_or(TreeAttachedCircuitsSort::CircuitName); + let descending = query.descending.unwrap_or(false); + let node_name = resolve_node_name(&query); + + let snapshot = fresh_circuit_live_snapshot(); + let ethernet_badges = ethernet_cap_badge_map(); + let mut rows: Vec = match node_name.as_deref() { + Some(node_name) => snapshot + .circuit_ids_by_parent_node + .get(node_name) + .into_iter() + .flat_map(|ids| ids.iter()) + .filter_map(|id| snapshot.by_circuit_id.get(id)) + .map(|row| TreeAttachedCircuitRow { + circuit_id: row.circuit_id.clone(), + circuit_name: row.circuit_name.clone(), + ethernet_cap_badge: ethernet_badges + .get(&row.circuit_id.to_ascii_lowercase()) + .cloned(), + parent_node: row.parent_node.clone(), + device_names: row.device_names.clone(), + ip_addrs: row.ip_addrs.clone(), + plan_mbps: row.plan_mbps, + bytes_per_second: row.bytes_per_second, + rtt_current_p50_nanos: row.rtt_current_p50_nanos, + qoo: row.qoo, + tcp_retransmit_sample: row.tcp_retransmit_sample, + last_seen_nanos: row.last_seen_nanos, + }) + .collect(), + None => Vec::new(), + }; + + if !search.is_empty() { + rows.retain(|row| { + row.circuit_name.to_lowercase().contains(&search) + || row.circuit_id.to_lowercase().contains(&search) + || row.parent_node.to_lowercase().contains(&search) + || row + .device_names + .iter() + .any(|name| name.to_lowercase().contains(&search)) + || row + .ip_addrs + .iter() + .any(|ip| ip.to_lowercase().contains(&search)) + }); + } + + rows.sort_by(|left, right| match sort { + TreeAttachedCircuitsSort::CircuitName => left + .circuit_name + .cmp(&right.circuit_name) + .then_with(|| left.circuit_id.cmp(&right.circuit_id)), + TreeAttachedCircuitsSort::LastSeen => left + .last_seen_nanos + .cmp(&right.last_seen_nanos) + .then_with(|| left.circuit_name.cmp(&right.circuit_name)), + TreeAttachedCircuitsSort::ThroughputDown => left + .bytes_per_second + .down + .cmp(&right.bytes_per_second.down) + .then_with(|| left.circuit_name.cmp(&right.circuit_name)), + TreeAttachedCircuitsSort::ThroughputUp => left + .bytes_per_second + .up + .cmp(&right.bytes_per_second.up) + .then_with(|| left.circuit_name.cmp(&right.circuit_name)), + }); + if descending { + rows.reverse(); + } + + let total_rows = rows.len(); + let start = page.saturating_mul(page_size); + let rows = if start >= total_rows { + Vec::new() + } else { + let end = (start + page_size).min(total_rows); + rows[start..end].to_vec() + }; + + TreeAttachedCircuitsPage { + query: TreeAttachedCircuitsQuery { + node_id: query.node_id, + node_path: query.node_path, + page: Some(page), + page_size: Some(page_size), + search: if search.is_empty() { + None + } else { + query.search + }, + sort: Some(sort), + descending: Some(descending), + }, + total_rows, + rows, + } +} diff --git a/src/rust/lqosd/src/node_manager/run.rs b/src/rust/lqosd/src/node_manager/run.rs index 9b398351..13075237 100644 --- a/src/rust/lqosd/src/node_manager/run.rs +++ b/src/rust/lqosd/src/node_manager/run.rs @@ -52,6 +52,8 @@ pub async fn spawn_webserver( // Construct the router from parts let router = Router::new() .route("/", get(redirect_to_index)) + .route("/login.html", get(auth::login_page)) + .route("/first-run.html", get(auth::first_run_page)) .route("/doLogin", post(auth::try_login)) .route("/firstLogin", post(auth::first_user)) .route("/health", get(health_check)) diff --git a/src/rust/lqosd/src/node_manager/static2/01_executive_summary.png b/src/rust/lqosd/src/node_manager/static2/01_executive_summary.png new file mode 100644 index 00000000..9d67d725 Binary files /dev/null and b/src/rust/lqosd/src/node_manager/static2/01_executive_summary.png differ diff --git a/src/rust/lqosd/src/node_manager/static2/02_site_overview.png b/src/rust/lqosd/src/node_manager/static2/02_site_overview.png new file mode 100644 index 00000000..6bcae2b6 Binary files /dev/null and b/src/rust/lqosd/src/node_manager/static2/02_site_overview.png differ diff --git a/src/rust/lqosd/src/node_manager/static2/03_site_rankings.png b/src/rust/lqosd/src/node_manager/static2/03_site_rankings.png new file mode 100644 index 00000000..73c27c73 Binary files /dev/null and b/src/rust/lqosd/src/node_manager/static2/03_site_rankings.png differ diff --git a/src/rust/lqosd/src/node_manager/static2/04_subscriber_view.png b/src/rust/lqosd/src/node_manager/static2/04_subscriber_view.png new file mode 100644 index 00000000..bb8847e8 Binary files /dev/null and b/src/rust/lqosd/src/node_manager/static2/04_subscriber_view.png differ diff --git a/src/rust/lqosd/src/node_manager/static2/05_issue_triage.png b/src/rust/lqosd/src/node_manager/static2/05_issue_triage.png new file mode 100644 index 00000000..fe21a2b1 Binary files /dev/null and b/src/rust/lqosd/src/node_manager/static2/05_issue_triage.png differ diff --git a/src/rust/lqosd/src/node_manager/static2/06_api_only.png b/src/rust/lqosd/src/node_manager/static2/06_api_only.png new file mode 100644 index 00000000..c52d3151 Binary files /dev/null and b/src/rust/lqosd/src/node_manager/static2/06_api_only.png differ diff --git a/src/rust/lqosd/src/node_manager/static2/all_tree_sankey.html b/src/rust/lqosd/src/node_manager/static2/all_tree_sankey.html index b38ff086..bee1aed2 100644 --- a/src/rust/lqosd/src/node_manager/static2/all_tree_sankey.html +++ b/src/rust/lqosd/src/node_manager/static2/all_tree_sankey.html @@ -1,11 +1,12 @@ -
    -
    +
    +
    Top node::
    -
    +
    - + +
    diff --git a/src/rust/lqosd/src/node_manager/static2/circuit.html b/src/rust/lqosd/src/node_manager/static2/circuit.html index b049b49d..13abb7e0 100644 --- a/src/rust/lqosd/src/node_manager/static2/circuit.html +++ b/src/rust/lqosd/src/node_manager/static2/circuit.html @@ -3,21 +3,34 @@
    - +
    - - + + - - + + - + - + - - + + @@ -587,6 +595,7 @@

    Network.Json - Network Layout

    +

    Network Adjustment Options

    @@ -613,9 +622,9 @@

    Shaped Devices

    -

    All Shaped Devices. Note that if you have an integration enabled, this will be overwritten - periodically.

    - diff --git a/src/rust/lqosd/src/node_manager/static2/cpu_weights.html b/src/rust/lqosd/src/node_manager/static2/cpu_weights.html index 3d3c7ea2..63a9dadf 100644 --- a/src/rust/lqosd/src/node_manager/static2/cpu_weights.html +++ b/src/rust/lqosd/src/node_manager/static2/cpu_weights.html @@ -1,52 +1,362 @@ -
    -
    -
    -
    -
    CPU Affinity
    -

    View current circuit placement per CPU/queue from the active TC plan.

    -
    -
    -
    -
    - - -
    -
    - Binpacking: … -
    -
    - - Site Tree - -
    -
    - -
    -
    -
    + + +
    +
    +
    +

    CPU Affinity

    +

    Live per-core load, promoted runtime branches, and effective site/AP ownership after TreeGuard virtualization.

    -
    - Loading CPU summary... +
    +
    + + +
    +
    + Binpacking: … +
    +
    + + +
    +
    + + Site Tree + +
    +
    + +
    +
    +
    + +
    +
    + Loading core assignment...
    -
    +
    +
    +
    +
    +
    +

    Effective Assignment

    +

    Current nodes, promoted branches, and circuit detail for the selected CPU.

    +
    +
    + + +
    +
    -
    -
    -
    - Circuits - +
    + Select a core above. +
    + +
    + Select a CPU from the list. +
    + +
    -
    - - -
    -
    -
    - Select a CPU from the table above.
    diff --git a/src/rust/lqosd/src/node_manager/static2/ethernet_caps.html b/src/rust/lqosd/src/node_manager/static2/ethernet_caps.html new file mode 100644 index 00000000..c0501691 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/static2/ethernet_caps.html @@ -0,0 +1,51 @@ +
    +
    +
    +
    +
    Ethernet Caps
    +
    Circuits automatically down-rated because detected Ethernet speed was below the requested plan.
    +
    +
    +
    + +
    +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    +
    +
    Loading Ethernet cap data…
    +
    +
    +
    +
    +
    + + diff --git a/src/rust/lqosd/src/node_manager/static2/first-run.html b/src/rust/lqosd/src/node_manager/static2/first-run.html index 95683af3..7f4c89f2 100644 --- a/src/rust/lqosd/src/node_manager/static2/first-run.html +++ b/src/rust/lqosd/src/node_manager/static2/first-run.html @@ -72,6 +72,11 @@
    +
    + + Unable to initialize web authentication. +
    +
    diff --git a/src/rust/lqosd/src/node_manager/static2/login.html b/src/rust/lqosd/src/node_manager/static2/login.html index 5867c5d3..48e872a8 100644 --- a/src/rust/lqosd/src/node_manager/static2/login.html +++ b/src/rust/lqosd/src/node_manager/static2/login.html @@ -55,7 +55,7 @@
    - Login failed. You can manage users via the lqusers CLI tool on the LibreQoS server. + Login failed. You can manage users via the lqusers CLI tool on the LibreQoS server.
    diff --git a/src/rust/lqosd/src/node_manager/static2/lts_trial.html b/src/rust/lqosd/src/node_manager/static2/lts_trial.html index 22840a2f..d623a401 100644 --- a/src/rust/lqosd/src/node_manager/static2/lts_trial.html +++ b/src/rust/lqosd/src/node_manager/static2/lts_trial.html @@ -123,6 +123,9 @@ body.dark-mode .card-header.bg-light {

    LibreQoS Insight

    Unlock the full potential of your network with advanced analytics, reporting, and monitoring.

    +
    @@ -326,6 +334,9 @@ body.dark-mode .card-header.bg-light { +

    + Free 30-day trial. You will not be charged until the trial ends. Cancel any time. +

    @@ -346,10 +357,10 @@ body.dark-mode .card-header.bg-light {
    -

    Welcome to LibreQoS Insight!

    -

    Your license has been activated successfully.

    +

    Insight Trial Created

    +

    Your trial is almost active.

    -

    LibreQoS is now being configured to connect to Insight. This process may take a few moments.

    +

    LibreQoS is saving your Insight configuration now. Finish the billing step from the Insight email to keep the free trial active.

    diff --git a/src/rust/lqosd/src/node_manager/static2/lts_trial_setup.html b/src/rust/lqosd/src/node_manager/static2/lts_trial_setup.html new file mode 100644 index 00000000..4808c4c6 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/static2/lts_trial_setup.html @@ -0,0 +1,37 @@ +
    +
    +
    +
    +

    + + Preparing Your Insight Sign-Up Session +

    +
    +
    +

    + LibreQoS will create a short-lived Insight claim and then hand your browser to the hosted signup flow. +

    +
    + + Preparing the Insight signup handoff… +
    +

    + Existing license validation and the direct signup form remain available if you need them. +

    +
    + + I have a license key + + + Use the direct signup form + + + Back + +
    +
    +
    +
    +
    + + diff --git a/src/rust/lqosd/src/node_manager/static2/lts_trial_success.html b/src/rust/lqosd/src/node_manager/static2/lts_trial_success.html index c748cfff..31e186b4 100644 --- a/src/rust/lqosd/src/node_manager/static2/lts_trial_success.html +++ b/src/rust/lqosd/src/node_manager/static2/lts_trial_success.html @@ -3,11 +3,11 @@
    -

    Insight Trial Activated

    +

    Insight Trial Created

    -

    Your LibreQoS Insight trial is now active.

    -

    You can access Insight from the sidebar, or return to the dashboard. If you don’t see Insight right away, try refreshing the page.

    +

    Your trial is almost active.

    +

    Watch for an email from Insight to add billing details within 2 days. No charge occurs until the trial ends, and you can cancel any time.

    Go to Dashboard @@ -21,4 +21,3 @@
    - diff --git a/src/rust/lqosd/src/node_manager/static2/node_manager.css b/src/rust/lqosd/src/node_manager/static2/node_manager.css index 19f58160..ccc6ff3e 100644 --- a/src/rust/lqosd/src/node_manager/static2/node_manager.css +++ b/src/rust/lqosd/src/node_manager/static2/node_manager.css @@ -405,6 +405,27 @@ html[data-bs-theme="light"] .sidebar .navbar-toggler-icon { .lqos-circuit-panel .lqos-table { height: 100%; } +.lqos-circuit-header-table.lqos-table-compact tbody td { + padding: 0.5rem 0.55rem; +} +.lqos-table.lqos-circuit-header-table .table-label-cell { + width: 6.75rem; + white-space: nowrap; + padding-right: 0.25rem; +} +.lqos-table.lqos-circuit-header-table .table-value-cell { + width: auto; + padding-left: 0.25rem; +} +.lqos-circuit-parent-link { + display: block; + max-width: 100%; +} +.flow-rtt-exclude-btn.btn.btn-sm { + padding-top: 0.175rem; + padding-bottom: 0.175rem; + line-height: 1.1; +} .lqos-circuit-gauge-panel { display: flex; align-items: center; @@ -619,6 +640,313 @@ html[data-bs-theme="light"] .sidebar .navbar-toggler-icon { .lqos-circuit-activity-kpi-value.is-active { color: #3fd279; } +.lqos-cake-panel { + margin-top: 1rem; + border: 1px solid color-mix(in srgb, var(--lqos-border) 80%, transparent); + border-radius: var(--lqos-radius-md); + background: linear-gradient( + 180deg, + color-mix(in srgb, var(--lqos-surface-2) 88%, transparent) 0%, + var(--lqos-surface-solid) 100% + ); + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.04), + 0 12px 24px rgba(6, 12, 20, 0.12); + padding: 0.95rem; +} +.lqos-cake-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 1rem; + margin-bottom: 0.9rem; +} +.lqos-cake-header-copy { + min-width: 0; +} +.lqos-cake-header-copy h5 { + margin-bottom: 0.25rem; +} +.lqos-cake-meta { + display: inline-flex; + align-items: center; + justify-content: flex-end; + flex-wrap: wrap; + gap: 0.4rem; +} +.lqos-cake-meta-pill { + display: inline-flex; + align-items: center; + padding: 0.34rem 0.72rem; + border-radius: 999px; + border: 1px solid color-mix(in srgb, var(--lqos-border) 84%, transparent); + background: color-mix(in srgb, var(--lqos-surface-2) 72%, transparent); + color: var(--bs-secondary-color); + font-size: 0.76rem; + font-weight: 600; + white-space: nowrap; +} +.lqos-cake-chart-panel { + height: 100%; + border: 1px solid color-mix(in srgb, var(--lqos-border) 76%, transparent); + border-radius: var(--lqos-radius-sm); + background: color-mix(in srgb, var(--lqos-surface-2) 66%, transparent); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.04); + padding: 0.35rem; +} +.lqos-cake-chart-panel.is-primary { + background: color-mix(in srgb, var(--lqos-surface-2) 78%, transparent); +} +.lqos-cake-info-card { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 0.75rem; + border: 1px solid color-mix(in srgb, var(--lqos-border) 76%, transparent); + border-radius: var(--lqos-radius-sm); + background: color-mix(in srgb, var(--lqos-surface-2) 62%, transparent); + padding: 0.8rem 0.9rem; +} +.lqos-cake-info-label { + color: var(--bs-secondary-color); + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; + margin-bottom: 0.28rem; +} +.lqos-cake-info-value { + color: var(--bs-emphasis-color); + font-size: 0.96rem; + font-weight: 600; + line-height: 1.35; +} +.lqos-circuit-devices-panel { + margin-top: 1rem; + border: 1px solid color-mix(in srgb, var(--lqos-border) 80%, transparent); + border-radius: var(--lqos-radius-md); + background: linear-gradient( + 180deg, + color-mix(in srgb, var(--lqos-surface-2) 88%, transparent) 0%, + var(--lqos-surface-solid) 100% + ); + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.04), + 0 12px 24px rgba(6, 12, 20, 0.12); + padding: 0.95rem; +} +.lqos-circuit-devices-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 1rem; + margin-bottom: 0.9rem; +} +.lqos-circuit-devices-header-copy { + min-width: 0; +} +.lqos-circuit-devices-header-copy h5 { + margin-bottom: 0.25rem; +} +.lqos-circuit-devices-meta { + display: inline-flex; + align-items: center; + justify-content: flex-end; + flex-wrap: wrap; + gap: 0.4rem; +} +.lqos-circuit-devices-meta-pill { + display: inline-flex; + align-items: center; + padding: 0.34rem 0.72rem; + border-radius: 999px; + border: 1px solid color-mix(in srgb, var(--lqos-border) 84%, transparent); + background: color-mix(in srgb, var(--lqos-surface-2) 72%, transparent); + color: var(--bs-secondary-color); + font-size: 0.76rem; + font-weight: 600; + white-space: nowrap; +} +.lqos-circuit-device-card { + border: 1px solid color-mix(in srgb, var(--lqos-border) 76%, transparent); + border-radius: var(--lqos-radius-sm); + background: color-mix(in srgb, var(--lqos-surface-2) 66%, transparent); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.04); + padding: 0.8rem; +} +.lqos-circuit-device-summary, +.lqos-circuit-device-graphs { + min-width: 0; +} +.lqos-circuit-device-summary h5 { + margin-bottom: 0.5rem; +} +.lqos-circuit-device-graph { + border: 1px solid color-mix(in srgb, var(--lqos-border) 76%, transparent); + border-radius: var(--lqos-radius-sm); + background: color-mix(in srgb, var(--lqos-surface-2) 76%, transparent); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.04); + padding: 0.35rem; +} +.lqos-funnel-panel { + margin-top: 1rem; + border: 1px solid color-mix(in srgb, var(--lqos-border) 80%, transparent); + border-radius: var(--lqos-radius-md); + background: linear-gradient( + 180deg, + color-mix(in srgb, var(--lqos-surface-2) 88%, transparent) 0%, + var(--lqos-surface-solid) 100% + ); + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.04), + 0 12px 24px rgba(6, 12, 20, 0.12); + padding: 0.95rem; +} +.lqos-funnel-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 1rem; + margin-bottom: 0.9rem; +} +.lqos-funnel-header-copy { + min-width: 0; +} +.lqos-funnel-header-copy h5 { + margin-bottom: 0.25rem; +} +.lqos-funnel-meta { + display: inline-flex; + align-items: center; + justify-content: flex-end; + flex-wrap: wrap; + gap: 0.4rem; +} +.lqos-funnel-meta-pill, +.lqos-funnel-path-pill, +.lqos-funnel-node-step { + display: inline-flex; + align-items: center; + padding: 0.34rem 0.72rem; + border-radius: 999px; + border: 1px solid color-mix(in srgb, var(--lqos-border) 84%, transparent); + background: color-mix(in srgb, var(--lqos-surface-2) 72%, transparent); + color: var(--bs-secondary-color); + font-size: 0.76rem; + font-weight: 600; + white-space: nowrap; +} +.lqos-funnel-stack { + display: grid; + gap: 0.95rem; +} +.lqos-funnel-path-card, +.lqos-funnel-node-card, +.lqos-funnel-empty-state { + border: 1px solid color-mix(in srgb, var(--lqos-border) 76%, transparent); + border-radius: var(--lqos-radius-sm); + background: color-mix(in srgb, var(--lqos-surface-2) 64%, transparent); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.04); +} +.lqos-funnel-path-card { + padding: 0.9rem; +} +.lqos-funnel-path-top, +.lqos-funnel-node-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.85rem; +} +.lqos-funnel-path-title { + color: var(--bs-emphasis-color); + font-size: 0.95rem; + font-weight: 700; +} +.lqos-funnel-path-meta, +.lqos-funnel-node-badges { + display: inline-flex; + align-items: center; + justify-content: flex-end; + flex-wrap: wrap; + gap: 0.45rem; +} +.lqos-funnel-path-chain { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.45rem; + margin: 0.72rem 0 0.55rem; +} +.lqos-funnel-path-node { + display: inline-flex; + align-items: center; + min-height: 2rem; + padding: 0.34rem 0.78rem; + border-radius: 999px; + border: 1px solid color-mix(in srgb, var(--lqos-border) 82%, transparent); + background: color-mix(in srgb, var(--lqos-surface) 86%, transparent); + color: var(--bs-emphasis-color); + font-size: 0.84rem; + font-weight: 600; +} +.lqos-funnel-path-node.is-origin { + background: color-mix(in srgb, #198754 18%, var(--lqos-surface) 82%); + border-color: color-mix(in srgb, #198754 34%, var(--lqos-border)); +} +.lqos-funnel-path-separator { + color: var(--bs-secondary-color); + font-size: 0.92rem; +} +.lqos-funnel-empty-state { + display: flex; + align-items: center; + gap: 0.6rem; + padding: 1rem 1.05rem; + color: var(--bs-secondary-color); +} +.lqos-funnel-empty-state i { + color: color-mix(in srgb, var(--bs-warning) 70%, var(--bs-secondary-color)); +} +.lqos-funnel-node-card { + padding: 0.9rem; +} +.lqos-funnel-node-title h5 { + display: flex; + align-items: center; + gap: 0.45rem; + margin-bottom: 0.25rem; +} +.lqos-funnel-node-title h5 .fa { + flex: 0 0 auto; +} +.lqos-funnel-node-badges .badge { + padding: 0.42rem 0.7rem; + font-weight: 600; +} +.lqos-funnel-node-type { + display: inline-flex; + align-items: center; + padding: 0.4rem 0.72rem; + border-radius: 999px; + border: 1px solid color-mix(in srgb, var(--lqos-border) 82%, transparent); + background: color-mix(in srgb, var(--lqos-surface) 84%, transparent); + color: var(--bs-emphasis-color); + font-size: 0.78rem; + font-weight: 600; + white-space: nowrap; +} +.lqos-funnel-node-type.is-virtual { + background: color-mix(in srgb, #6c757d 22%, var(--lqos-surface) 80%); + border-color: color-mix(in srgb, #6c757d 40%, var(--lqos-border)); +} +.lqos-funnel-chart-panel { + height: 100%; + border: 1px solid color-mix(in srgb, var(--lqos-border) 76%, transparent); + border-radius: var(--lqos-radius-sm); + background: color-mix(in srgb, var(--lqos-surface-2) 68%, transparent); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.04); + padding: 0.35rem; +} @media (max-width: 991.98px) { .lqos-circuit-activity-header { flex-direction: column; @@ -633,6 +961,27 @@ html[data-bs-theme="light"] .sidebar .navbar-toggler-icon { .lqos-circuit-activity-chart { height: 296px; } + .lqos-circuit-devices-header, + .lqos-cake-header { + flex-direction: column; + } + .lqos-circuit-devices-meta, + .lqos-cake-meta { + justify-content: flex-start; + } + .lqos-cake-info-card { + grid-template-columns: 1fr; + } + .lqos-funnel-header, + .lqos-funnel-path-top, + .lqos-funnel-node-header { + flex-direction: column; + } + .lqos-funnel-meta, + .lqos-funnel-path-meta, + .lqos-funnel-node-badges { + justify-content: flex-start; + } } @media (max-width: 575.98px) { .lqos-circuit-activity-panel { @@ -648,6 +997,16 @@ html[data-bs-theme="light"] .sidebar .navbar-toggler-icon { .lqos-circuit-activity-kpi-value { font-size: 1.05rem; } + .lqos-circuit-devices-panel, + .lqos-cake-panel { + padding: 0.8rem; + } + .lqos-circuit-device-card, + .lqos-funnel-panel, + .lqos-funnel-node-card, + .lqos-funnel-path-card { + padding: 0.8rem; + } } .tiny { font-size: 0.72rem; @@ -791,6 +1150,19 @@ h5 { .dash-table .muted-meta { color: var(--bs-tertiary-color); } +.lqos-topn-plain thead th { + padding: 0.32rem 0.46rem; + line-height: 1; +} +.lqos-topn-plain tbody td { + border-top: 0; + padding: 0.22rem 0.46rem; + line-height: 1; +} +.lqos-topn-plain tbody tr:nth-child(2n) td, +.lqos-topn-plain tbody tr:hover td { + background: transparent; +} .lqos-limit-block { display: flex; flex-direction: column; @@ -858,22 +1230,358 @@ h5 { .lqos-asn-cell .lqos-table-cell-ellipsis { max-width: 100%; } +.lqos-topflow-table { + width: 100%; + table-layout: fixed; +} +.lqos-topflow-table th, +.lqos-topflow-table td { + white-space: nowrap; +} +.lqos-topflow-table .lqos-topflow-key-cell, +.lqos-topflow-table .lqos-topflow-protocol-cell { + white-space: nowrap; +} +.lqos-topflow-table .lqos-topflow-key-cell { + width: 18%; +} +.lqos-topflow-table .lqos-topflow-protocol-cell { + width: 9%; +} +.lqos-topflow-table .lqos-topflow-key-cell .lqos-table-cell-ellipsis { + max-width: 7.5rem; +} +.lqos-topflow-table .lqos-topflow-protocol-cell .lqos-table-cell-ellipsis { + max-width: 100%; +} +.lqos-topflow-table .lqos-topflow-rate-cell { + width: 4.75rem; +} +.lqos-topflow-table .lqos-topflow-total-cell { + width: 5.5rem; +} +.lqos-topflow-table .lqos-topflow-rtt-cell { + width: 3.9rem; +} +.lqos-topflow-table .lqos-topflow-retrans-cell { + width: 4.5rem; +} +.lqos-topflow-table .lqos-asn-cell { + width: 7.5rem; + max-width: 7.5rem; +} +.lqos-circuit-traffic-table { + width: 100%; + table-layout: fixed; +} + +.lqos-circuit-traffic-table th, +.lqos-circuit-traffic-table td { + white-space: nowrap; +} + +.lqos-circuit-traffic-table .lqos-circuit-traffic-protocol-cell { + width: 10rem; + max-width: 10rem; +} + +.lqos-circuit-traffic-table .lqos-circuit-traffic-asn-cell { + width: 12rem; + max-width: 12rem; +} + +.lqos-circuit-traffic-table .lqos-circuit-traffic-country-cell { + width: 8rem; + max-width: 8rem; +} + +.lqos-circuit-traffic-table .lqos-circuit-traffic-protocol-cell .lqos-table-cell-ellipsis, +.lqos-circuit-traffic-table .lqos-circuit-traffic-asn-cell .lqos-table-cell-ellipsis, +.lqos-circuit-traffic-table .lqos-circuit-traffic-country-cell .lqos-table-cell-ellipsis { + max-width: 100%; +} + .lqos-tree-stack { display: flex; flex-direction: column; gap: 0.8rem; margin-top: 0.8rem; } +.lqos-tree-node-cell { + min-width: 17rem; +} +.lqos-tree-node-wrap { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.75rem; + min-width: 0; +} +.lqos-tree-node-lead { + display: flex; + align-items: center; + gap: 0.35rem; + min-width: 0; + flex: 1 1 auto; +} +.lqos-tree-toggle-slot { + width: 1.2rem; + flex: 0 0 1.2rem; + display: flex; + align-items: center; + justify-content: center; +} +.lqos-tree-toggle { + line-height: 1; +} +.lqos-tree-node-link { + min-width: 0; +} +.lqos-tree-circuit-name { + display: block; + max-width: 14rem; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.lqos-tree-plan-column, +.lqos-tree-plan-cell { + min-width: 8.25rem; + white-space: nowrap; +} +.lqos-tree-ip-cell { + white-space: nowrap; +} +.lqos-tree-row-statuses { + display: inline-flex; + align-items: center; + gap: 0.2rem; + flex: 0 0 auto; +} +.lqos-tree-row-status-icon { + color: var(--bs-secondary-color); + opacity: 0.95; + transition: opacity 120ms ease, color 120ms ease; +} +.lqos-tree-row-status-icon.is-inactive { + opacity: 0.35; +} +.lqos-tree-hero { + padding: 0.75rem 1rem; + border: 1px solid var(--lqos-border); + border-radius: var(--lqos-radius-md); + background: linear-gradient( + 180deg, + color-mix(in srgb, var(--lqos-surface-2) 82%, transparent) 0%, + color-mix(in srgb, var(--lqos-surface-solid) 98%, transparent) 100% + ); + box-shadow: var(--lqos-shadow-sm); +} +.lqos-tree-kicker { + display: inline-flex; + align-items: center; + gap: 0.45rem; + color: var(--bs-secondary-text-emphasis); + font-size: 1.55rem; + font-weight: 700; + letter-spacing: 0; + line-height: 1.2; + text-transform: none; +} +.lqos-tree-hero-bar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.85rem; + margin-top: 0.65rem; +} +.lqos-tree-hero-breadcrumb { + min-width: 0; +} +.lqos-tree-hero-status { + display: flex; + align-items: center; + justify-content: flex-end; +} +.lqos-tree-context-meta { + display: inline-flex; + align-items: center; + justify-content: flex-end; + flex-wrap: wrap; + gap: 0.45rem; +} +.lqos-tree-context-pill { + display: inline-flex; + align-items: center; + padding: 0.32rem 0.68rem; + border-radius: 999px; + border: 1px solid color-mix(in srgb, var(--lqos-border) 82%, transparent); + background: color-mix(in srgb, var(--lqos-surface-2) 90%, transparent); + color: var(--bs-secondary-color); + font-size: 0.8rem; + font-weight: 600; + white-space: nowrap; +} +.lqos-tree-compat-indicator { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 0.35rem; + min-width: 1.75rem; + height: 1.75rem; + padding: 0 0.5rem; + border-radius: 999px; + border: 1px solid color-mix(in srgb, var(--bs-warning) 48%, var(--lqos-border)); + background: color-mix(in srgb, var(--bs-warning-bg-subtle, #3a2d00) 72%, transparent); + color: var(--bs-warning-text-emphasis, var(--bs-warning)); + font-size: 0.82rem; + font-weight: 700; + cursor: help; +} +.lqos-tree-compat-count { + font-size: 0.76rem; + line-height: 1; +} +.lqos-tree-panel { + height: 100%; + display: flex; + flex-direction: column; + gap: 0.85rem; + min-height: 230px; + border: 1px solid var(--lqos-border); + border-radius: var(--lqos-radius-md); + background: linear-gradient( + 180deg, + color-mix(in srgb, var(--lqos-surface-2) 88%, transparent) 0%, + var(--lqos-surface-solid) 100% + ); + box-shadow: var(--lqos-shadow-sm); + padding: 0.85rem; +} +.lqos-tree-panel-heading { + min-height: 1.75rem; +} +.lqos-tree-panel h5, +.lqos-tree-snapshot-panel h5 { + font-size: 1.15rem; + font-weight: 650; +} +.lqos-tree-snapshot-panel { + display: flex; + flex-direction: column; + align-items: stretch; + justify-content: flex-start; + gap: 0.85rem; + padding: 0.85rem 0.85rem 0.6rem; +} +.lqos-tree-snapshot-panel .lqos-tree-panel-heading { + margin-bottom: 0; +} +.lqos-tree-snapshot-panel .lqos-circuit-gauge-grid { + flex: 1 1 auto; + align-content: center; +} +.lqos-tree-state-panel .lqos-table-wrap, +.lqos-tree-details-panel .lqos-table-wrap { + height: 100%; +} +.lqos-tree-detail-value { + display: inline-flex; + align-items: center; + flex-wrap: wrap; + gap: 0.4rem; +} +.lqos-tree-editor { + display: flex; + flex-direction: column; + gap: 0.85rem; + padding-top: 0.85rem; + border-top: 1px solid color-mix(in srgb, var(--lqos-border) 72%, transparent); +} +.lqos-tree-editor-heading-end { + display: inline-flex; + align-items: center; + justify-content: flex-end; + margin-left: auto; + min-height: 1.75rem; +} +.lqos-tree-editor[hidden] { + display: none !important; +} +.lqos-tree-editor #nodeOverrideFlash .alert:last-child, +.lqos-tree-details-panel #nodeOverrideDisabledReason .alert:last-child { + margin-bottom: 0; +} +.lqos-tree-editor #nodeOverrideFlash:empty, +.lqos-tree-details-panel #nodeOverrideDisabledReason:empty { + display: none; +} +.lqos-tree-editor-row { + display: flex; + align-items: center; + gap: 0.75rem; + flex-wrap: nowrap; + padding: 0.65rem 0.75rem; + border: 1px solid color-mix(in srgb, var(--lqos-border) 78%, transparent); + border-radius: var(--lqos-radius-md); + background: color-mix(in srgb, var(--lqos-surface-2) 76%, transparent); +} +.lqos-tree-editor-field { + flex: 1 1 0; + min-width: 0; +} +.lqos-tree-editor-field .input-group-text { + min-width: 5.9rem; + justify-content: center; + background: color-mix(in srgb, var(--lqos-surface-2) 88%, transparent); + border-color: color-mix(in srgb, var(--lqos-border) 78%, transparent); + color: var(--bs-secondary-color); + font-size: 0.82rem; + font-weight: 600; +} +.lqos-tree-editor-actions { + justify-content: flex-start; + flex: 0 0 auto; + white-space: nowrap; +} +.lqos-tree-state-badges { + display: inline-flex; + align-items: center; + flex-wrap: wrap; + gap: 0.35rem; +} +.lqos-tree-state-badges .fa { + display: inline-flex; + align-items: center; + justify-content: center; + width: 1.9rem; + height: 1.9rem; + border-radius: 999px; + border: 1px solid color-mix(in srgb, var(--lqos-border) 82%, transparent); + background: color-mix(in srgb, var(--lqos-surface-2) 88%, transparent); + box-shadow: var(--lqos-shadow-sm); +} +.lqos-tree-pending { + display: inline-flex; + align-items: center; + gap: 0.35rem; + color: var(--bs-warning-text-emphasis, var(--bs-warning)); + font-size: 0.88rem; + font-weight: 600; +} +.lqos-tree-pending-symbol { + font-size: 1rem; + line-height: 1; +} .lqos-tree-nav { display: flex; align-items: center; - margin-top: 0.15rem; } .lqos-tree-nav a { display: inline-flex; align-items: center; gap: 0.55rem; - padding: 0.55rem 0.85rem; + padding: 0.4rem 0.7rem; border-radius: 999px; border: 1px solid var(--lqos-border); background: color-mix(in srgb, var(--lqos-surface-2) 90%, transparent); @@ -889,6 +1597,28 @@ h5 { color: var(--bs-secondary-color); font-size: 0.82rem; } +.lqos-tree-breadcrumb { + display: inline-flex; + flex-wrap: wrap; + align-items: center; + gap: 0.45rem; +} +.lqos-tree-breadcrumb-link { + color: var(--bs-body-color); + text-decoration: none; +} +.lqos-tree-breadcrumb-link:hover { + color: var(--bs-body-color); + text-decoration: underline; +} +.lqos-tree-breadcrumb-current { + font-weight: 600; + color: var(--bs-body-color); +} +.lqos-tree-breadcrumb-separator { + color: var(--bs-secondary-color); + font-size: 0.72rem; +} .lqos-tree-section-label { display: inline-flex; align-items: center; @@ -900,6 +1630,27 @@ h5 { letter-spacing: 0.08em; text-transform: uppercase; } +@media (max-width: 991.98px) { + .lqos-tree-hero-status { + justify-content: flex-start; + } + .lqos-tree-hero-bar { + flex-direction: column; + align-items: flex-start; + } + .lqos-tree-context-meta { + justify-content: flex-start; + } +} +@media (max-width: 767.98px) { + .lqos-tree-editor-row { + flex-wrap: wrap; + } + .lqos-tree-editor-field, + .lqos-tree-editor-actions { + width: 100%; + } +} /* Funky tricks for the tree view */ .overlayThroughputWrapper { @@ -978,6 +1729,142 @@ table tr td a { /* For the config panel */ +.lqos-config-page { + display: grid; + gap: 1rem; +} + +.lqos-config-panel { + border: 1px solid var(--lqos-border); + border-radius: var(--lqos-radius-lg); + background: var(--lqos-surface); + box-shadow: var(--lqos-shadow-sm); + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + padding: 1.1rem 1.15rem; +} + +.lqos-config-panel-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.75rem; + margin-bottom: 1rem; +} + +.lqos-config-panel-title { + font-size: 1.05rem; + font-weight: 700; + margin: 0; +} + +.lqos-config-panel-subtitle { + margin-top: 0.2rem; + color: var(--bs-secondary-color); + font-size: 0.92rem; +} + +.lqos-config-grid { + display: grid; + gap: 1rem; +} + +.lqos-config-section { + border: 1px solid var(--lqos-border); + border-radius: var(--lqos-radius-md); + background: color-mix(in srgb, var(--lqos-surface-2) 82%, transparent); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.03); + padding: 1rem 1rem 0.9rem; +} + +.lqos-config-section-title { + font-size: 0.95rem; + font-weight: 700; + margin: 0 0 0.25rem; +} + +.lqos-config-section-subtitle { + color: var(--bs-secondary-color); + font-size: 0.87rem; + margin-bottom: 1rem; +} + +.lqos-config-note { + border: 1px solid color-mix(in srgb, var(--bs-info) 24%, var(--lqos-border)); + border-radius: var(--lqos-radius-md); + background: color-mix(in srgb, var(--bs-info-bg-subtle) 70%, var(--lqos-surface) 30%); + color: var(--bs-body-color); + padding: 0.85rem 1rem; +} + +.lqos-config-note-warning { + border-color: color-mix(in srgb, var(--bs-warning) 30%, var(--lqos-border)); + background: color-mix(in srgb, var(--bs-warning-bg-subtle) 76%, var(--lqos-surface) 24%); +} + +.lqos-config-note code { + color: inherit; +} + +.lqos-config-preview { + border: 1px solid var(--lqos-border); + border-radius: var(--lqos-radius-md); + background: color-mix(in srgb, var(--lqos-surface-2) 86%, transparent); + padding: 0.75rem 0.85rem; +} + +.lqos-config-actions { + display: flex; + align-items: center; + gap: 0.75rem; + margin-top: 1rem; +} + +.lqos-config-panel .card { + border: 1px solid var(--lqos-border); + border-radius: var(--lqos-radius-md); + background: color-mix(in srgb, var(--lqos-surface-2) 86%, transparent); + box-shadow: none; +} + +.lqos-config-panel .card-header { + background: transparent; + border-bottom-color: var(--lqos-border); + font-weight: 700; +} + +.lqos-config-panel .form-text, +.lqos-config-panel .text-muted, +.lqos-config-panel .small.text-muted { + color: var(--bs-secondary-color) !important; +} + +.lqos-config-network-editor { + display: grid; + gap: 0.9rem; +} + +.lqos-config-network-editor .lqos-network-node-card { + background: color-mix(in srgb, var(--lqos-surface-2) 88%, transparent); +} + +.lqos-config-network-editor .card-title { + font-size: 1rem; + font-weight: 700; +} + +.lqos-config-network-editor .card-body { + padding: 1rem 1rem 0.95rem; +} + +.lqos-config-network-editor .btn { + white-space: nowrap; +} + +.lqos-config-network-editor .badge { + font-weight: 600; +} + .nav-pills .nav-link { color: var(--bs-secondary-color); } @@ -1012,19 +1899,35 @@ table tr td a { .config-menu-item { margin: 0; - padding: 5px 10px; - background-color: var(--bs-tertiary-bg); + list-style: none; +} + +.config-menu-link { + display: inline-flex; + align-items: center; + gap: 0.45rem; + width: 100%; + padding: 6px 11px; + background: color-mix(in srgb, var(--lqos-surface-2) 80%, transparent); + border: 1px solid var(--lqos-border); border-radius: 999px; + color: inherit; cursor: pointer; + user-select: none; + -webkit-user-select: none; + transition: background-color 120ms ease, border-color 120ms ease, color 120ms ease, transform 120ms ease; } -.config-menu-item:hover { - background-color: var(--bs-info-bg-subtle); +.config-menu-link:hover { + background: color-mix(in srgb, var(--bs-info-bg-subtle) 72%, var(--lqos-surface) 28%); + border-color: color-mix(in srgb, var(--bs-info) 26%, var(--lqos-border)); + transform: translateY(-1px); } -.config-menu-item.active { - background-color: var(--bs-info); - color: var(--bs-info-text); +.config-menu-link.active { + background: color-mix(in srgb, var(--bs-info) 20%, var(--lqos-surface) 80%); + color: var(--bs-info-text-emphasis, var(--bs-info)); + border-color: color-mix(in srgb, var(--bs-info) 44%, var(--lqos-border)); font-weight: bold; } @@ -1205,3 +2108,83 @@ table tr td a { font-weight: 100 900; font-display: swap; } + +.bakery-pipeline-layout { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.bakery-pipeline-row { + display: grid; + gap: 0.5rem; + grid-template-columns: minmax(0, 1fr); +} + +.bakery-pipeline-cell { + min-width: 0; +} + +.bakery-pipeline-stage { + min-height: 6.6rem; + padding: 0.5rem 0.65rem; + gap: 0.25rem; +} + +.bakery-pipeline-stage-top, +.bakery-pipeline-stage-title, +.bakery-pipeline-stage-status, +.bakery-pipeline-stage-footer { + min-width: 0; +} + +.bakery-pipeline-stage-title { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + line-height: 1.15; +} + +.bakery-pipeline-stage-icon { + font-size: 0.875rem; +} + +.bakery-pipeline-stage-status { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + line-height: 1.3; + min-height: 1.3em; +} + +.bakery-pipeline-stage-footer { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + min-height: 1.2rem; + line-height: 1.2; +} + +@media (max-width: 767.98px) { + .bakery-pipeline-stage { + min-height: 7rem; + } + + .bakery-pipeline-stage-status { + display: -webkit-box; + -webkit-box-orient: vertical; + -webkit-line-clamp: 2; + white-space: normal; + min-height: 2.6em; + } +} + +@media (min-width: 768px) { + .bakery-pipeline-row-top { + grid-template-columns: repeat(3, minmax(0, 1fr)); + } + + .bakery-pipeline-row-bottom { + grid-template-columns: repeat(3, minmax(0, 1fr)); + } +} diff --git a/src/rust/lqosd/src/node_manager/static2/site_map.html b/src/rust/lqosd/src/node_manager/static2/site_map.html index e677b001..0fdf6717 100644 --- a/src/rust/lqosd/src/node_manager/static2/site_map.html +++ b/src/rust/lqosd/src/node_manager/static2/site_map.html @@ -176,6 +176,8 @@ } .site-map-size-dot { + width: 12px; + height: 12px; border-radius: 999px; border: 1px solid rgba(255,255,255,0.45); background: radial-gradient(circle at 35% 35%, rgba(255,255,255,0.9) 0%, rgba(100, 160, 255, 0.95) 42%, rgba(60, 108, 196, 0.92) 100%); @@ -238,6 +240,13 @@ font-size: 0.86rem; } +.site-map-inline-list { + display: flex; + flex-direction: column; + gap: 6px; + margin-bottom: 10px; +} + .site-map-empty { color: var(--bs-secondary-color); font-size: 0.88rem; @@ -247,6 +256,49 @@ min-width: 220px; } +.site-map-site-label-marker, +.site-map-cluster-count-marker { + user-select: none; + -webkit-user-select: none; + pointer-events: none; +} + +.site-map-site-label-marker { + max-width: 220px; + padding: 3px 10px; + border-radius: 999px; + border: 1px solid color-mix(in srgb, var(--lqos-border) 82%, transparent); + background: color-mix(in srgb, var(--lqos-surface-solid) 92%, transparent); + box-shadow: var(--lqos-shadow-sm); + color: var(--bs-body-color); + font-size: 0.78rem; + font-weight: 700; + line-height: 1.1; + text-align: center; + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; + backdrop-filter: blur(10px); + -webkit-backdrop-filter: blur(10px); +} + +.site-map-site-label-marker.is-selected { + border-color: color-mix(in srgb, var(--bs-primary) 60%, var(--lqos-border)); + background: color-mix(in srgb, var(--bs-primary) 22%, var(--lqos-surface-solid)); + box-shadow: 0 12px 28px rgba(37, 99, 235, 0.22); +} + +.site-map-cluster-count-marker { + min-width: 24px; + padding: 2px 6px; + color: #ffffff; + font-size: 0.78rem; + font-weight: 800; + line-height: 1; + text-align: center; + text-shadow: 0 1px 2px rgba(0, 0, 0, 0.35); +} + .maplibregl-popup-content { border-radius: 14px; border: 1px solid var(--lqos-border); @@ -335,6 +387,7 @@
    Click a site or AP marker to inspect it.
    +
    @@ -345,16 +398,6 @@ Poor Healthy
    -
    Marker Size
    -
    - - - -
    Combined recent throughput over the last 30 seconds.
    -
    -
    - Zoomed out views emphasize sites. As you zoom in, APs take over where coordinate coverage allows. -
    diff --git a/src/rust/lqosd/src/node_manager/static2/template.html b/src/rust/lqosd/src/node_manager/static2/template.html index 5eaae9df..1c092f4f 100644 --- a/src/rust/lqosd/src/node_manager/static2/template.html +++ b/src/rust/lqosd/src/node_manager/static2/template.html @@ -139,6 +139,7 @@ %%URGENT_STATUS%% %%SCHEDULER_STATUS%% + %%UPDATE_STATUS%%
  • @@ -167,6 +168,14 @@
    • + + @@ -261,7 +270,7 @@ ", + "", js_tf(script_has_lts), js_tf(script_has_insight), js_tf(script_has_support_tickets), - js_tf(new_version), node_id_js, rtt_thresholds.green_ms, rtt_thresholds.yellow_ms, @@ -218,6 +217,19 @@ pub async fn apply_templates( "##; let byte_string = byte_string.replace("%%URGENT_STATUS%%", urgent_placeholder); + let update_placeholder = if new_version { + r##" + +"## + } else { + "" + }; + let byte_string = byte_string.replace("%%UPDATE_STATUS%%", update_placeholder); + let byte_string = byte_string.replace("%CACHEBUSTERS%", &format!("?gh={}", GIT_HASH)); if let Some(length) = res_parts.headers.get_mut("content-length") { *length = HeaderValue::from(byte_string.len()); diff --git a/src/rust/lqosd/src/node_manager/ws.rs b/src/rust/lqosd/src/node_manager/ws.rs index aa53d983..6ddbf2b7 100644 --- a/src/rust/lqosd/src/node_manager/ws.rs +++ b/src/rust/lqosd/src/node_manager/ws.rs @@ -10,9 +10,10 @@ use std::sync::Arc; use crate::node_manager::auth::{LoginResult, login_from_token}; use crate::node_manager::local_api::{ - circuit, circuit_count, config, cpu_affinity, dashboard_themes, device_counts, flow_explorer, - flow_map, lts, network_tree, packet_analysis, reload_libreqos, scheduler, search, - shaped_device_api, unknown_ips, urgent, warnings, + circuit, circuit_count, config, cpu_affinity, dashboard_themes, device_counts, directories, + ethernet_caps, executive, flow_explorer, flow_map, lts, network_tree, network_tree_lite, + node_rate_overrides, packet_analysis, reload_libreqos, scheduler, search, shaped_device_api, + shaped_devices_page, unknown_ips, urgent, warnings, }; use crate::node_manager::shaper_queries_actor::ShaperQueryCommand; use crate::node_manager::ws::messages::{ @@ -20,6 +21,12 @@ use crate::node_manager::ws::messages::{ }; use crate::node_manager::ws::publish_subscribe::PubSub; use crate::node_manager::ws::published_channels::PublishedChannels; +use crate::node_manager::ws::single_user_channels::circuit::{ + circuit_devices_result, circuit_devices_snapshot, +}; +use crate::node_manager::ws::single_user_channels::flows_by_circuit::{ + circuit_flow_sankey_result, circuit_top_asns_result, circuit_traffic_flows_result, +}; use crate::node_manager::ws::ticker::channel_ticker; use crate::system_stats::SystemStats; use axum::{ @@ -181,7 +188,11 @@ async fn handle_socket( } } Some(Err(err)) => { - warn!("Websocket recv error: {err}"); + if is_benign_recv_error(&err, handshake_complete) { + info!("Websocket client disconnected: {err}"); + } else { + warn!("Websocket recv error: {err}"); + } break; } None => break, // The channel has closed @@ -194,6 +205,14 @@ async fn handle_socket( info!("Websocket disconnected"); } +fn is_benign_recv_error(err: &axum::Error, handshake_complete: bool) -> bool { + if !handshake_complete { + return false; + } + let err_text = err.to_string(); + err_text.contains("Connection reset by peer") || err_text.contains("Broken pipe") +} + struct WsRequestState<'a> { private_state: &'a mut single_user_channels::PrivateState, login: &'a mut LoginResult, @@ -222,7 +241,11 @@ async fn receive_channel_message( return false; } Message::Close(frame) => { - warn!("Websocket close frame received: {:?}", frame); + if is_benign_close_frame(frame.as_ref()) { + info!("Websocket close frame received: {:?}", frame); + } else { + warn!("Websocket close frame received: {:?}", frame); + } return true; } }; @@ -355,6 +378,38 @@ async fn receive_channel_message( return true; } } + WsRequest::ShapedDevicesPage { query } => { + let response = WsResponse::ShapedDevicesPage { + data: shaped_devices_page::shaped_devices_page(query), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + WsRequest::EthernetCapsPage { query } => { + let response = WsResponse::EthernetCapsPage { + data: ethernet_caps::ethernet_caps_page(query), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + WsRequest::ExecutiveHeatmapPage { query } => { + let response = WsResponse::ExecutiveHeatmapPage { + data: executive::executive_heatmap_page(query), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + WsRequest::ExecutiveLeaderboardPage { query } => { + let response = WsResponse::ExecutiveLeaderboardPage { + data: executive::executive_leaderboard_page(query), + }; + if send_ws_response(&tx, response).await { + return true; + } + } WsRequest::NetworkTree => { let response = WsResponse::NetworkTree { data: network_tree::network_tree_data(), @@ -363,6 +418,14 @@ async fn receive_channel_message( return true; } } + WsRequest::NetworkTreeLite => { + let response = WsResponse::NetworkTreeLite { + data: network_tree_lite::network_tree_lite_data(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } WsRequest::FlowMap => { let response = WsResponse::FlowMap { data: flow_map::flow_map_data(), @@ -372,11 +435,45 @@ async fn receive_channel_message( } } WsRequest::CircuitById { id } => { - let (ok, devices) = match circuit::circuit_by_id_data(&id) { - Some(devices) => (true, devices), - None => (false, Vec::new()), + let (ok, data) = match circuit::circuit_by_id_data(&id) { + Some(data) => (true, Some(data)), + None => (false, None), + }; + let response = WsResponse::CircuitByIdResult { id, data, ok }; + if send_ws_response(&tx, response).await { + return true; + } + } + WsRequest::CircuitDevices { circuit } => { + let devices = circuit_devices_snapshot(&circuit, request_state.private_state.bus_tx()).await; + let response = circuit_devices_result(circuit, devices); + if send_ws_response(&tx, response).await { + return true; + } + } + WsRequest::CircuitFlowSankey { circuit } => { + let response = WsResponse::CircuitFlowSankeyResult { + circuit_id: circuit.clone(), + flows: circuit_flow_sankey_result(&circuit), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + WsRequest::CircuitTopAsns { query } => { + let response = WsResponse::CircuitTopAsnsResult { + circuit_id: query.circuit.clone(), + data: circuit_top_asns_result(&query), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + WsRequest::CircuitTrafficFlowsPage { query } => { + let response = WsResponse::CircuitTrafficFlowsPageResult { + circuit_id: query.circuit.clone(), + data: circuit_traffic_flows_result(&query), }; - let response = WsResponse::CircuitByIdResult { id, devices, ok }; if send_ws_response(&tx, response).await { return true; } @@ -428,6 +525,14 @@ async fn receive_channel_message( return true; } } + WsRequest::CpuAffinityRuntimeSnapshot => { + let response = WsResponse::CpuAffinityRuntimeSnapshot { + data: cpu_affinity::cpu_affinity_runtime_snapshot_data(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } WsRequest::CpuAffinityCircuits { cpu, direction, @@ -931,6 +1036,34 @@ async fn receive_channel_message( return true; } } + WsRequest::LtsStartSignup => { + let result = lts::lts_trial_start_signup_data().await; + match result { + Ok(claim_id) => { + lts::spawn_signup_poll_loop(claim_id.clone()); + let response = WsResponse::LtsStartSignupResult { claim_id }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(StatusCode::FORBIDDEN) => { + let response = WsResponse::Error { + message: "Unauthorized".to_string(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(_) => { + let response = WsResponse::Error { + message: "Unable to start the Insight signup session.".to_string(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + } + } WsRequest::LtsSignUp { license_key } => { let result = lts::lts_trial_signup_data(license_key).await; let (ok, message) = match result { @@ -1351,6 +1484,17 @@ async fn receive_channel_message( return true; } } + WsRequest::UpdateNetworkJsonOnly { network_json } => { + let result = config::update_network_json_only_data(*request_state.login, network_json); + let (ok, message) = match result { + Ok(()) => (true, "Ok".to_string()), + Err(message) => (false, message), + }; + let response = WsResponse::UpdateNetworkJsonOnlyResult { ok, message }; + if send_ws_response(&tx, response).await { + return true; + } + } WsRequest::UpdateNetworkAndDevices { network_json, shaped_devices, @@ -1362,14 +1506,153 @@ async fn receive_channel_message( ); let (ok, message) = match result { Ok(()) => (true, "Ok".to_string()), - Err(StatusCode::FORBIDDEN) => (false, "Unauthorized".to_string()), - Err(_) => (false, "Error".to_string()), + Err(message) => (false, message), }; let response = WsResponse::UpdateNetworkAndDevicesResult { ok, message }; if send_ws_response(&tx, response).await { return true; } } + WsRequest::GetNodeRateOverride { query } => { + match node_rate_overrides::get_node_rate_override_data(*request_state.login, query) { + Ok(data) => { + let response = WsResponse::GetNodeRateOverride { data }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(StatusCode::FORBIDDEN) => { + let response = WsResponse::Error { + message: "Unauthorized".to_string(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(StatusCode::BAD_REQUEST) => { + let response = WsResponse::Error { + message: "Invalid tree node override request".to_string(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(_) => { + let response = WsResponse::Error { + message: "Unable to load tree node override state".to_string(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + } + } + WsRequest::SetNodeRateOverride { update } => { + let result = + node_rate_overrides::set_node_rate_override_data(*request_state.login, update); + match result { + Ok(data) => { + let response = WsResponse::SetNodeRateOverrideResult { + ok: true, + message: "Override saved".to_string(), + data, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(StatusCode::FORBIDDEN) => { + let response = WsResponse::SetNodeRateOverrideResult { + ok: false, + message: "Unauthorized".to_string(), + data: node_rate_overrides::NodeRateOverrideData { + writable: false, + can_edit: false, + disabled_reason: Some( + "Only administrators can edit node rate overrides.".to_string(), + ), + has_override: false, + override_node_id: None, + override_download_bandwidth_mbps: None, + override_upload_bandwidth_mbps: None, + legacy_warnings: Vec::new(), + }, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(StatusCode::BAD_REQUEST) => { + let response = WsResponse::Error { + message: "Invalid tree node override payload".to_string(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(_) => { + let response = WsResponse::SetNodeRateOverrideResult { + ok: false, + message: "Error saving override".to_string(), + data: node_rate_overrides::NodeRateOverrideData { + writable: true, + can_edit: false, + disabled_reason: Some( + "Unable to reload tree node override state.".to_string(), + ), + has_override: false, + override_node_id: None, + override_download_bandwidth_mbps: None, + override_upload_bandwidth_mbps: None, + legacy_warnings: Vec::new(), + }, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + } + } + WsRequest::ClearNodeRateOverride { query } => { + let result = + node_rate_overrides::clear_node_rate_override_data(*request_state.login, query); + match result { + Ok(data) => { + let response = WsResponse::ClearNodeRateOverrideResult { + ok: true, + message: "Override cleared".to_string(), + data, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(StatusCode::FORBIDDEN) => { + let response = WsResponse::Error { + message: "Unauthorized".to_string(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(StatusCode::BAD_REQUEST) => { + let response = WsResponse::Error { + message: "Invalid tree node override payload".to_string(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(_) => { + let response = WsResponse::Error { + message: "Unable to clear tree node override".to_string(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + } + } WsRequest::ListNics => match config::list_nics_data(*request_state.login) { Ok(data) => { let response = WsResponse::ListNics { data }; @@ -1410,6 +1693,146 @@ async fn receive_channel_message( return true; } } + WsRequest::GetShapedDevice { device_id } => { + match config::get_shaped_device_data(*request_state.login, device_id) { + Ok(device) => { + let response = WsResponse::GetShapedDeviceResult { + ok: device.is_some(), + message: if device.is_some() { + "Ok".to_string() + } else { + "Not found".to_string() + }, + device, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(StatusCode::FORBIDDEN) => { + let response = WsResponse::GetShapedDeviceResult { + ok: false, + message: "Unauthorized".to_string(), + device: None, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(_) => { + let response = WsResponse::GetShapedDeviceResult { + ok: false, + message: "Error".to_string(), + device: None, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + } + } + WsRequest::CreateShapedDevice { device } => { + match config::create_shaped_device_data(*request_state.login, device) { + Ok(device) => { + let response = WsResponse::CreateShapedDeviceResult { + ok: true, + message: "Ok".to_string(), + device: Some(device), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(message) => { + let response = WsResponse::CreateShapedDeviceResult { + ok: false, + message, + device: None, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + } + } + WsRequest::UpdateShapedDevice { + original_device_id, + device, + } => match config::update_shaped_device_data( + *request_state.login, + original_device_id, + device, + ) { + Ok(device) => { + let response = WsResponse::UpdateShapedDeviceResult { + ok: true, + message: "Ok".to_string(), + device: Some(device), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(message) => { + let response = WsResponse::UpdateShapedDeviceResult { + ok: false, + message, + device: None, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + }, + WsRequest::DeleteShapedDevice { device_id } => { + let device_id_clone = device_id.clone(); + match config::delete_shaped_device_data(*request_state.login, device_id) { + Ok(()) => { + let response = WsResponse::DeleteShapedDeviceResult { + ok: true, + message: "Ok".to_string(), + device_id: device_id_clone, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + Err(message) => { + let response = WsResponse::DeleteShapedDeviceResult { + ok: false, + message, + device_id: device_id_clone, + }; + if send_ws_response(&tx, response).await { + return true; + } + } + } + } + WsRequest::CircuitDirectoryPage { query } => { + let response = WsResponse::CircuitDirectoryPage { + data: directories::circuit_directory_page(query), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + WsRequest::NodeDirectory => { + let response = WsResponse::NodeDirectory { + data: directories::node_directory_data(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } + WsRequest::TreeGuardMetadataSummary => { + let response = WsResponse::TreeGuardMetadataSummary { + data: directories::treeguard_metadata_summary(), + }; + if send_ws_response(&tx, response).await { + return true; + } + } WsRequest::GetUsers => match config::get_users_data(*request_state.login) { Ok(data) => { let response = WsResponse::GetUsers { data }; @@ -1509,6 +1932,10 @@ async fn receive_channel_message( false } +fn is_benign_close_frame(frame: Option<&axum::extract::ws::CloseFrame<'_>>) -> bool { + frame.is_some_and(|frame| matches!(frame.code, 1000 | 1001)) +} + async fn send_ws_response(tx: &Sender>>, response: WsResponse) -> bool { let payload = match encode_ws_message(&response) { Ok(payload) => payload, diff --git a/src/rust/lqosd/src/node_manager/ws/messages.rs b/src/rust/lqosd/src/node_manager/ws/messages.rs index 79d6fd37..41afc0e3 100644 --- a/src/rust/lqosd/src/node_manager/ws/messages.rs +++ b/src/rust/lqosd/src/node_manager/ws/messages.rs @@ -1,17 +1,41 @@ use crate::lts2_sys::control_channel::{SupportTicket, SupportTicketSummary}; use crate::node_manager::WarningLevel; +use crate::node_manager::local_api::circuit::CircuitByIdData; +use crate::node_manager::local_api::circuit_activity::{ + CircuitFlowSankeyRow, CircuitSummaryData, CircuitTopAsnsData, CircuitTopAsnsQuery, + CircuitTrafficFlowsPage, CircuitTrafficFlowsQuery, +}; use crate::node_manager::local_api::dashboard_themes::{DashletIdentity, ThemeEntry}; use crate::node_manager::local_api::device_counts::DeviceCount; +use crate::node_manager::local_api::directories::{ + CircuitDirectoryPage, CircuitDirectoryQuery, NodeDirectoryEntry, TreeGuardMetadataSummary, +}; +use crate::node_manager::local_api::ethernet_caps::{EthernetCapsPage, EthernetCapsPageQuery}; +use crate::node_manager::local_api::executive::{ + ExecutiveDashboardSummary, ExecutiveHeatmapPage, ExecutiveHeatmapPageQuery, + ExecutiveLeaderboardPage, ExecutiveLeaderboardPageQuery, +}; +use crate::node_manager::local_api::network_tree_lite::NetworkTreeLiteNode; +use crate::node_manager::local_api::node_rate_overrides::{ + NodeRateOverrideData, NodeRateOverrideQuery, NodeRateOverrideUpdate, +}; use crate::node_manager::local_api::packet_analysis::RequestAnalysisResult; use crate::node_manager::local_api::scheduler::{SchedulerDetails, SchedulerStatus}; use crate::node_manager::local_api::search::SearchResult; +use crate::node_manager::local_api::shaped_devices_page::{ + ShapedDevicesPage, ShapedDevicesPageQuery, +}; +use crate::node_manager::local_api::tree_attached_circuits::{ + TreeAttachedCircuitsPage, TreeAttachedCircuitsQuery, +}; use crate::node_manager::local_api::unknown_ips::{ClearUnknownIpsResponse, UnknownIp}; use crate::node_manager::local_api::urgent::{UrgentList, UrgentStatus}; use crate::node_manager::local_api::{ circuit_count::CircuitCount, + circuit_live::{CircuitLiveMetrics, CircuitMetricsQuery}, cpu_affinity::{ - CircuitBrief, CpuAffinityCircuitsPage, CpuAffinitySiteTreeNode, CpuAffinitySummaryEntry, - PreviewWeightItem, + CircuitBrief, CpuAffinityCircuitsPage, CpuAffinityRuntimeSnapshot, CpuAffinitySiteTreeNode, + CpuAffinitySummaryEntry, PreviewWeightItem, }, flow_explorer::FlowTimeline, lts::{ @@ -26,15 +50,9 @@ use crate::throughput_tracker::TcpRetransmitTotal; use crate::throughput_tracker::flow_data::{ AsnCountryListEntry, AsnListEntry, AsnProtocolListEntry, }; -use crate::throughput_tracker::flow_data::{FlowAnalysis, FlowbeeLocalData}; -use lqos_bus::{ - AsnHeatmapData, Circuit, CircuitHeatmapData, ExecutiveSummaryHeader, FlowbeeSummaryData, - QueueStoreTransit, SiteHeatmapData, StormguardDebugEntry, -}; +use lqos_bus::{Circuit, FlowbeeSummaryData, QueueStoreTransit, StormguardDebugEntry}; use lqos_config::QooProfileInfo; use lqos_config::{Config, NetworkJsonTransport, ShapedDevice, WebUser}; -use lqos_utils::qoq_heatmap::QoqHeatmapBlocks; -use lqos_utils::temporal_heatmap::HeatmapBlocks; use lqos_utils::units::DownUpOrder; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -58,10 +76,15 @@ pub struct WsHelloReply { pub enum PrivateRequest { CircuitWatcher { circuit: String }, PingMonitor { ips: Vec<(String, String)> }, - FlowsByCircuit { circuit: String }, + StopCircuitWatcher, + StopPingMonitorWatch, CakeWatcher { circuit: String }, Chatbot { browser_ts_ms: Option }, ChatbotUserInput { text: String }, + WatchTreeAttachedCircuits { query: TreeAttachedCircuitsQuery }, + StopTreeAttachedCircuitsWatch, + WatchCircuitMetrics { query: CircuitMetricsQuery }, + StopCircuitMetricsWatch, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -90,7 +113,17 @@ pub enum WsRequest { SchedulerDetails, DeviceCount, DevicesAll, + ShapedDevicesPage { + query: ShapedDevicesPageQuery, + }, + ExecutiveHeatmapPage { + query: ExecutiveHeatmapPageQuery, + }, + ExecutiveLeaderboardPage { + query: ExecutiveLeaderboardPageQuery, + }, NetworkTree, + NetworkTreeLite, FlowMap, GlobalWarnings, Search { @@ -99,6 +132,7 @@ pub enum WsRequest { ReloadLibreQoS, LtsTrialConfig, CircuitCount, + LtsStartSignup, LtsSignUp { license_key: String, }, @@ -140,13 +174,46 @@ pub enum WsRequest { UpdateConfig { config: Config, }, + UpdateNetworkJsonOnly { + network_json: Value, + }, UpdateNetworkAndDevices { network_json: Value, shaped_devices: Vec, }, + GetNodeRateOverride { + query: NodeRateOverrideQuery, + }, + SetNodeRateOverride { + update: NodeRateOverrideUpdate, + }, + ClearNodeRateOverride { + query: NodeRateOverrideQuery, + }, ListNics, NetworkJson, AllShapedDevices, + GetShapedDevice { + device_id: String, + }, + CreateShapedDevice { + device: ShapedDevice, + }, + UpdateShapedDevice { + original_device_id: String, + device: ShapedDevice, + }, + DeleteShapedDevice { + device_id: String, + }, + CircuitDirectoryPage { + query: CircuitDirectoryQuery, + }, + EthernetCapsPage { + query: EthernetCapsPageQuery, + }, + NodeDirectory, + TreeGuardMetadataSummary, GetUsers, AddUser { username: String, @@ -164,6 +231,18 @@ pub enum WsRequest { CircuitById { id: String, }, + CircuitDevices { + circuit: String, + }, + CircuitFlowSankey { + circuit: String, + }, + CircuitTopAsns { + query: CircuitTopAsnsQuery, + }, + CircuitTrafficFlowsPage { + query: CircuitTrafficFlowsQuery, + }, SetCircuitRttExcluded { circuit_id: String, excluded: bool, @@ -172,6 +251,7 @@ pub enum WsRequest { ip: String, }, CpuAffinitySummary, + CpuAffinityRuntimeSnapshot, CpuAffinityCircuits { cpu: u32, direction: Option, @@ -260,32 +340,147 @@ pub struct RamData { pub used: u64, } -#[derive(Debug, Serialize)] -pub struct BakeryStatusState { - #[serde(rename = "activeCircuits")] - pub active_circuits: usize, +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BakeryCapacityInterfaceData { + pub name: String, + pub planned_qdiscs: usize, + pub infra_qdiscs: usize, + pub cake_qdiscs: usize, + pub fq_codel_qdiscs: usize, + pub estimated_memory_bytes: u64, } -#[derive(Debug, Serialize)] +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BakeryLiveCapacityInterfaceData { + pub name: String, + pub live_qdiscs: usize, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BakeryPreflightData { + pub ok: bool, + pub message: String, + pub safe_budget: usize, + pub hard_limit: usize, + pub estimated_total_memory_bytes: u64, + pub memory_available_bytes: Option, + pub memory_guard_min_available_bytes: u64, + pub memory_ok: bool, + pub interfaces: Vec, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BakeryStatusState { + pub active_circuits: usize, + pub mode: String, + pub current_action_started_unix: Option, + pub current_apply_phase: Option, + pub current_apply_total_tc_commands: usize, + pub current_apply_completed_tc_commands: usize, + pub current_apply_total_chunks: usize, + pub current_apply_completed_chunks: usize, + pub last_success_unix: Option, + pub last_full_reload_success_unix: Option, + pub last_failure_unix: Option, + pub last_failure_summary: Option, + pub last_apply_type: String, + pub last_total_tc_commands: usize, + pub last_class_commands: usize, + pub last_qdisc_commands: usize, + pub last_build_duration_ms: u64, + pub last_apply_duration_ms: u64, + pub avg_tc_io_interval_ms: Option, + pub last_tc_io_unix: Option, + pub tc_io_interval_samples: usize, + pub runtime_operations: BakeryRuntimeOperationsData, + pub queue_distribution: Vec, + pub live_capacity_interfaces: Vec, + pub live_capacity_safe_budget: usize, + pub live_capacity_updated_at_unix: Option, + pub preflight: Option, + pub reload_required: bool, + pub reload_required_reason: Option, + pub dirty_subtree_count: usize, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] pub struct BakeryStatusData { - #[serde(rename = "currentState")] pub current_state: BakeryStatusState, } +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BakeryQueueDistributionData { + pub queue: u32, + pub top_level_site_count: usize, + pub site_count: usize, + pub circuit_count: usize, + pub download_mbps: u64, + pub upload_mbps: u64, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BakeryRuntimeOperationsData { + pub submitted_count: usize, + pub deferred_count: usize, + pub applying_count: usize, + pub awaiting_cleanup_count: usize, + pub failed_count: usize, + pub dirty_count: usize, + pub latest: Option, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BakeryRuntimeOperationHeadlineData { + pub operation_id: u64, + pub site_hash: i64, + pub site_name: Option, + pub action: String, + pub status: String, + pub attempt_count: u32, + pub updated_at_unix: u64, + pub next_retry_at_unix: Option, + pub last_error: Option, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BakeryActivityEntry { + pub ts: u64, + pub event: String, + pub status: String, + pub summary: String, + pub site_hash: Option, + pub site_name: Option, +} + #[derive(Clone, Debug, Serialize)] pub struct TreeguardStatusData { pub enabled: bool, pub dry_run: bool, + pub paused_for_bakery_reload: bool, + pub pause_reason: Option, pub cpu_max_pct: Option, + pub total_nodes: usize, + pub total_circuits: usize, pub managed_nodes: usize, pub managed_circuits: usize, pub virtualized_nodes: usize, + pub cake_circuits: usize, + pub mixed_sqm_circuits: usize, pub fq_codel_circuits: usize, pub last_action_summary: Option, pub warnings: Vec, } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Default, Serialize)] pub struct TreeguardActivityEntry { pub time: String, pub entity_type: String, @@ -293,6 +488,10 @@ pub struct TreeguardActivityEntry { pub action: String, pub persisted: bool, pub reason: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub batch_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub batch_kind: Option, } #[derive(Debug, Serialize)] @@ -322,41 +521,11 @@ pub struct NodeCapacity { pub median_rtt: f32, } -#[derive(Debug, Serialize)] -pub struct OversubscribedSite { - pub site_name: String, - pub cap_down: f32, - pub cap_up: f32, - pub sub_down: f32, - pub sub_up: f32, - pub ratio_down: Option, - pub ratio_up: Option, - pub ratio_max: Option, -} - -#[derive(Debug, Serialize)] -pub struct ExecutiveHeatmapsData { - pub header: ExecutiveSummaryHeader, - pub global: HeatmapBlocks, - pub global_qoq: QoqHeatmapBlocks, - pub circuits: Vec, - pub sites: Vec, - pub asns: Vec, - pub oversubscribed_sites: Vec, -} - -#[derive(Debug, Serialize)] -pub struct FlowbeeKeyTransit { - pub remote_ip: String, - pub local_ip: String, - pub src_port: u16, - pub dst_port: u16, - pub ip_protocol: u8, - pub device_name: String, - pub asn_name: String, - pub asn_country: String, - pub protocol_name: String, - pub last_seen_nanos: u64, +#[derive(Debug, Serialize, Deserialize)] +pub struct CircuitDevicesResult { + pub circuit_id: String, + pub devices: Vec, + pub ok: bool, } #[derive(Debug, Serialize)] @@ -451,10 +620,59 @@ pub enum WsResponse { ok: bool, message: String, }, + UpdateNetworkJsonOnlyResult { + ok: bool, + message: String, + }, UpdateNetworkAndDevicesResult { ok: bool, message: String, }, + GetShapedDeviceResult { + ok: bool, + message: String, + device: Option, + }, + CreateShapedDeviceResult { + ok: bool, + message: String, + device: Option, + }, + UpdateShapedDeviceResult { + ok: bool, + message: String, + device: Option, + }, + DeleteShapedDeviceResult { + ok: bool, + message: String, + device_id: String, + }, + CircuitDirectoryPage { + data: CircuitDirectoryPage, + }, + EthernetCapsPage { + data: EthernetCapsPage, + }, + NodeDirectory { + data: Vec, + }, + TreeGuardMetadataSummary { + data: TreeGuardMetadataSummary, + }, + GetNodeRateOverride { + data: NodeRateOverrideData, + }, + SetNodeRateOverrideResult { + ok: bool, + message: String, + data: NodeRateOverrideData, + }, + ClearNodeRateOverrideResult { + ok: bool, + message: String, + data: NodeRateOverrideData, + }, GetUsers { data: Vec, }, @@ -476,6 +694,9 @@ pub enum WsResponse { CircuitCountResult { data: CircuitCount, }, + LtsStartSignupResult { + claim_id: String, + }, LtsSignUpResult { ok: bool, message: String, @@ -529,12 +750,24 @@ pub enum WsResponse { DevicesAll { data: Vec, }, + ShapedDevicesPage { + data: ShapedDevicesPage, + }, + ExecutiveDashboardSummary { + data: ExecutiveDashboardSummary, + }, + ExecutiveHeatmapPage { + data: ExecutiveHeatmapPage, + }, + ExecutiveLeaderboardPage { + data: ExecutiveLeaderboardPage, + }, FlowMap { data: Vec<(f64, f64, String, u64, f32)>, }, CircuitByIdResult { id: String, - devices: Vec, + data: Option, ok: bool, }, RequestAnalysisResult { @@ -543,6 +776,9 @@ pub enum WsResponse { CpuAffinitySummary { data: Vec, }, + CpuAffinityRuntimeSnapshot { + data: CpuAffinityRuntimeSnapshot, + }, CpuAffinityCircuits { data: CpuAffinityCircuitsPage, }, @@ -642,9 +878,24 @@ pub enum WsResponse { NetworkTree { data: Vec<(usize, NetworkJsonTransport)>, }, + NetworkTreeLite { + data: Vec<(usize, NetworkTreeLiteNode)>, + }, NetworkTreeClients { data: Vec, }, + TreeAttachedCircuitsSnapshot { + data: TreeAttachedCircuitsPage, + }, + TreeAttachedCircuitsUpdate { + data: TreeAttachedCircuitsPage, + }, + CircuitMetricsSnapshot { + data: Vec, + }, + CircuitMetricsUpdate { + data: Vec, + }, QueueStatsTotal { marks: DownUpOrder, drops: DownUpOrder, @@ -673,20 +924,20 @@ pub enum WsResponse { BakeryStatus { data: BakeryStatusData, }, + BakeryActivity { + data: Vec, + }, TreeGuardStatus { data: TreeguardStatusData, }, TreeGuardActivity { data: Vec, }, - ExecutiveHeatmaps { - data: ExecutiveHeatmapsData, - }, CircuitWatcher { - circuit_id: String, - devices: Vec, - qoo_score: Option, - rtt_excluded: bool, + data: CircuitSummaryData, + }, + CircuitDevicesResult { + data: CircuitDevicesResult, }, SetCircuitRttExcludedResult { ok: bool, @@ -698,9 +949,17 @@ pub enum WsResponse { ip: String, result: PingState, }, - FlowsByCircuit { + CircuitFlowSankeyResult { circuit_id: String, - flows: Vec<(FlowbeeKeyTransit, FlowbeeLocalData, FlowAnalysis)>, + flows: Vec, + }, + CircuitTopAsnsResult { + circuit_id: String, + data: CircuitTopAsnsData, + }, + CircuitTrafficFlowsPageResult { + circuit_id: String, + data: CircuitTrafficFlowsPage, }, CakeWatcher { #[serde(flatten)] diff --git a/src/rust/lqosd/src/node_manager/ws/published_channels.rs b/src/rust/lqosd/src/node_manager/ws/published_channels.rs index 648d983e..e736dcd6 100644 --- a/src/rust/lqosd/src/node_manager/ws/published_channels.rs +++ b/src/rust/lqosd/src/node_manager/ws/published_channels.rs @@ -39,16 +39,18 @@ pub enum PublishedChannels { TreeSummaryL2, QueueStatsTotal, NetworkTree, + NetworkTreeLite, NetworkTreeClients, CircuitCapacity, TreeCapacity, StormguardStatus, StormguardDebug, BakeryStatus, + BakeryActivity, TreeGuardStatus, TreeGuardActivity, EndpointLatLon, AsnTopDownload, AsnTopUpload, - ExecutiveHeatmaps, + ExecutiveDashboardSummary, } diff --git a/src/rust/lqosd/src/node_manager/ws/single_user_channels.rs b/src/rust/lqosd/src/node_manager/ws/single_user_channels.rs index 2401477a..15ffe0cf 100644 --- a/src/rust/lqosd/src/node_manager/ws/single_user_channels.rs +++ b/src/rust/lqosd/src/node_manager/ws/single_user_channels.rs @@ -1,14 +1,17 @@ mod cake_watcher; mod chatbot; -mod circuit; -mod flows_by_circuit; +pub(crate) mod circuit; +mod circuit_metrics; +pub(crate) mod flows_by_circuit; mod ping_monitor; +mod tree_attached_circuits; use crate::node_manager::ws::messages::{PrivateRequest, WsResponse, encode_ws_message}; use crate::node_manager::ws::single_user_channels::cake_watcher::cake_watcher; use crate::node_manager::ws::single_user_channels::circuit::circuit_watcher; -use crate::node_manager::ws::single_user_channels::flows_by_circuit::flows_by_circuit; +use crate::node_manager::ws::single_user_channels::circuit_metrics::watch_circuit_metrics; use crate::node_manager::ws::single_user_channels::ping_monitor::ping_monitor; +use crate::node_manager::ws::single_user_channels::tree_attached_circuits::watch_tree_attached_circuits; use tokio::spawn; use tokio::sync::mpsc::Sender; use tracing::info; @@ -22,6 +25,10 @@ pub struct PrivateState { control_tx: tokio::sync::mpsc::Sender, browser_language: Option, chatbot_request: Option, + circuit_watch: Option>, + ping_monitor_watch: Option>, + tree_attached_circuits_watch: Option>, + circuit_metrics_watch: Option>, } impl PrivateState { @@ -42,6 +49,10 @@ impl PrivateState { control_tx, browser_language, chatbot_request: None, + circuit_watch: None, + ping_monitor_watch: None, + tree_attached_circuits_watch: None, + circuit_metrics_watch: None, } } @@ -51,20 +62,28 @@ impl PrivateState { self.control_tx.clone() } + pub fn bus_tx( + &self, + ) -> Sender<( + tokio::sync::oneshot::Sender, + lqos_bus::BusRequest, + )> { + self.bus_tx.clone() + } + pub async fn handle_request(&mut self, request: PrivateRequest) { match request { PrivateRequest::CircuitWatcher { circuit } => { - spawn(circuit_watcher( - circuit, - self.tx.clone(), - self.bus_tx.clone(), - )); + self.replace_circuit_watch(circuit); } PrivateRequest::PingMonitor { ips } => { - spawn(ping_monitor(ips, self.tx.clone())); + self.replace_ping_monitor_watch(ips); } - PrivateRequest::FlowsByCircuit { circuit } => { - spawn(flows_by_circuit(circuit, self.tx.clone())); + PrivateRequest::StopCircuitWatcher => { + self.abort_circuit_watch(); + } + PrivateRequest::StopPingMonitorWatch => { + self.abort_ping_monitor_watch(); } PrivateRequest::CakeWatcher { circuit } => { spawn(cake_watcher(circuit, self.tx.clone())); @@ -76,6 +95,73 @@ impl PrivateState { PrivateRequest::ChatbotUserInput { text } => { self.forward_chatbot_input(text).await; } + PrivateRequest::WatchTreeAttachedCircuits { query } => { + self.replace_tree_attached_circuits_watch(query); + } + PrivateRequest::StopTreeAttachedCircuitsWatch => { + self.abort_tree_attached_circuits_watch(); + } + PrivateRequest::WatchCircuitMetrics { query } => { + self.replace_circuit_metrics_watch(query); + } + PrivateRequest::StopCircuitMetricsWatch => { + self.abort_circuit_metrics_watch(); + } + } + } + + fn replace_circuit_watch(&mut self, circuit: String) { + self.abort_circuit_watch(); + self.circuit_watch = Some(spawn(circuit_watcher( + circuit, + self.tx.clone(), + self.bus_tx.clone(), + ))); + } + + fn abort_circuit_watch(&mut self) { + if let Some(handle) = self.circuit_watch.take() { + handle.abort(); + } + } + + fn replace_ping_monitor_watch(&mut self, ips: Vec<(String, String)>) { + self.abort_ping_monitor_watch(); + self.ping_monitor_watch = Some(spawn(ping_monitor(ips, self.tx.clone()))); + } + + fn abort_ping_monitor_watch(&mut self) { + if let Some(handle) = self.ping_monitor_watch.take() { + handle.abort(); + } + } + + fn replace_tree_attached_circuits_watch( + &mut self, + query: crate::node_manager::local_api::tree_attached_circuits::TreeAttachedCircuitsQuery, + ) { + self.abort_tree_attached_circuits_watch(); + self.tree_attached_circuits_watch = + Some(spawn(watch_tree_attached_circuits(query, self.tx.clone()))); + } + + fn abort_tree_attached_circuits_watch(&mut self) { + if let Some(handle) = self.tree_attached_circuits_watch.take() { + handle.abort(); + } + } + + fn replace_circuit_metrics_watch( + &mut self, + query: crate::node_manager::local_api::circuit_live::CircuitMetricsQuery, + ) { + self.abort_circuit_metrics_watch(); + self.circuit_metrics_watch = Some(spawn(watch_circuit_metrics(query, self.tx.clone()))); + } + + fn abort_circuit_metrics_watch(&mut self) { + if let Some(handle) = self.circuit_metrics_watch.take() { + handle.abort(); } } @@ -136,6 +222,15 @@ impl PrivateState { } } +impl Drop for PrivateState { + fn drop(&mut self) { + self.abort_circuit_watch(); + self.abort_ping_monitor_watch(); + self.abort_tree_attached_circuits_watch(); + self.abort_circuit_metrics_watch(); + } +} + // JS CBOR encoder emits float64 for timestamps beyond 32-bit ranges; normalize to i64. fn normalize_browser_ts_ms(browser_ts_ms: Option) -> Option { let ts_ms = browser_ts_ms?; diff --git a/src/rust/lqosd/src/node_manager/ws/single_user_channels/circuit.rs b/src/rust/lqosd/src/node_manager/ws/single_user_channels/circuit.rs index 7f29ecd2..db1110f1 100644 --- a/src/rust/lqosd/src/node_manager/ws/single_user_channels/circuit.rs +++ b/src/rust/lqosd/src/node_manager/ws/single_user_channels/circuit.rs @@ -1,13 +1,153 @@ -use crate::node_manager::ws::messages::{WsResponse, encode_ws_message}; +use crate::node_manager::local_api::circuit_activity::{CircuitSummaryData, circuit_flow_counts}; +use crate::node_manager::ws::messages::{CircuitDevicesResult, WsResponse, encode_ws_message}; use crate::node_manager::ws::ticker::all_circuits; use crate::rtt_exclusions; use crate::shaped_devices_tracker::SHAPED_DEVICES; use crate::throughput_tracker::THROUGHPUT_TRACKER; -use lqos_bus::BusRequest; +use lqos_bus::{BusRequest, Circuit}; +use lqos_utils::units::{DownUpOrder, down_up_retransmit_sample}; use std::time::Duration; use tokio::time::MissedTickBehavior; use tracing::info; +const QUEUING_ACTIVITY_RTT_FLOOR_BPS: u64 = 200_000; + +fn qoo_score_for_circuit(circuit: &str) -> Option { + let shaped = SHAPED_DEVICES.load(); + let circuit_hash = shaped + .devices + .iter() + .find(|d| d.circuit_id == circuit) + .map(|d| d.circuit_hash); + circuit_hash.and_then(|hash| { + let qoq_heatmaps = THROUGHPUT_TRACKER.circuit_qoq_heatmaps.lock(); + qoq_heatmaps.get(&hash).and_then(|heatmap| { + let blocks = heatmap.blocks(); + let dl = blocks.download_total.last().copied().flatten(); + let ul = blocks.upload_total.last().copied().flatten(); + match (dl, ul) { + (Some(d), Some(u)) => Some(d.min(u)), + (Some(d), None) => Some(d), + (None, Some(u)) => Some(u), + (None, None) => None, + } + }) + }) +} + +fn weighted_directional_rtt_p50_nanos(devices: &[Circuit], direction: fn(&Circuit) -> (u64, Option)) -> Option { + let mut weighted_entries = Vec::new(); + let mut fallback_values = Vec::new(); + + for device in devices { + let (throughput_bps, rtt_nanos_opt) = direction(device); + let Some(rtt_nanos) = rtt_nanos_opt else { + continue; + }; + fallback_values.push(rtt_nanos); + if throughput_bps > QUEUING_ACTIVITY_RTT_FLOOR_BPS { + weighted_entries.push((rtt_nanos, throughput_bps)); + } + } + + if !weighted_entries.is_empty() { + weighted_entries.sort_by_key(|(rtt, _)| *rtt); + let total_weight: u128 = weighted_entries.iter().map(|(_, weight)| *weight as u128).sum(); + let threshold = total_weight / 2; + let mut running = 0_u128; + for (rtt, weight) in weighted_entries { + running += weight as u128; + if running >= threshold { + return Some(rtt); + } + } + } + + if fallback_values.is_empty() { + return None; + } + fallback_values.sort_unstable(); + let middle = fallback_values.len() / 2; + if fallback_values.len() % 2 == 1 { + Some(fallback_values[middle]) + } else { + Some((fallback_values[middle - 1] + fallback_values[middle]) / 2) + } +} + +fn summarize_circuit_devices(circuit: &str, devices: &[Circuit]) -> CircuitSummaryData { + let bytes_per_second = devices.iter().fold(DownUpOrder::default(), |mut acc, device| { + acc.down += device.bytes_per_second.down; + acc.up += device.bytes_per_second.up; + acc + }); + + let tcp_retransmit_sample = down_up_retransmit_sample( + DownUpOrder { + down: devices + .iter() + .map(|device| device.tcp_retransmit_sample.down.retransmits.get()) + .sum(), + up: devices + .iter() + .map(|device| device.tcp_retransmit_sample.up.retransmits.get()) + .sum(), + }, + DownUpOrder { + down: devices + .iter() + .map(|device| device.tcp_retransmit_sample.down.packets.get()) + .sum(), + up: devices + .iter() + .map(|device| device.tcp_retransmit_sample.up.packets.get()) + .sum(), + }, + ); + + let rtt_current_p50_nanos = DownUpOrder { + down: weighted_directional_rtt_p50_nanos(devices, |device| { + ( + device.bytes_per_second.down.saturating_mul(8), + device.rtt_current_p50_nanos.down, + ) + }), + up: weighted_directional_rtt_p50_nanos(devices, |device| { + ( + device.bytes_per_second.up.saturating_mul(8), + device.rtt_current_p50_nanos.up, + ) + }), + }; + + let (active_flow_count, active_asn_count) = circuit_flow_counts(circuit); + + CircuitSummaryData { + circuit_id: circuit.to_string(), + bytes_per_second, + rtt_current_p50_nanos, + tcp_retransmit_sample, + qoo_score: qoo_score_for_circuit(circuit), + rtt_excluded: rtt_exclusions::is_excluded_circuit_id(circuit), + active_flow_count, + active_asn_count, + } +} + +pub async fn circuit_devices_snapshot( + circuit: &str, + bus_tx: tokio::sync::mpsc::Sender<( + tokio::sync::oneshot::Sender, + BusRequest, + )>, +) -> Vec { + all_circuits(bus_tx) + .await + .into_iter() + .filter(|device| device.circuit_id.as_deref() == Some(circuit)) + .collect() +} + pub(super) async fn circuit_watcher( circuit: String, tx: tokio::sync::mpsc::Sender>>, @@ -20,53 +160,14 @@ pub(super) async fn circuit_watcher( ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); loop { ticker.tick().await; - - // Get all circuits and filter them - let devices_for_circuit: Vec<_> = all_circuits(bus_tx.clone()) - .await - .into_iter() - .filter(|c| { - if let Some(c) = c.circuit_id.as_ref() { - *c == circuit - } else { - false - } - }) - .collect(); - - let qoo_score = { - let shaped = SHAPED_DEVICES.load(); - let circuit_hash = shaped - .devices - .iter() - .find(|d| d.circuit_id == circuit) - .map(|d| d.circuit_hash); - circuit_hash.and_then(|hash| { - let qoq_heatmaps = THROUGHPUT_TRACKER.circuit_qoq_heatmaps.lock(); - qoq_heatmaps.get(&hash).and_then(|heatmap| { - let blocks = heatmap.blocks(); - let dl = blocks.download_total.last().copied().flatten(); - let ul = blocks.upload_total.last().copied().flatten(); - match (dl, ul) { - (Some(d), Some(u)) => Some(d.min(u)), - (Some(d), None) => Some(d), - (None, Some(u)) => Some(u), - (None, None) => None, - } - }) - }) + let devices_for_circuit = circuit_devices_snapshot(&circuit, bus_tx.clone()).await; + let response = WsResponse::CircuitWatcher { + data: summarize_circuit_devices(&circuit, &devices_for_circuit), }; - let result = WsResponse::CircuitWatcher { - circuit_id: circuit.clone(), - devices: devices_for_circuit, - qoo_score, - rtt_excluded: rtt_exclusions::is_excluded_circuit_id(&circuit), - }; - - if let Ok(payload) = encode_ws_message(&result) { + if let Ok(payload) = encode_ws_message(&response) { if tx.send(payload).await.is_err() { - info!("Channel is gone"); + info!("CircuitWatcher channel is gone"); break; } } else { @@ -75,3 +176,13 @@ pub(super) async fn circuit_watcher( } } } + +pub fn circuit_devices_result(circuit: String, devices: Vec) -> WsResponse { + WsResponse::CircuitDevicesResult { + data: CircuitDevicesResult { + circuit_id: circuit, + devices, + ok: true, + }, + } +} diff --git a/src/rust/lqosd/src/node_manager/ws/single_user_channels/circuit_metrics.rs b/src/rust/lqosd/src/node_manager/ws/single_user_channels/circuit_metrics.rs new file mode 100644 index 00000000..d03dcbd2 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/ws/single_user_channels/circuit_metrics.rs @@ -0,0 +1,47 @@ +use crate::node_manager::local_api::circuit_live::{ + CircuitLiveMetrics, CircuitMetricsQuery, circuit_live_metrics, +}; +use crate::node_manager::ws::messages::{WsResponse, encode_ws_message}; +use std::time::Duration; +use tokio::time::MissedTickBehavior; +use tracing::debug; + +pub(super) async fn watch_circuit_metrics( + query: CircuitMetricsQuery, + tx: tokio::sync::mpsc::Sender>>, +) { + let mut ticker = tokio::time::interval(Duration::from_secs(1)); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + let mut last_metrics: Option> = None; + + loop { + let metrics = circuit_live_metrics(&query); + let changed = last_metrics.as_ref() != Some(&metrics); + let response = if last_metrics.is_none() { + Some(WsResponse::CircuitMetricsSnapshot { + data: metrics.clone(), + }) + } else if changed { + Some(WsResponse::CircuitMetricsUpdate { + data: metrics.clone(), + }) + } else { + None + }; + last_metrics = Some(metrics); + + if let Some(response) = response { + match encode_ws_message(&response) { + Ok(payload) => { + if tx.send(payload).await.is_err() { + debug!("CircuitMetrics watcher channel closed"); + break; + } + } + Err(_) => break, + } + } + + ticker.tick().await; + } +} diff --git a/src/rust/lqosd/src/node_manager/ws/single_user_channels/flows_by_circuit.rs b/src/rust/lqosd/src/node_manager/ws/single_user_channels/flows_by_circuit.rs index 7c4c3667..aca1e4c5 100644 --- a/src/rust/lqosd/src/node_manager/ws/single_user_channels/flows_by_circuit.rs +++ b/src/rust/lqosd/src/node_manager/ws/single_user_channels/flows_by_circuit.rs @@ -1,103 +1,17 @@ -use crate::node_manager::ws::messages::{FlowbeeKeyTransit, WsResponse, encode_ws_message}; -use crate::shaped_devices_tracker::{SHAPED_DEVICE_HASH_CACHE, SHAPED_DEVICES}; -use crate::throughput_tracker::flow_data::{ - ALL_FLOWS, FlowAnalysis, FlowbeeLocalData, get_asn_name_and_country, +use crate::node_manager::local_api::circuit_activity::{ + CircuitFlowSankeyRow, CircuitTopAsnsData, CircuitTopAsnsQuery, CircuitTrafficFlowsPage, + CircuitTrafficFlowsQuery, circuit_flow_sankey_rows, circuit_top_asns_data, + circuit_traffic_flows_page, }; -use lqos_utils::hash_to_i64; -use lqos_utils::unix_time::time_since_boot; -use std::time::Duration; -use tokio::time::MissedTickBehavior; -use tracing::debug; -const RECENT_CIRCUIT_FLOWS_WINDOW_NANOS: u64 = 30 * 1_000_000_000; - -fn recent_flows_by_circuit( - circuit_id: &str, -) -> Vec<(FlowbeeKeyTransit, FlowbeeLocalData, FlowAnalysis)> { - let circuit_hash = hash_to_i64(circuit_id); - let shaped = SHAPED_DEVICES.load(); - let cache = SHAPED_DEVICE_HASH_CACHE.load(); - if let Ok(now) = time_since_boot() { - let now_as_nanos = Duration::from(now).as_nanos() as u64; - let recent_cutoff = now_as_nanos.saturating_sub(RECENT_CIRCUIT_FLOWS_WINDOW_NANOS); - - { - let all_flows = ALL_FLOWS.lock(); - let result: Vec<(FlowbeeKeyTransit, FlowbeeLocalData, FlowAnalysis)> = all_flows - .flow_data - .iter() - .filter_map(|(key, (local, analysis))| { - // Don't show older flows - if local.last_seen < recent_cutoff { - return None; - } - - if local.circuit_hash != Some(circuit_hash) { - return None; - } - - let device_name = local - .device_hash - .and_then(|hash| cache.index_by_device_hash(&shaped, hash)) - .and_then(|idx| shaped.devices.get(idx)) - .map(|d| d.device_name.clone()) - .unwrap_or_else(|| "Unknown".to_string()); - - let geo = get_asn_name_and_country(key.remote_ip.as_ip()); - let (local_ip_str, remote_ip_str, asn_name, asn_country) = ( - key.local_ip.to_string(), - key.remote_ip.to_string(), - geo.name, - geo.country, - ); - - Some(( - FlowbeeKeyTransit { - remote_ip: remote_ip_str, - local_ip: local_ip_str, - src_port: key.src_port, - dst_port: key.dst_port, - ip_protocol: key.ip_protocol, - device_name, - asn_name, - asn_country, - protocol_name: analysis.protocol_analysis.to_string(), - last_seen_nanos: now_as_nanos.saturating_sub(local.last_seen), - }, - local.clone(), - *analysis, - )) - }) - .collect(); - return result; - } - } - Vec::new() +pub fn circuit_flow_sankey_result(circuit: &str) -> Vec { + circuit_flow_sankey_rows(circuit) } -pub(super) async fn flows_by_circuit( - circuit: String, - tx: tokio::sync::mpsc::Sender>>, -) { - let mut ticker = tokio::time::interval(Duration::from_secs(1)); - ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); - loop { - let flows: Vec<(FlowbeeKeyTransit, FlowbeeLocalData, FlowAnalysis)> = - recent_flows_by_circuit(&circuit).into_iter().collect(); - - let result = WsResponse::FlowsByCircuit { - circuit_id: circuit.clone(), - flows, - }; - if let Ok(payload) = encode_ws_message(&result) { - if tx.send(payload).await.is_err() { - debug!("Channel is gone"); - break; - } - } else { - break; - } - - ticker.tick().await; - } +pub fn circuit_top_asns_result(query: &CircuitTopAsnsQuery) -> CircuitTopAsnsData { + circuit_top_asns_data(query) +} + +pub fn circuit_traffic_flows_result(query: &CircuitTrafficFlowsQuery) -> CircuitTrafficFlowsPage { + circuit_traffic_flows_page(query) } diff --git a/src/rust/lqosd/src/node_manager/ws/single_user_channels/tree_attached_circuits.rs b/src/rust/lqosd/src/node_manager/ws/single_user_channels/tree_attached_circuits.rs new file mode 100644 index 00000000..ee7e791f --- /dev/null +++ b/src/rust/lqosd/src/node_manager/ws/single_user_channels/tree_attached_circuits.rs @@ -0,0 +1,43 @@ +use crate::node_manager::local_api::tree_attached_circuits::{ + TreeAttachedCircuitsPage, TreeAttachedCircuitsQuery, tree_attached_circuits, +}; +use crate::node_manager::ws::messages::{WsResponse, encode_ws_message}; +use std::time::Duration; +use tokio::time::MissedTickBehavior; +use tracing::debug; + +pub(super) async fn watch_tree_attached_circuits( + query: TreeAttachedCircuitsQuery, + tx: tokio::sync::mpsc::Sender>>, +) { + let mut ticker = tokio::time::interval(Duration::from_secs(1)); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + let mut last_page: Option = None; + + loop { + let page = tree_attached_circuits(query.clone()); + let changed = last_page.as_ref() != Some(&page); + let response = if last_page.is_none() { + Some(WsResponse::TreeAttachedCircuitsSnapshot { data: page.clone() }) + } else if changed { + Some(WsResponse::TreeAttachedCircuitsUpdate { data: page.clone() }) + } else { + None + }; + last_page = Some(page); + + if let Some(response) = response { + match encode_ws_message(&response) { + Ok(payload) => { + if tx.send(payload).await.is_err() { + debug!("TreeAttachedCircuits watcher channel closed"); + break; + } + } + Err(_) => break, + } + } + + ticker.tick().await; + } +} diff --git a/src/rust/lqosd/src/node_manager/ws/ticker.rs b/src/rust/lqosd/src/node_manager/ws/ticker.rs index 1bedb10e..d12d44c1 100644 --- a/src/rust/lqosd/src/node_manager/ws/ticker.rs +++ b/src/rust/lqosd/src/node_manager/ws/ticker.rs @@ -13,11 +13,12 @@ mod bakery; mod cadence; mod circuit_capacity; mod endpoint_latlon; -mod executive_heatmaps; +mod executive_dashboard_summary; mod flow_counter; mod flow_endpoints; pub(crate) mod ipstats_conversion; mod network_tree; +mod network_tree_lite; mod queue_stats_total; mod retransmits; mod rtt_histogram; @@ -165,6 +166,10 @@ async fn one_second_cadence( "network_tree", network_tree::network_tree(channels.clone(), bus_tx.clone()) ), + ticker_with_timeout( + "network_tree_lite", + network_tree_lite::network_tree_lite(channels.clone()) + ), ticker_with_timeout( "circuit_capacity", circuit_capacity::circuit_capacity(channels.clone()) @@ -189,10 +194,8 @@ async fn one_second_cadence( "stormguard", stormguard::stormguard_ticker(channels.clone(), bus_tx.clone()) ), - ticker_with_timeout( - "bakery", - bakery::bakery_ticker(channels.clone(), bus_tx.clone()) - ), + ticker_with_timeout("bakery_status", bakery::bakery_status(channels.clone())), + ticker_with_timeout("bakery_activity", bakery::bakery_activity(channels.clone())), ticker_with_timeout( "treeguard_status", treeguard::treeguard_status(channels.clone()) @@ -202,8 +205,8 @@ async fn one_second_cadence( treeguard::treeguard_activity(channels.clone()) ), ticker_with_timeout( - "executive_heatmaps", - executive_heatmaps::executive_heatmaps(channels.clone(), bus_tx.clone()) + "executive_dashboard_summary", + executive_dashboard_summary::executive_dashboard_summary(channels.clone()) ), ); diff --git a/src/rust/lqosd/src/node_manager/ws/ticker/bakery.rs b/src/rust/lqosd/src/node_manager/ws/ticker/bakery.rs index afa69ff7..64d277ac 100644 --- a/src/rust/lqosd/src/node_manager/ws/ticker/bakery.rs +++ b/src/rust/lqosd/src/node_manager/ws/ticker/bakery.rs @@ -1,14 +1,203 @@ -use crate::node_manager::ws::messages::{BakeryStatusData, BakeryStatusState, WsResponse}; +use crate::node_manager::ws::messages::{ + BakeryActivityEntry, BakeryCapacityInterfaceData, BakeryLiveCapacityInterfaceData, + BakeryPreflightData, BakeryQueueDistributionData, BakeryRuntimeOperationHeadlineData, + BakeryRuntimeOperationsData, BakeryStatusData, BakeryStatusState, WsResponse, +}; use crate::node_manager::ws::publish_subscribe::PubSub; use crate::node_manager::ws::published_channels::PublishedChannels; -use lqos_bus::{BusReply, BusRequest, BusResponse}; +use crate::shaped_devices_tracker::NETWORK_JSON; +use lqos_bakery::{ + BakeryActivityEntry as BakeryActivitySnapshot, BakeryApplyType, BakeryMode, + BakeryPreflightSnapshot, BakeryRuntimeNodeOperationAction, BakeryRuntimeNodeOperationStatus, + BakeryStatusSnapshot, bakery_runtime_node_branch_snapshot, +}; +use lqos_utils::hash_to_i64; use std::sync::Arc; -use tokio::sync::mpsc::Sender; -pub async fn bakery_ticker( - pubsub: Arc, - bus_tx: Sender<(tokio::sync::oneshot::Sender, BusRequest)>, -) { +fn mode_to_string(mode: BakeryMode) -> String { + match mode { + BakeryMode::Idle => "Idle", + BakeryMode::ApplyingFullReload => "ApplyingFullReload", + BakeryMode::ApplyingLiveChange => "ApplyingLiveChange", + } + .to_string() +} + +fn apply_type_to_string(apply_type: BakeryApplyType) -> String { + match apply_type { + BakeryApplyType::None => "None", + BakeryApplyType::FullReload => "FullReload", + BakeryApplyType::LiveChange => "LiveChange", + } + .to_string() +} + +fn runtime_action_to_string(action: BakeryRuntimeNodeOperationAction) -> String { + match action { + BakeryRuntimeNodeOperationAction::Virtualize => "Virtualize", + BakeryRuntimeNodeOperationAction::Restore => "Restore", + } + .to_string() +} + +fn runtime_status_to_string(status: BakeryRuntimeNodeOperationStatus) -> String { + match status { + BakeryRuntimeNodeOperationStatus::Submitted => "Submitted", + BakeryRuntimeNodeOperationStatus::Deferred => "Deferred", + BakeryRuntimeNodeOperationStatus::Applying => "Applying", + BakeryRuntimeNodeOperationStatus::AppliedAwaitingCleanup => "AppliedAwaitingCleanup", + BakeryRuntimeNodeOperationStatus::Completed => "Completed", + BakeryRuntimeNodeOperationStatus::Failed => "Failed", + BakeryRuntimeNodeOperationStatus::Dirty => "Dirty", + } + .to_string() +} + +fn map_preflight(snapshot: BakeryPreflightSnapshot) -> BakeryPreflightData { + BakeryPreflightData { + ok: snapshot.ok, + message: snapshot.message, + safe_budget: snapshot.safe_budget, + hard_limit: snapshot.hard_limit, + estimated_total_memory_bytes: snapshot.estimated_total_memory_bytes, + memory_available_bytes: snapshot.memory_available_bytes, + memory_guard_min_available_bytes: snapshot.memory_guard_min_available_bytes, + memory_ok: snapshot.memory_ok, + interfaces: snapshot + .interfaces + .into_iter() + .map(|entry| BakeryCapacityInterfaceData { + name: entry.name, + planned_qdiscs: entry.planned_qdiscs, + infra_qdiscs: entry.infra_qdiscs, + cake_qdiscs: entry.cake_qdiscs, + fq_codel_qdiscs: entry.fq_codel_qdiscs, + estimated_memory_bytes: entry.estimated_memory_bytes, + }) + .collect(), + } +} + +fn resolve_site_name(site_hash: i64) -> Option { + let reader = NETWORK_JSON.read(); + reader + .get_nodes_when_ready() + .iter() + .find_map(|node| (hash_to_i64(&node.name) == site_hash).then(|| node.name.clone())) +} + +fn resolve_runtime_site_name(site_hash: i64) -> Option { + bakery_runtime_node_branch_snapshot(site_hash).map(|snapshot| snapshot.site_name) +} + +fn extract_site_hash_from_summary(summary: &str) -> Option { + let lower = summary.to_ascii_lowercase(); + let site_index = lower.find("site ")?; + let rest = &summary[site_index + 5..]; + let digits: String = rest + .chars() + .take_while(|ch| *ch == '-' || ch.is_ascii_digit()) + .collect(); + (!digits.is_empty()).then_some(digits)?.parse().ok() +} + +fn map_status(snapshot: BakeryStatusSnapshot) -> BakeryStatusData { + BakeryStatusData { + current_state: BakeryStatusState { + active_circuits: snapshot.active_circuits, + mode: mode_to_string(snapshot.mode), + current_action_started_unix: snapshot.current_action_started_unix, + current_apply_phase: snapshot.current_apply_phase, + current_apply_total_tc_commands: snapshot.current_apply_total_tc_commands, + current_apply_completed_tc_commands: snapshot.current_apply_completed_tc_commands, + current_apply_total_chunks: snapshot.current_apply_total_chunks, + current_apply_completed_chunks: snapshot.current_apply_completed_chunks, + last_success_unix: snapshot.last_success_unix, + last_full_reload_success_unix: snapshot.last_full_reload_success_unix, + last_failure_unix: snapshot.last_failure_unix, + last_failure_summary: snapshot.last_failure_summary, + last_apply_type: apply_type_to_string(snapshot.last_apply_type), + last_total_tc_commands: snapshot.last_total_tc_commands, + last_class_commands: snapshot.last_class_commands, + last_qdisc_commands: snapshot.last_qdisc_commands, + last_build_duration_ms: snapshot.last_build_duration_ms, + last_apply_duration_ms: snapshot.last_apply_duration_ms, + avg_tc_io_interval_ms: snapshot.avg_tc_io_interval_ms, + last_tc_io_unix: snapshot.last_tc_io_unix, + tc_io_interval_samples: snapshot.tc_io_interval_samples, + runtime_operations: BakeryRuntimeOperationsData { + submitted_count: snapshot.runtime_operations.submitted_count, + deferred_count: snapshot.runtime_operations.deferred_count, + applying_count: snapshot.runtime_operations.applying_count, + awaiting_cleanup_count: snapshot.runtime_operations.awaiting_cleanup_count, + failed_count: snapshot.runtime_operations.failed_count, + dirty_count: snapshot.runtime_operations.dirty_count, + latest: snapshot.runtime_operations.latest.map(|entry| { + BakeryRuntimeOperationHeadlineData { + operation_id: entry.operation_id, + site_hash: entry.site_hash, + site_name: entry + .site_name + .or_else(|| resolve_runtime_site_name(entry.site_hash)) + .or_else(|| resolve_site_name(entry.site_hash)), + action: runtime_action_to_string(entry.action), + status: runtime_status_to_string(entry.status), + attempt_count: entry.attempt_count, + updated_at_unix: entry.updated_at_unix, + next_retry_at_unix: entry.next_retry_at_unix, + last_error: entry.last_error, + } + }), + }, + reload_required: snapshot.reload_required, + reload_required_reason: snapshot.reload_required_reason, + dirty_subtree_count: snapshot.dirty_subtree_count, + queue_distribution: snapshot + .queue_distribution + .into_iter() + .map(|entry| BakeryQueueDistributionData { + queue: entry.queue, + top_level_site_count: entry.top_level_site_count, + site_count: entry.site_count, + circuit_count: entry.circuit_count, + download_mbps: entry.download_mbps, + upload_mbps: entry.upload_mbps, + }) + .collect(), + live_capacity_interfaces: snapshot + .live_capacity_interfaces + .into_iter() + .map(|entry| BakeryLiveCapacityInterfaceData { + name: entry.name, + live_qdiscs: entry.live_qdiscs, + }) + .collect(), + live_capacity_safe_budget: snapshot.live_capacity_safe_budget, + live_capacity_updated_at_unix: snapshot.live_capacity_updated_at_unix, + preflight: snapshot.preflight.map(map_preflight), + }, + } +} + +fn map_activity(entry: BakeryActivitySnapshot) -> BakeryActivityEntry { + let site_hash = entry + .site_hash + .or_else(|| extract_site_hash_from_summary(&entry.summary)); + let site_name = entry + .site_name + .or_else(|| site_hash.and_then(resolve_runtime_site_name)) + .or_else(|| site_hash.and_then(resolve_site_name)); + BakeryActivityEntry { + ts: entry.ts, + event: entry.event, + status: entry.status, + summary: entry.summary, + site_hash, + site_name, + } +} + +pub async fn bakery_status(pubsub: Arc) { if !pubsub .is_channel_alive(PublishedChannels::BakeryStatus) .await @@ -16,25 +205,23 @@ pub async fn bakery_ticker( return; } - // Request stats from bus - let (tx, rx) = tokio::sync::oneshot::channel::(); - let request = BusRequest::GetBakeryStats; - if let Ok(_) = bus_tx.send((tx, request)).await - && let Ok(replies) = rx.await - { - for response in replies.responses { - if let BusResponse::BakeryActiveCircuits(stats) = response { - // Format as JSON - let msg = WsResponse::BakeryStatus { - data: BakeryStatusData { - current_state: BakeryStatusState { - active_circuits: stats, - }, - }, - }; - - pubsub.send(PublishedChannels::BakeryStatus, msg).await; - } - } - } + let data = map_status(lqos_bakery::bakery_status_snapshot()); + let msg = WsResponse::BakeryStatus { data }; + pubsub.send(PublishedChannels::BakeryStatus, msg).await; +} + +pub async fn bakery_activity(pubsub: Arc) { + if !pubsub + .is_channel_alive(PublishedChannels::BakeryActivity) + .await + { + return; + } + + let data = lqos_bakery::bakery_activity_snapshot() + .into_iter() + .map(map_activity) + .collect(); + let msg = WsResponse::BakeryActivity { data }; + pubsub.send(PublishedChannels::BakeryActivity, msg).await; } diff --git a/src/rust/lqosd/src/node_manager/ws/ticker/executive_dashboard_summary.rs b/src/rust/lqosd/src/node_manager/ws/ticker/executive_dashboard_summary.rs new file mode 100644 index 00000000..6a4b04e4 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/ws/ticker/executive_dashboard_summary.rs @@ -0,0 +1,22 @@ +use std::sync::Arc; + +use crate::node_manager::local_api::executive; +use crate::node_manager::ws::messages::WsResponse; +use crate::node_manager::ws::publish_subscribe::PubSub; +use crate::node_manager::ws::published_channels::PublishedChannels; + +pub async fn executive_dashboard_summary(channels: Arc) { + if !channels + .is_channel_alive(PublishedChannels::ExecutiveDashboardSummary) + .await + { + return; + } + + let payload = WsResponse::ExecutiveDashboardSummary { + data: executive::executive_dashboard_summary(), + }; + channels + .send(PublishedChannels::ExecutiveDashboardSummary, payload) + .await; +} diff --git a/src/rust/lqosd/src/node_manager/ws/ticker/executive_heatmaps.rs b/src/rust/lqosd/src/node_manager/ws/ticker/executive_heatmaps.rs deleted file mode 100644 index 38e29edb..00000000 --- a/src/rust/lqosd/src/node_manager/ws/ticker/executive_heatmaps.rs +++ /dev/null @@ -1,271 +0,0 @@ -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use crate::node_manager::ws::messages::{ExecutiveHeatmapsData, OversubscribedSite, WsResponse}; -use crate::node_manager::ws::publish_subscribe::PubSub; -use crate::node_manager::ws::published_channels::PublishedChannels; -use crate::shaped_devices_tracker::{NETWORK_JSON, SHAPED_DEVICES}; -use crate::throughput_tracker::THROUGHPUT_TRACKER; -use lqos_bus::{BusReply, BusRequest, BusResponse, ExecutiveSummaryHeader}; -use lqos_utils::temporal_heatmap::{HeatmapBlocks, TemporalHeatmap}; -use once_cell::sync::Lazy; -use parking_lot::Mutex; -use std::collections::{HashMap, HashSet}; -use tokio::sync::mpsc::Sender; - -static LAST_PUBLISH: Lazy>> = Lazy::new(|| Mutex::new(None)); -const MIN_INTERVAL: Duration = Duration::from_secs(1); - -pub async fn executive_heatmaps( - channels: Arc, - bus_tx: Sender<(tokio::sync::oneshot::Sender, BusRequest)>, -) { - if !channels - .is_channel_alive(PublishedChannels::ExecutiveHeatmaps) - .await - { - return; - } - - if !should_publish() { - return; - } - - let circuits = fetch_circuit_heatmaps(bus_tx.clone()) - .await - .unwrap_or_default(); - let sites = fetch_site_heatmaps(bus_tx.clone()) - .await - .unwrap_or_default(); - let asns = fetch_asn_heatmaps(bus_tx.clone()).await.unwrap_or_default(); - let header = fetch_executive_header(bus_tx.clone()) - .await - .unwrap_or_else(empty_header); - let global = fetch_global_heatmap(bus_tx) - .await - .unwrap_or_else(empty_blocks); - let global_qoq = THROUGHPUT_TRACKER.global_qoq_heatmap.lock().blocks(); - let oversubscribed_sites = compute_oversubscribed_sites(); - - let payload = WsResponse::ExecutiveHeatmaps { - data: ExecutiveHeatmapsData { - header, - global, - global_qoq, - circuits, - sites, - asns, - oversubscribed_sites, - }, - }; - - channels - .send(PublishedChannels::ExecutiveHeatmaps, payload) - .await; -} - -fn should_publish() -> bool { - let now = Instant::now(); - let mut lock = LAST_PUBLISH.lock(); - if let Some(last) = *lock - && now.duration_since(last) < MIN_INTERVAL - { - return false; - } - *lock = Some(now); - true -} - -async fn fetch_circuit_heatmaps( - bus_tx: Sender<(tokio::sync::oneshot::Sender, BusRequest)>, -) -> Option> { - if let Some(BusResponse::CircuitHeatmaps(data)) = - fetch_single_response(bus_tx, BusRequest::GetCircuitHeatmaps).await - { - return Some(data); - } - None -} - -async fn fetch_site_heatmaps( - bus_tx: Sender<(tokio::sync::oneshot::Sender, BusRequest)>, -) -> Option> { - if let Some(BusResponse::SiteHeatmaps(data)) = - fetch_single_response(bus_tx, BusRequest::GetSiteHeatmaps).await - { - return Some(data); - } - None -} - -async fn fetch_asn_heatmaps( - bus_tx: Sender<(tokio::sync::oneshot::Sender, BusRequest)>, -) -> Option> { - if let Some(BusResponse::AsnHeatmaps(data)) = - fetch_single_response(bus_tx, BusRequest::GetAsnHeatmaps).await - { - return Some(data); - } - None -} - -async fn fetch_executive_header( - bus_tx: Sender<(tokio::sync::oneshot::Sender, BusRequest)>, -) -> Option { - if let Some(BusResponse::ExecutiveSummaryHeader(data)) = - fetch_single_response(bus_tx, BusRequest::GetExecutiveSummaryHeader).await - { - return Some(data); - } - None -} - -async fn fetch_global_heatmap( - bus_tx: Sender<(tokio::sync::oneshot::Sender, BusRequest)>, -) -> Option { - if let Some(BusResponse::GlobalHeatmap(data)) = - fetch_single_response(bus_tx, BusRequest::GetGlobalHeatmap).await - { - return Some(data); - } - None -} - -async fn fetch_single_response( - bus_tx: Sender<(tokio::sync::oneshot::Sender, BusRequest)>, - request: BusRequest, -) -> Option { - let (tx, rx) = tokio::sync::oneshot::channel::(); - bus_tx.send((tx, request)).await.ok()?; - let replies = rx.await.ok()?; - replies.responses.into_iter().next() -} - -fn empty_blocks() -> HeatmapBlocks { - TemporalHeatmap::new().blocks() -} - -fn empty_header() -> ExecutiveSummaryHeader { - ExecutiveSummaryHeader::default() -} - -#[derive(Default)] -struct OversubTally { - cap_down: f32, - cap_up: f32, - sub_down: f32, - sub_up: f32, -} - -fn compute_oversubscribed_sites() -> Vec { - let devices = SHAPED_DEVICES.load(); - let mut circuit_map: HashMap = HashMap::new(); - for device in devices.devices.iter() { - let entry = circuit_map - .entry(device.circuit_id.clone()) - .or_insert_with(|| (device.parent_node.clone(), 0.0_f32, 0.0_f32)); - // Keep the largest per-circuit subscription values. - entry.1 = entry.1.max(device.download_max_mbps); - entry.2 = entry.2.max(device.upload_max_mbps); - } - - let reader = NETWORK_JSON.read(); - let nodes = reader.get_nodes_when_ready(); - if nodes.is_empty() { - return Vec::new(); - } - - // Build children adjacency from immediate_parent. - let mut children: Vec> = vec![Vec::new(); nodes.len()]; - for (idx, node) in nodes.iter().enumerate() { - if let Some(parent) = node.immediate_parent - && parent < children.len() - { - children[parent].push(idx); - } - } - - // Helper to collect descendant names including the node itself. - let collect_descendants = - |start: usize, children: &Vec>, nodes: &Vec| { - let mut stack = vec![start]; - let mut visited = HashSet::new(); - let mut names = HashSet::new(); - while let Some(idx) = stack.pop() { - if !visited.insert(idx) { - continue; - } - if let Some(name) = nodes.get(idx).map(|n| n.name.clone()) { - names.insert(name); - } - if let Some(kids) = children.get(idx) { - for &child in kids { - stack.push(child); - } - } - } - names - }; - - let mut results: Vec<(String, OversubTally)> = Vec::new(); - for (idx, node) in nodes.iter().enumerate() { - if node.name == "Root" { - continue; - } - let node_type = node.node_type.as_deref().unwrap_or("").to_ascii_lowercase(); - if node_type != "site" && !node_type.is_empty() && node_type != "ap" { - continue; - } - - let mut tally = OversubTally { - cap_down: node.max_throughput.0 as f32, - cap_up: node.max_throughput.1 as f32, - ..Default::default() - }; - - let descendants = collect_descendants(idx, &children, nodes); - for (_circuit_id, (parent_name, down, up)) in circuit_map.iter() { - if descendants.contains(parent_name) { - tally.sub_down += *down; - tally.sub_up += *up; - } - } - - results.push((node.name.clone(), tally)); - } - - results - .into_iter() - .filter_map(|(site_name, t)| { - let ratio_down = if t.cap_down > 0.0 { - Some(t.sub_down / t.cap_down) - } else { - None - }; - let ratio_up = if t.cap_up > 0.0 { - Some(t.sub_up / t.cap_up) - } else { - None - }; - if ratio_down.is_none() && ratio_up.is_none() { - return None; - } - let ratio_max = match (ratio_down, ratio_up) { - (Some(d), Some(u)) => Some(d.max(u)), - (Some(d), None) => Some(d), - (None, Some(u)) => Some(u), - (None, None) => None, - }; - Some(OversubscribedSite { - site_name, - cap_down: t.cap_down, - cap_up: t.cap_up, - sub_down: t.sub_down, - sub_up: t.sub_up, - ratio_down, - ratio_up, - ratio_max, - }) - }) - .collect() -} diff --git a/src/rust/lqosd/src/node_manager/ws/ticker/ipstats_conversion.rs b/src/rust/lqosd/src/node_manager/ws/ticker/ipstats_conversion.rs index e34ca86f..644c0357 100644 --- a/src/rust/lqosd/src/node_manager/ws/ticker/ipstats_conversion.rs +++ b/src/rust/lqosd/src/node_manager/ws/ticker/ipstats_conversion.rs @@ -1,6 +1,6 @@ use crate::shaped_devices_tracker::SHAPED_DEVICES; use lqos_bus::{IpStats, TcHandle}; -use lqos_utils::units::DownUpOrder; +use lqos_utils::units::{DownUpOrder, TcpRetransmitSample}; use serde::{Deserialize, Serialize}; fn truncate_by_chars(input: &str, max_chars: usize) -> String { @@ -16,7 +16,7 @@ pub struct IpStatsWithPlan { pub tc_handle: TcHandle, pub circuit_id: String, pub plan: DownUpOrder, - pub tcp_retransmits: (f64, f64), + pub tcp_retransmit_sample: DownUpOrder, } impl From<&IpStats> for IpStatsWithPlan { @@ -29,7 +29,7 @@ impl From<&IpStats> for IpStatsWithPlan { tc_handle: i.tc_handle, circuit_id: i.circuit_id.clone(), plan: DownUpOrder { down: 0.0, up: 0.0 }, - tcp_retransmits: i.tcp_retransmits, + tcp_retransmit_sample: i.tcp_retransmit_sample, }; if !result.circuit_id.is_empty() { diff --git a/src/rust/lqosd/src/node_manager/ws/ticker/network_tree_lite.rs b/src/rust/lqosd/src/node_manager/ws/ticker/network_tree_lite.rs new file mode 100644 index 00000000..d9cc4af2 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/ws/ticker/network_tree_lite.rs @@ -0,0 +1,21 @@ +use crate::node_manager::local_api::network_tree_lite::network_tree_lite_data; +use crate::node_manager::ws::messages::WsResponse; +use crate::node_manager::ws::publish_subscribe::PubSub; +use crate::node_manager::ws::published_channels::PublishedChannels; +use std::sync::Arc; + +pub async fn network_tree_lite(channels: Arc) { + if !channels + .is_channel_alive(PublishedChannels::NetworkTreeLite) + .await + { + return; + } + + let message = WsResponse::NetworkTreeLite { + data: network_tree_lite_data(), + }; + channels + .send(PublishedChannels::NetworkTreeLite, message) + .await; +} diff --git a/src/rust/lqosd/src/reload_lock.rs b/src/rust/lqosd/src/reload_lock.rs deleted file mode 100644 index 2596422a..00000000 --- a/src/rust/lqosd/src/reload_lock.rs +++ /dev/null @@ -1,36 +0,0 @@ -use std::sync::atomic::{AtomicBool, Ordering}; - -static RELOAD_BUSY: AtomicBool = AtomicBool::new(false); - -pub(crate) struct ReloadGuard; - -impl Drop for ReloadGuard { - fn drop(&mut self) { - RELOAD_BUSY.store(false, Ordering::Release); - } -} - -pub(crate) enum ReloadExecOutcome { - Busy, - Success(String), - Failed(String), -} - -fn try_acquire_reload_guard() -> Option { - RELOAD_BUSY - .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) - .ok() - .map(|_| ReloadGuard) -} - -/// Attempts to reload LibreQoS, returning `Busy` if another reload is already running. -pub(crate) fn try_reload_libreqos_locked() -> ReloadExecOutcome { - let Some(_guard) = try_acquire_reload_guard() else { - return ReloadExecOutcome::Busy; - }; - - match lqos_config::load_libreqos() { - Ok(message) => ReloadExecOutcome::Success(message), - Err(e) => ReloadExecOutcome::Failed(e.to_string()), - } -} diff --git a/src/rust/lqosd/src/shaped_devices_tracker/circuit_live.rs b/src/rust/lqosd/src/shaped_devices_tracker/circuit_live.rs new file mode 100644 index 00000000..0da50d61 --- /dev/null +++ b/src/rust/lqosd/src/shaped_devices_tracker/circuit_live.rs @@ -0,0 +1,223 @@ +use crate::throughput_tracker::THROUGHPUT_TRACKER; +use fxhash::{FxHashMap, FxHashSet}; +use lqos_utils::rtt::{FlowbeeEffectiveDirection, RttBucket}; +use lqos_utils::units::{DownUpOrder, TcpRetransmitSample, down_up_retransmit_sample}; +use lqos_utils::unix_time::time_since_boot; +use serde::{Deserialize, Serialize}; +use std::net::IpAddr; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use super::{CIRCUIT_LIVE_LAST_REFRESH_SECS, CIRCUIT_LIVE_REFRESH_LOCK, CIRCUIT_LIVE_SNAPSHOT}; +use super::{SHAPED_DEVICE_HASH_CACHE, SHAPED_DEVICES}; + +/// Per-circuit live metrics aggregated from the device-level throughput tracker. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct CircuitLiveRollup { + pub circuit_id: String, + pub circuit_name: String, + pub parent_node: String, + pub device_names: Vec, + pub ip_addrs: Vec, + pub plan_mbps: DownUpOrder, + pub bytes_per_second: DownUpOrder, + pub rtt_current_p50_nanos: DownUpOrder>, + pub qoo: DownUpOrder>, + pub tcp_retransmit_sample: DownUpOrder, + pub last_seen_nanos: u64, +} + +/// Shared once-per-second snapshot of circuit rollups and parent-node indexes. +#[derive(Clone, Debug, Default)] +pub struct CircuitLiveSnapshot { + pub by_circuit_id: FxHashMap, + pub circuit_ids_by_parent_node: FxHashMap>, +} + +#[derive(Default)] +struct CircuitAccumulator { + circuit_name: String, + parent_node: String, + device_names: FxHashSet, + ip_addrs: FxHashSet, + plan_mbps: DownUpOrder, + bytes_per_second: DownUpOrder, + rtt_current_p50_nanos: DownUpOrder>, + qoo: DownUpOrder>, + tcp_packets: DownUpOrder, + tcp_retransmits: DownUpOrder, + last_seen_nanos: Option, +} + +fn current_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +fn max_opt_u64(left: Option, right: Option) -> Option { + match (left, right) { + (Some(a), Some(b)) => Some(a.max(b)), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + } +} + +fn min_opt_f32(left: Option, right: Option) -> Option { + match (left, right) { + (Some(a), Some(b)) => Some(a.min(b)), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + } +} + +fn sort_string_set(values: FxHashSet) -> Vec { + let mut out: Vec = values.into_iter().collect(); + out.sort_unstable(); + out +} + +fn ip_to_string(ip: IpAddr) -> String { + ip.to_string() +} + +fn kernel_age_from_last_seen(kernel_now: std::time::Duration, last_seen: u64) -> u64 { + if last_seen == 0 { + return u64::MAX; + } + let since_boot = kernel_now.as_nanos(); + since_boot.saturating_sub(last_seen as u128) as u64 +} + +/// Rebuilds the shared circuit-live snapshot from current tracker state. +/// +/// Side effects: stores the rebuilt snapshot in the global `CIRCUIT_LIVE_SNAPSHOT` +/// cache and updates the refresh timestamp used by `fresh_circuit_live_snapshot`. +pub fn rebuild_circuit_live_snapshot() -> Arc { + let Ok(kernel_now) = time_since_boot() else { + let empty = Arc::new(CircuitLiveSnapshot::default()); + CIRCUIT_LIVE_SNAPSHOT.store(empty.clone()); + return empty; + }; + let kernel_now: std::time::Duration = kernel_now.into(); + + let shaped_devices = SHAPED_DEVICES.load(); + let cache = SHAPED_DEVICE_HASH_CACHE.load(); + let mut by_circuit_id: FxHashMap = FxHashMap::default(); + + for (ip_key, data) in THROUGHPUT_TRACKER.raw_data.lock().iter() { + let device = data + .device_hash + .and_then(|device_hash| cache.index_by_device_hash(&shaped_devices, device_hash)) + .or_else(|| { + data.circuit_hash.and_then(|circuit_hash| { + cache.index_by_circuit_hash(&shaped_devices, circuit_hash) + }) + }) + .and_then(|idx| shaped_devices.devices.get(idx)); + let Some(device) = device else { + continue; + }; + if device.circuit_id.trim().is_empty() { + continue; + } + + let entry = by_circuit_id.entry(device.circuit_id.clone()).or_default(); + if entry.circuit_name.is_empty() { + entry.circuit_name = device.circuit_name.clone(); + } + if entry.parent_node.is_empty() { + entry.parent_node = device.parent_node.clone(); + } + if !device.device_name.trim().is_empty() { + entry.device_names.insert(device.device_name.clone()); + } + entry.ip_addrs.insert(ip_to_string(ip_key.as_ip())); + entry.plan_mbps.down = entry.plan_mbps.down.max(device.download_max_mbps.round()); + entry.plan_mbps.up = entry.plan_mbps.up.max(device.upload_max_mbps.round()); + entry.bytes_per_second.down += data.bytes_per_second.down; + entry.bytes_per_second.up += data.bytes_per_second.up; + entry.tcp_packets.down += data.tcp_retransmit_packets.down; + entry.tcp_packets.up += data.tcp_retransmit_packets.up; + entry.tcp_retransmits.down += data.tcp_retransmits.down; + entry.tcp_retransmits.up += data.tcp_retransmits.up; + entry.last_seen_nanos = Some(match entry.last_seen_nanos { + Some(current) => current.min(kernel_age_from_last_seen(kernel_now, data.last_seen)), + None => kernel_age_from_last_seen(kernel_now, data.last_seen), + }); + entry.rtt_current_p50_nanos.down = max_opt_u64( + entry.rtt_current_p50_nanos.down, + data.rtt_buffer + .percentile(RttBucket::Current, FlowbeeEffectiveDirection::Download, 50) + .map(|rtt| rtt.as_nanos()), + ); + entry.rtt_current_p50_nanos.up = max_opt_u64( + entry.rtt_current_p50_nanos.up, + data.rtt_buffer + .percentile(RttBucket::Current, FlowbeeEffectiveDirection::Upload, 50) + .map(|rtt| rtt.as_nanos()), + ); + entry.qoo.down = min_opt_f32(entry.qoo.down, data.qoq.download_total_f32()); + entry.qoo.up = min_opt_f32(entry.qoo.up, data.qoq.upload_total_f32()); + } + + let mut finalized: FxHashMap = FxHashMap::default(); + let mut circuit_ids_by_parent_node: FxHashMap> = FxHashMap::default(); + for (circuit_id, value) in by_circuit_id { + if !value.parent_node.trim().is_empty() { + circuit_ids_by_parent_node + .entry(value.parent_node.clone()) + .or_default() + .push(circuit_id.clone()); + } + finalized.insert( + circuit_id.clone(), + CircuitLiveRollup { + circuit_id, + circuit_name: value.circuit_name, + parent_node: value.parent_node, + device_names: sort_string_set(value.device_names), + ip_addrs: sort_string_set(value.ip_addrs), + plan_mbps: value.plan_mbps, + bytes_per_second: value.bytes_per_second, + rtt_current_p50_nanos: value.rtt_current_p50_nanos, + qoo: value.qoo, + tcp_retransmit_sample: down_up_retransmit_sample( + value.tcp_retransmits, + value.tcp_packets, + ), + last_seen_nanos: value.last_seen_nanos.unwrap_or(u64::MAX), + }, + ); + } + for ids in circuit_ids_by_parent_node.values_mut() { + ids.sort_unstable(); + } + + let snapshot = Arc::new(CircuitLiveSnapshot { + by_circuit_id: finalized, + circuit_ids_by_parent_node, + }); + CIRCUIT_LIVE_SNAPSHOT.store(snapshot.clone()); + CIRCUIT_LIVE_LAST_REFRESH_SECS + .store(current_epoch_secs(), std::sync::atomic::Ordering::Release); + snapshot +} + +/// Returns the current once-per-second circuit-live snapshot, rebuilding it if stale. +/// +/// Side effects: may rebuild and replace the global snapshot cache. +pub fn fresh_circuit_live_snapshot() -> Arc { + let now_secs = current_epoch_secs(); + if CIRCUIT_LIVE_LAST_REFRESH_SECS.load(std::sync::atomic::Ordering::Acquire) == now_secs { + return CIRCUIT_LIVE_SNAPSHOT.load_full(); + } + let _guard = CIRCUIT_LIVE_REFRESH_LOCK.lock(); + if CIRCUIT_LIVE_LAST_REFRESH_SECS.load(std::sync::atomic::Ordering::Acquire) == now_secs { + return CIRCUIT_LIVE_SNAPSHOT.load_full(); + } + rebuild_circuit_live_snapshot() +} diff --git a/src/rust/lqosd/src/shaped_devices_tracker/mod.rs b/src/rust/lqosd/src/shaped_devices_tracker/mod.rs index 9c6ce813..3ee3ee15 100644 --- a/src/rust/lqosd/src/shaped_devices_tracker/mod.rs +++ b/src/rust/lqosd/src/shaped_devices_tracker/mod.rs @@ -1,23 +1,31 @@ +use crate::node_manager::local_api::network_tree_lite::NetworkTreeLiteNode; +use crate::treeguard::actor::is_runtime_virtualized_node; use anyhow::Result; use arc_swap::ArcSwap; -use fxhash::FxHashMap; +use fxhash::{FxHashMap, FxHashSet}; use lqos_bus::{BusResponse, Circuit}; use lqos_config::{ConfigShapedDevices, NetworkJsonNode, NetworkJsonTransport, ShapedDevice}; use lqos_queue_tracker::EFFECTIVE_NODE_RATES; use lqos_utils::file_watcher::FileWatcher; use lqos_utils::hash_to_i64; use lqos_utils::rtt::{FlowbeeEffectiveDirection, RttBucket}; -use lqos_utils::units::DownUpOrder; +use lqos_utils::units::{DownUpOrder, down_up_retransmit_sample}; use lqos_utils::unix_time::time_since_boot; use once_cell::sync::Lazy; +use parking_lot::Mutex; use std::sync::Arc; +use std::sync::atomic::AtomicU64; use std::time::Duration; use tracing::{debug, error, info, warn}; // Removed rate_for_plan() function - no longer needed with f32 plan structures +const SHAPED_DEVICES_RELOAD_RETRY_DELAY_MS: u64 = 500; +const SHAPED_DEVICES_RELOAD_ATTEMPTS: usize = 2; +pub mod circuit_live; mod netjson; use crate::throughput_tracker::THROUGHPUT_TRACKER; +pub use circuit_live::CircuitLiveSnapshot; pub use netjson::*; pub static SHAPED_DEVICES: Lazy> = @@ -86,32 +94,127 @@ impl ShapedDeviceHashCache { pub static SHAPED_DEVICE_HASH_CACHE: Lazy> = Lazy::new(|| ArcSwap::new(Arc::new(ShapedDeviceHashCache::default()))); +pub static CIRCUIT_LIVE_SNAPSHOT: Lazy> = + Lazy::new(|| ArcSwap::new(Arc::new(CircuitLiveSnapshot::default()))); +pub static CIRCUIT_LIVE_LAST_REFRESH_SECS: AtomicU64 = AtomicU64::new(0); +pub static CIRCUIT_LIVE_REFRESH_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); + +pub(crate) fn invalidate_circuit_live_snapshot() { + CIRCUIT_LIVE_LAST_REFRESH_SECS.store(0, std::sync::atomic::Ordering::Release); +} + +pub(crate) fn invalidate_executive_cache_snapshot() { + crate::node_manager::invalidate_executive_cache_snapshot(); +} + +#[derive(Clone, Copy, Debug, Default)] +struct NetworkTreeSummary { + subtree_site_count: u32, + subtree_circuit_count: u32, + subtree_device_count: u32, +} /// Clones a network node into its transport form and overlays effective inherited limits when /// the active queue structure contains a matching node entry. pub fn node_to_transport(node: &NetworkJsonNode) -> NetworkJsonTransport { + node_to_transport_with_summary(node, NetworkTreeSummary::default()) +} + +fn node_to_transport_with_summary( + node: &NetworkJsonNode, + summary: NetworkTreeSummary, +) -> NetworkJsonTransport { let mut transport = node.clone_to_transit(); + transport.runtime_virtualized = is_runtime_virtualized_node(&node.name); transport.configured_max_throughput = node.max_throughput; transport.effective_max_throughput = EFFECTIVE_NODE_RATES.load().get(&node.name).copied(); + transport.subtree_site_count = summary.subtree_site_count; + transport.subtree_circuit_count = summary.subtree_circuit_count; + transport.subtree_device_count = summary.subtree_device_count; transport } +fn build_network_tree_summaries( + nodes: &[NetworkJsonNode], + shaped_devices: &ConfigShapedDevices, +) -> Vec { + let mut summaries = vec![NetworkTreeSummary::default(); nodes.len()]; + let mut direct_circuits = vec![FxHashSet::default(); nodes.len()]; + let mut node_index_by_name = FxHashMap::default(); + node_index_by_name.reserve(nodes.len()); + + for (idx, node) in nodes.iter().enumerate() { + node_index_by_name.entry(node.name.as_str()).or_insert(idx); + } + + for device in &shaped_devices.devices { + let Some(node_idx) = node_index_by_name.get(device.parent_node.as_str()).copied() else { + continue; + }; + summaries[node_idx].subtree_device_count = + summaries[node_idx].subtree_device_count.saturating_add(1); + direct_circuits[node_idx].insert(device.circuit_hash); + } + + for (idx, circuits) in direct_circuits.iter().enumerate() { + summaries[idx].subtree_circuit_count = circuits.len() as u32; + } + + for idx in (1..nodes.len()).rev() { + let Some(parent_idx) = nodes[idx].immediate_parent else { + continue; + }; + summaries[parent_idx].subtree_site_count = summaries[parent_idx] + .subtree_site_count + .saturating_add(1) + .saturating_add(summaries[idx].subtree_site_count); + summaries[parent_idx].subtree_circuit_count = summaries[parent_idx] + .subtree_circuit_count + .saturating_add(summaries[idx].subtree_circuit_count); + summaries[parent_idx].subtree_device_count = summaries[parent_idx] + .subtree_device_count + .saturating_add(summaries[idx].subtree_device_count); + } + + summaries +} + +fn publish_shaped_devices(new_file: ConfigShapedDevices) { + debug!("ShapedDevices.csv loaded"); + let cache = ShapedDeviceHashCache::from_devices(&new_file.devices); + SHAPED_DEVICES.store(Arc::new(new_file)); + SHAPED_DEVICE_HASH_CACHE.store(Arc::new(cache)); + invalidate_circuit_live_snapshot(); + invalidate_executive_cache_snapshot(); + let nj = NETWORK_JSON.read(); + crate::throughput_tracker::THROUGHPUT_TRACKER.refresh_circuit_ids(&nj); +} + fn load_shaped_devices() { debug!("ShapedDevices.csv has changed. Attempting to load it."); - let shaped_devices = ConfigShapedDevices::load(); - if let Ok(new_file) = shaped_devices { - debug!("ShapedDevices.csv loaded"); - let cache = ShapedDeviceHashCache::from_devices(&new_file.devices); - SHAPED_DEVICES.store(Arc::new(new_file)); - SHAPED_DEVICE_HASH_CACHE.store(Arc::new(cache)); - let nj = NETWORK_JSON.read(); - crate::throughput_tracker::THROUGHPUT_TRACKER.refresh_circuit_ids(&nj); - } else { - warn!( - "ShapedDevices.csv failed to load, see previous error messages. Reverting to empty set." - ); - SHAPED_DEVICES.store(Arc::new(ConfigShapedDevices::default())); - SHAPED_DEVICE_HASH_CACHE.store(Arc::new(ShapedDeviceHashCache::default())); + for attempt in 1..=SHAPED_DEVICES_RELOAD_ATTEMPTS { + match ConfigShapedDevices::load() { + Ok(new_file) => { + publish_shaped_devices(new_file); + return; + } + Err(err) => { + if attempt < SHAPED_DEVICES_RELOAD_ATTEMPTS { + warn!( + "ShapedDevices.csv reload attempt {attempt}/{} failed: {err}. Retrying after {} ms.", + SHAPED_DEVICES_RELOAD_ATTEMPTS, SHAPED_DEVICES_RELOAD_RETRY_DELAY_MS + ); + std::thread::sleep(Duration::from_millis(SHAPED_DEVICES_RELOAD_RETRY_DELAY_MS)); + } else { + let current = SHAPED_DEVICES.load(); + warn!( + "ShapedDevices.csv reload failed after {} attempts: {err}. Keeping last-known-good data with {} devices.", + SHAPED_DEVICES_RELOAD_ATTEMPTS, + current.devices.len() + ); + } + } + } } } @@ -152,14 +255,30 @@ fn watch_for_shaped_devices_changing() -> Result<()> { pub fn get_one_network_map_layer(parent_idx: usize) -> BusResponse { let net_json = NETWORK_JSON.read(); let nodes_ref = net_json.get_nodes_when_ready(); + let shaped_devices = SHAPED_DEVICES.load(); + let summaries = build_network_tree_summaries(nodes_ref, shaped_devices.as_ref()); if let Some(parent) = nodes_ref.get(parent_idx) { - let mut nodes = vec![(parent_idx, node_to_transport(parent))]; + let mut nodes = vec![( + parent_idx, + node_to_transport_with_summary( + parent, + summaries.get(parent_idx).copied().unwrap_or_default(), + ), + )]; nodes.extend( nodes_ref .iter() .enumerate() .filter(|(_, node)| node.immediate_parent == Some(parent_idx)) - .map(|(i, node)| (i, node_to_transport(node))), + .map(|(i, node)| { + ( + i, + node_to_transport_with_summary( + node, + summaries.get(i).copied().unwrap_or_default(), + ), + ) + }), ); BusResponse::NetworkMap(nodes) } else { @@ -169,10 +288,87 @@ pub fn get_one_network_map_layer(parent_idx: usize) -> BusResponse { pub fn full_network_map_snapshot() -> Vec<(usize, NetworkJsonTransport)> { let nj = NETWORK_JSON.read(); - nj.get_nodes_when_ready() + let nodes = nj.get_nodes_when_ready(); + let shaped_devices = SHAPED_DEVICES.load(); + let summaries = build_network_tree_summaries(nodes, shaped_devices.as_ref()); + nodes .iter() .enumerate() - .map(|(i, n)| (i, node_to_transport(n))) + .map(|(i, n)| { + ( + i, + node_to_transport_with_summary(n, summaries.get(i).copied().unwrap_or_default()), + ) + }) + .collect() +} + +fn node_to_transport_lite(node: &NetworkJsonNode) -> NetworkTreeLiteNode { + let download = + node.rtt_buffer + .percentile(RttBucket::Current, FlowbeeEffectiveDirection::Download, 50); + let upload = + node.rtt_buffer + .percentile(RttBucket::Current, FlowbeeEffectiveDirection::Upload, 50); + + let rtts = match (download, upload) { + (None, None) => Vec::new(), + (Some(d), None) => vec![d.as_millis() as f32; 2], + (None, Some(u)) => vec![u.as_millis() as f32; 2], + (Some(d), Some(u)) => vec![d.as_millis() as f32, u.as_millis() as f32], + }; + + let qoo = node + .qoq_heatmap + .as_ref() + .map(|heatmap| { + let blocks = heatmap.blocks(); + let latest = |values: &[Option]| values.iter().rev().find_map(|v| *v); + (latest(&blocks.download_total), latest(&blocks.upload_total)) + }) + .unwrap_or((None, None)); + + NetworkTreeLiteNode { + name: node.name.clone(), + id: node.id.clone(), + is_virtual: node.virtual_node, + runtime_virtualized: is_runtime_virtualized_node(&node.name), + max_throughput: node.max_throughput, + current_throughput: ( + node.current_throughput.get_down(), + node.current_throughput.get_up(), + ), + current_tcp_packets: ( + node.current_tcp_packets.get_down(), + node.current_tcp_packets.get_up(), + ), + current_tcp_retransmit_packets: ( + node.current_tcp_retransmit_packets.get_down(), + node.current_tcp_retransmit_packets.get_up(), + ), + current_retransmits: ( + node.current_tcp_retransmits.get_down(), + node.current_tcp_retransmits.get_up(), + ), + rtts, + qoo, + parents: node.parents.clone(), + immediate_parent: node.immediate_parent, + node_type: node.node_type.clone(), + latitude: node.latitude, + longitude: node.longitude, + } +} + +/// Returns a lightweight live snapshot of the network tree for pages that do not need the full +/// `NetworkJsonTransport` payload. +pub fn full_network_map_lite_snapshot() -> Vec<(usize, NetworkTreeLiteNode)> { + let nj = NETWORK_JSON.read(); + let nodes = nj.get_nodes_when_ready(); + nodes + .iter() + .enumerate() + .map(|(i, n)| (i, node_to_transport_lite(n))) .collect() } @@ -182,9 +378,29 @@ pub fn get_full_network_map() -> BusResponse { pub fn get_top_n_root_queues(n_queues: usize) -> BusResponse { let net_json = NETWORK_JSON.read(); - if let Some(parent) = net_json.get_cloned_entry_by_index(0) { - let mut nodes = vec![(0, parent)]; - nodes.extend_from_slice(&net_json.get_cloned_children(0)); + let nodes_ref = net_json.get_nodes_when_ready(); + let shaped_devices = SHAPED_DEVICES.load(); + let summaries = build_network_tree_summaries(nodes_ref, shaped_devices.as_ref()); + if let Some(parent) = nodes_ref.first() { + let mut nodes = vec![( + 0, + node_to_transport_with_summary(parent, summaries.first().copied().unwrap_or_default()), + )]; + nodes.extend( + nodes_ref + .iter() + .enumerate() + .filter(|(idx, node)| *idx != 0 && node.immediate_parent == Some(0)) + .map(|(idx, node)| { + ( + idx, + node_to_transport_with_summary( + node, + summaries.get(idx).copied().unwrap_or_default(), + ), + ) + }), + ); // Remove the top-level entry for root nodes.remove(0); // Sort by total bandwidth (up + down) descending @@ -198,6 +414,7 @@ pub fn get_top_n_root_queues(n_queues: usize) -> BusResponse { let mut other_bw = (0, 0); let mut other_packets = (0, 0); let mut other_tcp_packets = (0, 0); + let mut other_tcp_retransmit_packets = (0, 0); let mut other_udp_packets = (0, 0); let mut other_icmp_packets = (0, 0); let mut other_xmit = (0, 0); @@ -210,6 +427,8 @@ pub fn get_top_n_root_queues(n_queues: usize) -> BusResponse { other_packets.1 += n.1.current_packets.1; other_tcp_packets.0 += n.1.current_tcp_packets.0; other_tcp_packets.1 += n.1.current_tcp_packets.1; + other_tcp_retransmit_packets.0 += n.1.current_tcp_retransmit_packets.0; + other_tcp_retransmit_packets.1 += n.1.current_tcp_retransmit_packets.1; other_udp_packets.0 += n.1.current_udp_packets.0; other_udp_packets.1 += n.1.current_udp_packets.1; other_icmp_packets.0 += n.1.current_icmp_packets.0; @@ -228,12 +447,14 @@ pub fn get_top_n_root_queues(n_queues: usize) -> BusResponse { name: "Others".into(), id: None, is_virtual: false, + runtime_virtualized: false, max_throughput: (0.0, 0.0), configured_max_throughput: (0.0, 0.0), effective_max_throughput: None, current_throughput: other_bw, current_packets: other_packets, current_tcp_packets: other_tcp_packets, + current_tcp_retransmit_packets: other_tcp_retransmit_packets, current_udp_packets: other_udp_packets, current_icmp_packets: other_icmp_packets, current_retransmits: other_xmit, @@ -246,6 +467,9 @@ pub fn get_top_n_root_queues(n_queues: usize) -> BusResponse { node_type: None, latitude: None, longitude: None, + subtree_site_count: 0, + subtree_circuit_count: 0, + subtree_device_count: 0, }, )); } @@ -277,7 +501,10 @@ pub fn get_funnel(circuit_id: &str) -> BusResponse { .rev() .skip(1) { - result.push((*idx, node_to_transport(&reader.get_nodes_when_ready()[*idx]))); + result.push(( + *idx, + node_to_transport(&reader.get_nodes_when_ready()[*idx]), + )); } return BusResponse::NetworkMap(result); } @@ -378,8 +605,10 @@ pub fn get_all_circuits() -> BusResponse { down: v.qoq.download_total_f32(), up: v.qoq.upload_total_f32(), }, - tcp_retransmits: v.tcp_retransmits, - tcp_packets: v.tcp_packets.checked_sub_or_zero(v.prev_tcp_packets), + tcp_retransmit_sample: down_up_retransmit_sample( + v.tcp_retransmits, + v.tcp_retransmit_packets, + ), circuit_id, device_id, circuit_name, @@ -494,8 +723,10 @@ pub fn get_circuit_by_id(desired_circuit_id: String) -> BusResponse { down: v.qoq.download_total_f32(), up: v.qoq.upload_total_f32(), }, - tcp_retransmits: v.tcp_retransmits, - tcp_packets: v.tcp_packets.checked_sub_or_zero(v.prev_tcp_packets), + tcp_retransmit_sample: down_up_retransmit_sample( + v.tcp_retransmits, + v.tcp_retransmit_packets, + ), circuit_id, device_id, circuit_name, diff --git a/src/rust/lqosd/src/shaped_devices_tracker/netjson.rs b/src/rust/lqosd/src/shaped_devices_tracker/netjson.rs index b7e52e3d..5b263925 100644 --- a/src/rust/lqosd/src/shaped_devices_tracker/netjson.rs +++ b/src/rust/lqosd/src/shaped_devices_tracker/netjson.rs @@ -44,6 +44,8 @@ fn load_network_json() { if let Ok(njs) = njs { let mut nj = NETWORK_JSON.write(); *nj = njs; + super::invalidate_circuit_live_snapshot(); + super::invalidate_executive_cache_snapshot(); crate::throughput_tracker::THROUGHPUT_TRACKER.refresh_circuit_ids(&nj); } else { warn!("Unable to load network.json"); diff --git a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs index 5253faff..35005938 100644 --- a/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs +++ b/src/rust/lqosd/src/throughput_tracker/flow_data/flow_tracker.rs @@ -77,6 +77,9 @@ pub struct FlowbeeLocalData { pub packets_sent: DownUpOrder, /// Rate estimate pub rate_estimate_bps: DownUpOrder, + /// Optional UI-oriented display rate. This is populated by specific + /// websocket/API surfaces that need plan-aware presentation guards. + pub display_rate_bps: Option>, /// TCP Retransmission count (also counts duplicates) pub tcp_retransmits: DownUpOrder, /// Has the connection ended? @@ -102,12 +105,15 @@ impl Serialize for FlowbeeLocalData { S: Serializer, { // Note: Keep this wire format stable (UI compatibility) while we refactor internal storage. - let mut state = serializer.serialize_struct("FlowbeeLocalData", 13)?; + let mut state = serializer.serialize_struct("FlowbeeLocalData", 14)?; state.serialize_field("start_time", &self.start_time)?; state.serialize_field("last_seen", &self.last_seen)?; state.serialize_field("bytes_sent", &self.bytes_sent)?; state.serialize_field("packets_sent", &self.packets_sent)?; state.serialize_field("rate_estimate_bps", &self.rate_estimate_bps)?; + if let Some(display_rate_bps) = &self.display_rate_bps { + state.serialize_field("display_rate_bps", display_rate_bps)?; + } state.serialize_field("tcp_retransmits", &self.tcp_retransmits)?; state.serialize_field("end_status", &self.end_status)?; state.serialize_field("tos", &self.tos)?; @@ -133,6 +139,7 @@ impl FlowbeeLocalData { bytes_sent: data.bytes_sent, packets_sent: data.packets_sent, rate_estimate_bps: data.rate_estimate_bps, + display_rate_bps: None, tcp_retransmits: data.tcp_retransmits, end_status: data.end_status, tos: data.tos, diff --git a/src/rust/lqosd/src/throughput_tracker/mod.rs b/src/rust/lqosd/src/throughput_tracker/mod.rs index 7c67171e..fb4bd04f 100644 --- a/src/rust/lqosd/src/throughput_tracker/mod.rs +++ b/src/rust/lqosd/src/throughput_tracker/mod.rs @@ -18,14 +18,14 @@ use crate::{ use arc_swap::ArcSwap; pub(crate) use flow_data::RttBuffer; use fxhash::{FxHashMap, FxHashSet}; -use lqos_bakery::BakeryCommands; +use lqos_bakery::{BakeryCommands, full_reload_in_progress}; use lqos_bus::{ AsnHeatmapData, BusResponse, CircuitHeatmapData, ExecutiveSummaryHeader, FlowbeeProtocol, IpStats, SiteHeatmapData, TcHandle, TopFlowType, XdpPpingResult, }; -use lqos_queue_tracker::ALL_QUEUE_SUMMARY; +use lqos_queue_tracker::{ALL_QUEUE_SUMMARY, queue_stats_stale}; use lqos_sys::flowbee_data::FlowbeeKey; -use lqos_utils::units::{DownUpOrder, down_up_divide}; +use lqos_utils::units::{DownUpOrder, TcpRetransmitSample, down_up_retransmit_sample}; use lqos_utils::{XdpIpAddress, hash_to_i64, unix_time::time_since_boot}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; @@ -36,6 +36,7 @@ use tokio::time::{Duration, Instant}; use tracing::{debug, info, warn}; const RETIRE_AFTER_SECONDS: u64 = 30; +const RELOAD_THROUGHPUT_POLL_INTERVAL_SECONDS: u64 = 5; pub static THROUGHPUT_TRACKER: Lazy = Lazy::new(ThroughputTracker::new); pub(crate) static CIRCUIT_RTT_BUFFERS: Lazy>> = @@ -151,14 +152,41 @@ fn throughput_task( let mut rtt_circuit_tracker: FxHashMap = FxHashMap::default(); let mut rtt_by_circuit: FxHashMap = FxHashMap::default(); let mut tcp_retries: FxHashMap> = FxHashMap::default(); + let mut tcp_retry_packets: FxHashMap> = FxHashMap::default(); let mut expired_flows: Vec = Vec::new(); // Counter for occasional stats let mut stats_counter = 0; + let mut reload_backoff_logged = false; + let mut last_reload_poll = + Instant::now() - Duration::from_secs(RELOAD_THROUGHPUT_POLL_INTERVAL_SECONDS); loop { let start = Instant::now(); timer_metrics.zero(); + let bakery_reload_in_progress = full_reload_in_progress(); + if bakery_reload_in_progress { + if !reload_backoff_logged { + info!( + "Throughput monitor: backing off to every {} seconds while Bakery full reload is in progress", + RELOAD_THROUGHPUT_POLL_INTERVAL_SECONDS + ); + reload_backoff_logged = true; + } + if start.duration_since(last_reload_poll) + < Duration::from_secs(RELOAD_THROUGHPUT_POLL_INTERVAL_SECONDS) + { + let missed_ticks = tfd.read(); + if missed_ticks > 1 { + warn!("Missed {} ticks", missed_ticks - 1); + } + continue; + } + last_reload_poll = start; + } else if reload_backoff_logged { + info!("Throughput monitor: resuming 1-second polling after Bakery full reload"); + reload_backoff_logged = false; + } // Formerly a "spawn blocking" blob { @@ -179,6 +207,7 @@ fn throughput_task( rtt_circuit_tracker: &mut rtt_circuit_tracker, rtt_by_circuit: &mut rtt_by_circuit, tcp_retries: &mut tcp_retries, + tcp_retry_packets: &mut tcp_retry_packets, expired_keys: &mut expired_flows, }); CIRCUIT_RTT_BUFFERS.store(Arc::new(rtt_by_circuit.clone())); @@ -192,14 +221,22 @@ fn throughput_task( rtt_circuit_tracker.clear(); rtt_by_circuit.clear(); tcp_retries.clear(); + tcp_retry_packets.clear(); expired_flows.clear(); rtt_circuit_tracker.shrink_to_fit(); rtt_by_circuit.shrink_to_fit(); tcp_retries.shrink_to_fit(); + tcp_retry_packets.shrink_to_fit(); expired_flows.shrink_to_fit(); timer_metrics.apply_flow_data = timer_metrics.start.elapsed().as_secs_f64(); - THROUGHPUT_TRACKER.apply_queue_stats(&mut net_json_calc); + if bakery_reload_in_progress { + debug!( + "Throughput monitor: skipping queue-stat application during Bakery full reload" + ); + } else { + THROUGHPUT_TRACKER.apply_queue_stats(&mut net_json_calc); + } timer_metrics.apply_queue_stats = timer_metrics.start.elapsed().as_secs_f64(); THROUGHPUT_TRACKER.update_totals(); timer_metrics.update_totals = timer_metrics.start.elapsed().as_secs_f64(); @@ -316,7 +353,7 @@ type TopList = ( f32, TcHandle, String, - (f64, f64), + DownUpOrder, ); pub fn top_n(start: u32, end: u32) -> BusResponse { @@ -338,7 +375,7 @@ pub fn top_n(start: u32, end: u32) -> BusResponse { te.median_latency().unwrap_or(0.0), te.tc_handle, te.circuit_id.as_ref().unwrap_or(&String::new()).clone(), - down_up_divide(te.tcp_retransmits, te.tcp_packets), + down_up_retransmit_sample(te.tcp_retransmits, te.tcp_packets), ) }) .collect() @@ -356,7 +393,7 @@ pub fn top_n(start: u32, end: u32) -> BusResponse { packets_per_second: *packets, median_tcp_rtt: *median_rtt, tc_handle: *tc_handle, - tcp_retransmits: *tcp_retransmits, + tcp_retransmit_sample: *tcp_retransmits, }, ) .collect(); @@ -382,7 +419,7 @@ pub fn top_n_up(start: u32, end: u32) -> BusResponse { te.median_latency().unwrap_or(0.0), te.tc_handle, te.circuit_id.as_ref().unwrap_or(&String::new()).clone(), - down_up_divide(te.tcp_retransmits, te.tcp_packets), + down_up_retransmit_sample(te.tcp_retransmits, te.tcp_packets), ) }) .collect() @@ -400,7 +437,7 @@ pub fn top_n_up(start: u32, end: u32) -> BusResponse { packets_per_second: *packets, median_tcp_rtt: *median_rtt, tc_handle: *tc_handle, - tcp_retransmits: *tcp_retransmits, + tcp_retransmit_sample: *tcp_retransmits, }, ) .collect(); @@ -535,7 +572,7 @@ pub fn worst_n(start: u32, end: u32) -> BusResponse { te.median_latency().unwrap_or(0.0), te.tc_handle, te.circuit_id.as_ref().unwrap_or(&String::new()).clone(), - down_up_divide(te.tcp_retransmits, te.tcp_packets), + down_up_retransmit_sample(te.tcp_retransmits, te.tcp_packets), ) }) .collect() @@ -553,7 +590,7 @@ pub fn worst_n(start: u32, end: u32) -> BusResponse { packets_per_second: *packets, median_tcp_rtt: *median_rtt, tc_handle: *tc_handle, - tcp_retransmits: *tcp_retransmits, + tcp_retransmit_sample: *tcp_retransmits, }, ) .collect(); @@ -580,7 +617,7 @@ pub fn worst_n_retransmits(start: u32, end: u32) -> BusResponse { te.median_latency().unwrap_or(0.0), te.tc_handle, te.circuit_id.as_ref().unwrap_or(&String::new()).clone(), - down_up_divide(te.tcp_retransmits, te.tcp_packets), + down_up_retransmit_sample(te.tcp_retransmits, te.tcp_packets), ) }) .collect() @@ -588,8 +625,10 @@ pub fn worst_n_retransmits(start: u32, end: u32) -> BusResponse { // Use a total order for floating-point comparison to avoid panics // when NaN/Inf are present and ensure comparator transitivity. full_list.sort_by(|a, b| { - let total_a = a.6.0 + a.6.1; - let total_b = b.6.0 + b.6.1; + let total_a = a.6.down.fraction().map(|f| f.get()).unwrap_or(0.0) + + a.6.up.fraction().map(|f| f.get()).unwrap_or(0.0); + let total_b = b.6.down.fraction().map(|f| f.get()).unwrap_or(0.0) + + b.6.up.fraction().map(|f| f.get()).unwrap_or(0.0); total_b.total_cmp(&total_a) }); let result = full_list @@ -604,7 +643,7 @@ pub fn worst_n_retransmits(start: u32, end: u32) -> BusResponse { packets_per_second: *packets, median_tcp_rtt: *median_rtt, tc_handle: *tc_handle, - tcp_retransmits: *tcp_retransmits, + tcp_retransmit_sample: *tcp_retransmits, }, ) .collect(); @@ -636,7 +675,7 @@ pub fn best_n(start: u32, end: u32) -> BusResponse { te.median_latency().unwrap_or(0.0), te.tc_handle, te.circuit_id.as_ref().unwrap_or(&String::new()).clone(), - down_up_divide(te.tcp_retransmits, te.tcp_packets), + down_up_retransmit_sample(te.tcp_retransmits, te.tcp_packets), ) }) .collect() @@ -655,7 +694,7 @@ pub fn best_n(start: u32, end: u32) -> BusResponse { packets_per_second: *packets, median_tcp_rtt: *median_rtt, tc_handle: *tc_handle, - tcp_retransmits: *tcp_retransmits, + tcp_retransmit_sample: *tcp_retransmits, }, ) .collect(); @@ -873,6 +912,8 @@ pub fn executive_summary_header() -> BusResponse { let unmapped_ip_count = total_hosts.saturating_sub(shaped_hosts) as u64; let queue_counts = ALL_QUEUE_SUMMARY.queue_counts(); + let bakery_reload_in_progress = full_reload_in_progress(); + let queue_stats_stale = queue_stats_stale() || bakery_reload_in_progress; let insight_connected = !matches!( get_lts_license_status().0, LtsStatus::Invalid | LtsStatus::NotChecked @@ -886,6 +927,9 @@ pub fn executive_summary_header() -> BusResponse { unmapped_ip_count, htb_queue_count: queue_counts.htb as u64, cake_queue_count: queue_counts.cake as u64, + fq_codel_queue_count: queue_counts.fq_codel as u64, + queue_stats_stale, + bakery_reload_in_progress, insight_connected, }) } @@ -956,7 +1000,10 @@ pub fn all_unknown_ips() -> BusResponse { packets_per_second: *packets, median_tcp_rtt: *median_rtt, tc_handle: *tc_handle, - tcp_retransmits: (0.0, 0.0), + tcp_retransmit_sample: DownUpOrder::new( + TcpRetransmitSample::new(0, 0), + TcpRetransmitSample::new(0, 0), + ), }, ) .collect(); @@ -1320,6 +1367,7 @@ mod compatibility_tests { use super::Lts2Circuit; use serde::Deserialize; + #[allow(dead_code)] #[derive(Debug, Deserialize)] struct OldLts2Device { device_id: String, @@ -1331,6 +1379,7 @@ mod compatibility_tests { comment: String, } + #[allow(dead_code)] #[derive(Debug, Deserialize)] struct OldLts2Circuit { circuit_id: String, diff --git a/src/rust/lqosd/src/throughput_tracker/throughput_entry.rs b/src/rust/lqosd/src/throughput_tracker/throughput_entry.rs index 06c5b106..2cb0902f 100644 --- a/src/rust/lqosd/src/throughput_tracker/throughput_entry.rs +++ b/src/rust/lqosd/src/throughput_tracker/throughput_entry.rs @@ -30,7 +30,7 @@ pub(crate) struct ThroughputEntry { pub(crate) last_fresh_rtt_data_cycle: u64, pub(crate) last_seen: u64, // Last seen in kernel time since boot pub(crate) tcp_retransmits: DownUpOrder, - pub(crate) prev_tcp_retransmits: DownUpOrder, + pub(crate) tcp_retransmit_packets: DownUpOrder, pub(crate) qoq: QoqScores, } diff --git a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs index 9271cc5c..f19db921 100644 --- a/src/rust/lqosd/src/throughput_tracker/tracking_data.rs +++ b/src/rust/lqosd/src/throughput_tracker/tracking_data.rs @@ -45,6 +45,7 @@ pub(crate) struct FlowApplyContext<'a> { pub(crate) rtt_circuit_tracker: &'a mut FxHashMap, pub(crate) rtt_by_circuit: &'a mut FxHashMap, pub(crate) tcp_retries: &'a mut FxHashMap>, + pub(crate) tcp_retry_packets: &'a mut FxHashMap>, pub(crate) expired_keys: &'a mut Vec, } @@ -197,18 +198,12 @@ impl ThroughputTracker { let upload_delta = entry.bytes.up.saturating_sub(entry.prev_bytes.up); total_download_bytes = total_download_bytes.saturating_add(download_delta); total_upload_bytes = total_upload_bytes.saturating_add(upload_delta); - total_tcp_packets.down = total_tcp_packets.down.saturating_add( - entry - .tcp_packets - .down - .saturating_sub(entry.prev_tcp_packets.down), - ); - total_tcp_packets.up = total_tcp_packets.up.saturating_add( - entry - .tcp_packets - .up - .saturating_sub(entry.prev_tcp_packets.up), - ); + total_tcp_packets.down = total_tcp_packets + .down + .saturating_add(entry.tcp_retransmit_packets.down); + total_tcp_packets.up = total_tcp_packets + .up + .saturating_add(entry.tcp_retransmit_packets.up); total_retransmits.down = total_retransmits .down .saturating_add(entry.tcp_retransmits.down); @@ -219,18 +214,14 @@ impl ThroughputTracker { let agg = aggregates.entry(circuit_hash).or_default(); agg.download_bytes = agg.download_bytes.saturating_add(download_delta); agg.upload_bytes = agg.upload_bytes.saturating_add(upload_delta); - agg.tcp_packets.down = agg.tcp_packets.down.saturating_add( - entry - .tcp_packets - .down - .saturating_sub(entry.prev_tcp_packets.down), - ); - agg.tcp_packets.up = agg.tcp_packets.up.saturating_add( - entry - .tcp_packets - .up - .saturating_sub(entry.prev_tcp_packets.up), - ); + agg.tcp_packets.down = agg + .tcp_packets + .down + .saturating_add(entry.tcp_retransmit_packets.down); + agg.tcp_packets.up = agg + .tcp_packets + .up + .saturating_add(entry.tcp_retransmit_packets.up); agg.tcp_retransmits.down = agg .tcp_retransmits .down @@ -593,7 +584,7 @@ impl ThroughputTracker { last_fresh_rtt_data_cycle: 0, last_seen: reduced.last_seen, tcp_retransmits: DownUpOrder::zeroed(), - prev_tcp_retransmits: DownUpOrder::zeroed(), + tcp_retransmit_packets: DownUpOrder::zeroed(), qoq: QoqScores::default(), }; raw_data.insert(*xdp_ip, entry); @@ -640,6 +631,7 @@ impl ThroughputTracker { rtt_circuit_tracker, rtt_by_circuit, tcp_retries, + tcp_retry_packets, expired_keys, } = ctx; //log::debug!("Flowbee events this second: {}", get_flowbee_event_count_and_reset()); @@ -799,6 +791,19 @@ impl ThroughputTracker { flow_rtt, ); } + if key.ip_protocol == 6 + && data.end_status == 0 + && raw_data.contains_key(&key.local_ip) + { + tcp_retries + .entry(key.local_ip) + .or_insert_with(DownUpOrder::zeroed) + .checked_add(delta_retrans); + tcp_retry_packets + .entry(key.local_ip) + .or_insert_with(DownUpOrder::zeroed) + .checked_add(delta_packets); + } } else { // Check if we've hit the flow limit if all_flows_lock.flow_data.len() >= MAX_FLOWS { @@ -901,25 +906,6 @@ impl ThroughputTracker { } } - // TCP - we have RTT data? 6 is TCP - if key.ip_protocol == 6 - && data.end_status == 0 - && raw_data.contains_key(&key.local_ip) - { - // TCP Retries - if let Some(retries) = tcp_retries.get_mut(&key.local_ip) { - retries.down += data.tcp_retransmits.down as u64; - retries.up += data.tcp_retransmits.up as u64; - } else { - tcp_retries.insert( - key.local_ip, - DownUpOrder::new( - data.tcp_retransmits.down as u64, - data.tcp_retransmits.up as u64, - ), - ); - } - } if data.end_status != 0 { // The flow has ended. We need to remove it from the map. expired_keys.push(*key); @@ -971,21 +957,24 @@ impl ThroughputTracker { // Reset all entries in the tracker to 0 for (_k, circuit) in raw_data.iter_mut() { circuit.tcp_retransmits = DownUpOrder::zeroed(); + circuit.tcp_retransmit_packets = DownUpOrder::zeroed(); } // Apply the new ones for (local_ip, retries) in tcp_retries { if let Some(tracker) = raw_data.get_mut(local_ip) { - tracker.tcp_retransmits.down = retries - .down - .saturating_sub(tracker.prev_tcp_retransmits.down); - tracker.tcp_retransmits.up = - retries.up.saturating_sub(tracker.prev_tcp_retransmits.up); - tracker.prev_tcp_retransmits.down = retries.down; - tracker.prev_tcp_retransmits.up = retries.up; + tracker.tcp_retransmit_packets = tcp_retry_packets + .get(local_ip) + .copied() + .unwrap_or_else(DownUpOrder::zeroed); + tracker.tcp_retransmits = *retries; // Send it upstream if let Some(parents) = &tracker.network_json_parents { - net_json_calc.add_retransmit_cycle(parents, tracker.tcp_retransmits); + net_json_calc.add_retransmit_cycle( + parents, + tracker.tcp_retransmits, + tracker.tcp_retransmit_packets, + ); } } } @@ -1005,9 +994,7 @@ impl ThroughputTracker { tracker.qoq = QoqScores::default(); continue; } - let tcp_packets_delta = tracker - .tcp_packets - .checked_sub_or_zero(tracker.prev_tcp_packets); + let tcp_packets_delta = tracker.tcp_retransmit_packets; let loss_download = tcp_retransmit_loss_proxy( tracker.tcp_retransmits.down, tcp_packets_delta.down, diff --git a/src/rust/lqosd/src/treeguard/actor.rs b/src/rust/lqosd/src/treeguard/actor.rs index d06d3ae0..9fd74731 100644 --- a/src/rust/lqosd/src/treeguard/actor.rs +++ b/src/rust/lqosd/src/treeguard/actor.rs @@ -4,38 +4,100 @@ //! and applying (or dry-running) any decisions. use crate::node_manager::ws::messages::{TreeguardActivityEntry, TreeguardStatusData}; +use crate::shaped_devices_tracker::circuit_live::fresh_circuit_live_snapshot; use crate::shaped_devices_tracker::{NETWORK_JSON, SHAPED_DEVICES}; use crate::system_stats::SystemStats; -use crate::throughput_tracker::{CIRCUIT_RTT_BUFFERS, THROUGHPUT_TRACKER}; +use crate::throughput_tracker::CIRCUIT_RTT_BUFFERS; use crate::treeguard::TreeguardError; -use crate::treeguard::reload::{ReloadController, ReloadOutcome, ReloadPriority}; use crate::treeguard::state::{ - CircuitSqmState, CircuitState, LinkState, LinkVirtualState, is_sustained_idle, - is_sustained_window, + CircuitSqmState, CircuitState, LinkState, LinkStructuralIneligibleState, + LinkTopologyFingerprint, LinkVirtualState, is_sustained_idle, is_sustained_window, }; use crate::treeguard::{bakery, decisions, overrides}; -use crate::urgent; use crossbeam_channel::{Receiver, Sender}; use fxhash::{FxHashMap, FxHashSet}; -use lqos_bus::{UrgentSeverity, UrgentSource}; -use lqos_config::load_config; +use lqos_bakery::{BakeryRuntimeNodeOperationFailureReason, BakeryRuntimeNodeOperationStatus}; +use lqos_config::{NetworkJsonNode, ShapedDevice, load_config}; use lqos_overrides::{NetworkAdjustment, OverrideFile, OverrideLayer, OverrideStore}; use lqos_utils::hash_to_i64; use lqos_utils::units::DownUpOrder; use lqos_utils::unix_time::{time_since_boot, unix_now}; use parking_lot::RwLock; -use std::collections::VecDeque; -use std::sync::OnceLock; +use std::collections::{BTreeMap, VecDeque}; +use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; static TREEGUARD_SENDER: OnceLock> = OnceLock::new(); static TREEGUARD_STATUS_CACHE: OnceLock> = OnceLock::new(); static TREEGUARD_ACTIVITY_CACHE: OnceLock>> = OnceLock::new(); +static TREEGUARD_RUNTIME_VIRTUALIZED_NODES: OnceLock>> = OnceLock::new(); const ACTIVITY_RING_CAPACITY: usize = 200; const UTIL_EWMA_ALPHA: f64 = 0.1; const TOP_LEVEL_SAFE_SUSTAIN_MINUTES: u32 = 15; +const TOP_LEVEL_EMERGENCY_UTIL_PCT: f64 = 95.0; +const TOP_LEVEL_EMERGENCY_SUSTAIN_SECONDS: u64 = 5; +const TREEGUARD_LINK_CHANGE_BUDGET_PER_TICK: usize = 4; +const TREEGUARD_CIRCUIT_CHANGE_BUDGET_PER_TICK: usize = 512; +const TREEGUARD_CIRCUIT_TARGET_SWEEP_SECONDS: usize = 15; +const TREEGUARD_CIRCUIT_MIN_BATCH_SIZE: usize = 128; +const TREEGUARD_LINK_VIRTUALIZATION_BACKOFF_SECONDS: u64 = 15 * 60; +const TREEGUARD_LINK_VIRTUALIZATION_DEFERRED_BACKOFF_SECONDS: u64 = 60; +const TREEGUARD_MIN_AUTO_VIRTUALIZE_SUBTREE_NODES: usize = 8; +const TREEGUARD_MIN_AUTO_VIRTUALIZE_SUBTREE_NODES_LOW_THROUGHPUT: usize = 16; +const TREEGUARD_LOW_VALUE_THROUGHPUT_MBPS: f64 = 1.0; +const TREEGUARD_MAX_AUTO_VIRTUALIZED_NODES: usize = 64; +const TREEGUARD_TOP_LEVEL_VIRTUALIZATION_VALUE_BONUS: u64 = 1_000_000_000; +const TREEGUARD_WARNING_DETAIL_LIMIT_PER_GROUP: usize = 3; + +struct TreeguardWarningGroupState { + emitted: usize, + suppressed: usize, + suppression_summary: String, +} + +#[derive(Default)] +struct TreeguardWarningLimiter { + groups: BTreeMap, +} + +impl TreeguardWarningLimiter { + fn push( + &mut self, + status: &mut TreeguardStatusData, + group_key: impl Into, + warning: String, + suppression_summary: String, + ) { + let state = + self.groups + .entry(group_key.into()) + .or_insert_with(|| TreeguardWarningGroupState { + emitted: 0, + suppressed: 0, + suppression_summary, + }); + if state.emitted < TREEGUARD_WARNING_DETAIL_LIMIT_PER_GROUP { + status.warnings.push(warning); + state.emitted += 1; + } else { + state.suppressed += 1; + } + } + + fn flush(self, status: &mut TreeguardStatusData) { + for state in self.groups.into_values() { + if state.suppressed == 0 { + continue; + } + status.warnings.push(format!( + "Suppressed {} additional {} this tick.", + state.suppressed, state.suppression_summary + )); + } + } +} /// A message sent to the TreeGuard actor. #[derive(Debug)] @@ -65,6 +127,7 @@ pub(crate) fn start_treeguard_actor( let _ = TREEGUARD_STATUS_CACHE.set(RwLock::new(empty_status_snapshot())); let _ = TREEGUARD_ACTIVITY_CACHE.set(RwLock::new(Vec::new())); + let _ = TREEGUARD_RUNTIME_VIRTUALIZED_NODES.set(RwLock::new(FxHashSet::default())); let (tx, rx) = crossbeam_channel::bounded::(64); let _ = TREEGUARD_SENDER.set(tx); @@ -84,6 +147,12 @@ pub(crate) fn cached_activity_snapshot() -> Option> Some(TREEGUARD_ACTIVITY_CACHE.get()?.read().clone()) } +pub(crate) fn is_runtime_virtualized_node(node_name: &str) -> bool { + TREEGUARD_RUNTIME_VIRTUALIZED_NODES + .get() + .is_some_and(|cache| cache.read().contains(node_name)) +} + /// Requests a status snapshot from the TreeGuard actor. /// /// This function is not pure: it sends a message to the TreeGuard actor thread. @@ -149,6 +218,7 @@ fn treeguard_actor_loop( &mut runtime_state, ); update_cached_snapshots(&status, &activity); + update_runtime_virtualized_cache(&runtime_state.runtime_virtualized_nodes); } Err(crossbeam_channel::RecvTimeoutError::Disconnected) => { warn!("TreeGuard actor command channel disconnected; exiting actor"); @@ -181,16 +251,149 @@ fn empty_status_snapshot() -> TreeguardStatusData { TreeguardStatusData { enabled: false, dry_run: true, + paused_for_bakery_reload: false, + pause_reason: None, cpu_max_pct: None, + total_nodes: 0, + total_circuits: 0, managed_nodes: 0, managed_circuits: 0, virtualized_nodes: 0, + cake_circuits: 0, + mixed_sqm_circuits: 0, fq_codel_circuits: 0, last_action_summary: None, warnings: Vec::new(), } } +fn current_topology_totals() -> (usize, usize) { + let total_nodes = { + let reader = NETWORK_JSON.read(); + reader + .get_nodes_when_ready() + .iter() + .filter(|n| n.name != "Root") + .count() + }; + + let total_circuits = { + let shaped = SHAPED_DEVICES.load(); + let mut circuits: FxHashSet<&str> = FxHashSet::default(); + for d in shaped.devices.iter() { + let id = d.circuit_id.trim(); + if !id.is_empty() { + circuits.insert(id); + } + } + circuits.len() + }; + + (total_nodes, total_circuits) +} + +fn direct_child_site_counts_by_node(nodes: &[NetworkJsonNode]) -> Vec { + let mut counts = vec![0usize; nodes.len()]; + for node in nodes { + if let Some(parent) = node.immediate_parent + && parent < counts.len() + { + counts[parent] = counts[parent].saturating_add(1); + } + } + counts +} + +fn direct_circuit_counts_by_node(shaped_devices: &[ShapedDevice]) -> FxHashMap { + let mut circuits_by_node: FxHashMap> = FxHashMap::default(); + for device in shaped_devices { + let node_name = device.parent_node.trim(); + let circuit_id = device.circuit_id.trim(); + if node_name.is_empty() || circuit_id.is_empty() { + continue; + } + circuits_by_node + .entry(node_name.to_string()) + .or_default() + .insert(circuit_id); + } + + circuits_by_node + .into_iter() + .map(|(node_name, circuits)| (node_name, circuits.len())) + .collect() +} + +fn structural_failure_reason_label( + reason: BakeryRuntimeNodeOperationFailureReason, +) -> &'static str { + match reason { + BakeryRuntimeNodeOperationFailureReason::StructuralIneligibleNoPromotableChildren => { + "no promotable children" + } + BakeryRuntimeNodeOperationFailureReason::StructuralIneligibleSinglePromotableChild => { + "single promotable child" + } + } +} + +fn top_level_structural_ineligibility( + state: &LinkState, + target: LinkVirtualState, + is_top_level: bool, +) -> Option<(BakeryRuntimeNodeOperationFailureReason, String)> { + if target != LinkVirtualState::Virtual || !is_top_level { + return None; + } + + let direct_promotable_children = + state.topology_fingerprint.direct_child_sites + state.topology_fingerprint.direct_circuits; + + if direct_promotable_children == 0 { + return Some(( + BakeryRuntimeNodeOperationFailureReason::StructuralIneligibleNoPromotableChildren, + "has no direct child sites or direct circuits to promote safely".to_string(), + )); + } + + if direct_promotable_children == 1 { + return Some(( + BakeryRuntimeNodeOperationFailureReason::StructuralIneligibleSinglePromotableChild, + "has only one promotable direct child, so top-level runtime virtualization would not produce a deterministic v1 split point".to_string(), + )); + } + + None +} + +fn clear_structural_ineligible_if_topology_changed( + state: &mut LinkState, + new_topology_fingerprint: LinkTopologyFingerprint, +) -> bool { + if state + .structural_ineligible + .is_some_and(|latched| latched.topology_fingerprint != new_topology_fingerprint) + { + state.structural_ineligible = None; + return true; + } + false +} + +fn latched_structural_ineligible_reason( + state: &LinkState, + target: LinkVirtualState, +) -> Option { + if target != LinkVirtualState::Virtual { + return None; + } + + state + .structural_ineligible + .filter(|latched| latched.topology_fingerprint == state.topology_fingerprint) + .map(|latched| latched.reason) +} + fn update_cached_snapshots( status: &TreeguardStatusData, activity: &VecDeque, @@ -203,14 +406,366 @@ fn update_cached_snapshots( } } +fn update_runtime_virtualized_cache(runtime_virtualized_nodes: &FxHashSet) { + if let Some(cache) = TREEGUARD_RUNTIME_VIRTUALIZED_NODES.get() { + *cache.write() = runtime_virtualized_nodes.clone(); + } +} + #[derive(Default)] struct TreeguardRuntimeState { link_states: FxHashMap, circuit_states: FxHashMap, + circuit_inventory: CircuitInventory, + circuit_batch_cursor: usize, + next_sqm_batch_id: u64, + runtime_virtualized_nodes: FxHashSet, + pending_link_operations: FxHashMap, + link_virtualization_backoff_until_unix: FxHashMap, managed_nodes: FxHashSet, managed_device_ids: FxHashSet, + duplicate_device_conflict_circuits: FxHashSet, last_dry_run: Option, - reload_controller: ReloadController, + paused_for_bakery_reload: bool, +} + +#[derive(Clone, Debug)] +struct PendingLinkOperation { + target: LinkVirtualState, + reason: String, +} + +#[derive(Clone, Debug, Default)] +struct CircuitInventoryEntry { + circuit_hash: i64, + circuit_entity_id: String, + circuit_label: String, + devices: Vec, + device_ids: Vec, + cap_down: f32, + cap_up: f32, + duplicate_details: Vec<(String, Vec)>, +} + +#[derive(Clone, Debug, Default)] +struct CircuitInventory { + shaped_devices_ptr: usize, + circuit_ids: Vec, + entries: FxHashMap, + all_device_ids: FxHashSet, +} + +#[derive(Clone, Debug)] +struct PendingLinkVirtualizationDecision { + node_name: String, + node_index: usize, + target: LinkVirtualState, + reason: String, + subtree_nodes: usize, + current_subtree_throughput_mbps: f64, + explicit_allowlist: bool, + is_top_level: bool, + value_score: u64, +} + +fn build_subtree_node_counts(parent_by_index: &[Option]) -> Vec { + fn dfs(index: usize, children: &[Vec], memo: &mut [usize]) -> usize { + if memo[index] != 0 { + return memo[index]; + } + let total = 1usize + + children[index] + .iter() + .map(|child| dfs(*child, children, memo)) + .sum::(); + memo[index] = total; + total + } + + let mut children: Vec> = vec![Vec::new(); parent_by_index.len()]; + for (index, parent) in parent_by_index.iter().enumerate() { + if let Some(parent) = *parent + && parent < children.len() + { + children[parent].push(index); + } + } + + let mut memo = vec![0usize; parent_by_index.len()]; + for index in 0..parent_by_index.len() { + let _ = dfs(index, &children, &mut memo); + } + memo +} + +fn link_virtualization_value_score( + is_top_level: bool, + subtree_nodes: usize, + cap_down_mbps: f64, + cap_up_mbps: f64, +) -> u64 { + let top_level_bonus = if is_top_level { + TREEGUARD_TOP_LEVEL_VIRTUALIZATION_VALUE_BONUS + } else { + 0 + }; + let subtree_score = subtree_nodes as u64 * 1_000_000; + let capacity_score = ((cap_down_mbps.max(0.0) + cap_up_mbps.max(0.0)) * 100.0) as u64; + top_level_bonus + .saturating_add(subtree_score) + .saturating_add(capacity_score) +} + +fn has_ancestor_in_set( + node_index: usize, + parent_by_index: &[Option], + selected_ancestors: &FxHashSet, +) -> bool { + let mut current = parent_by_index.get(node_index).copied().flatten(); + while let Some(parent) = current { + if selected_ancestors.contains(&parent) { + return true; + } + current = parent_by_index.get(parent).copied().flatten(); + } + false +} + +fn select_link_virtualization_candidates( + mut candidates: Vec, + parent_by_index: &[Option], + existing_virtualized_indices: &FxHashSet, + current_virtualized_nodes: usize, +) -> (Vec, usize, usize) { + candidates.sort_by(|left, right| { + let left_restore = matches!(left.target, LinkVirtualState::Physical); + let right_restore = matches!(right.target, LinkVirtualState::Physical); + right_restore + .cmp(&left_restore) + .then_with(|| right.value_score.cmp(&left.value_score)) + .then_with(|| right.subtree_nodes.cmp(&left.subtree_nodes)) + .then_with(|| left.node_name.cmp(&right.node_name)) + }); + + let mut selected = Vec::new(); + let mut deferred = 0usize; + let mut skipped_low_value = 0usize; + let mut selected_virtualized_indices: FxHashSet = FxHashSet::default(); + let mut virtualized_nodes_total = current_virtualized_nodes; + + for candidate in candidates { + if selected.len() >= TREEGUARD_LINK_CHANGE_BUDGET_PER_TICK { + deferred += 1; + continue; + } + + if matches!(candidate.target, LinkVirtualState::Virtual) { + if has_ancestor_in_set( + candidate.node_index, + parent_by_index, + existing_virtualized_indices, + ) || has_ancestor_in_set( + candidate.node_index, + parent_by_index, + &selected_virtualized_indices, + ) { + deferred += 1; + continue; + } + + if !candidate.explicit_allowlist { + let required_subtree_nodes = if candidate.current_subtree_throughput_mbps + < TREEGUARD_LOW_VALUE_THROUGHPUT_MBPS + { + TREEGUARD_MIN_AUTO_VIRTUALIZE_SUBTREE_NODES_LOW_THROUGHPUT + } else { + TREEGUARD_MIN_AUTO_VIRTUALIZE_SUBTREE_NODES + }; + if !candidate.is_top_level && candidate.subtree_nodes < required_subtree_nodes { + skipped_low_value += 1; + continue; + } + if virtualized_nodes_total >= TREEGUARD_MAX_AUTO_VIRTUALIZED_NODES { + deferred += 1; + continue; + } + } + + selected_virtualized_indices.insert(candidate.node_index); + virtualized_nodes_total = virtualized_nodes_total.saturating_add(1); + } + + selected.push(candidate); + } + + (selected, deferred, skipped_low_value) +} + +fn ensure_circuit_inventory( + runtime_state: &mut TreeguardRuntimeState, + shaped: &Arc, +) { + let shaped_devices_ptr = Arc::as_ptr(shaped) as usize; + if runtime_state.circuit_inventory.shaped_devices_ptr == shaped_devices_ptr { + return; + } + + runtime_state.circuit_inventory = build_circuit_inventory(shaped.as_ref()); + runtime_state.circuit_batch_cursor = 0; +} + +fn build_circuit_inventory(shaped: &lqos_config::ConfigShapedDevices) -> CircuitInventory { + let mut circuits_by_device_id: FxHashMap> = FxHashMap::default(); + circuits_by_device_id.reserve(shaped.devices.len()); + for device in shaped.devices.iter() { + let device_id = device.device_id.trim(); + let circuit_id = device.circuit_id.trim(); + if device_id.is_empty() || circuit_id.is_empty() { + continue; + } + circuits_by_device_id + .entry(device_id.to_string()) + .or_default() + .insert(circuit_id.to_string()); + } + + let duplicate_device_ids: FxHashMap> = circuits_by_device_id + .into_iter() + .filter_map(|(device_id, circuits)| { + if circuits.len() <= 1 { + return None; + } + let mut circuits: Vec = circuits.into_iter().collect(); + circuits.sort(); + Some((device_id, circuits)) + }) + .collect(); + + let mut by_circuit_id: FxHashMap> = FxHashMap::default(); + by_circuit_id.reserve(shaped.devices.len()); + let mut all_device_ids = FxHashSet::default(); + all_device_ids.reserve(shaped.devices.len()); + for device in shaped.devices.iter() { + if device.circuit_id.trim().is_empty() { + continue; + } + by_circuit_id + .entry(device.circuit_id.clone()) + .or_default() + .push(device.clone()); + if !device.device_id.trim().is_empty() { + all_device_ids.insert(device.device_id.clone()); + } + } + + let mut circuit_ids: Vec = by_circuit_id.keys().cloned().collect(); + circuit_ids.sort(); + + let mut entries = FxHashMap::default(); + entries.reserve(circuit_ids.len()); + for circuit_id in circuit_ids.iter() { + let Some(mut devices) = by_circuit_id.remove(circuit_id) else { + continue; + }; + + devices.sort_by(|left, right| left.device_id.cmp(&right.device_id)); + + let circuit_name = devices.iter().find_map(|device| { + let name = device.circuit_name.trim(); + if name.is_empty() { + None + } else { + Some(name.to_string()) + } + }); + let circuit_entity_id = match circuit_name.as_deref() { + Some(name) => format!("{name} ({circuit_id})"), + None => circuit_id.clone(), + }; + let circuit_label = circuit_name.unwrap_or_else(|| circuit_id.clone()); + + let mut cap_down = 0.0f32; + let mut cap_up = 0.0f32; + let mut device_ids: Vec = devices + .iter() + .map(|device| device.device_id.clone()) + .collect(); + device_ids.sort(); + device_ids.dedup(); + for device in devices.iter() { + cap_down = cap_down.max(device.download_max_mbps); + cap_up = cap_up.max(device.upload_max_mbps); + } + let duplicate_details: Vec<(String, Vec)> = device_ids + .iter() + .filter_map(|device_id| { + duplicate_device_ids + .get(device_id) + .map(|circuits| (device_id.clone(), circuits.clone())) + }) + .collect(); + + entries.insert( + circuit_id.clone(), + CircuitInventoryEntry { + circuit_hash: hash_to_i64(circuit_id), + circuit_entity_id, + circuit_label, + devices, + device_ids, + cap_down, + cap_up, + duplicate_details, + }, + ); + } + + CircuitInventory { + shaped_devices_ptr: shaped as *const _ as usize, + circuit_ids, + entries, + all_device_ids, + } +} + +fn circuit_evaluation_batch_size(managed_circuits: usize, all_circuits: bool) -> usize { + if managed_circuits == 0 { + return 0; + } + if !all_circuits { + return managed_circuits; + } + + let target = managed_circuits.div_ceil(TREEGUARD_CIRCUIT_TARGET_SWEEP_SECONDS); + managed_circuits.min(target.max(TREEGUARD_CIRCUIT_MIN_BATCH_SIZE)) +} + +fn collect_circuit_batch<'a>( + enrolled_circuits: &'a [String], + cursor: &mut usize, + batch_size: usize, +) -> Vec<&'a str> { + if enrolled_circuits.is_empty() || batch_size == 0 { + *cursor = 0; + return Vec::new(); + } + + if *cursor >= enrolled_circuits.len() { + *cursor = 0; + } + + let mut batch: Vec<&str> = Vec::with_capacity(batch_size.min(enrolled_circuits.len())); + let mut index = *cursor; + for _ in 0..batch_size.min(enrolled_circuits.len()) { + batch.push(enrolled_circuits[index].as_str()); + index += 1; + if index >= enrolled_circuits.len() { + index = 0; + } + } + + *cursor = index; + batch } /// Executes a single TreeGuard tick. @@ -224,13 +779,6 @@ fn run_tick( tick_seconds: &mut u64, runtime_state: &mut TreeguardRuntimeState, ) { - let link_states = &mut runtime_state.link_states; - let circuit_states = &mut runtime_state.circuit_states; - let managed_nodes = &mut runtime_state.managed_nodes; - let managed_device_ids = &mut runtime_state.managed_device_ids; - let last_dry_run = &mut runtime_state.last_dry_run; - let reload_controller = &mut runtime_state.reload_controller; - let now_unix = unix_now().unwrap_or(0); let now_nanos_since_boot = time_since_boot() .ok() @@ -238,14 +786,19 @@ fn run_tick( .map(|d| d.as_nanos() as u64); let mut warnings = Vec::new(); + let mut warning_limiter = TreeguardWarningLimiter::default(); let Ok(config) = load_config() else { status.enabled = false; status.dry_run = true; + status.paused_for_bakery_reload = false; + status.pause_reason = None; status.cpu_max_pct = None; status.managed_nodes = 0; status.managed_circuits = 0; status.virtualized_nodes = 0; + status.cake_circuits = 0; + status.mixed_sqm_circuits = 0; status.fq_codel_circuits = 0; status.last_action_summary = None; status.warnings = vec!["Unable to load configuration; TreeGuard inactive.".to_string()]; @@ -255,9 +808,13 @@ fn run_tick( let tg = &config.treeguard; *tick_seconds = tg.tick_seconds.max(1); - if last_dry_run.is_some_and(|prev| prev != tg.dry_run) { - link_states.clear(); - circuit_states.clear(); + if runtime_state + .last_dry_run + .is_some_and(|prev| prev != tg.dry_run) + { + runtime_state.link_states.clear(); + runtime_state.circuit_states.clear(); + runtime_state.pending_link_operations.clear(); push_activity( activity, TreeguardActivityEntry { @@ -267,13 +824,28 @@ fn run_tick( action: "dry_run_toggled".to_string(), persisted: false, reason: "Dry-run mode changed; state machines reset.".to_string(), + ..Default::default() }, ); } - *last_dry_run = Some(tg.dry_run); + runtime_state.last_dry_run = Some(tg.dry_run); - let mut virtualized_nodes: usize = 0; - let mut fq_codel_circuits: usize = 0; + if pause_for_bakery_reload(status, tick_seconds, runtime_state, tg.enabled, tg.dry_run) { + return; + } + + let shaped = SHAPED_DEVICES.load(); + ensure_circuit_inventory(runtime_state, &shaped); + + let link_states = &mut runtime_state.link_states; + let circuit_states = &mut runtime_state.circuit_states; + let runtime_virtualized_nodes = &mut runtime_state.runtime_virtualized_nodes; + let pending_link_operations = &mut runtime_state.pending_link_operations; + let link_virtualization_backoff_until_unix = + &mut runtime_state.link_virtualization_backoff_until_unix; + let managed_nodes = &mut runtime_state.managed_nodes; + let managed_device_ids = &mut runtime_state.managed_device_ids; + let duplicate_device_conflict_circuits = &mut runtime_state.duplicate_device_conflict_circuits; let top_level_auto_virtualize = tg.links.enabled && tg.links.top_level_auto_virtualize; if tg.enabled @@ -312,14 +884,14 @@ fn run_tick( warnings .push("Unable to sample CPU usage; CPU-aware behavior may be degraded.".to_string()); } + if let Some(notice) = lqos_config::treeguard_cpu_mode_migration_notice() { + warnings.push(notice); + } + + let (total_nodes_count, total_circuits_count) = current_topology_totals(); let managed_nodes_count: usize = if tg.links.all_nodes { - let reader = NETWORK_JSON.read(); - reader - .get_nodes_when_ready() - .iter() - .filter(|n| n.name != "Root") - .count() + total_nodes_count } else { let mut enrolled: FxHashSet = tg.links.nodes.iter().cloned().collect(); if top_level_auto_virtualize { @@ -334,15 +906,7 @@ fn run_tick( }; let managed_circuits_count: usize = if tg.circuits.all_circuits { - let shaped = SHAPED_DEVICES.load(); - let mut circuits: FxHashSet<&str> = FxHashSet::default(); - for d in shaped.devices.iter() { - let id = d.circuit_id.trim(); - if !id.is_empty() { - circuits.insert(id); - } - } - circuits.len() + total_circuits_count } else { tg.circuits.circuits.len() }; @@ -350,10 +914,23 @@ fn run_tick( status.enabled = tg.enabled; status.dry_run = tg.dry_run; status.cpu_max_pct = cpu_max_pct; + status.total_nodes = total_nodes_count; + status.total_circuits = total_circuits_count; status.managed_nodes = managed_nodes_count; status.managed_circuits = managed_circuits_count; status.warnings = warnings; + reconcile_pending_link_operations( + status, + activity, + now_unix, + &mut warning_limiter, + pending_link_operations, + runtime_virtualized_nodes, + link_virtualization_backoff_until_unix, + link_states, + ); + let (operator_overrides_snapshot, treeguard_overrides_snapshot) = if tg.enabled && (tg.links.enabled || tg.circuits.enabled) { let operator = match OverrideStore::load_layer(OverrideLayer::Operator) { @@ -405,62 +982,48 @@ fn run_tick( // Cleanup for removed nodes or disabled links. if !manage_links { - let removed: Vec = match OverrideStore::load_layer(OverrideLayer::Treeguard) { - Ok(of) => of - .network_adjustments() - .iter() - .filter_map(|adj| match adj { - NetworkAdjustment::SetNodeVirtual { node_name, .. } => Some(node_name.clone()), - _ => None, - }) - .collect(), - Err(e) => { - status.warnings.push(format!( - "TreeGuard links: unable to load TreeGuard overrides for cleanup: {e}" - )); - Vec::new() - } - }; - for node_name in removed { - match overrides::clear_node_virtual(&node_name) { - Ok(changed) => { - if changed { - reload_controller.request_reload( - ReloadPriority::Normal, - format!("Cleared virtual override for node '{node_name}'"), - ); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "node".to_string(), - entity_id: node_name.clone(), - action: "clear_virtual_override".to_string(), - persisted: true, - reason: "TreeGuard disabled or links disabled".to_string(), - }, - ); - } - managed_nodes.remove(&node_name); - link_states.remove(&node_name); - } + let mut removed: FxHashSet = + match OverrideStore::load_layer(OverrideLayer::Treeguard) { + Ok(of) => of + .network_adjustments() + .iter() + .filter_map(|adj| match adj { + NetworkAdjustment::SetNodeVirtual { node_name, .. } => { + Some(node_name.clone()) + } + _ => None, + }) + .collect(), Err(e) => { status.warnings.push(format!( - "TreeGuard links: failed to clear virtual override for node '{node_name}': {e}" + "TreeGuard links: unable to load TreeGuard overrides for cleanup: {e}" )); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "node".to_string(), - entity_id: node_name, - action: "clear_virtual_override_failed".to_string(), - persisted: false, - reason: format!("Overrides write failed: {e}"), - }, - ); + FxHashSet::default() } - } + }; + removed.extend(runtime_virtualized_nodes.iter().cloned()); + for node_name in removed { + clear_legacy_treeguard_virtual_override( + status, + activity, + now_unix, + &node_name, + "TreeGuard disabled or links disabled", + ); + restore_runtime_virtualization_if_needed( + status, + activity, + now_unix, + &node_name, + "TreeGuard disabled or links disabled", + tg.dry_run, + runtime_virtualized_nodes, + pending_link_operations, + link_virtualization_backoff_until_unix, + link_states, + ); + managed_nodes.remove(&node_name); + link_states.remove(&node_name); } } else { let reader = NETWORK_JSON.read(); @@ -492,7 +1055,7 @@ fn run_tick( }) .unwrap_or_default(); - let removed: Vec = if tg.links.all_nodes { + let mut removed: FxHashSet = if tg.links.all_nodes { let current: FxHashSet<&str> = reader .get_nodes_when_ready() .iter() @@ -511,592 +1074,339 @@ fn run_tick( .cloned() .collect() }; - for node_name in removed { - match overrides::clear_node_virtual(&node_name) { - Ok(changed) => { - if changed { - reload_controller.request_reload( - ReloadPriority::Normal, - format!("Cleared virtual override for node '{node_name}'"), - ); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "node".to_string(), - entity_id: node_name.clone(), - action: "clear_virtual_override".to_string(), - persisted: true, - reason: "Node removed from allowlist".to_string(), - }, - ); - } - managed_nodes.remove(&node_name); - link_states.remove(&node_name); - } - Err(e) => { - status.warnings.push(format!( - "TreeGuard links: failed to clear virtual override for node '{node_name}': {e}" - )); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "node".to_string(), - entity_id: node_name, - action: "clear_virtual_override_failed".to_string(), - persisted: false, - reason: format!("Overrides write failed: {e}"), - }, - ); - } - } - } if tg.links.all_nodes { - for node in reader.get_nodes_when_ready().iter() { - let node_name = node.name.as_str(); - if node_name == "Root" { - continue; - } - - if operator_virtual_node_overrides.contains(node_name) { - status.warnings.push(format!( - "TreeGuard links: node '{node_name}' has an operator virtual override; TreeGuard will not manage it." - )); - match overrides::clear_node_virtual(node_name) { - Ok(changed) => { - if changed { - reload_controller.request_reload( - ReloadPriority::Normal, - format!( - "Cleared virtual override for node '{node_name}' due to operator conflict" - ), - ); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "node".to_string(), - entity_id: node_name.to_string(), - action: "clear_virtual_override_conflict".to_string(), - persisted: true, - reason: "Operator override present; TreeGuard will not manage this node.".to_string(), - }, - ); - } - } - Err(e) => { - status.warnings.push(format!( - "TreeGuard links: failed to clear virtual override for node '{node_name}' during conflict cleanup: {e}" - )); - } - } - managed_nodes.remove(node_name); - link_states.remove(node_name); - continue; - } - - let treeguard_virtual_override = treeguard_overrides_snapshot - .as_ref() - .and_then(|overrides| overrides_node_virtual(overrides, node_name)); - - if node.virtual_node && treeguard_virtual_override.is_none() { - status.warnings.push(format!( - "TreeGuard links: node '{node_name}' is marked virtual in base network.json; TreeGuard will not manage it." - )); - continue; - } - - let cap_down = node.max_throughput.0; - let cap_up = node.max_throughput.1; - if cap_down <= 0.0 || cap_up <= 0.0 { - status.warnings.push(format!( - "TreeGuard links: node '{node_name}' has unknown capacity; no changes will be made." - )); - continue; - } - - let bytes_down = node.current_throughput.get_down() as f64; - let bytes_up = node.current_throughput.get_up() as f64; - let mbps_down = (bytes_down * 8.0) / 1_000_000.0; - let mbps_up = (bytes_up * 8.0) / 1_000_000.0; - let util_down_pct = (mbps_down / cap_down) * 100.0; - let util_up_pct = (mbps_up / cap_up) * 100.0; - - let state = link_states.entry(node_name.to_string()).or_insert_with(|| { - let mut state = LinkState::default(); - if let Some(overrides) = treeguard_overrides_snapshot.as_ref() - && let Some(v) = overrides_node_virtual(overrides, node_name) - { - state.desired = if v { - LinkVirtualState::Virtual - } else { - LinkVirtualState::Physical - }; - } - state - }); - prune_recent_changes(&mut state.recent_changes_unix, now_unix); - - let ewma_down = state - .down - .util_ewma_pct - .update(util_down_pct, UTIL_EWMA_ALPHA); - let ewma_up = state.up.util_ewma_pct.update(util_up_pct, UTIL_EWMA_ALPHA); - - // Per-direction idle tracking (sustained-idle is evaluated across both directions). - update_idle_since( - &mut state.down.idle_since_unix, - now_unix, - ewma_down, - tg.links.idle_util_pct as f64, - ); - update_idle_since( - &mut state.up.idle_since_unix, - now_unix, - ewma_up, - tg.links.idle_util_pct as f64, - ); - - let sustained_idle = is_sustained_idle( - now_unix, - state.down.idle_since_unix, - state.up.idle_since_unix, - tg.links.idle_min_minutes, - ); - - let is_top_level = top_level_auto_virtualize && node.immediate_parent == Some(0); - let top_level_safe_util_pct = - tg.links.top_level_safe_util_pct.clamp(0.0, 100.0) as f64; - if is_top_level { - update_below_since( - &mut state.down.top_level_safe_since_unix, - now_unix, - ewma_down, - top_level_safe_util_pct, - ); - update_below_since( - &mut state.up.top_level_safe_since_unix, - now_unix, - ewma_up, - top_level_safe_util_pct, - ); - } - - let rtt_missing = match now_nanos_since_boot { - None => true, - Some(now_nanos) => { - if node.rtt_buffer.last_seen == 0 { - true - } else { - let age_nanos = now_nanos.saturating_sub(node.rtt_buffer.last_seen); - age_nanos - >= u64::from(tg.links.rtt_missing_seconds) - .saturating_mul(1_000_000_000) - } - } - }; - - // QoO (when available) from the node heatmap blocks (latest non-None sample). - let qoo = node - .qoq_heatmap - .as_ref() - .map(|heatmap| { - let blocks = heatmap.blocks(); - let latest = |values: &[Option]| values.iter().rev().find_map(|v| *v); - DownUpOrder { - down: latest(&blocks.download_total), - up: latest(&blocks.upload_total), - } - }) - .unwrap_or(DownUpOrder { - down: None, - up: None, - }); - - let util_ewma_pct = DownUpOrder { - down: ewma_down, - up: ewma_up, - }; - - let decision = if is_top_level { - let dwell_secs = u64::from(tg.links.min_state_dwell_minutes).saturating_mul(60); - let in_dwell_window = state - .last_change_unix - .is_some_and(|last| now_unix.saturating_sub(last) < dwell_secs); - let rate_limited = if tg.links.max_link_changes_per_hour == 0 { - true - } else { - state.recent_changes_unix.len() - >= tg.links.max_link_changes_per_hour as usize - }; - if in_dwell_window || rate_limited { - decisions::LinkVirtualDecision::NoChange - } else { - let sustained_safe = is_sustained_window( - now_unix, - state.down.top_level_safe_since_unix, - state.up.top_level_safe_since_unix, - TOP_LEVEL_SAFE_SUSTAIN_MINUTES, - ); - let util_high = util_ewma_pct.down >= top_level_safe_util_pct - || util_ewma_pct.up >= top_level_safe_util_pct; - match state.desired { - LinkVirtualState::Physical => { - if sustained_safe { - decisions::LinkVirtualDecision::Set(LinkVirtualState::Virtual) - } else { - decisions::LinkVirtualDecision::NoChange - } - } - LinkVirtualState::Virtual => { - if util_high { - decisions::LinkVirtualDecision::Set(LinkVirtualState::Physical) - } else { - decisions::LinkVirtualDecision::NoChange - } - } - } - } - } else { - decisions::decide_link_virtualization(decisions::LinkVirtualizationInput { - now_unix, - allowlisted: true, - cpu_max_pct, - cpu_cfg: &tg.cpu, - links_cfg: &tg.links, - qoo_cfg: &tg.qoo, - rtt_missing, - qoo, - util_ewma_pct, - sustained_idle, - state, - }) - }; - - if let decisions::LinkVirtualDecision::Set(target) = decision { - if target == state.desired { - continue; - } - - let persist = !tg.dry_run; - let mut persisted_ok = false; - let mut override_changed = false; - - if persist { - let new_virtual = target == LinkVirtualState::Virtual; - match overrides::set_node_virtual(node_name, new_virtual) { - Ok(changed) => { - persisted_ok = true; - override_changed = changed; - } - Err(e) => { - status.warnings.push(format!( - "TreeGuard links: failed to persist virtual override for node '{node_name}': {e}" - )); - managed_nodes.insert(node_name.to_string()); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "node".to_string(), - entity_id: node_name.to_string(), - action: "set_virtual_override_failed".to_string(), - persisted: false, - reason: format!("Overrides write failed: {e}"), - }, - ); - continue; - } - } - } - - if override_changed { - let priority = if target == LinkVirtualState::Physical { - ReloadPriority::Urgent - } else { - ReloadPriority::Normal - }; - reload_controller.request_reload( - priority, - format!("Node '{node_name}' virtualization changed"), - ); - } - - state.desired = target; - state.last_change_unix = Some(now_unix); - state.recent_changes_unix.push_back(now_unix); - prune_recent_changes(&mut state.recent_changes_unix, now_unix); - managed_nodes.insert(node_name.to_string()); - - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "node".to_string(), - entity_id: node_name.to_string(), - action: match target { - LinkVirtualState::Physical => "unvirtualize".to_string(), - LinkVirtualState::Virtual => "virtualize".to_string(), - }, - persisted: persist && persisted_ok, - reason: if is_top_level { - match target { - LinkVirtualState::Virtual => format!( - "Top-level safe: sustained utilization below {:.1}% for {} minutes", - top_level_safe_util_pct, TOP_LEVEL_SAFE_SUSTAIN_MINUTES - ), - LinkVirtualState::Physical => format!( - "Top-level unsafe: utilization above {:.1}%", - top_level_safe_util_pct - ), - } - } else { - "Decision policy matched".to_string() - }, - }, - ); - status.last_action_summary = Some(format!( - "{} node '{}'", - if target == LinkVirtualState::Virtual { - "Virtualized" - } else { - "Unvirtualized" - }, - node_name - )); - } else { - managed_nodes.insert(node_name.to_string()); - } - - if state.desired == LinkVirtualState::Virtual { - virtualized_nodes += 1; - } - } + let current: FxHashSet<&str> = reader + .get_nodes_when_ready() + .iter() + .filter(|n| n.name != "Root") + .map(|n| n.name.as_str()) + .collect(); + removed.extend( + runtime_virtualized_nodes + .iter() + .filter(|n| !current.contains(n.as_str())) + .cloned(), + ); } else { - let mut enrolled_nodes: Vec = tg.links.nodes.clone(); + removed.extend( + runtime_virtualized_nodes + .iter() + .filter(|n| !allowlisted_nodes.contains(*n) && !top_level_nodes.contains(*n)) + .cloned(), + ); + } + for node_name in removed { + clear_legacy_treeguard_virtual_override( + status, + activity, + now_unix, + &node_name, + "Node removed from allowlist", + ); + restore_runtime_virtualization_if_needed( + status, + activity, + now_unix, + &node_name, + "Node removed from allowlist", + tg.dry_run, + runtime_virtualized_nodes, + pending_link_operations, + link_virtualization_backoff_until_unix, + link_states, + ); + managed_nodes.remove(&node_name); + link_states.remove(&node_name); + } + let nodes = reader.get_nodes_when_ready(); + let parent_by_index: Vec> = + nodes.iter().map(|node| node.immediate_parent).collect(); + let subtree_node_counts = build_subtree_node_counts(&parent_by_index); + let direct_child_site_counts = direct_child_site_counts_by_node(nodes); + let direct_circuit_counts = direct_circuit_counts_by_node(&shaped.devices); + let existing_virtualized_indices: FxHashSet = runtime_virtualized_nodes + .iter() + .filter_map(|node_name| reader.get_index_for_name(node_name)) + .collect(); + + let mut enrolled_nodes: Vec = if tg.links.all_nodes { + nodes + .iter() + .filter(|node| node.name != "Root") + .map(|node| node.name.clone()) + .collect() + } else { + let mut enrolled = tg.links.nodes.clone(); if top_level_auto_virtualize { - enrolled_nodes.extend(top_level_nodes.iter().cloned()); + enrolled.extend(top_level_nodes.iter().cloned()); } + enrolled.sort(); + enrolled.dedup(); + enrolled + }; + + if tg.links.all_nodes { enrolled_nodes.sort(); - enrolled_nodes.dedup(); + } - for node_name in enrolled_nodes.iter() { - if operator_virtual_node_overrides.contains(node_name) { - status.warnings.push(format!( - "TreeGuard links: node '{node_name}' has an operator virtual override; TreeGuard will not manage it." - )); - match overrides::clear_node_virtual(node_name) { - Ok(changed) => { - if changed { - reload_controller.request_reload( - ReloadPriority::Normal, - format!( - "Cleared virtual override for node '{node_name}' due to operator conflict" - ), - ); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "node".to_string(), - entity_id: node_name.clone(), - action: "clear_virtual_override_conflict".to_string(), - persisted: true, - reason: "Operator override present; TreeGuard will not manage this node.".to_string(), - }, - ); - } - } - Err(e) => { - status.warnings.push(format!( - "TreeGuard links: failed to clear virtual override for node '{node_name}' during conflict cleanup: {e}" - )); - } - } - managed_nodes.remove(node_name); - link_states.remove(node_name); - continue; - } - let Some(index) = reader.get_index_for_name(node_name) else { - status.warnings.push(format!( - "TreeGuard links allowlist: node '{node_name}' not found in network.json." - )); - continue; - }; - let Some(node) = reader.get_nodes_when_ready().get(index) else { - status.warnings.push(format!( - "TreeGuard links allowlist: node '{node_name}' index not present." - )); - continue; - }; + let mut pending_link_decisions = Vec::new(); - let treeguard_virtual_override = treeguard_overrides_snapshot - .as_ref() - .and_then(|overrides| overrides_node_virtual(overrides, node_name)); + for node_name in enrolled_nodes.iter() { + if operator_virtual_node_overrides.contains(node_name) { + status.warnings.push(format!( + "TreeGuard links: node '{node_name}' has an operator virtual override; TreeGuard will not manage it." + )); + clear_legacy_treeguard_virtual_override( + status, + activity, + now_unix, + node_name, + "Operator override present; TreeGuard will not manage this node.", + ); + restore_runtime_virtualization_if_needed( + status, + activity, + now_unix, + node_name, + "Operator override present; TreeGuard will not manage this node.", + tg.dry_run, + runtime_virtualized_nodes, + pending_link_operations, + link_virtualization_backoff_until_unix, + link_states, + ); + managed_nodes.remove(node_name); + link_states.remove(node_name); + continue; + } - if node.virtual_node && treeguard_virtual_override.is_none() { - status.warnings.push(format!( + let Some(index) = reader.get_index_for_name(node_name) else { + status.warnings.push(format!( + "TreeGuard links allowlist: node '{node_name}' not found in network.json." + )); + clear_legacy_treeguard_virtual_override( + status, + activity, + now_unix, + node_name, + "Node no longer exists in network.json", + ); + restore_runtime_virtualization_if_needed( + status, + activity, + now_unix, + node_name, + "Node no longer exists in network.json", + tg.dry_run, + runtime_virtualized_nodes, + pending_link_operations, + link_virtualization_backoff_until_unix, + link_states, + ); + managed_nodes.remove(node_name); + link_states.remove(node_name); + continue; + }; + let Some(node) = nodes.get(index) else { + status.warnings.push(format!( + "TreeGuard links allowlist: node '{node_name}' index not present." + )); + clear_legacy_treeguard_virtual_override( + status, + activity, + now_unix, + node_name, + "Node index no longer exists in network.json", + ); + restore_runtime_virtualization_if_needed( + status, + activity, + now_unix, + node_name, + "Node index no longer exists in network.json", + tg.dry_run, + runtime_virtualized_nodes, + pending_link_operations, + link_virtualization_backoff_until_unix, + link_states, + ); + managed_nodes.remove(node_name); + link_states.remove(node_name); + continue; + }; + + if node.virtual_node { + status.warnings.push(format!( "TreeGuard links: node '{node_name}' is marked virtual in base network.json; TreeGuard will not manage it." )); - continue; - } + clear_legacy_treeguard_virtual_override( + status, + activity, + now_unix, + node_name, + "Node is marked virtual in base network.json; TreeGuard refuses to manage base-virtual nodes.", + ); + restore_runtime_virtualization_if_needed( + status, + activity, + now_unix, + node_name, + "Node is marked virtual in base network.json; TreeGuard refuses to manage base-virtual nodes.", + tg.dry_run, + runtime_virtualized_nodes, + pending_link_operations, + link_virtualization_backoff_until_unix, + link_states, + ); + managed_nodes.remove(node_name); + link_states.remove(node_name); + continue; + } - let cap_down = node.max_throughput.0; - let cap_up = node.max_throughput.1; - if cap_down <= 0.0 || cap_up <= 0.0 { - status.warnings.push(format!( + let cap_down = node.max_throughput.0; + let cap_up = node.max_throughput.1; + if cap_down <= 0.0 || cap_up <= 0.0 { + status.warnings.push(format!( "TreeGuard links: node '{node_name}' has unknown capacity; no changes will be made." )); - continue; + continue; + } + + let bytes_down = node.current_throughput.get_down() as f64; + let bytes_up = node.current_throughput.get_up() as f64; + let mbps_down = (bytes_down * 8.0) / 1_000_000.0; + let mbps_up = (bytes_up * 8.0) / 1_000_000.0; + let util_down_pct = (mbps_down / cap_down) * 100.0; + let util_up_pct = (mbps_up / cap_up) * 100.0; + + let state = link_states.entry(node_name.clone()).or_insert_with(|| { + let mut state = LinkState::default(); + if runtime_virtualized_nodes.contains(node_name) { + state.desired = LinkVirtualState::Virtual; } + state + }); + prune_recent_changes(&mut state.recent_changes_unix, now_unix); + let topology_fingerprint = LinkTopologyFingerprint { + direct_child_sites: direct_child_site_counts.get(index).copied().unwrap_or(0), + direct_circuits: direct_circuit_counts.get(node_name).copied().unwrap_or(0), + }; + if clear_structural_ineligible_if_topology_changed(state, topology_fingerprint) { + link_virtualization_backoff_until_unix.remove(node_name); + } + state.topology_fingerprint = topology_fingerprint; - let bytes_down = node.current_throughput.get_down() as f64; - let bytes_up = node.current_throughput.get_up() as f64; - let mbps_down = (bytes_down * 8.0) / 1_000_000.0; - let mbps_up = (bytes_up * 8.0) / 1_000_000.0; - let util_down_pct = (mbps_down / cap_down) * 100.0; - let util_up_pct = (mbps_up / cap_up) * 100.0; + let ewma_down = state + .down + .util_ewma_pct + .update(util_down_pct, UTIL_EWMA_ALPHA); + let ewma_up = state.up.util_ewma_pct.update(util_up_pct, UTIL_EWMA_ALPHA); - let state = link_states.entry(node_name.clone()).or_insert_with(|| { - let mut state = LinkState::default(); - if let Some(overrides) = treeguard_overrides_snapshot.as_ref() - && let Some(v) = overrides_node_virtual(overrides, node_name) - { - state.desired = if v { - LinkVirtualState::Virtual - } else { - LinkVirtualState::Physical - }; - } - state - }); - prune_recent_changes(&mut state.recent_changes_unix, now_unix); + update_idle_since( + &mut state.down.idle_since_unix, + now_unix, + ewma_down, + tg.links.idle_util_pct as f64, + ); + update_idle_since( + &mut state.up.idle_since_unix, + now_unix, + ewma_up, + tg.links.idle_util_pct as f64, + ); - let ewma_down = state - .down - .util_ewma_pct - .update(util_down_pct, UTIL_EWMA_ALPHA); - let ewma_up = state.up.util_ewma_pct.update(util_up_pct, UTIL_EWMA_ALPHA); + let sustained_idle = is_sustained_idle( + now_unix, + state.down.idle_since_unix, + state.up.idle_since_unix, + tg.links.idle_min_minutes, + ); - // Per-direction idle tracking (sustained-idle is evaluated across both directions). - update_idle_since( - &mut state.down.idle_since_unix, + let is_top_level = top_level_auto_virtualize && node.immediate_parent == Some(0); + let top_level_safe_util_pct = tg.links.top_level_safe_util_pct.clamp(0.0, 100.0) as f64; + if is_top_level { + update_below_since( + &mut state.down.top_level_safe_since_unix, now_unix, ewma_down, - tg.links.idle_util_pct as f64, + top_level_safe_util_pct, ); - update_idle_since( - &mut state.up.idle_since_unix, + update_below_since( + &mut state.up.top_level_safe_since_unix, now_unix, ewma_up, - tg.links.idle_util_pct as f64, + top_level_safe_util_pct, ); - - let sustained_idle = is_sustained_idle( + update_above_since( + &mut state.down.top_level_emergency_since_unix, now_unix, - state.down.idle_since_unix, - state.up.idle_since_unix, - tg.links.idle_min_minutes, + ewma_down, + TOP_LEVEL_EMERGENCY_UTIL_PCT, ); + update_above_since( + &mut state.up.top_level_emergency_since_unix, + now_unix, + ewma_up, + TOP_LEVEL_EMERGENCY_UTIL_PCT, + ); + } - let is_top_level = top_level_auto_virtualize && node.immediate_parent == Some(0); - let top_level_safe_util_pct = - tg.links.top_level_safe_util_pct.clamp(0.0, 100.0) as f64; - if is_top_level { - update_below_since( - &mut state.down.top_level_safe_since_unix, - now_unix, - ewma_down, - top_level_safe_util_pct, - ); - update_below_since( - &mut state.up.top_level_safe_since_unix, - now_unix, - ewma_up, - top_level_safe_util_pct, - ); - } - - let rtt_missing = match now_nanos_since_boot { - None => true, - Some(now_nanos) => { - if node.rtt_buffer.last_seen == 0 { - true - } else { - let age_nanos = now_nanos.saturating_sub(node.rtt_buffer.last_seen); - age_nanos - >= u64::from(tg.links.rtt_missing_seconds) - .saturating_mul(1_000_000_000) - } - } - }; - - // QoO (when available) from the node heatmap blocks (latest non-None sample). - let qoo = node - .qoq_heatmap - .as_ref() - .map(|heatmap| { - let blocks = heatmap.blocks(); - let latest = |values: &[Option]| values.iter().rev().find_map(|v| *v); - DownUpOrder { - down: latest(&blocks.download_total), - up: latest(&blocks.upload_total), - } - }) - .unwrap_or(DownUpOrder { - down: None, - up: None, - }); - - let util_ewma_pct = DownUpOrder { - down: ewma_down, - up: ewma_up, - }; - - let decision = if is_top_level { - let dwell_secs = u64::from(tg.links.min_state_dwell_minutes).saturating_mul(60); - let in_dwell_window = state - .last_change_unix - .is_some_and(|last| now_unix.saturating_sub(last) < dwell_secs); - let rate_limited = if tg.links.max_link_changes_per_hour == 0 { + let rtt_missing = match now_nanos_since_boot { + None => true, + Some(now_nanos) => { + if node.rtt_buffer.last_seen == 0 { true } else { - state.recent_changes_unix.len() - >= tg.links.max_link_changes_per_hour as usize - }; - if in_dwell_window || rate_limited { - decisions::LinkVirtualDecision::NoChange - } else { - let sustained_safe = is_sustained_window( - now_unix, - state.down.top_level_safe_since_unix, - state.up.top_level_safe_since_unix, - TOP_LEVEL_SAFE_SUSTAIN_MINUTES, - ); - let util_high = util_ewma_pct.down >= top_level_safe_util_pct - || util_ewma_pct.up >= top_level_safe_util_pct; - match state.desired { - LinkVirtualState::Physical => { - if sustained_safe { - decisions::LinkVirtualDecision::Set(LinkVirtualState::Virtual) - } else { - decisions::LinkVirtualDecision::NoChange - } - } - LinkVirtualState::Virtual => { - if util_high { - decisions::LinkVirtualDecision::Set(LinkVirtualState::Physical) - } else { - decisions::LinkVirtualDecision::NoChange - } - } - } + let age_nanos = now_nanos.saturating_sub(node.rtt_buffer.last_seen); + age_nanos + >= u64::from(tg.links.rtt_missing_seconds).saturating_mul(1_000_000_000) } - } else { - decisions::decide_link_virtualization(decisions::LinkVirtualizationInput { + } + }; + + let qoo = node + .qoq_heatmap + .as_ref() + .map(|heatmap| { + let blocks = heatmap.blocks(); + let latest = |values: &[Option]| values.iter().rev().find_map(|v| *v); + DownUpOrder { + down: latest(&blocks.download_total), + up: latest(&blocks.upload_total), + } + }) + .unwrap_or(DownUpOrder { + down: None, + up: None, + }); + + let util_ewma_pct = DownUpOrder { + down: ewma_down, + up: ewma_up, + }; + + let decision = if is_top_level { + let sustained_safe = is_sustained_window( + now_unix, + state.down.top_level_safe_since_unix, + state.up.top_level_safe_since_unix, + TOP_LEVEL_SAFE_SUSTAIN_MINUTES, + ); + let emergency_util_sustained = state + .down + .top_level_emergency_since_unix + .is_some_and(|since| { + now_unix.saturating_sub(since) >= TOP_LEVEL_EMERGENCY_SUSTAIN_SECONDS + }) + || state + .up + .top_level_emergency_since_unix + .is_some_and(|since| { + now_unix.saturating_sub(since) >= TOP_LEVEL_EMERGENCY_SUSTAIN_SECONDS + }); + decisions::decide_top_level_link_virtualization( + decisions::TopLevelLinkVirtualizationInput { now_unix, - allowlisted: allowlisted_nodes.contains(node_name), cpu_max_pct, cpu_cfg: &tg.cpu, links_cfg: &tg.links, @@ -1104,232 +1414,178 @@ fn run_tick( rtt_missing, qoo, util_ewma_pct, - sustained_idle, + safe_util_pct: top_level_safe_util_pct, + sustained_safe, + emergency_util_sustained, state, - }) - }; + }, + ) + } else { + decisions::decide_link_virtualization(decisions::LinkVirtualizationInput { + now_unix, + allowlisted: tg.links.all_nodes || allowlisted_nodes.contains(node_name), + cpu_max_pct, + cpu_cfg: &tg.cpu, + links_cfg: &tg.links, + qoo_cfg: &tg.qoo, + rtt_missing, + qoo, + util_ewma_pct, + sustained_idle, + state, + }) + }; - if let decisions::LinkVirtualDecision::Set(target) = decision { - if target == state.desired { - continue; - } - - let persist = !tg.dry_run; - let mut persisted_ok = false; - let mut override_changed = false; - - if persist { - let new_virtual = target == LinkVirtualState::Virtual; - match overrides::set_node_virtual(node_name, new_virtual) { - Ok(changed) => { - persisted_ok = true; - override_changed = changed; - } - Err(e) => { - status.warnings.push(format!( - "TreeGuard links: failed to persist virtual override for node '{node_name}': {e}" - )); - managed_nodes.insert(node_name.clone()); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "node".to_string(), - entity_id: node_name.clone(), - action: "set_virtual_override_failed".to_string(), - persisted: false, - reason: format!("Overrides write failed: {e}"), - }, - ); - continue; + if let decisions::LinkVirtualDecision::Set(target) = decision + && target != state.desired + { + if let Some(reason) = latched_structural_ineligible_reason(state, target) { + let details = structural_failure_reason_label(reason); + warning_limiter.push( + status, + format!( + "runtime_structural_ineligible|{}|{:?}", + node_name, reason + ), + format!( + "TreeGuard links: node '{node_name}' remains ineligible for runtime virtualization ({details}) until its direct topology changes." + ), + format!( + "TreeGuard runtime structural-ineligible warnings for reason={details}" + ), + ); + continue; + } + let reason = if is_top_level { + match target { + LinkVirtualState::Virtual => format!( + "Top-level safe: sustained utilization below {:.1}% for {} minutes", + top_level_safe_util_pct, TOP_LEVEL_SAFE_SUSTAIN_MINUTES + ), + LinkVirtualState::Physical => { + if state + .down + .top_level_emergency_since_unix + .is_some_and(|since| { + now_unix.saturating_sub(since) + >= TOP_LEVEL_EMERGENCY_SUSTAIN_SECONDS + }) + || state + .up + .top_level_emergency_since_unix + .is_some_and(|since| { + now_unix.saturating_sub(since) + >= TOP_LEVEL_EMERGENCY_SUSTAIN_SECONDS + }) + { + format!( + "Top-level emergency restore: utilization >= {:.1}% for {}s", + TOP_LEVEL_EMERGENCY_UTIL_PCT, + TOP_LEVEL_EMERGENCY_SUSTAIN_SECONDS + ) + } else { + format!( + "Top-level restore: utilization above {:.1}%", + top_level_safe_util_pct + ) } } } - - if override_changed { - let priority = if target == LinkVirtualState::Physical { - ReloadPriority::Urgent - } else { - ReloadPriority::Normal - }; - reload_controller.request_reload( - priority, - format!("Node '{}' virtualization changed", node_name.clone()), - ); - } - state.desired = target; - state.last_change_unix = Some(now_unix); - state.recent_changes_unix.push_back(now_unix); - prune_recent_changes(&mut state.recent_changes_unix, now_unix); - managed_nodes.insert(node_name.clone()); - - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "node".to_string(), - entity_id: node_name.clone(), - action: match target { - LinkVirtualState::Physical => "unvirtualize".to_string(), - LinkVirtualState::Virtual => "virtualize".to_string(), - }, - persisted: persist && persisted_ok, - reason: if is_top_level { - match target { - LinkVirtualState::Virtual => format!( - "Top-level safe: sustained utilization below {:.1}% for {} minutes", - top_level_safe_util_pct, TOP_LEVEL_SAFE_SUSTAIN_MINUTES - ), - LinkVirtualState::Physical => format!( - "Top-level unsafe: utilization above {:.1}%", - top_level_safe_util_pct - ), - } - } else { - "Decision policy matched".to_string() - }, - }, - ); - status.last_action_summary = Some(format!( - "{} node '{}'", - if target == LinkVirtualState::Virtual { - "Virtualized" - } else { - "Unvirtualized" - }, - node_name - )); } else { - managed_nodes.insert(node_name.clone()); - } - - if state.desired == LinkVirtualState::Virtual { - virtualized_nodes += 1; - } + "Decision policy matched".to_string() + }; + pending_link_decisions.push(PendingLinkVirtualizationDecision { + node_name: node_name.clone(), + node_index: index, + target, + reason, + subtree_nodes: subtree_node_counts[index], + current_subtree_throughput_mbps: mbps_down.max(mbps_up), + explicit_allowlist: allowlisted_nodes.contains(node_name), + is_top_level, + value_score: link_virtualization_value_score( + is_top_level, + subtree_node_counts[index], + cap_down, + cap_up, + ), + }); } + + managed_nodes.insert(node_name.clone()); } - } - if let Some(attempt) = reload_controller.poll_reload(now_unix, tg.links.reload_cooldown_minutes) - { - let crate::treeguard::reload::ReloadAttempt { - outcome, - request_reason, - } = attempt; + let (selected_link_decisions, deferred_link_decisions, skipped_low_value_decisions) = + select_link_virtualization_candidates( + pending_link_decisions, + &parent_by_index, + &existing_virtualized_indices, + runtime_virtualized_nodes.len(), + ); + if deferred_link_decisions > 0 { + status.warnings.push(format!( + "TreeGuard links: deferred {deferred_link_decisions} lower-value or over-budget node virtualization changes this tick." + )); + } + if skipped_low_value_decisions > 0 { + status.warnings.push(format!( + "TreeGuard links: skipped {skipped_low_value_decisions} low-value automatic node virtualization candidates this tick because the subtree was too small for its current throughput." + )); + } - match outcome { - ReloadOutcome::Success { message: _ } => { - let why = request_reason.unwrap_or_else(|| "Topology change".to_string()); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "treeguard".to_string(), - entity_id: "reload".to_string(), - action: "reload_success".to_string(), - persisted: true, - reason: why.clone(), - }, - ); - status.last_action_summary = Some(format!("Reloaded LibreQoS: {why}")); - } - ReloadOutcome::Skipped { - reason, - next_allowed_unix, - } => { - let extra = next_allowed_unix - .map(|t| format!(" next_allowed_unix={t}")) - .unwrap_or_default(); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "treeguard".to_string(), - entity_id: "reload".to_string(), - action: "reload_skipped".to_string(), - persisted: false, - reason: format!("{reason}.{extra}"), - }, - ); - } - ReloadOutcome::Failed { - error, - next_allowed_unix, - } => { - let why = request_reason.unwrap_or_else(|| "Topology change".to_string()); - let extra = next_allowed_unix - .map(|t| format!(" next_allowed_unix={t}")) - .unwrap_or_default(); - status - .warnings - .push(format!("TreeGuard reload failed: {error}.{extra}")); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "treeguard".to_string(), - entity_id: "reload".to_string(), - action: "reload_failed".to_string(), - persisted: false, - reason: format!("{error}.{extra}"), - }, - ); - urgent::submit( - UrgentSource::System, - UrgentSeverity::Error, - "treeguard_reload_failed".to_string(), - format!("TreeGuard failed to reload LibreQoS: {error}"), - Some(why), - Some("treeguard_reload".to_string()), - ); - } + for decision in selected_link_decisions { + let Some(state) = link_states.get_mut(&decision.node_name) else { + continue; + }; + apply_link_virtualization_decision( + status, + activity, + now_unix, + &decision.node_name, + decision.target, + decision.is_top_level, + decision.reason, + tg.dry_run, + state, + pending_link_operations, + link_virtualization_backoff_until_unix, + ); } } // --- Circuit sampling + decisions (SQM switching) --- let manage_circuits = tg.enabled && tg.circuits.enabled; - - // Snapshot shaped devices so we can compute the enrolled circuit set. - let shaped = SHAPED_DEVICES.load(); + let circuit_inventory = &runtime_state.circuit_inventory; let enrolled_circuits: Vec = if tg.circuits.all_circuits { - let mut set: FxHashSet = FxHashSet::default(); - for d in shaped.devices.iter() { - let id = d.circuit_id.trim(); - if !id.is_empty() { - set.insert(id.to_string()); - } - } - let mut v: Vec = set.into_iter().collect(); - v.sort(); - v + circuit_inventory.circuit_ids.clone() } else { let mut v = tg.circuits.circuits.clone(); v.sort(); v.dedup(); v }; - - let allowlisted_circuits: FxHashSet = enrolled_circuits.iter().cloned().collect(); status.managed_circuits = enrolled_circuits.len(); - // Compute desired device_id set from enrolled circuits. - let desired_device_ids: FxHashSet = if manage_circuits { - if tg.circuits.all_circuits { - shaped.devices.iter().map(|d| d.device_id.clone()).collect() - } else { - shaped - .devices - .iter() - .filter(|d| allowlisted_circuits.contains(&d.circuit_id)) - .map(|d| d.device_id.clone()) - .collect() + let allowlisted_circuits: FxHashSet = if tg.circuits.all_circuits { + FxHashSet::default() + } else { + enrolled_circuits.iter().cloned().collect() + }; + let desired_device_ids: FxHashSet = if manage_circuits && !tg.circuits.all_circuits { + let mut desired = FxHashSet::default(); + for circuit_id in enrolled_circuits.iter() { + if let Some(entry) = circuit_inventory.entries.get(circuit_id) { + desired.extend(entry.device_ids.iter().cloned()); + } } + desired } else { FxHashSet::default() }; + let mut circuit_change_budget_remaining = TREEGUARD_CIRCUIT_CHANGE_BUDGET_PER_TICK; + let mut deferred_circuit_sqm_changes = 0usize; - // Cleanup for removed circuits or disabled circuits/TreeGuard. if !manage_circuits { let removed: Vec = match OverrideStore::load_layer(OverrideLayer::Treeguard) { Ok(of) => overrides_sqm_device_ids(&of).into_iter().collect(), @@ -1353,6 +1609,7 @@ fn run_tick( action: "clear_sqm_overrides".to_string(), persisted: true, reason: "TreeGuard disabled or circuits disabled".to_string(), + ..Default::default() }, ); } @@ -1365,16 +1622,28 @@ fn run_tick( } } managed_device_ids.clear(); + duplicate_device_conflict_circuits.clear(); circuit_states.clear(); + runtime_state.circuit_batch_cursor = 0; } else { - // Reconcile device IDs removed from allowlisted circuits. + circuit_states.retain(|circuit_id, _| { + circuit_inventory.entries.contains_key(circuit_id) + && (tg.circuits.all_circuits || allowlisted_circuits.contains(circuit_id)) + }); + let treeguard_device_ids_with_overrides: FxHashSet = treeguard_overrides_snapshot .as_ref() .map(overrides_sqm_device_ids) .unwrap_or_default(); let removed: Vec = treeguard_device_ids_with_overrides .iter() - .filter(|d| !desired_device_ids.contains(*d)) + .filter(|device_id| { + if tg.circuits.all_circuits { + !circuit_inventory.all_device_ids.contains(*device_id) + } else { + !desired_device_ids.contains(*device_id) + } + }) .cloned() .collect(); if !removed.is_empty() { @@ -1390,6 +1659,7 @@ fn run_tick( action: "clear_sqm_overrides".to_string(), persisted: true, reason: "Device removed from allowlisted circuits".to_string(), + ..Default::default() }, ); } @@ -1405,182 +1675,58 @@ fn run_tick( } } - // Snapshot RTT buffers by circuit hash. let rtt_snapshot = CIRCUIT_RTT_BUFFERS.load(); + let live_snapshot = fresh_circuit_live_snapshot(); + let batch_size = + circuit_evaluation_batch_size(enrolled_circuits.len(), tg.circuits.all_circuits); + let circuit_batch = collect_circuit_batch( + &enrolled_circuits, + &mut runtime_state.circuit_batch_cursor, + batch_size, + ); + let sqm_batch_id = next_sqm_batch_id(&mut runtime_state.next_sqm_batch_id); - let allow_hashes: Option> = if tg.circuits.all_circuits { - None - } else { - Some(enrolled_circuits.iter().map(|id| hash_to_i64(id)).collect()) - }; - - // Capacity lookup by circuit hash (max down/up Mbps across devices in the circuit). - let mut capacity_by_circuit: FxHashMap = FxHashMap::default(); - capacity_by_circuit.reserve(shaped.devices.len()); - for device in shaped.devices.iter() { - let entry = capacity_by_circuit - .entry(device.circuit_hash) - .or_insert((device.download_max_mbps, device.upload_max_mbps)); - if device.download_max_mbps > entry.0 { - entry.0 = device.download_max_mbps; - } - if device.upload_max_mbps > entry.1 { - entry.1 = device.upload_max_mbps; - } - } - - // Aggregate worst (minimum) QoO and throughput per circuit hash across devices/hosts. - let mut qoo_by_circuit: FxHashMap>> = FxHashMap::default(); - let mut bps_by_circuit: FxHashMap> = FxHashMap::default(); - { - let raw = THROUGHPUT_TRACKER.raw_data.lock(); - for entry in raw.values() { - let Some(ch) = entry.circuit_hash else { - continue; - }; - if allow_hashes.as_ref().is_some_and(|h| !h.contains(&ch)) { - continue; - } - - let down = entry.qoq.download_total_f32(); - let up = entry.qoq.upload_total_f32(); - let slot = qoo_by_circuit.entry(ch).or_insert(DownUpOrder { - down: None, - up: None, - }); - slot.down = min_opt_f32(slot.down, down); - slot.up = min_opt_f32(slot.up, up); - - let bps = bps_by_circuit - .entry(ch) - .or_insert(DownUpOrder { down: 0, up: 0 }); - bps.down = bps.down.saturating_add(entry.bytes_per_second.down); - bps.up = bps.up.saturating_add(entry.bytes_per_second.up); - } - } - - for circuit_id in enrolled_circuits.iter() { - let state = circuit_states.entry(circuit_id.clone()).or_insert_with(|| { - let mut state = CircuitState::default(); - if let Some(token) = infer_circuit_sqm_override_token( - circuit_id, - &shaped.devices, - treeguard_overrides_snapshot.as_ref(), - ) { - let parsed = decisions::parse_directional_sqm_override(&token); - if let Some(down) = parsed.down { - state.down.desired = down; - } - if let Some(up) = parsed.up { - state.up.desired = up; - } - } - state - }); - prune_recent_changes(&mut state.down.recent_changes_unix, now_unix); - prune_recent_changes(&mut state.up.recent_changes_unix, now_unix); - - let circuit_hash = hash_to_i64(circuit_id); - let (cap_down, cap_up) = capacity_by_circuit - .get(&circuit_hash) - .copied() - .unwrap_or((0.0, 0.0)); - let capacity_known = cap_down > 0.0 && cap_up > 0.0; - if !capacity_known { - status.warnings.push(format!( - "TreeGuard circuits: circuit '{circuit_id}' has unknown capacity; no changes will be made." - )); - state.down.idle_since_unix = None; - state.up.idle_since_unix = None; - } else { - let bps = bps_by_circuit - .get(&circuit_hash) - .copied() - .unwrap_or(DownUpOrder { down: 0, up: 0 }); - let mbps_down = (bps.down as f64 * 8.0) / 1_000_000.0; - let mbps_up = (bps.up as f64 * 8.0) / 1_000_000.0; - let util_down_pct = (mbps_down / cap_down as f64) * 100.0; - let util_up_pct = (mbps_up / cap_up as f64) * 100.0; - - let ewma_down = state - .down - .util_ewma_pct - .update(util_down_pct, UTIL_EWMA_ALPHA); - let ewma_up = state.up.util_ewma_pct.update(util_up_pct, UTIL_EWMA_ALPHA); - - update_idle_since( - &mut state.down.idle_since_unix, - now_unix, - ewma_down, - tg.circuits.idle_util_pct as f64, - ); - update_idle_since( - &mut state.up.idle_since_unix, - now_unix, - ewma_up, - tg.circuits.idle_util_pct as f64, - ); - } - - let rtt_missing = match now_nanos_since_boot { - None => true, - Some(now_nanos) => match rtt_snapshot.get(&circuit_hash) { - None => true, - Some(buf) => { - if buf.last_seen == 0 { - true - } else { - let age_nanos = now_nanos.saturating_sub(buf.last_seen); - age_nanos - >= u64::from(tg.circuits.rtt_missing_seconds) - .saturating_mul(1_000_000_000) - } - } - }, + for circuit_id in circuit_batch { + let Some(entry) = circuit_inventory.entries.get(circuit_id) else { + continue; }; - let qoo = qoo_by_circuit - .get(&circuit_hash) - .cloned() - .unwrap_or(DownUpOrder { - down: None, - up: None, - }); - - let devices: Vec = shaped - .devices - .iter() - .filter(|d| d.circuit_id == circuit_id.as_str()) - .cloned() - .collect(); - - let circuit_name: Option = devices.iter().find_map(|d| { - let name = d.circuit_name.trim(); - if name.is_empty() { - None - } else { - Some(name.to_string()) - } - }); - let circuit_entity_id: String = match circuit_name.as_deref() { - Some(name) => format!("{name} ({circuit_id})"), - None => circuit_id.clone(), - }; - let circuit_label: String = circuit_name.unwrap_or_else(|| circuit_id.clone()); - - let mut circuit_device_ids: Vec = - devices.iter().map(|d| d.device_id.clone()).collect(); - circuit_device_ids.sort(); - circuit_device_ids.dedup(); - let operator_conflict = circuit_device_ids - .iter() - .any(|did| operator_sqm_device_overrides.contains(did)); - if operator_conflict { + if !entry.duplicate_details.is_empty() { + let was_conflicted = duplicate_device_conflict_circuits.contains(circuit_id); + duplicate_device_conflict_circuits.insert(circuit_id.to_string()); + let duplicate_reason = entry + .duplicate_details + .iter() + .map(|(device_id, circuits)| { + format!( + "device_id '{}' is shared by circuits [{}]", + device_id, + circuits.join(", ") + ) + }) + .collect::>() + .join("; "); status.warnings.push(format!( - "TreeGuard circuits: circuit '{circuit_id}' has operator SQM overrides; TreeGuard will not manage it." + "TreeGuard circuits: circuit '{circuit_id}' has duplicate device IDs; TreeGuard will not manage it. {duplicate_reason}" )); - if !circuit_device_ids.is_empty() { - match overrides::clear_device_overrides(&circuit_device_ids) { + if !was_conflicted { + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "circuit".to_string(), + entity_id: entry.circuit_entity_id.clone(), + action: "skip_duplicate_device_id".to_string(), + persisted: false, + reason: format!( + "TreeGuard refuses circuits with duplicate device IDs. {duplicate_reason}" + ), + ..Default::default() + }, + ); + } + if !entry.device_ids.is_empty() { + match overrides::clear_device_overrides(&entry.device_ids) { Ok(changed) => { if changed { push_activity( @@ -1588,10 +1734,82 @@ fn run_tick( TreeguardActivityEntry { time: now_unix.to_string(), entity_type: "circuit".to_string(), - entity_id: circuit_entity_id.clone(), + entity_id: entry.circuit_entity_id.clone(), + action: "clear_sqm_overrides_duplicate_device_id" + .to_string(), + persisted: true, + reason: "Duplicate device IDs detected; cleared TreeGuard SQM overlays and skipped management.".to_string(), + ..Default::default() + }, + ); + } + } + Err(e) => { + status.warnings.push(format!( + "TreeGuard circuits: failed to clear TreeGuard SQM overlays for duplicate device IDs on circuit '{circuit_id}': {e}" + )); + } + } + for device_id in entry.device_ids.iter() { + managed_device_ids.remove(device_id); + } + } + circuit_states.remove(circuit_id); + continue; + } + + duplicate_device_conflict_circuits.remove(circuit_id); + + let base_sqm = base_circuit_sqm_state( + &entry.devices, + operator_overrides_snapshot.as_ref(), + &config, + entry.cap_down, + entry.cap_up, + ); + let state = circuit_states + .entry(circuit_id.to_string()) + .or_insert_with(|| { + let mut state = CircuitState::default(); + state.down.desired = base_sqm.down; + state.up.desired = base_sqm.up; + if let Some(overrides) = treeguard_overrides_snapshot.as_ref() + && let Some(token) = + find_circuit_override_token_in_overrides(&entry.devices, overrides) + { + let parsed = decisions::parse_directional_sqm_override(&token); + if let Some(down) = parsed.down { + state.down.desired = down; + } + if let Some(up) = parsed.up { + state.up.desired = up; + } + } + state + }); + + let operator_conflict = entry + .device_ids + .iter() + .any(|device_id| operator_sqm_device_overrides.contains(device_id)); + if operator_conflict { + status.warnings.push(format!( + "TreeGuard circuits: circuit '{circuit_id}' has operator SQM overrides; TreeGuard will not manage it." + )); + if !entry.device_ids.is_empty() { + match overrides::clear_device_overrides(&entry.device_ids) { + Ok(changed) => { + if changed { + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "circuit".to_string(), + entity_id: entry.circuit_entity_id.clone(), action: "clear_sqm_overrides_conflict".to_string(), persisted: true, reason: "Operator SQM overrides present; cleared TreeGuard SQM overlays.".to_string(), + ..Default::default() }, ); } @@ -1602,212 +1820,710 @@ fn run_tick( )); } } - for did in circuit_device_ids.iter() { - managed_device_ids.remove(did); + for device_id in entry.device_ids.iter() { + managed_device_ids.remove(device_id); } } continue; } - let decision = decisions::decide_circuit_sqm(decisions::CircuitSqmInput { - now_unix, - allowlisted: allowlisted_circuits.contains(circuit_id) && capacity_known, - cpu_max_pct, - cpu_cfg: &tg.cpu, - circuits_cfg: &tg.circuits, - qoo_cfg: &tg.qoo, - rtt_missing, - qoo, - state, - }); - - let mut proposed_down = state.down.desired; - let mut proposed_up = state.up.desired; - if let Some(down) = decision.down { - proposed_down = down; - } - if let Some(up) = decision.up { - proposed_up = up; + for device_id in entry.device_ids.iter() { + managed_device_ids.insert(device_id.clone()); } - let changed_down = proposed_down != state.down.desired; - let changed_up = proposed_up != state.up.desired; - let changed = changed_down || changed_up; - - if devices.is_empty() { - status.warnings.push(format!( - "TreeGuard circuits: circuit '{circuit_id}' has no devices in ShapedDevices.csv." - )); - } else { - for dev in devices.iter() { - managed_device_ids.insert(dev.device_id.clone()); - } - } - - if changed && !devices.is_empty() { - let token = decisions::format_directional_sqm_override(proposed_down, proposed_up); - - if tg.dry_run { - if changed_down { - state.down.desired = proposed_down; - state.down.last_change_unix = Some(now_unix); - state.down.recent_changes_unix.push_back(now_unix); - prune_recent_changes(&mut state.down.recent_changes_unix, now_unix); - } - if changed_up { - state.up.desired = proposed_up; - state.up.last_change_unix = Some(now_unix); - state.up.recent_changes_unix.push_back(now_unix); - prune_recent_changes(&mut state.up.recent_changes_unix, now_unix); - } - - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "circuit".to_string(), - entity_id: circuit_entity_id.clone(), - action: format!("would_set_sqm_override:{token}"), - persisted: false, - reason: "Dry-run".to_string(), - }, - ); - status.last_action_summary = Some(format!( - "Would set SQM override for circuit '{}' -> {}", - circuit_label, token - )); - } else { - let mut persisted_ok = false; - let mut can_apply_live = true; - if tg.circuits.persist_sqm_overrides { - let device_ids: Vec = - devices.iter().map(|device| device.device_id.clone()).collect(); - match overrides::set_devices_sqm_override(&device_ids, &token) { - Ok(_) => { - persisted_ok = true; - } - Err(e) => { - can_apply_live = false; - status.warnings.push(format!( - "TreeGuard circuits: failed to persist SQM overrides for circuit '{circuit_id}': {e}" - )); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "circuit".to_string(), - entity_id: circuit_entity_id.clone(), - action: "set_sqm_override_failed".to_string(), - persisted: false, - reason: format!("Overrides write failed: {e}"), - }, - ); - } - } - } - - let live_ok = if can_apply_live { - match bakery::apply_circuit_sqm_override_live(circuit_id, &devices, &token) - { - Ok(()) => true, - Err(e) => { - status.warnings.push(format!( - "TreeGuard circuits: live SQM apply failed for circuit '{circuit_id}': {e}" - )); - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "circuit".to_string(), - entity_id: circuit_entity_id.clone(), - action: format!("apply_sqm_live_failed:{token}"), - persisted: persisted_ok, - reason: format!("Bakery live apply failed: {e}"), - }, - ); - false - } - } - } else { - false - }; - - if live_ok || persisted_ok { - if changed_down { - state.down.desired = proposed_down; - state.down.last_change_unix = Some(now_unix); - state.down.recent_changes_unix.push_back(now_unix); - prune_recent_changes(&mut state.down.recent_changes_unix, now_unix); - } - if changed_up { - state.up.desired = proposed_up; - state.up.last_change_unix = Some(now_unix); - state.up.recent_changes_unix.push_back(now_unix); - prune_recent_changes(&mut state.up.recent_changes_unix, now_unix); - } - - let (action, reason) = match (persisted_ok, live_ok) { - (true, true) => ( - "set_sqm_override".to_string(), - "Applied live + persisted".to_string(), - ), - (true, false) => ( - "set_sqm_override".to_string(), - "Persisted (live apply failed)".to_string(), - ), - (false, true) => { - ("set_sqm_live".to_string(), "Applied live".to_string()) - } - (false, false) => { - ("set_sqm_live".to_string(), "Not applied".to_string()) - } - }; - push_activity( - activity, - TreeguardActivityEntry { - time: now_unix.to_string(), - entity_type: "circuit".to_string(), - entity_id: circuit_entity_id.clone(), - action: format!("{action}:{token}"), - persisted: persisted_ok, - reason, - }, - ); - - status.last_action_summary = Some(format!( - "SQM override for circuit '{}' -> {}", - circuit_label, token - )); - } - } - } - - if state.down.desired == CircuitSqmState::FqCodel - || state.up.desired == CircuitSqmState::FqCodel + if !treeguard_manages_circuit_direction(base_sqm.down) + && !treeguard_manages_circuit_direction(base_sqm.up) + && state.down.desired == base_sqm.down + && state.up.desired == base_sqm.up { - fq_codel_circuits += 1; + continue; + } + + let live_rollup = live_snapshot.by_circuit_id.get(circuit_id); + process_circuit_tick( + CircuitTickContext { + status, + activity, + managed_device_ids, + now_unix, + now_nanos_since_boot, + cpu_max_pct, + dry_run: tg.dry_run, + circuit_id, + circuit_entity_id: &entry.circuit_entity_id, + circuit_label: &entry.circuit_label, + devices: &entry.devices, + sqm_batch_id: &sqm_batch_id, + allowlisted: tg.circuits.all_circuits + || allowlisted_circuits.contains(circuit_id), + cap_down: entry.cap_down, + cap_up: entry.cap_up, + bps: live_rollup + .map(|rollup| rollup.bytes_per_second) + .unwrap_or(DownUpOrder { down: 0, up: 0 }), + last_rtt_seen_nanos: rtt_snapshot + .get(&entry.circuit_hash) + .map(|buf| buf.last_seen), + qoo: live_rollup.map(|rollup| rollup.qoo).unwrap_or(DownUpOrder { + down: None, + up: None, + }), + cpu_cfg: &tg.cpu, + circuits_cfg: &tg.circuits, + qoo_cfg: &tg.qoo, + base_sqm, + circuit_change_budget_remaining: &mut circuit_change_budget_remaining, + deferred_circuit_sqm_changes: &mut deferred_circuit_sqm_changes, + }, + state, + overrides::set_devices_sqm_override, + overrides::clear_device_overrides, + |circuit_id, devices, token| { + bakery::apply_circuit_sqm_override_live(circuit_id, devices, token) + }, + ); + } + if deferred_circuit_sqm_changes > 0 { + status.warnings.push(format!( + "TreeGuard circuits: deferred {} SQM changes because the per-tick circuit change budget ({}) was exhausted.", + deferred_circuit_sqm_changes, TREEGUARD_CIRCUIT_CHANGE_BUDGET_PER_TICK + )); + } + } + + status.virtualized_nodes = runtime_virtualized_nodes.len(); + let mut cake_circuits = 0usize; + let mut mixed_sqm_circuits = 0usize; + let mut fq_codel_circuits = 0usize; + for state in circuit_states.values() { + match (state.down.desired, state.up.desired) { + (CircuitSqmState::Cake, CircuitSqmState::Cake) => { + cake_circuits = cake_circuits.saturating_add(1); + } + (CircuitSqmState::FqCodel, CircuitSqmState::FqCodel) => { + fq_codel_circuits = fq_codel_circuits.saturating_add(1); + } + _ => { + mixed_sqm_circuits = mixed_sqm_circuits.saturating_add(1); + } + } + } + status.cake_circuits = cake_circuits; + status.mixed_sqm_circuits = mixed_sqm_circuits; + status.fq_codel_circuits = fq_codel_circuits; + warning_limiter.flush(status); +} + +/// Applies TreeGuard backoff while Bakery is performing a structural full reload. +/// +/// This function is not pure: it mutates TreeGuard status/runtime state and may emit logs. +fn pause_for_bakery_reload( + status: &mut TreeguardStatusData, + tick_seconds: &mut u64, + runtime_state: &mut TreeguardRuntimeState, + enabled: bool, + dry_run: bool, +) -> bool { + pause_for_bakery_reload_with_flag( + status, + tick_seconds, + runtime_state, + enabled, + dry_run, + lqos_bakery::full_reload_in_progress(), + lqos_bakery::bakery_reload_required_reason(), + ) +} + +fn pause_for_bakery_reload_with_flag( + status: &mut TreeguardStatusData, + tick_seconds: &mut u64, + runtime_state: &mut TreeguardRuntimeState, + enabled: bool, + dry_run: bool, + bakery_reload_in_progress: bool, + bakery_reload_required_reason: Option, +) -> bool { + let pause_reason = if bakery_reload_in_progress { + Some("Bakery full reload in progress".to_string()) + } else { + bakery_reload_required_reason + }; + let paused = enabled && pause_reason.is_some(); + if paused { + let pause_reason = + pause_reason.unwrap_or_else(|| "Bakery full reload in progress".to_string()); + if !runtime_state.paused_for_bakery_reload { + info!("TreeGuard: pausing because {}", pause_reason); + runtime_state.paused_for_bakery_reload = true; + } + *tick_seconds = (*tick_seconds).max(5); + status.enabled = enabled; + status.dry_run = dry_run; + status.paused_for_bakery_reload = true; + status.pause_reason = Some(pause_reason.clone()); + status.cpu_max_pct = None; + status.last_action_summary = Some(format!("Paused while {}", pause_reason)); + status + .warnings + .push(format!("TreeGuard paused while {}.", pause_reason)); + return true; + } + + if runtime_state.paused_for_bakery_reload { + info!("TreeGuard: resuming after Bakery pause"); + runtime_state.paused_for_bakery_reload = false; + } + status.paused_for_bakery_reload = false; + status.pause_reason = None; + if status + .last_action_summary + .as_deref() + .is_some_and(|summary| summary.starts_with("Paused while Bakery ")) + { + status.last_action_summary = None; + } + + false +} + +fn clear_legacy_treeguard_virtual_override( + status: &mut TreeguardStatusData, + activity: &mut VecDeque, + now_unix: u64, + node_name: &str, + reason: &str, +) { + match overrides::clear_node_virtual(node_name) { + Ok(changed) => { + if changed { + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.to_string(), + action: "clear_virtual_override".to_string(), + persisted: true, + reason: reason.to_string(), + ..Default::default() + }, + ); + } + } + Err(e) => { + status.warnings.push(format!( + "TreeGuard links: failed to clear legacy virtual override for node '{node_name}': {e}" + )); + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.to_string(), + action: "clear_virtual_override_failed".to_string(), + persisted: false, + reason: format!("Overrides write failed: {e}"), + ..Default::default() + }, + ); + } + } +} + +fn current_link_virtual_state( + runtime_virtualized_nodes: &FxHashSet, + node_name: &str, +) -> LinkVirtualState { + if runtime_virtualized_nodes.contains(node_name) { + LinkVirtualState::Virtual + } else { + LinkVirtualState::Physical + } +} + +#[allow(clippy::too_many_arguments)] +fn reconcile_pending_link_operations( + status: &mut TreeguardStatusData, + activity: &mut VecDeque, + now_unix: u64, + warning_limiter: &mut TreeguardWarningLimiter, + pending_link_operations: &mut FxHashMap, + runtime_virtualized_nodes: &mut FxHashSet, + link_virtualization_backoff_until_unix: &mut FxHashMap, + link_states: &mut FxHashMap, +) { + let pending_nodes: Vec = pending_link_operations.keys().cloned().collect(); + for node_name in pending_nodes { + let Some(pending) = pending_link_operations.get(&node_name).cloned() else { + continue; + }; + let Some(snapshot) = bakery::node_virtualization_operation_status(&node_name) else { + continue; + }; + match snapshot.status { + BakeryRuntimeNodeOperationStatus::Submitted + | BakeryRuntimeNodeOperationStatus::Applying => {} + BakeryRuntimeNodeOperationStatus::Completed + | BakeryRuntimeNodeOperationStatus::AppliedAwaitingCleanup => { + if pending.target == LinkVirtualState::Virtual { + runtime_virtualized_nodes.insert(node_name.clone()); + } else { + runtime_virtualized_nodes.remove(&node_name); + } + link_virtualization_backoff_until_unix.remove(&node_name); + pending_link_operations.remove(&node_name); + if let Some(state) = link_states.get_mut(&node_name) { + state.desired = pending.target; + state.last_change_unix = Some(now_unix); + state.recent_changes_unix.push_back(now_unix); + state.structural_ineligible = None; + prune_recent_changes(&mut state.recent_changes_unix, now_unix); + } + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.clone(), + action: match pending.target { + LinkVirtualState::Physical => "unvirtualize".to_string(), + LinkVirtualState::Virtual => "virtualize".to_string(), + }, + persisted: true, + reason: if snapshot.status + == BakeryRuntimeNodeOperationStatus::AppliedAwaitingCleanup + { + format!( + "{}. Bakery operation {} applied; cleanup pending.", + pending.reason, snapshot.operation_id + ) + } else { + pending.reason + }, + ..Default::default() + }, + ); + status.last_action_summary = Some(match pending.target { + LinkVirtualState::Physical => { + format!("Unvirtualized node '{node_name}'") + } + LinkVirtualState::Virtual => { + format!("Virtualized node '{node_name}'") + } + }); + if snapshot.status == BakeryRuntimeNodeOperationStatus::AppliedAwaitingCleanup { + let target_label = if pending.target == LinkVirtualState::Virtual { + "virtualization" + } else { + "restore" + }; + warning_limiter.push( + status, + format!("runtime_cleanup_pending|{target_label}"), + format!( + "TreeGuard links: node '{node_name}' runtime {target_label} applied in Bakery operation {} and is awaiting cleanup.", + snapshot.operation_id + ), + format!("TreeGuard runtime {target_label} cleanup-pending warnings"), + ); + } + } + BakeryRuntimeNodeOperationStatus::Deferred => { + let until = snapshot.next_retry_at_unix.unwrap_or_else(|| { + now_unix.saturating_add(TREEGUARD_LINK_VIRTUALIZATION_DEFERRED_BACKOFF_SECONDS) + }); + link_virtualization_backoff_until_unix.insert(node_name.clone(), until); + pending_link_operations.remove(&node_name); + if let Some(state) = link_states.get_mut(&node_name) { + state.desired = + current_link_virtual_state(runtime_virtualized_nodes, &node_name); + } + let details = snapshot.last_error.unwrap_or_else(|| { + format!( + "Bakery operation {} deferred by capacity", + snapshot.operation_id + ) + }); + let target_label = if pending.target == LinkVirtualState::Virtual { + "virtualization" + } else { + "restore" + }; + warning_limiter.push( + status, + format!("runtime_deferred|{target_label}|{details}"), + format!( + "TreeGuard links: deferred runtime {target_label} for node '{node_name}': {details}. Retrying after {until}." + ), + format!("TreeGuard deferred runtime {target_label} warnings for reason={details}"), + ); + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.clone(), + action: match pending.target { + LinkVirtualState::Physical => "unvirtualize_deferred".to_string(), + LinkVirtualState::Virtual => "virtualize_deferred".to_string(), + }, + persisted: false, + reason: format!( + "{}. Bakery deferred the operation: {details}", + pending.reason + ), + ..Default::default() + }, + ); + } + BakeryRuntimeNodeOperationStatus::Failed | BakeryRuntimeNodeOperationStatus::Dirty => { + let structural_ineligible = snapshot.failure_reason; + let until = now_unix.saturating_add(TREEGUARD_LINK_VIRTUALIZATION_BACKOFF_SECONDS); + if structural_ineligible.is_none() { + link_virtualization_backoff_until_unix.insert(node_name.clone(), until); + } else { + link_virtualization_backoff_until_unix.remove(&node_name); + } + pending_link_operations.remove(&node_name); + if let Some(state) = link_states.get_mut(&node_name) { + state.desired = + current_link_virtual_state(runtime_virtualized_nodes, &node_name); + state.structural_ineligible = + structural_ineligible.map(|reason| LinkStructuralIneligibleState { + reason, + topology_fingerprint: state.topology_fingerprint, + }); + } + let details = snapshot.last_error.unwrap_or_else(|| { + format!("Bakery operation {} failed", snapshot.operation_id) + }); + let target_label = if pending.target == LinkVirtualState::Virtual { + "virtualization" + } else { + "restore" + }; + if let Some(reason) = structural_ineligible { + let reason_label = structural_failure_reason_label(reason); + warning_limiter.push( + status, + format!("runtime_structural_ineligible|{target_label}|{:?}", reason), + format!( + "TreeGuard links: node '{node_name}' is structurally ineligible for runtime {target_label} ({reason_label}): {details}. TreeGuard will retry only after the node's direct topology changes." + ), + format!( + "TreeGuard runtime structural-ineligible {target_label} warnings for reason={reason_label}" + ), + ); + } else { + warning_limiter.push( + status, + format!("runtime_failed|{target_label}|{details}"), + format!( + "TreeGuard links: failed to apply runtime {target_label} for node '{node_name}': {details}. Backing off until {until}." + ), + format!( + "TreeGuard failed runtime {target_label} warnings for reason={details}" + ), + ); + } + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.clone(), + action: match pending.target { + LinkVirtualState::Physical => "unvirtualize_failed".to_string(), + LinkVirtualState::Virtual => "virtualize_failed".to_string(), + }, + persisted: false, + reason: format!( + "{}. Bakery runtime operation failed: {details}", + pending.reason + ), + ..Default::default() + }, + ); + } + } + } +} + +#[allow(clippy::too_many_arguments)] +fn restore_runtime_virtualization_if_needed( + status: &mut TreeguardStatusData, + activity: &mut VecDeque, + now_unix: u64, + node_name: &str, + reason: &str, + dry_run: bool, + runtime_virtualized_nodes: &mut FxHashSet, + pending_link_operations: &mut FxHashMap, + link_virtualization_backoff_until_unix: &mut FxHashMap, + link_states: &mut FxHashMap, +) { + if !runtime_virtualized_nodes.contains(node_name) { + if let Some(state) = link_states.get_mut(node_name) { + state.desired = LinkVirtualState::Physical; + } + pending_link_operations.remove(node_name); + link_virtualization_backoff_until_unix.remove(node_name); + return; + } + + if dry_run { + status.warnings.push(format!( + "TreeGuard links: node '{node_name}' remains runtime-virtualized because dry-run mode will not restore live topology." + )); + return; + } + + if pending_link_operations + .get(node_name) + .is_some_and(|pending| pending.target == LinkVirtualState::Physical) + { + status.warnings.push(format!( + "TreeGuard links: restore for node '{node_name}' is already queued in Bakery." + )); + return; + } + + match bakery::submit_node_virtualization_live(node_name, false) { + Ok(()) => { + pending_link_operations.insert( + node_name.to_string(), + PendingLinkOperation { + target: LinkVirtualState::Physical, + reason: reason.to_string(), + }, + ); + if let Some(state) = link_states.get_mut(node_name) { + state.desired = LinkVirtualState::Physical; + } + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.to_string(), + action: "unvirtualize_requested".to_string(), + persisted: false, + reason: format!("{reason}. Queued in Bakery for live restore."), + ..Default::default() + }, + ); + status.last_action_summary = Some(format!("Queued restore for node '{node_name}'")); + } + Err(e) => { + let until = now_unix.saturating_add(TREEGUARD_LINK_VIRTUALIZATION_BACKOFF_SECONDS); + link_virtualization_backoff_until_unix.insert(node_name.to_string(), until); + status.warnings.push(format!( + "TreeGuard links: failed to submit restore for runtime-virtualized node '{node_name}': {e}. Backing off until {until}." + )); + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.to_string(), + action: "unvirtualize_failed".to_string(), + persisted: false, + reason: format!("{reason}. Bakery restore submission failed: {e}"), + ..Default::default() + }, + ); + } + } +} + +#[allow(clippy::too_many_arguments)] +fn apply_link_virtualization_decision( + status: &mut TreeguardStatusData, + activity: &mut VecDeque, + now_unix: u64, + node_name: &str, + target: LinkVirtualState, + is_top_level: bool, + reason: String, + dry_run: bool, + state: &mut LinkState, + pending_link_operations: &mut FxHashMap, + link_virtualization_backoff_until_unix: &mut FxHashMap, +) { + if target == state.desired + && pending_link_operations + .get(node_name) + .is_none_or(|pending| pending.target == target) + { + return; + } + + let persist = !dry_run; + if persist { + if let Some(until) = link_virtualization_backoff_until_unix + .get(node_name) + .copied() + && now_unix < until + { + status.warnings.push(format!( + "TreeGuard links: node '{node_name}' runtime virtualization is temporarily ineligible until {until}." + )); + return; + } + + if pending_link_operations + .get(node_name) + .is_some_and(|pending| pending.target == target) + { + status.warnings.push(format!( + "TreeGuard links: node '{node_name}' runtime {} is already queued in Bakery.", + if target == LinkVirtualState::Virtual { + "virtualization" + } else { + "restore" + } + )); + return; + } + + if let Some((failure_reason, details)) = + top_level_structural_ineligibility(state, target, is_top_level) + { + let reason_label = structural_failure_reason_label(failure_reason); + link_virtualization_backoff_until_unix.remove(node_name); + state.structural_ineligible = Some(LinkStructuralIneligibleState { + reason: failure_reason, + topology_fingerprint: state.topology_fingerprint, + }); + status.warnings.push(format!( + "TreeGuard links: node '{node_name}' is structurally ineligible for runtime virtualization ({reason_label}): {details}. TreeGuard rejected the change before submitting it to Bakery." + )); + status.last_action_summary = + Some(format!("Rejected virtualization for node '{node_name}'")); + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.to_string(), + action: "virtualize_rejected".to_string(), + persisted: false, + reason: format!( + "{}. TreeGuard rejected runtime virtualization before submitting to Bakery: node {details}", + reason + ), + ..Default::default() + }, + ); + return; + } + + match bakery::submit_node_virtualization_live( + node_name, + target == LinkVirtualState::Virtual, + ) { + Ok(()) => { + pending_link_operations.insert( + node_name.to_string(), + PendingLinkOperation { target, reason }, + ); + state.desired = target; + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.to_string(), + action: match target { + LinkVirtualState::Physical => "unvirtualize_requested".to_string(), + LinkVirtualState::Virtual => "virtualize_requested".to_string(), + }, + persisted: false, + reason: format!( + "{}. Queued in Bakery for live {}.", + pending_link_operations + .get(node_name) + .map(|pending| pending.reason.as_str()) + .unwrap_or("TreeGuard queued a runtime topology change"), + if target == LinkVirtualState::Virtual { + "virtualization" + } else { + "restore" + } + ), + ..Default::default() + }, + ); + status.last_action_summary = Some(format!( + "Queued {} for node '{}'", + if target == LinkVirtualState::Virtual { + "virtualization" + } else { + "restore" + }, + node_name + )); + return; + } + Err(e) => { + let until = now_unix.saturating_add(TREEGUARD_LINK_VIRTUALIZATION_BACKOFF_SECONDS); + link_virtualization_backoff_until_unix.insert(node_name.to_string(), until); + status.warnings.push(format!( + "TreeGuard links: failed to submit runtime {} for node '{node_name}': {e}. Backing off until {until}." + , + if target == LinkVirtualState::Virtual { + "virtualization" + } else { + "restore" + } + )); + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.to_string(), + action: match target { + LinkVirtualState::Physical => "unvirtualize_failed".to_string(), + LinkVirtualState::Virtual => "virtualize_failed".to_string(), + }, + persisted: false, + reason: format!("Bakery runtime virtualization submission failed: {e}"), + ..Default::default() + }, + ); + return; } } } - status.virtualized_nodes = virtualized_nodes; - status.fq_codel_circuits = fq_codel_circuits; -} + state.desired = target; -/// Returns the current `set_node_virtual` override value for `node_name`, if present. -/// -/// This function is pure: it has no side effects. -fn overrides_node_virtual(overrides: &OverrideFile, node_name: &str) -> Option { - overrides - .network_adjustments() - .iter() - .find_map(|adj| match adj { - NetworkAdjustment::SetNodeVirtual { - node_name: n, - virtual_node, - } if n == node_name => Some(*virtual_node), - _ => None, - }) + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "node".to_string(), + entity_id: node_name.to_string(), + action: match target { + LinkVirtualState::Physical => "unvirtualize".to_string(), + LinkVirtualState::Virtual => "virtualize".to_string(), + }, + persisted: persist, + reason, + ..Default::default() + }, + ); + status.last_action_summary = Some(format!( + "{} node '{}'", + if target == LinkVirtualState::Virtual { + "Virtualized" + } else { + "Unvirtualized" + }, + node_name + )); } /// Returns the current SQM override token for `device_id`, if present. @@ -1876,29 +2592,38 @@ fn overrides_sqm_device_ids(overrides: &OverrideFile) -> FxHashSet { out } +/// Looks up an SQM override token for a circuit from a specific overrides file. +/// +/// This function is pure: it has no side effects. +fn find_circuit_override_token_in_overrides( + devices: &[lqos_config::ShapedDevice], + overrides: &OverrideFile, +) -> Option { + for device_id in devices.iter().map(|device| device.device_id.as_str()) { + if let Some(token) = overrides_device_sqm(overrides, device_id) { + return Some(token); + } + } + + None +} + /// Infers an SQM override token for a circuit, preferring persisted override entries. /// /// This function is pure: it has no side effects. fn infer_circuit_sqm_override_token( - circuit_id: &str, - shaped_devices: &[lqos_config::ShapedDevice], + devices: &[lqos_config::ShapedDevice], overrides: Option<&OverrideFile>, ) -> Option { - let circuit_device_ids: FxHashSet<&str> = shaped_devices - .iter() - .filter(|d| d.circuit_id == circuit_id) - .map(|d| d.device_id.as_str()) - .collect(); - if let Some(overrides) = overrides { - for device_id in &circuit_device_ids { + for device_id in devices.iter().map(|device| device.device_id.as_str()) { if let Some(token) = overrides_device_sqm(overrides, device_id) { return Some(token); } } } - for dev in shaped_devices.iter().filter(|d| d.circuit_id == circuit_id) { + for dev in devices { if let Some(token) = dev.sqm_override.as_deref() { let token = token.trim(); if !token.is_empty() { @@ -1910,16 +2635,557 @@ fn infer_circuit_sqm_override_token( None } +/// Returns the default effective SQM state for a direction at the given configured rate. +/// +/// This function is pure: it has no side effects. +fn default_sqm_state_for_rate(rate_mbps: f32, config: &lqos_config::Config) -> CircuitSqmState { + let default_sqm = config.queues.default_sqm.trim().to_ascii_lowercase(); + if default_sqm.starts_with("cake") { + let threshold = config.queues.fast_queues_fq_codel.unwrap_or(1000.0) as f32; + if rate_mbps >= threshold { + CircuitSqmState::FqCodel + } else { + CircuitSqmState::Cake + } + } else { + CircuitSqmState::FqCodel + } +} + +/// Computes the base per-direction SQM state for a circuit before TreeGuard overlays are applied. +/// +/// This function is pure: it has no side effects. +fn base_circuit_sqm_state( + devices: &[lqos_config::ShapedDevice], + operator_overrides: Option<&OverrideFile>, + config: &lqos_config::Config, + cap_down: f32, + cap_up: f32, +) -> DownUpOrder { + let mut down = default_sqm_state_for_rate(cap_down, config); + let mut up = default_sqm_state_for_rate(cap_up, config); + + if let Some(token) = infer_circuit_sqm_override_token(devices, operator_overrides) { + let parsed = decisions::parse_directional_sqm_override(&token); + if let Some(v) = parsed.down { + down = v; + } + if let Some(v) = parsed.up { + up = v; + } + } + + DownUpOrder { down, up } +} + /// Appends an entry to the activity ring buffer. /// /// This function is not pure: it mutates `activity`. fn push_activity(activity: &mut VecDeque, entry: TreeguardActivityEntry) { + if entry.action.contains("failed") { + warn!( + "TreeGuard activity: entity_type={} entity_id={} action={} persisted={} reason={}", + entry.entity_type, entry.entity_id, entry.action, entry.persisted, entry.reason + ); + } else { + debug!( + "TreeGuard activity: entity_type={} entity_id={} action={} persisted={} reason={}", + entry.entity_type, entry.entity_id, entry.action, entry.persisted, entry.reason + ); + } if activity.len() >= ACTIVITY_RING_CAPACITY { activity.pop_front(); } activity.push_back(entry); } +struct CircuitSqmApplyContext<'a> { + status: &'a mut TreeguardStatusData, + activity: &'a mut VecDeque, + now_unix: u64, + dry_run: bool, + persist_sqm_overrides: bool, + circuit_id: &'a str, + circuit_entity_id: &'a str, + circuit_label: &'a str, + devices: &'a [lqos_config::ShapedDevice], + base_sqm: DownUpOrder, + batch_id: &'a str, +} + +struct CircuitSqmTransition { + proposed_down: CircuitSqmState, + proposed_up: CircuitSqmState, + changed_down: bool, + changed_up: bool, +} + +struct CircuitTickContext<'a> { + status: &'a mut TreeguardStatusData, + activity: &'a mut VecDeque, + managed_device_ids: &'a mut FxHashSet, + now_unix: u64, + now_nanos_since_boot: Option, + cpu_max_pct: Option, + dry_run: bool, + circuit_id: &'a str, + circuit_entity_id: &'a str, + circuit_label: &'a str, + devices: &'a [lqos_config::ShapedDevice], + sqm_batch_id: &'a str, + allowlisted: bool, + cap_down: f32, + cap_up: f32, + bps: DownUpOrder, + last_rtt_seen_nanos: Option, + qoo: DownUpOrder>, + cpu_cfg: &'a lqos_config::TreeguardCpuConfig, + circuits_cfg: &'a lqos_config::TreeguardCircuitsConfig, + qoo_cfg: &'a lqos_config::TreeguardQooConfig, + base_sqm: DownUpOrder, + circuit_change_budget_remaining: &'a mut usize, + deferred_circuit_sqm_changes: &'a mut usize, +} + +fn treeguard_manages_circuit_direction(base_sqm: CircuitSqmState) -> bool { + matches!(base_sqm, CircuitSqmState::Cake) +} + +fn circuit_sqm_transition_from_decision( + state: &CircuitState, + base_sqm: DownUpOrder, + decision: decisions::CircuitSqmDecision, +) -> CircuitSqmTransition { + let mut proposed_down = state.down.desired; + let mut proposed_up = state.up.desired; + + if treeguard_manages_circuit_direction(base_sqm.down) { + if let Some(down) = decision.down { + proposed_down = down; + } + } else { + proposed_down = base_sqm.down; + } + + if treeguard_manages_circuit_direction(base_sqm.up) { + if let Some(up) = decision.up { + proposed_up = up; + } + } else { + proposed_up = base_sqm.up; + } + + CircuitSqmTransition { + proposed_down, + proposed_up, + changed_down: proposed_down != state.down.desired, + changed_up: proposed_up != state.up.desired, + } +} + +fn try_consume_circuit_change_budget(remaining_budget: &mut usize) -> bool { + if *remaining_budget == 0 { + return false; + } + *remaining_budget -= 1; + true +} + +fn next_sqm_batch_id(next_sqm_batch_id: &mut u64) -> String { + *next_sqm_batch_id = next_sqm_batch_id.saturating_add(1); + format!("sqm-{}", *next_sqm_batch_id) +} + +fn is_retryable_live_mutation_unavailable(error: &TreeguardError) -> Option<&str> { + match error { + TreeguardError::LiveMutationUnavailable { details } => Some(details.as_str()), + _ => None, + } +} + +fn apply_circuit_sqm_change( + ctx: CircuitSqmApplyContext<'_>, + state: &mut CircuitState, + transition: CircuitSqmTransition, + mut persist_override: P, + mut clear_override: C, + mut live_apply: L, +) where + P: FnMut(&[String], &str) -> Result, + C: FnMut(&[String]) -> Result, + L: FnMut(&str, &[lqos_config::ShapedDevice], &str) -> Result<(), TreeguardError>, +{ + let CircuitSqmApplyContext { + status, + activity, + now_unix, + dry_run, + persist_sqm_overrides, + circuit_id, + circuit_entity_id, + circuit_label, + devices, + base_sqm, + batch_id, + } = ctx; + let CircuitSqmTransition { + proposed_down, + proposed_up, + changed_down, + changed_up, + } = transition; + + let token = decisions::format_directional_sqm_override(proposed_down, proposed_up); + let returning_to_base = proposed_down == base_sqm.down && proposed_up == base_sqm.up; + let live_token = if returning_to_base { + "/" + } else { + token.as_str() + }; + + if dry_run { + if changed_down { + state.down.desired = proposed_down; + state.down.last_change_unix = Some(now_unix); + state.down.recent_changes_unix.push_back(now_unix); + prune_recent_changes(&mut state.down.recent_changes_unix, now_unix); + } + if changed_up { + state.up.desired = proposed_up; + state.up.last_change_unix = Some(now_unix); + state.up.recent_changes_unix.push_back(now_unix); + prune_recent_changes(&mut state.up.recent_changes_unix, now_unix); + } + + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "circuit".to_string(), + entity_id: circuit_entity_id.to_string(), + action: if returning_to_base { + "would_clear_sqm_override".to_string() + } else { + format!("would_set_sqm_override:{token}") + }, + persisted: false, + reason: "Dry-run".to_string(), + batch_id: Some(batch_id.to_string()), + batch_kind: Some("sqm".to_string()), + }, + ); + status.last_action_summary = Some(if returning_to_base { + format!( + "Would clear TreeGuard SQM override for circuit '{}' (base {})", + circuit_label, token + ) + } else { + format!( + "Would set SQM override for circuit '{}' -> {}", + circuit_label, token + ) + }); + return; + } + + let mut persisted_ok = false; + let device_ids: Vec = devices + .iter() + .map(|device| device.device_id.clone()) + .collect(); + if persist_sqm_overrides { + let persist_result = if returning_to_base { + clear_override(&device_ids) + } else { + persist_override(&device_ids, &token) + }; + match persist_result { + Ok(_) => { + persisted_ok = true; + } + Err(e) => { + status.warnings.push(format!( + "TreeGuard circuits: failed to {} SQM overrides for circuit '{circuit_id}': {e}", + if returning_to_base { "clear" } else { "persist" } + )); + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "circuit".to_string(), + entity_id: circuit_entity_id.to_string(), + action: if returning_to_base { + "clear_sqm_override_failed".to_string() + } else { + "set_sqm_override_failed".to_string() + }, + persisted: false, + reason: format!("Overrides write failed: {e}"), + batch_id: Some(batch_id.to_string()), + batch_kind: Some("sqm".to_string()), + }, + ); + } + } + } + + let mut live_deferred = false; + let live_ok = match live_apply(circuit_id, devices, live_token) { + Ok(()) => true, + Err(e) => { + if let Some(details) = is_retryable_live_mutation_unavailable(&e) { + live_deferred = true; + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "circuit".to_string(), + entity_id: circuit_entity_id.to_string(), + action: if returning_to_base { + "clear_sqm_live_deferred".to_string() + } else { + format!("apply_sqm_live_deferred:{token}") + }, + persisted: persisted_ok, + reason: format!("Bakery live apply deferred: {details}"), + batch_id: Some(batch_id.to_string()), + batch_kind: Some("sqm".to_string()), + }, + ); + } else { + status.warnings.push(format!( + "TreeGuard circuits: live SQM apply failed for circuit '{circuit_id}': {e}" + )); + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "circuit".to_string(), + entity_id: circuit_entity_id.to_string(), + action: if returning_to_base { + "clear_sqm_live_failed".to_string() + } else { + format!("apply_sqm_live_failed:{token}") + }, + persisted: persisted_ok, + reason: format!("Bakery live apply failed: {e}"), + batch_id: Some(batch_id.to_string()), + batch_kind: Some("sqm".to_string()), + }, + ); + } + false + } + }; + + if live_ok || persisted_ok { + if changed_down { + state.down.desired = proposed_down; + state.down.last_change_unix = Some(now_unix); + state.down.recent_changes_unix.push_back(now_unix); + prune_recent_changes(&mut state.down.recent_changes_unix, now_unix); + } + if changed_up { + state.up.desired = proposed_up; + state.up.last_change_unix = Some(now_unix); + state.up.recent_changes_unix.push_back(now_unix); + prune_recent_changes(&mut state.up.recent_changes_unix, now_unix); + } + + let (action, reason) = match (returning_to_base, persisted_ok, live_ok) { + (false, true, true) => ( + "set_sqm_override".to_string(), + "Applied live + persisted".to_string(), + ), + (false, true, false) => ( + "set_sqm_override".to_string(), + if live_deferred { + "Persisted (live apply deferred)".to_string() + } else { + "Persisted (live apply failed)".to_string() + }, + ), + (false, false, true) => ("set_sqm_live".to_string(), "Applied live".to_string()), + (false, false, false) => ("set_sqm_live".to_string(), "Not applied".to_string()), + (true, true, true) => ( + "clear_sqm_override".to_string(), + "Cleared live + persisted overlay".to_string(), + ), + (true, true, false) => ( + "clear_sqm_override".to_string(), + if live_deferred { + "Persisted clear (live apply deferred)".to_string() + } else { + "Persisted clear (live apply failed)".to_string() + }, + ), + (true, false, true) => ( + "clear_sqm_live".to_string(), + "Applied live clear".to_string(), + ), + (true, false, false) => ("clear_sqm_live".to_string(), "Not applied".to_string()), + }; + push_activity( + activity, + TreeguardActivityEntry { + time: now_unix.to_string(), + entity_type: "circuit".to_string(), + entity_id: circuit_entity_id.to_string(), + action: format!("{action}:{token}"), + persisted: persisted_ok, + reason, + batch_id: Some(batch_id.to_string()), + batch_kind: Some("sqm".to_string()), + }, + ); + + status.last_action_summary = Some(if returning_to_base { + format!( + "Cleared TreeGuard SQM override for circuit '{}' (base {})", + circuit_label, token + ) + } else { + format!("SQM override for circuit '{}' -> {}", circuit_label, token) + }); + } +} + +fn process_circuit_tick( + ctx: CircuitTickContext<'_>, + state: &mut CircuitState, + persist_override: P, + clear_override: C, + live_apply: L, +) -> bool +where + P: FnMut(&[String], &str) -> Result, + C: FnMut(&[String]) -> Result, + L: FnMut(&str, &[lqos_config::ShapedDevice], &str) -> Result<(), TreeguardError>, +{ + let CircuitTickContext { + status, + activity, + managed_device_ids, + now_unix, + now_nanos_since_boot, + cpu_max_pct, + dry_run, + circuit_id, + circuit_entity_id, + circuit_label, + devices, + sqm_batch_id, + allowlisted, + cap_down, + cap_up, + bps, + last_rtt_seen_nanos, + qoo, + cpu_cfg, + circuits_cfg, + qoo_cfg, + base_sqm, + circuit_change_budget_remaining, + deferred_circuit_sqm_changes, + } = ctx; + + prune_recent_changes(&mut state.down.recent_changes_unix, now_unix); + prune_recent_changes(&mut state.up.recent_changes_unix, now_unix); + + let capacity_known = cap_down > 0.0 && cap_up > 0.0; + if !capacity_known { + status.warnings.push(format!( + "TreeGuard circuits: circuit '{circuit_id}' has unknown capacity; no changes will be made." + )); + state.down.idle_since_unix = None; + state.up.idle_since_unix = None; + } else { + let mbps_down = (bps.down as f64 * 8.0) / 1_000_000.0; + let mbps_up = (bps.up as f64 * 8.0) / 1_000_000.0; + let util_down_pct = (mbps_down / cap_down as f64) * 100.0; + let util_up_pct = (mbps_up / cap_up as f64) * 100.0; + + let ewma_down = state + .down + .util_ewma_pct + .update(util_down_pct, UTIL_EWMA_ALPHA); + let ewma_up = state.up.util_ewma_pct.update(util_up_pct, UTIL_EWMA_ALPHA); + + update_idle_since( + &mut state.down.idle_since_unix, + now_unix, + ewma_down, + circuits_cfg.idle_util_pct as f64, + ); + update_idle_since( + &mut state.up.idle_since_unix, + now_unix, + ewma_up, + circuits_cfg.idle_util_pct as f64, + ); + } + + let rtt_missing = match (now_nanos_since_boot, last_rtt_seen_nanos) { + (Some(now_nanos), Some(last_seen)) if last_seen > 0 => { + now_nanos.saturating_sub(last_seen) + >= u64::from(circuits_cfg.rtt_missing_seconds).saturating_mul(1_000_000_000) + } + _ => true, + }; + + let decision = decisions::decide_circuit_sqm(decisions::CircuitSqmInput { + now_unix, + allowlisted: allowlisted && capacity_known, + cpu_max_pct, + cpu_cfg, + circuits_cfg, + qoo_cfg, + rtt_missing, + qoo, + state, + }); + let transition = circuit_sqm_transition_from_decision(state, base_sqm, decision); + + if devices.is_empty() { + status.warnings.push(format!( + "TreeGuard circuits: circuit '{circuit_id}' has no devices in ShapedDevices.csv." + )); + } else { + for dev in devices { + managed_device_ids.insert(dev.device_id.clone()); + } + } + + if (transition.changed_down || transition.changed_up) && !devices.is_empty() { + if try_consume_circuit_change_budget(circuit_change_budget_remaining) { + apply_circuit_sqm_change( + CircuitSqmApplyContext { + status, + activity, + now_unix, + dry_run, + persist_sqm_overrides: circuits_cfg.persist_sqm_overrides, + circuit_id, + circuit_entity_id, + circuit_label, + devices, + base_sqm, + batch_id: sqm_batch_id, + }, + state, + transition, + persist_override, + clear_override, + live_apply, + ); + } else { + *deferred_circuit_sqm_changes = deferred_circuit_sqm_changes.saturating_add(1); + } + } + + state.down.desired == CircuitSqmState::FqCodel || state.up.desired == CircuitSqmState::FqCodel +} + /// Removes entries older than one hour from a recent-changes ring buffer. /// /// This function is not pure: it mutates `recent_changes`. @@ -1963,13 +3229,1201 @@ fn update_below_since( } } -/// Returns the minimum of two optional floats, treating `None` as "unknown". -/// -/// This function is pure: it has no side effects. -fn min_opt_f32(a: Option, b: Option) -> Option { - match (a, b) { - (Some(x), Some(y)) => Some(x.min(y)), - (Some(x), None) | (None, Some(x)) => Some(x), - (None, None) => None, +/// Updates an "above threshold since" timestamp based on utilization and a threshold. +fn update_above_since( + above_since: &mut Option, + now_unix: u64, + util_pct: f64, + threshold_pct: f64, +) { + if util_pct >= threshold_pct { + if above_since.is_none() { + *above_since = Some(now_unix); + } + } else { + *above_since = None; + } +} + +#[cfg(test)] +mod tests { + use super::{ + CircuitSqmApplyContext, CircuitSqmTransition, CircuitTickContext, LinkVirtualState, + PendingLinkVirtualizationDecision, TreeguardRuntimeState, apply_circuit_sqm_change, + apply_link_virtualization_decision, base_circuit_sqm_state, circuit_evaluation_batch_size, + circuit_sqm_transition_from_decision, clear_structural_ineligible_if_topology_changed, + collect_circuit_batch, empty_status_snapshot, latched_structural_ineligible_reason, + pause_for_bakery_reload_with_flag, process_circuit_tick, run_tick, + select_link_virtualization_candidates, treeguard_manages_circuit_direction, + try_consume_circuit_change_budget, + }; + use crate::node_manager::ws::messages::TreeguardActivityEntry; + use crate::shaped_devices_tracker::{NETWORK_JSON, SHAPED_DEVICES}; + use crate::system_stats::SystemStats; + use crate::throughput_tracker::CIRCUIT_RTT_BUFFERS; + use crate::treeguard::decisions; + use crate::treeguard::{ + bakery, + errors::TreeguardError, + state::{ + CircuitSqmState, LinkState, LinkStructuralIneligibleState, LinkTopologyFingerprint, + }, + }; + use crossbeam_channel::bounded; + use fxhash::{FxHashMap, FxHashSet}; + use lqos_bakery::{BakeryCommands, BakeryRuntimeNodeOperationFailureReason}; + use lqos_bus::TcHandle; + use lqos_config::ConfigShapedDevices; + use lqos_config::ShapedDevice; + use lqos_queue_tracker::{QUEUE_STRUCTURE, QueueNode, QueueStructure}; + use lqos_utils::rtt::RttBuffer; + use lqos_utils::units::DownUpOrder; + use std::collections::VecDeque; + use std::ffi::OsString; + use std::net::Ipv4Addr; + use std::path::PathBuf; + use std::sync::Arc; + use std::sync::{Once, OnceLock}; + + #[allow(clippy::too_many_arguments)] + fn pending_link_decision( + node_name: &str, + node_index: usize, + target: LinkVirtualState, + subtree_nodes: usize, + current_subtree_throughput_mbps: f64, + explicit_allowlist: bool, + is_top_level: bool, + value_score: u64, + ) -> PendingLinkVirtualizationDecision { + PendingLinkVirtualizationDecision { + node_name: node_name.to_string(), + node_index, + target, + reason: "test".to_string(), + subtree_nodes, + current_subtree_throughput_mbps, + explicit_allowlist, + is_top_level, + value_score, + } + } + + #[test] + fn bakery_reload_pause_updates_status_and_tick_backoff() { + let mut status = empty_status_snapshot(); + let mut tick_seconds = 1; + let mut runtime_state = TreeguardRuntimeState::default(); + + let paused = pause_for_bakery_reload_with_flag( + &mut status, + &mut tick_seconds, + &mut runtime_state, + true, + false, + true, + None, + ); + + assert!(paused); + assert_eq!(tick_seconds, 5); + assert!(status.enabled); + assert!(!status.dry_run); + assert_eq!( + status.last_action_summary.as_deref(), + Some("Paused while Bakery full reload in progress") + ); + assert!( + status + .warnings + .iter() + .any(|warning| warning.contains("TreeGuard paused while Bakery full reload")) + ); + assert!(runtime_state.paused_for_bakery_reload); + + let resumed = pause_for_bakery_reload_with_flag( + &mut status, + &mut tick_seconds, + &mut runtime_state, + true, + false, + false, + None, + ); + assert!(!resumed); + assert!(!runtime_state.paused_for_bakery_reload); + assert!(status.last_action_summary.is_none()); + } + + #[test] + fn bakery_reload_required_pause_sets_reason() { + let mut status = empty_status_snapshot(); + let mut tick_seconds = 1; + let mut runtime_state = TreeguardRuntimeState::default(); + + let paused = pause_for_bakery_reload_with_flag( + &mut status, + &mut tick_seconds, + &mut runtime_state, + true, + false, + false, + Some("Bakery requires full reload".to_string()), + ); + + assert!(paused); + assert_eq!( + status.pause_reason.as_deref(), + Some("Bakery requires full reload") + ); + assert_eq!( + status.last_action_summary.as_deref(), + Some("Paused while Bakery requires full reload") + ); + } + + #[test] + fn link_candidate_selection_prefers_larger_ancestor_and_budget() { + let parent_by_index = vec![None, Some(0), Some(1), Some(1), Some(0)]; + let existing_virtualized = FxHashSet::default(); + let candidates = vec![ + pending_link_decision( + "region", + 0, + LinkVirtualState::Virtual, + 5, + 10.0, + false, + true, + 500, + ), + pending_link_decision( + "pop-a", + 1, + LinkVirtualState::Virtual, + 3, + 10.0, + false, + false, + 300, + ), + pending_link_decision( + "ap-a1", + 2, + LinkVirtualState::Virtual, + 1, + 10.0, + false, + false, + 100, + ), + pending_link_decision( + "ap-a2", + 3, + LinkVirtualState::Virtual, + 1, + 10.0, + false, + false, + 90, + ), + pending_link_decision( + "pop-b", + 4, + LinkVirtualState::Virtual, + 1, + 10.0, + false, + false, + 80, + ), + ]; + + let (selected, deferred, skipped_low_value) = select_link_virtualization_candidates( + candidates, + &parent_by_index, + &existing_virtualized, + 0, + ); + + let selected_names: Vec<&str> = selected + .iter() + .map(|candidate| candidate.node_name.as_str()) + .collect(); + assert_eq!(selected_names, vec!["region"]); + assert_eq!(deferred, 4); + assert_eq!(skipped_low_value, 0); + } + + #[test] + fn link_candidate_selection_allows_small_explicit_allowlist_node() { + let parent_by_index = vec![None]; + let existing_virtualized = FxHashSet::default(); + let candidates = vec![pending_link_decision( + "small-explicit", + 0, + LinkVirtualState::Virtual, + 1, + 0.2, + true, + false, + 1, + )]; + + let (selected, deferred, skipped_low_value) = select_link_virtualization_candidates( + candidates, + &parent_by_index, + &existing_virtualized, + 0, + ); + + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].node_name, "small-explicit"); + assert_eq!(deferred, 0); + assert_eq!(skipped_low_value, 0); + } + + #[test] + fn link_candidate_selection_prioritizes_restore_before_new_virtualization() { + let parent_by_index = vec![None, None]; + let existing_virtualized = FxHashSet::default(); + let candidates = vec![ + pending_link_decision( + "virtualize-me", + 0, + LinkVirtualState::Virtual, + 10, + 10.0, + false, + true, + 1_000, + ), + pending_link_decision( + "restore-me", + 1, + LinkVirtualState::Physical, + 1, + 0.2, + false, + false, + 1, + ), + ]; + + let (selected, _deferred, skipped_low_value) = select_link_virtualization_candidates( + candidates, + &parent_by_index, + &existing_virtualized, + 0, + ); + + assert_eq!( + selected + .first() + .map(|candidate| candidate.node_name.as_str()), + Some("restore-me") + ); + assert_eq!(skipped_low_value, 0); + } + + #[test] + fn link_candidate_selection_skips_small_low_throughput_automatic_nodes() { + let parent_by_index = vec![None]; + let existing_virtualized = FxHashSet::default(); + let candidates = vec![pending_link_decision( + "small-auto", + 0, + LinkVirtualState::Virtual, + 8, + 0.4, + false, + false, + 10, + )]; + + let (selected, deferred, skipped_low_value) = select_link_virtualization_candidates( + candidates, + &parent_by_index, + &existing_virtualized, + 0, + ); + + assert!(selected.is_empty()); + assert_eq!(deferred, 0); + assert_eq!(skipped_low_value, 1); + } + + #[test] + fn link_candidate_selection_allows_large_low_throughput_automatic_nodes() { + let parent_by_index = vec![None]; + let existing_virtualized = FxHashSet::default(); + let candidates = vec![pending_link_decision( + "large-auto", + 0, + LinkVirtualState::Virtual, + 16, + 0.4, + false, + false, + 10, + )]; + + let (selected, deferred, skipped_low_value) = select_link_virtualization_candidates( + candidates, + &parent_by_index, + &existing_virtualized, + 0, + ); + + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].node_name, "large-auto"); + assert_eq!(deferred, 0); + assert_eq!(skipped_low_value, 0); + } + + #[test] + fn structural_ineligible_latch_blocks_virtualize_until_topology_changes() { + let mut state = LinkState { + topology_fingerprint: LinkTopologyFingerprint { + direct_child_sites: 0, + direct_circuits: 0, + }, + structural_ineligible: Some(LinkStructuralIneligibleState { + reason: + BakeryRuntimeNodeOperationFailureReason::StructuralIneligibleNoPromotableChildren, + topology_fingerprint: LinkTopologyFingerprint { + direct_child_sites: 0, + direct_circuits: 0, + }, + }), + ..Default::default() + }; + + assert_eq!( + latched_structural_ineligible_reason(&state, LinkVirtualState::Virtual), + Some(BakeryRuntimeNodeOperationFailureReason::StructuralIneligibleNoPromotableChildren) + ); + assert_eq!( + latched_structural_ineligible_reason(&state, LinkVirtualState::Physical), + None + ); + + let changed = clear_structural_ineligible_if_topology_changed( + &mut state, + LinkTopologyFingerprint { + direct_child_sites: 1, + direct_circuits: 0, + }, + ); + state.topology_fingerprint = LinkTopologyFingerprint { + direct_child_sites: 1, + direct_circuits: 0, + }; + + assert!(changed); + assert!(state.structural_ineligible.is_none()); + assert_eq!( + latched_structural_ineligible_reason(&state, LinkVirtualState::Virtual), + None + ); + } + + #[test] + fn actor_live_sqm_change_updates_state_activity_and_bakery_command() { + let (tx, rx) = bounded(1); + let queues = vec![QueueNode { + circuit_id: Some("circuit-1".to_string()), + class_minor: 0x14af, + class_major: 0x0003, + up_class_major: 0x0043, + parent_class_id: TcHandle::from_string("3:20").expect("valid down parent"), + up_parent_class_id: TcHandle::from_string("43:20").expect("valid up parent"), + download_bandwidth_mbps_min: 50, + upload_bandwidth_mbps_min: 10, + download_bandwidth_mbps: 200, + upload_bandwidth_mbps: 50, + ..QueueNode::default() + }]; + let devices = vec![ShapedDevice { + circuit_id: "circuit-1".to_string(), + circuit_name: "Circuit One".to_string(), + device_id: "device-1".to_string(), + ipv4: vec![(Ipv4Addr::new(192, 0, 2, 10), 32)], + ..ShapedDevice::default() + }]; + let mut status = empty_status_snapshot(); + let mut activity: VecDeque = VecDeque::new(); + let mut state = crate::treeguard::state::CircuitState::default(); + + apply_circuit_sqm_change( + CircuitSqmApplyContext { + status: &mut status, + activity: &mut activity, + now_unix: 1_000, + dry_run: false, + persist_sqm_overrides: false, + circuit_id: "circuit-1", + circuit_entity_id: "Circuit One (circuit-1)", + circuit_label: "Circuit One", + devices: &devices, + base_sqm: DownUpOrder { + down: CircuitSqmState::Cake, + up: CircuitSqmState::Cake, + }, + batch_id: "sqm-test-1", + }, + &mut state, + CircuitSqmTransition { + proposed_down: CircuitSqmState::FqCodel, + proposed_up: CircuitSqmState::Cake, + changed_down: true, + changed_up: false, + }, + |_device_ids, _token| Ok(false), + |_device_ids| Ok(false), + |circuit_id, devices, token| { + bakery::apply_circuit_sqm_override_live_with_sender_and_snapshot( + circuit_id, devices, token, &tx, &queues, + ) + }, + ); + + assert_eq!(state.down.desired, CircuitSqmState::FqCodel); + assert_eq!(state.up.desired, CircuitSqmState::Cake); + assert_eq!(state.down.last_change_unix, Some(1_000)); + assert_eq!(state.up.last_change_unix, None); + assert_eq!(state.down.recent_changes_unix.len(), 1); + assert!(state.up.recent_changes_unix.is_empty()); + assert_eq!( + status.last_action_summary.as_deref(), + Some("SQM override for circuit 'Circuit One' -> fq_codel/cake") + ); + + let last_activity = activity.back().expect("activity should be recorded"); + assert_eq!(last_activity.entity_type, "circuit"); + assert_eq!(last_activity.entity_id, "Circuit One (circuit-1)"); + assert_eq!(last_activity.action, "set_sqm_live:fq_codel/cake"); + assert!(!last_activity.persisted); + assert_eq!(last_activity.reason, "Applied live"); + assert_eq!(last_activity.batch_id.as_deref(), Some("sqm-test-1")); + assert_eq!(last_activity.batch_kind.as_deref(), Some("sqm")); + + let command = rx.try_recv().expect("bakery command should be sent"); + let BakeryCommands::AddCircuit { + circuit_hash, + sqm_override, + down_qdisc_handle, + up_qdisc_handle, + .. + } = command + else { + panic!("expected AddCircuit"); + }; + assert_eq!(circuit_hash, lqos_utils::hash_to_i64("circuit-1")); + assert_eq!(sqm_override, Some("fq_codel/cake".to_string())); + assert_eq!(down_qdisc_handle, None); + assert_eq!(up_qdisc_handle, None); + } + + #[test] + fn actor_return_to_base_clears_treeguard_overlay_and_uses_live_clear_token() { + let devices = vec![ShapedDevice { + circuit_id: "circuit-1".to_string(), + circuit_name: "Circuit One".to_string(), + device_id: "device-1".to_string(), + ..ShapedDevice::default() + }]; + let mut status = empty_status_snapshot(); + let mut activity: VecDeque = VecDeque::new(); + let mut state = crate::treeguard::state::CircuitState::default(); + state.down.desired = CircuitSqmState::FqCodel; + state.up.desired = CircuitSqmState::Cake; + let mut cleared_device_ids: Vec = Vec::new(); + let mut live_token: Option = None; + + apply_circuit_sqm_change( + CircuitSqmApplyContext { + status: &mut status, + activity: &mut activity, + now_unix: 1_000, + dry_run: false, + persist_sqm_overrides: true, + circuit_id: "circuit-1", + circuit_entity_id: "Circuit One (circuit-1)", + circuit_label: "Circuit One", + devices: &devices, + base_sqm: DownUpOrder { + down: CircuitSqmState::Cake, + up: CircuitSqmState::Cake, + }, + batch_id: "sqm-test-2", + }, + &mut state, + CircuitSqmTransition { + proposed_down: CircuitSqmState::Cake, + proposed_up: CircuitSqmState::Cake, + changed_down: true, + changed_up: false, + }, + |_device_ids, _token| Ok(false), + |device_ids| { + cleared_device_ids = device_ids.to_vec(); + Ok(true) + }, + |_circuit_id, _devices, token| { + live_token = Some(token.to_string()); + Ok(()) + }, + ); + + assert_eq!(state.down.desired, CircuitSqmState::Cake); + assert_eq!(state.up.desired, CircuitSqmState::Cake); + assert_eq!(cleared_device_ids, vec!["device-1".to_string()]); + assert_eq!(live_token.as_deref(), Some("/")); + assert_eq!( + status.last_action_summary.as_deref(), + Some("Cleared TreeGuard SQM override for circuit 'Circuit One' (base cake/cake)") + ); + + let last_activity = activity.back().expect("activity should be recorded"); + assert_eq!(last_activity.action, "clear_sqm_override:cake/cake"); + assert!(last_activity.persisted); + assert_eq!(last_activity.reason, "Cleared live + persisted overlay"); + assert_eq!(last_activity.batch_id.as_deref(), Some("sqm-test-2")); + assert_eq!(last_activity.batch_kind.as_deref(), Some("sqm")); + } + + #[test] + fn circuit_tick_snapshot_decides_and_applies_live_override() { + let (tx, rx) = bounded(1); + let queues = vec![QueueNode { + circuit_id: Some("circuit-2".to_string()), + class_minor: 0x22af, + class_major: 0x0005, + up_class_major: 0x0045, + parent_class_id: TcHandle::from_string("5:20").expect("valid down parent"), + up_parent_class_id: TcHandle::from_string("45:20").expect("valid up parent"), + download_bandwidth_mbps_min: 25, + upload_bandwidth_mbps_min: 5, + download_bandwidth_mbps: 100, + upload_bandwidth_mbps: 20, + ..QueueNode::default() + }]; + let devices = vec![ShapedDevice { + circuit_id: "circuit-2".to_string(), + circuit_name: "Circuit Two".to_string(), + device_id: "device-2".to_string(), + ipv4: vec![(Ipv4Addr::new(198, 51, 100, 20), 32)], + ..ShapedDevice::default() + }]; + let mut managed_device_ids = FxHashSet::default(); + let mut status = empty_status_snapshot(); + let mut activity: VecDeque = VecDeque::new(); + let mut state = crate::treeguard::state::CircuitState::default(); + state.down.idle_since_unix = Some(0); + state.up.idle_since_unix = Some(0); + + let circuits_cfg = lqos_config::TreeguardCircuitsConfig { + persist_sqm_overrides: false, + ..lqos_config::TreeguardCircuitsConfig::default() + }; + let mut circuit_change_budget_remaining = 1usize; + let mut deferred_circuit_sqm_changes = 0usize; + + let fq_codel = process_circuit_tick( + CircuitTickContext { + status: &mut status, + activity: &mut activity, + managed_device_ids: &mut managed_device_ids, + now_unix: 1_000, + now_nanos_since_boot: Some(2_000_000_000), + cpu_max_pct: Some(95), + dry_run: false, + circuit_id: "circuit-2", + circuit_entity_id: "Circuit Two (circuit-2)", + circuit_label: "Circuit Two", + devices: &devices, + sqm_batch_id: "sqm-test-3", + allowlisted: true, + cap_down: 100.0, + cap_up: 20.0, + bps: DownUpOrder { down: 0, up: 0 }, + last_rtt_seen_nanos: Some(1_900_000_000), + qoo: DownUpOrder { + down: Some(90.0), + up: Some(90.0), + }, + cpu_cfg: &lqos_config::TreeguardCpuConfig::default(), + circuits_cfg: &circuits_cfg, + qoo_cfg: &lqos_config::TreeguardQooConfig::default(), + base_sqm: DownUpOrder { + down: CircuitSqmState::Cake, + up: CircuitSqmState::Cake, + }, + circuit_change_budget_remaining: &mut circuit_change_budget_remaining, + deferred_circuit_sqm_changes: &mut deferred_circuit_sqm_changes, + }, + &mut state, + |_device_ids, _token| Ok(false), + |_device_ids| Ok(false), + |circuit_id, devices, token| { + bakery::apply_circuit_sqm_override_live_with_sender_and_snapshot( + circuit_id, devices, token, &tx, &queues, + ) + }, + ); + + assert!(fq_codel); + assert_eq!(state.down.desired, CircuitSqmState::FqCodel); + assert_eq!(state.up.desired, CircuitSqmState::FqCodel); + assert!(managed_device_ids.contains("device-2")); + assert_eq!( + status.last_action_summary.as_deref(), + Some("SQM override for circuit 'Circuit Two' -> fq_codel/fq_codel") + ); + + let command = rx.try_recv().expect("bakery command should be sent"); + let BakeryCommands::AddCircuit { + circuit_hash, + sqm_override, + .. + } = command + else { + panic!("expected AddCircuit"); + }; + assert_eq!(circuit_hash, lqos_utils::hash_to_i64("circuit-2")); + assert_eq!(sqm_override, Some("fq_codel/fq_codel".to_string())); + let last_activity = activity.back().expect("activity should be recorded"); + assert_eq!(last_activity.batch_id.as_deref(), Some("sqm-test-3")); + assert_eq!(last_activity.batch_kind.as_deref(), Some("sqm")); + } + + #[test] + fn base_circuit_sqm_state_uses_default_sqm_when_no_operator_override_exists() { + let mut config = lqos_config::Config::default(); + config.queues.default_sqm = "fq_codel".to_string(); + config.queues.fast_queues_fq_codel = Some(1000.0); + + let shaped_devices = vec![ShapedDevice { + circuit_id: "circuit-3".to_string(), + device_id: "device-3".to_string(), + ..ShapedDevice::default() + }]; + + let base = base_circuit_sqm_state(&shaped_devices, None, &config, 50.0, 20.0); + + assert_eq!(base.down, CircuitSqmState::FqCodel); + assert_eq!(base.up, CircuitSqmState::FqCodel); + } + + #[test] + fn circuit_tick_treats_bakery_live_mutation_unavailable_as_deferred() { + let devices = vec![ShapedDevice { + circuit_id: "circuit-deferred".to_string(), + circuit_name: "Circuit Deferred".to_string(), + device_id: "device-deferred".to_string(), + ipv4: vec![(Ipv4Addr::new(198, 51, 100, 30), 32)], + ..ShapedDevice::default() + }]; + let mut managed_device_ids = FxHashSet::default(); + let mut status = empty_status_snapshot(); + let mut activity: VecDeque = VecDeque::new(); + let mut state = crate::treeguard::state::CircuitState::default(); + state.down.idle_since_unix = Some(0); + state.up.idle_since_unix = Some(0); + + let circuits_cfg = lqos_config::TreeguardCircuitsConfig { + persist_sqm_overrides: false, + ..lqos_config::TreeguardCircuitsConfig::default() + }; + let mut circuit_change_budget_remaining = 1usize; + let mut deferred_circuit_sqm_changes = 0usize; + + let fq_codel = process_circuit_tick( + CircuitTickContext { + status: &mut status, + activity: &mut activity, + managed_device_ids: &mut managed_device_ids, + now_unix: 1_000, + now_nanos_since_boot: Some(2_000_000_000), + cpu_max_pct: Some(95), + dry_run: false, + circuit_id: "circuit-deferred", + circuit_entity_id: "Circuit Deferred (circuit-deferred)", + circuit_label: "Circuit Deferred", + devices: &devices, + sqm_batch_id: "sqm-test-deferred", + allowlisted: true, + cap_down: 100.0, + cap_up: 20.0, + bps: DownUpOrder { down: 0, up: 0 }, + last_rtt_seen_nanos: Some(1_900_000_000), + qoo: DownUpOrder { + down: Some(90.0), + up: Some(90.0), + }, + cpu_cfg: &lqos_config::TreeguardCpuConfig::default(), + circuits_cfg: &circuits_cfg, + qoo_cfg: &lqos_config::TreeguardQooConfig::default(), + base_sqm: DownUpOrder { + down: CircuitSqmState::Cake, + up: CircuitSqmState::Cake, + }, + circuit_change_budget_remaining: &mut circuit_change_budget_remaining, + deferred_circuit_sqm_changes: &mut deferred_circuit_sqm_changes, + }, + &mut state, + |_device_ids, _token| Ok(false), + |_device_ids| Ok(false), + |_circuit_id, _devices, _token| { + Err(TreeguardError::LiveMutationUnavailable { + details: "the shaping tree is not currently active".to_string(), + }) + }, + ); + + assert!(!fq_codel); + assert_eq!(state.down.desired, CircuitSqmState::Cake); + assert_eq!(state.up.desired, CircuitSqmState::Cake); + assert!(status.warnings.is_empty()); + assert!(managed_device_ids.contains("device-deferred")); + assert!(activity.iter().any(|entry| { + entry.action == "apply_sqm_live_deferred:fq_codel/fq_codel" + && entry.reason + == "Bakery live apply deferred: the shaping tree is not currently active" + })); + assert!(!activity.iter().any(|entry| entry.action.contains("failed"))); + } + + #[test] + fn base_fq_codel_directions_do_not_take_treeguard_circuit_switches() { + let mut state = crate::treeguard::state::CircuitState::default(); + state.down.desired = CircuitSqmState::Cake; + state.up.desired = CircuitSqmState::FqCodel; + + let transition = circuit_sqm_transition_from_decision( + &state, + DownUpOrder { + down: CircuitSqmState::FqCodel, + up: CircuitSqmState::FqCodel, + }, + decisions::CircuitSqmDecision { + down: Some(CircuitSqmState::FqCodel), + up: Some(CircuitSqmState::Cake), + }, + ); + + assert_eq!(transition.proposed_down, CircuitSqmState::FqCodel); + assert_eq!(transition.proposed_up, CircuitSqmState::FqCodel); + assert!(transition.changed_down); + assert!(!transition.changed_up); + } + + #[test] + fn treeguard_circuit_change_budget_consumes_then_stops() { + let mut remaining = 2usize; + assert!(try_consume_circuit_change_budget(&mut remaining)); + assert_eq!(remaining, 1); + assert!(try_consume_circuit_change_budget(&mut remaining)); + assert_eq!(remaining, 0); + assert!(!try_consume_circuit_change_budget(&mut remaining)); + assert_eq!(remaining, 0); + } + + #[test] + fn treeguard_only_manages_cake_based_circuit_directions() { + assert!(treeguard_manages_circuit_direction(CircuitSqmState::Cake)); + assert!(!treeguard_manages_circuit_direction( + CircuitSqmState::FqCodel + )); + } + + #[test] + fn treeguard_circuit_batch_size_spreads_large_all_circuit_sweeps() { + assert_eq!(circuit_evaluation_batch_size(0, true), 0); + assert_eq!(circuit_evaluation_batch_size(64, true), 64); + assert_eq!(circuit_evaluation_batch_size(10_000, true), 667); + assert_eq!(circuit_evaluation_batch_size(20_000, true), 1334); + assert_eq!(circuit_evaluation_batch_size(32, false), 32); + } + + #[test] + fn treeguard_collect_circuit_batch_wraps_cursor() { + let circuits = vec![ + "c1".to_string(), + "c2".to_string(), + "c3".to_string(), + "c4".to_string(), + ]; + let mut cursor = 3usize; + + let batch = collect_circuit_batch(&circuits, &mut cursor, 3); + + assert_eq!(batch, vec!["c4", "c1", "c2"]); + assert_eq!(cursor, 2); + } + + fn test_runtime_dir(name: &str) -> PathBuf { + let unique = format!( + "{}-{}-{}", + std::process::id(), + std::thread::current().name().unwrap_or("thread"), + name + ) + .replace(['/', ' '], "-"); + let dir = std::env::temp_dir().join(format!("libreqos-treeguard-{unique}")); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("temp runtime dir should be created"); + dir + } + + fn write_test_config(runtime_dir: &std::path::Path) -> PathBuf { + let config_path = runtime_dir.join("lqos.test.toml"); + let raw = include_str!("../../../lqos_config/src/etc/v15/example.toml") + .replace( + "lqos_directory = \"/opt/libreqos/src\"", + &format!("lqos_directory = \"{}\"", runtime_dir.display()), + ) + .replace("use_xdp_bridge = true", "use_xdp_bridge = false") + .replace( + "[treeguard.links]\nenabled = true", + "[treeguard.links]\nenabled = false", + ) + .replace( + "persist_sqm_overrides = true", + "persist_sqm_overrides = false", + ); + std::fs::write(&config_path, raw).expect("test config should be written"); + config_path + } + + fn treeguard_test_lock() -> &'static std::sync::Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| std::sync::Mutex::new(())) + } + + struct LiveBakeryTestContext { + _guard: std::sync::MutexGuard<'static, ()>, + old_lqos_config: Option, + old_lqos_directory: Option, + old_shaping_tree_active: bool, + } + + impl LiveBakeryTestContext { + fn new(name: &str) -> Self { + let guard = treeguard_test_lock() + .lock() + .expect("treeguard test lock should not be poisoned"); + let runtime_dir = test_runtime_dir(name); + let config_path = write_test_config(&runtime_dir); + let old_lqos_config = std::env::var_os("LQOS_CONFIG"); + let old_lqos_directory = std::env::var_os("LQOS_DIRECTORY"); + unsafe { + std::env::set_var("LQOS_CONFIG", &config_path); + std::env::set_var("LQOS_DIRECTORY", &runtime_dir); + } + lqos_config::clear_cached_config(); + let old_shaping_tree_active = lqos_bakery::set_shaping_tree_active_for_tests(true); + Self { + _guard: guard, + old_lqos_config, + old_lqos_directory, + old_shaping_tree_active, + } + } + } + + impl Drop for LiveBakeryTestContext { + fn drop(&mut self) { + match &self.old_lqos_config { + Some(value) => unsafe { std::env::set_var("LQOS_CONFIG", value) }, + None => unsafe { std::env::remove_var("LQOS_CONFIG") }, + } + match &self.old_lqos_directory { + Some(value) => unsafe { std::env::set_var("LQOS_DIRECTORY", value) }, + None => unsafe { std::env::remove_var("LQOS_DIRECTORY") }, + } + lqos_config::clear_cached_config(); + lqos_bakery::set_shaping_tree_active_for_tests(self.old_shaping_tree_active); + } + } + + fn install_test_bakery_sender() -> crossbeam_channel::Receiver { + static INIT: Once = Once::new(); + static RECEIVER: OnceLock> = OnceLock::new(); + INIT.call_once(|| { + let (tx, rx) = bounded(8); + let _ = lqos_bakery::BAKERY_SENDER.set(tx); + let _ = RECEIVER.set(rx); + }); + RECEIVER + .get() + .expect("test bakery receiver should be installed") + .clone() + } + + #[test] + fn run_tick_end_to_end_switches_circuit_and_emits_bakery_update() { + let _live_bakery = LiveBakeryTestContext::new("run-tick"); + + let devices = vec![ShapedDevice { + circuit_id: "circuit-3".to_string(), + circuit_name: "Circuit Three".to_string(), + device_id: "device-3".to_string(), + device_name: "Device Three".to_string(), + parent_node: "Node Three".to_string(), + ipv4: vec![(Ipv4Addr::new(203, 0, 113, 30), 32)], + download_min_mbps: 25.0, + upload_min_mbps: 5.0, + download_max_mbps: 100.0, + upload_max_mbps: 20.0, + circuit_hash: lqos_utils::hash_to_i64("circuit-3"), + device_hash: lqos_utils::hash_to_i64("device-3"), + parent_hash: lqos_utils::hash_to_i64("Node Three"), + ..ShapedDevice::default() + }]; + let mut shaped = ConfigShapedDevices::default(); + shaped.replace_with_new_data(devices.clone()); + let old_shaped = SHAPED_DEVICES.load_full(); + SHAPED_DEVICES.store(Arc::new(shaped)); + + let old_network = NETWORK_JSON.read().nodes.clone(); + NETWORK_JSON.write().nodes = Vec::new(); + + let old_queue_structure = QUEUE_STRUCTURE.load_full(); + QUEUE_STRUCTURE.store(Arc::new(QueueStructure { + maybe_queues: Some(vec![QueueNode { + circuit_id: Some("circuit-3".to_string()), + class_minor: 0x33af, + class_major: 0x0007, + up_class_major: 0x0047, + parent_class_id: TcHandle::from_string("7:20").expect("valid down parent"), + up_parent_class_id: TcHandle::from_string("47:20").expect("valid up parent"), + download_bandwidth_mbps_min: 25, + upload_bandwidth_mbps_min: 5, + download_bandwidth_mbps: 100, + upload_bandwidth_mbps: 20, + ..QueueNode::default() + }]), + })); + + let now_nanos = lqos_utils::unix_time::time_since_boot() + .map(std::time::Duration::from) + .map(|duration| duration.as_nanos() as u64) + .expect("time since boot should be available"); + let old_rtt = CIRCUIT_RTT_BUFFERS.load_full(); + let mut rtt = RttBuffer::default(); + rtt.last_seen = now_nanos.saturating_sub(100_000_000); + let mut rtt_map = FxHashMap::default(); + rtt_map.insert(lqos_utils::hash_to_i64("circuit-3"), rtt); + CIRCUIT_RTT_BUFFERS.store(Arc::new(rtt_map)); + + let rx = install_test_bakery_sender(); + while rx.try_recv().is_ok() {} + + let (system_tx, system_rx) = bounded::>(1); + let responder = std::thread::spawn(move || { + let reply = system_rx + .recv() + .expect("system stats request should arrive"); + let _ = reply.send(SystemStats { + cpu_usage: vec![95], + ram_used: 0, + total_ram: 0, + }); + }); + + let mut status = empty_status_snapshot(); + let mut activity: VecDeque = VecDeque::new(); + let mut runtime_state = TreeguardRuntimeState::default(); + runtime_state.circuit_states.insert( + "circuit-3".to_string(), + crate::treeguard::state::CircuitState { + down: crate::treeguard::state::CircuitDirectionState { + idle_since_unix: Some(0), + ..Default::default() + }, + up: crate::treeguard::state::CircuitDirectionState { + idle_since_unix: Some(0), + ..Default::default() + }, + }, + ); + let mut tick_seconds = 1; + run_tick( + &mut status, + &mut activity, + &system_tx, + &mut tick_seconds, + &mut runtime_state, + ); + responder + .join() + .expect("system stats responder should join"); + + assert!(status.enabled); + assert!(!status.dry_run); + assert_eq!(status.managed_circuits, 1); + assert_eq!(status.fq_codel_circuits, 1); + assert_eq!(status.cake_circuits, 0); + assert_eq!(status.mixed_sqm_circuits, 0); + assert_eq!( + status.last_action_summary.as_deref(), + Some("SQM override for circuit 'Circuit Three' -> fq_codel/fq_codel") + ); + let circuit_state = runtime_state + .circuit_states + .get("circuit-3") + .expect("circuit state should exist"); + assert_eq!(circuit_state.down.desired, CircuitSqmState::FqCodel); + assert_eq!(circuit_state.up.desired, CircuitSqmState::FqCodel); + assert!( + activity + .iter() + .any(|entry| entry.action == "set_sqm_live:fq_codel/fq_codel") + ); + + let command = rx + .try_recv() + .expect("run_tick should send a bakery command"); + let BakeryCommands::AddCircuit { + circuit_hash, + sqm_override, + .. + } = command + else { + panic!("expected AddCircuit"); + }; + assert_eq!(circuit_hash, lqos_utils::hash_to_i64("circuit-3")); + assert_eq!(sqm_override, Some("fq_codel/fq_codel".to_string())); + + SHAPED_DEVICES.store(old_shaped); + NETWORK_JSON.write().nodes = old_network; + QUEUE_STRUCTURE.store(old_queue_structure); + CIRCUIT_RTT_BUFFERS.store(old_rtt); + } + + #[test] + fn apply_link_virtualization_decision_emits_requested_activity_when_bakery_accepts_submit() { + let _live_bakery = LiveBakeryTestContext::new("link-virtualize"); + let rx = install_test_bakery_sender(); + while rx.try_recv().is_ok() {} + + let mut status = empty_status_snapshot(); + let mut activity: VecDeque = VecDeque::new(); + let mut pending_link_operations = FxHashMap::default(); + let mut link_virtualization_backoff_until_unix = FxHashMap::default(); + let mut state = LinkState::default(); + + apply_link_virtualization_decision( + &mut status, + &mut activity, + 1_000, + "Node Requested", + LinkVirtualState::Virtual, + false, + "High utilization".to_string(), + false, + &mut state, + &mut pending_link_operations, + &mut link_virtualization_backoff_until_unix, + ); + + assert_eq!(state.desired, LinkVirtualState::Virtual); + let pending = pending_link_operations + .get("Node Requested") + .expect("pending operation should be tracked"); + assert_eq!(pending.target, LinkVirtualState::Virtual); + assert_eq!(pending.reason, "High utilization"); + assert_eq!( + status.last_action_summary.as_deref(), + Some("Queued virtualization for node 'Node Requested'") + ); + + let last_activity = activity + .back() + .expect("requested activity should be recorded"); + assert_eq!(last_activity.action, "virtualize_requested"); + assert!(!last_activity.persisted); + assert!(last_activity.reason.contains("High utilization")); + assert!(last_activity.reason.contains("Queued in Bakery")); + + let command = rx.try_recv().expect("bakery command should be sent"); + let BakeryCommands::TreeGuardSetNodeVirtual { + site_hash, + virtualized, + .. + } = command + else { + panic!("expected TreeGuardSetNodeVirtual"); + }; + assert_eq!(site_hash, lqos_utils::hash_to_i64("Node Requested")); + assert!(virtualized); + } + + #[test] + fn apply_link_virtualization_decision_rejects_structurally_ineligible_top_level_node() { + let rx = install_test_bakery_sender(); + while rx.try_recv().is_ok() {} + + let mut status = empty_status_snapshot(); + let mut activity: VecDeque = VecDeque::new(); + let mut pending_link_operations = FxHashMap::default(); + let mut link_virtualization_backoff_until_unix = FxHashMap::default(); + let mut state = LinkState { + topology_fingerprint: LinkTopologyFingerprint { + direct_child_sites: 1, + direct_circuits: 0, + }, + ..Default::default() + }; + + apply_link_virtualization_decision( + &mut status, + &mut activity, + 1_000, + "Node Rejected", + LinkVirtualState::Virtual, + true, + "Top-level safe".to_string(), + false, + &mut state, + &mut pending_link_operations, + &mut link_virtualization_backoff_until_unix, + ); + + assert_eq!(state.desired, LinkVirtualState::Physical); + assert!(pending_link_operations.is_empty()); + assert!(link_virtualization_backoff_until_unix.is_empty()); + assert_eq!( + state.structural_ineligible, + Some(LinkStructuralIneligibleState { + reason: + BakeryRuntimeNodeOperationFailureReason::StructuralIneligibleSinglePromotableChild, + topology_fingerprint: LinkTopologyFingerprint { + direct_child_sites: 1, + direct_circuits: 0, + }, + }) + ); + assert_eq!( + status.last_action_summary.as_deref(), + Some("Rejected virtualization for node 'Node Rejected'") + ); + assert!(status.warnings.iter().any(|warning| { + warning.contains("Node Rejected") + && warning.contains("structurally ineligible") + && warning.contains("single promotable child") + })); + + let last_activity = activity + .back() + .expect("rejected activity should be recorded"); + assert_eq!(last_activity.action, "virtualize_rejected"); + assert!(!last_activity.persisted); + assert!(last_activity.reason.contains("Top-level safe")); + assert!(last_activity.reason.contains("before submitting to Bakery")); + + assert!(rx.try_recv().is_err()); } } diff --git a/src/rust/lqosd/src/treeguard/bakery.rs b/src/rust/lqosd/src/treeguard/bakery.rs index d64a755d..de4c9933 100644 --- a/src/rust/lqosd/src/treeguard/bakery.rs +++ b/src/rust/lqosd/src/treeguard/bakery.rs @@ -1,11 +1,14 @@ //! Bakery live-update helpers for TreeGuard. //! -//! This module implements live SQM switching via Bakery commands. +//! This module implements live SQM switching and runtime node virtualization via Bakery commands. use crate::treeguard::TreeguardError; -use lqos_bakery::BakeryCommands; +use crossbeam_channel::Sender; +use lqos_bakery::{ + BakeryCommands, BakeryRuntimeNodeBranchSnapshot, BakeryRuntimeNodeOperationSnapshot, +}; use lqos_config::ShapedDevice; -use lqos_queue_tracker::QUEUE_STRUCTURE; +use lqos_queue_tracker::{QUEUE_STRUCTURE, QueueNode}; use lqos_utils::hash_to_i64; /// Applies a per-circuit SQM override token live via Bakery. @@ -17,6 +20,9 @@ pub(crate) fn apply_circuit_sqm_override_live( devices: &[ShapedDevice], sqm_override: &str, ) -> Result<(), TreeguardError> { + if let Some(details) = lqos_bakery::bakery_live_tree_mutation_blocker() { + return Err(TreeguardError::LiveMutationUnavailable { details }); + } let Some(sender) = lqos_bakery::BAKERY_SENDER.get() else { return Err(TreeguardError::BakeryNotReady); }; @@ -28,17 +34,38 @@ pub(crate) fn apply_circuit_sqm_override_live( }); }; + apply_circuit_sqm_override_live_with_sender_and_snapshot( + circuit_id, + devices, + sqm_override, + sender, + queues, + ) +} + +#[doc(hidden)] +pub(crate) fn apply_circuit_sqm_override_live_with_sender_and_snapshot( + circuit_id: &str, + devices: &[ShapedDevice], + sqm_override: &str, + sender: &Sender, + queues: &[QueueNode], +) -> Result<(), TreeguardError> { + let node = find_circuit_queue_node(queues, circuit_id)?; + send_live_sqm_override(node, devices, sqm_override, sender) +} + +fn find_circuit_queue_node<'a>( + queues: &'a [QueueNode], + circuit_id: &str, +) -> Result<&'a QueueNode, TreeguardError> { // Find the circuit node in the queue structure. let mut stack = Vec::new(); - for q in queues.iter() { - stack.push(q); - } + stack.extend(queues.iter()); - let mut found = None; while let Some(node) = stack.pop() { if node.circuit_id.as_deref() == Some(circuit_id) && node.device_id.is_none() { - found = Some(node); - break; + return Ok(node); } for c in node.children.iter() { @@ -52,12 +79,23 @@ pub(crate) fn apply_circuit_sqm_override_live( } } - let Some(node) = found else { - return Err(TreeguardError::CircuitNotFound { - circuit_id: circuit_id.to_string(), - }); - }; + Err(TreeguardError::CircuitNotFound { + circuit_id: circuit_id.to_string(), + }) +} +fn send_live_sqm_override( + node: &QueueNode, + devices: &[ShapedDevice], + sqm_override: &str, + sender: &Sender, +) -> Result<(), TreeguardError> { + let circuit_id = node + .circuit_id + .as_deref() + .ok_or_else(|| TreeguardError::CircuitNotFound { + circuit_id: "".to_string(), + })?; let class_minor = u16::try_from(node.class_minor).map_err(|_| TreeguardError::InvalidClassId { details: format!("class_minor too large: {}", node.class_minor), @@ -77,6 +115,11 @@ pub(crate) fn apply_circuit_sqm_override_live( sender .send(BakeryCommands::AddCircuit { circuit_hash, + circuit_name: node + .circuit_name + .clone() + .or_else(|| Some(circuit_id.to_string())), + site_name: node.parent_node.clone(), parent_class_id: node.parent_class_id, up_parent_class_id: node.up_parent_class_id, class_minor, @@ -86,6 +129,8 @@ pub(crate) fn apply_circuit_sqm_override_live( upload_bandwidth_max: node.upload_bandwidth_mbps as f32, class_major, up_class_major, + down_qdisc_handle: None, + up_qdisc_handle: None, ip_addresses, sqm_override: Some(sqm_override.to_string()), }) @@ -96,6 +141,48 @@ pub(crate) fn apply_circuit_sqm_override_live( Ok(()) } +/// Submits a Bakery runtime-virtualization or restore intent for a single node without a full reload. +/// +/// This function has side effects: it sends a command to the Bakery thread. +pub(crate) fn submit_node_virtualization_live( + node_name: &str, + virtualized: bool, +) -> Result<(), TreeguardError> { + if let Some(details) = lqos_bakery::bakery_live_tree_mutation_blocker() { + return Err(TreeguardError::LiveMutationUnavailable { details }); + } + let Some(sender) = lqos_bakery::BAKERY_SENDER.get() else { + return Err(TreeguardError::BakeryNotReady); + }; + + let site_hash = hash_to_i64(node_name); + sender + .send(BakeryCommands::TreeGuardSetNodeVirtual { + site_hash, + virtualized, + reply: None, + }) + .map_err(|e| TreeguardError::BakerySend { + details: e.to_string(), + }) +} + +/// Returns the latest Bakery runtime-node operation snapshot for a node, if Bakery has processed one. +pub(crate) fn node_virtualization_operation_status( + node_name: &str, +) -> Option { + let site_hash = hash_to_i64(node_name); + lqos_bakery::bakery_runtime_node_operation_snapshot(site_hash) +} + +/// Returns the latest Bakery runtime branch-state snapshot for a node, if Bakery retains one. +pub(crate) fn node_virtualization_branch_state( + node_name: &str, +) -> Option { + let site_hash = hash_to_i64(node_name); + lqos_bakery::bakery_runtime_node_branch_snapshot(site_hash) +} + /// Builds a comma-separated IP list string from a circuit's shaped devices. /// /// This function is pure: it has no side effects. @@ -113,3 +200,83 @@ fn ip_list(devices: &[ShapedDevice]) -> String { ips.dedup(); ips.join(",") } + +#[cfg(test)] +mod tests { + use super::apply_circuit_sqm_override_live_with_sender_and_snapshot; + use crossbeam_channel::bounded; + use lqos_bakery::BakeryCommands; + use lqos_bus::TcHandle; + use lqos_config::ShapedDevice; + use lqos_queue_tracker::QueueNode; + use std::net::{Ipv4Addr, Ipv6Addr}; + + #[test] + fn live_override_builds_bakery_update_from_snapshot() { + let (tx, rx) = bounded(1); + let queues = vec![QueueNode { + circuit_id: Some("circuit-1".to_string()), + class_minor: 0x14af, + class_major: 0x0003, + up_class_major: 0x0043, + parent_class_id: TcHandle::from_string("3:20").expect("valid down parent"), + up_parent_class_id: TcHandle::from_string("43:20").expect("valid up parent"), + download_bandwidth_mbps_min: 50, + upload_bandwidth_mbps_min: 10, + download_bandwidth_mbps: 200, + upload_bandwidth_mbps: 50, + ..QueueNode::default() + }]; + let devices = vec![ShapedDevice { + circuit_id: "circuit-1".to_string(), + device_id: "device-1".to_string(), + ipv4: vec![(Ipv4Addr::new(192, 0, 2, 10), 32)], + ipv6: vec![(Ipv6Addr::LOCALHOST, 128)], + ..ShapedDevice::default() + }]; + + apply_circuit_sqm_override_live_with_sender_and_snapshot( + "circuit-1", + &devices, + "cake/fq_codel", + &tx, + &queues, + ) + .expect("live override should send bakery update"); + + let command = rx.try_recv().expect("bakery command should be sent"); + let BakeryCommands::AddCircuit { + circuit_hash, + parent_class_id, + up_parent_class_id, + class_minor, + class_major, + up_class_major, + ip_addresses, + sqm_override, + down_qdisc_handle, + up_qdisc_handle, + .. + } = command + else { + panic!("expected AddCircuit"); + }; + + assert_eq!(circuit_hash, lqos_utils::hash_to_i64("circuit-1")); + assert_eq!( + parent_class_id, + TcHandle::from_string("3:20").expect("valid down parent") + ); + assert_eq!( + up_parent_class_id, + TcHandle::from_string("43:20").expect("valid up parent") + ); + assert_eq!(class_minor, 0x14af); + assert_eq!(class_major, 0x0003); + assert_eq!(up_class_major, 0x0043); + assert_eq!(ip_addresses, "192.0.2.10/32,::1/128"); + assert_eq!(sqm_override, Some("cake/fq_codel".to_string())); + assert_eq!(down_qdisc_handle, None); + assert_eq!(up_qdisc_handle, None); + } +} diff --git a/src/rust/lqosd/src/treeguard/decisions.rs b/src/rust/lqosd/src/treeguard/decisions.rs index b9ea9833..98103d91 100644 --- a/src/rust/lqosd/src/treeguard/decisions.rs +++ b/src/rust/lqosd/src/treeguard/decisions.rs @@ -44,6 +44,23 @@ pub struct LinkVirtualizationInput<'a> { pub state: &'a LinkState, } +/// Input to top-level link virtualization decisions. +#[derive(Clone, Copy, Debug)] +pub struct TopLevelLinkVirtualizationInput<'a> { + pub now_unix: u64, + pub cpu_max_pct: Option, + pub cpu_cfg: &'a TreeguardCpuConfig, + pub links_cfg: &'a TreeguardLinksConfig, + pub qoo_cfg: &'a TreeguardQooConfig, + pub rtt_missing: bool, + pub qoo: DownUpOrder>, + pub util_ewma_pct: DownUpOrder, + pub safe_util_pct: f64, + pub sustained_safe: bool, + pub emergency_util_sustained: bool, + pub state: &'a LinkState, +} + /// Input to per-circuit SQM decisions. #[derive(Clone, Copy, Debug)] pub struct CircuitSqmInput<'a> { @@ -133,25 +150,24 @@ pub fn decide_link_virtualization(input: LinkVirtualizationInput<'_>) -> LinkVir return LinkVirtualDecision::NoChange; } - if in_dwell_window( - now_unix, - state.last_change_unix, - links_cfg.min_state_dwell_minutes, - ) { - return LinkVirtualDecision::NoChange; - } - - if rate_limited( - state.recent_changes_unix.len(), - links_cfg.max_link_changes_per_hour, - ) { - return LinkVirtualDecision::NoChange; - } - let qoo_bad = qoo_below_threshold(qoo_cfg, qoo); match state.desired { LinkVirtualState::Physical => { + if in_dwell_window( + now_unix, + state.last_change_unix, + links_cfg.min_state_dwell_minutes, + ) { + return LinkVirtualDecision::NoChange; + } + + if rate_limited( + state.recent_changes_unix.len(), + links_cfg.max_link_changes_per_hour, + ) { + return LinkVirtualDecision::NoChange; + } if !cpu_allows_saving(cpu_cfg, cpu_max_pct) { return LinkVirtualDecision::NoChange; } @@ -173,6 +189,60 @@ pub fn decide_link_virtualization(input: LinkVirtualizationInput<'_>) -> LinkVir } } +/// Decide whether to virtualize/unvirtualize a managed top-level node. +/// +/// This function is pure: it has no side effects. +pub fn decide_top_level_link_virtualization( + input: TopLevelLinkVirtualizationInput<'_>, +) -> LinkVirtualDecision { + let TopLevelLinkVirtualizationInput { + now_unix, + cpu_max_pct, + cpu_cfg, + links_cfg, + qoo_cfg: _qoo_cfg, + rtt_missing: _rtt_missing, + qoo: _qoo, + util_ewma_pct, + safe_util_pct, + sustained_safe, + emergency_util_sustained, + state, + } = input; + + let util_high = util_ewma_pct.down >= safe_util_pct || util_ewma_pct.up >= safe_util_pct; + + match state.desired { + LinkVirtualState::Physical => { + if in_dwell_window( + now_unix, + state.last_change_unix, + links_cfg.min_state_dwell_minutes, + ) { + return LinkVirtualDecision::NoChange; + } + if rate_limited( + state.recent_changes_unix.len(), + links_cfg.max_link_changes_per_hour, + ) { + return LinkVirtualDecision::NoChange; + } + if cpu_allows_saving(cpu_cfg, cpu_max_pct) && sustained_safe { + LinkVirtualDecision::Set(LinkVirtualState::Virtual) + } else { + LinkVirtualDecision::NoChange + } + } + LinkVirtualState::Virtual => { + if emergency_util_sustained || util_high { + LinkVirtualDecision::Set(LinkVirtualState::Physical) + } else { + LinkVirtualDecision::NoChange + } + } + } +} + /// Decide whether to switch a managed circuit's SQM profile per direction. /// /// This function is pure: it has no side effects. @@ -184,7 +254,7 @@ pub fn decide_circuit_sqm(input: CircuitSqmInput<'_>) -> CircuitSqmDecision { cpu_cfg, circuits_cfg, qoo_cfg, - rtt_missing, + rtt_missing: _rtt_missing, qoo, state, } = input; @@ -242,11 +312,7 @@ pub fn decide_circuit_sqm(input: CircuitSqmInput<'_>) -> CircuitSqmDecision { } } CircuitSqmState::FqCodel => { - if util_high - || qoo_bad - || cpu_calls_for_revert(cpu_cfg, cpu_max_pct) - || (rtt_missing && !sustained_idle) - { + if util_high || qoo_bad || cpu_calls_for_revert(cpu_cfg, cpu_max_pct) { Some(CircuitSqmState::Cake) } else { None @@ -300,11 +366,7 @@ pub fn decide_circuit_sqm(input: CircuitSqmInput<'_>) -> CircuitSqmDecision { } } CircuitSqmState::FqCodel => { - if util_high - || qoo_bad - || cpu_calls_for_revert(cpu_cfg, cpu_max_pct) - || (rtt_missing && !sustained_idle) - { + if util_high || qoo_bad || cpu_calls_for_revert(cpu_cfg, cpu_max_pct) { Some(CircuitSqmState::Cake) } else { None @@ -436,8 +498,11 @@ mod tests { } #[test] - fn link_does_not_virtualize_when_cpu_low() { - let cpu = TreeguardCpuConfig::default(); + fn link_does_not_virtualize_when_cpu_low_in_cpu_aware_mode() { + let cpu = TreeguardCpuConfig { + mode: TreeguardCpuMode::CpuAware, + ..TreeguardCpuConfig::default() + }; let links = TreeguardLinksConfig::default(); let qoo_cfg = TreeguardQooConfig::default(); let state = LinkState::default(); @@ -460,6 +525,203 @@ mod tests { assert_eq!(decision, LinkVirtualDecision::NoChange); } + #[test] + fn link_virtualizes_when_cpu_low_in_traffic_rtt_only_mode() { + let cpu = TreeguardCpuConfig { + mode: TreeguardCpuMode::TrafficRttOnly, + ..TreeguardCpuConfig::default() + }; + let links = TreeguardLinksConfig::default(); + let qoo_cfg = TreeguardQooConfig::default(); + let state = LinkState::default(); + let decision = decide_link_virtualization(LinkVirtualizationInput { + now_unix: 1000, + allowlisted: true, + cpu_max_pct: Some(10), + cpu_cfg: &cpu, + links_cfg: &links, + qoo_cfg: &qoo_cfg, + rtt_missing: false, + qoo: DownUpOrder { + down: Some(100.0), + up: Some(100.0), + }, + util_ewma_pct: DownUpOrder { down: 1.0, up: 1.0 }, + sustained_idle: true, + state: &state, + }); + assert_eq!( + decision, + LinkVirtualDecision::Set(LinkVirtualState::Virtual) + ); + } + + #[test] + fn top_level_link_does_not_virtualize_when_cpu_low_in_cpu_aware_mode() { + let state = LinkState { + desired: LinkVirtualState::Physical, + ..Default::default() + }; + let decision = decide_top_level_link_virtualization(TopLevelLinkVirtualizationInput { + now_unix: 1000, + cpu_max_pct: Some(10), + cpu_cfg: &TreeguardCpuConfig { + mode: TreeguardCpuMode::CpuAware, + ..TreeguardCpuConfig::default() + }, + links_cfg: &TreeguardLinksConfig::default(), + qoo_cfg: &TreeguardQooConfig::default(), + rtt_missing: false, + qoo: DownUpOrder { + down: Some(100.0), + up: Some(100.0), + }, + util_ewma_pct: DownUpOrder { + down: 10.0, + up: 12.0, + }, + safe_util_pct: 85.0, + sustained_safe: true, + emergency_util_sustained: false, + state: &state, + }); + assert_eq!(decision, LinkVirtualDecision::NoChange); + } + + #[test] + fn top_level_link_virtualizes_when_cpu_high_and_safe() { + let state = LinkState { + desired: LinkVirtualState::Physical, + ..Default::default() + }; + let decision = decide_top_level_link_virtualization(TopLevelLinkVirtualizationInput { + now_unix: 1000, + cpu_max_pct: Some(90), + cpu_cfg: &TreeguardCpuConfig { + mode: TreeguardCpuMode::CpuAware, + ..TreeguardCpuConfig::default() + }, + links_cfg: &TreeguardLinksConfig::default(), + qoo_cfg: &TreeguardQooConfig::default(), + rtt_missing: false, + qoo: DownUpOrder { + down: Some(100.0), + up: Some(100.0), + }, + util_ewma_pct: DownUpOrder { + down: 10.0, + up: 12.0, + }, + safe_util_pct: 85.0, + sustained_safe: true, + emergency_util_sustained: false, + state: &state, + }); + assert_eq!( + decision, + LinkVirtualDecision::Set(LinkVirtualState::Virtual) + ); + } + + #[test] + fn top_level_link_restores_on_util_spike_even_when_cpu_low() { + let state = LinkState { + desired: LinkVirtualState::Virtual, + ..Default::default() + }; + let decision = decide_top_level_link_virtualization(TopLevelLinkVirtualizationInput { + now_unix: 1000, + cpu_max_pct: Some(10), + cpu_cfg: &TreeguardCpuConfig { + mode: TreeguardCpuMode::CpuAware, + ..TreeguardCpuConfig::default() + }, + links_cfg: &TreeguardLinksConfig::default(), + qoo_cfg: &TreeguardQooConfig::default(), + rtt_missing: false, + qoo: DownUpOrder { + down: Some(100.0), + up: Some(100.0), + }, + util_ewma_pct: DownUpOrder { + down: 90.0, + up: 5.0, + }, + safe_util_pct: 85.0, + sustained_safe: false, + emergency_util_sustained: false, + state: &state, + }); + assert_eq!( + decision, + LinkVirtualDecision::Set(LinkVirtualState::Physical) + ); + } + + #[test] + fn top_level_link_stays_virtual_when_only_qoo_is_low() { + let state = LinkState { + desired: LinkVirtualState::Virtual, + ..Default::default() + }; + let decision = decide_top_level_link_virtualization(TopLevelLinkVirtualizationInput { + now_unix: 1000, + cpu_max_pct: Some(10), + cpu_cfg: &TreeguardCpuConfig { + mode: TreeguardCpuMode::CpuAware, + ..TreeguardCpuConfig::default() + }, + links_cfg: &TreeguardLinksConfig::default(), + qoo_cfg: &TreeguardQooConfig::default(), + rtt_missing: false, + qoo: DownUpOrder { + down: Some(65.0), + up: Some(100.0), + }, + util_ewma_pct: DownUpOrder { + down: 10.0, + up: 12.0, + }, + safe_util_pct: 85.0, + sustained_safe: false, + emergency_util_sustained: false, + state: &state, + }); + assert_eq!(decision, LinkVirtualDecision::NoChange); + } + + #[test] + fn top_level_link_stays_virtual_when_only_rtt_is_missing() { + let state = LinkState { + desired: LinkVirtualState::Virtual, + ..Default::default() + }; + let decision = decide_top_level_link_virtualization(TopLevelLinkVirtualizationInput { + now_unix: 1000, + cpu_max_pct: Some(10), + cpu_cfg: &TreeguardCpuConfig { + mode: TreeguardCpuMode::CpuAware, + ..TreeguardCpuConfig::default() + }, + links_cfg: &TreeguardLinksConfig::default(), + qoo_cfg: &TreeguardQooConfig::default(), + rtt_missing: true, + qoo: DownUpOrder { + down: Some(100.0), + up: Some(100.0), + }, + util_ewma_pct: DownUpOrder { + down: 10.0, + up: 12.0, + }, + safe_util_pct: 85.0, + sustained_safe: false, + emergency_util_sustained: false, + state: &state, + }); + assert_eq!(decision, LinkVirtualDecision::NoChange); + } + #[test] fn link_unvirtualizes_on_util_spike() { let cpu = TreeguardCpuConfig::default(); @@ -694,8 +956,11 @@ mod tests { } #[test] - fn circuit_reverts_when_cpu_low() { - let cpu = TreeguardCpuConfig::default(); + fn circuit_reverts_when_cpu_low_in_cpu_aware_mode() { + let cpu = TreeguardCpuConfig { + mode: TreeguardCpuMode::CpuAware, + ..TreeguardCpuConfig::default() + }; let circuits = TreeguardCircuitsConfig::default(); let qoo_cfg = TreeguardQooConfig::default(); let state = CircuitState { @@ -727,6 +992,43 @@ mod tests { assert_eq!(decision.up, Some(CircuitSqmState::Cake)); } + #[test] + fn circuit_does_not_revert_when_cpu_low_in_traffic_rtt_only_mode() { + let cpu = TreeguardCpuConfig { + mode: TreeguardCpuMode::TrafficRttOnly, + ..TreeguardCpuConfig::default() + }; + let circuits = TreeguardCircuitsConfig::default(); + let qoo_cfg = TreeguardQooConfig::default(); + let state = CircuitState { + down: CircuitDirectionState { + desired: CircuitSqmState::FqCodel, + ..Default::default() + }, + up: CircuitDirectionState { + desired: CircuitSqmState::FqCodel, + ..Default::default() + }, + }; + + let decision = decide_circuit_sqm(CircuitSqmInput { + now_unix: 1000, + allowlisted: true, + cpu_max_pct: Some(10), + cpu_cfg: &cpu, + circuits_cfg: &circuits, + qoo_cfg: &qoo_cfg, + rtt_missing: false, + qoo: DownUpOrder { + down: Some(90.0), + up: Some(90.0), + }, + state: &state, + }); + assert_eq!(decision.down, None); + assert_eq!(decision.up, None); + } + #[test] fn circuit_missing_rtt_does_not_block_downgrade_when_idle() { let cpu = TreeguardCpuConfig::default(); diff --git a/src/rust/lqosd/src/treeguard/errors.rs b/src/rust/lqosd/src/treeguard/errors.rs index e4f2bb81..209ab4e7 100644 --- a/src/rust/lqosd/src/treeguard/errors.rs +++ b/src/rust/lqosd/src/treeguard/errors.rs @@ -25,6 +25,10 @@ pub enum TreeguardError { #[error("failed to send command to bakery: {details}")] BakerySend { details: String }, + /// Bakery cannot apply live mutations because the shaping tree is unavailable. + #[error("bakery live mutation unavailable: {details}")] + LiveMutationUnavailable { details: String }, + /// The queue structure snapshot was unavailable. #[error("queue structure snapshot unavailable: {details}")] QueueStructureUnavailable { details: String }, diff --git a/src/rust/lqosd/src/treeguard/mod.rs b/src/rust/lqosd/src/treeguard/mod.rs index 41868a0f..2df709de 100644 --- a/src/rust/lqosd/src/treeguard/mod.rs +++ b/src/rust/lqosd/src/treeguard/mod.rs @@ -10,7 +10,6 @@ pub(crate) mod bakery; pub(crate) mod decisions; pub(crate) mod errors; pub(crate) mod overrides; -pub(crate) mod reload; pub(crate) mod state; pub(crate) mod status; diff --git a/src/rust/lqosd/src/treeguard/overrides.rs b/src/rust/lqosd/src/treeguard/overrides.rs index 8ccec28e..e7665497 100644 --- a/src/rust/lqosd/src/treeguard/overrides.rs +++ b/src/rust/lqosd/src/treeguard/overrides.rs @@ -3,51 +3,7 @@ //! This module writes TreeGuard-owned changes to `lqos_overrides.treeguard.json`. use crate::treeguard::TreeguardError; -use lqos_overrides::{NetworkAdjustment, OverrideFile, OverrideLayer, OverrideStore}; - -/// Returns the current virtual override value for `node_name`, if present. -/// -/// This function is pure: it does not perform I/O and has no side effects. -fn current_node_virtual_override(overrides: &OverrideFile, node_name: &str) -> Option { - overrides - .network_adjustments() - .iter() - .find_map(|adj| match adj { - NetworkAdjustment::SetNodeVirtual { - node_name: n, - virtual_node, - } if n == node_name => Some(*virtual_node), - _ => None, - }) -} - -/// Sets (adds or replaces) a node-virtualization override for a `network.json` node. -/// -/// This function is not pure: it reads and writes `lqos_overrides.treeguard.json`. -/// -/// Returns `Ok(true)` if the file was changed, `Ok(false)` if no change was needed. -pub(crate) fn set_node_virtual( - node_name: &str, - virtual_node: bool, -) -> Result { - let mut overrides = OverrideStore::load_layer(OverrideLayer::Treeguard).map_err(|e| { - TreeguardError::OverridesLoad { - details: e.to_string(), - } - })?; - - if current_node_virtual_override(&overrides, node_name) == Some(virtual_node) { - return Ok(false); - } - - overrides.set_network_node_virtual(node_name.to_string(), virtual_node); - OverrideStore::save_layer(OverrideLayer::Treeguard, &overrides).map_err(|e| { - TreeguardError::OverridesSave { - details: e.to_string(), - } - })?; - Ok(true) -} +use lqos_overrides::{OverrideLayer, OverrideStore}; /// Removes any node-virtualization overrides for a `network.json` node. /// diff --git a/src/rust/lqosd/src/treeguard/reload.rs b/src/rust/lqosd/src/treeguard/reload.rs deleted file mode 100644 index fe06544a..00000000 --- a/src/rust/lqosd/src/treeguard/reload.rs +++ /dev/null @@ -1,219 +0,0 @@ -//! Topology reload helpers for TreeGuard. -//! -//! Link virtualization changes require a scheduler reload to take effect. This module provides a -//! simple cooldown + exponential backoff controller to avoid flapping reloads. - -use crate::reload_lock::{ReloadExecOutcome, try_reload_libreqos_locked}; - -const MIN_COOLDOWN_SECONDS: u64 = 60; -const MAX_BACKOFF_SECONDS: u64 = 2 * 60 * 60; // 2 hours - -/// Reload request priority. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(crate) enum ReloadPriority { - /// Normal reload requests obey cooldown/backoff. - #[default] - Normal, - /// Urgent reload requests bypass cooldown (but still respect backoff). - Urgent, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum SkipKind { - Backoff, - Cooldown, - Busy, -} - -/// Result of polling the reload controller. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct ReloadAttempt { - pub(crate) outcome: ReloadOutcome, - pub(crate) request_reason: Option, -} - -/// Result of a reload attempt. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum ReloadOutcome { - /// Reload succeeded. - Success { - /// Output from the reload command. - message: String, - }, - /// Reload was skipped because cooldown/backoff is active. - Skipped { - /// Human-readable reason for skipping. - reason: String, - /// Next UNIX timestamp when a reload may be attempted. - next_allowed_unix: Option, - }, - /// Reload failed (and backoff has been applied). - Failed { - /// Human-readable error. - error: String, - /// Next UNIX timestamp when a reload may be attempted. - next_allowed_unix: Option, - }, -} - -/// Cooldown/backoff controller for topology reload requests. -#[derive(Clone, Debug, Default)] -pub(crate) struct ReloadController { - last_attempt_unix: Option, - consecutive_failures: u32, - backoff_until_unix: Option, - pending: bool, - pending_priority: ReloadPriority, - pending_reason: Option, - last_reported_skip: Option<(SkipKind, Option)>, -} - -impl ReloadController { - /// Enqueues a reload request with a given priority and human-readable reason. - /// - /// This function is not pure: it mutates internal pending state. - pub(crate) fn request_reload(&mut self, priority: ReloadPriority, reason: String) { - if !self.pending { - self.pending = true; - self.pending_priority = priority; - self.pending_reason = Some(reason); - } else { - if priority == ReloadPriority::Urgent { - self.pending_priority = ReloadPriority::Urgent; - } - match self.pending_reason.as_deref() { - None => self.pending_reason = Some(reason), - Some(existing) => { - if existing != reason && existing != "Multiple node topology changes" { - self.pending_reason = Some("Multiple node topology changes".to_string()); - } - } - } - } - // Ensure we emit at least one skip outcome for a newly requested reload. - self.last_reported_skip = None; - } - - /// Polls for a pending reload request and attempts it if allowed. - /// - /// Returns `Some(...)` when an outcome should be surfaced to the activity log; repeated - /// skip outcomes are suppressed to avoid flooding the activity ring. - pub(crate) fn poll_reload( - &mut self, - now_unix: u64, - cooldown_minutes: u32, - ) -> Option { - if !self.pending { - return None; - } - - let cooldown_seconds = u64::from(cooldown_minutes) - .saturating_mul(60) - .max(MIN_COOLDOWN_SECONDS); - - if let Some(until) = self.backoff_until_unix - && now_unix < until - { - return self.maybe_report_skip( - SkipKind::Backoff, - Some(until), - "Reload backoff active".to_string(), - Some(until), - ); - } - - if self.pending_priority == ReloadPriority::Normal - && let Some(last) = self.last_attempt_unix - { - let next = last.saturating_add(cooldown_seconds); - if now_unix < next { - return self.maybe_report_skip( - SkipKind::Cooldown, - Some(next), - "Reload cooldown active".to_string(), - Some(next), - ); - } - } - - match try_reload_libreqos_locked() { - ReloadExecOutcome::Busy => self.maybe_report_skip( - SkipKind::Busy, - None, - "Reload already in progress".to_string(), - None, - ), - ReloadExecOutcome::Success(message) => { - self.last_attempt_unix = Some(now_unix); - self.consecutive_failures = 0; - self.backoff_until_unix = None; - self.pending = false; - self.pending_priority = ReloadPriority::Normal; - let why = self - .pending_reason - .take() - .unwrap_or_else(|| "Topology change".to_string()); - self.last_reported_skip = None; - Some(ReloadAttempt { - outcome: ReloadOutcome::Success { message }, - request_reason: Some(why), - }) - } - ReloadExecOutcome::Failed(error) => { - self.last_attempt_unix = Some(now_unix); - self.consecutive_failures = self.consecutive_failures.saturating_add(1); - let backoff_seconds = backoff_seconds(cooldown_seconds, self.consecutive_failures); - let next = now_unix.saturating_add(backoff_seconds); - self.backoff_until_unix = Some(next); - self.last_reported_skip = None; - let why = self - .pending_reason - .clone() - .unwrap_or_else(|| "Topology change".to_string()); - Some(ReloadAttempt { - outcome: ReloadOutcome::Failed { - error, - next_allowed_unix: Some(next), - }, - request_reason: Some(why), - }) - } - } - } - - fn maybe_report_skip( - &mut self, - kind: SkipKind, - key_next_allowed_unix: Option, - reason: String, - next_allowed_unix: Option, - ) -> Option { - let key = (kind, key_next_allowed_unix); - if self.last_reported_skip == Some(key) { - return None; - } - self.last_reported_skip = Some(key); - Some(ReloadAttempt { - outcome: ReloadOutcome::Skipped { - reason, - next_allowed_unix, - }, - request_reason: None, - }) - } -} - -/// Computes the backoff window in seconds given a cooldown and consecutive failure count. -/// -/// This function is pure: it has no side effects. -fn backoff_seconds(cooldown_seconds: u64, consecutive_failures: u32) -> u64 { - if consecutive_failures == 0 { - return cooldown_seconds; - } - - let exp = consecutive_failures.saturating_sub(1).min(10); // cap exponent - let mult = 1u64 << exp; - cooldown_seconds - .saturating_mul(mult) - .min(MAX_BACKOFF_SECONDS) -} diff --git a/src/rust/lqosd/src/treeguard/state.rs b/src/rust/lqosd/src/treeguard/state.rs index f4699c79..737420c0 100644 --- a/src/rust/lqosd/src/treeguard/state.rs +++ b/src/rust/lqosd/src/treeguard/state.rs @@ -3,6 +3,7 @@ //! This module will hold per-node and per-circuit state such as dwell timers, //! last-seen timestamps, and smoothed telemetry. +use lqos_bakery::BakeryRuntimeNodeOperationFailureReason; use std::collections::VecDeque; /// Smoothed state using an exponential weighted moving average (EWMA). @@ -69,6 +70,27 @@ pub struct LinkDirectionState { /// When the link first went below the configured top-level safe-util threshold (seconds since /// UNIX epoch), if currently below. pub top_level_safe_since_unix: Option, + /// When the link first went above the fixed emergency-util threshold (seconds since UNIX + /// epoch), if currently above. + pub top_level_emergency_since_unix: Option, +} + +/// Minimal topology fingerprint for deciding whether a structural ineligibility latch is still valid. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct LinkTopologyFingerprint { + /// Number of direct child sites under the node. + pub direct_child_sites: usize, + /// Number of direct circuits directly attached to the node. + pub direct_circuits: usize, +} + +/// Latched structural ineligibility state for TreeGuard runtime virtualization. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LinkStructuralIneligibleState { + /// Structured Bakery failure reason. + pub reason: BakeryRuntimeNodeOperationFailureReason, + /// Topology fingerprint observed when Bakery rejected the node. + pub topology_fingerprint: LinkTopologyFingerprint, } /// Per-node link virtualization tracking state. @@ -80,6 +102,10 @@ pub struct LinkState { pub last_change_unix: Option, /// History of recent state changes (seconds since UNIX epoch), newest at the back. pub recent_changes_unix: VecDeque, + /// Current topology fingerprint for this node. + pub topology_fingerprint: LinkTopologyFingerprint, + /// Latched structural ineligibility, if Bakery rejected virtualization for this topology. + pub structural_ineligible: Option, /// Download direction tracking state. pub down: LinkDirectionState, /// Upload direction tracking state. diff --git a/src/rust/lqosd/src/treeguard/status.rs b/src/rust/lqosd/src/treeguard/status.rs index 5dccd36a..11498ba8 100644 --- a/src/rust/lqosd/src/treeguard/status.rs +++ b/src/rust/lqosd/src/treeguard/status.rs @@ -1,5 +1,6 @@ //! TreeGuard status snapshots for UI visibility. +use crate::node_manager::local_api::directories; use crate::node_manager::ws::messages::{TreeguardActivityEntry, TreeguardStatusData}; use crate::treeguard::actor; use lqos_config::load_config; @@ -18,13 +19,20 @@ pub async fn treeguard_status_snapshot() -> TreeguardStatusData { } let Ok(config) = load_config() else { + let totals = directories::treeguard_metadata_summary(); return TreeguardStatusData { enabled: false, dry_run: true, + paused_for_bakery_reload: false, + pause_reason: None, cpu_max_pct: None, + total_nodes: totals.total_nodes, + total_circuits: totals.total_circuits, managed_nodes: 0, managed_circuits: 0, virtualized_nodes: 0, + cake_circuits: 0, + mixed_sqm_circuits: 0, fq_codel_circuits: 0, last_action_summary: None, warnings: vec![ @@ -35,6 +43,7 @@ pub async fn treeguard_status_snapshot() -> TreeguardStatusData { let tg = &config.treeguard; let mut warnings = Vec::new(); + let totals = directories::treeguard_metadata_summary(); if tg.enabled && !tg.links.all_nodes @@ -52,18 +61,24 @@ pub async fn treeguard_status_snapshot() -> TreeguardStatusData { TreeguardStatusData { enabled: tg.enabled, dry_run: tg.dry_run, + paused_for_bakery_reload: false, + pause_reason: None, cpu_max_pct: None, + total_nodes: totals.total_nodes, + total_circuits: totals.total_circuits, managed_nodes: if tg.links.all_nodes { - 0 + totals.total_nodes } else { tg.links.nodes.len() }, managed_circuits: if tg.circuits.all_circuits { - 0 + totals.total_circuits } else { tg.circuits.circuits.len() }, virtualized_nodes: 0, + cake_circuits: 0, + mixed_sqm_circuits: 0, fq_codel_circuits: 0, last_action_summary: None, warnings, diff --git a/src/rust/lqosd/src/version_checks.rs b/src/rust/lqosd/src/version_checks.rs index 41ae310a..03beac9e 100644 --- a/src/rust/lqosd/src/version_checks.rs +++ b/src/rust/lqosd/src/version_checks.rs @@ -8,6 +8,7 @@ use std::sync::atomic::AtomicBool; use std::thread; const VERSION_STRING: &str = include_str!("../../../VERSION_STRING"); +const FORCE_UPDATE_ENV: &str = "LQOS_FORCE_UPDATE_AVAILABLE"; #[derive(Serialize, Debug)] struct VersionCheckRequest { @@ -23,6 +24,42 @@ pub struct VersionCheckResponse { static NEW_VERSION_AVAILABLE: AtomicBool = AtomicBool::new(false); +fn parse_force_update_override(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + + if matches!( + trimmed.to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) { + return Some(true); + } + + if matches!( + trimmed.to_ascii_lowercase().as_str(), + "0" | "false" | "no" | "off" + ) { + return Some(false); + } + + None +} + +fn force_update_available() -> Option { + let raw = std::env::var(FORCE_UPDATE_ENV).ok()?; + let parsed = parse_force_update_override(&raw); + if parsed.is_none() { + tracing::warn!( + "{} has invalid value {:?}; expected one of: 1/0 true/false yes/no on/off", + FORCE_UPDATE_ENV, + raw + ); + } + parsed +} + /// Initializes the version checking system. pub fn start_version_check() -> Result<()> { thread::Builder::new() @@ -64,6 +101,9 @@ pub fn start_version_check() -> Result<()> { /// Returns true if a new version is available. pub fn new_version_available() -> bool { + if let Some(forced) = force_update_available() { + return forced; + } NEW_VERSION_AVAILABLE.load(std::sync::atomic::Ordering::Relaxed) } @@ -74,3 +114,30 @@ fn check_version(request: VersionCheckRequest) -> Result { .into_json()?; Ok(response.update_available) } + +#[cfg(test)] +mod tests { + use super::parse_force_update_override; + + #[test] + fn parses_truthy_force_update_values() { + assert_eq!(parse_force_update_override("1"), Some(true)); + assert_eq!(parse_force_update_override("true"), Some(true)); + assert_eq!(parse_force_update_override("YES"), Some(true)); + assert_eq!(parse_force_update_override(" on "), Some(true)); + } + + #[test] + fn parses_falsey_force_update_values() { + assert_eq!(parse_force_update_override("0"), Some(false)); + assert_eq!(parse_force_update_override("false"), Some(false)); + assert_eq!(parse_force_update_override("NO"), Some(false)); + assert_eq!(parse_force_update_override(" off "), Some(false)); + } + + #[test] + fn rejects_invalid_force_update_values() { + assert_eq!(parse_force_update_override(""), None); + assert_eq!(parse_force_update_override("maybe"), None); + } +} diff --git a/src/rust/lqusers/Cargo.toml b/src/rust/lqusers/Cargo.toml index 28011337..821b6c62 100644 --- a/src/rust/lqusers/Cargo.toml +++ b/src/rust/lqusers/Cargo.toml @@ -8,3 +8,5 @@ license = "GPL-2.0-only" clap = { workspace = true } lqos_config = { path = "../lqos_config" } anyhow = { workspace = true } +lqos_bus = { path = "../lqos_bus" } +tokio = { workspace = true } diff --git a/src/rust/lqusers/src/main.rs b/src/rust/lqusers/src/main.rs index 13fcb644..7e484c0d 100644 --- a/src/rust/lqusers/src/main.rs +++ b/src/rust/lqusers/src/main.rs @@ -1,5 +1,6 @@ use anyhow::Result; use clap::{Parser, Subcommand}; +use lqos_bus::{BusRequest, bus_request}; use lqos_config::{UserRole, WebUsers}; use std::process::exit; @@ -35,6 +36,20 @@ enum Commands { List, } +fn notify_auth_cache_invalidated() { + let Ok(runtime) = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + else { + eprintln!("Warning: unable to create Tokio runtime for auth cache invalidation"); + return; + }; + + if let Err(e) = runtime.block_on(bus_request(vec![BusRequest::InvalidateAuthCache])) { + eprintln!("Warning: updated lqusers.toml but could not notify lqosd: {e}"); + } +} + fn main() -> Result<()> { let cli = Args::parse(); let mut users = WebUsers::load_or_create()?; @@ -45,9 +60,11 @@ fn main() -> Result<()> { password, }) => { users.add_or_update_user(&username, &password, role)?; + notify_auth_cache_invalidated(); } Some(Commands::Del { username }) => { users.remove_user(&username)?; + notify_auth_cache_invalidated(); } Some(Commands::List) => { println!("All Users\n"); diff --git a/src/rust/remove_pinned_maps.sh b/src/rust/remove_pinned_maps.sh index 7f6953c3..1a194886 100755 --- a/src/rust/remove_pinned_maps.sh +++ b/src/rust/remove_pinned_maps.sh @@ -1,18 +1,18 @@ #!/bin/bash -rm -v /sys/fs/bpf/map_traffic -rm -v /sys/fs/bpf/map_ip_to_cpu_and_tc -rm -v /sys/fs/bpf/cpu_map -rm -v /sys/fs/bpf/cpus_available -rm -v /sys/fs/bpf/packet_ts -rm -v /sys/fs/bpf/flow_state -rm -v /sys/fs/bpf/rtt_tracker -rm -v /sys/fs/bpf/map_ip_to_cpu_and_tc_recip -rm -v /sys/fs/bpf/map_txq_config -rm -v /sys/fs/bpf/bifrost_interface_map -rm -v /sys/fs/bpf/bifrost_vlan_map -rm -v /sys/fs/bpf/heimdall -rm -v /sys/fs/bpf/heimdall_config -rm -v /sys/fs/bpf/heimdall_watching -rm -v /sys/fs/bpf/flowbee -rm -v /sys/fs/bpf/ip_to_cpu_and_tc_hotcache -rm -v /sys/fs/bpf/ip_mapping_epoch +rm -vf /sys/fs/bpf/map_traffic +rm -vf /sys/fs/bpf/map_ip_to_cpu_and_tc +rm -vf /sys/fs/bpf/cpu_map +rm -vf /sys/fs/bpf/cpus_available +rm -vf /sys/fs/bpf/packet_ts +rm -vf /sys/fs/bpf/flow_state +rm -vf /sys/fs/bpf/rtt_tracker +rm -vf /sys/fs/bpf/map_ip_to_cpu_and_tc_recip +rm -vf /sys/fs/bpf/map_txq_config +rm -vf /sys/fs/bpf/bifrost_interface_map +rm -vf /sys/fs/bpf/bifrost_vlan_map +rm -vf /sys/fs/bpf/heimdall +rm -vf /sys/fs/bpf/heimdall_config +rm -vf /sys/fs/bpf/heimdall_watching +rm -vf /sys/fs/bpf/flowbee +rm -vf /sys/fs/bpf/ip_to_cpu_and_tc_hotcache +rm -vf /sys/fs/bpf/ip_mapping_epoch diff --git a/src/rust/uisp/src/device.rs b/src/rust/uisp/src/device.rs index 1c5b5e0f..6b2524e0 100644 --- a/src/rust/uisp/src/device.rs +++ b/src/rust/uisp/src/device.rs @@ -185,6 +185,7 @@ pub struct DeviceInterface { #[allow(non_snake_case)] #[derive(Serialize, Deserialize, Debug, Allocative)] pub struct InterfaceIdentification { + pub r#type: Option, pub name: Option, pub mac: Option, } @@ -200,6 +201,7 @@ pub struct DeviceAddress { pub struct InterfaceStatus { pub status: Option, pub speed: Option, + pub currentSpeed: Option, } #[allow(non_snake_case)] diff --git a/src/rust/uisp_integration/src/ethernet_advisory.rs b/src/rust/uisp_integration/src/ethernet_advisory.rs new file mode 100644 index 00000000..7b077e2b --- /dev/null +++ b/src/rust/uisp_integration/src/ethernet_advisory.rs @@ -0,0 +1,405 @@ +use crate::errors::UispIntegrationError; +use crate::uisp_types::UispDevice; +use lqos_config::{CIRCUIT_ETHERNET_METADATA_FILENAME, CircuitEthernetMetadata, Config}; +use std::path::Path; +use tracing::error; + +const ETH_PORT_HEADROOM_FACTOR_SUB_GIG: f32 = 0.95; + +/// Requested and applied circuit rates after negotiated-Ethernet evaluation. +#[derive(Clone, Debug, PartialEq)] +pub struct EthernetRateDecision { + /// Minimum download after any cap was applied. + pub download_min: f32, + /// Minimum upload after any cap was applied. + pub upload_min: f32, + /// Maximum download after any cap was applied. + pub download_max: f32, + /// Maximum upload after any cap was applied. + pub upload_max: f32, + /// Optional UI advisory for the affected circuit. + pub advisory: Option, +} + +/// Applies a negotiated-Ethernet cap to a circuit and returns the adjusted rates plus advisory. +pub fn apply_ethernet_rate_cap<'a>( + circuit_id: &str, + circuit_name: &str, + devices: impl IntoIterator, + download_min: f32, + upload_min: f32, + download_max: f32, + upload_max: f32, +) -> EthernetRateDecision { + let devices: Vec<&UispDevice> = devices.into_iter().collect(); + let related_device_ids = devices.iter().map(|device| device.id.clone()).collect(); + let candidate_devices: Vec<&UispDevice> = { + let station_devices: Vec<&UispDevice> = devices + .iter() + .copied() + .filter(|device| device.is_wireless_station_cpe()) + .collect(); + if !station_devices.is_empty() { + station_devices + } else { + devices + .iter() + .copied() + .filter(|device| !device.is_router_like()) + .collect() + } + }; + let mut limiting_device: Option<&UispDevice> = None; + for device in &candidate_devices { + let Some(speed_mbps) = device.negotiated_ethernet_mbps else { + continue; + }; + if limiting_device + .as_ref() + .is_none_or(|current| speed_mbps < current.negotiated_ethernet_mbps.unwrap_or(u64::MAX)) + { + limiting_device = Some(device); + } + } + + let Some(limiting_device) = limiting_device else { + return EthernetRateDecision { + download_min, + upload_min, + download_max, + upload_max, + advisory: None, + }; + }; + + let usable_cap = ethernet_usable_cap_mbps( + limiting_device + .negotiated_ethernet_mbps + .expect("limiting device must have ethernet speed"), + ); + let applied_download_max = download_max.min(usable_cap); + let applied_upload_max = upload_max.min(usable_cap); + let applied_download_min = download_min.min(applied_download_max); + let applied_upload_min = upload_min.min(applied_upload_max); + let auto_capped = applied_download_max < download_max || applied_upload_max < upload_max; + + EthernetRateDecision { + download_min: applied_download_min, + upload_min: applied_upload_min, + download_max: applied_download_max, + upload_max: applied_upload_max, + advisory: Some(CircuitEthernetMetadata { + circuit_id: circuit_id.to_string(), + circuit_name: circuit_name.to_string(), + device_ids: related_device_ids, + source: "uisp".to_string(), + negotiated_ethernet_mbps: limiting_device.negotiated_ethernet_mbps.unwrap_or_default(), + requested_download_mbps: download_max, + requested_upload_mbps: upload_max, + applied_download_mbps: applied_download_max, + applied_upload_mbps: applied_upload_max, + auto_capped, + limiting_device_id: Some(limiting_device.id.clone()), + limiting_device_name: Some(limiting_device.name.clone()), + limiting_interface_name: limiting_device.negotiated_ethernet_interface.clone(), + }), + } +} + +/// Persists circuit Ethernet advisories alongside other generated runtime files. +pub fn write_ethernet_advisories( + config: &Config, + advisories: &[CircuitEthernetMetadata], +) -> Result<(), UispIntegrationError> { + let path = Path::new(&config.lqos_directory).join(CIRCUIT_ETHERNET_METADATA_FILENAME); + let payload = serde_json::to_vec_pretty(&lqos_config::CircuitEthernetMetadataFile { + circuits: advisories + .iter() + .filter(|advisory| advisory.auto_capped) + .cloned() + .collect(), + }) + .map_err(|e| { + error!("Unable to serialize circuit Ethernet metadata: {e:?}"); + UispIntegrationError::CsvError + })?; + std::fs::write(path, payload).map_err(|e| { + error!("Unable to write circuit Ethernet metadata: {e:?}"); + UispIntegrationError::CsvError + })?; + Ok(()) +} + +fn ethernet_usable_cap_mbps(negotiated_mbps: u64) -> f32 { + match negotiated_mbps { + 0..=999 => negotiated_mbps as f32 * ETH_PORT_HEADROOM_FACTOR_SUB_GIG, + 1000..=2499 => negotiated_mbps.saturating_sub(5) as f32, + 2500..=9999 => negotiated_mbps.saturating_sub(10) as f32, + _ => negotiated_mbps.saturating_sub(50) as f32, + } +} + +#[cfg(test)] +mod tests { + use super::{apply_ethernet_rate_cap, write_ethernet_advisories}; + use crate::uisp_types::UispDevice; + use lqos_config::{CIRCUIT_ETHERNET_METADATA_FILENAME, CircuitEthernetMetadataFile, Config}; + use std::collections::HashSet; + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn sample_device( + id: &str, + name: &str, + speed: Option, + iface: Option<&str>, + role: Option<&str>, + wireless_mode: Option<&str>, + ) -> UispDevice { + UispDevice { + id: id.to_string(), + name: name.to_string(), + mac: String::new(), + role: role.map(str::to_string), + wireless_mode: wireless_mode.map(str::to_string), + site_id: "site-1".to_string(), + download: 1000, + upload: 1000, + ipv4: HashSet::new(), + ipv6: HashSet::new(), + negotiated_ethernet_mbps: speed, + negotiated_ethernet_interface: iface.map(str::to_string), + } + } + + #[test] + fn ethernet_cap_uses_lowest_detected_port_speed() { + let fast = sample_device( + "dev-fast", + "Fast", + Some(1000), + Some("eth0"), + Some("station"), + Some("sta-ptmp"), + ); + let slow = sample_device( + "dev-slow", + "Slow", + Some(100), + Some("eth1"), + Some("station"), + Some("sta-ptmp"), + ); + + let decision = apply_ethernet_rate_cap( + "circuit-1", + "Circuit 1", + [&fast, &slow], + 25.0, + 25.0, + 300.0, + 300.0, + ); + + assert_eq!(decision.download_max, 95.0); + assert_eq!(decision.upload_max, 95.0); + assert_eq!(decision.download_min, 25.0); + assert_eq!(decision.upload_min, 25.0); + let advisory = decision.advisory.expect("advisory should exist"); + assert!(advisory.auto_capped); + assert_eq!(advisory.negotiated_ethernet_mbps, 100); + assert_eq!(advisory.limiting_device_id.as_deref(), Some("dev-slow")); + } + + #[test] + fn ethernet_cap_preserves_lower_upload_plan() { + let slow = sample_device( + "dev-slow", + "Slow", + Some(100), + Some("eth1"), + Some("station"), + Some("sta-ptmp"), + ); + + let decision = + apply_ethernet_rate_cap("circuit-1", "Circuit 1", [&slow], 10.0, 10.0, 300.0, 50.0); + + assert_eq!(decision.download_max, 95.0); + assert_eq!(decision.upload_max, 50.0); + let advisory = decision.advisory.expect("advisory should exist"); + assert!(advisory.auto_capped); + assert_eq!(advisory.applied_download_mbps, 95.0); + assert_eq!(advisory.applied_upload_mbps, 50.0); + } + + #[test] + fn gigabit_ports_use_small_fixed_margin() { + let gig = sample_device( + "dev-gig", + "Gig", + Some(1000), + Some("eth0"), + Some("station"), + Some("sta-ptmp"), + ); + + let decision = + apply_ethernet_rate_cap("circuit-1", "Circuit 1", [&gig], 100.0, 100.0, 995.0, 995.0); + + assert_eq!(decision.download_max, 995.0); + assert_eq!(decision.upload_max, 995.0); + let advisory = decision.advisory.expect("advisory should exist"); + assert!(!advisory.auto_capped); + assert_eq!(advisory.applied_download_mbps, 995.0); + assert_eq!(advisory.applied_upload_mbps, 995.0); + } + + #[test] + fn gigabit_ports_cap_above_line_rate_to_995() { + let gig = sample_device( + "dev-gig", + "Gig", + Some(1000), + Some("eth0"), + Some("station"), + Some("sta-ptmp"), + ); + + let decision = apply_ethernet_rate_cap( + "circuit-1", + "Circuit 1", + [&gig], + 100.0, + 100.0, + 5000.0, + 5000.0, + ); + + assert_eq!(decision.download_max, 995.0); + assert_eq!(decision.upload_max, 995.0); + let advisory = decision.advisory.expect("advisory should exist"); + assert!(advisory.auto_capped); + assert_eq!(advisory.applied_download_mbps, 995.0); + assert_eq!(advisory.applied_upload_mbps, 995.0); + } + + #[test] + fn station_devices_win_over_router_devices() { + let station = sample_device( + "dev-station", + "Station", + Some(1000), + Some("data"), + Some("station"), + Some("sta-ptmp"), + ); + let router = sample_device( + "dev-router", + "Router", + Some(10), + Some("eth3"), + Some("homeWiFi"), + Some("ap"), + ); + + let decision = apply_ethernet_rate_cap( + "circuit-1", + "Circuit 1", + [&station, &router], + 10.0, + 10.0, + 115.0, + 22.0, + ); + + assert_eq!(decision.download_max, 115.0); + assert_eq!(decision.upload_max, 22.0); + let advisory = decision.advisory.expect("advisory should exist"); + assert!(!advisory.auto_capped); + assert_eq!(advisory.negotiated_ethernet_mbps, 1000); + assert_eq!(advisory.limiting_device_id.as_deref(), Some("dev-station")); + } + + #[test] + fn router_only_devices_do_not_drive_ethernet_cap() { + let router = sample_device( + "dev-router", + "Router", + Some(100), + Some("eth0"), + Some("router"), + None, + ); + + let decision = + apply_ethernet_rate_cap("circuit-1", "Circuit 1", [&router], 10.0, 10.0, 115.0, 22.0); + + assert_eq!(decision.download_max, 115.0); + assert_eq!(decision.upload_max, 22.0); + assert!(decision.advisory.is_none()); + } + + #[test] + fn writer_persists_only_auto_capped_advisories() { + let auto_capped = sample_device( + "dev-100m", + "Hundred", + Some(100), + Some("eth0"), + Some("station"), + Some("sta-ptmp"), + ); + let observed_only = sample_device( + "dev-1g", + "Gig", + Some(1000), + Some("eth0"), + Some("station"), + Some("sta-ptmp"), + ); + + let capped = apply_ethernet_rate_cap( + "circuit-capped", + "Capped Circuit", + [&auto_capped], + 10.0, + 10.0, + 115.0, + 22.0, + ) + .advisory + .expect("auto-capped advisory should exist"); + let observed = apply_ethernet_rate_cap( + "circuit-observed", + "Observed Circuit", + [&observed_only], + 100.0, + 100.0, + 995.0, + 995.0, + ) + .advisory + .expect("observed advisory should exist"); + + let mut config = Config::default(); + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time should move forward") + .as_nanos(); + let temp_dir = std::env::temp_dir().join(format!("lqos-ethernet-writer-{unique}")); + fs::create_dir_all(&temp_dir).expect("temp dir should be created"); + config.lqos_directory = temp_dir.to_string_lossy().to_string(); + + write_ethernet_advisories(&config, &[capped.clone(), observed]) + .expect("writer should succeed"); + + let payload = fs::read(temp_dir.join(CIRCUIT_ETHERNET_METADATA_FILENAME)) + .expect("metadata file should exist"); + let parsed: CircuitEthernetMetadataFile = + serde_json::from_slice(&payload).expect("metadata file should parse"); + assert_eq!(parsed.circuits, vec![capped]); + + let _ = fs::remove_dir_all(temp_dir); + } +} diff --git a/src/rust/uisp_integration/src/main.rs b/src/rust/uisp_integration/src/main.rs index dd486f8a..03f02f3c 100644 --- a/src/rust/uisp_integration/src/main.rs +++ b/src/rust/uisp_integration/src/main.rs @@ -5,6 +5,7 @@ mod blackboard; #[warn(missing_docs)] mod errors; +mod ethernet_advisory; pub mod ip_ranges; mod strategies; pub mod uisp_types; diff --git a/src/rust/uisp_integration/src/strategies/ap_only.rs b/src/rust/uisp_integration/src/strategies/ap_only.rs index 1ef045ac..704b72b4 100644 --- a/src/rust/uisp_integration/src/strategies/ap_only.rs +++ b/src/rust/uisp_integration/src/strategies/ap_only.rs @@ -1,8 +1,9 @@ use crate::errors::UispIntegrationError; +use crate::ethernet_advisory::{apply_ethernet_rate_cap, write_ethernet_advisories}; use crate::ip_ranges::IpRanges; use crate::strategies::common::UispData; use crate::strategies::full::shaped_devices_writer::ShapedDevice; -use lqos_config::Config; +use lqos_config::{CircuitEthernetMetadata, Config}; use std::collections::HashSet; use std::fs::write; use std::path::Path; @@ -68,6 +69,7 @@ pub async fn build_ap_only_network( // Write ShapedDevices.csv let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); let mut shaped_devices = Vec::new(); + let mut ethernet_advisories: Vec = Vec::new(); let mut seen_pairs = HashSet::new(); for (parent, client_ids) in mappings.iter() { for client_id in client_ids { @@ -77,50 +79,53 @@ pub async fn build_ap_only_network( .iter() .filter(|d| d.site_id == *client_id) .collect::>(); + let requested = if let Some((dl_min, dl_max, ul_min, ul_max)) = + site.burst_rates(&config) + { + ( + f32::max(0.1, dl_min), + f32::max(0.1, dl_max), + f32::max(0.1, ul_min), + f32::max(0.1, ul_max), + ) + } else if site.suspended && config.uisp_integration.suspended_strategy == "slow" { + (0.1, 0.1, 0.1, 0.1) + } else { + ( + f32::max( + 0.1, + site.max_down_mbps as f32 + * config.uisp_integration.commit_bandwidth_multiplier, + ), + f32::max( + 0.1, + site.max_down_mbps as f32 + * config.uisp_integration.bandwidth_overhead_factor, + ), + f32::max( + 0.1, + site.max_up_mbps as f32 + * config.uisp_integration.commit_bandwidth_multiplier, + ), + f32::max( + 0.1, + site.max_up_mbps as f32 * config.uisp_integration.bandwidth_overhead_factor, + ), + ) + }; + let ethernet_decision = apply_ethernet_rate_cap( + &site.id, + &site.name, + devices.iter().copied(), + requested.0, + requested.2, + requested.1, + requested.3, + ); + if let Some(advisory) = ethernet_decision.advisory.clone() { + ethernet_advisories.push(advisory); + } for device in devices.iter().filter(|d| d.has_address()) { - // Compute subscriber rates: prefer UISP QoS + burst - let (download_min, mut download_max, upload_min, mut upload_max) = - if let Some((dl_min, dl_max, ul_min, ul_max)) = site.burst_rates(&config) { - ( - f32::max(0.1, dl_min), - f32::max(0.1, dl_max), - f32::max(0.1, ul_min), - f32::max(0.1, ul_max), - ) - } else if site.suspended && config.uisp_integration.suspended_strategy == "slow" - { - (0.1, 0.1, 0.1, 0.1) - } else { - ( - f32::max( - 0.1, - site.max_down_mbps as f32 - * config.uisp_integration.commit_bandwidth_multiplier, - ), - f32::max( - 0.1, - site.max_down_mbps as f32 - * config.uisp_integration.bandwidth_overhead_factor, - ), - f32::max( - 0.1, - site.max_up_mbps as f32 - * config.uisp_integration.commit_bandwidth_multiplier, - ), - f32::max( - 0.1, - site.max_up_mbps as f32 - * config.uisp_integration.bandwidth_overhead_factor, - ), - ) - }; - if download_max < download_min { - download_max = download_min; - } - if upload_max < upload_min { - upload_max = upload_min; - } - let key = (site.id.clone(), device.id.clone()); if !seen_pairs.insert(key) { continue; @@ -135,10 +140,10 @@ pub async fn build_ap_only_network( mac: device.mac.clone(), ipv4: device.ipv4_list(), ipv6: device.ipv6_list(), - download_min, - upload_min, - download_max, - upload_max, + download_min: ethernet_decision.download_min, + upload_min: ethernet_decision.upload_min, + download_max: ethernet_decision.download_max, + upload_max: ethernet_decision.upload_max, comment: "".to_string(), }; shaped_devices.push(sd); @@ -158,6 +163,7 @@ pub async fn build_ap_only_network( error!("{e:?}"); UispIntegrationError::CsvError })?; + write_ethernet_advisories(&config, ðernet_advisories)?; info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); Ok(()) diff --git a/src/rust/uisp_integration/src/strategies/ap_site.rs b/src/rust/uisp_integration/src/strategies/ap_site.rs index 98664bca..2bc61a83 100644 --- a/src/rust/uisp_integration/src/strategies/ap_site.rs +++ b/src/rust/uisp_integration/src/strategies/ap_site.rs @@ -1,9 +1,10 @@ use crate::blackboard_blob; use crate::errors::UispIntegrationError; +use crate::ethernet_advisory::{apply_ethernet_rate_cap, write_ethernet_advisories}; use crate::ip_ranges::IpRanges; use crate::strategies::common::UispData; use crate::strategies::full::shaped_devices_writer::ShapedDevice; -use lqos_config::Config; +use lqos_config::{CircuitEthernetMetadata, Config}; use std::collections::{HashMap, HashSet}; use std::fs::write; use std::path::Path; @@ -52,7 +53,14 @@ pub async fn build_ap_site_network( //println!("{:#?}", root); let mut shaped_devices = Vec::new(); - let net_json = root.walk_children(None, &uisp_data, &mut shaped_devices, &config); + let mut ethernet_advisories = Vec::new(); + let net_json = root.walk_children( + None, + &uisp_data, + &mut shaped_devices, + &mut ethernet_advisories, + &config, + ); let network_path = Path::new(&config.lqos_directory).join("network.json"); if network_path.exists() && !config.integration_common.always_overwrite_network_json { @@ -69,7 +77,7 @@ pub async fn build_ap_site_network( info!("Written network.json"); } - let _ = write_shaped_devices(&config, &mut shaped_devices); + let _ = write_shaped_devices(&config, &mut shaped_devices, ðernet_advisories); info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); Ok(()) @@ -78,6 +86,7 @@ pub async fn build_ap_site_network( pub(crate) fn write_shaped_devices( config: &Arc, shaped_devices: &mut Vec, + ethernet_advisories: &[CircuitEthernetMetadata], ) -> Result<(), UispIntegrationError> { let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); let mut seen_pairs = HashSet::new(); @@ -95,6 +104,7 @@ pub(crate) fn write_shaped_devices( error!("{e:?}"); UispIntegrationError::CsvError })?; + write_ethernet_advisories(config, ethernet_advisories)?; info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); Ok(()) } @@ -166,6 +176,7 @@ impl Layer { parent: Option<&str>, uisp_data: &UispData, shaped_devices: &mut Vec, + ethernet_advisories: &mut Vec, config: &Config, ) -> serde_json::Map { let mut children = serde_json::Map::new(); @@ -181,13 +192,24 @@ impl Layer { children.insert( name.clone(), child - .walk_children(Some(&parent_name), uisp_data, shaped_devices, config) + .walk_children( + Some(&parent_name), + uisp_data, + shaped_devices, + ethernet_advisories, + config, + ) .into(), ); } GraphMapping::ClientById(_client_id) => { - let _ = - child.walk_children(Some(&parent_name), uisp_data, shaped_devices, config); + let _ = child.walk_children( + Some(&parent_name), + uisp_data, + shaped_devices, + ethernet_advisories, + config, + ); } _ => {} } @@ -209,10 +231,16 @@ impl Layer { root.insert("parent_site".to_string(), name.to_string().into()); if let (Some(latitude), Some(longitude)) = (site.latitude, site.longitude) { if let Some(number) = serde_json::Number::from_f64(latitude as f64) { - root.insert("latitude".to_string(), serde_json::Value::Number(number)); + root.insert( + "latitude".to_string(), + serde_json::Value::Number(number), + ); } if let Some(number) = serde_json::Number::from_f64(longitude as f64) { - root.insert("longitude".to_string(), serde_json::Value::Number(number)); + root.insert( + "longitude".to_string(), + serde_json::Value::Number(number), + ); } } } @@ -224,7 +252,10 @@ impl Layer { if let Some(device) = uisp_data.devices.iter().find(|d| d.name == *name) { root.insert("downloadBandwidthMbps".to_owned(), device.download.into()); root.insert("uploadBandwidthMbps".to_owned(), device.upload.into()); - root.insert("id".to_string(), format!("uisp:device:{}", device.id).into()); + root.insert( + "id".to_string(), + format!("uisp:device:{}", device.id).into(), + ); root.insert("uisp_device".to_string(), device.id.clone().into()); } } @@ -235,67 +266,61 @@ impl Layer { .iter() .filter(|d| d.site_id == *client_id) .collect::>(); + let requested = if let Some(site) = + uisp_data.sites.iter().find(|s| s.id == *client_id) + { + if let Some((dl_min, dl_max, ul_min, ul_max)) = site.burst_rates(config) + { + ( + f32::max(0.1, dl_min), + f32::max(0.1, dl_max), + f32::max(0.1, ul_min), + f32::max(0.1, ul_max), + ) + } else if site.suspended + && config.uisp_integration.suspended_strategy == "slow" + { + (0.1, 0.1, 0.1, 0.1) + } else { + ( + f32::max( + 0.1, + site.max_down_mbps as f32 + * config.uisp_integration.commit_bandwidth_multiplier, + ), + f32::max( + 0.1, + site.max_down_mbps as f32 + * config.uisp_integration.bandwidth_overhead_factor, + ), + f32::max( + 0.1, + site.max_up_mbps as f32 + * config.uisp_integration.commit_bandwidth_multiplier, + ), + f32::max( + 0.1, + site.max_up_mbps as f32 + * config.uisp_integration.bandwidth_overhead_factor, + ), + ) + } + } else { + (0.1, 0.1, 0.1, 0.1) + }; + let ethernet_decision = apply_ethernet_rate_cap( + &site.id, + &site.name, + devices.iter().copied(), + requested.0, + requested.2, + requested.1, + requested.3, + ); + if let Some(advisory) = ethernet_decision.advisory.clone() { + ethernet_advisories.push(advisory); + } for device in devices.iter().filter(|d| d.has_address()) { - // Compute subscriber rates: prefer UISP QoS + burst - let (download_min, mut download_max, upload_min, mut upload_max) = - if let Some(site) = - uisp_data.sites.iter().find(|s| s.id == *client_id) - { - if let Some((dl_min, dl_max, ul_min, ul_max)) = - site.burst_rates(config) - { - ( - f32::max(0.1, dl_min), - f32::max(0.1, dl_max), - f32::max(0.1, ul_min), - f32::max(0.1, ul_max), - ) - } else if site.suspended - && config.uisp_integration.suspended_strategy == "slow" - { - (0.1, 0.1, 0.1, 0.1) - } else { - ( - f32::max( - 0.1, - site.max_down_mbps as f32 - * config - .uisp_integration - .commit_bandwidth_multiplier, - ), - f32::max( - 0.1, - site.max_down_mbps as f32 - * config - .uisp_integration - .bandwidth_overhead_factor, - ), - f32::max( - 0.1, - site.max_up_mbps as f32 - * config - .uisp_integration - .commit_bandwidth_multiplier, - ), - f32::max( - 0.1, - site.max_up_mbps as f32 - * config - .uisp_integration - .bandwidth_overhead_factor, - ), - ) - } - } else { - (0.1, 0.1, 0.1, 0.1) - }; - if download_max < download_min { - download_max = download_min; - } - if upload_max < upload_min { - upload_max = upload_min; - } - let sd = ShapedDevice { circuit_id: site.id.clone(), circuit_name: site.name.clone(), @@ -305,10 +330,10 @@ impl Layer { mac: device.mac.clone(), ipv4: device.ipv4_list(), ipv6: device.ipv6_list(), - download_min, - upload_min, - download_max, - upload_max, + download_min: ethernet_decision.download_min, + upload_min: ethernet_decision.upload_min, + download_max: ethernet_decision.download_max, + upload_max: ethernet_decision.upload_max, comment: "".to_string(), }; shaped_devices.push(sd); @@ -358,6 +383,103 @@ pub(crate) async fn find_troublesome_sites(data: &UispData) -> anyhow::Result
    anyhow::Result> { let mut result = HashSet::new(); for client in data.find_client_sites() { diff --git a/src/rust/uisp_integration/src/strategies/common.rs b/src/rust/uisp_integration/src/strategies/common.rs index 7fd37ad1..c3dcfc98 100644 --- a/src/rust/uisp_integration/src/strategies/common.rs +++ b/src/rust/uisp_integration/src/strategies/common.rs @@ -19,7 +19,54 @@ pub(crate) struct UispData { //pub data_links: Vec, } -/// Ensure site names are unique by appending the site ID to duplicates. +fn short_address_segment(site: &Site) -> Option { + let address = site.address()?; + let first_segment = address.split(',').next()?.trim(); + if first_segment.is_empty() { + None + } else { + Some(first_segment.to_string()) + } +} + +fn service_name(site: &Site) -> Option { + let name = site.ucrm.as_ref()?.service.as_ref()?.name.trim(); + if name.is_empty() { + None + } else { + Some(name.to_string()) + } +} + +fn short_site_id(site: &Site) -> String { + site.id.chars().take(8).collect() +} + +fn disambiguation_candidates(site: &Site, base_name: &str) -> Vec { + let mut candidates = Vec::new(); + + if let Some(address_segment) = short_address_segment(site) + && address_segment != base_name + { + candidates.push(format!("{base_name} ({address_segment})")); + } + + if let Some(service_name) = service_name(site) + && service_name != base_name + { + candidates.push(format!("{base_name} ({service_name})")); + } + + let short_id = short_site_id(site); + if !short_id.is_empty() { + candidates.push(format!("{base_name} ({short_id})")); + } + candidates.push(format!("{base_name} ({})", site.id)); + + candidates +} + +/// Ensure site names are unique by appending a human-friendly disambiguator to duplicates. pub(crate) fn dedup_site_names(sites: &mut [Site]) { let mut name_counts: HashMap = HashMap::new(); for site in sites.iter() { @@ -30,6 +77,18 @@ pub(crate) fn dedup_site_names(sites: &mut [Site]) { *name_counts.entry(base_name).or_insert(0) += 1; } + let mut used_names: HashSet = sites + .iter() + .filter_map(|site| { + let base_name = site.name_or_blank(); + if base_name.is_empty() || name_counts.get(&base_name).copied().unwrap_or(0) > 1 { + None + } else { + Some(base_name) + } + }) + .collect(); + for site in sites.iter_mut() { let base_name = site.name_or_blank(); if base_name.is_empty() { @@ -37,7 +96,11 @@ pub(crate) fn dedup_site_names(sites: &mut [Site]) { } if name_counts.get(&base_name).copied().unwrap_or(0) > 1 { - let candidate = format!("{base_name} {}", site.id); + let candidate = disambiguation_candidates(site, &base_name) + .into_iter() + .find(|candidate| !used_names.contains(candidate)) + .unwrap_or_else(|| format!("{base_name} ({})", site.id)); + used_names.insert(candidate.clone()); if let Some(ident) = site.identification.as_mut() { ident.name = Some(candidate.clone()); } @@ -45,7 +108,7 @@ pub(crate) fn dedup_site_names(sites: &mut [Site]) { site_id = %site.id, original = %base_name, renamed = %candidate, - "Duplicate UISP site name detected; renaming with site ID for LibreQoS uniqueness" + "Duplicate UISP site name detected; renaming with human-friendly disambiguator for LibreQoS uniqueness" ); } } @@ -225,3 +288,94 @@ impl UispData { .collect() } } + +#[cfg(test)] +mod tests { + use super::dedup_site_names; + use serde_json::json; + use uisp::Site; + + fn mk_site(id: &str, name: &str, address: Option<&str>, service_name: Option<&str>) -> Site { + let mut value = json!({ + "id": id, + "identification": { + "id": id, + "name": name, + "type": "endpoint", + "suspended": false + } + }); + + if let Some(address) = address { + value["description"] = json!({ "address": address }); + } + if let Some(service_name) = service_name { + value["ucrm"] = json!({ + "service": { + "id": format!("svc-{id}"), + "name": service_name, + "status": 1, + "tariffId": "1", + "trafficShapingOverrideEnabled": false + } + }); + } + + serde_json::from_value(value).expect("site JSON must deserialize") + } + + #[test] + fn duplicate_site_names_prefer_address_segment() { + let mut sites = vec![ + mk_site( + "213f4b53-bddf-41dd-af65-2dbaa5bf0927", + "Pathway Communications", + Some("7100 Binational Way, Santa Teresa, 88008, New Mexico, United States"), + None, + ), + mk_site( + "96bb13cd-10cc-43a1-ae56-a389161178d7", + "Pathway Communications", + Some("175 Lindburgh Dr, El Paso, 79932, Texas, United States"), + None, + ), + ]; + + dedup_site_names(&mut sites); + + assert_eq!( + sites[0].name_or_blank(), + "Pathway Communications (7100 Binational Way)" + ); + assert_eq!( + sites[1].name_or_blank(), + "Pathway Communications (175 Lindburgh Dr)" + ); + } + + #[test] + fn duplicate_site_names_fall_back_to_service_name() { + let mut sites = vec![ + mk_site("site-1", "Acme", None, Some("1000/1000 Mbps - Primary")), + mk_site("site-2", "Acme", None, Some("1000/1000 Mbps - Backup")), + ]; + + dedup_site_names(&mut sites); + + assert_eq!(sites[0].name_or_blank(), "Acme (1000/1000 Mbps - Primary)"); + assert_eq!(sites[1].name_or_blank(), "Acme (1000/1000 Mbps - Backup)"); + } + + #[test] + fn duplicate_site_names_fall_back_to_short_id() { + let mut sites = vec![ + mk_site("abcd1234-0000-0000-0000-000000000000", "Acme", None, None), + mk_site("efgh5678-0000-0000-0000-000000000000", "Acme", None, None), + ]; + + dedup_site_names(&mut sites); + + assert_eq!(sites[0].name_or_blank(), "Acme (abcd1234)"); + assert_eq!(sites[1].name_or_blank(), "Acme (efgh5678)"); + } +} diff --git a/src/rust/uisp_integration/src/strategies/flat.rs b/src/rust/uisp_integration/src/strategies/flat.rs index 047d64d6..d8c95aa3 100644 --- a/src/rust/uisp_integration/src/strategies/flat.rs +++ b/src/rust/uisp_integration/src/strategies/flat.rs @@ -1,9 +1,10 @@ use crate::blackboard_blob; use crate::errors::UispIntegrationError; +use crate::ethernet_advisory::{apply_ethernet_rate_cap, write_ethernet_advisories}; use crate::ip_ranges::IpRanges; use crate::strategies::common::dedup_site_names; use crate::uisp_types::UispDevice; -use lqos_config::Config; +use lqos_config::{CircuitEthernetMetadata, Config}; use serde::Serialize; use std::collections::HashSet; use std::fs; @@ -82,6 +83,7 @@ pub async fn build_flat_network( // Simple Shaped Devices File let mut shaped_devices = Vec::new(); + let mut ethernet_advisories: Vec = Vec::new(); let mut seen_pairs = HashSet::new(); let ipv4_to_v6 = Vec::new(); for site in sites.iter() { @@ -96,7 +98,7 @@ pub async fn build_flat_network( .map(|id| id.suspended) .unwrap_or(false) && config.uisp_integration.suspended_strategy == "slow"; - let (download_min, upload_min, mut download_max, mut upload_max) = if suspended_slow { + let requested = if suspended_slow { (0.1, 0.1, 0.1, 0.1) } else if let Some(qos) = &site.qos { let base_down = qos @@ -172,16 +174,25 @@ pub async fn build_flat_network( f32::max(0.1, ul_max_f), ) }; - // Ensure max >= min - if download_max < download_min { - download_max = download_min; - } - if upload_max < upload_min { - upload_max = upload_min; + let site_devices: Vec = devices + .iter() + .map(|device| UispDevice::from_uisp(device, &config, &ip_ranges, &ipv4_to_v6)) + .filter(|device| device.site_id == site.id && device.has_address()) + .collect(); + let ethernet_decision = apply_ethernet_rate_cap( + &site.id, + &site.name_or_blank(), + site_devices.iter(), + requested.0, + requested.1, + requested.2, + requested.3, + ); + if let Some(advisory) = ethernet_decision.advisory.clone() { + ethernet_advisories.push(advisory); } for device in devices.iter() { - let dev = UispDevice::from_uisp(device, &config, &ip_ranges, &ipv4_to_v6); - if dev.site_id == site.id { + if let Some(dev) = site_devices.iter().find(|dev| dev.id == device.get_id()) { // We're an endpoint in the right sight. We're getting there let key = (site.id.clone(), device.get_id()); if !seen_pairs.insert(key) { @@ -197,10 +208,10 @@ pub async fn build_flat_network( mac: device.identification.mac.clone().unwrap_or("".to_string()), ipv4: dev.ipv4_list(), ipv6: dev.ipv6_list(), - download_min, - download_max, - upload_min, - upload_max, + download_min: ethernet_decision.download_min, + download_max: ethernet_decision.download_max, + upload_min: ethernet_decision.upload_min, + upload_max: ethernet_decision.upload_max, comment: "".to_string(), }; shaped_devices.push(sd); @@ -224,6 +235,7 @@ pub async fn build_flat_network( error!("{e:?}"); UispIntegrationError::CsvError })?; + write_ethernet_advisories(&config, ðernet_advisories)?; info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); Ok(()) diff --git a/src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs b/src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs index 751e1dfd..bd06db65 100644 --- a/src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs +++ b/src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs @@ -1,5 +1,7 @@ use crate::errors::UispIntegrationError; +use crate::ethernet_advisory::{apply_ethernet_rate_cap, write_ethernet_advisories}; use crate::uisp_types::{UispDevice, UispSite, UispSiteType}; +use lqos_config::CircuitEthernetMetadata; use lqos_config::Config; use serde::Serialize; use std::path::Path; @@ -24,6 +26,11 @@ pub struct ShapedDevice { pub comment: String, } +struct ShapedDeviceOutputs<'a> { + shaped_devices: &'a mut Vec, + ethernet_advisories: &'a mut Vec, +} + /// Writes the ShapedDevices.csv file for UISP /// /// # Arguments @@ -39,17 +46,14 @@ pub fn write_shaped_devices( ) -> Result<(), UispIntegrationError> { let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); let mut shaped_devices = Vec::new(); + let mut ethernet_advisories: Vec = Vec::new(); // Traverse - traverse( - sites, - root_idx, - 0, - devices, - &mut shaped_devices, - config, - root_idx, - ); + let mut outputs = ShapedDeviceOutputs { + shaped_devices: &mut shaped_devices, + ethernet_advisories: &mut ethernet_advisories, + }; + traverse(sites, root_idx, 0, devices, &mut outputs, config, root_idx); // Write the CSV let mut writer = csv::WriterBuilder::new() @@ -65,6 +69,7 @@ pub fn write_shaped_devices( error!("{e:?}"); UispIntegrationError::CsvError })?; + write_ethernet_advisories(config, ðernet_advisories)?; info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); Ok(()) @@ -75,52 +80,60 @@ fn traverse( idx: usize, depth: u32, devices: &[UispDevice], - shaped_devices: &mut Vec, + outputs: &mut ShapedDeviceOutputs<'_>, config: &Config, root_idx: usize, ) { if !sites[idx].device_indices.is_empty() { // We have devices! if sites[idx].site_type == UispSiteType::Client { + let site_devices: Vec<&UispDevice> = sites[idx] + .device_indices + .iter() + .filter_map(|device_idx| devices.get(*device_idx)) + .collect(); + + let requested = + if let Some((dl_min, dl_max, ul_min, ul_max)) = sites[idx].burst_rates(config) { + ( + f32::max(0.1, dl_min), + f32::max(0.1, dl_max), + f32::max(0.1, ul_min), + f32::max(0.1, ul_max), + ) + } else { + let download_max_f32 = sites[idx].max_down_mbps as f32 + * config.uisp_integration.bandwidth_overhead_factor; + let upload_max_f32 = sites[idx].max_up_mbps as f32 + * config.uisp_integration.bandwidth_overhead_factor; + let download_min_f32 = + download_max_f32 * config.uisp_integration.commit_bandwidth_multiplier; + let upload_min_f32 = + upload_max_f32 * config.uisp_integration.commit_bandwidth_multiplier; + ( + f32::max(0.1, download_min_f32), + f32::max(0.1, download_max_f32), + f32::max(0.1, upload_min_f32), + f32::max(0.1, upload_max_f32), + ) + }; + let ethernet_decision = apply_ethernet_rate_cap( + &sites[idx].id, + &sites[idx].name, + site_devices.iter().copied(), + requested.0, + requested.2, + requested.1, + requested.3, + ); + if let Some(advisory) = ethernet_decision.advisory.clone() { + outputs.ethernet_advisories.push(advisory); + } + // Add as normal clients for device in sites[idx].device_indices.iter() { let device = &devices[*device]; if device.has_address() { - // Prefer UISP QoS + burst if available; else fallback to capacity-based - let (download_min, mut download_max, upload_min, mut upload_max) = - if let Some((dl_min, dl_max, ul_min, ul_max)) = - sites[idx].burst_rates(config) - { - ( - f32::max(0.1, dl_min), - f32::max(0.1, dl_max), - f32::max(0.1, ul_min), - f32::max(0.1, ul_max), - ) - } else { - let download_max_f32 = sites[idx].max_down_mbps as f32 - * config.uisp_integration.bandwidth_overhead_factor; - let upload_max_f32 = sites[idx].max_up_mbps as f32 - * config.uisp_integration.bandwidth_overhead_factor; - let download_min_f32 = download_max_f32 - * config.uisp_integration.commit_bandwidth_multiplier; - let upload_min_f32 = upload_max_f32 - * config.uisp_integration.commit_bandwidth_multiplier; - ( - f32::max(0.1, download_min_f32), - f32::max(0.1, download_max_f32), - f32::max(0.1, upload_min_f32), - f32::max(0.1, upload_max_f32), - ) - }; - // Ensure max >= min per LibreQoS requirements - if download_max < download_min { - download_max = download_min; - } - if upload_max < upload_min { - upload_max = upload_min; - } - let sd = ShapedDevice { circuit_id: sites[idx].id.clone(), circuit_name: sites[idx].name.clone(), @@ -130,13 +143,13 @@ fn traverse( mac: device.mac.clone(), ipv4: device.ipv4_list(), ipv6: device.ipv6_list(), - download_min, - download_max, - upload_min, - upload_max, + download_min: ethernet_decision.download_min, + download_max: ethernet_decision.download_max, + upload_min: ethernet_decision.upload_min, + upload_max: ethernet_decision.upload_max, comment: "".to_string(), }; - shaped_devices.push(sd); + outputs.shaped_devices.push(sd); } } } else { @@ -178,7 +191,7 @@ fn traverse( upload_max, comment: "Infrastructure Entry".to_string(), }; - shaped_devices.push(sd); + outputs.shaped_devices.push(sd); } } } @@ -194,7 +207,7 @@ fn traverse( child_idx, depth + 1, devices, - shaped_devices, + outputs, config, root_idx, ); @@ -206,6 +219,8 @@ fn traverse( #[cfg(test)] mod tests { use super::*; + use crate::uisp_types::{UispDevice, UispSite, UispSiteType}; + use std::collections::HashSet; #[test] fn test_fractional_csv_serialization() { @@ -350,4 +365,54 @@ mod tests { println!("✅ Rate safeguard tests passed!"); } + + #[test] + fn ignored_only_client_device_produces_no_shaped_rows() { + let config = Config::default(); + let sites = vec![ + UispSite { + id: "root-site".to_string(), + name: "Root Site".to_string(), + site_type: UispSiteType::Site, + max_down_mbps: 1000, + max_up_mbps: 1000, + ..Default::default() + }, + UispSite { + id: "client-site".to_string(), + name: "Client Site".to_string(), + site_type: UispSiteType::Client, + max_down_mbps: 100, + max_up_mbps: 50, + selected_parent: Some(0), + device_indices: vec![0], + ..Default::default() + }, + ]; + let devices = vec![UispDevice { + id: "device-1".to_string(), + name: "CPE 1".to_string(), + mac: "".to_string(), + role: None, + wireless_mode: None, + site_id: "client-site".to_string(), + download: 100, + upload: 50, + ipv4: HashSet::new(), + ipv6: HashSet::new(), + negotiated_ethernet_mbps: None, + negotiated_ethernet_interface: None, + }]; + + let mut shaped_devices = Vec::new(); + let mut ethernet_advisories = Vec::new(); + let mut outputs = ShapedDeviceOutputs { + shaped_devices: &mut shaped_devices, + ethernet_advisories: &mut ethernet_advisories, + }; + + traverse(&sites, 0, 0, &devices, &mut outputs, &config, 0); + + assert!(outputs.shaped_devices.is_empty()); + } } diff --git a/src/rust/uisp_integration/src/strategies/full2.rs b/src/rust/uisp_integration/src/strategies/full2.rs index f3a17634..ce4781f0 100644 --- a/src/rust/uisp_integration/src/strategies/full2.rs +++ b/src/rust/uisp_integration/src/strategies/full2.rs @@ -6,6 +6,7 @@ mod net_json_parent; use crate::blackboard_blob; use crate::errors::UispIntegrationError; +use crate::ethernet_advisory::{apply_ethernet_rate_cap, write_ethernet_advisories}; use crate::ip_ranges::IpRanges; use crate::strategies::common::UispData; use crate::strategies::full::routes_override::RouteOverride; @@ -18,7 +19,7 @@ use crate::strategies::full2::graph_mapping::GraphMapping; use crate::strategies::full2::link_mapping::LinkMapping; use crate::strategies::full2::net_json_parent::{NetJsonParent, walk_parents}; use crate::uisp_types::UispDevice; -use lqos_config::Config; +use lqos_config::{CircuitEthernetMetadata, Config}; use petgraph::Directed; use petgraph::graph::NodeIndex; use petgraph::visit::{EdgeRef, NodeRef}; @@ -26,7 +27,7 @@ use std::collections::{HashMap, HashSet}; use std::fs::write; use std::path::Path; use std::sync::Arc; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; type GraphType = petgraph::Graph; @@ -292,11 +293,11 @@ pub async fn build_full_network_v2( if !bandwidth_overrides.is_empty() && let Some(bw_override) = bandwidth_overrides.get(name) { - info!("Applying bandwidth override for {}", name); - info!("Capacity was: {} / {}", download_capacity, upload_capacity); + debug!("Applying bandwidth override for {}", name); + debug!("Capacity was: {} / {}", download_capacity, upload_capacity); download_capacity = bw_override.0 as u64; upload_capacity = bw_override.1 as u64; - info!( + debug!( "Capacity is now: {} / {}", download_capacity, upload_capacity ); @@ -373,6 +374,9 @@ pub async fn build_full_network_v2( // Shaped Devices let mut shaped_devices = Vec::new(); let mut seen_pairs = HashSet::new(); + let mut processed_site_pairs = 0usize; + let mut shaped_device_count = 0usize; + let mut ethernet_advisories: Vec = Vec::new(); for (ap_id, client_sites) in client_mappings.iter() { for site_id in client_sites.iter() { @@ -382,51 +386,60 @@ pub async fn build_full_network_v2( let Some(site) = uisp_data.sites.iter().find(|s| s.id == *site_id) else { continue; }; - info!( + processed_site_pairs = processed_site_pairs.saturating_add(1); + debug!( "Processing site: {} (ID: {}) with AP: {} (ID: {})", site.name, site.id, ap_device.name, ap_device.id ); + let site_devices: Vec<&UispDevice> = uisp_data + .devices + .iter() + .filter(|d| d.site_id == *site_id && d.has_address()) + .collect(); + + let requested = + if let Some((dl_min, dl_max, ul_min, ul_max)) = site.burst_rates(&config) { + ( + f32::max(0.1, dl_min), + f32::max(0.1, dl_max), + f32::max(0.1, ul_min), + f32::max(0.1, ul_max), + ) + } else if site.suspended && config.uisp_integration.suspended_strategy == "slow" { + (0.1, 0.1, 0.1, 0.1) + } else { + let download_f32 = (site.max_down_mbps as f32) + * config.uisp_integration.bandwidth_overhead_factor; + let upload_f32 = (site.max_up_mbps as f32) + * config.uisp_integration.bandwidth_overhead_factor; + let download_min_f32 = + download_f32 * config.uisp_integration.commit_bandwidth_multiplier; + let upload_min_f32 = + upload_f32 * config.uisp_integration.commit_bandwidth_multiplier; + ( + f32::max(0.1, download_min_f32), + f32::max(0.1, download_f32), + f32::max(0.1, upload_min_f32), + f32::max(0.1, upload_f32), + ) + }; + let ethernet_decision = apply_ethernet_rate_cap( + &site.id, + &site.name, + site_devices.iter().copied(), + requested.0, + requested.2, + requested.1, + requested.3, + ); + if let Some(advisory) = ethernet_decision.advisory.clone() { + ethernet_advisories.push(advisory); + } for device in uisp_data.devices.iter().filter(|d| d.site_id == *site_id) { if !device.has_address() { continue; } - // Compute subscriber rates: prefer UISP QoS + burst; fallback to capacity-based - let (download_min, mut download_max, upload_min, mut upload_max) = - if let Some((dl_min, dl_max, ul_min, ul_max)) = site.burst_rates(&config) { - ( - f32::max(0.1, dl_min), - f32::max(0.1, dl_max), - f32::max(0.1, ul_min), - f32::max(0.1, ul_max), - ) - } else if site.suspended && config.uisp_integration.suspended_strategy == "slow" - { - (0.1, 0.1, 0.1, 0.1) - } else { - let download_f32 = (site.max_down_mbps as f32) - * config.uisp_integration.bandwidth_overhead_factor; - let upload_f32 = (site.max_up_mbps as f32) - * config.uisp_integration.bandwidth_overhead_factor; - let download_min_f32 = - download_f32 * config.uisp_integration.commit_bandwidth_multiplier; - let upload_min_f32 = - upload_f32 * config.uisp_integration.commit_bandwidth_multiplier; - ( - f32::max(0.1, download_min_f32), - f32::max(0.1, download_f32), - f32::max(0.1, upload_min_f32), - f32::max(0.1, upload_f32), - ) - }; - // Ensure max >= min - if download_max < download_min { - download_max = download_min; - } - if upload_max < upload_min { - upload_max = upload_min; - } - let parent_node = { if parents.contains_key(&ap_device.name) { ap_device.name.clone() @@ -453,20 +466,25 @@ pub async fn build_full_network_v2( mac: device.mac.to_owned(), ipv4: device.ipv4_list(), ipv6: device.ipv6_list(), - download_min, - upload_min, - download_max, - upload_max, + download_min: ethernet_decision.download_min, + upload_min: ethernet_decision.upload_min, + download_max: ethernet_decision.download_max, + upload_max: ethernet_decision.upload_max, comment: "".to_string(), }; - info!( + debug!( "Created shaped device for '{}' in site '{}' with parent '{}'", device.name, site.name, parent_node ); + shaped_device_count = shaped_device_count.saturating_add(1); shaped_devices.push(shaped_device); } } } + info!( + "UISP shaped devices: processed {} site/AP pair(s) and produced {} shaped device row(s).", + processed_site_pairs, shaped_device_count + ); let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); let mut writer = csv::WriterBuilder::new() .has_headers(true) @@ -481,6 +499,7 @@ pub async fn build_full_network_v2( error!("{e:?}"); UispIntegrationError::CsvError })?; + write_ethernet_advisories(&config, ðernet_advisories)?; info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); Ok(()) @@ -777,314 +796,155 @@ fn min_capacity_along_route(graph: &GraphType, path: &[NodeIndex]) -> u64 { #[derive(Debug)] struct SquashCandidate { endpoint_a: NodeIndex, + endpoint_a_id: String, endpoint_a_name: String, relay_a: NodeIndex, + relay_a_id: String, relay_a_name: String, relay_b: NodeIndex, + relay_b_id: String, relay_b_name: String, endpoint_b: NodeIndex, + endpoint_b_id: String, endpoint_b_name: String, } +impl SquashCandidate { + fn new( + graph: &GraphType, + endpoint_a: NodeIndex, + relay_a: NodeIndex, + relay_b: NodeIndex, + endpoint_b: NodeIndex, + ) -> Self { + let candidate = Self { + endpoint_a, + endpoint_a_id: graph[endpoint_a].network_json_id(), + endpoint_a_name: graph[endpoint_a].name(), + relay_a, + relay_a_id: graph[relay_a].network_json_id(), + relay_a_name: graph[relay_a].name(), + relay_b, + relay_b_id: graph[relay_b].network_json_id(), + relay_b_name: graph[relay_b].name(), + endpoint_b, + endpoint_b_id: graph[endpoint_b].network_json_id(), + endpoint_b_name: graph[endpoint_b].name(), + }; + + candidate.canonicalized() + } + + fn canonicalized(self) -> Self { + if self.reverse_key() < self.forward_key() { + self.reversed() + } else { + self + } + } + + fn canonical_key(&self) -> (String, String, String, String) { + let forward = self.forward_key(); + let reverse = self.reverse_key(); + if reverse < forward { reverse } else { forward } + } + + fn forward_key(&self) -> (String, String, String, String) { + ( + self.endpoint_a_id.clone(), + self.relay_a_id.clone(), + self.relay_b_id.clone(), + self.endpoint_b_id.clone(), + ) + } + + fn reverse_key(&self) -> (String, String, String, String) { + ( + self.endpoint_b_id.clone(), + self.relay_b_id.clone(), + self.relay_a_id.clone(), + self.endpoint_a_id.clone(), + ) + } + + fn reversed(self) -> Self { + Self { + endpoint_a: self.endpoint_b, + endpoint_a_id: self.endpoint_b_id, + endpoint_a_name: self.endpoint_b_name, + relay_a: self.relay_b, + relay_a_id: self.relay_b_id, + relay_a_name: self.relay_b_name, + relay_b: self.relay_a, + relay_b_id: self.relay_a_id, + relay_b_name: self.relay_a_name, + endpoint_b: self.endpoint_a, + endpoint_b_id: self.endpoint_a_id, + endpoint_b_name: self.endpoint_a_name, + } + } +} + +fn stable_node_key(graph: &GraphType, node: NodeIndex) -> (String, String) { + (graph[node].network_json_id(), graph[node].name()) +} + +fn sorted_unique_neighbors(graph: &GraphType, node: NodeIndex) -> Vec { + let mut neighbors: Vec = graph + .neighbors_directed(node, petgraph::Incoming) + .chain(graph.neighbors_directed(node, petgraph::Outgoing)) + .collect(); + neighbors.sort_unstable_by_key(|neighbor| stable_node_key(graph, *neighbor)); + neighbors.dedup(); + neighbors +} + +fn total_degree(graph: &GraphType, node: NodeIndex) -> usize { + graph.neighbors_directed(node, petgraph::Incoming).count() + + graph.neighbors_directed(node, petgraph::Outgoing).count() +} + +fn is_relay_node(graph: &GraphType, node: NodeIndex) -> bool { + total_degree(graph, node) == 4 && sorted_unique_neighbors(graph, node).len() == 2 +} + +fn is_meaningful_endpoint(graph: &GraphType, node: NodeIndex) -> bool { + let unique_neighbors = sorted_unique_neighbors(graph, node); + !(total_degree(graph, node) == 4 && unique_neighbors.len() == 2) + && unique_neighbors.len() >= 3 + && !matches!(graph[node], GraphMapping::AccessPoint { .. }) +} + +fn should_skip_squash( + graph: &GraphType, + config: &Arc, + chain_nodes: [NodeIndex; 4], +) -> bool { + let do_not_squash = config + .uisp_integration + .do_not_squash_sites + .clone() + .unwrap_or_default(); + + chain_nodes + .into_iter() + .map(|node| graph[node].name()) + .any(|name| do_not_squash.contains(&name)) +} + fn find_point_to_point_squash_candidates( graph: &mut GraphType, aps_with_clients: &HashSet, config: &Arc, ) { - let mut candidates = Vec::new(); - - // Find all nodes with exactly total degree 4 in bidirectional graph (2 unique neighbors) - // This accounts for bidirectional edges: A->B and B->A both exist - let relay_nodes: Vec = graph - .node_indices() - .filter(|&node| { - let incoming = graph.neighbors_directed(node, petgraph::Incoming).count(); - let outgoing = graph.neighbors_directed(node, petgraph::Outgoing).count(); - let total_degree = incoming + outgoing; - - // In bidirectional graph, relay nodes have degree 4 (2 in, 2 out) - // but only 2 unique neighbors - if total_degree == 4 { - let mut unique_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(node, petgraph::Incoming) { - unique_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(node, petgraph::Outgoing) { - unique_neighbors.insert(neighbor); - } - unique_neighbors.len() == 2 - } else { - false - } - }) - .collect(); - - // For each potential relay node, check if it's part of a 2-relay chain - for &relay_node in &relay_nodes { - // Skip if this relay node is an AP with clients - if let GraphMapping::AccessPoint { id, .. } = &graph[relay_node] - && aps_with_clients.contains(id) - { - continue; - } - // Get the unique neighbors for this relay node - let mut unique_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(relay_node, petgraph::Incoming) { - unique_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(relay_node, petgraph::Outgoing) { - unique_neighbors.insert(neighbor); - } - - if unique_neighbors.len() != 2 { - continue; - } - - let neighbors: Vec = unique_neighbors.into_iter().collect(); - let node_a = neighbors[0]; - let node_b = neighbors[1]; - - // Check if each neighbor is a relay or endpoint in bidirectional context - let is_node_a_relay = { - let mut a_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(node_a, petgraph::Incoming) { - a_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(node_a, petgraph::Outgoing) { - a_neighbors.insert(neighbor); - } - let a_incoming = graph.neighbors_directed(node_a, petgraph::Incoming).count(); - let a_outgoing = graph.neighbors_directed(node_a, petgraph::Outgoing).count(); - (a_incoming + a_outgoing) == 4 && a_neighbors.len() == 2 - }; - - let is_node_b_relay = { - let mut b_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(node_b, petgraph::Incoming) { - b_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(node_b, petgraph::Outgoing) { - b_neighbors.insert(neighbor); - } - let b_incoming = graph.neighbors_directed(node_b, petgraph::Incoming).count(); - let b_outgoing = graph.neighbors_directed(node_b, petgraph::Outgoing).count(); - (b_incoming + b_outgoing) == 4 && b_neighbors.len() == 2 - }; - - // We want exactly one of the neighbors to be a relay and one to be an endpoint - // Case 1: node_a is endpoint, node_b is relay - if !is_node_a_relay && is_node_b_relay { - // First check that node_a is a meaningful endpoint - let mut node_a_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(node_a, petgraph::Incoming) { - node_a_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(node_a, petgraph::Outgoing) { - node_a_neighbors.insert(neighbor); - } - let is_node_a_meaningful = node_a_neighbors.len() >= 3 - && !matches!(graph[node_a], GraphMapping::AccessPoint { .. }); - - if !is_node_a_meaningful { - continue; - } - - // Also verify that node_b is a pure relay (exactly 2 unique neighbors, no more) - let mut node_b_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(node_b, petgraph::Incoming) { - node_b_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(node_b, petgraph::Outgoing) { - node_b_neighbors.insert(neighbor); - } - if node_b_neighbors.len() != 2 { - continue; - } - - // Check if node_b (relay) is an AP with clients - if so, skip - if let GraphMapping::AccessPoint { id, .. } = &graph[node_b] - && aps_with_clients.contains(id) - { - continue; - } - - // Find the other neighbor of node_b (the endpoint on the far side) - let mut b_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(node_b, petgraph::Incoming) { - b_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(node_b, petgraph::Outgoing) { - b_neighbors.insert(neighbor); - } - b_neighbors.remove(&relay_node); // Remove the current relay node - - if b_neighbors.len() == 1 { - let endpoint_b = *b_neighbors.iter().next().unwrap(); - // Verify endpoint_b is not a relay - let mut endpoint_b_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(endpoint_b, petgraph::Incoming) { - endpoint_b_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(endpoint_b, petgraph::Outgoing) { - endpoint_b_neighbors.insert(neighbor); - } - let endpoint_b_degree = graph - .neighbors_directed(endpoint_b, petgraph::Incoming) - .count() - + graph - .neighbors_directed(endpoint_b, petgraph::Outgoing) - .count(); - - // Endpoint must not be a relay AND must have sufficient connections to be meaningful - // Also prefer Sites over AccessPoints as endpoints - let is_meaningful_endpoint = !(endpoint_b_degree == 4 && endpoint_b_neighbors.len() == 2) - && endpoint_b_neighbors.len() >= 3 // Must connect to at least 3 other nodes - && !matches!(graph[endpoint_b], GraphMapping::AccessPoint { .. }); // Avoid APs as endpoints - if is_meaningful_endpoint { - // Check do_not_squash_sites - skip if any node in the chain is in the exclusion list - let do_not_squash = &config - .uisp_integration - .do_not_squash_sites - .clone() - .unwrap_or_default(); - let node_names = [ - &graph[node_a].name(), - &graph[relay_node].name(), - &graph[node_b].name(), - &graph[endpoint_b].name(), - ]; - - if node_names.iter().any(|name| do_not_squash.contains(*name)) { - continue; - } - - candidates.push(SquashCandidate { - endpoint_a: node_a, - endpoint_a_name: graph[node_a].name(), - relay_a: relay_node, - relay_a_name: graph[relay_node].name(), - relay_b: node_b, - relay_b_name: graph[node_b].name(), - endpoint_b, - endpoint_b_name: graph[endpoint_b].name(), - }); - } - } - } - - // Case 2: node_a is relay, node_b is endpoint - if is_node_a_relay && !is_node_b_relay { - // First check that node_b is a meaningful endpoint - let mut node_b_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(node_b, petgraph::Incoming) { - node_b_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(node_b, petgraph::Outgoing) { - node_b_neighbors.insert(neighbor); - } - let is_node_b_meaningful = node_b_neighbors.len() >= 3 - && !matches!(graph[node_b], GraphMapping::AccessPoint { .. }); - - if !is_node_b_meaningful { - continue; - } - - // Also verify that node_a is a pure relay (exactly 2 unique neighbors, no more) - let mut node_a_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(node_a, petgraph::Incoming) { - node_a_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(node_a, petgraph::Outgoing) { - node_a_neighbors.insert(neighbor); - } - if node_a_neighbors.len() != 2 { - continue; - } - - // Check if node_a (relay) is an AP with clients - if so, skip - if let GraphMapping::AccessPoint { id, .. } = &graph[node_a] - && aps_with_clients.contains(id) - { - continue; - } - - // Find the other neighbor of node_a (the endpoint on the far side) - let mut a_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(node_a, petgraph::Incoming) { - a_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(node_a, petgraph::Outgoing) { - a_neighbors.insert(neighbor); - } - a_neighbors.remove(&relay_node); // Remove the current relay node - - if a_neighbors.len() == 1 { - let endpoint_a = *a_neighbors.iter().next().unwrap(); - // Verify endpoint_a is not a relay - let mut endpoint_a_neighbors = std::collections::HashSet::new(); - for neighbor in graph.neighbors_directed(endpoint_a, petgraph::Incoming) { - endpoint_a_neighbors.insert(neighbor); - } - for neighbor in graph.neighbors_directed(endpoint_a, petgraph::Outgoing) { - endpoint_a_neighbors.insert(neighbor); - } - let endpoint_a_degree = graph - .neighbors_directed(endpoint_a, petgraph::Incoming) - .count() - + graph - .neighbors_directed(endpoint_a, petgraph::Outgoing) - .count(); - - // Endpoint must not be a relay AND must have sufficient connections to be meaningful - // Also prefer Sites over AccessPoints as endpoints - let is_meaningful_endpoint = !(endpoint_a_degree == 4 && endpoint_a_neighbors.len() == 2) - && endpoint_a_neighbors.len() >= 3 // Must connect to at least 3 other nodes - && !matches!(graph[endpoint_a], GraphMapping::AccessPoint { .. }); // Avoid APs as endpoints - if is_meaningful_endpoint { - // Check do_not_squash_sites - skip if any node in the chain is in the exclusion list - let do_not_squash = &config - .uisp_integration - .do_not_squash_sites - .clone() - .unwrap_or_default(); - let node_names = [ - &graph[endpoint_a].name(), - &graph[node_a].name(), - &graph[relay_node].name(), - &graph[node_b].name(), - ]; - - if node_names.iter().any(|name| do_not_squash.contains(*name)) { - continue; - } - - candidates.push(SquashCandidate { - endpoint_a, - endpoint_a_name: graph[endpoint_a].name(), - relay_a: node_a, - relay_a_name: graph[node_a].name(), - relay_b: relay_node, - relay_b_name: graph[relay_node].name(), - endpoint_b: node_b, - endpoint_b_name: graph[node_b].name(), - }); - } - } - } - } - - // Remove duplicates (same chain detected from both relay nodes) - candidates.dedup_by(|a, b| { - (a.endpoint_a == b.endpoint_a && a.endpoint_b == b.endpoint_b) - || (a.endpoint_a == b.endpoint_b && a.endpoint_b == b.endpoint_a) - }); + let candidates = collect_point_to_point_squash_candidates(graph, aps_with_clients, config); info!( "Found {} point-to-point squash candidates:", candidates.len() ); for candidate in &candidates { - info!( + debug!( " {} -> {} -> {} -> {}", candidate.endpoint_a_name, candidate.relay_a_name, @@ -1097,13 +957,124 @@ fn find_point_to_point_squash_candidates( perform_squashing(graph, &candidates); } +fn collect_point_to_point_squash_candidates( + graph: &GraphType, + aps_with_clients: &HashSet, + config: &Arc, +) -> Vec { + let mut candidates = Vec::new(); + + // Find all nodes with exactly total degree 4 in bidirectional graph (2 unique neighbors) + // This accounts for bidirectional edges: A->B and B->A both exist + let mut relay_nodes: Vec = graph + .node_indices() + .filter(|&node| is_relay_node(graph, node)) + .collect(); + relay_nodes.sort_unstable_by_key(|node| stable_node_key(graph, *node)); + + // For each potential relay node, check if it's part of a 2-relay chain + for &relay_node in &relay_nodes { + // Skip if this relay node is an AP with clients + if let GraphMapping::AccessPoint { id, .. } = &graph[relay_node] + && aps_with_clients.contains(id) + { + continue; + } + let neighbors = sorted_unique_neighbors(graph, relay_node); + if neighbors.len() != 2 { + continue; + } + let node_a = neighbors[0]; + let node_b = neighbors[1]; + + // Check if each neighbor is a relay or endpoint in bidirectional context + let is_node_a_relay = is_relay_node(graph, node_a); + let is_node_b_relay = is_relay_node(graph, node_b); + + // We want exactly one of the neighbors to be a relay and one to be an endpoint + // Case 1: node_a is endpoint, node_b is relay + if !is_node_a_relay && is_node_b_relay { + if !is_meaningful_endpoint(graph, node_a) { + continue; + } + + if sorted_unique_neighbors(graph, node_b).len() != 2 { + continue; + } + + // Check if node_b (relay) is an AP with clients - if so, skip + if let GraphMapping::AccessPoint { id, .. } = &graph[node_b] + && aps_with_clients.contains(id) + { + continue; + } + + // Find the other neighbor of node_b (the endpoint on the far side) + let candidate_neighbors: Vec = sorted_unique_neighbors(graph, node_b) + .into_iter() + .filter(|neighbor| *neighbor != relay_node) + .collect(); + + if candidate_neighbors.len() == 1 { + let endpoint_b = candidate_neighbors[0]; + if is_meaningful_endpoint(graph, endpoint_b) + && !should_skip_squash(graph, config, [node_a, relay_node, node_b, endpoint_b]) + { + candidates.push(SquashCandidate::new( + graph, node_a, relay_node, node_b, endpoint_b, + )); + } + } + } + + // Case 2: node_a is relay, node_b is endpoint + if is_node_a_relay && !is_node_b_relay { + if !is_meaningful_endpoint(graph, node_b) { + continue; + } + + if sorted_unique_neighbors(graph, node_a).len() != 2 { + continue; + } + + // Check if node_a (relay) is an AP with clients - if so, skip + if let GraphMapping::AccessPoint { id, .. } = &graph[node_a] + && aps_with_clients.contains(id) + { + continue; + } + + // Find the other neighbor of node_a (the endpoint on the far side) + let candidate_neighbors: Vec = sorted_unique_neighbors(graph, node_a) + .into_iter() + .filter(|neighbor| *neighbor != relay_node) + .collect(); + + if candidate_neighbors.len() == 1 { + let endpoint_a = candidate_neighbors[0]; + if is_meaningful_endpoint(graph, endpoint_a) + && !should_skip_squash(graph, config, [endpoint_a, node_a, relay_node, node_b]) + { + candidates.push(SquashCandidate::new( + graph, endpoint_a, node_a, relay_node, node_b, + )); + } + } + } + } + + candidates.sort_unstable_by_key(SquashCandidate::canonical_key); + candidates.dedup_by(|a, b| a.canonical_key() == b.canonical_key()); + candidates +} + fn perform_squashing(graph: &mut GraphType, candidates: &[SquashCandidate]) { let mut nodes_to_remove = std::collections::HashSet::new(); info!("Performing squashing on {} candidates...", candidates.len()); for candidate in candidates { - info!( + debug!( "Squashing: {} -> {} -> {} -> {}", candidate.endpoint_a_name, candidate.relay_a_name, @@ -1124,7 +1095,7 @@ fn perform_squashing(graph: &mut GraphType, candidates: &[SquashCandidate]) { if graph.node_weight(candidate.endpoint_a).is_none() || graph.node_weight(candidate.endpoint_b).is_none() { - info!(" Skipping - endpoints no longer exist"); + debug!(" Skipping - endpoints no longer exist"); continue; } @@ -1133,7 +1104,7 @@ fn perform_squashing(graph: &mut GraphType, candidates: &[SquashCandidate]) { .find_edge(candidate.endpoint_a, candidate.endpoint_b) .is_some() { - info!(" Skipping - direct edge already exists"); + debug!(" Skipping - direct edge already exists"); continue; } @@ -1153,7 +1124,7 @@ fn perform_squashing(graph: &mut GraphType, candidates: &[SquashCandidate]) { nodes_to_remove.insert(candidate.relay_a); nodes_to_remove.insert(candidate.relay_b); - info!( + debug!( " Created direct link with {}/{}Mbps capacity (forward/reverse)", forward_capacity, reverse_capacity ); @@ -1161,11 +1132,13 @@ fn perform_squashing(graph: &mut GraphType, candidates: &[SquashCandidate]) { // Remove all relay nodes that are no longer needed let removed_count = nodes_to_remove.len(); + let mut nodes_to_remove: Vec = nodes_to_remove.into_iter().collect(); + nodes_to_remove.sort_unstable_by_key(|node| std::cmp::Reverse(node.index())); for node_to_remove in nodes_to_remove { if graph.node_weight(node_to_remove).is_some() { let node_name = graph[node_to_remove].name(); graph.remove_node(node_to_remove); - info!("Removed relay node: {}", node_name); + debug!("Removed relay node: {}", node_name); } } @@ -1212,3 +1185,163 @@ fn calculate_chain_capacity(graph: &GraphType, chain_nodes: &[NodeIndex]) -> (u6 (forward_capacity, reverse_capacity) } + +#[cfg(test)] +mod tests { + use super::*; + + fn add_site(graph: &mut GraphType, name: &str, id: &str) -> NodeIndex { + graph.add_node(GraphMapping::Site { + name: name.to_string(), + id: id.to_string(), + latitude: None, + longitude: None, + }) + } + + fn add_ap(graph: &mut GraphType, name: &str, id: &str, site_name: &str) -> NodeIndex { + graph.add_node(GraphMapping::AccessPoint { + name: name.to_string(), + id: id.to_string(), + site_name: site_name.to_string(), + download_mbps: 1000, + upload_mbps: 1000, + }) + } + + fn add_bidirectional_edge(graph: &mut GraphType, a: NodeIndex, b: NodeIndex, speed_mbps: u64) { + graph.add_edge(a, b, LinkMapping::ethernet(speed_mbps)); + graph.add_edge(b, a, LinkMapping::ethernet(speed_mbps)); + } + + fn build_point_to_point_graph(reverse_insertion: bool) -> GraphType { + let mut graph = GraphType::new(); + + let mut add_node = |label: &str| match label { + "left_endpoint" => add_site(&mut graph, "Left Endpoint", "site-left"), + "right_endpoint" => add_site(&mut graph, "Right Endpoint", "site-right"), + "left_extra_a" => add_site(&mut graph, "Left Extra A", "site-left-extra-a"), + "left_extra_b" => add_site(&mut graph, "Left Extra B", "site-left-extra-b"), + "right_extra_a" => add_site(&mut graph, "Right Extra A", "site-right-extra-a"), + "right_extra_b" => add_site(&mut graph, "Right Extra B", "site-right-extra-b"), + "relay_a" => add_ap(&mut graph, "Relay A", "device-relay-a", "Left Endpoint"), + "relay_b" => add_ap(&mut graph, "Relay B", "device-relay-b", "Right Endpoint"), + _ => unreachable!("unknown node label"), + }; + + let ordered_labels = if reverse_insertion { + vec![ + "right_extra_b", + "right_extra_a", + "relay_b", + "right_endpoint", + "left_extra_b", + "left_extra_a", + "relay_a", + "left_endpoint", + ] + } else { + vec![ + "left_endpoint", + "relay_a", + "left_extra_a", + "left_extra_b", + "right_endpoint", + "relay_b", + "right_extra_a", + "right_extra_b", + ] + }; + + let mut nodes = HashMap::new(); + for label in ordered_labels { + nodes.insert(label, add_node(label)); + } + + let mut edges = vec![ + ("left_endpoint", "relay_a", 500), + ("relay_a", "relay_b", 400), + ("relay_b", "right_endpoint", 300), + ("left_endpoint", "left_extra_a", 100), + ("left_endpoint", "left_extra_b", 100), + ("right_endpoint", "right_extra_a", 100), + ("right_endpoint", "right_extra_b", 100), + ]; + if reverse_insertion { + edges.reverse(); + } + + for (from, to, speed) in edges { + add_bidirectional_edge(&mut graph, nodes[from], nodes[to], speed); + } + + graph + } + + fn graph_node_ids(graph: &GraphType) -> Vec { + let mut ids: Vec = graph + .node_indices() + .map(|node| graph[node].network_json_id()) + .collect(); + ids.sort(); + ids + } + + fn find_node_by_id(graph: &GraphType, id: &str) -> NodeIndex { + graph + .node_indices() + .find(|node| graph[*node].network_json_id() == id) + .expect("expected node to exist") + } + + #[test] + fn squash_candidate_collection_is_deterministic() { + let config = Arc::new(Config::default()); + let aps_with_clients = HashSet::new(); + + let graph_a = build_point_to_point_graph(false); + let graph_b = build_point_to_point_graph(true); + + let candidates_a = + collect_point_to_point_squash_candidates(&graph_a, &aps_with_clients, &config); + let candidates_b = + collect_point_to_point_squash_candidates(&graph_b, &aps_with_clients, &config); + + assert_eq!(candidates_a.len(), 1); + assert_eq!(candidates_b.len(), 1); + assert_eq!( + candidates_a[0].canonical_key(), + candidates_b[0].canonical_key() + ); + assert_eq!(candidates_a[0].endpoint_a_id, "uisp:site:site-left"); + assert_eq!(candidates_a[0].relay_a_id, "uisp:device:device-relay-a"); + assert_eq!(candidates_a[0].relay_b_id, "uisp:device:device-relay-b"); + assert_eq!(candidates_a[0].endpoint_b_id, "uisp:site:site-right"); + } + + #[test] + fn squashing_result_is_stable_across_insertion_order() { + let config = Arc::new(Config::default()); + let aps_with_clients = HashSet::new(); + + let mut graph_a = build_point_to_point_graph(false); + let mut graph_b = build_point_to_point_graph(true); + + find_point_to_point_squash_candidates(&mut graph_a, &aps_with_clients, &config); + find_point_to_point_squash_candidates(&mut graph_b, &aps_with_clients, &config); + + assert_eq!(graph_node_ids(&graph_a), graph_node_ids(&graph_b)); + assert!(!graph_node_ids(&graph_a).contains(&"uisp:device:device-relay-a".to_string())); + assert!(!graph_node_ids(&graph_a).contains(&"uisp:device:device-relay-b".to_string())); + + let left_a = find_node_by_id(&graph_a, "uisp:site:site-left"); + let right_a = find_node_by_id(&graph_a, "uisp:site:site-right"); + let left_b = find_node_by_id(&graph_b, "uisp:site:site-left"); + let right_b = find_node_by_id(&graph_b, "uisp:site:site-right"); + + assert!(graph_a.find_edge(left_a, right_a).is_some()); + assert!(graph_a.find_edge(right_a, left_a).is_some()); + assert!(graph_b.find_edge(left_b, right_b).is_some()); + assert!(graph_b.find_edge(right_b, left_b).is_some()); + } +} diff --git a/src/rust/uisp_integration/src/strategies/full2/graph_mapping.rs b/src/rust/uisp_integration/src/strategies/full2/graph_mapping.rs index b6ae24cd..76f0d7cc 100644 --- a/src/rust/uisp_integration/src/strategies/full2/graph_mapping.rs +++ b/src/rust/uisp_integration/src/strategies/full2/graph_mapping.rs @@ -42,7 +42,10 @@ impl GraphMapping { format!("uisp:site:{id}") } GraphMapping::GeneratedSite { name } => { - format!("libreqos:generated:uisp:site:{}", slugify_generated_name(name)) + format!( + "libreqos:generated:uisp:site:{}", + slugify_generated_name(name) + ) } GraphMapping::AccessPoint { id, .. } => format!("uisp:device:{id}"), } diff --git a/src/rust/uisp_integration/src/uisp_types/uisp_device.rs b/src/rust/uisp_integration/src/uisp_types/uisp_device.rs index 75aa745e..ed1e04d2 100644 --- a/src/rust/uisp_integration/src/uisp_types/uisp_device.rs +++ b/src/rust/uisp_integration/src/uisp_types/uisp_device.rs @@ -15,14 +15,137 @@ pub struct UispDevice { pub id: String, pub name: String, pub mac: String, + pub role: Option, + pub wireless_mode: Option, pub site_id: String, pub download: u64, pub upload: u64, pub ipv4: HashSet, pub ipv6: HashSet, + pub negotiated_ethernet_mbps: Option, + pub negotiated_ethernet_interface: Option, } impl UispDevice { + fn parse_speed_mbps(raw: &str) -> Option { + let lower = raw.trim().to_lowercase(); + if lower.is_empty() { + return None; + } + + let digits: String = lower + .chars() + .take_while(|ch| ch.is_ascii_digit() || *ch == '.') + .collect(); + if digits.is_empty() { + return None; + } + + let value = digits.parse::().ok()?; + if !(value.is_finite() && value > 0.0) { + return None; + } + + let multiplier = if lower.contains("gbps") || lower.ends_with('g') { + 1000.0 + } else { + 1.0 + }; + + Some((value * multiplier).round() as u64) + } + + fn negotiated_ethernet_from_device(device: &Device) -> (Option, Option) { + let mut best: Option<(u64, Option)> = None; + let Some(interfaces) = &device.interfaces else { + return (None, None); + }; + + for interface in interfaces { + let interface_type = interface + .identification + .as_ref() + .and_then(|id| id.r#type.clone()) + .unwrap_or_default() + .to_lowercase(); + if matches!(interface_type.as_str(), "wlan" | "wifi" | "wireless") { + continue; + } + let Some(status) = &interface.status else { + continue; + }; + let speed_raw = if let Some(current_speed) = status.currentSpeed.as_deref() { + Some(current_speed) + } else if Self::allows_plain_speed(status.status.as_deref()) { + status.speed.as_deref() + } else { + None + }; + let Some(speed_raw) = speed_raw else { + continue; + }; + let Some(speed_mbps) = Self::parse_speed_mbps(speed_raw) else { + continue; + }; + let iface_name = interface + .identification + .as_ref() + .and_then(|id| id.name.clone()); + + if best + .as_ref() + .is_none_or(|(current_speed, _)| speed_mbps < *current_speed) + { + best = Some((speed_mbps, iface_name)); + } + } + + best.map_or((None, None), |(speed, iface)| (Some(speed), iface)) + } + + fn allows_plain_speed(status: Option<&str>) -> bool { + let Some(status) = status else { + return false; + }; + matches!( + status.trim().to_ascii_lowercase().as_str(), + "connected" | "up" | "active" + ) + } + + pub(crate) fn is_wireless_station_cpe(&self) -> bool { + let role = self + .role + .as_deref() + .map(str::trim) + .map(str::to_ascii_lowercase); + if matches!(role.as_deref(), Some("station" | "sta" | "cpe")) { + return true; + } + + let wireless_mode = self + .wireless_mode + .as_deref() + .map(str::trim) + .map(str::to_ascii_lowercase); + matches!( + wireless_mode.as_deref(), + Some(mode) if mode.starts_with("sta") || mode.starts_with("station") + ) + } + + pub(crate) fn is_router_like(&self) -> bool { + let role = self + .role + .as_deref() + .map(str::trim) + .map(str::to_ascii_lowercase); + matches!( + role.as_deref(), + Some("router" | "homewifi" | "home-wifi" | "home wifi") + ) + } + /// Check if an IP/CIDR represents a network address (all host bits are zero) /// Returns true if it's a network address, false if it's a host address #[cfg_attr(test, allow(dead_code))] @@ -209,16 +332,25 @@ impl UispDevice { site_id = exception.parent.clone(); } } + let (negotiated_ethernet_mbps, negotiated_ethernet_interface) = + Self::negotiated_ethernet_from_device(device); Self { id: device.get_id(), name: device.get_name().unwrap_or_default(), mac, + role: device.identification.role.clone(), + wireless_mode: device + .overview + .as_ref() + .and_then(|o| o.wirelessMode.clone()), site_id, upload, download, ipv4, ipv6, + negotiated_ethernet_mbps, + negotiated_ethernet_interface, } } @@ -269,6 +401,8 @@ impl UispDevice { #[cfg(test)] mod tests { use super::UispDevice; + use serde_json::json; + use uisp::Device; #[test] fn test_is_network_address() { @@ -302,4 +436,74 @@ mod tests { assert!(!UispDevice::is_network_address("192.168.1.0/invalid")); // Invalid prefix assert!(!UispDevice::is_network_address("192.168.1.0/33")); // Invalid IPv4 prefix length } + + #[test] + fn test_parse_speed_mbps() { + assert_eq!(UispDevice::parse_speed_mbps("100Mbps-Full"), Some(100)); + assert_eq!(UispDevice::parse_speed_mbps("1000"), Some(1000)); + assert_eq!(UispDevice::parse_speed_mbps("1Gbps"), Some(1000)); + assert_eq!(UispDevice::parse_speed_mbps("2.5Gbps"), Some(2500)); + assert_eq!(UispDevice::parse_speed_mbps(""), None); + assert_eq!(UispDevice::parse_speed_mbps("unknown"), None); + } + + fn mk_device_with_interfaces(interfaces: serde_json::Value) -> Device { + serde_json::from_value(json!({ + "identification": { + "id": "dev-1", + "hostname": "dev-1", + "role": null, + }, + "overview": { + "wirelessMode": null + }, + "interfaces": interfaces, + })) + .expect("device JSON must deserialize") + } + + #[test] + fn disconnected_plain_speed_is_ignored_for_negotiated_ethernet() { + let device = mk_device_with_interfaces(json!([ + { + "identification": { "name": "eth0@1", "type": "eth" }, + "status": { "status": "disconnected", "speed": "10-half", "currentSpeed": null }, + "wireless": {} + } + ])); + + let (speed, iface) = UispDevice::negotiated_ethernet_from_device(&device); + assert_eq!(speed, None); + assert_eq!(iface, None); + } + + #[test] + fn disconnected_currentspeed_is_still_accepted() { + let device = mk_device_with_interfaces(json!([ + { + "identification": { "name": "data", "type": "eth" }, + "status": { "status": "disconnected", "speed": "auto", "currentSpeed": "100-full" }, + "wireless": {} + } + ])); + + let (speed, iface) = UispDevice::negotiated_ethernet_from_device(&device); + assert_eq!(speed, Some(100)); + assert_eq!(iface.as_deref(), Some("data")); + } + + #[test] + fn connected_plain_speed_is_accepted() { + let device = mk_device_with_interfaces(json!([ + { + "identification": { "name": "eth0", "type": "eth" }, + "status": { "status": "connected", "speed": "1000-full", "currentSpeed": null }, + "wireless": {} + } + ])); + + let (speed, iface) = UispDevice::negotiated_ethernet_from_device(&device); + assert_eq!(speed, Some(1000)); + assert_eq!(iface.as_deref(), Some("eth0")); + } } diff --git a/src/rust/uisp_integration/src/uisp_types/uisp_site.rs b/src/rust/uisp_integration/src/uisp_types/uisp_site.rs index 85458f50..2e3643e2 100644 --- a/src/rust/uisp_integration/src/uisp_types/uisp_site.rs +++ b/src/rust/uisp_integration/src/uisp_types/uisp_site.rs @@ -78,7 +78,12 @@ impl UispSite { .description .as_ref() .and_then(|description| description.location.as_ref()) - .map(|location| (Some(location.latitude as f32), Some(location.longitude as f32))) + .map(|location| { + ( + Some(location.latitude as f32), + Some(location.longitude as f32), + ) + }) .unwrap_or((None, None)); // Extract UISP QoS base speeds (bps -> Mbps) and burst sizes (kB/s -> Mbps) let mut base_down_mbps: f32 = 0.0; diff --git a/src/scheduler.py b/src/scheduler.py index 7308d5e0..bdb90435 100644 --- a/src/scheduler.py +++ b/src/scheduler.py @@ -7,13 +7,15 @@ import chardet from LibreQoS import refreshShapers, refreshShapersUpdateOnly import subprocess import sys +import tempfile from io import StringIO from liblqos_python import automatic_import_uisp, automatic_import_splynx, queue_refresh_interval_mins, \ automatic_import_powercode, automatic_import_sonar, influx_db_enabled, get_libreqos_directory, \ blackboard_finish, blackboard_submit, automatic_import_wispgate, enable_insight_topology, insight_topology_role, \ - automatic_import_netzur, automatic_import_visp, calculate_hash, efficiency_core_ids, scheduler_alive, scheduler_error, \ - overrides_persistent_devices_effective, overrides_circuit_adjustments_effective, overrides_network_adjustments_effective, \ - scheduler_output + automatic_import_netzur, automatic_import_visp, calculate_shaping_runtime_hash, efficiency_core_ids, scheduler_alive, scheduler_error, \ + overrides_persistent_devices_materialized, overrides_circuit_adjustments_materialized, \ + overrides_network_adjustments_materialized, \ + scheduler_output, wait_for_bus_ready from apscheduler.schedulers.background import BlockingScheduler from apscheduler.executors.pool import ThreadPoolExecutor @@ -21,7 +23,9 @@ import os.path import os ads = BlockingScheduler(executors={'default': ThreadPoolExecutor(1)}) -network_hash = 0 +shaping_runtime_hash = 0 +INTEGRATION_FAILURE_PREVIEW_LINES = 30 +INTEGRATION_FAILURE_PREVIEW_CHARS = 4000 def clear_scheduler_error(): @@ -34,6 +38,82 @@ def clear_scheduler_output(): scheduler_output("") +def _integration_output_lines(output): + normalized = (output or "").replace("\r\n", "\n").strip() + if not normalized: + return [] + return normalized.split("\n") + + +def _summarize_output_preview(output, *, max_lines, max_chars): + lines = _integration_output_lines(output) + if not lines: + return "" + + preview_lines = lines[:max_lines] + preview = "\n".join(preview_lines) + truncated = len(lines) > max_lines + if len(preview) > max_chars: + preview = preview[:max_chars].rstrip() + truncated = True + if truncated: + preview += "\n..." + return preview + + +def _sanitize_label_for_filename(label): + token = "".join(ch.lower() if ch.isalnum() else "_" for ch in (label or "integration")) + token = token.strip("_") + return token or "integration" + + +def _write_integration_output_artifact(label, output): + normalized = (output or "").replace("\r\n", "\n").strip() + if not normalized: + return None + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + artifact = os.path.join( + tempfile.gettempdir(), + f"lqos_scheduler_{_sanitize_label_for_filename(label)}_{timestamp}.log", + ) + try: + with open(artifact, "w", encoding="utf-8") as handle: + handle.write(normalized) + handle.write("\n") + return artifact + except Exception as e: + print(f"Failed to write {label} output artifact: {e}") + return None + + +def _publish_integration_result(label, result): + output = ((result.stdout or "") + (result.stderr or "")).replace("\r\n", "\n").strip() + if result.returncode == 0: + line_count = len(_integration_output_lines(output)) + summary = ( + f"{label} completed successfully." + if line_count == 0 + else f"{label} completed successfully. Captured {line_count} line(s) of output." + ) + print(summary) + scheduler_output(summary) + return + + preview = _summarize_output_preview( + output, + max_lines=INTEGRATION_FAILURE_PREVIEW_LINES, + max_chars=INTEGRATION_FAILURE_PREVIEW_CHARS, + ) + artifact = _write_integration_output_artifact(label, output) + message = f"{label} exited with code {result.returncode}. Continuing." + if preview: + message += f"\nOutput preview:\n{preview}" + if artifact is not None: + message += f"\nFull output saved to {artifact}" + print(message) + scheduler_error(message) + + def get_integration_affinity_cpus(): """Return efficiency-core CPU IDs to prefer for integration subprocesses.""" try: @@ -118,8 +198,19 @@ def capture_output_and_run(func): sys.stderr = old_stderr output = captured_output.getvalue() if output: - print(output) - scheduler_output(output) + preview = _summarize_output_preview( + output, + max_lines=INTEGRATION_FAILURE_PREVIEW_LINES, + max_chars=INTEGRATION_FAILURE_PREVIEW_CHARS, + ) + artifact = _write_integration_output_artifact("captured_integration", output) + message = "Captured integration output." + if preview: + message += f"\nOutput preview:\n{preview}" + if artifact is not None: + message += f"\nFull output saved to {artifact}" + print(message) + scheduler_output(message) def run_python_integration(module_name: str, func_name: str, label: str = ""): @@ -137,15 +228,7 @@ def run_python_integration(module_name: str, func_name: str, label: str = ""): text=True, label=friendly, ) - output = (result.stdout or "") + (result.stderr or "") - if output: - print(output) - scheduler_output(output) - if result.returncode != 0: - # Non-zero exit shouldn't stop scheduling; log and continue - msg = f"Integration {friendly} exited with code {result.returncode}. Continuing." - print(msg) - scheduler_error(msg) + _publish_integration_result(friendly, result) except Exception as e: err = f"Failed to invoke integration {label or (module_name + '.' + func_name)}: {e}" print(err) @@ -165,14 +248,7 @@ def importFromCRM(): text=True, label="UISP integration", ) - output = (result.stdout or "") + (result.stderr or "") - if output: - print(output) - scheduler_output(output) - if result.returncode != 0: - msg = f"UISP integration exited with code {result.returncode}. Continuing." - print(msg) - scheduler_error(msg) + _publish_integration_result("UISP integration", result) blackboard_finish() except Exception as e: error_msg = f"Failed to run UISP integration: {str(e)}" @@ -224,6 +300,10 @@ SHAPED_DEVICES_HEADER = [ "Download Max Mbps", "Upload Max Mbps", "Comment", +] + +SHAPED_DEVICES_HEADER_WITH_SQM = [ + *SHAPED_DEVICES_HEADER, "sqm", ] @@ -272,8 +352,8 @@ def read_shaped_devices_csv(path: str): return header, data_rows -def override_devices_to_rows(devices): - """Convert override device dicts to CSV rows (14 columns, optional sqm).""" +def override_devices_to_rows(devices, include_sqm=False): + """Convert override device dicts to CSV rows, preserving the existing CSV shape by default.""" rows = [] for d in devices: ipv4s = d.get('ipv4s', []) @@ -294,8 +374,9 @@ def override_devices_to_rows(devices): str(d.get('maxDownload', '')), str(d.get('maxUpload', '')), d.get('comment', ''), - str(sqm), ] + if include_sqm: + row.append(str(sqm)) rows.append(row) return rows @@ -328,6 +409,19 @@ def write_shaped_devices_csv(path: str, header, rows): writer.writerows(rows) +def header_has_sqm(header): + return len(header) > 13 and header[13].strip().lower() == "sqm" + + +def operator_requires_sqm_column(devices, adjustments): + if any(((d.get('sqm', '') or d.get('sqm_override', '') or '').strip()) for d in devices or []): + return True + return any( + adj.get('type') == 'device_adjust_sqm' and (adj.get('sqm_override') or '').strip() + for adj in adjustments or [] + ) + + def apply_lqos_overrides(): """Load ShapedDevices.csv, apply persistent devices and circuit adjustments, and save back.""" path = shaped_devices_csv_path() @@ -335,23 +429,31 @@ def apply_lqos_overrides(): # 1) Persistent devices: replace by device_id or append try: - extra = overrides_persistent_devices_effective() + extra = overrides_persistent_devices_materialized() except Exception as e: # Persistent device overrides are optional. Keep the scheduler healthy # and continue applying the rest of the override sources if this loader # is unavailable or temporarily broken. print(f"Skipping persistent device overrides: {e}") extra = [] - override_rows = override_devices_to_rows(extra or []) - merged_rows, changed = merge_rows_replace_by_device_id(rows, override_rows) # 2) Circuit adjustments: speed changes, removals, reparenting try: - adjustments = overrides_circuit_adjustments_effective() + adjustments = overrides_circuit_adjustments_materialized() except Exception as e: print(f"Failed to read circuit adjustments: {e}") adjustments = [] + need_sqm_column = header_has_sqm(header) or operator_requires_sqm_column(extra, adjustments) + if need_sqm_column and not header_has_sqm(header): + header = list(header) + ["sqm"] + for row in rows: + if len(row) < len(header): + row.extend([""] * (len(header) - len(row))) + + override_rows = override_devices_to_rows(extra or [], include_sqm=need_sqm_column) + merged_rows, changed = merge_rows_replace_by_device_id(rows, override_rows) + def set_if_some(value_opt, current_str): if value_opt is None: return current_str @@ -387,6 +489,8 @@ def apply_lqos_overrides(): r[11] = set_if_some(adj.get('max_upload_bandwidth'), r[11] if len(r) > 11 else '') changed = True elif t == 'device_adjust_sqm': + if not need_sqm_column: + continue did = adj.get('device_id', '') sqm_override = (adj.get('sqm_override') or '').strip() for r in merged_rows: @@ -416,7 +520,8 @@ def apply_lqos_overrides(): changed = True if changed: - write_shaped_devices_csv(path, header if header else SHAPED_DEVICES_HEADER, merged_rows) + final_header = header if header else (SHAPED_DEVICES_HEADER_WITH_SQM if need_sqm_column else SHAPED_DEVICES_HEADER) + write_shaped_devices_csv(path, final_header, merged_rows) print("Updated ShapedDevices.csv with overrides") # 3) Load, adjust, and optionally save network.json @@ -452,13 +557,20 @@ def load_network_json(path: str): def apply_network_adjustments(network: dict) -> bool: """Apply network adjustments from overrides to the network JSON structure. - Currently supports: adjust_site_speed (by site_name) updating - downloadBandwidthMbps and uploadBandwidthMbps at the matching node, and - set_node_virtual (by node_name) updating the boolean 'virtual' flag. + Currently supports: adjust_site_speed (preferring node_id, with legacy + site_name fallback) updating downloadBandwidthMbps and + uploadBandwidthMbps at the matching node, and set_node_virtual (by + node_name) updating the boolean 'virtual' flag for operator-authored + topology overrides. + + This path intentionally excludes runtime automation changes such as + StormGuard adaptive site-speed overrides and TreeGuard virtual-node + decisions so they do not overwrite the operator-authored `network.json` + source of truth. Returns True if any changes were applied. """ try: - adjustments = overrides_network_adjustments_effective() + adjustments = overrides_network_adjustments_materialized() except Exception as e: print(f"Failed to read network adjustments: {e}") return False @@ -466,27 +578,40 @@ def apply_network_adjustments(network: dict) -> bool: if not adjustments: return False - def adjust_node(tree: dict, site: str, dl_opt, ul_opt) -> bool: + def normalize_bandwidth_value(value): + numeric = float(value) + if numeric.is_integer(): + return int(numeric) + return numeric + + def adjust_node(tree: dict, site: str, node_id, dl_opt, ul_opt) -> bool: changed_local = False for key in list(tree.keys()): if key == 'children': child = tree.get('children') if isinstance(child, dict): - if adjust_node(child, site, dl_opt, ul_opt): + if adjust_node(child, site, node_id, dl_opt, ul_opt): changed_local = True continue node = tree.get(key) if isinstance(node, dict): - if key == site: + current_node_id = node.get('id') + matches_target = False + if node_id: + matches_target = current_node_id == node_id + elif key == site: + matches_target = True + + if matches_target: if dl_opt is not None: - node['downloadBandwidthMbps'] = int(dl_opt) + node['downloadBandwidthMbps'] = normalize_bandwidth_value(dl_opt) changed_local = True if ul_opt is not None: - node['uploadBandwidthMbps'] = int(ul_opt) + node['uploadBandwidthMbps'] = normalize_bandwidth_value(ul_opt) changed_local = True # Recurse into children if 'children' in node and isinstance(node['children'], dict): - if adjust_node(node['children'], site, dl_opt, ul_opt): + if adjust_node(node['children'], site, node_id, dl_opt, ul_opt): changed_local = True return changed_local @@ -516,10 +641,11 @@ def apply_network_adjustments(network: dict) -> bool: for adj in adjustments: if adj.get('type') == 'adjust_site_speed': site = adj.get('site_name', '') + node_id = adj.get('node_id', None) dl = adj.get('download_bandwidth_mbps', None) ul = adj.get('upload_bandwidth_mbps', None) - if site: - if adjust_node(network, site, dl, ul): + if site or node_id: + if adjust_node(network, site, node_id, dl, ul): net_changed = True elif adj.get('type') == 'set_node_virtual': node_name = adj.get('node_name', '') @@ -543,26 +669,31 @@ def importAndShapeFullReload(): def importAndShapePartialReload(): - global network_hash + global shaping_runtime_hash importFromCRM() - # Calculate if the network.json or ShapedDevices.csv has changed and reload only if it has. - new_hash = calculate_hash() - if new_hash != network_hash: - refreshShapersUpdateOnly() - network_hash = new_hash - else: - print("No changes detected in network.json or ShapedDevices.csv, skipping shaper refresh.") + # Rebuild when runtime shaping inputs change, including effective adaptive + # circuit overrides that do not belong in source-of-truth files. + new_hash = calculate_shaping_runtime_hash() + if new_hash != shaping_runtime_hash: + refreshShapers() + shaping_runtime_hash = calculate_shaping_runtime_hash() def not_dead_yet(): #print(f"Scheduler alive at {datetime.datetime.now()}") scheduler_alive() + +def ensure_bus_ready(): + """Wait briefly for lqosd to finish binding the local bus socket.""" + wait_for_bus_ready(5000) + if __name__ == '__main__': try: + ensure_bus_ready() importAndShapeFullReload() - network_hash = calculate_hash() + shaping_runtime_hash = calculate_shaping_runtime_hash() print("Starting scheduler with jobs:") print(f"- not_dead_yet every 1 minute") diff --git a/src/systemd_hotfix.sh b/src/systemd_hotfix.sh index bff34887..cc5f3c57 100755 --- a/src/systemd_hotfix.sh +++ b/src/systemd_hotfix.sh @@ -2,12 +2,19 @@ set -euo pipefail -HOTFIX_BASE_URL="https://download.libreqos.com" -HOTFIX_PACKAGE_VERSION="${HOTFIX_PACKAGE_VERSION:-255.4-1ubuntu8.12+libreqos1}" +HOTFIX_REPO_URL="${HOTFIX_REPO_URL:-https://repo.libreqos.com}" +HOTFIX_REPO_DIST="${HOTFIX_REPO_DIST:-noble}" +HOTFIX_REPO_COMPONENT="${HOTFIX_REPO_COMPONENT:-main}" +HOTFIX_KEY_URL="${HOTFIX_KEY_URL:-${HOTFIX_REPO_URL}/keys/libreqos-archive-keyring.gpg}" +HOTFIX_KEYRING_PATH="${HOTFIX_KEYRING_PATH:-/usr/share/keyrings/libreqos-archive-keyring.gpg}" +HOTFIX_APT_SOURCE_PATH="${HOTFIX_APT_SOURCE_PATH:-/etc/apt/sources.list.d/libreqos-systemd-hotfix.list}" +HOTFIX_APT_PREFERENCES_PATH="${HOTFIX_APT_PREFERENCES_PATH:-/etc/apt/preferences.d/libreqos-systemd-hotfix}" +HOTFIX_APT_PIN_ORIGIN="${HOTFIX_APT_PIN_ORIGIN:-LibreQoS}" +HOTFIX_APT_PIN_LABEL="${HOTFIX_APT_PIN_LABEL:-LibreQoS}" +HOTFIX_PACKAGE_VERSION="${HOTFIX_PACKAGE_VERSION:-255.4-1ubuntu8.14+libreqos2}" SUPPORTED_UBUNTU_SYSTEMD_VERSION_GLOBS="${SUPPORTED_UBUNTU_SYSTEMD_VERSION_GLOBS:-255.4-1ubuntu8 255.4-1ubuntu8.*}" HOTFIX_MARKER="${HOTFIX_MARKER:-/opt/libreqos/src/.systemd_hotfix_installed}" HOTFIX_SKIP_REBOOT_PROMPT="${HOTFIX_SKIP_REBOOT_PROMPT:-0}" -ARCH="${ARCH:-amd64}" HOTFIX_CORE_PACKAGES=( "libsystemd0" @@ -35,17 +42,26 @@ Usage: $0 Commands: status Show whether this host should be offered the Noble systemd hotfix should-offer Exit 0 when the hotfix should be offered on this host - download Download the hotfix bundle into a temporary directory - install Download and install the hotfix bundle, then offer to schedule a reboot - packages Print the package filenames expected for this host - urls Print the expected package URLs for this host + bootstrap Configure the LibreQoS APT repo and package pin for this host + download Download the hotfix package set into a temporary directory via APT + install Configure the LibreQoS APT repo and install the hotfix package set + packages Print the package names managed by the hotfix on this host + urls Print the repo bootstrap URLs used for this host Environment: - HOTFIX_PACKAGE_VERSION Backported package version suffix + HOTFIX_REPO_URL LibreQoS APT repository URL + HOTFIX_REPO_DIST APT distribution, defaults to noble + HOTFIX_REPO_COMPONENT APT component, defaults to main + HOTFIX_KEY_URL Public key URL for the LibreQoS APT repository + HOTFIX_KEYRING_PATH Destination keyring path installed on the host + HOTFIX_APT_SOURCE_PATH Destination apt source list path installed on the host + HOTFIX_APT_PREFERENCES_PATH Destination apt pin file installed on the host + HOTFIX_APT_PIN_ORIGIN Expected Release Origin field, defaults to LibreQoS + HOTFIX_APT_PIN_LABEL Expected Release Label field, defaults to LibreQoS + HOTFIX_PACKAGE_VERSION Backported package version to install SUPPORTED_UBUNTU_SYSTEMD_VERSION_GLOBS Space-separated stock Ubuntu version globs eligible for replacement HOTFIX_MARKER Marker file written after install HOTFIX_SKIP_REBOOT_PROMPT Set to 1 to suppress the reboot prompt after install - ARCH Debian architecture suffix, defaults to amd64 EOF } @@ -112,41 +128,54 @@ package_is_installed() { dpkg-query -W -f='${db:Status-Abbrev}\n' "$package" 2>/dev/null | grep -q '^ii' } -package_arch_suffix() { - local package="$1" - - case "$package" in - systemd-dev) - printf 'all\n' - ;; - *) - printf '%s\n' "$ARCH" - ;; - esac -} - -package_filename() { - local package="$1" - printf '%s_%s_%s.deb\n' \ - "$package" \ - "$HOTFIX_PACKAGE_VERSION" \ - "$(package_arch_suffix "$package")" -} - -resolved_hotfix_packages() { +resolved_hotfix_package_names() { local package for package in "${HOTFIX_CORE_PACKAGES[@]}"; do - package_filename "$package" + printf '%s\n' "$package" done for package in "${HOTFIX_OPTIONAL_PACKAGES[@]}"; do if package_is_installed "$package"; then - package_filename "$package" + printf '%s\n' "$package" fi done } +resolved_hotfix_package_specs() { + local package + while IFS= read -r package; do + printf '%s=%s\n' "$package" "$HOTFIX_PACKAGE_VERSION" + done < <(resolved_hotfix_package_names) +} + +joined_hotfix_packages() { + local packages=() + local package + + while IFS= read -r package; do + packages+=("$package") + done < <(resolved_hotfix_package_names) + + printf '%s\n' "${packages[*]}" +} + +render_apt_source() { + printf 'deb [signed-by=%s] %s %s %s\n' \ + "$HOTFIX_KEYRING_PATH" \ + "$HOTFIX_REPO_URL" \ + "$HOTFIX_REPO_DIST" \ + "$HOTFIX_REPO_COMPONENT" +} + +render_apt_preferences() { + cat </dev/null + run_as_root chmod 644 "$HOTFIX_KEYRING_PATH" + + render_apt_source | run_as_root tee "$HOTFIX_APT_SOURCE_PATH" >/dev/null + render_apt_preferences | run_as_root tee "$HOTFIX_APT_PREFERENCES_PATH" >/dev/null + run_as_root chmod 644 "$HOTFIX_APT_SOURCE_PATH" "$HOTFIX_APT_PREFERENCES_PATH" + + run_as_root apt-get update + log "Configured LibreQoS APT hotfix repository: $HOTFIX_REPO_URL" +} + +download_bundle() { + local workdir + + require_command apt-get + bootstrap_repo + workdir="$(mktemp -d /tmp/libreqos-systemd-hotfix.XXXXXX)" - while IFS= read -r package; do - curl -fL --retry 3 --output "${workdir}/${package}" "${HOTFIX_BASE_URL}/${package}" - done < <(resolved_hotfix_packages) + ( + cd "$workdir" + while IFS= read -r package; do + apt-get download "$package" + done < <(resolved_hotfix_package_specs) + ) + printf '%s\n' "$workdir" } @@ -237,12 +302,15 @@ write_marker() { { printf 'installed_at=%s\n' "$(date -Iseconds)" printf 'package_version=%s\n' "$HOTFIX_PACKAGE_VERSION" - printf 'base_url=%s\n' "$HOTFIX_BASE_URL" + printf 'repo_url=%s\n' "$HOTFIX_REPO_URL" + printf 'key_url=%s\n' "$HOTFIX_KEY_URL" + printf 'apt_source_path=%s\n' "$HOTFIX_APT_SOURCE_PATH" + printf 'apt_preferences_path=%s\n' "$HOTFIX_APT_PREFERENCES_PATH" printf 'systemd_version=%s\n' "$(current_systemd_version)" while IFS= read -r package; do - printf 'package_file=%s\n' "$package" - printf 'package_url=%s/%s\n' "$HOTFIX_BASE_URL" "$package" - done < <(resolved_hotfix_packages) + printf 'package_name=%s\n' "$package" + printf 'package_spec=%s=%s\n' "$package" "$HOTFIX_PACKAGE_VERSION" + done < <(resolved_hotfix_package_names) } | run_as_root tee "$HOTFIX_MARKER" >/dev/null } @@ -268,17 +336,18 @@ offer_reboot() { } install_bundle() { - local workdir package_paths=() package + local package_specs=() + local package - status >/dev/null || fail "Hotfix is not applicable on this host." require_command apt-get + ensure_applicable_host + bootstrap_repo - workdir="$(download_bundle)" while IFS= read -r package; do - package_paths+=("${workdir}/${package}") - done < <(resolved_hotfix_packages) + package_specs+=("$package") + done < <(resolved_hotfix_package_specs) - run_as_root apt-get install -y "${package_paths[@]}" + run_as_root apt-get install -y "${package_specs[@]}" write_marker log "Hotfix installed." offer_reboot @@ -294,8 +363,10 @@ main() { should-offer) status >/dev/null ;; + bootstrap) + bootstrap_repo + ;; download) - status >/dev/null || fail "Hotfix is not applicable on this host." download_bundle ;; packages) diff --git a/src/testGraph.py b/src/testGraph.py index 132eb389..cf549fb7 100644 --- a/src/testGraph.py +++ b/src/testGraph.py @@ -68,6 +68,22 @@ class TestGraph(unittest.TestCase): self.assertEqual(graph.nodes[2].parentIndex, 1) self.assertEqual(graph.nodes[0].parentIndex, 0) + def test_add_child_by_named_parent_survives_reparent(self): + """ + Ensures addNodeAsChild persists parent identity so later reparenting + keeps the child attached to the intended parent. + """ + from integrationCommon import NetworkGraph, NetworkNode, NodeType + graph = NetworkGraph() + graph.addRawNode(NetworkNode("Site")) + graph.addNodeAsChild("Site", NetworkNode("Client", type=NodeType.client)) + + graph._NetworkGraph__reparentById() + + self.assertEqual(len(graph.nodes), 3) + self.assertEqual(graph.nodes[2].parentId, "Site") + self.assertEqual(graph.nodes[2].parentIndex, 1) + def test_reparent_by_name(self): """ Tests that re-parenting a tree by name is functional @@ -171,6 +187,25 @@ class TestGraph(unittest.TestCase): self.assertEqual(graph.nodes[7].type, NodeType.site) self.assertEqual(graph.nodes[7].id, "Client 3_gen") + def test_generated_site_keeps_original_parent(self): + """ + Ensures generated sites created from relay-style clients retain the + original non-root parent after the reparent pass. + """ + from integrationCommon import NetworkGraph, NetworkNode, NodeType + graph = NetworkGraph() + graph.addRawNode(NetworkNode("Parent Site")) + graph.addRawNode(NetworkNode("Relay Client", parentId="Parent Site", type=NodeType.client)) + graph.addRawNode(NetworkNode("Child Client", parentId="Relay Client", type=NodeType.client)) + + graph._NetworkGraph__reparentById() + graph._NetworkGraph__promoteClientsWithChildren() + graph._NetworkGraph__clientsWithChildrenToSites() + + generated_site = next(node for node in graph.nodes if node.id == "Relay Client_gen") + self.assertEqual(generated_site.parentId, "Parent Site") + self.assertEqual(generated_site.parentIndex, 1) + def test_find_unconnected(self): """ Tests traversing a tree and finding nodes that diff --git a/src/test_integration_common_ignore_subnets.py b/src/test_integration_common_ignore_subnets.py new file mode 100644 index 00000000..714df8fd --- /dev/null +++ b/src/test_integration_common_ignore_subnets.py @@ -0,0 +1,138 @@ +import csv +import importlib +import os +import sys +import tempfile +import types +import unittest + + +def install_common_stubs(): + lqlib = types.ModuleType("liblqos_python") + lqlib.allowed_subnets = lambda: ["0.0.0.0/0", "::/0"] + lqlib.ignore_subnets = lambda: [] + lqlib.generated_pn_download_mbps = lambda: 1000 + lqlib.generated_pn_upload_mbps = lambda: 1000 + lqlib.circuit_name_use_address = lambda: False + lqlib.upstream_bandwidth_capacity_download_mbps = lambda: 1000 + lqlib.upstream_bandwidth_capacity_upload_mbps = lambda: 1000 + lqlib.find_ipv6_using_mikrotik = lambda: False + lqlib.exclude_sites = lambda: [] + lqlib.bandwidth_overhead_factor = lambda: 1.0 + lqlib.committed_bandwidth_multiplier = lambda: 1.0 + lqlib.exception_cpes = lambda: [] + lqlib.promote_to_root_list = lambda: [] + lqlib.client_bandwidth_multiplier = lambda: 1.0 + sys.modules["liblqos_python"] = lqlib + + +install_common_stubs() +integrationCommon = importlib.import_module("integrationCommon") + + +class TestIntegrationCommonIgnoreSubnets(unittest.TestCase): + def setUp(self): + integrationCommon.allowed_subnets = lambda: ["0.0.0.0/0", "::/0"] + integrationCommon.ignore_subnets = lambda: [] + + def _build_client_with_device(self, ipv4=None, ipv6=None): + net = integrationCommon.NetworkGraph() + net.addRawNode( + integrationCommon.NetworkNode( + id="client-1", + displayName="Client 1", + type=integrationCommon.NodeType.client, + download=100, + upload=50, + ) + ) + net.addRawNode( + integrationCommon.NetworkNode( + id="device-1", + displayName="Device 1", + type=integrationCommon.NodeType.device, + parentId="client-1", + ipv4=ipv4 or [], + ipv6=ipv6 or [], + ) + ) + return net + + def test_ignored_only_device_removes_entire_circuit(self): + integrationCommon.ignore_subnets = lambda: ["100.64.0.0/10"] + net = self._build_client_with_device(ipv4=["100.64.1.10/32"]) + + net.prepareTree() + + self.assertEqual([node.id for node in net.nodes], ["FakeRoot"]) + + with tempfile.TemporaryDirectory() as tmpdir: + old_cwd = os.getcwd() + try: + os.chdir(tmpdir) + net.createShapedDevices() + with open("ShapedDevices.csv", newline="") as csvfile: + rows = list(csv.reader(csvfile)) + finally: + os.chdir(old_cwd) + + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0][0], "Circuit ID") + + def test_mixed_device_ips_keep_only_permitted_addresses(self): + integrationCommon.ignore_subnets = lambda: ["100.64.0.0/10"] + net = self._build_client_with_device( + ipv4=["100.64.1.10/32", "203.0.113.10/32"], + ipv6=["2001:db8::10/128"], + ) + + net.prepareTree() + + self.assertEqual(len(net.nodes), 3) + device = next(node for node in net.nodes if node.id == "device-1") + self.assertEqual(device.ipv4, ["203.0.113.10/32"]) + self.assertEqual(device.ipv6, ["2001:db8::10/128"]) + + with tempfile.TemporaryDirectory() as tmpdir: + old_cwd = os.getcwd() + try: + os.chdir(tmpdir) + net.createShapedDevices() + with open("ShapedDevices.csv", newline="") as csvfile: + rows = list(csv.reader(csvfile)) + finally: + os.chdir(old_cwd) + + self.assertEqual(len(rows), 2) + self.assertEqual(rows[1][0], "client-1") + self.assertIn("203.0.113.10/32", rows[1][6]) + self.assertNotIn("100.64.1.10/32", rows[1][6]) + + def test_prune_does_not_require_ip_to_be_in_allowed_subnets(self): + integrationCommon.allowed_subnets = lambda: ["10.0.0.0/8"] + integrationCommon.ignore_subnets = lambda: [] + net = self._build_client_with_device(ipv4=["203.0.113.10/32"]) + + net.prepareTree() + + self.assertEqual(len(net.nodes), 3) + device = next(node for node in net.nodes if node.id == "device-1") + self.assertEqual(device.ipv4, ["203.0.113.10/32"]) + + with tempfile.TemporaryDirectory() as tmpdir: + old_cwd = os.getcwd() + try: + os.chdir(tmpdir) + net.createShapedDevices() + with open("ShapedDevices.csv", newline="") as csvfile: + rows = list(csv.reader(csvfile)) + finally: + os.chdir(old_cwd) + + self.assertEqual(len(rows), 2) + self.assertEqual(rows[1][0], "client-1") + self.assertIn("203.0.113.10/32", rows[1][6]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/test_integration_sonar.py b/src/test_integration_sonar.py new file mode 100644 index 00000000..cb1c2268 --- /dev/null +++ b/src/test_integration_sonar.py @@ -0,0 +1,418 @@ +import importlib +import sys +import types +import unittest +import requests + + +def install_sonar_stubs(): + lqlib = types.ModuleType("liblqos_python") + lqlib.allowed_subnets = lambda: ["0.0.0.0/0"] + lqlib.ignore_subnets = lambda: [] + lqlib.generated_pn_download_mbps = lambda: 1000 + lqlib.generated_pn_upload_mbps = lambda: 1000 + lqlib.circuit_name_use_address = lambda: False + lqlib.upstream_bandwidth_capacity_download_mbps = lambda: 1000 + lqlib.upstream_bandwidth_capacity_upload_mbps = lambda: 1000 + lqlib.find_ipv6_using_mikrotik = lambda: False + lqlib.exclude_sites = lambda: [] + lqlib.bandwidth_overhead_factor = lambda: 1.0 + lqlib.committed_bandwidth_multiplier = lambda: 1.0 + lqlib.exception_cpes = lambda: [] + lqlib.promote_to_root_list = lambda: [] + lqlib.client_bandwidth_multiplier = lambda: 1.0 + lqlib.sonar_api_key = lambda: "" + lqlib.sonar_api_url = lambda: "http://example.invalid" + lqlib.snmp_community = lambda: "public" + lqlib.sonar_airmax_ap_model_ids = lambda: [] + lqlib.sonar_ltu_ap_model_ids = lambda: [] + lqlib.sonar_active_status_ids = lambda: [] + lqlib.sonar_recurring_service_rates = lambda: [] + lqlib.sonar_recurring_excluded_service_names = lambda: [] + sys.modules["liblqos_python"] = lqlib + + +install_sonar_stubs() +integrationSonar = importlib.import_module("integrationSonar") + + +def inventory_item(item_id, name, ips=None, mac=""): + ips = ips or [] + field_entities = [] + if mac: + field_entities.append({ + "value": mac, + "ip_assignments": {"entities": []}, + "inventory_model_field": { + "id": 1, + "name": "MAC Address", + "primary": True, + }, + }) + return { + "id": item_id, + "inventory_model": {"name": name}, + "inventory_model_field_data": {"entities": field_entities}, + "ip_assignments": { + "entities": [{"subnet": subnet} for subnet in ips] + }, + } + + +def sonar_account(address_items=None, radius_accounts=None): + return { + "addresses": { + "entities": [ + { + "line1": "1 Main", + "line2": "", + "city": "Town", + "subdivision": "TX", + "inventory_items": {"entities": address_items or []}, + } + ] + }, + "radius_accounts": {"entities": radius_accounts or []}, + "all_account_services": {"entities": []}, + } + + +def radius_account(radius_id, ips): + return { + "id": radius_id, + "ip_assignments": { + "entities": [{"subnet": subnet} for subnet in ips] + }, + } + + +class TestIntegrationSonarDevices(unittest.TestCase): + def setUp(self): + integrationSonar.sonar_recurring_service_rates = lambda: [] + integrationSonar.sonar_recurring_excluded_service_names = lambda: [] + + def test_inventory_only_devices_keep_existing_shape(self): + devices = integrationSonar.buildAccountDevices( + sonar_account( + address_items=[ + inventory_item( + 42, + "ONU", + ips=["100.64.1.10/32"], + mac="AA:BB:CC:DD:EE:FF", + ) + ] + ) + ) + + self.assertEqual(len(devices), 1) + self.assertEqual(devices[0]["id"], "sonar:device:42") + self.assertEqual(devices[0]["ips"], ["100.64.1.10/32"]) + self.assertEqual(devices[0]["mac"], "AA:BB:CC:DD:EE:FF") + + def test_radius_only_accounts_emit_stable_radius_devices(self): + devices = integrationSonar.buildAccountDevices( + sonar_account( + radius_accounts=[ + radius_account(77, ["100.64.2.20/32"]) + ] + ) + ) + + self.assertEqual(len(devices), 1) + self.assertEqual(devices[0]["id"], "sonar:radius-account:77") + self.assertEqual(devices[0]["name"], "Radius Account 77") + self.assertEqual(devices[0]["ips"], ["100.64.2.20/32"]) + self.assertEqual(devices[0]["mac"], "") + + def test_radius_ips_do_not_duplicate_inventory_ips(self): + devices = integrationSonar.buildAccountDevices( + sonar_account( + address_items=[ + inventory_item( + 42, + "ONU", + ips=["100.64.3.30/32"], + mac="AA:BB:CC:DD:EE:01", + ) + ], + radius_accounts=[ + radius_account(88, ["100.64.3.30/32", "100.64.3.31/32"]) + ], + ) + ) + + self.assertEqual(len(devices), 2) + self.assertEqual(devices[0]["id"], "sonar:device:42") + self.assertEqual(devices[1]["id"], "sonar:radius-account:88") + self.assertEqual(devices[1]["ips"], ["100.64.3.31/32"]) + + def test_inventory_mac_devices_are_preserved_when_radius_supplies_ips(self): + devices = integrationSonar.buildAccountDevices( + sonar_account( + address_items=[ + inventory_item( + 42, + "ONU", + ips=[], + mac="AA:BB:CC:DD:EE:02", + ) + ], + radius_accounts=[ + radius_account(99, ["100.64.4.40/32"]) + ], + ) + ) + + self.assertEqual(len(devices), 2) + self.assertEqual(devices[0]["id"], "sonar:device:42") + self.assertEqual(devices[0]["ips"], []) + self.assertEqual(devices[0]["mac"], "AA:BB:CC:DD:EE:02") + self.assertEqual(devices[1]["id"], "sonar:radius-account:99") + self.assertEqual(devices[1]["ips"], ["100.64.4.40/32"]) + + def test_child_account_inherits_parent_address_when_missing_own_address(self): + child = sonar_account( + address_items=[], + radius_accounts=[radius_account(100, ["100.64.5.50/32"])], + ) + child["addresses"] = {"entities": []} + child["id"] = 500 + child["name"] = "Suite 500" + child["account_services"] = { + "entities": [{ + "service": { + "data_service_detail": { + "download_speed_kilobits_per_second": 100000, + "upload_speed_kilobits_per_second": 20000, + } + } + }] + } + record = integrationSonar.buildAccountRecord(child, fallback_address="Parent Address") + + self.assertIsNotNone(record) + self.assertEqual(record["address"], "Parent Address") + self.assertEqual(record["id"], "sonar:account:500") + + def test_child_accounts_are_added_without_duplicate_top_level_records(self): + top_level = sonar_account( + address_items=[ + inventory_item( + 42, + "ONU", + ips=["100.64.6.60/32"], + mac="AA:BB:CC:DD:EE:06", + ) + ] + ) + top_level["id"] = 600 + top_level["name"] = "Parent" + top_level["account_services"] = { + "entities": [{ + "service": { + "data_service_detail": { + "download_speed_kilobits_per_second": 100000, + "upload_speed_kilobits_per_second": 20000, + } + } + }] + } + + child_only = sonar_account( + address_items=[], + radius_accounts=[radius_account(6010, ["100.64.6.61/32"])], + ) + child_only["addresses"] = {"entities": []} + child_only["id"] = 601 + child_only["name"] = "Child Only" + child_only["account_services"] = { + "entities": [{ + "service": { + "data_service_detail": { + "download_speed_kilobits_per_second": 50000, + "upload_speed_kilobits_per_second": 10000, + } + } + }] + } + + child_duplicate = sonar_account( + address_items=[], + radius_accounts=[radius_account(6020, ["100.64.6.62/32"])], + ) + child_duplicate["addresses"] = {"entities": []} + child_duplicate["id"] = 602 + child_duplicate["name"] = "Child Duplicate" + child_duplicate["account_services"] = { + "entities": [{ + "service": { + "data_service_detail": { + "download_speed_kilobits_per_second": 50000, + "upload_speed_kilobits_per_second": 10000, + } + } + }] + } + + top_level["child_accounts"] = {"entities": [child_only, child_duplicate]} + + duplicate_top_level = sonar_account( + address_items=[], + radius_accounts=[radius_account(6021, ["100.64.6.63/32"])], + ) + duplicate_top_level["id"] = 602 + duplicate_top_level["name"] = "Child Duplicate" + duplicate_top_level["account_services"] = { + "entities": [{ + "service": { + "data_service_detail": { + "download_speed_kilobits_per_second": 75000, + "upload_speed_kilobits_per_second": 15000, + } + } + }] + } + duplicate_top_level["child_accounts"] = {"entities": []} + + accounts = integrationSonar.buildAccountsFromSonarEntities([top_level, duplicate_top_level]) + + self.assertEqual(len(accounts), 3) + self.assertEqual([account["raw_id"] for account in accounts], [600, 602, 601]) + duplicate = next(account for account in accounts if account["raw_id"] == 602) + self.assertEqual(duplicate["download"], 75.0) + self.assertEqual(duplicate["address"], "1 Main, Town, TX") + + def test_recurring_rate_rule_is_used_when_data_service_is_missing(self): + integrationSonar.sonar_recurring_service_rates = lambda: [ + (True, "1 Gig Bulk Tenant Service", 1000.0, 1000.0) + ] + account = sonar_account( + radius_accounts=[radius_account(7000, ["100.64.7.70/32"])], + ) + account["id"] = 700 + account["name"] = "Bulk Tenant" + account["account_services"] = {"entities": []} + account["all_account_services"] = { + "entities": [{ + "service": { + "id": 1, + "name": "1 Gig Bulk Tenant Service", + "type": "RECURRING", + "enabled": True, + "data_service_detail": None, + } + }] + } + + record = integrationSonar.buildAccountRecord(account) + + self.assertIsNotNone(record) + self.assertEqual(record["download"], 1000.0) + self.assertEqual(record["upload"], 1000.0) + + def test_recurring_exclusion_blocks_recurring_fallback(self): + integrationSonar.sonar_recurring_service_rates = lambda: [ + (True, "Equipment Rental", 1000.0, 1000.0) + ] + integrationSonar.sonar_recurring_excluded_service_names = lambda: ["Equipment Rental"] + account = sonar_account( + radius_accounts=[radius_account(7100, ["100.64.7.71/32"])], + ) + account["id"] = 701 + account["name"] = "Rental" + account["account_services"] = {"entities": []} + account["all_account_services"] = { + "entities": [{ + "service": { + "id": 1, + "name": "Equipment Rental", + "type": "RECURRING", + "enabled": True, + "data_service_detail": None, + } + }] + } + + record = integrationSonar.buildAccountRecord(account) + + self.assertIsNone(record) + + +class FakeResponse: + def __init__(self, payload): + self._payload = payload + self.status_code = 200 + self.headers = {} + self.text = "" + + def json(self): + return self._payload + + +class FakeSession: + def __init__(self, outcomes): + self.outcomes = list(outcomes) + self.calls = [] + + def post(self, url, json, timeout): + self.calls.append({ + "url": url, + "json": json, + "timeout": timeout, + }) + outcome = self.outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + return outcome + + +class TestIntegrationSonarTimeoutHandling(unittest.TestCase): + def setUp(self): + self.original_session = integrationSonar._SONAR_SESSION + self.original_sleep = integrationSonar.time.sleep + self.sleep_calls = [] + integrationSonar.time.sleep = lambda seconds: self.sleep_calls.append(seconds) + + def tearDown(self): + integrationSonar._SONAR_SESSION = self.original_session + integrationSonar.time.sleep = self.original_sleep + + def test_paginated_request_retries_read_timeout(self): + fake_session = FakeSession([ + requests.exceptions.ReadTimeout("timed out"), + FakeResponse({ + "data": { + "accounts": { + "entities": [{"id": "1"}], + "page_info": {"total_pages": 1}, + } + } + }), + ]) + integrationSonar._SONAR_SESSION = fake_session + + entities = integrationSonar.sonarPaginatedRequest("query test {}", {}) + + self.assertEqual(entities, [{"id": "1"}]) + self.assertEqual(len(fake_session.calls), 2) + self.assertEqual(fake_session.calls[0]["timeout"], (10, 60)) + self.assertEqual(self.sleep_calls, [2]) + + def test_paginated_request_raises_after_retry_budget_exhausted(self): + fake_session = FakeSession([ + requests.exceptions.ReadTimeout("timed out 1"), + requests.exceptions.ReadTimeout("timed out 2"), + requests.exceptions.ReadTimeout("timed out 3"), + ]) + integrationSonar._SONAR_SESSION = fake_session + + with self.assertRaises(RuntimeError) as ctx: + integrationSonar.sonarPaginatedRequest("query test {}", {}) + + self.assertIn("after 3 attempts", str(ctx.exception)) + self.assertEqual(self.sleep_calls, [2, 5]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/test_scheduler.py b/src/test_scheduler.py index 491fd085..bc8df923 100644 --- a/src/test_scheduler.py +++ b/src/test_scheduler.py @@ -18,7 +18,8 @@ def install_scheduler_stubs(): lqlib.automatic_import_powercode = lambda: False lqlib.automatic_import_sonar = lambda: False lqlib.influx_db_enabled = lambda: False - lqlib.get_libreqos_directory = lambda: "/tmp/libreqos" + # Test-only fake install root. + lqlib.get_libreqos_directory = lambda: "/tmp/libreqos" # nosec B108 lqlib.blackboard_finish = Mock() lqlib.blackboard_submit = Mock() lqlib.automatic_import_wispgate = lambda: False @@ -27,13 +28,18 @@ def install_scheduler_stubs(): lqlib.automatic_import_netzur = lambda: False lqlib.automatic_import_visp = lambda: False lqlib.calculate_hash = lambda: 0 + lqlib.calculate_shaping_runtime_hash = lambda: 0 lqlib.efficiency_core_ids = lambda: [] lqlib.scheduler_alive = Mock() lqlib.scheduler_error = Mock() lqlib.scheduler_output = Mock() + lqlib.wait_for_bus_ready = Mock(return_value=True) lqlib.overrides_persistent_devices_effective = lambda: [] + lqlib.overrides_persistent_devices_materialized = lambda: [] lqlib.overrides_circuit_adjustments_effective = lambda: [] + lqlib.overrides_circuit_adjustments_materialized = lambda: [] lqlib.overrides_network_adjustments_effective = lambda: [] + lqlib.overrides_network_adjustments_materialized = lambda: [] sys.modules["liblqos_python"] = lqlib apscheduler_pkg = types.ModuleType("apscheduler") @@ -137,7 +143,8 @@ class TestSchedulerAffinity(unittest.TestCase): result = types.SimpleNamespace(returncode=0, stdout="", stderr="") with patch.object(scheduler, "automatic_import_uisp", return_value=True): - with patch.object(scheduler, "get_libreqos_directory", return_value="/tmp/libreqos"): + # Test-only fake install root. + with patch.object(scheduler, "get_libreqos_directory", return_value="/tmp/libreqos"): # nosec B108 with patch.object(scheduler, "run_integration_subprocess", return_value=result) as mock_run: with patch.object(scheduler, "apply_lqos_overrides"): with patch.object(scheduler.os.path, "isfile", return_value=True): @@ -146,8 +153,8 @@ class TestSchedulerAffinity(unittest.TestCase): mock_run.assert_called_once() mock_popen.assert_called_once_with( - "/tmp/libreqos/bin/post_integration_hook.sh", - cwd="/tmp/libreqos/bin", + "/tmp/libreqos/bin/post_integration_hook.sh", # nosec B108 + cwd="/tmp/libreqos/bin", # nosec B108 ) @@ -190,7 +197,8 @@ class TestSchedulerErrorReporting(unittest.TestCase): result = types.SimpleNamespace(returncode=0, stdout="uisp info\n", stderr="") with patch.object(scheduler, "automatic_import_uisp", return_value=True): - with patch.object(scheduler, "get_libreqos_directory", return_value="/tmp/libreqos"): + # Test-only fake install root. + with patch.object(scheduler, "get_libreqos_directory", return_value="/tmp/libreqos"): # nosec B108 with patch.object(scheduler, "run_integration_subprocess", return_value=result): with patch.object(scheduler, "apply_lqos_overrides"): with patch.object(scheduler.os.path, "isfile", return_value=False): @@ -209,7 +217,8 @@ class TestSchedulerErrorReporting(unittest.TestCase): result = types.SimpleNamespace(returncode=1, stdout="uisp info\n", stderr="") with patch.object(scheduler, "automatic_import_uisp", return_value=True): - with patch.object(scheduler, "get_libreqos_directory", return_value="/tmp/libreqos"): + # Test-only fake install root. + with patch.object(scheduler, "get_libreqos_directory", return_value="/tmp/libreqos"): # nosec B108 with patch.object(scheduler, "run_integration_subprocess", return_value=result): with patch.object(scheduler, "apply_lqos_overrides"): with patch.object(scheduler.os.path, "isfile", return_value=False): @@ -262,12 +271,13 @@ class TestSchedulerOverrideMerge(unittest.TestCase): "1", "1", "330", "330", "", "" ]] - with patch.object(scheduler, "shaped_devices_csv_path", return_value="/tmp/ShapedDevices.csv"): + # Test-only fake csv path. + with patch.object(scheduler, "shaped_devices_csv_path", return_value="/tmp/ShapedDevices.csv"): # nosec B108 with patch.object(scheduler, "read_shaped_devices_csv", return_value=(header, rows)): - with patch.object(scheduler, "overrides_persistent_devices_effective", return_value=[]): + with patch.object(scheduler, "overrides_persistent_devices_materialized", return_value=[]): with patch.object( scheduler, - "overrides_circuit_adjustments_effective", + "overrides_circuit_adjustments_materialized", return_value=[{ "type": "device_adjust_sqm", "device_id": "splynx_service_93", @@ -282,6 +292,166 @@ class TestSchedulerOverrideMerge(unittest.TestCase): self.assertEqual(written_rows[0][11], "330") self.assertEqual(written_rows[0][13], "fq_codel/fq_codel") + def test_apply_network_adjustments_uses_materialized_adjustments(self): + network = { + "Root": { + "downloadBandwidthMbps": 1000, + "uploadBandwidthMbps": 1000, + "children": { + "SiteA": { + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 50, + "children": {}, + }, + "NodeB": { + "id": "node-b", + "downloadBandwidthMbps": 200, + "uploadBandwidthMbps": 100, + "virtual": False, + "children": {}, + }, + }, + } + } + + with patch.object( + scheduler, + "overrides_network_adjustments_materialized", + return_value=[ + { + "type": "adjust_site_speed", + "node_id": "node-b", + "site_name": "NodeB", + "download_bandwidth_mbps": 80.5, + "upload_bandwidth_mbps": 40.25, + }, + { + "type": "set_node_virtual", + "node_name": "NodeB", + "virtual": True, + }, + ], + ): + changed = scheduler.apply_network_adjustments(network) + + self.assertTrue(changed) + node = network["Root"]["children"]["NodeB"] + self.assertEqual(node["downloadBandwidthMbps"], 80.5) + self.assertEqual(node["uploadBandwidthMbps"], 40.25) + self.assertTrue(network["Root"]["children"]["NodeB"]["virtual"]) + + def test_apply_network_adjustments_keeps_legacy_name_based_matching(self): + network = { + "Root": { + "downloadBandwidthMbps": 1000, + "uploadBandwidthMbps": 1000, + "children": { + "SiteA": { + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 50, + "children": {}, + }, + }, + } + } + + with patch.object( + scheduler, + "overrides_network_adjustments_materialized", + return_value=[ + { + "type": "adjust_site_speed", + "site_name": "SiteA", + "download_bandwidth_mbps": 80, + "upload_bandwidth_mbps": 40, + }, + ], + ): + changed = scheduler.apply_network_adjustments(network) + + self.assertTrue(changed) + site = network["Root"]["children"]["SiteA"] + self.assertEqual(site["downloadBandwidthMbps"], 80) + self.assertEqual(site["uploadBandwidthMbps"], 40) + + def test_apply_network_adjustments_does_not_use_effective_stormguard_speeds(self): + network = { + "Root": { + "downloadBandwidthMbps": 1000, + "uploadBandwidthMbps": 1000, + "children": { + "Pine Hills": { + "downloadBandwidthMbps": 940, + "uploadBandwidthMbps": 500, + "children": {}, + } + }, + } + } + + effective_adjustments = [ + { + "type": "adjust_site_speed", + "site_name": "Pine Hills", + "download_bandwidth_mbps": 4, + "upload_bandwidth_mbps": 4, + } + ] + + with patch.dict( + scheduler.apply_network_adjustments.__globals__, + {"overrides_network_adjustments_effective": lambda: effective_adjustments}, + clear=False, + ): + with patch.object( + scheduler, + "overrides_network_adjustments_materialized", + return_value=[], + ) as mock_materialized: + changed = scheduler.apply_network_adjustments(network) + + mock_materialized.assert_called_once_with() + self.assertFalse(changed) + site = network["Root"]["children"]["Pine Hills"] + self.assertEqual(site["downloadBandwidthMbps"], 940) + self.assertEqual(site["uploadBandwidthMbps"], 500) + + def test_apply_network_adjustments_does_not_materialize_runtime_treeguard_virtual_state(self): + network = { + "Root": { + "children": { + "REGION_01": { + "virtual": False, + "children": {}, + } + }, + } + } + + effective_adjustments = [ + { + "type": "set_node_virtual", + "node_name": "REGION_01", + "virtual": True, + } + ] + + with patch.dict( + scheduler.apply_network_adjustments.__globals__, + {"overrides_network_adjustments_effective": lambda: effective_adjustments}, + clear=False, + ): + with patch.object( + scheduler, + "overrides_network_adjustments_materialized", + return_value=[], + ) as mock_materialized: + changed = scheduler.apply_network_adjustments(network) + + mock_materialized.assert_called_once_with() + self.assertFalse(changed) + self.assertFalse(network["Root"]["children"]["REGION_01"]["virtual"]) + if __name__ == "__main__": unittest.main()
    Parent NodeParent
    Maximum SpeedMax + + + + +
    Minimum SpeedMin
    RTT MetricsRTT
    @@ -90,6 +103,11 @@ Flow Sankey 0 +
    -
    -
    -
    - - Loading devices… +
    +
    +
    +
    Devices
    +

    Per-device live throughput, RTT, retransmits, and address inventory for this circuit.

    +
    +
    +
    +
    +
    + + Loading devices… +
    @@ -188,6 +214,16 @@
    +
    +
    +
    +
    + + Loading ASN summary… +
    +
    +
    +
    @@ -221,8 +257,19 @@
    -
    -
    +
    +
    +
    +
    Queue Tree
    +

    Ancestor queue path from this circuit toward root, with live queue telemetry for each upstream node.

    +
    +
    + Ancestor path + Live updates + Throughput / Retransmits / Latency +
    +
    +
    Loading queue tree… @@ -231,28 +278,65 @@
    -
    -
    -
    +
    +
    +
    +
    Queue Stats
    +

    Raw 1s scatter samples over the last 3 minutes. Hover any queue chart to inspect the same timestamp across all six charts.

    +
    +
    + Last 3 minutes + 1s samples + Synchronized hover +
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - Queue Memory: ? -
    Queue Type: ?
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    Queue Memory
    +
    ?
    +
    +
    +
    Live Queue Type
    +
    ?
    +
    +
    +
    Interpretation
    +
    Download plots above zero, upload plots below zero. Tooltips always show absolute values with direction labels.
    +
    +
    +
    diff --git a/src/rust/lqosd/src/node_manager/static2/config_devices.html b/src/rust/lqosd/src/node_manager/static2/config_devices.html index 75b70c58..b6963cee 100644 --- a/src/rust/lqosd/src/node_manager/static2/config_devices.html +++ b/src/rust/lqosd/src/node_manager/static2/config_devices.html @@ -1,26 +1,21 @@ -
    -
    -
    - -
    -
    +
    +
    + -
    -
    +
    Shaped Devices
    Loading devices...
    +
    Adds, edits, and deletes save immediately.
    @@ -41,7 +36,7 @@
    -
    +
    SQM overrides can be set per direction. Leave a side blank to use the global default; set to "none" to disable that side.
    @@ -62,7 +57,7 @@
    -
    +
    + +
    + Observe removes the LibreQoS shaping tree so you can gather a true baseline. Changing modes requires a reload and may briefly interrupt traffic. +