From d1a63b21bf1b28a0e2b72bd4cecff3ce353b3cae Mon Sep 17 00:00:00 2001 From: rchac Date: Sat, 11 Apr 2026 04:15:10 +0000 Subject: [PATCH] changes --- .gitignore | 13 + AGENTS.md | 6 + docs-es/v2.0/TechnicalDocs/integrations.md | 12 +- docs-es/v2.0/configuration-es.md | 4 +- docs-es/v2.0/future-development-inputs-es.md | 4 +- docs-es/v2.0/integrations-es.md | 15 +- docs-es/v2.0/integrations-netzur-es.md | 6 +- docs-es/v2.0/integrations-reference-es.md | 24 +- docs-es/v2.0/integrations-sonar-es.md | 6 +- docs-es/v2.0/integrations-splynx-es.md | 12 +- docs-es/v2.0/integrations-uisp-es.md | 10 +- docs-es/v2.0/integrations-visp-es.md | 4 +- docs-es/v2.0/integrations-wispgate-es.md | 4 +- docs-es/v2.0/node-manager-ui-es.md | 12 +- docs-es/v2.0/operating-modes-es.md | 34 +- docs-es/v2.0/quickstart-es.md | 10 +- .../v2.0/recipes-wisp-fisp-integration-es.md | 2 +- docs-es/v2.0/troubleshooting-es.md | 6 + docs/v2.0/case-studies.md | 2 +- docs/v2.0/configuration.md | 6 +- docs/v2.0/future-development-inputs.md | 4 +- docs/v2.0/integrations-netzur.md | 5 +- docs/v2.0/integrations-reference.md | 45 +- docs/v2.0/integrations-sonar.md | 9 +- docs/v2.0/integrations-splynx.md | 38 +- docs/v2.0/integrations-uisp.md | 46 +- docs/v2.0/integrations-visp.md | 5 +- docs/v2.0/integrations-wispgate.md | 6 +- docs/v2.0/integrations.md | 37 +- docs/v2.0/node-manager-ui.md | 35 +- docs/v2.0/operating-modes.md | 46 +- docs/v2.0/quickstart.md | 32 +- docs/v2.0/recipes-wisp-fisp-integration.md | 2 +- docs/v2.0/treeguard.md | 2 +- docs/v2.0/troubleshooting.md | 19 +- src/LibreQoS.py | 89 +- src/configMigrator.py | 4 +- src/csvToNetworkJSON.py | 6 +- src/integrationCommon.py | 412 +++- src/integrationNetzur.py | 9 +- src/integrationPowercode.py | 6 +- src/integrationSonar.py | 7 +- src/integrationSplynx.py | 191 +- src/integrationVISP.py | 8 +- src/integrationWISPGate.py | 12 +- src/lqos.example | 1 - src/rust/Cargo.lock | 17 + src/rust/Cargo.toml | 1 + src/rust/lqos_bakery/src/lib.rs | 20 + src/rust/lqos_config/src/etc/migration.rs | 7 +- src/rust/lqos_config/src/etc/mod.rs | 359 +++- src/rust/lqos_config/src/etc/test_data.rs | 2 +- src/rust/lqos_config/src/etc/v15/example.toml | 4 +- .../src/etc/v15/integration_common.rs | 4 - src/rust/lqos_config/src/etc/v15/mod.rs | 2 + .../src/etc/v15/splynx_integration.rs | 4 +- .../lqos_config/src/etc/v15/top_config.rs | 84 + src/rust/lqos_config/src/etc/v15/topology.rs | 48 + src/rust/lqos_config/src/lib.rs | 26 +- src/rust/lqos_config/src/network_json.rs | 90 +- .../src/shaped_devices/shaped_device.rs | 6 +- .../src/topology_canonical_state.rs | 690 ++++++- .../lqos_config/src/topology_editor_state.rs | 71 +- .../src/topology_parent_candidates.rs | 3 + .../lqos_config/src/topology_runtime_state.rs | 537 +++++- src/rust/lqos_overrides/src/overrides_file.rs | 40 +- src/rust/lqos_python/Cargo.toml | 3 + src/rust/lqos_python/src/lib.rs | 434 ++++- src/rust/lqos_setup/src/main.rs | 154 +- src/rust/lqos_topology/Cargo.toml | 1 + src/rust/lqos_topology/src/lib.rs | 1656 ++++++++++++++++- src/rust/lqos_topology_compile/Cargo.toml | 11 + src/rust/lqos_topology_compile/src/bundle.rs | 456 +++++ .../lqos_topology_compile/src/compile_mode.rs | 1501 +++++++++++++++ src/rust/lqos_topology_compile/src/lib.rs | 14 + .../lqos_topology_compile/src/validation.rs | 157 ++ src/rust/lqosd/Cargo.toml | 1 + src/rust/lqosd/src/main.rs | 3 + .../src/node_manager/js_build/src/circuit.js | 29 +- .../js_build/src/config/config_helper.js | 158 +- .../js_build/src/config_integration.js | 21 +- .../js_build/src/config_netzur.js | 19 +- .../js_build/src/config_powercode.js | 20 +- .../node_manager/js_build/src/config_sonar.js | 20 +- .../js_build/src/config_splynx.js | 34 +- .../node_manager/js_build/src/config_uisp.js | 30 +- .../node_manager/js_build/src/config_visp.js | 30 +- .../node_manager/js_build/src/cpu_weights.js | 2 +- .../src/node_manager/js_build/src/template.js | 4 +- .../js_build/src/toasts/sponsor_us.js | 16 +- .../src/node_manager/js_build/src/tree.js | 2 +- .../src/node_manager/local_api/circuit.rs | 16 + .../src/node_manager/local_api/config.rs | 422 ++++- .../local_api/node_rate_overrides.rs | 2 +- .../local_api/tree_attached_circuits.rs | 348 +++- .../static2/config_integration.html | 23 +- .../node_manager/static2/config_netzur.html | 6 +- .../static2/config_powercode.html | 6 +- .../node_manager/static2/config_sonar.html | 6 +- .../node_manager/static2/config_splynx.html | 24 +- .../src/node_manager/static2/config_uisp.html | 18 +- .../src/node_manager/static2/config_visp.html | 12 +- src/rust/lqosd/src/node_manager/ws.rs | 8 +- .../lqosd/src/node_manager/ws/messages.rs | 5 +- .../shaped_devices_tracker/circuit_live.rs | 12 - .../lqosd/src/shaped_devices_tracker/mod.rs | 63 +- .../src/shaped_devices_tracker/netjson.rs | 34 +- .../throughput_tracker/stats_submission.rs | 43 +- src/rust/lqosd/src/validation.rs | 37 +- src/rust/uisp_integration/Cargo.toml | 1 + .../src/strategies/ap_only.rs | 398 +++- .../src/strategies/ap_site.rs | 23 +- .../uisp_integration/src/strategies/common.rs | 22 +- .../strategies/full/bandwidth_overrides.rs | 52 +- .../src/strategies/full/network_json.rs | 6 - .../uisp_integration/src/strategies/full2.rs | 181 +- .../strategies/legacy_bandwidth_overrides.rs | 267 +++ .../src/strategies/legacy_mikrotik.rs | 63 + .../src/strategies/legacy_parse.rs | 73 + .../src/strategies/legacy_routes_override.rs | 100 + .../src/strategies/legacy_uisp_fetch.rs | 70 + .../uisp_integration/src/strategies/mod.rs | 127 +- .../src/uisp_types/uisp_device.rs | 115 +- src/scheduler.py | 219 +-- src/shaping_skip_report.py | 2 +- src/testGraph.py | 278 +++ src/test_integration_splynx.py | 79 +- src/test_libreqos_shaping_inputs.py | 5 + src/test_scheduler.py | 59 + src/test_shaping_skip_report.py | 1 + 130 files changed, 10108 insertions(+), 1177 deletions(-) create mode 100644 src/rust/lqos_config/src/etc/v15/topology.rs create mode 100644 src/rust/lqos_topology_compile/Cargo.toml create mode 100644 src/rust/lqos_topology_compile/src/bundle.rs create mode 100644 src/rust/lqos_topology_compile/src/compile_mode.rs create mode 100644 src/rust/lqos_topology_compile/src/lib.rs create mode 100644 src/rust/lqos_topology_compile/src/validation.rs create mode 100644 src/rust/uisp_integration/src/strategies/legacy_bandwidth_overrides.rs create mode 100644 src/rust/uisp_integration/src/strategies/legacy_mikrotik.rs create mode 100644 src/rust/uisp_integration/src/strategies/legacy_parse.rs create mode 100644 src/rust/uisp_integration/src/strategies/legacy_routes_override.rs create mode 100644 src/rust/uisp_integration/src/strategies/legacy_uisp_fetch.rs diff --git a/.gitignore b/.gitignore index 14f7ce8e..1a0c1501 100644 --- a/.gitignore +++ b/.gitignore @@ -173,17 +173,30 @@ src/network.large.json /src/lqusers.session.key /src/network.effective.json /src/network.json.pre_* +/src/network.effective.json.stale-* /src/topology_attachment_health_state.json +/src/topology_attachment_health_state.json.stale-* /src/topology_editor_state.json +/src/topology_editor_state.json.stale-* /src/topology_effective_state.json +/src/topology_effective_state.json.stale-* /src/topology_overrides.json /src/topology_parent_candidates.json /src/topology_canonical_state.json +/src/topology_canonical_state.json.stale-* +/src/topology_runtime_status.json +/src/topology_runtime_status.json.stale-* +/src/topology_import.json /src/circuit_anchors.json /src/shaping_inputs.json +/src/shaping_inputs.json.stale-* +/src/.topology_stale/ +/src/.codex # Migrated integration compatibility artifacts /src/integrationUISPbandwidths.csv.migrated-* +/src/integrationUISPbandwidths.csv.backup* +/src/integrationSplynxBandwidths.csv.backup* /src/integrationUISProutes.csv.migrated-* # Local override snapshots diff --git a/AGENTS.md b/AGENTS.md index a044dd7b..456a737a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -169,3 +169,9 @@ Canonical shared Codex skills for this repo live in `.codex-repo/skills/`. ## Documentation - Always keep the documentation updated to match code changes. +- Public docs in `docs/` and `docs-es/` are ISP/operator-facing by default. Write them for customers using LibreQoS, not for contributors, Codex, or internal support staff. +- Do not write public docs like internal change logs, AGENTS notes, implementation plans, or support-only commentary. +- Prefer operator-visible behavior, setup steps, and validation guidance over browser/server mechanics, internal file-flow details, or code-architecture commentary. +- If a topic is mainly for contributors, support, or engineering, keep it in clearly separate technical/reference documentation rather than mixing it into customer pages. +- Keep Spanish public docs fully translated and idiomatic; avoid leaking English implementation terms unless they are the actual UI label. +- Follow `DOCS_STYLE.md` when editing public documentation. diff --git a/docs-es/v2.0/TechnicalDocs/integrations.md b/docs-es/v2.0/TechnicalDocs/integrations.md index 72c68696..e5462850 100644 --- a/docs-es/v2.0/TechnicalDocs/integrations.md +++ b/docs-es/v2.0/TechnicalDocs/integrations.md @@ -43,9 +43,7 @@ python3 integrationSplynx.py On the first successful run, it will create a ShapedDevices.csv file and network.json. ShapedDevices.csv will be overwritten every time the Splynx integration is run. -To ensure the network.json is always overwritten with the newest version pulled in by the integration, please edit `/etc/lqos.conf` with the command `sudo nano /etc/lqos.conf`. -Edit the file to set the value of `always_overwrite_network_json` to `true`. -Then, run `sudo systemctl restart lqosd`. +Built-in integrations do not overwrite `network.json`; keep DIY `network.json` operator-owned. You have the option to run integrationSplynx.py automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_splynx = true``` under the ```[splynx_integration]``` section in `/etc/lqos.conf`. Once set, run `sudo systemctl restart lqos_scheduler`. @@ -127,15 +125,13 @@ sudo /opt/libreqos/src/bin/uisp_integration ``` On the first successful run, it will create a network.json and ShapedDevices.csv file. -If a network.json file exists, it will not be overwritten, unless you set ```always_overwrite_network_json = true```. +Built-in integrations do not overwrite `network.json`; keep DIY `network.json` operator-owned. ShapedDevices.csv will be overwritten every time the UISP integration is run. Cuando varios sitios cliente de UISP comparten el mismo nombre, LibreQoS ahora intenta diferenciar los nombres visibles generados para circuitos/sitios con un sufijo más humano, como el primer segmento de la dirección, recurriendo al nombre del servicio y luego a un ID corto solo cuando hace falta. La identidad estable del circuito sigue viniendo del ID de sitio/servicio de UISP, no del nombre visible. -To ensure the network.json is always overwritten with the newest version pulled in by the integration, please edit `/etc/lqos.conf` with the command `sudo nano /etc/lqos.conf`. -Edit the file to set the value of `always_overwrite_network_json` to `true`. -Then, run `sudo systemctl restart lqosd`. +Built-in integrations do not overwrite `network.json`; keep DIY `network.json` operator-owned. You have the option to run `uisp_integration` automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_uisp = true``` in `/etc/lqos.conf`. Once set, run `sudo systemctl restart lqos_scheduler`. @@ -179,7 +175,7 @@ python3 integrationSonar.py ``` On the first successful run, it will create a ShapedDevices.csv file. -If a network.json file exists, it will not be overwritten, unless you set ```always_overwrite_network_json = true```. +Built-in integrations do not overwrite `network.json`; keep DIY `network.json` operator-owned. You can modify the network.json file to more accurately reflect bandwidth limits. ShapedDevices.csv will be overwritten every time the Sonar integration is run. You have the option to run integrationSonar.py automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_sonar = true``` in `/etc/lqos.conf`. diff --git a/docs-es/v2.0/configuration-es.md b/docs-es/v2.0/configuration-es.md index c92ddabc..07c1632c 100644 --- a/docs-es/v2.0/configuration-es.md +++ b/docs-es/v2.0/configuration-es.md @@ -25,7 +25,7 @@ Para la mayoría de operadores: La mayoría de cambios operativos diarios se realizan en la WebUI (`http://tu_ip_del_shaper:9123/config_general.html`). -Las versiones actuales usan paneles compartidos de configuración en las páginas General, RTT, Queues, TreeGuard, Network Mode, valores por defecto de integración, Network Layout, Insight, páginas de integración por proveedor, IP Ranges, Flow Tracking y Shaped Devices para que los ajustes del nodo y la afinación operativa sigan la misma presentación visual que el resto de Node Manager. La página de TreeGuard comienza con un resumen del comportamiento actual en lugar de un banner permanente de advertencia. Los valores por defecto de integración también incluyen la política compartida de margen para puertos Ethernet usada por integraciones que pueden detectar la velocidad negociada hacia el suscriptor. +Las versiones actuales usan un diseño coherente de configuración en las páginas General, RTT, Queues, TreeGuard, Network Mode, valores por defecto de integración, Network Layout, Insight, páginas de integración por proveedor, IP Ranges, Flow Tracking y Shaped Devices. Los valores por defecto de integración también incluyen la política compartida de margen para puertos Ethernet usada por integraciones que pueden detectar la velocidad negociada hacia el suscriptor. ### Dónde en la WebUI @@ -35,7 +35,7 @@ Las versiones actuales usan paneles compartidos de configuración en las página - Editor de dispositivos regulados: `Configuration -> Shaped Devices` - Validación operativa: páginas de `WebUI (Node Manager)` (dashboard/tree/flow/scheduler) -Cuando una integración está habilitada y actúa como fuente de verdad, los editores `Network Layout` y `Shaped Devices` siguen siendo visibles pero pasan a modo de solo lectura en la WebUI. +Cuando una integración está gestionando sus datos de shaping, los editores `Network Layout` y `Shaped Devices` siguen siendo visibles pero pasan a modo de solo lectura en la WebUI. Nota de topología: - Los nombres de nodo en `network.json` deben ser globalmente únicos en todo el árbol. Los nombres duplicados ahora fallan la validación y no son aceptados por el guardado de la WebUI ni por `LibreQoS.py`. diff --git a/docs-es/v2.0/future-development-inputs-es.md b/docs-es/v2.0/future-development-inputs-es.md index 09f8a5b1..08a79a47 100644 --- a/docs-es/v2.0/future-development-inputs-es.md +++ b/docs-es/v2.0/future-development-inputs-es.md @@ -47,8 +47,8 @@ Sintomas ejemplo: Issues representativos: -- [#860 - always_overwrite_network_json default behavior confusion](https://github.com/LibreQoE/LibreQoS/issues/860) -- [#899 - UISP: always_overwrite_network_json=false and missing ShapedDevices.csv](https://github.com/LibreQoE/LibreQoS/issues/899) +- [#860 - confusión histórica del valor por defecto de sobrescritura de network.json](https://github.com/LibreQoE/LibreQoS/issues/860) +- [#899 - UISP: ruta histórica con sobrescritura deshabilitada y ShapedDevices.csv ausente](https://github.com/LibreQoE/LibreQoS/issues/899) - [#845 - UISP: multi-services with same site name](https://github.com/LibreQoE/LibreQoS/issues/845) - [#699 - UISP: trailing spaces break matching](https://github.com/LibreQoE/LibreQoS/issues/699) diff --git a/docs-es/v2.0/integrations-es.md b/docs-es/v2.0/integrations-es.md index 4edc21a8..bca7020a 100644 --- a/docs-es/v2.0/integrations-es.md +++ b/docs-es/v2.0/integrations-es.md @@ -4,14 +4,14 @@ Use esta página para elegir y configurar integraciones CRM/NMS soportadas. Use las páginas por integración para detalles específicos. -¿Necesita definiciones de términos de sobrescritura/persistencia? Vea el [Glosario](glossary-es.md). +¿Necesita definiciones de términos comunes? Vea el [Glosario](glossary-es.md). La mayoría de operadores usan las integraciones incluidas. Si todavía no eligió ruta de despliegue, empiece por [Quickstart](quickstart-es.md). ## Elegir ruta de integración -| Ruta | Mejor para | Fuente de verdad principal | +| Ruta | Mejor para | Dónde deben hacerse los cambios permanentes | |---|---|---| | Integración incluida | La mayoría de operadores con sistemas soportados | CRM/NMS por jobs de integración de LibreQoS | | Fuente de verdad personalizada | Operadores con sincronización propia de CRM/NMS | Archivos externos generados (`network.json`, `ShapedDevices.csv`) | @@ -33,12 +33,13 @@ Si todavía no eligió ruta de despliegue, empiece por [Quickstart](quickstart-e - [Integración con Powercode](integrations-powercode-es.md) - [Integración con Sonar](integrations-sonar-es.md) -## Comportamiento importante de sobrescritura +## Comportamiento importante de refresco Cuando hay integraciones habilitadas: -- `ShapedDevices.csv` normalmente se regenera por jobs de sincronización. -- `network.json` también puede sobrescribirse según configuración (por ejemplo `always_overwrite_network_json`). -- Las ediciones manuales pueden sobrescribirse en el siguiente ciclo de refresco. +- La sincronización de la integración refresca los datos de topología y shaping que LibreQoS usa. +- Las integraciones incluidas no usan `network.json` ni `ShapedDevices.csv` como sus archivos normales de salida. +- Si trabaja en modo DIY o manual, `network.json` y `ShapedDevices.csv` siguen siendo los archivos que usted mantiene. +- Las ediciones directas sobre datos gestionados por la integración pueden sobrescribirse en el siguiente ciclo. ## Soporte para IDs de nodos de topología @@ -49,7 +50,7 @@ LibreQoS soporta un campo genérico opcional `"id"` en los nodos de `network.jso | UISP | Sí | Sitios/dispositivos reales de UISP exportan `id` genérico más la metadata existente `uisp_site` / `uisp_device`. Los nodos sintéticos de LibreQoS usan IDs generados estables. | | Splynx | Sí | Los nodos de topología de network sites y AP/site exportan `id` genérico. | | Sonar | Sí | Los nodos de topología de sitios y AP exportan `id` genérico. | -| Netzur | Parcial | Se exporta solo cuando el payload upstream de zonas incluye un ID de zona estable. | +| Netzur | Parcial | Se exporta solo cuando los datos upstream de zonas incluyen un ID de zona estable. | | VISP | No | El importador actual shapea servicios/dispositivos pero no construye nodos de topología en `network.json`. | | Powercode | No | El importador actual no construye nodos de topología en `network.json`. | | WISPGate | No | El importador actual no construye nodos de topología a partir de identificadores estables upstream. | diff --git a/docs-es/v2.0/integrations-netzur-es.md b/docs-es/v2.0/integrations-netzur-es.md index 9ec60fe3..88168991 100644 --- a/docs-es/v2.0/integrations-netzur-es.md +++ b/docs-es/v2.0/integrations-netzur-es.md @@ -12,9 +12,9 @@ Use esta integración cuando Netzur sea su fuente de verdad CRM/NMS. ## Notas operativas -- La integración regenera `ShapedDevices.csv`. -- `network.json` depende de su configuración de sobrescritura. -- Recomendado: mantener `always_overwrite_network_json = true` para alinear topología con los ciclos de Netzur. +- La integración refresca los datos importados y de shaping que LibreQoS usa con Netzur. +- `network.json` queda para despliegues DIY o manuales. +- Use la WebUI para validar que la importación terminó correctamente. ## Referencia completa diff --git a/docs-es/v2.0/integrations-reference-es.md b/docs-es/v2.0/integrations-reference-es.md index 54e4a34a..25971d17 100644 --- a/docs-es/v2.0/integrations-reference-es.md +++ b/docs-es/v2.0/integrations-reference-es.md @@ -25,7 +25,7 @@ Si usa scripts propios como fuente de verdad para `network.json` y `ShapedDevice Comportamiento importante cuando hay integraciones habilitadas: - `ShapedDevices.csv` normalmente se regenera por los jobs de sincronización. -- El comportamiento de sobrescritura de `network.json` depende de su configuración de integración (por ejemplo `always_overwrite_network_json`). +- `network.json` sigue siendo una entrada DIY/manual operada por el usuario. - Las ediciones manuales pueden sobrescribirse en el próximo ciclo de refresco. ## Integración con Splynx @@ -42,8 +42,8 @@ LibreQoS soporta múltiples estrategias de topología para la integración con S |------------|-------------|-------------|-------------| | `flat` | Solo regula suscriptores, sin jerarquía | Menor | Máximo rendimiento, regulación simple de suscriptores únicamente | | `ap_only` | Una capa: AP → Clientes | Bajo | **Predeterminado**. Mejor balance entre rendimiento y estructura | -| `ap_site` | Dos capas: Sitio → AP → Clientes | Medio | Agregación a nivel de sitio con complejidad moderada | -| `full` | Mapeo completo de topología | Mayor | Representación completa de jerarquía de red | +| `ap_site` | Dos capas: Sitio → AP → Clientes | Medio | Agregación a nivel de sitio usando Splynx Network Sites cuando estén disponibles | +| `full` | Mapeo completo de la jerarquía de monitoreo | Mayor | Conserva la jerarquía más rica de monitoreo de Splynx en vez de compactarla a Network Sites | Configure la estrategia en `/etc/lqos.conf` bajo la sección `[splynx]`: @@ -121,8 +121,7 @@ python3 integrationSplynx.py En la primera ejecución exitosa, se crearán los archivos ShapedDevices.csv y network.json. ShapedDevices.csv será sobrescrito cada vez que se ejecute la integración con Splynx. -Para asegurarse de que network.json siempre se sobrescriba con la versión más reciente obtenida por la integración, edite `/etc/lqos.conf` con el comando `sudo nano /etc/lqos.conf` y configure el valor `always_overwrite_network_json` a `true`. -Luego ejecute `sudo systemctl restart lqosd`. +Las integraciones incluidas no sobrescriben `network.json`; conserve `network.json` como entrada DIY/manual operada por el usuario. Tiene la opción de ejecutar integrationSplynx.py automáticamente al iniciar el equipo y cada X minutos (configurado con el parámetro `queue_refresh_interval_mins`), lo cual es altamente recomendado. Esto se habilita estableciendo ```enable_splynx = true``` bajo la sección ```[splynx_integration]``` en `/etc/lqos.conf`. Una vez configurado, ejecute `sudo systemctl restart lqos_scheduler`. @@ -162,7 +161,7 @@ Para una importación manual: python3 integrationNetzur.py ``` -La integración actualiza `ShapedDevices.csv` y, salvo que `always_overwrite_network_json` esté deshabilitado, también `network.json`. Ajuste la opción Integración → Común si necesita preservar un `network.json` existente entre ejecuciones de Netzur. +La integración actualiza sus artefactos propios de topología y shaping. `network.json` sigue siendo una entrada DIY/manual operada por el usuario. ## Integración con VISP @@ -186,7 +185,7 @@ Notas: - La importación VISP usa GraphQL y actualmente trabaja en estrategia `flat`. - La integración reescribe `ShapedDevices.csv` en cada ejecución. -- `network.json` solo se sobrescribe cuando `always_overwrite_network_json = true` (en `[integration_common]`). +- `network.json` sigue siendo una entrada DIY/manual operada por el usuario. - Los tokens de VISP se cachean en `/.visp_token_cache_*.json`. Importación manual: @@ -297,7 +296,6 @@ commit_bandwidth_multiplier = 0.98 # Establecer mínimo al 98% del máximo (CIR # Opciones Avanzadas ipv6_with_mikrotik = false # Habilitar si usa DHCPv6 con MikroTik -always_overwrite_network_json = true # Recomendado para despliegues guiados por integración exception_cpes = [] # Excepciones de CPE en formato ["cpe:parent"] squash_sites = [] # Opcional: sitios a compactar do_not_squash_sites = [] # Opcional: nombres de sitios que deben permanecer sin compactar en runtime/exportación @@ -338,14 +336,13 @@ cd /opt/libreqos/src sudo /opt/libreqos/src/bin/uisp_integration ``` En la primera ejecución exitosa, se crearán los archivos network.json y ShapedDevices.csv. -Si ya existe un archivo network.json, no será sobrescrito a menos que configure ```always_overwrite_network_json = true```. +Las integraciones incluidas no sobrescriben `network.json`; conserve `network.json` como entrada DIY/manual operada por el usuario. ShapedDevices.csv será sobrescrito cada vez que se ejecute la integración de UISP. Si UISP expone un sitio y un AP con el mismo nombre visible dentro de la misma topología, las compilaciones actuales conservan el nombre del sitio en `network.json` y diferencian el nombre exportado del AP para que la rama del sitio no desaparezca del árbol. -Para asegurarse de que network.json siempre sea sobrescrito con la versión más reciente obtenida por la integración, edite el archivo `/etc/lqos.conf` con el comando `sudo nano /etc/lqos.conf` y configure el valor `always_overwrite_network_json` a `true`. -Luego ejecute: `sudo systemctl restart lqosd`. +Las integraciones incluidas no sobrescriben `network.json`; conserve `network.json` como entrada DIY/manual operada por el usuario. Tiene la opción de ejecutar `uisp_integration` automáticamente al iniciar el equipo y cada X minutos (configurado con el parámetro `queue_refresh_interval_mins`), lo cual es altamente recomendado. Esto se habilita estableciendo ```enable_uisp = true``` en `/etc/lqos.conf`. Una vez configurado, ejecute `sudo systemctl restart lqos_scheduler`. @@ -390,8 +387,7 @@ python3 integrationWISPGate.py En la primera ejecución exitosa, se crearán los archivos network.json y ShapedDevices.csv. ShapedDevices.csv será sobrescrito cada vez que se ejecute la integración de WISPGate. -Para asegurarse de que network.json siempre sea sobrescrito con la versión más reciente obtenida por la integración, edite el archivo `/etc/lqos.conf` con el comando `sudo nano /etc/lqos.conf` y configure el valor `always_overwrite_network_json` a `true`. -Luego ejecute: `sudo systemctl restart lqosd`. +Las integraciones incluidas no sobrescriben `network.json`; conserve `network.json` como entrada DIY/manual operada por el usuario. Tiene la opción de ejecutar integrationWISPGate.py automáticamente al iniciar el equipo y cada X minutos (configurado con el parámetro `queue_refresh_interval_mins`), lo cual es altamente recomendado. Esto se habilita estableciendo ```enable_wispgate = true``` en `/etc/lqos.conf`. Una vez configurado, ejecute `sudo systemctl restart lqos_scheduler`. @@ -421,7 +417,7 @@ python3 integrationSonar.py ``` En la primera ejecución exitosa, se crearán los archivos network.json y ShapedDevices.csv. -Si ya existe un archivo network.json, no será sobrescrito a menos que configure ```always_overwrite_network_json = true```. +Las integraciones incluidas no sobrescriben `network.json`; conserve `network.json` como entrada DIY/manual operada por el usuario. Puede modificar el archivo network.json para reflejar con mayor precisión los límites de ancho de banda. El archivo ShapedDevices.csv se sobrescribirá cada vez que se ejecute la integración de Sonar. Tiene la opción de ejecutar integrationSonar.py automáticamente al iniciar el equipo y cada X minutos (configurado con el parámetro `queue_refresh_interval_mins`), lo cual es altamente recomendado. Esto se habilita estableciendo ```enable_sonar = true``` en `/etc/lqos.conf`. diff --git a/docs-es/v2.0/integrations-sonar-es.md b/docs-es/v2.0/integrations-sonar-es.md index 5d303741..284a56c2 100644 --- a/docs-es/v2.0/integrations-sonar-es.md +++ b/docs-es/v2.0/integrations-sonar-es.md @@ -12,9 +12,9 @@ Use esta integración cuando Sonar sea su fuente de verdad CRM/NMS. ## Notas operativas -- `ShapedDevices.csv` se regenera en cada sincronización. -- `network.json` depende de su configuración de sobrescritura. -- Recomendado: mantener `always_overwrite_network_json = true` para alinear topología con Sonar en cada ciclo. +- La integración refresca automáticamente los datos importados y de shaping que LibreQoS usa con Sonar. +- `network.json` queda para despliegues DIY o manuales. +- Use la WebUI para confirmar que la importación terminó correctamente. ## Referencia completa diff --git a/docs-es/v2.0/integrations-splynx-es.md b/docs-es/v2.0/integrations-splynx-es.md index ce4d5024..65a09651 100644 --- a/docs-es/v2.0/integrations-splynx-es.md +++ b/docs-es/v2.0/integrations-splynx-es.md @@ -16,15 +16,15 @@ Base recomendada: - `strategy = "ap_only"` (menos confusión inicial) - `enable_splynx = true` -- `always_overwrite_network_json = true` para despliegues guiados por integración +- `network.json` queda reservado para despliegues DIY o manuales Después, ejecute una sincronización manual y valide salidas antes de aumentar frecuencia de refresh. ## Notas operativas -- `ShapedDevices.csv` se regenera en cada sincronización. -- `network.json` depende de `always_overwrite_network_json`. -- Recomendado: mantener `always_overwrite_network_json = true` para alinear topología con Splynx en cada ciclo. +- La sincronización actualiza automáticamente los datos importados y de shaping que LibreQoS usa con Splynx. +- `network.json` queda para despliegues DIY o manuales. +- Use la WebUI para confirmar que la importación y la profundidad del árbol son las esperadas. - Use WebUI para ajustes operativos diarios. ## Validación en 5 minutos después de cambios Splynx @@ -35,7 +35,7 @@ python3 integrationSplynx.py ``` 2. Confirme archivos actualizados: ```bash -ls -lh /opt/libreqos/src/network.json /opt/libreqos/src/ShapedDevices.csv +ls -lh /opt/libreqos/src/topology_import.json /opt/libreqos/src/shaping_inputs.json ``` 3. Confirme salud de servicios: ```bash @@ -49,4 +49,4 @@ journalctl -u lqos_scheduler --since "30 minutes ago" - [Referencia detallada de Splynx](integrations-reference-es.md#integración-con-splynx) - [Modos de operación y fuente de verdad](operating-modes-es.md) -Las versiones actuales también exponen una política compartida de margen para puertos Ethernet en `Configuration -> Integrations -> Integration Defaults`. Las integraciones que pueden aportar la velocidad Ethernet negociada hacia el suscriptor usan un multiplicador conservador por defecto de `0.94`, salvo que el operador lo sobrescriba. +Las versiones actuales también exponen una política compartida de margen para puertos Ethernet en `Configuration -> Integrations -> Integration Defaults`. Las integraciones que pueden aportar la velocidad Ethernet negociada hacia el suscriptor usan un multiplicador conservador por defecto de `0.94`, salvo que el operador lo cambie. diff --git a/docs-es/v2.0/integrations-uisp-es.md b/docs-es/v2.0/integrations-uisp-es.md index 4a85158f..b2a233e1 100644 --- a/docs-es/v2.0/integrations-uisp-es.md +++ b/docs-es/v2.0/integrations-uisp-es.md @@ -27,9 +27,9 @@ Use esta guía para evitar confusión con opciones: ## Notas operativas -- `ShapedDevices.csv` se regenera en cada sincronización. -- `network.json` depende de `always_overwrite_network_json`. -- Para despliegues guiados por integración, use `always_overwrite_network_json = true` para mantener la topología alineada con UISP en cada ciclo. +- La sincronización actualiza automáticamente los datos importados y de shaping que LibreQoS usa con UISP. +- `network.json` queda para despliegues DIY o manuales. +- Use la WebUI para confirmar que la importación y la profundidad del árbol son las esperadas. - En modo integración, trate ediciones de archivos como temporales. ## Validación en 5 minutos después de cambios UISP @@ -41,7 +41,7 @@ sudo /opt/libreqos/src/bin/uisp_integration ``` 2. Confirme archivos generados/actualizados: ```bash -ls -lh /opt/libreqos/src/network.json /opt/libreqos/src/ShapedDevices.csv +ls -lh /opt/libreqos/src/topology_import.json /opt/libreqos/src/shaping_inputs.json ``` 3. Verifique salud de servicios: ```bash @@ -61,7 +61,7 @@ Los valores por defecto de integración también incluyen el límite compartido Las compilaciones UISP actuales también reutilizan ese mismo multiplicador conservador para límites de transporte en adjuntos de infraestructura cuando `infrastructure_transport_caps_enabled = true`. Para esos límites de infraestructura, LibreQoS prefiere la interfaz Ethernet/SFP de transporte activa con mayor velocidad reportada por UISP, con fallbacks exactos por modelo para techos de hardware conocidos como AF60-LR. -La compactación de runtime/exportación para UISP ahora siempre se aplica después de que Topology Manager resuelve la preferencia de adjuntos. La topología canónica de UISP y Topology Manager permanecen sin compactar; `tree.html` y shaping usan el árbol efectivo compactado. `do_not_squash_sites` sigue permitiendo excluir nombres de sitios concretos de esa compactación de runtime. +La compactación de runtime/exportación para UISP ahora siempre se aplica después de que Topology Manager resuelve la preferencia de adjuntos. `do_not_squash_sites` sigue permitiendo excluir nombres de sitios concretos de esa compactación. Nota heredada: - Los valores existentes de `enable_squashing` en `/etc/lqos.conf` se ignoran por compatibilidad hacia atrás. diff --git a/docs-es/v2.0/integrations-visp-es.md b/docs-es/v2.0/integrations-visp-es.md index 73017f12..bc7b2b95 100644 --- a/docs-es/v2.0/integrations-visp-es.md +++ b/docs-es/v2.0/integrations-visp-es.md @@ -13,8 +13,8 @@ Use esta integración cuando VISP sea su fuente de verdad CRM/NMS. ## Notas operativas - `ShapedDevices.csv` se reescribe en cada ejecución. -- `network.json` se sobrescribe solo cuando está habilitado. -- Recomendado: mantener `always_overwrite_network_json = true` para despliegues guiados por integración VISP. +- `network.json` sigue siendo una entrada DIY/manual operada por el usuario. +- Para despliegues guiados por integración, use `topology_import.json` y `network.effective.json`. ## Referencia completa diff --git a/docs-es/v2.0/integrations-wispgate-es.md b/docs-es/v2.0/integrations-wispgate-es.md index 8fa0170d..93d5c0e9 100644 --- a/docs-es/v2.0/integrations-wispgate-es.md +++ b/docs-es/v2.0/integrations-wispgate-es.md @@ -13,8 +13,8 @@ Use esta integración cuando WISPGate sea su fuente de verdad CRM/NMS. ## Notas operativas - `ShapedDevices.csv` se regenera en cada sincronización. -- `network.json` depende de su configuración de sobrescritura. -- Recomendado: mantener `always_overwrite_network_json = true` para alinear topología con WISPGate en cada ciclo. +- `network.json` sigue siendo una entrada DIY/manual operada por el usuario. +- Para despliegues guiados por integración, use `topology_import.json` y `network.effective.json`. ## Referencia completa diff --git a/docs-es/v2.0/node-manager-ui-es.md b/docs-es/v2.0/node-manager-ui-es.md index 8619153c..a88bf5ab 100644 --- a/docs-es/v2.0/node-manager-ui-es.md +++ b/docs-es/v2.0/node-manager-ui-es.md @@ -94,6 +94,7 @@ Esta página documenta las vistas clave de la WebUI (Node Manager) y su comporta - `Devices` muestra tablas de detalle por dispositivo y gráficos en vivo de throughput, retransmisiones y latencia. - `Queue Stats` muestra historial reciente de la cola del circuito, incluyendo backlog, delay, longitud de cola, tráfico, marcas ECN y drops. - Los gráficos de Queue Stats usan hover sincronizado para inspeccionar el mismo segundo en todos los gráficos de cola al mismo tiempo. +- `Queue Stats` debe seguir mostrando la telemetría en vivo de la cola del circuito en modo `flat` igual que en los demás modos de topología. - `Queue Tree` muestra la ruta ascendente de colas del circuito, incluyendo un resumen de ruta y contexto de throughput, retransmisiones y latencia para cada nodo aguas arriba. - `Traffic Flows` es una tabla operativa de flujos recientes, no una vista de historial a largo plazo. - `Traffic Flows` incluye paginación y un filtro `Hide Small Flows` para que los circuitos grandes y ocupados sigan siendo utilizables sin intentar renderizar cada fila. @@ -116,8 +117,13 @@ Esta página documenta las vistas clave de la WebUI (Node Manager) y su comporta - Incluye paginación y filtros. - En el editor dedicado, las acciones de agregar, editar y eliminar se guardan de inmediato. +### Páginas de configuración +- Las contraseñas, los tokens API y las claves guardadas permanecen ocultos al abrir una página de integración. +- Si ya existe una credencial guardada, el campo permanece vacío hasta que escriba un reemplazo. Use `Clear` solo cuando quiera quitar el valor guardado. +- Deje el campo vacío para conservar el valor actual. Escriba uno nuevo para reemplazarlo. + ### Problemas urgentes -- WebUI puede mostrar problemas operativos urgentes emitidos por servicios backend. +- WebUI puede mostrar problemas operativos urgentes informados por los servicios de LibreQoS. - Ejemplos: advertencias de límites de mapeo/licencia y errores de alta prioridad. - Operadores pueden reconocer/limpiar eventos desde la UI. - Códigos comunes: `MAPPED_CIRCUIT_LIMIT` y `TC_U16_OVERFLOW` (ver [Solución de Problemas](troubleshooting-es.md#códigos-de-problemas-urgentes-y-primeras-acciones)). @@ -132,10 +138,10 @@ Esta página documenta las vistas clave de la WebUI (Node Manager) y su comporta ## Modo privacidad / redacción - Se activa con el ícono de máscara en la barra superior. -- Es redacción del lado cliente y se guarda en `localStorage` del navegador. +- Es un ajuste visual que se guarda en su navegador. - Ayuda a ocultar PII en capturas/demos. - Site Map reemplaza los nombres de sitios mostrados por `[redacted]` mientras el modo de redacción está activo. -- No modifica `ShapedDevices.csv`, `network.json` ni datos backend. +- No cambia sus datos guardados ni sus archivos de configuración. ## Comportamiento de vistas vacías diff --git a/docs-es/v2.0/operating-modes-es.md b/docs-es/v2.0/operating-modes-es.md index 35af815a..80a96ad7 100644 --- a/docs-es/v2.0/operating-modes-es.md +++ b/docs-es/v2.0/operating-modes-es.md @@ -2,34 +2,44 @@ ## Propósito de esta página -Use esta página para elegir y aplicar su política de fuente de verdad (integraciones incluidas vs fuente personalizada) antes de cambios en producción. +Use esta página para decidir dónde deben hacerse los cambios permanentes de shaping antes de pasar a producción. ¿Necesita definiciones de términos usados aquí? Vea el [Glosario](glossary-es.md). -LibreQoS soporta dos modos de operación principales. +LibreQoS soporta tres modos de operación comunes. ## Integraciones incluidas (recomendado para la mayoría de operadores) -En este modo, su CRM/NMS es la fuente de verdad y los jobs de integración de LibreQoS generan los insumos de shaping. +En este modo, su CRM/NMS es el lugar donde deben hacerse los cambios permanentes de topología y suscriptores, y LibreQoS se mantiene sincronizado desde allí. Comportamiento clave: -- `ShapedDevices.csv` se regenera en ciclos de sincronización. -- El comportamiento de sobrescritura de `network.json` depende de la configuración de integración (por ejemplo `always_overwrite_network_json`). +- La sincronización de la integración refresca los datos de topología y shaping usados por LibreQoS. +- `network.json` queda reservado para despliegues DIY o manuales. - Las ediciones manuales directas pueden sobrescribirse en el siguiente refresco del scheduler. +- El modo de topología `flat` simplifica el árbol cuando necesita menor sobrecarga. ## Fuente de verdad personalizada -En este modo, sus propios scripts/sistemas generan `network.json` y `ShapedDevices.csv`. +En este modo, sus propios scripts o sistemas generan `network.json` y `ShapedDevices.csv`. Comportamiento clave: -- Su flujo externo controla la persistencia. +- Su flujo externo es donde deben hacerse los cambios permanentes. - Las ediciones por WebUI son válidas para cambios operativos rápidos. -- Los cambios permanentes deben mantenerse en su flujo externo de fuente de verdad. +- Mantenga los cambios de largo plazo en sus scripts o automatización. + +## Modo archivos manuales + +En este modo, usted mantiene `network.json` y `ShapedDevices.csv` directamente. + +Comportamiento clave: +- Es más adecuado para redes pequeñas, pilotos cortos o soluciones temporales. +- La WebUI le ayuda a validar lo que LibreQoS está usando. +- Requiere disciplina manual porque no hay un sistema superior que mantenga esos archivos sincronizados. ## Lista de verificación de modo (antes de producción) -1. Elija una fuente de verdad principal. -2. Confirme qué sistema puede escribir insumos de shaping en producción. +1. Elija un lugar principal para los cambios permanentes. +2. Confirme qué sistema escribe los datos de shaping en producción. 3. Confirme el comportamiento de refresco del scheduler y la cadencia de sobrescritura. 4. Documente su flujo de cambios rápidos (WebUI, editor externo, o ambos). 5. No mantenga ediciones en competencia en varios sistemas para los mismos objetos. @@ -37,8 +47,8 @@ Comportamiento clave: ## Expectativas de topología y modo - Diseños single-interface (on-a-stick) y con VLAN son válidos, pero requieren validación explícita de colas/interfaces después de cambios. -- Modo integración es ideal cuando CRM/NMS debe controlar topología y datos de suscriptores. -- Si necesita topología estrictamente personalizada no representada por integración, use modo fuente personalizada y mantenga propiedad clara. +- El modo integración es ideal cuando CRM/NMS debe controlar topología y datos de suscriptores. +- Si necesita una topología que su integración no puede representar, use modo fuente personalizada y mantenga clara la responsabilidad. Vea también: - [Referencia avanzada de configuración](configuration-advanced-es.md) diff --git a/docs-es/v2.0/quickstart-es.md b/docs-es/v2.0/quickstart-es.md index 3b5de29e..b16a54cd 100644 --- a/docs-es/v2.0/quickstart-es.md +++ b/docs-es/v2.0/quickstart-es.md @@ -83,15 +83,15 @@ Si elige laboratorio primero: 3. Valide Dashboard, Tree, Flow, Scheduler Status y Urgent Issues. 4. Continúe a la Decisión B. -## 4) Decisión B: Fuente de verdad (elija un dueño) +## 4) Decisión B: Elija dónde hará los cambios permanentes -| Si esto describe su caso | Modo | Dueño de datos de shaping durables | +| Si esto describe su caso | Modo | Dónde deben hacerse los cambios permanentes | |---|---|---| | Usa integración CRM/NMS soportada | Modo integración incluida | Jobs de integración | | Genera `network.json` y `ShapedDevices.csv` con scripts propios | Modo fuente de verdad personalizada | Sus scripts | | Mantiene archivos manualmente en red pequeña/simple | Modo archivos manuales | Edición manual | -Regla: mantenga un único dueño para insumos persistentes de shaping. +Regla: mantenga un solo lugar para los cambios permanentes de shaping. ## 5) Tarjetas de ruta @@ -118,7 +118,7 @@ Cuándo elegir: Haga esto ahora: 1. Implemente script/proceso para generar y refrescar archivos de shaping. -2. Declare salidas del script como su fuente de verdad. +2. Trate las salidas del script como el lugar donde hará los cambios permanentes. 3. Coloque LibreQoS inline para tráfico piloto. 4. Use WebUI para checks operativos y ajustes de corto plazo. 5. Mantenga cambios permanentes en su flujo externo de scripts. @@ -148,7 +148,7 @@ Siguiente: ## 6) Errores comunes en primera puesta en marcha -- Propiedad poco clara de la fuente de verdad entre integración y edición manual. +- No decidir con claridad dónde hará los cambios permanentes. - Cambiar profundidad topológica antes de pasar la puerta de salud. - Omitir validación de servicios/logs antes de tráfico piloto. diff --git a/docs-es/v2.0/recipes-wisp-fisp-integration-es.md b/docs-es/v2.0/recipes-wisp-fisp-integration-es.md index a826ad07..5c318881 100644 --- a/docs-es/v2.0/recipes-wisp-fisp-integration-es.md +++ b/docs-es/v2.0/recipes-wisp-fisp-integration-es.md @@ -16,7 +16,7 @@ Use este patron cuando su operacion este centrada en un CRM/NMS soportado (UISP, ## Implementacion 1. Configure credenciales y parametros en WebUI (`Configuration -> Integrations`). -2. Para despliegues guiados por integracion, mantenga `always_overwrite_network_json = true`. +2. Para despliegues guiados por integración, trate `network.json` como solo DIY/manual y deje que la integración publique `topology_import.json`. 3. Elija la estrategia de topologia mas liviana que cumpla su necesidad. | Necesidad | Estrategia sugerida | diff --git a/docs-es/v2.0/troubleshooting-es.md b/docs-es/v2.0/troubleshooting-es.md index 0dc21a88..f9970518 100644 --- a/docs-es/v2.0/troubleshooting-es.md +++ b/docs-es/v2.0/troubleshooting-es.md @@ -35,6 +35,12 @@ journalctl -u lqos_scheduler --since "30 minutes ago" Si el problema es de integración, agregue: +```bash +ls -lh /opt/libreqos/src/topology_import.json /opt/libreqos/src/shaping_inputs.json +``` + +Si usa un despliegue manual o con archivos propios en lugar de una integración incluida, agregue: + ```bash ls -lh /opt/libreqos/src/network.json /opt/libreqos/src/ShapedDevices.csv ``` diff --git a/docs/v2.0/case-studies.md b/docs/v2.0/case-studies.md index 6a32fc2b..0f2855a9 100644 --- a/docs/v2.0/case-studies.md +++ b/docs/v2.0/case-studies.md @@ -20,7 +20,7 @@ Situation: Approach: - Adopted built-in UISP integration as durable source of truth. -- Standardized on integration-owned `ShapedDevices.csv` with explicit overwrite policy. +- Standardized on integration-driven shaping data with a clear refresh policy. - Started with moderate hierarchy depth before considering deeper topology. Outcome: diff --git a/docs/v2.0/configuration.md b/docs/v2.0/configuration.md index fcfb2070..fd128073 100644 --- a/docs/v2.0/configuration.md +++ b/docs/v2.0/configuration.md @@ -27,7 +27,7 @@ For most operators: Most day-to-day LibreQoS configuration is handled in WebUI (`http://your_shaper_ip:9123/config_general.html`). -Current builds use shared configuration panels on the General, RTT, Queues, TreeGuard, Network Mode, Integration Defaults, Network Layout, Insight, provider-specific integration pages, IP Ranges, Flow Tracking, and Shaped Devices pages so node settings and operational tuning follow the same surface styling as the rest of Node Manager. The TreeGuard page starts with a live behavior summary rather than a permanent warning banner. Integration Defaults also carries the shared Ethernet port headroom policy used by integrations that can detect negotiated subscriber-facing port speeds. +Current builds use a consistent configuration layout across the General, RTT, Queues, TreeGuard, Network Mode, Integration Defaults, Network Layout, Insight, provider integration, IP Ranges, Flow Tracking, and Shaped Devices pages. Integration Defaults also includes the shared Ethernet port headroom policy used by integrations that can detect negotiated subscriber-facing port speeds. ### Where in WebUI @@ -37,11 +37,11 @@ Current builds use shared configuration panels on the General, RTT, Queues, Tree - Shaped devices editor: `Configuration -> Shaped Devices` - Runtime operational validation: `WebUI (Node Manager)` pages (dashboard/tree/flow/scheduler) -When an integration is enabled and acting as the source of truth, the `Network Layout` and `Shaped Devices` editors remain viewable but become read-only in WebUI. +When an integration is managing your shaping data, the `Network Layout` and `Shaped Devices` editors remain visible but become read-only in WebUI. Topology note: - `network.json` node names must be globally unique across the whole tree. Duplicate node names now fail validation and are not accepted by the WebUI save path or `LibreQoS.py`. -- When a node exposes a stable `id`, current builds prefer that ID for operator-owned site bandwidth overrides while keeping legacy name-only matching as a fallback. +- When a node exposes a stable `id`, LibreQoS prefers that ID for saved site bandwidth overrides while keeping legacy name-only matching as a fallback. Queue-mode note: - Current builds use `queue_mode` with `shape` and `observe` values. Older `monitor_only` wording is a compatibility alias rather than the primary operator-facing setting. diff --git a/docs/v2.0/future-development-inputs.md b/docs/v2.0/future-development-inputs.md index ee4ff426..b24f2265 100644 --- a/docs/v2.0/future-development-inputs.md +++ b/docs/v2.0/future-development-inputs.md @@ -42,8 +42,8 @@ Example symptoms: Representative issues: -- [#860 - always_overwrite_network_json default behavior confusion](https://github.com/LibreQoE/LibreQoS/issues/860) -- [#899 - UISP: always_overwrite_network_json=false and missing ShapedDevices.csv](https://github.com/LibreQoE/LibreQoS/issues/899) +- [#860 - legacy network.json overwrite-default confusion](https://github.com/LibreQoE/LibreQoS/issues/860) +- [#899 - UISP: legacy overwrite-disabled path and missing ShapedDevices.csv](https://github.com/LibreQoE/LibreQoS/issues/899) - [#845 - UISP: multi-services with same site name](https://github.com/LibreQoE/LibreQoS/issues/845) - [#699 - UISP: trailing spaces break matching](https://github.com/LibreQoE/LibreQoS/issues/699) diff --git a/docs/v2.0/integrations-netzur.md b/docs/v2.0/integrations-netzur.md index 56cb2c07..ee9284ca 100644 --- a/docs/v2.0/integrations-netzur.md +++ b/docs/v2.0/integrations-netzur.md @@ -23,10 +23,9 @@ Run a manual import with: python3 integrationNetzur.py ``` -The integration regenerates `ShapedDevices.csv` and, unless `always_overwrite_network_json` is disabled, updates `network.json`. Adjust the Integration → Common settings if you need to preserve an existing network map between Netzur syncs. +The integration regenerates `ShapedDevices.csv` for its legacy DIY-style path, but built-in integrations do not write `network.json`. Keep `network.json` for DIY/manual deployments. -Overwrite policy: -- Recommended: keep `always_overwrite_network_json = true` for integration-driven deployments so topology stays aligned with Netzur syncs. +For integration-driven workflows, validate the import in WebUI and through the current topology/shaping files rather than editing `network.json`. ## Related Pages diff --git a/docs/v2.0/integrations-reference.md b/docs/v2.0/integrations-reference.md index 4d89c594..80c82e32 100644 --- a/docs/v2.0/integrations-reference.md +++ b/docs/v2.0/integrations-reference.md @@ -25,16 +25,16 @@ If you use your own scripts as the source of truth for `network.json` and `Shape ## What these integrations do -These integrations work by synchronizing with your CRM/NMS system to produce the two files LibreQoS requires for shaping - [network.json](configuration.md#network-hierarchy) and [ShapedDevices.csv](configuration.md#circuits). +These integrations synchronize with your CRM/NMS system to produce integration-owned topology and shaping artifacts for LibreQoS. Important behavior when integrations are enabled: -- `ShapedDevices.csv` is typically regenerated by integration sync jobs. -- `network.json` overwrite behavior depends on your integration settings (for example `always_overwrite_network_json`). +- Built-in integrations publish integration-owned topology and shaping artifacts. +- `network.json` remains a DIY/manual ingress file and is not written by built-in integrations. - Manual edits may be overwritten on the next refresh cycle. ## Splynx Integration -> **⚠️ Breaking Change Notice**: Prior to v1.5-RC-2, the default Splynx strategy was `full`. Starting with v1.5-RC-2, the default strategy is `ap_only` for improved CPU performance. If you require the previous behavior, explicitly set `strategy = "full"` in your Splynx configuration section. +> **⚠️ Breaking Change Notice**: Prior to v1.5-RC-2, the default Splynx strategy was `full`. The default shared topology mode is now `ap_site` as the best balance of structure and operator clarity. If you require the previous behavior, explicitly set `strategy = "full"` in your Splynx configuration section. First, set the relevant parameters for Splynx (splynx_api_key, splynx_api_secret, etc.) in `/etc/lqos.conf`. @@ -45,8 +45,8 @@ LibreQoS supports multiple topology strategies for Splynx integration to balance | Strategy | Description | CPU Impact | Use Case | |----------|-------------|------------|----------| | `flat` | Only shapes subscribers, no hierarchy | Lowest | Maximum performance, simple subscriber-only shaping | -| `ap_only` | Single layer: AP → Clients | Low | **Default**. Best balance of performance and structure | -| `ap_site` | Two layers: Site → AP → Clients | Medium | Site-level aggregation with moderate complexity | +| `ap_only` | Single layer: AP → Clients | Low | Lowest overhead when site-level grouping is not needed | +| `ap_site` | Two layers: Site → AP → Clients | Medium | **Default**. Best balance of structure and operator clarity | | `full` | Complete topology mapping | Highest | Full network hierarchy representation | Configure the strategy in `/etc/lqos.conf` under the `[splynx_integration]` section: @@ -54,12 +54,12 @@ Configure the strategy in `/etc/lqos.conf` under the `[splynx_integration]` sect ```ini [splynx_integration] # ... other splynx settings ... -strategy = "ap_only" +strategy = "ap_site" ``` **Performance Considerations:** - `flat` and `ap_only` strategies significantly reduce CPU load by limiting network depth -- Choose `ap_only` for most deployments unless you need site-level traffic aggregation +- Choose `ap_site` by default, and move to `ap_only` only when you want lower overhead and can give up site-level grouping - Only use `full` if you require complete network topology representation and have adequate CPU resources ### Promote to Root Nodes (Performance Optimization) @@ -122,12 +122,8 @@ To test the Splynx Integration, use python3 integrationSplynx.py ``` -On the first successful run, it will create a ShapedDevices.csv file and network.json. -ShapedDevices.csv will be overwritten every time the Splynx integration is run. - -To ensure the network.json is always overwritten with the newest version pulled in by the integration, please edit `/etc/lqos.conf` with the command `sudo nano /etc/lqos.conf`. -Edit the file to set the value of `always_overwrite_network_json` to `true`. -Then, run `sudo systemctl restart lqosd`. +On the first successful run, it will create integration-owned shaping data. +Built-in integrations do not overwrite `network.json`; keep DIY `network.json` operator-owned. You have the option to run integrationSplynx.py automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_splynx = true``` under the ```[splynx_integration]``` section in `/etc/lqos.conf`. Once set, run `sudo systemctl restart lqos_scheduler`. @@ -167,7 +163,7 @@ Run a manual import with: python3 integrationNetzur.py ``` -The integration regenerates `ShapedDevices.csv` and, unless `always_overwrite_network_json` is disabled, updates `network.json`. Adjust the Integration → Common settings if you need to preserve an existing network map between Netzur syncs. +The integration regenerates `ShapedDevices.csv` for its legacy DIY-style path, but built-in integrations no longer write `network.json`. Keep DIY `network.json` operator-owned. ## VISP Integration @@ -190,7 +186,7 @@ timeout_secs = 20 Notes: - VISP import is GraphQL-based and currently defaults to a flat topology strategy. - The integration writes `ShapedDevices.csv` every run. -- `network.json` is only overwritten when `always_overwrite_network_json = true` (under `[integration_common]`). +- `network.json` remains a DIY/manual ingress file; built-in integrations do not overwrite it. - VISP auth tokens are cached in `/.visp_token_cache_*.json`. Run a manual import with: @@ -223,7 +219,7 @@ LibreQoS supports multiple topology strategies for UISP integration to balance C | `full` | Shapes entire network including backhauls, Sites, APs, and clients | Highest | Best after topology/overrides are validated and stable | **Choosing the Right Strategy:** -- Start with `ap_only` for new deployments and initial validation +- Start with `ap_site` for new deployments and initial validation - Move to `ap_site` when you need site-level control but not backhaul shaping - Move to `full` after topology quality and overrides are validated, and CPU headroom is confirmed - Use `flat` only when maximum performance is critical and you don't need any hierarchy @@ -278,7 +274,7 @@ url = "https://uisp.your_domain.com" site = "Root_Site_Name" # Root site for topology perspective # Topology Strategy (see table above) -strategy = "ap_only" # Recommended starting point for new UISP deployments +strategy = "ap_site" # Recommended starting point for new UISP deployments # Suspension Handling (see table above) suspended_strategy = "none" @@ -300,7 +296,6 @@ commit_bandwidth_multiplier = 0.98 # Set minimum to 98% of maximum (CIR) # Advanced Options ipv6_with_mikrotik = false # Enable if using DHCPv6 with MikroTik -always_overwrite_network_json = true # Recommended for integration-driven deployments exception_cpes = [] # CPE exceptions in ["cpe:parent"] format squash_sites = [] # Optional: sites to squash do_not_squash_sites = [] # Optional: keep these site names unsquashed in the runtime/export tree @@ -341,15 +336,13 @@ sudo /opt/libreqos/src/bin/uisp_integration ``` On the first successful run, it will create a network.json and ShapedDevices.csv file. -If a network.json file exists, it will not be overwritten, unless you set ```always_overwrite_network_json = true```. +Built-in integrations do not overwrite an existing `network.json`; keep DIY `network.json` operator-owned. ShapedDevices.csv will be overwritten every time the UISP integration is run. If UISP exposes a site and an AP with the same visible name in the same topology, current builds keep the site name stable in `network.json` and disambiguate the AP name during export so the site branch is not dropped from the tree. -To ensure the network.json is always overwritten with the newest version pulled in by the integration, please edit `/etc/lqos.conf` with the command `sudo nano /etc/lqos.conf`. -Edit the file to set the value of `always_overwrite_network_json` to `true`. -Then, run `sudo systemctl restart lqosd`. +Built-in integrations do not overwrite `network.json`; keep DIY `network.json` operator-owned. You have the option to run `uisp_integration` automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_uisp = true``` in `/etc/lqos.conf`. Once set, run `sudo systemctl restart lqos_scheduler`. @@ -394,9 +387,7 @@ python3 integrationWISPGate.py On the first successful run, it will create a ShapedDevices.csv file and network.json. ShapedDevices.csv will be overwritten every time the WISPGate integration is run. -To ensure the network.json is always overwritten with the newest version pulled in by the integration, please edit `/etc/lqos.conf` with the command `sudo nano /etc/lqos.conf`. -Edit the file to set the value of `always_overwrite_network_json` to `true`. -Then, run `sudo systemctl restart lqosd`. +Built-in integrations do not overwrite `network.json`; keep DIY `network.json` operator-owned. You have the option to run integrationWISPGate.py automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_wispgate = true``` in `/etc/lqos.conf`. Once set, run `sudo systemctl restart lqos_scheduler`. @@ -427,7 +418,7 @@ python3 integrationSonar.py ``` On the first successful run, it will create a ShapedDevices.csv file. -If a network.json file exists, it will not be overwritten, unless you set ```always_overwrite_network_json = true```. +Built-in integrations do not overwrite an existing `network.json`; keep DIY `network.json` operator-owned. You can modify the network.json file to more accurately reflect bandwidth limits. ShapedDevices.csv will be overwritten every time the Sonar integration is run. You have the option to run integrationSonar.py automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_sonar = true``` in `/etc/lqos.conf`. diff --git a/docs/v2.0/integrations-sonar.md b/docs/v2.0/integrations-sonar.md index d5c93ebb..12184f46 100644 --- a/docs/v2.0/integrations-sonar.md +++ b/docs/v2.0/integrations-sonar.md @@ -18,11 +18,10 @@ To test the Sonar Integration, use python3 integrationSonar.py ``` -On the first successful run, it will create a ShapedDevices.csv file. -If a network.json file exists, it will not be overwritten, unless you set ```always_overwrite_network_json = true```. -You can modify the network.json file to more accurately reflect bandwidth limits. -ShapedDevices.csv will be overwritten every time the Sonar integration is run. -Recommended: keep `always_overwrite_network_json = true` for integration-driven deployments so topology stays aligned with Sonar syncs. +On the first successful run, it creates the Sonar import and shaping data LibreQoS needs for scheduled refreshes. +Built-in Sonar integrations do not write `network.json`; keep that file for DIY/manual deployments. +You can maintain a separate DIY `network.json` when Sonar is not the system you want to use for topology. +The Sonar integration refreshes its imported topology and shaping data every run. You have the option to run integrationSonar.py automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_sonar = true``` in `/etc/lqos.conf`. diff --git a/docs/v2.0/integrations-splynx.md b/docs/v2.0/integrations-splynx.md index 058cf9eb..ac020c35 100644 --- a/docs/v2.0/integrations-splynx.md +++ b/docs/v2.0/integrations-splynx.md @@ -1,6 +1,6 @@ # Splynx Integration -> **⚠️ Breaking Change Notice**: Prior to v1.5-RC-2, the default Splynx strategy was `full`. Starting with v1.5-RC-2, the default strategy is `ap_only` for improved CPU performance. If you require the previous behavior, explicitly set `strategy = "full"` in your Splynx configuration section. +> **⚠️ Breaking Change Notice**: Prior to v1.5-RC-2, the default Splynx strategy was `full`. The recommended shared topology mode is now `ap_site` as the best default balance of structure and operator clarity. If you require the previous behavior, explicitly set `topology.compile_mode = "full"`. First, set the relevant parameters for Splynx (splynx_api_key, splynx_api_secret, etc.) in `/etc/lqos.conf`. @@ -8,15 +8,11 @@ First, set the relevant parameters for Splynx (splynx_api_key, splynx_api_secret Start with this baseline unless you have a known reason to deviate: -- `strategy = "ap_only"` (default, lowest confusion) +- `topology.compile_mode = "ap_site"` (default) - `enable_splynx = true` -- `always_overwrite_network_json = true` for integration-driven deployments Then run one manual sync and validate outputs before enabling frequent scheduler refresh cycles. -Overwrite policy: -- Recommended: keep `always_overwrite_network_json = true` so topology stays aligned with Splynx on each refresh cycle. - ### Topology Strategies LibreQoS supports multiple topology strategies for Splynx integration to balance CPU performance with network hierarchy needs: @@ -24,21 +20,22 @@ LibreQoS supports multiple topology strategies for Splynx integration to balance | Strategy | Description | CPU Impact | Use Case | |----------|-------------|------------|----------| | `flat` | Only shapes subscribers, no hierarchy | Lowest | Maximum performance, simple subscriber-only shaping | -| `ap_only` | Single layer: AP → Clients | Low | **Default**. Best balance of performance and structure | -| `ap_site` | Two layers: Site → AP → Clients | Medium | Site-level aggregation with moderate complexity | -| `full` | Complete topology mapping | Highest | Full network hierarchy representation | +| `ap_only` | Single layer: AP → Clients | Low | Lowest overhead when site-level grouping is not needed | +| `ap_site` | Two layers: Site → AP → Clients | Medium | **Default**. Best balance of structure and operator clarity | +| `full` | Complete monitoring-parent topology mapping | Highest | Preserves the richer Splynx monitoring hierarchy instead of flattening to Network Sites | -Configure the strategy in `/etc/lqos.conf` under the `[splynx_integration]` section: +When Splynx `full` cannot infer a circuit parent from `access_device` or router metadata, LibreQoS now groups those circuits under a single generated `LibreQoS Unattached [Site]` node instead of leaving them fully unresolved at runtime. + +Configure the shared topology mode in `/etc/lqos.conf` under the `[topology]` section: ```ini -[splynx_integration] -# ... other splynx settings ... -strategy = "ap_only" +[topology] +compile_mode = "ap_site" ``` **Performance Considerations:** - `flat` and `ap_only` strategies significantly reduce CPU load by limiting network depth -- Choose `ap_only` for most deployments unless you need site-level traffic aggregation +- Choose `ap_site` by default, and move to `ap_only` only when you want lower overhead and can give up site-level grouping - Only use `full` if you require complete network topology representation and have adequate CPU resources ### Promote to Root Nodes (Performance Optimization) @@ -101,10 +98,12 @@ To test the Splynx Integration, use python3 integrationSplynx.py ``` -On the first successful run, it creates `ShapedDevices.csv` and `network.json`. -ShapedDevices.csv will be overwritten every time the Splynx integration is run. +On the first successful run, it creates the Splynx import and shaping files LibreQoS needs for scheduled refreshes. +Legacy `integrationSplynxBandwidths.csv` values are migrated into operator `AdjustSiteSpeed` overrides in `lqos_overrides.json`, and the original CSV is renamed to a `.backup` file. Current builds also expose a shared Ethernet port headroom policy under `Configuration -> Integrations -> Integration Defaults`. Integrations that can supply negotiated subscriber-facing port speed use a conservative default multiplier of `0.94` unless the operator overrides it. +Current builds also expose the shared topology compile mode under `Configuration -> Integrations -> Integration Defaults`. Existing `splynx_integration.strategy` values are retained only as a compatibility mirror during upgrade. +Topology Manager now receives a native infrastructure topology export for Splynx imports, so real sites and APs use their resolved parent chain instead of being reconstructed from the compatibility tree. Customer-derived generated sites stay compatibility-only: they are excluded from canonical/editor topology and circuits shape under the nearest real infrastructure parent instead. Current Python-backed imports are intentionally conservative: infrastructure nodes import as fixed roots or fixed-parent branches unless the importer can provide a bounded, trustworthy set of alternative parents. When a bounded move set is available, it is limited to the node's current parent plus nearby sibling-parent alternatives under the same upstream branch, or peer root parents of the same type when the current parent is itself a root. You have the option to run integrationSplynx.py automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_splynx = true``` under the ```[splynx_integration]``` section in `/etc/lqos.conf`. Once set, run `sudo systemctl restart lqos_scheduler`. @@ -117,8 +116,9 @@ python3 integrationSplynx.py ``` 2. Confirm files exist and were updated: ```shell -ls -lh /opt/libreqos/src/network.json /opt/libreqos/src/ShapedDevices.csv +ls -lh /opt/libreqos/src/topology_import.json /opt/libreqos/src/shaping_inputs.json ``` +Built-in Splynx imports refresh LibreQoS's imported topology and shaping data automatically. They do not use `network.json` or `ShapedDevices.csv` as the normal working files for this integration. 3. Confirm services are healthy: ```shell sudo systemctl status lqosd lqos_scheduler @@ -132,7 +132,7 @@ If sync looks successful but data is incomplete, recheck API permissions and str You can also modify the the file `integrationSplynxBandwidths.csv` to override the default bandwidths for each Node (Site, AP). -Tree-page `Operator Override` edits are separate from this legacy Splynx file. Current builds write those operator-owned node rate changes to `lqos_overrides.json` and do not rewrite `integrationSplynxBandwidths.csv`. +Rate overrides saved from the tree page are separate from this legacy Splynx CSV workflow. Pick one long-term override method for a given node so your changes stay predictable. A template is available in the `/opt/libreqos/src` folder. To utilize the template, copy the file `integrationSplynxBandwidths.template.csv` (removing the `.template` part of the filename) and set the appropriate information inside each file. For example, if you want to change the set bandwidth for a site, you would do: ``` @@ -140,7 +140,7 @@ sudo cp /opt/libreqos/src/integrationSplynxBandwidths.template.csv /opt/libreqos ``` And edit the CSV using LibreOffice or your preferred CSV editor. -To avoid conflicting sources of truth, prefer one durable override path per node: either the legacy Splynx CSV workflow or the operator override layer. +To avoid conflicting edits, prefer one long-term override path per node: either the legacy Splynx CSV workflow or WebUI overrides. ## Related Pages diff --git a/docs/v2.0/integrations-uisp.md b/docs/v2.0/integrations-uisp.md index 9caddbdf..16a43337 100644 --- a/docs/v2.0/integrations-uisp.md +++ b/docs/v2.0/integrations-uisp.md @@ -4,9 +4,9 @@ First, set the relevant parameters for UISP in `/etc/lqos.conf`. ## New Operator Quick Chooser -Use this section first to avoid common strategy confusion. +Use this section first to avoid common topology-mode confusion. -1. If you are new to UISP integration, start with `strategy = "ap_only"`. +1. If you are new to UISP integration, start with `topology.compile_mode = "ap_site"` under `Integration - Common`. 2. Move to `ap_site` when you need explicit site-level aggregation. 3. Use `full` when you need full hierarchy/backhaul representation and have CPU headroom. 4. Use `flat` only when hierarchy is not needed and maximum performance is the priority. @@ -39,8 +39,9 @@ sudo /opt/libreqos/src/bin/uisp_integration ``` 2. Confirm outputs were generated/refreshed as expected: ```shell -ls -lh /opt/libreqos/src/network.json /opt/libreqos/src/ShapedDevices.csv +ls -lh /opt/libreqos/src/topology_import.json /opt/libreqos/src/shaping_inputs.json ``` +Built-in UISP imports refresh LibreQoS's imported topology and shaping data automatically. They do not use `network.json`, `ShapedDevices.csv`, or a standalone `circuit_anchors.json` as the normal working files for this integration. 3. Confirm scheduler and shaper health: ```shell sudo systemctl status lqosd lqos_scheduler @@ -48,9 +49,9 @@ journalctl -u lqos_scheduler --since "30 minutes ago" ``` 4. Validate result in WebUI: - Scheduler Status is healthy -- Tree/Flow views reflect expected hierarchy depth for selected strategy +- Tree/Flow views reflect expected hierarchy depth for the selected compile mode -If hierarchy depth or parent mapping is not what you expect, revisit `strategy`, `use_ptmp_as_parent`, `exclude_sites`, and `exception_cpes` before changing other settings. +If hierarchy depth or parent mapping is not what you expect, revisit `topology.compile_mode`, `use_ptmp_as_parent`, `exclude_sites`, and `exception_cpes` before changing other settings. ### Promote to Root Nodes (Performance Optimization) @@ -113,6 +114,10 @@ Configure how LibreQoS handles suspended customer accounts: ### Configuration Example ```ini +[topology] +# Shared topology compile mode (see table above) +compile_mode = "ap_site" # Recommended starting point for new UISP deployments + [uisp_integration] # Core Settings enable_uisp = true @@ -120,17 +125,14 @@ token = "your-api-token-here" url = "https://uisp.your_domain.com" site = "Root_Site_Name" # Root site for topology perspective -# Topology Strategy (see table above) -strategy = "ap_only" # Recommended starting point for new UISP deployments - # Suspension Handling (see table above) suspended_strategy = "none" # Capacity Adjustments -# UISP's reported AP capacities can be optimistic -airmax_capacity = 0.8 # Use 80% of reported AirMax capacity on new installs +# Current defaults normalize both AP capacity multipliers to 1.0 +airmax_capacity = 1.0 # Use 100% of reported AirMax capacity airmax_flexible_frame_download_ratio = 0.8 # Fallback split for AirMax flexible framing when UISP does not expose dlRatio -ltu_capacity = 1.0 # Use 100% of reported LTU capacity on new installs +ltu_capacity = 1.0 # Use 100% of reported LTU capacity infrastructure_transport_caps_enabled = true # Automatically cap radio capacity to active/model Ethernet transport ceilings # Site Management @@ -143,7 +145,7 @@ commit_bandwidth_multiplier = 0.98 # Set minimum to 98% of maximum (CIR) # Advanced Options ipv6_with_mikrotik = false # Enable if using DHCPv6 with MikroTik -always_overwrite_network_json = true # Recommended when using UISP integration in production +# `network.json` is for DIY/manual deployments; built-in integrations do not write it exception_cpes = [] # CPE exceptions in ["cpe:parent"] format squash_sites = [] # Optional: sites to squash do_not_squash_sites = [] # Optional: keep these site names unsquashed in the runtime/export tree @@ -184,32 +186,28 @@ Recommended use: Legacy note: - Existing `enable_squashing` values in `/etc/lqos.conf` are ignored for backward compatibility. +- Existing `uisp_integration.strategy` values are retained only as a compatibility mirror. Current builds read the shared `topology.compile_mode` first. -On the first successful run, the integration creates `network.json` and `ShapedDevices.csv`. -If a `network.json` file exists, it is only overwritten when `always_overwrite_network_json = true`. +On the first successful run, the integration creates the UISP import and shaping files LibreQoS needs for scheduled refreshes. -ShapedDevices.csv will be overwritten every time the UISP integration is run. - -If UISP exposes a site and an AP with the same visible name in the same topology, current builds keep the site name stable in `network.json` and disambiguate the AP name during export so the site branch is not dropped from the tree. +If UISP exposes a site and an AP with the same visible name in the same topology, current builds keep them distinct so the site branch is not dropped from runtime views. When UISP client sites share the same name, LibreQoS now tries to disambiguate the generated circuit/site display names with a human-friendly suffix such as the first street-address segment, falling back to service name and then a short ID only when needed. Stable circuit identity still comes from the UISP site/service ID, not the display name. -For integration-driven deployments, keep `always_overwrite_network_json = true` so topology stays aligned with UISP on each refresh cycle. - You have the option to run `uisp_integration` automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_uisp = true``` in `/etc/lqos.conf`. Once set, run `sudo systemctl restart lqos_scheduler`. ### UISP Overrides You can also use the following override inputs to more accurately reflect your network: -- Tree-page `Rate Override` edits, stored as operator `AdjustSiteSpeed` entries in `lqos_overrides.json` -- Tree-page `Topology Override` edits for supported UISP `full` nodes, stored in `lqos_overrides.json` +- Tree-page `Rate Override` edits, stored as `AdjustSiteSpeed` entries in `lqos_overrides.json` +- Tree-page `Topology Override` edits for supported UISP `full` nodes, also stored in `lqos_overrides.json` - integrationUISPbandwidths.csv as a legacy compatibility input only -Current UISP builds auto-migrate a legacy `integrationUISPbandwidths.csv` into operator `AdjustSiteSpeed` overrides on the next integration run when no operator rate overrides exist yet. If operator rate overrides already exist, the CSV is ignored and a warning is logged so there is only one active source of truth. -Deprecated legacy `uisp.bandwidth_overrides` JSON entries are ignored. The authoritative bandwidth override path is operator `AdjustSiteSpeed` in `lqos_overrides.json`. +Current UISP builds auto-migrate a legacy `integrationUISPbandwidths.csv` into `AdjustSiteSpeed` overrides on the next integration run when no newer rate overrides exist yet. If newer rate overrides already exist, the CSV is ignored and a warning is logged so only one override method stays active. +Deprecated legacy `uisp.bandwidth_overrides` JSON entries are ignored. The supported long-term bandwidth override path is `AdjustSiteSpeed` in `lqos_overrides.json`. Current UISP builds ignore legacy `uisp.route_overrides` entries in `lqos_overrides.json` and legacy `integrationUISProutes.csv` files. If either is present, LibreQoS logs a warning and uses detected topology plus Topology Manager overrides instead. -UISP `full` strategy builds also expose tree-page `Topology Override` editing for supported nodes. These overrides are stored in `lqos_overrides.json` and resolve before final `network.json` / `ShapedDevices.csv` emission. Current WebUI support is `Pinned Parent` only. +UISP `full` strategy builds also expose tree-page `Topology Override` editing for supported nodes. Current WebUI support is `Pinned Parent` only. Each of the files above have templates available in the `/opt/libreqos/src` folder. If you don't find them there, you can navigate [here](https://github.com/LibreQoE/LibreQoS/tree/develop/src). diff --git a/docs/v2.0/integrations-visp.md b/docs/v2.0/integrations-visp.md index b27b45ca..0320bcc0 100644 --- a/docs/v2.0/integrations-visp.md +++ b/docs/v2.0/integrations-visp.md @@ -18,9 +18,8 @@ timeout_secs = 20 Notes: - VISP import is GraphQL-based and currently defaults to a flat topology strategy. -- The integration writes `ShapedDevices.csv` every run. -- `network.json` is only overwritten when `always_overwrite_network_json = true` (under `[integration_common]`). -- Recommended: keep `always_overwrite_network_json = true` for integration-driven deployments so topology stays aligned with VISP syncs. +- The integration refreshes its shaping data every run. +- `network.json` is for DIY/manual deployments; built-in integrations do not write it. - VISP auth tokens are cached in `/.visp_token_cache_*.json`. Run a manual import with: diff --git a/docs/v2.0/integrations-wispgate.md b/docs/v2.0/integrations-wispgate.md index 896cdde0..2132c6a3 100644 --- a/docs/v2.0/integrations-wispgate.md +++ b/docs/v2.0/integrations-wispgate.md @@ -19,9 +19,9 @@ To test the WISPGate Integration, use python3 integrationWISPGate.py ``` -On the first successful run, it will create a ShapedDevices.csv file and network.json. -ShapedDevices.csv will be overwritten every time the WISPGate integration is run. -Recommended: keep `always_overwrite_network_json = true` for integration-driven deployments so topology stays aligned with WISPGate syncs. +On the first successful run, it creates the WISPGate import and shaping data LibreQoS needs for scheduled refreshes. +Those files are refreshed every time the WISPGate integration runs. +Built-in integrations do not write `network.json`; keep that file for DIY/manual deployments. You have the option to run integrationWISPGate.py automatically on boot and every X minutes (set by the parameter `queue_refresh_interval_mins`), which is highly recommended. This can be enabled by setting ```enable_wispgate = true``` in `/etc/lqos.conf`. Once set, run `sudo systemctl restart lqos_scheduler`. diff --git a/docs/v2.0/integrations.md b/docs/v2.0/integrations.md index 867c9cf1..4e2ac011 100644 --- a/docs/v2.0/integrations.md +++ b/docs/v2.0/integrations.md @@ -2,9 +2,9 @@ ## Page Purpose -Use this page to choose and configure supported CRM/NMS integrations. Use per-integration pages for specific setup details. +Use this page to choose and configure a supported CRM/NMS integration. Use the per-integration pages for setup details. -Need definitions for overwrite/persistence terms? See the [Glossary](glossary.md). +Need definitions for common terms? See the [Glossary](glossary.md). Most operators use built-in integrations. If you have not selected a deployment path yet, start with [Quickstart](quickstart.md). @@ -12,17 +12,15 @@ If you have not selected a deployment path yet, start with [Quickstart](quicksta ```{mermaid} flowchart LR A[CRM/NMS] --> B[LibreQoS Integration Job] - B --> C[ShapedDevices.csv] - B --> D[network.json] - C --> E[lqos_scheduler] - D --> E + B --> H[topology_import.json] + H --> E[lqos_scheduler] E --> F[Shaping Updates] E --> G[WebUI Status / Urgent Issues] ``` ## Choose Your Integration Path -| Path | Best fit | Primary source of truth | +| Path | Best fit | Where permanent shaping changes belong | |---|---|---| | Built-in integration | Most operators using supported systems | CRM/NMS via LibreQoS integration jobs | | Custom source of truth | Operators with in-house CRM/NMS sync logic | External generated files (`network.json`, `ShapedDevices.csv`) | @@ -44,20 +42,33 @@ flowchart LR - [Powercode Integration](integrations-powercode.md) - [Sonar Integration](integrations-sonar.md) -## Important Overwrite Behavior +## Important Refresh Behavior When integrations are enabled: -- `ShapedDevices.csv` is typically regenerated by integration sync jobs. -- `network.json` may also be overwritten depending on integration settings (for example `always_overwrite_network_json`). +- Integration sync refreshes the imported topology and shaping data LibreQoS uses. +- Built-in integrations do not use `network.json` or `ShapedDevices.csv` as their normal output files. +- Built-in integrations provide native infrastructure topology to Topology Manager; legacy compatibility trees remain derived/export-only data. +- Python-backed integrations currently import infrastructure conservatively as fixed roots or fixed-parent branches. When bounded alternative-parent candidates are available, they are limited to the node's current local parent neighborhood, plus peer root parents of the same type when the current parent is itself a root, rather than the whole imported graph. +- If you run a DIY or manual deployment, `network.json` and `ShapedDevices.csv` remain the files you maintain. +- Direct edits to integration-managed data may be replaced on the next refresh cycle. + +Mode capabilities are now explicit by integration: +- UISP: `flat`, `ap_only`, `ap_site`, `full` +- Splynx: `flat`, `ap_only`, `ap_site`, `full` +- Sonar: `flat`, `full` - Manual edits may be overwritten on the next refresh cycle. +Legacy `integrationUISPbandwidths.csv` and `integrationSplynxBandwidths.csv` files are auto-migrated into operator `AdjustSiteSpeed` entries in `lqos_overrides.json` and renamed to `.backup` files. + +DIY/manual deployments continue to use `network.json` and `ShapedDevices.csv` as their primary input files. + ## Topology Node ID Support LibreQoS supports an optional generic `"id"` field on `network.json` nodes. This field is intended to carry stable node identifiers from the integration source where possible. Current behavior: -- node IDs are the preferred match key for operator-owned site bandwidth overrides and tree-page node override editing when an ID is present -- legacy name-only matching is still supported as a fallback for older data +- when a node ID is available, LibreQoS uses it to match saved site bandwidth overrides more reliably +- older name-only matching still works as a fallback for older data - topology names still need to remain globally unique in `network.json` | Integration | `network.json` node ID support | Notes | @@ -65,7 +76,7 @@ Current behavior: | UISP | Yes | Real UISP sites/devices export generic `id` plus existing `uisp_site` / `uisp_device` metadata. Synthetic LibreQoS nodes use stable generated IDs. | | Splynx | Yes | Network sites and AP/site topology nodes export generic `id`. | | Sonar | Yes | Site and AP topology nodes export generic `id`. | -| Netzur | Partial | Exported only when the upstream zone payload includes a stable zone ID. | +| Netzur | Partial | Exported only when the upstream zone data includes a stable zone ID. | | VISP | No | Current importer shapes services/devices but does not build topology nodes in `network.json`. | | Powercode | No | Current importer does not build topology nodes in `network.json`. | | WISPGate | No | Current importer does not build topology nodes from stable upstream topology identifiers. | diff --git a/docs/v2.0/node-manager-ui.md b/docs/v2.0/node-manager-ui.md index 90dbd4c7..a3e0ebc4 100644 --- a/docs/v2.0/node-manager-ui.md +++ b/docs/v2.0/node-manager-ui.md @@ -28,16 +28,18 @@ This page documents key WebUI (Node Manager) views and operational behavior in t - `Node Details` summarizes the selected node’s configured rates, override state, and effective rate. - `Node Snapshot` provides a quick visual summary of throughput and QoO for the selected node. - Attached circuits are shown in a dedicated table for the selected node. +- The attached-circuits table is attachment-aware, not traffic-aware: idle circuits remain visible even when they have no current throughput, with live columns left empty or zeroed until traffic appears. - The attached-circuits IP column keeps rows compact by showing one address inline and collapsing additional addresses as `+X`, with the full list still available on hover. - Ethernet-limited attached circuits can show inline `10M`, `100M`, or `1G` warning badges beside the `Plan (Mbps)` value; hovering explains the auto-cap and clicking the badge opens the dedicated Ethernet review page. - Administrators can save or clear `Rate Override` values where node-level overrides are supported. Read-only users and unsupported nodes continue to display current values without edit controls. -- Tree-page operator rate edits write operator-owned `AdjustSiteSpeed` overrides to `lqos_overrides.json`. +- Tree-page rate edits save persistent `AdjustSiteSpeed` overrides in `lqos_overrides.json`. - On UISP `full` builds, legacy `integrationUISPbandwidths.csv` files are auto-migrated into those operator overrides on the next integration run when no operator rate overrides exist yet; otherwise the CSV is ignored. - Tree-page operator rate edits require an administrator session, a stable node ID, and a non-generated node. Generated/integration-synthetic nodes remain read-only in this editor. - The tree page keeps `Node Details` as a compact summary card and places the override editors as compact inline rows directly beneath the details table. - The tree page now shows topology override state as read-only summary only. Parent and attachment edits have moved to `Topology`, and the tree details panel links directly into Topology Manager for the selected node. - Runtime-selected attachment hops are collapsed out of the main tree hierarchy. When a site is currently using a specific backhaul or radio path, `Node Details` shows that as `Active Attachment` metadata on the site instead of exposing the attachment as its own branch node. - Inactive alternate PtP/wired backhaul attachment stubs are also pruned from the runtime tree. `tree.html` shows the effective path only; dormant alternates remain visible in Topology Manager rather than as standalone runtime nodes. +- For integration-backed topology, `Network Tree Overview`, Tree Overview sankey views, and HTB queue planning all follow the squashed runtime-effective tree from `network.effective.json`. Topology Manager intentionally keeps the fuller canonical/editor topology so transport hops such as backhauls can still be managed there. - On the synthetic `Root` node, topology override still shows as not applicable rather than as a generic missing-node-ID warning. ### Topology Manager @@ -57,12 +59,11 @@ This page documents key WebUI (Node Manager) views and operational behavior in t - Parent changes are gated behind an explicit `Start Move` step in the Details panel. In move mode, legal parent targets are highlighted in green and can be chosen from the target cards or by dragging the selected branch onto a highlighted target. - For UISP-backed sites, direct inter-site radio links can also surface sibling or alternate upstream sites as legal parent targets when the linked remote radio reaches the root without traversing the local site first. - Exported shaping/runtime trees anchor those cross-site moves under the parent-side peer attachment so the moved site stays visible after regeneration. -- Canonical integration snapshots stay integration-derived; saved Topology Manager re-parents are applied only in the runtime-effective topology layer, which now refuses to publish an effective tree that drops or duplicates a site. -- Saving a Topology Manager move, probe-policy change, manual attachment group, or attachment-rate override now triggers an immediate runtime-effective recompute and publish; routine topology edits do not need a fresh UISP import to take effect. -- When a move or attachment-preference save is in flight, the primary `Save` button switches to a short spinner/disabled state so operators can see that the runtime-effective publish is still underway. -- Effective publish validation is ID-based and fail-closed. Before validation, the runtime-effective compiler normalizes duplicate canonical attachment/device candidates by stable `node_id` so UISP duplicate radio rows do not block otherwise valid site moves. If a saved change would still select an invalid parent or attachment, create a cycle, duplicate a site, or drop a site from the exported tree, LibreQoS rejects the publish and keeps the last good effective topology in place. -- Canonical and effective topology snapshots are written with atomic file replacement so normal refreshes do not rely on partially written JSON artifacts. -- The Hierarchy Summary now shows three distinct states: `Canonical` for the latest integration snapshot, `Live` for the runtime-effective topology currently used for shaping, and `Saved`/`Proposed` for operator intent. When a saved move is already active in the runtime-effective topology, the status chip reads `Live Override`; otherwise it remains `Saved Override`. +- Saved Topology Manager moves and attachment preferences are applied on top of the latest imported topology, so you do not need to wait for another UISP sync before seeing the result. +- Saving a move, probe policy, manual attachment group, or attachment-rate override triggers an immediate refresh of the live topology used for shaping. +- While a save is being applied, the main `Save` button briefly shows a spinner and stays disabled. +- LibreQoS validates a move before making it active. If the change would create an invalid path, create a loop, duplicate a site, or remove a site from the tree, the save is rejected and the previous working topology stays in place. +- The Hierarchy Summary helps compare the latest imported topology, the currently active topology, and any saved override for the selected branch. - Main operator-facing hierarchy chips now prefer display names only. If a label cannot be resolved, the page shows a neutral placeholder instead of a raw internal node ID. - Attachment or radio preference edits for the current logical parent do not require move mode; when multiple explicit attachments are available, inspect mode exposes a lighter `Edit Attachment Preference` path in Details. - Attachment rows now surface runtime health directly in the page, including `Healthy`, `Suppressed`, `Probe Unavailable`, or `Probe Disabled` state, plus reason text, suppression hold-down timestamps, and per-pair probe enable/disable actions. @@ -78,7 +79,7 @@ This page documents key WebUI (Node Manager) views and operational behavior in t - When UISP exposes multiple parallel links between the same two sites, Topology Manager groups them under one logical parent target and exposes the concrete radios/devices as explicit attachment preferences. - `tree.html` only squashes runtime-selected UISP attachments when the effective attachment role is a true backhaul-style path (`PtP Backhaul` or `Wired Uplink`). PtMP access/uplink APs remain visible in the runtime tree. - When a chosen parent exposes only one valid explicit attachment option, the UI auto-saves that override immediately. -- Saved topology overrides are stored as operator-owned intent in `topology_overrides.json`, separate from integration-generated runtime editor state. +- Saved topology overrides are stored in `topology_overrides.json`. ### Topology Probes - Read-only debug page for system-wide topology probe troubleshooting, linked from Topology Manager rather than promoted as a primary sidebar destination. @@ -95,12 +96,12 @@ This page documents key WebUI (Node Manager) views and operational behavior in t - APs without explicit coordinates are represented through their parent site and can be expanded temporarily around the selected site for inspection. - When a site or AP is selected and the right-hand details panel opens, the selected node name links directly to that node's `tree.html` page when a stable node ID is available. - Visible unclustered sites show labels as the operator zooms in, and the selected site keeps its label visible while it is being inspected. -- When browser redaction mode is enabled, Site Map replaces displayed site names with `[redacted]` while leaving the underlying topology data unchanged. +- When redaction mode is enabled, Site Map replaces displayed site names with `[redacted]`. - Initial map framing prefers site coordinates for the first view, falling back to AP coordinates when no sites are mapped yet. - Site Map uses an Insight-hosted OpenStreetMap raster tile cache. -- In dark mode, the raster underlay is muted and tinted client-side toward the same cooler blue/cyan palette used by Flow Globe, so roads and geography stay visible without the bright light-theme basemap glare. +- In dark mode, the map is muted toward the same cooler blue/cyan palette used by Flow Globe, so roads and geography stay visible without the bright light-theme basemap glare. - Site Map depends on outbound access to `https://insight.libreqos.com` for initial bbox/bootstrap and raster tile fetches. -- When tiles are missing from the remote cache, the browser retries automatically for a short period instead of failing immediately, so initial map paint can lag briefly on cold tiles. +- When tiles are missing from the remote cache, the page retries automatically for a short period instead of failing immediately, so initial map paint can lag briefly on cold tiles. ### Flow Globe - Geographic flow visualization based on endpoint geolocation. @@ -124,6 +125,7 @@ This page documents key WebUI (Node Manager) views and operational behavior in t - `Devices` shows per-device detail tables and live charts for throughput, retransmits, and latency. - `Queue Stats` shows recent live queue history for the circuit, including backlog, delay, queue length, traffic, ECN marks, and drops. - Queue Stats charts use synchronized hover so operators can inspect the same second across all queue charts together. +- `Queue Stats` should continue to show the circuit's live queue telemetry in `flat` mode as well as other topology modes. - `Queue Tree` shows the circuit's live upstream queue path, including a path summary and per-node throughput, retransmit, and latency context. - `Traffic Flows` is a recent-flow operational table rather than a long-term history view. - `Traffic Flows` includes paging and a `Hide Small Flows` filter so large busy circuits remain usable without trying to render every row at once. @@ -146,8 +148,13 @@ This page documents key WebUI (Node Manager) views and operational behavior in t - Supports paging and filtering. - Add, edit, and delete actions save immediately in the dedicated editor. +### Configuration Pages +- Saved passwords, API tokens, and keys stay hidden when you open an integration page. +- If a credential is already configured, the field stays blank until you type a replacement. Use `Clear` only when you want to remove the saved value. +- Leave the field blank to keep the current value. Enter a new value to replace it. + ### Urgent Issues -- WebUI can display urgent operational issues generated by backend services. +- WebUI can display urgent operational issues reported by LibreQoS services. - Examples include mapping/license-limit warnings and other high-priority health events. - Operators can acknowledge/clear issues in the UI after review. - Common codes include `MAPPED_CIRCUIT_LIMIT` and `TC_U16_OVERFLOW` (see [Troubleshooting](troubleshooting.md#urgent-issue-codes-and-first-actions)). @@ -164,10 +171,10 @@ This page documents key WebUI (Node Manager) views and operational behavior in t ## Privacy / Redaction Mode - Toggle with the mask icon in the top navigation. -- Redaction is client-side and stored in browser local storage. +- Redaction is a display-only setting saved in your browser. - Redaction helps hide PII in screenshots/demos. - Site Map replaces displayed site names with `[redacted]` while redaction mode is enabled. -- Redaction does not modify `ShapedDevices.csv`, `network.json`, or any backend data. +- Redaction does not change your saved data or configuration files. ## Common Empty-State Behavior diff --git a/docs/v2.0/operating-modes.md b/docs/v2.0/operating-modes.md index 838f3c60..d5165b35 100644 --- a/docs/v2.0/operating-modes.md +++ b/docs/v2.0/operating-modes.md @@ -2,46 +2,56 @@ ## Page Purpose -Use this page to choose and enforce your source of truth policy (built-in integrations vs custom source of truth) before production changes. +Use this page to decide where permanent shaping changes should live before you go to production. Need definitions for terms on this page? See the [Glossary](glossary.md). ```{mermaid} flowchart TD - A[Choose Source of Truth] --> B{Using built-in CRM/NMS integration?} + A[Choose Where Changes Live] --> B{Using built-in CRM/NMS integration?} B -->|Yes| C[Built-in Integrations Mode] B -->|No| D{Generating files from your own scripts/systems?} D -->|Yes| E[Custom Source of Truth Mode] D -->|No| F[Manual Files Mode] - C --> G[Integration sync jobs own file refresh] - E --> H[External script pipeline owns persistence] - F --> I[Direct file maintenance; best for small networks] + C --> G[Integration sync keeps shaping data current] + E --> H[Your scripts refresh the shaping files] + F --> I[You maintain the files directly] ``` -LibreQoS supports two first-class operating modes. +LibreQoS supports three common operating modes. ## Built-in Integrations (Recommended for Most Operators) -In this mode, your CRM/NMS is the source of truth and LibreQoS integration jobs produce shaping inputs. +In this mode, your CRM/NMS is where permanent subscriber and topology changes belong, and LibreQoS keeps itself in sync from that system. Key behavior: -- `ShapedDevices.csv` is regenerated by integration sync cycles. -- `network.json` overwrite behavior depends on integration settings (for example `always_overwrite_network_json`). -- Direct manual edits may be overwritten on the next scheduler refresh. +- Integration sync refreshes the topology and shaping data LibreQoS uses. +- Built-in integrations do not use `network.json` as their normal working file. +- Direct file edits may be overwritten on the next scheduler refresh. +- `flat` mode intentionally simplifies the tree when you want lower overhead. ## Custom Source of Truth -In this mode, your own scripts/systems generate `network.json` and `ShapedDevices.csv`. +In this mode, your own scripts or systems generate `network.json` and `ShapedDevices.csv`. Key behavior: -- Your external workflow owns persistence. -- WebUI edits are valid for quick operational changes. -- Permanent changes should be made in your external source of truth workflow. +- Your external workflow is where permanent changes belong. +- WebUI edits are fine for quick operational changes. +- Keep your long-term changes in your own scripts or automation. + +## Manual Files Mode + +In this mode, you maintain `network.json` and `ShapedDevices.csv` yourself. + +Key behavior: +- Best fit for small networks, short pilots, or temporary workarounds. +- WebUI helps you validate what LibreQoS is using. +- Manual discipline matters because there is no upstream system keeping those files in sync for you. ## Mode Declaration Checklist (Before Go-Live) -1. Pick one primary source of truth. -2. Confirm which system can write production shaping inputs. +1. Pick one primary place for permanent shaping changes. +2. Confirm which system writes production shaping data. 3. Confirm scheduler refresh behavior and overwrite cadence. 4. Document your hotfix workflow (WebUI, external editor, or both). 5. Do not maintain competing edits in multiple systems for the same objects. @@ -49,8 +59,8 @@ Key behavior: ## Topology and Mode Expectations - Single-interface (on-a-stick) and VLAN-heavy designs are valid, but require explicit queue/interface planning and careful validation after changes. -- Integration mode is best when you want CRM/NMS-driven topology and subscriber lifecycle data to own shaping inputs. -- If you need strict custom topology behavior not represented by integration output, use custom source of truth mode and keep ownership explicit. +- Integration mode is best when you want CRM/NMS-driven topology and subscriber lifecycle data to control shaping. +- If you need topology behavior that your integration cannot represent, use custom source of truth mode and keep the responsibility clear. See: - [Advanced Configuration Reference](configuration-advanced.md) diff --git a/docs/v2.0/quickstart.md b/docs/v2.0/quickstart.md index 82088941..7293520e 100644 --- a/docs/v2.0/quickstart.md +++ b/docs/v2.0/quickstart.md @@ -8,8 +8,8 @@ Use this page in order: 1. Complete install and bridge setup. 2. Complete first-run WebUI access. 3. Pass the 10-minute health check. -4. Pick one source of truth owner. -5. Follow the recommended path for that owner. +4. Choose one system to manage your shaping data. +5. Follow the recommended path for that system. Need definitions for key terms? See the [Glossary](glossary.md). @@ -89,23 +89,23 @@ Confirm: If this fails, go to [Troubleshooting](troubleshooting.md) before proceeding. -## 3) Pick One Source of Truth Owner +## 3) Choose One System To Manage Your Shaping Data This is the most important early decision. -Source of truth means: the one place that should own persistent shaping data. +On this page, "source of truth" just means the place where permanent shaping changes should be made. If you make persistent changes somewhere else, they may be overwritten later. Choose one: -| If this describes you | Mode | Owner of durable shaping data | +| If this describes you | Mode | Where permanent shaping changes belong | |---|---|---| | You use a supported LibreQoS CRM/NMS integration | Built-In Integration Mode | Integration jobs | | You generate `network.json` and `ShapedDevices.csv` with your own automation | Custom Source of Truth Mode | Your scripts | | You intentionally maintain files by hand | Manual Files Mode | Manual edits | -Rule: keep one owner for persistent shaping inputs. +Rule: pick one place for permanent shaping changes. Most operators should stop here and choose **Built-In Integration Mode**. @@ -121,18 +121,14 @@ When to choose: Do this now: 1. Configure integration settings in WebUI. 2. Run the initial sync. -3. Confirm outputs were created or refreshed: - - `network.json` - - `ShapedDevices.csv` -4. Confirm Scheduler Status is healthy. -5. Confirm Network Tree reflects the expected hierarchy depth for your chosen integration strategy. -6. Confirm there are no urgent/fatal issues that block shaping. -7. Only then move to limited pilot traffic or inline use. +3. Confirm Scheduler Status is healthy. +4. Confirm Network Tree reflects the expected hierarchy depth for your chosen integration strategy. +5. Confirm there are no urgent or fatal issues that block shaping. +6. Only then move to limited pilot traffic or inline use. Do not do this: -- Do not hand-edit integration-owned files as your normal workflow. -- Do not split ownership between integration syncs and manual file edits unless you intentionally want that complexity. -- Do not change overwrite behavior unless you understand how your integration should own `network.json`. +- Do not hand-edit files that your integration refreshes as part of your normal workflow. +- Do not mix manual file edits with scheduled integration syncs unless you intentionally want that extra complexity. Next: - [CRM/NMS Integrations](integrations.md) @@ -147,7 +143,7 @@ When to choose: Do this now: 1. Implement script/process to generate and refresh shaping files. -2. Declare script outputs as your source of truth. +2. Treat your script outputs as the place for permanent shaping changes. 3. Place LibreQoS inline for pilot traffic. 4. Use WebUI for operational checks and short-term adjustments. 5. Keep permanent changes in your external script workflow. @@ -182,7 +178,7 @@ Next: ## 6) Common First-Run Mistakes -- Unclear source of truth ownership between integration and manual edits. +- Not deciding where permanent shaping changes should be made. - Hand-editing files even though your integration is supposed to own them. - Changing topology depth before passing the health gate. - Skipping post-change service/log validation before pilot traffic. diff --git a/docs/v2.0/recipes-wisp-fisp-integration.md b/docs/v2.0/recipes-wisp-fisp-integration.md index 32246421..889c05a1 100644 --- a/docs/v2.0/recipes-wisp-fisp-integration.md +++ b/docs/v2.0/recipes-wisp-fisp-integration.md @@ -16,7 +16,7 @@ Use this pattern when your operator workflow is already centered on a supported ## Implementation 1. Configure integration credentials/settings in WebUI (`Configuration -> Integrations`). -2. For integration-driven deployments, keep `always_overwrite_network_json = true`. +2. For integration-driven deployments, treat `network.json` as DIY/manual-only and let the integration publish `topology_import.json` instead. 3. Choose the lightest topology strategy that meets requirements. | Requirement | Suggested strategy | diff --git a/docs/v2.0/treeguard.md b/docs/v2.0/treeguard.md index 5ed0cc3e..3301d344 100644 --- a/docs/v2.0/treeguard.md +++ b/docs/v2.0/treeguard.md @@ -102,7 +102,7 @@ When enabled and not dry-run, TreeGuard may persist circuit SQM decisions to: - `lqos_overrides.treeguard.json` -TreeGuard is designed to avoid fighting operator-owned overrides. If operator overrides exist for enrolled entities, TreeGuard skips those entities and reports warnings. +TreeGuard is designed to avoid fighting manually saved overrides. If those overrides exist for enrolled entities, TreeGuard skips those entities and reports warnings. TreeGuard virtual-node decisions are runtime-only Bakery operations. They are not materialized back into the base `network.json` by the scheduler, and they are not persisted as TreeGuard-owned diff --git a/docs/v2.0/troubleshooting.md b/docs/v2.0/troubleshooting.md index 5466c84a..df06d1dd 100644 --- a/docs/v2.0/troubleshooting.md +++ b/docs/v2.0/troubleshooting.md @@ -37,6 +37,12 @@ journalctl -u lqos_scheduler --since "30 minutes ago" If integration-related, also include: +```bash +ls -lh /opt/libreqos/src/topology_import.json /opt/libreqos/src/shaping_inputs.json +``` + +If you run a manual or custom-file deployment instead of a built-in integration, include: + ```bash ls -lh /opt/libreqos/src/network.json /opt/libreqos/src/ShapedDevices.csv ``` @@ -150,9 +156,7 @@ Routine package upgrades now invoke `lqos_setup --skip-if-ready` during `postins If startup shaping fails because `shaping_inputs.json` is missing or stale, current builds leave the scheduler running in a degraded state and wait for the next scheduled full refresh to recover. The high-frequency topology refresh tick stays disabled until one shaping pass completes successfully, so repeated 3-second refresh attempts should not continue hammering a fresh install that has not produced runtime topology inputs yet. -Current builds no longer gate startup shaping on file mtimes. Instead, `lqos_topology` publishes `/opt/libreqos/src/topology_runtime_status.json` with a `source_generation` derived from the current shaping inputs, and `lqos_scheduler` waits for `ready: true` for that exact generation before calling `refreshShapers()`. - -If scheduler startup is degraded with a message about topology runtime still building, that does not necessarily mean the scheduler service died. It means the current generation of `network.json`, `ShapedDevices.csv`, `circuit_anchors.json`, and active topology state has not finished producing committed runtime outputs yet. Once topology finishes and the status file flips to `ready: true` for the same generation, scheduled shaping refreshes recover automatically without restarting the service. +If scheduler startup is degraded with a message about topology runtime still building, that usually means LibreQoS is still finishing an import or refreshing shaping data. Give the current cycle a little time to complete, then recheck Scheduler Status before changing unrelated settings. If scheduler startup is degraded with a message that topology runtime failed for the current generation, inspect: @@ -162,15 +166,15 @@ ls -lh /opt/libreqos/src/topology_effective_state.json /opt/libreqos/src/network journalctl -u lqos_scheduler --since "30 minutes ago" ``` -If Insight shows sites buried under AP/backhaul chains or only a narrow subset of sites loads cleanly, check whether the host is on a build before the separate Insight logical-topology export. Current builds submit an Insight-only logical-parent tree derived from canonical topology state, while `compatibility_network_json` remains reserved for local compatibility and `network.effective.json` generation. +If Topology Manager changes or imports seem stuck on older data, check whether LibreQoS set older snapshots aside under `/opt/libreqos/src/.topology_stale/`, then review recent scheduler logs before retrying. -To inspect the exact topology JSON that `lqosd` is preparing for Insight, review: +If Insight topology looks wrong, review the current troubleshooting snapshot that `lqosd` is preparing for Insight: ```bash cat /opt/libreqos/src/network.insight.debug.json ``` -This debug snapshot is separate from `network.insight.json`. The latter is an installed-runtime input path used by shaping components when Insight topology mode is enabled, while `network.insight.debug.json` is only a diagnostic snapshot of the payload source tree. +Treat `network.insight.debug.json` as a troubleshooting snapshot only; do not edit it. If specific APs or switches appear multiple times with suffixed names such as `... [AP deadbeef]`, check whether UISP is returning duplicate rows for the same device ID. Current builds defensively deduplicate raw UISP devices by `identification.id` before topology graph construction, and skip any residual duplicate device IDs during graph assembly. @@ -180,6 +184,7 @@ If an integration subprocess fails, current builds keep the scheduler alive, pub Recent builds expose scheduler readiness/state in the WebUI (Node Manager). If the scheduler is still starting, the sidebar now reports the current startup phase and a coarse progress ring rather than only a spinner. +Current builds also treat scheduler progress, output, and error bus messages as proof that the scheduler is alive, so the sidebar should not stay stuck on `Scheduler available: false` while the scheduler is actively reporting work. If scheduler status appears down/stale: 1. Verify both services: @@ -249,7 +254,7 @@ Site Map has one extra dependency beyond normal WebUI data feeds: current builds If Site Map alone is blank or slow: 1. Confirm `lqosd` is healthy. 2. Confirm the box can reach `insight.libreqos.com` from its management network. -3. Wait briefly and reload the page; the browser retries tile requests automatically while cold tiles are being populated upstream. +3. Wait briefly and reload the page; the map page retries tile requests automatically while cold tiles are being populated upstream. 4. Check recent `lqosd` logs: ``` journalctl -u lqosd --since "10 minutes ago" diff --git a/src/LibreQoS.py b/src/LibreQoS.py index 3b74f14c..2e1eddcc 100755 --- a/src/LibreQoS.py +++ b/src/LibreQoS.py @@ -45,7 +45,7 @@ from liblqos_python import is_lqosd_alive, clear_ip_mappings, delete_ip_mapping, overrides_circuit_adjustments_effective, \ automatic_import_uisp, automatic_import_splynx, automatic_import_powercode, automatic_import_sonar, \ automatic_import_wispgate, automatic_import_netzur, automatic_import_visp, \ - plan_top_level_cpu_bins, \ + plan_top_level_cpu_bins, topology_import_ingress_enabled, \ plan_class_identities, \ fast_queues_fq_codel, \ shaping_cpu_count, \ @@ -115,6 +115,9 @@ def get_network_json_path(): if os.path.exists(effective_path): return effective_path + if topology_import_ingress_enabled(): + return effective_path + if enable_insight_topology(): insight_path = os.path.join(base_dir, "network.insight.json") if os.path.exists(insight_path): @@ -260,6 +263,16 @@ def _resolve_effective_parent_node(circuit, parent_node_ids, parent_node_aliases return parent_node, parent_node_id +def _attachment_lookup_candidates(node_key, node_data): + candidates = [] + node_id = str(node_data.get('id', '') or '').strip() + node_name = str(node_data.get('name', '') or '').strip() + for candidate in (node_id, str(node_key).strip(), node_name): + if candidate and candidate not in candidates: + candidates.append(candidate) + return candidates + + def _shaping_inputs_are_fresh(shaping_inputs_path, shaped_devices_file, network_json_file, circuit_anchors_file=None): if not os.path.isfile(shaping_inputs_path): return False @@ -670,13 +683,20 @@ def validateNetworkAndDevices(): networkValidatedOrNot = True # Verify ShapedDevices.csv is valid devicesValidatedOrNot = True # True by default, switches to false if ANY entry in ShapedDevices.csv fails validation + integration_ingress = topology_import_ingress_enabled() - # Verify that the Rust side of things can read the CSV file + # Verify that the Rust side of things can read the active shaped-device ingress rustValid = validate_shaped_devices() if rustValid == "OK": - print("Rust validated ShapedDevices.csv") + if integration_ingress: + print("Rust validated integration shaping ingress") + else: + print("Rust validated ShapedDevices.csv") else: - warnings.warn("Rust failed to validate ShapedDevices.csv", stacklevel=2) + if integration_ingress: + warnings.warn("Rust failed to validate integration shaping ingress", stacklevel=2) + else: + warnings.warn("Rust failed to validate ShapedDevices.csv", stacklevel=2) warnings.warn(rustValid, stacklevel=2) devicesValidatedOrNot = False with open(get_network_json_path()) as file: @@ -707,6 +727,17 @@ def validateNetworkAndDevices(): except json.decoder.JSONDecodeError: warnings.warn("network.json is an invalid JSON file", stacklevel=2) # in case json is invalid networkValidatedOrNot = False + if integration_ingress: + if devicesValidatedOrNot == True: + print("integration shaping ingress passed validation") + else: + print("integration shaping ingress failed validation") + if networkValidatedOrNot == True: + print("network.json passed validation") + else: + print("network.json failed validation") + return devicesValidatedOrNot and networkValidatedOrNot + rowNum = 2 # Handle non-utf8 encoding in ShapedDevices.csv @@ -1121,12 +1152,19 @@ def refreshShapers(): safeToRunRefresh = False print("Validating input files '" + shapedDevicesFile + "' and '" + networkJSONfile + "'") if (validateNetworkAndDevices() == True): - shutil.copyfile(shapedDevicesFile, 'lastGoodConfig.csv') + if os.path.isfile(shapedDevicesFile): + shutil.copyfile(shapedDevicesFile, 'lastGoodConfig.csv') shutil.copyfile(networkJSONfile, 'lastGoodConfig.json') - print("Backed up good config as lastGoodConfig.csv and lastGoodConfig.json") + if os.path.isfile(shapedDevicesFile): + print("Backed up good config as lastGoodConfig.csv and lastGoodConfig.json") + else: + print("Backed up good config as lastGoodConfig.json") safeToRunRefresh = True else: - if (isThisFirstRunSinceBoot == False): + if topology_import_ingress_enabled(): + warnings.warn("Validation failed for integration ingress/runtime artifacts - will now exit.", stacklevel=2) + safeToRunRefresh = False + elif (isThisFirstRunSinceBoot == False): warnings.warn("Validation failed. Because this is not the first run since boot (queues already set up) - will now exit.", stacklevel=2) safeToRunRefresh = False else: @@ -2008,17 +2046,17 @@ def refreshShapers(): node_id = str(node_data.get('id', '') or '').strip() node_name = str(node_data.get('name', '') or '').strip() - parent_candidates = [] - for candidate in (node_id, node, node_name): - if candidate and candidate not in parent_candidates: - parent_candidates.append(candidate) + parent_candidates = _attachment_lookup_candidates(node, node_data) selected_circuits = [] if node_id and node_id in circuits_by_parent_id: selected_circuits = list(circuits_by_parent_id[node_id]) else: seen_circuit_ids = set() - for candidate in parent_candidates[1:]: + name_candidates = ( + parent_candidates[1:] if node_id and parent_candidates else parent_candidates + ) + for candidate in name_candidates: for circuit in circuits_by_parent_name.get(candidate, []): circuit_id = str(circuit.get('circuitID', '') or '') if circuit_id in seen_circuit_ids: @@ -2598,16 +2636,27 @@ def refreshShapers(): flat_network, ) if len(devicesSkipped) > 0: - warnings.warn( - str(len(devicesSkipped)) + " device(s) were not shaped. Detailed reasons are listed below.", - stacklevel=2, + flat_mode_generated_parent_only = flat_network and all( + entry.get("reasonCode") == "unattached_flat_network" for entry in devicesSkipped ) - print("Devices not shaped:") - for entry in devicesSkipped: - print(format_unshaped_device_line(entry)) + if flat_mode_generated_parent_only: + print( + f"Flat network mode assigned {len(devicesSkipped)} device(s) to generated parent queues." + ) + else: + warnings.warn( + str(len(devicesSkipped)) + " device(s) were not shaped. Detailed reasons are listed below.", + stacklevel=2, + ) + print("Devices not shaped:") + for entry in devicesSkipped: + print(format_unshaped_device_line(entry)) - # Save ShapedDevices.csv as ShapedDevices.lastLoaded.csv - shutil.copyfile('ShapedDevices.csv', 'ShapedDevices.lastLoaded.csv') + # DIY/manual mode snapshots the active ShapedDevices.csv for update-only diffs. + # Built-in integrations no longer emit ShapedDevices.csv, so skip the legacy + # snapshot there instead of failing the whole scheduler refresh. + if os.path.isfile(shapedDevicesFile): + shutil.copyfile(shapedDevicesFile, 'ShapedDevices.lastLoaded.csv') # Save for stats with open('statsByCircuit.json', 'w') as f: diff --git a/src/configMigrator.py b/src/configMigrator.py index d393400e..49dcfbb5 100644 --- a/src/configMigrator.py +++ b/src/configMigrator.py @@ -55,8 +55,8 @@ oldConfig = { 'uispSite': getIspConfigValue('uispSite', 'Main Site'), 'uispStrategy': getIspConfigValue('uispStrategy', 'full'), 'uispSuspendedStrategy': getIspConfigValue('uispSuspendedStrategy', 'none'), - 'airMax_capacity': getIspConfigValue('airMax_capacity', 0.65), - 'ltu_capacity': getIspConfigValue('ltu_capacity', 0.90), + 'airMax_capacity': getIspConfigValue('airMax_capacity', 1.0), + 'ltu_capacity': getIspConfigValue('ltu_capacity', 1.0), 'bandwidthOverheadFactor': getIspConfigValue('bandwidthOverheadFactor', 1.0), 'committedBandwidthMultiplier': getIspConfigValue('committedBandwidthMultiplier', 0.98), 'exceptionCPEs': getIspConfigValue('exceptionCPEs', {}), diff --git a/src/csvToNetworkJSON.py b/src/csvToNetworkJSON.py index a7ff5c2e..27a252b3 100644 --- a/src/csvToNetworkJSON.py +++ b/src/csvToNetworkJSON.py @@ -3,7 +3,6 @@ checkPythonVersion() import os import csv import json -from liblqos_python import overwrite_network_json_always from integrationCommon import NetworkGraph, NetworkNode, NodeType def csvToNetworkJSONfile(): @@ -45,10 +44,7 @@ def csvToNetworkJSONfile(): net.prepareTree() net.plotNetworkGraph(False) if net.doesNetworkJsonExist(): - if overwrite_network_json_always(): - net.createNetworkJson() - else: - print("network.json already exists and overwriteNetworkJSONalways set to False. Leaving in-place.") + print("network.json already exists. Leaving it in place; DIY/manual inputs are operator-owned.") else: net.createNetworkJson() net.createShapedDevices() diff --git a/src/integrationCommon.py b/src/integrationCommon.py index 09d3b48b..ef7b8af7 100644 --- a/src/integrationCommon.py +++ b/src/integrationCommon.py @@ -5,7 +5,8 @@ from typing import List, Any, Dict, Set from liblqos_python import allowed_subnets, ignore_subnets, generated_pn_download_mbps, generated_pn_upload_mbps, \ circuit_name_use_address, upstream_bandwidth_capacity_download_mbps, upstream_bandwidth_capacity_upload_mbps, \ find_ipv6_using_mikrotik, exclude_sites, bandwidth_overhead_factor, committed_bandwidth_multiplier, \ - exception_cpes, promote_to_root_list, client_bandwidth_multiplier + exception_cpes, promote_to_root_list, client_bandwidth_multiplier, \ + write_compiled_topology_from_python_graph_payload import ipaddress import enum import json @@ -378,6 +379,33 @@ class NetworkGraph: self.__reparentById() + def __liftTopologyChildrenOutOfClients(self) -> None: + # Keep exportable topology nodes as siblings of a circuit's generated site, + # never as children of the circuit itself. Otherwise nested generated sites + # become unreachable from network.json because client nodes are not exported. + if not self._cache_valid: + self._buildChildrenCache() + + reparent_map = {} + for i, node in enumerate(self.nodes): + if node.type != NodeType.client: + continue + new_parent_id = node.parentId + if new_parent_id in (None, ""): + continue + for child_idx in self._children_cache.get(i, []): + child = self.nodes[child_idx] + if child.type in (NodeType.client, NodeType.clientWithChildren, NodeType.site): + reparent_map[child_idx] = new_parent_id + + if not reparent_map: + return + + for child_idx, new_parent_id in reparent_map.items(): + self.nodes[child_idx].parentId = new_parent_id + + self.__reparentById() + def __findUnconnectedNodes(self) -> List: # Rebuild the cache each time because callers may have adjusted # parentIndex directly as part of repair/testing flows. @@ -437,6 +465,8 @@ class NetworkGraph: self.__promoteClientsWithChildren() print("PrepareTree: Converting clients with children to sites...") self.__clientsWithChildrenToSites() + print("PrepareTree: Lifting topology descendants out of client nodes...") + self.__liftTopologyChildrenOutOfClients() print("PrepareTree: Reconnecting unconnected nodes...") self.__reconnectUnconnected() print("PrepareTree: Pruning ignored-only devices and empty circuits...") @@ -453,6 +483,11 @@ class NetworkGraph: def createNetworkJson(self): import json + topLevelNode = self.buildNetworkJson() + with open('network.json', 'w') as f: + json.dump(topLevelNode, f, indent=4) + + def buildNetworkJson(self): topLevelNode = {} self.__visited = [] # Protection against loops - never visit twice @@ -476,9 +511,7 @@ class NetworkGraph: if 'children' in data[node]: inheritBandwidthMaxes(data[node]['children'], data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps']) inheritBandwidthMaxes(topLevelNode, parentMaxDL=upstream_bandwidth_capacity_download_mbps(), parentMaxUL=upstream_bandwidth_capacity_upload_mbps()) - - with open('network.json', 'w') as f: - json.dump(topLevelNode, f, indent=4) + return topLevelNode def __buildNetworkObject(self, idx): # Private: used to recurse down the network tree while building @@ -571,19 +604,38 @@ class NetworkGraph: self.__reparentById() def createShapedDevices(self): + shaped_devices_csv, circuit_anchor_file = self.buildShapedDevicesArtifacts() + with open('ShapedDevices.csv', 'w', newline='') as csvfile: + csvfile.write(shaped_devices_csv) + + with open('circuit_anchors.json', 'w', encoding='utf-8') as anchorfile: + json.dump(circuit_anchor_file, anchorfile, indent=2) + anchorfile.write('\n') + + def buildShapedDevicesArtifacts(self): import csv - # Builds ShapedDevices.csv from the network tree. + import io circuits = [] circuit_anchors = [] if not self._cache_valid: self._buildChildrenCache() + + def nearestRealTopologyParent(idx): + parent_idx = self.nodes[idx].parentIndex + while parent_idx > 0: + parent_node = self.nodes[parent_idx] + parent_id = parent_node.networkJsonId or "" + if parent_id and not parent_id.startswith("libreqos:generated:graph:"): + return parent_node + parent_idx = parent_node.parentIndex + return None for i, node in enumerate(self.nodes): if node.type == NodeType.client: - parent_node = self.nodes[node.parentIndex] - parent = parent_node.displayName - parent_id = parent_node.networkJsonId + parent_node = nearestRealTopologyParent(i) + parent = parent_node.displayName if parent_node is not None else "" + parent_id = parent_node.networkJsonId if parent_node is not None else "" if parent == "Shaper Root": parent = "" if parent == "": parent_id = "" @@ -614,7 +666,6 @@ class NetworkGraph: "anchor_node_name": node.displayName if node.displayName else None, }) - # O(1) lookup of children child_indices = self._children_cache.get(i, []) for child_idx in child_indices: child = self.nodes[child_idx] @@ -633,56 +684,309 @@ class NetworkGraph: if len(circuit["devices"]) > 0: circuits.append(circuit) - - with open('ShapedDevices.csv', 'w', newline='') as csvfile: - wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL) - wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'Parent Node ID', 'Anchor Node ID', 'MAC', - 'IPv4', 'IPv6', 'Download Min', 'Upload Min', 'Download Max', 'Upload Max', 'Comment']) - for circuit in circuits: - for device in circuit["devices"]: - #Remove brackets and quotes of list so LibreQoS.py can parse it - device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'') - device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'') - if circuit["upload"] is None: - circuit["upload"] = 0.0 - if circuit["download"] is None: - circuit["download"] = 0.0 - row = [ - circuit["id"], - circuit["name"], - device["id"], - device["name"], - circuit["parent"], - circuit.get("parent_id", ""), - "", - device["mac"], - device["ipv4"], - device["ipv6"], - int(1), - int(1), - max(1.0, round(float(circuit["download"]), 2)), - max(1.0, round(float(circuit["upload"]), 2)), - "" - ] - wr.writerow(row) - - # If we have an "appendToShapedDevices.csv" file, it gets appended to the end of the file. - # This is useful for adding devices that are not in the network tree, such as a - # "default" device that gets all the traffic that doesn't match any other device. - if os.path.isfile('appendToShapedDevices.csv'): - with open('appendToShapedDevices.csv', 'r') as f: - reader = csv.reader(f) - for row in reader: - wr.writerow(row) - with open('circuit_anchors.json', 'w', encoding='utf-8') as anchorfile: - json.dump({ + csv_buffer = io.StringIO() + wr = csv.writer(csv_buffer, quoting=csv.QUOTE_ALL) + wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'Parent Node ID', 'Anchor Node ID', 'MAC', + 'IPv4', 'IPv6', 'Download Min', 'Upload Min', 'Download Max', 'Upload Max', 'Comment']) + for circuit in circuits: + for device in circuit["devices"]: + ipv4 = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'') + ipv6 = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'') + if circuit["upload"] is None: + circuit["upload"] = 0.0 + if circuit["download"] is None: + circuit["download"] = 0.0 + row = [ + circuit["id"], + circuit["name"], + device["id"], + device["name"], + circuit["parent"], + circuit.get("parent_id", ""), + "", + device["mac"], + ipv4, + ipv6, + int(1), + int(1), + max(1.0, round(float(circuit["download"]), 2)), + max(1.0, round(float(circuit["upload"]), 2)), + "" + ] + wr.writerow(row) + + if os.path.isfile('appendToShapedDevices.csv'): + with open('appendToShapedDevices.csv', 'r') as f: + reader = csv.reader(f) + for row in reader: + wr.writerow(row) + + return ( + csv_buffer.getvalue(), + { "schema_version": 1, "source": "python/integration_common", "generated_unix": int(time.time()), "anchors": circuit_anchors, - }, anchorfile, indent=2) - anchorfile.write('\n') + }, + ) + + def _isInfrastructureTopologyNode(self, idx): + if idx <= 0: + return False + node = self.nodes[idx] + if node.type not in (NodeType.site, NodeType.ap): + return False + if not node.networkJsonId: + return False + # Customer-derived generated graph sites are compatibility-only and + # must never participate in native topology editing/runtime state. + if node.networkJsonId.startswith("libreqos:generated:graph:"): + return False + return True + + def _buildInfrastructureTopologyContext(self): + if not self._cache_valid: + self._buildChildrenCache() + + exportable_indices = [ + idx for idx, _node in enumerate(self.nodes) + if self._isInfrastructureTopologyNode(idx) + ] + exportable_set = set(exportable_indices) + + def nearestInfrastructureParentIndex(idx): + parent_idx = self.nodes[idx].parentIndex + while parent_idx > 0: + if parent_idx in exportable_set: + return parent_idx + parent_idx = self.nodes[parent_idx].parentIndex + return None + + resolved_parent_by_idx = {} + children_by_parent = {} + for idx in exportable_indices: + parent_idx = nearestInfrastructureParentIndex(idx) + resolved_parent_by_idx[idx] = parent_idx + if parent_idx is None: + continue + children_by_parent.setdefault(parent_idx, []).append(idx) + + descendant_cache = {} + + def topology_descendants(idx): + if idx in descendant_cache: + return descendant_cache[idx] + descendants = set() + for child_idx in children_by_parent.get(idx, []): + descendants.add(child_idx) + descendants.update(topology_descendants(child_idx)) + descendant_cache[idx] = descendants + return descendants + + return { + "exportable_indices": exportable_indices, + "resolved_parent_by_idx": resolved_parent_by_idx, + "children_by_parent": children_by_parent, + "topology_descendants": topology_descendants, + } + + def _autoAttachmentOption(self): + return { + "attachment_id": "auto", + "attachment_name": "Auto", + "attachment_kind": "auto", + "attachment_role": "unknown", + "rate_source": "unknown", + "can_override_rate": False, + "has_rate_override": False, + "probe_enabled": False, + "probeable": False, + "health_status": "disabled", + "effective_selected": False, + } + + def _isGeneratedInfrastructureNode(self, idx): + if idx <= 0: + return False + node_id = self.nodes[idx].networkJsonId or "" + return node_id.startswith("libreqos:generated:") + + def _classifyInfrastructureEditPolicy(self, idx, context): + parent_idx = context["resolved_parent_by_idx"][idx] + node = self.nodes[idx] + if parent_idx is None: + return "root" + if node.networkJsonId.startswith("libreqos:generated:"): + return "fixed_parent" + candidate_parent_indices = self._boundedAlternativeParentIndices(idx, context) + if len(candidate_parent_indices) > 1: + return "movable" + return "fixed_parent" + + def _boundedAlternativeParentIndices(self, idx, context): + parent_idx = context["resolved_parent_by_idx"][idx] + if parent_idx is None: + return [] + + candidate_indices = [] + seen = set() + + def include(candidate_idx): + if candidate_idx is None: + return + if candidate_idx == idx: + return + if candidate_idx in seen: + return + if not self._isInfrastructureTopologyNode(candidate_idx): + return + if self._isGeneratedInfrastructureNode(candidate_idx): + return + seen.add(candidate_idx) + candidate_indices.append(candidate_idx) + + # Always keep the current resolved parent. + include(parent_idx) + + # Conservative bounded-move policy for Python-backed integrations: + # allow reparenting only within the current parent's immediate neighborhood. + grandparent_idx = context["resolved_parent_by_idx"].get(parent_idx) + if grandparent_idx is not None: + for sibling_parent_idx in context["children_by_parent"].get(grandparent_idx, []): + if sibling_parent_idx == idx: + continue + include(sibling_parent_idx) + else: + parent_node = self.nodes[parent_idx] + for root_candidate_idx in context["exportable_indices"]: + if root_candidate_idx == parent_idx: + continue + if context["resolved_parent_by_idx"][root_candidate_idx] is not None: + continue + root_candidate = self.nodes[root_candidate_idx] + if root_candidate.type != parent_node.type: + continue + include(root_candidate_idx) + + return candidate_indices + + def buildNativeTopologyEditorState(self, source: str): + # Build the native infrastructure topology consumed by Topology Manager. + # This intentionally excludes generated customer graph nodes and preserves + # the resolved infrastructure parent chain from the prepared NetworkGraph. + context = self._buildInfrastructureTopologyContext() + exportable_indices = context["exportable_indices"] + resolved_parent_by_idx = context["resolved_parent_by_idx"] + topology_descendants = context["topology_descendants"] + + nodes = [] + for idx in exportable_indices: + node = self.nodes[idx] + parent_idx = resolved_parent_by_idx[idx] + parent_node = self.nodes[parent_idx] if parent_idx is not None else None + edit_policy = self._classifyInfrastructureEditPolicy(idx, context) + can_move = edit_policy == "movable" + + allowed_parents = [] + if can_move: + for candidate_idx in self._boundedAlternativeParentIndices(idx, context): + if candidate_idx in topology_descendants(idx) or candidate_idx == idx: + continue + candidate = self.nodes[candidate_idx] + allowed_parents.append({ + "parent_node_id": candidate.networkJsonId, + "parent_node_name": candidate.displayName, + "attachment_options": [self._autoAttachmentOption()], + "all_attachments_suppressed": False, + "has_probe_unavailable_attachments": False, + }) + allowed_parents.sort(key=lambda entry: (entry["parent_node_name"].lower(), entry["parent_node_id"])) + + nodes.append({ + "node_id": node.networkJsonId, + "node_name": node.displayName, + "current_parent_node_id": parent_node.networkJsonId if parent_node is not None else None, + "current_parent_node_name": parent_node.displayName if parent_node is not None else None, + "current_attachment_id": None, + "current_attachment_name": None, + "can_move": can_move, + "allowed_parents": allowed_parents, + "preferred_attachment_id": None, + "preferred_attachment_name": None, + "effective_attachment_id": None, + "effective_attachment_name": None, + }) + + return { + "schema_version": 1, + "source": source, + "generated_unix": int(time.time()), + "nodes": nodes, + } + + def buildTopologyParentCandidates(self): + # Build a legacy-compatible parent-candidate snapshot for topology editing. + # This gives Python-backed integrations the same basic "Start Move" capability + # as richer native importers by enumerating legal non-descendant topology parents + # for real infrastructure branches only. + context = self._buildInfrastructureTopologyContext() + exportable_indices = context["exportable_indices"] + resolved_parent_by_idx = context["resolved_parent_by_idx"] + topology_descendants = context["topology_descendants"] + + nodes = [] + for idx in exportable_indices: + node = self.nodes[idx] + parent_idx = resolved_parent_by_idx[idx] + edit_policy = self._classifyInfrastructureEditPolicy(idx, context) + if edit_policy != "movable": + continue + parent_node = self.nodes[parent_idx] + parent_node_id = parent_node.networkJsonId + parent_node_name = parent_node.displayName + + candidate_parents = [] + for candidate_idx in self._boundedAlternativeParentIndices(idx, context): + if candidate_idx in topology_descendants(idx) or candidate_idx == idx: + continue + candidate = self.nodes[candidate_idx] + candidate_parents.append({ + "node_id": candidate.networkJsonId, + "node_name": candidate.displayName, + }) + candidate_parents.sort(key=lambda entry: (entry["node_name"].lower(), entry["node_id"])) + + nodes.append({ + "node_id": node.networkJsonId, + "node_name": node.displayName, + "current_parent_node_id": parent_node_id, + "current_parent_node_name": parent_node_name, + "candidate_parents": candidate_parents, + }) + + return { + "source": "python/integration_common", + "nodes": nodes, + } + + def materializeCompiledTopology(self, source: str, compileMode: str) -> None: + # Materializes compiler-owned topology artifacts without writing integration + # network.json or ShapedDevices.csv. + compatibility_network_json = json.dumps(self.buildNetworkJson()) + shaped_devices_csv, circuit_anchor_file = self.buildShapedDevicesArtifacts() + parent_candidates_json = json.dumps(self.buildTopologyParentCandidates()) + native_editor_json = json.dumps(self.buildNativeTopologyEditorState(source)) + write_compiled_topology_from_python_graph_payload( + source, + compileMode, + compatibility_network_json, + shaped_devices_csv, + json.dumps(circuit_anchor_file), + parent_candidates_json, + native_editor_json, + ) def plotNetworkGraph(self, showClients=False): # Requires `pip install graphviz` to function. diff --git a/src/integrationNetzur.py b/src/integrationNetzur.py index db09e308..a6064af6 100644 --- a/src/integrationNetzur.py +++ b/src/integrationNetzur.py @@ -12,7 +12,6 @@ from liblqos_python import ( netzur_api_key, netzur_api_url, netzur_api_timeout, - overwrite_network_json_always, ) import json @@ -168,12 +167,8 @@ def createShaper() -> NetworkGraph: except json.JSONDecodeError as err: LOG.warning("[Netzur] Unable to parse Mikrotik IPv6 map: %s", err) - if net.doesNetworkJsonExist() and not overwrite_network_json_always(): - LOG.info("[Netzur] network.json exists and overwrite disabled; preserving current file") - else: - net.createNetworkJson() - net.createShapedDevices() - LOG.info("[Netzur] ShapedDevices.csv updated") + net.materializeCompiledTopology("python/netzur", "full") + LOG.info("[Netzur] Topology import artifacts updated") for error in net.getErrors(): LOG.error("[Netzur] %s", error) return net diff --git a/src/integrationPowercode.py b/src/integrationPowercode.py index ef31935a..bfd855eb 100644 --- a/src/integrationPowercode.py +++ b/src/integrationPowercode.py @@ -111,11 +111,7 @@ def createShaper(): print("Imported " + str(len(devicesByCustomerID)) + " customers") net.prepareTree() net.plotNetworkGraph(False) - if net.doesNetworkJsonExist(): - print("network.json already exists. Leaving in-place.") - else: - net.createNetworkJson() - net.createShapedDevices() + net.materializeCompiledTopology("python/powercode", "full") def importFromPowercode(): #createNetworkJSON() diff --git a/src/integrationSonar.py b/src/integrationSonar.py index 581e45b6..d9c07246 100644 --- a/src/integrationSonar.py +++ b/src/integrationSonar.py @@ -6,7 +6,7 @@ import time from urllib.parse import urlsplit, urlunsplit from liblqos_python import sonar_api_key, sonar_api_url, snmp_community, sonar_airmax_ap_model_ids, \ sonar_ltu_ap_model_ids, sonar_active_status_ids, sonar_recurring_service_rates, \ - sonar_recurring_excluded_service_names + sonar_recurring_excluded_service_names, sonar_strategy all_models = sonar_airmax_ap_model_ids() + sonar_ltu_ap_model_ids() from integrationCommon import NetworkGraph, NetworkNode, NodeType, apply_client_bandwidth_multiplier from multiprocessing.pool import ThreadPool @@ -777,8 +777,9 @@ def createShaper(): net.prepareTree() net.plotNetworkGraph(False) - net.createNetworkJson() - net.createShapedDevices() + mode = sonar_strategy() + print(f"Using Sonar topology mode: {mode}") + net.materializeCompiledTopology("python/sonar", mode) def importFromSonar(): createShaper() diff --git a/src/integrationSplynx.py b/src/integrationSplynx.py index a99c2a3c..e0d6064b 100644 --- a/src/integrationSplynx.py +++ b/src/integrationSplynx.py @@ -6,8 +6,8 @@ import requests import warnings import os import csv -from liblqos_python import exclude_sites, find_ipv6_using_mikrotik, splynx_api_key, \ - splynx_api_secret, splynx_api_url, splynx_strategy, overwrite_network_json_always +from liblqos_python import exclude_sites, find_ipv6_using_mikrotik, migrate_legacy_site_bandwidth_csv, \ + overrides_network_adjustments_materialized, splynx_api_key, splynx_api_secret, splynx_api_url, splynx_strategy from integrationCommon import apply_client_bandwidth_multiplier, isIpv4Permitted import base64 @@ -80,6 +80,46 @@ def stable_splynx_device_id(service_id, equipment_id=None): return base_id return f"{base_id}_equipment_{equipment_id}" + +def normalize_splynx_graph_id(value): + """ + Normalize upstream Splynx IDs into one graph-stable string form. + """ + if value in (None, "", 0, "0"): + return None + return str(value) + + +def splynx_generated_unattached_site_id(): + """ + Stable raw graph ID for Splynx full-mode circuits with no inferred parent. + """ + return "splynx_generated_unattached_site" + + +def splynx_generated_unattached_site_network_id(): + """ + Stable exported topology ID for Splynx full-mode fallback grouping. + """ + return "libreqos:generated:splynx:site:unattached" + + +def splynx_generated_unattached_site_name(): + """ + Display name for the explicit Splynx full-mode fallback site. + """ + return "LibreQoS Unattached [Site]" + + +def splynx_generated_unattached_parent_id(strategy_name: str): + """ + Only full mode should materialize a stable generated site for unresolved parents. + Other modes handle unresolved circuits through their own compiler/runtime fallbacks. + """ + if strategy_name == "full": + return splynx_generated_unattached_site_id() + return None + def supplement_existing_devices_with_online_ips(net, allServices, service_ids_handled, customersOnline, cust_id_to_name, allocated_ipv4s, allocated_ipv6s, device_by_service_id=None): """ For services already handled (devices created from static IPs), supplement missing @@ -183,6 +223,28 @@ def create_devices_from_online_for_unhandled_services(net, allServices, service_ matched_via_alternate_method += 1 return matched_via_alternate_method +def has_network_sites_in_monitoring(network_sites, monitoring): + """ + Return True when Splynx network sites are present and monitoring devices + actually reference them. + """ + if not network_sites: + return False + for dev in monitoring: + if dev.get('network_site_id') not in (None, 0, "0", ""): + return True + return False + + +def strategy_uses_network_sites_topology(strategy_name: str, network_sites, monitoring): + """ + Only ap_site should flatten infrastructure onto Splynx Network Sites. + + full mode is expected to preserve the richer monitoring parent hierarchy. + """ + return strategy_name == 'ap_site' and has_network_sites_in_monitoring(network_sites, monitoring) + + def run_splynx_pipeline(strategy_name: str): """ Unified pipeline to fetch data, build infrastructure, create static clients, @@ -230,8 +292,8 @@ def run_splynx_pipeline(strategy_name: str): print("Successfully fetched data from Splynx") # Precompute access_device IDs from services to improve AP detection for service in allServices: - ad = service.get('access_device') - if ad not in (None, 0, "0", ""): + ad = normalize_splynx_graph_id(service.get('access_device')) + if ad is not None: access_device_ids.add(ad) # Build basic customer map cust_id_to_name = {c['id']: c['name'] for c in customers} @@ -244,13 +306,14 @@ def run_splynx_pipeline(strategy_name: str): ap_nodes = {} if monitoring: for dev in monitoring: - dev_id = dev.get('id') + dev_id = normalize_splynx_graph_id(dev.get('id')) if dev_id is None: continue dev_name = dev.get('title') or dev.get('name') or dev.get('address') or dev.get('ip') or str(dev_id) hardware_name[dev_id] = dev_name - if 'parent_id' in dev: - hardware_parent[dev_id] = dev['parent_id'] + parent_id = normalize_splynx_graph_id(dev.get('parent_id')) + if parent_id is not None: + hardware_parent[dev_id] = parent_id # Determine AP vs Site: prefer access_device flag, fall back to legacy type is_ap = False if 'access_device' in dev: @@ -266,7 +329,7 @@ def run_splynx_pipeline(strategy_name: str): else: hardware_type[dev_id] = 'Site' for dev in monitoring: - dev_id = dev.get('id') + dev_id = normalize_splynx_graph_id(dev.get('id')) if dev_id is None: continue dev_title = hardware_name.get(dev_id, str(dev_id)) @@ -296,21 +359,18 @@ def run_splynx_pipeline(strategy_name: str): def ap_node_id(raw_id): return f"ap_{raw_id}" - def has_network_sites(): - if not network_sites: - return False - for dev in monitoring: - if dev.get('network_site_id') not in (None, 0, "0", ""): - return True - return False - - use_network_sites = strategy_name in ('ap_site', 'full') and has_network_sites() + use_network_sites = strategy_uses_network_sites_topology( + strategy_name, + network_sites, + monitoring, + ) # Infrastructure builder def build_infrastructure(): if strategy_name == 'flat': return - # Prefer network sites when available (ap_site/full) + # Prefer network sites only for ap_site. full mode should preserve the + # richer monitoring parent hierarchy instead of flattening to site -> AP. if use_network_sites: print(f"Creating site and AP infrastructure using Network Sites ({len(network_sites)} sites)") for site_id, node_id in site_id_to_node_id.items(): @@ -364,6 +424,19 @@ def run_splynx_pipeline(strategy_name: str): # ap_site and full print("Creating site and AP infrastructure") createInfrastructureNodes(net, monitoring, hardware_name, hardware_parent, hardware_type, siteBandwidth, hardware_name_extended) + if strategy_name == 'full': + net.addRawNode( + NetworkNode( + id=splynx_generated_unattached_site_id(), + displayName=splynx_generated_unattached_site_name(), + type=NodeType.site, + parentId=None, + download=10000, + upload=10000, + address=splynx_generated_unattached_site_name(), + networkJsonId=splynx_generated_unattached_site_network_id(), + ) + ) # Parent selector per strategy def select_parent(serviceItem): @@ -377,7 +450,7 @@ def run_splynx_pipeline(strategy_name: str): return None if strategy_name == 'ap_only': return parent_node_id if (parent_node_id in ap_nodes) else None - return parent_node_id + return parent_node_id or splynx_generated_unattached_parent_id(strategy_name) # Run pipeline build_infrastructure() @@ -418,14 +491,7 @@ def run_splynx_pipeline(strategy_name: str): # Finalize net.prepareTree() net.plotNetworkGraph(False) - if net.doesNetworkJsonExist(): - if overwrite_network_json_always: - net.createNetworkJson() - else: - print("network.json already exists. Leaving in-place.") - else: - net.createNetworkJson() - net.createShapedDevices() + net.materializeCompiledTopology("python/splynx", splynx_strategy()) def buildHeaders(): """ @@ -477,9 +543,40 @@ def getTariffs(headers): def buildSiteBandwidths(): """ - Build a dictionary of site bandwidths by reading data from a CSV file. + Build a dictionary of site bandwidths from operator overrides, with a legacy CSV fallback. """ siteBandwidth = {} + try: + migrate_legacy_site_bandwidth_csv("integrationSplynxBandwidths.csv") + except Exception as e: + warnings.warn( + f"Unable to migrate legacy Splynx bandwidth overrides into lqos_overrides.json: {e}", + stacklevel=2, + ) + + try: + for adjustment in overrides_network_adjustments_materialized(): + if adjustment.get('type') != 'adjust_site_speed': + continue + name = adjustment.get('site_name', '') + if not name: + continue + download = adjustment.get('download_bandwidth_mbps') + upload = adjustment.get('upload_bandwidth_mbps') + if download is None and upload is None: + continue + current = siteBandwidth.get(name, {"download": 10000, "upload": 10000}) + if download is not None: + current["download"] = int(float(download)) + if upload is not None: + current["upload"] = int(float(upload)) + siteBandwidth[name] = current + except Exception as e: + warnings.warn( + f"Unable to load site bandwidth overrides from lqos_overrides.json: {e}", + stacklevel=2, + ) + if os.path.isfile("integrationSplynxBandwidths.csv"): with open('integrationSplynxBandwidths.csv') as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') @@ -512,9 +609,11 @@ def getRouters(headers): ipForRouter = {} nameForRouterID = {} for router in data: - routerID = router['id'] - if router['id'] not in routerIdList: - routerIdList.append(router['id']) + routerID = normalize_splynx_graph_id(router.get('id')) + if routerID is None: + continue + if routerID not in routerIdList: + routerIdList.append(routerID) ipForRouter[routerID] = router['ip'] nameForRouterID[routerID] = router['title'] print("Router IPs found: " + str(len(ipForRouter))) @@ -527,7 +626,9 @@ def getSectors(headers): data = splynx_request("admin/networking/routers-sectors", headers) sectorForRouter = {} for sector in data: - routerID = sector['router_id'] + routerID = normalize_splynx_graph_id(sector.get('router_id')) + if routerID is None: + continue if routerID not in sectorForRouter: newList = [] newList.append(sector) @@ -682,13 +783,13 @@ def createInfrastructureNodes(net, monitoring, hardware_name, hardware_parent, h """ monitoring_gps = {} for dev in monitoring: - if 'id' in dev: - monitoring_gps[str(dev.get('id'))] = parse_gps_pair(dev.get('gps')) + dev_id = normalize_splynx_graph_id(dev.get('id')) + if dev_id is not None: + monitoring_gps[dev_id] = parse_gps_pair(dev.get('gps')) for device_num in hardware_name: parent_id = None if device_num in hardware_parent.keys(): - if hardware_parent[device_num] != 0: - parent_id = hardware_parent[device_num] + parent_id = normalize_splynx_graph_id(hardware_parent[device_num]) download = 10000 upload = 10000 nodeName = hardware_name_extended[device_num] @@ -696,7 +797,7 @@ def createInfrastructureNodes(net, monitoring, hardware_name, hardware_parent, h download = siteBandwidth[nodeName]["download"] upload = siteBandwidth[nodeName]["upload"] nodeType = hardware_type[device_num] - latitude, longitude = monitoring_gps.get(str(device_num), (None, None)) + latitude, longitude = monitoring_gps.get(device_num, (None, None)) if nodeType == 'AP': node = NetworkNode(id=device_num, displayName=nodeName, type=NodeType.ap, parentId=parent_id, download=download, upload=upload, address=None, @@ -716,15 +817,16 @@ def findBestParentNode(serviceItem, hardware_name, ipForRouter, sectorForRouter) assignment_method = 'none' # Method 1: Direct access_device assignment (highest priority) - if 'access_device' in serviceItem and serviceItem['access_device'] != 0: - if serviceItem['access_device'] in hardware_name: - parent_node_id = serviceItem['access_device'] + access_device = normalize_splynx_graph_id(serviceItem.get('access_device')) + if access_device is not None: + if access_device in hardware_name: + parent_node_id = access_device assignment_method = 'access_device' return parent_node_id, assignment_method # Method 2: Router ID assignment - if 'router_id' in serviceItem and serviceItem['router_id'] != 0: - router_id = serviceItem['router_id'] + router_id = normalize_splynx_graph_id(serviceItem.get('router_id')) + if router_id is not None: if router_id in hardware_name: parent_node_id = router_id assignment_method = 'router_id' @@ -735,8 +837,9 @@ def findBestParentNode(serviceItem, hardware_name, ipForRouter, sectorForRouter) if sectors and len(sectors) > 0: # Use first sector as parent sector = sectors[0] - if 'id' in sector and sector['id'] in hardware_name: - parent_node_id = sector['id'] + sector_id = normalize_splynx_graph_id(sector.get('id')) + if sector_id is not None and sector_id in hardware_name: + parent_node_id = sector_id assignment_method = 'sector_id' return parent_node_id, assignment_method diff --git a/src/integrationVISP.py b/src/integrationVISP.py index 972334a3..e38f1c06 100644 --- a/src/integrationVISP.py +++ b/src/integrationVISP.py @@ -12,7 +12,6 @@ from collections import Counter from liblqos_python import ( get_libreqos_directory, - overwrite_network_json_always, visp_client_id, visp_client_secret, visp_username, @@ -419,12 +418,7 @@ def createShaper() -> None: pass net.prepareTree() - - if net.doesNetworkJsonExist() and not overwrite_network_json_always(): - print("[VISP] network.json exists and overwrite disabled; preserving current file") - else: - net.createNetworkJson() - net.createShapedDevices() + net.materializeCompiledTopology("python/visp", "full") print( "[VISP] shaped={shaped} wifi_bulk_services={wifi_bulk_services} " diff --git a/src/integrationWISPGate.py b/src/integrationWISPGate.py index 8eb19807..d5667a92 100644 --- a/src/integrationWISPGate.py +++ b/src/integrationWISPGate.py @@ -5,7 +5,7 @@ import requests import warnings import os import csv -from liblqos_python import exclude_sites, find_ipv6_using_mikrotik, bandwidth_overhead_factor, overwrite_network_json_always, wispgate_api_token, wispgate_api_url +from liblqos_python import exclude_sites, find_ipv6_using_mikrotik, bandwidth_overhead_factor, wispgate_api_token, wispgate_api_url from integrationCommon import isIpv4Permitted if find_ipv6_using_mikrotik() == True: @@ -138,15 +138,7 @@ def createShaper(): print("Imported " + "{:.0%}".format(circuits_added_counter/len(shaped_devices)) + " of known shaped devices from WISPGate.") net.prepareTree() net.plotNetworkGraph(False) - - if net.doesNetworkJsonExist(): - if overwrite_network_json_always: - net.createNetworkJson() - else: - print("network.json already exists. Leaving in-place.") - else: - net.createNetworkJson() - net.createShapedDevices() + net.materializeCompiledTopology("python/wispgate", "full") def importFromWISPGate(): """ diff --git a/src/lqos.example b/src/lqos.example index 4bc99519..7eecbea4 100644 --- a/src/lqos.example +++ b/src/lqos.example @@ -68,7 +68,6 @@ do_not_track_subnets = [ "192.168.66.0/24" ] [integration_common] circuit_name_as_address = false -always_overwrite_network_json = true queue_refresh_interval_mins = 30 use_mikrotik_ipv6 = false diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index e8c923bd..e735c9d1 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -2132,11 +2132,13 @@ dependencies = [ "anyhow", "base64", "chrono", + "csv", "lqos_bakery", "lqos_bus", "lqos_config", "lqos_overrides", "lqos_sys", + "lqos_topology_compile", "lqos_utils", "miniz_oxide", "nix", @@ -2144,6 +2146,7 @@ dependencies = [ "reqwest", "serde", "serde_cbor", + "serde_json", "sysinfo", "tempfile", "tokio", @@ -2237,12 +2240,24 @@ dependencies = [ "lqos_config", "lqos_overrides", "lqos_probe", + "lqos_topology_compile", "serde_json", "tokio", "tracing", "tracing-subscriber", ] +[[package]] +name = "lqos_topology_compile" +version = "0.1.0" +dependencies = [ + "anyhow", + "lqos_config", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "lqos_utils" version = "0.1.0" @@ -2297,6 +2312,7 @@ dependencies = [ "lqos_stormguard", "lqos_sys", "lqos_topology", + "lqos_topology_compile", "lqos_utils", "miniz_oxide", "native-tls", @@ -4303,6 +4319,7 @@ dependencies = [ "lqos_bus", "lqos_config", "lqos_overrides", + "lqos_topology_compile", "lqos_utils", "petgraph", "serde", diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml index 03389574..2ea377bb 100644 --- a/src/rust/Cargo.toml +++ b/src/rust/Cargo.toml @@ -35,6 +35,7 @@ members = [ "lqos_bakery", # The bakery makes CAKEs - controls queue creation. "lqos_overrides", # A CLI tool and library for unifying the override system and allowing API support for changing network.json and ShapedDevices.csv "lqos_topology", # Shared topology runtime domain logic and health/effective-state helper. + "lqos_topology_compile", # Shared topology compilation and mode projection layer. ] [dependencies] diff --git a/src/rust/lqos_bakery/src/lib.rs b/src/rust/lqos_bakery/src/lib.rs index 739ae48c..3d07722d 100644 --- a/src/rust/lqos_bakery/src/lib.rs +++ b/src/rust/lqos_bakery/src/lib.rs @@ -2656,6 +2656,26 @@ fn migration_target_label(migration: &Migration) -> String { fn runtime_network_json_path(config: &Config) -> std::path::PathBuf { let base_path = Path::new(&config.lqos_directory); + let effective_path = base_path.join("network.effective.json"); + let integration_ingress_enabled = config.uisp_integration.enable_uisp + || config.splynx_integration.enable_splynx + || config + .netzur_integration + .as_ref() + .is_some_and(|integration| integration.enable_netzur) + || config + .visp_integration + .as_ref() + .is_some_and(|integration| integration.enable_visp) + || config.powercode_integration.enable_powercode + || config.sonar_integration.enable_sonar + || config + .wispgate_integration + .as_ref() + .is_some_and(|integration| integration.enable_wispgate); + if effective_path.exists() || integration_ingress_enabled { + return effective_path; + } if config .long_term_stats .enable_insight_topology diff --git a/src/rust/lqos_config/src/etc/migration.rs b/src/rust/lqos_config/src/etc/migration.rs index 4e04c636..d0d5e74b 100644 --- a/src/rust/lqos_config/src/etc/migration.rs +++ b/src/rust/lqos_config/src/etc/migration.rs @@ -254,8 +254,6 @@ fn migrate_integration_common( new_config: &mut Config, ) -> Result<(), MigrationError> { new_config.integration_common.circuit_name_as_address = python_config.circuit_name_use_address; - new_config.integration_common.always_overwrite_network_json = - python_config.overwrite_network_jsonalways; new_config.integration_common.queue_refresh_interval_mins = python_config.queue_refresh_interval_mins as u32; new_config.integration_common.use_mikrotik_ipv6 = python_config.find_ipv6using_mikrotik_api; @@ -270,6 +268,7 @@ fn migrate_splynx( new_config.splynx_integration.api_key = python_config.splynx_api_key.clone(); new_config.splynx_integration.api_secret = python_config.splynx_api_secret.clone(); new_config.splynx_integration.url = python_config.splynx_api_url.clone(); + new_config.topology.compile_mode = "ap_site".to_string(); Ok(()) } @@ -303,6 +302,10 @@ fn migrate_uisp( new_config.uisp_integration.url = python_config.uispbase_url.clone(); new_config.uisp_integration.site = python_config.uisp_site.clone(); new_config.uisp_integration.strategy = python_config.uisp_strategy.clone(); + new_config.topology.compile_mode = + crate::etc::v15::normalize_topology_compile_mode(&python_config.uisp_strategy) + .unwrap_or("full") + .to_string(); new_config.uisp_integration.suspended_strategy = python_config.uisp_suspended_strategy.clone(); new_config.uisp_integration.airmax_capacity = python_config.air_max_capacity as f32; new_config.uisp_integration.ltu_capacity = python_config.ltu_capacity as f32; diff --git a/src/rust/lqos_config/src/etc/mod.rs b/src/rust/lqos_config/src/etc/mod.rs index 77eeb215..03b855e5 100644 --- a/src/rust/lqos_config/src/etc/mod.rs +++ b/src/rust/lqos_config/src/etc/mod.rs @@ -10,7 +10,7 @@ use once_cell::sync::Lazy; use std::path::Path; use std::sync::Arc; use thiserror::Error; -use toml_edit::{DocumentMut, value}; +use toml_edit::{DocumentMut, Item, Table, value}; use tracing::{debug, error, info}; mod migration; @@ -20,9 +20,9 @@ pub mod test_data; mod v15; pub use v15::{ BridgeConfig, IntegrationConfig, LazyQueueMode, QueueMode, RttThresholds, - SingleInterfaceConfig, StormguardConfig, StormguardStrategy, TreeguardCircuitsConfig, - TreeguardConfig, TreeguardCpuConfig, TreeguardCpuMode, TreeguardLinksConfig, - TreeguardQooConfig, Tunables, + SingleInterfaceConfig, StormguardConfig, StormguardStrategy, TopologyConfig, + TreeguardCircuitsConfig, TreeguardConfig, TreeguardCpuConfig, TreeguardCpuMode, + TreeguardLinksConfig, TreeguardQooConfig, Tunables, }; static CONFIG: Lazy>>> = Lazy::new(|| ArcSwap::from_pointee(None)); @@ -64,6 +64,117 @@ fn treeguard_cpu_mode_migration_stamp_path(config_path: &str) -> String { format!("{config_path}.treeguard_cpu_mode_migrated") } +fn uisp_capacity_defaults_migration_stamp_path(config_path: &str) -> String { + format!("{config_path}.uisp_capacity_defaults_migrated") +} + +fn topology_compile_mode_migration_stamp_path(config_path: &str) -> String { + format!("{config_path}.topology_compile_mode_migrated") +} + +fn enabled_table_string( + doc: &DocumentMut, + table_name: &str, + enabled_key: &str, + value_key: &str, +) -> Option { + let table = doc.get(table_name)?.as_table_like()?; + if !table.get(enabled_key)?.as_bool().unwrap_or(false) { + return None; + } + let raw = table.get(value_key)?.as_str()?.trim(); + if raw.is_empty() { + return None; + } + crate::etc::v15::normalize_topology_compile_mode(raw).map(ToString::to_string) +} + +fn maybe_migrate_topology_compile_mode( + config_location: &str, + raw: String, +) -> Result { + let stamp_path = topology_compile_mode_migration_stamp_path(config_location); + if Path::new(&stamp_path).exists() { + return Ok(raw); + } + + let mut doc: DocumentMut = raw.parse().map_err(|e| LibreQoSConfigError::ParseError { + path: config_location.to_string(), + details: format!("Error parsing config: {e}"), + })?; + + if let Some(existing) = doc + .get("topology") + .and_then(|section| section.as_table_like()) + .and_then(|section| section.get("compile_mode")) + .and_then(|mode| mode.as_str()) + .and_then(crate::etc::v15::normalize_topology_compile_mode) + { + std::fs::write(&stamp_path, format!("{existing}\n")).map_err(|e| { + LibreQoSConfigError::CannotWrite { + path: stamp_path.clone(), + source: e, + } + })?; + return Ok(raw); + } + + let uisp_mode = enabled_table_string(&doc, "uisp_integration", "enable_uisp", "strategy"); + let splynx_mode = enabled_table_string(&doc, "splynx_integration", "enable_splynx", "strategy"); + + let selected = match (uisp_mode.as_deref(), splynx_mode.as_deref()) { + (Some(uisp), Some(splynx)) if uisp == splynx => Some(uisp.to_string()), + (Some(_), Some(_)) => None, + (Some(uisp), None) => Some(uisp.to_string()), + (None, Some(splynx)) => Some(splynx.to_string()), + (None, None) => None, + }; + + let Some(selected) = selected else { + if uisp_mode.is_some() && splynx_mode.is_some() { + info!( + "Topology compile mode migration skipped because UISP and Splynx legacy strategies differ." + ); + } + std::fs::write(&stamp_path, b"skipped\n").map_err(|e| { + LibreQoSConfigError::CannotWrite { + path: stamp_path, + source: e, + } + })?; + return Ok(raw); + }; + + if !doc.as_table().contains_key("topology") { + doc["topology"] = Item::Table(Table::new()); + } + doc["topology"]["compile_mode"] = value(selected.as_str()); + let migrated = doc.to_string(); + + let config_path = Path::new(config_location); + if config_path.exists() { + let backup_path = format!("{config_location}.topology_compile_mode_backup"); + std::fs::copy(config_path, &backup_path).map_err(|e| LibreQoSConfigError::CannotWrite { + path: backup_path, + source: e, + })?; + } + + std::fs::write(config_path, &migrated).map_err(|e| LibreQoSConfigError::CannotWrite { + path: config_location.to_string(), + source: e, + })?; + std::fs::write(&stamp_path, format!("{selected}\n")).map_err(|e| { + LibreQoSConfigError::CannotWrite { + path: stamp_path, + source: e, + } + })?; + + info!("Topology compile mode was automatically migrated to '{selected}'."); + Ok(migrated) +} + fn maybe_migrate_treeguard_cpu_mode( config_location: &str, raw: String, @@ -128,6 +239,58 @@ fn maybe_migrate_treeguard_cpu_mode( Ok(migrated) } +fn maybe_migrate_uisp_capacity_defaults( + config_location: &str, + raw: String, +) -> Result { + let stamp_path = uisp_capacity_defaults_migration_stamp_path(config_location); + if Path::new(&stamp_path).exists() { + return Ok(raw); + } + + let mut doc: DocumentMut = raw.parse().map_err(|e| LibreQoSConfigError::ParseError { + path: config_location.to_string(), + details: format!("Error parsing config: {e}"), + })?; + + if !doc.as_table().contains_key("uisp_integration") { + doc["uisp_integration"] = Item::Table(Table::new()); + } + + doc["uisp_integration"]["airmax_capacity"] = value(1.0); + doc["uisp_integration"]["ltu_capacity"] = value(1.0); + let migrated = doc.to_string(); + + if migrated != raw { + let config_path = Path::new(config_location); + if config_path.exists() { + let backup_path = format!("{config_location}.uisp_capacity_defaults_backup"); + std::fs::copy(config_path, &backup_path).map_err(|e| { + LibreQoSConfigError::CannotWrite { + path: backup_path, + source: e, + } + })?; + } + + std::fs::write(config_path, &migrated).map_err(|e| LibreQoSConfigError::CannotWrite { + path: config_location.to_string(), + source: e, + })?; + + info!( + "UISP AirMax and LTU capacity defaults were automatically normalized to 1.0 during upgrade." + ); + } + + std::fs::write(&stamp_path, b"1.0\n").map_err(|e| LibreQoSConfigError::CannotWrite { + path: stamp_path, + source: e, + })?; + + Ok(migrated) +} + fn actually_load_from_disk() -> Result, LibreQoSConfigError> { let config_location = if let Ok(lqos_config) = std::env::var("LQOS_CONFIG") { info!("Overriding lqos.conf location from environment variable."); @@ -169,6 +332,8 @@ fn actually_load_from_disk() -> Result, LibreQoSConfigError> { })?; let raw = maybe_migrate_treeguard_cpu_mode(&config_location, raw)?; + let raw = maybe_migrate_uisp_capacity_defaults(&config_location, raw)?; + let raw = maybe_migrate_topology_compile_mode(&config_location, raw)?; let mut final_config = Config::load_from_string(&raw).map_err(|e| { error!("Unable to parse {config_location}"); @@ -287,3 +452,189 @@ pub enum LibreQoSConfigError { source: std::io::Error, }, } + +#[cfg(test)] +mod tests { + use super::{ + maybe_migrate_topology_compile_mode, maybe_migrate_uisp_capacity_defaults, + topology_compile_mode_migration_stamp_path, uisp_capacity_defaults_migration_stamp_path, + }; + use std::fs; + use std::path::{Path, PathBuf}; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn unique_test_dir() -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + std::env::temp_dir().join(format!( + "lqos_config_uisp_capacity_test_{}_{}", + std::process::id(), + nanos + )) + } + + fn write_test_config(base_dir: &Path, raw: &str) -> PathBuf { + fs::create_dir_all(base_dir).expect("should create temp dir"); + let config_path = base_dir.join("lqos.conf"); + fs::write(&config_path, raw).expect("should write test config"); + config_path + } + + fn path_string(path: &Path) -> String { + path.to_string_lossy().into_owned() + } + + #[test] + fn uisp_capacity_migration_rewrites_existing_values_and_stamps() { + let test_dir = unique_test_dir(); + let raw = "[uisp_integration]\nairmax_capacity = 0.65\nltu_capacity = 0.9\n"; + let config_path = write_test_config(&test_dir, raw); + + let config_path_str = path_string(&config_path); + let migrated = maybe_migrate_uisp_capacity_defaults(&config_path_str, raw.to_string()) + .expect("migration should succeed"); + + assert!(migrated.contains("airmax_capacity = 1.0")); + assert!(migrated.contains("ltu_capacity = 1.0")); + assert_eq!( + fs::read_to_string(&config_path).expect("config should be rewritten"), + migrated + ); + assert_eq!( + fs::read_to_string(config_path.with_extension("conf.uisp_capacity_defaults_backup")) + .expect("backup should exist"), + raw + ); + assert!( + Path::new(&uisp_capacity_defaults_migration_stamp_path( + &config_path_str + )) + .exists() + ); + + fs::remove_dir_all(&test_dir).expect("should clean up temp dir"); + } + + #[test] + fn uisp_capacity_migration_stamps_already_correct_values_without_rewrite() { + let test_dir = unique_test_dir(); + let raw = "[uisp_integration]\nairmax_capacity = 1.0\nltu_capacity = 1.0\n"; + let config_path = write_test_config(&test_dir, raw); + + let config_path_str = path_string(&config_path); + let migrated = maybe_migrate_uisp_capacity_defaults(&config_path_str, raw.to_string()) + .expect("migration should succeed"); + + assert_eq!(migrated, raw); + assert_eq!( + fs::read_to_string(&config_path).expect("config should remain unchanged"), + raw + ); + assert!( + !config_path + .with_extension("conf.uisp_capacity_defaults_backup") + .exists() + ); + assert!( + Path::new(&uisp_capacity_defaults_migration_stamp_path( + &config_path_str + )) + .exists() + ); + + fs::remove_dir_all(&test_dir).expect("should clean up temp dir"); + } + + #[test] + fn uisp_capacity_migration_respects_manual_changes_after_stamp() { + let test_dir = unique_test_dir(); + let raw = "[uisp_integration]\nairmax_capacity = 0.65\nltu_capacity = 0.9\n"; + let config_path = write_test_config(&test_dir, raw); + + let config_path_str = path_string(&config_path); + maybe_migrate_uisp_capacity_defaults(&config_path_str, raw.to_string()) + .expect("initial migration should succeed"); + + let manual = "[uisp_integration]\nairmax_capacity = 0.72\nltu_capacity = 0.81\n"; + fs::write(&config_path, manual).expect("should simulate operator edit"); + + let second = maybe_migrate_uisp_capacity_defaults(&config_path_str, manual.to_string()) + .expect("second migration should skip after stamp"); + + assert_eq!(second, manual); + assert_eq!( + fs::read_to_string(&config_path).expect("manual edit should remain"), + manual + ); + + fs::remove_dir_all(&test_dir).expect("should clean up temp dir"); + } + + #[test] + fn topology_compile_mode_migration_copies_enabled_uisp_strategy() { + let test_dir = unique_test_dir(); + let raw = "[uisp_integration]\nenable_uisp = true\nstrategy = \"full2\"\n"; + let config_path = write_test_config(&test_dir, raw); + + let config_path_str = path_string(&config_path); + let migrated = maybe_migrate_topology_compile_mode(&config_path_str, raw.to_string()) + .expect("migration should succeed"); + + assert!(migrated.contains("[topology]")); + assert!(migrated.contains("compile_mode = \"full\"")); + assert!( + Path::new(&topology_compile_mode_migration_stamp_path( + &config_path_str + )) + .exists() + ); + + fs::remove_dir_all(&test_dir).expect("should clean up temp dir"); + } + + #[test] + fn topology_compile_mode_migration_copies_enabled_splynx_strategy() { + let test_dir = unique_test_dir(); + let raw = "[splynx_integration]\nenable_splynx = true\nstrategy = \"ap_site\"\n"; + let config_path = write_test_config(&test_dir, raw); + + let config_path_str = path_string(&config_path); + let migrated = maybe_migrate_topology_compile_mode(&config_path_str, raw.to_string()) + .expect("migration should succeed"); + + assert!(migrated.contains("[topology]")); + assert!(migrated.contains("compile_mode = \"ap_site\"")); + assert!( + Path::new(&topology_compile_mode_migration_stamp_path( + &config_path_str + )) + .exists() + ); + + fs::remove_dir_all(&test_dir).expect("should clean up temp dir"); + } + + #[test] + fn topology_compile_mode_migration_skips_conflicting_legacy_strategies() { + let test_dir = unique_test_dir(); + let raw = "[uisp_integration]\nenable_uisp = true\nstrategy = \"full\"\n\n[splynx_integration]\nenable_splynx = true\nstrategy = \"ap_only\"\n"; + let config_path = write_test_config(&test_dir, raw); + + let config_path_str = path_string(&config_path); + let migrated = maybe_migrate_topology_compile_mode(&config_path_str, raw.to_string()) + .expect("migration should succeed"); + + assert_eq!(migrated, raw); + assert!(!migrated.contains("[topology]")); + assert!( + Path::new(&topology_compile_mode_migration_stamp_path( + &config_path_str + )) + .exists() + ); + + fs::remove_dir_all(&test_dir).expect("should clean up temp dir"); + } +} diff --git a/src/rust/lqos_config/src/etc/test_data.rs b/src/rust/lqos_config/src/etc/test_data.rs index 6acd266c..6855e58c 100644 --- a/src/rust/lqos_config/src/etc/test_data.rs +++ b/src/rust/lqos_config/src/etc/test_data.rs @@ -122,7 +122,7 @@ influxDBtoken = \"\" # Use Customer Name or Address as Circuit Name circuitNameUseAddress = True -# Should integrationUISP overwrite network.json on each run? +# Legacy network.json overwrite flag from older configs. Modern integration mode ignores it. overwriteNetworkJSONalways = False # If a device shows a WAN IP within these subnets, assume they are behind NAT / un-shapable, and ignore them diff --git a/src/rust/lqos_config/src/etc/v15/example.toml b/src/rust/lqos_config/src/etc/v15/example.toml index 247f8ed3..117c45fd 100644 --- a/src/rust/lqos_config/src/etc/v15/example.toml +++ b/src/rust/lqos_config/src/etc/v15/example.toml @@ -98,7 +98,6 @@ allow_subnets = [ "172.16.0.0/12", "10.0.0.0/8", "100.64.0.0/16", "192.168.0.0/1 [integration_common] circuit_name_as_address = false -always_overwrite_network_json = false queue_refresh_interval_mins = 30 use_mikrotik_ipv6 = false ethernet_port_limits_enabled = true @@ -137,6 +136,9 @@ timeout_secs = 20 # Optional RADIUS domain for online session enrichment. #online_users_domain = "" +[topology] +compile_mode = "ap_site" + [uisp_integration] enable_uisp = false token = "" diff --git a/src/rust/lqos_config/src/etc/v15/integration_common.rs b/src/rust/lqos_config/src/etc/v15/integration_common.rs index 074d4817..06867725 100644 --- a/src/rust/lqos_config/src/etc/v15/integration_common.rs +++ b/src/rust/lqos_config/src/etc/v15/integration_common.rs @@ -78,9 +78,6 @@ pub struct IntegrationConfig { /// Replace names with addresses? pub circuit_name_as_address: bool, - /// Always overwrite network.json? - pub always_overwrite_network_json: bool, - /// Queue refresh interval in minutes pub queue_refresh_interval_mins: u32, @@ -113,7 +110,6 @@ impl Default for IntegrationConfig { fn default() -> Self { Self { circuit_name_as_address: false, - always_overwrite_network_json: true, queue_refresh_interval_mins: 30, use_mikrotik_ipv6: false, promote_to_root: None, diff --git a/src/rust/lqos_config/src/etc/v15/mod.rs b/src/rust/lqos_config/src/etc/v15/mod.rs index 10232418..6d4938ec 100644 --- a/src/rust/lqos_config/src/etc/v15/mod.rs +++ b/src/rust/lqos_config/src/etc/v15/mod.rs @@ -15,6 +15,7 @@ mod queues; mod sonar_integration; mod splynx_integration; mod stormguard; +mod topology; mod treeguard; mod tuning; mod uisp_integration; @@ -26,6 +27,7 @@ pub use integration_common::IntegrationConfig; pub use long_term_stats::LongTermStats; pub use queues::{LazyQueueMode, QueueMode}; pub use stormguard::{StormguardConfig, StormguardStrategy}; +pub use topology::{TopologyConfig, normalize_topology_compile_mode}; pub use treeguard::{ TreeguardCircuitsConfig, TreeguardConfig, TreeguardCpuConfig, TreeguardCpuMode, TreeguardLinksConfig, TreeguardQooConfig, diff --git a/src/rust/lqos_config/src/etc/v15/splynx_integration.rs b/src/rust/lqos_config/src/etc/v15/splynx_integration.rs index 480188bb..5b2a4c09 100644 --- a/src/rust/lqos_config/src/etc/v15/splynx_integration.rs +++ b/src/rust/lqos_config/src/etc/v15/splynx_integration.rs @@ -13,7 +13,7 @@ pub struct SplynxIntegration { } fn default_strategy() -> String { - "ap_only".to_string() + "ap_site".to_string() } impl Default for SplynxIntegration { @@ -23,7 +23,7 @@ impl Default for SplynxIntegration { api_key: "".to_string(), api_secret: "".to_string(), url: "".to_string(), - strategy: "ap_only".to_string(), + strategy: "ap_site".to_string(), } } } diff --git a/src/rust/lqos_config/src/etc/v15/top_config.rs b/src/rust/lqos_config/src/etc/v15/top_config.rs index 5713076c..0a5f940e 100644 --- a/src/rust/lqos_config/src/etc/v15/top_config.rs +++ b/src/rust/lqos_config/src/etc/v15/top_config.rs @@ -113,6 +113,10 @@ pub struct Config { #[serde(default)] pub integration_common: super::integration_common::IntegrationConfig, + /// Shared topology compiler settings. + #[serde(default)] + pub topology: super::topology::TopologyConfig, + /// Splynx Integration configuration. Optional so older configs without this /// section still deserialize cleanly. #[serde(default)] @@ -229,6 +233,14 @@ impl Config { .to_string(), ); } + if !self.topology.compile_mode.trim().is_empty() + && super::topology::normalize_topology_compile_mode(&self.topology.compile_mode) + .is_none() + { + return Err( + "topology.compile_mode must be one of flat, ap_only, ap_site, or full".to_string(), + ); + } let airmax_flexible_frame_download_ratio = self.uisp_integration.airmax_flexible_frame_download_ratio; if !airmax_flexible_frame_download_ratio.is_finite() @@ -321,6 +333,7 @@ impl Default for Config { long_term_stats: super::long_term_stats::LongTermStats::default(), ip_ranges: super::ip_ranges::IpRanges::default(), integration_common: super::integration_common::IntegrationConfig::default(), + topology: super::topology::TopologyConfig::default(), splynx_integration: super::splynx_integration::SplynxIntegration::default(), netzur_integration: Some(super::netzur_integration::NetzurIntegration::default()), visp_integration: Some(super::visp_integration::VispIntegration::default()), @@ -346,6 +359,50 @@ impl Default for Config { } impl Config { + /// Returns the explicitly configured shared topology compile mode, when set. + pub fn shared_topology_compile_mode(&self) -> Option<&'static str> { + super::topology::normalize_topology_compile_mode(&self.topology.compile_mode) + } + + /// Returns the topology compile mode that UISP should use during the transition period. + pub fn resolved_topology_compile_mode_for_uisp(&self) -> &'static str { + self.shared_topology_compile_mode() + .filter(|mode| { + super::topology::integration_supports_topology_compile_mode("uisp", mode) + }) + .or_else(|| { + super::topology::normalize_supported_topology_compile_mode( + "uisp", + &self.uisp_integration.strategy, + ) + }) + .unwrap_or("full") + } + + /// Returns the topology compile mode that Splynx should use during the transition period. + pub fn resolved_topology_compile_mode_for_splynx(&self) -> &'static str { + self.shared_topology_compile_mode() + .filter(|mode| { + super::topology::integration_supports_topology_compile_mode("splynx", mode) + }) + .or_else(|| { + super::topology::normalize_supported_topology_compile_mode( + "splynx", + &self.splynx_integration.strategy, + ) + }) + .unwrap_or("ap_site") + } + + /// Returns the topology compile mode that Sonar should use. + pub fn resolved_topology_compile_mode_for_sonar(&self) -> &'static str { + self.shared_topology_compile_mode() + .filter(|mode| { + super::topology::integration_supports_topology_compile_mode("sonar", mode) + }) + .unwrap_or("full") + } + /// Calculate the unterface facing the Internet pub fn internet_interface(&self) -> String { if let Some(bridge) = &self.bridge { @@ -478,6 +535,33 @@ mod test { assert!(!config.sonar_integration.enable_sonar); } + #[test] + fn sonar_mode_resolution_rejects_ap_site_and_ap_only() { + let cfg = Config { + topology: crate::etc::v15::TopologyConfig { + compile_mode: "ap_site".to_string(), + }, + ..Config::default() + }; + assert_eq!(cfg.resolved_topology_compile_mode_for_sonar(), "full"); + + let cfg = Config { + topology: crate::etc::v15::TopologyConfig { + compile_mode: "ap_only".to_string(), + }, + ..Config::default() + }; + assert_eq!(cfg.resolved_topology_compile_mode_for_sonar(), "full"); + + let cfg = Config { + topology: crate::etc::v15::TopologyConfig { + compile_mode: "flat".to_string(), + }, + ..Config::default() + }; + assert_eq!(cfg.resolved_topology_compile_mode_for_sonar(), "flat"); + } + #[test] fn serialize_uses_splynx_keys() { let config = diff --git a/src/rust/lqos_config/src/etc/v15/topology.rs b/src/rust/lqos_config/src/etc/v15/topology.rs new file mode 100644 index 00000000..ceb78ab4 --- /dev/null +++ b/src/rust/lqos_config/src/etc/v15/topology.rs @@ -0,0 +1,48 @@ +use allocative::Allocative; +use serde::{Deserialize, Serialize}; + +/// Shared topology compiler settings used by `lqos_topology`. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Allocative)] +pub struct TopologyConfig { + /// Selected topology compile mode. + /// + /// Empty means "use legacy per-integration fallback" during the transition period. + #[serde(default)] + pub compile_mode: String, +} + +/// Normalizes operator-facing topology compile mode strings. +/// +/// `full2` is accepted as a legacy UISP alias and maps to `full`. +pub fn normalize_topology_compile_mode(mode: &str) -> Option<&'static str> { + match mode.trim().to_ascii_lowercase().as_str() { + "flat" => Some("flat"), + "ap_only" => Some("ap_only"), + "ap_site" => Some("ap_site"), + "full" => Some("full"), + "full2" => Some("full"), + _ => None, + } +} + +/// Returns true when `mode` is supported by the named integration. +pub fn integration_supports_topology_compile_mode(integration: &str, mode: &str) -> bool { + let Some(mode) = normalize_topology_compile_mode(mode) else { + return false; + }; + match integration.trim().to_ascii_lowercase().as_str() { + "uisp" | "splynx" => matches!(mode, "flat" | "ap_only" | "ap_site" | "full"), + "sonar" => matches!(mode, "flat" | "full"), + _ => matches!(mode, "full"), + } +} + +/// Resolves one operator-facing compile mode for a specific integration, returning `None` +/// when the requested mode is unsupported. +pub fn normalize_supported_topology_compile_mode( + integration: &str, + mode: &str, +) -> Option<&'static str> { + let mode = normalize_topology_compile_mode(mode)?; + integration_supports_topology_compile_mode(integration, mode).then_some(mode) +} diff --git a/src/rust/lqos_config/src/lib.rs b/src/rust/lqos_config/src/lib.rs index b206a782..be6a27a7 100644 --- a/src/rust/lqos_config/src/lib.rs +++ b/src/rust/lqos_config/src/lib.rs @@ -4,6 +4,8 @@ //! * The `/etc/lqos.conf` file. //! * `ShapedDevices.csv` files. //! * `network.json` files. +//! * `topology_import.json` files for compiler-backed integration ingress. +//! * `topology_compiled_shaping.json` files for compiler-selected integration shaping ingress. #![deny(clippy::unwrap_used)] #![warn(missing_docs)] @@ -36,7 +38,7 @@ pub use cpu_topology::{ }; pub use etc::{ BridgeConfig, Config, LazyQueueMode, QueueMode, RttThresholds, SingleInterfaceConfig, - StormguardConfig, StormguardStrategy, TreeguardCircuitsConfig, TreeguardConfig, + StormguardConfig, StormguardStrategy, TopologyConfig, TreeguardCircuitsConfig, TreeguardConfig, TreeguardCpuConfig, TreeguardCpuMode, TreeguardLinksConfig, TreeguardQooConfig, Tunables, clear_cached_config, disable_xdp_bridge, enable_long_term_stats, load_config, treeguard_cpu_mode_migration_notice, update_config, @@ -64,6 +66,7 @@ pub use topology_canonical_state::{ TOPOLOGY_CANONICAL_STATE_FILENAME, TopologyCanonicalIngressKind, TopologyCanonicalNode, TopologyCanonicalRateInput, TopologyCanonicalRateInputSource, TopologyCanonicalStateError, TopologyCanonicalStateFile, topology_canonical_state_path, + topology_ingress_identity_from_tokens, }; pub use topology_editor_state::{ TOPOLOGY_ATTACHMENT_AUTO_ID, TOPOLOGY_EDITOR_STATE_FILENAME, TopologyAllowedParent, @@ -76,16 +79,17 @@ pub use topology_parent_candidates::{ TopologyParentCandidatesFile, TopologyParentCandidatesNode, topology_parent_candidates_path, }; pub use topology_runtime_state::{ - TOPOLOGY_ATTACHMENT_HEALTH_STATE_FILENAME, TOPOLOGY_EFFECTIVE_NETWORK_FILENAME, - TOPOLOGY_EFFECTIVE_STATE_FILENAME, TOPOLOGY_RUNTIME_STATUS_FILENAME, - TOPOLOGY_SHAPING_INPUTS_FILENAME, TopologyAttachmentEndpointStatus, - TopologyAttachmentHealthEntry, TopologyAttachmentHealthStateFile, - TopologyEffectiveAttachmentState, TopologyEffectiveNodeState, TopologyEffectiveStateFile, - TopologyRuntimeStateError, TopologyRuntimeStatusFile, TopologyShapingCircuitInput, - TopologyShapingDeviceInput, TopologyShapingInputsFile, TopologyShapingResolutionSource, - compute_topology_source_generation, topology_attachment_health_state_path, - topology_effective_network_path, topology_effective_state_path, topology_runtime_status_path, - topology_shaping_inputs_path, + TOPOLOGY_ATTACHMENT_HEALTH_STATE_FILENAME, TOPOLOGY_COMPILED_SHAPING_FILENAME, + TOPOLOGY_EFFECTIVE_NETWORK_FILENAME, TOPOLOGY_EFFECTIVE_STATE_FILENAME, + TOPOLOGY_IMPORT_FILENAME, TOPOLOGY_RUNTIME_STATUS_FILENAME, TOPOLOGY_SHAPING_INPUTS_FILENAME, + TopologyAttachmentEndpointStatus, TopologyAttachmentHealthEntry, + TopologyAttachmentHealthStateFile, TopologyEffectiveAttachmentState, + TopologyEffectiveNodeState, TopologyEffectiveStateFile, TopologyRuntimeStateError, + TopologyRuntimeStatusFile, TopologyShapingCircuitInput, TopologyShapingDeviceInput, + TopologyShapingInputsFile, TopologyShapingResolutionSource, compute_topology_source_generation, + topology_attachment_health_state_path, topology_compiled_shaping_path, + topology_effective_network_path, topology_effective_state_path, topology_import_path, + topology_runtime_status_path, topology_shaping_inputs_path, }; /// Used as a constant in determining buffer preallocation diff --git a/src/rust/lqos_config/src/network_json.rs b/src/rust/lqos_config/src/network_json.rs index 1a1dcbd4..128dc9ca 100644 --- a/src/rust/lqos_config/src/network_json.rs +++ b/src/rust/lqos_config/src/network_json.rs @@ -46,15 +46,29 @@ impl NetworkJson { (self.nodes.len(), self.nodes.capacity()) } - /// The path to the current `network.json` file, determined - /// by acquiring the prefix from the `/etc/lqos.conf` configuration - /// file. - pub fn path() -> Result { - let cfg = crate::load_config().map_err(|_| NetworkJsonError::ConfigLoadError)?; + fn path_for_config(cfg: &crate::Config) -> PathBuf { let base_path = Path::new(&cfg.lqos_directory); let effective_path = base_path.join("network.effective.json"); - let file_path = if effective_path.exists() { + let integration_ingress_enabled = cfg.uisp_integration.enable_uisp + || cfg.splynx_integration.enable_splynx + || cfg + .netzur_integration + .as_ref() + .is_some_and(|integration| integration.enable_netzur) + || cfg + .visp_integration + .as_ref() + .is_some_and(|integration| integration.enable_visp) + || cfg.powercode_integration.enable_powercode + || cfg.sonar_integration.enable_sonar + || cfg + .wispgate_integration + .as_ref() + .is_some_and(|integration| integration.enable_wispgate); + if effective_path.exists() { effective_path + } else if integration_ingress_enabled { + base_path.join("network.effective.json") } else if cfg.long_term_stats.enable_insight_topology.unwrap_or(false) { let tmp_path = base_path.join("network.insight.json"); if tmp_path.exists() { @@ -64,8 +78,15 @@ impl NetworkJson { } } else { base_path.join("network.json") - }; - Ok(file_path) + } + } + + /// The path to the current `network.json` file, determined + /// by acquiring the prefix from the `/etc/lqos.conf` configuration + /// file. + pub fn path() -> Result { + let cfg = crate::load_config().map_err(|_| NetworkJsonError::ConfigLoadError)?; + Ok(Self::path_for_config(cfg.as_ref())) } /// Does network.json exist? @@ -710,4 +731,57 @@ mod test { assert!((encoded_lat - 45.123).abs() < 0.001); assert!((encoded_lon + 111.75).abs() < 0.001); } + + #[test] + fn path_for_config_prefers_runtime_effective_tree_for_integration_ingress() { + let temp = unique_temp_dir("network-json-integration-effective"); + let mut config = crate::Config { + lqos_directory: temp.display().to_string(), + ..crate::Config::default() + }; + config.uisp_integration.enable_uisp = true; + + let selected = NetworkJson::path_for_config(&config); + assert_eq!(selected, temp.join("network.effective.json")); + } + + #[test] + fn path_for_config_uses_diy_network_json_when_integrations_are_disabled() { + let temp = unique_temp_dir("network-json-diy"); + let config = crate::Config { + lqos_directory: temp.display().to_string(), + ..crate::Config::default() + }; + + let selected = NetworkJson::path_for_config(&config); + assert_eq!(selected, temp.join("network.json")); + } + + #[test] + fn path_for_config_prefers_runtime_effective_tree_over_other_tree_inputs() { + let temp = unique_temp_dir("network-json-effective-priority"); + std::fs::write(temp.join("network.effective.json"), "{}") + .expect("effective tree should write"); + std::fs::write(temp.join("network.insight.json"), "{}") + .expect("insight tree should write"); + std::fs::write(temp.join("network.json"), "{}").expect("legacy tree should write"); + let mut config = crate::Config { + lqos_directory: temp.display().to_string(), + ..crate::Config::default() + }; + config.long_term_stats.enable_insight_topology = Some(true); + + let selected = NetworkJson::path_for_config(&config); + assert_eq!(selected, temp.join("network.effective.json")); + } + + fn unique_temp_dir(prefix: &str) -> PathBuf { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("time must advance") + .as_nanos(); + let path = std::env::temp_dir().join(format!("{prefix}-{unique}")); + std::fs::create_dir_all(&path).expect("temp dir should create"); + path + } } diff --git a/src/rust/lqos_config/src/shaped_devices/shaped_device.rs b/src/rust/lqos_config/src/shaped_devices/shaped_device.rs index 1e6a19ec..83082d44 100644 --- a/src/rust/lqos_config/src/shaped_devices/shaped_device.rs +++ b/src/rust/lqos_config/src/shaped_devices/shaped_device.rs @@ -34,9 +34,9 @@ pub struct ShapedDevice { /// Optional compatibility field for a stable topology node identifier. /// - /// Built-in integrations should prefer `circuit_anchors.json` for circuit - /// anchor resolution. This CSV field remains accepted for DIY and legacy - /// operator-managed inputs. + /// Built-in integrations should prefer `topology_compiled_shaping.json` + /// for circuit anchor resolution. This CSV field remains accepted for DIY + /// and legacy operator-managed inputs. #[serde(skip_serializing_if = "Option::is_none")] pub anchor_node_id: Option, diff --git a/src/rust/lqos_config/src/topology_canonical_state.rs b/src/rust/lqos_config/src/topology_canonical_state.rs index 2ff1ac61..d81a254f 100644 --- a/src/rust/lqos_config/src/topology_canonical_state.rs +++ b/src/rust/lqos_config/src/topology_canonical_state.rs @@ -1,15 +1,21 @@ use crate::{ - Config, TOPOLOGY_ATTACHMENT_AUTO_ID, TopologyAllowedParent, TopologyEditorNode, - TopologyEditorStateError, TopologyEditorStateFile, + Config, TOPOLOGY_ATTACHMENT_AUTO_ID, TOPOLOGY_ATTACHMENT_HEALTH_STATE_FILENAME, + TOPOLOGY_COMPILED_SHAPING_FILENAME, TOPOLOGY_EDITOR_STATE_FILENAME, + TOPOLOGY_EFFECTIVE_NETWORK_FILENAME, TOPOLOGY_EFFECTIVE_STATE_FILENAME, + TOPOLOGY_IMPORT_FILENAME, TOPOLOGY_RUNTIME_STATUS_FILENAME, TOPOLOGY_SHAPING_INPUTS_FILENAME, + TopologyAllowedParent, TopologyEditorNode, TopologyEditorStateError, TopologyEditorStateFile, + TopologyParentCandidatesError, TopologyParentCandidatesFile, }; use lqos_utils::hash_to_i64; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::Write; use std::path::{Path, PathBuf}; use thiserror::Error; +use tracing::warn; /// Runtime filename carrying canonical topology state used by `lqos_topology`. pub const TOPOLOGY_CANONICAL_STATE_FILENAME: &str = "topology_canonical_state.json"; @@ -108,6 +114,9 @@ pub struct TopologyCanonicalStateFile { /// Unix timestamp when the state was generated, if known. #[serde(default, skip_serializing_if = "Option::is_none")] pub generated_unix: Option, + /// Stable identity of the imported topology facts plus selected compile mode, when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ingress_identity: Option, /// How this canonical state entered LibreQoS. #[serde(default)] pub ingress_kind: TopologyCanonicalIngressKind, @@ -125,6 +134,7 @@ impl Default for TopologyCanonicalStateFile { schema_version: default_topology_canonical_schema_version(), source: String::new(), generated_unix: None, + ingress_identity: None, ingress_kind: TopologyCanonicalIngressKind::NativeIntegration, nodes: Vec::new(), compatibility_network_json: empty_json_object(), @@ -160,6 +170,233 @@ fn empty_json_object() -> Value { Value::Object(Map::new()) } +fn topology_import_ingress_enabled(config: &Config) -> bool { + config.uisp_integration.enable_uisp + || config.splynx_integration.enable_splynx + || config + .netzur_integration + .as_ref() + .is_some_and(|integration| integration.enable_netzur) + || config + .visp_integration + .as_ref() + .is_some_and(|integration| integration.enable_visp) + || config.powercode_integration.enable_powercode + || config.sonar_integration.enable_sonar + || config + .wispgate_integration + .as_ref() + .is_some_and(|integration| integration.enable_wispgate) +} + +pub(crate) fn topology_ingress_fingerprint_from_tokens(tokens: I) -> Option +where + I: IntoIterator, +{ + let mut identifiers = tokens + .into_iter() + .map(|token| token.trim().to_string()) + .filter(|token| !token.is_empty()) + .collect::>(); + if identifiers.is_empty() { + return None; + } + identifiers.sort_unstable(); + identifiers.dedup(); + + let mut hasher = Sha256::new(); + hasher.update(b"libreqos-topology-ingress-v1"); + for identifier in identifiers { + hasher.update([0]); + hasher.update(identifier.as_bytes()); + } + Some(format!("{:x}", hasher.finalize())) +} + +/// Returns a stable identity hash for imported topology facts and compile mode selection. +pub fn topology_ingress_identity_from_tokens(tokens: I) -> Option +where + I: IntoIterator, +{ + let mut identifiers = tokens + .into_iter() + .map(|token| token.trim().to_string()) + .filter(|token| !token.is_empty()) + .collect::>(); + if identifiers.is_empty() { + return None; + } + identifiers.sort_unstable(); + identifiers.dedup(); + + let mut hasher = Sha256::new(); + hasher.update(b"libreqos-topology-ingress-identity-v1"); + for identifier in identifiers { + hasher.update([0]); + hasher.update(identifier.as_bytes()); + } + Some(format!("{:x}", hasher.finalize())) +} + +fn collect_network_json_ingress_tokens(map: &Map, out: &mut Vec) { + for (key, value) in map { + let Some(node) = value.as_object() else { + continue; + }; + let node_name = node + .get("name") + .and_then(Value::as_str) + .unwrap_or(key) + .to_string(); + out.push(legacy_node_id(node, &node_name)); + if let Some(children) = node.get("children").and_then(Value::as_object) { + collect_network_json_ingress_tokens(children, out); + } + } +} + +pub(crate) fn topology_ingress_fingerprint_for_network_json( + network_json: &Value, +) -> Option { + let map = network_json.as_object()?; + let mut tokens = Vec::new(); + collect_network_json_ingress_tokens(map, &mut tokens); + topology_ingress_fingerprint_from_tokens(tokens) +} + +pub(crate) fn current_topology_ingress_identity( + config: &Config, +) -> Result, TopologyCanonicalStateError> { + let integration_ingress = topology_import_ingress_enabled(config); + if integration_ingress { + let topology_import_path = Path::new(&config.lqos_directory).join(TOPOLOGY_IMPORT_FILENAME); + if topology_import_path.exists() { + let raw = std::fs::read_to_string(topology_import_path)?; + let imported = serde_json::from_str::(&raw)?; + if let Some(identity) = imported + .get("ingress_identity") + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + { + return Ok(Some(identity.to_string())); + } + } + } + + let parent_candidates = + TopologyParentCandidatesFile::load(config).map_err(|err| match err { + TopologyParentCandidatesError::Io(inner) => TopologyCanonicalStateError::Io(inner), + TopologyParentCandidatesError::Json(inner) => TopologyCanonicalStateError::Json(inner), + })?; + if let Some(identity) = parent_candidates + .ingress_identity + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + return Ok(Some(identity.to_string())); + } + + if integration_ingress { + return Ok(None); + } + + let legacy_network_path = Path::new(&config.lqos_directory).join("network.json"); + if !legacy_network_path.exists() { + return Ok(None); + } + let raw = std::fs::read_to_string(legacy_network_path)?; + let network = serde_json::from_str::(&raw)?; + Ok(topology_ingress_fingerprint_for_network_json(&network)) +} + +fn quarantine_target_path(path: &Path, stamp: u64, attempt: usize) -> PathBuf { + let quarantine_directory = + topology_stale_directory_path(path.parent().unwrap_or_else(|| Path::new("."))); + let filename = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + if attempt == 0 { + quarantine_directory.join(format!("{filename}.stale-{stamp}")) + } else { + quarantine_directory.join(format!("{filename}.stale-{stamp}-{attempt}")) + } +} + +fn topology_stale_directory_path(base: &Path) -> PathBuf { + base.join(".topology_stale") +} + +pub(crate) fn quarantine_stale_topology_state( + config: &Config, + reason: &str, +) -> Result<(), TopologyCanonicalStateError> { + let base = Path::new(&config.lqos_directory); + let quarantine_directory = topology_stale_directory_path(base); + let stamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok() + .map(|duration| duration.as_secs()) + .unwrap_or(0); + let mut moved = Vec::new(); + let filenames = [ + TOPOLOGY_CANONICAL_STATE_FILENAME, + TOPOLOGY_EDITOR_STATE_FILENAME, + TOPOLOGY_COMPILED_SHAPING_FILENAME, + TOPOLOGY_ATTACHMENT_HEALTH_STATE_FILENAME, + TOPOLOGY_EFFECTIVE_NETWORK_FILENAME, + TOPOLOGY_EFFECTIVE_STATE_FILENAME, + TOPOLOGY_RUNTIME_STATUS_FILENAME, + TOPOLOGY_SHAPING_INPUTS_FILENAME, + ]; + + std::fs::create_dir_all(&quarantine_directory)?; + + for filename in filenames { + let path = base.join(filename); + if !path.exists() { + continue; + } + + let mut attempt = 0usize; + loop { + let target = quarantine_target_path(&path, stamp, attempt); + if target.exists() { + attempt += 1; + continue; + } + std::fs::rename(&path, &target)?; + moved.push(( + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or(filename) + .to_string(), + target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default() + .to_string(), + )); + break; + } + } + + if !moved.is_empty() { + let summary = moved + .iter() + .map(|(from, to)| format!("{from} -> {to}")) + .collect::>() + .join(", "); + warn!( + "Quarantined stale topology state because it does not match current ingress: {reason}. {summary}" + ); + } + + Ok(()) +} + #[derive(Clone)] struct InsightLogicalNodeEntry { node_id: String, @@ -424,7 +661,7 @@ fn network_node_snapshot_for_editor_node<'a>( .or_else(|| by_name.get(&node.node_name)) } -fn legacy_id_for_name(name: &str) -> String { +pub(crate) fn legacy_id_for_name(name: &str) -> String { format!( "libreqos:legacy-network-json:node:{:016x}", hash_to_i64(name) as u64 @@ -548,13 +785,36 @@ impl TopologyCanonicalStateFile { Ok(serde_json::from_str(&raw)?) } - /// Loads canonical topology state, falling back to importing legacy `network.json`. + /// Loads canonical topology state, falling back to importing legacy `network.json` + /// only when integration-import ingress is not enabled. /// - /// Side effects: reads `topology_canonical_state.json` and, if missing, may read `network.json` - /// from `config.lqos_directory`. + /// Side effects: reads `topology_canonical_state.json` and, in DIY/manual mode when missing, + /// may read `network.json` from `config.lqos_directory`. pub fn load_with_legacy_fallback(config: &Config) -> Result { let state = Self::load(config)?; if !state.nodes.is_empty() { + let is_current = match state.matches_current_ingress(config) { + Ok(is_current) => is_current, + Err(err) => { + warn!( + "Unable to validate topology canonical state against current ingress; preserving existing state: {err}" + ); + true + } + }; + if is_current { + return Ok(state); + } + quarantine_stale_topology_state( + config, + &format!( + "canonical topology source '{}' does not match current topology ingress identity", + state.source + ), + )?; + } + + if topology_import_ingress_enabled(config) { return Ok(state); } @@ -581,6 +841,48 @@ impl TopologyCanonicalStateFile { self.nodes.iter().find(|node| node.node_id == node_id) } + /// Returns a stable fingerprint of the topology ingress this canonical state represents. + /// + /// This function is pure: it has no side effects. + pub fn topology_ingress_fingerprint(&self) -> Option { + topology_ingress_fingerprint_from_tokens(self.nodes.iter().map(|node| { + let node_id = node.node_id.trim(); + if node_id.is_empty() { + legacy_id_for_name(&node.node_name) + } else { + node_id.to_string() + } + })) + } + + /// Returns true if this canonical state still matches the current topology ingress identity. + /// + /// Side effects: reads topology ingress inputs from `config.lqos_directory`. + pub fn matches_current_ingress( + &self, + config: &Config, + ) -> Result { + if let Some(saved_identity) = self + .ingress_identity + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + let Some(current_identity) = current_topology_ingress_identity(config)? else { + return Ok(true); + }; + return Ok(saved_identity == current_identity); + } + + let Some(current_fingerprint) = current_topology_ingress_identity(config)? else { + return Ok(true); + }; + let Some(saved_fingerprint) = self.topology_ingress_fingerprint() else { + return Ok(false); + }; + Ok(saved_fingerprint == current_fingerprint) + } + /// Converts canonical topology state into the UI-facing topology editor projection. /// /// This function is pure: it has no side effects. @@ -589,6 +891,7 @@ impl TopologyCanonicalStateFile { schema_version: self.schema_version, source: self.source.clone(), generated_unix: self.generated_unix, + ingress_identity: self.ingress_identity.clone(), nodes: self .nodes .iter() @@ -747,6 +1050,7 @@ impl TopologyCanonicalStateFile { schema_version: editor_state.schema_version, source: editor_state.source.clone(), generated_unix: editor_state.generated_unix, + ingress_identity: editor_state.ingress_identity.clone(), ingress_kind, nodes, compatibility_network_json: compatibility_network_json.clone(), @@ -766,6 +1070,7 @@ impl TopologyCanonicalStateFile { schema_version: default_topology_canonical_schema_version(), source: "legacy/network.json".to_string(), generated_unix: None, + ingress_identity: topology_ingress_fingerprint_for_network_json(network_json), ingress_kind: TopologyCanonicalIngressKind::LegacyNetworkJson, nodes, compatibility_network_json: network_json.clone(), @@ -785,15 +1090,32 @@ impl From for TopologyEditorStateError { #[cfg(test)] mod tests { use super::{ - TopologyCanonicalIngressKind, TopologyCanonicalRateInputSource, TopologyCanonicalStateFile, - legacy_id_for_name, + TOPOLOGY_CANONICAL_STATE_FILENAME, TopologyCanonicalIngressKind, + TopologyCanonicalRateInputSource, TopologyCanonicalStateFile, + current_topology_ingress_identity, legacy_id_for_name, + topology_ingress_identity_from_tokens, }; use crate::{ - TopologyAllowedParent, TopologyAttachmentHealthStatus, TopologyAttachmentOption, - TopologyAttachmentRateSource, TopologyAttachmentRole, TopologyEditorNode, - TopologyEditorStateFile, + Config, TOPOLOGY_EDITOR_STATE_FILENAME, TOPOLOGY_EFFECTIVE_NETWORK_FILENAME, + TOPOLOGY_IMPORT_FILENAME, TOPOLOGY_PARENT_CANDIDATES_FILENAME, TopologyAllowedParent, + TopologyAttachmentHealthStatus, TopologyAttachmentOption, TopologyAttachmentRateSource, + TopologyAttachmentRole, TopologyCanonicalNode, TopologyEditorNode, TopologyEditorStateFile, + TopologyParentCandidatesFile, }; use serde_json::{Value, json}; + use std::fs; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn unique_temp_dir(prefix: &str) -> PathBuf { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be monotonic enough for tests") + .as_nanos(); + let path = std::env::temp_dir().join(format!("{prefix}-{unique}")); + fs::create_dir_all(&path).expect("temp directory should be creatable"); + path + } fn sample_attachment_option( attachment_id: &str, @@ -835,6 +1157,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: Some(123), + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "child-site".to_string(), node_name: "Child Site".to_string(), @@ -975,12 +1298,357 @@ mod tests { ); } + #[test] + fn stale_canonical_state_is_quarantined_when_network_ingress_changes() { + let lqos_directory = unique_temp_dir("lqos-config-canonical-quarantine"); + fs::write( + lqos_directory.join("network.json"), + serde_json::to_string_pretty(&json!({ + "Current AP": { + "children": {}, + "id": "uisp:device:current-ap", + "type": "AP", + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 100 + } + })) + .expect("network json should serialize"), + ) + .expect("network json should write"); + fs::write( + lqos_directory.join(TOPOLOGY_CANONICAL_STATE_FILENAME), + serde_json::to_string_pretty(&TopologyCanonicalStateFile::from_legacy_network_json( + &json!({ + "Old AP": { + "children": {}, + "id": "uisp:device:old-ap", + "type": "AP", + "downloadBandwidthMbps": 50, + "uploadBandwidthMbps": 50 + } + }), + )) + .expect("canonical state should serialize"), + ) + .expect("canonical state should write"); + fs::write( + lqos_directory.join(TOPOLOGY_EDITOR_STATE_FILENAME), + "{\"schema_version\":1,\"source\":\"legacy:test\",\"nodes\":[{\"node_id\":\"uisp:device:old-ap\",\"node_name\":\"Old AP\"}]}", + ) + .expect("editor state should write"); + fs::write( + lqos_directory.join(TOPOLOGY_EFFECTIVE_NETWORK_FILENAME), + "{\"Old AP\":{\"children\":{}}}", + ) + .expect("effective network should write"); + + let config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + + let loaded = TopologyCanonicalStateFile::load_with_legacy_fallback(&config) + .expect("load with fallback should succeed"); + + assert!(loaded.find_node("uisp:device:current-ap").is_some()); + assert!(loaded.find_node("uisp:device:old-ap").is_none()); + assert!( + !lqos_directory + .join(TOPOLOGY_CANONICAL_STATE_FILENAME) + .exists() + ); + assert!(!lqos_directory.join(TOPOLOGY_EDITOR_STATE_FILENAME).exists()); + assert!( + !lqos_directory + .join(TOPOLOGY_EFFECTIVE_NETWORK_FILENAME) + .exists() + ); + + let quarantine_directory = lqos_directory.join(".topology_stale"); + let quarantined = fs::read_dir(&quarantine_directory) + .expect("directory should read") + .filter_map(|entry| entry.ok()) + .map(|entry| entry.file_name().to_string_lossy().to_string()) + .collect::>(); + assert!( + quarantined + .iter() + .any(|name| name.starts_with("topology_canonical_state.json.stale-")) + ); + assert!( + quarantined + .iter() + .any(|name| name.starts_with("topology_editor_state.json.stale-")) + ); + } + + #[test] + fn current_ingress_identity_prevents_quarantining_matching_native_state() { + let lqos_directory = unique_temp_dir("lqos-config-canonical-ingress-identity"); + let ingress_identity = topology_ingress_identity_from_tokens([ + "import:uisp/full2".to_string(), + "mode:ap_site".to_string(), + "node:uisp:device:current-ap".to_string(), + ]) + .expect("identity should hash"); + fs::write( + lqos_directory.join("network.json"), + serde_json::to_string_pretty(&json!({ + "Current AP": { + "children": { + "Orphans": { + "children": {}, + "id": "libreqos:generated:uisp:site:orphans", + "type": "Site", + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 100 + } + }, + "id": "uisp:device:current-ap", + "type": "AP", + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 100 + } + })) + .expect("network json should serialize"), + ) + .expect("network json should write"); + fs::write( + lqos_directory.join(TOPOLOGY_PARENT_CANDIDATES_FILENAME), + serde_json::to_string_pretty(&TopologyParentCandidatesFile { + source: "uisp/full2".to_string(), + ingress_identity: Some(ingress_identity.clone()), + nodes: Vec::new(), + }) + .expect("parent candidates should serialize"), + ) + .expect("parent candidates should write"); + + let canonical = TopologyCanonicalStateFile { + schema_version: 1, + source: "uisp/ap_site".to_string(), + generated_unix: Some(1), + ingress_identity: Some(ingress_identity.clone()), + ingress_kind: TopologyCanonicalIngressKind::NativeIntegration, + nodes: vec![TopologyCanonicalNode { + node_id: "uisp:device:current-ap".to_string(), + node_name: "Current AP".to_string(), + node_kind: "AP".to_string(), + is_virtual: false, + current_parent_node_id: None, + current_parent_node_name: None, + current_attachment_id: None, + current_attachment_name: None, + can_move: false, + allowed_parents: Vec::new(), + rate_input: super::TopologyCanonicalRateInput { + intrinsic_download_mbps: Some(100), + intrinsic_upload_mbps: Some(100), + legacy_imported_download_mbps: None, + legacy_imported_upload_mbps: None, + source: TopologyCanonicalRateInputSource::AttachmentMax, + }, + }], + compatibility_network_json: json!({}), + }; + fs::write( + lqos_directory.join(TOPOLOGY_CANONICAL_STATE_FILENAME), + serde_json::to_string_pretty(&canonical).expect("canonical should serialize"), + ) + .expect("canonical should write"); + fs::write( + lqos_directory.join(TOPOLOGY_EDITOR_STATE_FILENAME), + serde_json::to_string_pretty(&TopologyEditorStateFile { + schema_version: 1, + source: "uisp/ap_site".to_string(), + generated_unix: Some(1), + ingress_identity: Some(ingress_identity), + nodes: vec![TopologyEditorNode { + node_id: "uisp:device:current-ap".to_string(), + node_name: "Current AP".to_string(), + current_parent_node_id: None, + current_parent_node_name: None, + current_attachment_id: None, + current_attachment_name: None, + can_move: false, + allowed_parents: Vec::new(), + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }], + }) + .expect("editor should serialize"), + ) + .expect("editor should write"); + + let config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + + let loaded = TopologyCanonicalStateFile::load_with_legacy_fallback(&config) + .expect("load with fallback should succeed"); + + assert!(loaded.find_node("uisp:device:current-ap").is_some()); + assert!( + lqos_directory + .join(TOPOLOGY_CANONICAL_STATE_FILENAME) + .exists() + ); + assert!(lqos_directory.join(TOPOLOGY_EDITOR_STATE_FILENAME).exists()); + let quarantine_directory = lqos_directory.join(".topology_stale"); + let quarantined = if quarantine_directory.exists() { + fs::read_dir(&quarantine_directory) + .expect("directory should read") + .filter_map(|entry| entry.ok()) + .map(|entry| entry.file_name().to_string_lossy().to_string()) + .collect::>() + } else { + Vec::::new() + }; + assert!( + !quarantined + .iter() + .any(|name| name.starts_with("topology_canonical_state.json.stale-")) + ); + assert!( + !quarantined + .iter() + .any(|name| name.starts_with("topology_editor_state.json.stale-")) + ); + } + + #[test] + fn current_topology_ingress_identity_prefers_topology_import_artifact() { + let lqos_directory = unique_temp_dir("lqos-config-topology-import-identity"); + fs::write( + lqos_directory.join(TOPOLOGY_IMPORT_FILENAME), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "uisp/full2", + "compile_mode": "ap_site", + "generated_unix": 1, + "ingress_identity": "import-identity", + "imported": { + "source": "uisp/full2", + "generated_unix": 1, + "compatibility_network_json": {}, + "shaped_devices": [], + "circuit_anchors": { + "schema_version": 1, + "source": "uisp/ap_site", + "generated_unix": 1, + "anchors": [] + }, + "ethernet_advisories": [] + } + })) + .expect("topology import should serialize"), + ) + .expect("topology import should write"); + fs::write( + lqos_directory.join(TOPOLOGY_PARENT_CANDIDATES_FILENAME), + serde_json::to_string_pretty(&TopologyParentCandidatesFile { + source: "uisp/ap_site".to_string(), + ingress_identity: Some("parent-candidates-identity".to_string()), + nodes: Vec::new(), + }) + .expect("parent candidates should serialize"), + ) + .expect("parent candidates should write"); + fs::write( + lqos_directory.join("network.json"), + serde_json::to_string_pretty(&json!({ + "Legacy Tower": { + "children": {}, + "id": "legacy-tower", + "type": "Site", + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 100 + } + })) + .expect("network json should serialize"), + ) + .expect("network json should write"); + + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.uisp_integration.enable_uisp = true; + + let identity = current_topology_ingress_identity(&config) + .expect("ingress identity should load") + .expect("ingress identity should be present"); + assert_eq!(identity, "import-identity"); + } + + #[test] + fn current_topology_ingress_identity_does_not_fallback_to_network_json_for_integrations() { + let lqos_directory = unique_temp_dir("lqos-config-topology-import-no-network-fallback"); + fs::write( + lqos_directory.join("network.json"), + serde_json::to_string_pretty(&json!({ + "Legacy Tower": { + "children": {}, + "id": "legacy-tower", + "type": "Site", + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 100 + } + })) + .expect("network json should serialize"), + ) + .expect("network json should write"); + + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.uisp_integration.enable_uisp = true; + + let identity = + current_topology_ingress_identity(&config).expect("ingress identity should load"); + assert!(identity.is_none()); + } + + #[test] + fn load_with_legacy_fallback_skips_network_json_for_integration_ingress() { + let lqos_directory = unique_temp_dir("lqos-config-no-legacy-fallback-integration"); + fs::write( + lqos_directory.join("network.json"), + serde_json::to_string_pretty(&json!({ + "Legacy Tower": { + "children": {}, + "id": "legacy-tower", + "type": "Site", + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 100 + } + })) + .expect("network json should serialize"), + ) + .expect("network json should write"); + + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.uisp_integration.enable_uisp = true; + + let loaded = TopologyCanonicalStateFile::load_with_legacy_fallback(&config) + .expect("load with fallback should succeed"); + assert!(loaded.nodes.is_empty()); + } + #[test] fn insight_topology_network_json_uses_logical_parent_hierarchy() { let editor_state = TopologyEditorStateFile { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: Some(123), + ingress_identity: None, nodes: vec![ TopologyEditorNode { node_id: "site-router".to_string(), diff --git a/src/rust/lqos_config/src/topology_editor_state.rs b/src/rust/lqos_config/src/topology_editor_state.rs index aecb6ef7..450d2571 100644 --- a/src/rust/lqos_config/src/topology_editor_state.rs +++ b/src/rust/lqos_config/src/topology_editor_state.rs @@ -1,12 +1,17 @@ use crate::{ Config, TOPOLOGY_PARENT_CANDIDATES_FILENAME, TopologyCanonicalStateFile, TopologyParentCandidatesError, TopologyParentCandidatesFile, + topology_canonical_state::{ + current_topology_ingress_identity, legacy_id_for_name, quarantine_stale_topology_state, + topology_ingress_fingerprint_from_tokens, + }, }; use serde::{Deserialize, Serialize}; use std::fs::File; use std::io::Write; use std::path::{Path, PathBuf}; use thiserror::Error; +use tracing::warn; /// Runtime filename carrying integration-provided topology editor state for UI editing. pub const TOPOLOGY_EDITOR_STATE_FILENAME: &str = "topology_editor_state.json"; @@ -219,6 +224,9 @@ pub struct TopologyEditorStateFile { /// Unix timestamp when the snapshot was generated, if known. #[serde(default, skip_serializing_if = "Option::is_none")] pub generated_unix: Option, + /// Stable identity of the imported topology facts plus selected compile mode, when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ingress_identity: Option, /// Runtime topology-manager metadata keyed by node. #[serde(default)] pub nodes: Vec, @@ -265,7 +273,25 @@ impl TopologyEditorStateFile { pub fn load_with_legacy_fallback(config: &Config) -> Result { let state = Self::load(config)?; if !state.nodes.is_empty() { - return Ok(state); + let is_current = match state.matches_current_ingress(config) { + Ok(is_current) => is_current, + Err(err) => { + warn!( + "Unable to validate topology editor state against current ingress; preserving existing state: {err}" + ); + true + } + }; + if is_current { + return Ok(state); + } + quarantine_stale_topology_state( + config, + &format!( + "topology editor source '{}' does not match current topology ingress identity", + state.source + ), + )?; } let canonical = TopologyCanonicalStateFile::load_with_legacy_fallback(config)?; @@ -287,6 +313,48 @@ impl TopologyEditorStateFile { atomic_write_json(&topology_editor_state_path(config), self) } + /// Returns a stable fingerprint of the topology ingress this editor state represents. + /// + /// This function is pure: it has no side effects. + pub fn topology_ingress_fingerprint(&self) -> Option { + topology_ingress_fingerprint_from_tokens(self.nodes.iter().map(|node| { + let node_id = node.node_id.trim(); + if node_id.is_empty() { + legacy_id_for_name(&node.node_name) + } else { + node_id.to_string() + } + })) + } + + /// Returns true if this editor state still matches the current topology ingress identity. + /// + /// Side effects: reads topology ingress inputs from `config.lqos_directory`. + pub fn matches_current_ingress( + &self, + config: &Config, + ) -> Result { + if let Some(saved_identity) = self + .ingress_identity + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + let Some(current_identity) = current_topology_ingress_identity(config)? else { + return Ok(true); + }; + return Ok(saved_identity == current_identity); + } + + let Some(current_fingerprint) = current_topology_ingress_identity(config)? else { + return Ok(true); + }; + let Some(saved_fingerprint) = self.topology_ingress_fingerprint() else { + return Ok(false); + }; + Ok(saved_fingerprint == current_fingerprint) + } + /// Finds runtime editor metadata for `node_id`. /// /// This function is pure: it has no side effects. @@ -361,6 +429,7 @@ impl TopologyEditorStateFile { format!("legacy:{}", legacy.source.trim()) }, generated_unix: None, + ingress_identity: legacy.ingress_identity.clone(), nodes, } } diff --git a/src/rust/lqos_config/src/topology_parent_candidates.rs b/src/rust/lqos_config/src/topology_parent_candidates.rs index 9f288a71..efbbccb9 100644 --- a/src/rust/lqos_config/src/topology_parent_candidates.rs +++ b/src/rust/lqos_config/src/topology_parent_candidates.rs @@ -52,6 +52,9 @@ pub struct TopologyParentCandidatesFile { /// Human-readable source for the snapshot, such as `uisp/full`. #[serde(default)] pub source: String, + /// Stable identity of the imported topology facts plus selected compile mode, when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ingress_identity: Option, /// Candidate-parent metadata keyed by node. #[serde(default)] pub nodes: Vec, diff --git a/src/rust/lqos_config/src/topology_runtime_state.rs b/src/rust/lqos_config/src/topology_runtime_state.rs index e006cb18..ce3700f6 100644 --- a/src/rust/lqos_config/src/topology_runtime_state.rs +++ b/src/rust/lqos_config/src/topology_runtime_state.rs @@ -1,6 +1,7 @@ use crate::{ CIRCUIT_ANCHORS_FILENAME, Config, TOPOLOGY_CANONICAL_STATE_FILENAME, - TOPOLOGY_EDITOR_STATE_FILENAME, TopologyAttachmentHealthStatus, + TOPOLOGY_EDITOR_STATE_FILENAME, TopologyAttachmentHealthStatus, TopologyCanonicalStateFile, + TopologyEditorStateFile, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -18,12 +19,23 @@ pub const TOPOLOGY_EFFECTIVE_STATE_FILENAME: &str = "topology_effective_state.js /// Runtime filename carrying the effective network tree for shaping/export. pub const TOPOLOGY_EFFECTIVE_NETWORK_FILENAME: &str = "network.effective.json"; +/// Runtime filename carrying integration-import topology facts before mode projection. +pub const TOPOLOGY_IMPORT_FILENAME: &str = "topology_import.json"; + +/// Runtime filename carrying compiled shaping rows and anchors for integration ingress. +pub const TOPOLOGY_COMPILED_SHAPING_FILENAME: &str = "topology_compiled_shaping.json"; + /// Runtime filename carrying shaping-ready circuit inputs resolved from topology runtime. pub const TOPOLOGY_SHAPING_INPUTS_FILENAME: &str = "shaping_inputs.json"; /// Runtime filename carrying topology publication readiness for one source generation. pub const TOPOLOGY_RUNTIME_STATUS_FILENAME: &str = "topology_runtime_status.json"; +const OPERATOR_OVERRIDES_FILENAME: &str = "lqos_overrides.json"; +const STORMGUARD_OVERRIDES_FILENAME: &str = "lqos_overrides.stormguard.json"; +const TREEGUARD_OVERRIDES_FILENAME: &str = "lqos_overrides.treeguard.json"; +const LEGACY_AUTOPILOT_OVERRIDES_FILENAME: &str = "lqos_overrides.autopilot.json"; + /// Errors returned while reading or writing topology runtime snapshots. #[derive(Debug, Error)] pub enum TopologyRuntimeStateError { @@ -203,6 +215,11 @@ pub enum TopologyShapingResolutionSource { TopologyAnchor, /// Legacy `ParentNode`/`ParentNodeID` was used because no anchor was available. LegacyParent, + /// Runtime could not resolve the circuit to an exported effective queue node, so + /// shaping falls back to generated parent nodes during queue construction. + RuntimeFallback, + /// Flat topology mode assigned the circuit to an explicit generated parent bucket. + FlatBucket, } /// One shaped device row carried into `shaping_inputs.json`. @@ -333,6 +350,42 @@ fn default_runtime_schema_version() -> u32 { 1 } +fn topology_import_ingress_enabled(config: &Config) -> bool { + config.uisp_integration.enable_uisp + || config.splynx_integration.enable_splynx + || config + .netzur_integration + .as_ref() + .is_some_and(|integration| integration.enable_netzur) + || config + .visp_integration + .as_ref() + .is_some_and(|integration| integration.enable_visp) + || config.powercode_integration.enable_powercode + || config.sonar_integration.enable_sonar + || config + .wispgate_integration + .as_ref() + .is_some_and(|integration| integration.enable_wispgate) +} + +fn operator_overrides_path(config: &Config) -> PathBuf { + Path::new(&config.lqos_directory).join(OPERATOR_OVERRIDES_FILENAME) +} + +fn stormguard_overrides_path(config: &Config) -> PathBuf { + Path::new(&config.lqos_directory).join(STORMGUARD_OVERRIDES_FILENAME) +} + +fn treeguard_overrides_path(config: &Config) -> PathBuf { + let canonical = Path::new(&config.lqos_directory).join(TREEGUARD_OVERRIDES_FILENAME); + if canonical.exists() { + return canonical; + } + let legacy = Path::new(&config.lqos_directory).join(LEGACY_AUTOPILOT_OVERRIDES_FILENAME); + if legacy.exists() { legacy } else { canonical } +} + fn atomic_write_json( path: &Path, value: &T, @@ -361,6 +414,11 @@ pub fn topology_effective_network_path(config: &Config) -> PathBuf { Path::new(&config.lqos_directory).join(TOPOLOGY_EFFECTIVE_NETWORK_FILENAME) } +/// Returns the path of the topology import artifact file. +pub fn topology_import_path(config: &Config) -> PathBuf { + Path::new(&config.lqos_directory).join(TOPOLOGY_IMPORT_FILENAME) +} + /// Returns the path of the runtime shaping-input snapshot file. pub fn topology_shaping_inputs_path(config: &Config) -> PathBuf { Path::new(&config.lqos_directory).join(TOPOLOGY_SHAPING_INPUTS_FILENAME) @@ -371,6 +429,11 @@ pub fn topology_runtime_status_path(config: &Config) -> PathBuf { Path::new(&config.lqos_directory).join(TOPOLOGY_RUNTIME_STATUS_FILENAME) } +/// Returns the path of the integration compiled-shaping artifact. +pub fn topology_compiled_shaping_path(config: &Config) -> PathBuf { + Path::new(&config.lqos_directory).join(TOPOLOGY_COMPILED_SHAPING_FILENAME) +} + fn file_exists_with_nonempty_nodes(path: &Path) -> Result { if !path.exists() { return Ok(false); @@ -416,12 +479,31 @@ pub fn compute_topology_source_generation( let base = Path::new(&config.lqos_directory); let canonical_path = base.join(TOPOLOGY_CANONICAL_STATE_FILENAME); let editor_path = base.join(TOPOLOGY_EDITOR_STATE_FILENAME); + let topology_import_path = base.join(TOPOLOGY_IMPORT_FILENAME); + let topology_compiled_shaping_path = base.join(TOPOLOGY_COMPILED_SHAPING_FILENAME); + let use_topology_import = topology_import_ingress_enabled(config); let network_path = base.join("network.json"); let shaped_devices_path = base.join("ShapedDevices.csv"); let circuit_anchors_path = base.join(CIRCUIT_ANCHORS_FILENAME); - let canonical_active = file_exists_with_nonempty_nodes(&canonical_path)?; - let editor_active = !canonical_active && file_exists_with_nonempty_nodes(&editor_path)?; + let canonical_active = if file_exists_with_nonempty_nodes(&canonical_path)? { + let canonical = TopologyCanonicalStateFile::load(config) + .map_err(|err| TopologyRuntimeStateError::SourceGeneration(err.to_string()))?; + canonical + .matches_current_ingress(config) + .map_err(|err| TopologyRuntimeStateError::SourceGeneration(err.to_string()))? + } else { + false + }; + let editor_active = if !canonical_active && file_exists_with_nonempty_nodes(&editor_path)? { + let editor = TopologyEditorStateFile::load(config) + .map_err(|err| TopologyRuntimeStateError::SourceGeneration(err.to_string()))?; + editor + .matches_current_ingress(config) + .map_err(|err| TopologyRuntimeStateError::SourceGeneration(err.to_string()))? + } else { + false + }; let mut hasher = Sha256::new(); hasher.update(b"topology-runtime-source-generation"); @@ -430,14 +512,52 @@ pub fn compute_topology_source_generation( hasher.update(b"topology_source=canonical"); } else if editor_active { hasher.update(b"topology_source=editor"); + } else if use_topology_import { + hasher.update(b"topology_source=integration_import"); } else { hasher.update(b"topology_source=legacy_network"); } hasher.update([0xff]); - hash_file_state(&mut hasher, "network.json", &network_path)?; - hash_file_state(&mut hasher, "ShapedDevices.csv", &shaped_devices_path)?; - hash_file_state(&mut hasher, CIRCUIT_ANCHORS_FILENAME, &circuit_anchors_path)?; + if use_topology_import { + hash_file_state(&mut hasher, TOPOLOGY_IMPORT_FILENAME, &topology_import_path)?; + hash_file_state( + &mut hasher, + TOPOLOGY_COMPILED_SHAPING_FILENAME, + &topology_compiled_shaping_path, + )?; + } else { + hash_file_state(&mut hasher, "network.json", &network_path)?; + hash_file_state(&mut hasher, "ShapedDevices.csv", &shaped_devices_path)?; + hash_file_state(&mut hasher, CIRCUIT_ANCHORS_FILENAME, &circuit_anchors_path)?; + } + if use_topology_import { + hash_file_state( + &mut hasher, + "lqos_overrides.json", + &operator_overrides_path(config), + )?; + + if config + .stormguard + .as_ref() + .is_some_and(|stormguard| stormguard.enabled && !stormguard.dry_run) + { + hash_file_state( + &mut hasher, + "lqos_overrides.stormguard.json", + &stormguard_overrides_path(config), + )?; + } + + if config.treeguard.enabled { + hash_file_state( + &mut hasher, + "lqos_overrides.treeguard.json", + &treeguard_overrides_path(config), + )?; + } + } if canonical_active { hash_file_state( &mut hasher, @@ -524,11 +644,14 @@ impl TopologyRuntimeStatusFile { mod tests { use super::{ CIRCUIT_ANCHORS_FILENAME, Config, TOPOLOGY_CANONICAL_STATE_FILENAME, - TOPOLOGY_RUNTIME_STATUS_FILENAME, TopologyRuntimeStatusFile, + TOPOLOGY_COMPILED_SHAPING_FILENAME, TOPOLOGY_EDITOR_STATE_FILENAME, + TOPOLOGY_IMPORT_FILENAME, TOPOLOGY_RUNTIME_STATUS_FILENAME, TopologyRuntimeStatusFile, compute_topology_source_generation, topology_runtime_status_path, }; + use crate::{TopologyCanonicalStateFile, TopologyEditorNode, TopologyEditorStateFile}; + use serde_json::json; use std::fs; - use std::path::PathBuf; + use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; fn unique_temp_dir(prefix: &str) -> PathBuf { @@ -541,7 +664,7 @@ mod tests { path } - fn write_required_inputs(dir: &PathBuf) { + fn write_required_inputs(dir: &Path) { fs::write(dir.join("network.json"), "{\"root\":{}}\n").expect("network.json should write"); fs::write( dir.join("ShapedDevices.csv"), @@ -557,10 +680,11 @@ mod tests { fn topology_source_generation_is_stable_for_identical_inputs() { let lqos_directory = unique_temp_dir("lqos-config-topology-generation-stable"); write_required_inputs(&lqos_directory); - let config = Config { + let mut config = Config { lqos_directory: lqos_directory.to_string_lossy().to_string(), ..Config::default() }; + config.uisp_integration.enable_uisp = true; let first = compute_topology_source_generation(&config) .expect("generation should compute for stable inputs"); @@ -574,6 +698,20 @@ mod tests { fn topology_source_generation_changes_when_source_inputs_change() { let lqos_directory = unique_temp_dir("lqos-config-topology-generation-changes"); write_required_inputs(&lqos_directory); + fs::write( + lqos_directory.join("network.json"), + serde_json::to_string_pretty(&json!({ + "Tower 1": { + "children": {}, + "id": "tower-1", + "type": "Site", + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 100 + } + })) + .expect("network json should serialize"), + ) + .expect("network json should write"); let config = Config { lqos_directory: lqos_directory.to_string_lossy().to_string(), ..Config::default() @@ -600,7 +738,9 @@ mod tests { "\"nodes\":[{\"node_id\":\"tower-1\",\"node_name\":\"Tower 1\",", "\"node_kind\":\"Site\",\"is_virtual\":false,", "\"allowed_parents\":[],\"can_move\":false,", - "\"rate_input\":{\"download_mbps\":100,\"upload_mbps\":100,\"source\":\"native_integration\"}}]}\n" + "\"rate_input\":{\"intrinsic_download_mbps\":100,", + "\"intrinsic_upload_mbps\":100,", + "\"source\":\"attachment_max\"}}]}\n" ), ) .expect("topology_canonical_state.json should write"); @@ -609,6 +749,381 @@ mod tests { assert_ne!(after_anchors, after_canonical); } + #[test] + fn topology_source_generation_prefers_topology_import_over_network_json() { + let lqos_directory = unique_temp_dir("lqos-config-topology-generation-import"); + write_required_inputs(&lqos_directory); + fs::write( + lqos_directory.join(TOPOLOGY_IMPORT_FILENAME), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "uisp/full2", + "compile_mode": "ap_site", + "generated_unix": 1, + "ingress_identity": "ingress-1", + "imported": { + "source": "uisp/full2", + "generated_unix": 1, + "ingress_identity": "imported-1", + "compatibility_network_json": {}, + "shaped_devices": [], + "circuit_anchors": { + "schema_version": 1, + "source": "uisp/ap_site", + "generated_unix": 1, + "anchors": [] + }, + "ethernet_advisories": [] + } + })) + .expect("topology import should serialize"), + ) + .expect("topology_import.json should write"); + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.uisp_integration.enable_uisp = true; + + let before = compute_topology_source_generation(&config) + .expect("generation should compute before compatibility export change"); + fs::write( + lqos_directory.join("network.json"), + serde_json::to_string_pretty(&json!({ + "Tower 2": { + "children": {}, + "id": "tower-2", + "type": "Site", + "downloadBandwidthMbps": 200, + "uploadBandwidthMbps": 200 + } + })) + .expect("network json should serialize"), + ) + .expect("network json should write"); + let after = compute_topology_source_generation(&config) + .expect("generation should compute after compatibility export change"); + + assert_eq!(before, after); + } + + #[test] + fn topology_source_generation_ignores_network_json_when_integration_ingress_is_enabled() { + let lqos_directory = unique_temp_dir("lqos-config-topology-generation-no-network-fallback"); + write_required_inputs(&lqos_directory); + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.uisp_integration.enable_uisp = true; + + let before = compute_topology_source_generation(&config) + .expect("generation should compute before compatibility export change"); + fs::write( + lqos_directory.join("network.json"), + serde_json::to_string_pretty(&json!({ + "Tower 2": { + "children": {}, + "id": "tower-2", + "type": "Site", + "downloadBandwidthMbps": 200, + "uploadBandwidthMbps": 200 + } + })) + .expect("network json should serialize"), + ) + .expect("network json should write"); + let after = compute_topology_source_generation(&config) + .expect("generation should compute after compatibility export change"); + + assert_eq!(before, after); + } + + #[test] + fn topology_source_generation_tracks_override_files_for_integration_ingress() { + let lqos_directory = unique_temp_dir("lqos-config-topology-generation-override-files"); + write_required_inputs(&lqos_directory); + fs::write( + lqos_directory.join(TOPOLOGY_IMPORT_FILENAME), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "python/splynx", + "compile_mode": "full", + "generated_unix": 1, + "ingress_identity": "ingress-1", + "imported": { + "source": "python/splynx", + "generated_unix": 1, + "ingress_identity": "imported-1", + "compatibility_network_json": {}, + "shaped_devices": [], + "circuit_anchors": { + "schema_version": 1, + "source": "python/splynx", + "generated_unix": 1, + "anchors": [] + }, + "ethernet_advisories": [] + } + })) + .expect("topology import should serialize"), + ) + .expect("topology import should write"); + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.splynx_integration.enable_splynx = true; + + let before = compute_topology_source_generation(&config) + .expect("generation should compute before override change"); + fs::write( + lqos_directory.join("lqos_overrides.json"), + serde_json::to_string_pretty(&json!({ + "persistent_devices": [], + "circuit_adjustments": [ + { + "type": "device_adjust_speed", + "device_id": "device-1", + "max_download_bandwidth": 55.0 + } + ], + "network_adjustments": [] + })) + .expect("override json should serialize"), + ) + .expect("override file should write"); + let after = compute_topology_source_generation(&config) + .expect("generation should compute after override change"); + + assert_ne!(before, after); + } + + #[test] + fn topology_source_generation_tracks_compiled_shaping_for_integration_ingress() { + let lqos_directory = unique_temp_dir("lqos-config-topology-generation-compiled-shaping"); + write_required_inputs(&lqos_directory); + fs::write( + lqos_directory.join(TOPOLOGY_IMPORT_FILENAME), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "uisp/full2", + "compile_mode": "full", + "generated_unix": 1, + "ingress_identity": "ingress-1", + "imported": { + "source": "uisp/full2", + "generated_unix": 1, + "ingress_identity": "imported-1", + "compatibility_network_json": {}, + "shaped_devices": [], + "circuit_anchors": { + "schema_version": 1, + "source": "uisp/full", + "generated_unix": 1, + "anchors": [] + }, + "ethernet_advisories": [] + } + })) + .expect("topology import should serialize"), + ) + .expect("topology import should write"); + fs::write( + lqos_directory.join(TOPOLOGY_COMPILED_SHAPING_FILENAME), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "uisp/full", + "compile_mode": "full", + "generated_unix": 1, + "ingress_identity": "ingress-1", + "shaped_devices": [], + "circuit_anchors": { + "schema_version": 1, + "source": "uisp/full", + "generated_unix": 1, + "anchors": [] + } + })) + .expect("compiled shaping should serialize"), + ) + .expect("compiled shaping should write"); + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.uisp_integration.enable_uisp = true; + + let before = compute_topology_source_generation(&config) + .expect("generation should compute before compiled shaping change"); + fs::write( + lqos_directory.join(TOPOLOGY_COMPILED_SHAPING_FILENAME), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "uisp/full", + "compile_mode": "full", + "generated_unix": 2, + "ingress_identity": "ingress-1", + "shaped_devices": [ + { + "circuit_id": "circuit-1", + "circuit_name": "Circuit 1", + "device_id": "device-1", + "device_name": "Device 1", + "parent_node": "Tower 1", + "parent_node_id": "tower-1", + "anchor_node_id": null, + "mac": "", + "ipv4": [], + "ipv6": [], + "download_min_mbps": 10.0, + "upload_min_mbps": 10.0, + "download_max_mbps": 100.0, + "upload_max_mbps": 100.0, + "comment": "", + "sqm_override": null + } + ], + "circuit_anchors": { + "schema_version": 1, + "source": "uisp/full", + "generated_unix": 2, + "anchors": [] + } + })) + .expect("compiled shaping should serialize"), + ) + .expect("compiled shaping should rewrite"); + let after = compute_topology_source_generation(&config) + .expect("generation should compute after compiled shaping change"); + + assert_ne!(before, after); + } + + #[test] + fn topology_source_generation_ignores_shaped_devices_csv_for_integration_ingress() { + let lqos_directory = unique_temp_dir("lqos-config-topology-generation-no-shaped-devices"); + write_required_inputs(&lqos_directory); + fs::write( + lqos_directory.join(TOPOLOGY_IMPORT_FILENAME), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "uisp/full2", + "compile_mode": "full", + "generated_unix": 1, + "ingress_identity": "ingress-1", + "imported": { + "source": "uisp/full2", + "generated_unix": 1, + "ingress_identity": "imported-1", + "compatibility_network_json": {}, + "shaped_devices": [], + "circuit_anchors": { + "schema_version": 1, + "source": "uisp/full", + "generated_unix": 1, + "anchors": [] + }, + "ethernet_advisories": [] + } + })) + .expect("topology import should serialize"), + ) + .expect("topology import should write"); + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.uisp_integration.enable_uisp = true; + + let before = compute_topology_source_generation(&config) + .expect("generation should compute before ShapedDevices change"); + fs::write( + lqos_directory.join("ShapedDevices.csv"), + concat!( + "Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment\n", + "\"c2\",\"Circuit 2\",\"d2\",\"Device 2\",\"Tower 2\",\"aa:bb:cc:dd:ee:11\",\"192.0.2.11/32\",\"\",\"20\",\"20\",\"200\",\"200\",\"\"\n", + ), + ) + .expect("ShapedDevices.csv should rewrite"); + let after = compute_topology_source_generation(&config) + .expect("generation should compute after ShapedDevices change"); + + assert_eq!(before, after); + } + + #[test] + fn topology_source_generation_ignores_stale_canonical_and_editor_state() { + let lqos_directory = unique_temp_dir("lqos-config-topology-generation-stale-state"); + write_required_inputs(&lqos_directory); + fs::write( + lqos_directory.join("network.json"), + serde_json::to_string_pretty(&json!({ + "Current Tower": { + "children": {}, + "id": "uisp:device:current-tower", + "type": "AP", + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 100 + } + })) + .expect("network json should serialize"), + ) + .expect("network json should write"); + let config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + + let baseline = compute_topology_source_generation(&config) + .expect("baseline generation should compute"); + + let stale_canonical = TopologyCanonicalStateFile::from_legacy_network_json(&json!({ + "Old Tower": { + "children": {}, + "id": "uisp:device:old-tower", + "type": "AP", + "downloadBandwidthMbps": 50, + "uploadBandwidthMbps": 50 + } + })); + fs::write( + lqos_directory.join(TOPOLOGY_CANONICAL_STATE_FILENAME), + serde_json::to_string_pretty(&stale_canonical) + .expect("canonical json should serialize"), + ) + .expect("canonical state should write"); + fs::write( + lqos_directory.join(TOPOLOGY_EDITOR_STATE_FILENAME), + serde_json::to_string_pretty(&TopologyEditorStateFile { + schema_version: 1, + source: "uisp/full2".to_string(), + generated_unix: Some(1), + ingress_identity: None, + nodes: vec![TopologyEditorNode { + node_id: "uisp:device:old-tower".to_string(), + node_name: "Old Tower".to_string(), + current_parent_node_id: None, + current_parent_node_name: None, + current_attachment_id: None, + current_attachment_name: None, + can_move: false, + allowed_parents: vec![], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }], + }) + .expect("editor json should serialize"), + ) + .expect("editor state should write"); + + let with_stale = compute_topology_source_generation(&config) + .expect("generation should ignore stale state"); + assert_eq!(baseline, with_stale); + } + #[test] fn topology_runtime_status_round_trips() { let lqos_directory = unique_temp_dir("lqos-config-topology-status-roundtrip"); diff --git a/src/rust/lqos_overrides/src/overrides_file.rs b/src/rust/lqos_overrides/src/overrides_file.rs index 14ac8622..41344f35 100644 --- a/src/rust/lqos_overrides/src/overrides_file.rs +++ b/src/rust/lqos_overrides/src/overrides_file.rs @@ -1035,17 +1035,23 @@ fn site_speed_override_matches( } impl OverrideStore { + /// Returns the filesystem path for one overrides layer using a caller-supplied config snapshot. + /// + /// This function is pure: it has no side effects. + pub fn path_for_layer_config(config: &lqos_config::Config, layer: OverrideLayer) -> PathBuf { + match layer { + OverrideLayer::Operator | OverrideLayer::Stormguard => overrides_path(config, layer), + OverrideLayer::Treeguard => treeguard_read_path(config), + } + } + /// Loads a single overrides layer. /// /// Side effects: acquires the global overrides lock and may create the operator overrides file. pub fn load_layer(layer: OverrideLayer) -> Result { let lock = FileLock::new()?; let config = lqos_config::load_config()?; - let path = match layer { - OverrideLayer::Operator => overrides_path(&config, layer), - OverrideLayer::Stormguard => overrides_path(&config, layer), - OverrideLayer::Treeguard => treeguard_read_path(&config), - }; + let path = Self::path_for_layer_config(&config, layer); let overrides = match layer { OverrideLayer::Operator => { ensure_exists_default(&path)?; @@ -1083,33 +1089,43 @@ impl OverrideStore { pub fn load_effective(apply_stormguard: bool, apply_treeguard: bool) -> Result { let lock = FileLock::new()?; let config = lqos_config::load_config()?; + let merged = Self::load_effective_for_config(&config, apply_stormguard, apply_treeguard)?; + drop(lock); + Ok(merged) + } - let operator_path = overrides_path(&config, OverrideLayer::Operator); + /// Loads the effective overrides view using a caller-supplied config snapshot. + /// + /// This function reads override files from the config's LibreQoS directory without + /// reloading `/etc/lqos.conf`. + pub fn load_effective_for_config( + config: &lqos_config::Config, + apply_stormguard: bool, + apply_treeguard: bool, + ) -> Result { + let operator_path = overrides_path(config, OverrideLayer::Operator); ensure_exists_default(&operator_path)?; let operator = load_from_path(&operator_path)?; if !apply_stormguard && !apply_treeguard { - drop(lock); return Ok(operator); } - let stormguard_path = overrides_path(&config, OverrideLayer::Stormguard); + let stormguard_path = overrides_path(config, OverrideLayer::Stormguard); let stormguard = if !apply_stormguard || !stormguard_path.exists() { OverrideFile::default() } else { load_from_path(&stormguard_path)? }; - let treeguard_path = treeguard_read_path(&config); + let treeguard_path = treeguard_read_path(config); let treeguard = if !apply_treeguard || !treeguard_path.exists() { OverrideFile::default() } else { load_from_path(&treeguard_path)? }; - let merged = merge_owned_sections(operator, stormguard, treeguard); - drop(lock); - Ok(merged) + Ok(merge_owned_sections(operator, stormguard, treeguard)) } } diff --git a/src/rust/lqos_python/Cargo.toml b/src/rust/lqos_python/Cargo.toml index 7ffde011..411a746c 100644 --- a/src/rust/lqos_python/Cargo.toml +++ b/src/rust/lqos_python/Cargo.toml @@ -16,16 +16,19 @@ lqos_sys = { path = "../lqos_sys" } lqos_utils = { path = "../lqos_utils" } lqos_config = { path = "../lqos_config" } lqos_overrides = { path = "../lqos_overrides" } +lqos_topology_compile = { path = "../lqos_topology_compile" } tokio = { workspace = true } anyhow = { workspace = true } sysinfo = { workspace = true } nix = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } chrono = "0.4.33" serde_cbor = { workspace = true } base64 = "0.22" miniz_oxide = { workspace = true } +csv = { workspace = true } [dev-dependencies] tempfile = "3.10" diff --git a/src/rust/lqos_python/src/lib.rs b/src/rust/lqos_python/src/lib.rs index c27bd88e..e987076d 100644 --- a/src/rust/lqos_python/src/lib.rs +++ b/src/rust/lqos_python/src/lib.rs @@ -3,10 +3,15 @@ #![allow(non_local_definitions)] // Temporary: rewrite required for much of this, for newer PyO3. #![allow(unsafe_op_in_unsafe_fn)] #![warn(missing_docs)] +use csv::ReaderBuilder; use lqos_bus::{ BakeryCapacityReportInterface, BlackboardSystem, BusRequest, BusResponse, SchedulerProgressReport, TcHandle, UrgentSeverity, UrgentSource, }; +use lqos_topology_compile::{ + CompiledTopologyBundle, ImportedTopologyBundle, TopologyCompileMode, + TopologyCompiledShapingFile, TopologyImportFile, compile_topology, +}; use lqos_utils::hex_string::read_hex_string; use lqos_utils::rustls::ensure_rustls_crypto_provider; use nix::libc::getpid; @@ -29,7 +34,7 @@ use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; // ===== Planner CBOR I/O ===== @@ -77,6 +82,25 @@ fn default_algo_version() -> String { "v1".to_string() } +fn load_shaped_devices_from_csv_payload( + payload: &str, +) -> anyhow::Result { + let mut reader = ReaderBuilder::new() + .comment(Some(b'#')) + .trim(csv::Trim::All) + .from_reader(payload.as_bytes()); + let headers = reader.headers()?.clone(); + let mut devices = Vec::new(); + for record in reader.records() { + let record = record?; + let device = lqos_config::ShapedDevice::from_csv(&record, Some(&headers))?; + devices.push(device); + } + let mut shaped_devices = lqos_config::ConfigShapedDevices::default(); + shaped_devices.replace_with_new_data(devices); + Ok(shaped_devices) +} + fn to_i64_any(v: &pyo3::Bound<'_, pyo3::types::PyAny>) -> Option { if let Ok(n) = v.extract::() { return Some(n); @@ -841,6 +865,66 @@ fn store_planner_remote(py: Python, state: PyObject) -> PyResult { const LOCK_FILE: &str = "/run/lqos/libreqos.lock"; +fn parse_topology_compile_mode(mode: &str) -> PyResult { + match mode.trim().to_lowercase().as_str() { + "flat" => Ok(TopologyCompileMode::Flat), + "ap_only" => Ok(TopologyCompileMode::ApOnly), + "ap_site" => Ok(TopologyCompileMode::ApSite), + "full" => Ok(TopologyCompileMode::Full), + "full2" => Ok(TopologyCompileMode::Full2), + _ => Err(PyOSError::new_err(format!( + "Unknown topology compile mode '{mode}'" + ))), + } +} + +fn write_compiled_topology_outputs( + config: &lqos_config::Config, + topology_import: &TopologyImportFile, + compiled_shaping: &TopologyCompiledShapingFile, + compiled: CompiledTopologyBundle, +) -> PyResult<()> { + topology_import + .save(config) + .map_err(|e| PyOSError::new_err(format!("Unable to write topology_import.json: {e}")))?; + compiled_shaping.save(config).map_err(|e| { + PyOSError::new_err(format!( + "Unable to write topology_compiled_shaping.json: {e}" + )) + })?; + compiled.parent_candidates.save(config).map_err(|e| { + PyOSError::new_err(format!( + "Unable to write topology parent candidates snapshot: {e}" + )) + })?; + compiled.editor.save(config).map_err(|e| { + PyOSError::new_err(format!( + "Unable to write topology editor state snapshot: {e}" + )) + })?; + compiled.canonical.save(config).map_err(|e| { + PyOSError::new_err(format!( + "Unable to write topology canonical state snapshot: {e}" + )) + })?; + let anchors_path = lqos_config::circuit_anchors_path(config); + if let Err(err) = remove_file(&anchors_path) + && err.kind() != std::io::ErrorKind::NotFound + { + return Err(PyOSError::new_err(format!( + "Unable to remove stale duplicate circuit_anchors.json at {}: {err}", + anchors_path.display() + ))); + } + if !config_uses_topology_import_ingress(config) { + compiled + .shaped_devices + .write_csv("ShapedDevices.csv") + .map_err(|e| PyOSError::new_err(format!("Unable to write ShapedDevices.csv: {e}")))?; + } + Ok(()) +} + /// Defines the Python module exports. /// All exported functions have to be listed here. #[pymodule] @@ -885,7 +969,8 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(generated_pn_upload_mbps, m)?)?; m.add_function(wrap_pyfunction!(queues_available_override, m)?)?; m.add_function(wrap_pyfunction!(on_a_stick, m)?)?; - m.add_function(wrap_pyfunction!(overwrite_network_json_always, m)?)?; + m.add_function(wrap_pyfunction!(topology_import_ingress_enabled, m)?)?; + m.add_function(wrap_pyfunction!(migrate_legacy_site_bandwidth_csv, m)?)?; m.add_function(wrap_pyfunction!(allowed_subnets, m)?)?; m.add_function(wrap_pyfunction!(ignore_subnets, m)?)?; m.add_function(wrap_pyfunction!(circuit_name_use_address, m)?)?; @@ -907,6 +992,19 @@ fn liblqos_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(splynx_api_secret, m)?)?; m.add_function(wrap_pyfunction!(splynx_api_url, m)?)?; m.add_function(wrap_pyfunction!(splynx_strategy, m)?)?; + m.add_function(wrap_pyfunction!(sonar_strategy, m)?)?; + m.add_function(wrap_pyfunction!( + write_compiled_topology_from_legacy_artifacts, + m + )?)?; + m.add_function(wrap_pyfunction!( + write_compiled_topology_from_network_json_payload, + m + )?)?; + m.add_function(wrap_pyfunction!( + write_compiled_topology_from_python_graph_payload, + m + )?)?; m.add_function(wrap_pyfunction!(netzur_api_key, m)?)?; m.add_function(wrap_pyfunction!(netzur_api_url, m)?)?; m.add_function(wrap_pyfunction!(netzur_api_timeout, m)?)?; @@ -2070,12 +2168,6 @@ fn on_a_stick() -> PyResult { Ok(config.on_a_stick_mode()) } -#[pyfunction] -fn overwrite_network_json_always() -> PyResult { - let config = lqos_config::load_config().unwrap(); - Ok(config.integration_common.always_overwrite_network_json) -} - #[pyfunction] fn allowed_subnets() -> PyResult> { let config = lqos_config::load_config().unwrap(); @@ -2156,8 +2248,7 @@ fn uisp_site() -> PyResult { #[pyfunction] fn uisp_strategy() -> PyResult { let config = lqos_config::load_config().unwrap(); - let strategy = config.uisp_integration.strategy.clone(); - Ok(strategy) + Ok(config.resolved_topology_compile_mode_for_uisp().to_string()) } #[pyfunction] @@ -2224,11 +2315,172 @@ fn splynx_api_url() -> PyResult { fn splynx_strategy() -> PyResult { let config = lqos_config::load_config(); match config { - Ok(config) => Ok(config.splynx_integration.strategy.clone()), + Ok(config) => Ok(config + .resolved_topology_compile_mode_for_splynx() + .to_string()), Err(_) => Ok("ap_only".to_string()), // Default value when config can't be loaded } } +#[pyfunction] +fn sonar_strategy() -> PyResult { + let config = lqos_config::load_config(); + match config { + Ok(config) => Ok(config + .resolved_topology_compile_mode_for_sonar() + .to_string()), + Err(_) => Ok("full".to_string()), + } +} + +#[pyfunction] +fn write_compiled_topology_from_legacy_artifacts( + source: String, + compile_mode: String, +) -> PyResult { + let config = lqos_config::load_config().map_err(|e| PyOSError::new_err(e.to_string()))?; + let imported = ImportedTopologyBundle::from_legacy_artifacts(config.as_ref(), source.clone()) + .map_err(|e| { + PyOSError::new_err(format!( + "Unable to load legacy topology artifacts for '{source}': {e}" + )) + })?; + let topology_import = TopologyImportFile::from_imported_bundle(&imported, compile_mode.clone()); + let mode = parse_topology_compile_mode(&compile_mode)?; + let compiled = compile_topology(imported, mode).map_err(|e| { + PyOSError::new_err(format!( + "Unable to compile legacy topology artifacts for '{source}' using mode '{compile_mode}': {e}" + )) + })?; + let compiled_shaping = + TopologyCompiledShapingFile::from_compiled(&compiled, compile_mode.clone()); + write_compiled_topology_outputs( + config.as_ref(), + &topology_import, + &compiled_shaping, + compiled, + )?; + Ok(true) +} + +#[pyfunction] +fn write_compiled_topology_from_network_json_payload( + source: String, + compile_mode: String, + compatibility_network_json: String, +) -> PyResult { + let config = lqos_config::load_config().map_err(|e| PyOSError::new_err(e.to_string()))?; + let compatibility_network_json = + serde_json::from_str::(&compatibility_network_json).map_err(|e| { + PyOSError::new_err(format!("Unable to parse topology payload JSON: {e}")) + })?; + let imported = ImportedTopologyBundle { + source: source.clone(), + generated_unix: None, + ingress_identity: None, + native_canonical: None, + native_editor: None, + parent_candidates: None, + compatibility_network_json, + shaped_devices: lqos_config::ConfigShapedDevices::load_for_config(config.as_ref()) + .map_err(|e| { + PyOSError::new_err(format!( + "Unable to load ShapedDevices.csv for '{source}': {e}" + )) + })?, + circuit_anchors: lqos_config::CircuitAnchorsFile::load(config.as_ref()).map_err(|e| { + PyOSError::new_err(format!( + "Unable to load circuit_anchors.json for '{source}': {e}" + )) + })?, + ethernet_advisories: Vec::new(), + }; + let topology_import = TopologyImportFile::from_imported_bundle(&imported, compile_mode.clone()); + let mode = parse_topology_compile_mode(&compile_mode)?; + let compiled = compile_topology(imported, mode).map_err(|e| { + PyOSError::new_err(format!( + "Unable to compile topology payload for '{source}' using mode '{compile_mode}': {e}" + )) + })?; + let compiled_shaping = + TopologyCompiledShapingFile::from_compiled(&compiled, compile_mode.clone()); + write_compiled_topology_outputs( + config.as_ref(), + &topology_import, + &compiled_shaping, + compiled, + )?; + Ok(true) +} + +#[pyfunction] +fn write_compiled_topology_from_python_graph_payload( + source: String, + compile_mode: String, + compatibility_network_json: String, + shaped_devices_csv: String, + circuit_anchors_json: String, + parent_candidates_json: String, + native_editor_json: String, +) -> PyResult { + let config = lqos_config::load_config().map_err(|e| PyOSError::new_err(e.to_string()))?; + let compatibility_network_json = + serde_json::from_str::(&compatibility_network_json).map_err(|e| { + PyOSError::new_err(format!("Unable to parse topology payload JSON: {e}")) + })?; + let shaped_devices = + load_shaped_devices_from_csv_payload(&shaped_devices_csv).map_err(|e| { + PyOSError::new_err(format!("Unable to parse shaped-device CSV payload: {e}")) + })?; + let circuit_anchors = serde_json::from_str::( + &circuit_anchors_json, + ) + .map_err(|e| PyOSError::new_err(format!("Unable to parse circuit anchor payload JSON: {e}")))?; + let parent_candidates = + serde_json::from_str::(&parent_candidates_json) + .map_err(|e| { + PyOSError::new_err(format!( + "Unable to parse topology parent-candidate payload JSON: {e}" + )) + })?; + let native_editor = serde_json::from_str::( + &native_editor_json, + ) + .map_err(|e| { + PyOSError::new_err(format!( + "Unable to parse native topology editor payload JSON: {e}" + )) + })?; + let imported = ImportedTopologyBundle { + source: source.clone(), + generated_unix: None, + ingress_identity: None, + native_canonical: None, + native_editor: Some(native_editor), + parent_candidates: Some(parent_candidates), + compatibility_network_json, + shaped_devices, + circuit_anchors, + ethernet_advisories: Vec::new(), + }; + let topology_import = TopologyImportFile::from_imported_bundle(&imported, compile_mode.clone()); + let mode = parse_topology_compile_mode(&compile_mode)?; + let compiled = compile_topology(imported, mode).map_err(|e| { + PyOSError::new_err(format!( + "Unable to compile topology payload for '{source}' using mode '{compile_mode}': {e}" + )) + })?; + let compiled_shaping = + TopologyCompiledShapingFile::from_compiled(&compiled, compile_mode.clone()); + write_compiled_topology_outputs( + config.as_ref(), + &topology_import, + &compiled_shaping, + compiled, + )?; + Ok(true) +} + #[pyfunction] fn netzur_api_key() -> PyResult { let config = lqos_config::load_config().unwrap(); @@ -2581,6 +2833,157 @@ fn automatic_import_wispgate() -> PyResult { Ok(wisp_gate.enable_wispgate) } +fn config_uses_topology_import_ingress(config: &lqos_config::Config) -> bool { + config.uisp_integration.enable_uisp + || config.splynx_integration.enable_splynx + || config + .netzur_integration + .as_ref() + .is_some_and(|integration| integration.enable_netzur) + || config + .visp_integration + .as_ref() + .is_some_and(|integration| integration.enable_visp) + || config.powercode_integration.enable_powercode + || config.sonar_integration.enable_sonar + || config + .wispgate_integration + .as_ref() + .is_some_and(|integration| integration.enable_wispgate) +} + +#[pyfunction] +fn topology_import_ingress_enabled() -> PyResult { + let config = lqos_config::load_config().unwrap(); + Ok(config_uses_topology_import_ingress(config.as_ref())) +} + +fn rename_legacy_site_bandwidth_csv_to_backup(file_path: &Path) -> Result<()> { + let unix_secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let file_name = file_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("legacy_bandwidths.csv"); + let mut backup_path = file_path.with_file_name(format!("{file_name}.backup")); + if backup_path.exists() { + backup_path = file_path.with_file_name(format!("{file_name}.backup-{unix_secs}")); + } + std::fs::rename(file_path, backup_path)?; + Ok(()) +} + +#[pyfunction] +fn migrate_legacy_site_bandwidth_csv(csv_filename: &str) -> PyResult { + let trimmed = csv_filename.trim(); + if trimmed.is_empty() { + return Ok(false); + } + + let config = lqos_config::load_config().map_err(|e| PyOSError::new_err(e.to_string()))?; + let legacy_path = Path::new(&config.lqos_directory).join(trimmed); + if !legacy_path.exists() { + return Ok(false); + } + + let mut reader = ReaderBuilder::new() + .comment(Some(b'#')) + .trim(csv::Trim::All) + .from_path(&legacy_path) + .map_err(|e| { + PyOSError::new_err(format!("Unable to read {}: {e}", legacy_path.display())) + })?; + + let operator_path = lqos_overrides::OverrideFile::operator_path_for_config(config.as_ref()); + let mut overrides = if operator_path.exists() { + lqos_overrides::OverrideFile::load_from_explicit_path(&operator_path).map_err(|e| { + PyOSError::new_err(format!("Unable to load {}: {e}", operator_path.display())) + })? + } else { + lqos_overrides::OverrideFile::default() + }; + + let mut migrated_rows = 0_usize; + for (line, record) in reader.records().enumerate() { + let record = record.map_err(|e| { + PyOSError::new_err(format!( + "Unable to decode {} line {}: {e}", + legacy_path.display(), + line + 2 + )) + })?; + if record.len() != 3 { + return Err(PyOSError::new_err(format!( + "Wrong number of records in {} on line {}", + legacy_path.display(), + line + 2 + ))); + } + + let site_name = record.get(0).unwrap_or_default().trim().to_string(); + if site_name.is_empty() { + continue; + } + let download = record + .get(1) + .unwrap_or_default() + .trim() + .parse::() + .map_err(|e| { + PyOSError::new_err(format!( + "Unable to parse download bandwidth '{}' in {} on line {}: {e}", + record.get(1).unwrap_or_default(), + legacy_path.display(), + line + 2 + )) + })?; + let upload = record + .get(2) + .unwrap_or_default() + .trim() + .parse::() + .map_err(|e| { + PyOSError::new_err(format!( + "Unable to parse upload bandwidth '{}' in {} on line {}: {e}", + record.get(2).unwrap_or_default(), + legacy_path.display(), + line + 2 + )) + })?; + let already_present = overrides.network_adjustments().iter().any(|adjustment| { + matches!( + adjustment, + lqos_overrides::NetworkAdjustment::AdjustSiteSpeed { site_name: current_name, .. } + if current_name == &site_name + ) + }); + if already_present { + continue; + } + overrides.set_site_bandwidth_override(None, site_name, Some(download), Some(upload)); + migrated_rows += 1; + } + + if migrated_rows == 0 { + return Ok(false); + } + + overrides + .save_to_explicit_path(&operator_path) + .map_err(|e| { + PyOSError::new_err(format!("Unable to save {}: {e}", operator_path.display())) + })?; + rename_legacy_site_bandwidth_csv_to_backup(&legacy_path).map_err(|e| { + PyOSError::new_err(format!( + "Unable to rename {} to a backup: {e}", + legacy_path.display() + )) + })?; + Ok(true) +} + #[pyfunction] fn wispgate_api_token() -> PyResult { let config = lqos_config::load_config().unwrap(); @@ -2640,7 +3043,12 @@ fn calculate_hash() -> PyResult { let Ok(config) = lqos_config::load_config() else { return Ok(0); }; - let nj_path = Path::new(&config.lqos_directory).join("network.json"); + let base_path = Path::new(&config.lqos_directory); + let nj_path = if config_uses_topology_import_ingress(config.as_ref()) { + base_path.join("network.effective.json") + } else { + base_path.join("network.json") + }; let sd_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); let Ok(nj_as_string) = read_to_string(nj_path) else { @@ -2717,6 +3125,8 @@ fn calculate_shaping_runtime_hash() -> PyResult { let effective_path = base_path.join("network.effective.json"); let nj_path = if effective_path.exists() { effective_path + } else if config_uses_topology_import_ingress(config.as_ref()) { + base_path.join("network.effective.json") } else if config .long_term_stats .enable_insight_topology diff --git a/src/rust/lqos_setup/src/main.rs b/src/rust/lqos_setup/src/main.rs index f739371d..471d546f 100644 --- a/src/rust/lqos_setup/src/main.rs +++ b/src/rust/lqos_setup/src/main.rs @@ -253,82 +253,6 @@ fn main() { ui.run(); } -#[cfg(test)] -mod test { - use super::upgrade_readiness_for_config_with; - - fn configured_bridge_config(lqos_directory: String) -> lqos_config::Config { - let mut config = lqos_config::Config::default(); - config.lqos_directory = lqos_directory; - config.bridge = Some(lqos_config::BridgeConfig { - use_xdp_bridge: true, - to_internet: "wan0".to_string(), - to_network: "lan0".to_string(), - }); - config.single_interface = None; - config.queues.downlink_bandwidth_mbps = 1000; - config.queues.uplink_bandwidth_mbps = 1000; - config - } - - #[test] - fn upgrade_ready_when_existing_install_is_configured() { - let temp_dir = - std::env::temp_dir().join(format!("lqos-setup-upgrade-ready-{}", std::process::id())); - std::fs::create_dir_all(&temp_dir).unwrap(); - std::fs::write(temp_dir.join("network.json"), "{}").unwrap(); - std::fs::write(temp_dir.join("ShapedDevices.csv"), "Circuit ID\n").unwrap(); - - let config = configured_bridge_config(temp_dir.display().to_string()); - let readiness = upgrade_readiness_for_config_with(true, Some(&config), |_| true); - assert!(readiness.all_ready()); - - let _ = std::fs::remove_file(temp_dir.join("network.json")); - let _ = std::fs::remove_file(temp_dir.join("ShapedDevices.csv")); - let _ = std::fs::remove_dir_all(&temp_dir); - } - - #[test] - fn upgrade_not_ready_without_required_runtime_inputs() { - let temp_dir = std::env::temp_dir().join(format!( - "lqos-setup-upgrade-missing-files-{}", - std::process::id() - )); - std::fs::create_dir_all(&temp_dir).unwrap(); - - let config = configured_bridge_config(temp_dir.display().to_string()); - let readiness = upgrade_readiness_for_config_with(true, Some(&config), |_| true); - assert!(!readiness.all_ready()); - assert!(!readiness.network_json_present); - assert!(!readiness.shaped_devices_present); - - let _ = std::fs::remove_dir_all(&temp_dir); - } - - #[test] - fn upgrade_not_ready_when_bridge_interfaces_are_not_distinct() { - let temp_dir = std::env::temp_dir().join(format!( - "lqos-setup-upgrade-same-iface-{}", - std::process::id() - )); - std::fs::create_dir_all(&temp_dir).unwrap(); - std::fs::write(temp_dir.join("network.json"), "{}").unwrap(); - std::fs::write(temp_dir.join("ShapedDevices.csv"), "Circuit ID\n").unwrap(); - - let mut config = configured_bridge_config(temp_dir.display().to_string()); - if let Some(bridge) = config.bridge.as_mut() { - bridge.to_network = bridge.to_internet.clone(); - } - let readiness = upgrade_readiness_for_config_with(true, Some(&config), |_| true); - assert!(!readiness.all_ready()); - assert!(!readiness.interfaces_ready); - - let _ = std::fs::remove_file(temp_dir.join("network.json")); - let _ = std::fs::remove_file(temp_dir.join("ShapedDevices.csv")); - let _ = std::fs::remove_dir_all(&temp_dir); - } -} - fn finalize(ui: &mut cursive::Cursive) { // If we cannot load the config but a file exists, warn the user and // take a backup before proceeding to create a new config. @@ -483,3 +407,81 @@ fn continue_finalize(ui: &mut cursive::Cursive) { .button("OK", |ui| ui.quit()), ); } + +#[cfg(test)] +mod test { + use super::upgrade_readiness_for_config_with; + + fn configured_bridge_config(lqos_directory: String) -> lqos_config::Config { + let mut config = lqos_config::Config { + lqos_directory, + bridge: Some(lqos_config::BridgeConfig { + use_xdp_bridge: true, + to_internet: "wan0".to_string(), + to_network: "lan0".to_string(), + }), + single_interface: None, + ..lqos_config::Config::default() + }; + config.queues.downlink_bandwidth_mbps = 1000; + config.queues.uplink_bandwidth_mbps = 1000; + config + } + + #[test] + fn upgrade_ready_when_existing_install_is_configured() { + let temp_dir = + std::env::temp_dir().join(format!("lqos-setup-upgrade-ready-{}", std::process::id())); + std::fs::create_dir_all(&temp_dir).unwrap(); + std::fs::write(temp_dir.join("network.json"), "{}").unwrap(); + std::fs::write(temp_dir.join("ShapedDevices.csv"), "Circuit ID\n").unwrap(); + + let config = configured_bridge_config(temp_dir.display().to_string()); + let readiness = upgrade_readiness_for_config_with(true, Some(&config), |_| true); + assert!(readiness.all_ready()); + + let _ = std::fs::remove_file(temp_dir.join("network.json")); + let _ = std::fs::remove_file(temp_dir.join("ShapedDevices.csv")); + let _ = std::fs::remove_dir_all(&temp_dir); + } + + #[test] + fn upgrade_not_ready_without_required_runtime_inputs() { + let temp_dir = std::env::temp_dir().join(format!( + "lqos-setup-upgrade-missing-files-{}", + std::process::id() + )); + std::fs::create_dir_all(&temp_dir).unwrap(); + + let config = configured_bridge_config(temp_dir.display().to_string()); + let readiness = upgrade_readiness_for_config_with(true, Some(&config), |_| true); + assert!(!readiness.all_ready()); + assert!(!readiness.network_json_present); + assert!(!readiness.shaped_devices_present); + + let _ = std::fs::remove_dir_all(&temp_dir); + } + + #[test] + fn upgrade_not_ready_when_bridge_interfaces_are_not_distinct() { + let temp_dir = std::env::temp_dir().join(format!( + "lqos-setup-upgrade-same-iface-{}", + std::process::id() + )); + std::fs::create_dir_all(&temp_dir).unwrap(); + std::fs::write(temp_dir.join("network.json"), "{}").unwrap(); + std::fs::write(temp_dir.join("ShapedDevices.csv"), "Circuit ID\n").unwrap(); + + let mut config = configured_bridge_config(temp_dir.display().to_string()); + if let Some(bridge) = config.bridge.as_mut() { + bridge.to_network = bridge.to_internet.clone(); + } + let readiness = upgrade_readiness_for_config_with(true, Some(&config), |_| true); + assert!(!readiness.all_ready()); + assert!(!readiness.interfaces_ready); + + let _ = std::fs::remove_file(temp_dir.join("network.json")); + let _ = std::fs::remove_file(temp_dir.join("ShapedDevices.csv")); + let _ = std::fs::remove_dir_all(&temp_dir); + } +} diff --git a/src/rust/lqos_topology/Cargo.toml b/src/rust/lqos_topology/Cargo.toml index d025d35d..7eb62e05 100644 --- a/src/rust/lqos_topology/Cargo.toml +++ b/src/rust/lqos_topology/Cargo.toml @@ -13,3 +13,4 @@ lqos_bus = { path = "../lqos_bus" } lqos_config = { path = "../lqos_config" } lqos_overrides = { path = "../lqos_overrides" } lqos_probe = { path = "../lqos_probe" } +lqos_topology_compile = { path = "../lqos_topology_compile" } diff --git a/src/rust/lqos_topology/src/lib.rs b/src/rust/lqos_topology/src/lib.rs index 0ef8598d..c27ccf1f 100644 --- a/src/rust/lqos_topology/src/lib.rs +++ b/src/rust/lqos_topology/src/lib.rs @@ -5,18 +5,22 @@ use anyhow::{Context, Result}; use lqos_config::{ CircuitAnchor, CircuitAnchorsFile, Config, ConfigShapedDevices, TOPOLOGY_ATTACHMENT_AUTO_ID, - TopologyAllowedParent, TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus, - TopologyAttachmentOption, TopologyAttachmentRateSource, TopologyAttachmentRole, + TopLevelPlannerItem, TopLevelPlannerMode, TopLevelPlannerParams, TopologyAllowedParent, + TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus, TopologyAttachmentOption, + TopologyAttachmentRateSource, TopologyAttachmentRole, TopologyCanonicalIngressKind, TopologyCanonicalNode, TopologyCanonicalStateFile, TopologyEditorNode, TopologyEditorStateFile, TopologyEffectiveAttachmentState, TopologyEffectiveNodeState, TopologyEffectiveStateFile, TopologyRuntimeStatusFile, TopologyShapingCircuitInput, TopologyShapingDeviceInput, TopologyShapingInputsFile, TopologyShapingResolutionSource, circuit_anchors_path, - topology_effective_network_path, topology_effective_state_path, topology_runtime_status_path, - topology_shaping_inputs_path, + detect_shaping_cpus, plan_top_level_assignments, topology_effective_network_path, + topology_effective_state_path, topology_runtime_status_path, topology_shaping_inputs_path, }; -use lqos_overrides::{TopologyAttachmentMode, TopologyOverridesFile}; +use lqos_overrides::{ + CircuitAdjustment, OverrideStore, TopologyAttachmentMode, TopologyOverridesFile, +}; +use lqos_topology_compile::{TopologyCompiledShapingFile, TopologyImportFile}; use serde_json::{Map, Value}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs::{File, OpenOptions}; use std::io::Write; use std::net::IpAddr; @@ -25,6 +29,7 @@ use std::thread; use std::time::{SystemTime, UNIX_EPOCH}; const TOPOLOGY_EFFECTIVE_PUBLISH_LOCK_FILENAME: &str = "topology_effective_publish.lock"; +type EffectiveQueueAliasMap = HashMap; /// One unique probe pair emitted from topology state plus operator intent. #[derive(Clone, Debug, PartialEq, Eq)] @@ -97,7 +102,7 @@ fn collect_exported_effective_nodes(value: &Value, by_id: &mut HashMap, +) -> (EffectiveQueueAliasMap, EffectiveQueueAliasMap) { + let ui_by_node = ui_state + .nodes + .iter() + .map(|node| (node.node_id.as_str(), node)) + .collect::>(); + let mut aliases_by_id = HashMap::new(); + let mut aliases_by_name = HashMap::new(); + + for effective_node in &effective.nodes { + if exported_effective_nodes.contains_key(&effective_node.node_id) { + continue; + } + if effective_node.effective_attachment_id.is_some() { + continue; + } + let Some(parent_name) = exported_effective_nodes + .get(effective_node.logical_parent_node_id.as_str()) + .cloned() + else { + continue; + }; + let resolved = ( + effective_node.logical_parent_node_id.clone(), + parent_name.clone(), + ); + aliases_by_id + .entry(effective_node.node_id.clone()) + .or_insert_with(|| resolved.clone()); + if let Some(ui_node) = ui_by_node.get(effective_node.node_id.as_str()).copied() { + aliases_by_name + .entry(ui_node.node_name.clone()) + .or_insert_with(|| resolved.clone()); + } + } + + (aliases_by_id, aliases_by_name) +} + fn resolve_legacy_parent_from_effective_tree( parent_node: &str, parent_node_id: Option<&str>, exported_effective_nodes: &HashMap, exported_effective_aliases: &HashMap, + queue_aliases_by_id: &HashMap, + queue_aliases_by_name: &HashMap, ) -> Option<(String, String)> { let trimmed_id = parent_node_id.and_then(optional_non_empty); let trimmed_name = optional_non_empty(parent_node); @@ -166,6 +218,11 @@ fn resolve_legacy_parent_from_effective_tree( { return Some((parent_id.to_string(), parent_name)); } + if let Some(parent_id) = trimmed_id.as_deref() + && let Some(resolved) = queue_aliases_by_id.get(parent_id).cloned() + { + return Some(resolved); + } if let Some(parent_name) = trimmed_name.as_deref() && let Some(parent_id) = exported_effective_nodes .iter() @@ -173,6 +230,11 @@ fn resolve_legacy_parent_from_effective_tree( { return Some((parent_id, parent_name.to_string())); } + if let Some(parent_name) = trimmed_name.as_deref() + && let Some(resolved) = queue_aliases_by_name.get(parent_name).cloned() + { + return Some(resolved); + } trimmed_name.and_then(|alias| exported_effective_aliases.get(&alias).cloned()) } @@ -221,6 +283,7 @@ fn resolve_effective_parent_from_anchor( effective_by_node: &HashMap<&str, &TopologyEffectiveNodeState>, exported_effective_nodes: &HashMap, attachment_owner_by_attachment_id: &HashMap, + queue_aliases_by_id: &HashMap, ) -> Option<(String, String, Option, Option)> { let anchor_id = anchor_id.trim(); if anchor_id.is_empty() { @@ -243,9 +306,14 @@ fn resolve_effective_parent_from_anchor( attachment_name, )); } - - let (owner_node_id, owner_node_name) = - attachment_owner_by_attachment_id.get(anchor_id).cloned()?; + let Some((owner_node_id, owner_node_name)) = + attachment_owner_by_attachment_id.get(anchor_id).cloned() + else { + if let Some((parent_id, parent_name)) = queue_aliases_by_id.get(anchor_id).cloned() { + return Some((parent_id, parent_name, None, None)); + } + return None; + }; let owner_ui = ui_by_node.get(owner_node_id.as_str()).copied()?; let owner_effective = effective_by_node.get(owner_node_id.as_str()).copied()?; @@ -273,6 +341,10 @@ fn resolve_effective_parent_from_anchor( selected_attachment_name_for_node(owner_ui, owner_effective), )); } + if let Some((parent_id, parent_name)) = queue_aliases_by_id.get(owner_node_id.as_str()).cloned() + { + return Some((parent_id, parent_name, None, None)); + } Some(( owner_node_id, @@ -318,21 +390,85 @@ fn load_circuit_anchors( .unwrap_or_default() } +fn load_integration_shaping_artifacts( + config: &Config, +) -> Result)>> { + let topology_import = TopologyImportFile::load(config) + .with_context(|| "Unable to load topology_import.json while validating shaping ingress")?; + let Some(compiled_shaping) = TopologyCompiledShapingFile::load(config).with_context( + || "Unable to load topology_compiled_shaping.json while building shaping_inputs.json", + )? + else { + return Ok(None); + }; + let import_identity = topology_import + .as_ref() + .and_then(|file| file.ingress_identity.as_deref()) + .map(str::trim) + .filter(|value| !value.is_empty()); + let compiled_identity = compiled_shaping + .ingress_identity + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()); + if let (Some(import_identity), Some(compiled_identity)) = (import_identity, compiled_identity) + && import_identity != compiled_identity + { + anyhow::bail!( + "Integration compiled shaping ingress identity '{}' did not match topology import identity '{}'", + compiled_identity, + import_identity + ); + } + Ok(Some(compiled_shaping.shaping_artifacts())) +} + fn build_shaping_inputs( config: &Config, artifacts: &EffectiveTopologyArtifacts, ) -> Result> { - let shaped_devices_path = ConfigShapedDevices::path_for_config(config); - if !shaped_devices_path.exists() { - return Ok(None); - } + let integration_ingress = topology_import_ingress_enabled(config); + let (mut shaped_devices, circuit_anchor_rows) = if integration_ingress { + let Some((shaped_devices, circuit_anchors)) = load_integration_shaping_artifacts(config)? + else { + return Ok(None); + }; + (shaped_devices, circuit_anchors) + } else { + let shaped_devices_path = ConfigShapedDevices::path_for_config(config); + if !shaped_devices_path.exists() { + return Ok(None); + } - let shaped_devices_mtime = std::fs::metadata(&shaped_devices_path) - .ok() - .and_then(|metadata| metadata.modified().ok()); - let shaped_devices = ConfigShapedDevices::load_for_config(config) - .with_context(|| "Unable to load ShapedDevices.csv while building shaping_inputs.json")?; - let circuit_anchors = load_circuit_anchors(config, shaped_devices_mtime) + let shaped_devices_mtime = std::fs::metadata(&shaped_devices_path) + .ok() + .and_then(|metadata| metadata.modified().ok()); + ( + ConfigShapedDevices::load_for_config(config).with_context( + || "Unable to load ShapedDevices.csv while building shaping_inputs.json", + )?, + load_circuit_anchors(config, shaped_devices_mtime), + ) + }; + if integration_ingress { + let effective_overrides = load_runtime_shaping_overrides(config).with_context( + || "Unable to load effective overrides while building shaping_inputs.json", + )?; + let runtime_devices = apply_runtime_shaped_device_overrides( + shaped_devices.devices.clone(), + &effective_overrides, + ); + shaped_devices.replace_with_new_data(runtime_devices); + } + let flat_bucket_assignments = if runtime_flat_mode(config) { + Some(build_flat_bucket_assignments( + config, + &shaped_devices.devices, + )) + } else { + None + }; + let circuit_anchors = circuit_anchor_rows .into_iter() .map(|anchor| (anchor.circuit_id.clone(), anchor)) .collect::>(); @@ -355,9 +491,14 @@ fn build_shaping_inputs( collect_exported_effective_nodes(effective_network, &mut exported_effective_nodes); collect_exported_effective_aliases(effective_network, &mut exported_effective_aliases); } + let (queue_aliases_by_id, queue_aliases_by_name) = build_effective_queue_aliases( + &artifacts.ui_state, + &artifacts.effective, + &exported_effective_nodes, + ); let mut errors = Vec::new(); - let warnings = Vec::new(); + let mut warnings = Vec::new(); let mut circuits = Vec::::new(); let mut circuits_by_id = HashMap::::new(); @@ -379,13 +520,25 @@ fn build_shaping_inputs( effective_attachment_id, effective_attachment_name, resolution_source, - ) = if let Some(anchor_id) = anchor_node_id.as_deref() { + ) = if let Some((bucket_id, bucket_name)) = flat_bucket_assignments + .as_ref() + .and_then(|assignments| assignments.get(&device.circuit_id)) + { + ( + bucket_id.clone(), + bucket_name.clone(), + None, + None, + TopologyShapingResolutionSource::FlatBucket, + ) + } else if let Some(anchor_id) = anchor_node_id.as_deref() { match resolve_effective_parent_from_anchor( anchor_id, &ui_by_node, &effective_by_node, &exported_effective_nodes, &attachment_owner_by_attachment_id, + &queue_aliases_by_id, ) { Some(( resolved_parent_id, @@ -411,8 +564,8 @@ fn build_shaping_inputs( ) { (Some(ui_node), Some(_effective_node), _) => { anchor_node_name = Some(ui_node.node_name.clone()); - errors.push(format!( - "Circuit '{}' anchor '{}' ('{}') exists in topology metadata but does not resolve to an exported effective queue node.", + warnings.push(format!( + "Circuit '{}' anchor '{}' ('{}') did not resolve to an exported effective queue node. Falling back to generated parent-node shaping.", device.circuit_id, anchor_id, ui_node.node_name @@ -422,12 +575,12 @@ fn build_shaping_inputs( String::new(), None, None, - TopologyShapingResolutionSource::TopologyAnchor, + TopologyShapingResolutionSource::RuntimeFallback, ) } (None, None, Some(anchor)) => { - errors.push(format!( - "Circuit '{}' anchor '{}' ('{}') was not found in the effective topology.", + warnings.push(format!( + "Circuit '{}' anchor '{}' ('{}') was not found in the effective topology. Falling back to generated parent-node shaping.", device.circuit_id, anchor_id, anchor @@ -440,12 +593,12 @@ fn build_shaping_inputs( String::new(), None, None, - TopologyShapingResolutionSource::TopologyAnchor, + TopologyShapingResolutionSource::RuntimeFallback, ) } _ => { - errors.push(format!( - "Circuit '{}' anchor '{}' was not found in the effective topology.", + warnings.push(format!( + "Circuit '{}' anchor '{}' was not found in the effective topology. Falling back to generated parent-node shaping.", device.circuit_id, anchor_id )); ( @@ -453,7 +606,7 @@ fn build_shaping_inputs( String::new(), None, None, - TopologyShapingResolutionSource::TopologyAnchor, + TopologyShapingResolutionSource::RuntimeFallback, ) } }, @@ -464,6 +617,8 @@ fn build_shaping_inputs( device.parent_node_id.as_deref(), &exported_effective_nodes, &exported_effective_aliases, + &queue_aliases_by_id, + &queue_aliases_by_name, ) { ( @@ -477,8 +632,8 @@ fn build_shaping_inputs( if optional_non_empty(&device.parent_node).is_some() || optional_non_empty_owned(device.parent_node_id.clone()).is_some() { - errors.push(format!( - "Circuit '{}' parent reference '{}' ({}) was not found in the exported effective topology.", + warnings.push(format!( + "Circuit '{}' parent reference '{}' ({}) was not found in the exported effective topology. Falling back to generated parent-node shaping.", device.circuit_id, device.parent_node.trim(), device.parent_node_id.clone().unwrap_or_default().trim() @@ -489,7 +644,7 @@ fn build_shaping_inputs( String::new(), None, None, - TopologyShapingResolutionSource::LegacyParent, + TopologyShapingResolutionSource::RuntimeFallback, ) }; @@ -557,19 +712,29 @@ fn build_shaping_inputs( .sort_unstable_by(|left, right| left.device_id.cmp(&right.device_id)); } - for circuit in &circuits { - if circuit.anchor_node_id.is_some() && circuit.effective_parent_node_id.trim().is_empty() { - errors.push(format!( - "Circuit '{}' has an anchor but no resolved effective parent.", - circuit.circuit_id - )); - } else if circuit.logical_parent_node_name.is_some() - && circuit.effective_parent_node_id.trim().is_empty() - { - errors.push(format!( - "Circuit '{}' has a logical parent but no resolved effective parent.", - circuit.circuit_id + let unresolved_runtime_fallbacks = circuits + .iter() + .filter(|circuit| { + circuit.effective_parent_node_id.trim().is_empty() + && circuit.resolution_source == TopologyShapingResolutionSource::RuntimeFallback + }) + .count(); + if unresolved_runtime_fallbacks > 0 { + if config.shared_topology_compile_mode() == Some("flat") { + warnings.push(format!( + "Flat topology mode assigned {unresolved_runtime_fallbacks} circuit(s) to generated parent nodes during queue construction." )); + } else { + for circuit in &circuits { + if circuit.effective_parent_node_id.trim().is_empty() + && circuit.resolution_source == TopologyShapingResolutionSource::RuntimeFallback + { + warnings.push(format!( + "Circuit '{}' is unresolved in runtime topology and will be shaped under generated parent nodes.", + circuit.circuit_id + )); + } + } } } @@ -594,6 +759,298 @@ fn build_shaping_inputs( })) } +fn topology_import_ingress_enabled(config: &Config) -> bool { + config.uisp_integration.enable_uisp + || config.splynx_integration.enable_splynx + || config + .netzur_integration + .as_ref() + .is_some_and(|integration| integration.enable_netzur) + || config + .visp_integration + .as_ref() + .is_some_and(|integration| integration.enable_visp) + || config.powercode_integration.enable_powercode + || config.sonar_integration.enable_sonar + || config + .wispgate_integration + .as_ref() + .is_some_and(|integration| integration.enable_wispgate) +} + +fn runtime_flat_mode(config: &Config) -> bool { + config.shared_topology_compile_mode() == Some("flat") +} + +fn count_interface_tx_queues(interface_name: &str) -> Option { + let path = Path::new("/sys/class/net") + .join(interface_name) + .join("queues"); + let entries = std::fs::read_dir(path).ok()?; + let mut count = 0usize; + for entry in entries.flatten() { + let Some(name) = entry.file_name().to_str().map(str::to_string) else { + continue; + }; + if name.starts_with("tx-") { + count += 1; + } + } + Some(count) +} + +fn runtime_flat_bucket_count(config: &Config) -> usize { + let mut queues_available = config + .queues + .override_available_queues + .map(|value| value as usize); + if queues_available.is_none() { + queues_available = if config.queues.dry_run { + Some(16) + } else { + let internet_queues = count_interface_tx_queues(&config.internet_interface()); + let isp_queues = count_interface_tx_queues(&config.isp_interface()); + internet_queues + .zip(isp_queues) + .map(|(left, right)| left.min(right)) + }; + } + + let shaping_cpu_count = detect_shaping_cpus(config).shaping.len(); + let mut queue_count = queues_available.unwrap_or(shaping_cpu_count.max(1)); + if shaping_cpu_count > 0 { + queue_count = queue_count.min(shaping_cpu_count); + } + if config.on_a_stick_mode() { + queue_count = (queue_count / 2).max(1); + } + queue_count.max(1) +} + +fn runtime_flat_bucket_name(index: usize) -> String { + format!("Generated_PN_{}", index + 1) +} + +fn runtime_flat_bucket_id(index: usize) -> String { + format!("libreqos:generated:flat:bucket:{index}") +} + +fn runtime_flat_bucket_network(config: &Config) -> Value { + let mut root = Map::new(); + for index in 0..runtime_flat_bucket_count(config) { + let mut node = Map::new(); + node.insert("children".to_string(), Value::Object(Map::new())); + node.insert( + "downloadBandwidthMbps".to_string(), + Value::Number(config.queues.generated_pn_download_mbps.into()), + ); + node.insert( + "uploadBandwidthMbps".to_string(), + Value::Number(config.queues.generated_pn_upload_mbps.into()), + ); + node.insert( + "id".to_string(), + Value::String(runtime_flat_bucket_id(index)), + ); + node.insert( + "name".to_string(), + Value::String(runtime_flat_bucket_name(index)), + ); + node.insert("type".to_string(), Value::String("Site".to_string())); + root.insert(runtime_flat_bucket_name(index), Value::Object(node)); + } + Value::Object(root) +} + +fn build_flat_bucket_assignments( + config: &Config, + devices: &[lqos_config::ShapedDevice], +) -> HashMap { + let bucket_count = runtime_flat_bucket_count(config); + if bucket_count == 0 { + return HashMap::new(); + } + + let bucket_names = (0..bucket_count) + .map(runtime_flat_bucket_name) + .collect::>(); + let bucket_ids_by_name = bucket_names + .iter() + .enumerate() + .map(|(index, name)| (name.clone(), runtime_flat_bucket_id(index))) + .collect::>(); + + let mut item_weights = BTreeMap::::new(); + for device in devices { + let weight = f64::from(device.download_max_mbps.max(0.0) + device.upload_max_mbps.max(0.0)); + let sanitized = if weight.is_finite() && weight > 0.0 { + weight + } else { + 1.0 + }; + item_weights + .entry(device.circuit_id.clone()) + .and_modify(|current| { + if sanitized > *current { + *current = sanitized; + } + }) + .or_insert(sanitized); + } + + let items = item_weights + .into_iter() + .map(|(id, weight)| TopLevelPlannerItem { id, weight }) + .collect::>(); + let planner_mode = if config.queues.use_binpacking { + TopLevelPlannerMode::Greedy + } else { + TopLevelPlannerMode::RoundRobin + }; + let planner = plan_top_level_assignments( + &items, + &bucket_names, + &BTreeMap::new(), + &BTreeMap::new(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs_f64()) + .unwrap_or(0.0), + &TopLevelPlannerParams { + mode: planner_mode, + hysteresis_threshold: 0.0, + cooldown_seconds: 0.0, + move_budget_per_run: usize::MAX, + }, + ); + + planner + .assignment + .into_iter() + .filter_map(|(circuit_id, bucket_name)| { + bucket_ids_by_name + .get(&bucket_name) + .map(|bucket_id| (circuit_id, (bucket_id.clone(), bucket_name))) + }) + .collect() +} + +fn load_runtime_shaping_overrides(config: &Config) -> Result { + let apply_stormguard = config + .stormguard + .as_ref() + .is_some_and(|stormguard| stormguard.enabled && !stormguard.dry_run); + let apply_treeguard = config.treeguard.enabled; + OverrideStore::load_effective_for_config(config, apply_stormguard, apply_treeguard) + .with_context(|| "Unable to load effective override layers") +} + +fn apply_runtime_shaped_device_overrides( + base_devices: Vec, + overrides: &lqos_overrides::OverrideFile, +) -> Vec { + let mut devices = base_devices; + for override_device in overrides.persistent_devices() { + if let Some(existing_index) = devices + .iter() + .position(|device| device.device_id == override_device.device_id) + { + devices[existing_index] = override_device.clone(); + } else { + devices.push(override_device.clone()); + } + } + + for adjustment in overrides.circuit_adjustments() { + match adjustment { + CircuitAdjustment::CircuitAdjustSpeed { + circuit_id, + min_download_bandwidth, + max_download_bandwidth, + min_upload_bandwidth, + max_upload_bandwidth, + } => { + for device in devices + .iter_mut() + .filter(|device| device.circuit_id == *circuit_id) + { + if let Some(value) = min_download_bandwidth { + device.download_min_mbps = *value; + } + if let Some(value) = max_download_bandwidth { + device.download_max_mbps = *value; + } + if let Some(value) = min_upload_bandwidth { + device.upload_min_mbps = *value; + } + if let Some(value) = max_upload_bandwidth { + device.upload_max_mbps = *value; + } + } + } + CircuitAdjustment::DeviceAdjustSpeed { + device_id, + min_download_bandwidth, + max_download_bandwidth, + min_upload_bandwidth, + max_upload_bandwidth, + } => { + for device in devices + .iter_mut() + .filter(|device| device.device_id == *device_id) + { + if let Some(value) = min_download_bandwidth { + device.download_min_mbps = *value; + } + if let Some(value) = max_download_bandwidth { + device.download_max_mbps = *value; + } + if let Some(value) = min_upload_bandwidth { + device.upload_min_mbps = *value; + } + if let Some(value) = max_upload_bandwidth { + device.upload_max_mbps = *value; + } + } + } + CircuitAdjustment::DeviceAdjustSqm { + device_id, + sqm_override, + } => { + for device in devices + .iter_mut() + .filter(|device| device.device_id == *device_id) + { + device.sqm_override = sqm_override + .as_ref() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + } + } + CircuitAdjustment::RemoveCircuit { circuit_id } => { + devices.retain(|device| device.circuit_id != *circuit_id); + } + CircuitAdjustment::RemoveDevice { device_id } => { + devices.retain(|device| device.device_id != *device_id); + } + CircuitAdjustment::ReparentCircuit { + circuit_id, + parent_node, + } => { + for device in devices + .iter_mut() + .filter(|device| device.circuit_id == *circuit_id) + { + device.parent_node = parent_node.clone(); + device.parent_node_id = None; + } + } + } + } + + devices +} + /// Loads canonical topology state, falling back to importing legacy `network.json`. pub fn load_canonical_topology_state(config: &Config) -> TopologyCanonicalStateFile { TopologyCanonicalStateFile::load_with_legacy_fallback(config).unwrap_or_default() @@ -699,9 +1156,13 @@ fn build_effective_topology_artifacts_from_prepared( let effective = compute_effective_state_from_prepared(config, prepared, overrides, health); let ui_state = merged_topology_state_from_prepared(config, prepared, overrides, health, &effective); - let effective_network = canonical.compatibility_network_json().as_object().map(|_| { - apply_effective_topology_to_canonical_state(config, canonical, &ui_state, &effective) - }); + let effective_network = if runtime_flat_mode(config) { + Some(runtime_flat_bucket_network(config)) + } else { + canonical.compatibility_network_json().as_object().map(|_| { + apply_effective_topology_to_canonical_state(config, canonical, &ui_state, &effective) + }) + }; if let Some(effective_network) = effective_network.as_ref() { validate_effective_topology_network(&ui_state, &effective, effective_network)?; @@ -1227,6 +1688,10 @@ fn current_parent_for_node<'a>( .find(|parent| parent.parent_node_id == parent_id) } +fn runtime_may_infer_parent_from_candidates(source: &str) -> bool { + !(source.starts_with("uisp/") || source.starts_with("python/")) +} + fn merge_attachment_option( existing: &mut TopologyAttachmentOption, incoming: &TopologyAttachmentOption, @@ -1387,6 +1852,7 @@ fn normalize_topology_editor_state(canonical: &TopologyEditorStateFile) -> Topol schema_version: canonical.schema_version, source: canonical.source.clone(), generated_unix: canonical.generated_unix, + ingress_identity: canonical.ingress_identity.clone(), nodes, } } @@ -1445,6 +1911,7 @@ fn compute_effective_state_from_prepared( health: &TopologyAttachmentHealthStateFile, ) -> TopologyEffectiveStateFile { let health_by_pair = health_entries_by_pair(config, health); + let may_infer_parent = runtime_may_infer_parent_from_candidates(&prepared.source); let mut nodes = Vec::with_capacity(prepared.nodes.len()); for node in &prepared.nodes { @@ -1456,9 +1923,13 @@ fn compute_effective_state_from_prepared( }) .or_else(|| node.current_parent_node_id.clone()) .or_else(|| { - node.allowed_parents - .first() - .map(|parent| parent.parent_node_id.clone()) + may_infer_parent + .then(|| { + node.allowed_parents + .first() + .map(|parent| parent.parent_node_id.clone()) + }) + .flatten() }); let Some(selected_parent_id) = selected_parent_id else { @@ -1474,9 +1945,26 @@ fn compute_effective_state_from_prepared( continue; }; - let Some(selected_parent) = current_parent_for_node(node, &selected_parent_id) - .or_else(|| node.allowed_parents.first()) + let Some(selected_parent) = + current_parent_for_node(node, &selected_parent_id).or_else(|| { + may_infer_parent + .then(|| node.allowed_parents.first()) + .flatten() + }) else { + let fixed_attachment_id = node + .current_attachment_id + .clone() + .filter(|attachment_id| !attachment_id.is_empty()); + nodes.push(TopologyEffectiveNodeState { + node_id: node.node_id.clone(), + logical_parent_node_id: selected_parent_id, + preferred_attachment_id: fixed_attachment_id.clone(), + effective_attachment_id: fixed_attachment_id, + fallback_reason: None, + all_attachments_suppressed: false, + attachments: Vec::new(), + }); continue; }; let selected_parent_id = selected_parent.parent_node_id.clone(); @@ -2879,6 +3367,18 @@ fn validate_effective_node_identity_consistency( .iter() .find(|parent| parent.parent_node_id == node.logical_parent_node_id) else { + let fixed_attachment_id = ui_node + .current_attachment_id + .as_deref() + .filter(|attachment_id| !attachment_id.is_empty()); + let legacy_fixed_parent = ui_node.allowed_parents.is_empty() + && ui_node.current_parent_node_id.as_deref() + == Some(node.logical_parent_node_id.as_str()) + && node.preferred_attachment_id.as_deref() == fixed_attachment_id + && node.effective_attachment_id.as_deref() == fixed_attachment_id; + if legacy_fixed_parent { + continue; + } errors.push(format!( "Effective topology selected invalid parent '{}' for '{}'.", node.logical_parent_node_id, ui_node.node_name @@ -2989,8 +3489,27 @@ pub fn apply_effective_topology_to_network_json( else { continue; }; + let already_parented = find_parent_id_of_node(&out, &ui_node.node_id, None) + .flatten() + .as_deref() + == Some(selected_parent.parent_node_id.as_str()); let Some(effective_attachment_id) = effective_node.effective_attachment_id.as_deref() else { + if ui_node.current_parent_node_id.as_deref() + == Some(effective_node.logical_parent_node_id.as_str()) + && already_parented + { + continue; + } + let Some((node_key, node_value)) = remove_node_by_id(&mut out, &ui_node.node_id) else { + continue; + }; + let _ = insert_node_under_parent_id( + &mut out, + &selected_parent.parent_node_id, + &node_key, + node_value, + ); continue; }; let Some(target_attachment) = selected_parent @@ -3003,10 +3522,12 @@ pub fn apply_effective_topology_to_network_json( let current_anchor_attachment = find_node_by_id(&out, &ui_node.node_id) .map(|node_value| attachment_anchor_for_reparent(node_value, target_attachment)) .unwrap_or_else(|| target_attachment.clone()); - let already_anchored = find_parent_id_of_node(&out, &ui_node.node_id, None) - .flatten() - .as_deref() - == Some(current_anchor_attachment.attachment_id.as_str()); + let already_anchored = already_parented + && current_anchor_attachment.attachment_id == selected_parent.parent_node_id + || find_parent_id_of_node(&out, &ui_node.node_id, None) + .flatten() + .as_deref() + == Some(current_anchor_attachment.attachment_id.as_str()); if ui_node.current_parent_node_id.as_deref() == Some(effective_node.logical_parent_node_id.as_str()) && ui_node.current_attachment_id.as_deref() @@ -3048,12 +3569,14 @@ fn apply_effective_topology_to_canonical_state( ui_state: &TopologyEditorStateFile, effective: &TopologyEffectiveStateFile, ) -> Value { - let mut effective_network = apply_effective_topology_to_network_json( - config, - canonical.compatibility_network_json(), - ui_state, - effective, - ); + let canonical_network = + if canonical.ingress_kind == TopologyCanonicalIngressKind::NativeIntegration { + canonical.insight_topology_network_json() + } else { + canonical.compatibility_network_json().clone() + }; + let mut effective_network = + apply_effective_topology_to_network_json(config, &canonical_network, ui_state, effective); if let Some(root) = effective_network.as_object_mut() { recompile_effective_network_bandwidths(root, canonical, ui_state, effective); } @@ -3063,15 +3586,17 @@ fn apply_effective_topology_to_canonical_state( #[cfg(test)] mod tests { use super::{ - EffectiveTopologyArtifacts, apply_effective_topology_to_network_json, - build_effective_topology_artifacts, build_shaping_inputs, compute_effective_state, - publish_effective_topology_artifacts, publish_topology_runtime_error_status, - validate_effective_topology_network, + EffectiveTopologyArtifacts, apply_effective_topology_to_canonical_state, + apply_effective_topology_to_network_json, auto_attachment_option, + build_effective_topology_artifacts, build_effective_topology_artifacts_from_canonical, + build_shaping_inputs, compute_effective_state, publish_effective_topology_artifacts, + publish_topology_runtime_error_status, validate_effective_topology_network, }; use lqos_config::{ CircuitAnchor, CircuitAnchorsFile, Config, TopologyAllowedParent, TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus, TopologyAttachmentOption, TopologyAttachmentRateSource, TopologyAttachmentRole, + TopologyCanonicalIngressKind, TopologyCanonicalNode, TopologyCanonicalStateFile, TopologyEditorNode, TopologyEditorStateFile, TopologyEffectiveAttachmentState, TopologyEffectiveNodeState, TopologyEffectiveStateFile, TopologyRuntimeStatusFile, topology_effective_network_path, topology_effective_state_path, @@ -3093,6 +3618,32 @@ mod tests { path } + fn runtime_tree_max_depth(value: &Value) -> usize { + fn recurse(node: &Value, depth: usize) -> usize { + let Some(map) = node.as_object() else { + return depth; + }; + let Some(children) = map.get("children").and_then(Value::as_object) else { + return depth; + }; + children + .values() + .map(|child| recurse(child, depth + 1)) + .max() + .unwrap_or(depth) + } + + value + .as_object() + .map(|root| { + root.values() + .map(|child| recurse(child, 1)) + .max() + .unwrap_or(0) + }) + .unwrap_or(0) + } + fn sample_attachment_option( attachment_id: &str, attachment_name: &str, @@ -3146,6 +3697,7 @@ mod tests { schema_version: 1, source: "test".to_string(), generated_unix: Some(1), + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "tower-1".to_string(), node_name: "Tower 1".to_string(), @@ -3266,6 +3818,7 @@ mod tests { schema_version: 1, source: "test".to_string(), generated_unix: Some(1), + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "tower-1".to_string(), node_name: "Tower 1".to_string(), @@ -3275,7 +3828,6 @@ mod tests { effective_network: Some(json!({ "Tower 1": { "id": "tower-1", - "name": "Tower 1", "children": {} } })), @@ -3296,6 +3848,310 @@ mod tests { assert_eq!(circuit.effective_parent_node_name, "Tower 1"); } + #[test] + fn shaping_inputs_apply_effective_overrides_for_integration_ingress() { + let lqos_directory = unique_temp_dir("lqos-topology-runtime-overrides"); + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.splynx_integration.enable_splynx = true; + fs::write( + lqos_directory.join("topology_import.json"), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "splynx/full", + "generated_unix": 1, + "ingress_identity": "ingress-1", + "compile_mode": "full", + "imported": { + "source": "splynx/full", + "generated_unix": 1, + "ingress_identity": "ingress-1", + "compatibility_network_json": { + "Tower 1": { + "id": "tower-1", + "children": {} + } + }, + "shaped_devices": [ + { + "circuit_id": "circuit-1", + "circuit_name": "Circuit 1", + "device_id": "device-1", + "device_name": "Device 1", + "parent_node": "Tower 1", + "parent_node_id": "tower-1", + "anchor_node_id": null, + "mac": "", + "ipv4": [], + "ipv6": [], + "download_min_mbps": 10.0, + "upload_min_mbps": 10.0, + "download_max_mbps": 100.0, + "upload_max_mbps": 100.0, + "comment": "", + "sqm_override": null + } + ], + "circuit_anchors": { + "schema_version": 1, + "source": "splynx/full", + "generated_unix": 1, + "anchors": [] + }, + "ethernet_advisories": [] + } + })) + .expect("topology import should serialize"), + ) + .expect("topology import should write"); + fs::write( + lqos_directory.join("topology_compiled_shaping.json"), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "splynx/full", + "compile_mode": "full", + "generated_unix": 1, + "ingress_identity": "ingress-1", + "shaped_devices": [ + { + "circuit_id": "circuit-1", + "circuit_name": "Circuit 1", + "device_id": "device-1", + "device_name": "Device 1", + "parent_node": "Tower 1", + "parent_node_id": "tower-1", + "anchor_node_id": null, + "mac": "", + "ipv4": [], + "ipv6": [], + "download_min_mbps": 10.0, + "upload_min_mbps": 10.0, + "download_max_mbps": 100.0, + "upload_max_mbps": 100.0, + "comment": "", + "sqm_override": null + } + ], + "circuit_anchors": { + "schema_version": 1, + "source": "splynx/full", + "generated_unix": 1, + "anchors": [] + } + })) + .expect("compiled shaping should serialize"), + ) + .expect("compiled shaping should write"); + fs::write( + lqos_directory.join("lqos_overrides.json"), + serde_json::to_string_pretty(&json!({ + "persistent_devices": [ + { + "circuit_id": "circuit-2", + "circuit_name": "Circuit 2", + "device_id": "device-2", + "device_name": "Device 2", + "parent_node": "Tower 1", + "parent_node_id": "tower-1", + "anchor_node_id": null, + "mac": "", + "ipv4": [], + "ipv6": [], + "download_min_mbps": 5.0, + "upload_min_mbps": 5.0, + "download_max_mbps": 50.0, + "upload_max_mbps": 50.0, + "comment": "", + "sqm_override": null + } + ], + "circuit_adjustments": [ + { + "type": "device_adjust_speed", + "device_id": "device-1", + "max_download_bandwidth": 80.0, + "max_upload_bandwidth": 60.0 + } + ], + "network_adjustments": [] + })) + .expect("override json should serialize"), + ) + .expect("override file should write"); + + let artifacts = EffectiveTopologyArtifacts { + effective: TopologyEffectiveStateFile { + schema_version: 1, + generated_unix: Some(1), + canonical_generated_unix: Some(1), + health_generated_unix: Some(1), + nodes: Vec::new(), + }, + ui_state: TopologyEditorStateFile { + schema_version: 1, + source: "test".to_string(), + generated_unix: Some(1), + ingress_identity: None, + nodes: Vec::new(), + }, + effective_network: Some(json!({ + "Tower 1": { + "id": "tower-1", + "children": {} + } + })), + }; + + let shaping_inputs = build_shaping_inputs(&config, &artifacts) + .expect("shaping inputs should build") + .expect("shaping inputs should exist"); + let circuit_one = shaping_inputs + .circuits + .iter() + .find(|circuit| circuit.circuit_id == "circuit-1") + .expect("expected circuit-1"); + let circuit_two = shaping_inputs + .circuits + .iter() + .find(|circuit| circuit.circuit_id == "circuit-2") + .expect("expected circuit-2"); + + assert_eq!(circuit_one.download_max_mbps, 80.0); + assert_eq!(circuit_one.upload_max_mbps, 60.0); + assert_eq!(circuit_two.effective_parent_node_id, "tower-1"); + assert_eq!(circuit_two.effective_parent_node_name, "Tower 1"); + } + + #[test] + fn shaping_inputs_use_topology_import_without_shaped_devices_csv_for_integration_ingress() { + let lqos_directory = unique_temp_dir("lqos-topology-import-shaped-devices"); + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.uisp_integration.enable_uisp = true; + fs::write( + lqos_directory.join("topology_import.json"), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "uisp/full2", + "compile_mode": "full", + "generated_unix": 1, + "ingress_identity": "ingress-1", + "imported": { + "source": "uisp/full2", + "generated_unix": 1, + "ingress_identity": "imported-1", + "compatibility_network_json": {}, + "shaped_devices": [ + { + "circuit_id": "circuit-1", + "circuit_name": "Circuit 1", + "device_id": "device-1", + "device_name": "Device 1", + "parent_node": "Tower 1", + "parent_node_id": "tower-1", + "anchor_node_id": null, + "mac": "", + "ipv4": [], + "ipv6": [], + "download_min_mbps": 10.0, + "upload_min_mbps": 10.0, + "download_max_mbps": 100.0, + "upload_max_mbps": 100.0, + "comment": "", + "sqm_override": null + } + ], + "circuit_anchors": { + "schema_version": 1, + "source": "uisp/full", + "generated_unix": 1, + "anchors": [] + }, + "ethernet_advisories": [] + } + })) + .expect("topology import should serialize"), + ) + .expect("topology_import.json should write"); + fs::write( + lqos_directory.join("topology_compiled_shaping.json"), + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "source": "uisp/full", + "compile_mode": "full", + "generated_unix": 1, + "ingress_identity": "ingress-1", + "shaped_devices": [ + { + "circuit_id": "circuit-1", + "circuit_name": "Circuit 1", + "device_id": "device-1", + "device_name": "Device 1", + "parent_node": "Compiled Tower 1", + "parent_node_id": "compiled-tower-1", + "anchor_node_id": null, + "mac": "", + "ipv4": [], + "ipv6": [], + "download_min_mbps": 10.0, + "upload_min_mbps": 10.0, + "download_max_mbps": 100.0, + "upload_max_mbps": 100.0, + "comment": "", + "sqm_override": null + } + ], + "circuit_anchors": { + "schema_version": 1, + "source": "uisp/full", + "generated_unix": 1, + "anchors": [] + } + })) + .expect("compiled shaping should serialize"), + ) + .expect("topology_compiled_shaping.json should write"); + + let artifacts = EffectiveTopologyArtifacts { + effective: TopologyEffectiveStateFile { + schema_version: 1, + generated_unix: Some(1), + canonical_generated_unix: Some(1), + health_generated_unix: Some(1), + nodes: Vec::new(), + }, + ui_state: TopologyEditorStateFile { + schema_version: 1, + source: "test".to_string(), + generated_unix: Some(1), + ingress_identity: None, + nodes: Vec::new(), + }, + effective_network: Some(json!({ + "Compiled Tower 1": { + "id": "compiled-tower-1", + "children": {} + } + })), + }; + + let shaping_inputs = build_shaping_inputs(&config, &artifacts) + .expect("shaping inputs should build from topology_compiled_shaping.json") + .expect("shaping inputs should exist"); + let circuit = shaping_inputs + .circuits + .iter() + .find(|circuit| circuit.circuit_id == "circuit-1") + .expect("expected circuit"); + + assert_eq!(circuit.effective_parent_node_id, "compiled-tower-1"); + assert_eq!(circuit.effective_parent_node_name, "Compiled Tower 1"); + } + #[test] fn shaping_inputs_remap_non_selected_attachment_anchor_to_effective_attachment() { let lqos_directory = unique_temp_dir("lqos-topology-attachment-anchor-remap"); @@ -3356,6 +4212,7 @@ mod tests { schema_version: 1, source: "test".to_string(), generated_unix: Some(1), + ingress_identity: None, nodes: vec![ TopologyEditorNode { node_id: "attachment-old".to_string(), @@ -3456,12 +4313,12 @@ mod tests { schema_version: 1, source: "test".to_string(), generated_unix: Some(1), + ingress_identity: None, nodes: Vec::new(), }, effective_network: Some(json!({ "Tower 1": { "id": "tower-1", - "name": "Tower 1", "children": {} } })), @@ -3485,7 +4342,7 @@ mod tests { } #[test] - fn shaping_inputs_fail_when_anchor_does_not_resolve() { + fn shaping_inputs_fallback_to_generated_parents_when_anchor_does_not_resolve() { let lqos_directory = unique_temp_dir("lqos-topology-missing-anchor"); let config = Config { lqos_directory: lqos_directory.to_string_lossy().to_string(), @@ -3525,16 +4382,129 @@ mod tests { schema_version: 1, source: "test".to_string(), generated_unix: Some(1), + ingress_identity: None, nodes: Vec::new(), }, effective_network: Some(json!({})), }; - let err = build_shaping_inputs(&config, &artifacts) - .expect_err("missing anchor should fail shaping input generation"); - let message = format!("{err:#}"); - assert!(message.contains("circuit-1")); - assert!(message.contains("missing-anchor")); + let shaping_inputs = build_shaping_inputs(&config, &artifacts) + .expect("missing anchor should no longer fail shaping input generation") + .expect("shaping inputs should be present"); + let circuit = shaping_inputs + .circuits + .iter() + .find(|circuit| circuit.circuit_id == "circuit-1") + .expect("circuit should be present"); + assert_eq!(circuit.effective_parent_node_id, ""); + assert_eq!(circuit.effective_parent_node_name, ""); + assert_eq!( + circuit.resolution_source, + lqos_config::TopologyShapingResolutionSource::RuntimeFallback + ); + assert!( + shaping_inputs + .warnings + .iter() + .any(|warning| warning.contains("missing-anchor")) + ); + assert!( + shaping_inputs + .warnings + .iter() + .any(|warning| warning.contains("generated parent nodes")) + ); + } + + #[test] + fn flat_mode_assigns_explicit_generated_parent_buckets() { + let lqos_directory = unique_temp_dir("lqos-topology-flat-summary"); + let mut config = Config { + lqos_directory: lqos_directory.to_string_lossy().to_string(), + ..Config::default() + }; + config.topology.compile_mode = "flat".to_string(); + config.queues.override_available_queues = Some(2); + fs::write( + lqos_directory.join("ShapedDevices.csv"), + concat!( + "Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment\n", + "\"circuit-1\",\"Circuit 1\",\"device-1\",\"Device 1\",\"\",\"\",\"\",\"aa:bb:cc:dd:ee:ff\",\"192.0.2.10/32\",\"\",\"10\",\"10\",\"100\",\"100\",\"\"\n", + "\"circuit-2\",\"Circuit 2\",\"device-2\",\"Device 2\",\"\",\"\",\"\",\"aa:bb:cc:dd:ee:00\",\"192.0.2.11/32\",\"\",\"10\",\"10\",\"100\",\"100\",\"\"\n", + ), + ) + .expect("ShapedDevices.csv should write"); + + let artifacts = EffectiveTopologyArtifacts { + effective: TopologyEffectiveStateFile { + schema_version: 1, + generated_unix: Some(1), + canonical_generated_unix: Some(1), + health_generated_unix: Some(1), + nodes: Vec::new(), + }, + ui_state: TopologyEditorStateFile { + schema_version: 1, + source: "test".to_string(), + generated_unix: Some(1), + ingress_identity: None, + nodes: Vec::new(), + }, + effective_network: Some(json!({})), + }; + + let shaping_inputs = build_shaping_inputs(&config, &artifacts) + .expect("flat mode shaping inputs should build") + .expect("shaping inputs should be present"); + assert!(shaping_inputs.warnings.is_empty()); + assert_eq!(shaping_inputs.circuits.len(), 2); + assert!(shaping_inputs.circuits.iter().all(|circuit| { + circuit + .effective_parent_node_name + .starts_with("Generated_PN_") + })); + assert!(shaping_inputs.circuits.iter().all(|circuit| { + circuit.resolution_source == lqos_config::TopologyShapingResolutionSource::FlatBucket + })); + } + + #[test] + fn flat_mode_publishes_generated_parent_nodes_into_effective_network() { + let mut config = Config::default(); + config.topology.compile_mode = "flat".to_string(); + config.queues.override_available_queues = Some(3); + + let canonical = TopologyCanonicalStateFile::from_legacy_network_json(&json!({})); + let artifacts = build_effective_topology_artifacts_from_canonical( + &config, + &canonical, + &TopologyOverridesFile::default(), + &TopologyAttachmentHealthStateFile::default(), + ) + .expect("flat mode effective artifacts should build"); + let effective_network = artifacts + .effective_network + .expect("flat mode should publish an effective network"); + let root = effective_network + .as_object() + .expect("effective network should be an object"); + assert_eq!(root.len(), 3); + for index in 0..3 { + let name = format!("Generated_PN_{}", index + 1); + let expected_id = format!("libreqos:generated:flat:bucket:{index}"); + let node = root + .get(&name) + .and_then(Value::as_object) + .expect("generated parent node should exist"); + assert_eq!( + node.get("id").and_then(Value::as_str), + Some(expected_id.as_str()) + ); + assert_eq!( + node.get("name").and_then(Value::as_str), + Some(name.as_str()) + ); + } } #[test] @@ -3595,6 +4565,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "child-site".to_string(), node_name: "Child Site".to_string(), @@ -3707,6 +4678,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "child-site".to_string(), node_name: "Child Site".to_string(), @@ -3769,6 +4741,247 @@ mod tests { assert_eq!(child_site["uploadBandwidthMbps"].as_u64(), Some(400)); } + #[test] + fn runtime_squashing_reduces_export_tree_depth_for_queue_consumers() { + let mut config = Config::default(); + config.uisp_integration.enable_uisp = true; + let canonical = json!({ + "Parent Site": { + "children": { + "Relay A": { + "children": { + "Relay B": { + "children": { + "Child Site": { + "children": {}, + "downloadBandwidthMbps": 800, + "id": "child-site", + "name": "Child Site", + "parent_site": "Relay B", + "type": "Site", + "uploadBandwidthMbps": 700 + } + }, + "downloadBandwidthMbps": 600, + "id": "relay-b", + "name": "Relay B", + "parent_site": "Relay A", + "type": "AP", + "uploadBandwidthMbps": 500 + } + }, + "downloadBandwidthMbps": 900, + "id": "relay-a", + "name": "Relay A", + "parent_site": "Parent Site", + "type": "AP", + "uploadBandwidthMbps": 400 + } + }, + "downloadBandwidthMbps": 1000, + "id": "parent-site", + "name": "Parent Site", + "type": "Site", + "uploadBandwidthMbps": 1000 + } + }); + let ui_state = TopologyEditorStateFile { + schema_version: 1, + source: "uisp/full2".to_string(), + generated_unix: None, + ingress_identity: None, + nodes: vec![TopologyEditorNode { + node_id: "child-site".to_string(), + node_name: "Child Site".to_string(), + current_parent_node_id: Some("parent-site".to_string()), + current_parent_node_name: Some("Parent Site".to_string()), + current_attachment_id: Some("relay-b".to_string()), + current_attachment_name: Some("Relay B".to_string()), + can_move: true, + allowed_parents: vec![TopologyAllowedParent { + parent_node_id: "parent-site".to_string(), + parent_node_name: "Parent Site".to_string(), + attachment_options: vec![sample_attachment_option("relay-b", "Relay B")], + all_attachments_suppressed: false, + has_probe_unavailable_attachments: false, + }], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }], + }; + let effective = TopologyEffectiveStateFile { + schema_version: 1, + generated_unix: None, + canonical_generated_unix: None, + health_generated_unix: None, + nodes: vec![TopologyEffectiveNodeState { + node_id: "child-site".to_string(), + logical_parent_node_id: "parent-site".to_string(), + preferred_attachment_id: Some("relay-b".to_string()), + effective_attachment_id: Some("relay-b".to_string()), + fallback_reason: None, + all_attachments_suppressed: false, + attachments: vec![TopologyEffectiveAttachmentState { + attachment_id: "relay-b".to_string(), + health_status: TopologyAttachmentHealthStatus::Healthy, + health_reason: None, + suppressed_until_unix: None, + probe_enabled: false, + probeable: false, + effective_selected: true, + }], + }], + }; + + let canonical_depth = runtime_tree_max_depth(&canonical); + let squashed = + apply_effective_topology_to_network_json(&config, &canonical, &ui_state, &effective); + let squashed_depth = runtime_tree_max_depth(&squashed); + + assert_eq!(canonical_depth, 4); + assert_eq!(squashed_depth, 2); + assert!(squashed_depth < canonical_depth); + } + + #[test] + fn native_integration_effective_export_uses_logical_canonical_tree_before_squashing() { + let mut config = Config::default(); + config.uisp_integration.enable_uisp = true; + + let editor_state = TopologyEditorStateFile { + schema_version: 1, + source: "uisp/full2".to_string(), + generated_unix: None, + ingress_identity: None, + nodes: vec![ + TopologyEditorNode { + node_id: "site-west".to_string(), + node_name: "WestRedd".to_string(), + current_parent_node_id: None, + current_parent_node_name: None, + current_attachment_id: None, + current_attachment_name: None, + can_move: false, + allowed_parents: Vec::new(), + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }, + TopologyEditorNode { + node_id: "child-site".to_string(), + node_name: "Tuscany Ridge".to_string(), + current_parent_node_id: Some("site-west".to_string()), + current_parent_node_name: Some("WestRedd".to_string()), + current_attachment_id: Some("relay-b".to_string()), + current_attachment_name: Some("AVIAT_TuscanyRidge".to_string()), + can_move: true, + allowed_parents: vec![TopologyAllowedParent { + parent_node_id: "site-west".to_string(), + parent_node_name: "WestRedd".to_string(), + attachment_options: vec![sample_attachment_option( + "relay-b", + "AVIAT_TuscanyRidge", + )], + all_attachments_suppressed: false, + has_probe_unavailable_attachments: false, + }], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }, + ], + }; + + let mut canonical = TopologyCanonicalStateFile::from_editor_and_network( + &editor_state, + &json!({ + "AVIAT_WestRedd": { + "children": { + "AVIAT_TuscanyRidge": { + "children": { + "Tuscany Ridge": { + "children": {}, + "downloadBandwidthMbps": 900, + "id": "child-site", + "name": "Tuscany Ridge", + "type": "Site", + "uploadBandwidthMbps": 900 + } + }, + "downloadBandwidthMbps": 900, + "id": "relay-b", + "name": "AVIAT_TuscanyRidge", + "type": "AP", + "uploadBandwidthMbps": 900 + } + }, + "downloadBandwidthMbps": 1000, + "id": "relay-a", + "name": "AVIAT_WestRedd", + "type": "AP", + "uploadBandwidthMbps": 1000 + } + }), + TopologyCanonicalIngressKind::NativeIntegration, + ); + canonical.nodes.push(TopologyCanonicalNode { + node_id: "site-west".to_string(), + node_name: "WestRedd".to_string(), + node_kind: "Site".to_string(), + is_virtual: false, + current_parent_node_id: None, + current_parent_node_name: None, + current_attachment_id: None, + current_attachment_name: None, + can_move: false, + allowed_parents: Vec::new(), + rate_input: Default::default(), + }); + + let squashed = apply_effective_topology_to_canonical_state( + &config, + &canonical, + &editor_state, + &TopologyEffectiveStateFile { + schema_version: 1, + generated_unix: None, + canonical_generated_unix: None, + health_generated_unix: None, + nodes: vec![TopologyEffectiveNodeState { + node_id: "child-site".to_string(), + logical_parent_node_id: "site-west".to_string(), + preferred_attachment_id: Some("relay-b".to_string()), + effective_attachment_id: Some("relay-b".to_string()), + fallback_reason: None, + all_attachments_suppressed: false, + attachments: vec![TopologyEffectiveAttachmentState { + attachment_id: "relay-b".to_string(), + health_status: TopologyAttachmentHealthStatus::Healthy, + health_reason: None, + suppressed_until_unix: None, + probe_enabled: false, + probeable: false, + effective_selected: true, + }], + }], + }, + ); + + let root_children = squashed + .as_object() + .expect("native effective export should be an object"); + let west_children = root_children["WestRedd"]["children"] + .as_object() + .expect("WestRedd should stay as a logical root"); + assert!(root_children.get("AVIAT_WestRedd").is_none()); + assert!(west_children.get("AVIAT_WestRedd").is_none()); + assert!(west_children.get("Tuscany Ridge").is_some()); + } + #[test] fn runtime_squashing_respects_do_not_squash_sites() { let mut config = Config::default(); @@ -3818,6 +5031,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: Vec::new(), }; let effective = TopologyEffectiveStateFile::default(); @@ -3878,6 +5092,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "child-site".to_string(), node_name: "Child Site".to_string(), @@ -3937,6 +5152,85 @@ mod tests { assert!(parent_children.get("Child Site").is_none()); } + #[test] + fn effective_export_keeps_logical_children_without_explicit_attachment() { + let mut config = Config::default(); + config.uisp_integration.enable_uisp = true; + let canonical = json!({ + "Parent Site": { + "children": { + "Leaf AP": { + "children": {}, + "downloadBandwidthMbps": 150, + "id": "leaf-ap", + "name": "Leaf AP", + "parent_site": "Parent Site", + "type": "AP", + "uploadBandwidthMbps": 75 + } + }, + "downloadBandwidthMbps": 1000, + "id": "parent-site", + "name": "Parent Site", + "type": "Site", + "uploadBandwidthMbps": 1000 + } + }); + let ui_state = TopologyEditorStateFile { + schema_version: 1, + source: "uisp/full".to_string(), + generated_unix: None, + ingress_identity: None, + nodes: vec![TopologyEditorNode { + node_id: "leaf-ap".to_string(), + node_name: "Leaf AP".to_string(), + current_parent_node_id: Some("parent-site".to_string()), + current_parent_node_name: Some("Parent Site".to_string()), + current_attachment_id: Some("parent-site".to_string()), + current_attachment_name: Some("Parent Site".to_string()), + can_move: true, + allowed_parents: vec![TopologyAllowedParent { + parent_node_id: "parent-site".to_string(), + parent_node_name: "Parent Site".to_string(), + attachment_options: vec![auto_attachment_option()], + all_attachments_suppressed: false, + has_probe_unavailable_attachments: false, + }], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }], + }; + let effective = TopologyEffectiveStateFile { + schema_version: 1, + generated_unix: None, + canonical_generated_unix: None, + health_generated_unix: None, + nodes: vec![TopologyEffectiveNodeState { + node_id: "leaf-ap".to_string(), + logical_parent_node_id: "parent-site".to_string(), + preferred_attachment_id: None, + effective_attachment_id: None, + fallback_reason: None, + all_attachments_suppressed: false, + attachments: vec![], + }], + }; + + let exported = + apply_effective_topology_to_network_json(&config, &canonical, &ui_state, &effective); + let parent_children = exported["Parent Site"]["children"] + .as_object() + .expect("parent should keep children"); + assert!( + parent_children + .get("Leaf AP") + .and_then(Value::as_object) + .is_some() + ); + } + #[test] fn runtime_prunes_inactive_backhaul_attachment_stubs() { let mut config = Config::default(); @@ -4025,6 +5319,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "child-site".to_string(), node_name: "Child Site".to_string(), @@ -4136,6 +5431,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "site-beta".to_string(), node_name: "Site Beta".to_string(), @@ -4257,6 +5553,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: vec![ TopologyEditorNode { node_id: "site-beta".to_string(), @@ -4373,6 +5670,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "uisp:site:site-beta".to_string(), node_name: "Site Beta".to_string(), @@ -4423,6 +5721,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: vec![ TopologyEditorNode { node_id: "uisp:site:site-a".to_string(), @@ -4506,6 +5805,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "uisp:site:site-beta".to_string(), node_name: "Site Beta".to_string(), @@ -4570,6 +5870,203 @@ mod tests { ); } + #[test] + fn effective_topology_validation_accepts_fixed_parent_nodes_without_allowed_parents() { + let ui_state = TopologyEditorStateFile { + schema_version: 1, + source: "uisp/ap_site".to_string(), + generated_unix: None, + ingress_identity: None, + nodes: vec![ + TopologyEditorNode { + node_id: "site-root".to_string(), + node_name: "Site Root".to_string(), + current_parent_node_id: None, + current_parent_node_name: None, + current_attachment_id: None, + current_attachment_name: None, + can_move: false, + allowed_parents: vec![], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }, + TopologyEditorNode { + node_id: "ap-child".to_string(), + node_name: "AP Child".to_string(), + current_parent_node_id: Some("site-root".to_string()), + current_parent_node_name: Some("Site Root".to_string()), + current_attachment_id: Some("legacy-attachment".to_string()), + current_attachment_name: Some("Legacy Attachment".to_string()), + can_move: false, + allowed_parents: vec![], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }, + ], + }; + let effective = TopologyEffectiveStateFile { + schema_version: 1, + generated_unix: None, + canonical_generated_unix: None, + health_generated_unix: None, + nodes: vec![ + TopologyEffectiveNodeState { + node_id: "site-root".to_string(), + logical_parent_node_id: String::new(), + preferred_attachment_id: None, + effective_attachment_id: None, + fallback_reason: None, + all_attachments_suppressed: false, + attachments: vec![], + }, + TopologyEffectiveNodeState { + node_id: "ap-child".to_string(), + logical_parent_node_id: "site-root".to_string(), + preferred_attachment_id: Some("legacy-attachment".to_string()), + effective_attachment_id: Some("legacy-attachment".to_string()), + fallback_reason: None, + all_attachments_suppressed: false, + attachments: vec![], + }, + ], + }; + let exported = json!({ + "Site Root": { + "children": { + "AP Child": { + "children": {}, + "id": "ap-child", + "name": "AP Child", + "parent_site": "Site Root", + "type": "AP" + } + }, + "id": "site-root", + "name": "Site Root", + "type": "Site" + } + }); + + validate_effective_topology_network(&ui_state, &effective, &exported) + .expect("fixed-parent legacy nodes should validate"); + } + + #[test] + fn compute_effective_state_keeps_fixed_parent_nodes_without_allowed_parents() { + let config = Config::default(); + let canonical = TopologyEditorStateFile { + schema_version: 1, + source: "uisp/ap_site".to_string(), + generated_unix: None, + ingress_identity: None, + nodes: vec![ + TopologyEditorNode { + node_id: "site-root".to_string(), + node_name: "Site Root".to_string(), + current_parent_node_id: None, + current_parent_node_name: None, + current_attachment_id: None, + current_attachment_name: None, + can_move: false, + allowed_parents: vec![], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }, + TopologyEditorNode { + node_id: "ap-child".to_string(), + node_name: "AP Child".to_string(), + current_parent_node_id: Some("site-root".to_string()), + current_parent_node_name: Some("Site Root".to_string()), + current_attachment_id: Some("legacy-attachment".to_string()), + current_attachment_name: Some("Legacy Attachment".to_string()), + can_move: false, + allowed_parents: vec![], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }, + ], + }; + + let effective = compute_effective_state( + &config, + &canonical, + &TopologyOverridesFile::default(), + &TopologyAttachmentHealthStateFile::default(), + ); + + assert_eq!(effective.nodes.len(), 2); + let child = effective + .nodes + .iter() + .find(|node| node.node_id == "ap-child") + .expect("child node should remain in effective state"); + assert_eq!(child.logical_parent_node_id, "site-root"); + assert_eq!( + child.preferred_attachment_id.as_deref(), + Some("legacy-attachment") + ); + assert_eq!( + child.effective_attachment_id.as_deref(), + Some("legacy-attachment") + ); + assert!(child.attachments.is_empty()); + assert!(!child.all_attachments_suppressed); + } + + #[test] + fn compute_effective_state_does_not_infer_parent_for_native_integration_nodes() { + let config = Config::default(); + let canonical = TopologyEditorStateFile { + schema_version: 1, + source: "python/full".to_string(), + generated_unix: None, + ingress_identity: None, + nodes: vec![TopologyEditorNode { + node_id: "splynx:site:child".to_string(), + node_name: "Child Site".to_string(), + current_parent_node_id: None, + current_parent_node_name: None, + current_attachment_id: None, + current_attachment_name: None, + can_move: false, + allowed_parents: vec![TopologyAllowedParent { + parent_node_id: "splynx:site:parent".to_string(), + parent_node_name: "Parent Site".to_string(), + attachment_options: vec![], + all_attachments_suppressed: false, + has_probe_unavailable_attachments: false, + }], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }], + }; + + let effective = compute_effective_state( + &config, + &canonical, + &TopologyOverridesFile::default(), + &TopologyAttachmentHealthStateFile::default(), + ); + + assert_eq!(effective.nodes.len(), 1); + let child = &effective.nodes[0]; + assert_eq!(child.node_id, "splynx:site:child"); + assert!(child.logical_parent_node_id.is_empty()); + assert!(child.preferred_attachment_id.is_none()); + assert!(child.effective_attachment_id.is_none()); + assert!(child.attachments.is_empty()); + } + #[test] fn effective_state_fallback_does_not_keep_old_parent_attachment_after_reparent() { use lqos_overrides::TopologyOverridesFile; @@ -4579,6 +6076,7 @@ mod tests { schema_version: 1, source: "uisp/full2".to_string(), generated_unix: None, + ingress_identity: None, nodes: vec![TopologyEditorNode { node_id: "uisp:site:site-beta".to_string(), node_name: "Site Beta".to_string(), diff --git a/src/rust/lqos_topology_compile/Cargo.toml b/src/rust/lqos_topology_compile/Cargo.toml new file mode 100644 index 00000000..42b369e9 --- /dev/null +++ b/src/rust/lqos_topology_compile/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "lqos_topology_compile" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +lqos_config = { path = "../lqos_config" } diff --git a/src/rust/lqos_topology_compile/src/bundle.rs b/src/rust/lqos_topology_compile/src/bundle.rs new file mode 100644 index 00000000..cb035252 --- /dev/null +++ b/src/rust/lqos_topology_compile/src/bundle.rs @@ -0,0 +1,456 @@ +use lqos_config::{ + CircuitAnchorsFile, CircuitEthernetMetadata, Config, ConfigShapedDevices, ShapedDevice, + TopologyCanonicalStateFile, TopologyEditorStateFile, TopologyParentCandidatesFile, + topology_compiled_shaping_path, topology_import_path, topology_ingress_identity_from_tokens, +}; +use serde::{Deserialize, Serialize}; +use std::fs::File; +use std::io::Write; +use std::path::Path; + +fn default_topology_import_schema_version() -> u32 { + 1 +} + +fn atomic_write_json(path: &Path, value: &T) -> anyhow::Result<()> { + let raw = serde_json::to_string_pretty(value)?; + let temp_path = path.with_extension("tmp"); + let mut file = File::create(&temp_path)?; + file.write_all(raw.as_bytes())?; + file.sync_all()?; + std::fs::rename(&temp_path, path)?; + Ok(()) +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +struct ImportedTopologySnapshot { + source: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + generated_unix: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + ingress_identity: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + native_canonical: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + native_editor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + parent_candidates: Option, + #[serde(default)] + compatibility_network_json: serde_json::Value, + #[serde(default)] + shaped_devices: Vec, + #[serde(default)] + circuit_anchors: CircuitAnchorsFile, + #[serde(default)] + ethernet_advisories: Vec, +} + +impl ImportedTopologySnapshot { + fn from_bundle(imported: &ImportedTopologyBundle) -> Self { + Self { + source: imported.source.clone(), + generated_unix: imported.generated_unix, + ingress_identity: imported.ingress_identity(), + native_canonical: imported.native_canonical.clone(), + native_editor: imported.native_editor.clone(), + parent_candidates: imported.parent_candidates.clone(), + compatibility_network_json: imported.compatibility_network_json.clone(), + shaped_devices: imported.shaped_devices.devices.clone(), + circuit_anchors: imported.circuit_anchors.clone(), + ethernet_advisories: imported.ethernet_advisories.clone(), + } + } + + fn into_bundle(self) -> ImportedTopologyBundle { + let mut shaped_devices = ConfigShapedDevices::default(); + shaped_devices.replace_with_new_data(self.shaped_devices); + ImportedTopologyBundle { + source: self.source, + generated_unix: self.generated_unix, + ingress_identity: self.ingress_identity, + native_canonical: self.native_canonical, + native_editor: self.native_editor, + parent_candidates: self.parent_candidates, + compatibility_network_json: self.compatibility_network_json, + shaped_devices, + circuit_anchors: self.circuit_anchors, + ethernet_advisories: self.ethernet_advisories, + } + } +} + +/// Serialized compiler-selected shaping rows and circuit anchors for one integration mode. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TopologyCompiledShapingFile { + /// Schema version for compatibility checks. + #[serde(default = "default_topology_import_schema_version")] + pub schema_version: u32, + /// Human-readable compiler source such as `uisp/ap_only`. + #[serde(default)] + pub source: String, + /// Selected topology compile mode such as `ap_only`. + #[serde(default)] + pub compile_mode: String, + /// Generation timestamp when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub generated_unix: Option, + /// Stable identity of imported topology facts plus selected compile mode. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ingress_identity: Option, + #[serde(default)] + shaped_devices: Vec, + #[serde(default)] + circuit_anchors: CircuitAnchorsFile, +} + +impl TopologyCompiledShapingFile { + /// Builds a serialized compiled-shaping artifact from one compiled bundle. + /// + /// This function is pure: it has no side effects. + pub fn from_compiled( + compiled: &CompiledTopologyBundle, + compile_mode: impl Into, + ) -> Self { + Self { + schema_version: default_topology_import_schema_version(), + source: compiled.source.clone(), + compile_mode: compile_mode.into(), + generated_unix: compiled.generated_unix, + ingress_identity: compiled.ingress_identity.clone(), + shaped_devices: compiled.shaped_devices.devices.clone(), + circuit_anchors: compiled.circuit_anchors.clone(), + } + } + + /// Loads the compiled-shaping artifact when it exists. + /// + /// Side effects: reads `topology_compiled_shaping.json` from `config.lqos_directory`. + pub fn load(config: &Config) -> anyhow::Result> { + let path = topology_compiled_shaping_path(config); + if !path.exists() { + return Ok(None); + } + let raw = std::fs::read_to_string(path)?; + Ok(Some(serde_json::from_str(&raw)?)) + } + + /// Saves the serialized compiled-shaping artifact atomically. + /// + /// Side effects: writes `topology_compiled_shaping.json` into `config.lqos_directory`. + pub fn save(&self, config: &Config) -> anyhow::Result<()> { + atomic_write_json(&topology_compiled_shaping_path(config), self) + } + + /// Returns the shaping-device rows and circuit anchors for integration ingress. + /// + /// This function is pure: it has no side effects. + pub fn shaping_artifacts(self) -> (ConfigShapedDevices, Vec) { + let mut shaped_devices = ConfigShapedDevices::default(); + shaped_devices.replace_with_new_data(self.shaped_devices); + (shaped_devices, self.circuit_anchors.anchors) + } +} + +/// Serialized integration import artifact carrying imported facts before mode selection. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TopologyImportFile { + /// Schema version for compatibility checks. + #[serde(default = "default_topology_import_schema_version")] + pub schema_version: u32, + /// Human-readable importer source such as `uisp/full2`. + #[serde(default)] + pub source: String, + /// Selected topology compile mode such as `ap_only`. + #[serde(default)] + pub compile_mode: String, + /// Generation timestamp when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub generated_unix: Option, + /// Stable identity of imported topology facts plus selected compile mode. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ingress_identity: Option, + /// Imported topology facts before projection into a specific runtime shape. + imported: ImportedTopologySnapshot, +} + +/// Imported topology facts supplied by one integration adapter before mode selection. +pub struct ImportedTopologyBundle { + /// Human-readable source such as `uisp/full2` or `python/integration_common`. + pub source: String, + /// Generation timestamp when known. + pub generated_unix: Option, + /// Stable identity of the imported topology facts before mode projection. + pub ingress_identity: Option, + /// Rich canonical topology state, when the importer can provide it directly. + pub native_canonical: Option, + /// Rich editor/runtime topology state, when the importer can provide it directly. + pub native_editor: Option, + /// Legacy immediate-parent candidate snapshot, when available. + pub parent_candidates: Option, + /// Compatibility topology tree used by legacy readers. + pub compatibility_network_json: serde_json::Value, + /// Shaping-device rows associated with the imported topology. + pub shaped_devices: ConfigShapedDevices, + /// Circuit anchors associated with the imported topology. + pub circuit_anchors: CircuitAnchorsFile, + /// Ethernet advisories associated with the imported topology. + pub ethernet_advisories: Vec, +} + +/// Fully compiled topology outputs for one selected mode. +pub struct CompiledTopologyBundle { + /// Human-readable compiled source, such as `uisp/ap_site`. + pub source: String, + /// Generation timestamp when known. + pub generated_unix: Option, + /// Stable identity of the imported topology facts plus selected compile mode. + pub ingress_identity: Option, + /// Canonical topology state consumed by runtime compilation. + pub canonical: TopologyCanonicalStateFile, + /// Editor topology state presented to Topology Manager. + pub editor: TopologyEditorStateFile, + /// Legacy immediate-parent candidate snapshot kept for compatibility. + pub parent_candidates: TopologyParentCandidatesFile, + /// Compatibility topology tree emitted to `network.json`. + pub compatibility_network_json: serde_json::Value, + /// Shaping-device rows emitted to `ShapedDevices.csv`. + pub shaped_devices: ConfigShapedDevices, + /// Circuit anchors emitted to `circuit_anchors.json`. + pub circuit_anchors: CircuitAnchorsFile, + /// Ethernet advisories emitted to runtime metadata. + pub ethernet_advisories: Vec, +} + +impl ImportedTopologyBundle { + /// Returns the stable identity for this imported topology bundle. + /// + /// This function is pure: it has no side effects. + pub fn ingress_identity(&self) -> Option { + if let Some(identity) = self + .ingress_identity + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + return Some(identity.to_string()); + } + + let mut tokens = vec![format!("source:{}", self.source.trim())]; + if let Some(editor) = &self.native_editor { + tokens.extend( + editor + .nodes + .iter() + .map(|node| format!("editor-node:{}", node.node_id.trim())), + ); + } else if let Some(canonical) = &self.native_canonical { + tokens.extend( + canonical + .nodes + .iter() + .map(|node| format!("canonical-node:{}", node.node_id.trim())), + ); + } + if let Some(parent_candidates) = &self.parent_candidates { + tokens.extend( + parent_candidates + .nodes + .iter() + .map(|node| format!("parent-node:{}", node.node_id.trim())), + ); + } + tokens.extend(self.shaped_devices.devices.iter().flat_map(|device| { + [ + format!("circuit:{}", device.circuit_id.trim()), + format!("device:{}", device.device_id.trim()), + ] + })); + tokens.extend(self.circuit_anchors.anchors.iter().flat_map(|anchor| { + [ + format!("anchor-circuit:{}", anchor.circuit_id.trim()), + format!("anchor-node:{}", anchor.anchor_node_id.trim()), + ] + })); + topology_ingress_identity_from_tokens(tokens) + } + + /// Loads a legacy import bundle from the current runtime artifacts on disk. + /// + /// Side effects: reads `network.json`, `ShapedDevices.csv`, `circuit_anchors.json`, + /// `topology_canonical_state.json`, `topology_editor_state.json`, and + /// `topology_parent_candidates.json` from `config.lqos_directory` when they exist. + pub fn from_legacy_artifacts( + config: &Config, + source: impl Into, + ) -> anyhow::Result { + let network_path = Path::new(&config.lqos_directory).join("network.json"); + let compatibility_network_json = if network_path.exists() { + let raw = std::fs::read_to_string(&network_path)?; + serde_json::from_str(&raw)? + } else { + serde_json::Value::Object(serde_json::Map::new()) + }; + let native_canonical = { + let loaded = TopologyCanonicalStateFile::load(config)?; + (!loaded.nodes.is_empty()).then_some(loaded) + }; + let native_editor = { + let loaded = TopologyEditorStateFile::load(config)?; + (!loaded.nodes.is_empty()).then_some(loaded) + }; + let parent_candidates = { + let loaded = TopologyParentCandidatesFile::load(config)?; + (!loaded.nodes.is_empty()).then_some(loaded) + }; + Ok(Self { + source: source.into(), + generated_unix: native_canonical + .as_ref() + .and_then(|state| state.generated_unix) + .or(native_editor + .as_ref() + .and_then(|state| state.generated_unix)), + ingress_identity: native_canonical + .as_ref() + .and_then(|state| state.ingress_identity.clone()) + .or(native_editor + .as_ref() + .and_then(|state| state.ingress_identity.clone())) + .or(parent_candidates + .as_ref() + .and_then(|state| state.ingress_identity.clone())), + native_canonical, + native_editor, + parent_candidates, + compatibility_network_json, + shaped_devices: ConfigShapedDevices::load_for_config(config)?, + circuit_anchors: CircuitAnchorsFile::load(config)?, + ethernet_advisories: Vec::new(), + }) + } +} + +impl TopologyImportFile { + /// Builds a serialized import artifact from one imported bundle and selected compile mode. + /// + /// This function is pure: it has no side effects. + pub fn from_imported_bundle( + imported: &ImportedTopologyBundle, + compile_mode: impl Into, + ) -> Self { + let compile_mode = compile_mode.into(); + let imported_identity = imported.ingress_identity(); + Self { + schema_version: default_topology_import_schema_version(), + source: imported.source.clone(), + compile_mode: compile_mode.clone(), + generated_unix: imported.generated_unix, + ingress_identity: imported_identity.map(|identity| { + topology_ingress_identity_from_tokens([ + format!("import:{}", identity.trim()), + format!("mode:{}", compile_mode.trim()), + ]) + .unwrap_or(identity) + }), + imported: ImportedTopologySnapshot::from_bundle(imported), + } + } + + /// Loads the serialized topology import artifact when it exists. + /// + /// Side effects: reads `topology_import.json` from `config.lqos_directory`. + pub fn load(config: &Config) -> anyhow::Result> { + let path = topology_import_path(config); + if !path.exists() { + return Ok(None); + } + let raw = std::fs::read_to_string(path)?; + Ok(Some(serde_json::from_str(&raw)?)) + } + + /// Reconstructs the in-memory imported topology bundle from the serialized snapshot. + /// + /// This function is pure: it has no side effects. + pub fn into_imported_bundle(self) -> ImportedTopologyBundle { + self.imported.into_bundle() + } + + /// Saves the serialized topology import artifact atomically. + /// + /// Side effects: writes `topology_import.json` into `config.lqos_directory`. + pub fn save(&self, config: &Config) -> anyhow::Result<()> { + atomic_write_json(&topology_import_path(config), self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn imported_bundle_with_identity(identity: &str) -> ImportedTopologyBundle { + ImportedTopologyBundle { + source: "test/import".to_string(), + generated_unix: Some(123), + ingress_identity: Some(identity.to_string()), + native_canonical: None, + native_editor: None, + parent_candidates: None, + compatibility_network_json: json!({}), + shaped_devices: ConfigShapedDevices::default(), + circuit_anchors: CircuitAnchorsFile::default(), + ethernet_advisories: Vec::new(), + } + } + + #[test] + fn topology_import_identity_matches_compiled_identity_contract() { + let imported = imported_bundle_with_identity("import-base"); + let topology_import = TopologyImportFile::from_imported_bundle(&imported, "full"); + let expected = topology_ingress_identity_from_tokens([ + "import:import-base".to_string(), + "mode:full".to_string(), + ]) + .expect("expected ingress identity"); + assert_eq!( + topology_import.ingress_identity.as_deref(), + Some(expected.as_str()) + ); + } + + #[test] + fn compiled_shaping_file_round_trips_compiled_artifacts() { + let file = TopologyCompiledShapingFile { + schema_version: 1, + source: "test/full".to_string(), + compile_mode: "full".to_string(), + generated_unix: Some(123), + ingress_identity: Some("identity".to_string()), + shaped_devices: vec![ShapedDevice { + circuit_id: "circuit-1".to_string(), + circuit_name: "Circuit 1".to_string(), + device_id: "device-1".to_string(), + device_name: "Device 1".to_string(), + parent_node: "Tower 1".to_string(), + parent_node_id: Some("tower-1".to_string()), + anchor_node_id: None, + mac: String::new(), + ipv4: Vec::new(), + ipv6: Vec::new(), + download_min_mbps: 10.0, + upload_min_mbps: 10.0, + download_max_mbps: 100.0, + upload_max_mbps: 100.0, + comment: String::new(), + sqm_override: None, + circuit_hash: 0, + device_hash: 0, + parent_hash: 0, + }], + circuit_anchors: CircuitAnchorsFile::default(), + }; + let (shaped_devices, anchors) = file.shaping_artifacts(); + assert_eq!(shaped_devices.devices.len(), 1); + assert!(anchors.is_empty()); + } +} diff --git a/src/rust/lqos_topology_compile/src/compile_mode.rs b/src/rust/lqos_topology_compile/src/compile_mode.rs new file mode 100644 index 00000000..9bbf9b14 --- /dev/null +++ b/src/rust/lqos_topology_compile/src/compile_mode.rs @@ -0,0 +1,1501 @@ +use crate::bundle::{CompiledTopologyBundle, ImportedTopologyBundle}; +use crate::validation::validate_compiled_bundle; +use anyhow::Result; +use lqos_config::{ + CircuitAnchor, CircuitAnchorsFile, ConfigShapedDevices, ShapedDevice, + TopologyCanonicalIngressKind, TopologyCanonicalNode, TopologyCanonicalStateFile, + TopologyEditorNode, TopologyEditorStateFile, TopologyParentCandidatesFile, + topology_ingress_identity_from_tokens, +}; +use serde_json::{Map, Number, Value}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use thiserror::Error; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// Selects which topology projection to compile from the imported bundle. +pub enum TopologyCompileMode { + /// Preserve the richest imported topology representation. + Full2, + /// Preserve the legacy full topology representation. + Full, + /// Project to a site -> AP tree. + ApSite, + /// Project to AP-only nodes. + ApOnly, + /// Collapse all topology structure. + Flat, +} + +#[derive(Debug, Error)] +/// Errors returned while compiling one topology mode. +pub enum TopologyCompileError { + /// The imported topology bundle was missing required facts. + #[error("Unable to compile topology mode: {0}")] + Compile(String), +} + +#[derive(Clone, Debug)] +struct ExportedNode { + id: String, + name: String, + kind: String, + parent_id: Option, + download_mbps: Option, + upload_mbps: Option, +} + +#[derive(Clone)] +struct CircuitProjection { + node_id: String, + node_name: String, +} + +const GENERATED_UNATTACHED_SITE_ID: &str = "libreqos:generated:site:unattached"; +const GENERATED_UNATTACHED_SITE_NAME: &str = "LibreQoS Unattached [Site]"; +const GENERATED_UNATTACHED_AP_ID: &str = "libreqos:generated:ap:unattached"; +const GENERATED_UNATTACHED_AP_NAME: &str = "LibreQoS Unattached [AP]"; + +fn empty_parent_candidates(source: &str) -> TopologyParentCandidatesFile { + TopologyParentCandidatesFile { + source: source.to_string(), + ingress_identity: None, + nodes: Vec::new(), + } +} + +fn mode_name(mode: TopologyCompileMode) -> &'static str { + match mode { + TopologyCompileMode::Full2 => "full2", + TopologyCompileMode::Full => "full", + TopologyCompileMode::ApSite => "ap_site", + TopologyCompileMode::ApOnly => "ap_only", + TopologyCompileMode::Flat => "flat", + } +} + +fn compiled_source(import_source: &str, mode: TopologyCompileMode) -> String { + let prefix = import_source + .split('/') + .next() + .filter(|value| !value.is_empty()) + .unwrap_or("compiled"); + let mode_name = match mode { + TopologyCompileMode::Full2 => "full2", + TopologyCompileMode::Full => "full", + TopologyCompileMode::ApSite => "ap_site", + TopologyCompileMode::ApOnly => "ap_only", + TopologyCompileMode::Flat => "flat", + }; + format!("{prefix}/{mode_name}") +} + +fn compiled_ingress_identity( + imported: &ImportedTopologyBundle, + mode: TopologyCompileMode, +) -> Option { + let base = imported.ingress_identity()?; + topology_ingress_identity_from_tokens([ + format!("import:{base}"), + format!("mode:{}", mode_name(mode)), + ]) +} + +fn u64_field(node: &Map, key: &str) -> Option { + match node.get(key) { + Some(Value::Number(number)) => number + .as_u64() + .or_else(|| number.as_f64().map(|value| value.round() as u64)), + _ => None, + } +} + +fn collect_exported_nodes( + map: &Map, + parent: Option<(&str, &str)>, + by_id: &mut HashMap, + by_name: &mut HashMap, +) { + for (key, value) in map { + let Some(node) = value.as_object() else { + continue; + }; + let Some(node_id) = node + .get("id") + .and_then(Value::as_str) + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + else { + continue; + }; + let node_name = node + .get("name") + .and_then(Value::as_str) + .unwrap_or(key) + .to_string(); + let node_kind = node + .get("type") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + by_name + .entry(node_name.clone()) + .or_insert_with(|| node_id.to_string()); + by_id.insert( + node_id.to_string(), + ExportedNode { + id: node_id.to_string(), + name: node_name.clone(), + kind: node_kind, + parent_id: parent.map(|entry| entry.0.to_string()), + download_mbps: u64_field(node, "downloadBandwidthMbps"), + upload_mbps: u64_field(node, "uploadBandwidthMbps"), + }, + ); + if let Some(children) = node.get("children").and_then(Value::as_object) { + collect_exported_nodes( + children, + Some((node_id, node_name.as_str())), + by_id, + by_name, + ); + } + } +} + +fn exported_index( + network_json: &Value, +) -> (HashMap, HashMap) { + let mut by_id = HashMap::new(); + let mut by_name = HashMap::new(); + if let Some(map) = network_json.as_object() { + collect_exported_nodes(map, None, &mut by_id, &mut by_name); + } + (by_id, by_name) +} + +fn canonical_node_rates(node: &TopologyCanonicalNode) -> (Option, Option) { + ( + node.rate_input + .intrinsic_download_mbps + .or(node.rate_input.legacy_imported_download_mbps), + node.rate_input + .intrinsic_upload_mbps + .or(node.rate_input.legacy_imported_upload_mbps), + ) +} + +fn native_projection_index( + imported: &ImportedTopologyBundle, +) -> (HashMap, HashMap) { + let Some(canonical) = imported.native_canonical.as_ref() else { + return exported_index(&imported.compatibility_network_json); + }; + + let mut by_id = HashMap::new(); + let mut by_name = HashMap::new(); + for node in &canonical.nodes { + let (download_mbps, upload_mbps) = canonical_node_rates(node); + let exported = ExportedNode { + id: node.node_id.clone(), + name: node.node_name.clone(), + kind: node.node_kind.clone(), + parent_id: node.current_parent_node_id.clone(), + download_mbps, + upload_mbps, + }; + by_name + .entry(exported.name.clone()) + .or_insert_with(|| exported.id.clone()); + by_id.insert(exported.id.clone(), exported); + } + + if by_id.is_empty() { + exported_index(&imported.compatibility_network_json) + } else { + (by_id, by_name) + } +} + +fn nearest_kind( + nodes_by_id: &HashMap, + start_id: &str, + expected_kind: &str, +) -> Option { + let mut current = Some(start_id); + let mut seen = HashSet::new(); + while let Some(candidate) = current { + if !seen.insert(candidate.to_string()) { + return None; + } + let node = nodes_by_id.get(candidate)?; + if node.kind.eq_ignore_ascii_case(expected_kind) { + return Some(node.id.clone()); + } + current = node.parent_id.as_deref(); + } + None +} + +fn nearest_site_parent( + nodes_by_id: &HashMap, + start_id: &str, +) -> Option { + let mut current = nodes_by_id.get(start_id)?.parent_id.clone(); + let mut seen = HashSet::new(); + while let Some(candidate) = current { + if !seen.insert(candidate.clone()) { + return None; + } + let node = nodes_by_id.get(&candidate)?; + if node.kind.eq_ignore_ascii_case("site") { + return Some(node.id.clone()); + } + current = node.parent_id.clone(); + } + None +} + +fn resolve_reference( + parent_node_id: Option<&str>, + parent_node_name: &str, + nodes_by_id: &HashMap, + ids_by_name: &HashMap, +) -> Option { + if let Some(parent_id) = parent_node_id + .map(str::trim) + .filter(|value| !value.is_empty()) + && nodes_by_id.contains_key(parent_id) + { + return Some(parent_id.to_string()); + } + let trimmed_name = parent_node_name.trim(); + if trimmed_name.is_empty() { + None + } else { + ids_by_name.get(trimmed_name).cloned() + } +} + +fn circuit_anchor_map(anchors: &CircuitAnchorsFile) -> HashMap { + anchors + .anchors + .iter() + .map(|anchor| (anchor.circuit_id.clone(), anchor.clone())) + .collect() +} + +fn circuit_projection_map( + imported: &ImportedTopologyBundle, + transform: F, +) -> HashMap +where + F: Fn(&HashMap, &str) -> Option, +{ + let (nodes_by_id, ids_by_name) = native_projection_index(imported); + let anchors = circuit_anchor_map(&imported.circuit_anchors); + let mut circuits = HashMap::::new(); + for device in &imported.shaped_devices.devices { + let anchor = anchors.get(&device.circuit_id); + let candidate = anchor + .and_then(|anchor| { + resolve_reference( + Some(anchor.anchor_node_id.as_str()), + anchor.anchor_node_name.as_deref().unwrap_or_default(), + &nodes_by_id, + &ids_by_name, + ) + }) + .or_else(|| { + resolve_reference( + device.parent_node_id.as_deref(), + &device.parent_node, + &nodes_by_id, + &ids_by_name, + ) + }); + let Some(target_id) = candidate.and_then(|node_id| transform(&nodes_by_id, &node_id)) + else { + continue; + }; + let Some(target) = nodes_by_id.get(&target_id) else { + continue; + }; + circuits + .entry(device.circuit_id.clone()) + .or_insert_with(|| CircuitProjection { + node_id: target.id.clone(), + node_name: target.name.clone(), + }); + } + circuits +} + +fn unresolved_circuits( + imported: &ImportedTopologyBundle, + projection: &HashMap, +) -> Vec<(String, String)> { + let mut unresolved = Vec::new(); + let mut seen = HashSet::new(); + for device in &imported.shaped_devices.devices { + if projection.contains_key(&device.circuit_id) || !seen.insert(device.circuit_id.clone()) { + continue; + } + unresolved.push((device.circuit_id.clone(), device.circuit_name.clone())); + } + unresolved +} + +fn generated_ap_node() -> ExportedNode { + ExportedNode { + id: GENERATED_UNATTACHED_AP_ID.to_string(), + name: GENERATED_UNATTACHED_AP_NAME.to_string(), + kind: "AP".to_string(), + parent_id: None, + download_mbps: None, + upload_mbps: None, + } +} + +fn generated_site_node() -> ExportedNode { + ExportedNode { + id: GENERATED_UNATTACHED_SITE_ID.to_string(), + name: GENERATED_UNATTACHED_SITE_NAME.to_string(), + kind: "Site".to_string(), + parent_id: None, + download_mbps: None, + upload_mbps: None, + } +} + +fn exported_unattached_fallback_node( + nodes_by_id: &HashMap, +) -> Option { + let mut preferred_site = None; + let mut preferred_ap = None; + for node in nodes_by_id.values() { + let id = node.id.to_ascii_lowercase(); + let name = node.name.to_ascii_lowercase(); + let is_unattached = id.contains("unattach") + || name.contains("unattach") + || id.contains("orphan") + || name.contains("orphan"); + if !is_unattached { + continue; + } + if node.kind.eq_ignore_ascii_case("site") { + preferred_site = Some(node.clone()); + break; + } + if preferred_ap.is_none() && node.kind.eq_ignore_ascii_case("ap") { + preferred_ap = Some(node.clone()); + } + } + preferred_site.or(preferred_ap) +} + +fn attach_unresolved_to_projection( + projection: &mut HashMap, + unresolved: Vec<(String, String)>, + fallback_node: &ExportedNode, +) { + for (circuit_id, _circuit_name) in unresolved { + projection.insert( + circuit_id, + CircuitProjection { + node_id: fallback_node.id.clone(), + node_name: fallback_node.name.clone(), + }, + ); + } +} + +fn set_number(map: &mut Map, key: &str, value: Option, fallback: u64) { + map.insert( + key.to_string(), + Value::Number(Number::from(value.unwrap_or(fallback))), + ); +} + +fn ap_object(node: &ExportedNode, parent_site: Option<&str>) -> Value { + let mut out = Map::new(); + out.insert("children".to_string(), Value::Object(Map::new())); + out.insert("id".to_string(), node.id.clone().into()); + out.insert("name".to_string(), node.name.clone().into()); + out.insert("type".to_string(), "AP".into()); + set_number(&mut out, "downloadBandwidthMbps", node.download_mbps, 1); + set_number(&mut out, "uploadBandwidthMbps", node.upload_mbps, 1); + if let Some(parent_site) = parent_site.filter(|value| !value.is_empty()) { + out.insert("parent_site".to_string(), parent_site.to_string().into()); + } + Value::Object(out) +} + +fn site_object( + node_id: String, + node_name: String, + download_mbps: Option, + upload_mbps: Option, + children: Map, +) -> Value { + let mut out = Map::new(); + out.insert("children".to_string(), Value::Object(children)); + out.insert("id".to_string(), node_id.into()); + out.insert("name".to_string(), node_name.into()); + out.insert("type".to_string(), "Site".into()); + set_number(&mut out, "downloadBandwidthMbps", download_mbps, 1); + set_number(&mut out, "uploadBandwidthMbps", upload_mbps, 1); + Value::Object(out) +} + +fn make_shaped_devices(devices: Vec) -> ConfigShapedDevices { + let mut shaped_devices = ConfigShapedDevices::default(); + shaped_devices.replace_with_new_data(devices); + shaped_devices +} + +struct SanitizedTopologyIndex { + node_names_by_id: HashMap, + parent_by_node_id: HashMap>, +} + +fn sanitized_topology_index(imported: &ImportedTopologyBundle) -> SanitizedTopologyIndex { + if let Some(native) = imported.native_canonical.as_ref() { + return SanitizedTopologyIndex { + node_names_by_id: native + .nodes + .iter() + .map(|node| (node.node_id.clone(), node.node_name.clone())) + .collect(), + parent_by_node_id: native + .nodes + .iter() + .map(|node| (node.node_id.clone(), node.current_parent_node_id.clone())) + .collect(), + }; + } + if let Some(native) = imported.native_editor.as_ref() { + return SanitizedTopologyIndex { + node_names_by_id: native + .nodes + .iter() + .map(|node| (node.node_id.clone(), node.node_name.clone())) + .collect(), + parent_by_node_id: native + .nodes + .iter() + .map(|node| (node.node_id.clone(), node.current_parent_node_id.clone())) + .collect(), + }; + } + + let (nodes_by_id, _) = exported_index(&imported.compatibility_network_json); + SanitizedTopologyIndex { + node_names_by_id: nodes_by_id + .iter() + .map(|(node_id, node)| (node_id.clone(), node.name.clone())) + .collect(), + parent_by_node_id: nodes_by_id + .iter() + .map(|(node_id, node)| (node_id.clone(), node.parent_id.clone())) + .collect(), + } +} + +fn sanitize_editor_nodes( + nodes: Vec, + topology_index: &SanitizedTopologyIndex, +) -> Vec { + nodes + .into_iter() + .filter_map(|mut node| { + if node.node_id.starts_with("libreqos:generated:graph:") { + return None; + } + let exported_name = topology_index.node_names_by_id.get(&node.node_id)?; + node.allowed_parents.retain(|parent| { + topology_index + .node_names_by_id + .contains_key(parent.parent_node_id.as_str()) + }); + let legal_current_parent = + node.current_parent_node_id + .as_deref() + .and_then(|parent_id| { + if (node.allowed_parents.is_empty() + && topology_index.node_names_by_id.contains_key(parent_id)) + || node + .allowed_parents + .iter() + .any(|parent| parent.parent_node_id == parent_id) + { + Some(parent_id.to_string()) + } else { + None + } + }); + let exported_parent = topology_index + .parent_by_node_id + .get(&node.node_id) + .and_then(|parent| parent.as_deref()) + .and_then(|parent_id| { + if (node.allowed_parents.is_empty() + && topology_index.node_names_by_id.contains_key(parent_id)) + || node + .allowed_parents + .iter() + .any(|parent| parent.parent_node_id == parent_id) + { + Some(parent_id.to_string()) + } else { + None + } + }); + let selected_parent = legal_current_parent.or(exported_parent); + node.current_parent_node_name = selected_parent.as_deref().and_then(|parent_id| { + node.allowed_parents + .iter() + .find(|parent| parent.parent_node_id == parent_id) + .map(|parent| parent.parent_node_name.clone()) + .or_else(|| topology_index.node_names_by_id.get(parent_id).cloned()) + }); + node.current_parent_node_id = selected_parent; + + if node + .current_attachment_id + .as_deref() + .is_some_and(|attachment_id| { + !topology_index.node_names_by_id.contains_key(attachment_id) + }) + { + node.current_attachment_id = None; + node.current_attachment_name = None; + } + + node.node_name = exported_name.clone(); + Some(node) + }) + .collect() +} + +fn sanitized_editor_state( + source: &str, + generated_unix: Option, + imported: &ImportedTopologyBundle, +) -> TopologyEditorStateFile { + let editor = imported.native_editor.clone().unwrap_or_else(|| { + imported + .native_canonical + .clone() + .unwrap_or_else(|| { + TopologyCanonicalStateFile::from_legacy_network_json( + &imported.compatibility_network_json, + ) + }) + .to_editor_state() + }); + let editor = merge_legacy_parent_candidates(editor, imported.parent_candidates.as_ref()); + let topology_index = sanitized_topology_index(imported); + + TopologyEditorStateFile { + source: source.to_string(), + generated_unix, + nodes: sanitize_editor_nodes(editor.nodes, &topology_index), + ..editor + } +} + +fn merge_legacy_parent_candidates( + mut editor: TopologyEditorStateFile, + parent_candidates: Option<&TopologyParentCandidatesFile>, +) -> TopologyEditorStateFile { + let Some(parent_candidates) = parent_candidates else { + return editor; + }; + let legacy_editor = TopologyEditorStateFile::from_legacy_parent_candidates(parent_candidates); + let legacy_by_id = legacy_editor + .nodes + .into_iter() + .map(|node| (node.node_id.clone(), node)) + .collect::>(); + + for node in &mut editor.nodes { + let Some(legacy) = legacy_by_id.get(&node.node_id) else { + continue; + }; + if node.current_parent_node_id.is_none() { + node.current_parent_node_id = legacy.current_parent_node_id.clone(); + node.current_parent_node_name = legacy.current_parent_node_name.clone(); + } + if !node.can_move && node.allowed_parents.is_empty() { + node.can_move = legacy.can_move; + node.allowed_parents = legacy.allowed_parents.clone(); + } + } + + editor +} + +fn projected_outputs( + imported: ImportedTopologyBundle, + projection: HashMap, + network_json: Value, + source: String, + mode: TopologyCompileMode, +) -> CompiledTopologyBundle { + let ingress_identity = compiled_ingress_identity(&imported, mode); + let mut shaped_devices = Vec::with_capacity(imported.shaped_devices.devices.len()); + let circuit_names = imported + .shaped_devices + .devices + .iter() + .map(|device| (device.circuit_id.clone(), device.circuit_name.clone())) + .collect::>(); + for mut device in imported.shaped_devices.devices { + if let Some(parent) = projection.get(&device.circuit_id) { + device.parent_node = parent.node_name.clone(); + device.parent_node_id = Some(parent.node_id.clone()); + } else { + device.parent_node.clear(); + device.parent_node_id = None; + } + device.anchor_node_id = None; + shaped_devices.push(device); + } + + let mut anchors = projection + .into_iter() + .map(|(circuit_id, parent)| CircuitAnchor { + circuit_name: circuit_names.get(&circuit_id).cloned(), + circuit_id, + anchor_node_id: parent.node_id, + anchor_node_name: Some(parent.node_name), + }) + .collect::>(); + anchors.sort_unstable_by(|left, right| left.circuit_id.cmp(&right.circuit_id)); + anchors.dedup_by(|left, right| left.circuit_id == right.circuit_id); + + let editor = + TopologyCanonicalStateFile::from_legacy_network_json(&network_json).to_editor_state(); + let mut canonical = TopologyCanonicalStateFile::from_legacy_network_json(&network_json); + canonical.source = source.clone(); + canonical.generated_unix = imported.generated_unix; + canonical.ingress_identity = ingress_identity.clone(); + canonical.ingress_kind = TopologyCanonicalIngressKind::NativeIntegration; + let mut editor = editor; + editor.source = source.clone(); + editor.generated_unix = imported.generated_unix; + editor.ingress_identity = ingress_identity.clone(); + + CompiledTopologyBundle { + source: source.clone(), + generated_unix: imported.generated_unix, + ingress_identity: ingress_identity.clone(), + canonical, + editor, + parent_candidates: TopologyParentCandidatesFile { + source: source.clone(), + ingress_identity, + nodes: Vec::new(), + }, + compatibility_network_json: network_json, + shaped_devices: make_shaped_devices(shaped_devices), + circuit_anchors: CircuitAnchorsFile { + schema_version: 1, + source, + generated_unix: imported.generated_unix, + anchors, + }, + ethernet_advisories: imported.ethernet_advisories, + } +} + +fn compile_full_like( + imported: ImportedTopologyBundle, + mode: TopologyCompileMode, +) -> Result { + let source = compiled_source(&imported.source, mode); + let ingress_identity = compiled_ingress_identity(&imported, mode); + let (exported_nodes_by_id, _) = exported_index(&imported.compatibility_network_json); + let unresolved_fallback = exported_unattached_fallback_node(&exported_nodes_by_id); + let mut editor = sanitized_editor_state(&source, imported.generated_unix, &imported); + editor.ingress_identity = ingress_identity.clone(); + let mut canonical = TopologyCanonicalStateFile::from_editor_and_network( + &editor, + &imported.compatibility_network_json, + TopologyCanonicalIngressKind::NativeIntegration, + ); + canonical.source = source.clone(); + canonical.generated_unix = imported.generated_unix; + canonical.ingress_identity = ingress_identity.clone(); + let mut parent_candidates = imported + .parent_candidates + .unwrap_or_else(|| empty_parent_candidates(&source)); + parent_candidates.source = source.clone(); + parent_candidates.ingress_identity = ingress_identity.clone(); + let mut shaped_devices = imported.shaped_devices.devices; + if let Some(fallback) = unresolved_fallback { + for device in &mut shaped_devices { + let missing_parent = device.parent_node.trim().is_empty() + && device.parent_node_id.is_none() + && device.anchor_node_id.is_none(); + if missing_parent { + device.parent_node = fallback.name.clone(); + device.parent_node_id = Some(fallback.id.clone()); + } + } + } + Ok(CompiledTopologyBundle { + source: source.clone(), + generated_unix: imported.generated_unix, + ingress_identity, + canonical, + editor, + parent_candidates, + compatibility_network_json: imported.compatibility_network_json, + shaped_devices: make_shaped_devices(shaped_devices), + circuit_anchors: CircuitAnchorsFile { + schema_version: imported.circuit_anchors.schema_version, + source, + generated_unix: imported.generated_unix, + anchors: imported.circuit_anchors.anchors, + }, + ethernet_advisories: imported.ethernet_advisories, + }) +} + +fn compile_flat(imported: ImportedTopologyBundle) -> CompiledTopologyBundle { + let source = compiled_source(&imported.source, TopologyCompileMode::Flat); + projected_outputs( + imported, + HashMap::new(), + Value::Object(Map::new()), + source, + TopologyCompileMode::Flat, + ) +} + +fn compile_ap_only(imported: ImportedTopologyBundle) -> Result { + let source = compiled_source(&imported.source, TopologyCompileMode::ApOnly); + let mut projection = circuit_projection_map(&imported, |nodes_by_id, node_id| { + nearest_kind(nodes_by_id, node_id, "AP") + }); + let unresolved = unresolved_circuits(&imported, &projection); + let (nodes_by_id, _) = native_projection_index(&imported); + let mut root = BTreeMap::::new(); + let mut seen = HashSet::new(); + for parent in projection.values() { + if !seen.insert(parent.node_id.clone()) { + continue; + } + let Some(node) = nodes_by_id.get(&parent.node_id) else { + continue; + }; + root.insert(node.name.clone(), ap_object(node, None)); + } + if !unresolved.is_empty() { + let fallback_ap = generated_ap_node(); + attach_unresolved_to_projection(&mut projection, unresolved, &fallback_ap); + root.insert(fallback_ap.name.clone(), ap_object(&fallback_ap, None)); + } + Ok(projected_outputs( + imported, + projection, + Value::Object(root.into_iter().collect()), + source, + TopologyCompileMode::ApOnly, + )) +} + +fn compile_ap_site(imported: ImportedTopologyBundle) -> Result { + let source = compiled_source(&imported.source, TopologyCompileMode::ApSite); + let mut projection = circuit_projection_map(&imported, |nodes_by_id, node_id| { + nearest_kind(nodes_by_id, node_id, "AP") + }); + let unresolved = unresolved_circuits(&imported, &projection); + let (nodes_by_id, _) = native_projection_index(&imported); + let mut groups = BTreeMap::)>::new(); + let mut synthetic_counter = 0usize; + let mut seen = HashSet::new(); + for parent in projection.values() { + if !seen.insert(parent.node_id.clone()) { + continue; + } + let Some(ap) = nodes_by_id.get(&parent.node_id).cloned() else { + continue; + }; + let site_entry = nearest_site_parent(&nodes_by_id, &ap.id) + .and_then(|site_id| nodes_by_id.get(&site_id).cloned()) + .unwrap_or_else(|| { + synthetic_counter += 1; + ExportedNode { + id: format!("libreqos:generated:ap_site:{synthetic_counter}"), + name: ap.name.clone(), + kind: "Site".to_string(), + parent_id: None, + download_mbps: ap.download_mbps, + upload_mbps: ap.upload_mbps, + } + }); + let entry = groups + .entry(site_entry.id.clone()) + .or_insert_with(|| (site_entry.clone(), BTreeMap::new())); + entry.1.insert( + ap.name.clone(), + ap_object(&ap, Some(site_entry.name.as_str())), + ); + } + if !unresolved.is_empty() { + let fallback_site = generated_site_node(); + let fallback_ap = ExportedNode { + parent_id: Some(fallback_site.id.clone()), + ..generated_ap_node() + }; + attach_unresolved_to_projection(&mut projection, unresolved, &fallback_ap); + let entry = groups + .entry(fallback_site.id.clone()) + .or_insert_with(|| (fallback_site.clone(), BTreeMap::new())); + entry.1.insert( + fallback_ap.name.clone(), + ap_object(&fallback_ap, Some(fallback_site.name.as_str())), + ); + } + let mut root = BTreeMap::::new(); + for (_site_id, (site, children)) in groups { + root.insert( + site.name.clone(), + site_object( + site.id, + site.name, + site.download_mbps, + site.upload_mbps, + children.into_iter().collect(), + ), + ); + } + Ok(projected_outputs( + imported, + projection, + Value::Object(root.into_iter().collect()), + source, + TopologyCompileMode::ApSite, + )) +} + +/// Compiles one selectable topology mode from imported topology facts. +pub fn compile_topology( + imported: ImportedTopologyBundle, + mode: TopologyCompileMode, +) -> Result { + let bundle = match mode { + TopologyCompileMode::Full2 => compile_full_like(imported, TopologyCompileMode::Full2), + TopologyCompileMode::Full => compile_full_like(imported, TopologyCompileMode::Full), + TopologyCompileMode::ApOnly => compile_ap_only(imported), + TopologyCompileMode::ApSite => compile_ap_site(imported), + TopologyCompileMode::Flat => Ok(compile_flat(imported)), + } + .map_err(|err| TopologyCompileError::Compile(err.to_string()))?; + validate_compiled_bundle(&bundle) + .map_err(|err| TopologyCompileError::Compile(err.to_string()))?; + Ok(bundle) +} + +#[cfg(test)] +mod tests { + use super::{ + GENERATED_UNATTACHED_AP_ID, GENERATED_UNATTACHED_AP_NAME, GENERATED_UNATTACHED_SITE_NAME, + TopologyCompileMode, compile_topology, + }; + use crate::bundle::ImportedTopologyBundle; + use lqos_config::{ + CircuitAnchor, CircuitAnchorsFile, ConfigShapedDevices, ShapedDevice, + TopologyAllowedParent, TopologyAttachmentHealthStatus, TopologyAttachmentOption, + TopologyAttachmentRateSource, TopologyAttachmentRole, TopologyCanonicalIngressKind, + TopologyCanonicalStateFile, TopologyEditorNode, TopologyEditorStateFile, + TopologyParentCandidate, TopologyParentCandidatesFile, TopologyParentCandidatesNode, + }; + use serde_json::json; + + fn shaped_devices(rows: Vec) -> ConfigShapedDevices { + let mut shaped = ConfigShapedDevices::default(); + shaped.replace_with_new_data(rows); + shaped + } + + fn sample_device(circuit_id: &str, parent_name: &str, parent_id: Option<&str>) -> ShapedDevice { + ShapedDevice { + circuit_id: circuit_id.to_string(), + circuit_name: format!("Circuit {circuit_id}"), + device_id: format!("device-{circuit_id}"), + device_name: format!("Device {circuit_id}"), + parent_node: parent_name.to_string(), + parent_node_id: parent_id.map(ToOwned::to_owned), + anchor_node_id: None, + mac: String::new(), + ipv4: Vec::new(), + ipv6: Vec::new(), + download_min_mbps: 10.0, + upload_min_mbps: 10.0, + download_max_mbps: 20.0, + upload_max_mbps: 20.0, + comment: String::new(), + sqm_override: None, + circuit_hash: 0, + device_hash: 0, + parent_hash: 0, + } + } + + fn imported_bundle() -> ImportedTopologyBundle { + let network = json!({ + "Site Alpha": { + "children": { + "AP North": { + "children": {}, + "id": "ap-north", + "name": "AP North", + "type": "AP", + "downloadBandwidthMbps": 100, + "uploadBandwidthMbps": 50 + }, + "AP South": { + "children": {}, + "id": "ap-south", + "name": "AP South", + "type": "AP", + "downloadBandwidthMbps": 90, + "uploadBandwidthMbps": 45 + } + }, + "id": "site-alpha", + "name": "Site Alpha", + "type": "Site", + "downloadBandwidthMbps": 1000, + "uploadBandwidthMbps": 1000 + } + }); + ImportedTopologyBundle { + source: "uisp/full2".to_string(), + generated_unix: Some(1), + ingress_identity: None, + native_canonical: Some(TopologyCanonicalStateFile { + schema_version: 1, + source: "uisp/full2".to_string(), + generated_unix: Some(1), + ingress_identity: None, + ingress_kind: TopologyCanonicalIngressKind::NativeIntegration, + nodes: Vec::new(), + compatibility_network_json: network.clone(), + }), + native_editor: Some(TopologyEditorStateFile { + schema_version: 1, + source: "uisp/full2".to_string(), + generated_unix: Some(1), + ingress_identity: None, + nodes: Vec::new(), + }), + parent_candidates: None, + compatibility_network_json: network, + shaped_devices: shaped_devices(vec![ + sample_device("circuit-1", "AP North", Some("ap-north")), + sample_device("circuit-2", "AP South", Some("ap-south")), + sample_device("circuit-3", "Orphans", None), + ]), + circuit_anchors: CircuitAnchorsFile { + schema_version: 1, + source: "uisp/full2".to_string(), + generated_unix: Some(1), + anchors: vec![ + CircuitAnchor { + circuit_id: "circuit-1".to_string(), + circuit_name: Some("Circuit circuit-1".to_string()), + anchor_node_id: "ap-north".to_string(), + anchor_node_name: Some("AP North".to_string()), + }, + CircuitAnchor { + circuit_id: "circuit-2".to_string(), + circuit_name: Some("Circuit circuit-2".to_string()), + anchor_node_id: "ap-south".to_string(), + anchor_node_name: Some("AP South".to_string()), + }, + ], + }, + ethernet_advisories: Vec::new(), + } + } + + fn imported_bundle_with_hidden_logical_parent() -> ImportedTopologyBundle { + let mut bundle = imported_bundle(); + bundle.native_editor = Some(TopologyEditorStateFile { + schema_version: 1, + source: "uisp/full2".to_string(), + generated_unix: Some(1), + ingress_identity: None, + nodes: vec![TopologyEditorNode { + node_id: "ap-north".to_string(), + node_name: "Wrong Name".to_string(), + current_parent_node_id: Some("missing-site".to_string()), + current_parent_node_name: Some("Missing Site".to_string()), + current_attachment_id: Some("missing-attachment".to_string()), + current_attachment_name: Some("Missing Attachment".to_string()), + can_move: false, + allowed_parents: Vec::new(), + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }], + }); + bundle + } + + #[test] + fn ap_only_drops_unresolved_orphan_parents() { + let compiled = compile_topology(imported_bundle(), TopologyCompileMode::ApOnly) + .expect("ap_only compilation should succeed"); + assert_eq!(compiled.circuit_anchors.anchors.len(), 3); + assert!(compiled.compatibility_network_json[GENERATED_UNATTACHED_AP_NAME].is_object()); + let device = compiled + .shaped_devices + .devices + .iter() + .find(|device| device.circuit_id == "circuit-3") + .expect("unresolved circuit should still be emitted"); + assert_eq!( + device.parent_node_id.as_deref(), + Some(GENERATED_UNATTACHED_AP_ID) + ); + assert_eq!(device.parent_node, GENERATED_UNATTACHED_AP_NAME); + } + + #[test] + fn ap_site_groups_aps_under_nearest_site() { + let compiled = compile_topology(imported_bundle(), TopologyCompileMode::ApSite) + .expect("ap_site compilation should succeed"); + let root = compiled + .compatibility_network_json + .as_object() + .expect("compiled ap_site network should be an object"); + assert!(root.get("Site Alpha").is_some()); + let site = root["Site Alpha"] + .as_object() + .expect("site should be an object"); + let children = site["children"] + .as_object() + .expect("site should have children"); + assert!(children.get("AP North").is_some()); + assert!(children.get("AP South").is_some()); + let fallback_site = root[GENERATED_UNATTACHED_SITE_NAME] + .as_object() + .expect("fallback site should be present for unresolved circuits"); + let fallback_children = fallback_site["children"] + .as_object() + .expect("fallback site should have children"); + assert!( + fallback_children + .get(GENERATED_UNATTACHED_AP_NAME) + .is_some() + ); + let device = compiled + .shaped_devices + .devices + .iter() + .find(|device| device.circuit_id == "circuit-3") + .expect("unresolved circuit should still be emitted"); + assert_eq!( + device.parent_node_id.as_deref(), + Some(GENERATED_UNATTACHED_AP_ID) + ); + } + + #[test] + fn ap_site_projects_from_native_canonical_when_compatibility_tree_is_empty() { + let mut imported = imported_bundle(); + imported.native_canonical = Some(TopologyCanonicalStateFile::from_legacy_network_json( + &imported.compatibility_network_json, + )); + imported.compatibility_network_json = json!({}); + let compiled = compile_topology(imported, TopologyCompileMode::ApSite) + .expect("ap_site compilation should succeed from native canonical data"); + let root = compiled + .compatibility_network_json + .as_object() + .expect("compiled ap_site network should be an object"); + assert!(root.get("Site Alpha").is_some()); + let site = root["Site Alpha"] + .as_object() + .expect("site should be an object"); + let children = site["children"] + .as_object() + .expect("site should have children"); + assert!(children.get("AP North").is_some()); + assert!(children.get("AP South").is_some()); + } + + #[test] + fn full_mode_sanitizes_hidden_logical_parent_references() { + let compiled = compile_topology( + imported_bundle_with_hidden_logical_parent(), + TopologyCompileMode::Full, + ) + .expect("full compilation should succeed"); + let node = compiled + .editor + .nodes + .iter() + .find(|node| node.node_id == "ap-north") + .expect("compiled editor should retain exported node"); + assert_eq!(node.node_name, "AP North"); + assert_eq!(node.current_parent_node_id.as_deref(), Some("site-alpha")); + assert_eq!(node.current_parent_node_name.as_deref(), Some("Site Alpha")); + assert!(node.current_attachment_id.is_none()); + assert!(node.current_attachment_name.is_none()); + } + + #[test] + fn full_mode_preserves_legal_native_logical_parent_over_export_hop() { + let imported = ImportedTopologyBundle { + source: "uisp/full2".to_string(), + generated_unix: Some(1), + ingress_identity: None, + native_canonical: None, + native_editor: Some(TopologyEditorStateFile { + schema_version: 1, + source: "uisp/full2".to_string(), + generated_unix: Some(1), + ingress_identity: None, + nodes: vec![TopologyEditorNode { + node_id: "site-beta".to_string(), + node_name: "Site Beta".to_string(), + current_parent_node_id: Some("site-alpha".to_string()), + current_parent_node_name: Some("Site Alpha".to_string()), + current_attachment_id: Some("ap-alpha".to_string()), + current_attachment_name: Some("AP Alpha".to_string()), + can_move: true, + allowed_parents: vec![TopologyAllowedParent { + parent_node_id: "site-alpha".to_string(), + parent_node_name: "Site Alpha".to_string(), + attachment_options: vec![TopologyAttachmentOption { + attachment_id: "ap-alpha".to_string(), + attachment_name: "AP Alpha".to_string(), + attachment_kind: "wireless".to_string(), + attachment_role: TopologyAttachmentRole::PtpBackhaul, + pair_id: None, + peer_attachment_id: None, + peer_attachment_name: None, + capacity_mbps: None, + download_bandwidth_mbps: None, + upload_bandwidth_mbps: None, + transport_cap_mbps: None, + transport_cap_reason: None, + rate_source: TopologyAttachmentRateSource::Unknown, + can_override_rate: false, + rate_override_disabled_reason: None, + has_rate_override: false, + local_probe_ip: None, + remote_probe_ip: None, + probe_enabled: false, + probeable: false, + health_status: TopologyAttachmentHealthStatus::Healthy, + health_reason: None, + suppressed_until_unix: None, + effective_selected: false, + }], + all_attachments_suppressed: false, + has_probe_unavailable_attachments: false, + }], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }], + }), + parent_candidates: None, + compatibility_network_json: json!({ + "Site Alpha": { + "children": { + "AP Alpha": { + "children": { + "Site Beta": { + "children": {}, + "id": "site-beta", + "name": "Site Beta", + "type": "Site" + } + }, + "id": "ap-alpha", + "name": "AP Alpha", + "type": "AP" + } + }, + "id": "site-alpha", + "name": "Site Alpha", + "type": "Site" + } + }), + shaped_devices: shaped_devices(Vec::new()), + circuit_anchors: CircuitAnchorsFile::default(), + ethernet_advisories: Vec::new(), + }; + + let compiled = compile_topology(imported, TopologyCompileMode::Full) + .expect("full compilation should preserve imported logical parent"); + let node = compiled + .editor + .nodes + .iter() + .find(|node| node.node_id == "site-beta") + .expect("site beta should remain in editor state"); + + assert_eq!(node.current_parent_node_id.as_deref(), Some("site-alpha")); + assert_eq!(node.current_parent_node_name.as_deref(), Some("Site Alpha")); + assert_eq!(node.current_attachment_id.as_deref(), Some("ap-alpha")); + assert_eq!(node.current_attachment_name.as_deref(), Some("AP Alpha")); + } + + #[test] + fn full_mode_keeps_native_parent_when_compatibility_tree_flattens_backhaul_hops() { + let imported = ImportedTopologyBundle { + source: "uisp/full2".to_string(), + generated_unix: Some(1), + ingress_identity: None, + native_canonical: None, + native_editor: Some(TopologyEditorStateFile { + schema_version: 1, + source: "uisp/full2".to_string(), + generated_unix: Some(1), + ingress_identity: None, + nodes: vec![ + TopologyEditorNode { + node_id: "site-west".to_string(), + node_name: "WestRedd".to_string(), + current_parent_node_id: None, + current_parent_node_name: None, + current_attachment_id: None, + current_attachment_name: None, + can_move: false, + allowed_parents: Vec::new(), + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }, + TopologyEditorNode { + node_id: "aviat-west".to_string(), + node_name: "AVIAT_WestRedd".to_string(), + current_parent_node_id: Some("site-west".to_string()), + current_parent_node_name: Some("WestRedd".to_string()), + current_attachment_id: Some("site-west".to_string()), + current_attachment_name: Some("WestRedd".to_string()), + can_move: true, + allowed_parents: vec![TopologyAllowedParent { + parent_node_id: "site-west".to_string(), + parent_node_name: "WestRedd".to_string(), + attachment_options: vec![TopologyAttachmentOption { + attachment_id: "auto".to_string(), + attachment_name: "Auto".to_string(), + attachment_kind: "auto".to_string(), + attachment_role: TopologyAttachmentRole::Unknown, + pair_id: None, + peer_attachment_id: None, + peer_attachment_name: None, + capacity_mbps: None, + download_bandwidth_mbps: None, + upload_bandwidth_mbps: None, + transport_cap_mbps: None, + transport_cap_reason: None, + rate_source: TopologyAttachmentRateSource::Unknown, + can_override_rate: false, + rate_override_disabled_reason: None, + has_rate_override: false, + local_probe_ip: None, + remote_probe_ip: None, + probe_enabled: false, + probeable: false, + health_status: TopologyAttachmentHealthStatus::Disabled, + health_reason: None, + suppressed_until_unix: None, + effective_selected: false, + }], + all_attachments_suppressed: false, + has_probe_unavailable_attachments: false, + }], + preferred_attachment_id: None, + preferred_attachment_name: None, + effective_attachment_id: None, + effective_attachment_name: None, + }, + ], + }), + parent_candidates: None, + compatibility_network_json: json!({ + "AVIAT_WestRedd": { + "children": {}, + "id": "aviat-west", + "name": "AVIAT_WestRedd", + "parent_site": "WestRedd", + "type": "AP" + } + }), + shaped_devices: shaped_devices(Vec::new()), + circuit_anchors: CircuitAnchorsFile::default(), + ethernet_advisories: Vec::new(), + }; + + let compiled = compile_topology(imported, TopologyCompileMode::Full) + .expect("full compilation should preserve native site parent"); + let node = compiled + .editor + .nodes + .iter() + .find(|node| node.node_id == "aviat-west") + .expect("backhaul AP should remain in editor state"); + + assert_eq!(node.current_parent_node_id.as_deref(), Some("site-west")); + assert_eq!(node.current_parent_node_name.as_deref(), Some("WestRedd")); + } + + #[test] + fn full_mode_assigns_blank_parent_circuits_to_exported_unattached_site() { + let mut imported = imported_bundle(); + imported.compatibility_network_json = json!({ + "Site Alpha": { + "children": { + "AP North": { + "children": {}, + "id": "ap-north", + "name": "AP North", + "type": "AP" + }, + "AP South": { + "children": {}, + "id": "ap-south", + "name": "AP South", + "type": "AP" + } + }, + "id": "site-alpha", + "name": "Site Alpha", + "type": "Site" + }, + "LibreQoS Unattached [Site]": { + "children": {}, + "id": "libreqos:generated:splynx:site:unattached", + "name": "LibreQoS Unattached [Site]", + "type": "Site" + } + }); + imported.native_canonical = Some(TopologyCanonicalStateFile::from_legacy_network_json( + &imported.compatibility_network_json, + )); + if let Some(device) = imported + .shaped_devices + .devices + .iter_mut() + .find(|device| device.circuit_id == "circuit-3") + { + device.parent_node.clear(); + device.parent_node_id = None; + device.anchor_node_id = None; + } + + let compiled = compile_topology(imported, TopologyCompileMode::Full) + .expect("full compilation should succeed"); + let device = compiled + .shaped_devices + .devices + .iter() + .find(|device| device.circuit_id == "circuit-3") + .expect("unresolved circuit should still be emitted"); + + assert_eq!(device.parent_node, "LibreQoS Unattached [Site]"); + assert_eq!( + device.parent_node_id.as_deref(), + Some("libreqos:generated:splynx:site:unattached") + ); + } + + #[test] + fn full_mode_merges_legacy_parent_candidates_into_editor_state() { + let mut imported = imported_bundle(); + imported.native_editor = None; + imported.native_canonical = None; + imported.parent_candidates = Some(TopologyParentCandidatesFile { + source: "python/integration_common".to_string(), + ingress_identity: None, + nodes: vec![TopologyParentCandidatesNode { + node_id: "ap-north".to_string(), + node_name: "AP North".to_string(), + current_parent_node_id: Some("site-alpha".to_string()), + current_parent_node_name: Some("Site Alpha".to_string()), + candidate_parents: vec![TopologyParentCandidate { + node_id: "site-alpha".to_string(), + node_name: "Site Alpha".to_string(), + }], + }], + }); + + let compiled = + compile_topology(imported, TopologyCompileMode::Full).expect("full should compile"); + let node = compiled + .editor + .nodes + .iter() + .find(|node| node.node_id == "ap-north") + .expect("ap north should remain in editor state"); + + assert!(node.can_move); + assert_eq!(node.allowed_parents.len(), 1); + assert_eq!(node.current_parent_node_id.as_deref(), Some("site-alpha")); + assert_eq!(node.current_parent_node_name.as_deref(), Some("Site Alpha")); + } + + #[test] + fn full_mode_excludes_generated_graph_sites_from_editor_state() { + let network = json!({ + "Site Alpha": { + "children": { + "(Generated Site) Customer Alpha": { + "children": {}, + "id": "libreqos:generated:graph:site:customer-alpha", + "name": "(Generated Site) Customer Alpha", + "type": "Site" + }, + "AP North": { + "children": {}, + "id": "ap-north", + "name": "AP North", + "type": "AP" + } + }, + "id": "site-alpha", + "name": "Site Alpha", + "type": "Site" + } + }); + let imported = ImportedTopologyBundle { + source: "python/splynx".to_string(), + generated_unix: Some(1), + ingress_identity: None, + native_canonical: None, + native_editor: None, + parent_candidates: None, + compatibility_network_json: network, + shaped_devices: shaped_devices(Vec::new()), + circuit_anchors: CircuitAnchorsFile::default(), + ethernet_advisories: Vec::new(), + }; + + let compiled = + compile_topology(imported, TopologyCompileMode::Full).expect("full should compile"); + + assert!( + compiled + .editor + .nodes + .iter() + .all(|node| !node.node_id.starts_with("libreqos:generated:graph:")) + ); + assert!(compiled.editor.find_node("site-alpha").is_some()); + assert!(compiled.editor.find_node("ap-north").is_some()); + } +} diff --git a/src/rust/lqos_topology_compile/src/lib.rs b/src/rust/lqos_topology_compile/src/lib.rs new file mode 100644 index 00000000..2df740f1 --- /dev/null +++ b/src/rust/lqos_topology_compile/src/lib.rs @@ -0,0 +1,14 @@ +//! Shared topology compilation layer for projecting imported integration topology +//! into operator-selectable topology modes. + +#![warn(missing_docs)] + +mod bundle; +mod compile_mode; +mod validation; + +pub use bundle::{ + CompiledTopologyBundle, ImportedTopologyBundle, TopologyCompiledShapingFile, TopologyImportFile, +}; +pub use compile_mode::{TopologyCompileError, TopologyCompileMode, compile_topology}; +pub use validation::validate_compiled_bundle; diff --git a/src/rust/lqos_topology_compile/src/validation.rs b/src/rust/lqos_topology_compile/src/validation.rs new file mode 100644 index 00000000..fd1a2f63 --- /dev/null +++ b/src/rust/lqos_topology_compile/src/validation.rs @@ -0,0 +1,157 @@ +use crate::bundle::CompiledTopologyBundle; +use anyhow::{Result, bail}; +use lqos_config::{ShapedDevice, TopologyCanonicalIngressKind}; +use serde_json::{Map, Value}; +use std::collections::{HashMap, HashSet}; + +#[derive(Clone, Debug)] +struct ExportedNode { + name: String, + parent_id: Option, +} + +fn exported_node_index(network_json: &Value) -> HashMap { + let mut nodes = HashMap::new(); + if let Some(map) = network_json.as_object() { + collect_nodes(map, None, &mut nodes); + } + nodes +} + +fn collect_nodes( + map: &Map, + parent_id: Option<&str>, + out: &mut HashMap, +) { + for (key, value) in map { + let Some(node) = value.as_object() else { + continue; + }; + let Some(node_id) = node + .get("id") + .and_then(Value::as_str) + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + else { + continue; + }; + let node_name = node + .get("name") + .and_then(Value::as_str) + .unwrap_or(key) + .to_string(); + out.insert( + node_id.to_string(), + ExportedNode { + name: node_name, + parent_id: parent_id.map(ToOwned::to_owned), + }, + ); + if let Some(children) = node.get("children").and_then(Value::as_object) { + collect_nodes(children, Some(node_id), out); + } + } +} + +fn has_cycle(nodes: &HashMap) -> bool { + for node_id in nodes.keys() { + let mut seen = HashSet::new(); + let mut current = Some(node_id.as_str()); + while let Some(candidate) = current { + if !seen.insert(candidate) { + return true; + } + current = nodes + .get(candidate) + .and_then(|node| node.parent_id.as_deref()); + } + } + false +} + +fn validate_shaped_device( + device: &ShapedDevice, + nodes: &HashMap, +) -> Result<()> { + if let Some(parent_id) = device.parent_node_id.as_deref().map(str::trim) + && !parent_id.is_empty() + { + let Some(parent) = nodes.get(parent_id) else { + bail!( + "Compiled shaped device '{}' references missing parent node '{}'", + device.circuit_id, + parent_id + ); + }; + if !device.parent_node.trim().is_empty() && parent.name != device.parent_node { + bail!( + "Compiled shaped device '{}' parent name '{}' does not match exported node '{}'", + device.circuit_id, + device.parent_node, + parent.name + ); + } + } + if let Some(anchor_id) = device.anchor_node_id.as_deref().map(str::trim) + && !anchor_id.is_empty() + && !nodes.contains_key(anchor_id) + { + bail!( + "Compiled shaped device '{}' references missing anchor node '{}'", + device.circuit_id, + anchor_id + ); + } + Ok(()) +} + +/// Validates the compiled bundle's exported node references. +pub fn validate_compiled_bundle(bundle: &CompiledTopologyBundle) -> Result<()> { + let nodes = exported_node_index(&bundle.compatibility_network_json); + if has_cycle(&nodes) { + bail!("Compiled topology contains a parent cycle"); + } + let native_integration = + bundle.canonical.ingress_kind == TopologyCanonicalIngressKind::NativeIntegration; + for anchor in &bundle.circuit_anchors.anchors { + if !nodes.contains_key(anchor.anchor_node_id.as_str()) { + bail!( + "Compiled circuit anchor '{}' references missing node '{}'", + anchor.circuit_id, + anchor.anchor_node_id + ); + } + } + for device in &bundle.shaped_devices.devices { + validate_shaped_device(device, &nodes)?; + } + for node in &bundle.canonical.nodes { + if !node.node_id.trim().is_empty() + && !nodes.contains_key(node.node_id.as_str()) + && !bundle + .editor + .nodes + .iter() + .any(|editor| editor.node_id == node.node_id) + { + bail!("Compiled canonical node '{}' is not exported", node.node_id); + } + } + for node in &bundle.editor.nodes { + if !nodes.contains_key(node.node_id.as_str()) && !native_integration { + bail!("Compiled editor node '{}' is not exported", node.node_id); + } + if let Some(parent_id) = node.current_parent_node_id.as_deref() + && !parent_id.is_empty() + && !nodes.contains_key(parent_id) + && !native_integration + { + bail!( + "Compiled editor node '{}' references missing parent '{}'", + node.node_id, + parent_id + ); + } + } + Ok(()) +} diff --git a/src/rust/lqosd/Cargo.toml b/src/rust/lqosd/Cargo.toml index aecd11f7..9b0acdbe 100644 --- a/src/rust/lqosd/Cargo.toml +++ b/src/rust/lqosd/Cargo.toml @@ -17,6 +17,7 @@ lqos_queue_tracker = { path = "../lqos_queue_tracker" } lqos_utils = { path = "../lqos_utils" } lqos_overrides = { path = "../lqos_overrides" } lqos_topology = { path = "../lqos_topology" } +lqos_topology_compile = { path = "../lqos_topology_compile" } lqos_heimdall = { path = "../lqos_heimdall" } lqos_stormguard = { path = "../lqos_stormguard" } lqos_probe = { path = "../lqos_probe" } diff --git a/src/rust/lqosd/src/main.rs b/src/rust/lqosd/src/main.rs index de494a2a..bd095de8 100644 --- a/src/rust/lqosd/src/main.rs +++ b/src/rust/lqosd/src/main.rs @@ -897,14 +897,17 @@ fn handle_bus_requests(requests: &[BusRequest], responses: &mut Vec BusResponse::Ack } BusRequest::SchedulerError(error) => { + tool_status::scheduler_seen(); tool_status::scheduler_error(Some(error.clone())); BusResponse::Ack } BusRequest::SchedulerOutput(output) => { + tool_status::scheduler_seen(); tool_status::scheduler_output(Some(output.clone())); BusResponse::Ack } BusRequest::SchedulerProgress(progress) => { + tool_status::scheduler_seen(); let mut progress = progress.clone(); if progress.updated_unix.is_none() { progress.updated_unix = unix_now().ok(); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/circuit.js b/src/rust/lqosd/src/node_manager/js_build/src/circuit.js index a80fd344..5e1c3fd5 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/circuit.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/circuit.js @@ -59,6 +59,7 @@ let excludeRttBusy = false; let latestCakeMsg = null; let cakeGraphs = null; let cakeQueueUnavailable = false; +let queueStatsMode = "live"; let circuitSqmOverride = ""; let queuingActivityGraph = null; let latestCircuitDevices = []; @@ -864,6 +865,14 @@ function updateCakeTabAvailability(msg) { return true; } +function renderCakeStatusMessage(html) { + const cakeTab = document.getElementById("cake"); + if (!cakeTab) { + return; + } + cakeTab.innerHTML = html; +} + function renderCakeGraphShell() { const cakeTab = document.getElementById("cake"); if (!cakeTab || document.getElementById("cakeBacklog")) { @@ -1100,6 +1109,14 @@ function formatDirectionalQueueTypeDisplay(downToken, upToken) { return `${downLabel} / ${upLabel}`; } +function setQueueTypeDisplayText(label) { + const queueTypeEl = document.getElementById("cakeQueueType"); + if (!queueTypeEl) { + return; + } + queueTypeEl.textContent = label; +} + function setQueueTypeDisplay(sqmToken) { const queueTypeEl = document.getElementById("cakeQueueType"); if (!queueTypeEl) { @@ -1109,13 +1126,9 @@ function setQueueTypeDisplay(sqmToken) { } function setQueueTypeDisplayFromKinds(kindDown, kindUp) { - const queueTypeEl = document.getElementById("cakeQueueType"); - if (!queueTypeEl) { - return; - } const down = (kindDown ?? "").toString().trim().toLowerCase(); const up = (kindUp ?? "").toString().trim().toLowerCase(); - queueTypeEl.textContent = formatDirectionalQueueTypeDisplay(down, up); + setQueueTypeDisplayText(formatDirectionalQueueTypeDisplay(down, up)); } function requestCircuitById(onSuccess, onError) { @@ -2280,9 +2293,8 @@ function subscribeToCake() { // Function to show "Queue not loaded" message function showNoQueueMessage() { - const cakeTab = document.getElementById("cake"); - if (cakeTab && !hasReceivedData) { - cakeTab.innerHTML = '

Queue not loaded.

The shaper queue for this circuit has not been created yet.

'; + if (!hasReceivedData) { + renderCakeStatusMessage('

Queue not loaded.

The shaper queue for this circuit has not been created yet.

'); } } @@ -2400,6 +2412,7 @@ function loadInitial() { requestCircuitById((payload) => { const circuits = payload.devices || []; const advisory = payload.ethernet_advisory || null; + queueStatsMode = payload.queue_stats_mode || "live"; let circuit = circuits[0]; const parentNode = resolveCircuitParentNode(payload, circuits); circuitConfigDevices = circuits; diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config/config_helper.js b/src/rust/lqosd/src/node_manager/js_build/src/config/config_helper.js index 7a149f99..e4358346 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config/config_helper.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config/config_helper.js @@ -1,6 +1,7 @@ import { get_ws_client } from "../pubsub/ws"; const wsClient = get_ws_client(); +const secretBindings = []; function sendWsRequest(responseEvent, request, onComplete, onError) { let done = false; @@ -33,6 +34,9 @@ function ensureOptionalConfigSections(config) { if (!config.splynx_integration || typeof config.splynx_integration !== "object") { config.splynx_integration = {}; } + if (!config.topology || typeof config.topology !== "object") { + config.topology = {}; + } if (!config.sonar_integration || typeof config.sonar_integration !== "object") { config.sonar_integration = {}; @@ -59,7 +63,29 @@ function ensureOptionalConfigSections(config) { if (typeof splynx.api_secret !== "string") splynx.api_secret = ""; if (typeof splynx.url !== "string") splynx.url = ""; if (typeof splynx.strategy !== "string" || splynx.strategy.length === 0) { - splynx.strategy = "ap_only"; + splynx.strategy = "ap_site"; + } + + const topology = config.topology; + const normalizeMode = (mode) => { + if (typeof mode !== "string") { + return ""; + } + const lowered = mode.trim().toLowerCase(); + if (lowered === "full2") { + return "full"; + } + return ["flat", "ap_only", "ap_site", "full"].includes(lowered) ? lowered : ""; + }; + if (typeof topology.compile_mode !== "string" || topology.compile_mode.length === 0) { + topology.compile_mode = normalizeMode(topology.compile_mode) + || normalizeMode(config.uisp_integration?.strategy) + || normalizeMode(splynx.strategy) + || (config.uisp_integration?.enable_uisp ? "full" : "") + || (splynx.enable_splynx ? "ap_site" : "") + || "ap_site"; + } else { + topology.compile_mode = normalizeMode(topology.compile_mode) || "ap_site"; } return config; @@ -98,7 +124,10 @@ export function loadConfig(onComplete, onError) { "GetConfig", { GetConfig: {} }, (msg) => { - window.config = ensureOptionalConfigSections(msg.data); + const payload = msg && msg.data ? msg.data : {}; + window.config = ensureOptionalConfigSections(payload.config || {}); + window.configSecretState = payload.secret_state || {}; + window.configSecretClears = {}; if (onComplete) onComplete(msg); }, onError, @@ -106,10 +135,31 @@ export function loadConfig(onComplete, onError) { } export function saveConfig(onComplete, onError) { + const clearSecrets = window.configSecretClears || {}; sendWsRequest( "UpdateConfigResult", - { UpdateConfig: { config: window.config } }, + { + UpdateConfig: { + config: window.config, + clear_secrets: clearSecrets, + }, + }, (msg) => { + if (msg && msg.ok) { + secretBindings.forEach((binding) => { + const sectionState = secretFieldMap(window.configSecretState || {}, binding.section); + const inputValue = binding.input.value.trim(); + sectionState[binding.field] = clearSecrets?.[binding.section]?.[binding.field] + ? false + : (inputValue.length > 0 || sectionState[binding.field]); + if (window.config?.[binding.section] && typeof window.config[binding.section] === "object") { + window.config[binding.section][binding.field] = ""; + } + binding.input.value = ""; + setSecretClear(binding.section, binding.field, false); + binding.updateStatus(); + }); + } if (onComplete) onComplete(msg); }, (msg) => { @@ -122,6 +172,108 @@ export function saveConfig(onComplete, onError) { ); } +function secretFieldMap(container, section) { + if (!container[section] || typeof container[section] !== "object") { + container[section] = {}; + } + return container[section]; +} + +export function secretConfigured(section, field) { + return !!window.configSecretState?.[section]?.[field]; +} + +export function secretClearMarked(section, field) { + return !!window.configSecretClears?.[section]?.[field]; +} + +export function setSecretClear(section, field, clear) { + if (!window.configSecretClears || typeof window.configSecretClears !== "object") { + window.configSecretClears = {}; + } + const sectionMap = secretFieldMap(window.configSecretClears, section); + if (clear) { + sectionMap[field] = true; + } else { + delete sectionMap[field]; + if (Object.keys(sectionMap).length === 0) { + delete window.configSecretClears[section]; + } + } +} + +export function secretWillExistAfterSave(section, field, inputId) { + const input = document.getElementById(inputId); + const hasReplacement = !!(input && input.value.trim().length > 0); + if (hasReplacement) { + return true; + } + if (secretClearMarked(section, field)) { + return false; + } + return secretConfigured(section, field); +} + +export function bindSecretField({ + section, + field, + inputId, + statusId, + clearButtonId, + configuredMessage = "Stored on server. Leave blank to keep the current value.", + emptyMessage = "No value stored.", +}) { + const input = document.getElementById(inputId); + const status = document.getElementById(statusId); + const clearButton = clearButtonId ? document.getElementById(clearButtonId) : null; + if (!input || !status) { + return; + } + + const updateStatus = () => { + const hasReplacement = input.value.trim().length > 0; + const configured = secretConfigured(section, field); + const clearing = secretClearMarked(section, field); + + if (hasReplacement) { + status.textContent = "A new value will replace the stored secret when you save."; + } else if (clearing) { + status.textContent = "The stored secret will be cleared when you save."; + } else if (configured) { + status.textContent = configuredMessage; + } else { + status.textContent = emptyMessage; + } + + if (clearButton) { + clearButton.disabled = !configured && !hasReplacement && !clearing; + } + }; + + input.addEventListener("input", () => { + if (input.value.trim().length > 0) { + setSecretClear(section, field, false); + } + updateStatus(); + }); + + if (clearButton) { + clearButton.addEventListener("click", () => { + input.value = ""; + setSecretClear(section, field, secretConfigured(section, field)); + updateStatus(); + }); + } + + secretBindings.push({ + section, + field, + input, + updateStatus, + }); + updateStatus(); +} + export function saveNetworkAndDevices(network_json, shaped_devices, onComplete, onError) { if (!network_json || typeof network_json !== "object") { alert("Invalid network configuration"); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_integration.js b/src/rust/lqosd/src/node_manager/js_build/src/config_integration.js index 74391292..5b7da402 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_integration.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_integration.js @@ -32,6 +32,12 @@ function validateConfig() { return false; } } + + const topologyMode = document.getElementById("topologyCompileMode").value.trim(); + if (!["flat", "ap_only", "ap_site", "full"].includes(topologyMode)) { + alert("Topology Compile Mode must be one of Flat, AP Only, AP Site, or Full"); + return false; + } return true; } @@ -40,7 +46,6 @@ function updateConfig() { // Update only the integration_common section window.config.integration_common = { circuit_name_as_address: document.getElementById("circuitNameAsAddress").checked, - always_overwrite_network_json: document.getElementById("alwaysOverwriteNetworkJson").checked, use_mikrotik_ipv6: document.getElementById("useMikrotikIpv6").checked, queue_refresh_interval_mins: parseInt(document.getElementById("queueRefreshInterval").value), promote_to_root: (() => { @@ -64,6 +69,16 @@ function updateConfig() { return value === 0.94 ? null : value; })(), }; + window.config.topology = { + ...(window.config.topology || {}), + compile_mode: document.getElementById("topologyCompileMode").value.trim(), + }; + if (window.config.uisp_integration && typeof window.config.uisp_integration === "object") { + window.config.uisp_integration.strategy = window.config.topology.compile_mode; + } + if (window.config.splynx_integration && typeof window.config.splynx_integration === "object") { + window.config.splynx_integration.strategy = window.config.topology.compile_mode; + } } // Render the configuration menu @@ -78,8 +93,6 @@ loadConfig(() => { // Boolean fields document.getElementById("circuitNameAsAddress").checked = integration.circuit_name_as_address ?? false; - document.getElementById("alwaysOverwriteNetworkJson").checked = - integration.always_overwrite_network_json ?? false; document.getElementById("useMikrotikIpv6").checked = integration.use_mikrotik_ipv6 ?? false; @@ -96,6 +109,8 @@ loadConfig(() => { integration.ethernet_port_limits_enabled ?? true; document.getElementById("ethernetPortLimitMultiplier").value = (integration.ethernet_port_limit_multiplier ?? 0.94).toFixed(2); + document.getElementById("topologyCompileMode").value = + window.config.topology?.compile_mode ?? "ap_site"; // Add save button click handler document.getElementById('saveButton').addEventListener('click', () => { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_netzur.js b/src/rust/lqosd/src/node_manager/js_build/src/config_netzur.js index b5d19263..3a23fb1a 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_netzur.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_netzur.js @@ -1,4 +1,10 @@ -import {saveConfig, loadConfig, renderConfigMenu} from "./config/config_helper"; +import { + bindSecretField, + loadConfig, + renderConfigMenu, + saveConfig, + secretWillExistAfterSave, +} from "./config/config_helper"; function validateConfig() { const enabled = document.getElementById("enableNetzur").checked; @@ -6,8 +12,7 @@ function validateConfig() { return true; } - const apiKey = document.getElementById("netzurApiKey").value.trim(); - if (!apiKey) { + if (!secretWillExistAfterSave("netzur_integration", "api_key", "netzurApiKey")) { alert("API Key is required when Netzur integration is enabled"); return false; } @@ -53,9 +58,15 @@ loadConfig(() => { const cfg = window.config.netzur_integration ?? {}; document.getElementById("enableNetzur").checked = cfg.enable_netzur ?? false; - document.getElementById("netzurApiKey").value = cfg.api_key ?? ""; document.getElementById("netzurApiUrl").value = cfg.api_url ?? ""; document.getElementById("netzurTimeout").value = cfg.timeout_secs ?? 60; + bindSecretField({ + section: "netzur_integration", + field: "api_key", + inputId: "netzurApiKey", + statusId: "netzurApiKeyStatus", + clearButtonId: "clearNetzurApiKey", + }); document.getElementById("saveNetzur").addEventListener('click', () => { if (validateConfig()) { updateConfig(); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_powercode.js b/src/rust/lqosd/src/node_manager/js_build/src/config_powercode.js index 9d1e61b8..6501d331 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_powercode.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_powercode.js @@ -1,10 +1,15 @@ -import {saveConfig, loadConfig, renderConfigMenu} from "./config/config_helper"; +import { + bindSecretField, + loadConfig, + renderConfigMenu, + saveConfig, + secretWillExistAfterSave, +} from "./config/config_helper"; function validateConfig() { // Validate required fields when enabled if (document.getElementById("enablePowercode").checked) { - const apiKey = document.getElementById("powercodeApiKey").value.trim(); - if (!apiKey) { + if (!secretWillExistAfterSave("powercode_integration", "powercode_api_key", "powercodeApiKey")) { alert("API Key is required when Powercode integration is enabled"); return false; } @@ -47,10 +52,15 @@ loadConfig(() => { powercode.enable_powercode ?? false; // String fields - document.getElementById("powercodeApiKey").value = - powercode.powercode_api_key ?? ""; document.getElementById("powercodeApiUrl").value = powercode.powercode_api_url ?? ""; + bindSecretField({ + section: "powercode_integration", + field: "powercode_api_key", + inputId: "powercodeApiKey", + statusId: "powercodeApiKeyStatus", + clearButtonId: "clearPowercodeApiKey", + }); // Add save button click handler document.getElementById('saveButton').addEventListener('click', () => { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_sonar.js b/src/rust/lqosd/src/node_manager/js_build/src/config_sonar.js index c87c4d28..f33fa430 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_sonar.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_sonar.js @@ -1,4 +1,10 @@ -import {saveConfig, loadConfig, renderConfigMenu} from "./config/config_helper"; +import { + bindSecretField, + loadConfig, + renderConfigMenu, + saveConfig, + secretWillExistAfterSave, +} from "./config/config_helper"; function arrayToString(arr) { return arr ? arr.join(', ') : ''; @@ -67,8 +73,7 @@ function validateConfig() { return false; } - const apiKey = document.getElementById("sonarApiKey").value.trim(); - if (!apiKey) { + if (!secretWillExistAfterSave("sonar_integration", "sonar_api_key", "sonarApiKey")) { alert("API Key is required when Sonar integration is enabled"); return false; } @@ -124,8 +129,6 @@ loadConfig(() => { // String fields document.getElementById("sonarApiUrl").value = sonar.sonar_api_url ?? ""; - document.getElementById("sonarApiKey").value = - sonar.sonar_api_key ?? ""; document.getElementById("snmpCommunity").value = sonar.snmp_community ?? "public"; @@ -139,6 +142,13 @@ loadConfig(() => { document.getElementById("recurringExcludedServiceNames").value = arrayToString(sonar.recurring_excluded_service_names); renderRecurringRateRows(sonar.recurring_service_rates ?? []); + bindSecretField({ + section: "sonar_integration", + field: "sonar_api_key", + inputId: "sonarApiKey", + statusId: "sonarApiKeyStatus", + clearButtonId: "clearSonarApiKey", + }); document.getElementById("addRecurringRateRow").addEventListener("click", () => { document.getElementById("recurringServiceRatesBody").appendChild(buildRecurringRateRow()); }); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_splynx.js b/src/rust/lqosd/src/node_manager/js_build/src/config_splynx.js index b34129ed..02b25c78 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_splynx.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_splynx.js @@ -1,16 +1,20 @@ -import {saveConfig, loadConfig, renderConfigMenu} from "./config/config_helper"; +import { + bindSecretField, + loadConfig, + renderConfigMenu, + saveConfig, + secretWillExistAfterSave, +} from "./config/config_helper"; function validateConfig() { // Validate required fields when enabled if (document.getElementById("enableSplynx").checked) { - const apiKey = document.getElementById("apiKey").value.trim(); - if (!apiKey) { + if (!secretWillExistAfterSave("splynx_integration", "api_key", "apiKey")) { alert("API Key is required when Splynx integration is enabled"); return false; } - const apiSecret = document.getElementById("apiSecret").value.trim(); - if (!apiSecret) { + if (!secretWillExistAfterSave("splynx_integration", "api_secret", "apiSecret")) { alert("API Secret is required when Splynx integration is enabled"); return false; } @@ -37,7 +41,7 @@ function updateConfig() { api_key: document.getElementById("apiKey").value.trim(), api_secret: document.getElementById("apiSecret").value.trim(), url: document.getElementById("splynxUrl").value.trim(), - strategy: document.getElementById("topologyStrategy").value + strategy: window.config.topology?.compile_mode ?? document.getElementById("topologyStrategy").value }; } @@ -54,10 +58,22 @@ loadConfig(() => { // Populate form fields with config values const splynx = window.config.splynx_integration; document.getElementById("enableSplynx").checked = splynx.enable_splynx ?? false; - document.getElementById("apiKey").value = splynx.api_key ?? ""; - document.getElementById("apiSecret").value = splynx.api_secret ?? ""; document.getElementById("splynxUrl").value = splynx.url ?? ""; - document.getElementById("topologyStrategy").value = splynx.strategy ?? "ap_only"; + document.getElementById("topologyStrategy").value = window.config.topology?.compile_mode ?? splynx.strategy ?? "ap_site"; + bindSecretField({ + section: "splynx_integration", + field: "api_key", + inputId: "apiKey", + statusId: "apiKeyStatus", + clearButtonId: "clearApiKey", + }); + bindSecretField({ + section: "splynx_integration", + field: "api_secret", + inputId: "apiSecret", + statusId: "apiSecretStatus", + clearButtonId: "clearApiSecret", + }); // Add save button click handler document.getElementById('saveButton').addEventListener('click', () => { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_uisp.js b/src/rust/lqosd/src/node_manager/js_build/src/config_uisp.js index 6bafa5fc..c1d1763c 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_uisp.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_uisp.js @@ -1,10 +1,15 @@ -import {saveConfig, loadConfig, renderConfigMenu} from "./config/config_helper"; +import { + bindSecretField, + loadConfig, + renderConfigMenu, + saveConfig, + secretWillExistAfterSave, +} from "./config/config_helper"; function validateConfig() { // Validate required fields when enabled if (document.getElementById("enableUisp").checked) { - const token = document.getElementById("uispToken").value.trim(); - if (!token) { + if (!secretWillExistAfterSave("uisp_integration", "token", "uispToken")) { alert("API Token is required when UISP integration is enabled"); return false; } @@ -27,12 +32,6 @@ function validateConfig() { return false; } - const strategy = document.getElementById("uispStrategy").value.trim(); - if (!strategy) { - alert("Strategy is required when UISP integration is enabled"); - return false; - } - const suspendedStrategy = document.getElementById("uispSuspendedStrategy").value.trim(); if (!suspendedStrategy) { alert("Suspended Strategy is required when UISP integration is enabled"); @@ -112,7 +111,7 @@ function updateConfig() { token: document.getElementById("uispToken").value.trim(), url: document.getElementById("uispUrl").value.trim(), site: document.getElementById("uispSite").value.trim(), - strategy: document.getElementById("uispStrategy").value.trim(), + strategy: window.config.topology?.compile_mode ?? document.getElementById("uispStrategy").value.trim(), suspended_strategy: document.getElementById("uispSuspendedStrategy").value.trim(), airmax_capacity: parseFloat(document.getElementById("uispAirmaxCapacity").value), airmax_flexible_frame_download_ratio: parseFloat(document.getElementById("uispAirmaxFlexibleFrameDownloadRatio").value), @@ -147,10 +146,9 @@ loadConfig(() => { document.getElementById("uispInsecureSsl").checked = uisp.insecure_ssl ?? false; // String fields - document.getElementById("uispToken").value = uisp.token ?? ""; document.getElementById("uispUrl").value = uisp.url ?? ""; document.getElementById("uispSite").value = uisp.site ?? ""; - document.getElementById("uispStrategy").value = uisp.strategy ?? "full"; + document.getElementById("uispStrategy").value = window.config.topology?.compile_mode ?? uisp.strategy ?? "ap_site"; document.getElementById("uispSuspendedStrategy").value = uisp.suspended_strategy ?? "none"; // Numeric fields @@ -166,6 +164,14 @@ loadConfig(() => { document.getElementById("uispExceptionCpes").value = uisp.exception_cpes?.map(e => `${e.cpe}:${e.parent}`).join(", ") || ""; document.getElementById("uispDoNotSquashSites").value = uisp.do_not_squash_sites?.join(", ") || ""; + bindSecretField({ + section: "uisp_integration", + field: "token", + inputId: "uispToken", + statusId: "uispTokenStatus", + clearButtonId: "clearUispToken", + }); + // Add save button click handler document.getElementById('saveButton').addEventListener('click', () => { if (validateConfig()) { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/config_visp.js b/src/rust/lqosd/src/node_manager/js_build/src/config_visp.js index e6ddb77f..51a72694 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/config_visp.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/config_visp.js @@ -1,4 +1,10 @@ -import {saveConfig, loadConfig, renderConfigMenu} from "./config/config_helper"; +import { + bindSecretField, + loadConfig, + renderConfigMenu, + saveConfig, + secretWillExistAfterSave, +} from "./config/config_helper"; function validateConfig() { const enabled = document.getElementById("enableVisp").checked; @@ -12,8 +18,7 @@ function validateConfig() { return false; } - const clientSecret = document.getElementById("vispClientSecret").value.trim(); - if (!clientSecret) { + if (!secretWillExistAfterSave("visp_integration", "client_secret", "vispClientSecret")) { alert("Client Secret is required when VISP integration is enabled"); return false; } @@ -24,8 +29,7 @@ function validateConfig() { return false; } - const password = document.getElementById("vispPassword").value.trim(); - if (!password) { + if (!secretWillExistAfterSave("visp_integration", "password", "vispPassword")) { alert("Appuser Password is required when VISP integration is enabled"); return false; } @@ -66,12 +70,24 @@ loadConfig(() => { const cfg = window.config.visp_integration ?? {}; document.getElementById("enableVisp").checked = cfg.enable_visp ?? false; document.getElementById("vispClientId").value = cfg.client_id ?? ""; - document.getElementById("vispClientSecret").value = cfg.client_secret ?? ""; document.getElementById("vispUsername").value = cfg.username ?? ""; - document.getElementById("vispPassword").value = cfg.password ?? ""; document.getElementById("vispIspId").value = cfg.isp_id ?? ""; document.getElementById("vispTimeout").value = cfg.timeout_secs ?? 20; document.getElementById("vispOnlineUsersDomain").value = cfg.online_users_domain ?? ""; + bindSecretField({ + section: "visp_integration", + field: "client_secret", + inputId: "vispClientSecret", + statusId: "vispClientSecretStatus", + clearButtonId: "clearVispClientSecret", + }); + bindSecretField({ + section: "visp_integration", + field: "password", + inputId: "vispPassword", + statusId: "vispPasswordStatus", + clearButtonId: "clearVispPassword", + }); document.getElementById("saveVisp").addEventListener('click', () => { if (validateConfig()) { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/cpu_weights.js b/src/rust/lqosd/src/node_manager/js_build/src/cpu_weights.js index 90557136..6d9756e8 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/cpu_weights.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/cpu_weights.js @@ -38,7 +38,7 @@ function requestConfig() { const timeout = setTimeout(() => finish(null), 5000); listenOnce("GetConfig", (msg) => { clearTimeout(timeout); - finish(msg && msg.data ? msg.data : null); + finish(msg && msg.data ? (msg.data.config || null) : null); }); wsClient.send({ GetConfig: {} }); }); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/template.js b/src/rust/lqosd/src/node_manager/js_build/src/template.js index b195ab63..76ad97cb 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/template.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/template.js @@ -460,11 +460,11 @@ function loadObserveShapeControl() { } listenOnce("GetConfig", (cfgMsg) => { - if (!cfgMsg || !cfgMsg.data) { + if (!cfgMsg || !cfgMsg.data || !cfgMsg.data.config) { nav.classList.add("d-none"); return; } - cachedQueueModeConfig = cfgMsg.data; + cachedQueueModeConfig = cfgMsg.data.config; const mode = inferQueueMode(cachedQueueModeConfig); nav.classList.remove("d-none"); renderQueueModeState(mode, false); diff --git a/src/rust/lqosd/src/node_manager/js_build/src/toasts/sponsor_us.js b/src/rust/lqosd/src/node_manager/js_build/src/toasts/sponsor_us.js index 8564c874..f3fc9d20 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/toasts/sponsor_us.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/toasts/sponsor_us.js @@ -2,14 +2,14 @@ import { get_ws_client } from "../pubsub/ws"; const sponsorBtn = " Sponsor Us on GitHub"; const sponsorMessages = [ - "LibreQoS includes shaping and core controls. Insight adds historical dashboards and alerts so you can spot issues before tickets arrive. Start a free 30-day trial.", - "Need proof before or after changes? Insight keeps long-term latency, retransmit, and flow history in one place. Try it free.", - "Heatmaps in Insight make congestion trends obvious across sites and APs. Find busy hours fast. Start your free trial.", - "Managing multiple shapers? Insight gives you a single dashboard view across locations. Start free for 30 days.", - "Insight AI reports summarize what changed and where to look first, so troubleshooting takes minutes instead of hours.", - "When customers say internet is slow, Insight helps you verify latency, retransmits, and utilization quickly. Try it free.", - "LibreQoS handles shaping. Insight adds visibility, trends, and alerts to run operations proactively. Start a free trial.", - "See circuit and site behavior over time, not just right now. Insight gives you the historical context to make better decisions.", + "LibreQoS handles shaping. Insight adds history, dashboards, and alerts so you catch issues before tickets arrive.", + "Need proof before or after changes? Insight keeps latency, retransmit, and flow history in one place.", + "Insight heatmaps make congestion trends obvious across sites and APs. Find busy hours fast.", + "Managing multiple shapers? Insight gives you one dashboard view across locations.", + "Insight AI reports summarize what changed and where to look first, cutting troubleshooting time.", + "When customers say internet is slow, Insight helps you verify latency, retransmits, and utilization fast.", + "LibreQoS handles shaping. Insight adds visibility, trends, and alerts so you can run ops proactively.", + "See circuit and site behavior over time, not just right now. Insight gives you historical context.", ]; export function sponsorTag(parentId) { diff --git a/src/rust/lqosd/src/node_manager/js_build/src/tree.js b/src/rust/lqosd/src/node_manager/js_build/src/tree.js index c9bf628c..e3f4827c 100644 --- a/src/rust/lqosd/src/node_manager/js_build/src/tree.js +++ b/src/rust/lqosd/src/node_manager/js_build/src/tree.js @@ -98,7 +98,7 @@ function sendPrivateRequest(command) { async function loadRootGaugeConfigMax() { try { const msg = await sendWsRequest("GetConfig", {GetConfig: {}}); - const queues = msg?.data?.queues; + const queues = msg?.data?.config?.queues; if (!queues) { return; } diff --git a/src/rust/lqosd/src/node_manager/local_api/circuit.rs b/src/rust/lqosd/src/node_manager/local_api/circuit.rs index 0b62a00a..90df0bd8 100644 --- a/src/rust/lqosd/src/node_manager/local_api/circuit.rs +++ b/src/rust/lqosd/src/node_manager/local_api/circuit.rs @@ -14,6 +14,14 @@ pub struct CircuitParentNode { pub id: Option, } +/// Circuit-page queue-stats mode. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CircuitQueueStatsMode { + /// Standard per-circuit queue stats are expected to be available. + Live, +} + /// Circuit-page payload containing shaped devices plus optional Ethernet advisory metadata. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct CircuitByIdData { @@ -21,6 +29,8 @@ pub struct CircuitByIdData { pub devices: Vec, /// Canonical circuit parent resolved from the shaped-device parent and `network.json`. pub parent_node: Option, + /// Queue-stats behavior for the active topology mode. + pub queue_stats_mode: CircuitQueueStatsMode, /// Optional negotiated-Ethernet advisory derived from integration metadata. pub ethernet_advisory: Option, } @@ -71,6 +81,10 @@ fn circuit_parent_node( .or(canonical_parent) } +fn queue_stats_mode() -> CircuitQueueStatsMode { + CircuitQueueStatsMode::Live +} + pub fn circuit_by_id_data(id: &str) -> Option { let safe_id = id.to_lowercase().trim().to_string(); let reader = SHAPED_DEVICES.load(); @@ -85,10 +99,12 @@ pub fn circuit_by_id_data(id: &str) -> Option { None } else { let parent_node = circuit_parent_node(&safe_id, &mut devices); + let queue_stats_mode = queue_stats_mode(); let ethernet_advisory = load_ethernet_advisory(&safe_id, &devices); Some(CircuitByIdData { devices, parent_node, + queue_stats_mode, ethernet_advisory, }) } diff --git a/src/rust/lqosd/src/node_manager/local_api/config.rs b/src/rust/lqosd/src/node_manager/local_api/config.rs index 8838b79c..f2a1badc 100644 --- a/src/rust/lqosd/src/node_manager/local_api/config.rs +++ b/src/rust/lqosd/src/node_manager/local_api/config.rs @@ -39,16 +39,286 @@ const TOPOLOGY_SOURCE_INTEGRATIONS: [TopologySourceIntegration; 7] = [ }), ]; +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct UispSecretState { + #[serde(default)] + pub token: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct SplynxSecretState { + #[serde(default)] + pub api_key: bool, + #[serde(default)] + pub api_secret: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct VispSecretState { + #[serde(default)] + pub client_secret: bool, + #[serde(default)] + pub password: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct SonarSecretState { + #[serde(default)] + pub sonar_api_key: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct NetzurSecretState { + #[serde(default)] + pub api_key: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct PowercodeSecretState { + #[serde(default)] + pub powercode_api_key: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct ConfigSecretState { + #[serde(default)] + pub uisp_integration: UispSecretState, + #[serde(default)] + pub splynx_integration: SplynxSecretState, + #[serde(default)] + pub visp_integration: VispSecretState, + #[serde(default)] + pub sonar_integration: SonarSecretState, + #[serde(default)] + pub netzur_integration: NetzurSecretState, + #[serde(default)] + pub powercode_integration: PowercodeSecretState, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct UispSecretClearRequest { + #[serde(default)] + pub token: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct SplynxSecretClearRequest { + #[serde(default)] + pub api_key: bool, + #[serde(default)] + pub api_secret: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct VispSecretClearRequest { + #[serde(default)] + pub client_secret: bool, + #[serde(default)] + pub password: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct SonarSecretClearRequest { + #[serde(default)] + pub sonar_api_key: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct NetzurSecretClearRequest { + #[serde(default)] + pub api_key: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct PowercodeSecretClearRequest { + #[serde(default)] + pub powercode_api_key: bool, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct ConfigSecretClearRequest { + #[serde(default)] + pub uisp_integration: UispSecretClearRequest, + #[serde(default)] + pub splynx_integration: SplynxSecretClearRequest, + #[serde(default)] + pub visp_integration: VispSecretClearRequest, + #[serde(default)] + pub sonar_integration: SonarSecretClearRequest, + #[serde(default)] + pub netzur_integration: NetzurSecretClearRequest, + #[serde(default)] + pub powercode_integration: PowercodeSecretClearRequest, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ConfigView { + pub config: Config, + #[serde(default)] + pub secret_state: ConfigSecretState, +} + pub fn admin_check_data(login: LoginResult) -> bool { matches!(login, LoginResult::Admin) } -pub fn get_config_data(login: LoginResult) -> Result { +fn has_secret(value: &str) -> bool { + !value.trim().is_empty() +} + +fn redact_string_secret(value: &mut String) -> bool { + let configured = has_secret(value); + value.clear(); + configured +} + +fn redact_config_secrets(config: &mut Config) -> ConfigSecretState { + let mut secret_state = ConfigSecretState::default(); + + secret_state.uisp_integration.token = redact_string_secret(&mut config.uisp_integration.token); + secret_state.splynx_integration.api_key = + redact_string_secret(&mut config.splynx_integration.api_key); + secret_state.splynx_integration.api_secret = + redact_string_secret(&mut config.splynx_integration.api_secret); + secret_state.sonar_integration.sonar_api_key = + redact_string_secret(&mut config.sonar_integration.sonar_api_key); + secret_state.powercode_integration.powercode_api_key = + redact_string_secret(&mut config.powercode_integration.powercode_api_key); + + if let Some(netzur) = config.netzur_integration.as_mut() { + secret_state.netzur_integration.api_key = redact_string_secret(&mut netzur.api_key); + } + + if let Some(visp) = config.visp_integration.as_mut() { + secret_state.visp_integration.client_secret = + redact_string_secret(&mut visp.client_secret); + secret_state.visp_integration.password = redact_string_secret(&mut visp.password); + } + + secret_state +} + +fn merge_string_secret(incoming: &mut String, existing: &str, clear: bool) { + if clear { + incoming.clear(); + } else if incoming.trim().is_empty() { + *incoming = existing.to_string(); + } +} + +fn apply_secret_updates( + existing: &Config, + incoming: &mut Config, + clear_secrets: &ConfigSecretClearRequest, +) { + merge_string_secret( + &mut incoming.uisp_integration.token, + &existing.uisp_integration.token, + clear_secrets.uisp_integration.token, + ); + merge_string_secret( + &mut incoming.splynx_integration.api_key, + &existing.splynx_integration.api_key, + clear_secrets.splynx_integration.api_key, + ); + merge_string_secret( + &mut incoming.splynx_integration.api_secret, + &existing.splynx_integration.api_secret, + clear_secrets.splynx_integration.api_secret, + ); + merge_string_secret( + &mut incoming.sonar_integration.sonar_api_key, + &existing.sonar_integration.sonar_api_key, + clear_secrets.sonar_integration.sonar_api_key, + ); + merge_string_secret( + &mut incoming.powercode_integration.powercode_api_key, + &existing.powercode_integration.powercode_api_key, + clear_secrets.powercode_integration.powercode_api_key, + ); + + match ( + existing.netzur_integration.as_ref(), + incoming.netzur_integration.as_mut(), + ) { + (Some(existing_netzur), Some(incoming_netzur)) => { + merge_string_secret( + &mut incoming_netzur.api_key, + &existing_netzur.api_key, + clear_secrets.netzur_integration.api_key, + ); + } + (Some(existing_netzur), None) => { + incoming.netzur_integration = Some(existing_netzur.clone()); + if let Some(incoming_netzur) = incoming.netzur_integration.as_mut() { + merge_string_secret( + &mut incoming_netzur.api_key, + &existing_netzur.api_key, + clear_secrets.netzur_integration.api_key, + ); + } + } + (None, Some(incoming_netzur)) => { + if clear_secrets.netzur_integration.api_key { + incoming_netzur.api_key.clear(); + } + } + (None, None) => {} + } + + match (existing.visp_integration.as_ref(), incoming.visp_integration.as_mut()) { + (Some(existing_visp), Some(incoming_visp)) => { + merge_string_secret( + &mut incoming_visp.client_secret, + &existing_visp.client_secret, + clear_secrets.visp_integration.client_secret, + ); + merge_string_secret( + &mut incoming_visp.password, + &existing_visp.password, + clear_secrets.visp_integration.password, + ); + } + (Some(existing_visp), None) => { + incoming.visp_integration = Some(existing_visp.clone()); + if let Some(incoming_visp) = incoming.visp_integration.as_mut() { + merge_string_secret( + &mut incoming_visp.client_secret, + &existing_visp.client_secret, + clear_secrets.visp_integration.client_secret, + ); + merge_string_secret( + &mut incoming_visp.password, + &existing_visp.password, + clear_secrets.visp_integration.password, + ); + } + } + (None, Some(incoming_visp)) => { + if clear_secrets.visp_integration.client_secret { + incoming_visp.client_secret.clear(); + } + if clear_secrets.visp_integration.password { + incoming_visp.password.clear(); + } + } + (None, None) => {} + } +} + +pub fn get_config_data(login: LoginResult) -> Result { if login != LoginResult::Admin { return Err(StatusCode::FORBIDDEN); } lqos_config::load_config() - .map(|config| (*config).clone()) + .map(|config| { + let mut config = (*config).clone(); + let secret_state = redact_config_secrets(&mut config); + ConfigView { + config, + secret_state, + } + }) .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) } @@ -352,11 +622,14 @@ fn persist_shaped_devices(mut devices: Vec) -> Result<(), String> pub async fn update_lqosd_config_data( login: LoginResult, - config: Config, + mut config: Config, + clear_secrets: ConfigSecretClearRequest, ) -> Result<(), StatusCode> { if login != LoginResult::Admin { return Err(StatusCode::FORBIDDEN); } + let existing = lqos_config::load_config().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + apply_secret_updates(existing.as_ref(), &mut config, &clear_secrets); bus_request(vec![BusRequest::UpdateLqosdConfig(Box::new(config))]) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; @@ -578,7 +851,11 @@ pub struct UserRequest { #[cfg(test)] mod tests { - use super::validate_network_json; + use super::{ + ConfigSecretClearRequest, apply_secret_updates, redact_config_secrets, + validate_network_json, + }; + use lqos_config::Config; use serde_json::json; #[test] @@ -631,4 +908,141 @@ mod tests { let err = validate_network_json(&network).expect_err("duplicate names must fail"); assert!(err.contains("Duplicate")); } + + #[test] + fn redacts_secret_fields_and_marks_presence() { + let mut config = Config::default(); + config.uisp_integration.token = "uisp-token".to_string(); + config.splynx_integration.api_key = "splynx-key".to_string(); + config.splynx_integration.api_secret = "splynx-secret".to_string(); + config.sonar_integration.sonar_api_key = "sonar-key".to_string(); + config.powercode_integration.powercode_api_key = "powercode-key".to_string(); + if let Some(visp) = config.visp_integration.as_mut() { + visp.client_secret = "visp-secret".to_string(); + visp.password = "visp-password".to_string(); + } + config.netzur_integration = Some(Default::default()); + config + .netzur_integration + .as_mut() + .expect("netzur_integration should exist") + .api_key = "netzur-key".to_string(); + + let secret_state = redact_config_secrets(&mut config); + + assert!(secret_state.uisp_integration.token); + assert!(secret_state.splynx_integration.api_key); + assert!(secret_state.splynx_integration.api_secret); + assert!(secret_state.visp_integration.client_secret); + assert!(secret_state.visp_integration.password); + assert!(secret_state.sonar_integration.sonar_api_key); + assert!(secret_state.netzur_integration.api_key); + assert!(secret_state.powercode_integration.powercode_api_key); + assert!(config.uisp_integration.token.is_empty()); + assert!(config.splynx_integration.api_key.is_empty()); + assert!(config.splynx_integration.api_secret.is_empty()); + assert!(config.sonar_integration.sonar_api_key.is_empty()); + assert!(config.powercode_integration.powercode_api_key.is_empty()); + assert!( + config + .visp_integration + .as_ref() + .expect("visp_integration should exist") + .client_secret + .is_empty() + ); + assert!( + config + .visp_integration + .as_ref() + .expect("visp_integration should exist") + .password + .is_empty() + ); + assert!( + config + .netzur_integration + .as_ref() + .expect("netzur_integration should exist") + .api_key + .is_empty() + ); + } + + #[test] + fn applies_secret_updates_with_preserve_replace_and_clear() { + let mut existing = Config::default(); + existing.uisp_integration.token = "old-uisp".to_string(); + existing.splynx_integration.api_key = "old-key".to_string(); + existing.splynx_integration.api_secret = "old-secret".to_string(); + existing.sonar_integration.sonar_api_key = "old-sonar".to_string(); + existing.powercode_integration.powercode_api_key = "old-powercode".to_string(); + if let Some(visp) = existing.visp_integration.as_mut() { + visp.client_secret = "old-visp-secret".to_string(); + visp.password = "old-visp-password".to_string(); + } + existing.netzur_integration = Some(Default::default()); + existing + .netzur_integration + .as_mut() + .expect("netzur_integration should exist") + .api_key = "old-netzur".to_string(); + + let mut incoming = existing.clone(); + incoming.uisp_integration.token.clear(); + incoming.splynx_integration.api_key = "new-key".to_string(); + incoming.splynx_integration.api_secret.clear(); + incoming.sonar_integration.sonar_api_key.clear(); + incoming.powercode_integration.powercode_api_key.clear(); + if let Some(visp) = incoming.visp_integration.as_mut() { + visp.client_secret.clear(); + visp.password = "new-visp-password".to_string(); + } + incoming + .netzur_integration + .as_mut() + .expect("netzur_integration should exist") + .api_key + .clear(); + + let mut clear_secrets = ConfigSecretClearRequest::default(); + clear_secrets.splynx_integration.api_secret = true; + clear_secrets.sonar_integration.sonar_api_key = true; + clear_secrets.netzur_integration.api_key = true; + + apply_secret_updates(&existing, &mut incoming, &clear_secrets); + + assert_eq!(incoming.uisp_integration.token, "old-uisp"); + assert_eq!(incoming.splynx_integration.api_key, "new-key"); + assert!(incoming.splynx_integration.api_secret.is_empty()); + assert!(incoming.sonar_integration.sonar_api_key.is_empty()); + assert_eq!( + incoming + .visp_integration + .as_ref() + .expect("visp_integration should exist") + .client_secret, + "old-visp-secret" + ); + assert_eq!( + incoming + .visp_integration + .as_ref() + .expect("visp_integration should exist") + .password, + "new-visp-password" + ); + assert!( + incoming + .netzur_integration + .as_ref() + .expect("netzur_integration should exist") + .api_key + .is_empty() + ); + assert_eq!( + incoming.powercode_integration.powercode_api_key, + "old-powercode" + ); + } } diff --git a/src/rust/lqosd/src/node_manager/local_api/node_rate_overrides.rs b/src/rust/lqosd/src/node_manager/local_api/node_rate_overrides.rs index 025c127e..14bf58fd 100644 --- a/src/rust/lqosd/src/node_manager/local_api/node_rate_overrides.rs +++ b/src/rust/lqosd/src/node_manager/local_api/node_rate_overrides.rs @@ -221,7 +221,7 @@ fn legacy_warning_messages() -> Result, StatusCode> { } if config.splynx_integration.enable_splynx && base.join(LEGACY_SPLYNX_FILE).exists() { warnings.push(format!( - "Legacy Splynx bandwidth overrides are present in `{LEGACY_SPLYNX_FILE}`. Tree edits write operator overrides to `lqos_overrides.json` and do not modify that file." + "Legacy Splynx bandwidth overrides are present in `{LEGACY_SPLYNX_FILE}`. Current Splynx builds auto-migrate that file into operator `AdjustSiteSpeed` overrides in `lqos_overrides.json` and rename the CSV to a backup." )); } diff --git a/src/rust/lqosd/src/node_manager/local_api/tree_attached_circuits.rs b/src/rust/lqosd/src/node_manager/local_api/tree_attached_circuits.rs index c0937af4..ec800a64 100644 --- a/src/rust/lqosd/src/node_manager/local_api/tree_attached_circuits.rs +++ b/src/rust/lqosd/src/node_manager/local_api/tree_attached_circuits.rs @@ -1,8 +1,13 @@ use crate::node_manager::local_api::ethernet_caps::{EthernetCapBadge, ethernet_cap_badge_map}; -use crate::shaped_devices_tracker::NETWORK_JSON; +use crate::shaped_devices_tracker::{ + NETWORK_JSON, SHAPED_DEVICES, effective_parent_for_circuit, resolve_parent_node_reference, +}; use crate::shaped_devices_tracker::circuit_live::fresh_circuit_live_snapshot; +use fxhash::{FxHashMap, FxHashSet}; +use lqos_config::{ConfigShapedDevices, ShapedDevice}; use lqos_utils::units::{DownUpOrder, TcpRetransmitSample}; use serde::{Deserialize, Deserializer, Serialize}; +use std::net::{Ipv4Addr, Ipv6Addr}; const DEFAULT_ATTACHED_CIRCUITS_PAGE_SIZE: usize = 100; const MAX_ATTACHED_CIRCUITS_PAGE_SIZE: usize = 250; @@ -73,6 +78,22 @@ pub struct TreeAttachedCircuitsPage { pub rows: Vec, } +#[derive(Clone, Debug, Default)] +struct SubtreeMatcher { + node_names: FxHashSet, + node_ids: FxHashSet, +} + +#[derive(Clone, Debug, Default)] +struct CircuitRowAccumulator { + circuit_id: String, + circuit_name: String, + parent_node: String, + device_names: FxHashSet, + ip_addrs: FxHashSet, + plan_mbps: DownUpOrder, +} + fn normalized_page_size(query: &TreeAttachedCircuitsQuery) -> usize { query .page_size @@ -80,23 +101,23 @@ fn normalized_page_size(query: &TreeAttachedCircuitsQuery) -> usize { .clamp(1, MAX_ATTACHED_CIRCUITS_PAGE_SIZE) } -fn resolve_node_name(query: &TreeAttachedCircuitsQuery) -> Option { +fn resolve_selected_node_index(query: &TreeAttachedCircuitsQuery) -> Option { let reader = NETWORK_JSON.read(); let nodes = reader.get_nodes_when_ready(); if let Some(node_id) = query.node_id.as_deref() && let Some(node) = nodes .iter() - .find(|node| node.id.as_deref() == Some(node_id)) + .position(|node| node.id.as_deref() == Some(node_id)) { - return Some(node.name.clone()); + return Some(node); } let node_path = query.node_path.as_ref()?; if node_path.is_empty() { return None; } - nodes.iter().find_map(|node| { + nodes.iter().enumerate().find_map(|(idx, node)| { let path = if node.name == "Root" { vec!["Root".to_string()] } else { @@ -116,13 +137,159 @@ fn resolve_node_name(query: &TreeAttachedCircuitsQuery) -> Option { names }; if &path == node_path { - Some(node.name.clone()) + Some(idx) } else { None } }) } +fn build_subtree_matcher(query: &TreeAttachedCircuitsQuery) -> Option { + let selected_idx = resolve_selected_node_index(query)?; + let reader = NETWORK_JSON.read(); + let nodes = reader.get_nodes_when_ready(); + let selected = nodes.get(selected_idx)?; + + let mut node_names = FxHashSet::default(); + let mut node_ids = FxHashSet::default(); + node_names.insert(selected.name.clone()); + if let Some(node_id) = selected.id.clone() { + node_ids.insert(node_id); + } + + for node in nodes { + let mut current = Some(node); + let mut in_subtree = false; + while let Some(candidate) = current { + if candidate.name == selected.name && candidate.id == selected.id { + in_subtree = true; + break; + } + current = candidate + .immediate_parent + .and_then(|parent_idx| nodes.get(parent_idx)); + } + if !in_subtree { + continue; + } + node_names.insert(node.name.clone()); + if let Some(node_id) = node.id.clone() { + node_ids.insert(node_id); + } + } + + Some(SubtreeMatcher { + node_names, + node_ids, + }) +} + +fn format_ipv4(addr: Ipv4Addr, prefix: u32) -> String { + if prefix >= 32 { + addr.to_string() + } else { + format!("{addr}/{prefix}") + } +} + +fn format_ipv6(addr: Ipv6Addr, prefix: u32) -> String { + if prefix >= 128 { + addr.to_string() + } else { + format!("{addr}/{prefix}") + } +} + +fn circuit_parent_matches_subtree( + matcher: &SubtreeMatcher, + parent_name: &str, + parent_id: Option<&str>, +) -> bool { + let trimmed_name = parent_name.trim(); + if !trimmed_name.is_empty() && matcher.node_names.contains(trimmed_name) { + return true; + } + + let trimmed_id = parent_id.map(str::trim).filter(|id| !id.is_empty()); + if let Some(parent_id) = trimmed_id + && matcher.node_ids.contains(parent_id) + { + return true; + } + + false +} + +fn effective_or_canonical_parent(device: &ShapedDevice) -> Option<(String, Option)> { + if let Some(runtime_parent) = effective_parent_for_circuit(&device.circuit_id) { + return Some((runtime_parent.name, runtime_parent.id)); + } + + resolve_parent_node_reference(&device.parent_node, device.parent_node_id.as_deref()) + .map(|resolved| (resolved.name, resolved.id)) + .or_else(|| { + let trimmed = device.parent_node.trim(); + if trimmed.is_empty() { + None + } else { + Some((trimmed.to_string(), device.parent_node_id.clone())) + } + }) +} + +fn aggregate_attached_circuit_rows( + matcher: &SubtreeMatcher, + shaped_devices: &ConfigShapedDevices, +) -> Vec { + let mut rows: FxHashMap = FxHashMap::default(); + + for device in &shaped_devices.devices { + if device.circuit_id.trim().is_empty() { + continue; + } + let Some((parent_name, parent_id)) = effective_or_canonical_parent(device) else { + continue; + }; + if !circuit_parent_matches_subtree(matcher, &parent_name, parent_id.as_deref()) { + continue; + } + + let entry = rows + .entry(device.circuit_id.clone()) + .or_insert_with(|| CircuitRowAccumulator { + circuit_id: device.circuit_id.clone(), + circuit_name: device.circuit_name.clone(), + parent_node: parent_name.clone(), + ..CircuitRowAccumulator::default() + }); + if entry.circuit_name.is_empty() { + entry.circuit_name = device.circuit_name.clone(); + } + if entry.parent_node.is_empty() { + entry.parent_node = parent_name; + } + if !device.device_name.trim().is_empty() { + entry.device_names.insert(device.device_name.clone()); + } + for (addr, prefix) in &device.ipv4 { + entry.ip_addrs.insert(format_ipv4(*addr, *prefix)); + } + for (addr, prefix) in &device.ipv6 { + entry.ip_addrs.insert(format_ipv6(*addr, *prefix)); + } + entry.plan_mbps.down = entry.plan_mbps.down.max(device.download_max_mbps.round()); + entry.plan_mbps.up = entry.plan_mbps.up.max(device.upload_max_mbps.round()); + } + + let mut rows: Vec = rows.into_values().collect(); + rows.sort_by(|left, right| { + left.circuit_name + .cmp(&right.circuit_name) + .then_with(|| left.circuit_id.cmp(&right.circuit_id)) + }); + rows +} + /// Returns one filtered, sorted page of circuits attached to the selected tree node. pub fn tree_attached_circuits(query: TreeAttachedCircuitsQuery) -> TreeAttachedCircuitsPage { let page = query.page.unwrap_or(0); @@ -133,36 +300,62 @@ pub fn tree_attached_circuits(query: TreeAttachedCircuitsQuery) -> TreeAttachedC .clone() .unwrap_or(TreeAttachedCircuitsSort::CircuitName); let descending = query.descending.unwrap_or(false); - let node_name = resolve_node_name(&query); + let matcher = build_subtree_matcher(&query); let snapshot = fresh_circuit_live_snapshot(); let ethernet_badges = ethernet_cap_badge_map(); - let mut rows: Vec = match node_name.as_deref() { - Some(node_name) => snapshot - .circuit_ids_by_parent_node - .get(node_name) - .into_iter() - .flat_map(|ids| ids.iter()) - .filter_map(|id| snapshot.by_circuit_id.get(id)) - .map(|row| TreeAttachedCircuitRow { - circuit_id: row.circuit_id.clone(), - circuit_name: row.circuit_name.clone(), - ethernet_cap_badge: ethernet_badges - .get(&row.circuit_id.to_ascii_lowercase()) - .cloned(), - parent_node: row.parent_node.clone(), - device_names: row.device_names.clone(), - ip_addrs: row.ip_addrs.clone(), - plan_mbps: row.plan_mbps, - bytes_per_second: row.bytes_per_second, - rtt_current_p50_nanos: row.rtt_current_p50_nanos, - qoo: row.qoo, - tcp_retransmit_sample: row.tcp_retransmit_sample, - last_seen_nanos: row.last_seen_nanos, - }) - .collect(), - None => Vec::new(), - }; + let shaped_devices = SHAPED_DEVICES.load(); + let mut rows: Vec = matcher + .as_ref() + .map(|matcher| { + aggregate_attached_circuit_rows(matcher, &shaped_devices) + .into_iter() + .map(|row| { + let live = snapshot.by_circuit_id.get(&row.circuit_id); + TreeAttachedCircuitRow { + circuit_id: row.circuit_id.clone(), + circuit_name: row.circuit_name.clone(), + ethernet_cap_badge: ethernet_badges + .get(&row.circuit_id.to_ascii_lowercase()) + .cloned(), + parent_node: live + .map(|live| live.parent_node.clone()) + .filter(|parent| !parent.trim().is_empty()) + .unwrap_or(row.parent_node), + device_names: live + .map(|live| live.device_names.clone()) + .unwrap_or_else(|| { + let mut names: Vec = + row.device_names.into_iter().collect(); + names.sort_unstable(); + names + }), + ip_addrs: live + .map(|live| live.ip_addrs.clone()) + .unwrap_or_else(|| { + let mut addrs: Vec = row.ip_addrs.into_iter().collect(); + addrs.sort_unstable(); + addrs + }), + plan_mbps: live.map(|live| live.plan_mbps).unwrap_or(row.plan_mbps), + bytes_per_second: live + .map(|live| live.bytes_per_second) + .unwrap_or_default(), + rtt_current_p50_nanos: live + .map(|live| live.rtt_current_p50_nanos) + .unwrap_or_default(), + qoo: live.map(|live| live.qoo).unwrap_or_default(), + tcp_retransmit_sample: live + .map(|live| live.tcp_retransmit_sample) + .unwrap_or_default(), + last_seen_nanos: live + .map(|live| live.last_seen_nanos) + .unwrap_or(u64::MAX), + } + }) + .collect() + }) + .unwrap_or_default(); if !search.is_empty() { rows.retain(|row| { @@ -231,3 +424,92 @@ pub fn tree_attached_circuits(query: TreeAttachedCircuitsQuery) -> TreeAttachedC rows, } } + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + + fn sample_device( + circuit_id: &str, + circuit_name: &str, + device_id: &str, + device_name: &str, + parent_node: &str, + parent_node_id: Option<&str>, + ipv4: &[&str], + ) -> ShapedDevice { + ShapedDevice { + circuit_id: circuit_id.to_string(), + circuit_name: circuit_name.to_string(), + device_id: device_id.to_string(), + device_name: device_name.to_string(), + parent_node: parent_node.to_string(), + parent_node_id: parent_node_id.map(str::to_string), + ipv4: ipv4 + .iter() + .map(|value| { + ( + Ipv4Addr::from_str(value).expect("test IPv4 addresses should parse"), + 32, + ) + }) + .collect(), + download_max_mbps: 100.0, + upload_max_mbps: 50.0, + ..ShapedDevice::default() + } + } + + #[test] + fn aggregate_attached_rows_include_idle_circuits_in_selected_subtree() { + let matcher = SubtreeMatcher { + node_names: ["Parent A".to_string(), "Child A".to_string()] + .into_iter() + .collect(), + node_ids: ["node-a".to_string(), "node-child".to_string()] + .into_iter() + .collect(), + }; + let shaped_devices = ConfigShapedDevices { + devices: vec![ + sample_device( + "circuit-1", + "Circuit One", + "device-1", + "Device One", + "Parent A", + Some("node-a"), + &["100.64.0.1"], + ), + sample_device( + "circuit-1", + "Circuit One", + "device-2", + "Device Two", + "Parent A", + Some("node-a"), + &["100.64.0.2"], + ), + sample_device( + "circuit-2", + "Circuit Two", + "device-3", + "Device Three", + "Elsewhere", + Some("node-b"), + &["100.64.0.3"], + ), + ], + ..ConfigShapedDevices::default() + }; + + let rows = aggregate_attached_circuit_rows(&matcher, &shaped_devices); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].circuit_id, "circuit-1"); + assert_eq!(rows[0].circuit_name, "Circuit One"); + assert_eq!(rows[0].parent_node, "Parent A"); + assert_eq!(rows[0].device_names.len(), 2); + assert_eq!(rows[0].ip_addrs.len(), 2); + } +} diff --git a/src/rust/lqosd/src/node_manager/static2/config_integration.html b/src/rust/lqosd/src/node_manager/static2/config_integration.html index faeb44aa..d99b7dfc 100644 --- a/src/rust/lqosd/src/node_manager/static2/config_integration.html +++ b/src/rust/lqosd/src/node_manager/static2/config_integration.html @@ -6,7 +6,7 @@
Integration Defaults
-
Set import behavior shared across integrations, including naming, overwrite policy, queue-refresh timing, and bandwidth adjustments.
+
Set behavior shared across integrations, including topology compile mode, naming, DIY overwrite policy, queue-refresh timing, and bandwidth adjustments.
@@ -14,18 +14,23 @@
Import Behavior
-
Control how imported topology and subscriber metadata are written into LibreQoS state files.
+
Control how imported topology and subscriber metadata are turned into runtime shaping state.
-
- - -
Replace IP addresses with circuit names in network.json.
+
+ + +
Shared topology mode used by Topology Manager and the scheduler for integration-driven imports.
- - -
Force overwrite of network.json on every update.
+ + +
For Python-based integrations, write the circuit address into the generated Circuit Name in ShapedDevices.csv and circuit_anchors.json instead of the customer or display name. Some integrations do not provide a real address.
diff --git a/src/rust/lqosd/src/node_manager/static2/config_netzur.html b/src/rust/lqosd/src/node_manager/static2/config_netzur.html index 917e1cbe..d185de04 100644 --- a/src/rust/lqosd/src/node_manager/static2/config_netzur.html +++ b/src/rust/lqosd/src/node_manager/static2/config_netzur.html @@ -19,7 +19,11 @@
- +
+ + +
+
Bearer token generated within Netzur
diff --git a/src/rust/lqosd/src/node_manager/static2/config_powercode.html b/src/rust/lqosd/src/node_manager/static2/config_powercode.html index 30218129..06cbdf4a 100644 --- a/src/rust/lqosd/src/node_manager/static2/config_powercode.html +++ b/src/rust/lqosd/src/node_manager/static2/config_powercode.html @@ -19,7 +19,11 @@
- +
+ + +
+
Powercode API key for authentication
diff --git a/src/rust/lqosd/src/node_manager/static2/config_sonar.html b/src/rust/lqosd/src/node_manager/static2/config_sonar.html index 073d8c38..4f4eb5f1 100644 --- a/src/rust/lqosd/src/node_manager/static2/config_sonar.html +++ b/src/rust/lqosd/src/node_manager/static2/config_sonar.html @@ -25,7 +25,11 @@
- +
+ + +
+
Sonar API key for authentication
diff --git a/src/rust/lqosd/src/node_manager/static2/config_splynx.html b/src/rust/lqosd/src/node_manager/static2/config_splynx.html index a2799946..c579cfe1 100644 --- a/src/rust/lqosd/src/node_manager/static2/config_splynx.html +++ b/src/rust/lqosd/src/node_manager/static2/config_splynx.html @@ -6,7 +6,7 @@
Splynx Integration
-
Connect LibreQoS to Splynx and choose how much hierarchy should be imported into the shaping tree.
+
Connect LibreQoS to Splynx. Shared topology mode now lives under Integration - Common.
@@ -19,13 +19,21 @@
- +
+ + +
+
Splynx API key for authentication
- +
+ + +
+
Splynx API secret for authentication
@@ -36,14 +44,14 @@
- - - - + + -
Select the topology strategy for network hierarchy. AP Only provides the best balance of performance and structure.
+
Change this under Integration - Common. Splynx still keeps the legacy field for compatibility during upgrade.
diff --git a/src/rust/lqosd/src/node_manager/static2/config_uisp.html b/src/rust/lqosd/src/node_manager/static2/config_uisp.html index 09aee65c..f7bf7b61 100644 --- a/src/rust/lqosd/src/node_manager/static2/config_uisp.html +++ b/src/rust/lqosd/src/node_manager/static2/config_uisp.html @@ -6,7 +6,7 @@
UISP Integration
-
Connect to UISP, choose import strategy, and control how topology, suspended accounts, capacity, and site overrides are handled.
+
Connect to UISP and control suspended accounts, capacity, and site overrides. Topology mode now lives under Integration - Common.
@@ -24,7 +24,11 @@
- +
+ + +
+
UISP API token for authentication.
@@ -56,18 +60,18 @@
-
Strategy and Capacity
-
Choose the imported hierarchy depth and how LibreQoS should interpret suspended accounts and device capacity.
+
Topology and Capacity
+
Review the shared topology compile mode and choose how LibreQoS should interpret suspended accounts and device capacity.
- - -
Select the topology strategy for network hierarchy.
+
Change this under Integration - Common. UISP still keeps the legacy field for compatibility during upgrade.
diff --git a/src/rust/lqosd/src/node_manager/static2/config_visp.html b/src/rust/lqosd/src/node_manager/static2/config_visp.html index efc7738e..b799b9e8 100644 --- a/src/rust/lqosd/src/node_manager/static2/config_visp.html +++ b/src/rust/lqosd/src/node_manager/static2/config_visp.html @@ -24,7 +24,11 @@
- +
+ + +
+
@@ -35,7 +39,11 @@
- +
+ + +
+
diff --git a/src/rust/lqosd/src/node_manager/ws.rs b/src/rust/lqosd/src/node_manager/ws.rs index 70110f18..4ec523ad 100644 --- a/src/rust/lqosd/src/node_manager/ws.rs +++ b/src/rust/lqosd/src/node_manager/ws.rs @@ -1485,8 +1485,12 @@ async fn receive_channel_message( } } } - WsRequest::UpdateConfig { config: cfg } => { - let result = config::update_lqosd_config_data(*request_state.login, cfg).await; + WsRequest::UpdateConfig { + config: cfg, + clear_secrets, + } => { + let result = + config::update_lqosd_config_data(*request_state.login, cfg, clear_secrets).await; let (ok, message) = match result { Ok(()) => (true, "Ok".to_string()), Err(StatusCode::FORBIDDEN) => (false, "Unauthorized".to_string()), diff --git a/src/rust/lqosd/src/node_manager/ws/messages.rs b/src/rust/lqosd/src/node_manager/ws/messages.rs index 4c618a14..1ad8e10c 100644 --- a/src/rust/lqosd/src/node_manager/ws/messages.rs +++ b/src/rust/lqosd/src/node_manager/ws/messages.rs @@ -23,6 +23,7 @@ use crate::node_manager::local_api::node_topology_overrides::{ NodeTopologyOverrideData, NodeTopologyOverrideQuery, }; use crate::node_manager::local_api::packet_analysis::RequestAnalysisResult; +use crate::node_manager::local_api::config::{ConfigSecretClearRequest, ConfigView}; use crate::node_manager::local_api::scheduler::{SchedulerDetails, SchedulerStatus}; use crate::node_manager::local_api::search::SearchResult; use crate::node_manager::local_api::shaped_devices_page::{ @@ -183,6 +184,8 @@ pub enum WsRequest { QooProfiles, UpdateConfig { config: Config, + #[serde(default)] + clear_secrets: ConfigSecretClearRequest, }, UpdateNetworkJsonOnly { network_json: Value, @@ -639,7 +642,7 @@ pub enum WsResponse { ok: bool, }, GetConfig { - data: Config, + data: ConfigView, }, QooProfiles { data: QooProfilesSummary, diff --git a/src/rust/lqosd/src/shaped_devices_tracker/circuit_live.rs b/src/rust/lqosd/src/shaped_devices_tracker/circuit_live.rs index 2ba30e1c..3d805d29 100644 --- a/src/rust/lqosd/src/shaped_devices_tracker/circuit_live.rs +++ b/src/rust/lqosd/src/shaped_devices_tracker/circuit_live.rs @@ -31,7 +31,6 @@ pub struct CircuitLiveRollup { #[derive(Clone, Debug, Default)] pub struct CircuitLiveSnapshot { pub by_circuit_id: FxHashMap, - pub circuit_ids_by_parent_node: FxHashMap>, } #[derive(Default)] @@ -165,18 +164,11 @@ pub fn rebuild_circuit_live_snapshot() -> Arc { } let mut finalized: FxHashMap = FxHashMap::default(); - let mut circuit_ids_by_parent_node: FxHashMap> = FxHashMap::default(); for (circuit_id, value) in by_circuit_id { let parent_node = super::effective_parent_for_circuit(&circuit_id) .map(|parent| parent.name) .filter(|name| !name.trim().is_empty()) .unwrap_or(value.parent_node); - if !parent_node.trim().is_empty() { - circuit_ids_by_parent_node - .entry(parent_node.clone()) - .or_default() - .push(circuit_id.clone()); - } finalized.insert( circuit_id.clone(), CircuitLiveRollup { @@ -197,13 +189,9 @@ pub fn rebuild_circuit_live_snapshot() -> Arc { }, ); } - for ids in circuit_ids_by_parent_node.values_mut() { - ids.sort_unstable(); - } let snapshot = Arc::new(CircuitLiveSnapshot { by_circuit_id: finalized, - circuit_ids_by_parent_node, }); CIRCUIT_LIVE_SNAPSHOT.store(snapshot.clone()); CIRCUIT_LIVE_LAST_REFRESH_SECS diff --git a/src/rust/lqosd/src/shaped_devices_tracker/mod.rs b/src/rust/lqosd/src/shaped_devices_tracker/mod.rs index 8610e29a..e5dc12e9 100644 --- a/src/rust/lqosd/src/shaped_devices_tracker/mod.rs +++ b/src/rust/lqosd/src/shaped_devices_tracker/mod.rs @@ -6,9 +6,10 @@ use fxhash::{FxHashMap, FxHashSet}; use lqos_bus::{BusResponse, Circuit}; use lqos_config::{ ConfigShapedDevices, NetworkJsonNode, NetworkJsonTransport, ShapedDevice, - TopologyShapingInputsFile, load_config, topology_shaping_inputs_path, + TopologyShapingInputsFile, load_config, topology_import_path, topology_shaping_inputs_path, }; use lqos_queue_tracker::EFFECTIVE_NODE_RATES; +use lqos_topology_compile::TopologyImportFile; use lqos_utils::file_watcher::FileWatcher; use lqos_utils::hash_to_i64; use lqos_utils::rtt::{FlowbeeEffectiveDirection, RttBucket}; @@ -297,10 +298,39 @@ fn publish_shaped_devices(new_file: ConfigShapedDevices) { crate::throughput_tracker::THROUGHPUT_TRACKER.refresh_circuit_ids(&nj); } +fn integration_ingress_enabled(config: &lqos_config::Config) -> bool { + config.uisp_integration.enable_uisp + || config.splynx_integration.enable_splynx + || config + .netzur_integration + .as_ref() + .is_some_and(|integration| integration.enable_netzur) + || config + .visp_integration + .as_ref() + .is_some_and(|integration| integration.enable_visp) + || config.powercode_integration.enable_powercode + || config.sonar_integration.enable_sonar + || config + .wispgate_integration + .as_ref() + .is_some_and(|integration| integration.enable_wispgate) +} + +fn load_shaped_devices_from_active_ingress() -> Result { + let config = load_config()?; + if integration_ingress_enabled(config.as_ref()) + && let Some(topology_import) = TopologyImportFile::load(config.as_ref())? + { + return Ok(topology_import.into_imported_bundle().shaped_devices); + } + Ok(ConfigShapedDevices::load()?) +} + fn load_shaped_devices() { - debug!("ShapedDevices.csv has changed. Attempting to load it."); + debug!("Shaped-device ingress has changed. Attempting to load it."); for attempt in 1..=SHAPED_DEVICES_RELOAD_ATTEMPTS { - match ConfigShapedDevices::load() { + match load_shaped_devices_from_active_ingress() { Ok(new_file) => { publish_shaped_devices(new_file); return; @@ -308,14 +338,14 @@ fn load_shaped_devices() { Err(err) => { if attempt < SHAPED_DEVICES_RELOAD_ATTEMPTS { warn!( - "ShapedDevices.csv reload attempt {attempt}/{} failed: {err}. Retrying after {} ms.", + "Shaped-device ingress reload attempt {attempt}/{} failed: {err}. Retrying after {} ms.", SHAPED_DEVICES_RELOAD_ATTEMPTS, SHAPED_DEVICES_RELOAD_RETRY_DELAY_MS ); std::thread::sleep(Duration::from_millis(SHAPED_DEVICES_RELOAD_RETRY_DELAY_MS)); } else { let current = SHAPED_DEVICES.load(); warn!( - "ShapedDevices.csv reload failed after {} attempts: {err}. Keeping last-known-good data with {} devices.", + "Shaped-device ingress reload failed after {} attempts: {err}. Keeping last-known-good data with {} devices.", SHAPED_DEVICES_RELOAD_ATTEMPTS, current.devices.len() ); @@ -329,9 +359,9 @@ pub fn shaped_devices_watcher() -> Result<()> { std::thread::Builder::new() .name("ShapedDevices Watcher".to_string()) .spawn(|| { - debug!("Watching for ShapedDevices.csv changes"); + debug!("Watching for shaped-device ingress changes"); if let Err(e) = watch_for_shaped_devices_changing() { - error!("Error watching for ShapedDevices.csv: {:?}", e); + error!("Error watching for shaped-device ingress: {:?}", e); } })?; Ok(()) @@ -340,16 +370,17 @@ pub fn shaped_devices_watcher() -> Result<()> { /// Fires up a Linux file system watcher than notifies /// when `ShapedDevices.csv` changes, and triggers a reload. fn watch_for_shaped_devices_changing() -> Result<()> { - let watch_path = ConfigShapedDevices::path(); - if watch_path.is_err() { - error!("Unable to generate path for ShapedDevices.csv"); - return Err(anyhow::Error::msg( - "Unable to create path for ShapedDevices.csv", - )); - } - let watch_path = watch_path?; + let config = load_config()?; + let (watch_name, watch_path) = if integration_ingress_enabled(config.as_ref()) { + ( + "topology_import.json", + topology_import_path(config.as_ref()), + ) + } else { + ("ShapedDevices.csv", ConfigShapedDevices::path()?) + }; - let mut watcher = FileWatcher::new("ShapedDevices.csv", watch_path); + let mut watcher = FileWatcher::new(watch_name, watch_path); watcher.set_file_exists_callback(load_shaped_devices); watcher.set_file_created_callback(load_shaped_devices); watcher.set_file_changed_callback(load_shaped_devices); diff --git a/src/rust/lqosd/src/shaped_devices_tracker/netjson.rs b/src/rust/lqosd/src/shaped_devices_tracker/netjson.rs index 5b263925..7729d427 100644 --- a/src/rust/lqosd/src/shaped_devices_tracker/netjson.rs +++ b/src/rust/lqosd/src/shaped_devices_tracker/netjson.rs @@ -3,6 +3,7 @@ use lqos_config::NetworkJson; use lqos_utils::file_watcher::FileWatcher; use once_cell::sync::Lazy; use parking_lot::RwLock; +use std::path::PathBuf; use tracing::{debug, error, info, warn}; pub static NETWORK_JSON: Lazy> = @@ -10,32 +11,37 @@ pub static NETWORK_JSON: Lazy> = pub fn network_json_watcher() -> Result<()> { std::thread::Builder::new() - .name("NetJson Watcher".to_string()) + .name("Active Network Tree Watcher".to_string()) .spawn(|| { - debug!("Watching for network.kson changes"); + debug!("Watching for active network tree changes"); if let Err(e) = watch_for_network_json_changing() { - error!("Error watching for network.json changes: {:?}", e); + error!("Error watching for active network tree changes: {:?}", e); } })?; Ok(()) } -/// Fires up a Linux file system watcher that notifies -/// when `network.json` changes and triggers a reload. -fn watch_for_network_json_changing() -> Result<()> { - let watch_path = NetworkJson::path(); - let Ok(watch_path) = watch_path else { - error!("Unable to generate path for network.json"); - return Err(anyhow::Error::msg("Unable to create path for network.json")); - }; +fn active_network_tree_path() -> Result { + NetworkJson::path().map_err(|_| anyhow::Error::msg("Unable to create path for active network tree")) +} - let mut watcher = FileWatcher::new("network.json", watch_path); +/// Fires up a Linux file system watcher that notifies +/// when the active runtime tree changes and triggers a reload. +fn watch_for_network_json_changing() -> Result<()> { + let watch_path = active_network_tree_path()?; + let watch_name = watch_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("active network tree") + .to_string(); + + let mut watcher = FileWatcher::new(&watch_name, watch_path); watcher.set_file_exists_callback(load_network_json); watcher.set_file_created_callback(load_network_json); watcher.set_file_changed_callback(load_network_json); loop { let result = watcher.watch(); - info!("network.json watcher returned: {result:?}"); + info!("active network tree watcher returned: {result:?}"); } } @@ -48,6 +54,6 @@ fn load_network_json() { super::invalidate_executive_cache_snapshot(); crate::throughput_tracker::THROUGHPUT_TRACKER.refresh_circuit_ids(&nj); } else { - warn!("Unable to load network.json"); + warn!("Unable to load active runtime network tree"); } } diff --git a/src/rust/lqosd/src/throughput_tracker/stats_submission.rs b/src/rust/lqosd/src/throughput_tracker/stats_submission.rs index d3724822..1f100d0e 100644 --- a/src/rust/lqosd/src/throughput_tracker/stats_submission.rs +++ b/src/rust/lqosd/src/throughput_tracker/stats_submission.rs @@ -12,6 +12,7 @@ use csv::ReaderBuilder; use fxhash::{FxHashMap, FxHashSet}; use lqos_config::{ShapedDevice, TopologyCanonicalStateFile, load_config}; use lqos_queue_tracker::{ALL_QUEUE_SUMMARY, TOTAL_QUEUE_STATS}; +use lqos_topology_compile::TopologyImportFile; use lqos_utils::hash_to_i64; use lqos_utils::units::DownUpOrder; use lqos_utils::unix_time::unix_now; @@ -588,10 +589,21 @@ fn ip6_to_bytes(ip: (Ipv6Addr, u32)) -> ([u8; 16], u8) { /// runtime-effective topology or local compatibility tree. fn combined_devices_network_hash() -> anyhow::Result { let cfg = load_config()?; - let sd_path = std::path::Path::new(&cfg.lqos_directory).join("ShapedDevices.csv"); let nj_as_string = load_insight_logical_network_json()?; - let sd_as_string = read_to_string(sd_path)?; + let sd_as_string = if integration_ingress_enabled(cfg.as_ref()) { + let topology_import = TopologyImportFile::load(cfg.as_ref())? + .ok_or_else(|| anyhow::anyhow!("topology_import.json does not exist"))?; + serde_json::to_string( + &topology_import + .into_imported_bundle() + .shaped_devices + .devices, + )? + } else { + let sd_path = std::path::Path::new(&cfg.lqos_directory).join("ShapedDevices.csv"); + read_to_string(sd_path)? + }; let combined = format!("{}\n{}", nj_as_string, sd_as_string); let hash = hash_to_i64(&combined); @@ -612,6 +624,14 @@ static LTS2_HASH: AtomicI64 = AtomicI64::new(0); /// transmission to Insight fn load_local_shaped_devices() -> anyhow::Result> { let cfg = load_config()?; + if integration_ingress_enabled(cfg.as_ref()) { + let topology_import = TopologyImportFile::load(cfg.as_ref())? + .ok_or_else(|| anyhow::anyhow!("topology_import.json does not exist"))?; + return Ok(topology_import + .into_imported_bundle() + .shaped_devices + .devices); + } let sd_path = Path::new(&cfg.lqos_directory).join("ShapedDevices.csv"); if !sd_path.exists() { anyhow::bail!("ShapedDevices.csv does not exist"); @@ -634,6 +654,25 @@ fn load_local_shaped_devices() -> anyhow::Result> { Ok(devices) } +fn integration_ingress_enabled(config: &lqos_config::Config) -> bool { + config.uisp_integration.enable_uisp + || config.splynx_integration.enable_splynx + || config + .netzur_integration + .as_ref() + .is_some_and(|integration| integration.enable_netzur) + || config + .visp_integration + .as_ref() + .is_some_and(|integration| integration.enable_visp) + || config.powercode_integration.enable_powercode + || config.sonar_integration.enable_sonar + || config + .wispgate_integration + .as_ref() + .is_some_and(|integration| integration.enable_wispgate) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/rust/lqosd/src/validation.rs b/src/rust/lqosd/src/validation.rs index f604c548..de6b5516 100644 --- a/src/rust/lqosd/src/validation.rs +++ b/src/rust/lqosd/src/validation.rs @@ -1,10 +1,45 @@ use lqos_bus::BusResponse; use lqos_config::ConfigShapedDevices; +use lqos_topology_compile::TopologyImportFile; pub fn validate_shaped_devices_csv() -> BusResponse { - let result = ConfigShapedDevices::load(); + let result: anyhow::Result = match lqos_config::load_config() { + Ok(config) => { + if integration_ingress_enabled(config.as_ref()) { + match TopologyImportFile::load(config.as_ref()) { + Ok(Some(topology_import)) => { + Ok(topology_import.into_imported_bundle().shaped_devices) + } + Ok(None) => Ok(ConfigShapedDevices::default()), + Err(err) => Err(err), + } + } else { + ConfigShapedDevices::load().map_err(anyhow::Error::from) + } + } + Err(err) => Err(anyhow::Error::from(err)), + }; match result { Ok(..) => BusResponse::Ack, Err(e) => BusResponse::ShapedDevicesValidation(format!("{e:#?}")), } } + +fn integration_ingress_enabled(config: &lqos_config::Config) -> bool { + config.uisp_integration.enable_uisp + || config.splynx_integration.enable_splynx + || config + .netzur_integration + .as_ref() + .is_some_and(|integration| integration.enable_netzur) + || config + .visp_integration + .as_ref() + .is_some_and(|integration| integration.enable_visp) + || config.powercode_integration.enable_powercode + || config.sonar_integration.enable_sonar + || config + .wispgate_integration + .as_ref() + .is_some_and(|integration| integration.enable_wispgate) +} diff --git a/src/rust/uisp_integration/Cargo.toml b/src/rust/uisp_integration/Cargo.toml index b7de1961..4d5c5407 100644 --- a/src/rust/uisp_integration/Cargo.toml +++ b/src/rust/uisp_integration/Cargo.toml @@ -23,3 +23,4 @@ serde_cbor = { workspace = true } lqos_bus = { path = "../lqos_bus" } petgraph = { version = "0.7.1", features = ["serde-1"] } lqos_overrides = { path = "../lqos_overrides" } +lqos_topology_compile = { path = "../lqos_topology_compile" } diff --git a/src/rust/uisp_integration/src/strategies/ap_only.rs b/src/rust/uisp_integration/src/strategies/ap_only.rs index db9c17e8..5124a8db 100644 --- a/src/rust/uisp_integration/src/strategies/ap_only.rs +++ b/src/rust/uisp_integration/src/strategies/ap_only.rs @@ -12,79 +12,132 @@ use std::path::Path; use std::sync::Arc; use tracing::{error, info, warn}; -fn build_ap_only_network_json( +#[derive(Clone, Debug, PartialEq, Eq)] +struct ExportedApNode { + node_id: String, + node_name: String, + uisp_device_id: String, + download_mbps: u64, + upload_mbps: u64, +} + +fn short_device_id(device_id: &str) -> &str { + let trimmed = device_id.trim(); + let len = trimmed.len().min(8); + &trimmed[..len] +} + +fn disambiguate_exported_ap_names(exported: &mut HashMap) { + let mut ids_by_name = HashMap::>::new(); + for node in exported.values() { + ids_by_name + .entry(node.node_name.clone()) + .or_default() + .push(node.uisp_device_id.clone()); + } + + let mut used_names = HashSet::::new(); + for ids in ids_by_name.values_mut() { + ids.sort(); + for device_id in ids.iter() { + let Some(node) = exported.get_mut(device_id) else { + continue; + }; + let base_name = node.node_name.trim(); + if ids.len() == 1 && used_names.insert(base_name.to_string()) { + continue; + } + + let mut candidate = format!("{base_name} ({})", short_device_id(device_id)); + if used_names.contains(&candidate) { + candidate = format!("{base_name} ({device_id})"); + } + used_names.insert(candidate.clone()); + node.node_name = candidate; + } + } +} + +fn build_exported_ap_nodes( uisp_data: &UispData, mappings: &HashMap>, -) -> serde_json::Map { - let mut root = serde_json::Map::new(); +) -> HashMap { + let mut exported = HashMap::new(); for ap_id in mappings.keys() { let Some(ap_device) = uisp_data.find_uisp_device_by_id(ap_id) else { - warn!("Unable to find AP device for mapping key {ap_id}"); + warn!("Unable to export AP-only parent mapping key {ap_id} into network.json"); continue; }; - let ap_name = uisp_data.device_display_name(ap_id); + exported.insert( + ap_id.clone(), + ExportedApNode { + node_id: format!("uisp:device:{}", ap_device.id), + node_name: uisp_data.device_display_name(ap_id), + uisp_device_id: ap_device.id.clone(), + download_mbps: ap_device.download, + upload_mbps: ap_device.upload, + }, + ); + } + disambiguate_exported_ap_names(&mut exported); + exported +} +fn build_ap_only_network_json( + exported_aps: &HashMap, +) -> serde_json::Map { + let mut root = serde_json::Map::new(); + let mut exported_nodes = exported_aps.values().collect::>(); + exported_nodes.sort_unstable_by(|left, right| left.node_name.cmp(&right.node_name)); + for exported in exported_nodes { let mut ap_object = serde_json::Map::new(); ap_object.insert("children".to_string(), serde_json::Map::new().into()); ap_object.insert( "downloadBandwidthMbps".to_string(), - serde_json::Value::Number(ap_device.download.into()), + serde_json::Value::Number(exported.download_mbps.into()), ); ap_object.insert( "uploadBandwidthMbps".to_string(), - serde_json::Value::Number(ap_device.upload.into()), + serde_json::Value::Number(exported.upload_mbps.into()), ); ap_object.insert("type".to_string(), "AP".to_string().into()); + ap_object.insert("id".to_string(), exported.node_id.clone().into()); ap_object.insert( - "id".to_string(), - format!("uisp:device:{}", ap_device.id).into(), + "uisp_device".to_string(), + exported.uisp_device_id.clone().into(), ); - ap_object.insert("uisp_device".to_string(), ap_device.id.clone().into()); - root.insert(ap_name, ap_object.into()); + root.insert(exported.node_name.clone(), ap_object.into()); } root } -/// Creates a network with only APs detected -/// from clients. -pub async fn build_ap_only_network( - config: Arc, - ip_ranges: IpRanges, -) -> Result<(), UispIntegrationError> { - let uisp_data = UispData::fetch_uisp_data(config.clone(), ip_ranges).await?; - let ethernet_policy = EthernetPortLimitPolicy::from(&config.integration_common); - - // Find the clients - let mappings = uisp_data.map_clients_to_aps(); - - // Write network.json - let network_path = Path::new(&config.lqos_directory).join("network.json"); - if network_path.exists() && !config.integration_common.always_overwrite_network_json { - warn!( - "Network.json exists, and always overwrite network json is not true - not writing network.json" - ); - } else { - let root = build_ap_only_network_json(&uisp_data, &mappings); - let json = serde_json::to_string_pretty(&root).unwrap(); - write(network_path, json).map_err(|e| { - error!("Unable to write network.json"); - error!("{e:?}"); - UispIntegrationError::WriteNetJson - })?; - info!("Written network.json"); - } - - // Write ShapedDevices.csv - let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); +fn build_ap_only_shaping_outputs( + config: &Config, + ethernet_policy: EthernetPortLimitPolicy, + uisp_data: &UispData, + mappings: &HashMap>, + exported_aps: &HashMap, +) -> ( + Vec, + Vec, + Vec, +) { let mut shaped_devices = Vec::new(); let mut circuit_anchors = Vec::::new(); let mut ethernet_advisories: Vec = Vec::new(); let mut seen_pairs = HashSet::new(); let mut seen_circuits = HashSet::new(); - for (parent_id, client_ids) in mappings.iter() { - let parent_name = uisp_data.device_display_name(parent_id); - let parent_node_id = format!("uisp:device:{parent_id}"); + + for (parent_id, client_ids) in mappings { + let exported_parent = exported_aps.get(parent_id); + let parent_name = exported_parent + .map(|parent| parent.node_name.clone()) + .unwrap_or_default(); + let parent_node_id = exported_parent + .map(|parent| parent.node_id.clone()) + .unwrap_or_default(); + for client_id in client_ids { let site = uisp_data.sites.iter().find(|s| *client_id == s.id).unwrap(); let devices = uisp_data @@ -92,8 +145,7 @@ pub async fn build_ap_only_network( .iter() .filter(|d| d.site_id == *client_id) .collect::>(); - let requested = if let Some((dl_min, dl_max, ul_min, ul_max)) = - site.burst_rates(&config) + let requested = if let Some((dl_min, dl_max, ul_min, ul_max)) = site.burst_rates(config) { ( f32::max(0.1, dl_min), @@ -179,6 +231,43 @@ pub async fn build_ap_only_network( } } } + + (shaped_devices, circuit_anchors, ethernet_advisories) +} + +/// Creates a network with only APs detected +/// from clients. +pub async fn build_ap_only_network( + config: Arc, + ip_ranges: IpRanges, +) -> Result<(), UispIntegrationError> { + let uisp_data = UispData::fetch_uisp_data(config.clone(), ip_ranges).await?; + let ethernet_policy = EthernetPortLimitPolicy::from(&config.integration_common); + + // Find the clients + let mappings = uisp_data.map_clients_to_aps(); + let exported_aps = build_exported_ap_nodes(&uisp_data, &mappings); + + // Write network.json + let network_path = Path::new(&config.lqos_directory).join("network.json"); + let root = build_ap_only_network_json(&exported_aps); + let json = serde_json::to_string_pretty(&root).unwrap(); + write(network_path, json).map_err(|e| { + error!("Unable to write network.json"); + error!("{e:?}"); + UispIntegrationError::WriteNetJson + })?; + info!("Written network.json"); + + // Write ShapedDevices.csv + let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); + let (shaped_devices, circuit_anchors, ethernet_advisories) = build_ap_only_shaping_outputs( + &config, + ethernet_policy, + &uisp_data, + &mappings, + &exported_aps, + ); let mut writer = csv::WriterBuilder::new() .has_headers(true) .from_path(file_path) @@ -201,9 +290,12 @@ pub async fn build_ap_only_network( #[cfg(test)] mod tests { - use super::build_ap_only_network_json; + use super::{ + build_ap_only_network_json, build_ap_only_shaping_outputs, build_exported_ap_nodes, + }; use crate::strategies::common::UispData; - use crate::uisp_types::{UispAttachmentRateSource, UispDevice, UispSite}; + use crate::uisp_types::{UispAttachmentRateSource, UispDevice, UispSite, UispSiteType}; + use lqos_config::{Config, EthernetPortLimitPolicy}; use std::collections::{HashMap, HashSet}; #[test] @@ -240,8 +332,9 @@ mod tests { }], ); let mappings = HashMap::from([("ap-1".to_string(), vec!["client-site".to_string()])]); + let exported_aps = build_exported_ap_nodes(&uisp_data, &mappings); - let network_json = build_ap_only_network_json(&uisp_data, &mappings); + let network_json = build_ap_only_network_json(&exported_aps); assert!(network_json.get("Tower AP").is_some()); let ap = network_json @@ -250,4 +343,209 @@ mod tests { .unwrap(); assert_eq!(ap.get("uisp_device").and_then(|v| v.as_str()), Some("ap-1")); } + + #[test] + fn ap_only_shaping_outputs_skip_non_exported_parent_mappings() { + let uisp_data = UispData::from_parts( + vec![], + vec![], + vec![], + vec![ + UispSite { + id: "client-valid".to_string(), + name: "Client Valid".to_string(), + site_type: UispSiteType::Client, + max_down_mbps: 100, + max_up_mbps: 20, + ..Default::default() + }, + UispSite { + id: "client-orphan".to_string(), + name: "Client Orphan".to_string(), + site_type: UispSiteType::Client, + max_down_mbps: 80, + max_up_mbps: 10, + ..Default::default() + }, + ], + vec![ + UispDevice { + id: "ap-1".to_string(), + name: "Tower AP".to_string(), + mac: "".to_string(), + role: None, + wireless_mode: None, + site_id: "tower-site".to_string(), + raw_download: 500, + raw_upload: 400, + download: 500, + upload: 400, + ipv4: HashSet::new(), + ipv6: HashSet::new(), + probe_ipv4: HashSet::new(), + probe_ipv6: HashSet::new(), + negotiated_ethernet_mbps: None, + negotiated_ethernet_interface: None, + transport_cap_mbps: None, + transport_cap_reason: None, + attachment_rate_source: UispAttachmentRateSource::Static, + }, + UispDevice { + id: "device-valid".to_string(), + name: "Valid Device".to_string(), + mac: "aa:bb:cc:dd:ee:01".to_string(), + role: None, + wireless_mode: None, + site_id: "client-valid".to_string(), + raw_download: 100, + raw_upload: 20, + download: 100, + upload: 20, + ipv4: HashSet::from(["192.0.2.10/32".to_string()]), + ipv6: HashSet::new(), + probe_ipv4: HashSet::new(), + probe_ipv6: HashSet::new(), + negotiated_ethernet_mbps: None, + negotiated_ethernet_interface: None, + transport_cap_mbps: None, + transport_cap_reason: None, + attachment_rate_source: UispAttachmentRateSource::Static, + }, + UispDevice { + id: "device-orphan".to_string(), + name: "Orphan Device".to_string(), + mac: "aa:bb:cc:dd:ee:02".to_string(), + role: None, + wireless_mode: None, + site_id: "client-orphan".to_string(), + raw_download: 80, + raw_upload: 10, + download: 80, + upload: 10, + ipv4: HashSet::from(["192.0.2.20/32".to_string()]), + ipv6: HashSet::new(), + probe_ipv4: HashSet::new(), + probe_ipv6: HashSet::new(), + negotiated_ethernet_mbps: None, + negotiated_ethernet_interface: None, + transport_cap_mbps: None, + transport_cap_reason: None, + attachment_rate_source: UispAttachmentRateSource::Static, + }, + ], + ); + let mappings = HashMap::from([ + ("ap-1".to_string(), vec!["client-valid".to_string()]), + ("Orphans".to_string(), vec!["client-orphan".to_string()]), + ]); + let exported_aps = build_exported_ap_nodes(&uisp_data, &mappings); + + let (shaped_devices, circuit_anchors, _ethernet_advisories) = build_ap_only_shaping_outputs( + &Config::default(), + EthernetPortLimitPolicy::default(), + &uisp_data, + &mappings, + &exported_aps, + ); + + assert_eq!(circuit_anchors.len(), 1); + assert_eq!(circuit_anchors[0].circuit_id, "client-valid"); + assert_eq!(circuit_anchors[0].anchor_node_id, "uisp:device:ap-1"); + + let valid_device = shaped_devices + .iter() + .find(|device| device.device_id == "device-valid") + .expect("expected valid device"); + assert_eq!(valid_device.parent_node, "Tower AP"); + assert_eq!(valid_device.parent_node_id, "uisp:device:ap-1"); + + let orphan_device = shaped_devices + .iter() + .find(|device| device.device_id == "device-orphan") + .expect("expected orphan device"); + assert!(orphan_device.parent_node.is_empty()); + assert!(orphan_device.parent_node_id.is_empty()); + } + + #[test] + fn ap_only_exported_ap_names_are_disambiguated_when_display_names_collide() { + let uisp_data = UispData::from_parts( + vec![], + vec![], + vec![], + vec![], + vec![ + UispDevice { + id: "ap-dup-1-12345678".to_string(), + name: "Shared AP".to_string(), + mac: "".to_string(), + role: None, + wireless_mode: None, + site_id: "site-a".to_string(), + raw_download: 100, + raw_upload: 100, + download: 100, + upload: 100, + ipv4: HashSet::new(), + ipv6: HashSet::new(), + probe_ipv4: HashSet::new(), + probe_ipv6: HashSet::new(), + negotiated_ethernet_mbps: None, + negotiated_ethernet_interface: None, + transport_cap_mbps: None, + transport_cap_reason: None, + attachment_rate_source: UispAttachmentRateSource::Static, + }, + UispDevice { + id: "ap-dup-2-87654321".to_string(), + name: "Shared AP".to_string(), + mac: "".to_string(), + role: None, + wireless_mode: None, + site_id: "site-b".to_string(), + raw_download: 90, + raw_upload: 90, + download: 90, + upload: 90, + ipv4: HashSet::new(), + ipv6: HashSet::new(), + probe_ipv4: HashSet::new(), + probe_ipv6: HashSet::new(), + negotiated_ethernet_mbps: None, + negotiated_ethernet_interface: None, + transport_cap_mbps: None, + transport_cap_reason: None, + attachment_rate_source: UispAttachmentRateSource::Static, + }, + ], + ); + let mappings = HashMap::from([ + ( + "ap-dup-1-12345678".to_string(), + vec!["client-a".to_string()], + ), + ( + "ap-dup-2-87654321".to_string(), + vec!["client-b".to_string()], + ), + ]); + + let exported_aps = build_exported_ap_nodes(&uisp_data, &mappings); + let network_json = build_ap_only_network_json(&exported_aps); + + assert_eq!(network_json.len(), 2); + assert!( + network_json + .keys() + .all(|name| name.starts_with("Shared AP (")) + ); + assert!( + network_json.contains_key("Shared AP (ap-dup-2)") + || network_json.contains_key("Shared AP (ap-dup-2-87654321)") + ); + assert!( + network_json.contains_key("Shared AP (ap-dup-1)") + || network_json.contains_key("Shared AP (ap-dup-1-12345678)") + ); + } } diff --git a/src/rust/uisp_integration/src/strategies/ap_site.rs b/src/rust/uisp_integration/src/strategies/ap_site.rs index 1d00b792..52a0b770 100644 --- a/src/rust/uisp_integration/src/strategies/ap_site.rs +++ b/src/rust/uisp_integration/src/strategies/ap_site.rs @@ -71,19 +71,13 @@ pub async fn build_ap_site_network( ); let network_path = Path::new(&config.lqos_directory).join("network.json"); - if network_path.exists() && !config.integration_common.always_overwrite_network_json { - warn!( - "Network.json exists, and always overwrite network json is not true - not writing network.json" - ); - } else { - let json = serde_json::to_string_pretty(&net_json).unwrap(); - write(network_path, json).map_err(|e| { - error!("Unable to write network.json"); - error!("{e:?}"); - UispIntegrationError::WriteNetJson - })?; - info!("Written network.json"); - } + let json = serde_json::to_string_pretty(&net_json).unwrap(); + write(network_path, json).map_err(|e| { + error!("Unable to write network.json"); + error!("{e:?}"); + UispIntegrationError::WriteNetJson + })?; + info!("Written network.json"); let _ = write_shaped_devices(&config, &mut shaped_devices, ðernet_advisories); info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len()); @@ -142,8 +136,7 @@ pub(crate) fn map_sites_above_aps( let mut sites = HashMap::new(); for (ap_id, client_ids) in ap_mappings.iter() { if let Some(device) = uisp_data.find_uisp_device_by_id(ap_id) { - if let Some(device_site) = uisp_data.find_site_by_id(&device.site_id) - { + if let Some(device_site) = uisp_data.find_site_by_id(&device.site_id) { let site_entry = sites .entry(device_site.name.clone()) .or_insert_with(|| Layer { diff --git a/src/rust/uisp_integration/src/strategies/common.rs b/src/rust/uisp_integration/src/strategies/common.rs index 1f04af07..437d8ec7 100644 --- a/src/rust/uisp_integration/src/strategies/common.rs +++ b/src/rust/uisp_integration/src/strategies/common.rs @@ -1,8 +1,8 @@ use crate::blackboard_blob; use crate::errors::UispIntegrationError; use crate::ip_ranges::IpRanges; -use crate::strategies::full::parse::parse_uisp_datasets; -use crate::strategies::full::uisp_fetch::load_uisp_data; +use crate::strategies::legacy_parse::parse_uisp_datasets; +use crate::strategies::legacy_uisp_fetch::load_uisp_data; use crate::uisp_types::{UispDevice, UispSite, UispSiteType}; use lqos_config::Config; use serde_json::Value; @@ -351,7 +351,7 @@ impl UispData { } // If Mikrotik is enabled, we need to fetch the Mikrotik data - let ipv4_to_v6 = crate::strategies::full::mikrotik::mikrotik_data(&config) + let ipv4_to_v6 = crate::strategies::legacy_mikrotik::mikrotik_data(&config) .await .unwrap_or_else(|_| Vec::new()); @@ -426,22 +426,6 @@ impl UispData { .collect() } - pub fn find_uisp_device_by_id(&self, device_id: &str) -> Option<&UispDevice> { - self.find_parsed_device_by_id(device_id) - } - - pub fn device_display_name(&self, device_id: &str) -> String { - self.find_uisp_device_by_id(device_id) - .map(|device| { - if device.name.trim().is_empty() { - device.id.clone() - } else { - device.name.clone() - } - }) - .unwrap_or_else(|| device_id.to_string()) - } - pub fn map_clients_to_aps(&self) -> HashMap> { let started = Instant::now(); let client_site_count = self.find_client_sites().len(); diff --git a/src/rust/uisp_integration/src/strategies/full/bandwidth_overrides.rs b/src/rust/uisp_integration/src/strategies/full/bandwidth_overrides.rs index 6a1122ec..28ed9341 100644 --- a/src/rust/uisp_integration/src/strategies/full/bandwidth_overrides.rs +++ b/src/rust/uisp_integration/src/strategies/full/bandwidth_overrides.rs @@ -54,15 +54,18 @@ pub fn get_site_bandwidth_overrides( ); } - let mut materialized = materialize_operator_site_bandwidth_overrides(&operator_overrides); - if !materialized.is_empty() { - let legacy_path = legacy_bandwidth_csv_path(config); - if legacy_path.exists() { - warn!( - "Legacy {} is present but operator rate overrides already exist in lqos_overrides.json; the CSV is ignored.", - LEGACY_UISP_BANDWIDTH_FILE - ); + if let Some(migrated) = + try_migrate_legacy_csv_to_operator_overrides(config, &mut operator_overrides)? + { + let materialized = materialize_operator_site_bandwidth_overrides(&operator_overrides); + if materialized.is_empty() { + return Ok(migrated); } + return Ok(materialized); + } + + let materialized = materialize_operator_site_bandwidth_overrides(&operator_overrides); + if !materialized.is_empty() { info!( "Using {} UISP bandwidth override(s) from operator AdjustSiteSpeed entries in lqos_overrides.json", materialized.len() @@ -70,16 +73,6 @@ pub fn get_site_bandwidth_overrides( return Ok(materialized); } - if let Some(migrated) = - try_migrate_legacy_csv_to_operator_overrides(config, &mut operator_overrides)? - { - materialized = materialize_operator_site_bandwidth_overrides(&operator_overrides); - if materialized.is_empty() { - return Ok(migrated); - } - return Ok(materialized); - } - info!("No UISP bandwidth overrides loaded."); Ok(Vec::new()) } @@ -138,6 +131,19 @@ fn try_migrate_legacy_csv_to_operator_overrides( } for override_row in &migrated { + let already_present = operator_overrides + .network_adjustments() + .iter() + .any(|adjustment| { + matches!( + adjustment, + NetworkAdjustment::AdjustSiteSpeed { site_name, .. } + if site_name == &override_row.site_name + ) + }); + if already_present { + continue; + } operator_overrides.set_site_bandwidth_override( None, override_row.site_name.clone(), @@ -233,8 +239,11 @@ fn rename_legacy_csv_to_backup(file_path: &Path) -> std::io::Result<()> { .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_secs(); - let backup_name = format!("{LEGACY_UISP_BANDWIDTH_FILE}.migrated-{unix_secs}"); - let backup_path = file_path.with_file_name(backup_name); + let mut backup_path = file_path.with_file_name(format!("{LEGACY_UISP_BANDWIDTH_FILE}.backup")); + if backup_path.exists() { + backup_path = + file_path.with_file_name(format!("{LEGACY_UISP_BANDWIDTH_FILE}.backup-{unix_secs}")); + } fs::rename(file_path, backup_path) } @@ -374,7 +383,8 @@ mod test { upload_bandwidth_mbps: Some(45.0), }] ); - assert!(legacy_bandwidth_csv_path(&config).exists()); + assert!(!legacy_bandwidth_csv_path(&config).exists()); + assert!(dir.join("integrationUISPbandwidths.csv.backup").exists()); } #[test] diff --git a/src/rust/uisp_integration/src/strategies/full/network_json.rs b/src/rust/uisp_integration/src/strategies/full/network_json.rs index c1e32011..cabe6946 100644 --- a/src/rust/uisp_integration/src/strategies/full/network_json.rs +++ b/src/rust/uisp_integration/src/strategies/full/network_json.rs @@ -21,12 +21,6 @@ pub fn write_network_file( root_idx: usize, ) -> Result<(), UispIntegrationError> { let network_path = Path::new(&config.lqos_directory).join("network.json"); - if network_path.exists() && !config.integration_common.always_overwrite_network_json { - tracing::warn!( - "Network.json exists, and always overwrite network json is not true - not writing network.json" - ); - return Ok(()); - } // Write the network JSON file let root = traverse_sites(sites, root_idx, 0)?; diff --git a/src/rust/uisp_integration/src/strategies/full2.rs b/src/rust/uisp_integration/src/strategies/full2.rs index f1f02c9e..94c076aa 100644 --- a/src/rust/uisp_integration/src/strategies/full2.rs +++ b/src/rust/uisp_integration/src/strategies/full2.rs @@ -6,12 +6,13 @@ mod net_json_parent; use crate::blackboard_blob; use crate::errors::UispIntegrationError; -use crate::ethernet_advisory::{apply_ethernet_rate_cap, write_ethernet_advisories}; +use crate::ethernet_advisory::apply_ethernet_rate_cap; use crate::ip_ranges::IpRanges; use crate::strategies::common::UispData; -use crate::strategies::full::bandwidth_overrides::{BandwidthOverride, find_bandwidth_override}; -use crate::strategies::full::routes_override::RouteOverride; -use crate::strategies::full::shaped_devices_writer::{ShapedDevice, write_circuit_anchors}; +use crate::strategies::legacy_bandwidth_overrides::{ + BandwidthOverride, find_bandwidth_override, +}; +use crate::strategies::legacy_routes_override::RouteOverride; use crate::strategies::full2::directionality::{ build_device_capacity_map, build_device_link_meta_map, directed_caps_mbps, }; @@ -21,7 +22,8 @@ use crate::strategies::full2::link_mapping::LinkMapping; use crate::strategies::full2::net_json_parent::{NetJsonParent, assign_export_names, walk_parents}; use crate::uisp_types::{UispAttachmentRateSource, UispDevice}; use lqos_config::{ - CircuitAnchor, CircuitEthernetMetadata, Config, EthernetPortLimitPolicy, RequestedCircuitRates, + CircuitAnchor, CircuitAnchorsFile, CircuitEthernetMetadata, Config, ConfigShapedDevices, + EthernetPortLimitPolicy, RequestedCircuitRates, ShapedDevice as ConfigShapedDevice, TOPOLOGY_ATTACHMENT_AUTO_ID, TopologyAllowedParent, TopologyAttachmentOption, TopologyAttachmentRateSource, TopologyAttachmentRole, TopologyCanonicalIngressKind, TopologyCanonicalStateFile, TopologyEditorNode, TopologyEditorStateFile, @@ -29,21 +31,38 @@ use lqos_config::{ }; #[cfg(test)] use lqos_overrides::{TopologyAttachmentMode, TopologyParentOverrideMode}; +use lqos_topology_compile::ImportedTopologyBundle; use petgraph::Directed; use petgraph::graph::NodeIndex; use petgraph::visit::{EdgeRef, NodeRef}; use serde_json::Value; use std::collections::{HashMap, HashSet}; use std::env; -use std::fs::File; -use std::io::Write; -use std::path::Path; use std::sync::Arc; use std::time::Instant; -use tracing::{debug, error, info, warn}; +use tracing::{debug, info, warn}; type GraphType = petgraph::Graph; +#[derive(Clone, Debug)] +struct LegacyShapedDevice { + circuit_id: String, + circuit_name: String, + device_id: String, + device_name: String, + parent_node: String, + parent_node_id: String, + anchor_node_id: String, + mac: String, + ipv4: String, + ipv6: String, + download_min: f32, + upload_min: f32, + download_max: f32, + upload_max: f32, + comment: String, +} + #[cfg(test)] #[derive(Clone, Debug)] struct TopologyParentOverrideSelection { @@ -65,13 +84,56 @@ struct TopologyAllowedParentGroup { candidate_nodes: Vec, } -fn atomic_write_string(path: &Path, raw: &str) -> std::io::Result<()> { - let temp_path = path.with_extension("tmp"); - let mut file = File::create(&temp_path)?; - file.write_all(raw.as_bytes())?; - file.sync_all()?; - std::fs::rename(&temp_path, path)?; - Ok(()) +fn now_unix() -> Option { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok() + .map(|duration| duration.as_secs()) +} + +fn config_shaped_device(legacy: LegacyShapedDevice) -> ConfigShapedDevice { + let headers = csv::StringRecord::from(vec![ + "circuit_id", + "circuit_name", + "device_id", + "device_name", + "parent_node", + "parent_node_id", + "anchor_node_id", + "mac", + "ipv4", + "ipv6", + "download_min", + "upload_min", + "download_max", + "upload_max", + "comment", + ]); + let record = csv::StringRecord::from(vec![ + legacy.circuit_id, + legacy.circuit_name, + legacy.device_id, + legacy.device_name, + legacy.parent_node, + legacy.parent_node_id, + legacy.anchor_node_id, + legacy.mac, + legacy.ipv4, + legacy.ipv6, + legacy.download_min.to_string(), + legacy.upload_min.to_string(), + legacy.download_max.to_string(), + legacy.upload_max.to_string(), + legacy.comment, + ]); + ConfigShapedDevice::from_csv(&record, Some(&headers)) + .expect("UISP shaped-device rows must remain valid during compile conversion") +} + +fn config_shaped_devices(rows: Vec) -> ConfigShapedDevices { + let mut shaped_devices = ConfigShapedDevices::default(); + shaped_devices.replace_with_new_data(rows.into_iter().map(config_shaped_device).collect()); + shaped_devices } fn debug_graph_output_enabled() -> bool { @@ -93,27 +155,19 @@ fn anchor_for_site_ap( }) } -pub async fn build_full_network_v2( +pub async fn build_imported_full2_bundle( config: Arc, ip_ranges: IpRanges, -) -> Result<(), UispIntegrationError> { +) -> Result { let build_started = Instant::now(); let ethernet_policy = EthernetPortLimitPolicy::from(&config.integration_common); // Fetch the data let uisp_data = UispData::fetch_uisp_data(config.clone(), ip_ranges).await?; - // Report on obvious UISP errors that should be fixed - let _trouble = crate::strategies::ap_site::find_troublesome_sites(&uisp_data) - .await - .map_err(|e| { - error!("Error finding troublesome sites"); - error!("{e:?}"); - UispIntegrationError::UnknownSiteType - })?; - let bandwidth_overrides = - crate::strategies::full::bandwidth_overrides::get_site_bandwidth_overrides(&config)?; - let routing_overrides = crate::strategies::full::routes_override::get_route_overrides(&config)?; + crate::strategies::legacy_bandwidth_overrides::get_site_bandwidth_overrides(&config)?; + let routing_overrides = + crate::strategies::legacy_routes_override::get_route_overrides(&config)?; // Create a new graph let mut graph = GraphType::new(); @@ -543,50 +597,25 @@ pub async fn build_full_network_v2( walk_parents(&parents, node_id, &mut visited).into(), ); } - let network_path = Path::new(&config.lqos_directory).join("network.json"); - if network_path.exists() && !config.integration_common.always_overwrite_network_json { - warn!( - "Network.json exists, and always overwrite network json is not true - not writing network.json" - ); - } else { - let json = serde_json::to_string_pretty(&network_json).unwrap(); - atomic_write_string(&network_path, &json).map_err(|e| { - error!("Unable to write network.json"); - error!("{e:?}"); - UispIntegrationError::WriteNetJson - })?; - info!("Written network.json"); - } - topology_parent_candidates.sort_unstable_by(|left, right| left.node_id.cmp(&right.node_id)); let topology_candidates_file = TopologyParentCandidatesFile { source: "uisp/full".to_string(), + ingress_identity: None, nodes: topology_parent_candidates, }; - if let Err(err) = topology_candidates_file.save(&config) { - warn!("Unable to write topology parent candidates snapshot: {err:?}"); - } topology_editor_nodes.sort_unstable_by(|left, right| left.node_id.cmp(&right.node_id)); let topology_editor_state = TopologyEditorStateFile { schema_version: 1, source: "uisp/full2".to_string(), - generated_unix: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .ok() - .map(|duration| duration.as_secs()), + generated_unix: now_unix(), + ingress_identity: None, nodes: topology_editor_nodes, }; - if let Err(err) = topology_editor_state.save(&config) { - warn!("Unable to write topology editor state snapshot: {err:?}"); - } let canonical_state = TopologyCanonicalStateFile::from_editor_and_network( &topology_editor_state, &Value::Object(network_json.clone()), TopologyCanonicalIngressKind::NativeIntegration, ); - if let Err(err) = canonical_state.save(&config) { - warn!("Unable to write topology canonical state snapshot: {err:?}"); - } let export_elapsed_ms = export_started.elapsed().as_millis(); // Shaped Devices @@ -700,7 +729,7 @@ pub async fn build_full_network_v2( continue; } - let shaped_device = ShapedDevice { + let shaped_device = LegacyShapedDevice { circuit_id: site.id.to_owned(), circuit_name: site.name.to_owned(), device_id: device.id.to_owned(), @@ -730,31 +759,31 @@ pub async fn build_full_network_v2( "UISP shaped devices: processed {} site/AP pair(s) and produced {} shaped device row(s).", processed_site_pairs, shaped_device_count ); - let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv"); - let mut writer = csv::WriterBuilder::new() - .has_headers(true) - .from_path(file_path) - .unwrap(); - - for d in shaped_devices.iter() { - writer.serialize(d).unwrap(); - } - writer.flush().map_err(|e| { - error!("Unable to flush CSV file"); - error!("{e:?}"); - UispIntegrationError::CsvError - })?; - write_circuit_anchors(&config, "uisp/full2", &circuit_anchors)?; - write_ethernet_advisories(&config, ðernet_advisories)?; info!( shaped_devices = shaped_devices.len(), shaped_device_elapsed_ms = shaped_devices_started.elapsed().as_millis(), export_elapsed_ms, total_elapsed_ms = build_started.elapsed().as_millis(), - "Completed UISP full2 export" + "Completed UISP full2 import bundle" ); - Ok(()) + Ok(ImportedTopologyBundle { + source: "uisp/full2".to_string(), + generated_unix: topology_editor_state.generated_unix, + ingress_identity: None, + native_canonical: Some(canonical_state), + native_editor: Some(topology_editor_state), + parent_candidates: Some(topology_candidates_file), + compatibility_network_json: Value::Object(network_json), + shaped_devices: config_shaped_devices(shaped_devices), + circuit_anchors: CircuitAnchorsFile { + schema_version: 1, + source: "uisp/full2".to_string(), + generated_unix: now_unix(), + anchors: circuit_anchors, + }, + ethernet_advisories, + }) } fn add_device_links_to_graph( diff --git a/src/rust/uisp_integration/src/strategies/legacy_bandwidth_overrides.rs b/src/rust/uisp_integration/src/strategies/legacy_bandwidth_overrides.rs new file mode 100644 index 00000000..66d27208 --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/legacy_bandwidth_overrides.rs @@ -0,0 +1,267 @@ +use crate::errors::UispIntegrationError; +use csv::ReaderBuilder; +use lqos_config::Config; +use lqos_overrides::{NetworkAdjustment, OverrideFile}; +use serde::Serialize; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; +use tracing::{error, info, warn}; + +const LEGACY_UISP_BANDWIDTH_FILE: &str = "integrationUISPbandwidths.csv"; + +/// One UISP bandwidth override resolved from operator overrides or legacy CSV. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct BandwidthOverride { + /// Stable node identifier, when the override originated from the operator tree UI. + pub node_id: Option, + /// Display name of the target site or AP. + pub site_name: String, + /// Replacement download bandwidth in Mbps. + pub download_bandwidth_mbps: Option, + /// Replacement upload bandwidth in Mbps. + pub upload_bandwidth_mbps: Option, +} + +/// Attempts to load UISP bandwidth overrides. +/// +/// Current precedence: +/// 1. Operator `AdjustSiteSpeed` overrides from `lqos_overrides.json` +/// 2. One-time migration from `integrationUISPbandwidths.csv` into operator overrides +pub fn get_site_bandwidth_overrides( + config: &Config, +) -> Result, UispIntegrationError> { + let operator_path = OverrideFile::operator_path_for_config(config); + let mut operator_overrides = load_operator_override_file(&operator_path).map_err(|err| { + error!( + "Unable to load operator overrides from {}", + operator_path.display() + ); + error!("{err:?}"); + UispIntegrationError::CsvError + })?; + + if let Some(uisp) = operator_overrides.uisp() + && !uisp.bandwidth_overrides.is_empty() + { + warn!( + path = %operator_path.display(), + count = uisp.bandwidth_overrides.len(), + "Deprecated legacy uisp.bandwidth_overrides entries are present but ignored. Use operator AdjustSiteSpeed overrides in lqos_overrides.json instead" + ); + } + + if let Some(migrated) = + try_migrate_legacy_csv_to_operator_overrides(config, &mut operator_overrides)? + { + let materialized = materialize_operator_site_bandwidth_overrides(&operator_overrides); + if materialized.is_empty() { + return Ok(migrated); + } + return Ok(materialized); + } + + let materialized = materialize_operator_site_bandwidth_overrides(&operator_overrides); + if !materialized.is_empty() { + info!( + "Using {} UISP bandwidth override(s) from operator AdjustSiteSpeed entries in lqos_overrides.json", + materialized.len() + ); + return Ok(materialized); + } + + info!("No UISP bandwidth overrides loaded."); + Ok(Vec::new()) +} + +/// Finds the bandwidth override that best matches `site_name` and `node_id`. +pub fn find_bandwidth_override<'a>( + overrides: &'a [BandwidthOverride], + node_id: Option<&str>, + site_name: &str, +) -> Option<&'a BandwidthOverride> { + if let Some(node_id) = node_id + && let Some(found) = overrides + .iter() + .find(|entry| entry.node_id.as_deref() == Some(node_id)) + { + return Some(found); + } + + overrides + .iter() + .find(|entry| entry.node_id.is_none() && entry.site_name == site_name) +} + +fn load_operator_override_file(operator_path: &Path) -> anyhow::Result { + if !operator_path.exists() { + return Ok(OverrideFile::default()); + } + OverrideFile::load_from_explicit_path(operator_path) +} + +fn legacy_bandwidth_csv_path(config: &Config) -> PathBuf { + Path::new(&config.lqos_directory).join(LEGACY_UISP_BANDWIDTH_FILE) +} + +fn materialize_operator_site_bandwidth_overrides( + overrides: &OverrideFile, +) -> Vec { + overrides + .network_adjustments() + .iter() + .filter_map(|adjustment| match adjustment { + NetworkAdjustment::AdjustSiteSpeed { + node_id, + site_name, + download_bandwidth_mbps, + upload_bandwidth_mbps, + } => Some(BandwidthOverride { + node_id: node_id.clone(), + site_name: site_name.clone(), + download_bandwidth_mbps: *download_bandwidth_mbps, + upload_bandwidth_mbps: *upload_bandwidth_mbps, + }), + _ => None, + }) + .collect() +} + +fn try_migrate_legacy_csv_to_operator_overrides( + config: &Config, + operator_overrides: &mut OverrideFile, +) -> Result>, UispIntegrationError> { + let file_path = legacy_bandwidth_csv_path(config); + if !file_path.exists() { + return Ok(None); + } + + info!("Looking for {LEGACY_UISP_BANDWIDTH_FILE}"); + let migrated = load_bandwidth_overrides_from_csv(&file_path)?; + if migrated.is_empty() { + warn!( + "Legacy {} exists but no usable rows were found; leaving the file untouched.", + LEGACY_UISP_BANDWIDTH_FILE + ); + return Ok(Some(Vec::new())); + } + + for override_row in &migrated { + let already_present = operator_overrides + .network_adjustments() + .iter() + .any(|adjustment| { + matches!( + adjustment, + NetworkAdjustment::AdjustSiteSpeed { site_name, .. } + if site_name == &override_row.site_name + ) + }); + if already_present { + continue; + } + operator_overrides.set_site_bandwidth_override( + None, + override_row.site_name.clone(), + override_row.download_bandwidth_mbps, + override_row.upload_bandwidth_mbps, + ); + } + + let operator_path = OverrideFile::operator_path_for_config(config); + if let Err(err) = operator_overrides.save_to_explicit_path(&operator_path) { + warn!( + "Unable to save migrated UISP bandwidth overrides to {}: {err:?}. Falling back to the legacy CSV for this run.", + operator_path.display() + ); + return Ok(Some(migrated)); + } + + if let Err(err) = rename_legacy_csv_to_backup(&file_path) { + warn!( + "Migrated legacy UISP bandwidth overrides into lqos_overrides.json, but could not rename {} to a backup: {err:?}", + file_path.display() + ); + } else { + info!( + "Migrated legacy {} into operator AdjustSiteSpeed overrides in lqos_overrides.json", + LEGACY_UISP_BANDWIDTH_FILE + ); + } + + Ok(Some(migrated)) +} + +fn load_bandwidth_overrides_from_csv( + file_path: &Path, +) -> Result, UispIntegrationError> { + let reader = ReaderBuilder::new() + .comment(Some(b'#')) + .trim(csv::Trim::All) + .from_path(file_path); + let mut reader = match reader { + Ok(reader) => reader, + Err(err) => { + error!("Unable to read {}", file_path.display()); + error!("{err:?}"); + return Err(UispIntegrationError::CsvError); + } + }; + + let mut overrides = Vec::new(); + for (line, result) in reader.records().enumerate() { + if let Ok(result) = result { + if result.len() != 3 { + error!( + "Wrong number of records in {} on line {line}", + file_path.display() + ); + continue; + } + let site_name = result[0].to_string(); + if let Some(down) = numeric_string_to_f32(&result[1]) { + if let Some(up) = numeric_string_to_f32(&result[2]) { + overrides.push(BandwidthOverride { + node_id: None, + site_name, + download_bandwidth_mbps: Some(down), + upload_bandwidth_mbps: Some(up), + }); + } else { + error!("Cannot parse {} as float on line {line}", &result[2]); + } + } else { + error!("Cannot parse {} as float on line {line}", &result[1]); + } + } else { + error!("Error reading {} line", file_path.display()); + error!("{result:?}"); + } + } + + Ok(overrides) +} + +fn rename_legacy_csv_to_backup(file_path: &Path) -> std::io::Result<()> { + let unix_secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let mut backup_path = file_path.with_file_name(format!("{LEGACY_UISP_BANDWIDTH_FILE}.backup")); + if backup_path.exists() { + backup_path = + file_path.with_file_name(format!("{LEGACY_UISP_BANDWIDTH_FILE}.backup-{unix_secs}")); + } + fs::rename(file_path, backup_path) +} + +fn numeric_string_to_f32(text: &str) -> Option { + if let Ok(n) = text.parse::() { + Some(n) + } else if let Ok(n) = text.parse::() { + Some(n as f32) + } else { + error!("Unable to parse {text} as a numeric"); + None + } +} diff --git a/src/rust/uisp_integration/src/strategies/legacy_mikrotik.rs b/src/rust/uisp_integration/src/strategies/legacy_mikrotik.rs new file mode 100644 index 00000000..5131fd4a --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/legacy_mikrotik.rs @@ -0,0 +1,63 @@ +use crate::uisp_types::Ipv4ToIpv6; +use lqos_config::Config; +use std::path::Path; +use std::process::Command; + +pub async fn mikrotik_data(config: &Config) -> anyhow::Result> { + if config.uisp_integration.ipv6_with_mikrotik { + fetch_mikrotik_data(config).await + } else { + Ok(Vec::new()) + } +} + +async fn fetch_mikrotik_data(config: &Config) -> anyhow::Result> { + let base_path = Path::new(&config.lqos_directory); + let mikrotik_script_path = base_path.join("mikrotikFindIPv6.py"); + if !mikrotik_script_path.exists() { + tracing::error!("Mikrotik script not found at {:?}", mikrotik_script_path); + return Err(anyhow::anyhow!( + "Mikrotik script not found at {:?}", + mikrotik_script_path + )); + } + + let mikrotik_dhcp_router_list_path = base_path.join("mikrotikDHCPRouterList.csv"); + if !mikrotik_dhcp_router_list_path.exists() { + tracing::error!( + "Mikrotik DHCP router list not found at {:?}", + mikrotik_dhcp_router_list_path + ); + return Err(anyhow::anyhow!( + "Mikrotik DHCP router list not found at {:?}", + mikrotik_dhcp_router_list_path + )); + } + + let code = mikrotik_script_path.to_string_lossy().to_string(); + let csv_path = mikrotik_dhcp_router_list_path.to_string_lossy().to_string(); + + let output = Command::new("/usr/bin/python3") + .args([&code, &csv_path]) + .output(); + if let Err(e) = output { + tracing::error!("Python error: {:?}", e); + return Err(anyhow::anyhow!("Python error: {:?}", e)); + } + let output = output?; + let json_from_python = String::from_utf8(output.stdout)?; + let json = serde_json::from_str::(&json_from_python)?; + if let Some(map) = json.as_object() { + let mut result = Vec::new(); + for (ipv4, ipv6) in map { + result.push(Ipv4ToIpv6 { + ipv4: ipv4.to_string().replace("\"", ""), + ipv6: ipv6.to_string().replace("\"", ""), + }); + } + Ok(result) + } else { + tracing::error!("Mikrotik data is not an object"); + Err(anyhow::anyhow!("Mikrotik data is not an object")) + } +} diff --git a/src/rust/uisp_integration/src/strategies/legacy_parse.rs b/src/rust/uisp_integration/src/strategies/legacy_parse.rs new file mode 100644 index 00000000..a8a10969 --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/legacy_parse.rs @@ -0,0 +1,73 @@ +use crate::ip_ranges::IpRanges; +use crate::uisp_types::{Ipv4ToIpv6, UispDataLink, UispDevice, UispSite}; +use lqos_config::Config; +use tracing::info; +use uisp::{DataLink, Device, Site}; + +/// Parses the UISP datasets into a more usable format. +pub fn parse_uisp_datasets( + sites_raw: &[Site], + data_links_raw: &[DataLink], + devices_raw: &[Device], + config: &Config, + ip_ranges: &IpRanges, + ipv4_to_v6: Vec, +) -> (Vec, Vec, Vec) { + let (mut sites, data_links, devices) = ( + parse_sites(sites_raw, config), + parse_data_links(data_links_raw), + parse_devices(devices_raw, config, ip_ranges, ipv4_to_v6), + ); + + for site in sites.iter_mut() { + devices + .iter() + .enumerate() + .filter(|(_, device)| device.site_id == site.id) + .for_each(|(idx, _)| { + site.device_indices.push(idx); + }); + } + + (sites, data_links, devices) +} + +fn parse_sites(sites_raw: &[Site], config: &Config) -> Vec { + let sites: Vec = sites_raw + .iter() + .map(|s| UispSite::from_uisp(s, config)) + .collect(); + info!("{} sites have been successfully parsed", sites.len()); + sites +} + +fn parse_data_links(data_links_raw: &[DataLink]) -> Vec { + let mut data_links = Vec::with_capacity(data_links_raw.len() * 2); + for link in data_links_raw { + let uisp_link = UispDataLink::from_uisp(link); + if let Some(link) = uisp_link { + data_links.push(link.invert()); + data_links.push(link); + } + } + + info!( + "{} data-links have been successfully parsed (doubled)", + data_links.len() + ); + data_links +} + +fn parse_devices( + devices_raw: &[Device], + config: &Config, + ip_ranges: &IpRanges, + ipv4_to_v6: Vec, +) -> Vec { + let devices: Vec = devices_raw + .iter() + .map(|d| UispDevice::from_uisp(d, config, ip_ranges, &ipv4_to_v6)) + .collect(); + info!("{} devices have been sucessfully parsed", devices.len()); + devices +} diff --git a/src/rust/uisp_integration/src/strategies/legacy_routes_override.rs b/src/rust/uisp_integration/src/strategies/legacy_routes_override.rs new file mode 100644 index 00000000..96fa7b01 --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/legacy_routes_override.rs @@ -0,0 +1,100 @@ +use crate::errors::UispIntegrationError; +use lqos_config::Config; +use lqos_overrides::OverrideFile; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tracing::{info, warn}; + +const LEGACY_UISP_ROUTES_FILE: &str = "integrationUISProutes.csv"; + +/// Represents a route override in the integrationUISProutes.csv file. +#[derive(Serialize, Deserialize, Debug)] +pub struct RouteOverride { + /// The site to override the route from. + pub from_site: String, + /// The site to override the route to. + pub to_site: String, + /// The cost of the route. + pub cost: u32, +} + +/// Attempts to load UISP route overrides. +/// +/// UISP route overrides are deprecated and ignored by current UISP builds. +pub fn get_route_overrides(config: &Config) -> Result, UispIntegrationError> { + let operator_path = OverrideFile::operator_path_for_config(config); + match load_operator_override_file(&operator_path) { + Ok(operator_overrides) => { + let configured = materialize_uisp_route_overrides(&operator_overrides); + if !configured.is_empty() { + warn!( + path = %operator_path.display(), + count = configured.len(), + "UISP route overrides are deprecated and ignored; use Topology Manager parent or attachment preferences instead" + ); + } + } + Err(err) if operator_path.exists() => { + warn!( + path = %operator_path.display(), + "Unable to inspect operator overrides for deprecated UISP route overrides: {err:?}" + ); + } + Err(_) => {} + } + + let legacy_path = legacy_routes_csv_path(config); + if legacy_path.exists() { + warn!( + path = %legacy_path.display(), + "Legacy integrationUISProutes.csv is deprecated and ignored; use Topology Manager parent or attachment preferences instead" + ); + } + + info!( + "UISP route overrides are disabled; using detected topology and Topology Manager overrides only." + ); + Ok(Vec::new()) +} + +fn load_operator_override_file(operator_path: &Path) -> anyhow::Result { + if !operator_path.exists() { + return Ok(OverrideFile::default()); + } + OverrideFile::load_from_explicit_path(operator_path) +} + +fn legacy_routes_csv_path(config: &Config) -> PathBuf { + Path::new(&config.lqos_directory).join(LEGACY_UISP_ROUTES_FILE) +} + +fn materialize_uisp_route_overrides(overrides: &OverrideFile) -> Vec { + let Some(uisp) = overrides.uisp() else { + return Vec::new(); + }; + + uisp.route_overrides + .iter() + .map(|r| RouteOverride { + from_site: r.from_site.clone(), + to_site: r.to_site.clone(), + cost: r.cost, + }) + .collect() +} + +#[allow(dead_code)] +pub fn write_routing_overrides_template( + config: Arc, + natural_routes: &[RouteOverride], +) -> anyhow::Result<()> { + let file_path = Path::new(&config.lqos_directory).join("integrationUISProutes.template.csv"); + let mut writer = csv::Writer::from_path(file_path)?; + writer.write_record(["From Site", "To Site", "Cost"])?; + for route in natural_routes { + writer.write_record([&route.from_site, &route.to_site, &route.cost.to_string()])?; + } + writer.flush()?; + Ok(()) +} diff --git a/src/rust/uisp_integration/src/strategies/legacy_uisp_fetch.rs b/src/rust/uisp_integration/src/strategies/legacy_uisp_fetch.rs new file mode 100644 index 00000000..450068f5 --- /dev/null +++ b/src/rust/uisp_integration/src/strategies/legacy_uisp_fetch.rs @@ -0,0 +1,70 @@ +use crate::errors::UispIntegrationError; +use lqos_config::Config; +use std::sync::Arc; +use tokio::join; +use tracing::{error, info}; +use uisp::{DataLink, Device, Site}; + +/// Load required data from UISP, using the API. +pub async fn load_uisp_data( + config: Arc, +) -> Result< + ( + Vec, + Vec, + Vec, + Vec, + ), + UispIntegrationError, +> { + info!("Loading Devices, Sites and Data-Links from UISP"); + let (devices, sites, data_links) = join!( + uisp::load_all_devices_with_interfaces(config.clone()), + uisp::load_all_sites(config.clone()), + uisp::load_all_data_links(config.clone()), + ); + + if devices.is_err() { + error!("Error downloading devices list from UISP"); + error!("{:?}", devices); + return Err(UispIntegrationError::UispConnectError); + } + let (mut devices, raw_json_devices_data) = devices.unwrap(); + + if sites.is_err() { + error!("Error downloading sites list from UISP"); + error!("{:?}", sites); + return Err(UispIntegrationError::UispConnectError); + } + let sites = sites.unwrap(); + + if data_links.is_err() { + error!("Error downloading data_links list from UISP"); + error!("{:?}", data_links); + return Err(UispIntegrationError::UispConnectError); + } + let data_links = data_links.unwrap(); + + devices.retain(|dev| { + if let Some(site_id) = dev.get_site_id() { + if let Some(site) = sites.iter().find(|site| site.id == site_id) { + !config + .uisp_integration + .exclude_sites + .contains(&site.name_or_blank()) + } else { + true + } + } else { + true + } + }); + + info!( + "Loaded backing data: {} sites, {} devices, {} links", + sites.len(), + devices.len(), + data_links.len() + ); + Ok((sites, devices, data_links, raw_json_devices_data)) +} diff --git a/src/rust/uisp_integration/src/strategies/mod.rs b/src/rust/uisp_integration/src/strategies/mod.rs index 0debe960..78a62b98 100644 --- a/src/rust/uisp_integration/src/strategies/mod.rs +++ b/src/rust/uisp_integration/src/strategies/mod.rs @@ -1,64 +1,95 @@ -mod ap_only; -mod ap_site; mod common; -mod flat; -mod full; mod full2; +mod legacy_bandwidth_overrides; +mod legacy_mikrotik; +mod legacy_parse; +mod legacy_routes_override; +mod legacy_uisp_fetch; use crate::blackboard; use crate::errors::UispIntegrationError; +use crate::ethernet_advisory::write_ethernet_advisories; use crate::ip_ranges::IpRanges; use lqos_bus::BlackboardSystem; -use lqos_config::Config; +use lqos_config::{Config, circuit_anchors_path}; +use lqos_topology_compile::{ + CompiledTopologyBundle, TopologyCompileMode, TopologyCompiledShapingFile, TopologyImportFile, + compile_topology, +}; +use std::io::ErrorKind; use std::sync::Arc; -use tracing::{error, info}; +use tracing::{error, info, warn}; + +fn compile_mode_for_strategy(strategy: &str) -> Result { + match strategy { + "flat" => Ok(TopologyCompileMode::Flat), + "ap_only" => Ok(TopologyCompileMode::ApOnly), + "ap_site" => Ok(TopologyCompileMode::ApSite), + "full" => Ok(TopologyCompileMode::Full), + "full2" => Ok(TopologyCompileMode::Full2), + _ => Err(UispIntegrationError::UnknownIntegrationStrategy), + } +} + +fn write_compiled_outputs( + config: &Config, + topology_import: &TopologyImportFile, + compiled_shaping: &TopologyCompiledShapingFile, + compiled: CompiledTopologyBundle, +) -> Result<(), UispIntegrationError> { + topology_import.save(config).map_err(|e| { + error!("Unable to write topology_import.json: {e:?}"); + UispIntegrationError::WriteNetJson + })?; + compiled_shaping.save(config).map_err(|e| { + error!("Unable to write topology_compiled_shaping.json: {e:?}"); + UispIntegrationError::WriteNetJson + })?; + compiled.parent_candidates.save(config).map_err(|e| { + error!("Unable to write topology parent candidates snapshot: {e:?}"); + UispIntegrationError::WriteNetJson + })?; + compiled.editor.save(config).map_err(|e| { + error!("Unable to write topology editor state snapshot: {e:?}"); + UispIntegrationError::WriteNetJson + })?; + compiled.canonical.save(config).map_err(|e| { + error!("Unable to write topology canonical state snapshot: {e:?}"); + UispIntegrationError::WriteNetJson + })?; + let anchors_path = circuit_anchors_path(config); + if let Err(err) = std::fs::remove_file(&anchors_path) + && err.kind() != ErrorKind::NotFound + { + warn!( + "Unable to remove stale duplicate circuit_anchors.json at {}: {err:?}", + anchors_path.display() + ); + } + write_ethernet_advisories(config, &compiled.ethernet_advisories)?; + Ok(()) +} /// Builds the network using the selected strategy. pub async fn build_with_strategy( config: Arc, ip_ranges: IpRanges, ) -> Result<(), UispIntegrationError> { - // Select a Strategy - blackboard( - BlackboardSystem::System, - "UISP", - config.uisp_integration.strategy.to_lowercase().as_str(), + let strategy = config.resolved_topology_compile_mode_for_uisp().to_string(); + blackboard(BlackboardSystem::System, "UISP", strategy.as_str()).await; + let mode = compile_mode_for_strategy(&strategy)?; + info!("Strategy selected: {strategy}"); + let imported = full2::build_imported_full2_bundle(config.clone(), ip_ranges).await?; + let topology_import = TopologyImportFile::from_imported_bundle(&imported, strategy.clone()); + let compiled = compile_topology(imported, mode).map_err(|e| { + error!("Unable to compile UISP topology for mode '{strategy}': {e:?}"); + UispIntegrationError::WriteNetJson + })?; + let compiled_shaping = TopologyCompiledShapingFile::from_compiled(&compiled, strategy.clone()); + write_compiled_outputs( + config.as_ref(), + &topology_import, + &compiled_shaping, + compiled, ) - .await; - match config.uisp_integration.strategy.to_lowercase().as_str() { - "flat" => { - info!("Strategy selected: flat"); - flat::build_flat_network(config, ip_ranges).await?; - Ok(()) - } - /* "full" is now an alias for full2. The legacy full topology builder remains - * here only as shared support code and historical reference. - * "full" => { - info!("Strategy selected: full"); - full::build_full_network(config, ip_ranges).await?; - Ok(()) - }*/ - "ap_only" => { - info!("Strategy selected: ap_only"); - ap_only::build_ap_only_network(config, ip_ranges).await?; - Ok(()) - } - "ap_site" => { - info!("Strategy selected: ap_site"); - ap_site::build_ap_site_network(config, ip_ranges).await?; - Ok(()) - } - "full" | "full2" => { - info!("Strategy selected: full2"); - full2::build_full_network_v2(config, ip_ranges).await?; - Ok(()) - } - _ => { - error!( - "Unknown strategy: {}. Bailing.", - config.uisp_integration.strategy - ); - Err(UispIntegrationError::UnknownIntegrationStrategy) - } - } } diff --git a/src/rust/uisp_integration/src/uisp_types/uisp_device.rs b/src/rust/uisp_integration/src/uisp_types/uisp_device.rs index 44cc3cb1..b8cdbedd 100644 --- a/src/rust/uisp_integration/src/uisp_types/uisp_device.rs +++ b/src/rust/uisp_integration/src/uisp_types/uisp_device.rs @@ -1,7 +1,7 @@ use crate::ip_ranges::IpRanges; use lqos_config::{Config, EthernetPortLimitPolicy, usable_ethernet_cap_mbps}; use std::collections::HashSet; -use std::net::{IpAddr, Ipv4Addr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use uisp::Device; #[derive(Debug)] @@ -493,6 +493,44 @@ impl UispDevice { (ip_bits & host_mask) == 0 } + fn normalize_ipv6_entry(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + + if !trimmed.contains('/') { + let addr = trimmed.parse::().ok()?; + return Some(addr.to_string()); + } + + let parts: Vec<&str> = trimmed.split('/').collect(); + if parts.len() != 2 { + return None; + } + + let addr = parts[0].parse::().ok()?; + let prefix_len = parts[1].parse::().ok()?; + if prefix_len > 128 { + return None; + } + if prefix_len == 128 { + return Some(addr.to_string()); + } + + let host_bits = 128u32.saturating_sub(prefix_len as u32); + let addr_bits = u128::from_be_bytes(addr.octets()); + let host_mask = if host_bits == 0 { + 0 + } else { + (1u128 << host_bits) - 1 + }; + if addr_bits & host_mask == 0 { + return Some(format!("{addr}/{prefix_len}")); + } + Some(addr.to_string()) + } + /// Creates a new UispDevice from a UISP device /// /// # Arguments @@ -563,7 +601,9 @@ impl UispDevice { if let Some(ip) = &device.ipAddress { if ip.contains(':') { // It's IPv6 - ipv6.insert(ip.clone()); + if let Some(normalized) = Self::normalize_ipv6_entry(ip) { + ipv6.insert(normalized); + } if let Some(probe_ip) = Self::probe_ip_candidate(ip) { probe_ipv6.insert(probe_ip); } @@ -594,7 +634,9 @@ impl UispDevice { if let Some(address) = &address.cidr { if address.contains(':') { // It's IPv6 - ipv6.insert(address.clone()); + if let Some(normalized) = Self::normalize_ipv6_entry(address) { + ipv6.insert(normalized); + } if let Some(probe_ip) = Self::probe_ip_candidate(address) { probe_ipv6.insert(probe_ip); } @@ -743,20 +785,9 @@ impl UispDevice { if self.ipv6.is_empty() { return "".to_string(); } - if self.ipv6.len() == 1 { - let mut result = "".to_string(); - for ip in self.ipv6.iter() { - result = ip.clone(); - } - return result; - } - let mut result = "".to_string(); - for ip in self.ipv6.iter() { - result += &format!("{}, ", &ip); - } - result.truncate(result.len() - 2); - let result = format!("[{result}]"); - result + let mut ips: Vec<&str> = self.ipv6.iter().map(String::as_str).collect(); + ips.sort_unstable(); + ips.join(", ") } } @@ -1219,4 +1250,54 @@ mod tests { .is_some_and(|reason| reason.contains("sfp0") && reason.contains("2500 Mbps")) ); } + + #[test] + fn normalize_ipv6_entry_keeps_real_prefixes_and_flattens_host_cidrs() { + assert_eq!( + UispDevice::normalize_ipv6_entry("2602:fd47:1a:80::/60").as_deref(), + Some("2602:fd47:1a:80::/60") + ); + assert_eq!( + UispDevice::normalize_ipv6_entry("2602:fd47:1a:80:76ac:b9ff:fe73:e28/64").as_deref(), + Some("2602:fd47:1a:80:76ac:b9ff:fe73:e28") + ); + assert_eq!( + UispDevice::normalize_ipv6_entry("2602:fd47:1:1b::9/128").as_deref(), + Some("2602:fd47:1:1b::9") + ); + } + + #[test] + fn ipv6_list_is_plain_csv_without_brackets() { + let device = UispDevice { + id: "dev-1".to_string(), + name: "Device 1".to_string(), + mac: "".to_string(), + role: None, + wireless_mode: None, + site_id: "".to_string(), + raw_download: 0, + raw_upload: 0, + download: 0, + upload: 0, + ipv4: HashSet::new(), + ipv6: HashSet::from([ + "2602:fd47:1a:80::/60".to_string(), + "2602:fd47:1:1b::9".to_string(), + ]), + probe_ipv4: HashSet::new(), + probe_ipv6: HashSet::new(), + negotiated_ethernet_mbps: None, + negotiated_ethernet_interface: None, + transport_cap_mbps: None, + transport_cap_reason: None, + attachment_rate_source: super::UispAttachmentRateSource::Static, + }; + + let rendered = device.ipv6_list(); + assert!(!rendered.contains('[')); + assert!(!rendered.contains(']')); + assert!(rendered.contains("2602:fd47:1a:80::/60")); + assert!(rendered.contains("2602:fd47:1:1b::9")); + } } diff --git a/src/scheduler.py b/src/scheduler.py index 021115ea..e4b9298b 100644 --- a/src/scheduler.py +++ b/src/scheduler.py @@ -14,7 +14,7 @@ from liblqos_python import automatic_import_uisp, automatic_import_splynx, queue automatic_import_powercode, automatic_import_sonar, influx_db_enabled, get_libreqos_directory, \ blackboard_finish, blackboard_submit, automatic_import_wispgate, enable_insight_topology, insight_topology_role, \ automatic_import_netzur, automatic_import_visp, calculate_shaping_runtime_hash, efficiency_core_ids, scheduler_alive as _scheduler_alive_native, scheduler_error as _scheduler_error_native, \ - calculate_topology_source_generation, \ + calculate_topology_source_generation, topology_import_ingress_enabled, \ scheduler_progress as _scheduler_progress_native, overrides_persistent_devices_materialized, overrides_circuit_adjustments_materialized, \ overrides_network_adjustments_materialized, \ scheduler_output as _scheduler_output_native, wait_for_bus_ready @@ -589,124 +589,127 @@ def operator_requires_sqm_column(devices, adjustments): def apply_lqos_overrides(): - """Load ShapedDevices.csv, apply persistent devices and circuit adjustments, and save back.""" - path = shaped_devices_csv_path() - header, rows = read_shaped_devices_csv(path) + """Apply overrides to source topology files without mutating integration-owned shaping CSV.""" + if not topology_import_ingress_enabled(): + path = shaped_devices_csv_path() + header, rows = read_shaped_devices_csv(path) - # 1) Persistent devices: replace by device_id or append - try: - extra = overrides_persistent_devices_materialized() - except Exception as e: - # Persistent device overrides are optional. Keep the scheduler healthy - # and continue applying the rest of the override sources if this loader - # is unavailable or temporarily broken. - print(f"Skipping persistent device overrides: {e}") - extra = [] - - # 2) Circuit adjustments: speed changes, removals, reparenting - try: - adjustments = overrides_circuit_adjustments_materialized() - except Exception as e: - print(f"Failed to read circuit adjustments: {e}") - adjustments = [] - - need_sqm_column = header_has_sqm(header) or operator_requires_sqm_column(extra, adjustments) - if need_sqm_column and not header_has_sqm(header): - header = list(header) + ["sqm"] - for row in rows: - if len(row) < len(header): - row.extend([""] * (len(header) - len(row))) - - override_rows = override_devices_to_rows(extra or [], header, include_sqm=need_sqm_column) - merged_rows, changed = merge_rows_replace_by_device_id(rows, override_rows) - header_map = shaped_devices_header_index_map(header) - - def set_if_some(value_opt, current_str): - if value_opt is None: - return current_str + # 1) Persistent devices: replace by device_id or append try: - return str(float(value_opt)) - except Exception: - return current_str + extra = overrides_persistent_devices_materialized() + except Exception as e: + print(f"Skipping persistent device overrides: {e}") + extra = [] - def set_row_value(row, index, value): - while len(row) <= index: - row.append('') - row[index] = value + # 2) Circuit adjustments: speed changes, removals, reparenting + try: + adjustments = overrides_circuit_adjustments_materialized() + except Exception as e: + print(f"Failed to read circuit adjustments: {e}") + adjustments = [] - if adjustments: - for adj in adjustments: - t = adj.get('type') - if t == 'circuit_adjust_speed': - cid = adj.get('circuit_id', '') - for r in merged_rows: - if len(r) >= 12 and r[0] == cid: - r[8] = set_if_some(adj.get('min_download_bandwidth'), r[8] if len(r) > 8 else '') - r[10] = set_if_some(adj.get('max_download_bandwidth'), r[10] if len(r) > 10 else '') - r[9] = set_if_some(adj.get('min_upload_bandwidth'), r[9] if len(r) > 9 else '') - r[11] = set_if_some(adj.get('max_upload_bandwidth'), r[11] if len(r) > 11 else '') - changed = True - elif t == 'device_adjust_speed': - did = adj.get('device_id', '') - for r in merged_rows: - if len(r) >= 12 and r[2] == did: - r[8] = set_if_some(adj.get('min_download_bandwidth'), r[8] if len(r) > 8 else '') - r[10] = set_if_some(adj.get('max_download_bandwidth'), r[10] if len(r) > 10 else '') - r[9] = set_if_some(adj.get('min_upload_bandwidth'), r[9] if len(r) > 9 else '') - r[11] = set_if_some(adj.get('max_upload_bandwidth'), r[11] if len(r) > 11 else '') - changed = True - elif t == 'device_adjust_sqm': - if not need_sqm_column: - continue - did = adj.get('device_id', '') - sqm_override = (adj.get('sqm_override') or '').strip() - for r in merged_rows: - if len(r) >= 3 and r[2] == did: - current_sqm = r[13] if len(r) > 13 else '' - if current_sqm != sqm_override: - set_row_value(r, 13, sqm_override) + need_sqm_column = header_has_sqm(header) or operator_requires_sqm_column(extra, adjustments) + if need_sqm_column and not header_has_sqm(header): + header = list(header) + ["sqm"] + for row in rows: + if len(row) < len(header): + row.extend([""] * (len(header) - len(row))) + + override_rows = override_devices_to_rows(extra or [], header, include_sqm=need_sqm_column) + merged_rows, changed = merge_rows_replace_by_device_id(rows, override_rows) + header_map = shaped_devices_header_index_map(header) + + def set_if_some(value_opt, current_str): + if value_opt is None: + return current_str + try: + return str(float(value_opt)) + except Exception: + return current_str + + def set_row_value(row, index, value): + while len(row) <= index: + row.append('') + row[index] = value + + if adjustments: + for adj in adjustments: + t = adj.get('type') + if t == 'circuit_adjust_speed': + cid = adj.get('circuit_id', '') + for r in merged_rows: + if len(r) >= 12 and r[0] == cid: + r[8] = set_if_some(adj.get('min_download_bandwidth'), r[8] if len(r) > 8 else '') + r[10] = set_if_some(adj.get('max_download_bandwidth'), r[10] if len(r) > 10 else '') + r[9] = set_if_some(adj.get('min_upload_bandwidth'), r[9] if len(r) > 9 else '') + r[11] = set_if_some(adj.get('max_upload_bandwidth'), r[11] if len(r) > 11 else '') changed = True - elif t == 'remove_circuit': - cid = adj.get('circuit_id', '') - before = len(merged_rows) - merged_rows = [r for r in merged_rows if len(r) < 1 or r[0] != cid] - if len(merged_rows) != before: - changed = True - elif t == 'remove_device': - did = adj.get('device_id', '') - before = len(merged_rows) - merged_rows = [r for r in merged_rows if len(r) < 3 or r[2] != did] - if len(merged_rows) != before: - changed = True - elif t == 'reparent_circuit': - cid = adj.get('circuit_id', '') - parent_node = adj.get('parent_node', '') - for r in merged_rows: - circuit_id_idx = header_map.get('circuit_id', 0) - if len(r) > circuit_id_idx and r[circuit_id_idx] == cid: - set_row_value_by_header(r, header_map, 'parent_node', parent_node) - set_row_value_by_header(r, header_map, 'parent_node_id', '') + elif t == 'device_adjust_speed': + did = adj.get('device_id', '') + for r in merged_rows: + if len(r) >= 12 and r[2] == did: + r[8] = set_if_some(adj.get('min_download_bandwidth'), r[8] if len(r) > 8 else '') + r[10] = set_if_some(adj.get('max_download_bandwidth'), r[10] if len(r) > 10 else '') + r[9] = set_if_some(adj.get('min_upload_bandwidth'), r[9] if len(r) > 9 else '') + r[11] = set_if_some(adj.get('max_upload_bandwidth'), r[11] if len(r) > 11 else '') + changed = True + elif t == 'device_adjust_sqm': + if not need_sqm_column: + continue + did = adj.get('device_id', '') + sqm_override = (adj.get('sqm_override') or '').strip() + for r in merged_rows: + if len(r) >= 3 and r[2] == did: + current_sqm = r[13] if len(r) > 13 else '' + if current_sqm != sqm_override: + set_row_value(r, 13, sqm_override) + changed = True + elif t == 'remove_circuit': + cid = adj.get('circuit_id', '') + before = len(merged_rows) + merged_rows = [r for r in merged_rows if len(r) < 1 or r[0] != cid] + if len(merged_rows) != before: changed = True + elif t == 'remove_device': + did = adj.get('device_id', '') + before = len(merged_rows) + merged_rows = [r for r in merged_rows if len(r) < 3 or r[2] != did] + if len(merged_rows) != before: + changed = True + elif t == 'reparent_circuit': + cid = adj.get('circuit_id', '') + parent_node = adj.get('parent_node', '') + for r in merged_rows: + circuit_id_idx = header_map.get('circuit_id', 0) + if len(r) > circuit_id_idx and r[circuit_id_idx] == cid: + set_row_value_by_header(r, header_map, 'parent_node', parent_node) + set_row_value_by_header(r, header_map, 'parent_node_id', '') + changed = True - if changed: - final_header = header if header else (SHAPED_DEVICES_HEADER_WITH_SQM if need_sqm_column else SHAPED_DEVICES_HEADER) - write_shaped_devices_csv(path, final_header, merged_rows) - print("Updated ShapedDevices.csv with overrides") + if changed: + final_header = header if header else (SHAPED_DEVICES_HEADER_WITH_SQM if need_sqm_column else SHAPED_DEVICES_HEADER) + write_shaped_devices_csv(path, final_header, merged_rows) + print("Updated ShapedDevices.csv with overrides") - # 3) Load, adjust, and optionally save network.json - nj_path = network_json_path() - network = load_network_json(nj_path) + # 3) Load, adjust, and optionally save topology compatibility state try: adjustments = overrides_network_adjustments_materialized() except Exception as e: print(f"Failed to read network adjustments: {e}") adjustments = [] - net_changed = apply_network_adjustments(network, adjustments) - if net_changed: - write_network_json(nj_path, network) - print("Updated network.json with overrides") - canonical_path = topology_canonical_state_path() - canonical_state = load_topology_canonical_state(canonical_path) + canonical_path = topology_canonical_state_path() + canonical_state = load_topology_canonical_state(canonical_path) + if topology_import_ingress_enabled(): + if canonical_state and apply_network_adjustments_to_canonical_state(canonical_state, adjustments): + write_topology_canonical_state(canonical_path, canonical_state) + print("Updated topology_canonical_state.json with overrides") + else: + nj_path = network_json_path() + network = load_network_json(nj_path) + net_changed = apply_network_adjustments(network, adjustments) + if net_changed: + write_network_json(nj_path, network) + print("Updated network.json with overrides") if canonical_state and apply_network_adjustments_to_canonical_state(canonical_state, adjustments): write_topology_canonical_state(canonical_path, canonical_state) print("Updated topology_canonical_state.json with overrides") @@ -1209,8 +1212,10 @@ def run_scheduler_main(): atexit.register(stop_topology_runtime_process) set_scheduler_status_bus_enabled(False) - publish_scheduler_progress(True, "waiting_for_bus", "Waiting for lqosd bus", 1, SCHEDULER_STARTUP_STEP_COUNT) ensure_bus_ready() + set_scheduler_status_bus_enabled(True) + scheduler_alive() + publish_scheduler_progress(True, "waiting_for_bus", "Waiting for lqosd bus", 1, SCHEDULER_STARTUP_STEP_COUNT) try: importAndShapeFullReload() publish_scheduler_progress(False, "ready", "Scheduler ready", SCHEDULER_STARTUP_STEP_COUNT, SCHEDULER_STARTUP_STEP_COUNT, percent=100) diff --git a/src/shaping_skip_report.py b/src/shaping_skip_report.py index 5b04106e..ea9133d1 100644 --- a/src/shaping_skip_report.py +++ b/src/shaping_skip_report.py @@ -42,7 +42,7 @@ def _classify_unshaped_circuit(circuit, valid_parent_nodes, flat_network): if flat_network: return ( "unattached_flat_network", - "Circuit was not attached during shaping in flat-network mode.", + "Circuit is being shaped under generated parent queues in flat-network mode.", ) if logical_parent_str in ("", "none"): diff --git a/src/testGraph.py b/src/testGraph.py index cf549fb7..8d61962f 100644 --- a/src/testGraph.py +++ b/src/testGraph.py @@ -206,6 +206,46 @@ class TestGraph(unittest.TestCase): self.assertEqual(generated_site.parentId, "Parent Site") self.assertEqual(generated_site.parentIndex, 1) + def test_prepare_tree_lifts_nested_generated_sites_out_of_clients(self): + """ + Nested generated sites should remain reachable from exported topology + instead of being stranded below plain client nodes. + """ + from integrationCommon import NetworkGraph, NetworkNode, NodeType + + graph = NetworkGraph() + graph.addRawNode(NetworkNode("AP 1", type=NodeType.ap)) + graph.addRawNode(NetworkNode("Bonnie McBride", parentId="AP 1", type=NodeType.client)) + graph.addRawNode(NetworkNode("LeRoy Dozois", parentId="Bonnie McBride", type=NodeType.client)) + graph.addRawNode(NetworkNode("Nested Site", parentId="LeRoy Dozois", type=NodeType.site)) + graph.addRawNode( + NetworkNode( + "bonnie-device", + parentId="Bonnie McBride", + type=NodeType.device, + ipv4=["100.64.0.1"], + ) + ) + graph.addRawNode( + NetworkNode( + "leroy-device", + parentId="LeRoy Dozois", + type=NodeType.device, + ipv4=["100.64.0.2"], + ) + ) + + graph.prepareTree() + + bonnie = next(node for node in graph.nodes if node.id == "Bonnie McBride") + bonnie_site = next(node for node in graph.nodes if node.id == "Bonnie McBride_gen") + leroy_site = next(node for node in graph.nodes if node.id == "LeRoy Dozois_gen") + + self.assertEqual(bonnie.type, NodeType.client) + self.assertEqual(bonnie.parentId, "Bonnie McBride_gen") + self.assertEqual(bonnie_site.parentId, "AP 1") + self.assertEqual(leroy_site.parentId, "Bonnie McBride_gen") + def test_find_unconnected(self): """ Tests traversing a tree and finding nodes that @@ -411,6 +451,244 @@ class TestGraph(unittest.TestCase): net.prepareTree() self.assertEqual(net.nodes[2].parentIndex, 1) + def test_native_topology_editor_uses_nearest_real_infrastructure_parent(self): + from integrationCommon import NetworkGraph, NetworkNode, NodeType + + net = NetworkGraph() + net.addRawNode( + NetworkNode( + "site_1", + "Site 1", + "", + NodeType.site, + 1000, + 1000, + networkJsonId="splynx:site:1", + ) + ) + net.addRawNode( + NetworkNode( + "relay_client", + "Relay Client", + "site_1", + NodeType.client, + 100, + 100, + networkJsonId="splynx:circuit:relay", + ) + ) + net.addRawNode( + NetworkNode( + "child_site", + "Child Site", + "relay_client", + NodeType.site, + 500, + 500, + networkJsonId="splynx:site:2", + ) + ) + net.addRawNode( + NetworkNode( + "relay_device", + "Relay Device", + "relay_client", + NodeType.device, + ipv4=["100.64.0.10"], + ) + ) + + net.prepareTree() + editor = net.buildNativeTopologyEditorState("python/splynx") + nodes = {node["node_id"]: node for node in editor["nodes"]} + + self.assertEqual(set(nodes.keys()), {"splynx:site:1", "splynx:site:2"}) + self.assertIsNone(nodes["splynx:site:1"]["current_parent_node_id"]) + self.assertFalse(nodes["splynx:site:1"]["can_move"]) + self.assertEqual(nodes["splynx:site:1"]["allowed_parents"], []) + self.assertEqual(nodes["splynx:site:2"]["current_parent_node_id"], "splynx:site:1") + self.assertEqual(nodes["splynx:site:2"]["current_parent_node_name"], "Site 1") + self.assertFalse(nodes["splynx:site:2"]["can_move"]) + self.assertEqual(nodes["splynx:site:2"]["allowed_parents"], []) + + def test_legacy_parent_candidates_skip_fixed_roots(self): + from integrationCommon import NetworkGraph, NetworkNode, NodeType + + net = NetworkGraph() + net.addRawNode( + NetworkNode( + "site_1", + "Site 1", + "", + NodeType.site, + 1000, + 1000, + networkJsonId="splynx:site:1", + ) + ) + net.addRawNode( + NetworkNode( + "site_2", + "Site 2", + "site_1", + NodeType.site, + 500, + 500, + networkJsonId="splynx:site:2", + ) + ) + + net.prepareTree() + candidates = net.buildTopologyParentCandidates() + self.assertEqual(candidates["nodes"], []) + + def test_native_topology_editor_exposes_bounded_local_move_candidates(self): + from integrationCommon import NetworkGraph, NetworkNode, NodeType + + net = NetworkGraph() + net.addRawNode( + NetworkNode( + "root_site", + "Root Site", + "", + NodeType.site, + 1000, + 1000, + networkJsonId="splynx:site:root", + ) + ) + net.addRawNode( + NetworkNode( + "parent_a", + "Parent A", + "root_site", + NodeType.site, + 800, + 800, + networkJsonId="splynx:site:a", + ) + ) + net.addRawNode( + NetworkNode( + "parent_b", + "Parent B", + "root_site", + NodeType.site, + 800, + 800, + networkJsonId="splynx:site:b", + ) + ) + net.addRawNode( + NetworkNode( + "child_ap", + "Child AP", + "parent_a", + NodeType.ap, + 300, + 300, + networkJsonId="splynx:ap:child", + ) + ) + + net.prepareTree() + editor = net.buildNativeTopologyEditorState("python/splynx") + nodes = {node["node_id"]: node for node in editor["nodes"]} + child = nodes["splynx:ap:child"] + + self.assertEqual(child["current_parent_node_id"], "splynx:site:a") + self.assertTrue(child["can_move"]) + self.assertEqual( + [parent["parent_node_id"] for parent in child["allowed_parents"]], + ["splynx:site:a", "splynx:site:b"], + ) + + candidates = net.buildTopologyParentCandidates() + candidate_by_id = {node["node_id"]: node for node in candidates["nodes"]} + self.assertEqual( + [candidate["node_id"] for candidate in candidate_by_id["splynx:ap:child"]["candidate_parents"]], + ["splynx:site:a", "splynx:site:b"], + ) + + def test_native_topology_editor_exposes_root_peer_move_candidates(self): + from integrationCommon import NetworkGraph, NetworkNode, NodeType + + net = NetworkGraph() + net.addRawNode( + NetworkNode( + "gateway_a", + "Gateway A", + "", + NodeType.site, + 1000, + 1000, + networkJsonId="splynx:site:gateway-a", + ) + ) + net.addRawNode( + NetworkNode( + "gateway_b", + "Gateway B", + "", + NodeType.site, + 1000, + 1000, + networkJsonId="splynx:site:gateway-b", + ) + ) + net.addRawNode( + NetworkNode( + "gateway_c", + "Gateway C", + "", + NodeType.site, + 1000, + 1000, + networkJsonId="splynx:site:gateway-c", + ) + ) + net.addRawNode( + NetworkNode( + "unattached", + "LibreQoS Unattached [Site]", + "", + NodeType.site, + 1000, + 1000, + networkJsonId="libreqos:generated:splynx:site:unattached", + ) + ) + net.addRawNode( + NetworkNode( + "child_ap", + "Child AP", + "gateway_a", + NodeType.ap, + 300, + 300, + networkJsonId="splynx:ap:child", + ) + ) + + net.prepareTree() + editor = net.buildNativeTopologyEditorState("python/splynx") + nodes = {node["node_id"]: node for node in editor["nodes"]} + child = nodes["splynx:ap:child"] + + self.assertEqual(child["current_parent_node_id"], "splynx:site:gateway-a") + self.assertTrue(child["can_move"]) + self.assertEqual( + [parent["parent_node_id"] for parent in child["allowed_parents"]], + ["splynx:site:gateway-a", "splynx:site:gateway-b", "splynx:site:gateway-c"], + ) + + candidates = net.buildTopologyParentCandidates() + candidate_by_id = {node["node_id"]: node for node in candidates["nodes"]} + self.assertEqual( + [candidate["node_id"] for candidate in candidate_by_id["splynx:ap:child"]["candidate_parents"]], + ["splynx:site:gateway-a", "splynx:site:gateway-b", "splynx:site:gateway-c"], + ) + def test_graph_render_to_pdf(self): """ Requires that graphviz be installed with diff --git a/src/test_integration_splynx.py b/src/test_integration_splynx.py index 89250c7b..8ad53fc3 100644 --- a/src/test_integration_splynx.py +++ b/src/test_integration_splynx.py @@ -14,17 +14,19 @@ def install_splynx_stubs(): lqlib.upstream_bandwidth_capacity_download_mbps = lambda: 1000 lqlib.upstream_bandwidth_capacity_upload_mbps = lambda: 1000 lqlib.find_ipv6_using_mikrotik = lambda: False + lqlib.migrate_legacy_site_bandwidth_csv = lambda *_args, **_kwargs: None + lqlib.overrides_network_adjustments_materialized = lambda: [] lqlib.exclude_sites = lambda: [] lqlib.bandwidth_overhead_factor = lambda: 1.0 lqlib.committed_bandwidth_multiplier = lambda: 1.0 lqlib.exception_cpes = lambda: [] lqlib.promote_to_root_list = lambda: [] lqlib.client_bandwidth_multiplier = lambda: 1.0 + lqlib.write_compiled_topology_from_python_graph_payload = lambda *_args, **_kwargs: None lqlib.splynx_api_key = lambda: "" lqlib.splynx_api_secret = lambda: "" lqlib.splynx_api_url = lambda: "http://example.invalid" lqlib.splynx_strategy = lambda: "flat" - lqlib.overwrite_network_json_always = False sys.modules["liblqos_python"] = lqlib @@ -108,6 +110,81 @@ class TestIntegrationSplynxStableIds(unittest.TestCase): self.assertEqual(net.nodes[2].id, "splynx_service_93") self.assertEqual(net.nodes[2].ipv4, ["66.185.224.210"]) + def test_ap_site_prefers_network_sites_topology_when_present(self): + self.assertTrue( + integrationSplynx.strategy_uses_network_sites_topology( + "ap_site", + [{"id": 1, "title": "Site A"}], + [{"id": 5, "network_site_id": 1}], + ) + ) + + def test_full_preserves_monitoring_hierarchy_even_with_network_sites(self): + self.assertFalse( + integrationSplynx.strategy_uses_network_sites_topology( + "full", + [{"id": 1, "title": "Site A"}], + [{"id": 5, "network_site_id": 1}], + ) + ) + + def test_full_mode_uses_stable_generated_unattached_site(self): + self.assertEqual( + integrationSplynx.splynx_generated_unattached_parent_id("full"), + "splynx_generated_unattached_site", + ) + self.assertEqual( + integrationSplynx.splynx_generated_unattached_site_network_id(), + "libreqos:generated:splynx:site:unattached", + ) + + def test_non_full_modes_do_not_force_generated_unattached_site(self): + self.assertIsNone(integrationSplynx.splynx_generated_unattached_parent_id("ap_site")) + self.assertIsNone(integrationSplynx.splynx_generated_unattached_parent_id("ap_only")) + self.assertIsNone(integrationSplynx.splynx_generated_unattached_parent_id("flat")) + + def test_create_infrastructure_nodes_normalizes_parent_ids(self): + net = integrationCommon.NetworkGraph() + integrationSplynx.createInfrastructureNodes( + net, + monitoring=[ + {"id": 1, "gps": None}, + {"id": "2", "gps": None}, + ], + hardware_name={"1": "Parent Site", "2": "Child AP"}, + hardware_parent={"2": 1}, + hardware_type={"1": "Site", "2": "AP"}, + siteBandwidth={}, + hardware_name_extended={"1": "Parent Site", "2": "Child AP"}, + ) + + net.prepareTree() + + parent = next(node for node in net.nodes if node.id == "1") + child = next(node for node in net.nodes if node.id == "2") + self.assertEqual(parent.parentIndex, 0) + self.assertEqual(child.parentId, "1") + self.assertEqual(child.parentIndex, net.findNodeIndexById("1")) + + def test_find_best_parent_node_normalizes_service_and_sector_ids(self): + parent_node_id, assignment_method = integrationSplynx.findBestParentNode( + {"router_id": 10, "access_device": 0}, + hardware_name={"10": "Router A", "20": "Sector A"}, + ipForRouter={"10": "192.0.2.1"}, + sectorForRouter={"10": [{"id": 20}]}, + ) + self.assertEqual(parent_node_id, "10") + self.assertEqual(assignment_method, "router_id") + + parent_node_id, assignment_method = integrationSplynx.findBestParentNode( + {"router_id": 30, "access_device": 0}, + hardware_name={"20": "Sector A"}, + ipForRouter={"30": "192.0.2.2"}, + sectorForRouter={"30": [{"id": 20}]}, + ) + self.assertEqual(parent_node_id, "20") + self.assertEqual(assignment_method, "sector_id") + if __name__ == "__main__": unittest.main() diff --git a/src/test_libreqos_shaping_inputs.py b/src/test_libreqos_shaping_inputs.py index 0142cd7e..9d206254 100644 --- a/src/test_libreqos_shaping_inputs.py +++ b/src/test_libreqos_shaping_inputs.py @@ -61,6 +61,7 @@ def install_libreqos_stubs(): lqlib.automatic_import_wispgate = lambda: False lqlib.automatic_import_netzur = lambda: False lqlib.automatic_import_visp = lambda: False + lqlib.topology_import_ingress_enabled = lambda: False lqlib.plan_top_level_cpu_bins = lambda *_args, **_kwargs: {} lqlib.plan_class_identities = lambda *_args, **_kwargs: {} lqlib.fast_queues_fq_codel = lambda: False @@ -74,6 +75,10 @@ LibreQoS = importlib.import_module("LibreQoS") class TestLibreQoSShapingInputs(unittest.TestCase): + def test_attachment_lookup_candidates_preserve_generated_parent_name_without_node_id(self): + candidates = LibreQoS._attachment_lookup_candidates("Generated_PN_1", {}) + self.assertEqual(candidates, ["Generated_PN_1"]) + def test_shaping_inputs_freshness_tracks_circuit_anchors(self): with tempfile.TemporaryDirectory() as temp_dir: shaping_inputs = os.path.join(temp_dir, "shaping_inputs.json") diff --git a/src/test_scheduler.py b/src/test_scheduler.py index 0ca8fdaf..21592e7d 100644 --- a/src/test_scheduler.py +++ b/src/test_scheduler.py @@ -33,6 +33,7 @@ def install_scheduler_stubs(): lqlib.calculate_hash = lambda: 0 lqlib.calculate_shaping_runtime_hash = lambda: 0 lqlib.calculate_topology_source_generation = lambda: "test-generation" + lqlib.topology_import_ingress_enabled = lambda: False lqlib.efficiency_core_ids = lambda: [] lqlib.scheduler_alive = Mock() lqlib.scheduler_error = Mock() @@ -533,6 +534,64 @@ class TestSchedulerOverrideMerge(unittest.TestCase): self.assertEqual(written_rows[0][4], "AP-Updated") self.assertEqual(written_rows[0][5], "") + def test_apply_lqos_overrides_updates_canonical_only_for_integration_ingress(self): + header = [ + "Circuit ID", "Circuit Name", "Device ID", "Device Name", "Parent Node", "MAC", + "IPv4", "IPv6", "Download Min Mbps", "Upload Min Mbps", "Download Max Mbps", + "Upload Max Mbps", "Comment", + ] + rows = [[ + "93", "Name", "splynx_service_93", "Name", "AP", "MAC", "1.1.1.1", "", + "1", "1", "330", "330", "", + ]] + canonical_state = { + "compatibility_network_json": { + "NodeB": { + "downloadBandwidthMbps": 200, + "uploadBandwidthMbps": 100, + "children": {}, + } + }, + "nodes": [ + { + "node_id": "node-b", + "node_name": "NodeB", + "rate_input": { + "intrinsic_download_mbps": 200, + "intrinsic_upload_mbps": 100, + }, + } + ], + } + + with patch.object(scheduler, "shaped_devices_csv_path", return_value="/tmp/ShapedDevices.csv"): # nosec B108 + with patch.object(scheduler, "read_shaped_devices_csv", return_value=(header, rows)): + with patch.object(scheduler, "overrides_persistent_devices_materialized", return_value=[]): + with patch.object(scheduler, "overrides_circuit_adjustments_materialized", return_value=[]): + with patch.object( + scheduler, + "overrides_network_adjustments_materialized", + return_value=[{ + "type": "adjust_site_speed", + "node_id": "node-b", + "site_name": "NodeB", + "download_bandwidth_mbps": 80, + "upload_bandwidth_mbps": 40, + }], + ): + with patch.object(scheduler, "topology_import_ingress_enabled", return_value=True): + with patch.object(scheduler, "load_topology_canonical_state", return_value=canonical_state): + with patch.object(scheduler, "write_topology_canonical_state") as mock_write_canonical: + with patch.object(scheduler, "load_network_json") as mock_load_network: + with patch.object(scheduler, "write_network_json") as mock_write_network: + with patch.object(scheduler, "write_shaped_devices_csv") as mock_write_sd: + scheduler.apply_lqos_overrides() + + mock_load_network.assert_not_called() + mock_write_network.assert_not_called() + mock_write_sd.assert_not_called() + mock_write_canonical.assert_called_once() + def test_override_devices_to_rows_preserves_anchor_node_id(self): header = [ "Circuit ID", "Circuit Name", "Device ID", "Device Name", "Parent Node", diff --git a/src/test_shaping_skip_report.py b/src/test_shaping_skip_report.py index 0c2ec80e..74ce40c0 100644 --- a/src/test_shaping_skip_report.py +++ b/src/test_shaping_skip_report.py @@ -142,6 +142,7 @@ class TestShapingSkipReport(unittest.TestCase): self.assertEqual(len(skipped), 1) self.assertEqual(skipped[0]["reasonCode"], "unattached_flat_network") + self.assertIn("generated parent queues", skipped[0]["reasonText"]) def test_format_unshaped_device_line_contains_context(self): line = format_unshaped_device_line(