From 3dbcb0717297da9d843ee89fa5cf7f0717c95448 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 14 Mar 2026 10:42:54 -0500 Subject: [PATCH 1/2] chore: Remove the incremental flag from release mode compilation. Slightly slower but much smaller builds. --- src/rust/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml index fd2f241d..27aa52a1 100644 --- a/src/rust/Cargo.toml +++ b/src/rust/Cargo.toml @@ -5,9 +5,9 @@ edition = "2024" license = "GPL-2.0-only" [profile.release] -strip = "debuginfo" +strip = yes lto = "thin" -incremental = true +incremental = false [profile.release.build-override] opt-level = 3 From c99652ebe83805c32f65a729b0949aba16685f02 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Sat, 14 Mar 2026 10:44:24 -0500 Subject: [PATCH 2/2] Docs: Add shared agent workflows and build contract guidance --- .codex-repo/link-skills.sh | 51 ++++++ .../skills/libreqos-lqos-sys-ebpf/SKILL.md | 72 ++++++++ .../libreqos-lqos-sys-ebpf/agents/openai.yaml | 4 + .../libreqos-node-manager-frontend/SKILL.md | 67 +++++++ .../agents/openai.yaml | 4 + .../libreqos-packaging-release/SKILL.md | 54 ++++++ .../agents/openai.yaml | 4 + .../references/package-surface.md | 27 +++ .../skills/libreqos-python-workflow/SKILL.md | 62 +++++++ .../agents/openai.yaml | 4 + .../references/generated-runtime-files.md | 30 +++ .../references/integration-modes.md | 25 +++ .../skills/libreqos-rust-workflow/SKILL.md | 45 +++++ .../libreqos-rust-workflow/agents/openai.yaml | 4 + AGENTS.md | 171 ++++++++++++++++++ README.md | 10 + src/rust/lqosd/copy_files.sh | 1 + src/rust/lqosd/dev_build.sh | 1 + .../lqosd/src/node_manager/js_build/README.md | 15 +- .../src/node_manager/js_build/entrypoints.txt | 47 +++++ .../src/node_manager/js_build/esbuild.sh | 47 +++-- .../js_build/test-build-contract.sh | 106 +++++++++++ .../src/node_manager/static2/chatbot.html | 2 +- .../src/node_manager/static2/lts_trial.html | 2 +- .../src/node_manager/static2/template.html | 2 +- 25 files changed, 840 insertions(+), 17 deletions(-) create mode 100755 .codex-repo/link-skills.sh create mode 100755 .codex-repo/skills/libreqos-lqos-sys-ebpf/SKILL.md create mode 100644 .codex-repo/skills/libreqos-lqos-sys-ebpf/agents/openai.yaml create mode 100755 .codex-repo/skills/libreqos-node-manager-frontend/SKILL.md create mode 100644 .codex-repo/skills/libreqos-node-manager-frontend/agents/openai.yaml create mode 100644 .codex-repo/skills/libreqos-packaging-release/SKILL.md create mode 100644 .codex-repo/skills/libreqos-packaging-release/agents/openai.yaml create mode 100644 .codex-repo/skills/libreqos-packaging-release/references/package-surface.md create mode 100644 .codex-repo/skills/libreqos-python-workflow/SKILL.md create mode 100644 .codex-repo/skills/libreqos-python-workflow/agents/openai.yaml create mode 100644 .codex-repo/skills/libreqos-python-workflow/references/generated-runtime-files.md create mode 100644 .codex-repo/skills/libreqos-python-workflow/references/integration-modes.md create mode 100644 .codex-repo/skills/libreqos-rust-workflow/SKILL.md create mode 100644 .codex-repo/skills/libreqos-rust-workflow/agents/openai.yaml create mode 100644 AGENTS.md create mode 100644 src/rust/lqosd/src/node_manager/js_build/entrypoints.txt create mode 100755 src/rust/lqosd/src/node_manager/js_build/test-build-contract.sh diff --git a/.codex-repo/link-skills.sh b/.codex-repo/link-skills.sh new file mode 100755 index 00000000..f45f18b4 --- /dev/null +++ b/.codex-repo/link-skills.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_SKILLS_DIR="${SCRIPT_DIR}/skills" +CODEX_HOME_DIR="${CODEX_HOME:-$HOME/.codex}" +USER_SKILLS_DIR="${CODEX_HOME_DIR}/skills" + +mkdir -p "${USER_SKILLS_DIR}" + +if [ ! -d "${REPO_SKILLS_DIR}" ]; then + echo "Repo skills directory not found: ${REPO_SKILLS_DIR}" >&2 + exit 1 +fi + +linked_any=0 + +for skill_dir in "${REPO_SKILLS_DIR}"/*; do + [ -d "${skill_dir}" ] || continue + [ -f "${skill_dir}/SKILL.md" ] || continue + + skill_name="$(basename "${skill_dir}")" + target="${USER_SKILLS_DIR}/${skill_name}" + + if [ -L "${target}" ]; then + current_target="$(readlink -f "${target}")" + desired_target="$(readlink -f "${skill_dir}")" + if [ "${current_target}" = "${desired_target}" ]; then + echo "Already linked: ${skill_name}" + continue + fi + echo "Skipping ${skill_name}: ${target} already points to ${current_target}" >&2 + continue + fi + + if [ -e "${target}" ]; then + echo "Skipping ${skill_name}: ${target} already exists and is not a symlink" >&2 + continue + fi + + ln -s "${skill_dir}" "${target}" + echo "Linked ${skill_name} -> ${skill_dir}" + linked_any=1 +done + +if [ "${linked_any}" -eq 0 ]; then + echo "No new skills linked." +fi + +echo "Restart Codex to pick up skill changes." diff --git a/.codex-repo/skills/libreqos-lqos-sys-ebpf/SKILL.md b/.codex-repo/skills/libreqos-lqos-sys-ebpf/SKILL.md new file mode 100755 index 00000000..f61dc341 --- /dev/null +++ b/.codex-repo/skills/libreqos-lqos-sys-ebpf/SKILL.md @@ -0,0 +1,72 @@ +--- +name: libreqos-lqos-sys-ebpf +description: Shared LibreQoS workflow for lqos_sys eBPF, C wrapper, bindgen, and pinned-map ABI work. Use when changing src/rust/lqos_sys/build.rs, src/rust/lqos_sys/src/bpf/**, wrapper.c or wrapper.h, lqos_sys Rust mirror structs, or XDP/TC attachment and map ABI behavior. +--- + +# LibreQoS lqos_sys eBPF Workflow + +Use this skill for `lqos_sys` BPF/FFI work. + +## Scope + +- Source of truth lives under `src/rust/lqos_sys/`. +- eBPF/C code lives under `src/rust/lqos_sys/src/bpf/`. +- Rust includes generated bindings from `OUT_DIR`; do not edit generated artifacts directly. + +## Build Pipeline + +`build.rs` orchestrates this pipeline: + +1. Compile `src/bpf/lqos_kern.c` to LLVM IR with `clang` +2. Convert LLVM IR to a BPF object with `llc` +3. Generate `lqos_kern_skel.h` with `bpftool gen skeleton` +4. Compile `src/bpf/wrapper.c` +5. Archive the wrapper into a small static library +6. Run `bindgen` on `wrapper.h` +7. Include generated bindings from `OUT_DIR/bindings.rs` + +When changing any part of that chain, treat the whole chain as one linked surface. + +## High-Risk Changes + +- BPF map key/value structs +- Shared constants such as sizing limits +- `wrapper.h` / `wrapper.c` function signatures +- Skeleton-exposed program or map names +- Pinned-map ABI expectations in `lqos_kernel.rs` + +## Required Checklist + +1. Edit only source files under `src/rust/lqos_sys/`. +2. If a BPF-side struct changes, update the Rust mirror type in the same change. +3. Update or preserve size assertions for Rust mirror types. +4. Review pinned-map ABI cleanup in `src/rust/lqos_sys/src/lqos_kernel.rs`. +5. Run `cargo check -p lqos_sys`. +6. Run relevant tests for touched Rust mirror types and map readers. +7. If runtime attach/detach behavior changed, state clearly whether live root validation was performed. + +## Mirror Types To Check + +- `src/rust/lqos_sys/src/throughput.rs` +- `src/rust/lqos_sys/src/flowbee_data.rs` +- `src/rust/lqos_sys/src/ip_mapping/ip_hash_data.rs` +- `src/rust/lqos_sys/src/ip_mapping/ip_hash_key.rs` + +## Safety Rules + +- Do not edit `OUT_DIR` files. +- Do not assume a Rust-only change is safe if it reads a BPF map with a mirrored struct layout. +- Do not run live XDP/TC attach-detach or pinned-map cleanup without explicit user approval; this can disrupt networking on the host. +- Be careful with verifier-sensitive patterns in BPF code: bounds checks, scratch maps for large structs, and hot-path allocation avoidance are intentional. + +## Common Failure Modes + +- `clang`, `llc`, or `bpftool` missing from the host +- Skeleton generation succeeding but wrapper/bindgen drifting +- Rust mirror struct sizes no longer matching BPF-side layouts +- Stale pinned maps silently reusing an incompatible ABI + +## Notes + +- `build.rs` treats stderr from `clang` and `llc` as fatal. +- Pinned map compatibility cleanup already exists for several maps in `lqos_kernel.rs`; extend it when introducing new ABI-sensitive pinned maps. diff --git a/.codex-repo/skills/libreqos-lqos-sys-ebpf/agents/openai.yaml b/.codex-repo/skills/libreqos-lqos-sys-ebpf/agents/openai.yaml new file mode 100644 index 00000000..fb30d980 --- /dev/null +++ b/.codex-repo/skills/libreqos-lqos-sys-ebpf/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "LibreQoS lqos_sys eBPF" + short_description: "eBPF, wrapper, and ABI workflow" + default_prompt: "Use $libreqos-lqos-sys-ebpf to update LibreQoS eBPF, wrapper, or pinned-map ABI code safely." diff --git a/.codex-repo/skills/libreqos-node-manager-frontend/SKILL.md b/.codex-repo/skills/libreqos-node-manager-frontend/SKILL.md new file mode 100755 index 00000000..24189a8e --- /dev/null +++ b/.codex-repo/skills/libreqos-node-manager-frontend/SKILL.md @@ -0,0 +1,67 @@ +--- +name: libreqos-node-manager-frontend +description: Shared LibreQoS workflow for node_manager frontend work. Use when changing src/rust/lqosd/src/node_manager/js_build/**, static2/**, esbuild.sh, copy_files.sh, dev_build.sh, static_pages.rs, or the overall Bootstrap 5 and FontAwesome-based UI. +--- + +# LibreQoS Node Manager Frontend + +Use this skill for `node_manager` frontend work. + +## Scope + +- Bundled page JS lives under `src/rust/lqosd/src/node_manager/js_build/src/`. +- Static HTML, CSS, and vendored assets live under `src/rust/lqosd/src/node_manager/static2/`. +- Served-page routing lives in `src/rust/lqosd/src/node_manager/static_pages.rs`. +- Copy/build helpers live in `src/rust/lqosd/copy_files.sh` and `src/rust/lqosd/dev_build.sh`. + +## Build Contract + +The frontend is page-oriented, not a SPA: + +- one HTML page body per page in `static2/` +- one JS entrypoint per page in `js_build/src/` +- one bundled output per entrypoint in `js_build/out/` +- shared template shell in `static2/template.html` +- shared styling in `static2/node_manager.css` +- vendored third-party assets in `static2/vendor/` + +`js_build/esbuild.sh` reads `js_build/entrypoints.txt`, builds each listed page entrypoint, and writes bundled `.js` plus sourcemaps into `js_build/out/`. + +## Dependency Rules + +- Bootstrap 5 is the UI framework. +- FontAwesome is the icon system. +- Vendored assets under `static2/vendor/` are the default. +- Do not infer frontend dependencies from `src/package.json`; it is not the source of truth for this UI. +- Avoid introducing new CDN or npm dependencies unless explicitly required. Existing CDN fallback behavior should be treated as an exception, not the default. + +## Style Guide + +- Preserve the existing shell: left sidebar, top nav tabs, rounded cards, dense data tables, badges, and dark/light theme support. +- Use `static2/template.html` and `static2/node_manager.css` as the baseline visual language. +- Favor Bootstrap 5 components and utility classes over ad hoc widget systems. +- Use FontAwesome icons that fit the existing navigation and dashboard vocabulary. +- Keep new pages consistent with the screenshots and existing pages: operational dashboard first, not marketing UI. +- Respect `data-bs-theme`, existing theme helpers, and the current dark/light behavior. + +## Page Checklist + +When adding or renaming a page, update all of: + +1. `js_build/src/.js` +2. `js_build/entrypoints.txt` +3. `static2/.html` +4. `static_pages.rs` + +If the page is served through the templated/authenticated router, use `%CACHEBUSTERS%` on its local bundle reference. + +## Validation + +- Run `src/rust/lqosd/src/node_manager/js_build/test-build-contract.sh` +- Run `src/rust/lqosd/src/node_manager/js_build/esbuild.sh` +- For broader local iteration, run `src/rust/lqosd/dev_build.sh` + +## Notes + +- The build-contract test checks page list alignment, required vendored assets, entrypoint/source alignment, and bundle output presence. +- `copy_files.sh` and `dev_build.sh` should stay aligned with the same node_manager asset pipeline. diff --git a/.codex-repo/skills/libreqos-node-manager-frontend/agents/openai.yaml b/.codex-repo/skills/libreqos-node-manager-frontend/agents/openai.yaml new file mode 100644 index 00000000..0307c457 --- /dev/null +++ b/.codex-repo/skills/libreqos-node-manager-frontend/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "LibreQoS Node Manager UI" + short_description: "Node manager frontend workflow" + default_prompt: "Use $libreqos-node-manager-frontend to update the LibreQoS node_manager frontend, build contract, or static assets." diff --git a/.codex-repo/skills/libreqos-packaging-release/SKILL.md b/.codex-repo/skills/libreqos-packaging-release/SKILL.md new file mode 100644 index 00000000..7e3ad329 --- /dev/null +++ b/.codex-repo/skills/libreqos-packaging-release/SKILL.md @@ -0,0 +1,54 @@ +--- +name: libreqos-packaging-release +description: Shared LibreQoS workflow for packaging, release, and shipped-file changes. Use when changing src/build_dpkg.sh, src/build_rust.sh, src/rust/lqosd/copy_files.sh, packaged binaries/assets/templates/service files, or any file that must exist on installed LibreQoS systems. +--- + +# LibreQoS Packaging and Release + +Use this skill for packaging and shipped-file work. + +## Scope + +- Packaging scripts: `src/build_dpkg.sh`, `src/build_rust.sh` +- Node manager asset copy/build path: `src/rust/lqosd/copy_files.sh`, `src/rust/lqosd/dev_build.sh` +- Files copied into installed layouts such as `/opt/libreqos/src`, `/opt/libreqos/src/bin`, `/etc/systemd/system`, and packaged static assets + +## Invariants + +- `src/build_dpkg.sh` is the functional source of truth for shipped `.deb` installs. +- `src/build_rust.sh` and `src/build_dpkg.sh` should stay aligned where they overlap on binaries, static assets, helper scripts, and operator expectations. +- If a change adds, renames, moves, or newly requires a shipped file, update packaging in the same change. +- Never assume packaging will pick up a new file automatically. Verify the copy/install path explicitly. + +## Workflow + +1. Read `AGENTS.md` first for current packaging and safety rules. +2. Identify every new or changed file the feature needs at runtime, install time, or service start time. +3. Review `src/build_dpkg.sh` and any related copy/build helpers for the relevant copy/install path. +4. Update the packaging/build scripts in the same change as the feature. +5. If node_manager assets are involved, also review `copy_files.sh`, `dev_build.sh`, and the node_manager build contract. +6. If install behavior or service expectations changed, update docs in the same change. + +## References + +- Read `references/package-surface.md` when deciding whether a file is part of the shipped install or where it is copied from. + +## Validation + +- Run `bash -n` on changed shell scripts. +- If node_manager assets are involved, run `src/rust/lqosd/src/node_manager/js_build/test-build-contract.sh`. +- If you did not run a full package build, say so explicitly. +- Before finishing, answer this directly: "Does `src/build_dpkg.sh` include every new required file?" + +## High-Risk Changes + +- Service files and post-install behavior +- Static assets copied through intermediate build helpers +- Python/Rust files that work in-repo but are missing from the installed package +- `update_api.sh` / `lqos_api` packaging behavior +- Any change that assumes `/opt/libreqos/src` contains files that packaging does not copy + +## Notes + +- Package-content drift is a recurring failure mode in this repo. +- `src/build_rust.sh` can restart services on the local host; treat it as an operationally sensitive script, not a harmless build helper. diff --git a/.codex-repo/skills/libreqos-packaging-release/agents/openai.yaml b/.codex-repo/skills/libreqos-packaging-release/agents/openai.yaml new file mode 100644 index 00000000..4f454c63 --- /dev/null +++ b/.codex-repo/skills/libreqos-packaging-release/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "LibreQoS Packaging Release" + short_description: "Packaging and shipped-file workflow" + default_prompt: "Use $libreqos-packaging-release to review LibreQoS packaging, shipped files, and release-script changes." diff --git a/.codex-repo/skills/libreqos-packaging-release/references/package-surface.md b/.codex-repo/skills/libreqos-packaging-release/references/package-surface.md new file mode 100644 index 00000000..3ead1444 --- /dev/null +++ b/.codex-repo/skills/libreqos-packaging-release/references/package-surface.md @@ -0,0 +1,27 @@ +# Package Surface + +Use this reference when deciding whether a file is part of the shipped install. + +## Source Of Truth + +- `src/build_dpkg.sh` defines the `.deb` layout under `/opt/libreqos/src`, `/etc`, and related install hooks. +- `src/build_rust.sh` defines the in-repo build/install layout under `src/bin` and the local `liblqos_python.so` workflow. +- `src/rust/lqosd/copy_files.sh` builds and copies node_manager static assets before they are packaged or used locally. + +## Key Copy Paths + +- Python/runtime files copied into `/opt/libreqos/src` come from the explicit `LQOS_FILES` list in `src/build_dpkg.sh`. +- Example service files are copied from `src/bin/*.service.example` into `/opt/libreqos/src/bin`, then installed into `/etc/systemd/system` by `postinst`. +- Rust binaries are built in `src/rust/target/release/` and copied into `/opt/libreqos/src/bin`. +- `liblqos_python.so` is copied into `/opt/libreqos/src`. +- Node manager static assets are assembled into `src/bin/static2` and then copied into `/opt/libreqos/src/bin/static2`. +- `rust/remove_pinned_maps.sh` is copied into `/opt/libreqos/src/rust`. +- `update_api.sh` fetches `lqos_api` into the package output during `build_dpkg.sh`. + +## Common Drift Points + +- Adding a new Python helper or template that works in-repo but is not added to `LQOS_FILES` +- Adding frontend/static assets that are built locally but not copied into the package +- Adding a new binary or renaming an existing one without updating both `RUSTPROGS` and the related build path +- Depending on generated files in `src/bin/` without checking how they get there during packaging +- Changing install/service assumptions in `postinst` without reviewing operator impact diff --git a/.codex-repo/skills/libreqos-python-workflow/SKILL.md b/.codex-repo/skills/libreqos-python-workflow/SKILL.md new file mode 100644 index 00000000..d274cccf --- /dev/null +++ b/.codex-repo/skills/libreqos-python-workflow/SKILL.md @@ -0,0 +1,62 @@ +--- +name: libreqos-python-workflow +description: Shared LibreQoS workflow for Python orchestration, scheduler, integrations, and helper scripts under src/. Use when changing LibreQoS.py, scheduler.py, integration*.py, integrationCommon.py, Python helpers, or Python tests around shaping and generated config files. +--- + +# LibreQoS Python Workflow + +Use this skill for Python work under `src/`. + +## Scope + +- Main Python shaper entrypoint: `src/LibreQoS.py` +- Scheduler/orchestration: `src/scheduler.py` +- Shared importer graph/output logic: `src/integrationCommon.py` +- Importers and helper scripts: `src/integration*.py`, `src/lqTools.py`, `src/configMigrator.py`, `src/csvToNetworkJSON.py`, `src/cidrToHosts.py`, `src/mikrotikFindIPv6.py` +- Python tests: `src/test_*.py`, `src/bakery_integration_test.py`, `src/testGraph.py` + +## Architecture + +- `LibreQoS.py` is the live shaper/orchestrator. It validates inputs, builds queueing state, coordinates TC/XDP work through `liblqos_python`, and writes runtime artifacts such as `queuingStructure.json`, `statsByCircuit.json`, and `statsByParentNode.json`. +- `scheduler.py` runs integrations, applies overrides, updates scheduler status for the Web UI, and triggers full or partial refreshes. +- `integrationCommon.py` is the shared graph/output layer for integrations and writes `network.json` and `ShapedDevices.csv`. +- Python in this repo is operational glue around `liblqos_python` and installed runtime files, not an isolated app package. + +## Invariants + +- Do not require a venv. System-Python compatibility is intentional. +- Treat `ShapedDevices.csv` and `network.json` as shared contracts across integrations, scheduler, overrides, and shaping. +- Preserve stable circuit and device identities emitted by integrations unless the change explicitly includes a migration plan for overrides and downstream consumers. +- Preserve tolerant input handling for operator-managed files: BOMs, UTF-16/non-UTF8 CSVs, comment stripping, and uneven rows are all present in current workflows for a reason. +- Preserve scheduler resilience: importer failures should be logged/reported and should not kill the scheduler loop. +- Preserve existing external protocols and identity values for integrations unless the task explicitly includes coordinated changes on both sides. +- Be careful with paths: some code uses `get_libreqos_directory()`, while some helpers and tests intentionally use cwd-relative files. + +## Validation + +- Run targeted tests from `src/`, not repo root, for example: + - `python3 -m unittest test_scheduler.py` + - `python3 -m unittest test_shaping_skip_report.py` + - `python3 -m unittest test_virtual_tree_nodes.py` +- If changing scheduler/helpers, prefer the smallest focused unittest set that covers the touched logic. +- If changing generated file behavior, review both the writer and the reader side in the same change. + +## References + +- Read `references/generated-runtime-files.md` when deciding whether a file is source input, generated output, or runtime state. +- Read `references/integration-modes.md` when touching scheduler/importer selection, stable identities, or integration output assumptions. + +## High-Risk Changes + +- Anything that changes `ShapedDevices.csv` columns, row semantics, or encoding behavior +- Anything that changes `network.json` structure or virtual-node handling +- Scheduler subprocess/error-handling behavior +- Runtime path handling or current-working-directory assumptions +- Live shaping entrypoints that can touch TC/XDP state +- Broad "cleanup" refactors that try to package-ize or normalize the Python tree + +## Notes + +- `ispConfig.py` is useful as a historical/example config reference, but current runtime config is largely surfaced through `liblqos_python` and `/etc/lqos.conf`. +- Historical files such as `LibreQoS-old.py`, `LibreQoS-ancient.py`, and `LibreQoS.py.new` exist in-tree but are not the default source of truth. +- If a Python change adds or newly requires shipped files, update `src/build_dpkg.sh` in the same change. diff --git a/.codex-repo/skills/libreqos-python-workflow/agents/openai.yaml b/.codex-repo/skills/libreqos-python-workflow/agents/openai.yaml new file mode 100644 index 00000000..08ede03f --- /dev/null +++ b/.codex-repo/skills/libreqos-python-workflow/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "LibreQoS Python Workflow" + short_description: "Python shaping and integration workflow" + default_prompt: "Use $libreqos-python-workflow to work on LibreQoS Python orchestration, scheduler, integrations, or generated config files." diff --git a/.codex-repo/skills/libreqos-python-workflow/references/generated-runtime-files.md b/.codex-repo/skills/libreqos-python-workflow/references/generated-runtime-files.md new file mode 100644 index 00000000..e17cafd5 --- /dev/null +++ b/.codex-repo/skills/libreqos-python-workflow/references/generated-runtime-files.md @@ -0,0 +1,30 @@ +# Generated And Runtime Files + +Use this reference when deciding whether a file is source-of-truth input, generated output, or runtime state. + +## Source Inputs + +- `ShapedDevices.csv`: operator- or integration-managed shaping input consumed by `LibreQoS.py` +- `network.json`: operator- or integration-managed topology input consumed by `LibreQoS.py` +- `network.insight.json` / `ShapedDevices.insight.csv`: Insight-topology variants when enabled + +## Integration Writers + +- `integrationCommon.py` writes `network.json` and `ShapedDevices.csv` +- `scheduler.py` may rewrite `ShapedDevices.csv` and `network.json` when applying overrides + +## Shaper Outputs + +- `queuingStructure.json`: runtime queue/tree snapshot written by `LibreQoS.py` +- `statsByCircuit.json`, `statsByParentNode.json`: runtime stats snapshots written by `LibreQoS.py` +- `linux_tc.txt` and related `linux_tc*.txt` files: runtime/debug outputs from shaping work +- `lastRun.txt`: last successful shaper run marker +- `ShapedDevices.lastLoaded.csv`: snapshot of the last loaded shaping input +- `lastGoodConfig.csv`, `lastGoodConfig.json`: fallback copies of last validated good inputs +- `planner_state.json`: runtime planner state used by shaping logic + +## Agent Guidance + +- Prefer editing source inputs and code, not generated/runtime outputs. +- When changing a writer, review every reader of that file in the same change. +- Preserve stable file names and semantics unless the task explicitly includes a coordinated migration. diff --git a/.codex-repo/skills/libreqos-python-workflow/references/integration-modes.md b/.codex-repo/skills/libreqos-python-workflow/references/integration-modes.md new file mode 100644 index 00000000..8d46cddc --- /dev/null +++ b/.codex-repo/skills/libreqos-python-workflow/references/integration-modes.md @@ -0,0 +1,25 @@ +# Integration Modes + +Use this reference when working on scheduler/importer behavior. + +## Scheduler Selection + +- `scheduler.importFromCRM()` uses an `elif` chain for automatic integrations. +- In normal automatic mode, only the first enabled integration runs. +- This is not a fan-out scheduler for running multiple automatic importers in one pass. + +## UISP Special Case + +- Automatic UISP import is handled through the compiled `bin/uisp_integration` binary. +- `integrationUISP.py` still exists in-tree, but the scheduler's automatic UISP path does not call it directly. + +## Shared Contracts + +- Integrations normalize external systems into `network.json` and `ShapedDevices.csv`. +- Stable circuit IDs and device IDs are important for overrides, planner state, and partial reload behavior. +- Preserve existing external protocols and identity values unless the task explicitly includes coordinated work on both sides. + +## Agent Guidance + +- Do not assume all integrations are equally mature; error handling and path behavior vary across modules. +- When changing integration output shape or identity rules, review scheduler overrides and `LibreQoS.py` consumers in the same change. diff --git a/.codex-repo/skills/libreqos-rust-workflow/SKILL.md b/.codex-repo/skills/libreqos-rust-workflow/SKILL.md new file mode 100644 index 00000000..0f871ef9 --- /dev/null +++ b/.codex-repo/skills/libreqos-rust-workflow/SKILL.md @@ -0,0 +1,45 @@ +--- +name: libreqos-rust-workflow +description: Shared LibreQoS Rust workflow for repo contributors. Use when changing Rust under src/rust, validating Rust crates, deciding between workspace commands and --manifest-path, or applying LibreQoS-specific Rust conventions and verification steps. +--- + +# LibreQoS Rust Workflow + +Use this skill for Rust work in this repo. + +## Scope + +- Rust sources live under `src/rust/`. +- `src/rust/Cargo.toml` is the source of truth for current workspace members. +- Some crates exist in-tree but are not current workspace members. If a crate is outside `[workspace].members`, use `cargo --manifest-path path/to/Cargo.toml`. + +## Workflow + +1. Read `AGENTS.md` first for current repo rules and crate descriptions. +2. Identify whether the touched crate is a workspace member. +3. Validate the touched crate with `cargo check -p ` when possible. +4. Run relevant tests. +5. Run `cargo clippy` for the touched crate and fix actionable issues. +6. If dependencies changed, also run: + - `cargo machete` + - `cargo audit` + - `cargo tree` +7. If the change adds, renames, moves, or newly depends on runtime files, static assets, helper scripts, service files, templates, or install-time artifacts, review and update `src/build_dpkg.sh` in the same change. + +## Preferred Rust Direction + +- Prefer `parking_lot` for new `Mutex` and `RwLock` usage. +- Prefer `crossbeam_channel` for new MPSC/MPMC channels. +- Prefer `thiserror` for structured errors. +- Prefer `let else` and early returns over deeply nested `if let`. +- Avoid introducing new `pub static` values with locks when helper functions or actors are better. +- Avoid introducing new `#[inline(always)]`; prefer `#[inline]`. +- Keep RustDoc current for changed public items and note side effects for non-pure functions. +- Avoid allocation in hot paths. + +## Notes + +- Existing code does not fully match all preferred conventions yet. Treat these as direction for new and touched code, not as a reason to perform unrelated cleanup. +- Build/package scripts live under `src/`, not repo root. +- `src/build_dpkg.sh` is a functional packaging manifest for shipped installs. Forgetting to update it is a common failure mode; treat package-content drift as a bug. +- For Insight/LTS2 integrations, preserve the existing external protocol and identity values unless the task explicitly covers coordinated changes on both sides. Unilateral protocol drift is a breaking change. diff --git a/.codex-repo/skills/libreqos-rust-workflow/agents/openai.yaml b/.codex-repo/skills/libreqos-rust-workflow/agents/openai.yaml new file mode 100644 index 00000000..1ac2d205 --- /dev/null +++ b/.codex-repo/skills/libreqos-rust-workflow/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "LibreQoS Rust Workflow" + short_description: "Rust crate workflow and checks" + default_prompt: "Use $libreqos-rust-workflow to work on LibreQoS Rust crates, validation steps, or packaging-aware Rust changes." diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..a044dd7b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,171 @@ +LibreQoS is a Quality-of-Experience traffic management and analysis system. It's open source (GPL2). It is supported +by a paid Insight package (aka LTS2) and a paid API (lqos_api). + +# Layout + +src/ Main source tree: Python entrypoints, packaging/build scripts, templates, test data, generated/runtime files +src/rust/ Rust source tree and workspace root +docs/ English documentation +docs-es/ Spanish documentation +docker/ Container-related files and configs +sim/ Simulation/support assets + +## Python + +Most Python entrypoints live in `src/`, not the repo root. + +src/LibreQoS.py Main shaper/orchestrator. Validates inputs, builds the shaping tree, programs TC/XDP state, and writes runtime stats/artifacts. +src/scheduler.py Background scheduler. Runs integrations, applies overrides, updates Web UI scheduler status, and triggers refreshes. +src/integrationCommon.py Shared integration graph/model layer. Builds `network.json` and `ShapedDevices.csv` from importer data. +src/integration*.py CRM/NMS importers and related helpers. UISP may run via Rust binary; several others still run in Python. +src/virtual_tree_nodes.py Logical-to-physical network helpers for virtual nodes in `network.json`. +src/shaping_skip_report.py Helper for reporting devices/circuits that were not attached during shaping. +src/lqTools.py Operator/debug helper CLI for inspecting active shaping state. +src/configMigrator.py Compatibility helper for migrating old `ispConfig.py` settings into newer config flows. +src/csvToNetworkJSON.py Helper script for generating `network.json` from CSV input. +src/cidrToHosts.py Helper script for expanding CIDRs in `ShapedDevices.csv`. +src/mikrotikFindIPv6.py Optional Mikrotik helper used by some integrations for IPv6 enrichment. +src/pythonCheck.py Shared Python version gate used by executable Python entrypoints. +src/test_*.py Lightweight unit tests. These generally assume the working directory is `src/`. +src/bakery_integration_test.py Larger integration-oriented test harness for shaping/tree behavior. +src/LibreQoS-old.py Historical copy; not the primary source of truth for current work. +src/LibreQoS-ancient.py Historical copy; not the primary source of truth for current work. +src/LibreQoS.py.new Staging/alternate copy; not the primary source of truth unless the task explicitly targets it. +src/VERSION_STRING Current release string, also used by packaging/build scripts. + +## Rust + +`src/rust/Cargo.toml` is the source of truth for current workspace members. + +### Current workspace members + +lqosd Main daemon. Also contains the node manager / web UI server. +xdp_iphash_to_cpu_cmdline Compatibility CLI for managing IP mappings in the XDP cpumap system. +xdp_pping Legacy compatibility CLI for the earlier `xdp_pping` workflow. +uisp_integration Rust UISP integration binary. +uisp Shared UISP API/types used by the integration. +lqusers CLI for managing LibreQoS web logins. +lqtop CLI/TUI for displaying live network status. +lqos_utils Shared helpers and utility types. +lqos_sys XDP/TC/eBPF bridge and kernel-program management. +lqos_stormguard Experimental dynamic rate-adjustment system driven by live link-quality/capacity data. +lqos_setup First-run/setup CLI. +lqos_python PyO3 bindings exposing Rust config/bus/helpers to Python. +lqos_overrides Override library/CLI for persistent devices, circuit adjustments, network adjustments, and UISP overrides. +lqos_map_perf eBPF/XDP map performance CLI. +lqos_heimdall Heimdall packet/flow watchlist and capture support. +lqos_config Configuration management, including config, shaped devices, and network.json handling. +lqos_bus Local bus types and Unix-domain-socket support used to communicate with lqosd. +lqos_bakery Queue creation/update/delete and diff-driven queue management. + +### Other Rust crates/directories present in-tree + +lqos_queue_tracker Reads and tracks Linux `tc` queue state for the rest of the system. +lqos_support_tool Support/sanity-check CLI for gathering and submitting support dumps. +third_party/libbpf-sys Vendored libbpf-sys tree used by the Rust side. + +## Helper Scripts + +src/build_rust.sh Builds the Rust side in release mode, installs binaries/artifacts into `src/bin`, updates `liblqos_python.so`, and may restart services +src/build_dpkg.sh Builds LibreQoS and assembles a `.deb` package +src/rust/lqosd/dev_build.sh Builds node_manager JS bundles and copies static assets into `src/bin/static2` for local UI iteration +src/lqosd/src/node_manager/js_build/esbuild.sh Builds all JavaScript and puts the web system in the right place (must run from that directory) +src/rust/lqosd/src/node_manager/js_build/test-build-contract.sh Verifies the node_manager page/build/vendor contract before bundling +.codex-repo/link-skills.sh Symlinks repo-owned shared Codex skills into `~/.codex/skills` + +## Shared Codex Skills + +Canonical shared Codex skills for this repo live in `.codex-repo/skills/`. + +- These skills are intended to be committed to the repo and shared across developers. +- Developers should link them into `~/.codex/skills` with `.codex-repo/link-skills.sh`. +- After linking new skills, or pulling updates that change them, restart Codex so the skill list refreshes. +- When a LibreQoS repo skill is installed and the task matches it, use it. +- If your changes touch `.codex-repo/skills/` or `.codex-repo/link-skills.sh`, remind the user to run `./.codex-repo/link-skills.sh` and restart Codex so updated shared skills are picked up. + +# Code Rules + +## Generated Artifacts + +- Treat built outputs, generated files, and runtime artifacts as derived state unless the task explicitly targets the generator or a test fixture. +- Do not hand-edit generated/built outputs such as `src/bin/**`, `dist/**`, `target/**`, `src/rust/lqosd/src/node_manager/js_build/out/**`, generated `OUT_DIR` artifacts, packaged `.deb` trees, runtime graphs/images/logs, or shaping snapshots like `queuingStructure.json`, `statsByCircuit.json`, `statsByParentNode.json`, `lastRun.txt`, and `linux_tc*.txt`. +- Edit the source inputs, rebuild/regenerate, and then review the result. + +## Operational Safety + +- Any command that can affect live shaping, TC/XDP state, network interfaces, system services, package install/uninstall state, or installed runtime files under `/opt/libreqos` or `/etc` requires explicit user approval. +- Prefer non-destructive validation first: targeted tests, `cargo check`, shell syntax checks, build-contract checks, and static review before any live-host or service-affecting action. + +## Rust + +- For Rust changes, prefer validating touched workspace members with `cargo check -p package`, `cargo test`, and `cargo clippy`. +- Some crates present under `src/rust/` are not current workspace members. For those, use `cargo --manifest-path path/to/Cargo.toml`. +- Use `cargo machete` when changing Rust dependencies in the workspace. +- Use `cargo audit`. Failures with no mitigation, unmaintained crates are acceptable. +- Use `cargo tree` when adding dependencies and try to minimize the number of dependencies with different versions in the tree. +- Prefer `parking_lot` for new Mutex/RwLock usage. Existing code still contains some `std::sync` locks and atomics. +- Prefer `crossbeam_channel` for new MPSC/MPMC channels. Existing code still contains some `std::sync::mpsc`. +- Public Rust functions and types should have RustDoc. Update RustDoc when changing behavior. +- Avoid nested `if let` where `let else` with early return is clearer. +- Prefer `thiserror` defined errors. Log the error during `map_err` while converting to the error type. + - Avoid error messages that lack specificity. "File not found" is not ok; "File (filename) not found" is great. +- In most cases, prefer an actor (thread or async) over a fine-grained lock. +- Avoid introducing new `pub static` values with locks when helper functions or actors would work better. Existing code still has some shared globals. +- Prefer pure functions. If a pure function can be constified, do it. +- If a function is NOT pure, the RustDoc should document any side effects. +- Avoid introducing new `#[inline(always)]`; prefer `#[inline]`. Existing hot-path code may still contain `#[inline(always)]`. +- Consider inline markers for functions in the hot path, such as XDP ring-buffer/perf-buffer processing. +- Avoid allocation in the hot path. + +## Python + +- If a task touches `src/LibreQoS.py`, `src/scheduler.py`, `src/integration*.py`, `src/integrationCommon.py`, or Python helpers/tests around shaping and integrations, use the `libreqos-python-workflow` repo skill if installed. +- Do not add a mandatory venv/poetry/pipenv workflow. This repo intentionally supports system-Python installs for operational compatibility. +- Treat `ShapedDevices.csv` and `network.json` as functional contracts shared between integrations, overrides, the scheduler, and `LibreQoS.py`. Do not casually rename them, relocate them, or change their schema/semantics. +- Preserve stable circuit/device identity values emitted by integrations unless the task explicitly calls for a coordinated migration. Overrides, partial reload logic, and downstream tooling depend on those identifiers staying stable. +- Preserve tolerant file handling for operator-managed inputs. Existing code intentionally accepts BOMs, UTF-16/non-UTF8 CSVs, comment lines, and uneven row shapes in places like `ShapedDevices.csv`. +- Preserve scheduler fault tolerance. Integration failures should surface to scheduler output/error reporting and continue scheduling, not terminate the scheduler process. +- Preserve current path behavior carefully. Some Python code uses `get_libreqos_directory()` for installed runtime paths, while other helpers/tests intentionally read and write files relative to the current working directory. +- Prefer `get_libreqos_directory()` when adding new installed-runtime path lookups, but do not "normalize" existing cwd-based scripts without reviewing all callers, packaging, and operator workflows. +- Keep `pythonCheck.checkPythonVersion()` at the top of executable Python entrypoints unless the script is being intentionally retired. +- Prefer focused `python3 -m unittest ...` runs from `src/` for touched Python tests. Existing tests are not laid out as a package and often assume that working directory. +- Prefer incremental hardening over large Python rewrites. This code is operational glue with many install-time and user-data assumptions; avoid broad refactors unless the task explicitly calls for them. +- Do not run `src/LibreQoS.py` or other live-shaping Python entrypoints against the host without explicit user approval; they can modify active TC/XDP state and scheduler-visible files. + +### lqos_sys BPF/FFI + +- If a task touches `src/rust/lqos_sys/build.rs`, `src/rust/lqos_sys/src/bpf/**`, `src/rust/lqos_sys/src/bpf/wrapper.c`, `src/rust/lqos_sys/src/bpf/wrapper.h`, or Rust structs mirroring eBPF map layouts, use the `libreqos-lqos-sys-ebpf` repo skill if installed. +- Never edit generated `OUT_DIR` artifacts. Edit only source files under `src/rust/lqos_sys/`. +- Treat BPF-side structs, Rust mirror structs, bindgen output expectations, and pinned-map ABI handling as one linked surface. +- When changing BPF map key/value structs or shared constants, update the Rust mirror types and size assertions in the same change. +- When changing pinned-map ABI, review the map compatibility cleanup in `src/rust/lqos_sys/src/lqos_kernel.rs`. +- Do not run live XDP/TC attach-detach or pinned-map cleanup without explicit user approval. These actions can disrupt host networking. + +### Node Manager Frontend + +- If a task touches `src/rust/lqosd/src/node_manager/js_build/**`, `src/rust/lqosd/src/node_manager/static2/**`, `src/rust/lqosd/copy_files.sh`, `src/rust/lqosd/dev_build.sh`, or `src/rust/lqosd/src/node_manager/static_pages.rs`, use the `libreqos-node-manager-frontend` repo skill if installed. +- Treat `template.html`, `node_manager.css`, `static2/vendor/`, `js_build/entrypoints.txt`, `js_build/esbuild.sh`, `js_build/test-build-contract.sh`, `copy_files.sh`, `dev_build.sh`, and `static_pages.rs` as one linked surface. +- Node manager frontend dependencies are Bootstrap 5, FontAwesome, vendored assets, and page-specific bundled JS. Do not infer frontend dependencies from `src/package.json`. +- Preserve the existing node_manager look and feel. Use `src/rust/lqosd/src/node_manager/static2/template.html`, `src/rust/lqosd/src/node_manager/static2/node_manager.css`, and the existing pages as the baseline rather than introducing a new design system or framework. +- Prefer vendored frontend assets under `src/rust/lqosd/src/node_manager/static2/vendor/`. Avoid new CDN or npm dependencies unless explicitly required. +- When adding or renaming a node_manager page, update `js_build/src/.js`, `js_build/entrypoints.txt`, `static2/.html`, and `static_pages.rs` together. +- Run `src/rust/lqosd/src/node_manager/js_build/test-build-contract.sh` after changes to the node_manager frontend build path. + +### Insight Integration Compatibility + +- Any communication with Insight/LTS2 must preserve the existing external protocol, message shapes, field names, identity values, and compatibility expectations by default. +- Do not change Insight-facing protocol details or identity semantics unless the task explicitly includes coordinated work on both LibreQoS and the Insight side. +- If a task appears to require an Insight protocol or identity change but the paired Insight-side update is not in scope, stop and surface that as a blocker instead of making a unilateral compatibility break. +- Treat Insight protocol drift as a breaking change, even if the local LibreQoS side compiles or tests cleanly. + +## Scripts + +- If a task touches `src/build_rust.sh`, `src/build_dpkg.sh`, `src/rust/lqosd/copy_files.sh`, shipped service files, packaged templates/assets, or anything that must exist on installed LibreQoS systems, use the `libreqos-packaging-release` repo skill if installed. +- Any change to `src/build_rust.sh` or `src/build_dpkg.sh` MUST be reflected in the other where applicable. They should stay in sync on shared build/package assumptions. +- `src/build_dpkg.sh` is part of the functional source of truth for shipped LibreQoS installs. If a change adds, renames, moves, or newly requires files at runtime, packaging time, or install time, update `src/build_dpkg.sh` in the same change. +- Never assume a new file is "obvious enough" to be picked up automatically by packaging. Verify that `src/build_dpkg.sh` explicitly includes every new file or directory required for the feature to work in an installed `.deb`. +- When a change affects packaged assets, treat "is `src/build_dpkg.sh` still accurate?" as a mandatory review check before finishing. + +## Documentation + +- Always keep the documentation updated to match code changes. diff --git a/README.md b/README.md index d96d3511..0c9ceb5f 100644 --- a/README.md +++ b/README.md @@ -59,3 +59,13 @@ Dave served as LibreQoS' Chief Science Officer and championed the global fight a ## External Pull Request Policy We can only accept PRs that address one specific change or topic each. We ask that you keep all changes small and focused per-PR to help our review and testing process. + +## LLM Agent Setup + +If you use an LLM coding agent with shared LibreQoS repo skills: + +```bash +./.codex-repo/link-skills.sh +``` + +This links the repo-owned shared skills into `~/.codex/skills`. After linking, or after pulling changes to `.codex-repo/skills/`, restart your agent so it reloads the updated skills. diff --git a/src/rust/lqosd/copy_files.sh b/src/rust/lqosd/copy_files.sh index 9c445cf5..3384b44c 100755 --- a/src/rust/lqosd/copy_files.sh +++ b/src/rust/lqosd/copy_files.sh @@ -5,6 +5,7 @@ mkdir -p ../../bin/static2/ cp -v -R src/node_manager/static2/* ../../bin/static2/ echo "Done" pushd src/node_manager/js_build || exit +./test-build-contract.sh ./esbuild.sh popd || exit cp -R src/node_manager/js_build/out/* ../../bin/static2/ diff --git a/src/rust/lqosd/dev_build.sh b/src/rust/lqosd/dev_build.sh index c4a3d3a6..7871b35b 100755 --- a/src/rust/lqosd/dev_build.sh +++ b/src/rust/lqosd/dev_build.sh @@ -9,6 +9,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" echo "[dev_build] Building JS bundles..." pushd "$SCRIPT_DIR/src/node_manager/js_build" >/dev/null +./test-build-contract.sh ./esbuild.sh popd >/dev/null diff --git a/src/rust/lqosd/src/node_manager/js_build/README.md b/src/rust/lqosd/src/node_manager/js_build/README.md index 252dbbbe..08a1bbcd 100644 --- a/src/rust/lqosd/src/node_manager/js_build/README.md +++ b/src/rust/lqosd/src/node_manager/js_build/README.md @@ -1,5 +1,16 @@ # Site_build -This assembles the JavaScript for the site using `esbuild`. You need to have installed -`esbuild` with `sudo apt install esbuild` for this to work. +This assembles the JavaScript for the node_manager site using `esbuild`. +## Source of truth + +- Page entrypoints live in `entrypoints.txt` +- Page source files live in `src/` +- Bundled outputs are written to `out/` + +## Commands + +- `./test-build-contract.sh` +- `./esbuild.sh` + +`esbuild.sh` prefers a locally installed `esbuild` binary when available. If none is found, it falls back to downloading a standalone binary into `/tmp/esbuild`. diff --git a/src/rust/lqosd/src/node_manager/js_build/entrypoints.txt b/src/rust/lqosd/src/node_manager/js_build/entrypoints.txt new file mode 100644 index 00000000..469429c3 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/entrypoints.txt @@ -0,0 +1,47 @@ +index.js +template.js +login.js +first-run.js +shaped-devices.js +tree.js +help.js +unknown-ips.js +configuration.js +circuit.js +flow_map.js +all_tree_sankey.js +asn_explorer.js +lts_trial.js +config_general.js +config_rtt.js +config_tuning.js +config_queues.js +config_stormguard.js +config_treeguard.js +config_lts.js +config_iprange.js +config_flows.js +config_integration.js +config_splynx.js +config_netzur.js +config_visp.js +config_uisp.js +config_powercode.js +config_sonar.js +config_interface.js +config_network.js +config_devices.js +config_users.js +config_wispgate.js +chatbot.js +cpu_weights.js +cpu_tree.js +executive_worst_sites.js +executive_oversubscribed_sites.js +executive_sites_due_upgrade.js +executive_circuits_due_upgrade.js +executive_top_asns.js +executive_heatmap_rtt.js +executive_heatmap_retransmit.js +executive_heatmap_download.js +executive_heatmap_upload.js diff --git a/src/rust/lqosd/src/node_manager/js_build/esbuild.sh b/src/rust/lqosd/src/node_manager/js_build/esbuild.sh index 7550819f..80366b9d 100755 --- a/src/rust/lqosd/src/node_manager/js_build/esbuild.sh +++ b/src/rust/lqosd/src/node_manager/js_build/esbuild.sh @@ -1,17 +1,40 @@ -#!/bin/bash -set -e -#ESBUILD_BIN="$(command -v esbuild || true)" -#if [[ -z "$ESBUILD_BIN" ]]; then +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENTRYPOINTS_FILE="${SCRIPT_DIR}/entrypoints.txt" +SRC_DIR="${SCRIPT_DIR}/src" +OUT_DIR="${SCRIPT_DIR}/out" + +if [[ ! -f "${ENTRYPOINTS_FILE}" ]]; then + echo "Missing entrypoints file: ${ENTRYPOINTS_FILE}" >&2 + exit 1 +fi + +if [[ -z "${ESBUILD_BIN:-}" ]]; then + ESBUILD_BIN="$(command -v esbuild || true)" +fi + +if [[ -z "${ESBUILD_BIN}" ]]; then mkdir -p /tmp/esbuild - pushd /tmp/esbuild + pushd /tmp/esbuild >/dev/null curl -fsSL https://esbuild.github.io/dl/latest | sh - popd + popd >/dev/null chmod a+x /tmp/esbuild/esbuild ESBUILD_BIN="/tmp/esbuild/esbuild" -#fi -scripts=( index.js template.js login.js first-run.js shaped-devices.js tree.js help.js unknown-ips.js configuration.js circuit.js flow_map.js all_tree_sankey.js asn_explorer.js lts_trial.js config_general.js config_rtt.js config_tuning.js config_queues.js config_stormguard.js config_treeguard.js config_lts.js config_iprange.js config_flows.js config_integration.js config_splynx.js config_netzur.js config_visp.js config_uisp.js config_powercode.js config_sonar.js config_interface.js config_network.js config_devices.js config_users.js config_wispgate.js chatbot.js cpu_weights.js cpu_tree.js executive_worst_sites.js executive_oversubscribed_sites.js executive_sites_due_upgrade.js executive_circuits_due_upgrade.js executive_top_asns.js executive_heatmap_rtt.js executive_heatmap_retransmit.js executive_heatmap_download.js executive_heatmap_upload.js) -for script in "${scripts[@]}" -do - echo "Building {$script}" - "$ESBUILD_BIN" src/"$script" --bundle --minify --sourcemap --target=chrome58,firefox57,safari11 --outdir=out || exit +fi + +mkdir -p "${OUT_DIR}" +find "${OUT_DIR}" -maxdepth 1 -type f \( -name '*.js' -o -name '*.js.map' \) -delete + +mapfile -t scripts < <(grep -Ev '^\s*(#|$)' "${ENTRYPOINTS_FILE}") + +for script in "${scripts[@]}"; do + if [[ ! -f "${SRC_DIR}/${script}" ]]; then + echo "Missing source entrypoint: ${SRC_DIR}/${script}" >&2 + exit 1 + fi + + echo "Building ${script}" + "${ESBUILD_BIN}" "${SRC_DIR}/${script}" --bundle --minify --sourcemap --target=chrome58,firefox57,safari11 --outdir="${OUT_DIR}" done diff --git a/src/rust/lqosd/src/node_manager/js_build/test-build-contract.sh b/src/rust/lqosd/src/node_manager/js_build/test-build-contract.sh new file mode 100755 index 00000000..0740be56 --- /dev/null +++ b/src/rust/lqosd/src/node_manager/js_build/test-build-contract.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENTRYPOINTS_FILE="${SCRIPT_DIR}/entrypoints.txt" +SRC_DIR="${SCRIPT_DIR}/src" +OUT_DIR="${SCRIPT_DIR}/out" +STATIC_DIR="$(cd "${SCRIPT_DIR}/../static2" && pwd)" +STATIC_PAGES_RS="${SCRIPT_DIR}/../static_pages.rs" + +errors=0 + +fail() { + echo "[FAIL] $*" >&2 + errors=$((errors + 1)) +} + +if [[ ! -f "${ENTRYPOINTS_FILE}" ]]; then + fail "Missing entrypoints file: ${ENTRYPOINTS_FILE}" +fi + +if [[ ! -f "${STATIC_PAGES_RS}" ]]; then + fail "Missing static page router source: ${STATIC_PAGES_RS}" +fi + +mapfile -t entrypoints < <(grep -Ev '^\s*(#|$)' "${ENTRYPOINTS_FILE}") +declare -A entrypoint_set=() + +for entrypoint in "${entrypoints[@]}"; do + entrypoint_set["${entrypoint}"]=1 + [[ -f "${SRC_DIR}/${entrypoint}" ]] || fail "Missing JS source entrypoint: ${SRC_DIR}/${entrypoint}" + [[ -f "${OUT_DIR}/${entrypoint}" ]] || fail "Missing built bundle: ${OUT_DIR}/${entrypoint}" + [[ -f "${OUT_DIR}/${entrypoint}.map" ]] || fail "Missing sourcemap: ${OUT_DIR}/${entrypoint}.map" +done + +for artifact in "${OUT_DIR}"/*.js; do + [[ -e "${artifact}" ]] || continue + name="$(basename "${artifact}")" + [[ -n "${entrypoint_set[${name}]:-}" ]] || fail "Unexpected built JS artifact not listed in entrypoints.txt: ${name}" +done + +for artifact in "${OUT_DIR}"/*.js.map; do + [[ -e "${artifact}" ]] || continue + name="$(basename "${artifact}" .map)" + [[ -n "${entrypoint_set[${name}]:-}" ]] || fail "Unexpected sourcemap not listed in entrypoints.txt: $(basename "${artifact}")" +done + +mapfile -t served_pages < <(sed -n 's/^[[:space:]]*"\([A-Za-z0-9_.-]\+\.html\)",$/\1/p' "${STATIC_PAGES_RS}") + +for page in "${served_pages[@]}"; do + [[ -f "${STATIC_DIR}/${page}" ]] || fail "Page is served by static_pages.rs but missing from static2: ${page}" +done + +required_template_assets=( + "node_manager.css" + "vendor/bootstrap.min.css" + "vendor/bootstrap.bundle.min.js" + "vendor/jquery-3.7.1.min.js" + "vendor/echarts.min.js" + "vendor/echarts-gl.min.js" + "vendor/echarts_dark.js" + "vendor/echart_vintage.js" + "vendor/sortable.min.js" + "vendor/fontawesome/css/all.css" +) + +for asset in "${required_template_assets[@]}"; do + [[ -f "${STATIC_DIR}/${asset}" ]] || fail "Missing required node_manager asset referenced by template.html: ${asset}" +done + +check_cachebusted_bundle_refs() { + local file_path="$1" + local label="$2" + mapfile -t refs < <(sed -n 's#.*#\1#p' "${file_path}") + for bundle in "${refs[@]}"; do + [[ -n "${entrypoint_set[${bundle}]:-}" ]] || fail "${label} references bundle not listed in entrypoints.txt: ${bundle}" + [[ -f "${SRC_DIR}/${bundle}" ]] || fail "${label} references bundle with no source entrypoint: ${bundle}" + done +} + +check_direct_bundle_refs() { + local file_path="$1" + local label="$2" + mapfile -t refs < <(sed -n 's#.*#\1#p' "${file_path}") + for bundle in "${refs[@]}"; do + [[ -n "${entrypoint_set[${bundle}]:-}" ]] || fail "${label} references bundle not listed in entrypoints.txt: ${bundle}" + [[ -f "${SRC_DIR}/${bundle}" ]] || fail "${label} references bundle with no source entrypoint: ${bundle}" + done +} + +check_cachebusted_bundle_refs "${STATIC_DIR}/template.html" "template.html" + +for page in "${served_pages[@]}"; do + check_cachebusted_bundle_refs "${STATIC_DIR}/${page}" "${page}" +done + +for standalone_page in "login.html" "first-run.html"; do + check_direct_bundle_refs "${STATIC_DIR}/${standalone_page}" "${standalone_page}" +done + +if (( errors > 0 )); then + echo "[FAIL] node_manager build contract check failed with ${errors} issue(s)." >&2 + exit 1 +fi + +echo "[OK] node_manager build contract is consistent." diff --git a/src/rust/lqosd/src/node_manager/static2/chatbot.html b/src/rust/lqosd/src/node_manager/static2/chatbot.html index fe15f42c..d3c7778a 100644 --- a/src/rust/lqosd/src/node_manager/static2/chatbot.html +++ b/src/rust/lqosd/src/node_manager/static2/chatbot.html @@ -39,4 +39,4 @@ - + diff --git a/src/rust/lqosd/src/node_manager/static2/lts_trial.html b/src/rust/lqosd/src/node_manager/static2/lts_trial.html index 6594e9e1..22840a2f 100644 --- a/src/rust/lqosd/src/node_manager/static2/lts_trial.html +++ b/src/rust/lqosd/src/node_manager/static2/lts_trial.html @@ -396,4 +396,4 @@ body.dark-mode .card-header.bg-light { - \ No newline at end of file + diff --git a/src/rust/lqosd/src/node_manager/static2/template.html b/src/rust/lqosd/src/node_manager/static2/template.html index e93a0370..d9d0c658 100644 --- a/src/rust/lqosd/src/node_manager/static2/template.html +++ b/src/rust/lqosd/src/node_manager/static2/template.html @@ -240,7 +240,7 @@ LibreQoS %%VERSION%%. © Copyright 2022-, LibreQoE LLC. All Rights Reserved. - +