IntenseWebs/LibreQoS
3
0 Fork 0
mirror of https://github.com/LibreQoE/LibreQoS.git synced 2026-11-02 08:10:02 -06:00
Code Issues Packages Projects Releases Wiki Activity

changes

Browse Source
This commit is contained in:
rchac committed 2026-04-08 17:47:56 +00:00
1 parent 6b56bd47f1
commit 36aad13590
37 files changed
+2958 -288
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+2
View File
@@ -179,6 +179,8 @@ src/network.large.json
/src/topology_overrides.json
/src/topology_parent_candidates.json
/src/topology_canonical_state.json
/src/circuit_anchors.json
/src/shaping_inputs.json
# Migrated integration compatibility artifacts
/src/integrationUISPbandwidths.csv.migrated-*
docs/v2.0/configuration-advanced.md
+20 -19
View File
@@ -307,10 +307,10 @@ LibreQoS knows how to shape these devices, and what Node (AP, Site, etc) they ar
The ShapedDevices.csv file correlates device IP addresses to Circuits (each internet subscriber's unique service).
The base format has 13 columns, with an optional 14th `sqm` column for per-circuit queue overrides:
The base format has 15 columns, with an optional 16th `sqm` column for per-circuit queue overrides:
```
Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment[,sqm]
Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment[,sqm]
```
##### Optional `sqm` column
@@ -340,34 +340,34 @@ Important:
See [TreeGuard](treeguard.md).
Here is an example of an entry in the ShapedDevices.csv file:
| Circuit ID | Circuit Name | Device ID | Device Name | Parent Node | MAC | IPv4 | IPv6 | Download Min Mbps | Upload Min Mbps | Download Max Mbps | Upload Max Mbps | Comment |
|------------|-----------------------------------------------------|-----------|-------------|-------------|-----|-------------------------|----------------------|-------------------|-----------------|-------------------|-----------------|---------|
| 1 | 968 Circle St., Gurnee, IL 60031 | 1 | Device 1 | AP_A | | 100.64.0.1, 100.64.0.14 | fdd7:b724:0:100::/56 | 1 | 1 | 155 | 20 | |
| 2 | 31 Marconi Street, Lake In The Hills, IL 60156 | 2 | Device 2 | AP_A | | 100.64.0.2 | fdd7:b724:0:200::/56 | 1 | 1 | 105 | 18 | |
| 3 | 255 NW. Newport Ave., Jamestown, NY 14701 | 3 | Device 3 | AP_9 | | 100.64.0.3 | fdd7:b724:0:300::/56 | 1 | 1 | 105 | 18 | |
| 4 | 8493 Campfire Street, Peabody, MA 01960 | 4 | Device 4 | AP_9 | | 100.64.0.4 | fdd7:b724:0:400::/56 | 1 | 1 | 105 | 18 | |
| 2794 | 6 Littleton Drive, Ringgold, GA 30736 | 5 | Device 5 | AP_11 | | 100.64.0.5 | fdd7:b724:0:500::/56 | 1 | 1 | 105 | 18 | |
| 2794 | 6 Littleton Drive, Ringgold, GA 30736 | 6 | Device 6 | AP_11 | | 100.64.0.6 | fdd7:b724:0:600::/56 | 1 | 1 | 105 | 18 | |
| 5 | 93 Oklahoma Ave., Parsippany, NJ 07054 | 7 | Device 7 | AP_1 | | 100.64.0.7 | fdd7:b724:0:700::/56 | 1 | 1 | 155 | 20 | |
| 6 | 74 Bishop Ave., Bakersfield, CA 93306 | 8 | Device 8 | AP_1 | | 100.64.0.8 | fdd7:b724:0:800::/56 | 1 | 1 | 105 | 18 | |
| 7 | 9598 Peg Shop Drive, Lutherville Timonium, MD 21093 | 9 | Device 9 | AP_7 | | 100.64.0.9 | fdd7:b724:0:900::/56 | 1 | 1 | 105 | 18 | |
| 8 | 115 Gartner Rd., Gettysburg, PA 17325 | 10 | Device 10 | AP_7 | | 100.64.0.10 | fdd7:b724:0:a00::/56 | 1 | 1 | 105 | 18 | |
| 9 | 525 Birchpond St., Romulus, MI 48174 | 11 | Device 11 | Site_1 | | 100.64.0.11 | fdd7:b724:0:b00::/56 | 1 | 1 | 105 | 18 | |
| Circuit ID | Circuit Name | Device ID | Device Name | Parent Node | Parent Node ID | Anchor Node ID | MAC | IPv4 | IPv6 | Download Min Mbps | Upload Min Mbps | Download Max Mbps | Upload Max Mbps | Comment |
|------------|-----------------------------------------------------|-----------|-------------|-------------|----------------|----------------|-----|-------------------------|----------------------|-------------------|-----------------|-------------------|-----------------|---------|
| 1 | 968 Circle St., Gurnee, IL 60031 | 1 | Device 1 | AP_A | | site_1 | | 100.64.0.1, 100.64.0.14 | fdd7:b724:0:100::/56 | 1 | 1 | 155 | 20 | |
| 2 | 31 Marconi Street, Lake In The Hills, IL 60156 | 2 | Device 2 | AP_A | | site_2 | | 100.64.0.2 | fdd7:b724:0:200::/56 | 1 | 1 | 105 | 18 | |
| 3 | 255 NW. Newport Ave., Jamestown, NY 14701 | 3 | Device 3 | AP_9 | | site_3 | | 100.64.0.3 | fdd7:b724:0:300::/56 | 1 | 1 | 105 | 18 | |
| 4 | 8493 Campfire Street, Peabody, MA 01960 | 4 | Device 4 | AP_9 | | site_4 | | 100.64.0.4 | fdd7:b724:0:400::/56 | 1 | 1 | 105 | 18 | |
| 2794 | 6 Littleton Drive, Ringgold, GA 30736 | 5 | Device 5 | AP_11 | | site_2794 | | 100.64.0.5 | fdd7:b724:0:500::/56 | 1 | 1 | 105 | 18 | |
| 2794 | 6 Littleton Drive, Ringgold, GA 30736 | 6 | Device 6 | AP_11 | | site_2794 | | 100.64.0.6 | fdd7:b724:0:600::/56 | 1 | 1 | 105 | 18 | |
| 5 | 93 Oklahoma Ave., Parsippany, NJ 07054 | 7 | Device 7 | AP_1 | | site_5 | | 100.64.0.7 | fdd7:b724:0:700::/56 | 1 | 1 | 155 | 20 | |
| 6 | 74 Bishop Ave., Bakersfield, CA 93306 | 8 | Device 8 | AP_1 | | site_6 | | 100.64.0.8 | fdd7:b724:0:800::/56 | 1 | 1 | 105 | 18 | |
| 7 | 9598 Peg Shop Drive, Lutherville Timonium, MD 21093 | 9 | Device 9 | AP_7 | | site_7 | | 100.64.0.9 | fdd7:b724:0:900::/56 | 1 | 1 | 105 | 18 | |
| 8 | 115 Gartner Rd., Gettysburg, PA 17325 | 10 | Device 10 | AP_7 | | site_8 | | 100.64.0.10 | fdd7:b724:0:a00::/56 | 1 | 1 | 105 | 18 | |
| 9 | 525 Birchpond St., Romulus, MI 48174 | 11 | Device 11 | Site_1 | | site_9 | | 100.64.0.11 | fdd7:b724:0:b00::/56 | 1 | 1 | 105 | 18 | |
If you are using one of our CRM integrations, this file will be automatically generated. If you are not using an integration, you can manually edit the file using either the WebUI or by directly editing the ShapedDevices.csv file through the CLI.
Directional SQM examples:
```
2794,"6 Littleton Drive, Ringgold, GA 30736",5,Device 5,AP_11,,100.64.0.5,"fdd7:b724:0:500::/56",1,1,105,18,"",cake/fq_codel
2795,"7 Example Ave",6,Device 6,AP_11,,100.64.0.6,,1,1,105,18,"",/none
2794,"6 Littleton Drive, Ringgold, GA 30736",5,Device 5,AP_11,,site_2794,,100.64.0.5,"fdd7:b724:0:500::/56",1,1,105,18,"",cake/fq_codel
2795,"7 Example Ave",6,Device 6,AP_11,,site_2795,,100.64.0.6,,1,1,105,18,"",/none
```
##### Multiple IPs per Circuit
If you need to list multiple IPv4s in the IPv4 field, or multiple IPv6s in the IPv6 field, add a comma between them. If you are editing with a CSV editor (LibreOffice Calc, Excel), the CSV editor will automatically wrap these comma-seperated items with quotes for you. If you are editing the file manually with a utility like notepad or nano, please add quotes surrounding the comma-seperated entries.
```
2794,"6 Littleton Drive, Ringgold, GA 30736",5,Device 5,AP_11,,100.64.0.5,"fdd7:b724:0:500::/56,fdd7:b724:0:600::/56",1,1,105,18,""
2794,"6 Littleton Drive, Ringgold, GA 30736",5,Device 5,AP_11,,site_2794,,100.64.0.5,"fdd7:b724:0:500::/56,fdd7:b724:0:600::/56",1,1,105,18,""
```
##### Manual Editing by WebUI
@@ -378,7 +378,8 @@ Navigate to the LibreQoS WebUI (http://a.b.c.d:9123) and select Configuration >
- Modify the ShapedDevices.csv file using your preferred spreadsheet editor (LibreOffice Calc, Excel, etc), following the template file - ShapedDevices.example.csv
- Circuit ID is required. The Circuit ID can be a number or string. This field must NOT include any number symbols (#). Every circuit requires a unique CircuitID - they cannot be reused. Here, circuit essentially refers to a customer's service. If a customer has multiple locations on different parts of your network, use a unique CircuitID for each of those locations.
- At least one IPv4 address or IPv6 address is required for each entry.
- The Access Point or Site name should be set in the Parent Node field. Parent Node can be left blank for flat networks.
- `Anchor Node ID` is the preferred topology key. When present, it should contain the stable node ID for the circuit's own topology node so LibreQoS runtime can derive the effective parent from canonical topology, overrides, and attachment health.
- `Parent Node` and `Parent Node ID` remain supported for compatibility and display/grouping, but new integrations should treat them as legacy fields rather than the primary shaping key.
- The ShapedDevices.csv file allows you to set minimum (guaranteed), and maximum allowed bandwidth per subscriber.
- The Download Min and Upload Min for each Circuit must be 1 Mbps or greater. Generally, these should be set to 1 Mbps by default.
- The Download Max and Upload Max for each Circuit must be 2 Mbps or greater. Generally, these correspond to the customer's speed plan.
docs/v2.0/recipes-education.md
+1 -1
View File
@@ -39,7 +39,7 @@ Recommended patterns:
Example row:
```text
Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment,sqm
Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment,sqm
UNI-BLDG-A-10.40.12.77,BLDG-A-10.40.12.77,UNI-BLDG-A-10.40.12.77,Host-10.40.12.77,BLDG-A,,10.40.12.77,,1,1,150,60,Education per-IP circuit,cake
```
docs/v2.0/recipes-event-wifi.md
+1 -1
View File
@@ -16,7 +16,7 @@ Use this pattern for short-lived, high-density event networks where grouping cli
Example row (one subnet as one shaped circuit):
```text
Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment
Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment
EVT-24-101,Event Hall A Subnet,EVT-24-101,HallA-Clients,Event_Core,,100.64.101.0/24,,10,10,300,300,Temporary event subnet group
```
docs/v2.0/recipes-hospitality.md
+1 -1
View File
@@ -25,7 +25,7 @@ Per-device shaping can raise circuit count quickly. Validate:
## Example Entry
```text
Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment,sqm
Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment,sqm
HTL-ROOM-1204,Room1204-Device,HTL-ROOM-1204,Room1204-DeviceA,Floor12,,100.70.12.44,,2,2,50,20,Hospitality per-device plan,cake
```
src/LibreQoS.py
+298 -9
View File
@@ -25,6 +25,7 @@ from virtual_tree_nodes import (
build_logical_to_physical_node_map,
build_physical_network,
collect_physical_parent_node_aliases,
collect_physical_parent_node_ids,
is_virtual_node,
)
from shaping_skip_report import (
@@ -42,6 +43,8 @@ from liblqos_python import is_lqosd_alive, clear_ip_mappings, delete_ip_mapping,
on_a_stick, get_tree_weights, get_weights, is_network_flat, get_libreqos_directory, enable_insight_topology, \
is_insight_enabled, scheduler_error, xdp_ip_mapping_capacity, \
overrides_circuit_adjustments_effective, \
automatic_import_uisp, automatic_import_splynx, automatic_import_powercode, automatic_import_sonar, \
automatic_import_wispgate, automatic_import_netzur, automatic_import_visp, \
plan_top_level_cpu_bins, \
plan_class_identities, \
fast_queues_fq_codel, \
@@ -121,6 +124,14 @@ def get_network_json_path():
return os.path.join(base_dir, "network.json")
def get_shaping_inputs_path():
return os.path.join(get_libreqos_directory(), "shaping_inputs.json")
def get_circuit_anchors_path():
return os.path.join(get_libreqos_directory(), "circuit_anchors.json")
def get_planner_state_path():
return os.path.join(get_libreqos_directory(), "planner_state.json")
@@ -151,6 +162,197 @@ def loaded_network_is_flat(network):
return not isinstance(network, dict) or len(network) == 0
def _normalize_shaped_devices_header(header_value):
return ''.join(ch for ch in str(header_value).lower() if ch.isalnum())
_SHAPED_DEVICES_LEGACY_LAYOUT = {
'circuitID': 0,
'circuitName': 1,
'deviceID': 2,
'deviceName': 3,
'ParentNode': 4,
'mac': 5,
'ipv4_input': 6,
'ipv6_input': 7,
'downloadMin': 8,
'uploadMin': 9,
'downloadMax': 10,
'uploadMax': 11,
'comment': 12,
'sqm': 13,
}
_SHAPED_DEVICES_HEADER_ALIASES = {
'circuitID': {'circuitid'},
'circuitName': {'circuitname'},
'deviceID': {'deviceid'},
'deviceName': {'devicename'},
'ParentNode': {'parentnode'},
'ParentNodeID': {'parentnodeid'},
'AnchorNodeID': {'anchornodeid', 'id'},
'mac': {'mac'},
'ipv4_input': {'ipv4'},
'ipv6_input': {'ipv6'},
'downloadMin': {'downloadmin', 'downloadminmbps'},
'uploadMin': {'uploadmin', 'uploadminmbps'},
'downloadMax': {'downloadmax', 'downloadmaxmbps'},
'uploadMax': {'uploadmax', 'uploadmaxmbps'},
'comment': {'comment'},
'sqm': {'sqm'},
}
def _build_shaped_devices_layout(header_row):
layout = dict(_SHAPED_DEVICES_LEGACY_LAYOUT)
layout['ParentNodeID'] = None
layout['AnchorNodeID'] = None
for idx, header in enumerate(header_row):
normalized = _normalize_shaped_devices_header(header)
for field, aliases in _SHAPED_DEVICES_HEADER_ALIASES.items():
if normalized in aliases:
layout[field] = idx
break
return layout
def _shaped_devices_row_value(row, layout, field):
idx = layout.get(field)
if idx is None or idx >= len(row):
return ''
return row[idx]
def _resolve_effective_parent_node(circuit, parent_node_ids, parent_node_aliases):
parent_node_id = str(circuit.get('ParentNodeID', '') or '').strip()
if parent_node_id:
resolved_parent = parent_node_ids.get(parent_node_id)
if resolved_parent:
return resolved_parent, parent_node_id
parent_node = str(circuit.get('ParentNode', '') or '').strip()
if not parent_node or parent_node == 'none':
return parent_node, ''
resolved_parent = parent_node_aliases.get(parent_node)
if resolved_parent:
resolved_parent_id = ''
for node_id, node_name in parent_node_ids.items():
if node_name == resolved_parent:
resolved_parent_id = node_id
break
return resolved_parent, resolved_parent_id
return parent_node, parent_node_id
def _shaping_inputs_are_fresh(shaping_inputs_path, shaped_devices_file, network_json_file, circuit_anchors_file=None):
if not os.path.isfile(shaping_inputs_path):
return False
try:
shaping_inputs_mtime = os.path.getmtime(shaping_inputs_path)
if os.path.isfile(shaped_devices_file) and shaping_inputs_mtime < os.path.getmtime(shaped_devices_file):
return False
if os.path.isfile(network_json_file) and shaping_inputs_mtime < os.path.getmtime(network_json_file):
return False
if circuit_anchors_file and os.path.isfile(circuit_anchors_file) and shaping_inputs_mtime < os.path.getmtime(circuit_anchors_file):
return False
return True
except OSError:
return False
def loadSubscriberCircuitsFromShapingInputs(shapingInputsPath):
payload = _load_json_dict(shapingInputsPath)
circuits = payload.get('circuits', [])
if not isinstance(circuits, list):
raise ValueError(f"Invalid shaping_inputs.json at {shapingInputsPath}: circuits must be a list")
subscriberCircuits = []
dictForCircuitsWithoutParentNodes = {}
counterForCircuitsWithoutParentNodes = 0
warnings_from_file = payload.get('warnings', [])
if isinstance(warnings_from_file, list):
for warning_text in warnings_from_file:
if isinstance(warning_text, str) and warning_text.strip():
warnings.warn(f"shaping_inputs.json: {warning_text}", stacklevel=2)
for circuit in circuits:
if not isinstance(circuit, dict):
continue
circuitID = str(circuit.get('circuit_id', '') or '').strip()
if circuitID == '':
raise ValueError("Missing circuit_id in shaping_inputs.json")
parent_node = str(circuit.get('effective_parent_node_name', '') or '').strip()
parent_node_id = str(circuit.get('effective_parent_node_id', '') or '').strip()
logical_parent_node = str(circuit.get('logical_parent_node_name', '') or '').strip()
logical_parent_node_id = str(circuit.get('logical_parent_node_id', '') or '').strip()
anchor_node_id = str(circuit.get('anchor_node_id', '') or '').strip()
devices = []
for device in circuit.get('devices', []):
if not isinstance(device, dict):
continue
devices.append(
{
"deviceID": str(device.get('device_id', '') or '').strip(),
"deviceName": str(device.get('device_name', '') or '').strip(),
"mac": str(device.get('mac', '') or '').strip(),
"ipv4s": [str(entry).strip() for entry in device.get('ipv4', []) if str(entry).strip() != ''],
"ipv6s": [str(entry).strip() for entry in device.get('ipv6', []) if str(entry).strip() != ''],
"comment": str(device.get('comment', '') or ''),
}
)
thisCircuit = {
"circuitID": circuitID,
"circuitName": str(circuit.get('circuit_name', '') or ''),
"ParentNode": parent_node if parent_node != '' else 'none',
"ParentNodeID": parent_node_id,
"AnchorNodeID": anchor_node_id,
"devices": devices,
"minDownload": float(circuit.get('download_min_mbps', 0.0) or 0.0),
"minUpload": float(circuit.get('upload_min_mbps', 0.0) or 0.0),
"maxDownload": float(circuit.get('download_max_mbps', 0.0) or 0.0),
"maxUpload": float(circuit.get('upload_max_mbps', 0.0) or 0.0),
"classid": '',
"comment": str(circuit.get('comment', '') or ''),
"logicalParentNode": logical_parent_node if logical_parent_node != '' else (parent_node if parent_node != '' else 'none'),
"logicalParentNodeID": logical_parent_node_id,
"effectiveAttachmentID": str(circuit.get('effective_attachment_id', '') or '').strip(),
"effectiveAttachmentName": str(circuit.get('effective_attachment_name', '') or '').strip(),
"parentResolutionSource": str(circuit.get('resolution_source', '') or '').strip(),
"parentResolvedByShapingInputs": True,
}
sqm_override = normalize_sqm_override_token(str(circuit.get('sqm_override', '') or ''))
if sqm_override != '':
thisCircuit['sqm'] = sqm_override
if thisCircuit['ParentNode'] == 'none':
thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes
dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = (
thisCircuit['maxDownload'] + thisCircuit['maxUpload']
)
counterForCircuitsWithoutParentNodes += 1
subscriberCircuits.append(thisCircuit)
return subscriberCircuits, dictForCircuitsWithoutParentNodes
def loadSubscriberCircuitsForShaping(shapedDevicesFile, networkJSONfile):
shaping_inputs_path = get_shaping_inputs_path()
circuit_anchors_path = get_circuit_anchors_path()
if _shaping_inputs_are_fresh(shaping_inputs_path, shapedDevicesFile, networkJSONfile, circuit_anchors_path):
try:
subscriberCircuits, dictForCircuitsWithoutParentNodes = loadSubscriberCircuitsFromShapingInputs(shaping_inputs_path)
print("Loaded shaping inputs from " + shaping_inputs_path)
return subscriberCircuits, dictForCircuitsWithoutParentNodes
except Exception as e:
raise RefreshFailure(
f"Unable to load required shaping_inputs.json at {shaping_inputs_path}: {e}"
) from e
raise RefreshFailure(
"Missing or stale shaping_inputs.json. Run topology runtime before shaping."
)
def load_planner_state(state_path=None, planner_module=None):
if state_path is None:
state_path = get_planner_state_path()
@@ -470,6 +672,7 @@ def validateNetworkAndDevices():
with io.StringIO(text_content) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
header_consumed = False
layout = None
seenTheseIPsAlready = set()
for row in csv_reader:
if not row:
@@ -478,9 +681,23 @@ def validateNetworkAndDevices():
continue
if not header_consumed:
header_consumed = True
layout = _build_shaped_devices_layout(row)
continue
# Accept optional 14th column 'sqm' but ignore here (validation focuses on core fields)
circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row[0:13]
# Accept optional Anchor Node ID / Parent Node ID columns and trailing
# 'sqm' override columns while validating the core shaping inputs.
circuitID = _shaped_devices_row_value(row, layout, 'circuitID')
circuitName = _shaped_devices_row_value(row, layout, 'circuitName')
deviceID = _shaped_devices_row_value(row, layout, 'deviceID')
deviceName = _shaped_devices_row_value(row, layout, 'deviceName')
ParentNode = _shaped_devices_row_value(row, layout, 'ParentNode')
mac = _shaped_devices_row_value(row, layout, 'mac')
ipv4_input = _shaped_devices_row_value(row, layout, 'ipv4_input')
ipv6_input = _shaped_devices_row_value(row, layout, 'ipv6_input')
downloadMin = _shaped_devices_row_value(row, layout, 'downloadMin')
uploadMin = _shaped_devices_row_value(row, layout, 'uploadMin')
downloadMax = _shaped_devices_row_value(row, layout, 'downloadMax')
uploadMax = _shaped_devices_row_value(row, layout, 'uploadMax')
comment = _shaped_devices_row_value(row, layout, 'comment')
# Must have circuitID, it's a unique identifier required for stateful changes to queue structure
if circuitID == '':
warnings.warn("No Circuit ID provided in ShapedDevices.csv at row " + str(rowNum), stacklevel=2)
@@ -598,6 +815,7 @@ def loadSubscriberCircuits(shapedDevicesFile):
with open(shapedDevicesFile) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
header_consumed = False
layout = None
for row in csv_reader:
if not row:
continue
@@ -605,12 +823,16 @@ def loadSubscriberCircuits(shapedDevicesFile):
continue
if not header_consumed:
header_consumed = True
layout = _build_shaped_devices_layout(row)
continue
# Optional per-circuit SQM override in last column
sqm_override_token = ''
if len(row) > 13:
if layout is not None:
raw_token = _shaped_devices_row_value(row, layout, 'sqm')
else:
raw_token = ''
if raw_token != '':
# Normalize: lowercase, trim, collapse spaces around '/'
raw_token = row[13]
token = raw_token.strip().lower()
if '/' in token:
parts = token.split('/', 1)
@@ -618,7 +840,21 @@ def loadSubscriberCircuits(shapedDevicesFile):
right = parts[1].strip()
token = left + '/' + right
sqm_override_token = token
circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row[0:13]
circuitID = _shaped_devices_row_value(row, layout, 'circuitID')
circuitName = _shaped_devices_row_value(row, layout, 'circuitName')
deviceID = _shaped_devices_row_value(row, layout, 'deviceID')
deviceName = _shaped_devices_row_value(row, layout, 'deviceName')
ParentNode = _shaped_devices_row_value(row, layout, 'ParentNode')
ParentNodeID = _shaped_devices_row_value(row, layout, 'ParentNodeID')
AnchorNodeID = _shaped_devices_row_value(row, layout, 'AnchorNodeID')
mac = _shaped_devices_row_value(row, layout, 'mac')
ipv4_input = _shaped_devices_row_value(row, layout, 'ipv4_input')
ipv6_input = _shaped_devices_row_value(row, layout, 'ipv6_input')
downloadMin = _shaped_devices_row_value(row, layout, 'downloadMin')
uploadMin = _shaped_devices_row_value(row, layout, 'uploadMin')
downloadMax = _shaped_devices_row_value(row, layout, 'downloadMax')
uploadMax = _shaped_devices_row_value(row, layout, 'uploadMax')
comment = _shaped_devices_row_value(row, layout, 'comment')
ipv4_subnets_and_hosts = []
# Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s separated by commas. Split them up and parse each
if ipv4_input != "":
@@ -647,6 +883,20 @@ def loadSubscriberCircuits(shapedDevicesFile):
if circuit['ParentNode'] != ParentNode:
errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID
raise ValueError(errorMessageString)
existing_parent_node_id = str(circuit.get('ParentNodeID', '') or '').strip()
candidate_parent_node_id = str(ParentNodeID or '').strip()
if existing_parent_node_id and candidate_parent_node_id and existing_parent_node_id != candidate_parent_node_id:
errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node ID from other devices of circuit ID #" + circuitID
raise ValueError(errorMessageString)
if not existing_parent_node_id and candidate_parent_node_id:
circuit['ParentNodeID'] = candidate_parent_node_id
existing_anchor_node_id = str(circuit.get('AnchorNodeID', '') or '').strip()
candidate_anchor_node_id = str(AnchorNodeID or '').strip()
if existing_anchor_node_id and candidate_anchor_node_id and existing_anchor_node_id != candidate_anchor_node_id:
errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Anchor Node ID from other devices of circuit ID #" + circuitID
raise ValueError(errorMessageString)
if not existing_anchor_node_id and candidate_anchor_node_id:
circuit['AnchorNodeID'] = candidate_anchor_node_id
if ((circuit['minDownload'] != float(downloadMin))
or (circuit['minUpload'] != float(uploadMin))
or (circuit['maxDownload'] != float(downloadMax))
@@ -689,6 +939,8 @@ def loadSubscriberCircuits(shapedDevicesFile):
"circuitID": circuitID,
"circuitName": circuitName,
"ParentNode": ParentNode,
"ParentNodeID": str(ParentNodeID or '').strip(),
"AnchorNodeID": str(AnchorNodeID or '').strip(),
"devices": deviceListForCircuit,
"minDownload": float(downloadMin),
"minUpload": float(uploadMin),
@@ -822,7 +1074,7 @@ def refreshShapers():
if safeToRunRefresh == True:
# Load Subscriber Circuits & Devices
subscriberCircuits, dictForCircuitsWithoutParentNodes = loadSubscriberCircuits(shapedDevicesFile)
subscriberCircuits, dictForCircuitsWithoutParentNodes = loadSubscriberCircuitsForShaping(shapedDevicesFile, networkJSONfile)
runtime_override_count = apply_effective_runtime_circuit_overrides(subscriberCircuits)
if runtime_override_count > 0:
print(
@@ -833,7 +1085,10 @@ def refreshShapers():
# Preserve the logical parent (as configured in ShapedDevices.csv) before any shaping-time rewrites.
for circuit in subscriberCircuits:
circuit['logicalParentNode'] = circuit.get('ParentNode')
if 'logicalParentNode' not in circuit:
circuit['logicalParentNode'] = circuit.get('ParentNode')
if 'logicalParentNodeID' not in circuit:
circuit['logicalParentNodeID'] = circuit.get('ParentNodeID', '')
# Load network hierarchy
with open(networkJSONfile, 'r') as j:
@@ -877,6 +1132,8 @@ def refreshShapers():
stacklevel=2,
)
circuit['ParentNode'] = 'none'
circuit['ParentNodeID'] = ''
circuit['effectiveParentNodeID'] = ''
else:
circuit['ParentNode'] = physical_parent
@@ -895,6 +1152,8 @@ def refreshShapers():
for circuit in subscriberCircuits:
if circuit.get('ParentNode') != 'none':
circuit['ParentNode'] = 'none'
circuit['ParentNodeID'] = ''
circuit['effectiveParentNodeID'] = ''
if circuit.get('ParentNode') == 'none' and 'idForCircuitsWithoutParentNodes' not in circuit:
try:
weight = float(circuit.get('maxDownload', 0)) + float(circuit.get('maxUpload', 0))
@@ -1062,6 +1321,8 @@ def refreshShapers():
item_key = str(item_id)
if item_key in assignments:
circuit['ParentNode'] = assignments[item_key]
circuit['ParentNodeID'] = ''
circuit['effectiveParentNodeID'] = ''
# Update and save state
if bin_planner is not None and isinstance(state, dict):
@@ -1088,6 +1349,8 @@ def refreshShapers():
for circuit in subscriberCircuits:
if circuit['ParentNode'] == 'none':
circuit['ParentNode'] = generatedPNs[genPNcounter]
circuit['ParentNodeID'] = ''
circuit['effectiveParentNodeID'] = ''
genPNcounter += 1
if genPNcounter >= queuesAvailable:
genPNcounter = 0
@@ -1213,20 +1476,41 @@ def refreshShapers():
# After flattening, some attachment-style names may only survive as
# metadata on the physical node that now owns that branch.
parent_node_ids = collect_physical_parent_node_ids(network)
parent_node_aliases = collect_physical_parent_node_aliases(network)
for circuit in subscriberCircuits:
if circuit.get('parentResolvedByShapingInputs') == True:
continue
parent_node = str(circuit.get('ParentNode', '') or '').strip()
if not parent_node or parent_node == 'none':
continue
resolved_parent = parent_node_aliases.get(parent_node)
resolved_parent, resolved_parent_id = _resolve_effective_parent_node(
circuit,
parent_node_ids,
parent_node_aliases,
)
if resolved_parent_id:
circuit['effectiveParentNodeID'] = resolved_parent_id
if resolved_parent and resolved_parent != parent_node:
logging.info(
"Resolved circuit parent alias '%s' to physical queue node '%s' for circuit '%s'",
"Resolved circuit parent reference '%s' (%s) to physical queue node '%s' for circuit '%s'",
parent_node,
str(circuit.get('ParentNodeID', '') or '').strip(),
resolved_parent,
circuit.get('circuitID', ''),
)
circuit['ParentNode'] = resolved_parent
elif resolved_parent_id and str(circuit.get('ParentNodeID', '') or '').strip() != resolved_parent_id:
logging.info(
"Resolved circuit parent ID '%s' to physical queue node '%s' for circuit '%s'",
str(circuit.get('ParentNodeID', '') or '').strip(),
resolved_parent,
circuit.get('circuitID', ''),
)
circuit['ParentNode'] = resolved_parent
circuit['ParentNodeID'] = resolved_parent_id
elif resolved_parent_id:
circuit['ParentNodeID'] = resolved_parent_id
# Group circuits by parent node. Reduces runtime for section below this one.
circuits_by_parent_node = {}
@@ -1689,7 +1973,10 @@ def refreshShapers():
"circuitID": circuit['circuitID'],
"circuitName": circuit['circuitName'],
"ParentNode": circuit['ParentNode'],
"ParentNodeID": circuit.get('ParentNodeID', ''),
"logicalParentNode": circuit.get('logicalParentNode', circuit['ParentNode']),
"logicalParentNodeID": circuit.get('logicalParentNodeID', circuit.get('ParentNodeID', '')),
"effectiveParentNodeID": circuit.get('effectiveParentNodeID', circuit.get('ParentNodeID', '')),
"devices": circuit['devices'],
"classid": flowIDstring,
"up_classid": upFlowIDstring,
@@ -2289,6 +2576,8 @@ def refreshShapersUpdateOnly():
devicesChanged = True
if newlyUpdatedSubscriberCircuitsByID[circuitID]['ParentNode'] != lastLoadedSubscriberCircuitsByID[circuitID]['ParentNode']:
devicesChanged = True
if newlyUpdatedSubscriberCircuitsByID[circuitID].get('ParentNodeID', '') != lastLoadedSubscriberCircuitsByID[circuitID].get('ParentNodeID', ''):
devicesChanged = True
else:
devicesChanged = True
for circuitID, circuit in newlyUpdatedSubscriberCircuitsByID.items():
src/ShapedDevices.example.csv
+1 -1
View File
@@ -1,4 +1,4 @@
Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment
Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment
10001,"968 Circle St., Gurnee, IL 60031",1,Device 1,AP_A,,"100.64.0.1, 100.64.0.2",fdd7:b724:0:100::/56,1,1,155,20,
10002,"31 Marconi Street, Lake In The Hills, IL 60156",2,Device 2,AP_A,,100.64.0.3,fdd7:b724:0:200::/56,0.5,0.5,2.5,1,
10003,"255 NW. Newport Ave., Jamestown, NY 14701",3,Device 3,AP_9,,100.64.0.4,fdd7:b724:0:300::/56,1.25,0.75,10.5,5.25,
1 Circuit ID Circuit Name Device ID Device Name Parent Node Parent Node ID Anchor Node ID MAC IPv4 IPv6 Download Min Mbps Upload Min Mbps Download Max Mbps Upload Max Mbps Comment
2 10001 968 Circle St., Gurnee, IL 60031 1 Device 1 AP_A 100.64.0.1, 100.64.0.2 fdd7:b724:0:100::/56 100.64.0.1, 100.64.0.2 1 fdd7:b724:0:100::/56 1 1 155 1 20 155
3 10002 31 Marconi Street, Lake In The Hills, IL 60156 2 Device 2 AP_A 100.64.0.3 fdd7:b724:0:200::/56 100.64.0.3 0.5 fdd7:b724:0:200::/56 0.5 0.5 2.5 0.5 1 2.5
4 10003 255 NW. Newport Ave., Jamestown, NY 14701 3 Device 3 AP_9 100.64.0.4 fdd7:b724:0:300::/56 100.64.0.4 1.25 fdd7:b724:0:300::/56 0.75 1.25 10.5 0.75 5.25 10.5
src/cidrToHosts.py
+9 -5
View File
@@ -14,7 +14,11 @@ def cidrToHosts():
csv_reader = csv.reader(csv_file, delimiter=',')
next(csv_reader)
for row in csv_reader:
circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row
if len(row) >= 14:
circuitID, circuitName, deviceID, deviceName, ParentNode, ParentNodeID, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row[:14]
else:
circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row
ParentNodeID = ''
ipv4_list = ipv4_input.replace(' ','').split(',')
counter = 10001
if len(ipv4_list) > 1:
@@ -26,12 +30,12 @@ def cidrToHosts():
extended_circuit_id = circuitID + "_host_" + str(counter)
extended_device_id = deviceID + "_host_" + str(counter)
counter += 1
revised_rows.append((extended_circuit_id, circuitName, extended_device_id, deviceName, ParentNode, mac, str_host, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment))
revised_rows.append((extended_circuit_id, circuitName, extended_device_id, deviceName, ParentNode, ParentNodeID, mac, str_host, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment))
else:
extended_circuit_id = circuitID + "_host_" + str(counter)
extended_device_id = deviceID + "_host_" + str(counter)
counter += 1
revised_rows.append((extended_circuit_id, circuitName, extended_device_id, deviceName, ParentNode, mac, ipv4, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment))
revised_rows.append((extended_circuit_id, circuitName, extended_device_id, deviceName, ParentNode, ParentNodeID, mac, ipv4, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment))
changed_made = True
else:
if ('/' in ipv4_input) and not ('/32' in ipv4_input):
@@ -41,7 +45,7 @@ def cidrToHosts():
extended_circuit_id = circuitID + "_host_" + str(counter)
extended_device_id = deviceID + "_host_" + str(counter)
counter += 1
revised_rows.append((extended_circuit_id, circuitName, extended_device_id, deviceName, ParentNode, mac, str_host, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment))
revised_rows.append((extended_circuit_id, circuitName, extended_device_id, deviceName, ParentNode, ParentNodeID, mac, str_host, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment))
changed_made = True
else:
revised_rows.append(row)
@@ -49,7 +53,7 @@ def cidrToHosts():
if changed_made:
with open('ShapedDevices.csv','w') as file:
writer = csv.writer(file)
writer.writerow(('Circuit ID (Do Not Duplicate)', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC', 'IPv4', 'IPv6', 'Download Min Mbps', 'Upload Min Mbps', 'Download Max Mbps', 'Upload Max Mbps', 'Comment'))
writer.writerow(('Circuit ID (Do Not Duplicate)', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'Parent Node ID', 'Anchor Node ID', 'MAC', 'IPv4', 'IPv6', 'Download Min Mbps', 'Upload Min Mbps', 'Download Max Mbps', 'Upload Max Mbps', 'Comment'))
for row in revised_rows:
writer.writerow(row)
src/integrationCommon.py
+28 -2
View File
@@ -10,6 +10,7 @@ import ipaddress
import enum
import json
import os
import time
def isInAllowedSubnets(inputIP):
# Check whether an IP address occurs inside the allowedSubnets list
@@ -573,14 +574,19 @@ class NetworkGraph:
import csv
# Builds ShapedDevices.csv from the network tree.
circuits = []
circuit_anchors = []
if not self._cache_valid:
self._buildChildrenCache()
for i, node in enumerate(self.nodes):
if node.type == NodeType.client:
parent = self.nodes[node.parentIndex].displayName
parent_node = self.nodes[node.parentIndex]
parent = parent_node.displayName
parent_id = parent_node.networkJsonId
if parent == "Shaper Root": parent = ""
if parent == "":
parent_id = ""
if circuit_name_use_address():
displayNameToUse = node.address
@@ -594,10 +600,19 @@ class NetworkGraph:
"id": node.id,
"name": displayNameToUse,
"parent": parent,
"parent_id": parent_id,
"download": node.downloadMbps,
"upload": node.uploadMbps,
"devices": []
}
anchor_id = node.networkJsonId
if anchor_id:
circuit_anchors.append({
"circuit_id": node.id,
"circuit_name": displayNameToUse,
"anchor_node_id": anchor_id,
"anchor_node_name": node.displayName if node.displayName else None,
})
# O(1) lookup of children
child_indices = self._children_cache.get(i, [])
@@ -621,7 +636,7 @@ class NetworkGraph:
with open('ShapedDevices.csv', 'w', newline='') as csvfile:
wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC',
wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'Parent Node ID', 'Anchor Node ID', 'MAC',
'IPv4', 'IPv6', 'Download Min', 'Upload Min', 'Download Max', 'Upload Max', 'Comment'])
for circuit in circuits:
for device in circuit["devices"]:
@@ -638,6 +653,8 @@ class NetworkGraph:
device["id"],
device["name"],
circuit["parent"],
circuit.get("parent_id", ""),
"",
device["mac"],
device["ipv4"],
device["ipv6"],
@@ -658,6 +675,15 @@ class NetworkGraph:
for row in reader:
wr.writerow(row)
with open('circuit_anchors.json', 'w', encoding='utf-8') as anchorfile:
json.dump({
"schema_version": 1,
"source": "python/integration_common",
"generated_unix": int(time.time()),
"anchors": circuit_anchors,
}, anchorfile, indent=2)
anchorfile.write('\n')
def plotNetworkGraph(self, showClients=False):
# Requires `pip install graphviz` to function.
# You also need to install graphviz on your PC.
src/rust/lqos_config/src/circuit_anchors.rs
+94
View File
@@ -0,0 +1,94 @@
use crate::Config;
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::io::Write;
use std::path::{Path, PathBuf};
use thiserror::Error;
/// Runtime filename carrying circuit-to-topology anchor assignments emitted by integrations.
pub const CIRCUIT_ANCHORS_FILENAME: &str = "circuit_anchors.json";
/// Errors returned while reading or writing circuit anchor snapshots.
#[derive(Debug, Error)]
pub enum CircuitAnchorsError {
/// Reading or writing the snapshot file failed.
#[error("Unable to access circuit anchors file: {0}")]
Io(#[from] std::io::Error),
/// Serializing or deserializing the snapshot failed.
#[error("Unable to parse circuit anchors JSON: {0}")]
Json(#[from] serde_json::Error),
}
fn default_circuit_anchors_schema_version() -> u32 {
1
}
/// One circuit-to-topology anchor assignment.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)]
pub struct CircuitAnchor {
/// Stable circuit identifier.
pub circuit_id: String,
/// Human-readable circuit name, when available.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub circuit_name: Option<String>,
/// Stable topology node identifier the circuit should attach beneath.
pub anchor_node_id: String,
/// Human-readable topology node name, when available.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub anchor_node_name: Option<String>,
}
/// Integration-generated circuit anchor snapshot.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)]
pub struct CircuitAnchorsFile {
/// Schema version for compatibility checks.
#[serde(default = "default_circuit_anchors_schema_version")]
pub schema_version: u32,
/// Human-readable source such as `uisp/full2` or `python/integration_common`.
#[serde(default)]
pub source: String,
/// Unix timestamp when the file was generated, if known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub generated_unix: Option<u64>,
/// Ordered circuit-to-anchor assignments.
#[serde(default)]
pub anchors: Vec<CircuitAnchor>,
}
/// Returns the path of the circuit anchor runtime file.
///
/// This function is pure: it has no side effects.
pub fn circuit_anchors_path(config: &Config) -> PathBuf {
Path::new(&config.lqos_directory).join(CIRCUIT_ANCHORS_FILENAME)
}
fn atomic_write_json<T: Serialize>(path: &Path, value: &T) -> Result<(), CircuitAnchorsError> {
let raw = serde_json::to_string_pretty(value)?;
let temp_path = path.with_extension("tmp");
let mut file = File::create(&temp_path)?;
file.write_all(raw.as_bytes())?;
file.sync_all()?;
std::fs::rename(&temp_path, path)?;
Ok(())
}
impl CircuitAnchorsFile {
/// Loads the circuit anchor snapshot if it exists.
///
/// Side effects: reads `circuit_anchors.json` from `config.lqos_directory`.
pub fn load(config: &Config) -> Result<Self, CircuitAnchorsError> {
let path = circuit_anchors_path(config);
if !path.exists() {
return Ok(Self::default());
}
let raw = std::fs::read_to_string(path)?;
Ok(serde_json::from_str(&raw)?)
}
/// Saves the circuit anchor snapshot atomically.
///
/// Side effects: writes `circuit_anchors.json` into `config.lqos_directory`.
pub fn save(&self, config: &Config) -> Result<(), CircuitAnchorsError> {
atomic_write_json(&circuit_anchors_path(config), self)
}
}
src/rust/lqos_config/src/lib.rs
+12 -5
View File
@@ -8,6 +8,7 @@
#![deny(clippy::unwrap_used)]
#![warn(missing_docs)]
pub mod authentication;
mod circuit_anchors;
mod circuit_ethernet_metadata;
mod cpu_topology;
mod etc;
@@ -23,6 +24,10 @@ mod topology_parent_candidates;
mod topology_runtime_state;
pub use authentication::{AuthenticatedUser, UserRole, WebUser, WebUsers};
pub use circuit_anchors::{
CIRCUIT_ANCHORS_FILENAME, CircuitAnchor, CircuitAnchorsError, CircuitAnchorsFile,
circuit_anchors_path,
};
pub use circuit_ethernet_metadata::{
CIRCUIT_ETHERNET_METADATA_FILENAME, CircuitEthernetMetadata, CircuitEthernetMetadataFile,
};
@@ -72,11 +77,13 @@ pub use topology_parent_candidates::{
};
pub use topology_runtime_state::{
TOPOLOGY_ATTACHMENT_HEALTH_STATE_FILENAME, TOPOLOGY_EFFECTIVE_NETWORK_FILENAME,
TOPOLOGY_EFFECTIVE_STATE_FILENAME, TopologyAttachmentEndpointStatus,
TopologyAttachmentHealthEntry, TopologyAttachmentHealthStateFile,
TopologyEffectiveAttachmentState, TopologyEffectiveNodeState, TopologyEffectiveStateFile,
TopologyRuntimeStateError, topology_attachment_health_state_path,
topology_effective_network_path, topology_effective_state_path,
TOPOLOGY_EFFECTIVE_STATE_FILENAME, TOPOLOGY_SHAPING_INPUTS_FILENAME,
TopologyAttachmentEndpointStatus, TopologyAttachmentHealthEntry,
TopologyAttachmentHealthStateFile, TopologyEffectiveAttachmentState,
TopologyEffectiveNodeState, TopologyEffectiveStateFile, TopologyRuntimeStateError,
TopologyShapingCircuitInput, TopologyShapingDeviceInput, TopologyShapingInputsFile,
TopologyShapingResolutionSource, topology_attachment_health_state_path,
topology_effective_network_path, topology_effective_state_path, topology_shaping_inputs_path,
};
/// Used as a constant in determining buffer preallocation
src/rust/lqos_config/src/shaped_devices/mod.rs
+27 -9
View File
@@ -31,13 +31,12 @@ impl Default for ConfigShapedDevices {
}
impl ConfigShapedDevices {
/// The path to the current `ShapedDevices.csv` file, determined
/// by acquiring the prefix from the `/etc/lqos.conf` configuration
/// file.
pub fn path() -> Result<PathBuf, ShapedDevicesError> {
let cfg = crate::load_config().map_err(|_| ShapedDevicesError::ConfigLoadError)?;
/// Computes the shaped-devices path for one concrete config snapshot.
///
/// This function is pure: it has no side effects.
pub fn path_for_config(cfg: &crate::Config) -> PathBuf {
let base_path = Path::new(&cfg.lqos_directory);
let full_path = if cfg.long_term_stats.enable_insight_topology.unwrap_or(false) {
if cfg.long_term_stats.enable_insight_topology.unwrap_or(false) {
let tmp_path = base_path.join("ShapedDevices.insight.csv");
if tmp_path.exists() {
tmp_path
@@ -46,7 +45,15 @@ impl ConfigShapedDevices {
}
} else {
base_path.join("ShapedDevices.csv")
};
}
}
/// The path to the current `ShapedDevices.csv` file, determined
/// by acquiring the prefix from the `/etc/lqos.conf` configuration
/// file.
pub fn path() -> Result<PathBuf, ShapedDevicesError> {
let cfg = crate::load_config().map_err(|_| ShapedDevicesError::ConfigLoadError)?;
let full_path = Self::path_for_config(cfg.as_ref());
debug!("ShapedDevices.csv path: {:?}", full_path);
Ok(full_path)
}
@@ -119,7 +126,13 @@ impl ConfigShapedDevices {
/// Loads `ShapedDevices.csv` and constructs a `ConfigShapedDevices`
/// object containing the resulting data.
pub fn load() -> Result<Self, ShapedDevicesError> {
let final_path = ConfigShapedDevices::path()?;
let cfg = crate::load_config().map_err(|_| ShapedDevicesError::ConfigLoadError)?;
Self::load_for_config(cfg.as_ref())
}
/// Loads `ShapedDevices.csv` using a caller-supplied config snapshot.
pub fn load_for_config(cfg: &crate::Config) -> Result<Self, ShapedDevicesError> {
let final_path = ConfigShapedDevices::path_for_config(cfg);
// Load the CSV file as a byte array
if !final_path.exists() {
@@ -138,12 +151,17 @@ impl ConfigShapedDevices {
.flexible(true)
.from_reader(utf8_bytes.as_slice());
let headers = reader
.headers()
.map_err(|e| ShapedDevicesError::GenericCsvError(format!("CSV HEADERS: {e:?}")))?
.clone();
// Example: StringRecord(["1", "968 Circle St., Gurnee, IL 60031", "1", "Device 1", "", "", "192.168.101.2", "", "25", "5", "10000", "10000", ""])
let mut devices = Vec::new(); // Note that this used to be supported_customers, but we're going to let it grow organically
for result in reader.records() {
if let Ok(result) = result {
let device = ShapedDevice::from_csv(&result);
let device = ShapedDevice::from_csv(&result, Some(&headers));
if let Ok(device) = device {
devices.push(device);
} else {
src/rust/lqos_config/src/shaped_devices/serializable.rs
+4
View File
@@ -11,6 +11,8 @@ pub(crate) struct SerializableShapedDevice {
pub device_id: String,
pub device_name: String,
pub parent_node: String,
pub parent_node_id: String,
pub anchor_node_id: String,
pub mac: String,
pub ipv4: String,
pub ipv6: String,
@@ -32,6 +34,8 @@ impl From<&ShapedDevice> for SerializableShapedDevice {
device_id: d.device_id.clone(),
device_name: d.device_name.clone(),
parent_node: d.parent_node.clone(),
parent_node_id: d.parent_node_id.clone().unwrap_or_default(),
anchor_node_id: d.anchor_node_id.clone().unwrap_or_default(),
mac: d.mac.clone(),
ipv4: ipv4_list_to_string(&d.ipv4),
ipv6: ipv6_list_to_string(&d.ipv6),
src/rust/lqos_config/src/shaped_devices/shaped_device.rs
+282 -39
View File
@@ -3,13 +3,14 @@ use allocative::Allocative;
use csv::StringRecord;
use lqos_utils::hash_to_i64;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::net::{Ipv4Addr, Ipv6Addr};
use tracing::error;
/// Represents a row in the `ShapedDevices.csv` file.
#[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq, Allocative)]
pub struct ShapedDevice {
// Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment
// Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment[,sqm]
/// The ID of the circuit to which the device belongs. Circuits are 1:many,
/// multiple devices may be in a single circuit.
pub circuit_id: String,
@@ -27,6 +28,18 @@ pub struct ShapedDevice {
/// The parent node of the device, derived from `network.json`
pub parent_node: String,
/// Optional stable `network.json` identifier for the parent node.
#[serde(skip_serializing_if = "Option::is_none")]
pub parent_node_id: Option<String>,
/// Optional compatibility field for a stable topology node identifier.
///
/// Built-in integrations should prefer `circuit_anchors.json` for circuit
/// anchor resolution. This CSV field remains accepted for DIY and legacy
/// operator-managed inputs.
#[serde(skip_serializing_if = "Option::is_none")]
pub anchor_node_id: Option<String>,
/// The device's MAC address. This isn't actually used, it exists for
/// convenient mapping/seraching.
pub mac: String,
@@ -78,26 +91,129 @@ pub struct ShapedDevice {
}
impl ShapedDevice {
fn normalize_header(value: &str) -> String {
value
.chars()
.filter(|ch| ch.is_ascii_alphanumeric())
.flat_map(|ch| ch.to_lowercase())
.collect()
}
fn csv_layout(headers: Option<&StringRecord>) -> HashMap<&'static str, usize> {
let mut layout = HashMap::from([
("circuit_id", 0usize),
("circuit_name", 1usize),
("device_id", 2usize),
("device_name", 3usize),
("parent_node", 4usize),
("mac", 5usize),
("ipv4", 6usize),
("ipv6", 7usize),
("download_min", 8usize),
("upload_min", 9usize),
("download_max", 10usize),
("upload_max", 11usize),
("comment", 12usize),
("sqm", 13usize),
]);
let Some(headers) = headers else {
return layout;
};
layout.remove("sqm");
for (idx, header) in headers.iter().enumerate() {
match Self::normalize_header(header).as_str() {
"circuitid" | "circuit_id" => {
layout.insert("circuit_id", idx);
}
"circuitname" | "circuit_name" => {
layout.insert("circuit_name", idx);
}
"deviceid" | "device_id" => {
layout.insert("device_id", idx);
}
"devicename" | "device_name" => {
layout.insert("device_name", idx);
}
"parentnode" | "parent_node" => {
layout.insert("parent_node", idx);
}
"parentnodeid" | "parent_node_id" => {
layout.insert("parent_node_id", idx);
}
"anchornodeid" | "anchor_node_id" | "id" => {
layout.insert("anchor_node_id", idx);
}
"mac" => {
layout.insert("mac", idx);
}
"ipv4" => {
layout.insert("ipv4", idx);
}
"ipv6" => {
layout.insert("ipv6", idx);
}
"downloadmin" | "downloadminmbps" | "download_min_mbps" => {
layout.insert("download_min", idx);
}
"uploadmin" | "uploadminmbps" | "upload_min_mbps" => {
layout.insert("upload_min", idx);
}
"downloadmax" | "downloadmaxmbps" | "download_max_mbps" => {
layout.insert("download_max", idx);
}
"uploadmax" | "uploadmaxmbps" | "upload_max_mbps" => {
layout.insert("upload_max", idx);
}
"comment" => {
layout.insert("comment", idx);
}
"sqm" => {
layout.insert("sqm", idx);
}
_ => {}
}
}
layout
}
fn field<'a>(
record: &'a StringRecord,
layout: &HashMap<&'static str, usize>,
key: &str,
) -> &'a str {
layout
.get(key)
.and_then(|idx| record.get(*idx))
.unwrap_or_default()
}
/// Creates a new `ShapedDevice` instance from a CSV string record.
///
/// This function parses a CSV record containing device configuration data and constructs
/// a `ShapedDevice` with all necessary fields populated. The CSV record must contain
/// exactly 13 fields in the following order (optionally a 14th `sqm` field may be present):
/// a `ShapedDevice` with all necessary fields populated. The CSV record uses header-aware
/// parsing and supports the legacy 13-column shape or the newer optional `Parent Node ID`,
/// `Anchor Node ID`/`id`, and trailing `sqm` fields:
///
/// 1. Circuit ID
/// 2. Circuit Name
/// 3. Device ID
/// 4. Device Name
/// 5. Parent Node
/// 6. MAC Address
/// 7. IPv4 Addresses (comma-separated, CIDR notation supported)
/// 8. IPv6 Addresses (comma-separated, CIDR notation supported)
/// 9. Download Min Mbps
/// 10. Upload Min Mbps
/// 11. Download Max Mbps
/// 12. Upload Max Mbps
/// 13. Comment
/// 14. sqm (optional; allowed values: "cake", "fq_codel", "none", or
/// 6. Parent Node ID (optional)
/// 7. Anchor Node ID (optional)
/// 8. MAC Address
/// 9. IPv4 Addresses (comma-separated, CIDR notation supported)
/// 10. IPv6 Addresses (comma-separated, CIDR notation supported)
/// 11. Download Min Mbps
/// 12. Upload Min Mbps
/// 13. Download Max Mbps
/// 14. Upload Max Mbps
/// 15. Comment
/// 16. sqm (optional; allowed values: "cake", "fq_codel", "none", or
/// a directional override in the form "down_sqm/up_sqm". Either side
/// may be empty to indicate no override for that direction, e.g.
/// "cake/" or "/fq_codel".)
@@ -116,21 +232,37 @@ impl ShapedDevice {
/// This function will return an error if:
/// * The bandwidth values (min/max upload/download) cannot be parsed as unsigned integers
/// * The CSV record doesn't contain the expected number of fields
pub fn from_csv(record: &StringRecord) -> Result<Self, ShapedDevicesError> {
pub fn from_csv(
record: &StringRecord,
headers: Option<&StringRecord>,
) -> Result<Self, ShapedDevicesError> {
let layout = Self::csv_layout(headers);
// Parse mandatory fields (first 13 entries)
let mut device = Self {
circuit_id: record[0].to_string(),
circuit_name: record[1].to_string(),
device_id: record[2].to_string(),
device_name: record[3].to_string(),
parent_node: record[4].to_string(),
mac: record[5].to_string(),
ipv4: ShapedDevice::parse_ipv4(&record[6]),
ipv6: ShapedDevice::parse_ipv6(&record[7]),
circuit_id: Self::field(record, &layout, "circuit_id").to_string(),
circuit_name: Self::field(record, &layout, "circuit_name").to_string(),
device_id: Self::field(record, &layout, "device_id").to_string(),
device_name: Self::field(record, &layout, "device_name").to_string(),
parent_node: Self::field(record, &layout, "parent_node").to_string(),
parent_node_id: match Self::field(record, &layout, "parent_node_id").trim() {
"" => None,
value => Some(value.to_string()),
},
anchor_node_id: match Self::field(record, &layout, "anchor_node_id").trim() {
"" => None,
value => Some(value.to_string()),
},
mac: Self::field(record, &layout, "mac").to_string(),
ipv4: ShapedDevice::parse_ipv4(Self::field(record, &layout, "ipv4")),
ipv6: ShapedDevice::parse_ipv6(Self::field(record, &layout, "ipv6")),
download_min_mbps: {
let rate = record[8]
let rate = Self::field(record, &layout, "download_min")
.parse::<f32>()
.map_err(|_| ShapedDevicesError::CsvEntryParseError(record[8].to_string()))?;
.map_err(|_| {
ShapedDevicesError::CsvEntryParseError(
Self::field(record, &layout, "download_min").to_string(),
)
})?;
if rate < 0.01 {
return Err(ShapedDevicesError::CsvEntryParseError(format!(
"Download min rate {} too small (minimum 0.01 Mbps)",
@@ -140,9 +272,13 @@ impl ShapedDevice {
rate
},
upload_min_mbps: {
let rate = record[9]
let rate = Self::field(record, &layout, "upload_min")
.parse::<f32>()
.map_err(|_| ShapedDevicesError::CsvEntryParseError(record[9].to_string()))?;
.map_err(|_| {
ShapedDevicesError::CsvEntryParseError(
Self::field(record, &layout, "upload_min").to_string(),
)
})?;
if rate < 0.01 {
return Err(ShapedDevicesError::CsvEntryParseError(format!(
"Upload min rate {} too small (minimum 0.01 Mbps)",
@@ -152,9 +288,13 @@ impl ShapedDevice {
rate
},
download_max_mbps: {
let rate = record[10]
let rate = Self::field(record, &layout, "download_max")
.parse::<f32>()
.map_err(|_| ShapedDevicesError::CsvEntryParseError(record[10].to_string()))?;
.map_err(|_| {
ShapedDevicesError::CsvEntryParseError(
Self::field(record, &layout, "download_max").to_string(),
)
})?;
if rate < 0.01 {
return Err(ShapedDevicesError::CsvEntryParseError(format!(
"Download max rate {} too small (minimum 0.01 Mbps)",
@@ -164,9 +304,13 @@ impl ShapedDevice {
rate
},
upload_max_mbps: {
let rate = record[11]
let rate = Self::field(record, &layout, "upload_max")
.parse::<f32>()
.map_err(|_| ShapedDevicesError::CsvEntryParseError(record[11].to_string()))?;
.map_err(|_| {
ShapedDevicesError::CsvEntryParseError(
Self::field(record, &layout, "upload_max").to_string(),
)
})?;
if rate < 0.01 {
return Err(ShapedDevicesError::CsvEntryParseError(format!(
"Upload max rate {} too small (minimum 0.01 Mbps)",
@@ -175,16 +319,16 @@ impl ShapedDevice {
}
rate
},
comment: record[12].to_string(),
comment: Self::field(record, &layout, "comment").to_string(),
sqm_override: None,
circuit_hash: hash_to_i64(&record[0]),
device_hash: hash_to_i64(&record[2]),
parent_hash: hash_to_i64(&record[4]),
circuit_hash: hash_to_i64(Self::field(record, &layout, "circuit_id")),
device_hash: hash_to_i64(Self::field(record, &layout, "device_id")),
parent_hash: hash_to_i64(Self::field(record, &layout, "parent_node")),
};
// Optional 14th field: per-circuit SQM override token
if record.len() >= 14 {
let raw = record[13].trim();
if let Some(raw) = layout.get("sqm").and_then(|idx| record.get(*idx)) {
let raw = raw.trim();
if !raw.is_empty() {
// Normalize case and whitespace around optional '/'
let token = raw.to_lowercase();
@@ -346,7 +490,7 @@ mod tests {
"Test fractional rates",
]);
let device = ShapedDevice::from_csv(&record).expect("Should parse fractional rates");
let device = ShapedDevice::from_csv(&record, None).expect("Should parse fractional rates");
assert_eq!(device.download_min_mbps, 0.5);
assert_eq!(device.upload_min_mbps, 1.0);
@@ -373,7 +517,7 @@ mod tests {
"Integer rates",
]);
let device = ShapedDevice::from_csv(&record).expect("Should parse integer rates");
let device = ShapedDevice::from_csv(&record, None).expect("Should parse integer rates");
assert_eq!(device.download_min_mbps, 10.0);
assert_eq!(device.upload_min_mbps, 20.0);
@@ -381,6 +525,105 @@ mod tests {
assert_eq!(device.upload_max_mbps, 200.0);
}
#[test]
fn test_header_aware_parent_node_id_parsing() {
let headers = StringRecord::from(vec![
"Circuit ID",
"Circuit Name",
"Device ID",
"Device Name",
"Parent Node",
"Parent Node ID",
"Anchor Node ID",
"MAC",
"IPv4",
"IPv6",
"Download Min Mbps",
"Upload Min Mbps",
"Download Max Mbps",
"Upload Max Mbps",
"Comment",
]);
let record = StringRecord::from(vec![
"test3",
"Test Circuit 3",
"device3",
"Test Device 3",
"Tower-A",
"uisp:device:tower-a",
"uisp:site:test-site",
"00:00:00:00:00:03",
"192.168.1.3",
"",
"10",
"20",
"100",
"200",
"Header aware",
]);
let device = ShapedDevice::from_csv(&record, Some(&headers))
.expect("Should parse parent node ID from header-aware CSV");
assert_eq!(device.parent_node, "Tower-A");
assert_eq!(
device.parent_node_id.as_deref(),
Some("uisp:device:tower-a")
);
assert_eq!(
device.anchor_node_id.as_deref(),
Some("uisp:site:test-site")
);
assert_eq!(device.mac, "00:00:00:00:00:03");
assert_eq!(device.ipv4.len(), 1);
}
#[test]
fn test_header_aware_id_alias_parsing() {
let headers = StringRecord::from(vec![
"Circuit ID",
"Circuit Name",
"Device ID",
"Device Name",
"Parent Node",
"Parent Node ID",
"id",
"MAC",
"IPv4",
"IPv6",
"Download Min Mbps",
"Upload Min Mbps",
"Download Max Mbps",
"Upload Max Mbps",
"Comment",
]);
let record = StringRecord::from(vec![
"test4",
"Test Circuit 4",
"device4",
"Test Device 4",
"Tower-B",
"uisp:device:tower-b",
"uisp:site:test-site-b",
"00:00:00:00:00:04",
"192.168.1.4",
"",
"10",
"20",
"100",
"200",
"Header aware id alias",
]);
let device = ShapedDevice::from_csv(&record, Some(&headers))
.expect("Should parse anchor node ID from id header alias");
assert_eq!(
device.anchor_node_id.as_deref(),
Some("uisp:site:test-site-b")
);
}
#[test]
fn test_rate_validation_too_small() {
// Test that rates below 0.01 are rejected
@@ -400,7 +643,7 @@ mod tests {
"Rate too small",
]);
let result = ShapedDevice::from_csv(&record);
let result = ShapedDevice::from_csv(&record, None);
assert!(result.is_err(), "Should reject rates below 0.01 Mbps");
}
@@ -423,7 +666,7 @@ mod tests {
"Invalid rate",
]);
let result = ShapedDevice::from_csv(&record);
let result = ShapedDevice::from_csv(&record, None);
assert!(result.is_err(), "Should reject invalid rate strings");
}
}
src/rust/lqos_config/src/topology_canonical_state.rs
+53 -35
View File
@@ -188,13 +188,15 @@ fn canonical_rate_input_from_network_snapshot(
.download_bandwidth_mbps
.or(attachment.capacity_mbps)
{
explicit_download = Some(explicit_download.map_or(download, |current: u64| current.max(download)));
explicit_download =
Some(explicit_download.map_or(download, |current: u64| current.max(download)));
}
if let Some(upload) = attachment
.upload_bandwidth_mbps
.or(attachment.capacity_mbps)
{
explicit_upload = Some(explicit_upload.map_or(upload, |current: u64| current.max(upload)));
explicit_upload =
Some(explicit_upload.map_or(upload, |current: u64| current.max(upload)));
}
}
@@ -256,7 +258,10 @@ fn build_network_node_index(
.and_then(Value::as_str)
.unwrap_or_default()
.to_string(),
is_virtual: node.get("virtual").and_then(Value::as_bool).unwrap_or(false),
is_virtual: node
.get("virtual")
.and_then(Value::as_bool)
.unwrap_or(false),
download_mbps: read_u64_field(node, "downloadBandwidthMbps"),
upload_mbps: read_u64_field(node, "uploadBandwidthMbps"),
};
@@ -275,7 +280,9 @@ fn network_node_snapshot_for_editor_node<'a>(
by_id: &'a HashMap<String, NetworkNodeSnapshot>,
by_name: &'a HashMap<String, NetworkNodeSnapshot>,
) -> Option<&'a NetworkNodeSnapshot> {
by_id.get(&node.node_id).or_else(|| by_name.get(&node.node_name))
by_id
.get(&node.node_id)
.or_else(|| by_name.get(&node.node_name))
}
fn legacy_id_for_name(name: &str) -> String {
@@ -313,30 +320,37 @@ fn import_legacy_network_children(
.and_then(Value::as_str)
.unwrap_or_default()
.to_string();
let is_virtual = node.get("virtual").and_then(Value::as_bool).unwrap_or(false);
let is_virtual = node
.get("virtual")
.and_then(Value::as_bool)
.unwrap_or(false);
let download = read_u64_field(node, "downloadBandwidthMbps");
let upload = read_u64_field(node, "uploadBandwidthMbps");
let (current_parent_node_id, current_parent_node_name, current_attachment_id, current_attachment_name) =
if node_kind.eq_ignore_ascii_case("site") {
let logical = logical_parent.clone();
let attachment = attachment_context.clone().or_else(|| logical.clone());
(
logical.as_ref().map(|entry| entry.0.clone()),
logical.as_ref().map(|entry| entry.1.clone()),
attachment.as_ref().map(|entry| entry.0.clone()),
attachment.as_ref().map(|entry| entry.1.clone()),
)
} else {
let parent = logical_parent.clone();
let attachment = attachment_context.clone().or_else(|| parent.clone());
(
parent.as_ref().map(|entry| entry.0.clone()),
parent.as_ref().map(|entry| entry.1.clone()),
attachment.as_ref().map(|entry| entry.0.clone()),
attachment.as_ref().map(|entry| entry.1.clone()),
)
};
let (
current_parent_node_id,
current_parent_node_name,
current_attachment_id,
current_attachment_name,
) = if node_kind.eq_ignore_ascii_case("site") {
let logical = logical_parent.clone();
let attachment = attachment_context.clone().or_else(|| logical.clone());
(
logical.as_ref().map(|entry| entry.0.clone()),
logical.as_ref().map(|entry| entry.1.clone()),
attachment.as_ref().map(|entry| entry.0.clone()),
attachment.as_ref().map(|entry| entry.1.clone()),
)
} else {
let parent = logical_parent.clone();
let attachment = attachment_context.clone().or_else(|| parent.clone());
(
parent.as_ref().map(|entry| entry.0.clone()),
parent.as_ref().map(|entry| entry.1.clone()),
attachment.as_ref().map(|entry| entry.0.clone()),
attachment.as_ref().map(|entry| entry.1.clone()),
)
};
out.push(TopologyCanonicalNode {
node_id: node_id.clone(),
@@ -638,7 +652,9 @@ mod tests {
&compatibility_network,
TopologyCanonicalIngressKind::NativeIntegration,
);
let node = canonical.find_node("child-site").expect("expected canonical node");
let node = canonical
.find_node("child-site")
.expect("expected canonical node");
assert_eq!(node.rate_input.intrinsic_download_mbps, Some(350));
assert_eq!(node.rate_input.intrinsic_upload_mbps, Some(275));
assert_eq!(node.rate_input.legacy_imported_download_mbps, Some(900));
@@ -672,8 +688,13 @@ mod tests {
}
}));
assert_eq!(canonical.ingress_kind, TopologyCanonicalIngressKind::LegacyNetworkJson);
let child = canonical.find_node("child-site").expect("expected child site");
assert_eq!(
canonical.ingress_kind,
TopologyCanonicalIngressKind::LegacyNetworkJson
);
let child = canonical
.find_node("child-site")
.expect("expected child site");
assert!(!child.can_move);
assert!(child.allowed_parents.is_empty());
assert_eq!(child.current_parent_node_id.as_deref(), Some("parent-site"));
@@ -719,14 +740,11 @@ mod tests {
.find(|node| node.node_name == "Child Site")
.expect("expected child site");
assert_eq!(parent.node_id, legacy_id_for_name("Parent Site"));
assert_eq!(child.node_id, legacy_id_for_name("Child Site"));
assert_eq!(
parent.node_id,
legacy_id_for_name("Parent Site")
child.current_parent_node_id.as_deref(),
Some(parent.node_id.as_str())
);
assert_eq!(
child.node_id,
legacy_id_for_name("Child Site")
);
assert_eq!(child.current_parent_node_id.as_deref(), Some(parent.node_id.as_str()));
}
}
src/rust/lqos_config/src/topology_runtime_state.rs
+132
View File
@@ -14,6 +14,9 @@ pub const TOPOLOGY_EFFECTIVE_STATE_FILENAME: &str = "topology_effective_state.js
/// Runtime filename carrying the effective network tree for shaping/export.
pub const TOPOLOGY_EFFECTIVE_NETWORK_FILENAME: &str = "network.effective.json";
/// Runtime filename carrying shaping-ready circuit inputs resolved from topology runtime.
pub const TOPOLOGY_SHAPING_INPUTS_FILENAME: &str = "shaping_inputs.json";
/// Errors returned while reading or writing topology runtime snapshots.
#[derive(Debug, Error)]
pub enum TopologyRuntimeStateError {
@@ -180,6 +183,113 @@ pub struct TopologyEffectiveStateFile {
pub nodes: Vec<TopologyEffectiveNodeState>,
}
/// Source used to resolve one circuit's shaping parent.
#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum TopologyShapingResolutionSource {
/// Runtime topology resolved the effective parent from `circuit_anchors.json`
/// or a compatible anchor input.
#[default]
TopologyAnchor,
/// Legacy `ParentNode`/`ParentNodeID` was used because no anchor was available.
LegacyParent,
}
/// One shaped device row carried into `shaping_inputs.json`.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Default)]
pub struct TopologyShapingDeviceInput {
/// Stable device identifier.
pub device_id: String,
/// Human-readable device name.
pub device_name: String,
/// Device MAC address when known.
pub mac: String,
/// All IPv4 addresses/subnets associated with the device.
#[serde(default)]
pub ipv4: Vec<String>,
/// All IPv6 addresses/subnets associated with the device.
#[serde(default)]
pub ipv6: Vec<String>,
/// Free-form operator/integration comment.
#[serde(default)]
pub comment: String,
}
/// One shaping-ready circuit compiled from `ShapedDevices.csv` plus effective topology.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)]
pub struct TopologyShapingCircuitInput {
/// Stable circuit identifier.
pub circuit_id: String,
/// Human-readable circuit name.
pub circuit_name: String,
/// Stable topology node identifier the circuit attaches beneath, when provided
/// by the integration/operator input.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub anchor_node_id: Option<String>,
/// Human-readable topology anchor node name, when resolved.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub anchor_node_name: Option<String>,
/// Legacy logical parent name from `ShapedDevices.csv`, when present.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub logical_parent_node_name: Option<String>,
/// Legacy logical parent node identifier from `ShapedDevices.csv`, when present.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub logical_parent_node_id: Option<String>,
/// Runtime-effective parent node name used for shaping.
pub effective_parent_node_name: String,
/// Runtime-effective parent node identifier used for shaping.
pub effective_parent_node_id: String,
/// Runtime-effective attachment identifier, when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub effective_attachment_id: Option<String>,
/// Runtime-effective attachment label, when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub effective_attachment_name: Option<String>,
/// How the effective parent was resolved.
#[serde(default)]
pub resolution_source: TopologyShapingResolutionSource,
/// Guaranteed minimum download rate in Mbps.
pub download_min_mbps: f32,
/// Guaranteed minimum upload rate in Mbps.
pub upload_min_mbps: f32,
/// Maximum download rate in Mbps.
pub download_max_mbps: f32,
/// Maximum upload rate in Mbps.
pub upload_max_mbps: f32,
/// Free-form operator/integration comment.
#[serde(default)]
pub comment: String,
/// Optional per-circuit SQM override token.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sqm_override: Option<String>,
/// Device rows belonging to this circuit.
#[serde(default)]
pub devices: Vec<TopologyShapingDeviceInput>,
}
/// Full runtime shaping-input snapshot consumed by `LibreQoS.py`.
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)]
pub struct TopologyShapingInputsFile {
/// Schema version for compatibility checks.
#[serde(default = "default_runtime_schema_version")]
pub schema_version: u32,
/// Unix timestamp when the file was generated.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub generated_unix: Option<u64>,
/// Generation timestamp of the canonical topology used as input.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub canonical_generated_unix: Option<u64>,
/// Generation timestamp of the effective topology used as input.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub effective_generated_unix: Option<u64>,
/// Non-fatal generation warnings.
#[serde(default)]
pub warnings: Vec<String>,
/// Shaping-ready circuits.
#[serde(default)]
pub circuits: Vec<TopologyShapingCircuitInput>,
}
fn default_runtime_schema_version() -> u32 {
1
}
@@ -212,6 +322,11 @@ pub fn topology_effective_network_path(config: &Config) -> PathBuf {
Path::new(&config.lqos_directory).join(TOPOLOGY_EFFECTIVE_NETWORK_FILENAME)
}
/// Returns the path of the runtime shaping-input snapshot file.
pub fn topology_shaping_inputs_path(config: &Config) -> PathBuf {
Path::new(&config.lqos_directory).join(TOPOLOGY_SHAPING_INPUTS_FILENAME)
}
impl TopologyAttachmentHealthStateFile {
/// Loads the transient attachment-health state file if it exists.
pub fn load(config: &Config) -> Result<Self, TopologyRuntimeStateError> {
@@ -245,3 +360,20 @@ impl TopologyEffectiveStateFile {
atomic_write_json(&topology_effective_state_path(config), self)
}
}
impl TopologyShapingInputsFile {
/// Loads the runtime shaping-input snapshot if it exists.
pub fn load(config: &Config) -> Result<Self, TopologyRuntimeStateError> {
let path = topology_shaping_inputs_path(config);
if !path.exists() {
return Ok(Self::default());
}
let raw = std::fs::read_to_string(path)?;
Ok(serde_json::from_str(&raw)?)
}
/// Saves the runtime shaping-input snapshot.
pub fn save(&self, config: &Config) -> Result<(), TopologyRuntimeStateError> {
atomic_write_json(&topology_shaping_inputs_path(config), self)
}
}
src/rust/lqos_overrides/src/main.rs
+2
View File
@@ -229,6 +229,8 @@ impl AddArgs {
device_id: self.device_id,
device_name: self.device_name,
parent_node: self.parent_node,
parent_node_id: None,
anchor_node_id: None,
mac: self.mac,
ipv4,
ipv6,
src/rust/lqos_python/src/lib.rs
+25 -1
View File
@@ -1322,7 +1322,7 @@ fn wait_for_bus_ready(timeout_ms: u64) -> PyResult<bool> {
/// Returns a Python list of dictionaries representing persistent devices for ShapedDevices.csv
/// The dictionary keys mirror the normalized loader used in LibreQoS.py:
/// circuitID, circuitName, deviceID, deviceName, ParentNode, mac,
/// circuitID, circuitName, deviceID, deviceName, ParentNode, ParentNodeID, mac,
/// ipv4s (list[str]), ipv6s (list[str]), minDownload, minUpload, maxDownload,
/// maxUpload, comment, sqm.
#[pyfunction]
@@ -1351,6 +1351,14 @@ fn overrides_persistent_devices(py: Python<'_>) -> PyResult<Vec<PyObject>> {
d.set_item("deviceID", dev.device_id.clone())?;
d.set_item("deviceName", dev.device_name.clone())?;
d.set_item("ParentNode", dev.parent_node.clone())?;
d.set_item(
"ParentNodeID",
dev.parent_node_id.clone().unwrap_or_default(),
)?;
d.set_item(
"AnchorNodeID",
dev.anchor_node_id.clone().unwrap_or_default(),
)?;
d.set_item("mac", dev.mac.clone())?;
d.set_item("ipv4s", ipv4s)?;
d.set_item("ipv6s", ipv6s)?;
@@ -1409,6 +1417,14 @@ fn overrides_persistent_devices_effective(py: Python<'_>) -> PyResult<Vec<PyObje
d.set_item("deviceID", dev.device_id.clone())?;
d.set_item("deviceName", dev.device_name.clone())?;
d.set_item("ParentNode", dev.parent_node.clone())?;
d.set_item(
"ParentNodeID",
dev.parent_node_id.clone().unwrap_or_default(),
)?;
d.set_item(
"AnchorNodeID",
dev.anchor_node_id.clone().unwrap_or_default(),
)?;
d.set_item("mac", dev.mac.clone())?;
d.set_item("ipv4s", ipv4s)?;
d.set_item("ipv6s", ipv6s)?;
@@ -1462,6 +1478,14 @@ fn overrides_persistent_devices_materialized(py: Python<'_>) -> PyResult<Vec<PyO
d.set_item("deviceID", dev.device_id.clone())?;
d.set_item("deviceName", dev.device_name.clone())?;
d.set_item("ParentNode", dev.parent_node.clone())?;
d.set_item(
"ParentNodeID",
dev.parent_node_id.clone().unwrap_or_default(),
)?;
d.set_item(
"AnchorNodeID",
dev.anchor_node_id.clone().unwrap_or_default(),
)?;
d.set_item("mac", dev.mac.clone())?;
d.set_item("ipv4s", ipv4s)?;
d.set_item("ipv6s", ipv6s)?;
src/rust/lqos_topology/src/lib.rs
+906 -19
View File
@@ -4,11 +4,14 @@
use anyhow::{Context, Result};
use lqos_config::{
Config, TOPOLOGY_ATTACHMENT_AUTO_ID, TopologyAllowedParent, TopologyAttachmentHealthStateFile,
TopologyAttachmentHealthStatus, TopologyAttachmentOption, TopologyAttachmentRateSource,
TopologyAttachmentRole, TopologyCanonicalNode, TopologyCanonicalStateFile, TopologyEditorNode,
TopologyEditorStateFile, TopologyEffectiveAttachmentState, TopologyEffectiveNodeState,
TopologyEffectiveStateFile, topology_effective_network_path, topology_effective_state_path,
CircuitAnchor, CircuitAnchorsFile, Config, ConfigShapedDevices, TOPOLOGY_ATTACHMENT_AUTO_ID,
TopologyAllowedParent, TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus,
TopologyAttachmentOption, TopologyAttachmentRateSource, TopologyAttachmentRole,
TopologyCanonicalNode, TopologyCanonicalStateFile, TopologyEditorNode, TopologyEditorStateFile,
TopologyEffectiveAttachmentState, TopologyEffectiveNodeState, TopologyEffectiveStateFile,
TopologyShapingCircuitInput, TopologyShapingDeviceInput, TopologyShapingInputsFile,
TopologyShapingResolutionSource, circuit_anchors_path, topology_effective_network_path,
topology_effective_state_path, topology_shaping_inputs_path,
};
use lqos_overrides::{TopologyAttachmentMode, TopologyOverridesFile};
use serde_json::{Map, Value};
@@ -80,6 +83,516 @@ fn effective_state_payload_equals(
left == right
}
fn optional_non_empty(raw: &str) -> Option<String> {
let trimmed = raw.trim();
(!trimmed.is_empty()).then(|| trimmed.to_string())
}
fn optional_non_empty_owned(raw: Option<String>) -> Option<String> {
raw.and_then(|value| optional_non_empty(&value))
}
fn collect_exported_effective_nodes(value: &Value, by_id: &mut HashMap<String, String>) {
let Some(nodes) = value.as_object() else {
return;
};
for node in nodes.values() {
let Some(map) = node.as_object() else {
continue;
};
let node_id = map
.get("id")
.and_then(Value::as_str)
.and_then(optional_non_empty);
let node_name = map
.get("name")
.and_then(Value::as_str)
.and_then(optional_non_empty);
if let (Some(node_id), Some(node_name)) = (node_id, node_name) {
by_id.insert(node_id, node_name);
}
if let Some(children) = map.get("children") {
collect_exported_effective_nodes(children, by_id);
}
}
}
fn collect_exported_effective_aliases(
value: &Value,
aliases: &mut HashMap<String, (String, String)>,
) {
let Some(nodes) = value.as_object() else {
return;
};
for node in nodes.values() {
let Some(map) = node.as_object() else {
continue;
};
let node_id = map
.get("id")
.and_then(Value::as_str)
.and_then(optional_non_empty);
let node_name = map
.get("name")
.and_then(Value::as_str)
.and_then(optional_non_empty);
let active_attachment_name = map
.get("active_attachment_name")
.and_then(Value::as_str)
.and_then(optional_non_empty);
if let (Some(alias), Some(node_id), Some(node_name)) =
(active_attachment_name, node_id, node_name)
{
aliases.entry(alias).or_insert((node_id, node_name));
}
if let Some(children) = map.get("children") {
collect_exported_effective_aliases(children, aliases);
}
}
}
fn resolve_legacy_parent_from_effective_tree(
parent_node: &str,
parent_node_id: Option<&str>,
exported_effective_nodes: &HashMap<String, String>,
exported_effective_aliases: &HashMap<String, (String, String)>,
) -> Option<(String, String)> {
let trimmed_id = parent_node_id.and_then(optional_non_empty);
let trimmed_name = optional_non_empty(parent_node);
if let Some(parent_id) = trimmed_id.as_deref()
&& let Some(parent_name) = exported_effective_nodes.get(parent_id).cloned()
{
return Some((parent_id.to_string(), parent_name));
}
if let Some(parent_name) = trimmed_name.as_deref()
&& let Some(parent_id) = exported_effective_nodes
.iter()
.find_map(|(node_id, node_name)| (node_name == parent_name).then(|| node_id.clone()))
{
return Some((parent_id, parent_name.to_string()));
}
trimmed_name.and_then(|alias| exported_effective_aliases.get(&alias).cloned())
}
fn selected_attachment_name_for_node(
ui_node: &TopologyEditorNode,
effective_node: &TopologyEffectiveNodeState,
) -> Option<String> {
effective_node
.effective_attachment_id
.as_deref()
.and_then(|attachment_id| {
ui_node
.allowed_parents
.iter()
.find_map(|parent| option_name(parent, attachment_id))
})
.or_else(|| optional_non_empty_owned(ui_node.effective_attachment_name.clone()))
.or_else(|| optional_non_empty_owned(ui_node.current_attachment_name.clone()))
}
fn build_attachment_owner_map(
ui_state: &TopologyEditorStateFile,
) -> HashMap<String, (String, String)> {
let mut owners = HashMap::new();
for node in &ui_state.nodes {
for parent in &node.allowed_parents {
for option in &parent.attachment_options {
let Some(attachment_id) = optional_non_empty(&option.attachment_id) else {
continue;
};
if attachment_id == TOPOLOGY_ATTACHMENT_AUTO_ID {
continue;
}
owners
.entry(attachment_id)
.or_insert_with(|| (node.node_id.clone(), node.node_name.clone()));
}
}
}
owners
}
fn resolve_effective_parent_from_anchor(
anchor_id: &str,
ui_by_node: &HashMap<&str, &TopologyEditorNode>,
effective_by_node: &HashMap<&str, &TopologyEffectiveNodeState>,
exported_effective_nodes: &HashMap<String, String>,
attachment_owner_by_attachment_id: &HashMap<String, (String, String)>,
) -> Option<(String, String, Option<String>, Option<String>)> {
let anchor_id = anchor_id.trim();
if anchor_id.is_empty() {
return None;
}
if let Some(parent_name) = exported_effective_nodes.get(anchor_id).cloned() {
let attachment_id = effective_by_node
.get(anchor_id)
.and_then(|node| optional_non_empty_owned(node.effective_attachment_id.clone()));
let attachment_name = ui_by_node.get(anchor_id).and_then(|ui_node| {
effective_by_node.get(anchor_id).and_then(|effective_node| {
selected_attachment_name_for_node(ui_node, effective_node)
})
});
return Some((
anchor_id.to_string(),
parent_name,
attachment_id,
attachment_name,
));
}
let (owner_node_id, owner_node_name) =
attachment_owner_by_attachment_id.get(anchor_id).cloned()?;
let owner_ui = ui_by_node.get(owner_node_id.as_str()).copied()?;
let owner_effective = effective_by_node.get(owner_node_id.as_str()).copied()?;
if let Some(selected_attachment_id) = owner_effective.effective_attachment_id.as_deref()
&& let Some(parent_name) = exported_effective_nodes
.get(selected_attachment_id)
.cloned()
{
return Some((
selected_attachment_id.to_string(),
parent_name,
optional_non_empty(selected_attachment_id),
selected_attachment_name_for_node(owner_ui, owner_effective),
));
}
if let Some(parent_name) = exported_effective_nodes
.get(owner_node_id.as_str())
.cloned()
{
return Some((
owner_node_id,
parent_name,
optional_non_empty_owned(owner_effective.effective_attachment_id.clone()),
selected_attachment_name_for_node(owner_ui, owner_effective),
));
}
Some((
owner_node_id,
owner_node_name,
optional_non_empty_owned(owner_effective.effective_attachment_id.clone()),
selected_attachment_name_for_node(owner_ui, owner_effective),
))
}
fn ipv4_with_prefix_to_string(entry: &(std::net::Ipv4Addr, u32)) -> String {
if entry.1 >= 32 {
entry.0.to_string()
} else {
format!("{}/{}", entry.0, entry.1)
}
}
fn ipv6_with_prefix_to_string(entry: &(std::net::Ipv6Addr, u32)) -> String {
if entry.1 >= 128 {
entry.0.to_string()
} else {
format!("{}/{}", entry.0, entry.1)
}
}
fn load_circuit_anchors(
config: &Config,
shaped_devices_mtime: Option<std::time::SystemTime>,
) -> Vec<CircuitAnchor> {
let anchors_path = circuit_anchors_path(config);
let anchors_metadata = std::fs::metadata(&anchors_path).ok();
if let Some(shaped_devices_mtime) = shaped_devices_mtime
&& let Some(anchors_mtime) = anchors_metadata
.as_ref()
.and_then(|metadata| metadata.modified().ok())
&& anchors_mtime < shaped_devices_mtime
{
return Vec::new();
}
CircuitAnchorsFile::load(config)
.map(|file| file.anchors)
.unwrap_or_default()
}
fn build_shaping_inputs(
config: &Config,
artifacts: &EffectiveTopologyArtifacts,
) -> Result<Option<TopologyShapingInputsFile>> {
let shaped_devices_path = ConfigShapedDevices::path_for_config(config);
if !shaped_devices_path.exists() {
return Ok(None);
}
let shaped_devices_mtime = std::fs::metadata(&shaped_devices_path)
.ok()
.and_then(|metadata| metadata.modified().ok());
let shaped_devices = ConfigShapedDevices::load_for_config(config)
.with_context(|| "Unable to load ShapedDevices.csv while building shaping_inputs.json")?;
let circuit_anchors = load_circuit_anchors(config, shaped_devices_mtime)
.into_iter()
.map(|anchor| (anchor.circuit_id.clone(), anchor))
.collect::<HashMap<_, _>>();
let effective_by_node = artifacts
.effective
.nodes
.iter()
.map(|node| (node.node_id.as_str(), node))
.collect::<HashMap<_, _>>();
let ui_by_node = artifacts
.ui_state
.nodes
.iter()
.map(|node| (node.node_id.as_str(), node))
.collect::<HashMap<_, _>>();
let mut exported_effective_nodes = HashMap::<String, String>::new();
let mut exported_effective_aliases = HashMap::<String, (String, String)>::new();
let attachment_owner_by_attachment_id = build_attachment_owner_map(&artifacts.ui_state);
if let Some(effective_network) = artifacts.effective_network.as_ref() {
collect_exported_effective_nodes(effective_network, &mut exported_effective_nodes);
collect_exported_effective_aliases(effective_network, &mut exported_effective_aliases);
}
let mut errors = Vec::new();
let warnings = Vec::new();
let mut circuits = Vec::<TopologyShapingCircuitInput>::new();
let mut circuits_by_id = HashMap::<String, usize>::new();
for device in &shaped_devices.devices {
let anchor_from_file = circuit_anchors.get(&device.circuit_id);
let anchor_node_id = anchor_from_file
.map(|anchor| anchor.anchor_node_id.trim().to_string())
.filter(|value| !value.is_empty())
.or_else(|| optional_non_empty_owned(device.anchor_node_id.clone()));
let mut anchor_node_name = anchor_from_file.and_then(|anchor| {
anchor
.anchor_node_name
.as_ref()
.and_then(|value| optional_non_empty(value))
});
let (
effective_parent_node_id,
effective_parent_node_name,
effective_attachment_id,
effective_attachment_name,
resolution_source,
) = if let Some(anchor_id) = anchor_node_id.as_deref() {
match resolve_effective_parent_from_anchor(
anchor_id,
&ui_by_node,
&effective_by_node,
&exported_effective_nodes,
&attachment_owner_by_attachment_id,
) {
Some((
resolved_parent_id,
resolved_parent_name,
resolved_attachment_id,
resolved_attachment_name,
)) => {
if let Some(ui_node) = ui_by_node.get(anchor_id) {
anchor_node_name = Some(ui_node.node_name.clone());
}
(
resolved_parent_id,
resolved_parent_name,
resolved_attachment_id,
resolved_attachment_name,
TopologyShapingResolutionSource::TopologyAnchor,
)
}
None => match (
ui_by_node.get(anchor_id),
effective_by_node.get(anchor_id),
anchor_from_file,
) {
(Some(ui_node), Some(_effective_node), _) => {
anchor_node_name = Some(ui_node.node_name.clone());
errors.push(format!(
"Circuit '{}' anchor '{}' ('{}') exists in topology metadata but does not resolve to an exported effective queue node.",
device.circuit_id,
anchor_id,
ui_node.node_name
));
(
String::new(),
String::new(),
None,
None,
TopologyShapingResolutionSource::TopologyAnchor,
)
}
(None, None, Some(anchor)) => {
errors.push(format!(
"Circuit '{}' anchor '{}' ('{}') was not found in the effective topology.",
device.circuit_id,
anchor_id,
anchor
.anchor_node_name
.as_deref()
.unwrap_or_default()
));
(
String::new(),
String::new(),
None,
None,
TopologyShapingResolutionSource::TopologyAnchor,
)
}
_ => {
errors.push(format!(
"Circuit '{}' anchor '{}' was not found in the effective topology.",
device.circuit_id, anchor_id
));
(
String::new(),
String::new(),
None,
None,
TopologyShapingResolutionSource::TopologyAnchor,
)
}
},
}
} else if let Some((resolved_parent_id, resolved_parent_name)) =
resolve_legacy_parent_from_effective_tree(
&device.parent_node,
device.parent_node_id.as_deref(),
&exported_effective_nodes,
&exported_effective_aliases,
)
{
(
resolved_parent_id,
resolved_parent_name,
None,
None,
TopologyShapingResolutionSource::LegacyParent,
)
} else {
if optional_non_empty(&device.parent_node).is_some()
|| optional_non_empty_owned(device.parent_node_id.clone()).is_some()
{
errors.push(format!(
"Circuit '{}' parent reference '{}' ({}) was not found in the exported effective topology.",
device.circuit_id,
device.parent_node.trim(),
device.parent_node_id.clone().unwrap_or_default().trim()
));
}
(
String::new(),
String::new(),
None,
None,
TopologyShapingResolutionSource::LegacyParent,
)
};
let logical_parent_node_name = optional_non_empty(&device.parent_node);
let logical_parent_node_id = optional_non_empty_owned(device.parent_node_id.clone());
let circuit_index = if let Some(index) = circuits_by_id.get(&device.circuit_id).copied() {
let circuit = &mut circuits[index];
if circuit.anchor_node_id != anchor_node_id {
errors.push(format!(
"Circuit '{}' had multiple AnchorNodeID values while building shaping inputs.",
device.circuit_id
));
}
if circuit.effective_parent_node_id != effective_parent_node_id
|| circuit.effective_parent_node_name != effective_parent_node_name
{
errors.push(format!(
"Circuit '{}' resolved to multiple effective parents while building shaping_inputs.json.",
device.circuit_id
));
}
index
} else {
let index = circuits.len();
circuits_by_id.insert(device.circuit_id.clone(), index);
circuits.push(TopologyShapingCircuitInput {
circuit_id: device.circuit_id.clone(),
circuit_name: device.circuit_name.clone(),
anchor_node_id: anchor_node_id.clone(),
anchor_node_name,
logical_parent_node_name,
logical_parent_node_id,
effective_parent_node_name,
effective_parent_node_id,
effective_attachment_id,
effective_attachment_name,
resolution_source,
download_min_mbps: device.download_min_mbps,
upload_min_mbps: device.upload_min_mbps,
download_max_mbps: device.download_max_mbps,
upload_max_mbps: device.upload_max_mbps,
comment: device.comment.clone(),
sqm_override: device.sqm_override.clone(),
devices: Vec::new(),
});
index
};
circuits[circuit_index]
.devices
.push(TopologyShapingDeviceInput {
device_id: device.device_id.clone(),
device_name: device.device_name.clone(),
mac: device.mac.clone(),
ipv4: device.ipv4.iter().map(ipv4_with_prefix_to_string).collect(),
ipv6: device.ipv6.iter().map(ipv6_with_prefix_to_string).collect(),
comment: device.comment.clone(),
});
}
circuits.sort_unstable_by(|left, right| left.circuit_id.cmp(&right.circuit_id));
for circuit in &mut circuits {
circuit
.devices
.sort_unstable_by(|left, right| left.device_id.cmp(&right.device_id));
}
for circuit in &circuits {
if circuit.anchor_node_id.is_some() && circuit.effective_parent_node_id.trim().is_empty() {
errors.push(format!(
"Circuit '{}' has an anchor but no resolved effective parent.",
circuit.circuit_id
));
} else if circuit.logical_parent_node_name.is_some()
&& circuit.effective_parent_node_id.trim().is_empty()
{
errors.push(format!(
"Circuit '{}' has a logical parent but no resolved effective parent.",
circuit.circuit_id
));
}
}
if !errors.is_empty() {
let mut message = String::from(
"Unable to build shaping_inputs.json due to runtime topology contract errors:",
);
for error in errors {
message.push_str("\n- ");
message.push_str(&error);
}
return Err(anyhow::anyhow!(message));
}
Ok(Some(TopologyShapingInputsFile {
schema_version: 1,
generated_unix: now_unix(),
canonical_generated_unix: artifacts.effective.canonical_generated_unix,
effective_generated_unix: artifacts.effective.generated_unix,
warnings,
circuits,
}))
}
/// Loads canonical topology state, falling back to importing legacy `network.json`.
pub fn load_canonical_topology_state(config: &Config) -> TopologyCanonicalStateFile {
TopologyCanonicalStateFile::load_with_legacy_fallback(config).unwrap_or_default()
@@ -202,7 +715,8 @@ fn build_effective_topology_artifacts_from_prepared(
/// Publishes validated effective-topology artifacts under a single writer lock.
///
/// Side effects: writes `topology_effective_state.json` and, when present, `network.effective.json`.
/// Side effects: writes `topology_effective_state.json`, `network.effective.json` when present,
/// and `shaping_inputs.json` when `ShapedDevices.csv` is available.
/// If no effective network export is present, any stale `network.effective.json` is removed so
/// runtime consumers fall back to canonical integration output.
pub fn publish_effective_topology_artifacts(
@@ -250,6 +764,43 @@ pub fn publish_effective_topology_artifacts(
})?;
}
let shaping_inputs_path = topology_shaping_inputs_path(config);
let shaping_inputs = match build_shaping_inputs(config, artifacts) {
Ok(value) => value,
Err(err) => {
if shaping_inputs_path.exists() {
std::fs::remove_file(&shaping_inputs_path).with_context(|| {
format!(
"Unable to remove stale runtime shaping inputs at {:?}",
shaping_inputs_path
)
})?;
}
return Err(err);
}
};
if let Some(shaping_inputs) = shaping_inputs {
let shaping_inputs_value = serde_json::to_value(&shaping_inputs)?;
let current_shaping_inputs = read_json_value(&shaping_inputs_path);
if current_shaping_inputs.as_ref() != Some(&shaping_inputs_value) {
atomic_write_json_value(&shaping_inputs_path, &shaping_inputs_value).with_context(
|| {
format!(
"Unable to publish runtime shaping inputs at {:?}",
shaping_inputs_path
)
},
)?;
}
} else if shaping_inputs_path.exists() {
std::fs::remove_file(&shaping_inputs_path).with_context(|| {
format!(
"Unable to remove stale runtime shaping inputs at {:?}",
shaping_inputs_path
)
})?;
}
Ok(())
}
@@ -1164,11 +1715,8 @@ fn find_parent_id_of_node(
return Some(current_parent_id.map(ToOwned::to_owned));
}
if let Some(children) = node.get("children").and_then(Value::as_object)
&& let Some(found) = find_parent_id_of_node(
children,
target_id,
node.get("id").and_then(Value::as_str),
)
&& let Some(found) =
find_parent_id_of_node(children, target_id, node.get("id").and_then(Value::as_str))
{
return Some(found);
}
@@ -2457,17 +3005,32 @@ fn apply_effective_topology_to_canonical_state(
#[cfg(test)]
mod tests {
use super::{
apply_effective_topology_to_network_json, build_effective_topology_artifacts,
compute_effective_state, validate_effective_topology_network,
EffectiveTopologyArtifacts, apply_effective_topology_to_network_json,
build_effective_topology_artifacts, build_shaping_inputs, compute_effective_state,
validate_effective_topology_network,
};
use lqos_config::{
Config, TopologyAllowedParent, TopologyAttachmentHealthStateFile,
TopologyAttachmentHealthStatus, TopologyAttachmentOption, TopologyAttachmentRateSource,
TopologyAttachmentRole, TopologyEditorNode, TopologyEditorStateFile,
TopologyEffectiveAttachmentState, TopologyEffectiveNodeState, TopologyEffectiveStateFile,
CircuitAnchor, CircuitAnchorsFile, Config, TopologyAllowedParent,
TopologyAttachmentHealthStateFile, TopologyAttachmentHealthStatus,
TopologyAttachmentOption, TopologyAttachmentRateSource, TopologyAttachmentRole,
TopologyEditorNode, TopologyEditorStateFile, TopologyEffectiveAttachmentState,
TopologyEffectiveNodeState, TopologyEffectiveStateFile,
};
use lqos_overrides::{TopologyAttachmentMode, TopologyOverridesFile};
use serde_json::{Value, json};
use std::fs;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
fn unique_temp_dir(prefix: &str) -> PathBuf {
let unique = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("clock should be monotonic enough for tests")
.as_nanos();
let path = std::env::temp_dir().join(format!("{prefix}-{unique}"));
fs::create_dir_all(&path).expect("temp directory should be creatable");
path
}
fn sample_attachment_option(
attachment_id: &str,
@@ -2501,6 +3064,326 @@ mod tests {
}
}
#[test]
fn shaping_inputs_prefer_circuit_anchors_over_csv_anchor_fields() {
let lqos_directory = unique_temp_dir("lqos-topology-circuit-anchors");
let config = Config {
lqos_directory: lqos_directory.to_string_lossy().to_string(),
..Config::default()
};
fs::write(
lqos_directory.join("ShapedDevices.csv"),
concat!(
"Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment\n",
"\"circuit-1\",\"Circuit 1\",\"device-1\",\"Device 1\",\"Legacy Parent\",\"legacy-parent\",\"legacy-anchor\",\"aa:bb:cc:dd:ee:ff\",\"192.0.2.10/32\",\"\",\"10\",\"10\",\"100\",\"100\",\"\"\n",
),
)
.expect("ShapedDevices.csv should write");
CircuitAnchorsFile {
schema_version: 1,
source: "test".to_string(),
generated_unix: Some(1),
anchors: vec![CircuitAnchor {
circuit_id: "circuit-1".to_string(),
circuit_name: Some("Circuit 1".to_string()),
anchor_node_id: "tower-1".to_string(),
anchor_node_name: Some("Tower 1".to_string()),
}],
}
.save(&config)
.expect("circuit_anchors.json should write");
let artifacts = EffectiveTopologyArtifacts {
effective: TopologyEffectiveStateFile {
schema_version: 1,
generated_unix: Some(1),
canonical_generated_unix: Some(1),
health_generated_unix: Some(1),
nodes: vec![TopologyEffectiveNodeState {
node_id: "tower-1".to_string(),
logical_parent_node_id: "site-a".to_string(),
preferred_attachment_id: None,
effective_attachment_id: None,
fallback_reason: None,
all_attachments_suppressed: false,
attachments: Vec::new(),
}],
},
ui_state: TopologyEditorStateFile {
schema_version: 1,
source: "test".to_string(),
generated_unix: Some(1),
nodes: vec![TopologyEditorNode {
node_id: "tower-1".to_string(),
node_name: "Tower 1".to_string(),
..TopologyEditorNode::default()
}],
},
effective_network: Some(json!({
"Tower 1": {
"id": "tower-1",
"name": "Tower 1",
"children": {}
}
})),
};
let shaping_inputs = build_shaping_inputs(&config, &artifacts)
.expect("shaping inputs should build")
.expect("shaping inputs should exist");
let circuit = shaping_inputs
.circuits
.iter()
.find(|circuit| circuit.circuit_id == "circuit-1")
.expect("expected circuit");
assert_eq!(circuit.anchor_node_id.as_deref(), Some("tower-1"));
assert_eq!(circuit.anchor_node_name.as_deref(), Some("Tower 1"));
assert_eq!(circuit.effective_parent_node_id, "tower-1");
assert_eq!(circuit.effective_parent_node_name, "Tower 1");
}
#[test]
fn shaping_inputs_remap_non_selected_attachment_anchor_to_effective_attachment() {
let lqos_directory = unique_temp_dir("lqos-topology-attachment-anchor-remap");
let config = Config {
lqos_directory: lqos_directory.to_string_lossy().to_string(),
..Config::default()
};
fs::write(
lqos_directory.join("ShapedDevices.csv"),
concat!(
"Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment\n",
"\"circuit-1\",\"Circuit 1\",\"device-1\",\"Device 1\",\"Legacy Parent\",\"legacy-parent\",\"legacy-anchor\",\"aa:bb:cc:dd:ee:ff\",\"192.0.2.10/32\",\"\",\"10\",\"10\",\"100\",\"100\",\"\"\n",
),
)
.expect("ShapedDevices.csv should write");
CircuitAnchorsFile {
schema_version: 1,
source: "test".to_string(),
generated_unix: Some(1),
anchors: vec![CircuitAnchor {
circuit_id: "circuit-1".to_string(),
circuit_name: Some("Circuit 1".to_string()),
anchor_node_id: "attachment-old".to_string(),
anchor_node_name: Some("Old Attachment".to_string()),
}],
}
.save(&config)
.expect("circuit_anchors.json should write");
let artifacts = EffectiveTopologyArtifacts {
effective: TopologyEffectiveStateFile {
schema_version: 1,
generated_unix: Some(1),
canonical_generated_unix: Some(1),
health_generated_unix: Some(1),
nodes: vec![
TopologyEffectiveNodeState {
node_id: "attachment-old".to_string(),
logical_parent_node_id: "site-parent".to_string(),
preferred_attachment_id: None,
effective_attachment_id: None,
fallback_reason: None,
all_attachments_suppressed: false,
attachments: Vec::new(),
},
TopologyEffectiveNodeState {
node_id: "site-child".to_string(),
logical_parent_node_id: "site-parent".to_string(),
preferred_attachment_id: Some("attachment-new".to_string()),
effective_attachment_id: Some("attachment-new".to_string()),
fallback_reason: None,
all_attachments_suppressed: false,
attachments: Vec::new(),
},
],
},
ui_state: TopologyEditorStateFile {
schema_version: 1,
source: "test".to_string(),
generated_unix: Some(1),
nodes: vec![
TopologyEditorNode {
node_id: "attachment-old".to_string(),
node_name: "Old Attachment".to_string(),
..TopologyEditorNode::default()
},
TopologyEditorNode {
node_id: "attachment-new".to_string(),
node_name: "New Attachment".to_string(),
..TopologyEditorNode::default()
},
TopologyEditorNode {
node_id: "site-child".to_string(),
node_name: "Child Site".to_string(),
allowed_parents: vec![TopologyAllowedParent {
parent_node_id: "site-parent".to_string(),
parent_node_name: "Parent Site".to_string(),
attachment_options: vec![
sample_attachment_option("attachment-old", "Old Attachment"),
sample_attachment_option("attachment-new", "New Attachment"),
],
all_attachments_suppressed: false,
has_probe_unavailable_attachments: false,
}],
effective_attachment_name: Some("New Attachment".to_string()),
..TopologyEditorNode::default()
},
],
},
effective_network: Some(json!({
"Parent Site": {
"id": "site-parent",
"name": "Parent Site",
"children": {
"New Attachment": {
"id": "attachment-new",
"name": "New Attachment",
"children": {
"Child Site": {
"id": "site-child",
"name": "Child Site",
"children": {}
}
}
}
}
}
})),
};
let shaping_inputs = build_shaping_inputs(&config, &artifacts)
.expect("shaping inputs should build")
.expect("shaping inputs should exist");
let circuit = shaping_inputs
.circuits
.iter()
.find(|circuit| circuit.circuit_id == "circuit-1")
.expect("expected circuit");
assert_eq!(circuit.anchor_node_id.as_deref(), Some("attachment-old"));
assert_eq!(circuit.effective_parent_node_id, "attachment-new");
assert_eq!(circuit.effective_parent_node_name, "New Attachment");
assert_eq!(
circuit.effective_attachment_id.as_deref(),
Some("attachment-new")
);
assert_eq!(
circuit.effective_attachment_name.as_deref(),
Some("New Attachment")
);
}
#[test]
fn shaping_inputs_resolve_legacy_parent_against_exported_effective_tree() {
let lqos_directory = unique_temp_dir("lqos-topology-legacy-parent-resolution");
let config = Config {
lqos_directory: lqos_directory.to_string_lossy().to_string(),
..Config::default()
};
fs::write(
lqos_directory.join("ShapedDevices.csv"),
concat!(
"Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment\n",
"\"circuit-1\",\"Circuit 1\",\"device-1\",\"Device 1\",\"Tower 1\",\"tower-1\",\"\",\"aa:bb:cc:dd:ee:ff\",\"192.0.2.10/32\",\"\",\"10\",\"10\",\"100\",\"100\",\"\"\n",
),
)
.expect("ShapedDevices.csv should write");
let artifacts = EffectiveTopologyArtifacts {
effective: TopologyEffectiveStateFile {
schema_version: 1,
generated_unix: Some(1),
canonical_generated_unix: Some(1),
health_generated_unix: Some(1),
nodes: Vec::new(),
},
ui_state: TopologyEditorStateFile {
schema_version: 1,
source: "test".to_string(),
generated_unix: Some(1),
nodes: Vec::new(),
},
effective_network: Some(json!({
"Tower 1": {
"id": "tower-1",
"name": "Tower 1",
"children": {}
}
})),
};
let shaping_inputs = build_shaping_inputs(&config, &artifacts)
.expect("shaping inputs should build")
.expect("shaping inputs should exist");
let circuit = shaping_inputs
.circuits
.iter()
.find(|circuit| circuit.circuit_id == "circuit-1")
.expect("expected circuit");
assert_eq!(circuit.effective_parent_node_id, "tower-1");
assert_eq!(circuit.effective_parent_node_name, "Tower 1");
assert_eq!(
circuit.resolution_source,
lqos_config::TopologyShapingResolutionSource::LegacyParent
);
}
#[test]
fn shaping_inputs_fail_when_anchor_does_not_resolve() {
let lqos_directory = unique_temp_dir("lqos-topology-missing-anchor");
let config = Config {
lqos_directory: lqos_directory.to_string_lossy().to_string(),
..Config::default()
};
fs::write(
lqos_directory.join("ShapedDevices.csv"),
concat!(
"Circuit ID,Circuit Name,Device ID,Device Name,Parent Node,Parent Node ID,Anchor Node ID,MAC,IPv4,IPv6,Download Min Mbps,Upload Min Mbps,Download Max Mbps,Upload Max Mbps,Comment\n",
"\"circuit-1\",\"Circuit 1\",\"device-1\",\"Device 1\",\"Legacy Parent\",\"legacy-parent\",\"\",\"aa:bb:cc:dd:ee:ff\",\"192.0.2.10/32\",\"\",\"10\",\"10\",\"100\",\"100\",\"\"\n",
),
)
.expect("ShapedDevices.csv should write");
CircuitAnchorsFile {
schema_version: 1,
source: "test".to_string(),
generated_unix: Some(1),
anchors: vec![CircuitAnchor {
circuit_id: "circuit-1".to_string(),
circuit_name: Some("Circuit 1".to_string()),
anchor_node_id: "missing-anchor".to_string(),
anchor_node_name: Some("Missing Anchor".to_string()),
}],
}
.save(&config)
.expect("circuit_anchors.json should write");
let artifacts = EffectiveTopologyArtifacts {
effective: TopologyEffectiveStateFile {
schema_version: 1,
generated_unix: Some(1),
canonical_generated_unix: Some(1),
health_generated_unix: Some(1),
nodes: Vec::new(),
},
ui_state: TopologyEditorStateFile {
schema_version: 1,
source: "test".to_string(),
generated_unix: Some(1),
nodes: Vec::new(),
},
effective_network: Some(json!({})),
};
let err = build_shaping_inputs(&config, &artifacts)
.expect_err("missing anchor should fail shaping input generation");
let message = format!("{err:#}");
assert!(message.contains("circuit-1"));
assert!(message.contains("missing-anchor"));
}
#[test]
fn runtime_squashing_collapses_backhaul_pairs_after_attachment_selection() {
let mut config = Config::default();
@@ -3588,7 +4471,9 @@ mod tests {
attachment_kind: "device".to_string(),
attachment_role: TopologyAttachmentRole::PtpBackhaul,
pair_id: None,
peer_attachment_id: Some("uisp:device:device-alpha-beta".to_string()),
peer_attachment_id: Some(
"uisp:device:device-alpha-beta".to_string(),
),
peer_attachment_name: Some("Alpha-Beta-60".to_string()),
capacity_mbps: Some(940),
download_bandwidth_mbps: Some(940),
@@ -3648,7 +4533,9 @@ mod tests {
attachment_kind: "device".to_string(),
attachment_role: TopologyAttachmentRole::PtpBackhaul,
pair_id: None,
peer_attachment_id: Some("uisp:device:device-gamma-beta".to_string()),
peer_attachment_id: Some(
"uisp:device:device-gamma-beta".to_string(),
),
peer_attachment_name: Some("Gamma - Beta MLO6".to_string()),
capacity_mbps: Some(230),
download_bandwidth_mbps: Some(230),
src/rust/lqosd/src/lts2_sys/control_channel.rs
+4 -3
View File
@@ -1329,9 +1329,10 @@ async fn shaper_snapshot_streaming(
// Mirror node_manager ticker throughput: send compact tuple data.
// Note: field names are historical; values are bits per second.
{
let bits_per_second = crate::throughput_tracker::THROUGHPUT_TRACKER.actual_bits_per_second();
let shaped_bits_per_second = crate::throughput_tracker::THROUGHPUT_TRACKER
.shaped_actual_bits_per_second();
let bits_per_second =
crate::throughput_tracker::THROUGHPUT_TRACKER.actual_bits_per_second();
let shaped_bits_per_second =
crate::throughput_tracker::THROUGHPUT_TRACKER.shaped_actual_bits_per_second();
let packets_per_second = crate::throughput_tracker::THROUGHPUT_TRACKER
.packets_per_second
.as_down_up();
src/rust/lqosd/src/main.rs
+1
View File
@@ -243,6 +243,7 @@ fn main() -> Result<()> {
spawn_queue_structure_monitor()?;
shaped_devices_tracker::shaped_devices_watcher()?;
shaped_devices_tracker::network_json_watcher()?;
shaped_devices_tracker::shaping_inputs_watcher()?;
let system_usage_tx = system_stats::start_system_stats()?;
throughput_tracker::spawn_throughput_monitor(
flow_tx,
src/rust/lqosd/src/node_manager/local_api/circuit.rs
+21 -4
View File
@@ -1,6 +1,7 @@
use crate::node_manager::local_api::ethernet_caps::ethernet_advisory_for_circuit;
use crate::shaped_devices_tracker::resolve_parent_node;
use crate::shaped_devices_tracker::SHAPED_DEVICES;
use crate::shaped_devices_tracker::{
SHAPED_DEVICES, effective_parent_for_circuit, resolve_parent_node_reference,
};
use lqos_config::{CircuitEthernetMetadata, ShapedDevice};
use serde::{Deserialize, Serialize};
@@ -39,7 +40,9 @@ fn canonical_parent_node(devices: &mut [ShapedDevice]) -> Option<CircuitParentNo
let mut resolved_parent = None;
for device in devices.iter_mut() {
let Some(resolved) = resolve_parent_node(&device.parent_node) else {
let Some(resolved) =
resolve_parent_node_reference(&device.parent_node, device.parent_node_id.as_deref())
else {
continue;
};
if resolved_parent.is_none() {
@@ -49,11 +52,25 @@ fn canonical_parent_node(devices: &mut [ShapedDevice]) -> Option<CircuitParentNo
});
}
device.parent_node = resolved.name;
device.parent_node_id = resolved.id;
}
resolved_parent
}
fn circuit_parent_node(
circuit_id: &str,
devices: &mut [ShapedDevice],
) -> Option<CircuitParentNode> {
let canonical_parent = canonical_parent_node(devices);
effective_parent_for_circuit(circuit_id)
.map(|parent| CircuitParentNode {
name: parent.name,
id: parent.id,
})
.or(canonical_parent)
}
pub fn circuit_by_id_data(id: &str) -> Option<CircuitByIdData> {
let safe_id = id.to_lowercase().trim().to_string();
let reader = SHAPED_DEVICES.load();
@@ -67,7 +84,7 @@ pub fn circuit_by_id_data(id: &str) -> Option<CircuitByIdData> {
if devices.is_empty() {
None
} else {
let parent_node = canonical_parent_node(&mut devices);
let parent_node = circuit_parent_node(&safe_id, &mut devices);
let ethernet_advisory = load_ethernet_advisory(&safe_id, &devices);
Some(CircuitByIdData {
devices,
src/rust/lqosd/src/shaped_devices_tracker/circuit_live.rs
+7 -3
View File
@@ -167,9 +167,13 @@ pub fn rebuild_circuit_live_snapshot() -> Arc<CircuitLiveSnapshot> {
let mut finalized: FxHashMap<String, CircuitLiveRollup> = FxHashMap::default();
let mut circuit_ids_by_parent_node: FxHashMap<String, Vec<String>> = FxHashMap::default();
for (circuit_id, value) in by_circuit_id {
if !value.parent_node.trim().is_empty() {
let parent_node = super::effective_parent_for_circuit(&circuit_id)
.map(|parent| parent.name)
.filter(|name| !name.trim().is_empty())
.unwrap_or(value.parent_node);
if !parent_node.trim().is_empty() {
circuit_ids_by_parent_node
.entry(value.parent_node.clone())
.entry(parent_node.clone())
.or_default()
.push(circuit_id.clone());
}
@@ -178,7 +182,7 @@ pub fn rebuild_circuit_live_snapshot() -> Arc<CircuitLiveSnapshot> {
CircuitLiveRollup {
circuit_id,
circuit_name: value.circuit_name,
parent_node: value.parent_node,
parent_node,
device_names: sort_string_set(value.device_names),
ip_addrs: sort_string_set(value.ip_addrs),
plan_mbps: value.plan_mbps,
src/rust/lqosd/src/shaped_devices_tracker/mod.rs
+184 -7
View File
@@ -4,7 +4,10 @@ use anyhow::Result;
use arc_swap::ArcSwap;
use fxhash::{FxHashMap, FxHashSet};
use lqos_bus::{BusResponse, Circuit};
use lqos_config::{ConfigShapedDevices, NetworkJsonNode, NetworkJsonTransport, ShapedDevice};
use lqos_config::{
ConfigShapedDevices, NetworkJsonNode, NetworkJsonTransport, ShapedDevice,
TopologyShapingInputsFile, load_config, topology_shaping_inputs_path,
};
use lqos_queue_tracker::EFFECTIVE_NODE_RATES;
use lqos_utils::file_watcher::FileWatcher;
use lqos_utils::hash_to_i64;
@@ -98,6 +101,14 @@ pub static CIRCUIT_LIVE_SNAPSHOT: Lazy<ArcSwap<CircuitLiveSnapshot>> =
Lazy::new(|| ArcSwap::new(Arc::new(CircuitLiveSnapshot::default())));
pub static CIRCUIT_LIVE_LAST_REFRESH_SECS: AtomicU64 = AtomicU64::new(0);
pub static CIRCUIT_LIVE_REFRESH_LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
pub static EFFECTIVE_CIRCUIT_PARENTS: Lazy<ArcSwap<FxHashMap<String, RuntimeCircuitParent>>> =
Lazy::new(|| ArcSwap::new(Arc::new(FxHashMap::default())));
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RuntimeCircuitParent {
pub name: String,
pub id: Option<String>,
}
pub(crate) fn invalidate_circuit_live_snapshot() {
CIRCUIT_LIVE_LAST_REFRESH_SECS.store(0, std::sync::atomic::Ordering::Release);
@@ -107,6 +118,102 @@ pub(crate) fn invalidate_executive_cache_snapshot() {
crate::node_manager::invalidate_executive_cache_snapshot();
}
fn normalize_circuit_id_key(circuit_id: &str) -> Option<String> {
let trimmed = circuit_id.trim();
if trimmed.is_empty() {
None
} else {
Some(trimmed.to_ascii_lowercase())
}
}
fn optional_trimmed_string(value: &str) -> Option<String> {
let trimmed = value.trim();
if trimmed.is_empty() {
None
} else {
Some(trimmed.to_string())
}
}
fn build_effective_circuit_parent_map(
shaping_inputs: &TopologyShapingInputsFile,
) -> FxHashMap<String, RuntimeCircuitParent> {
let mut by_circuit_id = FxHashMap::default();
by_circuit_id.reserve(shaping_inputs.circuits.len());
for circuit in &shaping_inputs.circuits {
let Some(circuit_key) = normalize_circuit_id_key(&circuit.circuit_id) else {
continue;
};
let Some(parent_name) = optional_trimmed_string(&circuit.effective_parent_node_name) else {
continue;
};
by_circuit_id
.entry(circuit_key)
.or_insert_with(|| RuntimeCircuitParent {
name: parent_name,
id: optional_trimmed_string(&circuit.effective_parent_node_id),
});
}
by_circuit_id
}
fn publish_shaping_inputs(shaping_inputs: TopologyShapingInputsFile) {
let effective_parents = build_effective_circuit_parent_map(&shaping_inputs);
EFFECTIVE_CIRCUIT_PARENTS.store(Arc::new(effective_parents));
invalidate_circuit_live_snapshot();
invalidate_executive_cache_snapshot();
}
fn load_shaping_inputs() {
let Ok(config) = load_config() else {
warn!("Unable to load LibreQoS config while loading shaping_inputs.json");
return;
};
match TopologyShapingInputsFile::load(config.as_ref()) {
Ok(shaping_inputs) => publish_shaping_inputs(shaping_inputs),
Err(err) => {
warn!("Unable to load shaping_inputs.json: {err}");
}
}
}
pub fn shaping_inputs_watcher() -> Result<()> {
std::thread::Builder::new()
.name("Shaping Inputs Watcher".to_string())
.spawn(|| {
debug!("Watching for shaping_inputs.json changes");
if let Err(e) = watch_for_shaping_inputs_changing() {
error!("Error watching for shaping_inputs.json changes: {:?}", e);
}
})?;
Ok(())
}
fn watch_for_shaping_inputs_changing() -> Result<()> {
let Ok(config) = load_config() else {
error!("Unable to load LibreQoS config to watch shaping_inputs.json");
return Err(anyhow::Error::msg(
"Unable to load LibreQoS config for shaping_inputs.json",
));
};
let watch_path = topology_shaping_inputs_path(config.as_ref());
let mut watcher = FileWatcher::new("shaping_inputs.json", watch_path);
watcher.set_file_exists_callback(load_shaping_inputs);
watcher.set_file_created_callback(load_shaping_inputs);
watcher.set_file_changed_callback(load_shaping_inputs);
loop {
let result = watcher.watch();
info!("shaping_inputs.json watcher returned: {result:?}");
}
}
pub fn effective_parent_for_circuit(circuit_id: &str) -> Option<RuntimeCircuitParent> {
let circuit_key = normalize_circuit_id_key(circuit_id)?;
EFFECTIVE_CIRCUIT_PARENTS.load().get(&circuit_key).cloned()
}
#[derive(Clone, Copy, Debug, Default)]
struct NetworkTreeSummary {
subtree_site_count: u32,
@@ -500,17 +607,32 @@ pub struct ResolvedParentNode {
pub id: Option<String>,
}
/// Resolve a shaped-device parent node or active attachment alias into canonical `network.json`
/// parent metadata.
pub fn resolve_parent_node(parent_node: &str) -> Option<ResolvedParentNode> {
/// Resolve a shaped-device parent reference into canonical `network.json`
/// parent metadata, preferring a stable node ID when one is available.
pub fn resolve_parent_node_reference(
parent_node: &str,
parent_node_id: Option<&str>,
) -> Option<ResolvedParentNode> {
let trimmed_id = parent_node_id.map(str::trim).filter(|id| !id.is_empty());
let trimmed = parent_node.trim();
if trimmed.is_empty() {
if trimmed.is_empty() && trimmed_id.is_none() {
return None;
}
let reader = NETWORK_JSON.read();
let nodes = reader.get_nodes_when_ready();
if let Some(parent_node_id) = trimmed_id
&& let Some(node) = nodes
.iter()
.find(|node| node.id.as_deref() == Some(parent_node_id))
{
return Some(ResolvedParentNode {
name: node.name.clone(),
id: node.id.clone(),
});
}
if let Some(node) = nodes.iter().find(|node| node.name == trimmed) {
return Some(ResolvedParentNode {
name: node.name.clone(),
@@ -529,6 +651,12 @@ pub fn resolve_parent_node(parent_node: &str) -> Option<ResolvedParentNode> {
})
}
/// Resolve a shaped-device parent node or active attachment alias into canonical `network.json`
/// parent metadata.
pub fn resolve_parent_node(parent_node: &str) -> Option<ResolvedParentNode> {
resolve_parent_node_reference(parent_node, None)
}
pub fn resolve_parent_node_alias(parent_node: &str) -> Option<String> {
resolve_parent_node(parent_node).map(|resolved| resolved.name)
}
@@ -596,7 +724,9 @@ pub fn get_all_circuits() -> BusResponse {
device_id = Some(device.device_id.clone());
device_name = Some(device.device_name.clone());
parent_node = Some(
resolve_parent_node_alias(&device.parent_node)
effective_parent_for_circuit(&device.circuit_id)
.map(|parent| parent.name)
.or_else(|| resolve_parent_node_alias(&device.parent_node))
.unwrap_or_else(|| device.parent_node.clone()),
);
plan.down = device.download_max_mbps.round();
@@ -716,7 +846,12 @@ pub fn get_circuit_by_id(desired_circuit_id: String) -> BusResponse {
circuit_name = Some(device.circuit_name.clone());
device_id = Some(device.device_id.clone());
device_name = Some(device.device_name.clone());
parent_node = Some(device.parent_node.clone());
parent_node = Some(
effective_parent_for_circuit(&device.circuit_id)
.map(|parent| parent.name)
.or_else(|| resolve_parent_node_alias(&device.parent_node))
.unwrap_or_else(|| device.parent_node.clone()),
);
plan.down = device.download_max_mbps.round();
plan.up = device.upload_max_mbps.round();
}
@@ -790,3 +925,45 @@ pub fn get_circuit_by_id(desired_circuit_id: String) -> BusResponse {
BusResponse::CircuitData(Vec::new())
}
}
#[cfg(test)]
mod tests {
use super::*;
use lqos_config::{TopologyShapingCircuitInput, TopologyShapingInputsFile};
#[test]
fn effective_circuit_parent_map_uses_effective_parent_fields() {
let shaping_inputs = TopologyShapingInputsFile {
circuits: vec![TopologyShapingCircuitInput {
circuit_id: "Circuit-100".to_string(),
effective_parent_node_name: "Live Parent".to_string(),
effective_parent_node_id: "node-100".to_string(),
..TopologyShapingCircuitInput::default()
}],
..TopologyShapingInputsFile::default()
};
let map = build_effective_circuit_parent_map(&shaping_inputs);
let parent = map
.get("circuit-100")
.expect("expected normalized circuit id entry");
assert_eq!(parent.name, "Live Parent");
assert_eq!(parent.id.as_deref(), Some("node-100"));
}
#[test]
fn effective_circuit_parent_map_skips_empty_parent_names() {
let shaping_inputs = TopologyShapingInputsFile {
circuits: vec![TopologyShapingCircuitInput {
circuit_id: "Circuit-200".to_string(),
effective_parent_node_name: " ".to_string(),
effective_parent_node_id: "node-200".to_string(),
..TopologyShapingCircuitInput::default()
}],
..TopologyShapingInputsFile::default()
};
let map = build_effective_circuit_parent_map(&shaping_inputs);
assert!(map.is_empty());
}
}
src/rust/lqosd/src/throughput_tracker/stats_submission.rs
+2 -1
View File
@@ -598,11 +598,12 @@ fn load_local_shaped_devices() -> anyhow::Result<Vec<ShapedDevice>> {
.comment(Some(b'#'))
.trim(csv::Trim::All)
.from_path(sd_path)?;
let headers = reader.headers()?.clone();
let mut devices = Vec::new(); // Note that this used to be supported_customers, but we're going to let it grow organically
for result in reader.records() {
if let Ok(result) = result {
let device = ShapedDevice::from_csv(&result)?;
let device = ShapedDevice::from_csv(&result, Some(&headers))?;
devices.push(device);
} else {
anyhow::bail!("Error reading ShapedDevices.csv");
src/rust/lqosd/src/throughput_tracker/tracking_data.rs
+6 -8
View File
@@ -103,8 +103,7 @@ impl ReducedHostCounters {
for c in counts {
bytes.checked_add_direct(c.download_bytes, c.upload_bytes);
actual_bytes
.checked_add_direct(c.actual_download_bytes, c.actual_upload_bytes);
actual_bytes.checked_add_direct(c.actual_download_bytes, c.actual_upload_bytes);
packets.checked_add_direct(c.download_packets, c.upload_packets);
tcp_packets.checked_add_direct(c.tcp_download_packets, c.tcp_upload_packets);
udp_packets.checked_add_direct(c.udp_download_packets, c.udp_upload_packets);
@@ -383,8 +382,7 @@ impl ThroughputTracker {
raw_data.iter_mut().for_each(|(_k, v)| {
if v.first_cycle < self_cycle {
v.bytes_per_second = v.bytes.checked_sub_or_zero(v.prev_bytes);
v.actual_bytes_per_second =
v.actual_bytes.checked_sub_or_zero(v.prev_actual_bytes);
v.actual_bytes_per_second = v.actual_bytes.checked_sub_or_zero(v.prev_actual_bytes);
v.packets_per_second = v.packets.checked_sub_or_zero(v.prev_packets);
}
v.prev_bytes = v.bytes;
@@ -1092,9 +1090,7 @@ impl ThroughputTracker {
(
v.bytes.down.saturating_sub(v.prev_bytes.down),
v.bytes.up.saturating_sub(v.prev_bytes.up),
v.actual_bytes
.down
.saturating_sub(v.prev_actual_bytes.down),
v.actual_bytes.down.saturating_sub(v.prev_actual_bytes.down),
v.actual_bytes.up.saturating_sub(v.prev_actual_bytes.up),
v.packets.down.saturating_sub(v.prev_packets.down),
v.packets.up.saturating_sub(v.prev_packets.up),
@@ -1190,7 +1186,9 @@ impl ThroughputTracker {
#[allow(dead_code)]
pub(crate) fn actual_bits_per_second(&self) -> DownUpOrder<u64> {
self.actual_bytes_per_second.as_down_up().to_bits_from_bytes()
self.actual_bytes_per_second
.as_down_up()
.to_bits_from_bytes()
}
#[allow(dead_code)]
src/rust/uisp_integration/src/errors.rs
+2
View File
@@ -19,6 +19,8 @@ pub enum UispIntegrationError {
CsvError,
#[error("Unable to write network.json")]
WriteNetJson,
#[error("Unable to write circuit_anchors.json")]
WriteCircuitAnchors,
#[error("Bad IP")]
BadIp,
}
src/rust/uisp_integration/src/strategies/ap_only.rs
+30 -2
View File
@@ -2,9 +2,9 @@ use crate::errors::UispIntegrationError;
use crate::ethernet_advisory::{apply_ethernet_rate_cap, write_ethernet_advisories};
use crate::ip_ranges::IpRanges;
use crate::strategies::common::UispData;
use crate::strategies::full::shaped_devices_writer::ShapedDevice;
use crate::strategies::full::shaped_devices_writer::{ShapedDevice, write_circuit_anchors};
use lqos_config::{
CircuitEthernetMetadata, Config, EthernetPortLimitPolicy, RequestedCircuitRates,
CircuitAnchor, CircuitEthernetMetadata, Config, EthernetPortLimitPolicy, RequestedCircuitRates,
};
use std::collections::HashSet;
use std::fs::write;
@@ -72,8 +72,10 @@ pub async fn build_ap_only_network(
// Write ShapedDevices.csv
let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv");
let mut shaped_devices = Vec::new();
let mut circuit_anchors = Vec::<CircuitAnchor>::new();
let mut ethernet_advisories: Vec<CircuitEthernetMetadata> = Vec::new();
let mut seen_pairs = HashSet::new();
let mut seen_circuits = HashSet::new();
for (parent, client_ids) in mappings.iter() {
for client_id in client_ids {
let site = uisp_data.sites.iter().find(|s| *client_id == s.id).unwrap();
@@ -131,6 +133,24 @@ pub async fn build_ap_only_network(
if let Some(advisory) = ethernet_decision.advisory.clone() {
ethernet_advisories.push(advisory);
}
if devices.iter().any(|device| device.has_address())
&& seen_circuits.insert(site.id.clone())
{
let anchor_node_id = uisp_data
.devices
.iter()
.find(|entry| entry.name == *parent)
.map(|entry| format!("uisp:device:{}", entry.id))
.unwrap_or_default();
if !anchor_node_id.is_empty() {
circuit_anchors.push(CircuitAnchor {
circuit_id: site.id.clone(),
circuit_name: Some(site.name.clone()),
anchor_node_id,
anchor_node_name: Some(parent.clone()),
});
}
}
for device in devices.iter().filter(|d| d.has_address()) {
let key = (site.id.clone(), device.id.clone());
if !seen_pairs.insert(key) {
@@ -143,6 +163,13 @@ pub async fn build_ap_only_network(
device_id: device.id.clone(),
device_name: device.name.clone(),
parent_node: parent.clone(),
parent_node_id: uisp_data
.devices
.iter()
.find(|entry| entry.name == *parent)
.map(|entry| format!("uisp:device:{}", entry.id))
.unwrap_or_default(),
anchor_node_id: String::new(),
mac: device.mac.clone(),
ipv4: device.ipv4_list(),
ipv6: device.ipv6_list(),
@@ -169,6 +196,7 @@ pub async fn build_ap_only_network(
error!("{e:?}");
UispIntegrationError::CsvError
})?;
write_circuit_anchors(&config, "uisp/ap_only", &circuit_anchors)?;
write_ethernet_advisories(&config, &ethernet_advisories)?;
info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len());
src/rust/uisp_integration/src/strategies/ap_site.rs
+31 -2
View File
@@ -3,9 +3,9 @@ use crate::errors::UispIntegrationError;
use crate::ethernet_advisory::{apply_ethernet_rate_cap, write_ethernet_advisories};
use crate::ip_ranges::IpRanges;
use crate::strategies::common::UispData;
use crate::strategies::full::shaped_devices_writer::ShapedDevice;
use crate::strategies::full::shaped_devices_writer::{ShapedDevice, write_circuit_anchors};
use lqos_config::{
CircuitEthernetMetadata, Config, EthernetPortLimitPolicy, RequestedCircuitRates,
CircuitAnchor, CircuitEthernetMetadata, Config, EthernetPortLimitPolicy, RequestedCircuitRates,
};
use std::collections::{HashMap, HashSet};
use std::fs::write;
@@ -95,6 +95,22 @@ pub(crate) fn write_shaped_devices(
let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv");
let mut seen_pairs = HashSet::new();
shaped_devices.retain(|sd| seen_pairs.insert((sd.circuit_id.clone(), sd.device_id.clone())));
let mut seen_circuits = HashSet::new();
let circuit_anchors = shaped_devices
.iter()
.filter_map(|sd| {
let anchor_node_id = sd.parent_node_id.trim();
if anchor_node_id.is_empty() || !seen_circuits.insert(sd.circuit_id.clone()) {
return None;
}
Some(CircuitAnchor {
circuit_id: sd.circuit_id.clone(),
circuit_name: Some(sd.circuit_name.clone()),
anchor_node_id: anchor_node_id.to_string(),
anchor_node_name: Some(sd.parent_node.clone()),
})
})
.collect::<Vec<_>>();
let mut writer = csv::WriterBuilder::new()
.has_headers(true)
.from_path(file_path)
@@ -108,6 +124,7 @@ pub(crate) fn write_shaped_devices(
error!("{e:?}");
UispIntegrationError::CsvError
})?;
write_circuit_anchors(config.as_ref(), "uisp/ap_site", &circuit_anchors)?;
write_ethernet_advisories(config, ethernet_advisories)?;
info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len());
Ok(())
@@ -174,6 +191,16 @@ pub(crate) struct Layer {
pub(crate) children: Vec<Layer>,
}
fn parent_node_id_by_name(uisp_data: &UispData, name: &str) -> String {
if let Some(device) = uisp_data.devices.iter().find(|device| device.name == name) {
return format!("uisp:device:{}", device.id);
}
if let Some(site) = uisp_data.sites.iter().find(|site| site.name == name) {
return format!("uisp:site:{}", site.id);
}
String::new()
}
impl Layer {
pub(crate) fn walk_children(
&self,
@@ -337,6 +364,8 @@ impl Layer {
device_id: device.id.clone(),
device_name: device.name.clone(),
parent_node: parent.to_owned(),
parent_node_id: parent_node_id_by_name(uisp_data, parent),
anchor_node_id: String::new(),
mac: device.mac.clone(),
ipv4: device.ipv4_list(),
ipv6: device.ipv6_list(),
src/rust/uisp_integration/src/strategies/flat.rs
+4 -19
View File
@@ -3,35 +3,17 @@ use crate::errors::UispIntegrationError;
use crate::ethernet_advisory::{apply_ethernet_rate_cap, write_ethernet_advisories};
use crate::ip_ranges::IpRanges;
use crate::strategies::common::dedup_site_names;
use crate::strategies::full::shaped_devices_writer::{ShapedDevice, write_circuit_anchors};
use crate::uisp_types::UispDevice;
use lqos_config::{
CircuitEthernetMetadata, Config, EthernetPortLimitPolicy, RequestedCircuitRates,
};
use serde::Serialize;
use std::collections::HashSet;
use std::fs;
use std::path::Path;
use std::sync::Arc;
use tracing::{error, info, warn};
/// Represents a shaped device in the ShapedDevices.csv file.
#[derive(Serialize, Debug)]
struct ShapedDevice {
pub circuit_id: String,
pub circuit_name: String,
pub device_id: String,
pub device_name: String,
pub parent_node: String,
pub mac: String,
pub ipv4: String,
pub ipv6: String,
pub download_min: f32,
pub upload_min: f32,
pub download_max: f32,
pub upload_max: f32,
pub comment: String,
}
/// Builds a flat network for UISP
///
/// # Arguments
@@ -211,6 +193,8 @@ pub async fn build_flat_network(
device_id: device.get_id(),
device_name: device.get_name().unwrap_or("".to_string()),
parent_node: "".to_string(),
parent_node_id: "".to_string(),
anchor_node_id: "".to_string(),
mac: device.identification.mac.clone().unwrap_or("".to_string()),
ipv4: dev.ipv4_list(),
ipv6: dev.ipv6_list(),
@@ -241,6 +225,7 @@ pub async fn build_flat_network(
error!("{e:?}");
UispIntegrationError::CsvError
})?;
write_circuit_anchors(&config, "uisp/flat", &[])?;
write_ethernet_advisories(&config, &ethernet_advisories)?;
info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len());
src/rust/uisp_integration/src/strategies/full/shaped_devices_writer.rs
+114 -1
View File
@@ -2,10 +2,13 @@ use crate::errors::UispIntegrationError;
use crate::ethernet_advisory::{apply_ethernet_rate_cap, write_ethernet_advisories};
use crate::uisp_types::{UispDevice, UispSite, UispSiteType};
use lqos_config::{
CircuitEthernetMetadata, Config, EthernetPortLimitPolicy, RequestedCircuitRates,
CircuitAnchor, CircuitAnchorsFile, CircuitEthernetMetadata, Config, EthernetPortLimitPolicy,
RequestedCircuitRates,
};
use serde::Serialize;
use std::collections::HashMap;
use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::{error, info};
/// Represents a shaped device in the ShapedDevices.csv file.
@@ -17,6 +20,8 @@ pub struct ShapedDevice {
pub device_id: String,
pub device_name: String,
pub parent_node: String,
pub parent_node_id: String,
pub anchor_node_id: String,
pub mac: String,
pub ipv4: String,
pub ipv6: String,
@@ -27,8 +32,46 @@ pub struct ShapedDevice {
pub comment: String,
}
fn now_unix() -> Option<u64> {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.map(|duration| duration.as_secs())
}
/// Writes `circuit_anchors.json` for built-in UISP integrations.
pub(crate) fn write_circuit_anchors(
config: &Config,
source: &str,
anchors: &[CircuitAnchor],
) -> Result<(), UispIntegrationError> {
let mut deduped = HashMap::<String, CircuitAnchor>::new();
for anchor in anchors {
if anchor.anchor_node_id.trim().is_empty() {
continue;
}
deduped
.entry(anchor.circuit_id.clone())
.or_insert_with(|| anchor.clone());
}
let mut anchors = deduped.into_values().collect::<Vec<_>>();
anchors.sort_unstable_by(|left, right| left.circuit_id.cmp(&right.circuit_id));
let payload = CircuitAnchorsFile {
schema_version: 1,
source: source.to_string(),
generated_unix: now_unix(),
anchors,
};
payload.save(config).map_err(|e| {
error!("Unable to write circuit_anchors.json");
error!("{e:?}");
UispIntegrationError::WriteCircuitAnchors
})
}
struct ShapedDeviceOutputs<'a> {
shaped_devices: &'a mut Vec<ShapedDevice>,
circuit_anchors: &'a mut Vec<CircuitAnchor>,
ethernet_advisories: &'a mut Vec<CircuitEthernetMetadata>,
}
@@ -38,6 +81,25 @@ struct TraverseContext<'a> {
ethernet_policy: EthernetPortLimitPolicy,
}
pub(crate) fn generic_site_node_id(site: &UispSite) -> String {
if site.site_type == UispSiteType::ClientWithChildren {
let mut slug = String::with_capacity(site.name.len());
let mut last_was_dash = false;
for ch in site.name.chars() {
if ch.is_ascii_alphanumeric() {
slug.push(ch.to_ascii_lowercase());
last_was_dash = false;
} else if !last_was_dash {
slug.push('-');
last_was_dash = true;
}
}
format!("libreqos:generated:uisp:site:{}", slug.trim_matches('-'))
} else {
format!("uisp:site:{}", site.id)
}
}
/// Writes the ShapedDevices.csv file for UISP
///
/// # Arguments
@@ -54,11 +116,13 @@ pub fn write_shaped_devices(
let ethernet_policy = EthernetPortLimitPolicy::from(&config.integration_common);
let file_path = Path::new(&config.lqos_directory).join("ShapedDevices.csv");
let mut shaped_devices = Vec::new();
let mut circuit_anchors = Vec::new();
let mut ethernet_advisories: Vec<CircuitEthernetMetadata> = Vec::new();
// Traverse
let mut outputs = ShapedDeviceOutputs {
shaped_devices: &mut shaped_devices,
circuit_anchors: &mut circuit_anchors,
ethernet_advisories: &mut ethernet_advisories,
};
let context = TraverseContext {
@@ -82,6 +146,7 @@ pub fn write_shaped_devices(
error!("{e:?}");
UispIntegrationError::CsvError
})?;
write_circuit_anchors(config, "uisp/full", &circuit_anchors)?;
write_ethernet_advisories(config, &ethernet_advisories)?;
info!("Wrote {} lines to ShapedDevices.csv", shaped_devices.len());
@@ -99,6 +164,18 @@ fn traverse(
if !sites[idx].device_indices.is_empty() {
// We have devices!
if sites[idx].site_type == UispSiteType::Client {
let has_shapable_device = sites[idx]
.device_indices
.iter()
.copied()
.filter_map(|device_idx| devices.get(device_idx))
.any(UispDevice::has_address);
let circuit_anchor = CircuitAnchor {
circuit_id: sites[idx].id.clone(),
circuit_name: Some(sites[idx].name.clone()),
anchor_node_id: generic_site_node_id(&sites[idx]),
anchor_node_name: Some(sites[idx].name.clone()),
};
let site_devices: Vec<&UispDevice> = sites[idx]
.device_indices
.iter()
@@ -145,6 +222,9 @@ fn traverse(
if let Some(advisory) = ethernet_decision.advisory.clone() {
outputs.ethernet_advisories.push(advisory);
}
if has_shapable_device {
outputs.circuit_anchors.push(circuit_anchor);
}
// Add as normal clients
for device in sites[idx].device_indices.iter() {
@@ -156,6 +236,10 @@ fn traverse(
device_id: device.id.clone(),
device_name: device.name.clone(),
parent_node: sites[sites[idx].selected_parent.unwrap()].name.clone(),
parent_node_id: generic_site_node_id(
&sites[sites[idx].selected_parent.unwrap()],
),
anchor_node_id: String::new(),
mac: device.mac.clone(),
ipv4: device.ipv4_list(),
ipv6: device.ipv6_list(),
@@ -169,6 +253,18 @@ fn traverse(
}
}
} else {
let has_shapable_device = sites[idx]
.device_indices
.iter()
.copied()
.filter_map(|device_idx| devices.get(device_idx))
.any(UispDevice::has_address);
let infrastructure_anchor = (idx != context.root_idx).then(|| CircuitAnchor {
circuit_id: format!("{}-inf", sites[idx].id),
circuit_name: Some(format!("{} Infrastructure", sites[idx].name)),
anchor_node_id: generic_site_node_id(&sites[idx]),
anchor_node_name: Some(sites[idx].name.clone()),
});
// It's an infrastructure node
for device in sites[idx].device_indices.iter() {
let device = &devices[*device];
@@ -198,6 +294,12 @@ fn traverse(
device_id: device.id.clone(),
device_name: device.name.clone(),
parent_node,
parent_node_id: if idx != context.root_idx {
generic_site_node_id(&sites[idx])
} else {
String::new()
},
anchor_node_id: String::new(),
mac: device.mac.clone(),
ipv4: device.ipv4_list(),
ipv6: device.ipv6_list(),
@@ -210,6 +312,9 @@ fn traverse(
outputs.shaped_devices.push(sd);
}
}
if has_shapable_device && let Some(anchor) = infrastructure_anchor {
outputs.circuit_anchors.push(anchor);
}
}
}
@@ -240,6 +345,8 @@ mod tests {
device_id: "device-001".to_string(),
device_name: "CPE-001".to_string(),
parent_node: "Tower-A".to_string(),
parent_node_id: "uisp:site:tower-a".to_string(),
anchor_node_id: "uisp:site:test-001".to_string(),
mac: "00:11:22:33:44:55".to_string(),
ipv4: "192.168.1.100".to_string(),
ipv6: "".to_string(),
@@ -255,6 +362,8 @@ mod tests {
device_id: "device-002".to_string(),
device_name: "CPE-002".to_string(),
parent_node: "Tower-B".to_string(),
parent_node_id: "uisp:site:tower-b".to_string(),
anchor_node_id: "uisp:site:test-002".to_string(),
mac: "00:11:22:33:44:66".to_string(),
ipv4: "192.168.1.101".to_string(),
ipv6: "2001:db8::1".to_string(),
@@ -270,6 +379,8 @@ mod tests {
device_id: "device-003".to_string(),
device_name: "AP-003".to_string(),
parent_node: "Root".to_string(),
parent_node_id: "".to_string(),
anchor_node_id: "uisp:site:test-003".to_string(),
mac: "00:11:22:33:44:77".to_string(),
ipv4: "10.0.1.1".to_string(),
ipv6: "".to_string(),
@@ -420,9 +531,11 @@ mod tests {
}];
let mut shaped_devices = Vec::new();
let mut circuit_anchors = Vec::new();
let mut ethernet_advisories = Vec::new();
let mut outputs = ShapedDeviceOutputs {
shaped_devices: &mut shaped_devices,
circuit_anchors: &mut circuit_anchors,
ethernet_advisories: &mut ethernet_advisories,
};
src/rust/uisp_integration/src/strategies/full2.rs
+118 -49
View File
@@ -11,7 +11,7 @@ use crate::ip_ranges::IpRanges;
use crate::strategies::common::UispData;
use crate::strategies::full::bandwidth_overrides::{BandwidthOverride, find_bandwidth_override};
use crate::strategies::full::routes_override::RouteOverride;
use crate::strategies::full::shaped_devices_writer::ShapedDevice;
use crate::strategies::full::shaped_devices_writer::{ShapedDevice, write_circuit_anchors};
use crate::strategies::full2::directionality::{
build_device_capacity_map, build_device_link_meta_map, directed_caps_mbps,
};
@@ -21,7 +21,7 @@ use crate::strategies::full2::link_mapping::LinkMapping;
use crate::strategies::full2::net_json_parent::{NetJsonParent, assign_export_names, walk_parents};
use crate::uisp_types::{UispAttachmentRateSource, UispDevice};
use lqos_config::{
CircuitEthernetMetadata, Config, EthernetPortLimitPolicy, RequestedCircuitRates,
CircuitAnchor, CircuitEthernetMetadata, Config, EthernetPortLimitPolicy, RequestedCircuitRates,
TOPOLOGY_ATTACHMENT_AUTO_ID, TopologyAllowedParent, TopologyAttachmentOption,
TopologyAttachmentRateSource, TopologyAttachmentRole, TopologyCanonicalIngressKind,
TopologyCanonicalStateFile, TopologyEditorNode, TopologyEditorStateFile,
@@ -78,6 +78,21 @@ fn debug_graph_output_enabled() -> bool {
env::var_os("LIBREQOS_UISP_DEBUG_GRAPH").is_some()
}
fn anchor_for_site_ap(
site_id: &str,
site_name: &str,
ap_device: &UispDevice,
parents: &HashMap<String, NetJsonParent<'_>>,
) -> Option<CircuitAnchor> {
let ap_node_id = format!("uisp:device:{}", ap_device.id);
parents.contains_key(&ap_node_id).then(|| CircuitAnchor {
circuit_id: site_id.to_owned(),
circuit_name: Some(site_name.to_owned()),
anchor_node_id: ap_node_id,
anchor_node_name: Some(ap_device.name.to_owned()),
})
}
pub async fn build_full_network_v2(
config: Arc<Config>,
ip_ranges: IpRanges,
@@ -577,7 +592,9 @@ pub async fn build_full_network_v2(
// Shaped Devices
let shaped_devices_started = Instant::now();
let mut shaped_devices = Vec::new();
let mut circuit_anchors = Vec::<CircuitAnchor>::new();
let mut seen_pairs = HashSet::new();
let mut seen_circuits = HashSet::new();
let mut processed_site_pairs = 0usize;
let mut shaped_device_count = 0usize;
let mut ethernet_advisories: Vec<CircuitEthernetMetadata> = Vec::new();
@@ -642,6 +659,17 @@ pub async fn build_full_network_v2(
if let Some(advisory) = ethernet_decision.advisory.clone() {
ethernet_advisories.push(advisory);
}
if !site_devices.is_empty() && seen_circuits.insert(site.id.clone()) {
if let Some(anchor) = anchor_for_site_ap(&site.id, &site.name, ap_device, &parents)
{
circuit_anchors.push(anchor);
} else {
debug!(
"Skipping circuit anchor for '{}' because AP '{}' ({}) is not present in topology; circuit will resolve from ingress parent data.",
site.name, ap_device.name, ap_device.id
);
}
}
for device in uisp_data.find_parsed_devices_in_site(site_id) {
if !device.has_address() {
continue;
@@ -659,6 +687,13 @@ pub async fn build_full_network_v2(
orphans_node_name.clone()
}
};
let parent_node_id = {
let ap_node_id = format!("uisp:device:{}", ap_device.id);
parents
.get(&ap_node_id)
.map(|parent| parent.node_id.clone())
.unwrap_or_default()
};
let key = (site.id.clone(), device.id.clone());
if !seen_pairs.insert(key) {
@@ -671,6 +706,8 @@ pub async fn build_full_network_v2(
device_id: device.id.to_owned(),
device_name: device.name.to_owned(),
parent_node: parent_node.clone(),
parent_node_id,
anchor_node_id: String::new(),
mac: device.mac.to_owned(),
ipv4: device.ipv4_list(),
ipv6: device.ipv6_list(),
@@ -707,6 +744,7 @@ pub async fn build_full_network_v2(
error!("{e:?}");
UispIntegrationError::CsvError
})?;
write_circuit_anchors(&config, "uisp/full2", &circuit_anchors)?;
write_ethernet_advisories(&config, &ethernet_advisories)?;
info!(
shaped_devices = shaped_devices.len(),
@@ -2402,31 +2440,14 @@ mod topology_override_tests {
});
let beta_site = graph.add_node(site("Site Beta", "site-beta"));
let alpha_site = graph.add_node(site("Site Alpha", "site-alpha"));
let gamma_to_beta = graph.add_node(ap(
"Gamma - Beta MLO6",
"device-gamma-beta",
"Site Gamma",
));
let beta_to_gamma = graph.add_node(ap(
"Beta - Gamma MLO6",
"device-beta-gamma",
"Site Beta",
));
let gamma_to_alpha = graph.add_node(ap(
"Gamma-Alpha",
"device-gamma-alpha",
"Site Gamma",
));
let alpha_to_gamma = graph.add_node(ap(
"Alpha-Gamma",
"device-alpha-gamma",
"Site Alpha",
));
let beta_to_alpha = graph.add_node(ap(
"Beta - Alpha MLO5",
"device-beta-alpha",
"Site Beta",
));
let gamma_to_beta =
graph.add_node(ap("Gamma - Beta MLO6", "device-gamma-beta", "Site Gamma"));
let beta_to_gamma =
graph.add_node(ap("Beta - Gamma MLO6", "device-beta-gamma", "Site Beta"));
let gamma_to_alpha = graph.add_node(ap("Gamma-Alpha", "device-gamma-alpha", "Site Gamma"));
let alpha_to_gamma = graph.add_node(ap("Alpha-Gamma", "device-alpha-gamma", "Site Alpha"));
let beta_to_alpha =
graph.add_node(ap("Beta - Alpha MLO5", "device-beta-alpha", "Site Beta"));
let alpha_to_beta =
graph.add_node(ap("Alpha - Beta MLO5", "device-alpha-beta", "Site Alpha"));
@@ -2464,14 +2485,8 @@ mod topology_override_tests {
&[],
));
let groups = topology_allowed_parent_groups_for_node(
&graph,
root,
beta_site,
&candidates,
&[],
&[],
);
let groups =
topology_allowed_parent_groups_for_node(&graph, root, beta_site, &candidates, &[], &[]);
assert_eq!(groups.len(), 2);
assert_eq!(groups[0].logical_parent, alpha_site);
assert_eq!(groups[0].candidate_nodes, vec![beta_to_alpha]);
@@ -2512,20 +2527,10 @@ mod topology_override_tests {
});
let beta_site = graph.add_node(site("Site Beta", "site-beta"));
let alpha_site = graph.add_node(site("Site Alpha", "site-alpha"));
let gamma_to_alpha = graph.add_node(ap(
"Gamma-Alpha",
"device-gamma-alpha",
"Site Gamma",
));
let alpha_to_gamma = graph.add_node(ap(
"Alpha-Gamma",
"device-alpha-gamma",
"Site Alpha",
));
let beta_to_alpha =
graph.add_node(ap("Beta - Alpha 60", "device-beta-alpha", "Site Beta"));
let alpha_to_beta =
graph.add_node(ap("Alpha-Beta-60", "device-alpha-beta", "Site Alpha"));
let gamma_to_alpha = graph.add_node(ap("Gamma-Alpha", "device-gamma-alpha", "Site Gamma"));
let alpha_to_gamma = graph.add_node(ap("Alpha-Gamma", "device-alpha-gamma", "Site Alpha"));
let beta_to_alpha = graph.add_node(ap("Beta - Alpha 60", "device-beta-alpha", "Site Beta"));
let alpha_to_beta = graph.add_node(ap("Alpha-Beta-60", "device-alpha-beta", "Site Alpha"));
for (from, to) in [
(root, gamma_to_alpha),
@@ -2897,6 +2902,31 @@ fn calculate_chain_capacity(graph: &GraphType, chain_nodes: &[NodeIndex]) -> (u6
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashSet;
fn test_uisp_device(id: &str, name: &str) -> UispDevice {
UispDevice {
id: id.to_string(),
name: name.to_string(),
mac: String::new(),
role: None,
wireless_mode: None,
site_id: String::new(),
raw_download: 0,
raw_upload: 0,
download: 0,
upload: 0,
ipv4: HashSet::new(),
ipv6: HashSet::new(),
probe_ipv4: HashSet::new(),
probe_ipv6: HashSet::new(),
negotiated_ethernet_mbps: None,
negotiated_ethernet_interface: None,
transport_cap_mbps: None,
transport_cap_reason: None,
attachment_rate_source: UispAttachmentRateSource::Static,
}
}
fn add_site(graph: &mut GraphType, name: &str, id: &str) -> NodeIndex {
graph.add_node(GraphMapping::Site {
@@ -3052,4 +3082,43 @@ mod tests {
assert!(graph_b.find_edge(left_b, right_b).is_some());
assert!(graph_b.find_edge(right_b, left_b).is_some());
}
#[test]
fn full2_skips_orphaned_circuit_anchor_when_ap_is_not_in_topology() {
let device = test_uisp_device("device-missing", "Missing AP");
let parents = HashMap::<String, NetJsonParent<'_>>::new();
let anchor = anchor_for_site_ap("circuit-1", "Circuit 1", &device, &parents);
assert!(anchor.is_none());
}
#[test]
fn full2_emits_circuit_anchor_when_ap_exists_in_topology() {
let mut graph = GraphType::new();
let ap = add_ap(&mut graph, "Present AP", "device-present", "Present Site");
let mapping = &graph[ap];
let mut parents = HashMap::<String, NetJsonParent<'_>>::new();
let node_id = mapping.network_json_id();
parents.insert(
node_id.clone(),
NetJsonParent {
node_id: node_id.clone(),
node_name: mapping.name(),
export_name: mapping.name(),
parent_id: None,
parent_name: "Parent".to_string(),
mapping,
download: 1000,
upload: 1000,
},
);
let device = test_uisp_device("device-present", "Present AP");
let anchor = anchor_for_site_ap("circuit-1", "Circuit 1", &device, &parents)
.expect("expected anchor to be emitted");
assert_eq!(anchor.anchor_node_id, "uisp:device:device-present");
assert_eq!(anchor.anchor_node_name.as_deref(), Some("Present AP"));
}
}
src/scheduler.py
+159 -42
View File
@@ -68,6 +68,21 @@ def publish_scheduler_progress(active: bool, phase: str, phase_label: str, step_
print(f"Failed to publish scheduler progress: {e}")
def report_scheduler_runtime_failure(context: str, exc: Exception, *, startup: bool = False):
message = f"{context}: {exc}"
print(message)
scheduler_error(message)
if startup:
publish_scheduler_progress(
False,
"degraded",
"Scheduler running with topology/runtime error",
SCHEDULER_STARTUP_STEP_COUNT,
SCHEDULER_STARTUP_STEP_COUNT,
percent=100,
)
def _integration_output_lines(output):
normalized = (output or "").replace("\r\n", "\n").strip()
if not normalized:
@@ -345,6 +360,8 @@ SHAPED_DEVICES_HEADER = [
"Device ID",
"Device Name",
"Parent Node",
"Parent Node ID",
"Anchor Node ID",
"MAC",
"IPv4",
"IPv6",
@@ -361,6 +378,66 @@ SHAPED_DEVICES_HEADER_WITH_SQM = [
]
def normalize_shaped_devices_header(header_value):
return ''.join(ch for ch in str(header_value).lower() if ch.isalnum())
def shaped_devices_header_index_map(header):
legacy = {
'circuit_id': 0,
'circuit_name': 1,
'device_id': 2,
'device_name': 3,
'parent_node': 4,
'mac': 5,
'ipv4': 6,
'ipv6': 7,
'download_min': 8,
'upload_min': 9,
'download_max': 10,
'upload_max': 11,
'comment': 12,
'sqm': 13,
}
aliases = {
'circuit_id': {'circuitid', 'circuit_id'},
'circuit_name': {'circuitname', 'circuit_name'},
'device_id': {'deviceid', 'device_id'},
'device_name': {'devicename', 'device_name'},
'parent_node': {'parentnode', 'parent_node'},
'parent_node_id': {'parentnodeid', 'parent_node_id'},
'anchor_node_id': {'anchornodeid', 'anchor_node_id', 'id'},
'mac': {'mac'},
'ipv4': {'ipv4'},
'ipv6': {'ipv6'},
'download_min': {'downloadminmbps', 'downloadmin', 'download_min_mbps'},
'upload_min': {'uploadminmbps', 'uploadmin', 'upload_min_mbps'},
'download_max': {'downloadmaxmbps', 'downloadmax', 'download_max_mbps'},
'upload_max': {'uploadmaxmbps', 'uploadmax', 'upload_max_mbps'},
'comment': {'comment'},
'sqm': {'sqm'},
}
layout = dict(legacy)
layout['parent_node_id'] = None
layout['anchor_node_id'] = None
for idx, name in enumerate(header or []):
normalized = normalize_shaped_devices_header(name)
for field, names in aliases.items():
if normalized in names:
layout[field] = idx
break
return layout
def set_row_value_by_header(row, header_map, field, value):
index = header_map.get(field)
if index is None:
return
while len(row) <= index:
row.append('')
row[index] = value
def shaped_devices_csv_path() -> str:
base_dir = get_libreqos_directory()
return base_dir + "/ShapedDevices.csv"
@@ -405,31 +482,34 @@ def read_shaped_devices_csv(path: str):
return header, data_rows
def override_devices_to_rows(devices, include_sqm=False):
def override_devices_to_rows(devices, header, include_sqm=False):
"""Convert override device dicts to CSV rows, preserving the existing CSV shape by default."""
rows = []
header_map = shaped_devices_header_index_map(header)
row_len = max(len(header), len(SHAPED_DEVICES_HEADER_WITH_SQM if include_sqm else SHAPED_DEVICES_HEADER))
for d in devices:
ipv4s = d.get('ipv4s', [])
ipv6s = d.get('ipv6s', [])
sqm = d.get('sqm', '') or d.get('sqm_override', '')
sqm = sqm or ""
row = [
d.get('circuitID', ''),
d.get('circuitName', ''),
d.get('deviceID', ''),
d.get('deviceName', ''),
d.get('ParentNode', ''),
d.get('mac', ''),
','.join(ipv4s),
','.join(ipv6s),
str(d.get('minDownload', '')),
str(d.get('minUpload', '')),
str(d.get('maxDownload', '')),
str(d.get('maxUpload', '')),
d.get('comment', ''),
]
row = [''] * row_len
set_row_value_by_header(row, header_map, 'circuit_id', d.get('circuitID', ''))
set_row_value_by_header(row, header_map, 'circuit_name', d.get('circuitName', ''))
set_row_value_by_header(row, header_map, 'device_id', d.get('deviceID', ''))
set_row_value_by_header(row, header_map, 'device_name', d.get('deviceName', ''))
set_row_value_by_header(row, header_map, 'parent_node', d.get('ParentNode', ''))
set_row_value_by_header(row, header_map, 'parent_node_id', d.get('ParentNodeID', ''))
set_row_value_by_header(row, header_map, 'anchor_node_id', d.get('AnchorNodeID', ''))
set_row_value_by_header(row, header_map, 'mac', d.get('mac', ''))
set_row_value_by_header(row, header_map, 'ipv4', ','.join(ipv4s))
set_row_value_by_header(row, header_map, 'ipv6', ','.join(ipv6s))
set_row_value_by_header(row, header_map, 'download_min', str(d.get('minDownload', '')))
set_row_value_by_header(row, header_map, 'upload_min', str(d.get('minUpload', '')))
set_row_value_by_header(row, header_map, 'download_max', str(d.get('maxDownload', '')))
set_row_value_by_header(row, header_map, 'upload_max', str(d.get('maxUpload', '')))
set_row_value_by_header(row, header_map, 'comment', d.get('comment', ''))
if include_sqm:
row.append(str(sqm))
set_row_value_by_header(row, header_map, 'sqm', str(sqm))
rows.append(row)
return rows
@@ -463,7 +543,9 @@ def write_shaped_devices_csv(path: str, header, rows):
def header_has_sqm(header):
return len(header) > 13 and header[13].strip().lower() == "sqm"
return shaped_devices_header_index_map(header).get('sqm') is not None and any(
normalize_shaped_devices_header(value) == 'sqm' for value in (header or [])
)
def operator_requires_sqm_column(devices, adjustments):
@@ -504,8 +586,9 @@ def apply_lqos_overrides():
if len(row) < len(header):
row.extend([""] * (len(header) - len(row)))
override_rows = override_devices_to_rows(extra or [], include_sqm=need_sqm_column)
override_rows = override_devices_to_rows(extra or [], header, include_sqm=need_sqm_column)
merged_rows, changed = merge_rows_replace_by_device_id(rows, override_rows)
header_map = shaped_devices_header_index_map(header)
def set_if_some(value_opt, current_str):
if value_opt is None:
@@ -568,8 +651,10 @@ def apply_lqos_overrides():
cid = adj.get('circuit_id', '')
parent_node = adj.get('parent_node', '')
for r in merged_rows:
if len(r) >= 5 and r[0] == cid:
r[4] = parent_node
circuit_id_idx = header_map.get('circuit_id', 0)
if len(r) > circuit_id_idx and r[circuit_id_idx] == cid:
set_row_value_by_header(r, header_map, 'parent_node', parent_node)
set_row_value_by_header(r, header_map, 'parent_node_id', '')
changed = True
if changed:
@@ -845,7 +930,11 @@ def importAndShapePartialReload():
new_hash = calculate_shaping_runtime_hash()
if new_hash != shaping_runtime_hash:
publish_scheduler_progress(True, "partial_reload", "Applying incremental shaper refresh", 4, SCHEDULER_REFRESH_STEP_COUNT)
refreshShapers()
try:
refreshShapers()
except Exception as e:
report_scheduler_runtime_failure("Scheduled shaping refresh failed", e)
return
shaping_runtime_hash = calculate_shaping_runtime_hash()
publish_scheduler_progress(False, "ready", "Scheduler ready", SCHEDULER_REFRESH_STEP_COUNT, SCHEDULER_REFRESH_STEP_COUNT, percent=100)
@@ -860,6 +949,7 @@ def topology_runtime_output_paths():
os.path.join(base_dir, "topology_attachment_health_state.json"),
os.path.join(base_dir, "topology_effective_state.json"),
os.path.join(base_dir, "network.effective.json"),
os.path.join(base_dir, "shaping_inputs.json"),
]
@@ -895,8 +985,12 @@ def _topology_runtime_freshness_target():
def _topology_runtime_outputs_are_fresh():
_, effective_state_path, effective_network_path = topology_runtime_output_paths()
if not (os.path.isfile(effective_state_path) and os.path.isfile(effective_network_path)):
_, effective_state_path, effective_network_path, shaping_inputs_path = topology_runtime_output_paths()
if not (
os.path.isfile(effective_state_path)
and os.path.isfile(effective_network_path)
and os.path.isfile(shaping_inputs_path)
):
return False
target_kind, target_value = _topology_runtime_freshness_target()
@@ -911,9 +1005,16 @@ def _topology_runtime_outputs_are_fresh():
try:
effective_state_mtime = os.path.getmtime(effective_state_path)
effective_network_mtime = os.path.getmtime(effective_network_path)
shaping_inputs_mtime = os.path.getmtime(shaping_inputs_path)
except OSError:
return False
return min(effective_state_mtime, effective_network_mtime) >= target_value
shaped_devices_path = os.path.join(get_libreqos_directory(), "ShapedDevices.csv")
circuit_anchors_path = os.path.join(get_libreqos_directory(), "circuit_anchors.json")
if os.path.isfile(shaped_devices_path) and shaping_inputs_mtime < os.path.getmtime(shaped_devices_path):
return False
if os.path.isfile(circuit_anchors_path) and shaping_inputs_mtime < os.path.getmtime(circuit_anchors_path):
return False
return min(effective_state_mtime, effective_network_mtime, shaping_inputs_mtime) >= target_value
def clear_topology_runtime_outputs():
@@ -1005,7 +1106,11 @@ def topology_runtime_refresh_tick():
if new_hash == 0 or new_hash == shaping_runtime_hash:
return
refreshShapers()
try:
refreshShapers()
except Exception as e:
report_scheduler_runtime_failure("Topology runtime refresh failed", e)
return
shaping_runtime_hash = calculate_shaping_runtime_hash()
@@ -1018,28 +1123,40 @@ def ensure_bus_ready():
"""Wait briefly for lqosd to finish binding the local bus socket."""
wait_for_bus_ready(5000)
if __name__ == '__main__':
def run_scheduler_main():
global shaping_runtime_hash
atexit.register(stop_topology_runtime_process)
publish_scheduler_progress(True, "waiting_for_bus", "Waiting for lqosd bus", 1, SCHEDULER_STARTUP_STEP_COUNT)
ensure_bus_ready()
try:
atexit.register(stop_topology_runtime_process)
publish_scheduler_progress(True, "waiting_for_bus", "Waiting for lqosd bus", 1, SCHEDULER_STARTUP_STEP_COUNT)
ensure_bus_ready()
importAndShapeFullReload()
shaping_runtime_hash = calculate_shaping_runtime_hash()
publish_scheduler_progress(False, "ready", "Scheduler ready", SCHEDULER_STARTUP_STEP_COUNT, SCHEDULER_STARTUP_STEP_COUNT, percent=100)
except Exception as e:
report_scheduler_runtime_failure("Scheduler startup shaping refresh failed", e, startup=True)
import traceback
traceback.print_exc()
shaping_runtime_hash = 0
print("Starting scheduler with jobs:")
print(f"- not_dead_yet every 1 minute")
refresh_interval = queue_refresh_interval_mins()
print(f"- topology_runtime_refresh_tick every {TOPOLOGY_RUNTIME_REFRESH_SECONDS} seconds")
print(f"- importAndShapePartialReload every {refresh_interval} minutes")
not_dead_yet()
ads.add_job(not_dead_yet, 'interval', minutes=1, max_instances=1)
ads.add_job(topology_runtime_refresh_tick, 'interval', seconds=TOPOLOGY_RUNTIME_REFRESH_SECONDS, max_instances=1)
ads.add_job(importAndShapePartialReload, 'interval', minutes=refresh_interval, max_instances=1)
print("Starting scheduler with jobs:")
print(f"- not_dead_yet every 1 minute")
refresh_interval = queue_refresh_interval_mins()
print(f"- topology_runtime_refresh_tick every {TOPOLOGY_RUNTIME_REFRESH_SECONDS} seconds")
print(f"- importAndShapePartialReload every {refresh_interval} minutes")
print("Scheduler starting...")
ads.start()
not_dead_yet()
ads.add_job(not_dead_yet, 'interval', minutes=1, max_instances=1)
ads.add_job(topology_runtime_refresh_tick, 'interval', seconds=TOPOLOGY_RUNTIME_REFRESH_SECONDS, max_instances=1)
ads.add_job(importAndShapePartialReload, 'interval', minutes=refresh_interval, max_instances=1)
print("Scheduler starting...")
ads.start()
if __name__ == '__main__':
try:
run_scheduler_main()
except Exception as e:
print(f"Error starting scheduler: {e}")
import traceback
src/test_libreqos_shaping_inputs.py
+173
View File
@@ -0,0 +1,173 @@
import csv
import importlib
import os
import sys
import tempfile
import time
import types
import unittest
from unittest.mock import patch
def install_libreqos_stubs():
python_check = types.ModuleType("pythonCheck")
python_check.checkPythonVersion = lambda: None
sys.modules["pythonCheck"] = python_check
lqlib = types.ModuleType("liblqos_python")
lqlib.is_lqosd_alive = lambda: True
lqlib.clear_ip_mappings = lambda: None
lqlib.delete_ip_mapping = lambda *_args, **_kwargs: None
lqlib.validate_shaped_devices = lambda: "OK"
lqlib.is_libre_already_running = lambda: False
lqlib.create_lock_file = lambda: None
lqlib.free_lock_file = lambda: None
lqlib.add_ip_mapping = lambda *_args, **_kwargs: None
class DummyBatchedCommands:
pass
class DummyBakery:
pass
lqlib.BatchedCommands = DummyBatchedCommands
lqlib.check_config = lambda: None
lqlib.sqm = lambda: "cake"
lqlib.upstream_bandwidth_capacity_download_mbps = lambda: 1000
lqlib.upstream_bandwidth_capacity_upload_mbps = lambda: 1000
lqlib.interface_a = lambda: "eth0"
lqlib.interface_b = lambda: "eth1"
lqlib.enable_actual_shell_commands = lambda: False
lqlib.use_bin_packing_to_balance_cpu = lambda: False
lqlib.queue_mode = lambda: "shape"
lqlib.run_shell_commands_as_sudo = lambda: False
lqlib.generated_pn_download_mbps = lambda: 1000
lqlib.generated_pn_upload_mbps = lambda: 1000
lqlib.queues_available_override = lambda: 0
lqlib.on_a_stick = lambda: False
lqlib.get_tree_weights = lambda: {}
lqlib.get_weights = lambda: {}
lqlib.is_network_flat = lambda: False
lqlib.get_libreqos_directory = lambda: "/tmp/libreqos" # nosec B108
lqlib.enable_insight_topology = lambda: False
lqlib.is_insight_enabled = lambda: False
lqlib.scheduler_error = lambda *_args, **_kwargs: None
lqlib.xdp_ip_mapping_capacity = lambda: 1024
lqlib.overrides_circuit_adjustments_effective = lambda: []
lqlib.automatic_import_uisp = lambda: False
lqlib.automatic_import_splynx = lambda: False
lqlib.automatic_import_powercode = lambda: False
lqlib.automatic_import_sonar = lambda: False
lqlib.automatic_import_wispgate = lambda: False
lqlib.automatic_import_netzur = lambda: False
lqlib.automatic_import_visp = lambda: False
lqlib.plan_top_level_cpu_bins = lambda *_args, **_kwargs: {}
lqlib.plan_class_identities = lambda *_args, **_kwargs: {}
lqlib.fast_queues_fq_codel = lambda: False
lqlib.shaping_cpu_count = lambda: 16
lqlib.Bakery = DummyBakery
sys.modules["liblqos_python"] = lqlib
install_libreqos_stubs()
LibreQoS = importlib.import_module("LibreQoS")
class TestLibreQoSShapingInputs(unittest.TestCase):
def test_shaping_inputs_freshness_tracks_circuit_anchors(self):
with tempfile.TemporaryDirectory() as temp_dir:
shaping_inputs = os.path.join(temp_dir, "shaping_inputs.json")
shaped_devices = os.path.join(temp_dir, "ShapedDevices.csv")
network_json = os.path.join(temp_dir, "network.json")
circuit_anchors = os.path.join(temp_dir, "circuit_anchors.json")
for path in (shaping_inputs, shaped_devices, network_json, circuit_anchors):
with open(path, "w", encoding="utf-8") as handle:
handle.write("{}\n")
now = time.time()
os.utime(shaped_devices, (now - 20, now - 20))
os.utime(network_json, (now - 20, now - 20))
os.utime(shaping_inputs, (now - 10, now - 10))
os.utime(circuit_anchors, (now - 5, now - 5))
self.assertFalse(
LibreQoS._shaping_inputs_are_fresh(
shaping_inputs, shaped_devices, network_json, circuit_anchors
)
)
def test_load_subscriber_circuits_accepts_diy_id_alias(self):
header = [
"Circuit ID",
"Circuit Name",
"Device ID",
"Device Name",
"Parent Node",
"Parent Node ID",
"id",
"MAC",
"IPv4",
"IPv6",
"Download Min Mbps",
"Upload Min Mbps",
"Download Max Mbps",
"Upload Max Mbps",
"Comment",
]
row = [
"100",
"Subscriber 100",
"device-100",
"Radio 100",
"Tower-A",
"uisp:device:tower-a",
"uisp:site:site-100",
"aa:bb:cc:dd:ee:ff",
"100.64.0.10/32",
"",
"10",
"10",
"100",
"100",
"DIY id alias",
]
with tempfile.NamedTemporaryFile("w", encoding="utf-8", newline="", delete=False) as handle:
writer = csv.writer(handle)
writer.writerow(header)
writer.writerow(row)
path = handle.name
try:
circuits, _ = LibreQoS.loadSubscriberCircuits(path)
finally:
os.remove(path)
self.assertEqual(len(circuits), 1)
self.assertEqual(circuits[0]["AnchorNodeID"], "uisp:site:site-100")
def test_load_subscriber_circuits_for_shaping_requires_runtime_artifacts_for_diy(self):
with patch.object(LibreQoS, "_shaping_inputs_are_fresh", return_value=False):
with self.assertRaises(LibreQoS.RefreshFailure):
LibreQoS.loadSubscriberCircuitsForShaping(
"/tmp/ShapedDevices.csv", # nosec B108
"/tmp/network.json", # nosec B108
)
def test_load_subscriber_circuits_for_shaping_requires_valid_runtime_payload(self):
with patch.object(LibreQoS, "_shaping_inputs_are_fresh", return_value=True):
with patch.object(
LibreQoS,
"loadSubscriberCircuitsFromShapingInputs",
side_effect=ValueError("bad payload"),
):
with self.assertRaises(LibreQoS.RefreshFailure):
LibreQoS.loadSubscriberCircuitsForShaping(
"/tmp/ShapedDevices.csv", # nosec B108
"/tmp/network.json", # nosec B108
)
if __name__ == "__main__":
unittest.main()
src/test_scheduler.py
+151
View File
@@ -33,6 +33,7 @@ def install_scheduler_stubs():
lqlib.scheduler_alive = Mock()
lqlib.scheduler_error = Mock()
lqlib.scheduler_output = Mock()
lqlib.scheduler_progress = Mock()
lqlib.wait_for_bus_ready = Mock(return_value=True)
lqlib.overrides_persistent_devices_effective = lambda: []
lqlib.overrides_persistent_devices_materialized = lambda: []
@@ -236,6 +237,54 @@ class TestSchedulerErrorReporting(unittest.TestCase):
[(( "",),), (("uisp info\n",),)],
)
def test_run_scheduler_main_stays_alive_on_startup_refresh_failure(self):
fake_ads = Mock()
with patch.object(scheduler, "ads", fake_ads):
with patch.object(scheduler, "ensure_bus_ready"):
with patch.object(
scheduler,
"importAndShapeFullReload",
side_effect=RuntimeError("runtime contract failed"),
):
with patch.object(scheduler, "scheduler_error") as mock_scheduler_error:
with patch.object(scheduler, "publish_scheduler_progress") as mock_progress:
with patch.object(scheduler.atexit, "register"):
with patch.object(scheduler, "not_dead_yet"):
with patch("traceback.print_exc"):
with patch("builtins.print"):
scheduler.run_scheduler_main()
self.assertEqual(scheduler.shaping_runtime_hash, 0)
mock_scheduler_error.assert_called_once_with(
"Scheduler startup shaping refresh failed: runtime contract failed"
)
self.assertTrue(
any(
call.args[:3] == (
False,
"degraded",
"Scheduler running with topology/runtime error",
)
for call in mock_progress.call_args_list
)
)
self.assertEqual(fake_ads.add_job.call_count, 3)
fake_ads.start.assert_called_once()
def test_topology_runtime_refresh_tick_reports_refresh_failure(self):
with patch.object(scheduler, "ensure_topology_runtime_process"):
with patch.object(scheduler, "calculate_shaping_runtime_hash", return_value=5):
with patch.object(scheduler, "refreshShapers", side_effect=RuntimeError("bad runtime")):
with patch.object(scheduler, "scheduler_error") as mock_scheduler_error:
with patch("builtins.print"):
scheduler.shaping_runtime_hash = 0
scheduler.topology_runtime_refresh_tick()
mock_scheduler_error.assert_called_once_with(
"Topology runtime refresh failed: bad runtime"
)
class TestSchedulerOverrideMerge(unittest.TestCase):
def test_merge_rows_replaces_matching_device_id(self):
@@ -292,6 +341,108 @@ class TestSchedulerOverrideMerge(unittest.TestCase):
self.assertEqual(written_rows[0][11], "330")
self.assertEqual(written_rows[0][13], "fq_codel/fq_codel")
def test_apply_lqos_overrides_reparent_clears_parent_node_id_when_present(self):
header = [
"Circuit ID", "Circuit Name", "Device ID", "Device Name", "Parent Node",
"Parent Node ID", "MAC", "IPv4", "IPv6", "Download Min Mbps",
"Upload Min Mbps", "Download Max Mbps", "Upload Max Mbps", "Comment",
]
rows = [[
"93", "Name", "splynx_service_93", "Name", "AP",
"uisp:device:ap-1", "MAC", "1.1.1.1", "",
"1", "1", "330", "330", "",
]]
with patch.object(scheduler, "shaped_devices_csv_path", return_value="/tmp/ShapedDevices.csv"): # nosec B108
with patch.object(scheduler, "read_shaped_devices_csv", return_value=(header, rows)):
with patch.object(scheduler, "overrides_persistent_devices_materialized", return_value=[]):
with patch.object(
scheduler,
"overrides_circuit_adjustments_materialized",
return_value=[{
"type": "reparent_circuit",
"circuit_id": "93",
"parent_node": "AP-Updated",
}],
):
with patch.object(scheduler, "write_shaped_devices_csv") as mock_write:
scheduler.apply_lqos_overrides()
written_rows = mock_write.call_args.args[2]
self.assertEqual(written_rows[0][4], "AP-Updated")
self.assertEqual(written_rows[0][5], "")
def test_override_devices_to_rows_preserves_anchor_node_id(self):
header = [
"Circuit ID", "Circuit Name", "Device ID", "Device Name", "Parent Node",
"Parent Node ID", "Anchor Node ID", "MAC", "IPv4", "IPv6",
"Download Min Mbps", "Upload Min Mbps", "Download Max Mbps", "Upload Max Mbps",
"Comment", "SQM",
]
rows = scheduler.override_devices_to_rows(
[{
"circuitID": "93",
"circuitName": "Name",
"deviceID": "device-93",
"deviceName": "Name",
"ParentNode": "AP",
"ParentNodeID": "uisp:device:ap-1",
"AnchorNodeID": "uisp:site:site-93",
"mac": "MAC",
"ipv4s": ["1.1.1.1"],
"ipv6s": [],
"minDownload": 1,
"minUpload": 1,
"maxDownload": 330,
"maxUpload": 330,
"comment": "",
"sqm": "fq_codel/fq_codel",
}],
header,
include_sqm=True,
)
self.assertEqual(rows[0][6], "uisp:site:site-93")
self.assertEqual(rows[0][15], "fq_codel/fq_codel")
def test_override_devices_to_rows_preserves_diy_id_header_alias(self):
header = [
"Circuit ID", "Circuit Name", "Device ID", "Device Name", "Parent Node",
"Parent Node ID", "id", "MAC", "IPv4", "IPv6",
"Download Min Mbps", "Upload Min Mbps", "Download Max Mbps", "Upload Max Mbps",
"Comment",
]
rows = scheduler.override_devices_to_rows(
[{
"circuitID": "93",
"circuitName": "Name",
"deviceID": "device-93",
"deviceName": "Name",
"ParentNode": "AP",
"ParentNodeID": "uisp:device:ap-1",
"AnchorNodeID": "uisp:site:site-93",
"mac": "MAC",
"ipv4s": ["1.1.1.1"],
"ipv6s": [],
"minDownload": 1,
"minUpload": 1,
"maxDownload": 330,
"maxUpload": 330,
"comment": "",
}],
header,
include_sqm=False,
)
self.assertEqual(rows[0][6], "uisp:site:site-93")
def test_topology_runtime_output_paths_include_shaping_inputs(self):
# Test-only fake install root.
with patch.object(scheduler, "get_libreqos_directory", return_value="/tmp/libreqos"): # nosec B108
paths = scheduler.topology_runtime_output_paths()
self.assertIn("/tmp/libreqos/shaping_inputs.json", paths) # nosec B108
def test_apply_network_adjustments_uses_materialized_adjustments(self):
network = {
"Root": {
src/test_virtual_tree_nodes.py
+21
View File
@@ -4,6 +4,7 @@ from virtual_tree_nodes import (
build_logical_to_physical_node_map,
build_physical_network,
collect_physical_parent_node_aliases,
collect_physical_parent_node_ids,
is_virtual_node,
)
@@ -94,6 +95,26 @@ class TestVirtualTreeNodes(unittest.TestCase):
self.assertEqual(aliases["Thunderhill Core"], "Thunderhill Core")
self.assertEqual(aliases["Thunderhill-Hoodoo"], "Thunderhill Core")
def test_collect_physical_parent_node_ids_includes_network_json_id(self):
physical = {
"Hoodoo Hill": {
"children": {
"Thunderhill Core": {
"id": "uisp:device:245a86b7-203d-41cf-9c57-e3ff82079d44",
"name": "Thunderhill Core",
"children": {},
}
}
}
}
node_ids = collect_physical_parent_node_ids(physical)
self.assertEqual(
node_ids["uisp:device:245a86b7-203d-41cf-9c57-e3ff82079d44"],
"Thunderhill Core",
)
if __name__ == "__main__":
unittest.main()
src/virtual_tree_nodes.py
+32
View File
@@ -125,3 +125,35 @@ def collect_physical_parent_node_aliases(network):
recurse(network)
return aliases
def collect_physical_parent_node_ids(network):
"""
Returns a mapping of `network.json` node IDs to the physical node key that
exists in the shaping tree.
This lets shaping-time code resolve circuits by stable node identifier
instead of relying only on display names.
"""
node_ids = {}
def recurse(level):
if not isinstance(level, dict):
return
for key, node in level.items():
if not isinstance(key, str) or not isinstance(node, dict):
continue
node_id = node.get("id")
if node_id is not None:
node_id = str(node_id).strip()
if node_id:
node_ids.setdefault(node_id, key)
children = node.get("children")
if isinstance(children, dict):
recurse(children)
recurse(network)
return node_ids