IntenseWebs/LibreQoS
3
0 Fork 0
mirror of https://github.com/LibreQoE/LibreQoS.git synced 2026-10-25 22:30:05 -05:00
Code Issues Packages Projects Releases Wiki Activity
Files
691ff7dda4c1477201d85f22eda527338c3b5663
LibreQoS/src/LibreQoS.py
T
Name 691ff7dda4 changes
2026-04-29 13:58:19 -06:00

3031 lines
147 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/python3
from pythonCheck import checkPythonVersion
checkPythonVersion()
import csv
import io
import chardet
import ipaddress
import json
import math
import os
import os.path
import subprocess
from subprocess import PIPE, STDOUT
from datetime import datetime, timedelta
import multiprocessing
import warnings
import psutil
import argparse
import logging
import shutil
import time
from deepdiff import DeepDiff
from virtual_tree_nodes import (
build_logical_to_physical_node_map,
build_physical_network,
collect_physical_parent_node_aliases,
collect_physical_parent_node_ids,
is_virtual_node,
)
from shaping_skip_report import (
build_unshaped_device_report,
collect_parent_node_names,
device_shaping_key,
format_unshaped_device_line,
)
from liblqos_python import is_lqosd_alive, clear_ip_mappings, delete_ip_mapping, validate_shaped_devices, \
is_libre_already_running, create_lock_file, free_lock_file, add_ip_mapping, BatchedCommands, \
check_config, sqm, upstream_bandwidth_capacity_download_mbps, upstream_bandwidth_capacity_upload_mbps, \
interface_a, interface_b, enable_actual_shell_commands, use_bin_packing_to_balance_cpu, queue_mode, \
run_shell_commands_as_sudo, generated_pn_download_mbps, generated_pn_upload_mbps, queues_available_override, \
on_a_stick, get_tree_weights, get_weights, is_network_flat, get_libreqos_directory, enable_insight_topology, \
is_insight_enabled, scheduler_error, xdp_ip_mapping_capacity, \
overrides_circuit_adjustments_effective, \
automatic_import_uisp, automatic_import_splynx, automatic_import_powercode, automatic_import_sonar, \
automatic_import_wispgate, automatic_import_netzur, automatic_import_visp, \
plan_top_level_cpu_bins, topology_import_ingress_enabled, \
plan_class_identities, \
fast_queues_fq_codel, \
shaping_cpu_count, \
Bakery
# Optional: urgent issue submission (available in newer liblqos_python)
try:
from liblqos_python import submit_urgent_issue # type: ignore
except Exception:
def submit_urgent_issue(*_args, **_kwargs):
return False
try:
from liblqos_python import calculate_topology_source_generation # type: ignore
except Exception:
def calculate_topology_source_generation():
return None
try:
from liblqos_python import get_libreqos_state_directory as _get_state_dir_native
except Exception:
_get_state_dir_native = None
class RefreshFailure(Exception):
pass
class ValidationFailure(RefreshFailure):
pass
def report_refresh_failure(code, message, context=None, dedupe_key=None):
logging.error(message)
print("ERROR: " + message)
try:
scheduler_error(message)
except Exception:
pass
try:
submit_urgent_issue(
"LibreQoS",
"Error",
code,
message,
json.dumps(context) if context is not None else None,
dedupe_key,
)
except Exception:
pass
raise RefreshFailure(message)
R2Q = 10
#MAX_R2Q = 200_000
MAX_R2Q = 60_000 # See https://lartc.vger.kernel.narkive.com/NKaH1ZNG/htb-quantum-of-class-100001-is-small-consider-r2q-change
MIN_QUANTUM = 1522
# Gap after each node's circuits for future additions
# Can be overridden by setting CIRCUIT_PADDING in ispConfig.py
# Setting to 0 disables padding (not recommended for production)
# Higher values provide more room for growth but reduce total capacity
try:
from ispConfig import CIRCUIT_PADDING
except ImportError:
CIRCUIT_PADDING = 8 # Default value if not configured
def get_shaped_devices_path():
base_dir = get_libreqos_directory()
if enable_insight_topology():
insight_path = os.path.join(base_dir, "ShapedDevices.insight.csv")
if os.path.exists(insight_path):
return insight_path
# Either insight not enabled, or file doesn't exist
return os.path.join(base_dir, "ShapedDevices.csv")
def get_state_directory():
if _get_state_dir_native is not None:
return _get_state_dir_native()
base_dir = get_libreqos_directory()
if os.path.basename(base_dir.rstrip("/")) == "src":
parent = os.path.dirname(base_dir.rstrip("/"))
if parent:
return os.path.join(parent, "state")
return os.path.join(base_dir, "state")
def get_state_path(category, filename):
return os.path.join(get_state_directory(), category, filename)
def get_runtime_state_path(category, filename):
return get_state_path(category, filename)
def ensure_parent_dir(path):
parent = os.path.dirname(path)
if parent:
os.makedirs(parent, exist_ok=True)
def get_network_json_path():
base_dir = get_libreqos_directory()
effective_path = get_runtime_state_path("topology", "network.effective.json")
if topology_import_ingress_enabled():
return effective_path
if enable_insight_topology():
insight_path = os.path.join(base_dir, "network.insight.json")
if os.path.exists(insight_path):
return insight_path
# Either insight not enabled, or file doesn't exist
return os.path.join(base_dir, "network.json")
def get_shaping_inputs_path():
return get_runtime_state_path("shaping", "shaping_inputs.json")
def get_circuit_anchors_path():
return get_runtime_state_path("topology", "circuit_anchors.json")
def get_planner_state_path():
return get_runtime_state_path("shaping", "planner_state.json")
def get_topology_runtime_status_path():
return get_runtime_state_path("topology", "topology_runtime_status.json")
def get_queuing_structure_path():
return get_runtime_state_path("shaping", "queuingStructure.json")
def get_last_run_path():
return get_runtime_state_path("stats", "lastRun.txt")
def get_last_good_config_json_path():
return get_runtime_state_path("shaping", "lastGoodConfig.json")
def get_last_good_config_csv_path():
return get_runtime_state_path("shaping", "lastGoodConfig.csv")
def get_last_loaded_shaped_devices_path():
return get_runtime_state_path("shaping", "ShapedDevices.lastLoaded.csv")
def get_stats_by_circuit_path():
return get_runtime_state_path("stats", "statsByCircuit.json")
def get_stats_by_parent_node_path():
return get_runtime_state_path("stats", "statsByParentNode.json")
def get_linux_tc_path():
return get_state_path("debug", "linux_tc.txt")
def observe_mode_enabled():
return queue_mode() == "observe"
def _load_json_dict(path):
try:
with open(path, "r") as infile:
data = json.load(infile)
if isinstance(data, dict):
return data
except Exception:
pass
return {}
def loaded_network_is_flat(network):
"""Evaluate flat-network mode from one in-memory topology snapshot.
refreshShapers() should make all topology decisions from the same loaded
network tree. Re-reading the preferred network path mid-run can observe a
different runtime-effective snapshot and produce internally inconsistent
queue builds.
"""
return not isinstance(network, dict) or len(network) == 0
def _normalize_shaped_devices_header(header_value):
return ''.join(ch for ch in str(header_value).lower() if ch.isalnum())
_SHAPED_DEVICES_LEGACY_LAYOUT = {
'circuitID': 0,
'circuitName': 1,
'deviceID': 2,
'deviceName': 3,
'ParentNode': 4,
'mac': 5,
'ipv4_input': 6,
'ipv6_input': 7,
'downloadMin': 8,
'uploadMin': 9,
'downloadMax': 10,
'uploadMax': 11,
'comment': 12,
'sqm': 13,
}
_SHAPED_DEVICES_HEADER_ALIASES = {
'circuitID': {'circuitid'},
'circuitName': {'circuitname'},
'deviceID': {'deviceid'},
'deviceName': {'devicename'},
'ParentNode': {'parentnode'},
'ParentNodeID': {'parentnodeid'},
'AnchorNodeID': {'anchornodeid', 'id'},
'mac': {'mac'},
'ipv4_input': {'ipv4'},
'ipv6_input': {'ipv6'},
'downloadMin': {'downloadmin', 'downloadminmbps'},
'uploadMin': {'uploadmin', 'uploadminmbps'},
'downloadMax': {'downloadmax', 'downloadmaxmbps'},
'uploadMax': {'uploadmax', 'uploadmaxmbps'},
'comment': {'comment'},
'sqm': {'sqm'},
}
def _build_shaped_devices_layout(header_row):
layout = dict(_SHAPED_DEVICES_LEGACY_LAYOUT)
layout['ParentNodeID'] = None
layout['AnchorNodeID'] = None
for idx, header in enumerate(header_row):
normalized = _normalize_shaped_devices_header(header)
for field, aliases in _SHAPED_DEVICES_HEADER_ALIASES.items():
if normalized in aliases:
layout[field] = idx
break
return layout
def _shaped_devices_row_value(row, layout, field):
idx = layout.get(field)
if idx is None or idx >= len(row):
return ''
return row[idx]
def _resolve_effective_parent_node(circuit, parent_node_ids, parent_node_aliases):
for candidate_id in (
circuit.get('ParentNodeID', ''),
circuit.get('effectiveAttachmentID', ''),
):
parent_node_id = str(candidate_id or '').strip()
if not parent_node_id:
continue
resolved_parent = parent_node_ids.get(parent_node_id)
if resolved_parent:
return resolved_parent, parent_node_id
resolved_parent_id = ''
for candidate_name in (
circuit.get('ParentNode', ''),
circuit.get('effectiveAttachmentName', ''),
circuit.get('logicalParentNode', ''),
):
parent_node = str(candidate_name or '').strip()
if not parent_node or parent_node == 'none':
continue
resolved_parent = parent_node_aliases.get(parent_node)
if not resolved_parent:
continue
for node_id, node_name in parent_node_ids.items():
if node_name == resolved_parent:
resolved_parent_id = node_id
break
return resolved_parent, resolved_parent_id
parent_node = str(circuit.get('ParentNode', '') or '').strip()
parent_node_id = str(circuit.get('ParentNodeID', '') or '').strip()
if not parent_node or parent_node == 'none':
return parent_node, ''
return parent_node, parent_node_id
def _attachment_lookup_candidates(node_key, node_data):
candidates = []
node_id = str(node_data.get('id', '') or '').strip()
node_name = str(node_data.get('name', '') or '').strip()
for candidate in (node_id, str(node_key).strip(), node_name):
if candidate and candidate not in candidates:
candidates.append(candidate)
return candidates
def _circuit_attachment_name_candidates(circuit):
candidates = []
for candidate_name in (
circuit.get('ParentNode', ''),
circuit.get('effectiveParentNodeName', ''),
):
candidate = str(candidate_name or '').strip()
if candidate and candidate != 'none' and candidate not in candidates:
candidates.append(candidate)
if circuit.get('parentResolvedByShapingInputs') is not True:
logical_parent = str(circuit.get('logicalParentNode', '') or '').strip()
if logical_parent and logical_parent != 'none' and logical_parent not in candidates:
candidates.append(logical_parent)
return candidates
def _normalize_circuit_shaping_parent(circuit, parent_node_ids=None):
parent_name = str(circuit.get('ParentNode', '') or '').strip()
effective_name = str(circuit.get('effectiveParentNodeName', '') or '').strip()
parent_id = str(circuit.get('effectiveParentNodeID', '') or circuit.get('ParentNodeID', '') or '').strip()
resolved_id_parent = ''
if parent_node_ids and parent_id:
resolved_id_parent = str(parent_node_ids.get(parent_id, '') or '').strip()
if not parent_name:
parent_name = effective_name
if (not parent_name or parent_name == 'none') and resolved_id_parent:
parent_name = resolved_id_parent
if not parent_name:
parent_name = 'none'
if is_generated_parent_node_name(parent_name) or (
resolved_id_parent and resolved_id_parent != parent_name
):
parent_id = ''
circuit['shapingParentNode'] = parent_name
circuit['shapingParentNodeID'] = parent_id
circuit['shapingParentKey'] = f"id:{parent_id}" if parent_id else f"name:{parent_name}"
return parent_name, parent_id
def _validate_planned_circuit_attachment(node_name, node_data, circuit, planned_identity):
site_major = _parse_int_token(node_data.get('classMajor'))
site_up_major = _parse_int_token(node_data.get('up_classMajor'))
planned_major = _parse_int_token(planned_identity.get('class_major'))
planned_up_major = _parse_int_token(planned_identity.get('up_class_major'))
if None in (site_major, site_up_major, planned_major, planned_up_major):
raise ValueError(
f"Incomplete class identity while attaching circuit '{circuit.get('circuitID', '')}' to parent '{node_name}'"
)
if site_major != planned_major or site_up_major != planned_up_major:
raise ValueError(
"Planned circuit class majors do not match the selected parent node: "
f"circuit='{circuit.get('circuitID', '')}', parent='{node_name}', "
f"parentMajor={hex(site_major)}, parentUpMajor={hex(site_up_major)}, "
f"plannedMajor={hex(planned_major)}, plannedUpMajor={hex(planned_up_major)}, "
f"ParentNode='{circuit.get('ParentNode', '')}', "
f"logicalParentNode='{circuit.get('logicalParentNode', '')}', "
f"resolutionSource='{circuit.get('parentResolutionSource', '')}'"
)
return planned_major, planned_up_major
def _current_topology_source_generation():
try:
generation = calculate_topology_source_generation()
except Exception:
return None
if generation is None:
return None
generation = str(generation).strip()
return generation if generation != '' else None
def _topology_runtime_status_supports_shaping_inputs(shaping_inputs_path):
current_generation = _current_topology_source_generation()
if current_generation is None:
return None
status_path = get_topology_runtime_status_path()
try:
with open(status_path, 'r', encoding='utf-8') as handle:
status = json.load(handle)
except FileNotFoundError:
return None
except Exception:
return None
if not isinstance(status, dict):
return False
status_generation = str(status.get('source_generation', '') or '').strip()
if status_generation != current_generation:
return False
if not bool(status.get('ready')):
return False
status_shaping_generation = str(status.get('shaping_generation', '') or '').strip()
if status_shaping_generation == '':
return False
status_shaping_inputs_path = str(status.get('shaping_inputs_path', '') or '').strip()
if status_shaping_inputs_path != '':
try:
if os.path.abspath(status_shaping_inputs_path) != os.path.abspath(shaping_inputs_path):
return False
except Exception:
return False
if not os.path.isfile(status_shaping_inputs_path):
return False
return True
def _shaping_inputs_are_fresh(shaping_inputs_path, shaped_devices_file, network_json_file, circuit_anchors_file=None):
if not os.path.isfile(shaping_inputs_path):
return False
topology_runtime_ready = _topology_runtime_status_supports_shaping_inputs(shaping_inputs_path)
if topology_runtime_ready is True:
return True
if topology_runtime_ready is False:
return False
try:
shaping_inputs_mtime = os.path.getmtime(shaping_inputs_path)
if os.path.isfile(shaped_devices_file) and shaping_inputs_mtime < os.path.getmtime(shaped_devices_file):
return False
if os.path.isfile(network_json_file) and shaping_inputs_mtime < os.path.getmtime(network_json_file):
return False
if circuit_anchors_file and os.path.isfile(circuit_anchors_file) and shaping_inputs_mtime < os.path.getmtime(circuit_anchors_file):
return False
return True
except OSError:
return False
def loadSubscriberCircuitsFromShapingInputs(shapingInputsPath):
payload = _load_json_dict(shapingInputsPath)
circuits = payload.get('circuits', [])
if not isinstance(circuits, list):
raise ValueError(f"Invalid shaping_inputs.json at {shapingInputsPath}: circuits must be a list")
subscriberCircuits = []
dictForCircuitsWithoutParentNodes = {}
counterForCircuitsWithoutParentNodes = 0
warnings_from_file = payload.get('warnings', [])
if isinstance(warnings_from_file, list):
for warning_text in warnings_from_file:
if isinstance(warning_text, str) and warning_text.strip():
warnings.warn(f"shaping_inputs.json: {warning_text}", stacklevel=2)
for circuit in circuits:
if not isinstance(circuit, dict):
continue
circuitID = str(circuit.get('circuit_id', '') or '').strip()
if circuitID == '':
raise ValueError("Missing circuit_id in shaping_inputs.json")
parent_node = str(circuit.get('effective_parent_node_name', '') or '').strip()
parent_node_id = str(circuit.get('effective_parent_node_id', '') or '').strip()
logical_parent_node = str(circuit.get('logical_parent_node_name', '') or '').strip()
logical_parent_node_id = str(circuit.get('logical_parent_node_id', '') or '').strip()
anchor_node_id = str(circuit.get('anchor_node_id', '') or '').strip()
devices = []
for device in circuit.get('devices', []):
if not isinstance(device, dict):
continue
devices.append(
{
"deviceID": str(device.get('device_id', '') or '').strip(),
"deviceName": str(device.get('device_name', '') or '').strip(),
"mac": str(device.get('mac', '') or '').strip(),
"ipv4s": [str(entry).strip() for entry in device.get('ipv4', []) if str(entry).strip() != ''],
"ipv6s": [str(entry).strip() for entry in device.get('ipv6', []) if str(entry).strip() != ''],
"comment": str(device.get('comment', '') or ''),
}
)
thisCircuit = {
"circuitID": circuitID,
"circuitName": str(circuit.get('circuit_name', '') or ''),
"ParentNode": parent_node if parent_node != '' else 'none',
"effectiveParentNodeName": parent_node,
"ParentNodeID": parent_node_id,
"AnchorNodeID": anchor_node_id,
"devices": devices,
"minDownload": float(circuit.get('download_min_mbps', 0.0) or 0.0),
"minUpload": float(circuit.get('upload_min_mbps', 0.0) or 0.0),
"maxDownload": float(circuit.get('download_max_mbps', 0.0) or 0.0),
"maxUpload": float(circuit.get('upload_max_mbps', 0.0) or 0.0),
"classid": '',
"comment": str(circuit.get('comment', '') or ''),
"logicalParentNode": logical_parent_node if logical_parent_node != '' else (parent_node if parent_node != '' else 'none'),
"logicalParentNodeID": logical_parent_node_id,
"effectiveAttachmentID": str(circuit.get('effective_attachment_id', '') or '').strip(),
"effectiveAttachmentName": str(circuit.get('effective_attachment_name', '') or '').strip(),
"parentResolutionSource": str(circuit.get('resolution_source', '') or '').strip(),
"parentResolvedByShapingInputs": True,
}
sqm_override = normalize_sqm_override_token(str(circuit.get('sqm_override', '') or ''))
if sqm_override != '':
thisCircuit['sqm'] = sqm_override
if thisCircuit['ParentNode'] == 'none':
thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes
dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = (
thisCircuit['maxDownload'] + thisCircuit['maxUpload']
)
counterForCircuitsWithoutParentNodes += 1
subscriberCircuits.append(thisCircuit)
return subscriberCircuits, dictForCircuitsWithoutParentNodes
def loadSubscriberCircuitsForShaping(shapedDevicesFile, networkJSONfile):
shaping_inputs_path = get_shaping_inputs_path()
circuit_anchors_path = get_circuit_anchors_path()
if _shaping_inputs_are_fresh(shaping_inputs_path, shapedDevicesFile, networkJSONfile, circuit_anchors_path):
try:
subscriberCircuits, dictForCircuitsWithoutParentNodes = loadSubscriberCircuitsFromShapingInputs(shaping_inputs_path)
print("Loaded shaping inputs from " + shaping_inputs_path)
return subscriberCircuits, dictForCircuitsWithoutParentNodes
except Exception as e:
raise RefreshFailure(
f"Unable to load required shaping_inputs.json at {shaping_inputs_path}: {e}"
) from e
raise RefreshFailure(
"Missing or stale shaping_inputs.json. Run topology runtime before shaping."
)
def load_planner_state(state_path=None, planner_module=None):
if state_path is None:
state_path = get_planner_state_path()
if planner_module is not None:
try:
state = planner_module.load_state(state_path)
if isinstance(state, dict):
return sanitize_planner_state(state)
except Exception:
pass
return sanitize_planner_state(_load_json_dict(state_path))
def save_planner_state(state, state_path=None, planner_module=None):
if state_path is None:
state_path = get_planner_state_path()
if planner_module is not None:
planner_module.save_state(state_path, state)
return
parent = os.path.dirname(state_path)
if parent:
os.makedirs(parent, exist_ok=True)
temp_path = state_path + ".tmp"
with open(temp_path, "w") as outfile:
json.dump(state, outfile, indent=2, sort_keys=True)
os.replace(temp_path, state_path)
def sanitize_planner_state(state):
if not isinstance(state, dict):
return {}
sanitized = False
def _sanitize_entry(entry):
nonlocal sanitized
if not isinstance(entry, dict):
return
raw_minor = None
for key in ('class_minor', 'minor', 'classMinor'):
if key in entry:
raw_minor = _parse_int_token(entry.get(key))
if raw_minor is not None:
break
# TC minor 0xffff is not safe for leaf classes here. If stale planner
# state reuses it, Bakery later emits qdisc parents like 1:ffff and the
# kernel rejects them. Drop the stored identity and let the planner
# allocate a new minor on this run.
if raw_minor is not None and (raw_minor < 3 or raw_minor >= 0xFFFF):
for key in ('class_minor', 'minor', 'classMinor', 'class_major', 'up_class_major'):
if key in entry:
del entry[key]
sanitized = True
for value in entry.values():
if isinstance(value, dict):
_sanitize_entry(value)
elif isinstance(value, list):
for item in value:
if isinstance(item, dict):
_sanitize_entry(item)
state_copy = json.loads(json.dumps(state))
for section in ('sites', 'circuits'):
section_value = state_copy.get(section)
if isinstance(section_value, dict):
_sanitize_entry(section_value)
if sanitized:
logging.warning(
"Sanitized invalid planner state identities; removed reserved or out-of-range minor assignments before planning."
)
return state_copy
def _parse_int_token(value):
if value is None:
return None
try:
if isinstance(value, int):
return value
token = str(value).strip()
if not token:
return None
if token.lower().startswith("0x"):
return int(token, 16)
return int(token)
except Exception:
return None
def is_generated_parent_node_name(node_name):
return isinstance(node_name, str) and node_name.startswith("Generated_PN_")
def generated_parent_node_queue_key(node_name, queues_available):
if not is_generated_parent_node_name(node_name) or queues_available <= 0:
return None
suffix = _parse_int_token(str(node_name).rsplit("_", 1)[-1])
if suffix is None or suffix <= 0:
return None
return "CpueQueue" + str((suffix - 1) % queues_available)
def planner_circuit_identity_key(circuit):
circuit_id = str(circuit.get("circuitID", "") or "").strip()
if not circuit_id:
raise ValueError("Missing circuitID is unsupported for planner identity")
return circuit_id
def calculateR2q(maxRateInMbps):
# So we've learned that r2q defaults to 10, and is used to calculate quantum. Quantum is rateInBytes/r2q by
# default. This default gives errors at high rates, and tc clamps the quantum to 200000. Setting a high quantum
# directly gives no errors. So we want to calculate r2q to default to 10, but not exceed 200000 for the highest
# specified rate (which will be the available bandwidth rate).
maxRateInBytesPerSecond = maxRateInMbps * 125000
r2q = 10
quantum = maxRateInBytesPerSecond / r2q
while quantum > MAX_R2Q:
r2q += 1
quantum = maxRateInBytesPerSecond / r2q
global R2Q
R2Q = r2q
def quantum(rateInMbps):
# Attempt to calculate an appropriate quantum for an HTB queue, given
# that `mq` does not appear to carry a valid `r2q` value to individual
# root nodes.
rateInBytesPerSecond = rateInMbps * 125000
quantum = max(MIN_QUANTUM, int(rateInBytesPerSecond / R2Q))
#print("R2Q=" + str(R2Q) + ", quantum: " + str(quantum))
quantrumString = " quantum " + str(quantum)
return quantrumString
def format_rate_for_tc(rate_mbps):
"""
Format a rate in Mbps for TC commands with smart unit selection.
- Rates >= 1000 Mbps use 'gbit'
- Rates >= 1 Mbps use 'mbit'
- Rates < 1 Mbps use 'kbit'
"""
if rate_mbps >= 1000:
return f"{rate_mbps/1000:.1f}gbit"
elif rate_mbps >= 1:
return f"{rate_mbps:.1f}mbit"
else:
return f"{rate_mbps*1000:.0f}kbit"
def shell(command):
if enable_actual_shell_commands():
if run_shell_commands_as_sudo():
command = 'sudo ' + command
logging.info(command)
commands = command.split(' ')
proc = subprocess.Popen(commands, stdout=subprocess.PIPE)
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding
if logging.DEBUG <= logging.root.level:
print(line)
if ("RTNETLINK answers" in line) or ("We have an error talking to the kernel" in line):
warnings.warn("Command: '" + command + "' resulted in " + line, stacklevel=2)
else:
logging.info(command)
def shellReturn(command):
returnableString = ''
if enable_actual_shell_commands():
commands = command.split(' ')
proc = subprocess.Popen(commands, stdout=subprocess.PIPE)
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding
returnableString = returnableString + line + '\n'
return returnableString
def checkIfFirstRunSinceBoot():
last_run_path = get_last_run_path()
if os.path.isfile(last_run_path):
with open(last_run_path, 'r') as file:
lastRun = datetime.strptime(file.read(), "%d-%b-%Y (%H:%M:%S.%f)")
systemRunningSince = datetime.fromtimestamp(psutil.boot_time())
if systemRunningSince > lastRun:
print("First time run since system boot.")
return True
else:
print("Not first time run since system boot.")
return False
else:
print("First time run since system boot.")
return True
def clearPriorSettings(interfaceA, interfaceB):
if enable_actual_shell_commands():
if 'mq' in shellReturn('tc qdisc show dev ' + interfaceA + ' root'):
print('MQ detected. Will delete and recreate mq qdisc.')
# Clear tc filter
if on_a_stick() == True:
shell('tc qdisc delete dev ' + interfaceA + ' root')
else:
shell('tc qdisc delete dev ' + interfaceA + ' root')
shell('tc qdisc delete dev ' + interfaceB + ' root')
def tearDown(interfaceA, interfaceB):
# Full teardown of everything for exiting LibreQoS
if enable_actual_shell_commands():
# Clear IP filters and remove xdp program from interfaces
# The bakery tracks and prunes mappings; avoid clearing everything here.
clearPriorSettings(interfaceA, interfaceB)
def findQueuesAvailable(interfaceName):
# Find queues and CPU cores available. Use min between those two as queuesAvailable
if enable_actual_shell_commands():
if queues_available_override() == 0:
queuesAvailable = 0
path = '/sys/class/net/' + interfaceName + '/queues/'
directory_contents = os.listdir(path)
for item in directory_contents:
if "tx-" in str(item):
queuesAvailable += 1
print(f"Interface {interfaceName} NIC queues:\t\t\t" + str(queuesAvailable))
else:
queuesAvailable = queues_available_override()
print(f"Interface {interfaceName} NIC queues (Override):\t\t\t" + str(queuesAvailable))
try:
cpuCount = shaping_cpu_count()
except Exception:
cpuCount = multiprocessing.cpu_count()
print("CPU cores:\t\t\t" + str(cpuCount))
if queuesAvailable < 2:
raise SystemError(f'Only 1 NIC rx/tx queue available for interface {interfaceName}. You will need to use a NIC with 2 or more rx/tx queues available.')
if cpuCount < 2:
raise SystemError('Only 1 CPU core available. You will need to use a CPU with 2 or more CPU cores.')
queuesAvailable = min(queuesAvailable,cpuCount)
print(f"queuesAvailable for interface {interfaceName} set to:\t" + str(queuesAvailable))
else:
print("As enableActualShellCommands is False, CPU core / queue count has been set to 16")
logging.info(f"Interface {interfaceName} NIC queues:\t\t\t" + str(16))
cpuCount = multiprocessing.cpu_count()
logging.info("CPU cores:\t\t\t" + str(16))
logging.info(f"queuesAvailable for interface {interfaceName} set to:\t" + str(16))
queuesAvailable = 16
return queuesAvailable
def validateNetworkAndDevices():
# Verify Network.json is valid json
networkValidatedOrNot = True
# Verify ShapedDevices.csv is valid
devicesValidatedOrNot = True # True by default, switches to false if ANY entry in ShapedDevices.csv fails validation
integration_ingress = topology_import_ingress_enabled()
# Verify that the Rust side of things can read the active shaped-device ingress
rustValid = validate_shaped_devices()
if rustValid == "OK":
if integration_ingress:
print("Rust validated integration shaping ingress")
else:
print("Rust validated ShapedDevices.csv")
else:
if integration_ingress:
warnings.warn("Rust failed to validate integration shaping ingress", stacklevel=2)
else:
warnings.warn("Rust failed to validate ShapedDevices.csv", stacklevel=2)
warnings.warn(rustValid, stacklevel=2)
devicesValidatedOrNot = False
network_json_path = get_network_json_path()
if not os.path.isfile(network_json_path):
if integration_ingress:
warnings.warn(
f"Integration topology has not published an effective network tree yet: {network_json_path}",
stacklevel=2,
)
else:
warnings.warn(f"network.json is missing: {network_json_path}", stacklevel=2)
networkValidatedOrNot = False
else:
with open(network_json_path) as file:
try:
data = json.load(file) # put JSON-data to a variable
if data != {}:
#Traverse
observedNodes = set()
duplicateNodes = set()
def traverseToVerifyValidity(data):
for elem in data:
if isinstance(elem, str):
if (isinstance(data[elem], dict)) and (elem != 'children'):
if elem not in observedNodes:
observedNodes.add(elem)
if 'children' in data[elem]:
traverseToVerifyValidity(data[elem]['children'])
else:
duplicateNodes.add(elem)
traverseToVerifyValidity(data)
if len(duplicateNodes) > 0:
for elem in sorted(duplicateNodes):
warnings.warn("Non-unique Node name in network.json: " + elem, stacklevel=2)
networkValidatedOrNot = False
if len(observedNodes) < 1:
warnings.warn("network.json had 0 valid nodes. Only {} is accepted for that scenario.", stacklevel=2)
networkValidatedOrNot = False
except json.decoder.JSONDecodeError:
warnings.warn("network.json is an invalid JSON file", stacklevel=2) # in case json is invalid
networkValidatedOrNot = False
if integration_ingress:
if devicesValidatedOrNot == True:
print("integration shaping ingress passed validation")
else:
print("integration shaping ingress failed validation")
if networkValidatedOrNot == True:
print("network.json passed validation")
else:
print("network.json failed validation")
return devicesValidatedOrNot and networkValidatedOrNot
rowNum = 2
# Handle non-utf8 encoding in ShapedDevices.csv
with open(get_shaped_devices_path(), 'rb') as f:
raw_bytes = f.read()
# Handle BOM if present
if raw_bytes.startswith(b'\xef\xbb\xbf'): # UTF-8 BOM
raw_bytes = raw_bytes[3:]
text_content = raw_bytes.decode('utf-8')
elif raw_bytes.startswith(b'\xff\xfe'): # UTF-16 LE BOM
text_content = raw_bytes.decode('utf-16')
elif raw_bytes.startswith(b'\xfe\xff'): # UTF-16 BE BOM
text_content = raw_bytes.decode('utf-16')
else:
# Try UTF-8 first
try:
text_content = raw_bytes.decode('utf-8')
except UnicodeDecodeError:
# Detect encoding
detected = chardet.detect(raw_bytes)
encoding = detected['encoding'] or 'utf-8'
text_content = raw_bytes.decode(encoding, errors='replace')
# Create a StringIO object to mimic a file
# And read from the sanitized byte stream
with io.StringIO(text_content) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
header_consumed = False
layout = None
seenTheseIPsAlready = set()
for row in csv_reader:
if not row:
continue
if row[0].startswith('#'):
continue
if not header_consumed:
header_consumed = True
layout = _build_shaped_devices_layout(row)
continue
# Accept optional Anchor Node ID / Parent Node ID columns and trailing
# 'sqm' override columns while validating the core shaping inputs.
circuitID = _shaped_devices_row_value(row, layout, 'circuitID')
circuitName = _shaped_devices_row_value(row, layout, 'circuitName')
deviceID = _shaped_devices_row_value(row, layout, 'deviceID')
deviceName = _shaped_devices_row_value(row, layout, 'deviceName')
ParentNode = _shaped_devices_row_value(row, layout, 'ParentNode')
mac = _shaped_devices_row_value(row, layout, 'mac')
ipv4_input = _shaped_devices_row_value(row, layout, 'ipv4_input')
ipv6_input = _shaped_devices_row_value(row, layout, 'ipv6_input')
downloadMin = _shaped_devices_row_value(row, layout, 'downloadMin')
uploadMin = _shaped_devices_row_value(row, layout, 'uploadMin')
downloadMax = _shaped_devices_row_value(row, layout, 'downloadMax')
uploadMax = _shaped_devices_row_value(row, layout, 'uploadMax')
comment = _shaped_devices_row_value(row, layout, 'comment')
# Must have circuitID, it's a unique identifier required for stateful changes to queue structure
if circuitID == '':
warnings.warn("No Circuit ID provided in ShapedDevices.csv at row " + str(rowNum), stacklevel=2)
devicesValidatedOrNot = False
# Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s separated by commas. Split them up and parse each to ensure valid
ipv4_subnets_and_hosts = []
ipv6_subnets_and_hosts = []
if ipv4_input != "":
try:
ipv4_input = ipv4_input.replace(' ','')
if "," in ipv4_input:
ipv4_list = ipv4_input.split(',')
else:
ipv4_list = [ipv4_input]
for ipEntry in ipv4_list:
if ipEntry in seenTheseIPsAlready:
warnings.warn("Provided IPv4 '" + ipEntry + "' in ShapedDevices.csv at row " + str(rowNum) + " is duplicate.", stacklevel=2)
#devicesValidatedOrNot = False
seenTheseIPsAlready.add(ipEntry)
else:
if (type(ipaddress.ip_network(ipEntry)) is ipaddress.IPv4Network) or (type(ipaddress.ip_address(ipEntry)) is ipaddress.IPv4Address):
ipv4_subnets_and_hosts.extend(ipEntry)
else:
warnings.warn("Provided IPv4 '" + ipEntry + "' in ShapedDevices.csv at row " + str(rowNum) + " is not valid.", stacklevel=2)
devicesValidatedOrNot = False
seenTheseIPsAlready.add(ipEntry)
except:
warnings.warn("Provided IPv4 '" + ipv4_input + "' in ShapedDevices.csv at row " + str(rowNum) + " is not valid.", stacklevel=2)
devicesValidatedOrNot = False
if ipv6_input != "":
try:
ipv6_input = ipv6_input.replace(' ','')
if "," in ipv6_input:
ipv6_list = ipv6_input.split(',')
else:
ipv6_list = [ipv6_input]
for ipEntry in ipv6_list:
if ipEntry in seenTheseIPsAlready:
warnings.warn("Provided IPv6 '" + ipEntry + "' in ShapedDevices.csv at row " + str(rowNum) + " is duplicate.", stacklevel=2)
devicesValidatedOrNot = False
seenTheseIPsAlready.add(ipEntry)
else:
if (type(ipaddress.ip_network(ipEntry)) is ipaddress.IPv6Network) or (type(ipaddress.ip_address(ipEntry)) is ipaddress.IPv6Address):
ipv6_subnets_and_hosts.extend(ipEntry)
else:
warnings.warn("Provided IPv6 '" + ipEntry + "' in ShapedDevices.csv at row " + str(rowNum) + " is not valid.", stacklevel=2)
devicesValidatedOrNot = False
seenTheseIPsAlready.add(ipEntry)
except:
warnings.warn("Provided IPv6 '" + ipv6_input + "' in ShapedDevices.csv at row " + str(rowNum) + " is not valid.", stacklevel=2)
devicesValidatedOrNot = False
try:
a = float(downloadMin)
if a < 0.1:
warnings.warn("Provided downloadMin '" + downloadMin + "' in ShapedDevices.csv at row " + str(rowNum) + " is < 0.1 Mbps.", stacklevel=2)
devicesValidatedOrNot = False
except:
warnings.warn("Provided downloadMin '" + downloadMin + "' in ShapedDevices.csv at row " + str(rowNum) + " is not a valid number.", stacklevel=2)
devicesValidatedOrNot = False
try:
a = float(uploadMin)
if a < 0.1:
warnings.warn("Provided uploadMin '" + uploadMin + "' in ShapedDevices.csv at row " + str(rowNum) + " is < 0.1 Mbps.", stacklevel=2)
devicesValidatedOrNot = False
except:
warnings.warn("Provided uploadMin '" + uploadMin + "' in ShapedDevices.csv at row " + str(rowNum) + " is not a valid number.", stacklevel=2)
devicesValidatedOrNot = False
try:
a = float(downloadMax)
if a < 0.1:
warnings.warn("Provided downloadMax '" + downloadMax + "' in ShapedDevices.csv at row " + str(rowNum) + " is < 0.2 Mbps.", stacklevel=2)
devicesValidatedOrNot = False
except:
warnings.warn("Provided downloadMax '" + downloadMax + "' in ShapedDevices.csv at row " + str(rowNum) + " is not a valid number.", stacklevel=2)
devicesValidatedOrNot = False
try:
a = float(uploadMax)
if a < 0.1:
warnings.warn("Provided uploadMax '" + uploadMax + "' in ShapedDevices.csv at row " + str(rowNum) + " is < 0.2 Mbps.", stacklevel=2)
devicesValidatedOrNot = False
except:
warnings.warn("Provided uploadMax '" + uploadMax + "' in ShapedDevices.csv at row " + str(rowNum) + " is not a valid number.", stacklevel=2)
devicesValidatedOrNot = False
try:
if float(downloadMin) > float(downloadMax):
warnings.warn("Provided downloadMin '" + downloadMin + "' in ShapedDevices.csv at row " + str(rowNum) + " is greater than downloadMax", stacklevel=2)
devicesValidatedOrNot = False
if float(uploadMin) > float(uploadMax):
warnings.warn("Provided uploadMin '" + uploadMin + "' in ShapedDevices.csv at row " + str(rowNum) + " is greater than uploadMax", stacklevel=2)
devicesValidatedOrNot = False
except:
devicesValidatedOrNot = False
rowNum += 1
if devicesValidatedOrNot == True:
print("ShapedDevices.csv passed validation")
else:
print("ShapedDevices.csv failed validation")
if networkValidatedOrNot == True:
print("network.json passed validation")
else:
print("network.json failed validation")
if (devicesValidatedOrNot == True) and (networkValidatedOrNot == True):
return True
else:
return False
def loadSubscriberCircuits(shapedDevicesFile):
# Load Subscriber Circuits & Devices
subscriberCircuits = []
circuitsById = {}
counterForCircuitsWithoutParentNodes = 0
dictForCircuitsWithoutParentNodes = {}
with open(shapedDevicesFile) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
header_consumed = False
layout = None
for row in csv_reader:
if not row:
continue
if row[0].startswith('#'):
continue
if not header_consumed:
header_consumed = True
layout = _build_shaped_devices_layout(row)
continue
# Optional per-circuit SQM override in last column
sqm_override_token = ''
if layout is not None:
raw_token = _shaped_devices_row_value(row, layout, 'sqm')
else:
raw_token = ''
if raw_token != '':
# Normalize: lowercase, trim, collapse spaces around '/'
token = raw_token.strip().lower()
if '/' in token:
parts = token.split('/', 1)
left = parts[0].strip()
right = parts[1].strip()
token = left + '/' + right
sqm_override_token = token
circuitID = _shaped_devices_row_value(row, layout, 'circuitID')
circuitName = _shaped_devices_row_value(row, layout, 'circuitName')
deviceID = _shaped_devices_row_value(row, layout, 'deviceID')
deviceName = _shaped_devices_row_value(row, layout, 'deviceName')
ParentNode = _shaped_devices_row_value(row, layout, 'ParentNode')
ParentNodeID = _shaped_devices_row_value(row, layout, 'ParentNodeID')
AnchorNodeID = _shaped_devices_row_value(row, layout, 'AnchorNodeID')
mac = _shaped_devices_row_value(row, layout, 'mac')
ipv4_input = _shaped_devices_row_value(row, layout, 'ipv4_input')
ipv6_input = _shaped_devices_row_value(row, layout, 'ipv6_input')
downloadMin = _shaped_devices_row_value(row, layout, 'downloadMin')
uploadMin = _shaped_devices_row_value(row, layout, 'uploadMin')
downloadMax = _shaped_devices_row_value(row, layout, 'downloadMax')
uploadMax = _shaped_devices_row_value(row, layout, 'uploadMax')
comment = _shaped_devices_row_value(row, layout, 'comment')
ipv4_subnets_and_hosts = []
# Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s separated by commas. Split them up and parse each
if ipv4_input != "":
ipv4_input = ipv4_input.replace(' ','')
if "," in ipv4_input:
ipv4_list = ipv4_input.split(',')
else:
ipv4_list = [ipv4_input]
for ipEntry in ipv4_list:
ipv4_subnets_and_hosts.append(ipEntry)
ipv6_subnets_and_hosts = []
if ipv6_input != "":
ipv6_input = ipv6_input.replace(' ','')
if "," in ipv6_input:
ipv6_list = ipv6_input.split(',')
else:
ipv6_list = [ipv6_input]
for ipEntry in ipv6_list:
ipv6_subnets_and_hosts.append(ipEntry)
# If there is something in the circuit ID field
if circuitID != "":
# Seen circuit before
circuit = circuitsById.get(circuitID)
if circuit is not None:
if circuit['ParentNode'] != "none":
if circuit['ParentNode'] != ParentNode:
errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID
raise ValueError(errorMessageString)
existing_parent_node_id = str(circuit.get('ParentNodeID', '') or '').strip()
candidate_parent_node_id = str(ParentNodeID or '').strip()
if existing_parent_node_id and candidate_parent_node_id and existing_parent_node_id != candidate_parent_node_id:
errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node ID from other devices of circuit ID #" + circuitID
raise ValueError(errorMessageString)
if not existing_parent_node_id and candidate_parent_node_id:
circuit['ParentNodeID'] = candidate_parent_node_id
existing_anchor_node_id = str(circuit.get('AnchorNodeID', '') or '').strip()
candidate_anchor_node_id = str(AnchorNodeID or '').strip()
if existing_anchor_node_id and candidate_anchor_node_id and existing_anchor_node_id != candidate_anchor_node_id:
errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Anchor Node ID from other devices of circuit ID #" + circuitID
raise ValueError(errorMessageString)
if not existing_anchor_node_id and candidate_anchor_node_id:
circuit['AnchorNodeID'] = candidate_anchor_node_id
if ((circuit['minDownload'] != float(downloadMin))
or (circuit['minUpload'] != float(uploadMin))
or (circuit['maxDownload'] != float(downloadMax))
or (circuit['maxUpload'] != float(uploadMax))):
warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different bandwidth parameters than other devices on this circuit. Will instead use the bandwidth parameters defined by the first device added to its circuit.", stacklevel=2)
# If this row specifies an SQM override, but the circuit already has a different one, warn and keep the first.
if sqm_override_token != '':
if 'sqm' in circuit:
if circuit['sqm'] != sqm_override_token:
warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different SQM override than other devices on this circuit. Will instead use the SQM defined by the first device added to its circuit.", stacklevel=2)
else:
circuit['sqm'] = sqm_override_token
devicesListForCircuit = circuit['devices']
thisDevice = {
"deviceID": deviceID,
"deviceName": deviceName,
"mac": mac,
"ipv4s": ipv4_subnets_and_hosts,
"ipv6s": ipv6_subnets_and_hosts,
"comment": comment
}
devicesListForCircuit.append(thisDevice)
circuit['devices'] = devicesListForCircuit
# Have not seen circuit before
else:
if ParentNode == "":
ParentNode = "none"
#ParentNode = ParentNode.strip()
deviceListForCircuit = []
thisDevice = {
"deviceID": deviceID,
"deviceName": deviceName,
"mac": mac,
"ipv4s": ipv4_subnets_and_hosts,
"ipv6s": ipv6_subnets_and_hosts,
"comment": comment
}
deviceListForCircuit.append(thisDevice)
thisCircuit = {
"circuitID": circuitID,
"circuitName": circuitName,
"ParentNode": ParentNode,
"ParentNodeID": str(ParentNodeID or '').strip(),
"AnchorNodeID": str(AnchorNodeID or '').strip(),
"devices": deviceListForCircuit,
"minDownload": float(downloadMin),
"minUpload": float(uploadMin),
"maxDownload": float(downloadMax),
"maxUpload": float(uploadMax),
"classid": '',
"comment": comment
}
if sqm_override_token != '':
thisCircuit['sqm'] = sqm_override_token
if thisCircuit['ParentNode'] == 'none':
thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes
dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((float(downloadMax))+(float(uploadMax)))
counterForCircuitsWithoutParentNodes += 1
subscriberCircuits.append(thisCircuit)
circuitsById[circuitID] = thisCircuit
else:
raise ValueError(
"Missing circuitID is unsupported in ShapedDevices.csv "
f"(deviceID={deviceID}, deviceName={deviceName}, parent={ParentNode})"
)
return (subscriberCircuits, dictForCircuitsWithoutParentNodes)
def normalize_sqm_override_token(raw_token):
token = (raw_token or '').strip().lower()
if token == '':
return ''
if '/' in token:
left, right = token.split('/', 1)
token = left.strip() + '/' + right.strip()
return token
def apply_effective_runtime_circuit_overrides(subscriberCircuits):
"""
Overlay adaptive runtime circuit adjustments in memory without mutating
ShapedDevices.csv, so rebuilds can honor TreeGuard/StormGuard state.
"""
try:
adjustments = overrides_circuit_adjustments_effective()
except Exception as e:
warnings.warn(f"Unable to load effective runtime circuit overrides: {e}", stacklevel=2)
return 0
sqm_by_device_id = {}
for adj in adjustments:
if adj.get('type') != 'device_adjust_sqm':
continue
device_id = (adj.get('device_id') or '').strip()
sqm_override = normalize_sqm_override_token(adj.get('sqm_override'))
if device_id == '' or sqm_override == '':
continue
sqm_by_device_id[device_id] = sqm_override
if not sqm_by_device_id:
return 0
overlay_count = 0
for circuit in subscriberCircuits:
circuit_override = None
for device in circuit.get('devices', []):
override = sqm_by_device_id.get(device.get('deviceID', ''))
if override is None:
continue
if circuit_override is not None and circuit_override != override:
warnings.warn(
"Effective runtime SQM override conflict on circuit "
+ circuit.get('circuitID', 'unknown')
+ ". Will instead use the first runtime SQM override discovered.",
stacklevel=2,
)
continue
circuit_override = override
if circuit_override is None:
continue
if circuit.get('sqm') != circuit_override:
overlay_count += 1
circuit['sqm'] = circuit_override
return overlay_count
def refreshShapers():
# Starting
print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
observe_mode = observe_mode_enabled()
# Create a single batch of xdp update commands to execute together
ipMapBatch = BatchedCommands()
requiredIpMappings = 0
# Warn user if enableActualShellCommands is False, because that would mean no actual commands are executing
if enable_actual_shell_commands() == False:
warnings.warn("enableActualShellCommands is set to False. None of the commands below will actually be executed. Simulated run.", stacklevel=2)
if observe_mode:
warnings.warn(
"queue_mode is set to observe. LibreQoS will keep the root MQ and remove the shaping tree for a true baseline. "
"Switching between Observe and Shape may briefly interrupt traffic.",
stacklevel=2,
)
# Check if first run since boot
isThisFirstRunSinceBoot = checkIfFirstRunSinceBoot()
# Files
shapedDevicesFile = get_shaped_devices_path()
networkJSONfile = get_network_json_path()
# Check validation
safeToRunRefresh = False
print("Validating input files '" + shapedDevicesFile + "' and '" + networkJSONfile + "'")
if (validateNetworkAndDevices() == True):
if os.path.isfile(shapedDevicesFile):
last_good_csv_path = get_state_path("shaping", "lastGoodConfig.csv")
ensure_parent_dir(last_good_csv_path)
shutil.copyfile(shapedDevicesFile, last_good_csv_path)
last_good_json_path = get_state_path("shaping", "lastGoodConfig.json")
ensure_parent_dir(last_good_json_path)
shutil.copyfile(networkJSONfile, last_good_json_path)
if os.path.isfile(shapedDevicesFile):
print("Backed up good config as lastGoodConfig.csv and lastGoodConfig.json")
else:
print("Backed up good config as lastGoodConfig.json")
safeToRunRefresh = True
else:
if topology_import_ingress_enabled():
message = "Validation failed for integration ingress/runtime artifacts - will now exit."
warnings.warn(message, stacklevel=2)
raise ValidationFailure(message)
elif (isThisFirstRunSinceBoot == False):
message = "Validation failed. Because this is not the first run since boot (queues already set up) - will now exit."
warnings.warn(message, stacklevel=2)
raise ValidationFailure(message)
else:
warnings.warn("Validation failed. However - because this is the first run since boot - will load queues from last good config", stacklevel=2)
shapedDevicesFile = get_last_good_config_csv_path()
networkJSONfile = get_last_good_config_json_path()
safeToRunRefresh = True
if safeToRunRefresh == True:
# Load Subscriber Circuits & Devices
subscriberCircuits, dictForCircuitsWithoutParentNodes = loadSubscriberCircuitsForShaping(shapedDevicesFile, networkJSONfile)
runtime_override_count = apply_effective_runtime_circuit_overrides(subscriberCircuits)
if runtime_override_count > 0:
print(
"Applied "
+ str(runtime_override_count)
+ " effective runtime circuit SQM override(s) in memory"
)
# Preserve the logical parent (as configured in ShapedDevices.csv) before any shaping-time rewrites.
for circuit in subscriberCircuits:
if 'logicalParentNode' not in circuit:
circuit['logicalParentNode'] = circuit.get('ParentNode')
if 'logicalParentNodeID' not in circuit:
circuit['logicalParentNodeID'] = circuit.get('ParentNodeID', '')
# Load network hierarchy
with open(networkJSONfile, 'r') as j:
network = json.loads(j.read())
# Flat networks ({}) don't require ParentNode entries. Treat every circuit as
# unparented so they can be distributed across generated parent nodes / CPUs.
flat_network = loaded_network_is_flat(network)
# Virtual Nodes (logical-only): build a physical shaping topology that skips them,
# while leaving ShapedDevices.csv (and monitoring) unchanged.
logical_to_physical_node = {}
virtual_nodes = []
if not flat_network and isinstance(network, dict) and len(network) > 0:
logical_to_physical_node, virtual_nodes = build_logical_to_physical_node_map(network)
if len(virtual_nodes) > 0:
print(
f"Detected {len(virtual_nodes)} virtual node(s) in network.json; building physical HTB tree without them."
)
network = build_physical_network(network)
if len(network) == 0:
warnings.warn(
"All nodes were removed from the physical shaping tree after virtual-node promotion. Treating this as a flat network for shaping.",
stacklevel=2,
)
flat_network = True
else:
# Avoid bloating queuingStructure.json when there are no virtual nodes.
logical_to_physical_node = {}
# Re-map circuits that are directly parented to a virtual node to the nearest real ancestor (milestone c).
if not flat_network and len(virtual_nodes) > 0 and isinstance(logical_to_physical_node, dict):
next_id = max(dictForCircuitsWithoutParentNodes.keys(), default=-1) + 1
for circuit in subscriberCircuits:
logical_parent = circuit.get('logicalParentNode', circuit.get('ParentNode'))
if logical_parent and logical_parent != 'none' and logical_parent in logical_to_physical_node:
physical_parent = logical_to_physical_node.get(logical_parent)
if physical_parent is None:
warnings.warn(
f"Circuit '{circuit.get('circuitID','')}' is parented to virtual top-level node '{logical_parent}'. Attaching it as unparented for shaping.",
stacklevel=2,
)
circuit['ParentNode'] = 'none'
circuit['ParentNodeID'] = ''
circuit['effectiveParentNodeID'] = ''
else:
circuit['ParentNode'] = physical_parent
# If virtual-node mapping created new unparented circuits, ensure they have planner IDs.
if circuit.get('ParentNode') == 'none' and 'idForCircuitsWithoutParentNodes' not in circuit:
try:
weight = float(circuit.get('maxDownload', 0)) + float(circuit.get('maxUpload', 0))
except Exception:
weight = 0.0
dictForCircuitsWithoutParentNodes[next_id] = weight
circuit['idForCircuitsWithoutParentNodes'] = next_id
next_id += 1
if flat_network:
print("Flat network detected; assigning circuits to generated parent nodes")
next_id = max(dictForCircuitsWithoutParentNodes.keys(), default=-1) + 1
for circuit in subscriberCircuits:
if circuit.get('ParentNode') != 'none':
circuit['ParentNode'] = 'none'
circuit['ParentNodeID'] = ''
circuit['effectiveParentNodeID'] = ''
if circuit.get('ParentNode') == 'none' and 'idForCircuitsWithoutParentNodes' not in circuit:
try:
weight = float(circuit.get('maxDownload', 0)) + float(circuit.get('maxUpload', 0))
except Exception:
weight = 0.0
dictForCircuitsWithoutParentNodes[next_id] = weight
circuit['idForCircuitsWithoutParentNodes'] = next_id
next_id += 1
# Normalize any zero or missing bandwidths in the network model early
# Some users may specify 0 for site bandwidths. HTB requires positive
# rates, so bump zeros to the parent/default capacity and log a warning.
def fix_zero_bandwidths(data, parentMaxDL, parentMaxUL):
for node in data:
if isinstance(node, str):
if (isinstance(data[node], dict)) and (node != 'children'):
# Ensure max bandwidths are positive. If 0 or missing, use parent's defaults.
dl = data[node].get('downloadBandwidthMbps', None)
ul = data[node].get('uploadBandwidthMbps', None)
if dl is None or (isinstance(dl, (int, float)) and dl <= 0):
logging.warning(f"Node '{node}' has downloadBandwidthMbps set to 0 or missing; using parent/default {parentMaxDL} Mbps.")
data[node]['downloadBandwidthMbps'] = parentMaxDL
if ul is None or (isinstance(ul, (int, float)) and ul <= 0):
logging.warning(f"Node '{node}' has uploadBandwidthMbps set to 0 or missing; using parent/default {parentMaxUL} Mbps.")
data[node]['uploadBandwidthMbps'] = parentMaxUL
# Recurse into children with this node's maxima as the new parent defaults
if 'children' in data[node]:
fix_zero_bandwidths(
data[node]['children'],
data[node]['downloadBandwidthMbps'],
data[node]['uploadBandwidthMbps'],
)
fix_zero_bandwidths(
network,
upstream_bandwidth_capacity_download_mbps(),
upstream_bandwidth_capacity_upload_mbps(),
)
# Pull rx/tx queues / CPU cores available
# Handling the case when the number of queues for interfaces are different
InterfaceAQueuesAvailable = findQueuesAvailable(interface_a())
InterfaceBQueuesAvailable = findQueuesAvailable(interface_b())
queuesAvailable = min(InterfaceAQueuesAvailable, InterfaceBQueuesAvailable)
stickOffset = 0
if on_a_stick():
print("On-a-stick override dividing queues")
# The idea here is that download use queues 0 - n/2, upload uses the other half
queuesAvailable = math.floor(queuesAvailable / 2)
stickOffset = queuesAvailable
# Generate Parent Nodes. Spread ShapedDevices.csv which lack defined ParentNode across these (balance across CPUs)
print("Generating parent nodes")
generatedPNs = []
numberOfGeneratedPNs = queuesAvailable
chosenDownloadMbps = generated_pn_download_mbps()
chosenUploadMbps = generated_pn_upload_mbps()
for x in range(numberOfGeneratedPNs):
genPNname = "Generated_PN_" + str(x+1)
network[genPNname] = {
"downloadBandwidthMbps": chosenDownloadMbps,
"uploadBandwidthMbps": chosenUploadMbps
}
generatedPNs.append(genPNname)
# Planner/device weights (fetched only when planner/binpacking is enabled).
# When disabled, we keep this empty and fall back to rate-based weights later.
weight_by_circuit_id = {}
if use_bin_packing_to_balance_cpu():
print("Using internal planner to sort circuits by CPU core")
# Build item list with weights for circuits lacking a ParentNode
items = []
try:
weights = get_weights()
except Exception as e:
warnings.warn("get_weights() failed; defaulting to equal weights (" + str(e) + ")", stacklevel=2)
weights = None
weight_by_circuit_id = {}
if weights is not None:
try:
for w in weights:
weight_by_circuit_id[str(w.circuit_id)] = float(w.weight)
except Exception:
pass
for circuit in subscriberCircuits:
if circuit.get('ParentNode') == 'none' and 'idForCircuitsWithoutParentNodes' in circuit:
item_id = circuit['idForCircuitsWithoutParentNodes']
# Prefer provided weights; default to 1.0
w = dictForCircuitsWithoutParentNodes.get(item_id, 1.0)
# If a specific circuit weight exists, prefer it
if 'circuitID' in circuit and str(circuit['circuitID']) in weight_by_circuit_id:
w = weight_by_circuit_id[str(circuit['circuitID'])]
# Ignore placeholder default rates for weight purposes
try:
default_rate = float(generated_pn_download_mbps())
max_dl = float(circuit.get('maxDownload', 0))
if abs(max_dl - default_rate) < 1e-6:
w = 0.0
except Exception:
pass
items.append({"id": item_id, "weight": float(w)})
# Prepare bins and capacities
bins_list = [{"id": pn} for pn in generatedPNs]
capacities = {pn: 1.0 for pn in generatedPNs}
# Load planner state
try:
import bin_planner
except ImportError:
bin_planner = None
# Store planner state directly in lqos_directory (no hidden subdirs)
state_path = get_planner_state_path()
state = {}
if bin_planner is not None:
state = load_planner_state(state_path, bin_planner)
now_ts = time.time()
prev_assign = {}
last_change_ts = {}
if isinstance(state, dict):
prev_assign = state.get("assignments", {}) or {}
last_change_ts = state.get("last_change_ts", {}) or {}
# Filter previous assignments to only items/bins in this context
item_ids = {str(it["id"]) for it in items}
valid_bins = set(capacities.keys())
prev_assign = {iid: b for iid, b in prev_assign.items() if iid in item_ids and b in valid_bins}
last_change_ts = {iid: last_change_ts.get(iid, 0.0) for iid in item_ids}
# Planner parameters
params = {
"candidate_set_size": 4,
"headroom": 0.05,
"alpha": 0.1,
"hysteresis_threshold": 0.03,
"cooldown_seconds": 3600,
"move_budget_per_run": max(1, min(32, int(0.01 * max(1, len(items))))),
"salt": state.get("salt", "default_salt") if isinstance(state, dict) else "default_salt",
"last_change_ts_by_item": last_change_ts,
}
if observe_mode:
params["move_budget_per_run"] = 0
if bin_planner is not None:
assignments, changed = bin_planner.plan_assignments(
items, bins_list, capacities, prev_assign, now_ts, params
)
else:
# Fallback to simple greedy if planner unavailable
bin_loads = {pn: 0.0 for pn in generatedPNs}
pairs = [(str(it["id"]), float(it["weight"])) for it in items]
pairs.sort(key=lambda iw: (-iw[1], str(iw[0])))
assignments = {}
for item_id, w in pairs:
target_pn = min(bin_loads.items(), key=lambda kv: (kv[1], kv[0]))[0]
assignments[item_id] = target_pn
bin_loads[target_pn] += w
changed = list(assignments.keys())
# Apply assignments to circuits
for circuit in subscriberCircuits:
if circuit.get('ParentNode') == 'none' and 'idForCircuitsWithoutParentNodes' in circuit:
item_id = circuit['idForCircuitsWithoutParentNodes']
item_key = str(item_id)
if item_key in assignments:
circuit['ParentNode'] = assignments[item_key]
circuit['ParentNodeID'] = ''
circuit['effectiveParentNodeID'] = ''
# Update and save state
if bin_planner is not None and isinstance(state, dict):
if state.get("salt") is None:
state["salt"] = "default_salt"
if "assignments" not in state or not isinstance(state["assignments"], dict):
state["assignments"] = {}
if "last_change_ts" not in state or not isinstance(state["last_change_ts"], dict):
state["last_change_ts"] = {}
for iid, b in assignments.items():
# record last change time if changed
if iid in changed:
state["last_change_ts"][iid] = now_ts
state["assignments"][iid] = b
try:
print(f"Saving planner state to {state_path} (generated PNs)")
save_planner_state(state, state_path, bin_planner)
except Exception as e:
warnings.warn(f"Failed to save planner state at {state_path}: {e}", stacklevel=2)
print("Finished planning generated parent nodes")
else:
genPNcounter = 0
for circuit in subscriberCircuits:
if circuit['ParentNode'] == 'none':
circuit['ParentNode'] = generatedPNs[genPNcounter]
circuit['ParentNodeID'] = ''
circuit['effectiveParentNodeID'] = ''
genPNcounter += 1
if genPNcounter >= queuesAvailable:
genPNcounter = 0
print("Generated parent nodes created")
# Find the bandwidth minimums for each node by combining mimimums of devices lower in that node's hierarchy
def findBandwidthMins(data, depth):
tabs = ' ' * depth
minDownload = 0
minUpload = 0
for elem in data:
for circuit in subscriberCircuits:
if elem == circuit['ParentNode']:
minDownload += circuit['minDownload']
minUpload += circuit['minUpload']
if 'children' in data[elem]:
minDL, minUL = findBandwidthMins(data[elem]['children'], depth+1)
minDownload += minDL
minUpload += minUL
if 'downloadBandwidthMbpsMin' in data[elem]:
data[elem]['downloadBandwidthMbpsMin'] = max(data[elem]['downloadBandwidthMbpsMin'], minDownload)
else:
data[elem]['downloadBandwidthMbpsMin'] = max(data[elem]['downloadBandwidthMbps'], minUpload)
if 'uploadBandwidthMbpsMin' in data[elem]:
data[elem]['uploadBandwidthMbpsMin'] = max(data[elem]['uploadBandwidthMbpsMin'], minUpload)
else:
data[elem]['uploadBandwidthMbpsMin'] = max(data[elem]['uploadBandwidthMbps'], minUpload)
return minDownload, minUpload
logging.info("Finding the bandwidth minimums for each node")
minDownload, minUpload = findBandwidthMins(network, 0)
logging.info("Found the bandwidth minimums for each node")
# Child nodes inherit bandwidth maximums of parents. We apply this here to avoid bugs when compression is applied with flattenA().
def inheritBandwidthMaxes(data, parentMaxDL, parentMaxUL, parentMinDL, parentMinUL):
for node in data:
if isinstance(node, str):
if (isinstance(data[node], dict)) and (node != 'children'):
# Cap based on this node's max bandwidth, or parent node's max bandwidth, whichever is lower
data[node]['downloadBandwidthMbps'] = min(int(data[node]['downloadBandwidthMbps']),int(parentMaxDL))
data[node]['uploadBandwidthMbps'] = min(int(data[node]['uploadBandwidthMbps']),int(parentMaxUL))
data[node]['downloadBandwidthMbpsMin'] = min(int(data[node]['downloadBandwidthMbpsMin']),int(data[node]['downloadBandwidthMbps']),int(parentMinDL))
data[node]['uploadBandwidthMbpsMin'] = min(int(data[node]['uploadBandwidthMbpsMin']),int(data[node]['uploadBandwidthMbps']),int(parentMinUL))
# Recursive call this function for children nodes attached to this node
if 'children' in data[node]:
# We need to keep tabs on the minor counter, because we can't have repeating class IDs. Here, we bring back the minor counter from the recursive function
inheritBandwidthMaxes(data[node]['children'], data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps'], data[node]['downloadBandwidthMbpsMin'], data[node]['uploadBandwidthMbpsMin'])
#return data
# Here is the actual call to the recursive function
inheritBandwidthMaxes(network, parentMaxDL=upstream_bandwidth_capacity_download_mbps(), parentMaxUL=upstream_bandwidth_capacity_upload_mbps(), parentMinDL=upstream_bandwidth_capacity_download_mbps(), parentMinUL=upstream_bandwidth_capacity_upload_mbps())
# Ensure site-level minimums are strictly below maximums for HTB classes
def ensure_min_less_than_max(data):
for node in data:
if isinstance(node, str):
if (isinstance(data[node], dict)) and (node != 'children'):
try:
dl_max = float(data[node].get('downloadBandwidthMbps', 0))
ul_max = float(data[node].get('uploadBandwidthMbps', 0))
dl_min = float(data[node].get('downloadBandwidthMbpsMin', dl_max))
ul_min = float(data[node].get('uploadBandwidthMbpsMin', ul_max))
except Exception:
# If parsing fails, skip adjustment for this node
dl_max = data[node].get('downloadBandwidthMbps', 0)
ul_max = data[node].get('uploadBandwidthMbps', 0)
dl_min = data[node].get('downloadBandwidthMbpsMin', dl_max)
ul_min = data[node].get('uploadBandwidthMbpsMin', ul_max)
def adjust(min_v, max_v):
# Keep min strictly lower than max; support small max with fractional step
if min_v >= max_v:
if max_v >= 1.0:
return max_v - 1.0
else:
return max(0.01, max_v - 0.01)
return min_v
new_dl_min = adjust(dl_min, dl_max)
new_ul_min = adjust(ul_min, ul_max)
if new_dl_min != dl_min:
# Too noisy in practice; keep as debug for diagnostics
logging.debug(f"Node '{node}' min download ({dl_min}) >= max ({dl_max}); lowering min to {new_dl_min}")
data[node]['downloadBandwidthMbpsMin'] = new_dl_min
if new_ul_min != ul_min:
# Too noisy in practice; keep as debug for diagnostics
logging.debug(f"Node '{node}' min upload ({ul_min}) >= max ({ul_max}); lowering min to {new_ul_min}")
data[node]['uploadBandwidthMbpsMin'] = new_ul_min
if 'children' in data[node]:
ensure_min_less_than_max(data[node]['children'])
ensure_min_less_than_max(network)
# Compress network.json. HTB only supports 8 levels of HTB depth. Compress to 8 layers if beyond 8.
def flattenB(data):
newDict = {}
for node in data:
if isinstance(node, str):
if (isinstance(data[node], dict)) and (node != 'children'):
newDict[node] = dict(data[node])
if 'children' in data[node]:
result = flattenB(data[node]['children'])
del newDict[node]['children']
newDict.update(result)
return newDict
def flattenA(data, depth):
newDict = {}
for node in data:
if isinstance(node, str):
if (isinstance(data[node], dict)) and (node != 'children'):
newDict[node] = dict(data[node])
if 'children' in data[node]:
result = flattenA(data[node]['children'], depth+2)
del newDict[node]['children']
if depth <= 8:
newDict[node]['children'] = result
else:
flattened = flattenB(data[node]['children'])
if 'children' in newDict[node]:
newDict[node]['children'].update(flattened)
else:
newDict[node]['children'] = flattened
return newDict
network = flattenA(network, 1)
# After flattening, some attachment-style names may only survive as
# metadata on the physical node that now owns that branch.
parent_node_ids = collect_physical_parent_node_ids(network)
parent_node_aliases = collect_physical_parent_node_aliases(network)
for circuit in subscriberCircuits:
parent_node = str(circuit.get('ParentNode', '') or '').strip()
if (
(not parent_node or parent_node == 'none')
and not str(circuit.get('effectiveAttachmentName', '') or '').strip()
and not str(circuit.get('effectiveAttachmentID', '') or '').strip()
):
continue
resolved_parent, resolved_parent_id = _resolve_effective_parent_node(
circuit,
parent_node_ids,
parent_node_aliases,
)
if resolved_parent_id:
circuit['effectiveParentNodeID'] = resolved_parent_id
if resolved_parent and resolved_parent != parent_node:
logging.info(
"Resolved circuit parent reference '%s' (%s) to physical queue node '%s' for circuit '%s'",
parent_node,
str(circuit.get('ParentNodeID', '') or '').strip(),
resolved_parent,
circuit.get('circuitID', ''),
)
circuit['ParentNode'] = resolved_parent
elif resolved_parent_id and str(circuit.get('ParentNodeID', '') or '').strip() != resolved_parent_id:
logging.info(
"Resolved circuit parent ID '%s' to physical queue node '%s' for circuit '%s'",
str(circuit.get('ParentNodeID', '') or '').strip(),
resolved_parent,
circuit.get('circuitID', ''),
)
circuit['ParentNode'] = resolved_parent
circuit['ParentNodeID'] = resolved_parent_id
elif resolved_parent_id:
circuit['ParentNodeID'] = resolved_parent_id
# Group circuits by stable parent identity first, then by names as fallback.
circuits_by_parent_id = {}
circuits_by_parent_name = {}
for circuit in subscriberCircuits:
parent_name, parent_id = _normalize_circuit_shaping_parent(circuit, parent_node_ids)
if parent_id:
circuits_by_parent_id.setdefault(parent_id, []).append(circuit)
elif parent_name and parent_name != 'none':
circuits_by_parent_name.setdefault(parent_name, []).append(circuit)
# Parse network structure and add devices from ShapedDevices.csv
print("Parsing network structure and tallying devices")
parentNodes = []
minorByCPUpreloaded = {}
nodes_requiring_min_squashing = {}
# Track minor counter by CPU. This way we can have > 32000 hosts (htb has u16 limit to minor handle)
# Minor numbers start at 3 to reserve 1 for root qdisc and 2 for default class
# With CIRCUIT_PADDING, we leave gaps between nodes to allow future circuit additions
# without disrupting existing ClassID assignments. This maintains stability across reloads.
for x in range(queuesAvailable):
minorByCPUpreloaded[x+1] = 3
def report_minor_overflow(queue, minor):
msg = f"Minor class ID overflow on CPU {queue}: {minor} exceeds TC's u16 limit (65535). Consider increasing queue count or restructuring network hierarchy."
logging.error(msg)
try:
ctx = json.dumps({"cpu": queue, "minor": minor})
submit_urgent_issue("LibreQoS", "Error", "TC_U16_OVERFLOW", msg, ctx, f"TC_U16_OVERFLOW_CPU_{queue}")
except Exception:
pass
raise ValueError(msg)
def ensure_minor_capacity(queue, minor):
if minor >= 0xFFFF:
report_minor_overflow(queue, minor)
def next_free_minor(start_minor, reserved):
candidate = max(3, start_minor)
while candidate in reserved:
candidate += 1
return candidate
def sorted_node_keys(data, depth):
keys = list(data.keys())
if depth == 0 and len(keys) > 0 and all(k.startswith("CpueQueue") for k in keys):
try:
keys.sort(key=lambda k: int(k.replace("CpueQueue", "")))
except Exception:
keys = sorted(keys)
else:
keys = sorted(keys)
return keys
# If we're in binpacking mode, we need to sort the network structure a bit
if use_bin_packing_to_balance_cpu() and not flat_network:
# Binpacking is an Insight feature; if Insight is not enabled/licensed, fall back to
# deterministic round-robin placement so "virtual node promotion" can still spread
# the physical tree across CPUs.
insight_enabled = False
try:
insight_enabled = bool(is_insight_enabled())
except Exception:
insight_enabled = False
if insight_enabled:
print("Planner is enabled, so we're going to sort your network across CPU queues.")
else:
warnings.warn(
"Binpacking is enabled but Insight is not available; using round-robin CPU distribution.",
stacklevel=2,
)
# Build items from top-level nodes with weights
items = []
try:
weights = get_tree_weights()
except Exception as e:
warnings.warn(
"get_tree_weights() failed; defaulting to equal weights (" + str(e) + ")",
stacklevel=2,
)
weights = None
weight_by_name = {}
if weights is not None:
try:
for w in weights:
weight_by_name[str(w.name)] = float(w.weight)
except Exception:
pass
for node in network:
if is_generated_parent_node_name(node):
continue
w = weight_by_name.get(str(node), 1.0)
try:
w = float(w)
except Exception:
w = 1.0
# Ensure we always spread items. Zero/negative weights can cause all items
# to collapse into a single CPU bin in tie cases.
if not math.isfinite(w) or w <= 0.0:
w = 1.0
items.append({"id": str(node), "weight": w})
cpu_keys = ["CpueQueue" + str(cpu) for cpu in range(queuesAvailable)]
valid_bins = set(cpu_keys)
planner_used = False
state_path = get_planner_state_path()
state = {}
now_ts = time.time()
assignment = {}
changed = []
try:
state = load_planner_state(state_path, None)
except Exception:
state = {}
prev_assign = {}
last_change_ts = {}
if isinstance(state, dict):
prev_assign = state.get("assignments", {}) or {}
last_change_ts = state.get("last_change_ts", {}) or {}
item_ids = {str(it["id"]) for it in items}
prev_assign = {
iid: b for iid, b in prev_assign.items() if iid in item_ids and b in valid_bins
}
last_change_ts = {iid: last_change_ts.get(iid, 0.0) for iid in item_ids}
move_budget = max(1, min(32, int(0.01 * max(1, len(items)))))
if observe_mode:
move_budget = 0
planner_mode = "stable_greedy" if insight_enabled else "round_robin"
try:
plan_result = plan_top_level_cpu_bins(
items,
queuesAvailable,
prev_assign=prev_assign,
last_change_ts=last_change_ts,
now_ts=now_ts,
mode=planner_mode,
move_budget_per_run=move_budget,
cooldown_seconds=3600.0,
hysteresis_threshold=0.03,
)
assignment = dict(plan_result.get("assignment", {}) or {})
changed = list(plan_result.get("changed", []) or [])
planner_used = bool(plan_result.get("planner_used", False))
except Exception as e:
warnings.warn(
f"Shared Rust planner failed ({e}); falling back to deterministic local assignment.",
stacklevel=2,
)
assignment = {}
names = sorted(str(it["id"]) for it in items)
if cpu_keys:
for idx, name in enumerate(names):
if insight_enabled:
assignment[name] = cpu_keys[idx % len(cpu_keys)]
else:
assignment[name] = cpu_keys[idx % len(cpu_keys)]
changed = list(assignment.keys())
planner_used = False
resolved_assignment = {}
for node in network:
tgt = generated_parent_node_queue_key(node, queuesAvailable)
if tgt is None:
tgt = assignment.get(node)
if tgt is None:
tgt = "CpueQueue" + str(queuesAvailable - 1)
resolved_assignment[str(node)] = tgt
for x in range(queuesAvailable):
key = "CpueQueue" + str(x)
assigned = [name for name, tgt in resolved_assignment.items() if tgt == key]
print("Bin " + str(x) + " = ", assigned)
# Build the binned network structure
binnedNetwork = {}
for cpu in range(queuesAvailable):
cpuKey = "CpueQueue" + str(cpu)
binnedNetwork[cpuKey] = {
'downloadBandwidthMbps': generated_pn_download_mbps(),
'uploadBandwidthMbps': generated_pn_upload_mbps(),
'type': 'site',
'downloadBandwidthMbpsMin': generated_pn_download_mbps(),
'uploadBandwidthMbpsMin': generated_pn_upload_mbps(),
'children': {},
'name': cpuKey
}
for node in network:
tgt = resolved_assignment.get(str(node))
if tgt is None:
tgt = "CpueQueue" + str(queuesAvailable - 1)
binnedNetwork[tgt]['children'][node] = network[node]
network = binnedNetwork
# Update and save state
if planner_used and isinstance(state, dict):
if state.get("salt") is None:
state["salt"] = "default_salt"
if "assignments" not in state or not isinstance(state["assignments"], dict):
state["assignments"] = {}
if "last_change_ts" not in state or not isinstance(state["last_change_ts"], dict):
state["last_change_ts"] = {}
stale_generated = [
iid for iid in list(state["assignments"].keys())
if is_generated_parent_node_name(iid)
]
for iid in stale_generated:
state["assignments"].pop(iid, None)
state["last_change_ts"].pop(iid, None)
for iid, b in assignment.items():
if iid in changed:
state["last_change_ts"][iid] = now_ts
state["assignments"][iid] = b
try:
print(f"Saving planner state to {state_path} (top-level CPU binning)")
save_planner_state(state, state_path, None)
except Exception as e:
warnings.warn(
f"Failed to save planner state at {state_path}: {e}", stacklevel=2
)
# Load persisted top-level planner state. Full reloads rebuild TC, so
# site/circuit class minors are runtime artifacts and are not reused below.
try:
state # noqa: B018
except NameError:
state = {}
state_path = get_planner_state_path()
if not isinstance(state, dict) or len(state.keys()) == 0:
state = load_planner_state(state_path, None)
circuit_state_updated = {}
site_state_updated = {}
planner_site_inputs = []
planner_circuit_groups = []
def collect_identity_planner_inputs(data, depth, queue, path=()):
for node in sorted_node_keys(data, depth):
current_queue = queue
node_path = path + (node,)
parent_path = '/'.join(path)
has_children = bool(data[node].get('children'))
planner_site_inputs.append(
{
"site_key": '/'.join(node_path),
"parent_path": parent_path,
"queue": current_queue,
"has_children": has_children,
}
)
node_id = str(data[node].get('id', '') or '').strip()
node_name = str(data[node].get('name', '') or '').strip()
selected_circuits = []
if node_id and node_id in circuits_by_parent_id:
selected_circuits = list(circuits_by_parent_id[node_id])
else:
seen_circuit_ids = set()
for candidate in (node, node_name):
for circuit in circuits_by_parent_name.get(candidate, []):
circuit_id = planner_circuit_identity_key(circuit)
if circuit_id in seen_circuit_ids:
continue
selected_circuits.append(circuit)
seen_circuit_ids.add(circuit_id)
if selected_circuits:
sorted_circuits = sorted(
selected_circuits,
key=lambda c: c.get('circuitName', c.get('circuitID', '')),
)
planner_circuit_groups.append(
{
"parent_node": node,
"queue": current_queue,
"circuit_ids": [
planner_circuit_identity_key(circuit)
for circuit in sorted_circuits
],
}
)
if has_children:
sorted_children = dict(sorted(data[node]['children'].items()))
collect_identity_planner_inputs(
sorted_children,
depth + 1,
current_queue,
node_path,
)
if depth == 0:
if queue >= queuesAvailable:
queue = 1
else:
queue += 1
collect_identity_planner_inputs(network, 0, 1)
identity_plan = plan_class_identities(
planner_site_inputs,
planner_circuit_groups,
site_state={},
circuit_state={},
stick_offset=stickOffset,
circuit_padding=CIRCUIT_PADDING,
)
site_assignment_by_key = {
entry["site_key"]: entry for entry in identity_plan.get("sites", [])
}
circuit_assignment_by_key = {
entry["circuit_id"]: entry for entry in identity_plan.get("circuits", [])
}
site_state_updated = identity_plan.get("site_state", {}) or {}
circuit_state_updated = identity_plan.get("circuit_state", {}) or {}
def apply_site_assignments(data, depth, queue, parentClassID, upParentClassID, parentMaxDL, parentMaxUL, parentMinDL, parentMinUL, path=()):
for node in sorted_node_keys(data, depth):
current_queue = queue
node_path = path + (node,)
site_key = '/'.join(node_path)
assignment = site_assignment_by_key.get(site_key)
if assignment is None:
raise ValueError(f"Missing planned site identity for {site_key}")
assigned_site_minor = int(assignment["class_minor"])
major = int(assignment["class_major"])
up_major = int(assignment["up_class_major"])
ensure_minor_capacity(current_queue, assigned_site_minor)
nodeClassID = hex(major) + ':' + hex(assigned_site_minor)
upNodeClassID = hex(up_major) + ':' + hex(assigned_site_minor)
data[node]['classid'] = nodeClassID
data[node]['up_classid'] = upNodeClassID
current_parent_classid = parentClassID
current_up_parent_classid = upParentClassID
if depth == 0:
current_parent_classid = hex(major) + ':'
current_up_parent_classid = hex(up_major) + ':'
data[node]['parentClassID'] = current_parent_classid
data[node]['up_parentClassID'] = current_up_parent_classid
data[node]['downloadBandwidthMbps'] = min(data[node]['downloadBandwidthMbps'], parentMaxDL)
data[node]['uploadBandwidthMbps'] = min(data[node]['uploadBandwidthMbps'], parentMaxUL)
data[node]['downloadBandwidthMbpsMin'] = min(data[node]['downloadBandwidthMbpsMin'], data[node]['downloadBandwidthMbps'], parentMinDL)
data[node]['uploadBandwidthMbpsMin'] = min(data[node]['uploadBandwidthMbpsMin'], data[node]['uploadBandwidthMbps'], parentMinUL)
data[node]['classMajor'] = hex(major)
data[node]['up_classMajor'] = hex(up_major)
data[node]['classMinor'] = hex(assigned_site_minor)
data[node]['cpuNum'] = hex(current_queue-1)
data[node]['up_cpuNum'] = hex(current_queue-1+stickOffset)
parentNodes.append(
{
"parentNodeName": node,
"classID": nodeClassID,
"maxDownload": data[node]['downloadBandwidthMbps'],
"maxUpload": data[node]['uploadBandwidthMbps'],
}
)
if 'children' in data[node]:
sorted_children = dict(sorted(data[node]['children'].items()))
apply_site_assignments(
sorted_children,
depth+1,
current_queue,
nodeClassID,
upNodeClassID,
data[node]['downloadBandwidthMbps'],
data[node]['uploadBandwidthMbps'],
data[node]['downloadBandwidthMbpsMin'],
data[node]['uploadBandwidthMbpsMin'],
node_path,
)
if depth == 0:
if queue >= queuesAvailable:
queue = 1
else:
queue += 1
apply_site_assignments(
network,
0,
queue=1,
parentClassID=None,
upParentClassID=None,
parentMaxDL=upstream_bandwidth_capacity_download_mbps(),
parentMaxUL=upstream_bandwidth_capacity_upload_mbps(),
parentMinDL=upstream_bandwidth_capacity_download_mbps(),
parentMinUL=upstream_bandwidth_capacity_upload_mbps(),
)
attached_circuit_ids = set()
def attach_circuits(data, depth, path=()):
for node in sorted_node_keys(data, depth):
node_data = data[node]
queue_token = _parse_int_token(node_data.get('cpuNum'))
site_major = _parse_int_token(node_data.get('classMajor'))
site_up_major = _parse_int_token(node_data.get('up_classMajor'))
if queue_token is None or site_major is None or site_up_major is None:
continue
queue = queue_token + 1
circuitsForThisNetworkNode = []
node_id = str(node_data.get('id', '') or '').strip()
node_name = str(node_data.get('name', '') or '').strip()
parent_candidates = _attachment_lookup_candidates(node, node_data)
selected_circuits = []
if node_id and node_id in circuits_by_parent_id:
selected_circuits = list(circuits_by_parent_id[node_id])
else:
seen_circuit_ids = set()
name_candidates = (
parent_candidates[1:] if node_id and parent_candidates else parent_candidates
)
for candidate in name_candidates:
for circuit in circuits_by_parent_name.get(candidate, []):
circuit_id = str(circuit.get('circuitID', '') or '')
if circuit_id in seen_circuit_ids:
continue
selected_circuits.append(circuit)
seen_circuit_ids.add(circuit_id)
if selected_circuits:
override_min_down = None
override_min_up = None
combined_min_down = sum(float(circuit.get('minDownload', 0) or 0) for circuit in selected_circuits)
combined_min_up = sum(float(circuit.get('minUpload', 0) or 0) for circuit in selected_circuits)
if (combined_min_down > node_data['downloadBandwidthMbpsMin']) or (combined_min_up > node_data['uploadBandwidthMbpsMin']):
override_min_down = 1
override_min_up = 1
logging.info("The combined minimums of circuits in Parent Node [" + node + "] exceeded that of the parent node. Reducing these circuits' minimums to 1 now.", stacklevel=2)
if ((override_min_down * len(selected_circuits)) > node_data['downloadBandwidthMbpsMin']) or ((override_min_up * len(selected_circuits)) > node_data['uploadBandwidthMbpsMin']):
logging.info("Even with this change, minimums will exceed the min rate of the parent node. Using 10 kbps as the minimum for these circuits instead.", stacklevel=2)
nodes_requiring_min_squashing[node] = True
sorted_circuits = sorted(selected_circuits, key=lambda c: c.get('circuitName', c.get('circuitID', '')))
for circuit in sorted_circuits:
circuit_id = str(circuit.get('circuitID', '') or '')
if circuit_id in attached_circuit_ids:
continue
if circuit['maxDownload'] > node_data['downloadBandwidthMbps']:
logging.info("downloadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2)
if circuit['maxUpload'] > node_data['uploadBandwidthMbps']:
logging.info("uploadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2)
planner_key = planner_circuit_identity_key(circuit)
planned_identity = circuit_assignment_by_key.get(planner_key)
if planned_identity is None:
raise ValueError(f"Missing planned circuit identity for {planner_key}")
candidate_minor = int(planned_identity["class_minor"])
planned_major, planned_up_major = _validate_planned_circuit_attachment(
node,
node_data,
circuit,
planned_identity,
)
ensure_minor_capacity(queue, candidate_minor)
flowIDstring = hex(planned_major) + ':' + hex(candidate_minor)
upFlowIDstring = hex(planned_up_major) + ':' + hex(candidate_minor)
circuit['classid'] = flowIDstring
circuit['up_classid'] = upFlowIDstring
attached_circuit_ids.add(circuit_id)
logging.info("Added up_classid to circuit: " + circuit['up_classid'])
maxDownload = min(circuit['maxDownload'], node_data['downloadBandwidthMbps'])
maxUpload = min(circuit['maxUpload'], node_data['uploadBandwidthMbps'])
if override_min_down:
circuit['minDownload'] = 1
if override_min_up:
circuit['minUpload'] = 1
minDownload = min(circuit['minDownload'], maxDownload)
minUpload = min(circuit['minUpload'], maxUpload)
thisNewCircuitItemForNetwork = {
'maxDownload': maxDownload,
'maxUpload': maxUpload,
'minDownload': minDownload,
'minUpload': minUpload,
"circuitID": circuit['circuitID'],
"circuitName": circuit['circuitName'],
"ParentNode": circuit['ParentNode'],
"ParentNodeID": circuit.get('ParentNodeID', ''),
"logicalParentNode": circuit.get('logicalParentNode', circuit['ParentNode']),
"logicalParentNodeID": circuit.get('logicalParentNodeID', circuit.get('ParentNodeID', '')),
"effectiveParentNodeID": circuit.get('effectiveParentNodeID', circuit.get('ParentNodeID', '')),
"devices": circuit['devices'],
"classid": flowIDstring,
"up_classid": upFlowIDstring,
"classMajor": hex(planned_major),
"up_classMajor": hex(planned_up_major),
"classMinor": hex(candidate_minor),
"comment": circuit['comment'],
}
try:
cid = str(circuit.get('circuitID', ''))
w = None
if cid in weight_by_circuit_id:
w = float(weight_by_circuit_id[cid])
if w is None:
w = float(maxDownload)
if abs(w - 1000.0) < 1e-6:
w = float(maxDownload)
try:
default_rate = float(generated_pn_download_mbps())
if abs(float(maxDownload) - default_rate) < 1e-6:
w = 0.0
except Exception:
pass
thisNewCircuitItemForNetwork['planner_weight'] = w
except Exception:
pass
if 'sqm' in circuit and circuit['sqm']:
thisNewCircuitItemForNetwork['sqm'] = circuit['sqm']
thisNewCircuitItemForNetwork['devices'] = circuit['devices']
circuitsForThisNetworkNode.append(thisNewCircuitItemForNetwork)
if len(circuitsForThisNetworkNode) > 0:
node_data['circuits'] = circuitsForThisNetworkNode
else:
node_data.pop('circuits', None)
if 'children' in node_data:
sorted_children = dict(sorted(node_data['children'].items()))
attach_circuits(sorted_children, depth+1, path + (node,))
attach_circuits(network, 0)
unattached_circuits = []
for circuit in subscriberCircuits:
circuit_id = str(circuit.get('circuitID', '') or '')
if circuit_id and circuit_id not in attached_circuit_ids:
unattached_circuits.append(
f"{circuit_id} ({circuit.get('circuitName', '')}) parent={circuit.get('ParentNode', '')} parent_id={circuit.get('effectiveParentNodeID', circuit.get('ParentNodeID', ''))}"
)
if unattached_circuits:
examples = unattached_circuits[:20]
warnings.warn(
"Some shaped circuits did not attach to the live queue tree. First examples: " + "; ".join(examples),
stacklevel=2,
)
logging.warning(
"Unattached shaped circuits after queue build: %s total. Examples: %s",
len(unattached_circuits),
"; ".join(examples),
)
try:
submit_urgent_issue(
"LibreQoS",
"Warning",
"UNATTACHED_SHAPED_CIRCUITS",
(
f"{len(unattached_circuits)} shaped circuit(s) did not attach to the live queue tree. "
"These circuits will not have class IDs or queue stats until topology/shaping inputs are fixed."
),
json.dumps({
"count": len(unattached_circuits),
"examples": examples,
}),
"UNATTACHED_SHAPED_CIRCUITS",
)
except Exception as exc:
logging.debug("Unable to submit unattached shaped circuits urgent issue: %s", exc)
minorByCPU = {
int(queue): int(minor)
for queue, minor in (identity_plan.get("last_used_minor_by_queue", {}) or {}).items()
}
for cpu in range(queuesAvailable):
minorByCPU.setdefault(cpu + 1, 3)
if not isinstance(state, dict):
state = {}
state['circuits'] = circuit_state_updated
state['sites'] = site_state_updated
try:
print(f"Saving planner state to {state_path} (circuit/site minors)")
save_planner_state(state, state_path, None)
except Exception as e:
warnings.warn(f"Failed to save planner circuit state at {state_path}: {e}", stacklevel=2)
bakery = Bakery()
bakery.start_batch() # Initializes the bakery transaction
linuxTCcommands = []
shapedDeviceKeys = set()
# Root HTB Setup
# Create MQ qdisc for each CPU core / rx-tx queue. Generate commands to create corresponding HTB and leaf classes. Prepare commands for execution later
thisInterface = interface_a()
logging.info("# MQ Setup for " + thisInterface)
command = 'qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq'
bakery.setup_mq(queuesAvailable, stickOffset)
linuxTCcommands.append(command)
maxBandwidth = max(upstream_bandwidth_capacity_upload_mbps(), upstream_bandwidth_capacity_download_mbps())
calculateR2q(maxBandwidth)
for queue in range(queuesAvailable):
command = 'qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2'
linuxTCcommands.append(command)
command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ format_rate_for_tc(upstream_bandwidth_capacity_download_mbps()) + ' ceil ' + format_rate_for_tc(upstream_bandwidth_capacity_download_mbps()) + quantum(upstream_bandwidth_capacity_download_mbps())
linuxTCcommands.append(command)
command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + sqm()
linuxTCcommands.append(command)
# Default class - traffic gets passed through this limiter with lower priority if it enters the top HTB without a specific class.
# Technically, that should not even happen. So don't expect much if any traffic in this default class.
# Only 1/4 of defaultClassCapacity is guaranteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling.
command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + format_rate_for_tc(round((upstream_bandwidth_capacity_download_mbps()-1)/4)) + ' ceil ' + format_rate_for_tc(upstream_bandwidth_capacity_download_mbps()-1) + ' prio 5' + quantum(upstream_bandwidth_capacity_download_mbps())
linuxTCcommands.append(command)
command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + sqm()
linuxTCcommands.append(command)
# Note the use of stickOffset, and not replacing the root queue if we're on a stick
thisInterface = interface_b()
logging.info("# MQ Setup for " + thisInterface)
if not on_a_stick():
command = 'qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq'
linuxTCcommands.append(command)
for queue in range(queuesAvailable):
command = 'qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+stickOffset+1) + ' handle ' + hex(queue+stickOffset+1) + ': htb default 2'
linuxTCcommands.append(command)
command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ': classid ' + hex(queue+stickOffset+1) + ':1 htb rate '+ format_rate_for_tc(upstream_bandwidth_capacity_upload_mbps()) + ' ceil ' + format_rate_for_tc(upstream_bandwidth_capacity_upload_mbps()) + quantum(upstream_bandwidth_capacity_upload_mbps())
linuxTCcommands.append(command)
command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ':1 ' + sqm()
linuxTCcommands.append(command)
# Default class - traffic gets passed through this limiter with lower priority if it enters the top HTB without a specific class.
# Technically, that should not even happen. So don't expect much if any traffic in this default class.
# Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling.
command = 'class add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ':1 classid ' + hex(queue+stickOffset+1) + ':2 htb rate ' + format_rate_for_tc(round((upstream_bandwidth_capacity_upload_mbps()-1)/4)) + ' ceil ' + format_rate_for_tc(upstream_bandwidth_capacity_upload_mbps()-1) + ' prio 5' + quantum(upstream_bandwidth_capacity_upload_mbps())
linuxTCcommands.append(command)
command = 'qdisc add dev ' + thisInterface + ' parent ' + hex(queue+stickOffset+1) + ':2 ' + sqm()
linuxTCcommands.append(command)
# Parse network structure. For each tier, generate commands to create corresponding HTB and leaf classes. Prepare commands for execution later
# Define lists for hash filters
print("Preparing TC commands")
def traverseNetwork(data):
nonlocal requiredIpMappings
# Cake needs help handling rates lower than 5 Mbps
def sqmFixupRate(rate:int, sqm:str) -> str:
# If we aren't using cake, just return the sqm string
if not sqm.startswith("cake") or "rtt" in sqm:
return sqm
# If we are using cake, we need to fixup the rate
# Based on: 1 MTU is 1500 bytes, or 12,000 bits.
# At 1 Mbps, (1,000 bits per ms) transmitting an MTU takes 12ms. Add 3ms for overhead, and we get 15ms.
# So 15ms divided by 5 (for 1%) multiplied by 100 yields 300ms.
# The same formula gives 180ms at 2Mbps
# 140ms at 3Mbps
# 120ms at 4Mbps
match rate:
case 1: return sqm + " rtt 300"
case 2: return sqm + " rtt 180"
case 3: return sqm + " rtt 140"
case 4: return sqm + " rtt 120"
case _: return sqm
for node in sorted(data.keys()):
site_name = data[node]['name'] if 'name' in data[node] else node
bakery.add_site(
site_name,
data[node]['parentClassID'],
data[node]['up_parentClassID'],
int(data[node]['classMinor'], 16),
data[node]['downloadBandwidthMbpsMin'],
data[node]['uploadBandwidthMbpsMin'],
data[node]['downloadBandwidthMbps'],
data[node]['uploadBandwidthMbps'],
)
command = 'class add dev ' + interface_a() + ' parent ' + data[node]['parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ format_rate_for_tc(data[node]['downloadBandwidthMbpsMin']) + ' ceil '+ format_rate_for_tc(data[node]['downloadBandwidthMbps']) + ' prio 3' + quantum(data[node]['downloadBandwidthMbps'])
linuxTCcommands.append(command)
logging.info("Up ParentClassID: " + data[node]['up_parentClassID'])
logging.info("ClassMinor: " + data[node]['classMinor'])
command = 'class add dev ' + interface_b() + ' parent ' + data[node]['up_parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ format_rate_for_tc(data[node]['uploadBandwidthMbpsMin']) + ' ceil '+ format_rate_for_tc(data[node]['uploadBandwidthMbps']) + ' prio 3' + quantum(data[node]['uploadBandwidthMbps'])
linuxTCcommands.append(command)
if 'circuits' in data[node]:
# Sort circuits by name for stable ordering
sorted_circuits = sorted(data[node]['circuits'],
key=lambda c: c.get('circuitName', c.get('circuitID', '')))
for circuit in sorted_circuits:
# If circuit mins exceed node mins - handle low min rates of 1 to mean 10 kbps.
# Avoid changing minDownload or minUpload because they are used in queuingStructure.json, and must remain integers.
min_down = circuit['minDownload']
min_up = circuit['minUpload']
if node in nodes_requiring_min_squashing:
if min_down == 1:
min_down = 0.01
if min_up == 1:
min_up = 0.01
# Ensure min < max for circuits as well
try:
max_down = float(circuit['maxDownload'])
max_up = float(circuit['maxUpload'])
md = float(min_down)
mu = float(min_up)
except Exception:
max_down = circuit['maxDownload']
max_up = circuit['maxUpload']
md = min_down
mu = min_up
if md >= max_down:
new_md = (max_down - 1.0) if max_down >= 1.0 else max(0.01, max_down - 0.01)
# Too noisy in practice; keep as debug for diagnostics
logging.debug(f"Circuit '{circuit.get('circuitID','unknown')}' min download ({md}) >= max ({max_down}); lowering min to {new_md}")
min_down = new_md
if mu >= max_up:
new_mu = (max_up - 1.0) if max_up >= 1.0 else max(0.01, max_up - 0.01)
# Too noisy in practice; keep as debug for diagnostics
logging.debug(f"Circuit '{circuit.get('circuitID','unknown')}' min upload ({mu}) >= max ({max_up}); lowering min to {new_mu}")
min_up = new_mu
# Generate TC commands to be executed later
tcComment = " # CircuitID: " + circuit['circuitID'] + " DeviceIDs: "
for device in circuit['devices']:
tcComment = '' #tcComment + device['deviceID'] + ', '
if 'devices' in circuit:
if 'comment' in circuit['devices'][0]:
tcComment = '' # tcComment + '| Comment: ' + circuit['devices'][0]['comment']
tcComment = tcComment.replace("\n", "")
circuit_name = circuit['circuitID'] if 'circuitID' in circuit else "unknown"
# Collect all IP addresses for this circuit
ip_list = []
for device in circuit['devices']:
if device['ipv4s']:
ip_list.extend(device['ipv4s'])
if device['ipv6s']:
ip_list.extend(device['ipv6s'])
# Concatenate IPs with comma separator
ip_addresses_str = ','.join(ip_list)
sqm_override = circuit['sqm'] if 'sqm' in circuit else None
bakery.add_circuit(
circuit_name,
node,
data[node]['classid'],
data[node]['up_classid'],
int(circuit['classMinor'], 16),
min_down,
min_up,
circuit['maxDownload'],
circuit['maxUpload'],
int(circuit['classMajor'], 16),
int(circuit['up_classMajor'], 16),
ip_addresses_str,
sqm_override,
)
command = 'class add dev ' + interface_a() + ' parent ' + data[node]['classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ format_rate_for_tc(min_down) + ' ceil '+ format_rate_for_tc(circuit['maxDownload']) + ' prio 3' + quantum(circuit['maxDownload']) + tcComment
linuxTCcommands.append(command)
# SQM Fixup for lower rates (and per-circuit override)
def effective_sqm_str(rate, override, direction):
base = sqm()
# Resolve per-direction token from override string
chosen = None
if override:
try:
ov = str(override).strip().lower()
if '/' in ov:
left, right = ov.split('/', 1)
left = left.strip()
right = right.strip()
chosen = left if direction == 'down' else right
else:
chosen = ov
except Exception:
chosen = None
# If no explicit token for this direction, use default behavior
if not chosen:
try:
thresh = fast_queues_fq_codel()
except Exception:
thresh = 1000.0
if rate >= thresh:
return 'fq_codel'
return sqmFixupRate(rate, base)
if chosen == 'none':
return ''
if chosen == 'fq_codel':
return 'fq_codel'
if chosen == 'cake':
cake_base = base if base.startswith('cake') else 'cake diffserv4'
return sqmFixupRate(rate, cake_base)
return sqmFixupRate(rate, base)
sqm_override = circuit['sqm'] if 'sqm' in circuit else None
useSqm = effective_sqm_str(circuit['maxDownload'], sqm_override, 'down')
if useSqm != '':
command = 'qdisc add dev ' + interface_a() + ' parent ' + circuit['classMajor'] + ':' + circuit['classMinor'] + ' ' + useSqm
linuxTCcommands.append(command)
command = 'class add dev ' + interface_b() + ' parent ' + data[node]['up_classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ format_rate_for_tc(min_up) + ' ceil '+ format_rate_for_tc(circuit['maxUpload']) + ' prio 3' + quantum(circuit['maxUpload'])
linuxTCcommands.append(command)
sqm_override = circuit['sqm'] if 'sqm' in circuit else None
useSqm = effective_sqm_str(circuit['maxUpload'], sqm_override, 'up')
if useSqm != '':
command = 'qdisc add dev ' + interface_b() + ' parent ' + circuit['up_classMajor'] + ':' + circuit['classMinor'] + ' ' + useSqm
linuxTCcommands.append(command)
for device in circuit['devices']:
if device['ipv4s']:
for ipv4 in device['ipv4s']:
ipMapBatch.add_ip_mapping(
str(ipv4),
circuit['classid'],
data[node]['cpuNum'],
False,
circuit.get('circuitID', ''),
device.get('deviceID', ''),
)
requiredIpMappings += 1
#xdpCPUmapCommands.append('./bin/xdp_iphash_to_cpu_cmdline add --ip ' + str(ipv4) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['classid'])
if on_a_stick():
ipMapBatch.add_ip_mapping(
str(ipv4),
circuit['up_classid'],
data[node]['up_cpuNum'],
True,
circuit.get('circuitID', ''),
device.get('deviceID', ''),
)
#xdpCPUmapCommands.append('./bin/xdp_iphash_to_cpu_cmdline add --ip ' + str(ipv4) + ' --cpu ' + data[node]['up_cpuNum'] + ' --classid ' + circuit['up_classid'] + ' --upload 1')
if device['ipv6s']:
for ipv6 in device['ipv6s']:
ipMapBatch.add_ip_mapping(
str(ipv6),
circuit['classid'],
data[node]['cpuNum'],
False,
circuit.get('circuitID', ''),
device.get('deviceID', ''),
)
requiredIpMappings += 1
#xdpCPUmapCommands.append('./bin/xdp_iphash_to_cpu_cmdline add --ip ' + str(ipv6) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['classid'])
if on_a_stick():
ipMapBatch.add_ip_mapping(
str(ipv6),
circuit['up_classid'],
data[node]['up_cpuNum'],
True,
circuit.get('circuitID', ''),
device.get('deviceID', ''),
)
#xdpCPUmapCommands.append('./bin/xdp_iphash_to_cpu_cmdline add --ip ' + str(ipv6) + ' --cpu ' + data[node]['up_cpuNum'] + ' --classid ' + circuit['up_classid'] + ' --upload 1')
shapedDeviceKeys.add(device_shaping_key(circuit, device))
# Recursive call this function for children nodes attached to this node
if 'children' in data[node]:
# Sort children to ensure consistent traversal order
sorted_children = dict(sorted(data[node]['children'].items()))
traverseNetwork(sorted_children)
# Here is the actual call to the recursive traverseNetwork() function.
traverseNetwork(network)
if enable_actual_shell_commands():
ipMappingCapacity = xdp_ip_mapping_capacity()
print(
"Prepared "
+ str(requiredIpMappings)
+ " unique XDP IP mappings against capacity "
+ str(ipMappingCapacity)
)
if requiredIpMappings > ipMappingCapacity:
report_refresh_failure(
"XDP_IP_MAPPING_CAPACITY",
"Required XDP IP mappings ("
+ str(requiredIpMappings)
+ ") exceed current kernel map capacity ("
+ str(ipMappingCapacity)
+ "). Aborting refresh before apply.",
{
"required_ip_mappings": requiredIpMappings,
"kernel_map_capacity": ipMappingCapacity,
"queued_requests": ipMapBatch.length(),
"on_a_stick": on_a_stick(),
"shaped_devices_file": shapedDevicesFile,
"network_json_file": networkJSONfile,
},
"XDP_IP_MAPPING_CAPACITY",
)
qdiscBudgetEstimate = bakery.estimate_qdisc_budget()
if not qdiscBudgetEstimate["ok"]:
interfaceCounts = qdiscBudgetEstimate["interfaces"]
sortedInterfaces = sorted(interfaceCounts.items())
preflightSummary = qdiscBudgetEstimate.get("summary") or ""
report_refresh_failure(
"TC_QDISC_CAPACITY",
(preflightSummary + " Aborting refresh before apply.").strip(),
{
"interfaces": dict(sortedInterfaces),
"interface_details": qdiscBudgetEstimate.get("interface_details", {}),
"safe_budget": qdiscBudgetEstimate["safe_budget"],
"hard_limit": qdiscBudgetEstimate["hard_limit"],
"estimated_total_memory_bytes": qdiscBudgetEstimate.get("estimated_total_memory_bytes"),
"memory_ok": qdiscBudgetEstimate.get("memory_ok"),
"memory_guard_min_available_bytes": qdiscBudgetEstimate.get("memory_guard_min_available_bytes"),
"memory_total_bytes": qdiscBudgetEstimate.get("memory_total_bytes"),
"memory_available_bytes": qdiscBudgetEstimate.get("memory_available_bytes"),
"on_a_stick": on_a_stick(),
"queue_mode": queue_mode(),
"shaped_devices_file": shapedDevicesFile,
"network_json_file": networkJSONfile,
},
"TC_QDISC_CAPACITY",
)
# Save queuingStructure
queuingStructure = {}
queuingStructure['Network'] = network
queuingStructure['lastUsedClassIDCounterByCPU'] = minorByCPU
queuingStructure['generatedPNs'] = generatedPNs
queuingStructure['logical_to_physical_node'] = logical_to_physical_node
queuingStructure['virtual_nodes'] = virtual_nodes
queuing_structure_path = get_state_path("shaping", "queuingStructure.json")
ensure_parent_dir(queuing_structure_path)
with open(queuing_structure_path, 'w') as infile:
json.dump(queuingStructure, infile, indent=4)
# Record start time of actual filter reload
reloadStartTime = datetime.now()
# Clear Prior Settings
# We don't want to do this every time, with lqosd managing queues statefully.
# clearPriorSettings(interface_a(), interface_b())
# Setup XDP and disable XPS regardless of whether it is first run or not (necessary to handle cases where systemctl stop was used)
xdpStartTime = datetime.now()
#if enable_actual_shell_commands():
# Observe mode removes the shaping tree entirely, so clear any live
# per-circuit IP mappings before the commit to avoid pointing packets
# at handles that are about to disappear.
if enable_actual_shell_commands() and observe_mode:
logging.info("# Observe Mode Mapping Quiesce")
try:
clear_ip_mappings()
except Exception as e:
report_refresh_failure(
"XDP_IP_MAPPING_CLEAR_FAILED",
"Failed to clear XDP IP mappings before entering Observe mode: " + str(e),
{
"required_ip_mappings": requiredIpMappings,
"queued_requests": ipMapBatch.length(),
"on_a_stick": on_a_stick(),
"queue_mode": queue_mode(),
},
"XDP_IP_MAPPING_CLEAR_FAILED",
)
# Set up XDP-CPUMAP-TC
logging.info("# XDP Setup")
# Commented out - the daemon does this
#shell('./cpumap-pping/bin/xps_setup.sh -d ' + interfaceA + ' --default --disable')
#shell('./cpumap-pping/bin/xps_setup.sh -d ' + interfaceB + ' --default --disable')
#shell('./cpumap-pping/src/xdp_iphash_to_cpu --dev ' + interfaceA + ' --lan')
#shell('./cpumap-pping/src/xdp_iphash_to_cpu --dev ' + interfaceB + ' --wan')
#shell('./cpumap-pping/src/tc_classify --dev-egress ' + interfaceA)
#shell('./cpumap-pping/src/tc_classify --dev-egress ' + interfaceB)
xdpEndTime = datetime.now()
# Execute actual Linux TC commands
tcStartTime = datetime.now()
# print("Executing linux TC class/qdisc commands")
if observe_mode:
linuxTCcommands = []
linux_tc_path = get_linux_tc_path()
ensure_parent_dir(linux_tc_path)
with open(linux_tc_path, 'w') as f:
for command in linuxTCcommands:
logging.info(command)
f.write(f"{command}\n")
# if logging.DEBUG <= logging.root.level:
# # Do not --force in debug mode, so we can see any errors
# shell("/sbin/tc -b linux_tc.txt")
# else:
# shell("/sbin/tc -f -b linux_tc.txt")
bakery.commit()
tcEndTime = datetime.now()
# print("Executed " + str(len(linuxTCcommands)) + " linux TC class/qdisc commands")
# Execute actual XDP-CPUMAP-TC filter commands
xdpFilterStartTime = datetime.now()
print("Executing XDP-CPUMAP-TC IP filter commands")
numXdpCommands = ipMapBatch.length()
if enable_actual_shell_commands():
if observe_mode:
print("Observe mode active; skipping XDP-CPUMAP-TC IP filter apply after clearing mappings")
numXdpCommands = 0
else:
ipMapBatch.finish_ip_mappings()
try:
ipMapBatch.submit()
except Exception as e:
report_refresh_failure(
"XDP_IP_MAPPING_APPLY_FAILED",
"Failed to apply XDP IP mappings: " + str(e),
{
"required_ip_mappings": requiredIpMappings,
"queued_requests": numXdpCommands,
"on_a_stick": on_a_stick(),
},
"XDP_IP_MAPPING_APPLY_FAILED",
)
#for command in xdpCPUmapCommands:
# logging.info(command)
# commands = command.split(' ')
# proc = subprocess.Popen(commands, stdout=subprocess.DEVNULL)
else:
ipMapBatch.log()
#for command in xdpCPUmapCommands:
# logging.info(command)
print("Executed " + str(numXdpCommands) + " XDP-CPUMAP-TC IP filter commands")
#print(xdpCPUmapCommands)
xdpFilterEndTime = datetime.now()
# Record end time of all reload commands
reloadEndTime = datetime.now()
# Recap - warn operator if devices were skipped
validParentNodes = collect_parent_node_names(network)
devicesSkipped = build_unshaped_device_report(
subscriberCircuits,
shapedDeviceKeys,
validParentNodes,
flat_network,
)
if len(devicesSkipped) > 0:
flat_mode_generated_parent_only = flat_network and all(
entry.get("reasonCode") == "unattached_flat_network" for entry in devicesSkipped
)
if flat_mode_generated_parent_only:
print(
f"Flat network mode assigned {len(devicesSkipped)} device(s) to generated parent queues."
)
else:
warnings.warn(
str(len(devicesSkipped)) + " device(s) were not shaped. Detailed reasons are listed below.",
stacklevel=2,
)
print("Devices not shaped:")
for entry in devicesSkipped:
print(format_unshaped_device_line(entry))
# DIY/manual mode snapshots the active ShapedDevices.csv for update-only diffs.
# Built-in integrations no longer emit ShapedDevices.csv, so skip the legacy
# snapshot there instead of failing the whole scheduler refresh.
if os.path.isfile(shapedDevicesFile):
last_loaded_path = get_state_path("shaping", "ShapedDevices.lastLoaded.csv")
ensure_parent_dir(last_loaded_path)
shutil.copyfile(shapedDevicesFile, last_loaded_path)
# Save for stats
stats_by_circuit_path = get_state_path("stats", "statsByCircuit.json")
ensure_parent_dir(stats_by_circuit_path)
with open(stats_by_circuit_path, 'w') as f:
f.write(json.dumps(subscriberCircuits, indent=4))
stats_by_parent_node_path = get_state_path("stats", "statsByParentNode.json")
ensure_parent_dir(stats_by_parent_node_path)
with open(stats_by_parent_node_path, 'w') as f:
f.write(json.dumps(parentNodes, indent=4))
# Record time this run completed at
# filename = os.path.join(_here, 'lastRun.txt')
last_run_path = get_state_path("stats", "lastRun.txt")
ensure_parent_dir(last_run_path)
with open(last_run_path, 'w') as file:
file.write(datetime.now().strftime("%d-%b-%Y (%H:%M:%S.%f)"))
# Report reload time
reloadTimeSeconds = ((reloadEndTime - reloadStartTime).seconds) + (((reloadEndTime - reloadStartTime).microseconds) / 1000000)
tcTimeSeconds = ((tcEndTime - tcStartTime).seconds) + (((tcEndTime - tcStartTime).microseconds) / 1000000)
xdpSetupTimeSeconds = ((xdpEndTime - xdpStartTime).seconds) + (((xdpEndTime - xdpStartTime).microseconds) / 1000000)
xdpFilterTimeSeconds = ((xdpFilterEndTime - xdpFilterStartTime).seconds) + (((xdpFilterEndTime - xdpFilterStartTime).microseconds) / 1000000)
print("Queue and IP filter reload completed in " + "{:g}".format(round(reloadTimeSeconds,1)) + " seconds")
print("\tTC commands: \t" + "{:g}".format(round(tcTimeSeconds,1)) + " seconds")
print("\tXDP setup: \t " + "{:g}".format(round(xdpSetupTimeSeconds,1)) + " seconds")
print("\tXDP filters: \t " + "{:g}".format(round(xdpFilterTimeSeconds,4)) + " seconds")
# Done
print("refreshShapers completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
def refreshShapersUpdateOnly():
# Starting
print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
# Warn user if enableActualShellCommands is False, because that would mean no actual commands are executing
if enable_actual_shell_commands() == False:
warnings.warn("enableActualShellCommands is set to False. None of the commands below will actually be executed. Simulated run.", stacklevel=2)
# Files
shapedDevicesFile = get_shaped_devices_path()
networkJSONfile = get_network_json_path()
# Check validation
safeToRunRefresh = False
if (validateNetworkAndDevices() == True):
safeToRunRefresh = True
else:
message = "Validation failed. Will now exit."
warnings.warn(message, stacklevel=2)
raise ValidationFailure(message)
if safeToRunRefresh == True:
networkChanged = False
devicesChanged = False
# Check for changes to network.json
last_good_config_json_path = get_last_good_config_json_path()
if os.path.isfile(last_good_config_json_path):
with open(last_good_config_json_path, 'r') as j:
originalNetwork = json.loads(j.read())
with open(networkJSONfile, 'r') as j:
newestNetwork = json.loads(j.read())
ddiff = DeepDiff(originalNetwork, newestNetwork, ignore_order=True)
if ddiff != {}:
networkChanged = True
# Check for changes to ShapedDevices.csv
newlyUpdatedSubscriberCircuits, newlyUpdatedDictForCircuitsWithoutParentNodes = loadSubscriberCircuits(shapedDevicesFile)
lastLoadedSubscriberCircuits, lastLoadedDictForCircuitsWithoutParentNodes = loadSubscriberCircuits(get_last_loaded_shaped_devices_path())
newlyUpdatedSubscriberCircuitsByID = {}
for circuit in newlyUpdatedSubscriberCircuits:
circuitid = circuit['circuitID']
newlyUpdatedSubscriberCircuitsByID[circuitid] = circuit
lastLoadedSubscriberCircuitsByID = {}
for circuit in lastLoadedSubscriberCircuits:
circuitid = circuit['circuitID']
lastLoadedSubscriberCircuitsByID[circuitid] = circuit
for circuitID, circuit in lastLoadedSubscriberCircuitsByID.items():
if (circuitID in newlyUpdatedSubscriberCircuitsByID) and (circuitID in lastLoadedSubscriberCircuitsByID):
if newlyUpdatedSubscriberCircuitsByID[circuitID]['maxDownload'] != lastLoadedSubscriberCircuitsByID[circuitID]['maxDownload']:
devicesChanged = True
if newlyUpdatedSubscriberCircuitsByID[circuitID]['maxUpload'] != lastLoadedSubscriberCircuitsByID[circuitID]['maxUpload']:
devicesChanged = True
if newlyUpdatedSubscriberCircuitsByID[circuitID]['minDownload'] != lastLoadedSubscriberCircuitsByID[circuitID]['minDownload']:
devicesChanged = True
if newlyUpdatedSubscriberCircuitsByID[circuitID]['minUpload'] != lastLoadedSubscriberCircuitsByID[circuitID]['minUpload']:
devicesChanged = True
if newlyUpdatedSubscriberCircuitsByID[circuitID]['devices'] != lastLoadedSubscriberCircuitsByID[circuitID]['devices']:
devicesChanged = True
if newlyUpdatedSubscriberCircuitsByID[circuitID]['ParentNode'] != lastLoadedSubscriberCircuitsByID[circuitID]['ParentNode']:
devicesChanged = True
if newlyUpdatedSubscriberCircuitsByID[circuitID].get('ParentNodeID', '') != lastLoadedSubscriberCircuitsByID[circuitID].get('ParentNodeID', ''):
devicesChanged = True
else:
devicesChanged = True
for circuitID, circuit in newlyUpdatedSubscriberCircuitsByID.items():
if (circuitID not in lastLoadedSubscriberCircuitsByID):
devicesChanged = True
if devicesChanged or networkChanged:
print('Observed changes to runtime shaping inputs. Applying full reload now')
refreshShapers()
else:
print('Observed no changes to runtime shaping inputs. Leaving queues as is.')
# Done
print("refreshShapersUpdateOnly completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
if __name__ == '__main__':
# Check that the host lqosd is running
if is_lqosd_alive():
print("lqosd is running")
else:
print("ERROR: lqosd is not running. Aborting")
os._exit(-1)
# Check that the configuration file is usable
if check_config():
print("Configuration from /etc/lqos.conf is usable")
else:
print("ERROR: Unable to load configuration from /etc/lqos.conf")
os._exit(-1)
# Check that we aren't running LibreQoS.py more than once at a time
if is_libre_already_running():
print("LibreQoS.py is already running in another process. Aborting.")
os._exit(-1)
# We've got this far, so create a lock file
create_lock_file()
parser = argparse.ArgumentParser()
parser.add_argument(
'-d', '--debug',
help="Print lots of debugging statements",
action="store_const", dest="loglevel", const=logging.DEBUG,
default=logging.WARNING,
)
parser.add_argument(
'-v', '--verbose',
help="Be verbose",
action="store_const", dest="loglevel", const=logging.INFO,
)
parser.add_argument(
'--updateonly',
help="Only update to reflect changes in ShapedDevices.csv (partial reload)",
action=argparse.BooleanOptionalAction,
)
parser.add_argument(
'--validate',
help="Just validate network.json and ShapedDevices.csv",
action=argparse.BooleanOptionalAction,
)
parser.add_argument(
'--clearrules',
help="Clear ip filters, qdiscs, and xdp setup if any",
action=argparse.BooleanOptionalAction,
)
parser.add_argument(
'--planner-reset',
help="Delete planner state file before running",
action=argparse.BooleanOptionalAction,
)
args = parser.parse_args()
logging.basicConfig(level=args.loglevel)
if getattr(args, 'planner_reset', False):
try:
state_path = get_planner_state_path()
if os.path.exists(state_path):
os.remove(state_path)
print(f"Removed planner state: {state_path}")
except Exception as e:
print(f"Warning: could not remove planner state: {e}")
exit_code = 0
try:
if args.validate:
status = validateNetworkAndDevices()
elif args.clearrules:
tearDown(interface_a(), interface_b())
elif args.updateonly:
# Single-interface updates don't work at all right now.
if on_a_stick():
print("--updateonly is not supported for single-interface configurations")
exit_code = -1
else:
refreshShapersUpdateOnly()
else:
# Refresh and/or set up queues
refreshShapers()
except RefreshFailure:
exit_code = 1
finally:
free_lock_file()
if exit_code != 0:
os._exit(exit_code)
Reference in new issue View Git Blame Copy Permalink