mirror of
https://github.com/LibreQoE/LibreQoS.git
synced 2025-02-25 18:55:32 -06:00
Add --validate option. Do not refreshShapers if validation fails, unless first boot
This commit is contained in:
Robert Chacón
GitHub
parent
7fe9537be4
commit
aa4c6b5125
639
v1.2/LibreQoS.py
639
v1.2/LibreQoS.py
@ -214,93 +214,130 @@ def refreshShapers():
|
|||||||
networkJSONfile = 'network.json'
|
networkJSONfile = 'network.json'
|
||||||
|
|
||||||
# Check validation
|
# Check validation
|
||||||
|
safeToRunRefresh = False
|
||||||
if (validateNetworkAndDevices() == True):
|
if (validateNetworkAndDevices() == True):
|
||||||
shutil.copyfile('ShapedDevices.csv', 'lastGoodConfig.csv')
|
shutil.copyfile('ShapedDevices.csv', 'lastGoodConfig.csv')
|
||||||
shutil.copyfile('network.json', 'lastGoodConfig.json')
|
shutil.copyfile('network.json', 'lastGoodConfig.json')
|
||||||
print("Backed up good config as lastGoodConfig.csv and lastGoodConfig.json")
|
print("Backed up good config as lastGoodConfig.csv and lastGoodConfig.json")
|
||||||
|
safeToRunRefresh = True
|
||||||
else:
|
else:
|
||||||
warnings.warn("Validation failed - pulling from last good conifg")
|
if (isThisFirstRunSinceBoot == False):
|
||||||
shapedDevicesFile = 'lastGoodConfig.csv'
|
warnings.warn("Validation failed. Because this is not the first run since boot - will exit.")
|
||||||
networkJSONfile = 'lastGoodConfig.json'
|
safeToRunRefresh = False
|
||||||
|
else:
|
||||||
|
warnings.warn("Validation failed. However - because this is the first run since boot - will load queues from last good config")
|
||||||
|
shapedDevicesFile = 'lastGoodConfig.csv'
|
||||||
|
networkJSONfile = 'lastGoodConfig.json'
|
||||||
|
safeToRunRefresh = True
|
||||||
|
|
||||||
# Load Subscriber Circuits & Devices
|
if safeToRunRefresh == True:
|
||||||
subscriberCircuits = []
|
# Load Subscriber Circuits & Devices
|
||||||
knownCircuitIDs = []
|
subscriberCircuits = []
|
||||||
with open(shapedDevicesFile) as csv_file:
|
knownCircuitIDs = []
|
||||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
with open(shapedDevicesFile) as csv_file:
|
||||||
# Remove comments if any
|
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||||
commentsRemoved = []
|
# Remove comments if any
|
||||||
for row in csv_reader:
|
commentsRemoved = []
|
||||||
if not row[0].startswith('#'):
|
for row in csv_reader:
|
||||||
commentsRemoved.append(row)
|
if not row[0].startswith('#'):
|
||||||
# Remove header
|
commentsRemoved.append(row)
|
||||||
commentsRemoved.pop(0)
|
# Remove header
|
||||||
for row in commentsRemoved:
|
commentsRemoved.pop(0)
|
||||||
circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row
|
for row in commentsRemoved:
|
||||||
ipv4_hosts = []
|
circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row
|
||||||
# Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s seperated by commas. Split them up and parse each
|
ipv4_hosts = []
|
||||||
if ipv4_input != "":
|
# Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s seperated by commas. Split them up and parse each
|
||||||
ipv4_input = ipv4_input.replace(' ','')
|
if ipv4_input != "":
|
||||||
if "," in ipv4_input:
|
ipv4_input = ipv4_input.replace(' ','')
|
||||||
ipv4_list = ipv4_input.split(',')
|
if "," in ipv4_input:
|
||||||
else:
|
ipv4_list = ipv4_input.split(',')
|
||||||
ipv4_list = [ipv4_input]
|
|
||||||
for ipEntry in ipv4_list:
|
|
||||||
if '/32' in ipEntry:
|
|
||||||
ipv4_hosts.append(ipEntry.replace('/32',''))
|
|
||||||
elif '/' in ipEntry:
|
|
||||||
theseHosts = ipaddress.ip_network(ipEntry).hosts()
|
|
||||||
for host in theseHosts:
|
|
||||||
host = str(host)
|
|
||||||
if '/32' in host:
|
|
||||||
host = host.replace('/32','')
|
|
||||||
ipv4_hosts.append(host)
|
|
||||||
else:
|
else:
|
||||||
ipv4_hosts.append(ipEntry)
|
ipv4_list = [ipv4_input]
|
||||||
ipv6_hosts = []
|
for ipEntry in ipv4_list:
|
||||||
if ipv6_input != "":
|
if '/32' in ipEntry:
|
||||||
ipv6_input = ipv6_input.replace(' ','')
|
ipv4_hosts.append(ipEntry.replace('/32',''))
|
||||||
if "," in ipv6_input:
|
elif '/' in ipEntry:
|
||||||
ipv6_list = ipv6_input.split(',')
|
theseHosts = ipaddress.ip_network(ipEntry).hosts()
|
||||||
else:
|
for host in theseHosts:
|
||||||
ipv6_list = [ipv6_input]
|
host = str(host)
|
||||||
for ipEntry in ipv6_list:
|
if '/32' in host:
|
||||||
if '/128' in ipEntry:
|
host = host.replace('/32','')
|
||||||
ipv6_hosts.append(ipEntry)
|
ipv4_hosts.append(host)
|
||||||
elif '/' in ipEntry:
|
else:
|
||||||
theseHosts = ipaddress.ip_network(ipEntry).hosts()
|
ipv4_hosts.append(ipEntry)
|
||||||
for host in theseHosts:
|
ipv6_hosts = []
|
||||||
ipv6_hosts.append(str(host))
|
if ipv6_input != "":
|
||||||
|
ipv6_input = ipv6_input.replace(' ','')
|
||||||
|
if "," in ipv6_input:
|
||||||
|
ipv6_list = ipv6_input.split(',')
|
||||||
else:
|
else:
|
||||||
ipv6_hosts.append(ipEntry)
|
ipv6_list = [ipv6_input]
|
||||||
# If there is something in the circuit ID field
|
for ipEntry in ipv6_list:
|
||||||
if circuitID != "":
|
if '/128' in ipEntry:
|
||||||
# Seen circuit before
|
ipv6_hosts.append(ipEntry)
|
||||||
if circuitID in knownCircuitIDs:
|
elif '/' in ipEntry:
|
||||||
for circuit in subscriberCircuits:
|
theseHosts = ipaddress.ip_network(ipEntry).hosts()
|
||||||
if circuit['circuitID'] == circuitID:
|
for host in theseHosts:
|
||||||
if circuit['ParentNode'] != "none":
|
ipv6_hosts.append(str(host))
|
||||||
if circuit['ParentNode'] != ParentNode:
|
else:
|
||||||
errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID
|
ipv6_hosts.append(ipEntry)
|
||||||
raise ValueError(errorMessageString)
|
# If there is something in the circuit ID field
|
||||||
if ((circuit['downloadMin'] != round(int(downloadMin)*tcpOverheadFactor))
|
if circuitID != "":
|
||||||
or (circuit['uploadMin'] != round(int(uploadMin)*tcpOverheadFactor))
|
# Seen circuit before
|
||||||
or (circuit['downloadMax'] != round(int(downloadMax)*tcpOverheadFactor))
|
if circuitID in knownCircuitIDs:
|
||||||
or (circuit['uploadMax'] != round(int(uploadMax)*tcpOverheadFactor))):
|
for circuit in subscriberCircuits:
|
||||||
warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different bandwidth parameters than other devices on this circuit. Will instead use the bandwidth parameters defined by the first device added to its circuit.")
|
if circuit['circuitID'] == circuitID:
|
||||||
devicesListForCircuit = circuit['devices']
|
if circuit['ParentNode'] != "none":
|
||||||
thisDevice = {
|
if circuit['ParentNode'] != ParentNode:
|
||||||
"deviceID": deviceID,
|
errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID
|
||||||
"deviceName": deviceName,
|
raise ValueError(errorMessageString)
|
||||||
"mac": mac,
|
if ((circuit['downloadMin'] != round(int(downloadMin)*tcpOverheadFactor))
|
||||||
"ipv4s": ipv4_hosts,
|
or (circuit['uploadMin'] != round(int(uploadMin)*tcpOverheadFactor))
|
||||||
"ipv6s": ipv6_hosts,
|
or (circuit['downloadMax'] != round(int(downloadMax)*tcpOverheadFactor))
|
||||||
}
|
or (circuit['uploadMax'] != round(int(uploadMax)*tcpOverheadFactor))):
|
||||||
devicesListForCircuit.append(thisDevice)
|
warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different bandwidth parameters than other devices on this circuit. Will instead use the bandwidth parameters defined by the first device added to its circuit.")
|
||||||
circuit['devices'] = devicesListForCircuit
|
devicesListForCircuit = circuit['devices']
|
||||||
# Have not seen circuit before
|
thisDevice = {
|
||||||
|
"deviceID": deviceID,
|
||||||
|
"deviceName": deviceName,
|
||||||
|
"mac": mac,
|
||||||
|
"ipv4s": ipv4_hosts,
|
||||||
|
"ipv6s": ipv6_hosts,
|
||||||
|
}
|
||||||
|
devicesListForCircuit.append(thisDevice)
|
||||||
|
circuit['devices'] = devicesListForCircuit
|
||||||
|
# Have not seen circuit before
|
||||||
|
else:
|
||||||
|
knownCircuitIDs.append(circuitID)
|
||||||
|
if ParentNode == "":
|
||||||
|
ParentNode = "none"
|
||||||
|
ParentNode = ParentNode.strip()
|
||||||
|
deviceListForCircuit = []
|
||||||
|
thisDevice = {
|
||||||
|
"deviceID": deviceID,
|
||||||
|
"deviceName": deviceName,
|
||||||
|
"mac": mac,
|
||||||
|
"ipv4s": ipv4_hosts,
|
||||||
|
"ipv6s": ipv6_hosts,
|
||||||
|
}
|
||||||
|
deviceListForCircuit.append(thisDevice)
|
||||||
|
thisCircuit = {
|
||||||
|
"circuitID": circuitID,
|
||||||
|
"circuitName": circuitName,
|
||||||
|
"ParentNode": ParentNode,
|
||||||
|
"devices": deviceListForCircuit,
|
||||||
|
"downloadMin": round(int(downloadMin)*tcpOverheadFactor),
|
||||||
|
"uploadMin": round(int(uploadMin)*tcpOverheadFactor),
|
||||||
|
"downloadMax": round(int(downloadMax)*tcpOverheadFactor),
|
||||||
|
"uploadMax": round(int(uploadMax)*tcpOverheadFactor),
|
||||||
|
"qdisc": '',
|
||||||
|
}
|
||||||
|
subscriberCircuits.append(thisCircuit)
|
||||||
|
# If there is nothing in the circuit ID field
|
||||||
else:
|
else:
|
||||||
knownCircuitIDs.append(circuitID)
|
# Copy deviceName to circuitName if none defined already
|
||||||
|
if circuitName == "":
|
||||||
|
circuitName = deviceName
|
||||||
if ParentNode == "":
|
if ParentNode == "":
|
||||||
ParentNode = "none"
|
ParentNode = "none"
|
||||||
ParentNode = ParentNode.strip()
|
ParentNode = ParentNode.strip()
|
||||||
@ -325,238 +362,209 @@ def refreshShapers():
|
|||||||
"qdisc": '',
|
"qdisc": '',
|
||||||
}
|
}
|
||||||
subscriberCircuits.append(thisCircuit)
|
subscriberCircuits.append(thisCircuit)
|
||||||
# If there is nothing in the circuit ID field
|
|
||||||
else:
|
|
||||||
# Copy deviceName to circuitName if none defined already
|
|
||||||
if circuitName == "":
|
|
||||||
circuitName = deviceName
|
|
||||||
if ParentNode == "":
|
|
||||||
ParentNode = "none"
|
|
||||||
ParentNode = ParentNode.strip()
|
|
||||||
deviceListForCircuit = []
|
|
||||||
thisDevice = {
|
|
||||||
"deviceID": deviceID,
|
|
||||||
"deviceName": deviceName,
|
|
||||||
"mac": mac,
|
|
||||||
"ipv4s": ipv4_hosts,
|
|
||||||
"ipv6s": ipv6_hosts,
|
|
||||||
}
|
|
||||||
deviceListForCircuit.append(thisDevice)
|
|
||||||
thisCircuit = {
|
|
||||||
"circuitID": circuitID,
|
|
||||||
"circuitName": circuitName,
|
|
||||||
"ParentNode": ParentNode,
|
|
||||||
"devices": deviceListForCircuit,
|
|
||||||
"downloadMin": round(int(downloadMin)*tcpOverheadFactor),
|
|
||||||
"uploadMin": round(int(uploadMin)*tcpOverheadFactor),
|
|
||||||
"downloadMax": round(int(downloadMax)*tcpOverheadFactor),
|
|
||||||
"uploadMax": round(int(uploadMax)*tcpOverheadFactor),
|
|
||||||
"qdisc": '',
|
|
||||||
}
|
|
||||||
subscriberCircuits.append(thisCircuit)
|
|
||||||
|
|
||||||
# Load network heirarchy
|
# Load network heirarchy
|
||||||
with open(networkJSONfile, 'r') as j:
|
with open(networkJSONfile, 'r') as j:
|
||||||
network = json.loads(j.read())
|
network = json.loads(j.read())
|
||||||
|
|
||||||
# Pull rx/tx queues / CPU cores avaialble
|
# Pull rx/tx queues / CPU cores avaialble
|
||||||
queuesAvailable = findQueuesAvailable()
|
queuesAvailable = findQueuesAvailable()
|
||||||
|
|
||||||
# Generate Parent Nodes. Spread ShapedDevices.csv which lack defined ParentNode across these (balance across CPUs)
|
# Generate Parent Nodes. Spread ShapedDevices.csv which lack defined ParentNode across these (balance across CPUs)
|
||||||
generatedPNs = []
|
generatedPNs = []
|
||||||
for x in range(queuesAvailable):
|
for x in range(queuesAvailable):
|
||||||
genPNname = "Generated_PN_" + str(x+1)
|
genPNname = "Generated_PN_" + str(x+1)
|
||||||
network[genPNname] = {
|
network[genPNname] = {
|
||||||
"downloadBandwidthMbps":generatedPNDownloadMbps,
|
"downloadBandwidthMbps":generatedPNDownloadMbps,
|
||||||
"uploadBandwidthMbps":generatedPNUploadMbps
|
"uploadBandwidthMbps":generatedPNUploadMbps
|
||||||
}
|
}
|
||||||
generatedPNs.append(genPNname)
|
generatedPNs.append(genPNname)
|
||||||
genPNcounter = 0
|
genPNcounter = 0
|
||||||
for circuit in subscriberCircuits:
|
for circuit in subscriberCircuits:
|
||||||
if circuit['ParentNode'] == 'none':
|
if circuit['ParentNode'] == 'none':
|
||||||
circuit['ParentNode'] = generatedPNs[genPNcounter]
|
circuit['ParentNode'] = generatedPNs[genPNcounter]
|
||||||
genPNcounter += 1
|
genPNcounter += 1
|
||||||
if genPNcounter >= queuesAvailable:
|
if genPNcounter >= queuesAvailable:
|
||||||
genPNcounter = 0
|
genPNcounter = 0
|
||||||
|
|
||||||
# Find the bandwidth minimums for each node by combining mimimums of devices lower in that node's heirarchy
|
# Find the bandwidth minimums for each node by combining mimimums of devices lower in that node's heirarchy
|
||||||
def findBandwidthMins(data, depth):
|
def findBandwidthMins(data, depth):
|
||||||
tabs = ' ' * depth
|
tabs = ' ' * depth
|
||||||
minDownload = 0
|
minDownload = 0
|
||||||
minUpload = 0
|
minUpload = 0
|
||||||
for elem in data:
|
for elem in data:
|
||||||
for circuit in subscriberCircuits:
|
for circuit in subscriberCircuits:
|
||||||
if elem == circuit['ParentNode']:
|
if elem == circuit['ParentNode']:
|
||||||
minDownload += circuit['downloadMin']
|
minDownload += circuit['downloadMin']
|
||||||
minUpload += circuit['uploadMin']
|
minUpload += circuit['uploadMin']
|
||||||
if 'children' in data[elem]:
|
if 'children' in data[elem]:
|
||||||
minDL, minUL = findBandwidthMins(data[elem]['children'], depth+1)
|
minDL, minUL = findBandwidthMins(data[elem]['children'], depth+1)
|
||||||
minDownload += minDL
|
minDownload += minDL
|
||||||
minUpload += minUL
|
minUpload += minUL
|
||||||
data[elem]['downloadBandwidthMbpsMin'] = minDownload
|
data[elem]['downloadBandwidthMbpsMin'] = minDownload
|
||||||
data[elem]['uploadBandwidthMbpsMin'] = minUpload
|
data[elem]['uploadBandwidthMbpsMin'] = minUpload
|
||||||
return minDownload, minUpload
|
return minDownload, minUpload
|
||||||
|
|
||||||
minDownload, minUpload = findBandwidthMins(network, 0)
|
minDownload, minUpload = findBandwidthMins(network, 0)
|
||||||
|
|
||||||
# Parse network structure. For each tier, create corresponding HTB and leaf classes. Prepare for execution later
|
# Parse network structure. For each tier, create corresponding HTB and leaf classes. Prepare for execution later
|
||||||
linuxTCcommands = []
|
linuxTCcommands = []
|
||||||
xdpCPUmapCommands = []
|
xdpCPUmapCommands = []
|
||||||
devicesShaped = []
|
devicesShaped = []
|
||||||
parentNodes = []
|
parentNodes = []
|
||||||
def traverseNetwork(data, depth, major, minor, queue, parentClassID, parentMaxDL, parentMaxUL):
|
def traverseNetwork(data, depth, major, minor, queue, parentClassID, parentMaxDL, parentMaxUL):
|
||||||
tabs = ' ' * depth
|
tabs = ' ' * depth
|
||||||
for elem in data:
|
for elem in data:
|
||||||
elemClassID = hex(major) + ':' + hex(minor)
|
elemClassID = hex(major) + ':' + hex(minor)
|
||||||
# Cap based on this node's max bandwidth, or parent node's max bandwidth, whichever is lower
|
# Cap based on this node's max bandwidth, or parent node's max bandwidth, whichever is lower
|
||||||
elemDownloadMax = min(data[elem]['downloadBandwidthMbps'],parentMaxDL)
|
elemDownloadMax = min(data[elem]['downloadBandwidthMbps'],parentMaxDL)
|
||||||
elemUploadMax = min(data[elem]['uploadBandwidthMbps'],parentMaxUL)
|
elemUploadMax = min(data[elem]['uploadBandwidthMbps'],parentMaxUL)
|
||||||
# Calculations are done in findBandwidthMins(), determine optimal HTB rates (mins) and ceils (maxs)
|
# Calculations are done in findBandwidthMins(), determine optimal HTB rates (mins) and ceils (maxs)
|
||||||
# For some reason that doesn't always yield the expected result, so it's better to play with ceil more than rate
|
# For some reason that doesn't always yield the expected result, so it's better to play with ceil more than rate
|
||||||
# Here we override the rate as 95% of ceil.
|
# Here we override the rate as 95% of ceil.
|
||||||
elemDownloadMin = round(elemDownloadMax*.95)
|
elemDownloadMin = round(elemDownloadMax*.95)
|
||||||
elemUploadMin = round(elemUploadMax*.95)
|
elemUploadMin = round(elemUploadMax*.95)
|
||||||
linuxTCcommands.append('class add dev ' + interfaceA + ' parent ' + parentClassID + ' classid ' + hex(minor) + ' htb rate '+ str(round(elemDownloadMin)) + 'mbit ceil '+ str(round(elemDownloadMax)) + 'mbit prio 3')
|
linuxTCcommands.append('class add dev ' + interfaceA + ' parent ' + parentClassID + ' classid ' + hex(minor) + ' htb rate '+ str(round(elemDownloadMin)) + 'mbit ceil '+ str(round(elemDownloadMax)) + 'mbit prio 3')
|
||||||
linuxTCcommands.append('class add dev ' + interfaceB + ' parent ' + parentClassID + ' classid ' + hex(minor) + ' htb rate '+ str(round(elemUploadMin)) + 'mbit ceil '+ str(round(elemUploadMax)) + 'mbit prio 3')
|
linuxTCcommands.append('class add dev ' + interfaceB + ' parent ' + parentClassID + ' classid ' + hex(minor) + ' htb rate '+ str(round(elemUploadMin)) + 'mbit ceil '+ str(round(elemUploadMax)) + 'mbit prio 3')
|
||||||
thisParentNode = {
|
thisParentNode = {
|
||||||
"parentNodeName": elem,
|
"parentNodeName": elem,
|
||||||
"classID": elemClassID,
|
"classID": elemClassID,
|
||||||
"downloadMax": elemDownloadMax,
|
"downloadMax": elemDownloadMax,
|
||||||
"uploadMax": elemUploadMax,
|
"uploadMax": elemUploadMax,
|
||||||
}
|
}
|
||||||
parentNodes.append(thisParentNode)
|
parentNodes.append(thisParentNode)
|
||||||
minor += 1
|
minor += 1
|
||||||
for circuit in subscriberCircuits:
|
for circuit in subscriberCircuits:
|
||||||
#If a device from ShapedDevices.csv lists this elem as its Parent Node, attach it as a leaf to this elem HTB
|
#If a device from ShapedDevices.csv lists this elem as its Parent Node, attach it as a leaf to this elem HTB
|
||||||
if elem == circuit['ParentNode']:
|
if elem == circuit['ParentNode']:
|
||||||
maxDownload = min(circuit['downloadMax'],elemDownloadMax)
|
maxDownload = min(circuit['downloadMax'],elemDownloadMax)
|
||||||
maxUpload = min(circuit['uploadMax'],elemUploadMax)
|
maxUpload = min(circuit['uploadMax'],elemUploadMax)
|
||||||
minDownload = min(circuit['downloadMin'],maxDownload)
|
minDownload = min(circuit['downloadMin'],maxDownload)
|
||||||
minUpload = min(circuit['uploadMin'],maxUpload)
|
minUpload = min(circuit['uploadMin'],maxUpload)
|
||||||
linuxTCcommands.append('class add dev ' + interfaceA + ' parent ' + elemClassID + ' classid ' + hex(minor) + ' htb rate '+ str(minDownload) + 'mbit ceil '+ str(maxDownload) + 'mbit prio 3')
|
linuxTCcommands.append('class add dev ' + interfaceA + ' parent ' + elemClassID + ' classid ' + hex(minor) + ' htb rate '+ str(minDownload) + 'mbit ceil '+ str(maxDownload) + 'mbit prio 3')
|
||||||
linuxTCcommands.append('qdisc add dev ' + interfaceA + ' parent ' + hex(major) + ':' + hex(minor) + ' ' + fqOrCAKE)
|
linuxTCcommands.append('qdisc add dev ' + interfaceA + ' parent ' + hex(major) + ':' + hex(minor) + ' ' + fqOrCAKE)
|
||||||
linuxTCcommands.append('class add dev ' + interfaceB + ' parent ' + elemClassID + ' classid ' + hex(minor) + ' htb rate '+ str(minUpload) + 'mbit ceil '+ str(maxUpload) + 'mbit prio 3')
|
linuxTCcommands.append('class add dev ' + interfaceB + ' parent ' + elemClassID + ' classid ' + hex(minor) + ' htb rate '+ str(minUpload) + 'mbit ceil '+ str(maxUpload) + 'mbit prio 3')
|
||||||
linuxTCcommands.append('qdisc add dev ' + interfaceB + ' parent ' + hex(major) + ':' + hex(minor) + ' ' + fqOrCAKE)
|
linuxTCcommands.append('qdisc add dev ' + interfaceB + ' parent ' + hex(major) + ':' + hex(minor) + ' ' + fqOrCAKE)
|
||||||
parentString = hex(major) + ':'
|
parentString = hex(major) + ':'
|
||||||
flowIDstring = hex(major) + ':' + hex(minor)
|
flowIDstring = hex(major) + ':' + hex(minor)
|
||||||
circuit['qdisc'] = flowIDstring
|
circuit['qdisc'] = flowIDstring
|
||||||
for device in circuit['devices']:
|
for device in circuit['devices']:
|
||||||
if device['ipv4s']:
|
if device['ipv4s']:
|
||||||
for ipv4 in device['ipv4s']:
|
for ipv4 in device['ipv4s']:
|
||||||
xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + hex(queue-1) + ' --classid ' + flowIDstring)
|
xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + hex(queue-1) + ' --classid ' + flowIDstring)
|
||||||
if device['deviceName'] not in devicesShaped:
|
if device['deviceName'] not in devicesShaped:
|
||||||
devicesShaped.append(device['deviceName'])
|
devicesShaped.append(device['deviceName'])
|
||||||
minor += 1
|
minor += 1
|
||||||
# Recursive call this function for children nodes attached to this node
|
# Recursive call this function for children nodes attached to this node
|
||||||
if 'children' in data[elem]:
|
if 'children' in data[elem]:
|
||||||
# We need to keep tabs on the minor counter, because we can't have repeating class IDs. Here, we bring back the minor counter from the recursive function
|
# We need to keep tabs on the minor counter, because we can't have repeating class IDs. Here, we bring back the minor counter from the recursive function
|
||||||
minor = traverseNetwork(data[elem]['children'], depth+1, major, minor+1, queue, elemClassID, elemDownloadMax, elemUploadMax)
|
minor = traverseNetwork(data[elem]['children'], depth+1, major, minor+1, queue, elemClassID, elemDownloadMax, elemUploadMax)
|
||||||
# If top level node, increment to next queue / cpu core
|
# If top level node, increment to next queue / cpu core
|
||||||
if depth == 0:
|
if depth == 0:
|
||||||
if queue >= queuesAvailable:
|
if queue >= queuesAvailable:
|
||||||
queue = 1
|
queue = 1
|
||||||
major = queue
|
major = queue
|
||||||
else:
|
else:
|
||||||
queue += 1
|
queue += 1
|
||||||
major += 1
|
major += 1
|
||||||
return minor
|
return minor
|
||||||
|
|
||||||
# Print structure of network.json in debug or verbose mode
|
# Print structure of network.json in debug or verbose mode
|
||||||
logging.info(json.dumps(network, indent=4))
|
logging.info(json.dumps(network, indent=4))
|
||||||
|
|
||||||
# Here is the actual call to the recursive traverseNetwork() function. finalMinor is not used.
|
# Here is the actual call to the recursive traverseNetwork() function. finalMinor is not used.
|
||||||
finalMinor = traverseNetwork(network, 0, major=1, minor=3, queue=1, parentClassID="1:1", parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps)
|
finalMinor = traverseNetwork(network, 0, major=1, minor=3, queue=1, parentClassID="1:1", parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps)
|
||||||
|
|
||||||
# Record start time of actual filter reload
|
# Record start time of actual filter reload
|
||||||
reloadStartTime = datetime.now()
|
reloadStartTime = datetime.now()
|
||||||
|
|
||||||
# Clear Prior Settings
|
# Clear Prior Settings
|
||||||
clearPriorSettings(interfaceA, interfaceB)
|
clearPriorSettings(interfaceA, interfaceB)
|
||||||
|
|
||||||
# If this is the first time LibreQoS.py has run since system boot, load the XDP program and disable XPS
|
# If this is the first time LibreQoS.py has run since system boot, load the XDP program and disable XPS
|
||||||
# Otherwise, just clear the existing IP filter rules for xdp
|
# Otherwise, just clear the existing IP filter rules for xdp
|
||||||
if isThisFirstRunSinceBoot:
|
if isThisFirstRunSinceBoot:
|
||||||
# Set up XDP-CPUMAP-TC
|
# Set up XDP-CPUMAP-TC
|
||||||
shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceA + ' --default --disable')
|
shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceA + ' --default --disable')
|
||||||
shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceB + ' --default --disable')
|
shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceB + ' --default --disable')
|
||||||
shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceA + ' --lan')
|
shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceA + ' --lan')
|
||||||
shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceB + ' --wan')
|
shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceB + ' --wan')
|
||||||
if enableActualShellCommands:
|
if enableActualShellCommands:
|
||||||
# Here we use os.system for the command, because otherwise it sometimes gltiches out with Popen in shell()
|
# Here we use os.system for the command, because otherwise it sometimes gltiches out with Popen in shell()
|
||||||
result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear')
|
result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear')
|
||||||
shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceA)
|
shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceA)
|
||||||
shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceB)
|
shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceB)
|
||||||
else:
|
else:
|
||||||
if enableActualShellCommands:
|
if enableActualShellCommands:
|
||||||
result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear')
|
result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear')
|
||||||
|
|
||||||
# Create MQ qdisc for each interface
|
# Create MQ qdisc for each interface
|
||||||
thisInterface = interfaceA
|
thisInterface = interfaceA
|
||||||
shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq')
|
shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq')
|
||||||
for queue in range(queuesAvailable):
|
for queue in range(queuesAvailable):
|
||||||
shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2')
|
shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2')
|
||||||
shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityDownloadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityDownloadMbps) + 'mbit')
|
shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityDownloadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityDownloadMbps) + 'mbit')
|
||||||
shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE)
|
shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE)
|
||||||
# Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv
|
# Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv
|
||||||
# Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling.
|
# Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling.
|
||||||
# Default class can use up to defaultClassCapacityDownloadMbps when that bandwidth isn't used by known hosts.
|
# Default class can use up to defaultClassCapacityDownloadMbps when that bandwidth isn't used by known hosts.
|
||||||
shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityDownloadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityDownloadMbps) + 'mbit prio 5')
|
shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityDownloadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityDownloadMbps) + 'mbit prio 5')
|
||||||
shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE)
|
shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE)
|
||||||
|
|
||||||
thisInterface = interfaceB
|
thisInterface = interfaceB
|
||||||
shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq')
|
shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq')
|
||||||
for queue in range(queuesAvailable):
|
for queue in range(queuesAvailable):
|
||||||
shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2')
|
shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2')
|
||||||
shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityUploadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityUploadMbps) + 'mbit')
|
shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityUploadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityUploadMbps) + 'mbit')
|
||||||
shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE)
|
shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE)
|
||||||
# Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv.
|
# Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv.
|
||||||
# Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling.
|
# Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling.
|
||||||
# Default class can use up to defaultClassCapacityUploadMbps when that bandwidth isn't used by known hosts.
|
# Default class can use up to defaultClassCapacityUploadMbps when that bandwidth isn't used by known hosts.
|
||||||
shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityUploadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityUploadMbps) + 'mbit prio 5')
|
shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityUploadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityUploadMbps) + 'mbit prio 5')
|
||||||
shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE)
|
shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE)
|
||||||
|
|
||||||
# Execute actual Linux TC and XDP-CPUMAP-TC filter commands
|
# Execute actual Linux TC and XDP-CPUMAP-TC filter commands
|
||||||
print("Executing linux TC class/qdisc commands")
|
print("Executing linux TC class/qdisc commands")
|
||||||
with open('linux_tc.txt', 'w') as f:
|
with open('linux_tc.txt', 'w') as f:
|
||||||
for line in linuxTCcommands:
|
for line in linuxTCcommands:
|
||||||
f.write(f"{line}\n")
|
f.write(f"{line}\n")
|
||||||
logging.info(line)
|
logging.info(line)
|
||||||
shell("/sbin/tc -f -b linux_tc.txt")
|
shell("/sbin/tc -f -b linux_tc.txt")
|
||||||
print("Executed " + str(len(linuxTCcommands)) + " linux TC class/qdisc commands")
|
print("Executed " + str(len(linuxTCcommands)) + " linux TC class/qdisc commands")
|
||||||
print("Executing XDP-CPUMAP-TC IP filter commands")
|
print("Executing XDP-CPUMAP-TC IP filter commands")
|
||||||
for command in xdpCPUmapCommands:
|
for command in xdpCPUmapCommands:
|
||||||
logging.info(command)
|
logging.info(command)
|
||||||
shell(command)
|
shell(command)
|
||||||
print("Executed " + str(len(xdpCPUmapCommands)) + " XDP-CPUMAP-TC IP filter commands")
|
print("Executed " + str(len(xdpCPUmapCommands)) + " XDP-CPUMAP-TC IP filter commands")
|
||||||
reloadEndTime = datetime.now()
|
reloadEndTime = datetime.now()
|
||||||
|
|
||||||
# Recap - warn operator if devices were skipped
|
# Recap - warn operator if devices were skipped
|
||||||
devicesSkipped = []
|
devicesSkipped = []
|
||||||
for circuit in subscriberCircuits:
|
for circuit in subscriberCircuits:
|
||||||
for device in circuit['devices']:
|
for device in circuit['devices']:
|
||||||
if device['deviceName'] not in devicesShaped:
|
if device['deviceName'] not in devicesShaped:
|
||||||
devicesSkipped.append((device['deviceName'],device['deviceID']))
|
devicesSkipped.append((device['deviceName'],device['deviceID']))
|
||||||
if len(devicesSkipped) > 0:
|
if len(devicesSkipped) > 0:
|
||||||
warnings.warn('Some devices were not shaped. Please check to ensure they have a valid ParentNode listed in ShapedDevices.csv:', stacklevel=2)
|
warnings.warn('Some devices were not shaped. Please check to ensure they have a valid ParentNode listed in ShapedDevices.csv:', stacklevel=2)
|
||||||
print("Devices not shaped:")
|
print("Devices not shaped:")
|
||||||
for entry in devicesSkipped:
|
for entry in devicesSkipped:
|
||||||
name, idNum = entry
|
name, idNum = entry
|
||||||
print('DeviceID: ' + idNum + '\t DeviceName: ' + name)
|
print('DeviceID: ' + idNum + '\t DeviceName: ' + name)
|
||||||
|
|
||||||
# Save for stats
|
# Save for stats
|
||||||
with open('statsByCircuit.json', 'w') as infile:
|
with open('statsByCircuit.json', 'w') as infile:
|
||||||
json.dump(subscriberCircuits, infile)
|
json.dump(subscriberCircuits, infile)
|
||||||
with open('statsByParentNode.json', 'w') as infile:
|
with open('statsByParentNode.json', 'w') as infile:
|
||||||
json.dump(parentNodes, infile)
|
json.dump(parentNodes, infile)
|
||||||
|
|
||||||
# Record time this run completed at
|
# Record time this run completed at
|
||||||
# filename = os.path.join(_here, 'lastRun.txt')
|
# filename = os.path.join(_here, 'lastRun.txt')
|
||||||
with open("lastRun.txt", 'w') as file:
|
with open("lastRun.txt", 'w') as file:
|
||||||
file.write(datetime.now().strftime("%d-%b-%Y (%H:%M:%S.%f)"))
|
file.write(datetime.now().strftime("%d-%b-%Y (%H:%M:%S.%f)"))
|
||||||
|
|
||||||
# Report reload time
|
# Report reload time
|
||||||
reloadTimeSeconds = (reloadEndTime - reloadStartTime).seconds
|
reloadTimeSeconds = (reloadEndTime - reloadStartTime).seconds
|
||||||
print("Queue and IP filter reload completed in " + str(reloadTimeSeconds) + " seconds")
|
print("Queue and IP filter reload completed in " + str(reloadTimeSeconds) + " seconds")
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
@ -571,11 +579,20 @@ if __name__ == '__main__':
|
|||||||
help="Be verbose",
|
help="Be verbose",
|
||||||
action="store_const", dest="loglevel", const=logging.INFO,
|
action="store_const", dest="loglevel", const=logging.INFO,
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--validate',
|
||||||
|
help="Just validate network.json and ShapedDevices.csv",
|
||||||
|
action=argparse.BooleanOptionalAction,
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
logging.basicConfig(level=args.loglevel)
|
logging.basicConfig(level=args.loglevel)
|
||||||
# Starting
|
|
||||||
print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
|
if args.validate:
|
||||||
# Refresh and/or set up queues
|
status = validateNetworkAndDevices()
|
||||||
refreshShapers()
|
else:
|
||||||
# Done
|
# Starting
|
||||||
print("refreshShapers completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
|
print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
|
||||||
|
# Refresh and/or set up queues
|
||||||
|
refreshShapers()
|
||||||
|
# Done
|
||||||
|
print("refreshShapers completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user