mirror of
https://github.com/LibreQoE/LibreQoS.git
synced 2025-02-25 18:55:32 -06:00
Update LibreQoS.py
This commit is contained in:
Robert Chacón
GitHub
parent
5aab722ab4
commit
2261610723
797
v1.3/LibreQoS.py
797
v1.3/LibreQoS.py
@ -1,5 +1,6 @@
|
||||
#!/usr/bin/python3
|
||||
# v1.2.1
|
||||
#!/usr/bin/python3
|
||||
# v1.3
|
||||
|
||||
import csv
|
||||
import io
|
||||
@ -19,9 +20,13 @@ import shutil
|
||||
import binpacking
|
||||
|
||||
from ispConfig import fqOrCAKE, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps, \
|
||||
interfaceA, interfaceB, enableActualShellCommands, \
|
||||
interfaceA, interfaceB, enableActualShellCommands, useBinPackingToBalanceCPU, \
|
||||
runShellCommandsAsSudo, generatedPNDownloadMbps, generatedPNUploadMbps, queuesAvailableOverride
|
||||
|
||||
# Automatically account for TCP overhead of plans. For example a 100Mbps plan needs to be set to 109Mbps for the user to ever see that result on a speed test
|
||||
# Does not apply to nodes of any sort, just endpoint devices
|
||||
tcpOverheadFactor = 1.09
|
||||
|
||||
def shell(command):
|
||||
if enableActualShellCommands:
|
||||
if runShellCommandsAsSudo:
|
||||
@ -30,12 +35,25 @@ def shell(command):
|
||||
commands = command.split(' ')
|
||||
proc = subprocess.Popen(commands, stdout=subprocess.PIPE)
|
||||
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding
|
||||
logging.info(line)
|
||||
if logging.DEBUG <= logging.root.level:
|
||||
print(line)
|
||||
if ("RTNETLINK answers" in line) or ("We have an error talking to the kernel" in line):
|
||||
warnings.warn("Command: '" + command + "' resulted in " + line, stacklevel=2)
|
||||
else:
|
||||
logging.info(command)
|
||||
|
||||
def shellTC(command):
|
||||
if enableActualShellCommands:
|
||||
print(command)
|
||||
commands = command.split(' ')
|
||||
proc = subprocess.Popen(commands, stdout=subprocess.PIPE)
|
||||
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding
|
||||
if logging.DEBUG <= logging.root.level:
|
||||
print(line)
|
||||
if ("RTNETLINK answers" in line) or ("We have an error talking to the kernel" in line):
|
||||
warnings.warn("Command: '" + command + "' resulted in " + line, stacklevel=2)
|
||||
raise SystemError("Command: '" + command + "' resulted in " + line)
|
||||
|
||||
def checkIfFirstRunSinceBoot():
|
||||
if os.path.isfile("lastRun.txt"):
|
||||
with open("lastRun.txt", 'r') as file:
|
||||
@ -84,6 +102,10 @@ def findQueuesAvailable():
|
||||
print("NIC queues (Override):\t\t\t" + str(queuesAvailable))
|
||||
cpuCount = multiprocessing.cpu_count()
|
||||
print("CPU cores:\t\t\t" + str(cpuCount))
|
||||
if queuesAvailable < 2:
|
||||
raise SystemError('Only 1 NIC rx/tx queue avaialable. You will need to use a NIC with 2 or more rx/tx queues available.')
|
||||
if queuesAvailable < 2:
|
||||
raise SystemError('Only 1 CPU core avaialable. You will need to use a CPU with 2 or more CPU cores.')
|
||||
queuesAvailable = min(queuesAvailable,cpuCount)
|
||||
print("queuesAvailable set to:\t" + str(queuesAvailable))
|
||||
else:
|
||||
@ -121,6 +143,10 @@ def validateNetworkAndDevices():
|
||||
seenTheseIPsAlready = []
|
||||
for row in commentsRemoved:
|
||||
circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row
|
||||
# Must have circuitID, it's a unique identifier requried for stateful changes to queue structure
|
||||
if circuitID == '':
|
||||
warnings.warn("No Circuit ID provided in ShapedDevices.csv at row " + str(rowNum), stacklevel=2)
|
||||
devicesValidatedOrNot = False
|
||||
# Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s seperated by commas. Split them up and parse each to ensure valid
|
||||
ipv4_subnets_and_hosts = []
|
||||
ipv6_subnets_and_hosts = []
|
||||
@ -222,6 +248,137 @@ def validateNetworkAndDevices():
|
||||
else:
|
||||
return False
|
||||
|
||||
def loadSubscriberCircuits(shapedDevicesFile):
|
||||
# Load Subscriber Circuits & Devices
|
||||
subscriberCircuits = []
|
||||
knownCircuitIDs = []
|
||||
counterForCircuitsWithoutParentNodes = 0
|
||||
dictForCircuitsWithoutParentNodes = {}
|
||||
with open(shapedDevicesFile) as csv_file:
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
# Remove comments if any
|
||||
commentsRemoved = []
|
||||
for row in csv_reader:
|
||||
if not row[0].startswith('#'):
|
||||
commentsRemoved.append(row)
|
||||
# Remove header
|
||||
commentsRemoved.pop(0)
|
||||
for row in commentsRemoved:
|
||||
circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row
|
||||
ipv4_subnets_and_hosts = []
|
||||
# Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s seperated by commas. Split them up and parse each
|
||||
if ipv4_input != "":
|
||||
ipv4_input = ipv4_input.replace(' ','')
|
||||
if "," in ipv4_input:
|
||||
ipv4_list = ipv4_input.split(',')
|
||||
else:
|
||||
ipv4_list = [ipv4_input]
|
||||
for ipEntry in ipv4_list:
|
||||
ipv4_subnets_and_hosts.append(ipEntry)
|
||||
ipv6_subnets_and_hosts = []
|
||||
if ipv6_input != "":
|
||||
ipv6_input = ipv6_input.replace(' ','')
|
||||
if "," in ipv6_input:
|
||||
ipv6_list = ipv6_input.split(',')
|
||||
else:
|
||||
ipv6_list = [ipv6_input]
|
||||
for ipEntry in ipv6_list:
|
||||
ipv6_subnets_and_hosts.append(ipEntry)
|
||||
# If there is something in the circuit ID field
|
||||
if circuitID != "":
|
||||
# Seen circuit before
|
||||
if circuitID in knownCircuitIDs:
|
||||
for circuit in subscriberCircuits:
|
||||
if circuit['circuitID'] == circuitID:
|
||||
if circuit['ParentNode'] != "none":
|
||||
if circuit['ParentNode'] != ParentNode:
|
||||
errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID
|
||||
raise ValueError(errorMessageString)
|
||||
if ((circuit['minDownload'] != round(int(downloadMin)*tcpOverheadFactor))
|
||||
or (circuit['minUpload'] != round(int(uploadMin)*tcpOverheadFactor))
|
||||
or (circuit['maxDownload'] != round(int(downloadMax)*tcpOverheadFactor))
|
||||
or (circuit['maxUpload'] != round(int(uploadMax)*tcpOverheadFactor))):
|
||||
warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different bandwidth parameters than other devices on this circuit. Will instead use the bandwidth parameters defined by the first device added to its circuit.", stacklevel=2)
|
||||
devicesListForCircuit = circuit['devices']
|
||||
thisDevice = {
|
||||
"deviceID": deviceID,
|
||||
"deviceName": deviceName,
|
||||
"mac": mac,
|
||||
"ipv4s": ipv4_subnets_and_hosts,
|
||||
"ipv6s": ipv6_subnets_and_hosts,
|
||||
"comment": comment
|
||||
}
|
||||
devicesListForCircuit.append(thisDevice)
|
||||
circuit['devices'] = devicesListForCircuit
|
||||
# Have not seen circuit before
|
||||
else:
|
||||
knownCircuitIDs.append(circuitID)
|
||||
if ParentNode == "":
|
||||
ParentNode = "none"
|
||||
ParentNode = ParentNode.strip()
|
||||
deviceListForCircuit = []
|
||||
thisDevice = {
|
||||
"deviceID": deviceID,
|
||||
"deviceName": deviceName,
|
||||
"mac": mac,
|
||||
"ipv4s": ipv4_subnets_and_hosts,
|
||||
"ipv6s": ipv6_subnets_and_hosts,
|
||||
"comment": comment
|
||||
}
|
||||
deviceListForCircuit.append(thisDevice)
|
||||
thisCircuit = {
|
||||
"circuitID": circuitID,
|
||||
"circuitName": circuitName,
|
||||
"ParentNode": ParentNode,
|
||||
"devices": deviceListForCircuit,
|
||||
"minDownload": round(int(downloadMin)*tcpOverheadFactor),
|
||||
"minUpload": round(int(uploadMin)*tcpOverheadFactor),
|
||||
"maxDownload": round(int(downloadMax)*tcpOverheadFactor),
|
||||
"maxUpload": round(int(uploadMax)*tcpOverheadFactor),
|
||||
"classid": '',
|
||||
"comment": comment
|
||||
}
|
||||
if thisCircuit['ParentNode'] == 'none':
|
||||
thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes
|
||||
dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((round(int(downloadMax)*tcpOverheadFactor))+(round(int(uploadMax)*tcpOverheadFactor)))
|
||||
counterForCircuitsWithoutParentNodes += 1
|
||||
subscriberCircuits.append(thisCircuit)
|
||||
# If there is nothing in the circuit ID field
|
||||
else:
|
||||
# Copy deviceName to circuitName if none defined already
|
||||
if circuitName == "":
|
||||
circuitName = deviceName
|
||||
if ParentNode == "":
|
||||
ParentNode = "none"
|
||||
ParentNode = ParentNode.strip()
|
||||
deviceListForCircuit = []
|
||||
thisDevice = {
|
||||
"deviceID": deviceID,
|
||||
"deviceName": deviceName,
|
||||
"mac": mac,
|
||||
"ipv4s": ipv4_subnets_and_hosts,
|
||||
"ipv6s": ipv6_subnets_and_hosts,
|
||||
}
|
||||
deviceListForCircuit.append(thisDevice)
|
||||
thisCircuit = {
|
||||
"circuitID": circuitID,
|
||||
"circuitName": circuitName,
|
||||
"ParentNode": ParentNode,
|
||||
"devices": deviceListForCircuit,
|
||||
"minDownload": round(int(downloadMin)*tcpOverheadFactor),
|
||||
"minUpload": round(int(uploadMin)*tcpOverheadFactor),
|
||||
"maxDownload": round(int(downloadMax)*tcpOverheadFactor),
|
||||
"maxUpload": round(int(uploadMax)*tcpOverheadFactor),
|
||||
"classid": '',
|
||||
"comment": comment
|
||||
}
|
||||
if thisCircuit['ParentNode'] == 'none':
|
||||
thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes
|
||||
dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((round(int(downloadMax)*tcpOverheadFactor))+(round(int(uploadMax)*tcpOverheadFactor)))
|
||||
counterForCircuitsWithoutParentNodes += 1
|
||||
subscriberCircuits.append(thisCircuit)
|
||||
return (subscriberCircuits, dictForCircuitsWithoutParentNodes)
|
||||
|
||||
def refreshShapers():
|
||||
|
||||
# Starting
|
||||
@ -237,11 +394,6 @@ def refreshShapers():
|
||||
isThisFirstRunSinceBoot = checkIfFirstRunSinceBoot()
|
||||
|
||||
|
||||
# Automatically account for TCP overhead of plans. For example a 100Mbps plan needs to be set to 109Mbps for the user to ever see that result on a speed test
|
||||
# Does not apply to nodes of any sort, just endpoint devices
|
||||
tcpOverheadFactor = 1.09
|
||||
|
||||
|
||||
# Files
|
||||
shapedDevicesFile = 'ShapedDevices.csv'
|
||||
networkJSONfile = 'network.json'
|
||||
@ -249,6 +401,7 @@ def refreshShapers():
|
||||
|
||||
# Check validation
|
||||
safeToRunRefresh = False
|
||||
print("Validating input files '" + shapedDevicesFile + "' and '" + networkJSONfile + "'")
|
||||
if (validateNetworkAndDevices() == True):
|
||||
shutil.copyfile('ShapedDevices.csv', 'lastGoodConfig.csv')
|
||||
shutil.copyfile('network.json', 'lastGoodConfig.json')
|
||||
@ -267,133 +420,7 @@ def refreshShapers():
|
||||
if safeToRunRefresh == True:
|
||||
|
||||
# Load Subscriber Circuits & Devices
|
||||
subscriberCircuits = []
|
||||
knownCircuitIDs = []
|
||||
counterForCircuitsWithoutParentNodes = 0
|
||||
dictForCircuitsWithoutParentNodes = {}
|
||||
with open(shapedDevicesFile) as csv_file:
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
# Remove comments if any
|
||||
commentsRemoved = []
|
||||
for row in csv_reader:
|
||||
if not row[0].startswith('#'):
|
||||
commentsRemoved.append(row)
|
||||
# Remove header
|
||||
commentsRemoved.pop(0)
|
||||
for row in commentsRemoved:
|
||||
circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row
|
||||
ipv4_subnets_and_hosts = []
|
||||
# Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s seperated by commas. Split them up and parse each
|
||||
if ipv4_input != "":
|
||||
ipv4_input = ipv4_input.replace(' ','')
|
||||
if "," in ipv4_input:
|
||||
ipv4_list = ipv4_input.split(',')
|
||||
else:
|
||||
ipv4_list = [ipv4_input]
|
||||
for ipEntry in ipv4_list:
|
||||
ipv4_subnets_and_hosts.append(ipEntry)
|
||||
ipv6_subnets_and_hosts = []
|
||||
if ipv6_input != "":
|
||||
ipv6_input = ipv6_input.replace(' ','')
|
||||
if "," in ipv6_input:
|
||||
ipv6_list = ipv6_input.split(',')
|
||||
else:
|
||||
ipv6_list = [ipv6_input]
|
||||
for ipEntry in ipv6_list:
|
||||
ipv6_subnets_and_hosts.append(ipEntry)
|
||||
# If there is something in the circuit ID field
|
||||
if circuitID != "":
|
||||
# Seen circuit before
|
||||
if circuitID in knownCircuitIDs:
|
||||
for circuit in subscriberCircuits:
|
||||
if circuit['circuitID'] == circuitID:
|
||||
if circuit['ParentNode'] != "none":
|
||||
if circuit['ParentNode'] != ParentNode:
|
||||
errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID
|
||||
raise ValueError(errorMessageString)
|
||||
if ((circuit['downloadMin'] != round(int(downloadMin)*tcpOverheadFactor))
|
||||
or (circuit['uploadMin'] != round(int(uploadMin)*tcpOverheadFactor))
|
||||
or (circuit['downloadMax'] != round(int(downloadMax)*tcpOverheadFactor))
|
||||
or (circuit['uploadMax'] != round(int(uploadMax)*tcpOverheadFactor))):
|
||||
warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different bandwidth parameters than other devices on this circuit. Will instead use the bandwidth parameters defined by the first device added to its circuit.", stacklevel=2)
|
||||
devicesListForCircuit = circuit['devices']
|
||||
thisDevice = {
|
||||
"deviceID": deviceID,
|
||||
"deviceName": deviceName,
|
||||
"mac": mac,
|
||||
"ipv4s": ipv4_subnets_and_hosts,
|
||||
"ipv6s": ipv6_subnets_and_hosts,
|
||||
"comment": comment
|
||||
}
|
||||
devicesListForCircuit.append(thisDevice)
|
||||
circuit['devices'] = devicesListForCircuit
|
||||
# Have not seen circuit before
|
||||
else:
|
||||
knownCircuitIDs.append(circuitID)
|
||||
if ParentNode == "":
|
||||
ParentNode = "none"
|
||||
ParentNode = ParentNode.strip()
|
||||
deviceListForCircuit = []
|
||||
thisDevice = {
|
||||
"deviceID": deviceID,
|
||||
"deviceName": deviceName,
|
||||
"mac": mac,
|
||||
"ipv4s": ipv4_subnets_and_hosts,
|
||||
"ipv6s": ipv6_subnets_and_hosts,
|
||||
"comment": comment
|
||||
}
|
||||
deviceListForCircuit.append(thisDevice)
|
||||
thisCircuit = {
|
||||
"circuitID": circuitID,
|
||||
"circuitName": circuitName,
|
||||
"ParentNode": ParentNode,
|
||||
"devices": deviceListForCircuit,
|
||||
"downloadMin": round(int(downloadMin)*tcpOverheadFactor),
|
||||
"uploadMin": round(int(uploadMin)*tcpOverheadFactor),
|
||||
"downloadMax": round(int(downloadMax)*tcpOverheadFactor),
|
||||
"uploadMax": round(int(uploadMax)*tcpOverheadFactor),
|
||||
"qdisc": '',
|
||||
"comment": comment
|
||||
}
|
||||
if thisCircuit['ParentNode'] == 'none':
|
||||
thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes
|
||||
dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((round(int(downloadMax)*tcpOverheadFactor))+(round(int(uploadMax)*tcpOverheadFactor)))
|
||||
counterForCircuitsWithoutParentNodes += 1
|
||||
subscriberCircuits.append(thisCircuit)
|
||||
# If there is nothing in the circuit ID field
|
||||
else:
|
||||
# Copy deviceName to circuitName if none defined already
|
||||
if circuitName == "":
|
||||
circuitName = deviceName
|
||||
if ParentNode == "":
|
||||
ParentNode = "none"
|
||||
ParentNode = ParentNode.strip()
|
||||
deviceListForCircuit = []
|
||||
thisDevice = {
|
||||
"deviceID": deviceID,
|
||||
"deviceName": deviceName,
|
||||
"mac": mac,
|
||||
"ipv4s": ipv4_subnets_and_hosts,
|
||||
"ipv6s": ipv6_subnets_and_hosts,
|
||||
}
|
||||
deviceListForCircuit.append(thisDevice)
|
||||
thisCircuit = {
|
||||
"circuitID": circuitID,
|
||||
"circuitName": circuitName,
|
||||
"ParentNode": ParentNode,
|
||||
"devices": deviceListForCircuit,
|
||||
"downloadMin": round(int(downloadMin)*tcpOverheadFactor),
|
||||
"uploadMin": round(int(uploadMin)*tcpOverheadFactor),
|
||||
"downloadMax": round(int(downloadMax)*tcpOverheadFactor),
|
||||
"uploadMax": round(int(uploadMax)*tcpOverheadFactor),
|
||||
"qdisc": '',
|
||||
"comment": comment
|
||||
}
|
||||
if thisCircuit['ParentNode'] == 'none':
|
||||
thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes
|
||||
dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((round(int(downloadMax)*tcpOverheadFactor))+(round(int(uploadMax)*tcpOverheadFactor)))
|
||||
counterForCircuitsWithoutParentNodes += 1
|
||||
subscriberCircuits.append(thisCircuit)
|
||||
subscriberCircuits, dictForCircuitsWithoutParentNodes = loadSubscriberCircuits(shapedDevicesFile)
|
||||
|
||||
|
||||
# Load network heirarchy
|
||||
@ -404,30 +431,45 @@ def refreshShapers():
|
||||
# Pull rx/tx queues / CPU cores available
|
||||
queuesAvailable = findQueuesAvailable()
|
||||
|
||||
|
||||
# Generate Parent Nodes. Spread ShapedDevices.csv which lack defined ParentNode across these (balance across CPUs)
|
||||
print("Generating parent nodes")
|
||||
existingPNs = 0
|
||||
for node in network:
|
||||
existingPNs += 1
|
||||
generatedPNs = []
|
||||
for x in range(queuesAvailable):
|
||||
numberOfGeneratedPNs = queuesAvailable-existingPNs
|
||||
for x in range(numberOfGeneratedPNs):
|
||||
genPNname = "Generated_PN_" + str(x+1)
|
||||
network[genPNname] = {
|
||||
"downloadBandwidthMbps":generatedPNDownloadMbps,
|
||||
"uploadBandwidthMbps":generatedPNUploadMbps
|
||||
}
|
||||
generatedPNs.append(genPNname)
|
||||
bins = binpacking.to_constant_bin_number(dictForCircuitsWithoutParentNodes, queuesAvailable)
|
||||
genPNcounter = 0
|
||||
for binItem in bins:
|
||||
sumItem = 0
|
||||
logging.info(generatedPNs[genPNcounter] + " will contain " + str(len(binItem)) + " circuits")
|
||||
for key in binItem.keys():
|
||||
for circuit in subscriberCircuits:
|
||||
if circuit['ParentNode'] == 'none':
|
||||
if circuit['idForCircuitsWithoutParentNodes'] == key:
|
||||
circuit['ParentNode'] = generatedPNs[genPNcounter]
|
||||
genPNcounter += 1
|
||||
if genPNcounter >= queuesAvailable:
|
||||
genPNcounter = 0
|
||||
|
||||
if useBinPackingToBalanceCPU:
|
||||
print("Using binpacking module to sort circuits by CPU core")
|
||||
bins = binpacking.to_constant_bin_number(dictForCircuitsWithoutParentNodes, numberOfGeneratedPNs)
|
||||
genPNcounter = 0
|
||||
for binItem in bins:
|
||||
sumItem = 0
|
||||
logging.info(generatedPNs[genPNcounter] + " will contain " + str(len(binItem)) + " circuits")
|
||||
for key in binItem.keys():
|
||||
for circuit in subscriberCircuits:
|
||||
if circuit['ParentNode'] == 'none':
|
||||
if circuit['idForCircuitsWithoutParentNodes'] == key:
|
||||
circuit['ParentNode'] = generatedPNs[genPNcounter]
|
||||
genPNcounter += 1
|
||||
if genPNcounter >= queuesAvailable:
|
||||
genPNcounter = 0
|
||||
print("Finished binpacking")
|
||||
else:
|
||||
genPNcounter = 0
|
||||
for circuit in subscriberCircuits:
|
||||
if circuit['ParentNode'] == 'none':
|
||||
circuit['ParentNode'] = generatedPNs[genPNcounter]
|
||||
genPNcounter += 1
|
||||
if genPNcounter >= queuesAvailable:
|
||||
genPNcounter = 0
|
||||
print("Generated parent nodes created")
|
||||
|
||||
# Find the bandwidth minimums for each node by combining mimimums of devices lower in that node's heirarchy
|
||||
def findBandwidthMins(data, depth):
|
||||
@ -437,8 +479,8 @@ def refreshShapers():
|
||||
for elem in data:
|
||||
for circuit in subscriberCircuits:
|
||||
if elem == circuit['ParentNode']:
|
||||
minDownload += circuit['downloadMin']
|
||||
minUpload += circuit['uploadMin']
|
||||
minDownload += circuit['minDownload']
|
||||
minUpload += circuit['minUpload']
|
||||
if 'children' in data[elem]:
|
||||
minDL, minUL = findBandwidthMins(data[elem]['children'], depth+1)
|
||||
minDownload += minDL
|
||||
@ -446,18 +488,24 @@ def refreshShapers():
|
||||
data[elem]['downloadBandwidthMbpsMin'] = minDownload
|
||||
data[elem]['uploadBandwidthMbpsMin'] = minUpload
|
||||
return minDownload, minUpload
|
||||
logging.info("Finding the bandwidth minimums for each node")
|
||||
minDownload, minUpload = findBandwidthMins(network, 0)
|
||||
|
||||
logging.info("Found the bandwidth minimums for each node")
|
||||
|
||||
# Parse network structure and add devices from ShapedDevices.csv
|
||||
linuxTCcommands = []
|
||||
xdpCPUmapCommands = []
|
||||
parentNodes = []
|
||||
def traverseNetwork(data, depth, major, minor, queue, parentClassID, parentMaxDL, parentMaxUL):
|
||||
minorByCPUpreloaded = {}
|
||||
knownClassIDs = []
|
||||
# Track minor counter by CPU. This way we can have > 32000 hosts (htb has u16 limit to minor handle)
|
||||
for x in range(queuesAvailable):
|
||||
minorByCPUpreloaded[x+1] = 3
|
||||
def traverseNetwork(data, depth, major, minorByCPU, queue, parentClassID, parentMaxDL, parentMaxUL):
|
||||
for node in data:
|
||||
circuitsForThisNetworkNode = []
|
||||
nodeClassID = hex(major) + ':' + hex(minor)
|
||||
nodeClassID = hex(major) + ':' + hex(minorByCPU[queue])
|
||||
data[node]['classid'] = nodeClassID
|
||||
if depth == 0:
|
||||
parentClassID = hex(major) + ':'
|
||||
data[node]['parentClassID'] = parentClassID
|
||||
# Cap based on this node's max bandwidth, or parent node's max bandwidth, whichever is lower
|
||||
data[node]['downloadBandwidthMbps'] = min(data[node]['downloadBandwidthMbps'],parentMaxDL)
|
||||
@ -468,31 +516,31 @@ def refreshShapers():
|
||||
data[node]['downloadBandwidthMbpsMin'] = round(data[node]['downloadBandwidthMbps']*.95)
|
||||
data[node]['uploadBandwidthMbpsMin'] = round(data[node]['uploadBandwidthMbps']*.95)
|
||||
data[node]['classMajor'] = hex(major)
|
||||
data[node]['classMinor'] = hex(minor)
|
||||
data[node]['classMinor'] = hex(minorByCPU[queue])
|
||||
data[node]['cpuNum'] = hex(queue-1)
|
||||
thisParentNode = {
|
||||
"parentNodeName": node,
|
||||
"classID": nodeClassID,
|
||||
"downloadMax": data[node]['downloadBandwidthMbps'],
|
||||
"uploadMax": data[node]['uploadBandwidthMbps'],
|
||||
"maxDownload": data[node]['downloadBandwidthMbps'],
|
||||
"maxUpload": data[node]['uploadBandwidthMbps'],
|
||||
}
|
||||
parentNodes.append(thisParentNode)
|
||||
minor += 1
|
||||
minorByCPU[queue] = minorByCPU[queue] + 1
|
||||
for circuit in subscriberCircuits:
|
||||
#If a device from ShapedDevices.csv lists this node as its Parent Node, attach it as a leaf to this node HTB
|
||||
if node == circuit['ParentNode']:
|
||||
if circuit['downloadMax'] > data[node]['downloadBandwidthMbps']:
|
||||
if circuit['maxDownload'] > data[node]['downloadBandwidthMbps']:
|
||||
warnings.warn("downloadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2)
|
||||
if circuit['uploadMax'] > data[node]['uploadBandwidthMbps']:
|
||||
if circuit['maxUpload'] > data[node]['uploadBandwidthMbps']:
|
||||
warnings.warn("uploadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2)
|
||||
parentString = hex(major) + ':'
|
||||
flowIDstring = hex(major) + ':' + hex(minor)
|
||||
circuit['qdisc'] = flowIDstring
|
||||
flowIDstring = hex(major) + ':' + hex(minorByCPU[queue])
|
||||
circuit['classid'] = flowIDstring
|
||||
# Create circuit dictionary to be added to network structure, eventually output as queuingStructure.json
|
||||
maxDownload = min(circuit['downloadMax'],data[node]['downloadBandwidthMbps'])
|
||||
maxUpload = min(circuit['uploadMax'],data[node]['uploadBandwidthMbps'])
|
||||
minDownload = min(circuit['downloadMin'],maxDownload)
|
||||
minUpload = min(circuit['uploadMin'],maxUpload)
|
||||
maxDownload = min(circuit['maxDownload'],data[node]['downloadBandwidthMbps'])
|
||||
maxUpload = min(circuit['maxUpload'],data[node]['uploadBandwidthMbps'])
|
||||
minDownload = min(circuit['minDownload'],maxDownload)
|
||||
minUpload = min(circuit['minUpload'],maxUpload)
|
||||
thisNewCircuitItemForNetwork = {
|
||||
'maxDownload' : maxDownload,
|
||||
'maxUpload' : maxUpload,
|
||||
@ -502,21 +550,22 @@ def refreshShapers():
|
||||
"circuitName": circuit['circuitName'],
|
||||
"ParentNode": circuit['ParentNode'],
|
||||
"devices": circuit['devices'],
|
||||
"qdisc": flowIDstring,
|
||||
"classid": flowIDstring,
|
||||
"classMajor": hex(major),
|
||||
"classMinor": hex(minor),
|
||||
"classMinor": hex(minorByCPU[queue]),
|
||||
"comment": circuit['comment']
|
||||
}
|
||||
# Generate TC commands to be executed later
|
||||
thisNewCircuitItemForNetwork['devices'] = circuit['devices']
|
||||
circuitsForThisNetworkNode.append(thisNewCircuitItemForNetwork)
|
||||
minor += 1
|
||||
minorByCPU[queue] = minorByCPU[queue] + 1
|
||||
if len(circuitsForThisNetworkNode) > 0:
|
||||
data[node]['circuits'] = circuitsForThisNetworkNode
|
||||
# Recursive call this function for children nodes attached to this node
|
||||
if 'children' in data[node]:
|
||||
# We need to keep tabs on the minor counter, because we can't have repeating class IDs. Here, we bring back the minor counter from the recursive function
|
||||
minor = traverseNetwork(data[node]['children'], depth+1, major, minor+1, queue, nodeClassID, data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps'])
|
||||
minorByCPU[queue] = minorByCPU[queue] + 1
|
||||
minorByCPU = traverseNetwork(data[node]['children'], depth+1, major, minorByCPU, queue, nodeClassID, data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps'])
|
||||
# If top level node, increment to next queue / cpu core
|
||||
if depth == 0:
|
||||
if queue >= queuesAvailable:
|
||||
@ -525,16 +574,16 @@ def refreshShapers():
|
||||
else:
|
||||
queue += 1
|
||||
major += 1
|
||||
return minor
|
||||
return minorByCPU
|
||||
# Here is the actual call to the recursive traverseNetwork() function. finalMinor is not used.
|
||||
finalMinor = traverseNetwork(network, 0, major=1, minor=3, queue=1, parentClassID="1:1", parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps)
|
||||
minorByCPU = traverseNetwork(network, 0, major=1, minorByCPU=minorByCPUpreloaded, queue=1, parentClassID=None, parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps)
|
||||
|
||||
|
||||
linuxTCcommands = []
|
||||
xdpCPUmapCommands = []
|
||||
devicesShaped = []
|
||||
# Root HTB Setup
|
||||
# Create MQ qdisc for each CPU core / rx-tx queue (XDP method - requires IPv4)
|
||||
# Create MQ qdisc for each CPU core / rx-tx queue. Generate commands to create corresponding HTB and leaf classes. Prepare commands for execution later
|
||||
thisInterface = interfaceA
|
||||
logging.info("# MQ Setup for " + thisInterface)
|
||||
command = 'qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq'
|
||||
@ -578,7 +627,7 @@ def refreshShapers():
|
||||
# Define lists for hash filters
|
||||
def traverseNetwork(data):
|
||||
for node in data:
|
||||
command = 'class add dev ' + interfaceA + ' parent ' + data[node]['parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ str(data[node]['downloadBandwidthMbpsMin']) + 'mbit ceil '+ str(data[node]['downloadBandwidthMbps']) + 'mbit prio 3' + " # Node: " + node
|
||||
command = 'class add dev ' + interfaceA + ' parent ' + data[node]['parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ str(data[node]['downloadBandwidthMbpsMin']) + 'mbit ceil '+ str(data[node]['downloadBandwidthMbps']) + 'mbit prio 3'
|
||||
linuxTCcommands.append(command)
|
||||
command = 'class add dev ' + interfaceB + ' parent ' + data[node]['parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ str(data[node]['uploadBandwidthMbpsMin']) + 'mbit ceil '+ str(data[node]['uploadBandwidthMbps']) + 'mbit prio 3'
|
||||
linuxTCcommands.append(command)
|
||||
@ -591,7 +640,7 @@ def refreshShapers():
|
||||
if 'devices' in circuit:
|
||||
if 'comment' in circuit['devices'][0]:
|
||||
comment = comment + '| Comment: ' + circuit['devices'][0]['comment']
|
||||
command = 'class add dev ' + interfaceA + ' parent ' + data[node]['classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ str(circuit['minDownload']) + 'mbit ceil '+ str(circuit['maxDownload']) + 'mbit prio 3' + comment
|
||||
command = 'class add dev ' + interfaceA + ' parent ' + data[node]['classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ str(circuit['minDownload']) + 'mbit ceil '+ str(circuit['maxDownload']) + 'mbit prio 3'
|
||||
linuxTCcommands.append(command)
|
||||
command = 'qdisc add dev ' + interfaceA + ' parent ' + circuit['classMajor'] + ':' + circuit['classMinor'] + ' ' + fqOrCAKE
|
||||
linuxTCcommands.append(command)
|
||||
@ -602,30 +651,25 @@ def refreshShapers():
|
||||
for device in circuit['devices']:
|
||||
if device['ipv4s']:
|
||||
for ipv4 in device['ipv4s']:
|
||||
if '/' in ipv4:
|
||||
ipv4AddressOnly, prefixOnly = ipv4.split('/')
|
||||
xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4AddressOnly) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['qdisc'] + ' --prefix ' + prefixOnly)
|
||||
else:
|
||||
xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['qdisc'])
|
||||
xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['classid'])
|
||||
if device['ipv6s']:
|
||||
for ipv6 in device['ipv6s']:
|
||||
if '/' in ipv6:
|
||||
ipv6AddressOnly, prefixOnly = ipv6.split('/')
|
||||
xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv6AddressOnly) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['qdisc'] + ' --prefix ' + prefixOnly)
|
||||
else:
|
||||
xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv6) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['qdisc'])
|
||||
xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv6) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['classid'])
|
||||
if device['deviceName'] not in devicesShaped:
|
||||
devicesShaped.append(device['deviceName'])
|
||||
# Recursive call this function for children nodes attached to this node
|
||||
if 'children' in data[node]:
|
||||
traverseNetwork(data[node]['children'])
|
||||
# Here is the actual call to the recursive traverseNetwork() function. finalResult is not used.
|
||||
# Here is the actual call to the recursive traverseNetwork() function.
|
||||
traverseNetwork(network)
|
||||
|
||||
|
||||
# Save queuingStructure
|
||||
queuingStructure = {}
|
||||
queuingStructure['Network'] = network
|
||||
queuingStructure['lastUsedClassIDCounterByCPU'] = minorByCPU
|
||||
queuingStructure['generatedPNs'] = generatedPNs
|
||||
with open('queuingStructure.json', 'w') as infile:
|
||||
json.dump(network, infile, indent=4)
|
||||
json.dump(queuingStructure, infile, indent=4)
|
||||
|
||||
|
||||
# Record start time of actual filter reload
|
||||
@ -668,6 +712,10 @@ def refreshShapers():
|
||||
print("Executed " + str(len(linuxTCcommands)) + " linux TC class/qdisc commands")
|
||||
|
||||
|
||||
tcEndTime = datetime.now()
|
||||
print("Executed " + str(len(linuxTCcommands)) + " linux TC class/qdisc commands")
|
||||
|
||||
|
||||
# Execute actual XDP-CPUMAP-TC filter commands
|
||||
xdpFilterStartTime = datetime.now()
|
||||
print("Executing XDP-CPUMAP-TC IP filter commands")
|
||||
@ -700,6 +748,8 @@ def refreshShapers():
|
||||
name, idNum = entry
|
||||
print('DeviceID: ' + idNum + '\t DeviceName: ' + name)
|
||||
|
||||
# Save ShapedDevices.csv as ShapedDevices.lastLoaded.csv
|
||||
shutil.copyfile('ShapedDevices.csv', 'ShapedDevices.lastLoaded.csv')
|
||||
|
||||
# Save for stats
|
||||
with open('statsByCircuit.json', 'w') as f:
|
||||
@ -728,6 +778,362 @@ def refreshShapers():
|
||||
# Done
|
||||
print("refreshShapers completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
|
||||
|
||||
def refreshShapersUpdateOnly():
|
||||
# Starting
|
||||
print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
|
||||
|
||||
|
||||
# Warn user if enableActualShellCommands is False, because that would mean no actual commands are executing
|
||||
if enableActualShellCommands == False:
|
||||
warnings.warn("enableActualShellCommands is set to False. None of the commands below will actually be executed. Simulated run.", stacklevel=2)
|
||||
|
||||
|
||||
# Files
|
||||
shapedDevicesFile = 'ShapedDevices.csv'
|
||||
networkJSONfile = 'network.json'
|
||||
|
||||
|
||||
# Check validation
|
||||
safeToRunRefresh = False
|
||||
if (validateNetworkAndDevices() == True):
|
||||
shutil.copyfile('ShapedDevices.csv', 'lastGoodConfig.csv')
|
||||
shutil.copyfile('network.json', 'lastGoodConfig.json')
|
||||
print("Backed up good config as lastGoodConfig.csv and lastGoodConfig.json")
|
||||
safeToRunRefresh = True
|
||||
else:
|
||||
warnings.warn("Validation failed. Will now exit.", stacklevel=2)
|
||||
|
||||
if safeToRunRefresh == True:
|
||||
|
||||
# Load queuingStructure
|
||||
with open('queuingStructure.json', 'r') as infile:
|
||||
queuingStructure = json.loads(infile.read())
|
||||
|
||||
|
||||
# Load queuingStructure
|
||||
with open('queuingStructure.json', 'r') as infile:
|
||||
queuingStructure = json.loads(infile.read())
|
||||
|
||||
network = queuingStructure['Network']
|
||||
lastUsedClassIDCounterByCPU = queuingStructure['lastUsedClassIDCounterByCPU']
|
||||
generatedPNs = queuingStructure['generatedPNs']
|
||||
|
||||
newlyUpdatedSubscriberCircuits, newlyUpdatedDictForCircuitsWithoutParentNodes = loadSubscriberCircuits('ShapedDevices.csv')
|
||||
lastLoadedSubscriberCircuits, lastLoadedDictForCircuitsWithoutParentNodes = loadSubscriberCircuits('ShapedDevices.lastLoaded.csv')
|
||||
|
||||
|
||||
# Load stats files
|
||||
with open('statsByParentNode.json', 'r') as j:
|
||||
parentNodes = json.loads(j.read())
|
||||
|
||||
with open('statsByCircuit.json', 'r') as j:
|
||||
subscriberCircuits = json.loads(j.read())
|
||||
|
||||
newlyUpdatedSubscriberCircuitsByID = {}
|
||||
for circuit in newlyUpdatedSubscriberCircuits:
|
||||
circuitid = circuit['circuitID']
|
||||
newlyUpdatedSubscriberCircuitsByID[circuitid] = circuit
|
||||
|
||||
lastLoadedSubscriberCircuitsByID = {}
|
||||
for circuit in lastLoadedSubscriberCircuits:
|
||||
circuitid = circuit['circuitID']
|
||||
lastLoadedSubscriberCircuitsByID[circuitid] = circuit
|
||||
|
||||
|
||||
def removeDeviceIPsFromFilter(circuit):
|
||||
for device in circuit['devices']:
|
||||
for ipv4 in device['ipv4s']:
|
||||
shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --del --ip ' + str(ipv4))
|
||||
for ipv6 in device['ipv6s']:
|
||||
shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --del --ip ' + str(ipv6))
|
||||
|
||||
|
||||
def addDeviceIPsToFilter(circuit, cpuNumHex):
|
||||
for device in circuit['devices']:
|
||||
for ipv4 in device['ipv4s']:
|
||||
shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + cpuNumHex + ' --classid ' + circuit['classid'])
|
||||
for ipv6 in device['ipv6s']:
|
||||
shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv6) + ' --cpu ' + cpuNumHex + ' --classid ' + circuit['classid'])
|
||||
|
||||
|
||||
def getAllParentNodes(data, allParentNodes):
|
||||
for node in data:
|
||||
if isinstance(node, str):
|
||||
thisParentNode = node
|
||||
if thisParentNode not in allParentNodes:
|
||||
allParentNodes.append(thisParentNode)
|
||||
if 'children' in data[node]:
|
||||
for child in data[node]['children']:
|
||||
result = getAllParentNodes(data[node]['children'][child], allParentNodes)
|
||||
allParentNodes = allParentNodes + result
|
||||
return allParentNodes
|
||||
allParentNodes = getAllParentNodes(network, [])
|
||||
|
||||
|
||||
def getClassIDofParentNodes(data, classIDOfParentNodes):
|
||||
for node in data:
|
||||
if isinstance(node, str):
|
||||
thisParentNode = node
|
||||
classIDOfParentNodes[thisParentNode] = data[node]['classid']
|
||||
if 'children' in data[node]:
|
||||
for child in data[node]['children']:
|
||||
result = getClassIDofParentNodes(data[node]['children'][child], classIDOfParentNodes)
|
||||
classIDOfParentNodes.update(result)
|
||||
return classIDOfParentNodes
|
||||
classIDOfParentNodes = getClassIDofParentNodes(network, {})
|
||||
|
||||
|
||||
def getAllCircuitIDs(data, allCircuitIDs):
|
||||
for node in data:
|
||||
if isinstance(node, str):
|
||||
thisParentNode = node
|
||||
if 'circuits' in data[node]:
|
||||
for circuit in data[node]['circuits']:
|
||||
if circuit['circuitID'] not in allCircuitIDs:
|
||||
allCircuitIDs.append(circuit['circuitID'])
|
||||
if 'children' in data[node]:
|
||||
for child in data[node]['children']:
|
||||
result = getAllCircuitIDs(data[node]['children'][child], allCircuitIDs)
|
||||
for entry in result:
|
||||
if entry not in allCircuitIDs:
|
||||
allCircuitIDs.append(result)
|
||||
return allCircuitIDs
|
||||
allCircuitIDs = getAllCircuitIDs(network, [])
|
||||
|
||||
|
||||
def getClassIDofExistingCircuitID(data, classIDofExistingCircuitID):
|
||||
for node in data:
|
||||
if isinstance(node, str):
|
||||
thisParentNode = node
|
||||
if 'circuits' in data[node]:
|
||||
for circuit in data[node]['circuits']:
|
||||
classIDofExistingCircuitID[circuit['circuitID']] = circuit['classid']
|
||||
if 'children' in data[node]:
|
||||
for child in data[node]['children']:
|
||||
result = getClassIDofExistingCircuitID(data[node]['children'][child], allCircuitIDs)
|
||||
classIDofExistingCircuitID.update(result)
|
||||
return classIDofExistingCircuitID
|
||||
classIDofExistingCircuitID = getClassIDofExistingCircuitID(network, {})
|
||||
|
||||
|
||||
def getParentNodeOfCircuitID(data, parentNodeOfCircuitID, allCircuitIDs):
|
||||
for node in data:
|
||||
if isinstance(node, str):
|
||||
thisParentNode = node
|
||||
if 'circuits' in data[node]:
|
||||
for circuit in data[node]['circuits']:
|
||||
parentNodeOfCircuitID[circuit['circuitID']] = thisParentNode
|
||||
if 'children' in data[node]:
|
||||
for child in data[node]['children']:
|
||||
result = getParentNodeOfCircuitID(data[node]['children'][child], parentNodeOfCircuitID, allCircuitIDs)
|
||||
parentNodeOfCircuitID.update(result)
|
||||
return parentNodeOfCircuitID
|
||||
parentNodeOfCircuitID = getParentNodeOfCircuitID(network, {}, allCircuitIDs)
|
||||
|
||||
|
||||
def getCPUnumOfParentNodes(data, cpuNumOfParentNode):
|
||||
for node in data:
|
||||
if isinstance(node, str):
|
||||
thisParentNode = node
|
||||
cpuNumOfParentNode[thisParentNode] = data[node]['cpuNum']
|
||||
if 'children' in data[node]:
|
||||
for child in data[node]['children']:
|
||||
result = getCPUnumOfParentNodes(data[node]['children'][child], cpuNumOfParentNode)
|
||||
cpuNumOfParentNode.update(result)
|
||||
return cpuNumOfParentNode
|
||||
cpuNumOfParentNodeHex = getCPUnumOfParentNodes(network, {})
|
||||
cpuNumOfParentNodeInt = {}
|
||||
for key, value in cpuNumOfParentNodeHex.items():
|
||||
cpuNumOfParentNodeInt[key] = int(value,16) + 1
|
||||
|
||||
|
||||
def addCircuitHTBandQdisc(circuit, parentNodeClassID):
|
||||
minor = circuit['classid'].split(':')[1]
|
||||
for interface in [interfaceA, interfaceB]:
|
||||
if interface == interfaceA:
|
||||
rate = str(circuit['minDownload'])
|
||||
ceil = str(circuit['maxDownload'])
|
||||
else:
|
||||
rate = str(circuit['minUpload'])
|
||||
ceil = str(circuit['maxUpload'])
|
||||
command = 'tc class add dev ' + interface + ' parent ' + parentNodeClassID + ' classid ' + minor + ' htb rate ' + rate + 'Mbit ceil ' + ceil + 'Mbit'
|
||||
print(command)
|
||||
shell(command)
|
||||
command = 'tc qdisc add dev ' + interface + ' parent ' + classID + ' ' + fqOrCAKE
|
||||
print(command)
|
||||
shell(command)
|
||||
|
||||
|
||||
def delHTBclass(classid):
|
||||
for interface in [interfaceA, interfaceB]:
|
||||
command = 'tc class del dev ' + interface + ' classid ' + classid
|
||||
print(command)
|
||||
shell(command)
|
||||
|
||||
|
||||
generatedPNcounter = 1
|
||||
circuitsIDsToRemove = []
|
||||
circuitsToUpdateByID = {}
|
||||
circuitsToAddByParentNode = {}
|
||||
for circuitID, circuit in lastLoadedSubscriberCircuitsByID.items():
|
||||
# Same circuit, update parameters (bandwidth, devices)
|
||||
bandwidthChanged = False
|
||||
devicesChanged = False
|
||||
if (circuitID in newlyUpdatedSubscriberCircuitsByID) and (circuitID in lastLoadedSubscriberCircuitsByID):
|
||||
if newlyUpdatedSubscriberCircuitsByID[circuitID]['maxDownload'] != lastLoadedSubscriberCircuitsByID[circuitID]['maxDownload']:
|
||||
bandwidthChanged = True
|
||||
if newlyUpdatedSubscriberCircuitsByID[circuitID]['maxUpload'] != lastLoadedSubscriberCircuitsByID[circuitID]['maxUpload']:
|
||||
bandwidthChanged = True
|
||||
if newlyUpdatedSubscriberCircuitsByID[circuitID]['minDownload'] != lastLoadedSubscriberCircuitsByID[circuitID]['minDownload']:
|
||||
bandwidthChanged = True
|
||||
if newlyUpdatedSubscriberCircuitsByID[circuitID]['minUpload'] != lastLoadedSubscriberCircuitsByID[circuitID]['minUpload']:
|
||||
bandwidthChanged = True
|
||||
if newlyUpdatedSubscriberCircuitsByID[circuitID]['devices'] != lastLoadedSubscriberCircuitsByID[circuitID]['devices']:
|
||||
devicesChanged = True
|
||||
if bandwidthChanged == True:
|
||||
if newlyUpdatedSubscriberCircuitsByID[circuitID]['ParentNode'] == lastLoadedSubscriberCircuitsByID[circuitID]['ParentNode']:
|
||||
parentNodeActual = circuit['ParentNode']
|
||||
if parentNodeActual == 'none':
|
||||
parentNodeActual = parentNodeOfCircuitID[circuitID]
|
||||
parentNodeClassID = classIDOfParentNodes[parentNodeActual]
|
||||
classid = classIDofExistingCircuitID[circuitID]
|
||||
minor = classid.split(':')[1]
|
||||
for interface in [interfaceA, interfaceB]:
|
||||
if interface == interfaceA:
|
||||
rate = str(newlyUpdatedSubscriberCircuitsByID[circuitID]['minDownload'])
|
||||
ceil = str(newlyUpdatedSubscriberCircuitsByID[circuitID]['maxDownload'])
|
||||
else:
|
||||
rate = str(newlyUpdatedSubscriberCircuitsByID[circuitID]['minUpload'])
|
||||
ceil = str(newlyUpdatedSubscriberCircuitsByID[circuitID]['maxUpload'])
|
||||
command = 'tc class change dev ' + interface + ' parent ' + parentNodeClassID + ' classid ' + minor + ' htb rate ' + rate + 'Mbit ceil ' + ceil + 'Mbit'
|
||||
shell(command)
|
||||
else:
|
||||
removeDeviceIPsFromFilter(lastLoadedSubscriberCircuitsByID[circuitID])
|
||||
# Delete HTB class, qdisc. Then recreat it.
|
||||
classid = classIDofExistingCircuitID[circuitID]
|
||||
circuit['classid'] = classid
|
||||
delHTBclass(classID)
|
||||
parentNodeClassID = classIDOfParentNodes[parentNodeOfCircuitID[circuitID]]
|
||||
addCircuitHTBandQdisc(circuit, parentNodeClassID)
|
||||
addDeviceIPsToFilter(newlyUpdatedSubscriberCircuitsByID[circuitID], cpuNum)
|
||||
elif devicesChanged:
|
||||
removeDeviceIPsFromFilter(lastLoadedSubscriberCircuitsByID[circuitID])
|
||||
parentNodeActual = lastLoadedSubscriberCircuitsByID[circuitID]['ParentNode']
|
||||
if parentNodeActual == 'none':
|
||||
parentNodeActual = getParentNodeOfCircuitID(network, circuitID)
|
||||
cpuNum, parentNodeClassID = getCPUnumAndClassIDOfParentNode(network, parentNodeActual)
|
||||
addDeviceIPsToFilter(newlyUpdatedSubscriberCircuitsByID[circuitID], cpuNum)
|
||||
|
||||
newlyUpdatedSubscriberCircuitsByID[circuitID]['classid'] = lastLoadedSubscriberCircuitsByID[circuitID]['classid']
|
||||
if (bandwidthChanged) or (devicesChanged):
|
||||
circuitsToUpdateByID[circuitID] = newlyUpdatedSubscriberCircuitsByID[circuitID]
|
||||
|
||||
|
||||
# Removed circuit
|
||||
if (circuitID in lastLoadedSubscriberCircuitsByID) and (circuitID not in newlyUpdatedSubscriberCircuitsByID):
|
||||
circuitsIDsToRemove.append(circuitID)
|
||||
removeDeviceIPsFromFilter(lastLoadedSubscriberCircuitsByID[circuitID])
|
||||
classid = classIDofExistingCircuitID[circuitID]
|
||||
delHTBclass(classid)
|
||||
|
||||
|
||||
# New circuit
|
||||
for circuitID, circuit in newlyUpdatedSubscriberCircuitsByID.items():
|
||||
if (circuitID in newlyUpdatedSubscriberCircuitsByID) and (circuitID not in lastLoadedSubscriberCircuitsByID):
|
||||
if newlyUpdatedSubscriberCircuitsByID[circuitID]['ParentNode'] == 'none':
|
||||
newlyUpdatedSubscriberCircuitsByID[circuitID]['ParentNode'] = 'Generated_PN_' + str(generatedPNcounter)
|
||||
generatedPNcounter += 1
|
||||
if generatedPNcounter > len(generatedPNs):
|
||||
generatedPNcounter = 1
|
||||
cpuNumHex = cpuNumOfParentNodeHex[circuit['ParentNode']]
|
||||
cpuNumInt = cpuNumOfParentNodeInt[circuit['ParentNode']]
|
||||
parentNodeClassID = classIDOfParentNodes[circuit['ParentNode']]
|
||||
classID = parentNodeClassID.split(':')[0] + ':' + hex(lastUsedClassIDCounterByCPU[str(cpuNumInt)])
|
||||
lastUsedClassIDCounterByCPU[str(cpuNumInt)] = lastUsedClassIDCounterByCPU[str(cpuNumInt)] + 1
|
||||
circuit['classid'] = classID
|
||||
# Add HTB class, qdisc
|
||||
addCircuitHTBandQdisc(circuit, parentNodeClassID)
|
||||
addDeviceIPsToFilter(circuit, cpuNumHex)
|
||||
if circuit['ParentNode'] in circuitsToAddByParentNode:
|
||||
temp = circuitsToAddByParentNode[circuit['ParentNode']]
|
||||
temp.append(circuit)
|
||||
circuitsToAddByParentNode[circuit['ParentNode']] = temp
|
||||
else:
|
||||
temp = []
|
||||
temp.append(circuit)
|
||||
circuitsToAddByParentNode[circuit['ParentNode']] = temp
|
||||
|
||||
|
||||
# Update network structure reflecting new circuits, removals, and changes
|
||||
itemsToChange = (circuitsIDsToRemove, circuitsToUpdateByID, circuitsToAddByParentNode)
|
||||
def updateNetworkStructure(data, depth, itemsToChange):
|
||||
circuitsIDsToRemove, circuitsToUpdateByID, circuitsToAddByParentNode = itemsToChange
|
||||
for node in data:
|
||||
if isinstance(node, str):
|
||||
thisParentNode = node
|
||||
if 'circuits' in data[node]:
|
||||
for circuit in data[node]['circuits']:
|
||||
if circuit['circuitID'] in circuitsToUpdateByID:
|
||||
circuit = circuitsToUpdateByID[circuit['circuitID']]
|
||||
print('updated')
|
||||
if circuit['circuitID'] in circuitsIDsToRemove:
|
||||
data[node]['circuits'].remove(circuit)
|
||||
if thisParentNode in circuitsToAddByParentNode:
|
||||
if 'circuits' in data[node]:
|
||||
temp = data[node]['circuits']
|
||||
for circuit in circuitsToAddByParentNode[thisParentNode]:
|
||||
temp.append(circuit)
|
||||
data[node]['circuits'] = temp
|
||||
else:
|
||||
temp = []
|
||||
for circuit in circuitsToAddByParentNode[thisParentNode]:
|
||||
temp.append(circuit)
|
||||
data[node]['circuits'] = temp
|
||||
if 'children' in data[node]:
|
||||
for child in data[node]['children']:
|
||||
result = updateNetworkStructure(data[node]['children'][child], depth+1, itemsToChange)
|
||||
data[node]['children'][child] = result
|
||||
return data
|
||||
network = updateNetworkStructure(network, 0 , itemsToChange)
|
||||
|
||||
|
||||
# Record start time of actual filter reload
|
||||
reloadStartTime = datetime.now()
|
||||
|
||||
|
||||
# Record end time of all reload commands
|
||||
reloadEndTime = datetime.now()
|
||||
|
||||
|
||||
queuingStructure = {}
|
||||
queuingStructure['Network'] = network
|
||||
queuingStructure['lastUsedClassIDCounterByCPU'] = lastUsedClassIDCounterByCPU
|
||||
queuingStructure['generatedPNs'] = generatedPNs
|
||||
# Save queuingStructure
|
||||
with open('queuingStructure.json', 'w') as infile:
|
||||
json.dump(queuingStructure, infile, indent=4)
|
||||
|
||||
|
||||
# copy ShapedDevices.csv and save as ShapedDevices.lastLoaded.csv and lastGoodConfig.csv
|
||||
shutil.copyfile('ShapedDevices.csv', 'ShapedDevices.lastLoaded.csv')
|
||||
shutil.copyfile('ShapedDevices.csv', 'lastGoodConfig.csv')
|
||||
|
||||
|
||||
# Save for stats
|
||||
with open('statsByCircuit.json', 'w') as f:
|
||||
f.write(json.dumps(subscriberCircuits, indent=4))
|
||||
with open('statsByParentNode.json', 'w') as f:
|
||||
f.write(json.dumps(parentNodes, indent=4))
|
||||
|
||||
|
||||
# Report reload time
|
||||
reloadTimeSeconds = ((reloadEndTime - reloadStartTime).seconds) + (((reloadEndTime - reloadStartTime).microseconds) / 1000000)
|
||||
print("Queue and IP filter partial reload completed in " + "{:g}".format(round(reloadTimeSeconds,1)) + " seconds")
|
||||
|
||||
|
||||
# Done
|
||||
print("refreshShapersUpdateOnly completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
@ -741,6 +1147,11 @@ if __name__ == '__main__':
|
||||
help="Be verbose",
|
||||
action="store_const", dest="loglevel", const=logging.INFO,
|
||||
)
|
||||
parser.add_argument(
|
||||
'--updateonly',
|
||||
help="Only update to reflect changes in ShapedDevices.csv (partial reload)",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
)
|
||||
parser.add_argument(
|
||||
'--validate',
|
||||
help="Just validate network.json and ShapedDevices.csv",
|
||||
@ -758,6 +1169,8 @@ if __name__ == '__main__':
|
||||
status = validateNetworkAndDevices()
|
||||
elif args.clearrules:
|
||||
tearDown(interfaceA, interfaceB)
|
||||
elif args.updateonly:
|
||||
refreshShapersUpdateOnly()
|
||||
else:
|
||||
# Refresh and/or set up queues
|
||||
refreshShapers()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user