From 22616107236acf53a24b3d9b33dc1046820c9761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Thu, 20 Oct 2022 15:59:34 -0600 Subject: [PATCH] Update LibreQoS.py --- v1.3/LibreQoS.py | 799 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 606 insertions(+), 193 deletions(-) diff --git a/v1.3/LibreQoS.py b/v1.3/LibreQoS.py index 2edea901..a6d9641b 100755 --- a/v1.3/LibreQoS.py +++ b/v1.3/LibreQoS.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 -# v1.2.1 +#!/usr/bin/python3 +# v1.3 import csv import io @@ -19,9 +20,13 @@ import shutil import binpacking from ispConfig import fqOrCAKE, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps, \ - interfaceA, interfaceB, enableActualShellCommands, \ + interfaceA, interfaceB, enableActualShellCommands, useBinPackingToBalanceCPU, \ runShellCommandsAsSudo, generatedPNDownloadMbps, generatedPNUploadMbps, queuesAvailableOverride +# Automatically account for TCP overhead of plans. For example a 100Mbps plan needs to be set to 109Mbps for the user to ever see that result on a speed test +# Does not apply to nodes of any sort, just endpoint devices +tcpOverheadFactor = 1.09 + def shell(command): if enableActualShellCommands: if runShellCommandsAsSudo: @@ -30,12 +35,25 @@ def shell(command): commands = command.split(' ') proc = subprocess.Popen(commands, stdout=subprocess.PIPE) for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding - logging.info(line) + if logging.DEBUG <= logging.root.level: + print(line) if ("RTNETLINK answers" in line) or ("We have an error talking to the kernel" in line): warnings.warn("Command: '" + command + "' resulted in " + line, stacklevel=2) else: logging.info(command) +def shellTC(command): + if enableActualShellCommands: + print(command) + commands = command.split(' ') + proc = subprocess.Popen(commands, stdout=subprocess.PIPE) + for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding + if logging.DEBUG <= logging.root.level: + print(line) + if ("RTNETLINK answers" in line) or ("We have an error talking to the kernel" in line): + warnings.warn("Command: '" + command + "' resulted in " + line, stacklevel=2) + raise SystemError("Command: '" + command + "' resulted in " + line) + def checkIfFirstRunSinceBoot(): if os.path.isfile("lastRun.txt"): with open("lastRun.txt", 'r') as file: @@ -84,6 +102,10 @@ def findQueuesAvailable(): print("NIC queues (Override):\t\t\t" + str(queuesAvailable)) cpuCount = multiprocessing.cpu_count() print("CPU cores:\t\t\t" + str(cpuCount)) + if queuesAvailable < 2: + raise SystemError('Only 1 NIC rx/tx queue avaialable. You will need to use a NIC with 2 or more rx/tx queues available.') + if queuesAvailable < 2: + raise SystemError('Only 1 CPU core avaialable. You will need to use a CPU with 2 or more CPU cores.') queuesAvailable = min(queuesAvailable,cpuCount) print("queuesAvailable set to:\t" + str(queuesAvailable)) else: @@ -121,6 +143,10 @@ def validateNetworkAndDevices(): seenTheseIPsAlready = [] for row in commentsRemoved: circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row + # Must have circuitID, it's a unique identifier requried for stateful changes to queue structure + if circuitID == '': + warnings.warn("No Circuit ID provided in ShapedDevices.csv at row " + str(rowNum), stacklevel=2) + devicesValidatedOrNot = False # Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s seperated by commas. Split them up and parse each to ensure valid ipv4_subnets_and_hosts = [] ipv6_subnets_and_hosts = [] @@ -222,6 +248,137 @@ def validateNetworkAndDevices(): else: return False +def loadSubscriberCircuits(shapedDevicesFile): + # Load Subscriber Circuits & Devices + subscriberCircuits = [] + knownCircuitIDs = [] + counterForCircuitsWithoutParentNodes = 0 + dictForCircuitsWithoutParentNodes = {} + with open(shapedDevicesFile) as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + # Remove comments if any + commentsRemoved = [] + for row in csv_reader: + if not row[0].startswith('#'): + commentsRemoved.append(row) + # Remove header + commentsRemoved.pop(0) + for row in commentsRemoved: + circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row + ipv4_subnets_and_hosts = [] + # Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s seperated by commas. Split them up and parse each + if ipv4_input != "": + ipv4_input = ipv4_input.replace(' ','') + if "," in ipv4_input: + ipv4_list = ipv4_input.split(',') + else: + ipv4_list = [ipv4_input] + for ipEntry in ipv4_list: + ipv4_subnets_and_hosts.append(ipEntry) + ipv6_subnets_and_hosts = [] + if ipv6_input != "": + ipv6_input = ipv6_input.replace(' ','') + if "," in ipv6_input: + ipv6_list = ipv6_input.split(',') + else: + ipv6_list = [ipv6_input] + for ipEntry in ipv6_list: + ipv6_subnets_and_hosts.append(ipEntry) + # If there is something in the circuit ID field + if circuitID != "": + # Seen circuit before + if circuitID in knownCircuitIDs: + for circuit in subscriberCircuits: + if circuit['circuitID'] == circuitID: + if circuit['ParentNode'] != "none": + if circuit['ParentNode'] != ParentNode: + errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID + raise ValueError(errorMessageString) + if ((circuit['minDownload'] != round(int(downloadMin)*tcpOverheadFactor)) + or (circuit['minUpload'] != round(int(uploadMin)*tcpOverheadFactor)) + or (circuit['maxDownload'] != round(int(downloadMax)*tcpOverheadFactor)) + or (circuit['maxUpload'] != round(int(uploadMax)*tcpOverheadFactor))): + warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different bandwidth parameters than other devices on this circuit. Will instead use the bandwidth parameters defined by the first device added to its circuit.", stacklevel=2) + devicesListForCircuit = circuit['devices'] + thisDevice = { + "deviceID": deviceID, + "deviceName": deviceName, + "mac": mac, + "ipv4s": ipv4_subnets_and_hosts, + "ipv6s": ipv6_subnets_and_hosts, + "comment": comment + } + devicesListForCircuit.append(thisDevice) + circuit['devices'] = devicesListForCircuit + # Have not seen circuit before + else: + knownCircuitIDs.append(circuitID) + if ParentNode == "": + ParentNode = "none" + ParentNode = ParentNode.strip() + deviceListForCircuit = [] + thisDevice = { + "deviceID": deviceID, + "deviceName": deviceName, + "mac": mac, + "ipv4s": ipv4_subnets_and_hosts, + "ipv6s": ipv6_subnets_and_hosts, + "comment": comment + } + deviceListForCircuit.append(thisDevice) + thisCircuit = { + "circuitID": circuitID, + "circuitName": circuitName, + "ParentNode": ParentNode, + "devices": deviceListForCircuit, + "minDownload": round(int(downloadMin)*tcpOverheadFactor), + "minUpload": round(int(uploadMin)*tcpOverheadFactor), + "maxDownload": round(int(downloadMax)*tcpOverheadFactor), + "maxUpload": round(int(uploadMax)*tcpOverheadFactor), + "classid": '', + "comment": comment + } + if thisCircuit['ParentNode'] == 'none': + thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes + dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((round(int(downloadMax)*tcpOverheadFactor))+(round(int(uploadMax)*tcpOverheadFactor))) + counterForCircuitsWithoutParentNodes += 1 + subscriberCircuits.append(thisCircuit) + # If there is nothing in the circuit ID field + else: + # Copy deviceName to circuitName if none defined already + if circuitName == "": + circuitName = deviceName + if ParentNode == "": + ParentNode = "none" + ParentNode = ParentNode.strip() + deviceListForCircuit = [] + thisDevice = { + "deviceID": deviceID, + "deviceName": deviceName, + "mac": mac, + "ipv4s": ipv4_subnets_and_hosts, + "ipv6s": ipv6_subnets_and_hosts, + } + deviceListForCircuit.append(thisDevice) + thisCircuit = { + "circuitID": circuitID, + "circuitName": circuitName, + "ParentNode": ParentNode, + "devices": deviceListForCircuit, + "minDownload": round(int(downloadMin)*tcpOverheadFactor), + "minUpload": round(int(uploadMin)*tcpOverheadFactor), + "maxDownload": round(int(downloadMax)*tcpOverheadFactor), + "maxUpload": round(int(uploadMax)*tcpOverheadFactor), + "classid": '', + "comment": comment + } + if thisCircuit['ParentNode'] == 'none': + thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes + dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((round(int(downloadMax)*tcpOverheadFactor))+(round(int(uploadMax)*tcpOverheadFactor))) + counterForCircuitsWithoutParentNodes += 1 + subscriberCircuits.append(thisCircuit) + return (subscriberCircuits, dictForCircuitsWithoutParentNodes) + def refreshShapers(): # Starting @@ -237,11 +394,6 @@ def refreshShapers(): isThisFirstRunSinceBoot = checkIfFirstRunSinceBoot() - # Automatically account for TCP overhead of plans. For example a 100Mbps plan needs to be set to 109Mbps for the user to ever see that result on a speed test - # Does not apply to nodes of any sort, just endpoint devices - tcpOverheadFactor = 1.09 - - # Files shapedDevicesFile = 'ShapedDevices.csv' networkJSONfile = 'network.json' @@ -249,6 +401,7 @@ def refreshShapers(): # Check validation safeToRunRefresh = False + print("Validating input files '" + shapedDevicesFile + "' and '" + networkJSONfile + "'") if (validateNetworkAndDevices() == True): shutil.copyfile('ShapedDevices.csv', 'lastGoodConfig.csv') shutil.copyfile('network.json', 'lastGoodConfig.json') @@ -267,134 +420,8 @@ def refreshShapers(): if safeToRunRefresh == True: # Load Subscriber Circuits & Devices - subscriberCircuits = [] - knownCircuitIDs = [] - counterForCircuitsWithoutParentNodes = 0 - dictForCircuitsWithoutParentNodes = {} - with open(shapedDevicesFile) as csv_file: - csv_reader = csv.reader(csv_file, delimiter=',') - # Remove comments if any - commentsRemoved = [] - for row in csv_reader: - if not row[0].startswith('#'): - commentsRemoved.append(row) - # Remove header - commentsRemoved.pop(0) - for row in commentsRemoved: - circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row - ipv4_subnets_and_hosts = [] - # Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s seperated by commas. Split them up and parse each - if ipv4_input != "": - ipv4_input = ipv4_input.replace(' ','') - if "," in ipv4_input: - ipv4_list = ipv4_input.split(',') - else: - ipv4_list = [ipv4_input] - for ipEntry in ipv4_list: - ipv4_subnets_and_hosts.append(ipEntry) - ipv6_subnets_and_hosts = [] - if ipv6_input != "": - ipv6_input = ipv6_input.replace(' ','') - if "," in ipv6_input: - ipv6_list = ipv6_input.split(',') - else: - ipv6_list = [ipv6_input] - for ipEntry in ipv6_list: - ipv6_subnets_and_hosts.append(ipEntry) - # If there is something in the circuit ID field - if circuitID != "": - # Seen circuit before - if circuitID in knownCircuitIDs: - for circuit in subscriberCircuits: - if circuit['circuitID'] == circuitID: - if circuit['ParentNode'] != "none": - if circuit['ParentNode'] != ParentNode: - errorMessageString = "Device " + deviceName + " with deviceID " + deviceID + " had different Parent Node from other devices of circuit ID #" + circuitID - raise ValueError(errorMessageString) - if ((circuit['downloadMin'] != round(int(downloadMin)*tcpOverheadFactor)) - or (circuit['uploadMin'] != round(int(uploadMin)*tcpOverheadFactor)) - or (circuit['downloadMax'] != round(int(downloadMax)*tcpOverheadFactor)) - or (circuit['uploadMax'] != round(int(uploadMax)*tcpOverheadFactor))): - warnings.warn("Device " + deviceName + " with ID " + deviceID + " had different bandwidth parameters than other devices on this circuit. Will instead use the bandwidth parameters defined by the first device added to its circuit.", stacklevel=2) - devicesListForCircuit = circuit['devices'] - thisDevice = { - "deviceID": deviceID, - "deviceName": deviceName, - "mac": mac, - "ipv4s": ipv4_subnets_and_hosts, - "ipv6s": ipv6_subnets_and_hosts, - "comment": comment - } - devicesListForCircuit.append(thisDevice) - circuit['devices'] = devicesListForCircuit - # Have not seen circuit before - else: - knownCircuitIDs.append(circuitID) - if ParentNode == "": - ParentNode = "none" - ParentNode = ParentNode.strip() - deviceListForCircuit = [] - thisDevice = { - "deviceID": deviceID, - "deviceName": deviceName, - "mac": mac, - "ipv4s": ipv4_subnets_and_hosts, - "ipv6s": ipv6_subnets_and_hosts, - "comment": comment - } - deviceListForCircuit.append(thisDevice) - thisCircuit = { - "circuitID": circuitID, - "circuitName": circuitName, - "ParentNode": ParentNode, - "devices": deviceListForCircuit, - "downloadMin": round(int(downloadMin)*tcpOverheadFactor), - "uploadMin": round(int(uploadMin)*tcpOverheadFactor), - "downloadMax": round(int(downloadMax)*tcpOverheadFactor), - "uploadMax": round(int(uploadMax)*tcpOverheadFactor), - "qdisc": '', - "comment": comment - } - if thisCircuit['ParentNode'] == 'none': - thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes - dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((round(int(downloadMax)*tcpOverheadFactor))+(round(int(uploadMax)*tcpOverheadFactor))) - counterForCircuitsWithoutParentNodes += 1 - subscriberCircuits.append(thisCircuit) - # If there is nothing in the circuit ID field - else: - # Copy deviceName to circuitName if none defined already - if circuitName == "": - circuitName = deviceName - if ParentNode == "": - ParentNode = "none" - ParentNode = ParentNode.strip() - deviceListForCircuit = [] - thisDevice = { - "deviceID": deviceID, - "deviceName": deviceName, - "mac": mac, - "ipv4s": ipv4_subnets_and_hosts, - "ipv6s": ipv6_subnets_and_hosts, - } - deviceListForCircuit.append(thisDevice) - thisCircuit = { - "circuitID": circuitID, - "circuitName": circuitName, - "ParentNode": ParentNode, - "devices": deviceListForCircuit, - "downloadMin": round(int(downloadMin)*tcpOverheadFactor), - "uploadMin": round(int(uploadMin)*tcpOverheadFactor), - "downloadMax": round(int(downloadMax)*tcpOverheadFactor), - "uploadMax": round(int(uploadMax)*tcpOverheadFactor), - "qdisc": '', - "comment": comment - } - if thisCircuit['ParentNode'] == 'none': - thisCircuit['idForCircuitsWithoutParentNodes'] = counterForCircuitsWithoutParentNodes - dictForCircuitsWithoutParentNodes[counterForCircuitsWithoutParentNodes] = ((round(int(downloadMax)*tcpOverheadFactor))+(round(int(uploadMax)*tcpOverheadFactor))) - counterForCircuitsWithoutParentNodes += 1 - subscriberCircuits.append(thisCircuit) - + subscriberCircuits, dictForCircuitsWithoutParentNodes = loadSubscriberCircuits(shapedDevicesFile) + # Load network heirarchy with open(networkJSONfile, 'r') as j: @@ -404,30 +431,45 @@ def refreshShapers(): # Pull rx/tx queues / CPU cores available queuesAvailable = findQueuesAvailable() - # Generate Parent Nodes. Spread ShapedDevices.csv which lack defined ParentNode across these (balance across CPUs) + print("Generating parent nodes") + existingPNs = 0 + for node in network: + existingPNs += 1 generatedPNs = [] - for x in range(queuesAvailable): + numberOfGeneratedPNs = queuesAvailable-existingPNs + for x in range(numberOfGeneratedPNs): genPNname = "Generated_PN_" + str(x+1) network[genPNname] = { "downloadBandwidthMbps":generatedPNDownloadMbps, "uploadBandwidthMbps":generatedPNUploadMbps } generatedPNs.append(genPNname) - bins = binpacking.to_constant_bin_number(dictForCircuitsWithoutParentNodes, queuesAvailable) - genPNcounter = 0 - for binItem in bins: - sumItem = 0 - logging.info(generatedPNs[genPNcounter] + " will contain " + str(len(binItem)) + " circuits") - for key in binItem.keys(): - for circuit in subscriberCircuits: - if circuit['ParentNode'] == 'none': - if circuit['idForCircuitsWithoutParentNodes'] == key: - circuit['ParentNode'] = generatedPNs[genPNcounter] - genPNcounter += 1 - if genPNcounter >= queuesAvailable: - genPNcounter = 0 - + if useBinPackingToBalanceCPU: + print("Using binpacking module to sort circuits by CPU core") + bins = binpacking.to_constant_bin_number(dictForCircuitsWithoutParentNodes, numberOfGeneratedPNs) + genPNcounter = 0 + for binItem in bins: + sumItem = 0 + logging.info(generatedPNs[genPNcounter] + " will contain " + str(len(binItem)) + " circuits") + for key in binItem.keys(): + for circuit in subscriberCircuits: + if circuit['ParentNode'] == 'none': + if circuit['idForCircuitsWithoutParentNodes'] == key: + circuit['ParentNode'] = generatedPNs[genPNcounter] + genPNcounter += 1 + if genPNcounter >= queuesAvailable: + genPNcounter = 0 + print("Finished binpacking") + else: + genPNcounter = 0 + for circuit in subscriberCircuits: + if circuit['ParentNode'] == 'none': + circuit['ParentNode'] = generatedPNs[genPNcounter] + genPNcounter += 1 + if genPNcounter >= queuesAvailable: + genPNcounter = 0 + print("Generated parent nodes created") # Find the bandwidth minimums for each node by combining mimimums of devices lower in that node's heirarchy def findBandwidthMins(data, depth): @@ -437,8 +479,8 @@ def refreshShapers(): for elem in data: for circuit in subscriberCircuits: if elem == circuit['ParentNode']: - minDownload += circuit['downloadMin'] - minUpload += circuit['uploadMin'] + minDownload += circuit['minDownload'] + minUpload += circuit['minUpload'] if 'children' in data[elem]: minDL, minUL = findBandwidthMins(data[elem]['children'], depth+1) minDownload += minDL @@ -446,18 +488,24 @@ def refreshShapers(): data[elem]['downloadBandwidthMbpsMin'] = minDownload data[elem]['uploadBandwidthMbpsMin'] = minUpload return minDownload, minUpload + logging.info("Finding the bandwidth minimums for each node") minDownload, minUpload = findBandwidthMins(network, 0) - + logging.info("Found the bandwidth minimums for each node") # Parse network structure and add devices from ShapedDevices.csv - linuxTCcommands = [] - xdpCPUmapCommands = [] parentNodes = [] - def traverseNetwork(data, depth, major, minor, queue, parentClassID, parentMaxDL, parentMaxUL): + minorByCPUpreloaded = {} + knownClassIDs = [] + # Track minor counter by CPU. This way we can have > 32000 hosts (htb has u16 limit to minor handle) + for x in range(queuesAvailable): + minorByCPUpreloaded[x+1] = 3 + def traverseNetwork(data, depth, major, minorByCPU, queue, parentClassID, parentMaxDL, parentMaxUL): for node in data: circuitsForThisNetworkNode = [] - nodeClassID = hex(major) + ':' + hex(minor) + nodeClassID = hex(major) + ':' + hex(minorByCPU[queue]) data[node]['classid'] = nodeClassID + if depth == 0: + parentClassID = hex(major) + ':' data[node]['parentClassID'] = parentClassID # Cap based on this node's max bandwidth, or parent node's max bandwidth, whichever is lower data[node]['downloadBandwidthMbps'] = min(data[node]['downloadBandwidthMbps'],parentMaxDL) @@ -468,31 +516,31 @@ def refreshShapers(): data[node]['downloadBandwidthMbpsMin'] = round(data[node]['downloadBandwidthMbps']*.95) data[node]['uploadBandwidthMbpsMin'] = round(data[node]['uploadBandwidthMbps']*.95) data[node]['classMajor'] = hex(major) - data[node]['classMinor'] = hex(minor) + data[node]['classMinor'] = hex(minorByCPU[queue]) data[node]['cpuNum'] = hex(queue-1) thisParentNode = { "parentNodeName": node, "classID": nodeClassID, - "downloadMax": data[node]['downloadBandwidthMbps'], - "uploadMax": data[node]['uploadBandwidthMbps'], + "maxDownload": data[node]['downloadBandwidthMbps'], + "maxUpload": data[node]['uploadBandwidthMbps'], } parentNodes.append(thisParentNode) - minor += 1 + minorByCPU[queue] = minorByCPU[queue] + 1 for circuit in subscriberCircuits: #If a device from ShapedDevices.csv lists this node as its Parent Node, attach it as a leaf to this node HTB if node == circuit['ParentNode']: - if circuit['downloadMax'] > data[node]['downloadBandwidthMbps']: + if circuit['maxDownload'] > data[node]['downloadBandwidthMbps']: warnings.warn("downloadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2) - if circuit['uploadMax'] > data[node]['uploadBandwidthMbps']: + if circuit['maxUpload'] > data[node]['uploadBandwidthMbps']: warnings.warn("uploadMax of Circuit ID [" + circuit['circuitID'] + "] exceeded that of its parent node. Reducing to that of its parent node now.", stacklevel=2) parentString = hex(major) + ':' - flowIDstring = hex(major) + ':' + hex(minor) - circuit['qdisc'] = flowIDstring + flowIDstring = hex(major) + ':' + hex(minorByCPU[queue]) + circuit['classid'] = flowIDstring # Create circuit dictionary to be added to network structure, eventually output as queuingStructure.json - maxDownload = min(circuit['downloadMax'],data[node]['downloadBandwidthMbps']) - maxUpload = min(circuit['uploadMax'],data[node]['uploadBandwidthMbps']) - minDownload = min(circuit['downloadMin'],maxDownload) - minUpload = min(circuit['uploadMin'],maxUpload) + maxDownload = min(circuit['maxDownload'],data[node]['downloadBandwidthMbps']) + maxUpload = min(circuit['maxUpload'],data[node]['uploadBandwidthMbps']) + minDownload = min(circuit['minDownload'],maxDownload) + minUpload = min(circuit['minUpload'],maxUpload) thisNewCircuitItemForNetwork = { 'maxDownload' : maxDownload, 'maxUpload' : maxUpload, @@ -502,21 +550,22 @@ def refreshShapers(): "circuitName": circuit['circuitName'], "ParentNode": circuit['ParentNode'], "devices": circuit['devices'], - "qdisc": flowIDstring, + "classid": flowIDstring, "classMajor": hex(major), - "classMinor": hex(minor), + "classMinor": hex(minorByCPU[queue]), "comment": circuit['comment'] } # Generate TC commands to be executed later thisNewCircuitItemForNetwork['devices'] = circuit['devices'] circuitsForThisNetworkNode.append(thisNewCircuitItemForNetwork) - minor += 1 + minorByCPU[queue] = minorByCPU[queue] + 1 if len(circuitsForThisNetworkNode) > 0: data[node]['circuits'] = circuitsForThisNetworkNode # Recursive call this function for children nodes attached to this node if 'children' in data[node]: # We need to keep tabs on the minor counter, because we can't have repeating class IDs. Here, we bring back the minor counter from the recursive function - minor = traverseNetwork(data[node]['children'], depth+1, major, minor+1, queue, nodeClassID, data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps']) + minorByCPU[queue] = minorByCPU[queue] + 1 + minorByCPU = traverseNetwork(data[node]['children'], depth+1, major, minorByCPU, queue, nodeClassID, data[node]['downloadBandwidthMbps'], data[node]['uploadBandwidthMbps']) # If top level node, increment to next queue / cpu core if depth == 0: if queue >= queuesAvailable: @@ -525,16 +574,16 @@ def refreshShapers(): else: queue += 1 major += 1 - return minor + return minorByCPU # Here is the actual call to the recursive traverseNetwork() function. finalMinor is not used. - finalMinor = traverseNetwork(network, 0, major=1, minor=3, queue=1, parentClassID="1:1", parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps) + minorByCPU = traverseNetwork(network, 0, major=1, minorByCPU=minorByCPUpreloaded, queue=1, parentClassID=None, parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps) linuxTCcommands = [] xdpCPUmapCommands = [] devicesShaped = [] # Root HTB Setup - # Create MQ qdisc for each CPU core / rx-tx queue (XDP method - requires IPv4) + # Create MQ qdisc for each CPU core / rx-tx queue. Generate commands to create corresponding HTB and leaf classes. Prepare commands for execution later thisInterface = interfaceA logging.info("# MQ Setup for " + thisInterface) command = 'qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq' @@ -578,7 +627,7 @@ def refreshShapers(): # Define lists for hash filters def traverseNetwork(data): for node in data: - command = 'class add dev ' + interfaceA + ' parent ' + data[node]['parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ str(data[node]['downloadBandwidthMbpsMin']) + 'mbit ceil '+ str(data[node]['downloadBandwidthMbps']) + 'mbit prio 3' + " # Node: " + node + command = 'class add dev ' + interfaceA + ' parent ' + data[node]['parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ str(data[node]['downloadBandwidthMbpsMin']) + 'mbit ceil '+ str(data[node]['downloadBandwidthMbps']) + 'mbit prio 3' linuxTCcommands.append(command) command = 'class add dev ' + interfaceB + ' parent ' + data[node]['parentClassID'] + ' classid ' + data[node]['classMinor'] + ' htb rate '+ str(data[node]['uploadBandwidthMbpsMin']) + 'mbit ceil '+ str(data[node]['uploadBandwidthMbps']) + 'mbit prio 3' linuxTCcommands.append(command) @@ -591,7 +640,7 @@ def refreshShapers(): if 'devices' in circuit: if 'comment' in circuit['devices'][0]: comment = comment + '| Comment: ' + circuit['devices'][0]['comment'] - command = 'class add dev ' + interfaceA + ' parent ' + data[node]['classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ str(circuit['minDownload']) + 'mbit ceil '+ str(circuit['maxDownload']) + 'mbit prio 3' + comment + command = 'class add dev ' + interfaceA + ' parent ' + data[node]['classid'] + ' classid ' + circuit['classMinor'] + ' htb rate '+ str(circuit['minDownload']) + 'mbit ceil '+ str(circuit['maxDownload']) + 'mbit prio 3' linuxTCcommands.append(command) command = 'qdisc add dev ' + interfaceA + ' parent ' + circuit['classMajor'] + ':' + circuit['classMinor'] + ' ' + fqOrCAKE linuxTCcommands.append(command) @@ -602,30 +651,25 @@ def refreshShapers(): for device in circuit['devices']: if device['ipv4s']: for ipv4 in device['ipv4s']: - if '/' in ipv4: - ipv4AddressOnly, prefixOnly = ipv4.split('/') - xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4AddressOnly) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['qdisc'] + ' --prefix ' + prefixOnly) - else: - xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['qdisc']) + xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['classid']) if device['ipv6s']: for ipv6 in device['ipv6s']: - if '/' in ipv6: - ipv6AddressOnly, prefixOnly = ipv6.split('/') - xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv6AddressOnly) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['qdisc'] + ' --prefix ' + prefixOnly) - else: - xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv6) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['qdisc']) + xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv6) + ' --cpu ' + data[node]['cpuNum'] + ' --classid ' + circuit['classid']) if device['deviceName'] not in devicesShaped: devicesShaped.append(device['deviceName']) # Recursive call this function for children nodes attached to this node if 'children' in data[node]: traverseNetwork(data[node]['children']) - # Here is the actual call to the recursive traverseNetwork() function. finalResult is not used. + # Here is the actual call to the recursive traverseNetwork() function. traverseNetwork(network) - # Save queuingStructure + queuingStructure = {} + queuingStructure['Network'] = network + queuingStructure['lastUsedClassIDCounterByCPU'] = minorByCPU + queuingStructure['generatedPNs'] = generatedPNs with open('queuingStructure.json', 'w') as infile: - json.dump(network, infile, indent=4) + json.dump(queuingStructure, infile, indent=4) # Record start time of actual filter reload @@ -666,6 +710,10 @@ def refreshShapers(): shell("/sbin/tc -f -b linux_tc.txt") tcEndTime = datetime.now() print("Executed " + str(len(linuxTCcommands)) + " linux TC class/qdisc commands") + + + tcEndTime = datetime.now() + print("Executed " + str(len(linuxTCcommands)) + " linux TC class/qdisc commands") # Execute actual XDP-CPUMAP-TC filter commands @@ -700,6 +748,8 @@ def refreshShapers(): name, idNum = entry print('DeviceID: ' + idNum + '\t DeviceName: ' + name) + # Save ShapedDevices.csv as ShapedDevices.lastLoaded.csv + shutil.copyfile('ShapedDevices.csv', 'ShapedDevices.lastLoaded.csv') # Save for stats with open('statsByCircuit.json', 'w') as f: @@ -728,6 +778,362 @@ def refreshShapers(): # Done print("refreshShapers completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) +def refreshShapersUpdateOnly(): + # Starting + print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + + + # Warn user if enableActualShellCommands is False, because that would mean no actual commands are executing + if enableActualShellCommands == False: + warnings.warn("enableActualShellCommands is set to False. None of the commands below will actually be executed. Simulated run.", stacklevel=2) + + + # Files + shapedDevicesFile = 'ShapedDevices.csv' + networkJSONfile = 'network.json' + + + # Check validation + safeToRunRefresh = False + if (validateNetworkAndDevices() == True): + shutil.copyfile('ShapedDevices.csv', 'lastGoodConfig.csv') + shutil.copyfile('network.json', 'lastGoodConfig.json') + print("Backed up good config as lastGoodConfig.csv and lastGoodConfig.json") + safeToRunRefresh = True + else: + warnings.warn("Validation failed. Will now exit.", stacklevel=2) + + if safeToRunRefresh == True: + + # Load queuingStructure + with open('queuingStructure.json', 'r') as infile: + queuingStructure = json.loads(infile.read()) + + + # Load queuingStructure + with open('queuingStructure.json', 'r') as infile: + queuingStructure = json.loads(infile.read()) + + network = queuingStructure['Network'] + lastUsedClassIDCounterByCPU = queuingStructure['lastUsedClassIDCounterByCPU'] + generatedPNs = queuingStructure['generatedPNs'] + + newlyUpdatedSubscriberCircuits, newlyUpdatedDictForCircuitsWithoutParentNodes = loadSubscriberCircuits('ShapedDevices.csv') + lastLoadedSubscriberCircuits, lastLoadedDictForCircuitsWithoutParentNodes = loadSubscriberCircuits('ShapedDevices.lastLoaded.csv') + + + # Load stats files + with open('statsByParentNode.json', 'r') as j: + parentNodes = json.loads(j.read()) + + with open('statsByCircuit.json', 'r') as j: + subscriberCircuits = json.loads(j.read()) + + newlyUpdatedSubscriberCircuitsByID = {} + for circuit in newlyUpdatedSubscriberCircuits: + circuitid = circuit['circuitID'] + newlyUpdatedSubscriberCircuitsByID[circuitid] = circuit + + lastLoadedSubscriberCircuitsByID = {} + for circuit in lastLoadedSubscriberCircuits: + circuitid = circuit['circuitID'] + lastLoadedSubscriberCircuitsByID[circuitid] = circuit + + + def removeDeviceIPsFromFilter(circuit): + for device in circuit['devices']: + for ipv4 in device['ipv4s']: + shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --del --ip ' + str(ipv4)) + for ipv6 in device['ipv6s']: + shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --del --ip ' + str(ipv6)) + + + def addDeviceIPsToFilter(circuit, cpuNumHex): + for device in circuit['devices']: + for ipv4 in device['ipv4s']: + shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + cpuNumHex + ' --classid ' + circuit['classid']) + for ipv6 in device['ipv6s']: + shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv6) + ' --cpu ' + cpuNumHex + ' --classid ' + circuit['classid']) + + + def getAllParentNodes(data, allParentNodes): + for node in data: + if isinstance(node, str): + thisParentNode = node + if thisParentNode not in allParentNodes: + allParentNodes.append(thisParentNode) + if 'children' in data[node]: + for child in data[node]['children']: + result = getAllParentNodes(data[node]['children'][child], allParentNodes) + allParentNodes = allParentNodes + result + return allParentNodes + allParentNodes = getAllParentNodes(network, []) + + + def getClassIDofParentNodes(data, classIDOfParentNodes): + for node in data: + if isinstance(node, str): + thisParentNode = node + classIDOfParentNodes[thisParentNode] = data[node]['classid'] + if 'children' in data[node]: + for child in data[node]['children']: + result = getClassIDofParentNodes(data[node]['children'][child], classIDOfParentNodes) + classIDOfParentNodes.update(result) + return classIDOfParentNodes + classIDOfParentNodes = getClassIDofParentNodes(network, {}) + + + def getAllCircuitIDs(data, allCircuitIDs): + for node in data: + if isinstance(node, str): + thisParentNode = node + if 'circuits' in data[node]: + for circuit in data[node]['circuits']: + if circuit['circuitID'] not in allCircuitIDs: + allCircuitIDs.append(circuit['circuitID']) + if 'children' in data[node]: + for child in data[node]['children']: + result = getAllCircuitIDs(data[node]['children'][child], allCircuitIDs) + for entry in result: + if entry not in allCircuitIDs: + allCircuitIDs.append(result) + return allCircuitIDs + allCircuitIDs = getAllCircuitIDs(network, []) + + + def getClassIDofExistingCircuitID(data, classIDofExistingCircuitID): + for node in data: + if isinstance(node, str): + thisParentNode = node + if 'circuits' in data[node]: + for circuit in data[node]['circuits']: + classIDofExistingCircuitID[circuit['circuitID']] = circuit['classid'] + if 'children' in data[node]: + for child in data[node]['children']: + result = getClassIDofExistingCircuitID(data[node]['children'][child], allCircuitIDs) + classIDofExistingCircuitID.update(result) + return classIDofExistingCircuitID + classIDofExistingCircuitID = getClassIDofExistingCircuitID(network, {}) + + + def getParentNodeOfCircuitID(data, parentNodeOfCircuitID, allCircuitIDs): + for node in data: + if isinstance(node, str): + thisParentNode = node + if 'circuits' in data[node]: + for circuit in data[node]['circuits']: + parentNodeOfCircuitID[circuit['circuitID']] = thisParentNode + if 'children' in data[node]: + for child in data[node]['children']: + result = getParentNodeOfCircuitID(data[node]['children'][child], parentNodeOfCircuitID, allCircuitIDs) + parentNodeOfCircuitID.update(result) + return parentNodeOfCircuitID + parentNodeOfCircuitID = getParentNodeOfCircuitID(network, {}, allCircuitIDs) + + + def getCPUnumOfParentNodes(data, cpuNumOfParentNode): + for node in data: + if isinstance(node, str): + thisParentNode = node + cpuNumOfParentNode[thisParentNode] = data[node]['cpuNum'] + if 'children' in data[node]: + for child in data[node]['children']: + result = getCPUnumOfParentNodes(data[node]['children'][child], cpuNumOfParentNode) + cpuNumOfParentNode.update(result) + return cpuNumOfParentNode + cpuNumOfParentNodeHex = getCPUnumOfParentNodes(network, {}) + cpuNumOfParentNodeInt = {} + for key, value in cpuNumOfParentNodeHex.items(): + cpuNumOfParentNodeInt[key] = int(value,16) + 1 + + + def addCircuitHTBandQdisc(circuit, parentNodeClassID): + minor = circuit['classid'].split(':')[1] + for interface in [interfaceA, interfaceB]: + if interface == interfaceA: + rate = str(circuit['minDownload']) + ceil = str(circuit['maxDownload']) + else: + rate = str(circuit['minUpload']) + ceil = str(circuit['maxUpload']) + command = 'tc class add dev ' + interface + ' parent ' + parentNodeClassID + ' classid ' + minor + ' htb rate ' + rate + 'Mbit ceil ' + ceil + 'Mbit' + print(command) + shell(command) + command = 'tc qdisc add dev ' + interface + ' parent ' + classID + ' ' + fqOrCAKE + print(command) + shell(command) + + + def delHTBclass(classid): + for interface in [interfaceA, interfaceB]: + command = 'tc class del dev ' + interface + ' classid ' + classid + print(command) + shell(command) + + + generatedPNcounter = 1 + circuitsIDsToRemove = [] + circuitsToUpdateByID = {} + circuitsToAddByParentNode = {} + for circuitID, circuit in lastLoadedSubscriberCircuitsByID.items(): + # Same circuit, update parameters (bandwidth, devices) + bandwidthChanged = False + devicesChanged = False + if (circuitID in newlyUpdatedSubscriberCircuitsByID) and (circuitID in lastLoadedSubscriberCircuitsByID): + if newlyUpdatedSubscriberCircuitsByID[circuitID]['maxDownload'] != lastLoadedSubscriberCircuitsByID[circuitID]['maxDownload']: + bandwidthChanged = True + if newlyUpdatedSubscriberCircuitsByID[circuitID]['maxUpload'] != lastLoadedSubscriberCircuitsByID[circuitID]['maxUpload']: + bandwidthChanged = True + if newlyUpdatedSubscriberCircuitsByID[circuitID]['minDownload'] != lastLoadedSubscriberCircuitsByID[circuitID]['minDownload']: + bandwidthChanged = True + if newlyUpdatedSubscriberCircuitsByID[circuitID]['minUpload'] != lastLoadedSubscriberCircuitsByID[circuitID]['minUpload']: + bandwidthChanged = True + if newlyUpdatedSubscriberCircuitsByID[circuitID]['devices'] != lastLoadedSubscriberCircuitsByID[circuitID]['devices']: + devicesChanged = True + if bandwidthChanged == True: + if newlyUpdatedSubscriberCircuitsByID[circuitID]['ParentNode'] == lastLoadedSubscriberCircuitsByID[circuitID]['ParentNode']: + parentNodeActual = circuit['ParentNode'] + if parentNodeActual == 'none': + parentNodeActual = parentNodeOfCircuitID[circuitID] + parentNodeClassID = classIDOfParentNodes[parentNodeActual] + classid = classIDofExistingCircuitID[circuitID] + minor = classid.split(':')[1] + for interface in [interfaceA, interfaceB]: + if interface == interfaceA: + rate = str(newlyUpdatedSubscriberCircuitsByID[circuitID]['minDownload']) + ceil = str(newlyUpdatedSubscriberCircuitsByID[circuitID]['maxDownload']) + else: + rate = str(newlyUpdatedSubscriberCircuitsByID[circuitID]['minUpload']) + ceil = str(newlyUpdatedSubscriberCircuitsByID[circuitID]['maxUpload']) + command = 'tc class change dev ' + interface + ' parent ' + parentNodeClassID + ' classid ' + minor + ' htb rate ' + rate + 'Mbit ceil ' + ceil + 'Mbit' + shell(command) + else: + removeDeviceIPsFromFilter(lastLoadedSubscriberCircuitsByID[circuitID]) + # Delete HTB class, qdisc. Then recreat it. + classid = classIDofExistingCircuitID[circuitID] + circuit['classid'] = classid + delHTBclass(classID) + parentNodeClassID = classIDOfParentNodes[parentNodeOfCircuitID[circuitID]] + addCircuitHTBandQdisc(circuit, parentNodeClassID) + addDeviceIPsToFilter(newlyUpdatedSubscriberCircuitsByID[circuitID], cpuNum) + elif devicesChanged: + removeDeviceIPsFromFilter(lastLoadedSubscriberCircuitsByID[circuitID]) + parentNodeActual = lastLoadedSubscriberCircuitsByID[circuitID]['ParentNode'] + if parentNodeActual == 'none': + parentNodeActual = getParentNodeOfCircuitID(network, circuitID) + cpuNum, parentNodeClassID = getCPUnumAndClassIDOfParentNode(network, parentNodeActual) + addDeviceIPsToFilter(newlyUpdatedSubscriberCircuitsByID[circuitID], cpuNum) + + newlyUpdatedSubscriberCircuitsByID[circuitID]['classid'] = lastLoadedSubscriberCircuitsByID[circuitID]['classid'] + if (bandwidthChanged) or (devicesChanged): + circuitsToUpdateByID[circuitID] = newlyUpdatedSubscriberCircuitsByID[circuitID] + + + # Removed circuit + if (circuitID in lastLoadedSubscriberCircuitsByID) and (circuitID not in newlyUpdatedSubscriberCircuitsByID): + circuitsIDsToRemove.append(circuitID) + removeDeviceIPsFromFilter(lastLoadedSubscriberCircuitsByID[circuitID]) + classid = classIDofExistingCircuitID[circuitID] + delHTBclass(classid) + + + # New circuit + for circuitID, circuit in newlyUpdatedSubscriberCircuitsByID.items(): + if (circuitID in newlyUpdatedSubscriberCircuitsByID) and (circuitID not in lastLoadedSubscriberCircuitsByID): + if newlyUpdatedSubscriberCircuitsByID[circuitID]['ParentNode'] == 'none': + newlyUpdatedSubscriberCircuitsByID[circuitID]['ParentNode'] = 'Generated_PN_' + str(generatedPNcounter) + generatedPNcounter += 1 + if generatedPNcounter > len(generatedPNs): + generatedPNcounter = 1 + cpuNumHex = cpuNumOfParentNodeHex[circuit['ParentNode']] + cpuNumInt = cpuNumOfParentNodeInt[circuit['ParentNode']] + parentNodeClassID = classIDOfParentNodes[circuit['ParentNode']] + classID = parentNodeClassID.split(':')[0] + ':' + hex(lastUsedClassIDCounterByCPU[str(cpuNumInt)]) + lastUsedClassIDCounterByCPU[str(cpuNumInt)] = lastUsedClassIDCounterByCPU[str(cpuNumInt)] + 1 + circuit['classid'] = classID + # Add HTB class, qdisc + addCircuitHTBandQdisc(circuit, parentNodeClassID) + addDeviceIPsToFilter(circuit, cpuNumHex) + if circuit['ParentNode'] in circuitsToAddByParentNode: + temp = circuitsToAddByParentNode[circuit['ParentNode']] + temp.append(circuit) + circuitsToAddByParentNode[circuit['ParentNode']] = temp + else: + temp = [] + temp.append(circuit) + circuitsToAddByParentNode[circuit['ParentNode']] = temp + + + # Update network structure reflecting new circuits, removals, and changes + itemsToChange = (circuitsIDsToRemove, circuitsToUpdateByID, circuitsToAddByParentNode) + def updateNetworkStructure(data, depth, itemsToChange): + circuitsIDsToRemove, circuitsToUpdateByID, circuitsToAddByParentNode = itemsToChange + for node in data: + if isinstance(node, str): + thisParentNode = node + if 'circuits' in data[node]: + for circuit in data[node]['circuits']: + if circuit['circuitID'] in circuitsToUpdateByID: + circuit = circuitsToUpdateByID[circuit['circuitID']] + print('updated') + if circuit['circuitID'] in circuitsIDsToRemove: + data[node]['circuits'].remove(circuit) + if thisParentNode in circuitsToAddByParentNode: + if 'circuits' in data[node]: + temp = data[node]['circuits'] + for circuit in circuitsToAddByParentNode[thisParentNode]: + temp.append(circuit) + data[node]['circuits'] = temp + else: + temp = [] + for circuit in circuitsToAddByParentNode[thisParentNode]: + temp.append(circuit) + data[node]['circuits'] = temp + if 'children' in data[node]: + for child in data[node]['children']: + result = updateNetworkStructure(data[node]['children'][child], depth+1, itemsToChange) + data[node]['children'][child] = result + return data + network = updateNetworkStructure(network, 0 , itemsToChange) + + + # Record start time of actual filter reload + reloadStartTime = datetime.now() + + + # Record end time of all reload commands + reloadEndTime = datetime.now() + + + queuingStructure = {} + queuingStructure['Network'] = network + queuingStructure['lastUsedClassIDCounterByCPU'] = lastUsedClassIDCounterByCPU + queuingStructure['generatedPNs'] = generatedPNs + # Save queuingStructure + with open('queuingStructure.json', 'w') as infile: + json.dump(queuingStructure, infile, indent=4) + + + # copy ShapedDevices.csv and save as ShapedDevices.lastLoaded.csv and lastGoodConfig.csv + shutil.copyfile('ShapedDevices.csv', 'ShapedDevices.lastLoaded.csv') + shutil.copyfile('ShapedDevices.csv', 'lastGoodConfig.csv') + + + # Save for stats + with open('statsByCircuit.json', 'w') as f: + f.write(json.dumps(subscriberCircuits, indent=4)) + with open('statsByParentNode.json', 'w') as f: + f.write(json.dumps(parentNodes, indent=4)) + + + # Report reload time + reloadTimeSeconds = ((reloadEndTime - reloadStartTime).seconds) + (((reloadEndTime - reloadStartTime).microseconds) / 1000000) + print("Queue and IP filter partial reload completed in " + "{:g}".format(round(reloadTimeSeconds,1)) + " seconds") + + + # Done + print("refreshShapersUpdateOnly completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( @@ -741,6 +1147,11 @@ if __name__ == '__main__': help="Be verbose", action="store_const", dest="loglevel", const=logging.INFO, ) + parser.add_argument( + '--updateonly', + help="Only update to reflect changes in ShapedDevices.csv (partial reload)", + action=argparse.BooleanOptionalAction, + ) parser.add_argument( '--validate', help="Just validate network.json and ShapedDevices.csv", @@ -758,6 +1169,8 @@ if __name__ == '__main__': status = validateNetworkAndDevices() elif args.clearrules: tearDown(interfaceA, interfaceB) + elif args.updateonly: + refreshShapersUpdateOnly() else: # Refresh and/or set up queues refreshShapers()