From 0370f7ced8d2abe8bc897832d1e0887f8ae93540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Fri, 9 Sep 2022 14:28:58 -0600 Subject: [PATCH 1/9] Update LibreQoS.py --- v1.2/LibreQoS.py | 245 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 183 insertions(+), 62 deletions(-) diff --git a/v1.2/LibreQoS.py b/v1.2/LibreQoS.py index ad01d1d6..a464ed70 100644 --- a/v1.2/LibreQoS.py +++ b/v1.2/LibreQoS.py @@ -18,7 +18,7 @@ import shutil from ispConfig import fqOrCAKE, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps, \ defaultClassCapacityDownloadMbps, defaultClassCapacityUploadMbps, interfaceA, interfaceB, enableActualShellCommands, \ - runShellCommandsAsSudo, generatedPNDownloadMbps, generatedPNUploadMbps + runShellCommandsAsSudo, generatedPNDownloadMbps, generatedPNUploadMbps, usingXDP def shell(command): if enableActualShellCommands: @@ -38,24 +38,27 @@ def checkIfFirstRunSinceBoot(): lastRun = datetime.strptime(file.read(), "%d-%b-%Y (%H:%M:%S.%f)") systemRunningSince = datetime.fromtimestamp(psutil.boot_time()) if systemRunningSince > lastRun: - print("First time run since system boot. Will load xdp-cpumap-tc from scratch.") + print("First time run since system boot.") return True else: - print("Not first time run since system boot. Will clear xdp filter rules and reload.") + print("Not first time run since system boot.") return False else: - print("First time run since system boot. Will attach xdp.") + print("First time run since system boot.") return True def clearPriorSettings(interfaceA, interfaceB): if enableActualShellCommands: - #shell('tc filter delete dev ' + interfaceA) - #shell('tc filter delete dev ' + interfaceA + ' root') + # If not using XDP, clear tc filter + if usingXDP == False: + #two of these are probably redundant. Will remove later once determined which those are. + shell('tc filter delete dev ' + interfaceA) + shell('tc filter delete dev ' + interfaceA + ' root') + shell('tc filter delete dev ' + interfaceB) + shell('tc filter delete dev ' + interfaceB + ' root') shell('tc qdisc delete dev ' + interfaceA + ' root') - #shell('tc qdisc delete dev ' + interfaceA) - #shell('tc filter delete dev ' + interfaceB) - #shell('tc filter delete dev ' + interfaceB + ' root') shell('tc qdisc delete dev ' + interfaceB + ' root') + #shell('tc qdisc delete dev ' + interfaceA) #shell('tc qdisc delete dev ' + interfaceB) def findQueuesAvailable(): @@ -198,6 +201,9 @@ def validateNetworkAndDevices(): return False def refreshShapers(): + # Starting + print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + # Warn user if enableActualShellCommands is False, because that would mean no actual commands are executing if enableActualShellCommands == False: warnings.warn("enableActualShellCommands is set to False. None of the commands below will actually be executed. Simulated run.", stacklevel=2) @@ -222,7 +228,7 @@ def refreshShapers(): safeToRunRefresh = True else: if (isThisFirstRunSinceBoot == False): - warnings.warn("Validation failed. Because this is not the first run since boot - will exit.") + warnings.warn("Validation failed. Because this is not the first run since boot (queues already set up) - will now exit.") safeToRunRefresh = False else: warnings.warn("Validation failed. However - because this is the first run since boot - will load queues from last good config") @@ -367,8 +373,11 @@ def refreshShapers(): with open(networkJSONfile, 'r') as j: network = json.loads(j.read()) - # Pull rx/tx queues / CPU cores avaialble - queuesAvailable = findQueuesAvailable() + # Pull rx/tx queues / CPU cores available + if usingXDP: + queuesAvailable = findQueuesAvailable() + else: + queuesAvailable = 1 # Generate Parent Nodes. Spread ShapedDevices.csv which lack defined ParentNode across these (balance across CPUs) generatedPNs = [] @@ -407,6 +416,12 @@ def refreshShapers(): minDownload, minUpload = findBandwidthMins(network, 0) + # Define lists for hash filters + ipv4FiltersSrc = [] + ipv4FiltersDst = [] + ipv6FiltersSrc = [] + ipv6FiltersDst = [] + # Parse network structure. For each tier, create corresponding HTB and leaf classes. Prepare for execution later linuxTCcommands = [] xdpCPUmapCommands = [] @@ -451,7 +466,16 @@ def refreshShapers(): for device in circuit['devices']: if device['ipv4s']: for ipv4 in device['ipv4s']: - xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + hex(queue-1) + ' --classid ' + flowIDstring) + if usingXDP: + xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + hex(queue-1) + ' --classid ' + flowIDstring) + else: + ipv4FiltersSrc.append((ipv4, parentString, flowIDstring)) + ipv4FiltersDst.append((ipv4, parentString, flowIDstring)) + if not usingXDP: + if device['ipv6s']: + for ipv6 in device['ipv6s']: + ipv6FiltersSrc.append((ipv6, parentString, flowIDstring)) + ipv6FiltersDst.append((ipv6, parentString, flowIDstring)) if device['deviceName'] not in devicesShaped: devicesShaped.append(device['deviceName']) minor += 1 @@ -468,13 +492,78 @@ def refreshShapers(): queue += 1 major += 1 return minor - + # Print structure of network.json in debug or verbose mode logging.info(json.dumps(network, indent=4)) # Here is the actual call to the recursive traverseNetwork() function. finalMinor is not used. finalMinor = traverseNetwork(network, 0, major=1, minor=3, queue=1, parentClassID="1:1", parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps) + # If XDP off - prepare commands for Hash Tables + + # IPv4 Hash Filters + # Dst + linuxTCcommands.append('filter add dev ' + interfaceA + ' parent 0x1: protocol all u32') + linuxTCcommands.append('filter add dev ' + interfaceA + ' parent 0x1: protocol ip handle 3: u32 divisor 256') + filterHandleCounter = 101 + for i in range (256): + hexID = str(hex(i))#.replace('0x','') + for ipv4Filter in ipv4FiltersDst: + ipv4, parent, classid = ipv4Filter + if '/' in ipv4: + ipv4 = ipv4.split('/')[0] + if (ipv4.split('.', 3)[3]) == str(i): + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceA + ' handle ' + filterHandle + ' protocol ip parent 0x1: u32 ht 3:' + hexID + ': match ip dst ' + ipv4 + ' flowid ' + classid) + filterHandleCounter += 1 + linuxTCcommands.append('filter add dev ' + interfaceA + ' protocol ip parent 0x1: u32 ht 800: match ip dst 0.0.0.0/0 hashkey mask 0x000000ff at 16 link 3:') + # Src + linuxTCcommands.append('filter add dev ' + interfaceB + ' parent 0x1: protocol all u32') + linuxTCcommands.append('filter add dev ' + interfaceB + ' parent 0x1: protocol ip handle 4: u32 divisor 256') + filterHandleCounter = 101 + for i in range (256): + hexID = str(hex(i))#.replace('0x','') + for ipv4Filter in ipv4FiltersSrc: + ipv4, parent, classid = ipv4Filter + if '/' in ipv4: + ipv4 = ipv4.split('/')[0] + if (ipv4.split('.', 3)[3]) == str(i): + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceB + ' handle ' + filterHandle + ' protocol ip parent 0x1: u32 ht 4:' + hexID + ': match ip src ' + ipv4 + ' flowid ' + classid) + filterHandleCounter += 1 + linuxTCcommands.append('filter add dev ' + interfaceB + ' protocol ip parent 0x1: u32 ht 800: match ip src 0.0.0.0/0 hashkey mask 0x000000ff at 12 link 4:') + # IPv6 Hash Filters + # Dst + linuxTCcommands.append('tc filter add dev ' + interfaceA + ' parent 0x1: handle 5: protocol ipv6 u32 divisor 256') + filterHandleCounter = 101 + for ipv6Filter in ipv6FiltersDst: + ipv6, parent, classid = ipv6Filter + withoutCIDR = ipv6.split('/')[0] + third = str(ipaddress.IPv6Address(withoutCIDR).exploded).split(':',5)[3] + usefulPart = third[:2] + hexID = usefulPart + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceA + ' handle ' + filterHandle + ' protocol ipv6 parent 0x1: u32 ht 5:' + hexID + ': match ip6 dst ' + ipv6 + ' flowid ' + classid) + filterHandleCounter += 1 + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceA + ' protocol ipv6 parent 0x1: u32 ht 800:: match ip6 dst ::/0 hashkey mask 0x0000ff00 at 28 link 5:') + filterHandleCounter += 1 + # Src + linuxTCcommands.append('tc filter add dev ' + interfaceB + ' parent 0x1: handle 6: protocol ipv6 u32 divisor 256') + filterHandleCounter = 101 + for ipv6Filter in ipv6FiltersSrc: + ipv6, parent, classid = ipv6Filter + withoutCIDR = ipv6.split('/')[0] + third = str(ipaddress.IPv6Address(withoutCIDR).exploded).split(':',5)[3] + usefulPart = third[:2] + hexID = usefulPart + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceB + ' handle ' + filterHandle + ' protocol ipv6 parent 0x1: u32 ht 6:' + hexID + ': match ip6 src ' + ipv6 + ' flowid ' + classid) + filterHandleCounter += 1 + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceB + ' protocol ipv6 parent 0x1: u32 ht 800:: match ip6 src ::/0 hashkey mask 0x0000ff00 at 12 link 6:') + filterHandleCounter += 1 + # Record start time of actual filter reload reloadStartTime = datetime.now() @@ -483,47 +572,75 @@ def refreshShapers(): # If this is the first time LibreQoS.py has run since system boot, load the XDP program and disable XPS # Otherwise, just clear the existing IP filter rules for xdp - if isThisFirstRunSinceBoot: - # Set up XDP-CPUMAP-TC - shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceA + ' --default --disable') - shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceB + ' --default --disable') - shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceA + ' --lan') - shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceB + ' --wan') - if enableActualShellCommands: - # Here we use os.system for the command, because otherwise it sometimes gltiches out with Popen in shell() - result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear') - shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceA) - shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceB) + # If XDP is disabled, skips this entirely + if usingXDP: + if isThisFirstRunSinceBoot: + # Set up XDP-CPUMAP-TC + shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceA + ' --default --disable') + shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceB + ' --default --disable') + shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceA + ' --lan') + shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceB + ' --wan') + if enableActualShellCommands: + # Here we use os.system for the command, because otherwise it sometimes gltiches out with Popen in shell() + result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear') + shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceA) + shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceB) + else: + if enableActualShellCommands: + result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear') + + if usingXDP: + # Create MQ qdisc for each CPU core / rx-tx queue (XDP method - requires IPv4) + thisInterface = interfaceA + shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq') + for queue in range(queuesAvailable): + shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityDownloadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityDownloadMbps) + 'mbit') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) + # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv + # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. + # Default class can use up to defaultClassCapacityDownloadMbps when that bandwidth isn't used by known hosts. + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityDownloadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityDownloadMbps) + 'mbit prio 5') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) + + thisInterface = interfaceB + shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq') + for queue in range(queuesAvailable): + shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityUploadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityUploadMbps) + 'mbit') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) + # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv. + # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. + # Default class can use up to defaultClassCapacityUploadMbps when that bandwidth isn't used by known hosts. + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityUploadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityUploadMbps) + 'mbit prio 5') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) else: - if enableActualShellCommands: - result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear') - - # Create MQ qdisc for each interface - thisInterface = interfaceA - shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq') - for queue in range(queuesAvailable): - shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') - shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityDownloadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityDownloadMbps) + 'mbit') - shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) - # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv - # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. - # Default class can use up to defaultClassCapacityDownloadMbps when that bandwidth isn't used by known hosts. - shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityDownloadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityDownloadMbps) + 'mbit prio 5') - shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) + # Create single HTB qdisc (non XDP method - allows IPv6) + thisInterface = interfaceA + shell('tc qdisc replace dev ' + thisInterface + ' root handle 0x1: htb default 2 r2q 1514') + for queue in range(queuesAvailable): + shell('tc qdisc add dev ' + thisInterface + ' parent 0x1:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityDownloadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityDownloadMbps) + 'mbit') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) + # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv + # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. + # Default class can use up to defaultClassCapacityDownloadMbps when that bandwidth isn't used by known hosts. + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityDownloadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityDownloadMbps) + 'mbit prio 5') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) + + thisInterface = interfaceB + shell('tc qdisc replace dev ' + thisInterface + ' root handle 0x1: htb default 2 r2q 1514') + for queue in range(queuesAvailable): + shell('tc qdisc add dev ' + thisInterface + ' parent 0x1:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityUploadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityUploadMbps) + 'mbit') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) + # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv. + # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. + # Default class can use up to defaultClassCapacityUploadMbps when that bandwidth isn't used by known hosts. + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityUploadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityUploadMbps) + 'mbit prio 5') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) - thisInterface = interfaceB - shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq') - for queue in range(queuesAvailable): - shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') - shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityUploadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityUploadMbps) + 'mbit') - shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) - # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv. - # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. - # Default class can use up to defaultClassCapacityUploadMbps when that bandwidth isn't used by known hosts. - shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityUploadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityUploadMbps) + 'mbit prio 5') - shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) - - # Execute actual Linux TC and XDP-CPUMAP-TC filter commands + # Execute actual Linux TC commands print("Executing linux TC class/qdisc commands") with open('linux_tc.txt', 'w') as f: for line in linuxTCcommands: @@ -531,11 +648,16 @@ def refreshShapers(): logging.info(line) shell("/sbin/tc -f -b linux_tc.txt") print("Executed " + str(len(linuxTCcommands)) + " linux TC class/qdisc commands") - print("Executing XDP-CPUMAP-TC IP filter commands") - for command in xdpCPUmapCommands: - logging.info(command) - shell(command) - print("Executed " + str(len(xdpCPUmapCommands)) + " XDP-CPUMAP-TC IP filter commands") + + # Execute actual XDP-CPUMAP-TC filter commands + if usingXDP: + print("Executing XDP-CPUMAP-TC IP filter commands") + for command in xdpCPUmapCommands: + logging.info(command) + shell(command) + print("Executed " + str(len(xdpCPUmapCommands)) + " XDP-CPUMAP-TC IP filter commands") + + # Record end time reloadEndTime = datetime.now() # Recap - warn operator if devices were skipped @@ -565,6 +687,9 @@ def refreshShapers(): # Report reload time reloadTimeSeconds = (reloadEndTime - reloadStartTime).seconds print("Queue and IP filter reload completed in " + str(reloadTimeSeconds) + " seconds") + + # Done + print("refreshShapers completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -590,9 +715,5 @@ if __name__ == '__main__': if args.validate: status = validateNetworkAndDevices() else: - # Starting - print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) # Refresh and/or set up queues refreshShapers() - # Done - print("refreshShapers completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) From 0ad0d84635b3aedd796542165ca57d32ba4764e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Fri, 9 Sep 2022 14:30:12 -0600 Subject: [PATCH 2/9] XDP toggle --- v1.2/ispConfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v1.2/ispConfig.py b/v1.2/ispConfig.py index ba1f7103..308cdb78 100644 --- a/v1.2/ispConfig.py +++ b/v1.2/ispConfig.py @@ -24,9 +24,9 @@ interfaceA = 'eth1' # Interface connected to edge router interfaceB = 'eth2' -# Shape by Site in addition to by AP and Client -# Now deprecated, was only used prior to v1.1 -# shapeBySite = True +# Use XDP? If yes, multiple CPU cores can be used. Limits to IPv4 only. Throughput of 11 Gbps+ +# If using IPv6, choose False. False will limit throughput to 3-6 Gbps +usingXDP = False # Allow shell commands. False causes commands print to console only without being executed. # MUST BE ENABLED FOR PROGRAM TO FUNCTION From 2549316708c69977592dcce51a15193b4ee10b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Fri, 9 Sep 2022 14:49:19 -0600 Subject: [PATCH 3/9] Better handle IPv6 validation --- v1.2/LibreQoS.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/v1.2/LibreQoS.py b/v1.2/LibreQoS.py index a464ed70..a7388bdb 100644 --- a/v1.2/LibreQoS.py +++ b/v1.2/LibreQoS.py @@ -111,7 +111,7 @@ def validateNetworkAndDevices(): circuitID, circuitName, deviceID, deviceName, ParentNode, mac, ipv4_input, ipv6_input, downloadMin, uploadMin, downloadMax, uploadMax, comment = row # Each entry in ShapedDevices.csv can have multiple IPv4s or IPv6s seperated by commas. Split them up and parse each to ensure valid ipv4_hosts = [] - ipv6_hosts = [] + ipv6_subnets_and_hosts = [] if ipv4_input != "": try: ipv4_input = ipv4_input.replace(' ','') @@ -138,13 +138,11 @@ def validateNetworkAndDevices(): else: ipv6_list = [ipv6_input] for ipEntry in ipv6_list: - if '/128' in ipEntry: - ipEntry = ipEntry.replace('/128','') - ipv6_hosts.append(ipaddress.ip_address(ipEntry)) - elif '/' in ipEntry: - ipv6_hosts.extend(list(ipaddress.ip_network(ipEntry).hosts())) + if (type(ipaddress.ip_network(ipEntry)) is ipaddress.IPv6Network) or (type(ipaddress.ip_address(ipEntry)) is ipaddress.IPv6Address): + ipv6_subnets_and_hosts.extend(ipEntry) else: - ipv6_hosts.append(ipaddress.ip_address(ipEntry)) + warnings.warn("Provided IPv6 '" + ipv6_input + "' in ShapedDevices.csv at row " + str(rowNum) + " is not valid.") + devicesValidatedOrNot = False except: warnings.warn("Provided IPv6 '" + ipv6_input + "' in ShapedDevices.csv at row " + str(rowNum) + " is not valid.") devicesValidatedOrNot = False @@ -271,7 +269,7 @@ def refreshShapers(): ipv4_hosts.append(host) else: ipv4_hosts.append(ipEntry) - ipv6_hosts = [] + ipv6_subnets_and_hosts = [] if ipv6_input != "": ipv6_input = ipv6_input.replace(' ','') if "," in ipv6_input: @@ -279,14 +277,7 @@ def refreshShapers(): else: ipv6_list = [ipv6_input] for ipEntry in ipv6_list: - if '/128' in ipEntry: - ipv6_hosts.append(ipEntry) - elif '/' in ipEntry: - theseHosts = ipaddress.ip_network(ipEntry).hosts() - for host in theseHosts: - ipv6_hosts.append(str(host)) - else: - ipv6_hosts.append(ipEntry) + ipv6_subnets_and_hosts.append(ipEntry) # If there is something in the circuit ID field if circuitID != "": # Seen circuit before @@ -308,7 +299,7 @@ def refreshShapers(): "deviceName": deviceName, "mac": mac, "ipv4s": ipv4_hosts, - "ipv6s": ipv6_hosts, + "ipv6s": ipv6_subnets_and_hosts, } devicesListForCircuit.append(thisDevice) circuit['devices'] = devicesListForCircuit @@ -324,7 +315,7 @@ def refreshShapers(): "deviceName": deviceName, "mac": mac, "ipv4s": ipv4_hosts, - "ipv6s": ipv6_hosts, + "ipv6s": ipv6_subnets_and_hosts, } deviceListForCircuit.append(thisDevice) thisCircuit = { @@ -353,7 +344,7 @@ def refreshShapers(): "deviceName": deviceName, "mac": mac, "ipv4s": ipv4_hosts, - "ipv6s": ipv6_hosts, + "ipv6s": ipv6_subnets_and_hosts, } deviceListForCircuit.append(thisDevice) thisCircuit = { From 6f39280cfe79ebaf17c05343b89205fe92c1e4a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Fri, 9 Sep 2022 15:09:04 -0600 Subject: [PATCH 4/9] Add command line option to clear rules only --- v1.2/LibreQoS.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/v1.2/LibreQoS.py b/v1.2/LibreQoS.py index a7388bdb..a43dd7e4 100644 --- a/v1.2/LibreQoS.py +++ b/v1.2/LibreQoS.py @@ -60,6 +60,15 @@ def clearPriorSettings(interfaceA, interfaceB): shell('tc qdisc delete dev ' + interfaceB + ' root') #shell('tc qdisc delete dev ' + interfaceA) #shell('tc qdisc delete dev ' + interfaceB) + +def tearDown(interfaceA, interfaceB): + # Full teardown of everything for exiting LibreQoS + if enableActualShellCommands: + # If using XDP, remove xdp program from interfaces + if usingXDP == True: + shell('ip link set dev ' + interfaceA + ' xdp off') + shell('ip link set dev ' + interfaceB + ' xdp off') + clearPriorSettings(interfaceA, interfaceB) def findQueuesAvailable(): # Find queues and CPU cores available. Use min between those two as queuesAvailable @@ -700,11 +709,18 @@ if __name__ == '__main__': help="Just validate network.json and ShapedDevices.csv", action=argparse.BooleanOptionalAction, ) + parser.add_argument( + '--clearrules', + help="Clear ip filters, qdiscs, and xdp setup if any", + action=argparse.BooleanOptionalAction, + ) args = parser.parse_args() logging.basicConfig(level=args.loglevel) if args.validate: status = validateNetworkAndDevices() + elif args.clearrules: + tearDown(interfaceA, interfaceB) else: # Refresh and/or set up queues refreshShapers() From ee3b6df893cf9142d3e9606be749dd3bcedae041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Sat, 10 Sep 2022 06:10:33 -0600 Subject: [PATCH 5/9] Update ispConfig.py --- v1.2/ispConfig.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/v1.2/ispConfig.py b/v1.2/ispConfig.py index 308cdb78..ee89213f 100644 --- a/v1.2/ispConfig.py +++ b/v1.2/ispConfig.py @@ -61,6 +61,11 @@ uispAuthToken = '' shapeRouterOrStation = 'router' # List any sites that should not be included, with each site name surrounded by '' and seperated by commas excludeSites = [] +# If you use IPv6, this can be used to find associated IPv6 prefixes for your clients' IPv4 addresses, and match them to those devices +findIPv6usingMikrotik = False +# If you want to provide a safe cushion for speed test results to prevent customer complains, you can set this to 1.15 (15% above plan rate). +# If not, you can leave as 1.0 +bandwidthOverheadFactor = 1.0 # API Auth apiUsername = "testUser" From 1be00adaa94b472cc904517b4ee77a0c57e8e65f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Sat, 10 Sep 2022 06:13:43 -0600 Subject: [PATCH 6/9] Add IPv6 support for UISP integration --- v1.2/integrationUISP.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/v1.2/integrationUISP.py b/v1.2/integrationUISP.py index 2f77c268..be14df00 100644 --- a/v1.2/integrationUISP.py +++ b/v1.2/integrationUISP.py @@ -2,15 +2,19 @@ import requests import os import csv import ipaddress -from ispConfig import UISPbaseURL, uispAuthToken, shapeRouterOrStation, allowedSubnets, ignoreSubnets, excludeSites +from ispConfig import UISPbaseURL, uispAuthToken, shapeRouterOrStation, allowedSubnets, ignoreSubnets, excludeSites, findIPv6usingMikrotik, bandwidthOverheadFactor import shutil import json +if findIPv6usingMikrotik == True: + from mikrotikFindIPv6 import pullMikrotikIPv6 knownRouterModels = ['ACB-AC', 'ACB-ISP'] -knownAPmodels = ['LTU-Rocket', 'RP-5AC-Gen2', 'LAP-GPS', 'Wave-AP'] +knownAPmodels = ['LTU-Rocket', 'RP-5AC', 'RP-5AC-Gen2', 'LAP-GPS', 'Wave-AP'] def isInAllowedSubnets(inputIP): isAllowed = False + if '/' in inputIP: + inputIP = inputIP.split('/')[0] for subnet in allowedSubnets: if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)): isAllowed = True @@ -167,6 +171,9 @@ def createShaper(): headers = {'accept':'application/json', 'x-auth-token': uispAuthToken} r = requests.get(url, headers=headers) allDevices = r.json() + ipv4ToIPv6 = {} + if findIPv6usingMikrotik: + ipv4ToIPv6 = pullMikrotikIPv6() for uispClientSite in clientSites: #if (uispClientSite['identification']['status'] == 'active') and (uispClientSite['identification']['suspended'] == False): if (uispClientSite['identification']['suspended'] == False): @@ -204,25 +211,26 @@ def createShaper(): else: deviceMAC = '' if (deviceRole == 'router') or (deviceModel in knownRouterModels): - deviceIPstring = device['ipAddress'] - if '/' in deviceIPstring: - deviceIPstring = deviceIPstring.split("/")[0] - if isInAllowedSubnets(deviceIPstring): + ipv4 = device['ipAddress'] + if '/' in ipv4: + ipv4 = ipv4.split("/")[0] + ipv6 = '' + if ipv4 in ipv4ToIPv6.keys(): + ipv6 = ipv4ToIPv6[ipv4] + if isInAllowedSubnets(ipv4): deviceModel = device['identification']['model'] deviceModelName = device['identification']['modelName'] - - minSpeedDown = min(3,downloadSpeedMbps) - minSpeedUp = min(3,uploadSpeedMbps) - maxSpeedDown = round(1.15*downloadSpeedMbps) - maxSpeedUp = round(1.15*uploadSpeedMbps) - + minSpeedDown = min(1,downloadSpeedMbps) + minSpeedUp = min(1,uploadSpeedMbps) + maxSpeedDown = round(bandwidthOverheadFactor*downloadSpeedMbps) + maxSpeedUp = round(bandwidthOverheadFactor*uploadSpeedMbps) #Customers directly connected to Sites if AP == 'none': try: AP = siteIDtoName[uispClientSite['identification']['parent']['id']] except: AP = 'none' - devicesToImport.append((uispClientSiteID, address, '', deviceName, AP, deviceMAC, deviceIPstring,'', str(minSpeedDown), str(minSpeedUp), str(maxSpeedDown),str(maxSpeedUp),'')) + devicesToImport.append((uispClientSiteID, address, '', deviceName, AP, deviceMAC, ipv4, ipv6, str(minSpeedDown), str(minSpeedUp), str(maxSpeedDown),str(maxSpeedUp),'')) foundCPEforThisClientSite = True else: print("Failed to import devices from " + uispClientSite['description']['address'] + ". Missing QoS.") From 2ac2bbd71ce032c745e46d8558c670ca1b211e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Sat, 10 Sep 2022 06:16:04 -0600 Subject: [PATCH 7/9] Add UISP Integration IPv6 support --- v1.2/mikrotikDHCPRouterList.csv | 2 ++ v1.2/mikrotikFindIPv6.py | 52 +++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 v1.2/mikrotikDHCPRouterList.csv create mode 100644 v1.2/mikrotikFindIPv6.py diff --git a/v1.2/mikrotikDHCPRouterList.csv b/v1.2/mikrotikDHCPRouterList.csv new file mode 100644 index 00000000..62d3a49e --- /dev/null +++ b/v1.2/mikrotikDHCPRouterList.csv @@ -0,0 +1,2 @@ +Router Name / ID,IP,Username,Password +main,100.64.0.1,admin, diff --git a/v1.2/mikrotikFindIPv6.py b/v1.2/mikrotikFindIPv6.py new file mode 100644 index 00000000..0a38cd52 --- /dev/null +++ b/v1.2/mikrotikFindIPv6.py @@ -0,0 +1,52 @@ +#!/usr/bin/python3 +import routeros_api +import csv + +def pullMikrotikIPv6(): + ipv4ToIPv6 = {} + routerList = [] + with open('mikrotikDHCPRouterList.csv') as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + next(csv_reader) + for row in csv_reader: + RouterName, IP, Username, Password = row + routerList.append((RouterName, IP, Username, Password)) + for router in routerList: + RouterName, IP, inputUsername, inputPassword = router + connection = routeros_api.RouterOsApiPool(IP, username=inputUsername, password=inputPassword, use_ssl=True, ssl_verify=False, ssl_verify_hostname=False, plaintext_login=True) + api = connection.get_api() + macToIPv4 = {} + macToIPv6 = {} + clientAddressToIPv6 = {} + list_dhcp = api.get_resource('/ip/dhcp-server/lease') + entries = list_dhcp.get() + for entry in entries: + try: + macToIPv4[entry['mac-address']] = entry['address'] + except: + pass + list_dhcp = api.get_resource('/ipv6/dhcp-server/binding') + entries = list_dhcp.get() + for entry in entries: + try: + clientAddressToIPv6[entry['client-address']] = entry['address'] + except: + pass + list_dhcp = api.get_resource('/ipv6/neighbor') + entries = list_dhcp.get() + for entry in entries: + try: + realIPv6 = clientAddressToIPv6[entry['address']] + macToIPv6[entry['mac-address']] = realIPv6 + except: + pass + for mac, ipv6 in macToIPv6.items(): + try: + ipv4 = macToIPv4[mac] + ipv4ToIPv6[ipv4] = ipv6 + except: + print('Failed to find associated IPv4 for ' + ipv6) + return ipv4ToIPv6 + +if __name__ == '__main__': + print(pullMikrotikIPv6()) From bfb98d3975ebc879ef82278f62e1d72778331d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Sat, 10 Sep 2022 06:20:52 -0600 Subject: [PATCH 8/9] Update README.md --- v1.2/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/v1.2/README.md b/v1.2/README.md index 21cfaea3..ea79ad02 100644 --- a/v1.2/README.md +++ b/v1.2/README.md @@ -24,6 +24,7 @@ ShapedDevices.csv now has a field for Circuit ID. If the listed Circuit ID is th ## UISP Integration This integration fully maps out your entire UISP network. +Add UISP info under "Optional UISP integration" in ispConfig.py To use: 1. Delete network.json and, if you have it, integrationUISPbandwidths.csv @@ -31,3 +32,10 @@ To use: It will create a network.json with approximated bandwidths for APs based on UISP's reported capacities, and fixed bandwidth of 1000/1000 for sites. You can modify integrationUISPbandwidths.csv to correct bandwidth rates. It will load integrationUISPbandwidths.csv on each run and use those listed bandwidths to create network.json. It will always overwrite ShapedDevices.csv on each run by pulling devices from UISP. + +### UISP Integration - IPv6 Support +This will match IPv4 MAC addresses in the DHCP server leases of your mikrotik to DHCPv6 bindings, and include those IPv6 addresses with their respective devices. + +To enable: +* Edit mikrotikDHCPRouterList.csv to list of your mikrotik DHCPv6 servers +* Set findIPv6usingMikrotik in ispConfig.py to True From b88f5af47732fcf12008c5f83397b27e52cf2716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Sat, 10 Sep 2022 06:21:30 -0600 Subject: [PATCH 9/9] Update README.md --- v1.2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v1.2/README.md b/v1.2/README.md index ea79ad02..0b10adb0 100644 --- a/v1.2/README.md +++ b/v1.2/README.md @@ -6,7 +6,7 @@ - Support for multiple IPv4s or IPv6s per device -- Reduced reload time by 80% +- Reduced reload time by 80%. Actual packet loss is <25ms on reload of queues. - Command line arguments ```--debug```, ```--verbose```, and ```--validate```.