diff --git a/v1.2/LibreQoS.py b/v1.2/LibreQoS.py index ad01d1d6..a464ed70 100644 --- a/v1.2/LibreQoS.py +++ b/v1.2/LibreQoS.py @@ -18,7 +18,7 @@ import shutil from ispConfig import fqOrCAKE, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps, \ defaultClassCapacityDownloadMbps, defaultClassCapacityUploadMbps, interfaceA, interfaceB, enableActualShellCommands, \ - runShellCommandsAsSudo, generatedPNDownloadMbps, generatedPNUploadMbps + runShellCommandsAsSudo, generatedPNDownloadMbps, generatedPNUploadMbps, usingXDP def shell(command): if enableActualShellCommands: @@ -38,24 +38,27 @@ def checkIfFirstRunSinceBoot(): lastRun = datetime.strptime(file.read(), "%d-%b-%Y (%H:%M:%S.%f)") systemRunningSince = datetime.fromtimestamp(psutil.boot_time()) if systemRunningSince > lastRun: - print("First time run since system boot. Will load xdp-cpumap-tc from scratch.") + print("First time run since system boot.") return True else: - print("Not first time run since system boot. Will clear xdp filter rules and reload.") + print("Not first time run since system boot.") return False else: - print("First time run since system boot. Will attach xdp.") + print("First time run since system boot.") return True def clearPriorSettings(interfaceA, interfaceB): if enableActualShellCommands: - #shell('tc filter delete dev ' + interfaceA) - #shell('tc filter delete dev ' + interfaceA + ' root') + # If not using XDP, clear tc filter + if usingXDP == False: + #two of these are probably redundant. Will remove later once determined which those are. + shell('tc filter delete dev ' + interfaceA) + shell('tc filter delete dev ' + interfaceA + ' root') + shell('tc filter delete dev ' + interfaceB) + shell('tc filter delete dev ' + interfaceB + ' root') shell('tc qdisc delete dev ' + interfaceA + ' root') - #shell('tc qdisc delete dev ' + interfaceA) - #shell('tc filter delete dev ' + interfaceB) - #shell('tc filter delete dev ' + interfaceB + ' root') shell('tc qdisc delete dev ' + interfaceB + ' root') + #shell('tc qdisc delete dev ' + interfaceA) #shell('tc qdisc delete dev ' + interfaceB) def findQueuesAvailable(): @@ -198,6 +201,9 @@ def validateNetworkAndDevices(): return False def refreshShapers(): + # Starting + print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + # Warn user if enableActualShellCommands is False, because that would mean no actual commands are executing if enableActualShellCommands == False: warnings.warn("enableActualShellCommands is set to False. None of the commands below will actually be executed. Simulated run.", stacklevel=2) @@ -222,7 +228,7 @@ def refreshShapers(): safeToRunRefresh = True else: if (isThisFirstRunSinceBoot == False): - warnings.warn("Validation failed. Because this is not the first run since boot - will exit.") + warnings.warn("Validation failed. Because this is not the first run since boot (queues already set up) - will now exit.") safeToRunRefresh = False else: warnings.warn("Validation failed. However - because this is the first run since boot - will load queues from last good config") @@ -367,8 +373,11 @@ def refreshShapers(): with open(networkJSONfile, 'r') as j: network = json.loads(j.read()) - # Pull rx/tx queues / CPU cores avaialble - queuesAvailable = findQueuesAvailable() + # Pull rx/tx queues / CPU cores available + if usingXDP: + queuesAvailable = findQueuesAvailable() + else: + queuesAvailable = 1 # Generate Parent Nodes. Spread ShapedDevices.csv which lack defined ParentNode across these (balance across CPUs) generatedPNs = [] @@ -407,6 +416,12 @@ def refreshShapers(): minDownload, minUpload = findBandwidthMins(network, 0) + # Define lists for hash filters + ipv4FiltersSrc = [] + ipv4FiltersDst = [] + ipv6FiltersSrc = [] + ipv6FiltersDst = [] + # Parse network structure. For each tier, create corresponding HTB and leaf classes. Prepare for execution later linuxTCcommands = [] xdpCPUmapCommands = [] @@ -451,7 +466,16 @@ def refreshShapers(): for device in circuit['devices']: if device['ipv4s']: for ipv4 in device['ipv4s']: - xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + hex(queue-1) + ' --classid ' + flowIDstring) + if usingXDP: + xdpCPUmapCommands.append('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --add --ip ' + str(ipv4) + ' --cpu ' + hex(queue-1) + ' --classid ' + flowIDstring) + else: + ipv4FiltersSrc.append((ipv4, parentString, flowIDstring)) + ipv4FiltersDst.append((ipv4, parentString, flowIDstring)) + if not usingXDP: + if device['ipv6s']: + for ipv6 in device['ipv6s']: + ipv6FiltersSrc.append((ipv6, parentString, flowIDstring)) + ipv6FiltersDst.append((ipv6, parentString, flowIDstring)) if device['deviceName'] not in devicesShaped: devicesShaped.append(device['deviceName']) minor += 1 @@ -468,13 +492,78 @@ def refreshShapers(): queue += 1 major += 1 return minor - + # Print structure of network.json in debug or verbose mode logging.info(json.dumps(network, indent=4)) # Here is the actual call to the recursive traverseNetwork() function. finalMinor is not used. finalMinor = traverseNetwork(network, 0, major=1, minor=3, queue=1, parentClassID="1:1", parentMaxDL=upstreamBandwidthCapacityDownloadMbps, parentMaxUL=upstreamBandwidthCapacityUploadMbps) + # If XDP off - prepare commands for Hash Tables + + # IPv4 Hash Filters + # Dst + linuxTCcommands.append('filter add dev ' + interfaceA + ' parent 0x1: protocol all u32') + linuxTCcommands.append('filter add dev ' + interfaceA + ' parent 0x1: protocol ip handle 3: u32 divisor 256') + filterHandleCounter = 101 + for i in range (256): + hexID = str(hex(i))#.replace('0x','') + for ipv4Filter in ipv4FiltersDst: + ipv4, parent, classid = ipv4Filter + if '/' in ipv4: + ipv4 = ipv4.split('/')[0] + if (ipv4.split('.', 3)[3]) == str(i): + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceA + ' handle ' + filterHandle + ' protocol ip parent 0x1: u32 ht 3:' + hexID + ': match ip dst ' + ipv4 + ' flowid ' + classid) + filterHandleCounter += 1 + linuxTCcommands.append('filter add dev ' + interfaceA + ' protocol ip parent 0x1: u32 ht 800: match ip dst 0.0.0.0/0 hashkey mask 0x000000ff at 16 link 3:') + # Src + linuxTCcommands.append('filter add dev ' + interfaceB + ' parent 0x1: protocol all u32') + linuxTCcommands.append('filter add dev ' + interfaceB + ' parent 0x1: protocol ip handle 4: u32 divisor 256') + filterHandleCounter = 101 + for i in range (256): + hexID = str(hex(i))#.replace('0x','') + for ipv4Filter in ipv4FiltersSrc: + ipv4, parent, classid = ipv4Filter + if '/' in ipv4: + ipv4 = ipv4.split('/')[0] + if (ipv4.split('.', 3)[3]) == str(i): + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceB + ' handle ' + filterHandle + ' protocol ip parent 0x1: u32 ht 4:' + hexID + ': match ip src ' + ipv4 + ' flowid ' + classid) + filterHandleCounter += 1 + linuxTCcommands.append('filter add dev ' + interfaceB + ' protocol ip parent 0x1: u32 ht 800: match ip src 0.0.0.0/0 hashkey mask 0x000000ff at 12 link 4:') + # IPv6 Hash Filters + # Dst + linuxTCcommands.append('tc filter add dev ' + interfaceA + ' parent 0x1: handle 5: protocol ipv6 u32 divisor 256') + filterHandleCounter = 101 + for ipv6Filter in ipv6FiltersDst: + ipv6, parent, classid = ipv6Filter + withoutCIDR = ipv6.split('/')[0] + third = str(ipaddress.IPv6Address(withoutCIDR).exploded).split(':',5)[3] + usefulPart = third[:2] + hexID = usefulPart + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceA + ' handle ' + filterHandle + ' protocol ipv6 parent 0x1: u32 ht 5:' + hexID + ': match ip6 dst ' + ipv6 + ' flowid ' + classid) + filterHandleCounter += 1 + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceA + ' protocol ipv6 parent 0x1: u32 ht 800:: match ip6 dst ::/0 hashkey mask 0x0000ff00 at 28 link 5:') + filterHandleCounter += 1 + # Src + linuxTCcommands.append('tc filter add dev ' + interfaceB + ' parent 0x1: handle 6: protocol ipv6 u32 divisor 256') + filterHandleCounter = 101 + for ipv6Filter in ipv6FiltersSrc: + ipv6, parent, classid = ipv6Filter + withoutCIDR = ipv6.split('/')[0] + third = str(ipaddress.IPv6Address(withoutCIDR).exploded).split(':',5)[3] + usefulPart = third[:2] + hexID = usefulPart + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceB + ' handle ' + filterHandle + ' protocol ipv6 parent 0x1: u32 ht 6:' + hexID + ': match ip6 src ' + ipv6 + ' flowid ' + classid) + filterHandleCounter += 1 + filterHandle = hex(filterHandleCounter) + linuxTCcommands.append('filter add dev ' + interfaceB + ' protocol ipv6 parent 0x1: u32 ht 800:: match ip6 src ::/0 hashkey mask 0x0000ff00 at 12 link 6:') + filterHandleCounter += 1 + # Record start time of actual filter reload reloadStartTime = datetime.now() @@ -483,47 +572,75 @@ def refreshShapers(): # If this is the first time LibreQoS.py has run since system boot, load the XDP program and disable XPS # Otherwise, just clear the existing IP filter rules for xdp - if isThisFirstRunSinceBoot: - # Set up XDP-CPUMAP-TC - shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceA + ' --default --disable') - shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceB + ' --default --disable') - shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceA + ' --lan') - shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceB + ' --wan') - if enableActualShellCommands: - # Here we use os.system for the command, because otherwise it sometimes gltiches out with Popen in shell() - result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear') - shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceA) - shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceB) + # If XDP is disabled, skips this entirely + if usingXDP: + if isThisFirstRunSinceBoot: + # Set up XDP-CPUMAP-TC + shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceA + ' --default --disable') + shell('./xdp-cpumap-tc/bin/xps_setup.sh -d ' + interfaceB + ' --default --disable') + shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceA + ' --lan') + shell('./xdp-cpumap-tc/src/xdp_iphash_to_cpu --dev ' + interfaceB + ' --wan') + if enableActualShellCommands: + # Here we use os.system for the command, because otherwise it sometimes gltiches out with Popen in shell() + result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear') + shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceA) + shell('./xdp-cpumap-tc/src/tc_classify --dev-egress ' + interfaceB) + else: + if enableActualShellCommands: + result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear') + + if usingXDP: + # Create MQ qdisc for each CPU core / rx-tx queue (XDP method - requires IPv4) + thisInterface = interfaceA + shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq') + for queue in range(queuesAvailable): + shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityDownloadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityDownloadMbps) + 'mbit') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) + # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv + # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. + # Default class can use up to defaultClassCapacityDownloadMbps when that bandwidth isn't used by known hosts. + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityDownloadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityDownloadMbps) + 'mbit prio 5') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) + + thisInterface = interfaceB + shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq') + for queue in range(queuesAvailable): + shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityUploadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityUploadMbps) + 'mbit') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) + # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv. + # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. + # Default class can use up to defaultClassCapacityUploadMbps when that bandwidth isn't used by known hosts. + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityUploadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityUploadMbps) + 'mbit prio 5') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) else: - if enableActualShellCommands: - result = os.system('./xdp-cpumap-tc/src/xdp_iphash_to_cpu_cmdline --clear') - - # Create MQ qdisc for each interface - thisInterface = interfaceA - shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq') - for queue in range(queuesAvailable): - shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') - shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityDownloadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityDownloadMbps) + 'mbit') - shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) - # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv - # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. - # Default class can use up to defaultClassCapacityDownloadMbps when that bandwidth isn't used by known hosts. - shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityDownloadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityDownloadMbps) + 'mbit prio 5') - shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) + # Create single HTB qdisc (non XDP method - allows IPv6) + thisInterface = interfaceA + shell('tc qdisc replace dev ' + thisInterface + ' root handle 0x1: htb default 2 r2q 1514') + for queue in range(queuesAvailable): + shell('tc qdisc add dev ' + thisInterface + ' parent 0x1:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityDownloadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityDownloadMbps) + 'mbit') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) + # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv + # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. + # Default class can use up to defaultClassCapacityDownloadMbps when that bandwidth isn't used by known hosts. + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityDownloadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityDownloadMbps) + 'mbit prio 5') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) + + thisInterface = interfaceB + shell('tc qdisc replace dev ' + thisInterface + ' root handle 0x1: htb default 2 r2q 1514') + for queue in range(queuesAvailable): + shell('tc qdisc add dev ' + thisInterface + ' parent 0x1:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityUploadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityUploadMbps) + 'mbit') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) + # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv. + # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. + # Default class can use up to defaultClassCapacityUploadMbps when that bandwidth isn't used by known hosts. + shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityUploadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityUploadMbps) + 'mbit prio 5') + shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) - thisInterface = interfaceB - shell('tc qdisc replace dev ' + thisInterface + ' root handle 7FFF: mq') - for queue in range(queuesAvailable): - shell('tc qdisc add dev ' + thisInterface + ' parent 7FFF:' + hex(queue+1) + ' handle ' + hex(queue+1) + ': htb default 2') - shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ': classid ' + hex(queue+1) + ':1 htb rate '+ str(upstreamBandwidthCapacityUploadMbps) + 'mbit ceil ' + str(upstreamBandwidthCapacityUploadMbps) + 'mbit') - shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 ' + fqOrCAKE) - # Default class - traffic gets passed through this limiter with lower priority if not otherwise classified by the Shaper.csv. - # Only 1/4 of defaultClassCapacity is guarenteed (to prevent hitting ceiling of upstream), for the most part it serves as an "up to" ceiling. - # Default class can use up to defaultClassCapacityUploadMbps when that bandwidth isn't used by known hosts. - shell('tc class add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':1 classid ' + hex(queue+1) + ':2 htb rate ' + str(defaultClassCapacityUploadMbps/4) + 'mbit ceil ' + str(defaultClassCapacityUploadMbps) + 'mbit prio 5') - shell('tc qdisc add dev ' + thisInterface + ' parent ' + hex(queue+1) + ':2 ' + fqOrCAKE) - - # Execute actual Linux TC and XDP-CPUMAP-TC filter commands + # Execute actual Linux TC commands print("Executing linux TC class/qdisc commands") with open('linux_tc.txt', 'w') as f: for line in linuxTCcommands: @@ -531,11 +648,16 @@ def refreshShapers(): logging.info(line) shell("/sbin/tc -f -b linux_tc.txt") print("Executed " + str(len(linuxTCcommands)) + " linux TC class/qdisc commands") - print("Executing XDP-CPUMAP-TC IP filter commands") - for command in xdpCPUmapCommands: - logging.info(command) - shell(command) - print("Executed " + str(len(xdpCPUmapCommands)) + " XDP-CPUMAP-TC IP filter commands") + + # Execute actual XDP-CPUMAP-TC filter commands + if usingXDP: + print("Executing XDP-CPUMAP-TC IP filter commands") + for command in xdpCPUmapCommands: + logging.info(command) + shell(command) + print("Executed " + str(len(xdpCPUmapCommands)) + " XDP-CPUMAP-TC IP filter commands") + + # Record end time reloadEndTime = datetime.now() # Recap - warn operator if devices were skipped @@ -565,6 +687,9 @@ def refreshShapers(): # Report reload time reloadTimeSeconds = (reloadEndTime - reloadStartTime).seconds print("Queue and IP filter reload completed in " + str(reloadTimeSeconds) + " seconds") + + # Done + print("refreshShapers completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -590,9 +715,5 @@ if __name__ == '__main__': if args.validate: status = validateNetworkAndDevices() else: - # Starting - print("refreshShapers starting at " + datetime.now().strftime("%d/%m/%Y %H:%M:%S")) # Refresh and/or set up queues refreshShapers() - # Done - print("refreshShapers completed on " + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))