From b5d1f6574aa93af9edfc4802ff48f936192c897d Mon Sep 17 00:00:00 2001 From: rchac Date: Thu, 14 Jan 2021 00:02:19 -0700 Subject: [PATCH] Add files via upload --- LibreQoS.py | 39 +++++++----- Shaper.csv | 33 ++-------- stats.py | 176 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 41 deletions(-) create mode 100644 stats.py diff --git a/LibreQoS.py b/LibreQoS.py index 8f8f8a93..cdb5349e 100644 --- a/LibreQoS.py +++ b/LibreQoS.py @@ -19,7 +19,7 @@ # | | | | '_ \| '__/ _ \ | | |/ _ \___ \ # | |___| | |_) | | | __/ |_| | (_) |__) | # |_____|_|_.__/|_| \___|\__\_\\___/____/ -# v.0.75-alpha +# v.0.76-alpha # import random import logging @@ -74,9 +74,13 @@ def refreshShapers(): csv_reader = csv.reader(csv_file, delimiter=',') next(csv_reader) for row in csv_reader: - deviceID, hostname, ipv4, ipv6, download, upload = row + deviceID, AP, mac, hostname,ipv4, ipv6, download, upload = row + ipv4 = ipv4.strip() + ipv6 = ipv6.strip() thisDevice = { "id": deviceID, + "mac": mac, + "AP": AP, "hostname": hostname, "ipv4": ipv4, "ipv6": ipv6, @@ -104,12 +108,12 @@ def refreshShapers(): thisInterface = interfaceA classIDCounter = 101 hashIDCounter = parentIDFirstPart + 1 - shell('tc qdisc replace dev ' + thisInterface + ' root handle ' + str(parentIDFirstPart) + ': htb default 15') - shell('tc class add dev ' + thisInterface + ' parent ' + str(parentIDFirstPart) + ': classid ' + str(parentIDFirstPart) + ':1 htb rate '+ str(pipeBandwidthCapacityMbps) + 'mbit') - shell('tc qdisc add dev ' + thisInterface + ' parent 1:1 fq_codel') + shell('tc qdisc replace dev ' + thisInterface + ' root handle 1: htb default 15') + shell('tc class add dev ' + thisInterface + ' parent 1: classid 1:1 htb rate '+ str(pipeBandwidthCapacityMbps) + 'mbit') + shell('tc qdisc add dev ' + thisInterface + ' parent 1:1 ' + fqOrCAKE) #Default class - traffic gets passed through this limiter if not otherwise classified by the Shaper.csv shell('tc class add dev ' + thisInterface + ' parent 1:1 classid 1:15 htb rate 750mbit ceil 750mbit prio 5') - shell('tc qdisc add dev ' + thisInterface + ' parent 1:15 fq_codel') + shell('tc qdisc add dev ' + thisInterface + ' parent 1:15 ' + fqOrCAKE) handleIDSecond = 1 for device in devices: speedcap = 0 @@ -118,7 +122,7 @@ def refreshShapers(): elif srcOrDst == 'src': speedcap = device['upload'] #Create Hash Table - shell('tc class add dev ' + thisInterface + ' parent 1:1 classid 1:' + str(classIDCounter) + ' htb rate '+ str(speedcap) + 'mbit ceil '+ str(round(speedcap*1.1)) + 'mbit prio 3') + shell('tc class add dev ' + thisInterface + ' parent 1:1 classid 1:' + str(classIDCounter) + ' htb rate '+ str(speedcap) + 'mbit ceil '+ str(round(speedcap*1.05)) + 'mbit prio 3') shell('tc qdisc add dev ' + thisInterface + ' parent 1:' + str(classIDCounter) + ' ' + fqOrCAKE) if device['ipv4']: parentString = '1:' @@ -128,7 +132,8 @@ def refreshShapers(): parentString = '1:' flowIDstring = str(parentIDFirstPart) + ':' + str(classIDCounter) ipv6FiltersDst.append((device['ipv6'], parentString, flowIDstring)) - deviceQDiscID = str(parentIDFirstPart) + ':' + str(classIDCounter) + deviceQDiscID = '1:' + str(classIDCounter) + device['qdiscDst'] = deviceQDiscID if srcOrDst == 'src': device['qdiscSrc'] = deviceQDiscID elif srcOrDst == 'dst': @@ -142,11 +147,12 @@ def refreshShapers(): thisInterface = interfaceB classIDCounter = 101 hashIDCounter = parentIDFirstPart + 1 - shell('tc qdisc replace dev ' + thisInterface + ' root handle ' + str(parentIDFirstPart) + ': htb default 15') - shell('tc class add dev ' + thisInterface + ' parent ' + str(parentIDFirstPart) + ': classid ' + str(parentIDFirstPart) + ':1 htb rate '+ str(pipeBandwidthCapacityMbps) + 'mbit') + shell('tc qdisc replace dev ' + thisInterface + ' root handle 2: htb default 15') + shell('tc class add dev ' + thisInterface + ' parent 2: classid 2:1 htb rate '+ str(pipeBandwidthCapacityMbps) + 'mbit') + shell('tc qdisc add dev ' + thisInterface + ' parent 2:1 ' + fqOrCAKE) #Default class - traffic gets passed through this limiter if not otherwise classified by the Shaper.csv shell('tc class add dev ' + thisInterface + ' parent 2:1 classid 2:15 htb rate 750mbit ceil 750mbit prio 5') - shell('tc qdisc add dev ' + thisInterface + ' parent 2:15 fq_codel') + shell('tc qdisc add dev ' + thisInterface + ' parent 2:15 ' + fqOrCAKE) handleIDSecond = 1 for device in devices: speedcap = 0 @@ -155,7 +161,7 @@ def refreshShapers(): elif srcOrDst == 'src': speedcap = device['upload'] #Create Hash Table - shell('tc class add dev ' + thisInterface + ' parent 2:1 classid 2:' + str(classIDCounter) + ' htb rate '+ str(speedcap) + 'mbit ceil '+ str(round(speedcap*1.1)) + 'mbit prio 3') + shell('tc class add dev ' + thisInterface + ' parent 2:1 classid 2:' + str(classIDCounter) + ' htb rate '+ str(speedcap) + 'mbit ceil '+ str(round(speedcap*1.05)) + 'mbit prio 3') shell('tc qdisc add dev ' + thisInterface + ' parent 2:' + str(classIDCounter) + ' ' + fqOrCAKE) if device['ipv4']: parentString = '2:' @@ -165,7 +171,8 @@ def refreshShapers(): parentString = '2:' flowIDstring = str(parentIDFirstPart) + ':' + str(classIDCounter) ipv6FiltersSrc.append((device['ipv6'], parentString, flowIDstring)) - deviceQDiscID = str(parentIDFirstPart) + ':' + str(classIDCounter) + deviceQDiscID = '2:' + str(classIDCounter) + device['qdiscSrc'] = deviceQDiscID if srcOrDst == 'src': device['qdiscSrc'] = deviceQDiscID elif srcOrDst == 'dst': @@ -243,7 +250,11 @@ def refreshShapers(): filterHandle = hex(filterHandleCounter) shell('tc filter add dev ' + interface + ' protocol ipv6 parent 2: u32 ht 800:: match ip6 src ::/0 hashkey mask 0x0000ff00 at 12 link 6:') filterHandleCounter += 1 - + + #Save devices to file to allow for statistics runs + with open('devices.json', 'w') as outfile: + json.dump(devices, outfile) + #Done currentTimeString = datetime.now().strftime("%d/%m/%Y %H:%M:%S") print("Successful run completed on " + currentTimeString) diff --git a/Shaper.csv b/Shaper.csv index ebca0c94..04ce08fe 100644 --- a/Shaper.csv +++ b/Shaper.csv @@ -1,27 +1,6 @@ -ID,Hostname,IPv4,IPv6,Download,Upload -,Hostname-1,100.65.0.0,8e95:12ce::/56,100,10 -,Hostname-2,100.65.0.1,8e95:12ce:0:100::/56,100,10 -,Hostname-3,100.65.0.2,8e95:12ce:0:200::/56,100,10 -,Hostname-4,100.65.0.3,8e95:12ce:0:300::/56,100,10 -,Hostname-5,100.65.0.4,8e95:12ce:0:400::/56,100,10 -,Hostname-6,100.65.0.5,8e95:12ce:0:500::/56,100,10 -,Hostname-7,100.65.0.6,8e95:12ce:0:600::/56,100,10 -,Hostname-8,100.65.0.7,8e95:12ce:0:700::/56,100,10 -,Hostname-9,100.65.0.8,8e95:12ce:0:800::/56,100,10 -,Hostname-10,100.65.0.9,8e95:12ce:0:900::/56,100,10 -,Hostname-11,100.65.0.10,8e95:12ce:0:a00::/56,100,10 -,Hostname-12,100.65.0.11,8e95:12ce:0:b00::/56,100,10 -,Hostname-13,100.65.0.12,8e95:12ce:0:c00::/56,100,10 -,Hostname-14,100.65.0.13,8e95:12ce:0:d00::/56,100,10 -,Hostname-15,100.65.0.14,8e95:12ce:0:e00::/56,100,10 -,Hostname-16,100.65.0.15,8e95:12ce:0:f00::/56,100,10 -,Hostname-17,100.65.0.16,8e95:12ce:0:1000::/56,100,10 -,Hostname-18,100.65.0.17,8e95:12ce:0:1100::/56,100,10 -,Hostname-19,100.65.0.18,8e95:12ce:0:1200::/56,100,10 -,Hostname-20,100.65.0.19,8e95:12ce:0:1300::/56,100,10 -,Hostname-21,100.65.0.20,8e95:12ce:0:1400::/56,100,10 -,Hostname-22,100.65.0.21,8e95:12ce:0:1500::/56,100,10 -,Hostname-23,100.65.0.22,8e95:12ce:0:1600::/56,100,10 -,Hostname-24,100.65.0.23,8e95:12ce:0:1700::/56,100,10 -,Hostname-25,100.65.0.24,8e95:12ce:0:1800::/56,100,10 -,Hostname-26,100.65.0.25,8e95:12ce:0:1900::/56,100,10 +ID,AP,MAC,Hostname,IPv4,IPv6,Download,Upload +3001,A,32:3B:FE:B0:92:C1,CPE-Customer1,100.126.0.77,2001:495:1f0f:58a::4/64,25,5 +3002,C,AE:EC:D3:70:DD:36,CPE-Customer2,100.126.0.78,2001:495:1f0f:58a::8/64,50,10 +3003,F,1C:1E:60:69:88:9A,CPE-Customer3,100.126.0.79,2001:495:1f0f:58a::12/64,100,15 +3004,R,11:B1:63:C4:DA:4C,CPE-Customer4,100.126.0.80,2001:495:1f0f:58a::16/64,200,30 +3005,X,46:2F:B5:C2:0B:15,CPE-Customer5,100.126.0.81,2001:495:1f0f:58a::20/64,300,45 diff --git a/stats.py b/stats.py new file mode 100644 index 00000000..a02f2150 --- /dev/null +++ b/stats.py @@ -0,0 +1,176 @@ +import os +import subprocess +from subprocess import PIPE +import io +import decimal +import json +from operator import itemgetter +from prettytable import PrettyTable +from ispConfig import fqOrCAKE + +def getStatistics(): + tcShowResults = [] + command = 'tc -s qdisc show' + commands = command.split(' ') + proc = subprocess.Popen(commands, stdout=subprocess.PIPE) + for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding + tcShowResults.append(line) + allQDiscStats = [] + thisFlow = {} + thisFlowStats = {} + withinCorrectChunk = False + for line in tcShowResults: + expecting = "qdisc " + fqOrCAKE + if expecting in line: + thisFlow['qDiscID'] = line.split(' ')[6] + withinCorrectChunk = True + elif ("Sent " in line) and withinCorrectChunk: + items = line.split(' ') + thisFlowStats['GigabytesSent'] = str(round((int(items[2]) * 0.000000001), 1)) + thisFlowStats['PacketsSent'] = int(items[4]) + thisFlowStats['droppedPackets'] = int(items[7].replace(',','')) + thisFlowStats['overlimitsPackets'] = int(items[9]) + thisFlowStats['requeuedPackets'] = int(items[11].replace(')','')) + if thisFlowStats['PacketsSent'] > 0: + overlimitsFreq = (thisFlowStats['overlimitsPackets']/thisFlowStats['PacketsSent']) + else: + overlimitsFreq = -1 + elif ('backlog' in line) and withinCorrectChunk: + items = line.split(' ') + thisFlowStats['backlogBytes'] = int(items[2].replace('b','')) + thisFlowStats['backlogPackets'] = int(items[3].replace('p','')) + thisFlowStats['requeues'] = int(items[5]) + elif ('maxpacket' in line) and withinCorrectChunk: + items = line.split(' ') + thisFlowStats['maxPacket'] = int(items[3]) + thisFlowStats['dropOverlimit'] = int(items[5]) + thisFlowStats['newFlowCount'] = int(items[7]) + thisFlowStats['ecnMark'] = int(items[9]) + elif ("new_flows_len" in line) and withinCorrectChunk: + items = line.split(' ') + thisFlowStats['newFlowsLen'] = int(items[3]) + thisFlowStats['oldFlowsLen'] = int(items[5]) + if thisFlowStats['PacketsSent'] == 0: + thisFlowStats['percentageDropped'] = 0 + else: + thisFlowStats['percentageDropped'] = thisFlowStats['droppedPackets']/thisFlowStats['PacketsSent'] + withinCorrectChunk = False + thisFlow['stats'] = thisFlowStats + allQDiscStats.append(thisFlow) + thisFlowStats = {} + thisFlow = {} + #Load shapableDevices + updatedFlowStats = [] + with open('devices.json', 'r') as infile: + devices = json.load(infile) + for shapableDevice in devices: + shapableDeviceqdiscSrc = shapableDevice['qdiscSrc'] + shapableDeviceqdiscDst = shapableDevice['qdiscDst'] + for device in allQDiscStats: + deviceFlowID = device['qDiscID'] + if shapableDeviceqdiscSrc == deviceFlowID: + name = shapableDevice['hostname'] + AP = shapableDevice['AP'] + ipv4 = shapableDevice['ipv4'] + ipv6 = shapableDevice['ipv6'] + srcOrDst = 'src' + tempDict = {'name': name, 'AP': AP, 'ipv4': ipv4, 'ipv6': ipv6, 'srcOrDst': srcOrDst} + device['identification'] = tempDict + updatedFlowStats.append(device) + if shapableDeviceqdiscDst == deviceFlowID: + name = shapableDevice['hostname'] + AP = shapableDevice['AP'] + ipv4 = shapableDevice['ipv4'] + ipv6 = shapableDevice['ipv6'] + srcOrDst = 'dst' + tempDict = {'name': name, 'AP': AP, 'ipv4': ipv4, 'ipv6': ipv6, 'srcOrDst': srcOrDst} + device['identification'] = tempDict + updatedFlowStats.append(device) + mergedStats = [] + for item in updatedFlowStats: + if item['identification']['srcOrDst'] == 'src': + newStat = { + 'identification': { + 'name': item['identification']['name'], + 'AP': item['identification']['AP'], + 'ipv4': item['identification']['ipv4'], + 'ipv6': item['identification']['ipv6'] + }, + 'src': { + 'GigabytesSent': item['stats']['GigabytesSent'], + 'PacketsSent': item['stats']['PacketsSent'], + 'droppedPackets': item['stats']['droppedPackets'], + 'overlimitsPackets': item['stats']['overlimitsPackets'], + 'requeuedPackets': item['stats']['requeuedPackets'], + 'backlogBytes': item['stats']['backlogBytes'], + 'backlogPackets': item['stats']['backlogPackets'], + 'requeues': item['stats']['requeues'], + 'maxPacket': item['stats']['maxPacket'], + 'dropOverlimit': item['stats']['dropOverlimit'], + 'newFlowCount': item['stats']['newFlowCount'], + 'ecnMark': item['stats']['ecnMark'], + 'newFlowsLen': item['stats']['newFlowsLen'], + 'oldFlowsLen': item['stats']['oldFlowsLen'], + 'percentageDropped': item['stats']['percentageDropped'], + } + } + mergedStats.append(newStat) + for item in updatedFlowStats: + if item['identification']['srcOrDst'] == 'dst': + ipv4 = item['identification']['ipv4'] + ipv6 = item['identification']['ipv6'] + newStat = { + 'dst': { + 'GigabytesSent': item['stats']['GigabytesSent'], + 'PacketsSent': item['stats']['PacketsSent'], + 'droppedPackets': item['stats']['droppedPackets'], + 'overlimitsPackets': item['stats']['overlimitsPackets'], + 'requeuedPackets': item['stats']['requeuedPackets'] , + 'backlogBytes': item['stats']['backlogBytes'], + 'backlogPackets': item['stats']['backlogPackets'], + 'requeues': item['stats']['requeues'], + 'maxPacket': item['stats']['maxPacket'], + 'dropOverlimit': item['stats']['dropOverlimit'], + 'newFlowCount': item['stats']['newFlowCount'], + 'ecnMark': item['stats']['ecnMark'], + 'newFlowsLen': item['stats']['newFlowsLen'], + 'oldFlowsLen': item['stats']['oldFlowsLen'], + 'percentageDropped': item['stats']['percentageDropped'] + } + } + for item2 in mergedStats: + if ipv4 in item2['identification']['ipv4']: + item2 = item2.update(newStat) + elif ipv6 in item2['identification']['ipv6']: + item2 = item2.update(newStat) + return mergedStats + +if __name__ == '__main__': + mergedStats = getStatistics() + + # Display table of Customer CPEs with most packets dropped + x = PrettyTable() + x.field_names = ["Device", "AP", "IPv4", "IPv6", "UL Dropped", "DL Dropped", "GB Down/Up"] + sortableList = [] + pickTop = 30 + for stat in mergedStats: + name = stat['identification']['name'] + AP = stat['identification']['AP'] + ipv4 = stat['identification']['ipv4'] + ipv6 = stat['identification']['ipv6'] + srcDropped = stat['src']['percentageDropped'] + dstDropped = stat['dst']['percentageDropped'] + GBuploadedString = stat['src']['GigabytesSent'] + GBdownloadedString = stat['dst']['GigabytesSent'] + GBstring = GBuploadedString + '/' + GBdownloadedString + avgDropped = (srcDropped + dstDropped)/2 + sortableList.append((name, AP, ipv4, ipv6, srcDropped, dstDropped, avgDropped, GBstring)) + res = sorted(sortableList, key = itemgetter(4), reverse = True)[:pickTop] + for stat in res: + name, AP, ipv4, ipv6, srcDropped, dstDropped, avgDropped, GBstring = stat + if not name: + name = ipv4 + srcDroppedString = "{0:.4%}".format(srcDropped) + dstDroppedString = "{0:.4%}".format(dstDropped) + x.add_row([name, AP, ipv4, ipv6, srcDroppedString, dstDroppedString, GBstring]) + print(x)