IntenseWebs/LibreQoS
3
0
Fork 0
mirror of https://github.com/LibreQoE/LibreQoS.git synced 2025-01-08 13:33:02 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add files via upload

Browse Source
This commit is contained in:
rchac 2021-01-14 00:02:19 -07:00 committed by GitHub
parent a88be48095
commit b5d1f6574a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 207 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
LibreQoS.py
View File

@ -19,7 +19,7 @@
# | | | | '_ \| '__/ _ \ | | |/ _ \___ \
# | |___| | |_) | | | __/ |_| | (_) |__) |
# |_____|_|_.__/|_| \___|\__\_\\___/____/
# v.0.75-alpha
# v.0.76-alpha
#
import random
import logging
@ -74,9 +74,13 @@ def refreshShapers():
csv_reader = csv.reader(csv_file, delimiter=',')
next(csv_reader)
for row in csv_reader:
deviceID, hostname, ipv4, ipv6, download, upload = row
deviceID, AP, mac, hostname,ipv4, ipv6, download, upload = row
ipv4 = ipv4.strip()
ipv6 = ipv6.strip()
thisDevice = {
"id": deviceID,
"mac": mac,
"AP": AP,
"hostname": hostname,
"ipv4": ipv4,
"ipv6": ipv6,
@ -104,12 +108,12 @@ def refreshShapers():
thisInterface = interfaceA
classIDCounter = 101
hashIDCounter = parentIDFirstPart + 1
shell('tc qdisc replace dev ' + thisInterface + ' root handle ' + str(parentIDFirstPart) + ': htb default 15')
shell('tc class add dev ' + thisInterface + ' parent ' + str(parentIDFirstPart) + ': classid ' + str(parentIDFirstPart) + ':1 htb rate '+ str(pipeBandwidthCapacityMbps) + 'mbit')
shell('tc qdisc add dev ' + thisInterface + ' parent 1:1 fq_codel')
shell('tc qdisc replace dev ' + thisInterface + ' root handle 1: htb default 15')
shell('tc class add dev ' + thisInterface + ' parent 1: classid 1:1 htb rate '+ str(pipeBandwidthCapacityMbps) + 'mbit')
shell('tc qdisc add dev ' + thisInterface + ' parent 1:1 ' + fqOrCAKE)
#Default class - traffic gets passed through this limiter if not otherwise classified by the Shaper.csv
shell('tc class add dev ' + thisInterface + ' parent 1:1 classid 1:15 htb rate 750mbit ceil 750mbit prio 5')
shell('tc qdisc add dev ' + thisInterface + ' parent 1:15 fq_codel')
shell('tc qdisc add dev ' + thisInterface + ' parent 1:15 ' + fqOrCAKE)
handleIDSecond = 1
for device in devices:
speedcap = 0
@ -118,7 +122,7 @@ def refreshShapers():
elif srcOrDst == 'src':
speedcap = device['upload']
#Create Hash Table
shell('tc class add dev ' + thisInterface + ' parent 1:1 classid 1:' + str(classIDCounter) + ' htb rate '+ str(speedcap) + 'mbit ceil '+ str(round(speedcap*1.1)) + 'mbit prio 3')
shell('tc class add dev ' + thisInterface + ' parent 1:1 classid 1:' + str(classIDCounter) + ' htb rate '+ str(speedcap) + 'mbit ceil '+ str(round(speedcap*1.05)) + 'mbit prio 3')
shell('tc qdisc add dev ' + thisInterface + ' parent 1:' + str(classIDCounter) + ' ' + fqOrCAKE)
if device['ipv4']:
parentString = '1:'
@ -128,7 +132,8 @@ def refreshShapers():
parentString = '1:'
flowIDstring = str(parentIDFirstPart) + ':' + str(classIDCounter)
ipv6FiltersDst.append((device['ipv6'], parentString, flowIDstring))
deviceQDiscID = str(parentIDFirstPart) + ':' + str(classIDCounter)
deviceQDiscID = '1:' + str(classIDCounter)
device['qdiscDst'] = deviceQDiscID
if srcOrDst == 'src':
device['qdiscSrc'] = deviceQDiscID
elif srcOrDst == 'dst':
@ -142,11 +147,12 @@ def refreshShapers():
thisInterface = interfaceB
classIDCounter = 101
hashIDCounter = parentIDFirstPart + 1
shell('tc qdisc replace dev ' + thisInterface + ' root handle ' + str(parentIDFirstPart) + ': htb default 15')
shell('tc class add dev ' + thisInterface + ' parent ' + str(parentIDFirstPart) + ': classid ' + str(parentIDFirstPart) + ':1 htb rate '+ str(pipeBandwidthCapacityMbps) + 'mbit')
shell('tc qdisc replace dev ' + thisInterface + ' root handle 2: htb default 15')
shell('tc class add dev ' + thisInterface + ' parent 2: classid 2:1 htb rate '+ str(pipeBandwidthCapacityMbps) + 'mbit')
shell('tc qdisc add dev ' + thisInterface + ' parent 2:1 ' + fqOrCAKE)
#Default class - traffic gets passed through this limiter if not otherwise classified by the Shaper.csv
shell('tc class add dev ' + thisInterface + ' parent 2:1 classid 2:15 htb rate 750mbit ceil 750mbit prio 5')
shell('tc qdisc add dev ' + thisInterface + ' parent 2:15 fq_codel')
shell('tc qdisc add dev ' + thisInterface + ' parent 2:15 ' + fqOrCAKE)
handleIDSecond = 1
for device in devices:
speedcap = 0
@ -155,7 +161,7 @@ def refreshShapers():
elif srcOrDst == 'src':
speedcap = device['upload']
#Create Hash Table
shell('tc class add dev ' + thisInterface + ' parent 2:1 classid 2:' + str(classIDCounter) + ' htb rate '+ str(speedcap) + 'mbit ceil '+ str(round(speedcap*1.1)) + 'mbit prio 3')
shell('tc class add dev ' + thisInterface + ' parent 2:1 classid 2:' + str(classIDCounter) + ' htb rate '+ str(speedcap) + 'mbit ceil '+ str(round(speedcap*1.05)) + 'mbit prio 3')
shell('tc qdisc add dev ' + thisInterface + ' parent 2:' + str(classIDCounter) + ' ' + fqOrCAKE)
if device['ipv4']:
parentString = '2:'
@ -165,7 +171,8 @@ def refreshShapers():
parentString = '2:'
flowIDstring = str(parentIDFirstPart) + ':' + str(classIDCounter)
ipv6FiltersSrc.append((device['ipv6'], parentString, flowIDstring))
deviceQDiscID = str(parentIDFirstPart) + ':' + str(classIDCounter)
deviceQDiscID = '2:' + str(classIDCounter)
device['qdiscSrc'] = deviceQDiscID
if srcOrDst == 'src':
device['qdiscSrc'] = deviceQDiscID
elif srcOrDst == 'dst':
@ -243,7 +250,11 @@ def refreshShapers():
filterHandle = hex(filterHandleCounter)
shell('tc filter add dev ' + interface + ' protocol ipv6 parent 2: u32 ht 800:: match ip6 src ::/0 hashkey mask 0x0000ff00 at 12 link 6:')
filterHandleCounter += 1
#Save devices to file to allow for statistics runs
with open('devices.json', 'w') as outfile:
json.dump(devices, outfile)
#Done
currentTimeString = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
print("Successful run completed on " + currentTimeString)

33
Shaper.csv
View File

@ -1,27 +1,6 @@
ID,Hostname,IPv4,IPv6,Download,Upload
,Hostname-1,100.65.0.0,8e95:12ce::/56,100,10
,Hostname-2,100.65.0.1,8e95:12ce:0:100::/56,100,10
,Hostname-3,100.65.0.2,8e95:12ce:0:200::/56,100,10
,Hostname-4,100.65.0.3,8e95:12ce:0:300::/56,100,10
,Hostname-5,100.65.0.4,8e95:12ce:0:400::/56,100,10
,Hostname-6,100.65.0.5,8e95:12ce:0:500::/56,100,10
,Hostname-7,100.65.0.6,8e95:12ce:0:600::/56,100,10
,Hostname-8,100.65.0.7,8e95:12ce:0:700::/56,100,10
,Hostname-9,100.65.0.8,8e95:12ce:0:800::/56,100,10
,Hostname-10,100.65.0.9,8e95:12ce:0:900::/56,100,10
,Hostname-11,100.65.0.10,8e95:12ce:0:a00::/56,100,10
,Hostname-12,100.65.0.11,8e95:12ce:0:b00::/56,100,10
,Hostname-13,100.65.0.12,8e95:12ce:0:c00::/56,100,10
,Hostname-14,100.65.0.13,8e95:12ce:0:d00::/56,100,10
,Hostname-15,100.65.0.14,8e95:12ce:0:e00::/56,100,10
,Hostname-16,100.65.0.15,8e95:12ce:0:f00::/56,100,10
,Hostname-17,100.65.0.16,8e95:12ce:0:1000::/56,100,10
,Hostname-18,100.65.0.17,8e95:12ce:0:1100::/56,100,10
,Hostname-19,100.65.0.18,8e95:12ce:0:1200::/56,100,10
,Hostname-20,100.65.0.19,8e95:12ce:0:1300::/56,100,10
,Hostname-21,100.65.0.20,8e95:12ce:0:1400::/56,100,10
,Hostname-22,100.65.0.21,8e95:12ce:0:1500::/56,100,10
,Hostname-23,100.65.0.22,8e95:12ce:0:1600::/56,100,10
,Hostname-24,100.65.0.23,8e95:12ce:0:1700::/56,100,10
,Hostname-25,100.65.0.24,8e95:12ce:0:1800::/56,100,10
,Hostname-26,100.65.0.25,8e95:12ce:0:1900::/56,100,10
ID,AP,MAC,Hostname,IPv4,IPv6,Download,Upload
3001,A,32:3B:FE:B0:92:C1,CPE-Customer1,100.126.0.77,2001:495:1f0f:58a::4/64,25,5
3002,C,AE:EC:D3:70:DD:36,CPE-Customer2,100.126.0.78,2001:495:1f0f:58a::8/64,50,10
3003,F,1C:1E:60:69:88:9A,CPE-Customer3,100.126.0.79,2001:495:1f0f:58a::12/64,100,15
3004,R,11:B1:63:C4:DA:4C,CPE-Customer4,100.126.0.80,2001:495:1f0f:58a::16/64,200,30
3005,X,46:2F:B5:C2:0B:15,CPE-Customer5,100.126.0.81,2001:495:1f0f:58a::20/64,300,45

1 ID AP MAC Hostname IPv4 IPv6 Download Upload
2 3001 A 32:3B:FE:B0:92:C1 Hostname-1 CPE-Customer1 100.65.0.0 100.126.0.77 8e95:12ce::/56 2001:495:1f0f:58a::4/64 100 25 10 5
3 3002 C AE:EC:D3:70:DD:36 Hostname-2 CPE-Customer2 100.65.0.1 100.126.0.78 8e95:12ce:0:100::/56 2001:495:1f0f:58a::8/64 100 50 10
4 3003 F 1C:1E:60:69:88:9A Hostname-3 CPE-Customer3 100.65.0.2 100.126.0.79 8e95:12ce:0:200::/56 2001:495:1f0f:58a::12/64 100 10 15
5 3004 R 11:B1:63:C4:DA:4C Hostname-4 CPE-Customer4 100.65.0.3 100.126.0.80 8e95:12ce:0:300::/56 2001:495:1f0f:58a::16/64 100 200 10 30
6 3005 X 46:2F:B5:C2:0B:15 Hostname-5 CPE-Customer5 100.65.0.4 100.126.0.81 8e95:12ce:0:400::/56 2001:495:1f0f:58a::20/64 100 300 10 45
Hostname-6 100.65.0.5 8e95:12ce:0:500::/56 100 10
Hostname-7 100.65.0.6 8e95:12ce:0:600::/56 100 10
Hostname-8 100.65.0.7 8e95:12ce:0:700::/56 100 10
Hostname-9 100.65.0.8 8e95:12ce:0:800::/56 100 10
Hostname-10 100.65.0.9 8e95:12ce:0:900::/56 100 10
Hostname-11 100.65.0.10 8e95:12ce:0:a00::/56 100 10
Hostname-12 100.65.0.11 8e95:12ce:0:b00::/56 100 10
Hostname-13 100.65.0.12 8e95:12ce:0:c00::/56 100 10
Hostname-14 100.65.0.13 8e95:12ce:0:d00::/56 100 10
Hostname-15 100.65.0.14 8e95:12ce:0:e00::/56 100 10
Hostname-16 100.65.0.15 8e95:12ce:0:f00::/56 100 10
Hostname-17 100.65.0.16 8e95:12ce:0:1000::/56 100 10
Hostname-18 100.65.0.17 8e95:12ce:0:1100::/56 100 10
Hostname-19 100.65.0.18 8e95:12ce:0:1200::/56 100 10
Hostname-20 100.65.0.19 8e95:12ce:0:1300::/56 100 10
Hostname-21 100.65.0.20 8e95:12ce:0:1400::/56 100 10
Hostname-22 100.65.0.21 8e95:12ce:0:1500::/56 100 10
Hostname-23 100.65.0.22 8e95:12ce:0:1600::/56 100 10
Hostname-24 100.65.0.23 8e95:12ce:0:1700::/56 100 10
Hostname-25 100.65.0.24 8e95:12ce:0:1800::/56 100 10
Hostname-26 100.65.0.25 8e95:12ce:0:1900::/56 100 10

176
stats.py Normal file
View File

@ -0,0 +1,176 @@
import os
import subprocess
from subprocess import PIPE
import io
import decimal
import json
from operator import itemgetter
from prettytable import PrettyTable
from ispConfig import fqOrCAKE
def getStatistics():
tcShowResults = []
command = 'tc -s qdisc show'
commands = command.split(' ')
proc = subprocess.Popen(commands, stdout=subprocess.PIPE)
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): # or another encoding
tcShowResults.append(line)
allQDiscStats = []
thisFlow = {}
thisFlowStats = {}
withinCorrectChunk = False
for line in tcShowResults:
expecting = "qdisc " + fqOrCAKE
if expecting in line:
thisFlow['qDiscID'] = line.split(' ')[6]
withinCorrectChunk = True
elif ("Sent " in line) and withinCorrectChunk:
items = line.split(' ')
thisFlowStats['GigabytesSent'] = str(round((int(items[2]) * 0.000000001), 1))
thisFlowStats['PacketsSent'] = int(items[4])
thisFlowStats['droppedPackets'] = int(items[7].replace(',',''))
thisFlowStats['overlimitsPackets'] = int(items[9])
thisFlowStats['requeuedPackets'] = int(items[11].replace(')',''))
if thisFlowStats['PacketsSent'] > 0:
overlimitsFreq = (thisFlowStats['overlimitsPackets']/thisFlowStats['PacketsSent'])
else:
overlimitsFreq = -1
elif ('backlog' in line) and withinCorrectChunk:
items = line.split(' ')
thisFlowStats['backlogBytes'] = int(items[2].replace('b',''))
thisFlowStats['backlogPackets'] = int(items[3].replace('p',''))
thisFlowStats['requeues'] = int(items[5])
elif ('maxpacket' in line) and withinCorrectChunk:
items = line.split(' ')
thisFlowStats['maxPacket'] = int(items[3])
thisFlowStats['dropOverlimit'] = int(items[5])
thisFlowStats['newFlowCount'] = int(items[7])
thisFlowStats['ecnMark'] = int(items[9])
elif ("new_flows_len" in line) and withinCorrectChunk:
items = line.split(' ')
thisFlowStats['newFlowsLen'] = int(items[3])
thisFlowStats['oldFlowsLen'] = int(items[5])
if thisFlowStats['PacketsSent'] == 0:
thisFlowStats['percentageDropped'] = 0
else:
thisFlowStats['percentageDropped'] = thisFlowStats['droppedPackets']/thisFlowStats['PacketsSent']
withinCorrectChunk = False
thisFlow['stats'] = thisFlowStats
allQDiscStats.append(thisFlow)
thisFlowStats = {}
thisFlow = {}
#Load shapableDevices
updatedFlowStats = []
with open('devices.json', 'r') as infile:
devices = json.load(infile)
for shapableDevice in devices:
shapableDeviceqdiscSrc = shapableDevice['qdiscSrc']
shapableDeviceqdiscDst = shapableDevice['qdiscDst']
for device in allQDiscStats:
deviceFlowID = device['qDiscID']
if shapableDeviceqdiscSrc == deviceFlowID:
name = shapableDevice['hostname']
AP = shapableDevice['AP']
ipv4 = shapableDevice['ipv4']
ipv6 = shapableDevice['ipv6']
srcOrDst = 'src'
tempDict = {'name': name, 'AP': AP, 'ipv4': ipv4, 'ipv6': ipv6, 'srcOrDst': srcOrDst}
device['identification'] = tempDict
updatedFlowStats.append(device)
if shapableDeviceqdiscDst == deviceFlowID:
name = shapableDevice['hostname']
AP = shapableDevice['AP']
ipv4 = shapableDevice['ipv4']
ipv6 = shapableDevice['ipv6']
srcOrDst = 'dst'
tempDict = {'name': name, 'AP': AP, 'ipv4': ipv4, 'ipv6': ipv6, 'srcOrDst': srcOrDst}
device['identification'] = tempDict
updatedFlowStats.append(device)
mergedStats = []
for item in updatedFlowStats:
if item['identification']['srcOrDst'] == 'src':
newStat = {
'identification': {
'name': item['identification']['name'],
'AP': item['identification']['AP'],
'ipv4': item['identification']['ipv4'],
'ipv6': item['identification']['ipv6']
},
'src': {
'GigabytesSent': item['stats']['GigabytesSent'],
'PacketsSent': item['stats']['PacketsSent'],
'droppedPackets': item['stats']['droppedPackets'],
'overlimitsPackets': item['stats']['overlimitsPackets'],
'requeuedPackets': item['stats']['requeuedPackets'],
'backlogBytes': item['stats']['backlogBytes'],
'backlogPackets': item['stats']['backlogPackets'],
'requeues': item['stats']['requeues'],
'maxPacket': item['stats']['maxPacket'],
'dropOverlimit': item['stats']['dropOverlimit'],
'newFlowCount': item['stats']['newFlowCount'],
'ecnMark': item['stats']['ecnMark'],
'newFlowsLen': item['stats']['newFlowsLen'],
'oldFlowsLen': item['stats']['oldFlowsLen'],
'percentageDropped': item['stats']['percentageDropped'],
}
}
mergedStats.append(newStat)
for item in updatedFlowStats:
if item['identification']['srcOrDst'] == 'dst':
ipv4 = item['identification']['ipv4']
ipv6 = item['identification']['ipv6']
newStat = {
'dst': {
'GigabytesSent': item['stats']['GigabytesSent'],
'PacketsSent': item['stats']['PacketsSent'],
'droppedPackets': item['stats']['droppedPackets'],
'overlimitsPackets': item['stats']['overlimitsPackets'],
'requeuedPackets': item['stats']['requeuedPackets'] ,
'backlogBytes': item['stats']['backlogBytes'],
'backlogPackets': item['stats']['backlogPackets'],
'requeues': item['stats']['requeues'],
'maxPacket': item['stats']['maxPacket'],
'dropOverlimit': item['stats']['dropOverlimit'],
'newFlowCount': item['stats']['newFlowCount'],
'ecnMark': item['stats']['ecnMark'],
'newFlowsLen': item['stats']['newFlowsLen'],
'oldFlowsLen': item['stats']['oldFlowsLen'],
'percentageDropped': item['stats']['percentageDropped']
}
}
for item2 in mergedStats:
if ipv4 in item2['identification']['ipv4']:
item2 = item2.update(newStat)
elif ipv6 in item2['identification']['ipv6']:
item2 = item2.update(newStat)
return mergedStats
if __name__ == '__main__':
mergedStats = getStatistics()
# Display table of Customer CPEs with most packets dropped
x = PrettyTable()
x.field_names = ["Device", "AP", "IPv4", "IPv6", "UL Dropped", "DL Dropped", "GB Down/Up"]
sortableList = []
pickTop = 30
for stat in mergedStats:
name = stat['identification']['name']
AP = stat['identification']['AP']
ipv4 = stat['identification']['ipv4']
ipv6 = stat['identification']['ipv6']
srcDropped = stat['src']['percentageDropped']
dstDropped = stat['dst']['percentageDropped']
GBuploadedString = stat['src']['GigabytesSent']
GBdownloadedString = stat['dst']['GigabytesSent']
GBstring = GBuploadedString + '/' + GBdownloadedString
avgDropped = (srcDropped + dstDropped)/2
sortableList.append((name, AP, ipv4, ipv6, srcDropped, dstDropped, avgDropped, GBstring))
res = sorted(sortableList, key = itemgetter(4), reverse = True)[:pickTop]
for stat in res:
name, AP, ipv4, ipv6, srcDropped, dstDropped, avgDropped, GBstring = stat
if not name:
name = ipv4
srcDroppedString = "{0:.4%}".format(srcDropped)
dstDroppedString = "{0:.4%}".format(dstDropped)
x.add_row([name, AP, ipv4, ipv6, srcDroppedString, dstDroppedString, GBstring])
print(x)