Add files via upload

2025-02-25 18:55:32 -06:00 · 2020-10-07 04:02:44 -06:00
parent e31001632a
commit 9f58ee4c91
6 changed files with 219 additions and 44 deletions
--- a/LibreNMS_Integration.py
+++ b/LibreNMS_Integration.py
@@ -19,7 +19,7 @@
 #           | |   | | '_ \| '__/ _ \ | | |/ _ \___ \ 
 #           | |___| | |_) | | |  __/ |_| | (_) |__) |
 #           |_____|_|_.__/|_|  \___|\__\_\\___/____/
-#                          v.0.65-alpha
+#                          v.0.7-alpha
 #
 import requests
 from ispConfig import orgLibreNMSxAuthToken, libreNMSBaseURL, libreNMSDeviceGroups
--- a/LibreQoS.py
+++ b/LibreQoS.py
@@ -19,18 +19,20 @@
 #           | |   | | '_ \| '__/ _ \ | | |/ _ \___ \ 
 #           | |___| | |_) | | |  __/ |_| | (_) |__) |
 #           |_____|_|_.__/|_|  \___|\__\_\\___/____/
-#                          v.0.65-alpha
+#                          v.0.7-alpha
 #
 import random
 import logging
 import os
+import json
 import subprocess
 from subprocess import PIPE
+import ipaddress
 import time
-from datetime import date
+from datetime import date, datetime
 from UNMS_Integration import pullUNMSDevices
 from LibreNMS_Integration import pullLibreNMSDevices
-from ispConfig import fqOrCAKE, pipeBandwidthCapacityMbps, interfaceA, interfaceB, enableActualShellCommands, runShellCommandsAsSudo, importFromUNMS, importFromLibreNMS
+from ispConfig import fqOrCAKE, pipeBandwidthCapacityMbps, interfaceA, interfaceB, addTheseSubnets, enableActualShellCommands, runShellCommandsAsSudo, importFromUNMS, importFromLibreNMS

 def shell(inputCommand):
 	if enableActualShellCommands:
@@ -61,39 +63,53 @@ def getHashList():
 			twoDigitHash.append(str(i) + letters[x])
 	return twoDigitHash

-def createTestClientsPool(slash16, quantity):
-	if quantity<65534:
-		tempList = []
-		counterC = 0
-		counterD = 1
-		mainCounter = 0
-		while mainCounter < quantity:
-			if counterD <= 255:
-				ipAddr = slash16.replace('X.X', '') + str(counterC) + '.' + str(counterD)
-				tempList.append((ipAddr, 100, 15))
-				counterD += 1
-			else:
-				counterC += 1
-				counterD = 1
-			mainCounter += 1
-		return tempList
-	else:
-		raise Exception
+def addCustomersBySubnet(inputBlock):
+	addTheseSubnets, existingShapableDevices = inputBlock
+	customersToAdd = []
+	for subnetItem in addTheseSubnets:
+		ipcidr, downloadMbps, uploadMbps = subnetItem
+		theseHosts = list(ipaddress.ip_network(ipcidr).hosts())
+		for host in theseHosts:
+			deviceIP = str(host)
+			alreadyAssigned = False
+			for device in existingShapableDevices:
+				if deviceIP == device['identification']['ipAddr']:
+					alreadyAssigned = True
+			if not alreadyAssigned:
+				thisShapedDevice = {
+					"identification": {
+					  "name": None,
+					  "hostname": None,
+					  "ipAddr": deviceIP,
+					  "mac": None,
+					  "model": None,
+					  "modelName": None,
+					  "unmsSiteID": None,
+					  "libreNMSSiteID": None
+					},
+					"qos": {
+					  "downloadMbps": downloadMbps,
+					  "uploadMbps": uploadMbps,
+					  "accessPoint": None
+					},
+				}
+				customersToAdd.append(thisShapedDevice)
+	return customersToAdd

 def refreshShapers():
 	#Clients
 	shapableDevices = []
-	#Add arbitrary number of test clients in /16 subnet
-	#clientsList = createTestClientsPool('100.64.X.X', 5)
-	#Add specific test clients
-	#clientsList.append((100, '100.65.1.1'))
-
+	
 	#Bring in clients from UNMS or LibreNMS if enabled
 	if importFromUNMS:
 		shapableDevices.extend(pullUNMSDevices())
 	if importFromLibreNMS:
 		shapableDevices.extend(pullLibreNMSDevices())

+	#Add customers by subnet. Will not replace those that already exist
+	if addTheseSubnets:
+		shapableDevices.extend(addCustomersBySubnet((addTheseSubnets, shapableDevices)))
+
 	#Categorize Clients By IPv4 /16
 	listOfSlash16SubnetsInvolved = []
 	shapableDevicesListWithSubnet = []
@@ -104,8 +120,10 @@ def refreshShapers():
 		if slash16 not in listOfSlash16SubnetsInvolved:
 			listOfSlash16SubnetsInvolved.append(slash16)
 		shapableDevicesListWithSubnet.append((ipAddr))
+	
 	#Clear Prior Configs
 	clearPriorSettings(interfaceA, interfaceB)
+	
 	#InterfaceA
 	parentIDFirstPart = 1
 	srcOrDst = 'dst'
@@ -118,7 +136,7 @@ def refreshShapers():
 		thisSlash16Dec1 = slash16.split('.')[0]
 		thisSlash16Dec2 = slash16.split('.')[1]
 		groupedCustomers = []	
-		for i in range(255):
+		for i in range(256):
 			tempList = []
 			for ipAddr in shapableDevicesListWithSubnet:
 				dec1, dec2, dec3, dec4 = ipAddr.split('.')
@@ -144,10 +162,14 @@ def refreshShapers():
 					shell('tc class add dev ' + interfaceA + ' parent ' + str(parentIDFirstPart) + ':1 classid ' + str(parentIDFirstPart) + ':' + str(classIDCounter) + ' htb rate '+ str(downloadSpeed) + 'mbit ceil '+ str(downloadSpeed) + 'mbit prio 3') 
 					shell('tc qdisc add dev ' + interfaceA + ' parent ' + str(parentIDFirstPart) + ':' + str(classIDCounter) + ' ' + fqOrCAKE)
 					shell('tc filter add dev ' + interfaceA + ' parent ' + str(parentIDFirstPart) + ': prio 5 u32 ht ' + str(hashIDCounter) + ':' + twoDigitHashString + ' match ip ' + srcOrDst + ' ' + ipAddr + ' flowid ' + str(parentIDFirstPart) + ':' + str(classIDCounter))
-					deviceFlowID = str(parentIDFirstPart) + ':' + str(classIDCounter)
+					deviceQDiscID = str(parentIDFirstPart) + ':' + str(classIDCounter)
 					for device in shapableDevices:
 						if device['identification']['ipAddr'] == ipAddr:
-							device['identification']['flowID'] = deviceFlowID
+							if srcOrDst == 'src':
+								qdiscDict ={'qDiscSrc': deviceQDiscID}
+							elif srcOrDst == 'dst':
+								qdiscDict ={'qDiscDst': deviceQDiscID}
+							device['identification'].update(qdiscDict)
 					classIDCounter += 1
 			thirdDigitCounter += 1
 		if (srcOrDst == 'dst'):
@@ -156,6 +178,7 @@ def refreshShapers():
 			startPointForHash = '12' #Position of src-address in IP header
 		shell('tc filter add dev ' + interfaceA + ' parent ' + str(parentIDFirstPart) + ': prio 5 u32 ht 800:: match ip ' + srcOrDst + ' '+ thisSlash16Dec1 + '.' + thisSlash16Dec2 + '.0.0/16 hashkey mask 0x000000ff at ' + startPointForHash + ' link ' + str(hashIDCounter) + ':')
 		hashIDCounter += 1
+	
 	#InterfaceB
 	parentIDFirstPart = hashIDCounter + 1
 	hashIDCounter = parentIDFirstPart + 1
@@ -167,7 +190,7 @@ def refreshShapers():
 		thisSlash16Dec1 = slash16.split('.')[0]
 		thisSlash16Dec2 = slash16.split('.')[1]
 		groupedCustomers = []	
-		for i in range(255):
+		for i in range(256):
 			tempList = []
 			for ipAddr in shapableDevicesListWithSubnet:
 				dec1, dec2, dec3, dec4 = ipAddr.split('.')
@@ -193,10 +216,14 @@ def refreshShapers():
 					shell('tc class add dev ' + interfaceB + ' parent ' + str(parentIDFirstPart) + ':1 classid ' + str(parentIDFirstPart) + ':' + str(classIDCounter) + ' htb rate '+ str(uploadSpeed) + 'mbit ceil '+ str(uploadSpeed) + 'mbit prio 3') 
 					shell('tc qdisc add dev ' + interfaceB + ' parent ' + str(parentIDFirstPart) + ':' + str(classIDCounter) + ' ' + fqOrCAKE)
 					shell('tc filter add dev ' + interfaceB + ' parent ' + str(parentIDFirstPart) + ': prio 5 u32 ht ' + str(hashIDCounter) + ':' + twoDigitHashString + ' match ip ' + srcOrDst + ' ' + ipAddr + ' flowid ' + str(parentIDFirstPart) + ':' + str(classIDCounter))
-					deviceFlowID = str(parentIDFirstPart) + ':' + str(classIDCounter)
+					deviceQDiscID = str(parentIDFirstPart) + ':' + str(classIDCounter)
 					for device in shapableDevices:
 						if device['identification']['ipAddr'] == ipAddr:
-							device['identification']['flowID'] = deviceFlowID
+							if srcOrDst == 'src':
+								qdiscDict ={'qDiscSrc': deviceQDiscID}
+							elif srcOrDst == 'dst':
+								qdiscDict ={'qDiscDst': deviceQDiscID}
+							device['identification'].update(qdiscDict)
 					classIDCounter += 1
 			thirdDigitCounter += 1
 		if (srcOrDst == 'dst'):
@@ -205,6 +232,11 @@ def refreshShapers():
 			startPointForHash = '12' #Position of src-address in IP header
 		shell('tc filter add dev ' + interfaceB + ' parent ' + str(parentIDFirstPart) + ': prio 5 u32 ht 800:: match ip ' + srcOrDst + ' '+ thisSlash16Dec1 + '.' + thisSlash16Dec2 + '.0.0/16 hashkey mask 0x000000ff at ' + startPointForHash + ' link ' + str(hashIDCounter) + ':')
 		hashIDCounter += 1
+	
+	#Save shapableDevices to file to allow for debugging and statistics runs
+	with open('shapableDevices.json', 'w') as outfile:
+		json.dump(shapableDevices, outfile)
+	
 	#Recap and log
 	logging.basicConfig(level=logging.INFO, filename="log", filemode="a+",	format="%(asctime)-15s %(levelname)-8s %(message)s")
 	for device in shapableDevices:
@@ -212,13 +244,17 @@ def refreshShapers():
 		hostname = device['identification']['hostname']
 		downloadSpeed = str(device['qos']['downloadMbps'])
 		uploadSpeed = str(device['qos']['uploadMbps'])
-		recap = "Applied rate limiting of " + downloadSpeed + " down " + uploadSpeed + " up to device " + hostname
+		if hostname:
+			recap = "Applied rate limiting of " + downloadSpeed + " down " + uploadSpeed + " up to device " + hostname
+		else:
+			recap = "Applied rate limiting of " + downloadSpeed + " down " + uploadSpeed + " up to device " + ipAddr
 		logging.info(recap)
 		print(recap)
+	print(str(len(shapableDevices)) + " device rules (" + str(len(shapableDevices)*2) + " filter rules) were applied this round")
+	
 	#Done
-	today = date.today()
-	d1 = today.strftime("%d/%m/%Y")
-	print("Successful run completed at ", d1)
+	currentTimeString = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
+	print("Successful run completed on " + currentTimeString)

 if __name__ == '__main__':
 	refreshShapers()
--- a/UNMS_Integration.py
+++ b/UNMS_Integration.py
@@ -19,7 +19,7 @@
 #           | |   | | '_ \| '__/ _ \ | | |/ _ \___ \ 
 #           | |___| | |_) | | |  __/ |_| | (_) |__) |
 #           |_____|_|_.__/|_|  \___|\__\_\\___/____/
-#                          v.0.65-alpha
+#                          v.0.7-alpha
 #
 import requests
 from ispConfig import orgUNMSxAuthToken, unmsBaseURL, deviceModelBlacklistEnabled
@@ -63,12 +63,12 @@ def pullUNMSDevices():
 						  "model": deviceModel,
 						  "modelName": deviceModelName,
 						  "unmsSiteID": device['identification']['site']['id'],
-						  "libreNMSSiteID": ""
+						  "libreNMSSiteID": None
 						},
 						"qos": {
 						  "downloadMbps": downloadSpeedMbps,
 						  "uploadMbps": uploadSpeedMbps,
-						  "accessPoint": ""
+						  "accessPoint": None
 						},
 					}
 					unmsDevicesToImport.append(thisShapedDevice)
--- a/ispConfig.py
+++ b/ispConfig.py
@@ -4,7 +4,7 @@
 # 	https://github.com/dtaht/tc-adv
 fqOrCAKE = 'fq_codel'

-# How many symmetrical Mbps are available to the edge of this test network
+# How many symmetrical Mbps are available to the edge of this network
 pipeBandwidthCapacityMbps = 500

 # Interface connected to edge
@@ -13,10 +13,10 @@ interfaceA = 'eth4'
 # Interface connected to core
 interfaceB = 'eth5'

-# Allow shell commands. Default is False where commands print to console. Must be enabled to function
+# Allow shell commands. Default is False where commands print to console. MUST BE ENABLED FOR PROGRAM TO FUNCTION
 enableActualShellCommands = False

-# Add 'sudo' before execution of any shell commands. Default is False.
+# Add 'sudo' before execution of any shell commands. May be required depending on distribution and environment.
 runShellCommandsAsSudo = False

 # Import customer QoS rules from UNMS
@@ -25,6 +25,15 @@ importFromUNMS = False
 # Import customer QoS rules from LibreNMS
 importFromLibreNMS = False

+# So that new clients are client shaped with something by default, and don't get their traffic de-prioritized,
+# you can add specific subnets of hosts to be set to specific speeds.
+# These will not override any imports from actual customer data via UNMS or LibreNMS
+
+addTheseSubnets = [
+	('100.64.0.0/22', 115, 20),
+	('100.72.4.0/22', 115, 20)
+]
+
 # Available on LibreNMS site as https://exampleLibreNMSsite.net/api-access
 orgLibreNMSxAuthToken = ''

--- a/scheduled.py
+++ b/scheduled.py
@@ -19,7 +19,7 @@
 #           | |   | | '_ \| '__/ _ \ | | |/ _ \___ \ 
 #           | |___| | |_) | | |  __/ |_| | (_) |__) |
 #           |_____|_|_.__/|_|  \___|\__\_\\___/____/
-#                           v.0.6-alpha
+#                           v.0.7-alpha
 #
 import time
 import schedule
--- a/stats.py
+++ b/stats.py
@@ -0,0 +1,130 @@
+# Copyright (C) 2020  Robert Chacón
+# This file is part of LibreQoS.
+#
+# LibreQoS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# LibreQoS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with LibreQoS.  If not, see <http://www.gnu.org/licenses/>.
+#
+#            _     _ _               ___       ____  
+#           | |   (_) |__  _ __ ___ / _ \  ___/ ___| 
+#           | |   | | '_ \| '__/ _ \ | | |/ _ \___ \ 
+#           | |___| | |_) | | |  __/ |_| | (_) |__) |
+#           |_____|_|_.__/|_|  \___|\__\_\\___/____/
+#                          v.0.7-alpha
+#
+import os
+import subprocess
+from subprocess import PIPE
+import io
+import json
+from operator import itemgetter 
+
+def getStatistics():
+	tcShowResults = []
+	command = 'tc -s qdisc show'
+	commands = command.split(' ')
+	proc = subprocess.Popen(commands, stdout=subprocess.PIPE)
+	for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"):  # or another encoding
+		tcShowResults.append(line)
+	allQDiscStats = []
+	thisFlow = {}
+	thisFlowStats = {}
+	withinCorrectChunk = False
+	for line in tcShowResults:
+		if "qdisc fq_codel" in line:
+			thisFlow['qDiscID'] = line.split(' ')[6]
+			withinCorrectChunk = True
+		elif ("Sent " in line) and withinCorrectChunk:
+			items = line.split(' ')
+			thisFlowStats['MegabytesSent'] = int(int(items[2]) * 0.000001)
+			thisFlowStats['PacketsSent'] = int(items[4])
+			thisFlowStats['droppedPackets'] = int(items[7].replace(',',''))
+			thisFlowStats['overlimitsPackets'] = int(items[9])
+			thisFlowStats['requeuedPackets'] = int(items[11].replace(')',''))
+			if thisFlowStats['PacketsSent'] > 0:
+				overlimitsFreq = (thisFlowStats['overlimitsPackets']/thisFlowStats['PacketsSent'])
+			else:
+				overlimitsFreq = -1
+		elif ('backlog' in line) and withinCorrectChunk:
+			items = line.split(' ')
+			thisFlowStats['backlogBytes'] = int(items[2].replace('b',''))
+			thisFlowStats['backlogPackets'] = int(items[3].replace('p',''))
+			thisFlowStats['requeues'] = int(items[5])
+		elif ('maxpacket' in line) and withinCorrectChunk:
+			items = line.split(' ')
+			thisFlowStats['maxPacket'] = int(items[3])
+			thisFlowStats['dropOverlimit'] = int(items[5])
+			thisFlowStats['newFlowCount'] = int(items[7])
+			thisFlowStats['ecnMark'] = int(items[9])
+		elif ("new_flows_len" in line) and withinCorrectChunk:
+			items = line.split(' ')
+			thisFlowStats['newFlowsLen'] = int(items[3])
+			thisFlowStats['oldFlowsLen'] = int(items[5])
+			if thisFlowStats['PacketsSent'] == 0:
+				thisFlowStats['percentageDropped'] = 0
+			else:
+				thisFlowStats['percentageDropped'] = thisFlowStats['droppedPackets']/thisFlowStats['PacketsSent']
+			withinCorrectChunk = False
+			thisFlow['stats'] = thisFlowStats
+			allQDiscStats.append(thisFlow)
+			thisFlowStats = {}
+			thisFlow = {}
+	#Load identifiers from json file
+	updatedFlowStats = []
+	with open('shapableDevices.json', 'r') as infile:
+		shapableDevices = json.load(infile)
+	for shapableDevice in shapableDevices:
+		shapableDeviceQDiscSrc = shapableDevice['identification']['qDiscSrc']
+		shapableDeviceQDiscDst = shapableDevice['identification']['qDiscDst']
+		for device in allQDiscStats:
+			deviceDiscID = device['qDiscID']
+			if shapableDeviceQDiscSrc == deviceDiscID:
+				name = shapableDevice['identification']['name']
+				ipAddr = shapableDevice['identification']['ipAddr']
+				srcOrDst = 'src'
+				tempDict = {'name': name, 'ipAddr': ipAddr, 'srcOrDst': srcOrDst}
+				device['identification'] = tempDict
+				updatedFlowStats.append(device)
+			if shapableDeviceQDiscDst == deviceFlowID:
+				name = shapableDevice['identification']['name']
+				ipAddr = shapableDevice['identification']['ipAddr']
+				srcOrDst = 'dst'
+				tempDict = {'name': name, 'ipAddr': ipAddr, 'srcOrDst': srcOrDst}
+				device['identification'] = tempDict
+				updatedFlowStats.append(device)
+	return updatedFlowStats
+			
+if __name__ == '__main__':
+	allQDiscStats = getStatistics()
+	#Customer CPEs with most packet drops
+	packetDropsCPEs = []
+	sumOfPercentDropped  = 0
+	pickTop = 10
+	for item in allQDiscStats:
+		packetDropsCPEs.append((item['identification']['name'], item['identification']['ipAddr'], item['stats']['percentageDropped'], item['identification']['srcOrDst']))
+		sumOfPercentDropped += item['stats']['percentageDropped']
+	averageOfPercentDropped = sumOfPercentDropped/len(allQDiscStats)
+	res = sorted(packetDropsCPEs, key = itemgetter(2), reverse = True)[:pickTop]
+	for item in res:
+		name, ipAddr, percentageDropped, srcOrDst = item
+		v1 = percentageDropped
+		v2 = averageOfPercentDropped
+		difference = abs(v1-v2)/((v1+v2)/2)
+		downOrUp = ''
+		if srcOrDst == 'src':
+			downOrUp = ' upload'
+		elif srcOrDst == 'dst':
+			downOrUp = ' download'
+		if name:
+			print(name + downOrUp + " has high packet drop rate of {0:.2%}".format(percentageDropped) + ", {0:.0%}".format(difference) + " above the average of {0:.2%}".format(averageOfPercentDropped))
+		else:
+			print(ipAddr + downOrUp + " has high packet drop rate of {0:.2%}".format(percentageDropped) + ", {0:.0%}".format(difference) + " above the average of {0:.2%}".format(averageOfPercentDropped))