Merge pull request #335 from LibreQoE/fix334

UISP Integration Improvements

src/csvToNetworkJSON.py

@@ -0,0 +1,58 @@
+from pythonCheck import checkPythonVersion
+checkPythonVersion()
+import os
+import csv
+import json
+from ispConfig import uispSite, uispStrategy, overwriteNetworkJSONalways
+from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps
+from integrationCommon import NetworkGraph, NetworkNode, NodeType
+
+def csvToNetworkJSONfile():
+	sites = []
+	with open('manualNetwork.csv') as csv_file:
+		csv_reader = csv.reader(csv_file, delimiter=',')
+		for row in csv_reader:
+			if 'Site Name' in row[0]:
+				header = row
+			else:
+				name, down, up, parent = row
+				site = {'name': name,
+				'download': down,
+				'upload': up,
+				'parent': parent}
+				sites.append(site)
+	
+	net = NetworkGraph()
+	idCounter = 1000
+	nameToID = {}
+	for site in sites:
+		site['id'] = idCounter
+		idCounter = idCounter + 1
+		nameToID[site['name']] = site['id']
+	for site in sites:
+		id = site['id']
+		if site['parent'] == '':
+			parentID = None
+		else:
+			parentID = nameToID[site['parent']]
+		name = site['name']
+		parent = site['parent']
+		download = site['download']
+		upload = site['upload']
+		nodeType = NodeType.site
+		node = NetworkNode(id=id, displayName=name, type=nodeType,
+						   parentId=parentID, download=download, upload=upload, address=None, customerName=None)
+		net.addRawNode(node)
+	net.prepareTree()
+	net.plotNetworkGraph(False)
+	if net.doesNetworkJsonExist():
+		if overwriteNetworkJSONalways:
+			net.createNetworkJson()
+		else:
+			print("network.json already exists and overwriteNetworkJSONalways set to False. Leaving in-place.")
+	else:
+		net.createNetworkJson()
+	net.createShapedDevices()
+
+if __name__ == '__main__':
+	csvToNetworkJSONfile()

src/integrationUISP.py

@@ -131,11 +131,23 @@ def findApCapacities(devices, siteBandwidth):
 		if device['identification']['role'] == "ap":
 			name = device['identification']['name']
 			if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']:
+				safeToUse = True
 				download = int(device['overview']
 							   ['downlinkCapacity'] / 1000000)
 				upload = int(device['overview']['uplinkCapacity'] / 1000000)
-				siteBandwidth[device['identification']['name']] = {
+				if download < 15:
-					"download": download, "upload": upload}
+					print("WARNING: Device '" + device['identification']['hostname'] + "' has unusually low download capacity (" + str(upload) + " Mbps). Discarding in favor of parent site rates.")
+					safeToUse = False
+				if upload < 15:
+					print("WARNING: Device '" + device['identification']['hostname'] + "' has unusually low upload capacity (" + str(download) + " Mbps). Discarding in favor of parent site rates.")
+					safeToUse = False
+				if device['identification']['model'] == 'WaveAP':
+					if (download < 500) or (upload < 500):
+						download = 2450
+						upload = 2450
+				if safeToUse:
+					siteBandwidth[device['identification']['name']] = {
+						"download": download, "upload": upload}
 
 def findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps):
 	foundAirFibersBySite = {}
@@ -147,19 +159,17 @@ def findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps):
 						if device['overview']['downlinkCapacity'] is not None and device['overview']['uplinkCapacity'] is not None:
 							download = int(device['overview']['downlinkCapacity']/ 1000000)
 							upload = int(device['overview']['uplinkCapacity']/ 1000000)
-						else:
+							# Make sure to use half of reported bandwidth for AF60/AF60-LRs
-							download = generatedPNDownloadMbps
+							if (device['identification']['model'] == "AF60-LR") or (device['identification']['model'] == "AF60"):
-							upload = generatedPNUploadMbps
+								download = int(download / 2)
-						# Make sure to use half of reported bandwidth for AF60-LRs
+								upload = int(download / 2)
-						if device['identification']['model'] == "AF60-LR":
+							if device['identification']['site']['id'] in foundAirFibersBySite:
-							download = int(download / 2)
+								if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']):
-							upload = int(download / 2)
+									foundAirFibersBySite[device['identification']['site']['id']]['download'] = download
-						if device['identification']['site']['id'] in foundAirFibersBySite:
+									foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
-							if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']):
+									#print(device['identification']['name'] + ' will override bandwidth for site ' + device['identification']['site']['name'])
-								foundAirFibersBySite[device['identification']['site']['id']]['download'] = download
+							else:
-								foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
+								foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload}
-						else:
-							foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload}
 	return foundAirFibersBySite
 
 def buildSiteList(sites, dataLinks):
@@ -240,6 +250,39 @@ def loadRoutingOverrides():
 	#print(overrides)
 	return overrides
 
+def findNodesBranchedOffPtMP(siteList, dataLinks, sites, rootSite):
+	nodeOffPtMP = {}
+	for site in sites:
+		id = site['identification']['id']
+		name = site['identification']['name']
+		type = site['identification']['type']
+		if id != rootSite['id']:
+			trueParent = findInSiteListById(siteList, id)['parent']
+			if type == 'site':
+				#parent = findInSiteListById(siteList, id)['parent']
+				if ('parent' in site['identification']) and (site['identification']['parent'] is not None):
+					parent = site['identification']['parent']['id']
+					for link in dataLinks:
+						if (link['to']['site'] is not None) and (link['to']['site']['identification'] is not None):
+							if ('identification' in link['to']['site']) and (link['to']['site']['identification'] is not None):
+								if ('identification' in link['from']['site']) and (link['from']['site']['identification'] is not None):
+									# Respect parent defined by topology and overrides
+									if link['from']['site']['identification']['id'] == trueParent:
+										if link['to']['site']['identification']['id'] == id:
+											if link['from']['device']['overview']['wirelessMode'] == 'ap-ptmp':
+												if 'overview' in link['to']['device']:
+													if ('downlinkCapacity' in link['to']['device']['overview']) and ('uplinkCapacity' in link['to']['device']['overview']):
+														if (link['to']['device']['overview']['downlinkCapacity'] is not None) and (link['to']['device']['overview']['uplinkCapacity'] is not None): 
+															# Capacity of the PtMP client radio feeding the PoP will be used as the site bandwidth limit
+															download = int(round(link['to']['device']['overview']['downlinkCapacity']/1000000))
+															upload = int(round(link['to']['device']['overview']['uplinkCapacity']/1000000))
+															nodeOffPtMP[id] = { 'parent': link['from']['device']['identification']['id'],
+																		'parentName': link['from']['device']['identification']['name'],
+																		'download': download,
+																		'upload': upload
+																		}
+	return nodeOffPtMP
+
 def buildFullGraph():
 	# Attempts to build a full network graph, incorporating as much of the UISP
 	# hierarchy as possible.
@@ -253,18 +296,28 @@ def buildFullGraph():
 	dataLinks = uispRequest("data-links?siteLinksOnly=true")
 
 	# Build Site Capacities
+	print("Compiling Site Bandwidths")
 	siteBandwidth = buildSiteBandwidths()
+	print("Finding AP Capacities")
 	findApCapacities(devices, siteBandwidth)
-	foundAirFibersBySite = findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps)
-	
 	# Create a list of just network sites
+	print('Creating list of sites')
 	siteList = buildSiteList(sites, dataLinks)
 	rootSite = findInSiteList(siteList, uispSite)
+	print("Finding PtP Capacities")
+	foundAirFibersBySite = findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps)
+	
+	print('Creating list of route overrides')
 	routeOverrides = loadRoutingOverrides()
 	if rootSite is None:
 		print("ERROR: Unable to find root site in UISP")
 		return
+	print('Creating graph')
 	walkGraphOutwards(siteList, rootSite, routeOverrides)
+	
+	print("Finding PtMP Capacities")
+	nodeOffPtMP = findNodesBranchedOffPtMP(siteList, dataLinks, sites, rootSite)
+	
 	# Debug code: dump the list of site parents
 	# for s in siteList:
 	# 	if s['parent'] == "":
@@ -273,25 +326,8 @@ def buildFullGraph():
 	# 		p = findInSiteListById(siteList, s['parent'])['name']
 	# 		print(s['name'] + " (" + str(s['cost']) + ") <-- " + p)
 	
-	# Find Nodes Connected By PtMP
+	
-	nodeOffPtMP = {}
+
-	for site in sites:
-		id = site['identification']['id']
-		name = site['identification']['name']
-		type = site['identification']['type']
-		parent = findInSiteListById(siteList, id)['parent']
-		if type == 'site':
-			for link in dataLinks:
-				if link['from']['site'] is not None and link['from']['site']['identification']['id'] == parent:
-					if link['to']['site'] is not None and link['to']['site']['identification']['id'] == id:
-						if link['from']['device'] is not None and link['to']['device'] is not None and link['to']['device']['overview'] is not None and link['to']['device']['overview']['downlinkCapacity'] is not None and link['from']['device']['overview']['wirelessMode'] == 'ap-ptmp':
-							# Capacity of the PtMP client radio feeding the PoP will be used as the site bandwidth limit
-							download = int(round(link['to']['device']['overview']['downlinkCapacity']/1000000))
-							upload = int(round(link['to']['device']['overview']['uplinkCapacity']/1000000))
-							nodeOffPtMP[id] = { 'parent': link['from']['device']['identification']['id'],
-												'download': download,
-												'upload': upload
-												}
 	print("Building Topology")
 	net = NetworkGraph()
 	# Add all sites and client sites
@@ -318,15 +354,19 @@ def buildFullGraph():
 					# Use the CSV bandwidth values
 					download = siteBandwidth[name]["download"]
 					upload = siteBandwidth[name]["upload"]
-				elif id in nodeOffPtMP:
+				else:					
-					download = nodeOffPtMP[id]['download']
+					# Use limits from foundAirFibersBySite
-					upload = nodeOffPtMP[id]['upload']
+					if id in foundAirFibersBySite:
-				elif id in foundAirFibersBySite:
+						download = foundAirFibersBySite[id]['download']
-					download = foundAirFibersBySite[id]['download']
+						upload = foundAirFibersBySite[id]['upload']
-					upload = foundAirFibersBySite[id]['upload']
+						#print('Site ' + name + ' will use bandwidth from foundAirFibersBySite.')
-				else:
+					# Use limits from nodeOffPtMP only if they're greater than what is already set
-					# Add them just in case
+					if id in nodeOffPtMP:
-					siteBandwidth[name] = {
+						if (nodeOffPtMP[id]['download'] >= download) or (nodeOffPtMP[id]['upload'] >= upload):
+							download = nodeOffPtMP[id]['download']
+							upload = nodeOffPtMP[id]['upload']
+							#print('Site ' + name + ' will use bandwidth from nodeOffPtMP.')
+				siteBandwidth[name] = {
 						"download": download, "upload": upload}
 			case default:
 				nodeType = NodeType.client
@@ -364,7 +404,7 @@ def buildFullGraph():
 
 				# TODO: Figure out Mikrotik IPv6?
 				mac = device['identification']['mac']
-
+				
 				net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification']
 							   ['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac))
 

src/manualNetwork.template.csv

@@ -0,0 +1,4 @@
+Site Name,Download Mbps,Upload Mbps,Parent Site Name
+SiteA,1000,1000,
+SiteB,975,975,SiteA
+SiteC,950,950,SiteB