Merge pull request #328 from LibreQoE/uisp_integration_enhancements

Uisp integration enhancements: Topology

src/integrationCommon.py

@@ -383,6 +383,10 @@ class NetworkGraph:
 						#Remove brackets and quotes of list so LibreQoS.py can parse it
 						device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'')
 						device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'')
+						if circuit["upload"] is None: 
+							circuit["upload"] = 0.0
+						if circuit["download"] is None: 
+							circuit["download"] = 0.0
 						row = [
 							circuit["id"],
 							circuit["name"],
@@ -392,10 +396,10 @@ class NetworkGraph:
 							device["mac"],
 							device["ipv4"],
 							device["ipv6"],
-							int(circuit["download"] * 0.98),
+							int(float(circuit["download"]) * 0.98),
-							int(circuit["upload"] * 0.98),
+							int(float(circuit["upload"]) * 0.98),
-							int(circuit["download"] * bandwidthOverheadFactor),
+							int(float(circuit["download"]) * bandwidthOverheadFactor),
-							int(circuit["upload"] * bandwidthOverheadFactor),
+							int(float(circuit["upload"]) * bandwidthOverheadFactor),
 							""
 						]
 						wr.writerow(row)
@@ -414,7 +418,7 @@ class NetworkGraph:
 
 		import graphviz
 		dot = graphviz.Digraph(
-			'network', comment="Network Graph", engine="fdp")
+			'network', comment="Network Graph", engine="dot")
 
 		for (i, node) in enumerate(self.nodes):
 			if ((node.type != NodeType.client and node.type != NodeType.device) or showClients):

src/integrationUISP.py

@@ -76,6 +76,163 @@ def buildFlatGraph():
 		net.createNetworkJson()
 	net.createShapedDevices()
 
+def linkSiteTarget(link, direction):
+	# Helper function to extract the site ID from a data link. Returns
+	# None if not present.
+	if link[direction]['site'] is not None:
+		return link[direction]['site']['identification']['id']
+	
+	return None
+
+def findSiteLinks(dataLinks, siteId):
+	# Searches the Data Links for any links to/from the specified site.
+	# Returns a list of site IDs that are linked to the specified site.
+	links = []
+	for dl in dataLinks:
+		fromSiteId = linkSiteTarget(dl, "from")
+		if fromSiteId is not None and fromSiteId == siteId:
+			# We have a link originating in this site.
+			target = linkSiteTarget(dl, "to")
+			if target is not None:
+				links.append(target)
+
+		toSiteId = linkSiteTarget(dl, "to")
+		if toSiteId is not None and toSiteId == siteId:
+			# We have a link originating in this site.
+			target = linkSiteTarget(dl, "from")
+			if target is not None:
+				links.append(target)
+	return links
+
+def buildSiteBandwidths():
+	# Builds a dictionary of site bandwidths from the integrationUISPbandwidths.csv file.
+	siteBandwidth = {}
+	if os.path.isfile("integrationUISPbandwidths.csv"):
+		with open('integrationUISPbandwidths.csv') as csv_file:
+			csv_reader = csv.reader(csv_file, delimiter=',')
+			next(csv_reader)
+			for row in csv_reader:
+				name, download, upload = row
+				download = int(download)
+				upload = int(upload)
+				siteBandwidth[name] = {"download": download, "upload": upload}
+	return siteBandwidth
+
+def findApCapacities(devices, siteBandwidth):
+	# Searches the UISP devices for APs and adds their capacities to the siteBandwidth dictionary.
+	for device in devices:
+		if device['identification']['role'] == "ap":
+			name = device['identification']['name']
+			if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']:
+				download = int(device['overview']
+							   ['downlinkCapacity'] / 1000000)
+				upload = int(device['overview']['uplinkCapacity'] / 1000000)
+				siteBandwidth[device['identification']['name']] = {
+					"download": download, "upload": upload}
+
+def findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps):
+	foundAirFibersBySite = {}
+	for device in devices:
+		if device['identification']['site']['type'] == 'site':
+			if device['identification']['role'] == "station":
+				if device['identification']['type'] == "airFiber":
+					if device['overview']['status'] == 'active':
+						if device['overview']['downlinkCapacity'] is not None and device['overview']['uplinkCapacity'] is not None:
+							download = int(device['overview']['downlinkCapacity']/ 1000000)
+							upload = int(device['overview']['uplinkCapacity']/ 1000000)
+						else:
+							download = generatedPNDownloadMbps
+							upload = generatedPNUploadMbps
+						# Make sure to use half of reported bandwidth for AF60-LRs
+						if device['identification']['model'] == "AF60-LR":
+							download = int(download / 2)
+							upload = int(download / 2)
+						if device['identification']['site']['id'] in foundAirFibersBySite:
+							if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']):
+								foundAirFibersBySite[device['identification']['site']['id']]['download'] = download
+								foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
+						else:
+							foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload}
+	return foundAirFibersBySite
+
+def buildSiteList(sites, dataLinks):
+	# Builds a list of sites, including their IDs, names, and connections.
+	# Connections are determined by the dataLinks list.
+	siteList = []
+	for site in sites:
+		newSite = {
+			'id': site['identification']['id'], 
+			'name': site['identification']['name'],
+			'connections': findSiteLinks(dataLinks, site['identification']['id']),
+			'cost': 10000,
+			'parent': "",
+			'type': type,
+		}
+		siteList.append(newSite)
+	return siteList
+
+def findInSiteList(siteList, name):
+	# Searches the siteList for a site with the specified name.
+	for site in siteList:
+		if site['name'] == name:
+			return site
+	return None
+
+def findInSiteListById(siteList, id):
+	# Searches the siteList for a site with the specified name.
+	for site in siteList:
+		if site['id'] == id:
+			return site
+	return None
+
+def debugSpaces(n):
+	# Helper function to print n spaces.
+	spaces = ""
+	for i in range(int(n)):
+		spaces = spaces + " "
+	return spaces
+
+def walkGraphOutwards(siteList, root, routeOverrides):
+	def walkGraph(node, parent, cost, backPath):
+		site = findInSiteListById(siteList, node)
+		routeName = parent['name'] + "->" + site['name']
+		if routeName in routeOverrides:
+			# We have an overridden cost for this route, so use it instead
+			#print("--> Using overridden cost for " + routeName + ". New cost: " + str(routeOverrides[routeName]) + ".")
+			cost = routeOverrides[routeName]
+		if cost < site['cost']:
+			# It's cheaper to get here this way, so update the cost and parent.
+			site['cost'] = cost
+			site['parent'] = parent['id']
+			#print(debugSpaces(cost/10) + parent['name'] + "->" + site['name'] + " -> New cost: " + str(cost))
+
+		for connection in site['connections']:
+			if not connection in backPath:
+				#target = findInSiteListById(siteList, connection)
+				#print(debugSpaces((cost+10)/10) + site['name'] + " -> " + target['name'] + " (" + str(target['cost']) + ")")
+				newBackPath = backPath.copy()
+				newBackPath.append(site['id'])
+				walkGraph(connection, site, cost+10, newBackPath)
+
+	for connection in root['connections']:
+		# Force the parent since we're at the top
+		site = findInSiteListById(siteList, connection)
+		site['parent'] = root['id']
+		walkGraph(connection, root, 10, [root['id']])
+
+def loadRoutingOverrides():
+	# Loads integrationUISProutes.csv and returns a set of "from", "to", "cost"
+	overrides = {}
+	if os.path.isfile("integrationUISProutes.csv"):
+		with open("integrationUISProutes.csv", "r") as f:
+			reader = csv.reader(f)
+			for row in reader:
+				if not row[0].startswith("#") and len(row) == 3:
+					fromSite, toSite, cost = row
+					overrides[fromSite.strip() + "->" + toSite.strip()] = int(cost)
+	#print(overrides)
+	return overrides
+
 def buildFullGraph():
 	# Attempts to build a full network graph, incorporating as much of the UISP
 	# hierarchy as possible.
@@ -88,48 +245,26 @@ def buildFullGraph():
 	devices = uispRequest("devices?withInterfaces=true&authorized=true")
 	dataLinks = uispRequest("data-links?siteLinksOnly=true")
 
-	# Do we already have a integrationUISPbandwidths.csv file?
+	# Build Site Capacities
-	siteBandwidth = {}
+	siteBandwidth = buildSiteBandwidths()
-	if os.path.isfile("integrationUISPbandwidths.csv"):
+	findApCapacities(devices, siteBandwidth)
-		with open('integrationUISPbandwidths.csv') as csv_file:
+	foundAirFibersBySite = findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps)
-			csv_reader = csv.reader(csv_file, delimiter=',')
-			next(csv_reader)
-			for row in csv_reader:
-				name, download, upload = row
-				download = int(download)
-				upload = int(upload)
-				siteBandwidth[name] = {"download": download, "upload": upload}
 	
-	# Find AP capacities from UISP
+	# Create a list of just network sites
-	for device in devices:
+	siteList = buildSiteList(sites, dataLinks)
-		if device['identification']['role'] == "ap":
+	rootSite = findInSiteList(siteList, uispSite)
-			name = device['identification']['name']
+	routeOverrides = loadRoutingOverrides()
-			if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']:
+	if rootSite is None:
-				download = int(device['overview']
+		print("ERROR: Unable to find root site in UISP")
-							   ['downlinkCapacity'] / 1000000)
+		return
-				upload = int(device['overview']['uplinkCapacity'] / 1000000)
+	walkGraphOutwards(siteList, rootSite, routeOverrides)
-				siteBandwidth[device['identification']['name']] = {
+	# Debug code: dump the list of site parents
-					"download": download, "upload": upload}
+	# for s in siteList:
-	
+	# 	if s['parent'] == "":
-	# Find Site Capacities by AirFiber capacities
+	# 		p = "None"
-	foundAirFibersBySite = {}
+	# 	else:
-	for device in devices:
+	# 		p = findInSiteListById(siteList, s['parent'])['name']
-		if device['identification']['site']['type'] == 'site':
+	# 		print(s['name'] + " (" + str(s['cost']) + ") <-- " + p)
-			if device['identification']['role'] == "station":
-				if device['identification']['type'] == "airFiber":
-					if device['overview']['status'] == 'active':
-						download = int(device['overview']['downlinkCapacity']/ 1000000)
-						upload = int(device['overview']['uplinkCapacity']/ 1000000)
-						# Make sure to use half of reported bandwidth for AF60-LRs
-						if device['identification']['model'] == "AF60-LR":
-							download = int(download / 2)
-							upload = int(download / 2)
-						if device['identification']['site']['id'] in foundAirFibersBySite:
-							if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']):
-								foundAirFibersBySite[device['identification']['site']['id']]['download'] = download
-								foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
-						else:
-							foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload}
 
 	print("Building Topology")
 	net = NetworkGraph()
@@ -142,10 +277,12 @@ def buildFullGraph():
 		upload = generatedPNUploadMbps
 		address = ""
 		customerName = ""
-		if site['identification']['parent'] is None:
+		parent = findInSiteListById(siteList, id)['parent']
-			parent = ""
+		if parent == "":
-		else:
+			if site['identification']['parent'] is None:
-			parent = site['identification']['parent']['id']
+				parent = ""
+			else:
+				parent = site['identification']['parent']['id']
 		match type:
 			case "site":
 				nodeType = NodeType.site

src/integrationUISProutes.csv

@@ -0,0 +1,5 @@
+# Allows you to override route costs in the UISP integration, to better
+# represent your network. Costs by default increment 10 at each hop.
+# So if you want to skip 10 links, put a cost of 100 in.
+# From Site Name, To Site Name, New Cost
+# MYCORE, MYTOWER, 100