diff --git a/src/integrationCommon.py b/src/integrationCommon.py index 33f20e31..6e89c3c9 100644 --- a/src/integrationCommon.py +++ b/src/integrationCommon.py @@ -383,6 +383,10 @@ class NetworkGraph: #Remove brackets and quotes of list so LibreQoS.py can parse it device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'') device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'') + if circuit["upload"] is None: + circuit["upload"] = 0.0 + if circuit["download"] is None: + circuit["download"] = 0.0 row = [ circuit["id"], circuit["name"], @@ -392,10 +396,10 @@ class NetworkGraph: device["mac"], device["ipv4"], device["ipv6"], - int(circuit["download"] * 0.98), - int(circuit["upload"] * 0.98), - int(circuit["download"] * bandwidthOverheadFactor), - int(circuit["upload"] * bandwidthOverheadFactor), + int(float(circuit["download"]) * 0.98), + int(float(circuit["upload"]) * 0.98), + int(float(circuit["download"]) * bandwidthOverheadFactor), + int(float(circuit["upload"]) * bandwidthOverheadFactor), "" ] wr.writerow(row) @@ -414,7 +418,7 @@ class NetworkGraph: import graphviz dot = graphviz.Digraph( - 'network', comment="Network Graph", engine="fdp") + 'network', comment="Network Graph", engine="dot") for (i, node) in enumerate(self.nodes): if ((node.type != NodeType.client and node.type != NodeType.device) or showClients): diff --git a/src/integrationUISP.py b/src/integrationUISP.py index 36937fb7..55fb3411 100644 --- a/src/integrationUISP.py +++ b/src/integrationUISP.py @@ -76,6 +76,163 @@ def buildFlatGraph(): net.createNetworkJson() net.createShapedDevices() +def linkSiteTarget(link, direction): + # Helper function to extract the site ID from a data link. Returns + # None if not present. + if link[direction]['site'] is not None: + return link[direction]['site']['identification']['id'] + + return None + +def findSiteLinks(dataLinks, siteId): + # Searches the Data Links for any links to/from the specified site. + # Returns a list of site IDs that are linked to the specified site. + links = [] + for dl in dataLinks: + fromSiteId = linkSiteTarget(dl, "from") + if fromSiteId is not None and fromSiteId == siteId: + # We have a link originating in this site. + target = linkSiteTarget(dl, "to") + if target is not None: + links.append(target) + + toSiteId = linkSiteTarget(dl, "to") + if toSiteId is not None and toSiteId == siteId: + # We have a link originating in this site. + target = linkSiteTarget(dl, "from") + if target is not None: + links.append(target) + return links + +def buildSiteBandwidths(): + # Builds a dictionary of site bandwidths from the integrationUISPbandwidths.csv file. + siteBandwidth = {} + if os.path.isfile("integrationUISPbandwidths.csv"): + with open('integrationUISPbandwidths.csv') as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + next(csv_reader) + for row in csv_reader: + name, download, upload = row + download = int(download) + upload = int(upload) + siteBandwidth[name] = {"download": download, "upload": upload} + return siteBandwidth + +def findApCapacities(devices, siteBandwidth): + # Searches the UISP devices for APs and adds their capacities to the siteBandwidth dictionary. + for device in devices: + if device['identification']['role'] == "ap": + name = device['identification']['name'] + if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']: + download = int(device['overview'] + ['downlinkCapacity'] / 1000000) + upload = int(device['overview']['uplinkCapacity'] / 1000000) + siteBandwidth[device['identification']['name']] = { + "download": download, "upload": upload} + +def findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps): + foundAirFibersBySite = {} + for device in devices: + if device['identification']['site']['type'] == 'site': + if device['identification']['role'] == "station": + if device['identification']['type'] == "airFiber": + if device['overview']['status'] == 'active': + if device['overview']['downlinkCapacity'] is not None and device['overview']['uplinkCapacity'] is not None: + download = int(device['overview']['downlinkCapacity']/ 1000000) + upload = int(device['overview']['uplinkCapacity']/ 1000000) + else: + download = generatedPNDownloadMbps + upload = generatedPNUploadMbps + # Make sure to use half of reported bandwidth for AF60-LRs + if device['identification']['model'] == "AF60-LR": + download = int(download / 2) + upload = int(download / 2) + if device['identification']['site']['id'] in foundAirFibersBySite: + if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']): + foundAirFibersBySite[device['identification']['site']['id']]['download'] = download + foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload + else: + foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload} + return foundAirFibersBySite + +def buildSiteList(sites, dataLinks): + # Builds a list of sites, including their IDs, names, and connections. + # Connections are determined by the dataLinks list. + siteList = [] + for site in sites: + newSite = { + 'id': site['identification']['id'], + 'name': site['identification']['name'], + 'connections': findSiteLinks(dataLinks, site['identification']['id']), + 'cost': 10000, + 'parent': "", + 'type': type, + } + siteList.append(newSite) + return siteList + +def findInSiteList(siteList, name): + # Searches the siteList for a site with the specified name. + for site in siteList: + if site['name'] == name: + return site + return None + +def findInSiteListById(siteList, id): + # Searches the siteList for a site with the specified name. + for site in siteList: + if site['id'] == id: + return site + return None + +def debugSpaces(n): + # Helper function to print n spaces. + spaces = "" + for i in range(int(n)): + spaces = spaces + " " + return spaces + +def walkGraphOutwards(siteList, root, routeOverrides): + def walkGraph(node, parent, cost, backPath): + site = findInSiteListById(siteList, node) + routeName = parent['name'] + "->" + site['name'] + if routeName in routeOverrides: + # We have an overridden cost for this route, so use it instead + #print("--> Using overridden cost for " + routeName + ". New cost: " + str(routeOverrides[routeName]) + ".") + cost = routeOverrides[routeName] + if cost < site['cost']: + # It's cheaper to get here this way, so update the cost and parent. + site['cost'] = cost + site['parent'] = parent['id'] + #print(debugSpaces(cost/10) + parent['name'] + "->" + site['name'] + " -> New cost: " + str(cost)) + + for connection in site['connections']: + if not connection in backPath: + #target = findInSiteListById(siteList, connection) + #print(debugSpaces((cost+10)/10) + site['name'] + " -> " + target['name'] + " (" + str(target['cost']) + ")") + newBackPath = backPath.copy() + newBackPath.append(site['id']) + walkGraph(connection, site, cost+10, newBackPath) + + for connection in root['connections']: + # Force the parent since we're at the top + site = findInSiteListById(siteList, connection) + site['parent'] = root['id'] + walkGraph(connection, root, 10, [root['id']]) + +def loadRoutingOverrides(): + # Loads integrationUISProutes.csv and returns a set of "from", "to", "cost" + overrides = {} + if os.path.isfile("integrationUISProutes.csv"): + with open("integrationUISProutes.csv", "r") as f: + reader = csv.reader(f) + for row in reader: + if not row[0].startswith("#") and len(row) == 3: + fromSite, toSite, cost = row + overrides[fromSite.strip() + "->" + toSite.strip()] = int(cost) + #print(overrides) + return overrides + def buildFullGraph(): # Attempts to build a full network graph, incorporating as much of the UISP # hierarchy as possible. @@ -88,49 +245,27 @@ def buildFullGraph(): devices = uispRequest("devices?withInterfaces=true&authorized=true") dataLinks = uispRequest("data-links?siteLinksOnly=true") - # Do we already have a integrationUISPbandwidths.csv file? - siteBandwidth = {} - if os.path.isfile("integrationUISPbandwidths.csv"): - with open('integrationUISPbandwidths.csv') as csv_file: - csv_reader = csv.reader(csv_file, delimiter=',') - next(csv_reader) - for row in csv_reader: - name, download, upload = row - download = int(download) - upload = int(upload) - siteBandwidth[name] = {"download": download, "upload": upload} - - # Find AP capacities from UISP - for device in devices: - if device['identification']['role'] == "ap": - name = device['identification']['name'] - if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']: - download = int(device['overview'] - ['downlinkCapacity'] / 1000000) - upload = int(device['overview']['uplinkCapacity'] / 1000000) - siteBandwidth[device['identification']['name']] = { - "download": download, "upload": upload} - - # Find Site Capacities by AirFiber capacities - foundAirFibersBySite = {} - for device in devices: - if device['identification']['site']['type'] == 'site': - if device['identification']['role'] == "station": - if device['identification']['type'] == "airFiber": - if device['overview']['status'] == 'active': - download = int(device['overview']['downlinkCapacity']/ 1000000) - upload = int(device['overview']['uplinkCapacity']/ 1000000) - # Make sure to use half of reported bandwidth for AF60-LRs - if device['identification']['model'] == "AF60-LR": - download = int(download / 2) - upload = int(download / 2) - if device['identification']['site']['id'] in foundAirFibersBySite: - if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']): - foundAirFibersBySite[device['identification']['site']['id']]['download'] = download - foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload - else: - foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload} + # Build Site Capacities + siteBandwidth = buildSiteBandwidths() + findApCapacities(devices, siteBandwidth) + foundAirFibersBySite = findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps) + # Create a list of just network sites + siteList = buildSiteList(sites, dataLinks) + rootSite = findInSiteList(siteList, uispSite) + routeOverrides = loadRoutingOverrides() + if rootSite is None: + print("ERROR: Unable to find root site in UISP") + return + walkGraphOutwards(siteList, rootSite, routeOverrides) + # Debug code: dump the list of site parents + # for s in siteList: + # if s['parent'] == "": + # p = "None" + # else: + # p = findInSiteListById(siteList, s['parent'])['name'] + # print(s['name'] + " (" + str(s['cost']) + ") <-- " + p) + print("Building Topology") net = NetworkGraph() # Add all sites and client sites @@ -142,10 +277,12 @@ def buildFullGraph(): upload = generatedPNUploadMbps address = "" customerName = "" - if site['identification']['parent'] is None: - parent = "" - else: - parent = site['identification']['parent']['id'] + parent = findInSiteListById(siteList, id)['parent'] + if parent == "": + if site['identification']['parent'] is None: + parent = "" + else: + parent = site['identification']['parent']['id'] match type: case "site": nodeType = NodeType.site diff --git a/src/integrationUISProutes.csv b/src/integrationUISProutes.csv new file mode 100644 index 00000000..643c4eb0 --- /dev/null +++ b/src/integrationUISProutes.csv @@ -0,0 +1,5 @@ +# Allows you to override route costs in the UISP integration, to better +# represent your network. Costs by default increment 10 at each hop. +# So if you want to skip 10 links, put a cost of 100 in. +# From Site Name, To Site Name, New Cost +# MYCORE, MYTOWER, 100