From 6c84100832ec019f227fcbf5e5dd550f5a3ae276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Fri, 3 Mar 2023 20:27:36 -0700 Subject: [PATCH 1/5] Update integrationUISP.py --- src/integrationUISP.py | 372 +++++++++++++++++++++-------------------- 1 file changed, 189 insertions(+), 183 deletions(-) diff --git a/src/integrationUISP.py b/src/integrationUISP.py index 177df1a6..993447e7 100644 --- a/src/integrationUISP.py +++ b/src/integrationUISP.py @@ -7,216 +7,222 @@ from ispConfig import uispSite, uispStrategy from integrationCommon import isIpv4Permitted, fixSubnet def uispRequest(target): - # Sends an HTTP request to UISP and returns the - # result in JSON. You only need to specify the - # tail end of the URL, e.g. "sites" - from ispConfig import UISPbaseURL, uispAuthToken - url = UISPbaseURL + "/nms/api/v2.1/" + target - headers = {'accept': 'application/json', 'x-auth-token': uispAuthToken} - r = requests.get(url, headers=headers) - return r.json() + # Sends an HTTP request to UISP and returns the + # result in JSON. You only need to specify the + # tail end of the URL, e.g. "sites" + from ispConfig import UISPbaseURL, uispAuthToken + url = UISPbaseURL + "/nms/api/v2.1/" + target + headers = {'accept': 'application/json', 'x-auth-token': uispAuthToken} + r = requests.get(url, headers=headers) + return r.json() def buildFlatGraph(): - # Builds a high-performance (but lacking in site or AP bandwidth control) - # network. - from integrationCommon import NetworkGraph, NetworkNode, NodeType - from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps + # Builds a high-performance (but lacking in site or AP bandwidth control) + # network. + from integrationCommon import NetworkGraph, NetworkNode, NodeType + from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps - # Load network sites - print("Loading Data from UISP") - sites = uispRequest("sites") - devices = uispRequest("devices?withInterfaces=true&authorized=true") + # Load network sites + print("Loading Data from UISP") + sites = uispRequest("sites") + devices = uispRequest("devices?withInterfaces=true&authorized=true") - # Build a basic network adding every client to the tree - print("Building Flat Topology") - net = NetworkGraph() + # Build a basic network adding every client to the tree + print("Building Flat Topology") + net = NetworkGraph() - for site in sites: - type = site['identification']['type'] - if type == "endpoint": - id = site['identification']['id'] - address = site['description']['address'] - name = site['identification']['name'] - type = site['identification']['type'] - download = generatedPNDownloadMbps - upload = generatedPNUploadMbps - if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']): - download = int(round(site['qos']['downloadSpeed']/1000000)) - upload = int(round(site['qos']['uploadSpeed']/1000000)) + for site in sites: + type = site['identification']['type'] + if type == "endpoint": + id = site['identification']['id'] + address = site['description']['address'] + customerName = '' + name = site['identification']['name'] + type = site['identification']['type'] + download = generatedPNDownloadMbps + upload = generatedPNUploadMbps + if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']): + download = int(round(site['qos']['downloadSpeed']/1000000)) + upload = int(round(site['qos']['uploadSpeed']/1000000)) - node = NetworkNode(id=id, displayName=name, type=NodeType.client, download=download, upload=upload, address=address) - net.addRawNode(node) - for device in devices: - if device['identification']['site'] is not None and device['identification']['site']['id'] == id: - # The device is at this site, so add it - ipv4 = [] - ipv6 = [] + node = NetworkNode(id=id, displayName=name, type=NodeType.client, download=download, upload=upload, address=address, customerName=customerName) + net.addRawNode(node) + for device in devices: + if device['identification']['site'] is not None and device['identification']['site']['id'] == id: + # The device is at this site, so add it + ipv4 = [] + ipv6 = [] - for interface in device["interfaces"]: - for ip in interface["addresses"]: - ip = ip["cidr"] - if isIpv4Permitted(ip): - ip = fixSubnet(ip) - if ip not in ipv4: - ipv4.append(ip) + for interface in device["interfaces"]: + for ip in interface["addresses"]: + ip = ip["cidr"] + if isIpv4Permitted(ip): + ip = fixSubnet(ip) + if ip not in ipv4: + ipv4.append(ip) - # TODO: Figure out Mikrotik IPv6? - mac = device['identification']['mac'] + # TODO: Figure out Mikrotik IPv6? + mac = device['identification']['mac'] - net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification'] - ['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac)) + net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification'] + ['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac)) - # Finish up - net.prepareTree() - net.plotNetworkGraph(False) - if net.doesNetworkJsonExist(): - print("network.json already exists. Leaving in-place.") - else: - net.createNetworkJson() - net.createShapedDevices() + # Finish up + net.prepareTree() + net.plotNetworkGraph(False) + if net.doesNetworkJsonExist(): + print("network.json already exists. Leaving in-place.") + else: + net.createNetworkJson() + net.createShapedDevices() def buildFullGraph(): - # Attempts to build a full network graph, incorporating as much of the UISP - # hierarchy as possible. - from integrationCommon import NetworkGraph, NetworkNode, NodeType - from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps + # Attempts to build a full network graph, incorporating as much of the UISP + # hierarchy as possible. + from integrationCommon import NetworkGraph, NetworkNode, NodeType + from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps - # Load network sites - print("Loading Data from UISP") - sites = uispRequest("sites") - devices = uispRequest("devices?withInterfaces=true&authorized=true") - dataLinks = uispRequest("data-links?siteLinksOnly=true") + # Load network sites + print("Loading Data from UISP") + sites = uispRequest("sites") + devices = uispRequest("devices?withInterfaces=true&authorized=true") + dataLinks = uispRequest("data-links?siteLinksOnly=true") - # Do we already have a integrationUISPbandwidths.csv file? - siteBandwidth = {} - if os.path.isfile("integrationUISPbandwidths.csv"): - with open('integrationUISPbandwidths.csv') as csv_file: - csv_reader = csv.reader(csv_file, delimiter=',') - next(csv_reader) - for row in csv_reader: - name, download, upload = row - download = int(download) - upload = int(upload) - siteBandwidth[name] = {"download": download, "upload": upload} + # Do we already have a integrationUISPbandwidths.csv file? + siteBandwidth = {} + if os.path.isfile("integrationUISPbandwidths.csv"): + with open('integrationUISPbandwidths.csv') as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + next(csv_reader) + for row in csv_reader: + name, download, upload = row + download = int(download) + upload = int(upload) + siteBandwidth[name] = {"download": download, "upload": upload} - # Find AP capacities from UISP - for device in devices: - if device['identification']['role'] == "ap": - name = device['identification']['name'] - if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']: - download = int(device['overview'] - ['downlinkCapacity'] / 1000000) - upload = int(device['overview']['uplinkCapacity'] / 1000000) - siteBandwidth[device['identification']['name']] = { - "download": download, "upload": upload} + # Find AP capacities from UISP + for device in devices: + if device['identification']['role'] == "ap": + name = device['identification']['name'] + if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']: + download = int(device['overview'] + ['downlinkCapacity'] / 1000000) + upload = int(device['overview']['uplinkCapacity'] / 1000000) + siteBandwidth[device['identification']['name']] = { + "download": download, "upload": upload} - print("Building Topology") - net = NetworkGraph() - # Add all sites and client sites - for site in sites: - id = site['identification']['id'] - name = site['identification']['name'] - type = site['identification']['type'] - download = generatedPNDownloadMbps - upload = generatedPNUploadMbps - address = "" - if site['identification']['parent'] is None: - parent = "" - else: - parent = site['identification']['parent']['id'] - match type: - case "site": - nodeType = NodeType.site - if name in siteBandwidth: - # Use the CSV bandwidth values - download = siteBandwidth[name]["download"] - upload = siteBandwidth[name]["upload"] - else: - # Add them just in case - siteBandwidth[name] = { - "download": download, "upload": upload} - case default: - nodeType = NodeType.client - address = site['description']['address'] - if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']): - download = int(round(site['qos']['downloadSpeed']/1000000)) - upload = int(round(site['qos']['uploadSpeed']/1000000)) + print("Building Topology") + net = NetworkGraph() + # Add all sites and client sites + for site in sites: + id = site['identification']['id'] + name = site['identification']['name'] + type = site['identification']['type'] + download = generatedPNDownloadMbps + upload = generatedPNUploadMbps + address = "" + customerName = "" + if site['identification']['parent'] is None: + parent = "" + else: + parent = site['identification']['parent']['id'] + match type: + case "site": + nodeType = NodeType.site + if name in siteBandwidth: + # Use the CSV bandwidth values + download = siteBandwidth[name]["download"] + upload = siteBandwidth[name]["upload"] + else: + # Add them just in case + siteBandwidth[name] = { + "download": download, "upload": upload} + case default: + nodeType = NodeType.client + address = site['description']['address'] + try: + customerName = site["ucrm"]["client"]["name"] + except: + customerName = "" + if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']): + download = int(round(site['qos']['downloadSpeed']/1000000)) + upload = int(round(site['qos']['uploadSpeed']/1000000)) - node = NetworkNode(id=id, displayName=name, type=nodeType, - parentId=parent, download=download, upload=upload, address=address) - # If this is the uispSite node, it becomes the root. Otherwise, add it to the - # node soup. - if name == uispSite: - net.replaceRootNote(node) - else: - net.addRawNode(node) + node = NetworkNode(id=id, displayName=name, type=nodeType, + parentId=parent, download=download, upload=upload, address=address, customerName=customerName) + # If this is the uispSite node, it becomes the root. Otherwise, add it to the + # node soup. + if name == uispSite: + net.replaceRootNote(node) + else: + net.addRawNode(node) - for device in devices: - if device['identification']['site'] is not None and device['identification']['site']['id'] == id: - # The device is at this site, so add it - ipv4 = [] - ipv6 = [] + for device in devices: + if device['identification']['site'] is not None and device['identification']['site']['id'] == id: + # The device is at this site, so add it + ipv4 = [] + ipv6 = [] - for interface in device["interfaces"]: - for ip in interface["addresses"]: - ip = ip["cidr"] - if isIpv4Permitted(ip): - ip = fixSubnet(ip) - if ip not in ipv4: - ipv4.append(ip) + for interface in device["interfaces"]: + for ip in interface["addresses"]: + ip = ip["cidr"] + if isIpv4Permitted(ip): + ip = fixSubnet(ip) + if ip not in ipv4: + ipv4.append(ip) - # TODO: Figure out Mikrotik IPv6? - mac = device['identification']['mac'] + # TODO: Figure out Mikrotik IPv6? + mac = device['identification']['mac'] - net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification'] - ['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac)) + net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification'] + ['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac)) - # Now iterate access points, and look for connections to sites - for node in net.nodes: - if node.type == NodeType.device: - for dl in dataLinks: - if dl['from']['device'] is not None and dl['from']['device']['identification']['id'] == node.id: - if dl['to']['site'] is not None and dl['from']['site']['identification']['id'] != dl['to']['site']['identification']['id']: - target = net.findNodeIndexById( - dl['to']['site']['identification']['id']) - if target > -1: - # We found the site - if net.nodes[target].type == NodeType.client or net.nodes[target].type == NodeType.clientWithChildren: - net.nodes[target].parentId = node.id - node.type = NodeType.ap - if node.displayName in siteBandwidth: - # Use the bandwidth numbers from the CSV file - node.uploadMbps = siteBandwidth[node.displayName]["upload"] - node.downloadMbps = siteBandwidth[node.displayName]["download"] - else: - # Add some defaults in case they want to change them - siteBandwidth[node.displayName] = { - "download": generatedPNDownloadMbps, "upload": generatedPNUploadMbps} + # Now iterate access points, and look for connections to sites + for node in net.nodes: + if node.type == NodeType.device: + for dl in dataLinks: + if dl['from']['device'] is not None and dl['from']['device']['identification']['id'] == node.id: + if dl['to']['site'] is not None and dl['from']['site']['identification']['id'] != dl['to']['site']['identification']['id']: + target = net.findNodeIndexById( + dl['to']['site']['identification']['id']) + if target > -1: + # We found the site + if net.nodes[target].type == NodeType.client or net.nodes[target].type == NodeType.clientWithChildren: + net.nodes[target].parentId = node.id + node.type = NodeType.ap + if node.displayName in siteBandwidth: + # Use the bandwidth numbers from the CSV file + node.uploadMbps = siteBandwidth[node.displayName]["upload"] + node.downloadMbps = siteBandwidth[node.displayName]["download"] + else: + # Add some defaults in case they want to change them + siteBandwidth[node.displayName] = { + "download": generatedPNDownloadMbps, "upload": generatedPNUploadMbps} - net.prepareTree() - net.plotNetworkGraph(False) - if net.doesNetworkJsonExist(): - print("network.json already exists. Leaving in-place.") - else: - net.createNetworkJson() - net.createShapedDevices() + net.prepareTree() + net.plotNetworkGraph(False) + if net.doesNetworkJsonExist(): + print("network.json already exists. Leaving in-place.") + else: + net.createNetworkJson() + net.createShapedDevices() - # Save integrationUISPbandwidths.csv - # (the newLine fixes generating extra blank lines) - with open('integrationUISPbandwidths.csv', 'w', newline='') as csvfile: - wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL) - wr.writerow(['ParentNode', 'Download Mbps', 'Upload Mbps']) - for device in siteBandwidth: - entry = ( - device, siteBandwidth[device]["download"], siteBandwidth[device]["upload"]) - wr.writerow(entry) + # Save integrationUISPbandwidths.csv + # (the newLine fixes generating extra blank lines) + with open('integrationUISPbandwidths.csv', 'w', newline='') as csvfile: + wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL) + wr.writerow(['ParentNode', 'Download Mbps', 'Upload Mbps']) + for device in siteBandwidth: + entry = ( + device, siteBandwidth[device]["download"], siteBandwidth[device]["upload"]) + wr.writerow(entry) def importFromUISP(): - match uispStrategy: - case "full": buildFullGraph() - case default: buildFlatGraph() + match uispStrategy: + case "full": buildFullGraph() + case default: buildFlatGraph() if __name__ == '__main__': - importFromUISP() + importFromUISP() From b100870e3cd87a4a593fd60e78d259f4dad46789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Fri, 3 Mar 2023 20:28:22 -0700 Subject: [PATCH 2/5] Update integrationCommon.py --- src/integrationCommon.py | 726 ++++++++++++++++++++------------------- 1 file changed, 366 insertions(+), 360 deletions(-) diff --git a/src/integrationCommon.py b/src/integrationCommon.py index 710713ef..f9c4920b 100644 --- a/src/integrationCommon.py +++ b/src/integrationCommon.py @@ -2,420 +2,426 @@ # integrations. from typing import List, Any -from ispConfig import allowedSubnets, ignoreSubnets, generatedPNUploadMbps, generatedPNDownloadMbps +from ispConfig import allowedSubnets, ignoreSubnets, generatedPNUploadMbps, generatedPNDownloadMbps, circuitNameUseAddress import ipaddress import enum def isInAllowedSubnets(inputIP): - # Check whether an IP address occurs inside the allowedSubnets list - isAllowed = False - if '/' in inputIP: - inputIP = inputIP.split('/')[0] - for subnet in allowedSubnets: - if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)): - isAllowed = True - return isAllowed + # Check whether an IP address occurs inside the allowedSubnets list + isAllowed = False + if '/' in inputIP: + inputIP = inputIP.split('/')[0] + for subnet in allowedSubnets: + if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)): + isAllowed = True + return isAllowed def isInIgnoredSubnets(inputIP): - # Check whether an IP address occurs within the ignoreSubnets list - isIgnored = False - if '/' in inputIP: - inputIP = inputIP.split('/')[0] - for subnet in ignoreSubnets: - if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)): - isIgnored = True - return isIgnored + # Check whether an IP address occurs within the ignoreSubnets list + isIgnored = False + if '/' in inputIP: + inputIP = inputIP.split('/')[0] + for subnet in ignoreSubnets: + if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)): + isIgnored = True + return isIgnored def isIpv4Permitted(inputIP): - # Checks whether an IP address is in Allowed Subnets. - # If it is, check that it isn't in Ignored Subnets. - # If it is allowed and not ignored, returns true. - # Otherwise, returns false. - return isInIgnoredSubnets(inputIP) == False and isInAllowedSubnets(inputIP) + # Checks whether an IP address is in Allowed Subnets. + # If it is, check that it isn't in Ignored Subnets. + # If it is allowed and not ignored, returns true. + # Otherwise, returns false. + return isInIgnoredSubnets(inputIP) == False and isInAllowedSubnets(inputIP) def fixSubnet(inputIP): - # If an IP address has a CIDR other than /32 (e.g. 192.168.1.1/24), - # but doesn't appear as a network address (e.g. 192.168.1.0/24) - # then it probably isn't actually serving that whole subnet. - # This allows you to specify e.g. 192.168.1.0/24 is "the client - # on port 3" in the device, without falling afoul of UISP's inclusion - # of subnet masks in device IPs. - [rawIp, cidr] = inputIP.split('/') - if cidr != "32": - try: - subnet = ipaddress.ip_network(inputIP) - except: - # Not a network address - return rawIp + "/32" - return inputIP + # If an IP address has a CIDR other than /32 (e.g. 192.168.1.1/24), + # but doesn't appear as a network address (e.g. 192.168.1.0/24) + # then it probably isn't actually serving that whole subnet. + # This allows you to specify e.g. 192.168.1.0/24 is "the client + # on port 3" in the device, without falling afoul of UISP's inclusion + # of subnet masks in device IPs. + [rawIp, cidr] = inputIP.split('/') + if cidr != "32": + try: + subnet = ipaddress.ip_network(inputIP) + except: + # Not a network address + return rawIp + "/32" + return inputIP class NodeType(enum.IntEnum): - # Enumeration to define what type of node - # a NetworkNode is. - root = 1 - site = 2 - ap = 3 - client = 4 - clientWithChildren = 5 - device = 6 + # Enumeration to define what type of node + # a NetworkNode is. + root = 1 + site = 2 + ap = 3 + client = 4 + clientWithChildren = 5 + device = 6 class NetworkNode: - # Defines a node on a LibreQoS network graph. - # Nodes default to being disconnected, and - # will be mapped to the root of the overall - # graph. + # Defines a node on a LibreQoS network graph. + # Nodes default to being disconnected, and + # will be mapped to the root of the overall + # graph. - id: str - displayName: str - parentIndex: int - parentId: str - type: NodeType - downloadMbps: int - uploadMbps: int - ipv4: List - ipv6: List - address: str - mac: str + id: str + displayName: str + parentIndex: int + parentId: str + type: NodeType + downloadMbps: int + uploadMbps: int + ipv4: List + ipv6: List + address: str + mac: str - def __init__(self, id: str, displayName: str = "", parentId: str = "", type: NodeType = NodeType.site, download: int = generatedPNDownloadMbps, upload: int = generatedPNUploadMbps, ipv4: List = [], ipv6: List = [], address: str = "", mac: str = "") -> None: - self.id = id - self.parentIndex = 0 - self.type = type - self.parentId = parentId - if displayName == "": - self.displayName = id - else: - self.displayName = displayName - self.downloadMbps = download - self.uploadMbps = upload - self.ipv4 = ipv4 - self.ipv6 = ipv6 - self.address = address - self.mac = mac + def __init__(self, id: str, displayName: str = "", parentId: str = "", type: NodeType = NodeType.site, download: int = generatedPNDownloadMbps, upload: int = generatedPNUploadMbps, ipv4: List = [], ipv6: List = [], address: str = "", mac: str = "", customerName: str = "") -> None: + self.id = id + self.parentIndex = 0 + self.type = type + self.parentId = parentId + if displayName == "": + self.displayName = id + else: + self.displayName = displayName + self.downloadMbps = download + self.uploadMbps = upload + self.ipv4 = ipv4 + self.ipv6 = ipv6 + self.address = address + self.customerName = customerName + self.mac = mac class NetworkGraph: - # Defines a network as a graph topology - # allowing any integration to build the - # graph via a common API, emitting - # ShapedDevices and network.json files - # via a common interface. + # Defines a network as a graph topology + # allowing any integration to build the + # graph via a common API, emitting + # ShapedDevices and network.json files + # via a common interface. - nodes: List - ipv4ToIPv6: Any - excludeSites: List # Copied to allow easy in-test patching - exceptionCPEs: Any + nodes: List + ipv4ToIPv6: Any + excludeSites: List # Copied to allow easy in-test patching + exceptionCPEs: Any - def __init__(self) -> None: - from ispConfig import findIPv6usingMikrotik, excludeSites, exceptionCPEs - self.nodes = [ - NetworkNode("FakeRoot", type=NodeType.root, - parentId="", displayName="Shaper Root") - ] - self.excludeSites = excludeSites - self.exceptionCPEs = exceptionCPEs - if findIPv6usingMikrotik: - from mikrotikFindIPv6 import pullMikrotikIPv6 - self.ipv4ToIPv6 = pullMikrotikIPv6() - else: - self.ipv4ToIPv6 = {} + def __init__(self) -> None: + from ispConfig import findIPv6usingMikrotik, excludeSites, exceptionCPEs + self.nodes = [ + NetworkNode("FakeRoot", type=NodeType.root, + parentId="", displayName="Shaper Root") + ] + self.excludeSites = excludeSites + self.exceptionCPEs = exceptionCPEs + if findIPv6usingMikrotik: + from mikrotikFindIPv6 import pullMikrotikIPv6 + self.ipv4ToIPv6 = pullMikrotikIPv6() + else: + self.ipv4ToIPv6 = {} - def addRawNode(self, node: NetworkNode) -> None: - # Adds a NetworkNode to the graph, unchanged. - # If a site is excluded (via excludedSites in ispConfig) - # it won't be added - if not node.displayName in self.excludeSites: - if node.displayName in self.exceptionCPEs.keys(): - node.parentId = self.exceptionCPEs[node.displayName] - self.nodes.append(node) + def addRawNode(self, node: NetworkNode) -> None: + # Adds a NetworkNode to the graph, unchanged. + # If a site is excluded (via excludedSites in ispConfig) + # it won't be added + if not node.displayName in self.excludeSites: + if node.displayName in self.exceptionCPEs.keys(): + node.parentId = self.exceptionCPEs[node.displayName] + self.nodes.append(node) - def replaceRootNote(self, node: NetworkNode) -> None: - # Replaces the automatically generated root node - # with a new node. Useful when you have a top-level - # node specified (e.g. "uispSite" in the UISP - # integration) - self.nodes[0] = node + def replaceRootNote(self, node: NetworkNode) -> None: + # Replaces the automatically generated root node + # with a new node. Useful when you have a top-level + # node specified (e.g. "uispSite" in the UISP + # integration) + self.nodes[0] = node - def addNodeAsChild(self, parent: str, node: NetworkNode) -> None: - # Searches the existing graph for a named parent, - # adjusts the new node's parentIndex to match the new - # node. The parented node is then inserted. - # - # Exceptions are NOT applied, since we're explicitly - # specifying the parent - we're assuming you really - # meant it. - if node.displayName in self.excludeSites: return - parentIdx = 0 - for (i, node) in enumerate(self.nodes): - if node.id == parent: - parentIdx = i - node.parentIndex = parentIdx - self.nodes.append(node) + def addNodeAsChild(self, parent: str, node: NetworkNode) -> None: + # Searches the existing graph for a named parent, + # adjusts the new node's parentIndex to match the new + # node. The parented node is then inserted. + # + # Exceptions are NOT applied, since we're explicitly + # specifying the parent - we're assuming you really + # meant it. + if node.displayName in self.excludeSites: return + parentIdx = 0 + for (i, node) in enumerate(self.nodes): + if node.id == parent: + parentIdx = i + node.parentIndex = parentIdx + self.nodes.append(node) - def __reparentById(self) -> None: - # Scans the entire node tree, searching for parents - # by name. Entries are re-mapped to match the named - # parents. You can use this to build a tree from a - # blob of raw data. - for child in self.nodes: - if child.parentId != "": - for (i, node) in enumerate(self.nodes): - if node.id == child.parentId: - child.parentIndex = i + def __reparentById(self) -> None: + # Scans the entire node tree, searching for parents + # by name. Entries are re-mapped to match the named + # parents. You can use this to build a tree from a + # blob of raw data. + for child in self.nodes: + if child.parentId != "": + for (i, node) in enumerate(self.nodes): + if node.id == child.parentId: + child.parentIndex = i - def findNodeIndexById(self, id: str) -> int: - # Finds a single node by identity(id) - # Return -1 if not found - for (i, node) in enumerate(self.nodes): - if node.id == id: - return i - return -1 + def findNodeIndexById(self, id: str) -> int: + # Finds a single node by identity(id) + # Return -1 if not found + for (i, node) in enumerate(self.nodes): + if node.id == id: + return i + return -1 - def findNodeIndexByName(self, name: str) -> int: - # Finds a single node by identity(name) - # Return -1 if not found - for (i, node) in enumerate(self.nodes): - if node.displayName == name: - return i - return -1 + def findNodeIndexByName(self, name: str) -> int: + # Finds a single node by identity(name) + # Return -1 if not found + for (i, node) in enumerate(self.nodes): + if node.displayName == name: + return i + return -1 - def findChildIndices(self, parentIndex: int) -> List: - # Returns the indices of all nodes with a - # parentIndex equal to the specified parameter - result = [] - for (i, node) in enumerate(self.nodes): - if node.parentIndex == parentIndex: - result.append(i) - return result + def findChildIndices(self, parentIndex: int) -> List: + # Returns the indices of all nodes with a + # parentIndex equal to the specified parameter + result = [] + for (i, node) in enumerate(self.nodes): + if node.parentIndex == parentIndex: + result.append(i) + return result - def __promoteClientsWithChildren(self) -> None: - # Searches for client sites that have children, - # and changes their node type to clientWithChildren - for (i, node) in enumerate(self.nodes): - if node.type == NodeType.client: - for child in self.findChildIndices(i): - if self.nodes[child].type != NodeType.device: - node.type = NodeType.clientWithChildren + def __promoteClientsWithChildren(self) -> None: + # Searches for client sites that have children, + # and changes their node type to clientWithChildren + for (i, node) in enumerate(self.nodes): + if node.type == NodeType.client: + for child in self.findChildIndices(i): + if self.nodes[child].type != NodeType.device: + node.type = NodeType.clientWithChildren - def __clientsWithChildrenToSites(self) -> None: - toAdd = [] - for (i, node) in enumerate(self.nodes): - if node.type == NodeType.clientWithChildren: - siteNode = NetworkNode( - id=node.id + "_gen", - displayName="(Generated Site) " + node.displayName, - type=NodeType.site - ) - siteNode.parentIndex = node.parentIndex - node.parentId = siteNode.id - if node.type == NodeType.clientWithChildren: - node.type = NodeType.client - for child in self.findChildIndices(i): - if self.nodes[child].type == NodeType.client or self.nodes[child].type == NodeType.clientWithChildren or self.nodes[child].type == NodeType.site: - self.nodes[child].parentId = siteNode.id - toAdd.append(siteNode) + def __clientsWithChildrenToSites(self) -> None: + toAdd = [] + for (i, node) in enumerate(self.nodes): + if node.type == NodeType.clientWithChildren: + siteNode = NetworkNode( + id=node.id + "_gen", + displayName="(Generated Site) " + node.displayName, + type=NodeType.site + ) + siteNode.parentIndex = node.parentIndex + node.parentId = siteNode.id + if node.type == NodeType.clientWithChildren: + node.type = NodeType.client + for child in self.findChildIndices(i): + if self.nodes[child].type == NodeType.client or self.nodes[child].type == NodeType.clientWithChildren or self.nodes[child].type == NodeType.site: + self.nodes[child].parentId = siteNode.id + toAdd.append(siteNode) - for n in toAdd: - self.addRawNode(n) + for n in toAdd: + self.addRawNode(n) - self.__reparentById() + self.__reparentById() - def __findUnconnectedNodes(self) -> List: - # Performs a tree-traversal and finds any nodes that - # aren't connected to the root. This is a "sanity check", - # and also an easy way to handle "flat" topologies and - # ensure that the unconnected nodes are re-connected to - # the root. - visited = [] - next = [0] + def __findUnconnectedNodes(self) -> List: + # Performs a tree-traversal and finds any nodes that + # aren't connected to the root. This is a "sanity check", + # and also an easy way to handle "flat" topologies and + # ensure that the unconnected nodes are re-connected to + # the root. + visited = [] + next = [0] - while len(next) > 0: - nextTraversal = next.pop() - visited.append(nextTraversal) - for idx in self.findChildIndices(nextTraversal): - if idx not in visited: - next.append(idx) + while len(next) > 0: + nextTraversal = next.pop() + visited.append(nextTraversal) + for idx in self.findChildIndices(nextTraversal): + if idx not in visited: + next.append(idx) - result = [] - for i, n in enumerate(self.nodes): - if i not in visited: - result.append(i) - return result + result = [] + for i, n in enumerate(self.nodes): + if i not in visited: + result.append(i) + return result - def __reconnectUnconnected(self): - # Finds any unconnected nodes and reconnects - # them to the root - for idx in self.__findUnconnectedNodes(): - if self.nodes[idx].type == NodeType.site: - self.nodes[idx].parentIndex = 0 - for idx in self.__findUnconnectedNodes(): - if self.nodes[idx].type == NodeType.clientWithChildren: - self.nodes[idx].parentIndex = 0 - for idx in self.__findUnconnectedNodes(): - if self.nodes[idx].type == NodeType.client: - self.nodes[idx].parentIndex = 0 + def __reconnectUnconnected(self): + # Finds any unconnected nodes and reconnects + # them to the root + for idx in self.__findUnconnectedNodes(): + if self.nodes[idx].type == NodeType.site: + self.nodes[idx].parentIndex = 0 + for idx in self.__findUnconnectedNodes(): + if self.nodes[idx].type == NodeType.clientWithChildren: + self.nodes[idx].parentIndex = 0 + for idx in self.__findUnconnectedNodes(): + if self.nodes[idx].type == NodeType.client: + self.nodes[idx].parentIndex = 0 - def prepareTree(self) -> None: - # Helper function that calls all the cleanup and mapping - # functions in the right order. Unless you are doing - # something special, you can use this instead of - # calling the functions individually - self.__reparentById() - self.__promoteClientsWithChildren() - self.__clientsWithChildrenToSites() - self.__reconnectUnconnected() + def prepareTree(self) -> None: + # Helper function that calls all the cleanup and mapping + # functions in the right order. Unless you are doing + # something special, you can use this instead of + # calling the functions individually + self.__reparentById() + self.__promoteClientsWithChildren() + self.__clientsWithChildrenToSites() + self.__reconnectUnconnected() - def doesNetworkJsonExist(self): - # Returns true if "network.json" exists, false otherwise - import os - return os.path.isfile("network.json") + def doesNetworkJsonExist(self): + # Returns true if "network.json" exists, false otherwise + import os + return os.path.isfile("network.json") - def __isSite(self, index) -> bool: - return self.nodes[index].type == NodeType.ap or self.nodes[index].type == NodeType.site or self.nodes[index].type == NodeType.clientWithChildren + def __isSite(self, index) -> bool: + return self.nodes[index].type == NodeType.ap or self.nodes[index].type == NodeType.site or self.nodes[index].type == NodeType.clientWithChildren - def createNetworkJson(self): - import json - topLevelNode = {} - self.__visited = [] # Protection against loops - never visit twice + def createNetworkJson(self): + import json + topLevelNode = {} + self.__visited = [] # Protection against loops - never visit twice - for child in self.findChildIndices(0): - if child > 0 and self.__isSite(child): - topLevelNode[self.nodes[child].displayName] = self.__buildNetworkObject( - child) + for child in self.findChildIndices(0): + if child > 0 and self.__isSite(child): + topLevelNode[self.nodes[child].displayName] = self.__buildNetworkObject( + child) - del self.__visited + del self.__visited - with open('network.json', 'w') as f: - json.dump(topLevelNode, f, indent=4) + with open('network.json', 'w') as f: + json.dump(topLevelNode, f, indent=4) - def __buildNetworkObject(self, idx): - # Private: used to recurse down the network tree while building - # network.json - self.__visited.append(idx) - node = { - "downloadBandwidthMbps": self.nodes[idx].downloadMbps, - "uploadBandwidthMbps": self.nodes[idx].uploadMbps, - } - children = {} - hasChildren = False - for child in self.findChildIndices(idx): - if child > 0 and self.__isSite(child) and child not in self.__visited: - children[self.nodes[child].displayName] = self.__buildNetworkObject( - child) - hasChildren = True - if hasChildren: - node["children"] = children - return node + def __buildNetworkObject(self, idx): + # Private: used to recurse down the network tree while building + # network.json + self.__visited.append(idx) + node = { + "downloadBandwidthMbps": self.nodes[idx].downloadMbps, + "uploadBandwidthMbps": self.nodes[idx].uploadMbps, + } + children = {} + hasChildren = False + for child in self.findChildIndices(idx): + if child > 0 and self.__isSite(child) and child not in self.__visited: + children[self.nodes[child].displayName] = self.__buildNetworkObject( + child) + hasChildren = True + if hasChildren: + node["children"] = children + return node - def __addIpv6FromMap(self, ipv4, ipv6) -> None: - # Scans each address in ipv4. If its present in the - # IPv4 to Ipv6 map (currently pulled from Mikrotik devices - # if findIPv6usingMikrotik is enabled), then matching - # IPv6 networks are appended to the ipv6 list. - # This is explicitly non-destructive of the existing IPv6 - # list, in case you already have some. - for ipCidr in ipv4: - if '/' in ipCidr: ip = ipCidr.split('/')[0] - else: ip = ipCidr - if ip in self.ipv4ToIPv6.keys(): - ipv6.append(self.ipv4ToIPv6[ip]) + def __addIpv6FromMap(self, ipv4, ipv6) -> None: + # Scans each address in ipv4. If its present in the + # IPv4 to Ipv6 map (currently pulled from Mikrotik devices + # if findIPv6usingMikrotik is enabled), then matching + # IPv6 networks are appended to the ipv6 list. + # This is explicitly non-destructive of the existing IPv6 + # list, in case you already have some. + for ipCidr in ipv4: + if '/' in ipCidr: ip = ipCidr.split('/')[0] + else: ip = ipCidr + if ip in self.ipv4ToIPv6.keys(): + ipv6.append(self.ipv4ToIPv6[ip]) - def createShapedDevices(self): - import csv - from ispConfig import bandwidthOverheadFactor - # Builds ShapedDevices.csv from the network tree. - circuits = [] - for (i, node) in enumerate(self.nodes): - if node.type == NodeType.client: - parent = self.nodes[node.parentIndex].displayName - if parent == "Shaper Root": parent = "" - circuit = { - "id": node.id, - "name": node.address, - "parent": parent, - "download": node.downloadMbps, - "upload": node.uploadMbps, - "devices": [] - } - for child in self.findChildIndices(i): - if self.nodes[child].type == NodeType.device and (len(self.nodes[child].ipv4)+len(self.nodes[child].ipv6)>0): - ipv4 = self.nodes[child].ipv4 - ipv6 = self.nodes[child].ipv6 - self.__addIpv6FromMap(ipv4, ipv6) - device = { - "id": self.nodes[child].id, - "name": self.nodes[child].displayName, - "mac": self.nodes[child].mac, - "ipv4": ipv4, - "ipv6": ipv6, - } - circuit["devices"].append(device) - if len(circuit["devices"]) > 0: - circuits.append(circuit) + def createShapedDevices(self): + import csv + from ispConfig import bandwidthOverheadFactor + # Builds ShapedDevices.csv from the network tree. + circuits = [] + for (i, node) in enumerate(self.nodes): + if node.type == NodeType.client: + parent = self.nodes[node.parentIndex].displayName + if parent == "Shaper Root": parent = "" + + if circuitNameUseAddress: + displayNameToUse = node.address + else: + displayNameToUse = node.customerName + circuit = { + "id": node.id, + "name": displayNameToUse, + "parent": parent, + "download": node.downloadMbps, + "upload": node.uploadMbps, + "devices": [] + } + for child in self.findChildIndices(i): + if self.nodes[child].type == NodeType.device and (len(self.nodes[child].ipv4)+len(self.nodes[child].ipv6)>0): + ipv4 = self.nodes[child].ipv4 + ipv6 = self.nodes[child].ipv6 + self.__addIpv6FromMap(ipv4, ipv6) + device = { + "id": self.nodes[child].id, + "name": self.nodes[child].displayName, + "mac": self.nodes[child].mac, + "ipv4": ipv4, + "ipv6": ipv6, + } + circuit["devices"].append(device) + if len(circuit["devices"]) > 0: + circuits.append(circuit) - with open('ShapedDevices.csv', 'w', newline='') as csvfile: - wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL) - wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC', - 'IPv4', 'IPv6', 'Download Min', 'Upload Min', 'Download Max', 'Upload Max', 'Comment']) - for circuit in circuits: - for device in circuit["devices"]: - #Remove brackets and quotes of list so LibreQoS.py can parse it - device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'') - device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'') - row = [ - circuit["id"], - circuit["name"], - device["id"], - device["name"], - circuit["parent"], - device["mac"], - device["ipv4"], - device["ipv6"], - int(circuit["download"] * 0.98), - int(circuit["upload"] * 0.98), - int(circuit["download"] * bandwidthOverheadFactor), - int(circuit["upload"] * bandwidthOverheadFactor), - "" - ] - wr.writerow(row) + with open('ShapedDevices.csv', 'w', newline='') as csvfile: + wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL) + wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC', + 'IPv4', 'IPv6', 'Download Min', 'Upload Min', 'Download Max', 'Upload Max', 'Comment']) + for circuit in circuits: + for device in circuit["devices"]: + #Remove brackets and quotes of list so LibreQoS.py can parse it + device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'') + device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'') + row = [ + circuit["id"], + circuit["name"], + device["id"], + device["name"], + circuit["parent"], + device["mac"], + device["ipv4"], + device["ipv6"], + int(circuit["download"] * 0.98), + int(circuit["upload"] * 0.98), + int(circuit["download"] * bandwidthOverheadFactor), + int(circuit["upload"] * bandwidthOverheadFactor), + "" + ] + wr.writerow(row) - def plotNetworkGraph(self, showClients=False): - # Requires `pip install graphviz` to function. - # You also need to install graphviz on your PC. - # In Ubuntu, apt install graphviz will do it. - # Plots the network graph to a PDF file, allowing - # visual verification that the graph makes sense. - # Could potentially be useful in a future - # web interface. - import importlib.util - if (spec := importlib.util.find_spec('graphviz')) is None: - return + def plotNetworkGraph(self, showClients=False): + # Requires `pip install graphviz` to function. + # You also need to install graphviz on your PC. + # In Ubuntu, apt install graphviz will do it. + # Plots the network graph to a PDF file, allowing + # visual verification that the graph makes sense. + # Could potentially be useful in a future + # web interface. + import importlib.util + if (spec := importlib.util.find_spec('graphviz')) is None: + return - import graphviz - dot = graphviz.Digraph( - 'network', comment="Network Graph", engine="fdp") + import graphviz + dot = graphviz.Digraph( + 'network', comment="Network Graph", engine="fdp") - for (i, node) in enumerate(self.nodes): - if ((node.type != NodeType.client and node.type != NodeType.device) or showClients): - color = "white" - match node.type: - case NodeType.root: color = "green" - case NodeType.site: color = "red" - case NodeType.ap: color = "blue" - case NodeType.clientWithChildren: color = "magenta" - case NodeType.device: color = "white" - case default: color = "grey" - dot.node("N" + str(i), node.displayName, color=color) - children = self.findChildIndices(i) - for child in children: - if child != i: - if (self.nodes[child].type != NodeType.client and self.nodes[child].type != NodeType.device) or showClients: - dot.edge("N" + str(i), "N" + str(child)) + for (i, node) in enumerate(self.nodes): + if ((node.type != NodeType.client and node.type != NodeType.device) or showClients): + color = "white" + match node.type: + case NodeType.root: color = "green" + case NodeType.site: color = "red" + case NodeType.ap: color = "blue" + case NodeType.clientWithChildren: color = "magenta" + case NodeType.device: color = "white" + case default: color = "grey" + dot.node("N" + str(i), node.displayName, color=color) + children = self.findChildIndices(i) + for child in children: + if child != i: + if (self.nodes[child].type != NodeType.client and self.nodes[child].type != NodeType.device) or showClients: + dot.edge("N" + str(i), "N" + str(child)) - dot.render("network.pdf") + dot.render("network.pdf") From 0f9a2b0e2142f7b658e7d33b78c82e0466639de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Fri, 3 Mar 2023 20:34:18 -0700 Subject: [PATCH 3/5] Update integrationSplynx.py --- src/integrationSplynx.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/integrationSplynx.py b/src/integrationSplynx.py index 33fb6af9..d9aac5fb 100644 --- a/src/integrationSplynx.py +++ b/src/integrationSplynx.py @@ -86,6 +86,7 @@ def createShaper(): id=combinedId, displayName=customerJson["name"], address=combineAddress(customerJson), + customerName = customerJson["name"] download=downloadForTariffID[tariff_id], upload=uploadForTariffID[tariff_id], ) From fae48c85a58c418dbba038cb66793133b56a4ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Fri, 3 Mar 2023 20:35:15 -0700 Subject: [PATCH 4/5] Update ispConfig.example.py --- src/ispConfig.example.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ispConfig.example.py b/src/ispConfig.example.py index faa8c87a..cbca50ff 100644 --- a/src/ispConfig.example.py +++ b/src/ispConfig.example.py @@ -60,6 +60,9 @@ influxDBtoken = "" # NMS/CRM Integration +# Use Customer Name or Address as Circuit Name +circuitNameUseAddress = True + # If a device shows a WAN IP within these subnets, assume they are behind NAT / un-shapable, and ignore them ignoreSubnets = ['192.168.0.0/16'] allowedSubnets = ['100.64.0.0/10'] From 2e50a2a108c94542c46775b428b9ae60767b7b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Fri, 3 Mar 2023 20:38:32 -0700 Subject: [PATCH 5/5] Update integrationSplynx.py --- src/integrationSplynx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integrationSplynx.py b/src/integrationSplynx.py index d9aac5fb..73639b0b 100644 --- a/src/integrationSplynx.py +++ b/src/integrationSplynx.py @@ -86,7 +86,7 @@ def createShaper(): id=combinedId, displayName=customerJson["name"], address=combineAddress(customerJson), - customerName = customerJson["name"] + customerName=customerJson["name"], download=downloadForTariffID[tariff_id], upload=uploadForTariffID[tariff_id], )