IntenseWebs/LibreQoS
3
0
Fork 0
mirror of https://github.com/LibreQoE/LibreQoS.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #278 from LibreQoE/toggleDisplayNameOrAddress

Browse Source 
Integrations Improvement - Toggle using display name or address for Circuit Name
This commit is contained in:
Robert Chacón 2023-03-03 20:40:28 -07:00 committed by GitHub
commit 2843061bbd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 559 additions and 543 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

726
src/integrationCommon.py
View File

@ -2,420 +2,426 @@
# integrations. # integrations.
from typing import List, Any from typing import List, Any
from ispConfig import allowedSubnets, ignoreSubnets, generatedPNUploadMbps, generatedPNDownloadMbps from ispConfig import allowedSubnets, ignoreSubnets, generatedPNUploadMbps, generatedPNDownloadMbps, circuitNameUseAddress
import ipaddress import ipaddress
import enum import enum
def isInAllowedSubnets(inputIP): def isInAllowedSubnets(inputIP):
# Check whether an IP address occurs inside the allowedSubnets list # Check whether an IP address occurs inside the allowedSubnets list
isAllowed = False isAllowed = False
if '/' in inputIP: if '/' in inputIP:
inputIP = inputIP.split('/')[0] inputIP = inputIP.split('/')[0]
for subnet in allowedSubnets: for subnet in allowedSubnets:
if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)): if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)):
isAllowed = True isAllowed = True
return isAllowed return isAllowed
def isInIgnoredSubnets(inputIP): def isInIgnoredSubnets(inputIP):
# Check whether an IP address occurs within the ignoreSubnets list # Check whether an IP address occurs within the ignoreSubnets list
isIgnored = False isIgnored = False
if '/' in inputIP: if '/' in inputIP:
inputIP = inputIP.split('/')[0] inputIP = inputIP.split('/')[0]
for subnet in ignoreSubnets: for subnet in ignoreSubnets:
if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)): if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)):
isIgnored = True isIgnored = True
return isIgnored return isIgnored
def isIpv4Permitted(inputIP): def isIpv4Permitted(inputIP):
# Checks whether an IP address is in Allowed Subnets. # Checks whether an IP address is in Allowed Subnets.
# If it is, check that it isn't in Ignored Subnets. # If it is, check that it isn't in Ignored Subnets.
# If it is allowed and not ignored, returns true. # If it is allowed and not ignored, returns true.
# Otherwise, returns false. # Otherwise, returns false.
return isInIgnoredSubnets(inputIP) == False and isInAllowedSubnets(inputIP) return isInIgnoredSubnets(inputIP) == False and isInAllowedSubnets(inputIP)
def fixSubnet(inputIP): def fixSubnet(inputIP):
# If an IP address has a CIDR other than /32 (e.g. 192.168.1.1/24), # If an IP address has a CIDR other than /32 (e.g. 192.168.1.1/24),
# but doesn't appear as a network address (e.g. 192.168.1.0/24) # but doesn't appear as a network address (e.g. 192.168.1.0/24)
# then it probably isn't actually serving that whole subnet. # then it probably isn't actually serving that whole subnet.
# This allows you to specify e.g. 192.168.1.0/24 is "the client # This allows you to specify e.g. 192.168.1.0/24 is "the client
# on port 3" in the device, without falling afoul of UISP's inclusion # on port 3" in the device, without falling afoul of UISP's inclusion
# of subnet masks in device IPs. # of subnet masks in device IPs.
[rawIp, cidr] = inputIP.split('/') [rawIp, cidr] = inputIP.split('/')
if cidr != "32": if cidr != "32":
try: try:
subnet = ipaddress.ip_network(inputIP) subnet = ipaddress.ip_network(inputIP)
except: except:
# Not a network address # Not a network address
return rawIp + "/32" return rawIp + "/32"
return inputIP return inputIP
class NodeType(enum.IntEnum): class NodeType(enum.IntEnum):
# Enumeration to define what type of node # Enumeration to define what type of node
# a NetworkNode is. # a NetworkNode is.
root = 1 root = 1
site = 2 site = 2
ap = 3 ap = 3
client = 4 client = 4
clientWithChildren = 5 clientWithChildren = 5
device = 6 device = 6
class NetworkNode: class NetworkNode:
# Defines a node on a LibreQoS network graph. # Defines a node on a LibreQoS network graph.
# Nodes default to being disconnected, and # Nodes default to being disconnected, and
# will be mapped to the root of the overall # will be mapped to the root of the overall
# graph. # graph.
id: str id: str
displayName: str displayName: str
parentIndex: int parentIndex: int
parentId: str parentId: str
type: NodeType type: NodeType
downloadMbps: int downloadMbps: int
uploadMbps: int uploadMbps: int
ipv4: List ipv4: List
ipv6: List ipv6: List
address: str address: str
mac: str mac: str
def __init__(self, id: str, displayName: str = "", parentId: str = "", type: NodeType = NodeType.site, download: int = generatedPNDownloadMbps, upload: int = generatedPNUploadMbps, ipv4: List = [], ipv6: List = [], address: str = "", mac: str = "") -> None: def __init__(self, id: str, displayName: str = "", parentId: str = "", type: NodeType = NodeType.site, download: int = generatedPNDownloadMbps, upload: int = generatedPNUploadMbps, ipv4: List = [], ipv6: List = [], address: str = "", mac: str = "", customerName: str = "") -> None:
self.id = id self.id = id
self.parentIndex = 0 self.parentIndex = 0
self.type = type self.type = type
self.parentId = parentId self.parentId = parentId
if displayName == "": if displayName == "":
self.displayName = id self.displayName = id
else: else:
self.displayName = displayName self.displayName = displayName
self.downloadMbps = download self.downloadMbps = download
self.uploadMbps = upload self.uploadMbps = upload
self.ipv4 = ipv4 self.ipv4 = ipv4
self.ipv6 = ipv6 self.ipv6 = ipv6
self.address = address self.address = address
self.mac = mac self.customerName = customerName
self.mac = mac
class NetworkGraph: class NetworkGraph:
# Defines a network as a graph topology # Defines a network as a graph topology
# allowing any integration to build the # allowing any integration to build the
# graph via a common API, emitting # graph via a common API, emitting
# ShapedDevices and network.json files # ShapedDevices and network.json files
# via a common interface. # via a common interface.
nodes: List nodes: List
ipv4ToIPv6: Any ipv4ToIPv6: Any
excludeSites: List # Copied to allow easy in-test patching excludeSites: List # Copied to allow easy in-test patching
exceptionCPEs: Any exceptionCPEs: Any
def __init__(self) -> None: def __init__(self) -> None:
from ispConfig import findIPv6usingMikrotik, excludeSites, exceptionCPEs from ispConfig import findIPv6usingMikrotik, excludeSites, exceptionCPEs
self.nodes = [ self.nodes = [
NetworkNode("FakeRoot", type=NodeType.root, NetworkNode("FakeRoot", type=NodeType.root,
parentId="", displayName="Shaper Root") parentId="", displayName="Shaper Root")
] ]
self.excludeSites = excludeSites self.excludeSites = excludeSites
self.exceptionCPEs = exceptionCPEs self.exceptionCPEs = exceptionCPEs
if findIPv6usingMikrotik: if findIPv6usingMikrotik:
from mikrotikFindIPv6 import pullMikrotikIPv6 from mikrotikFindIPv6 import pullMikrotikIPv6
self.ipv4ToIPv6 = pullMikrotikIPv6() self.ipv4ToIPv6 = pullMikrotikIPv6()
else: else:
self.ipv4ToIPv6 = {} self.ipv4ToIPv6 = {}
def addRawNode(self, node: NetworkNode) -> None: def addRawNode(self, node: NetworkNode) -> None:
# Adds a NetworkNode to the graph, unchanged. # Adds a NetworkNode to the graph, unchanged.
# If a site is excluded (via excludedSites in ispConfig) # If a site is excluded (via excludedSites in ispConfig)
# it won't be added # it won't be added
if not node.displayName in self.excludeSites: if not node.displayName in self.excludeSites:
if node.displayName in self.exceptionCPEs.keys(): if node.displayName in self.exceptionCPEs.keys():
node.parentId = self.exceptionCPEs[node.displayName] node.parentId = self.exceptionCPEs[node.displayName]
self.nodes.append(node) self.nodes.append(node)
def replaceRootNote(self, node: NetworkNode) -> None: def replaceRootNote(self, node: NetworkNode) -> None:
# Replaces the automatically generated root node # Replaces the automatically generated root node
# with a new node. Useful when you have a top-level # with a new node. Useful when you have a top-level
# node specified (e.g. "uispSite" in the UISP # node specified (e.g. "uispSite" in the UISP
# integration) # integration)
self.nodes[0] = node self.nodes[0] = node
def addNodeAsChild(self, parent: str, node: NetworkNode) -> None: def addNodeAsChild(self, parent: str, node: NetworkNode) -> None:
# Searches the existing graph for a named parent, # Searches the existing graph for a named parent,
# adjusts the new node's parentIndex to match the new # adjusts the new node's parentIndex to match the new
# node. The parented node is then inserted. # node. The parented node is then inserted.
# #
# Exceptions are NOT applied, since we're explicitly # Exceptions are NOT applied, since we're explicitly
# specifying the parent - we're assuming you really # specifying the parent - we're assuming you really
# meant it. # meant it.
if node.displayName in self.excludeSites: return if node.displayName in self.excludeSites: return
parentIdx = 0 parentIdx = 0
for (i, node) in enumerate(self.nodes): for (i, node) in enumerate(self.nodes):
if node.id == parent: if node.id == parent:
parentIdx = i parentIdx = i
node.parentIndex = parentIdx node.parentIndex = parentIdx
self.nodes.append(node) self.nodes.append(node)
def __reparentById(self) -> None: def __reparentById(self) -> None:
# Scans the entire node tree, searching for parents # Scans the entire node tree, searching for parents
# by name. Entries are re-mapped to match the named # by name. Entries are re-mapped to match the named
# parents. You can use this to build a tree from a # parents. You can use this to build a tree from a
# blob of raw data. # blob of raw data.
for child in self.nodes: for child in self.nodes:
if child.parentId != "": if child.parentId != "":
for (i, node) in enumerate(self.nodes): for (i, node) in enumerate(self.nodes):
if node.id == child.parentId: if node.id == child.parentId:
child.parentIndex = i child.parentIndex = i
def findNodeIndexById(self, id: str) -> int: def findNodeIndexById(self, id: str) -> int:
# Finds a single node by identity(id) # Finds a single node by identity(id)
# Return -1 if not found # Return -1 if not found
for (i, node) in enumerate(self.nodes): for (i, node) in enumerate(self.nodes):
if node.id == id: if node.id == id:
return i return i
return -1 return -1
def findNodeIndexByName(self, name: str) -> int: def findNodeIndexByName(self, name: str) -> int:
# Finds a single node by identity(name) # Finds a single node by identity(name)
# Return -1 if not found # Return -1 if not found
for (i, node) in enumerate(self.nodes): for (i, node) in enumerate(self.nodes):
if node.displayName == name: if node.displayName == name:
return i return i
return -1 return -1
def findChildIndices(self, parentIndex: int) -> List: def findChildIndices(self, parentIndex: int) -> List:
# Returns the indices of all nodes with a # Returns the indices of all nodes with a
# parentIndex equal to the specified parameter # parentIndex equal to the specified parameter
result = [] result = []
for (i, node) in enumerate(self.nodes): for (i, node) in enumerate(self.nodes):
if node.parentIndex == parentIndex: if node.parentIndex == parentIndex:
result.append(i) result.append(i)
return result return result
def __promoteClientsWithChildren(self) -> None: def __promoteClientsWithChildren(self) -> None:
# Searches for client sites that have children, # Searches for client sites that have children,
# and changes their node type to clientWithChildren # and changes their node type to clientWithChildren
for (i, node) in enumerate(self.nodes): for (i, node) in enumerate(self.nodes):
if node.type == NodeType.client: if node.type == NodeType.client:
for child in self.findChildIndices(i): for child in self.findChildIndices(i):
if self.nodes[child].type != NodeType.device: if self.nodes[child].type != NodeType.device:
node.type = NodeType.clientWithChildren node.type = NodeType.clientWithChildren
def __clientsWithChildrenToSites(self) -> None: def __clientsWithChildrenToSites(self) -> None:
toAdd = [] toAdd = []
for (i, node) in enumerate(self.nodes): for (i, node) in enumerate(self.nodes):
if node.type == NodeType.clientWithChildren: if node.type == NodeType.clientWithChildren:
siteNode = NetworkNode( siteNode = NetworkNode(
id=node.id + "_gen", id=node.id + "_gen",
displayName="(Generated Site) " + node.displayName, displayName="(Generated Site) " + node.displayName,
type=NodeType.site type=NodeType.site
) )
siteNode.parentIndex = node.parentIndex siteNode.parentIndex = node.parentIndex
node.parentId = siteNode.id node.parentId = siteNode.id
if node.type == NodeType.clientWithChildren: if node.type == NodeType.clientWithChildren:
node.type = NodeType.client node.type = NodeType.client
for child in self.findChildIndices(i): for child in self.findChildIndices(i):
if self.nodes[child].type == NodeType.client or self.nodes[child].type == NodeType.clientWithChildren or self.nodes[child].type == NodeType.site: if self.nodes[child].type == NodeType.client or self.nodes[child].type == NodeType.clientWithChildren or self.nodes[child].type == NodeType.site:
self.nodes[child].parentId = siteNode.id self.nodes[child].parentId = siteNode.id
toAdd.append(siteNode) toAdd.append(siteNode)
for n in toAdd: for n in toAdd:
self.addRawNode(n) self.addRawNode(n)
self.__reparentById() self.__reparentById()
def __findUnconnectedNodes(self) -> List: def __findUnconnectedNodes(self) -> List:
# Performs a tree-traversal and finds any nodes that # Performs a tree-traversal and finds any nodes that
# aren't connected to the root. This is a "sanity check", # aren't connected to the root. This is a "sanity check",
# and also an easy way to handle "flat" topologies and # and also an easy way to handle "flat" topologies and
# ensure that the unconnected nodes are re-connected to # ensure that the unconnected nodes are re-connected to
# the root. # the root.
visited = [] visited = []
next = [0] next = [0]
while len(next) > 0: while len(next) > 0:
nextTraversal = next.pop() nextTraversal = next.pop()
visited.append(nextTraversal) visited.append(nextTraversal)
for idx in self.findChildIndices(nextTraversal): for idx in self.findChildIndices(nextTraversal):
if idx not in visited: if idx not in visited:
next.append(idx) next.append(idx)
result = [] result = []
for i, n in enumerate(self.nodes): for i, n in enumerate(self.nodes):
if i not in visited: if i not in visited:
result.append(i) result.append(i)
return result return result
def __reconnectUnconnected(self): def __reconnectUnconnected(self):
# Finds any unconnected nodes and reconnects # Finds any unconnected nodes and reconnects
# them to the root # them to the root
for idx in self.__findUnconnectedNodes(): for idx in self.__findUnconnectedNodes():
if self.nodes[idx].type == NodeType.site: if self.nodes[idx].type == NodeType.site:
self.nodes[idx].parentIndex = 0 self.nodes[idx].parentIndex = 0
for idx in self.__findUnconnectedNodes(): for idx in self.__findUnconnectedNodes():
if self.nodes[idx].type == NodeType.clientWithChildren: if self.nodes[idx].type == NodeType.clientWithChildren:
self.nodes[idx].parentIndex = 0 self.nodes[idx].parentIndex = 0
for idx in self.__findUnconnectedNodes(): for idx in self.__findUnconnectedNodes():
if self.nodes[idx].type == NodeType.client: if self.nodes[idx].type == NodeType.client:
self.nodes[idx].parentIndex = 0 self.nodes[idx].parentIndex = 0
def prepareTree(self) -> None: def prepareTree(self) -> None:
# Helper function that calls all the cleanup and mapping # Helper function that calls all the cleanup and mapping
# functions in the right order. Unless you are doing # functions in the right order. Unless you are doing
# something special, you can use this instead of # something special, you can use this instead of
# calling the functions individually # calling the functions individually
self.__reparentById() self.__reparentById()
self.__promoteClientsWithChildren() self.__promoteClientsWithChildren()
self.__clientsWithChildrenToSites() self.__clientsWithChildrenToSites()
self.__reconnectUnconnected() self.__reconnectUnconnected()
def doesNetworkJsonExist(self): def doesNetworkJsonExist(self):
# Returns true if "network.json" exists, false otherwise # Returns true if "network.json" exists, false otherwise
import os import os
return os.path.isfile("network.json") return os.path.isfile("network.json")
def __isSite(self, index) -> bool: def __isSite(self, index) -> bool:
return self.nodes[index].type == NodeType.ap or self.nodes[index].type == NodeType.site or self.nodes[index].type == NodeType.clientWithChildren return self.nodes[index].type == NodeType.ap or self.nodes[index].type == NodeType.site or self.nodes[index].type == NodeType.clientWithChildren
def createNetworkJson(self): def createNetworkJson(self):
import json import json
topLevelNode = {} topLevelNode = {}
self.__visited = [] # Protection against loops - never visit twice self.__visited = [] # Protection against loops - never visit twice
for child in self.findChildIndices(0): for child in self.findChildIndices(0):
if child > 0 and self.__isSite(child): if child > 0 and self.__isSite(child):
topLevelNode[self.nodes[child].displayName] = self.__buildNetworkObject( topLevelNode[self.nodes[child].displayName] = self.__buildNetworkObject(
child) child)
del self.__visited del self.__visited
with open('network.json', 'w') as f: with open('network.json', 'w') as f:
json.dump(topLevelNode, f, indent=4) json.dump(topLevelNode, f, indent=4)
def __buildNetworkObject(self, idx): def __buildNetworkObject(self, idx):
# Private: used to recurse down the network tree while building # Private: used to recurse down the network tree while building
# network.json # network.json
self.__visited.append(idx) self.__visited.append(idx)
node = { node = {
"downloadBandwidthMbps": self.nodes[idx].downloadMbps, "downloadBandwidthMbps": self.nodes[idx].downloadMbps,
"uploadBandwidthMbps": self.nodes[idx].uploadMbps, "uploadBandwidthMbps": self.nodes[idx].uploadMbps,
} }
children = {} children = {}
hasChildren = False hasChildren = False
for child in self.findChildIndices(idx): for child in self.findChildIndices(idx):
if child > 0 and self.__isSite(child) and child not in self.__visited: if child > 0 and self.__isSite(child) and child not in self.__visited:
children[self.nodes[child].displayName] = self.__buildNetworkObject( children[self.nodes[child].displayName] = self.__buildNetworkObject(
child) child)
hasChildren = True hasChildren = True
if hasChildren: if hasChildren:
node["children"] = children node["children"] = children
return node return node
def __addIpv6FromMap(self, ipv4, ipv6) -> None: def __addIpv6FromMap(self, ipv4, ipv6) -> None:
# Scans each address in ipv4. If its present in the # Scans each address in ipv4. If its present in the
# IPv4 to Ipv6 map (currently pulled from Mikrotik devices # IPv4 to Ipv6 map (currently pulled from Mikrotik devices
# if findIPv6usingMikrotik is enabled), then matching # if findIPv6usingMikrotik is enabled), then matching
# IPv6 networks are appended to the ipv6 list. # IPv6 networks are appended to the ipv6 list.
# This is explicitly non-destructive of the existing IPv6 # This is explicitly non-destructive of the existing IPv6
# list, in case you already have some. # list, in case you already have some.
for ipCidr in ipv4: for ipCidr in ipv4:
if '/' in ipCidr: ip = ipCidr.split('/')[0] if '/' in ipCidr: ip = ipCidr.split('/')[0]
else: ip = ipCidr else: ip = ipCidr
if ip in self.ipv4ToIPv6.keys(): if ip in self.ipv4ToIPv6.keys():
ipv6.append(self.ipv4ToIPv6[ip]) ipv6.append(self.ipv4ToIPv6[ip])
def createShapedDevices(self): def createShapedDevices(self):
import csv import csv
from ispConfig import bandwidthOverheadFactor from ispConfig import bandwidthOverheadFactor
# Builds ShapedDevices.csv from the network tree. # Builds ShapedDevices.csv from the network tree.
circuits = [] circuits = []
for (i, node) in enumerate(self.nodes): for (i, node) in enumerate(self.nodes):
if node.type == NodeType.client: if node.type == NodeType.client:
parent = self.nodes[node.parentIndex].displayName parent = self.nodes[node.parentIndex].displayName
if parent == "Shaper Root": parent = "" if parent == "Shaper Root": parent = ""
circuit = {
"id": node.id, if circuitNameUseAddress:
"name": node.address, displayNameToUse = node.address
"parent": parent, else:
"download": node.downloadMbps, displayNameToUse = node.customerName
"upload": node.uploadMbps, circuit = {
"devices": [] "id": node.id,
} "name": displayNameToUse,
for child in self.findChildIndices(i): "parent": parent,
if self.nodes[child].type == NodeType.device and (len(self.nodes[child].ipv4)+len(self.nodes[child].ipv6)>0): "download": node.downloadMbps,
ipv4 = self.nodes[child].ipv4 "upload": node.uploadMbps,
ipv6 = self.nodes[child].ipv6 "devices": []
self.__addIpv6FromMap(ipv4, ipv6) }
device = { for child in self.findChildIndices(i):
"id": self.nodes[child].id, if self.nodes[child].type == NodeType.device and (len(self.nodes[child].ipv4)+len(self.nodes[child].ipv6)>0):
"name": self.nodes[child].displayName, ipv4 = self.nodes[child].ipv4
"mac": self.nodes[child].mac, ipv6 = self.nodes[child].ipv6
"ipv4": ipv4, self.__addIpv6FromMap(ipv4, ipv6)
"ipv6": ipv6, device = {
} "id": self.nodes[child].id,
circuit["devices"].append(device) "name": self.nodes[child].displayName,
if len(circuit["devices"]) > 0: "mac": self.nodes[child].mac,
circuits.append(circuit) "ipv4": ipv4,
"ipv6": ipv6,
}
circuit["devices"].append(device)
if len(circuit["devices"]) > 0:
circuits.append(circuit)
with open('ShapedDevices.csv', 'w', newline='') as csvfile: with open('ShapedDevices.csv', 'w', newline='') as csvfile:
wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL) wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC', wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC',
'IPv4', 'IPv6', 'Download Min', 'Upload Min', 'Download Max', 'Upload Max', 'Comment']) 'IPv4', 'IPv6', 'Download Min', 'Upload Min', 'Download Max', 'Upload Max', 'Comment'])
for circuit in circuits: for circuit in circuits:
for device in circuit["devices"]: for device in circuit["devices"]:
#Remove brackets and quotes of list so LibreQoS.py can parse it #Remove brackets and quotes of list so LibreQoS.py can parse it
device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'') device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'')
device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'') device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'')
row = [ row = [
circuit["id"], circuit["id"],
circuit["name"], circuit["name"],
device["id"], device["id"],
device["name"], device["name"],
circuit["parent"], circuit["parent"],
device["mac"], device["mac"],
device["ipv4"], device["ipv4"],
device["ipv6"], device["ipv6"],
int(circuit["download"] * 0.98), int(circuit["download"] * 0.98),
int(circuit["upload"] * 0.98), int(circuit["upload"] * 0.98),
int(circuit["download"] * bandwidthOverheadFactor), int(circuit["download"] * bandwidthOverheadFactor),
int(circuit["upload"] * bandwidthOverheadFactor), int(circuit["upload"] * bandwidthOverheadFactor),
"" ""
] ]
wr.writerow(row) wr.writerow(row)
def plotNetworkGraph(self, showClients=False): def plotNetworkGraph(self, showClients=False):
# Requires `pip install graphviz` to function. # Requires `pip install graphviz` to function.
# You also need to install graphviz on your PC. # You also need to install graphviz on your PC.
# In Ubuntu, apt install graphviz will do it. # In Ubuntu, apt install graphviz will do it.
# Plots the network graph to a PDF file, allowing # Plots the network graph to a PDF file, allowing
# visual verification that the graph makes sense. # visual verification that the graph makes sense.
# Could potentially be useful in a future # Could potentially be useful in a future
# web interface. # web interface.
import importlib.util import importlib.util
if (spec := importlib.util.find_spec('graphviz')) is None: if (spec := importlib.util.find_spec('graphviz')) is None:
return return
import graphviz import graphviz
dot = graphviz.Digraph( dot = graphviz.Digraph(
'network', comment="Network Graph", engine="fdp") 'network', comment="Network Graph", engine="fdp")
for (i, node) in enumerate(self.nodes): for (i, node) in enumerate(self.nodes):
if ((node.type != NodeType.client and node.type != NodeType.device) or showClients): if ((node.type != NodeType.client and node.type != NodeType.device) or showClients):
color = "white" color = "white"
match node.type: match node.type:
case NodeType.root: color = "green" case NodeType.root: color = "green"
case NodeType.site: color = "red" case NodeType.site: color = "red"
case NodeType.ap: color = "blue" case NodeType.ap: color = "blue"
case NodeType.clientWithChildren: color = "magenta" case NodeType.clientWithChildren: color = "magenta"
case NodeType.device: color = "white" case NodeType.device: color = "white"
case default: color = "grey" case default: color = "grey"
dot.node("N" + str(i), node.displayName, color=color) dot.node("N" + str(i), node.displayName, color=color)
children = self.findChildIndices(i) children = self.findChildIndices(i)
for child in children: for child in children:
if child != i: if child != i:
if (self.nodes[child].type != NodeType.client and self.nodes[child].type != NodeType.device) or showClients: if (self.nodes[child].type != NodeType.client and self.nodes[child].type != NodeType.device) or showClients:
dot.edge("N" + str(i), "N" + str(child)) dot.edge("N" + str(i), "N" + str(child))
dot.render("network.pdf") dot.render("network.pdf")

1
src/integrationSplynx.py
View File

@ -86,6 +86,7 @@ def createShaper():
id=combinedId, id=combinedId,
displayName=customerJson["name"], displayName=customerJson["name"],
address=combineAddress(customerJson), address=combineAddress(customerJson),
customerName=customerJson["name"],
download=downloadForTariffID[tariff_id], download=downloadForTariffID[tariff_id],
upload=uploadForTariffID[tariff_id], upload=uploadForTariffID[tariff_id],
) )

372
src/integrationUISP.py
View File

@ -7,216 +7,222 @@ from ispConfig import uispSite, uispStrategy
from integrationCommon import isIpv4Permitted, fixSubnet from integrationCommon import isIpv4Permitted, fixSubnet
def uispRequest(target): def uispRequest(target):
# Sends an HTTP request to UISP and returns the # Sends an HTTP request to UISP and returns the
# result in JSON. You only need to specify the # result in JSON. You only need to specify the
# tail end of the URL, e.g. "sites" # tail end of the URL, e.g. "sites"
from ispConfig import UISPbaseURL, uispAuthToken from ispConfig import UISPbaseURL, uispAuthToken
url = UISPbaseURL + "/nms/api/v2.1/" + target url = UISPbaseURL + "/nms/api/v2.1/" + target
headers = {'accept': 'application/json', 'x-auth-token': uispAuthToken} headers = {'accept': 'application/json', 'x-auth-token': uispAuthToken}
r = requests.get(url, headers=headers) r = requests.get(url, headers=headers)
return r.json() return r.json()
def buildFlatGraph(): def buildFlatGraph():
# Builds a high-performance (but lacking in site or AP bandwidth control) # Builds a high-performance (but lacking in site or AP bandwidth control)
# network. # network.
from integrationCommon import NetworkGraph, NetworkNode, NodeType from integrationCommon import NetworkGraph, NetworkNode, NodeType
from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps
# Load network sites # Load network sites
print("Loading Data from UISP") print("Loading Data from UISP")
sites = uispRequest("sites") sites = uispRequest("sites")
devices = uispRequest("devices?withInterfaces=true&authorized=true") devices = uispRequest("devices?withInterfaces=true&authorized=true")
# Build a basic network adding every client to the tree # Build a basic network adding every client to the tree
print("Building Flat Topology") print("Building Flat Topology")
net = NetworkGraph() net = NetworkGraph()
for site in sites: for site in sites:
type = site['identification']['type'] type = site['identification']['type']
if type == "endpoint": if type == "endpoint":
id = site['identification']['id'] id = site['identification']['id']
address = site['description']['address'] address = site['description']['address']
name = site['identification']['name'] customerName = ''
type = site['identification']['type'] name = site['identification']['name']
download = generatedPNDownloadMbps type = site['identification']['type']
upload = generatedPNUploadMbps download = generatedPNDownloadMbps
if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']): upload = generatedPNUploadMbps
download = int(round(site['qos']['downloadSpeed']/1000000)) if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']):
upload = int(round(site['qos']['uploadSpeed']/1000000)) download = int(round(site['qos']['downloadSpeed']/1000000))
upload = int(round(site['qos']['uploadSpeed']/1000000))
node = NetworkNode(id=id, displayName=name, type=NodeType.client, download=download, upload=upload, address=address) node = NetworkNode(id=id, displayName=name, type=NodeType.client, download=download, upload=upload, address=address, customerName=customerName)
net.addRawNode(node) net.addRawNode(node)
for device in devices: for device in devices:
if device['identification']['site'] is not None and device['identification']['site']['id'] == id: if device['identification']['site'] is not None and device['identification']['site']['id'] == id:
# The device is at this site, so add it # The device is at this site, so add it
ipv4 = [] ipv4 = []
ipv6 = [] ipv6 = []
for interface in device["interfaces"]: for interface in device["interfaces"]:
for ip in interface["addresses"]: for ip in interface["addresses"]:
ip = ip["cidr"] ip = ip["cidr"]
if isIpv4Permitted(ip): if isIpv4Permitted(ip):
ip = fixSubnet(ip) ip = fixSubnet(ip)
if ip not in ipv4: if ip not in ipv4:
ipv4.append(ip) ipv4.append(ip)
# TODO: Figure out Mikrotik IPv6? # TODO: Figure out Mikrotik IPv6?
mac = device['identification']['mac'] mac = device['identification']['mac']
net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification'] net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification']
['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac)) ['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac))
# Finish up # Finish up
net.prepareTree() net.prepareTree()
net.plotNetworkGraph(False) net.plotNetworkGraph(False)
if net.doesNetworkJsonExist(): if net.doesNetworkJsonExist():
print("network.json already exists. Leaving in-place.") print("network.json already exists. Leaving in-place.")
else: else:
net.createNetworkJson() net.createNetworkJson()
net.createShapedDevices() net.createShapedDevices()
def buildFullGraph(): def buildFullGraph():
# Attempts to build a full network graph, incorporating as much of the UISP # Attempts to build a full network graph, incorporating as much of the UISP
# hierarchy as possible. # hierarchy as possible.
from integrationCommon import NetworkGraph, NetworkNode, NodeType from integrationCommon import NetworkGraph, NetworkNode, NodeType
from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps
# Load network sites # Load network sites
print("Loading Data from UISP") print("Loading Data from UISP")
sites = uispRequest("sites") sites = uispRequest("sites")
devices = uispRequest("devices?withInterfaces=true&authorized=true") devices = uispRequest("devices?withInterfaces=true&authorized=true")
dataLinks = uispRequest("data-links?siteLinksOnly=true") dataLinks = uispRequest("data-links?siteLinksOnly=true")
# Do we already have a integrationUISPbandwidths.csv file? # Do we already have a integrationUISPbandwidths.csv file?
siteBandwidth = {} siteBandwidth = {}
if os.path.isfile("integrationUISPbandwidths.csv"): if os.path.isfile("integrationUISPbandwidths.csv"):
with open('integrationUISPbandwidths.csv') as csv_file: with open('integrationUISPbandwidths.csv') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',') csv_reader = csv.reader(csv_file, delimiter=',')
next(csv_reader) next(csv_reader)
for row in csv_reader: for row in csv_reader:
name, download, upload = row name, download, upload = row
download = int(download) download = int(download)
upload = int(upload) upload = int(upload)
siteBandwidth[name] = {"download": download, "upload": upload} siteBandwidth[name] = {"download": download, "upload": upload}
# Find AP capacities from UISP # Find AP capacities from UISP
for device in devices: for device in devices:
if device['identification']['role'] == "ap": if device['identification']['role'] == "ap":
name = device['identification']['name'] name = device['identification']['name']
if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']: if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']:
download = int(device['overview'] download = int(device['overview']
['downlinkCapacity'] / 1000000) ['downlinkCapacity'] / 1000000)
upload = int(device['overview']['uplinkCapacity'] / 1000000) upload = int(device['overview']['uplinkCapacity'] / 1000000)
siteBandwidth[device['identification']['name']] = { siteBandwidth[device['identification']['name']] = {
"download": download, "upload": upload} "download": download, "upload": upload}
print("Building Topology") print("Building Topology")
net = NetworkGraph() net = NetworkGraph()
# Add all sites and client sites # Add all sites and client sites
for site in sites: for site in sites:
id = site['identification']['id'] id = site['identification']['id']
name = site['identification']['name'] name = site['identification']['name']
type = site['identification']['type'] type = site['identification']['type']
download = generatedPNDownloadMbps download = generatedPNDownloadMbps
upload = generatedPNUploadMbps upload = generatedPNUploadMbps
address = "" address = ""
if site['identification']['parent'] is None: customerName = ""
parent = "" if site['identification']['parent'] is None:
else: parent = ""
parent = site['identification']['parent']['id'] else:
match type: parent = site['identification']['parent']['id']
case "site": match type:
nodeType = NodeType.site case "site":
if name in siteBandwidth: nodeType = NodeType.site
# Use the CSV bandwidth values if name in siteBandwidth:
download = siteBandwidth[name]["download"] # Use the CSV bandwidth values
upload = siteBandwidth[name]["upload"] download = siteBandwidth[name]["download"]
else: upload = siteBandwidth[name]["upload"]
# Add them just in case else:
siteBandwidth[name] = { # Add them just in case
"download": download, "upload": upload} siteBandwidth[name] = {
case default: "download": download, "upload": upload}
nodeType = NodeType.client case default:
address = site['description']['address'] nodeType = NodeType.client
if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']): address = site['description']['address']
download = int(round(site['qos']['downloadSpeed']/1000000)) try:
upload = int(round(site['qos']['uploadSpeed']/1000000)) customerName = site["ucrm"]["client"]["name"]
except:
customerName = ""
if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']):
download = int(round(site['qos']['downloadSpeed']/1000000))
upload = int(round(site['qos']['uploadSpeed']/1000000))
node = NetworkNode(id=id, displayName=name, type=nodeType, node = NetworkNode(id=id, displayName=name, type=nodeType,
parentId=parent, download=download, upload=upload, address=address) parentId=parent, download=download, upload=upload, address=address, customerName=customerName)
# If this is the uispSite node, it becomes the root. Otherwise, add it to the # If this is the uispSite node, it becomes the root. Otherwise, add it to the
# node soup. # node soup.
if name == uispSite: if name == uispSite:
net.replaceRootNote(node) net.replaceRootNote(node)
else: else:
net.addRawNode(node) net.addRawNode(node)
for device in devices: for device in devices:
if device['identification']['site'] is not None and device['identification']['site']['id'] == id: if device['identification']['site'] is not None and device['identification']['site']['id'] == id:
# The device is at this site, so add it # The device is at this site, so add it
ipv4 = [] ipv4 = []
ipv6 = [] ipv6 = []
for interface in device["interfaces"]: for interface in device["interfaces"]:
for ip in interface["addresses"]: for ip in interface["addresses"]:
ip = ip["cidr"] ip = ip["cidr"]
if isIpv4Permitted(ip): if isIpv4Permitted(ip):
ip = fixSubnet(ip) ip = fixSubnet(ip)
if ip not in ipv4: if ip not in ipv4:
ipv4.append(ip) ipv4.append(ip)
# TODO: Figure out Mikrotik IPv6? # TODO: Figure out Mikrotik IPv6?
mac = device['identification']['mac'] mac = device['identification']['mac']
net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification'] net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification']
['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac)) ['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac))
# Now iterate access points, and look for connections to sites # Now iterate access points, and look for connections to sites
for node in net.nodes: for node in net.nodes:
if node.type == NodeType.device: if node.type == NodeType.device:
for dl in dataLinks: for dl in dataLinks:
if dl['from']['device'] is not None and dl['from']['device']['identification']['id'] == node.id: if dl['from']['device'] is not None and dl['from']['device']['identification']['id'] == node.id:
if dl['to']['site'] is not None and dl['from']['site']['identification']['id'] != dl['to']['site']['identification']['id']: if dl['to']['site'] is not None and dl['from']['site']['identification']['id'] != dl['to']['site']['identification']['id']:
target = net.findNodeIndexById( target = net.findNodeIndexById(
dl['to']['site']['identification']['id']) dl['to']['site']['identification']['id'])
if target > -1: if target > -1:
# We found the site # We found the site
if net.nodes[target].type == NodeType.client or net.nodes[target].type == NodeType.clientWithChildren: if net.nodes[target].type == NodeType.client or net.nodes[target].type == NodeType.clientWithChildren:
net.nodes[target].parentId = node.id net.nodes[target].parentId = node.id
node.type = NodeType.ap node.type = NodeType.ap
if node.displayName in siteBandwidth: if node.displayName in siteBandwidth:
# Use the bandwidth numbers from the CSV file # Use the bandwidth numbers from the CSV file
node.uploadMbps = siteBandwidth[node.displayName]["upload"] node.uploadMbps = siteBandwidth[node.displayName]["upload"]
node.downloadMbps = siteBandwidth[node.displayName]["download"] node.downloadMbps = siteBandwidth[node.displayName]["download"]
else: else:
# Add some defaults in case they want to change them # Add some defaults in case they want to change them
siteBandwidth[node.displayName] = { siteBandwidth[node.displayName] = {
"download": generatedPNDownloadMbps, "upload": generatedPNUploadMbps} "download": generatedPNDownloadMbps, "upload": generatedPNUploadMbps}
net.prepareTree() net.prepareTree()
net.plotNetworkGraph(False) net.plotNetworkGraph(False)
if net.doesNetworkJsonExist(): if net.doesNetworkJsonExist():
print("network.json already exists. Leaving in-place.") print("network.json already exists. Leaving in-place.")
else: else:
net.createNetworkJson() net.createNetworkJson()
net.createShapedDevices() net.createShapedDevices()
# Save integrationUISPbandwidths.csv # Save integrationUISPbandwidths.csv
# (the newLine fixes generating extra blank lines) # (the newLine fixes generating extra blank lines)
with open('integrationUISPbandwidths.csv', 'w', newline='') as csvfile: with open('integrationUISPbandwidths.csv', 'w', newline='') as csvfile:
wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL) wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
wr.writerow(['ParentNode', 'Download Mbps', 'Upload Mbps']) wr.writerow(['ParentNode', 'Download Mbps', 'Upload Mbps'])
for device in siteBandwidth: for device in siteBandwidth:
entry = ( entry = (
device, siteBandwidth[device]["download"], siteBandwidth[device]["upload"]) device, siteBandwidth[device]["download"], siteBandwidth[device]["upload"])
wr.writerow(entry) wr.writerow(entry)
def importFromUISP(): def importFromUISP():
match uispStrategy: match uispStrategy:
case "full": buildFullGraph() case "full": buildFullGraph()
case default: buildFlatGraph() case default: buildFlatGraph()
if __name__ == '__main__': if __name__ == '__main__':
importFromUISP() importFromUISP()

3
src/ispConfig.example.py
View File

@ -60,6 +60,9 @@ influxDBtoken = ""
# NMS/CRM Integration # NMS/CRM Integration
# Use Customer Name or Address as Circuit Name
circuitNameUseAddress = True
# If a device shows a WAN IP within these subnets, assume they are behind NAT / un-shapable, and ignore them # If a device shows a WAN IP within these subnets, assume they are behind NAT / un-shapable, and ignore them
ignoreSubnets = ['192.168.0.0/16'] ignoreSubnets = ['192.168.0.0/16']
allowedSubnets = ['100.64.0.0/10'] allowedSubnets = ['100.64.0.0/10']