IntenseWebs/LibreQoS
3
0
Fork 0
mirror of https://github.com/LibreQoE/LibreQoS.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #278 from LibreQoE/toggleDisplayNameOrAddress

Browse Source 
Integrations Improvement - Toggle using display name or address for Circuit Name
This commit is contained in:
Robert Chacón 2023-03-03 20:40:28 -07:00 committed by GitHub
commit 2843061bbd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 559 additions and 543 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

726
src/integrationCommon.py
View File

@ -2,420 +2,426 @@
# integrations.
from typing import List, Any
from ispConfig import allowedSubnets, ignoreSubnets, generatedPNUploadMbps, generatedPNDownloadMbps
from ispConfig import allowedSubnets, ignoreSubnets, generatedPNUploadMbps, generatedPNDownloadMbps, circuitNameUseAddress
import ipaddress
import enum
def isInAllowedSubnets(inputIP):
# Check whether an IP address occurs inside the allowedSubnets list
isAllowed = False
if '/' in inputIP:
inputIP = inputIP.split('/')[0]
for subnet in allowedSubnets:
if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)):
isAllowed = True
return isAllowed
# Check whether an IP address occurs inside the allowedSubnets list
isAllowed = False
if '/' in inputIP:
inputIP = inputIP.split('/')[0]
for subnet in allowedSubnets:
if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)):
isAllowed = True
return isAllowed
def isInIgnoredSubnets(inputIP):
# Check whether an IP address occurs within the ignoreSubnets list
isIgnored = False
if '/' in inputIP:
inputIP = inputIP.split('/')[0]
for subnet in ignoreSubnets:
if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)):
isIgnored = True
return isIgnored
# Check whether an IP address occurs within the ignoreSubnets list
isIgnored = False
if '/' in inputIP:
inputIP = inputIP.split('/')[0]
for subnet in ignoreSubnets:
if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)):
isIgnored = True
return isIgnored
def isIpv4Permitted(inputIP):
# Checks whether an IP address is in Allowed Subnets.
# If it is, check that it isn't in Ignored Subnets.
# If it is allowed and not ignored, returns true.
# Otherwise, returns false.
return isInIgnoredSubnets(inputIP) == False and isInAllowedSubnets(inputIP)
# Checks whether an IP address is in Allowed Subnets.
# If it is, check that it isn't in Ignored Subnets.
# If it is allowed and not ignored, returns true.
# Otherwise, returns false.
return isInIgnoredSubnets(inputIP) == False and isInAllowedSubnets(inputIP)
def fixSubnet(inputIP):
# If an IP address has a CIDR other than /32 (e.g. 192.168.1.1/24),
# but doesn't appear as a network address (e.g. 192.168.1.0/24)
# then it probably isn't actually serving that whole subnet.
# This allows you to specify e.g. 192.168.1.0/24 is "the client
# on port 3" in the device, without falling afoul of UISP's inclusion
# of subnet masks in device IPs.
[rawIp, cidr] = inputIP.split('/')
if cidr != "32":
try:
subnet = ipaddress.ip_network(inputIP)
except:
# Not a network address
return rawIp + "/32"
return inputIP
# If an IP address has a CIDR other than /32 (e.g. 192.168.1.1/24),
# but doesn't appear as a network address (e.g. 192.168.1.0/24)
# then it probably isn't actually serving that whole subnet.
# This allows you to specify e.g. 192.168.1.0/24 is "the client
# on port 3" in the device, without falling afoul of UISP's inclusion
# of subnet masks in device IPs.
[rawIp, cidr] = inputIP.split('/')
if cidr != "32":
try:
subnet = ipaddress.ip_network(inputIP)
except:
# Not a network address
return rawIp + "/32"
return inputIP
class NodeType(enum.IntEnum):
# Enumeration to define what type of node
# a NetworkNode is.
root = 1
site = 2
ap = 3
client = 4
clientWithChildren = 5
device = 6
# Enumeration to define what type of node
# a NetworkNode is.
root = 1
site = 2
ap = 3
client = 4
clientWithChildren = 5
device = 6
class NetworkNode:
# Defines a node on a LibreQoS network graph.
# Nodes default to being disconnected, and
# will be mapped to the root of the overall
# graph.
# Defines a node on a LibreQoS network graph.
# Nodes default to being disconnected, and
# will be mapped to the root of the overall
# graph.
id: str
displayName: str
parentIndex: int
parentId: str
type: NodeType
downloadMbps: int
uploadMbps: int
ipv4: List
ipv6: List
address: str
mac: str
id: str
displayName: str
parentIndex: int
parentId: str
type: NodeType
downloadMbps: int
uploadMbps: int
ipv4: List
ipv6: List
address: str
mac: str
def __init__(self, id: str, displayName: str = "", parentId: str = "", type: NodeType = NodeType.site, download: int = generatedPNDownloadMbps, upload: int = generatedPNUploadMbps, ipv4: List = [], ipv6: List = [], address: str = "", mac: str = "") -> None:
self.id = id
self.parentIndex = 0
self.type = type
self.parentId = parentId
if displayName == "":
self.displayName = id
else:
self.displayName = displayName
self.downloadMbps = download
self.uploadMbps = upload
self.ipv4 = ipv4
self.ipv6 = ipv6
self.address = address
self.mac = mac
def __init__(self, id: str, displayName: str = "", parentId: str = "", type: NodeType = NodeType.site, download: int = generatedPNDownloadMbps, upload: int = generatedPNUploadMbps, ipv4: List = [], ipv6: List = [], address: str = "", mac: str = "", customerName: str = "") -> None:
self.id = id
self.parentIndex = 0
self.type = type
self.parentId = parentId
if displayName == "":
self.displayName = id
else:
self.displayName = displayName
self.downloadMbps = download
self.uploadMbps = upload
self.ipv4 = ipv4
self.ipv6 = ipv6
self.address = address
self.customerName = customerName
self.mac = mac
class NetworkGraph:
# Defines a network as a graph topology
# allowing any integration to build the
# graph via a common API, emitting
# ShapedDevices and network.json files
# via a common interface.
# Defines a network as a graph topology
# allowing any integration to build the
# graph via a common API, emitting
# ShapedDevices and network.json files
# via a common interface.
nodes: List
ipv4ToIPv6: Any
excludeSites: List # Copied to allow easy in-test patching
exceptionCPEs: Any
nodes: List
ipv4ToIPv6: Any
excludeSites: List # Copied to allow easy in-test patching
exceptionCPEs: Any
def __init__(self) -> None:
from ispConfig import findIPv6usingMikrotik, excludeSites, exceptionCPEs
self.nodes = [
NetworkNode("FakeRoot", type=NodeType.root,
parentId="", displayName="Shaper Root")
]
self.excludeSites = excludeSites
self.exceptionCPEs = exceptionCPEs
if findIPv6usingMikrotik:
from mikrotikFindIPv6 import pullMikrotikIPv6
self.ipv4ToIPv6 = pullMikrotikIPv6()
else:
self.ipv4ToIPv6 = {}
def __init__(self) -> None:
from ispConfig import findIPv6usingMikrotik, excludeSites, exceptionCPEs
self.nodes = [
NetworkNode("FakeRoot", type=NodeType.root,
parentId="", displayName="Shaper Root")
]
self.excludeSites = excludeSites
self.exceptionCPEs = exceptionCPEs
if findIPv6usingMikrotik:
from mikrotikFindIPv6 import pullMikrotikIPv6
self.ipv4ToIPv6 = pullMikrotikIPv6()
else:
self.ipv4ToIPv6 = {}
def addRawNode(self, node: NetworkNode) -> None:
# Adds a NetworkNode to the graph, unchanged.
# If a site is excluded (via excludedSites in ispConfig)
# it won't be added
if not node.displayName in self.excludeSites:
if node.displayName in self.exceptionCPEs.keys():
node.parentId = self.exceptionCPEs[node.displayName]
self.nodes.append(node)
def addRawNode(self, node: NetworkNode) -> None:
# Adds a NetworkNode to the graph, unchanged.
# If a site is excluded (via excludedSites in ispConfig)
# it won't be added
if not node.displayName in self.excludeSites:
if node.displayName in self.exceptionCPEs.keys():
node.parentId = self.exceptionCPEs[node.displayName]
self.nodes.append(node)
def replaceRootNote(self, node: NetworkNode) -> None:
# Replaces the automatically generated root node
# with a new node. Useful when you have a top-level
# node specified (e.g. "uispSite" in the UISP
# integration)
self.nodes[0] = node
def replaceRootNote(self, node: NetworkNode) -> None:
# Replaces the automatically generated root node
# with a new node. Useful when you have a top-level
# node specified (e.g. "uispSite" in the UISP
# integration)
self.nodes[0] = node
def addNodeAsChild(self, parent: str, node: NetworkNode) -> None:
# Searches the existing graph for a named parent,
# adjusts the new node's parentIndex to match the new
# node. The parented node is then inserted.
#
# Exceptions are NOT applied, since we're explicitly
# specifying the parent - we're assuming you really
# meant it.
if node.displayName in self.excludeSites: return
parentIdx = 0
for (i, node) in enumerate(self.nodes):
if node.id == parent:
parentIdx = i
node.parentIndex = parentIdx
self.nodes.append(node)
def addNodeAsChild(self, parent: str, node: NetworkNode) -> None:
# Searches the existing graph for a named parent,
# adjusts the new node's parentIndex to match the new
# node. The parented node is then inserted.
#
# Exceptions are NOT applied, since we're explicitly
# specifying the parent - we're assuming you really
# meant it.
if node.displayName in self.excludeSites: return
parentIdx = 0
for (i, node) in enumerate(self.nodes):
if node.id == parent:
parentIdx = i
node.parentIndex = parentIdx
self.nodes.append(node)
def __reparentById(self) -> None:
# Scans the entire node tree, searching for parents
# by name. Entries are re-mapped to match the named
# parents. You can use this to build a tree from a
# blob of raw data.
for child in self.nodes:
if child.parentId != "":
for (i, node) in enumerate(self.nodes):
if node.id == child.parentId:
child.parentIndex = i
def __reparentById(self) -> None:
# Scans the entire node tree, searching for parents
# by name. Entries are re-mapped to match the named
# parents. You can use this to build a tree from a
# blob of raw data.
for child in self.nodes:
if child.parentId != "":
for (i, node) in enumerate(self.nodes):
if node.id == child.parentId:
child.parentIndex = i
def findNodeIndexById(self, id: str) -> int:
# Finds a single node by identity(id)
# Return -1 if not found
for (i, node) in enumerate(self.nodes):
if node.id == id:
return i
return -1
def findNodeIndexById(self, id: str) -> int:
# Finds a single node by identity(id)
# Return -1 if not found
for (i, node) in enumerate(self.nodes):
if node.id == id:
return i
return -1
def findNodeIndexByName(self, name: str) -> int:
# Finds a single node by identity(name)
# Return -1 if not found
for (i, node) in enumerate(self.nodes):
if node.displayName == name:
return i
return -1
def findNodeIndexByName(self, name: str) -> int:
# Finds a single node by identity(name)
# Return -1 if not found
for (i, node) in enumerate(self.nodes):
if node.displayName == name:
return i
return -1
def findChildIndices(self, parentIndex: int) -> List:
# Returns the indices of all nodes with a
# parentIndex equal to the specified parameter
result = []
for (i, node) in enumerate(self.nodes):
if node.parentIndex == parentIndex:
result.append(i)
return result
def findChildIndices(self, parentIndex: int) -> List:
# Returns the indices of all nodes with a
# parentIndex equal to the specified parameter
result = []
for (i, node) in enumerate(self.nodes):
if node.parentIndex == parentIndex:
result.append(i)
return result
def __promoteClientsWithChildren(self) -> None:
# Searches for client sites that have children,
# and changes their node type to clientWithChildren
for (i, node) in enumerate(self.nodes):
if node.type == NodeType.client:
for child in self.findChildIndices(i):
if self.nodes[child].type != NodeType.device:
node.type = NodeType.clientWithChildren
def __promoteClientsWithChildren(self) -> None:
# Searches for client sites that have children,
# and changes their node type to clientWithChildren
for (i, node) in enumerate(self.nodes):
if node.type == NodeType.client:
for child in self.findChildIndices(i):
if self.nodes[child].type != NodeType.device:
node.type = NodeType.clientWithChildren
def __clientsWithChildrenToSites(self) -> None:
toAdd = []
for (i, node) in enumerate(self.nodes):
if node.type == NodeType.clientWithChildren:
siteNode = NetworkNode(
id=node.id + "_gen",
displayName="(Generated Site) " + node.displayName,
type=NodeType.site
)
siteNode.parentIndex = node.parentIndex
node.parentId = siteNode.id
if node.type == NodeType.clientWithChildren:
node.type = NodeType.client
for child in self.findChildIndices(i):
if self.nodes[child].type == NodeType.client or self.nodes[child].type == NodeType.clientWithChildren or self.nodes[child].type == NodeType.site:
self.nodes[child].parentId = siteNode.id
toAdd.append(siteNode)
def __clientsWithChildrenToSites(self) -> None:
toAdd = []
for (i, node) in enumerate(self.nodes):
if node.type == NodeType.clientWithChildren:
siteNode = NetworkNode(
id=node.id + "_gen",
displayName="(Generated Site) " + node.displayName,
type=NodeType.site
)
siteNode.parentIndex = node.parentIndex
node.parentId = siteNode.id
if node.type == NodeType.clientWithChildren:
node.type = NodeType.client
for child in self.findChildIndices(i):
if self.nodes[child].type == NodeType.client or self.nodes[child].type == NodeType.clientWithChildren or self.nodes[child].type == NodeType.site:
self.nodes[child].parentId = siteNode.id
toAdd.append(siteNode)
for n in toAdd:
self.addRawNode(n)
for n in toAdd:
self.addRawNode(n)
self.__reparentById()
self.__reparentById()
def __findUnconnectedNodes(self) -> List:
# Performs a tree-traversal and finds any nodes that
# aren't connected to the root. This is a "sanity check",
# and also an easy way to handle "flat" topologies and
# ensure that the unconnected nodes are re-connected to
# the root.
visited = []
next = [0]
def __findUnconnectedNodes(self) -> List:
# Performs a tree-traversal and finds any nodes that
# aren't connected to the root. This is a "sanity check",
# and also an easy way to handle "flat" topologies and
# ensure that the unconnected nodes are re-connected to
# the root.
visited = []
next = [0]
while len(next) > 0:
nextTraversal = next.pop()
visited.append(nextTraversal)
for idx in self.findChildIndices(nextTraversal):
if idx not in visited:
next.append(idx)
while len(next) > 0:
nextTraversal = next.pop()
visited.append(nextTraversal)
for idx in self.findChildIndices(nextTraversal):
if idx not in visited:
next.append(idx)
result = []
for i, n in enumerate(self.nodes):
if i not in visited:
result.append(i)
return result
result = []
for i, n in enumerate(self.nodes):
if i not in visited:
result.append(i)
return result
def __reconnectUnconnected(self):
# Finds any unconnected nodes and reconnects
# them to the root
for idx in self.__findUnconnectedNodes():
if self.nodes[idx].type == NodeType.site:
self.nodes[idx].parentIndex = 0
for idx in self.__findUnconnectedNodes():
if self.nodes[idx].type == NodeType.clientWithChildren:
self.nodes[idx].parentIndex = 0
for idx in self.__findUnconnectedNodes():
if self.nodes[idx].type == NodeType.client:
self.nodes[idx].parentIndex = 0
def __reconnectUnconnected(self):
# Finds any unconnected nodes and reconnects
# them to the root
for idx in self.__findUnconnectedNodes():
if self.nodes[idx].type == NodeType.site:
self.nodes[idx].parentIndex = 0
for idx in self.__findUnconnectedNodes():
if self.nodes[idx].type == NodeType.clientWithChildren:
self.nodes[idx].parentIndex = 0
for idx in self.__findUnconnectedNodes():
if self.nodes[idx].type == NodeType.client:
self.nodes[idx].parentIndex = 0
def prepareTree(self) -> None:
# Helper function that calls all the cleanup and mapping
# functions in the right order. Unless you are doing
# something special, you can use this instead of
# calling the functions individually
self.__reparentById()
self.__promoteClientsWithChildren()
self.__clientsWithChildrenToSites()
self.__reconnectUnconnected()
def prepareTree(self) -> None:
# Helper function that calls all the cleanup and mapping
# functions in the right order. Unless you are doing
# something special, you can use this instead of
# calling the functions individually
self.__reparentById()
self.__promoteClientsWithChildren()
self.__clientsWithChildrenToSites()
self.__reconnectUnconnected()
def doesNetworkJsonExist(self):
# Returns true if "network.json" exists, false otherwise
import os
return os.path.isfile("network.json")
def doesNetworkJsonExist(self):
# Returns true if "network.json" exists, false otherwise
import os
return os.path.isfile("network.json")
def __isSite(self, index) -> bool:
return self.nodes[index].type == NodeType.ap or self.nodes[index].type == NodeType.site or self.nodes[index].type == NodeType.clientWithChildren
def __isSite(self, index) -> bool:
return self.nodes[index].type == NodeType.ap or self.nodes[index].type == NodeType.site or self.nodes[index].type == NodeType.clientWithChildren
def createNetworkJson(self):
import json
topLevelNode = {}
self.__visited = [] # Protection against loops - never visit twice
def createNetworkJson(self):
import json
topLevelNode = {}
self.__visited = [] # Protection against loops - never visit twice
for child in self.findChildIndices(0):
if child > 0 and self.__isSite(child):
topLevelNode[self.nodes[child].displayName] = self.__buildNetworkObject(
child)
for child in self.findChildIndices(0):
if child > 0 and self.__isSite(child):
topLevelNode[self.nodes[child].displayName] = self.__buildNetworkObject(
child)
del self.__visited
del self.__visited
with open('network.json', 'w') as f:
json.dump(topLevelNode, f, indent=4)
with open('network.json', 'w') as f:
json.dump(topLevelNode, f, indent=4)
def __buildNetworkObject(self, idx):
# Private: used to recurse down the network tree while building
# network.json
self.__visited.append(idx)
node = {
"downloadBandwidthMbps": self.nodes[idx].downloadMbps,
"uploadBandwidthMbps": self.nodes[idx].uploadMbps,
}
children = {}
hasChildren = False
for child in self.findChildIndices(idx):
if child > 0 and self.__isSite(child) and child not in self.__visited:
children[self.nodes[child].displayName] = self.__buildNetworkObject(
child)
hasChildren = True
if hasChildren:
node["children"] = children
return node
def __buildNetworkObject(self, idx):
# Private: used to recurse down the network tree while building
# network.json
self.__visited.append(idx)
node = {
"downloadBandwidthMbps": self.nodes[idx].downloadMbps,
"uploadBandwidthMbps": self.nodes[idx].uploadMbps,
}
children = {}
hasChildren = False
for child in self.findChildIndices(idx):
if child > 0 and self.__isSite(child) and child not in self.__visited:
children[self.nodes[child].displayName] = self.__buildNetworkObject(
child)
hasChildren = True
if hasChildren:
node["children"] = children
return node
def __addIpv6FromMap(self, ipv4, ipv6) -> None:
# Scans each address in ipv4. If its present in the
# IPv4 to Ipv6 map (currently pulled from Mikrotik devices
# if findIPv6usingMikrotik is enabled), then matching
# IPv6 networks are appended to the ipv6 list.
# This is explicitly non-destructive of the existing IPv6
# list, in case you already have some.
for ipCidr in ipv4:
if '/' in ipCidr: ip = ipCidr.split('/')[0]
else: ip = ipCidr
if ip in self.ipv4ToIPv6.keys():
ipv6.append(self.ipv4ToIPv6[ip])
def __addIpv6FromMap(self, ipv4, ipv6) -> None:
# Scans each address in ipv4. If its present in the
# IPv4 to Ipv6 map (currently pulled from Mikrotik devices
# if findIPv6usingMikrotik is enabled), then matching
# IPv6 networks are appended to the ipv6 list.
# This is explicitly non-destructive of the existing IPv6
# list, in case you already have some.
for ipCidr in ipv4:
if '/' in ipCidr: ip = ipCidr.split('/')[0]
else: ip = ipCidr
if ip in self.ipv4ToIPv6.keys():
ipv6.append(self.ipv4ToIPv6[ip])
def createShapedDevices(self):
import csv
from ispConfig import bandwidthOverheadFactor
# Builds ShapedDevices.csv from the network tree.
circuits = []
for (i, node) in enumerate(self.nodes):
if node.type == NodeType.client:
parent = self.nodes[node.parentIndex].displayName
if parent == "Shaper Root": parent = ""
circuit = {
"id": node.id,
"name": node.address,
"parent": parent,
"download": node.downloadMbps,
"upload": node.uploadMbps,
"devices": []
}
for child in self.findChildIndices(i):
if self.nodes[child].type == NodeType.device and (len(self.nodes[child].ipv4)+len(self.nodes[child].ipv6)>0):
ipv4 = self.nodes[child].ipv4
ipv6 = self.nodes[child].ipv6
self.__addIpv6FromMap(ipv4, ipv6)
device = {
"id": self.nodes[child].id,
"name": self.nodes[child].displayName,
"mac": self.nodes[child].mac,
"ipv4": ipv4,
"ipv6": ipv6,
}
circuit["devices"].append(device)
if len(circuit["devices"]) > 0:
circuits.append(circuit)
def createShapedDevices(self):
import csv
from ispConfig import bandwidthOverheadFactor
# Builds ShapedDevices.csv from the network tree.
circuits = []
for (i, node) in enumerate(self.nodes):
if node.type == NodeType.client:
parent = self.nodes[node.parentIndex].displayName
if parent == "Shaper Root": parent = ""
if circuitNameUseAddress:
displayNameToUse = node.address
else:
displayNameToUse = node.customerName
circuit = {
"id": node.id,
"name": displayNameToUse,
"parent": parent,
"download": node.downloadMbps,
"upload": node.uploadMbps,
"devices": []
}
for child in self.findChildIndices(i):
if self.nodes[child].type == NodeType.device and (len(self.nodes[child].ipv4)+len(self.nodes[child].ipv6)>0):
ipv4 = self.nodes[child].ipv4
ipv6 = self.nodes[child].ipv6
self.__addIpv6FromMap(ipv4, ipv6)
device = {
"id": self.nodes[child].id,
"name": self.nodes[child].displayName,
"mac": self.nodes[child].mac,
"ipv4": ipv4,
"ipv6": ipv6,
}
circuit["devices"].append(device)
if len(circuit["devices"]) > 0:
circuits.append(circuit)
with open('ShapedDevices.csv', 'w', newline='') as csvfile:
wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC',
'IPv4', 'IPv6', 'Download Min', 'Upload Min', 'Download Max', 'Upload Max', 'Comment'])
for circuit in circuits:
for device in circuit["devices"]:
#Remove brackets and quotes of list so LibreQoS.py can parse it
device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'')
device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'')
row = [
circuit["id"],
circuit["name"],
device["id"],
device["name"],
circuit["parent"],
device["mac"],
device["ipv4"],
device["ipv6"],
int(circuit["download"] * 0.98),
int(circuit["upload"] * 0.98),
int(circuit["download"] * bandwidthOverheadFactor),
int(circuit["upload"] * bandwidthOverheadFactor),
""
]
wr.writerow(row)
with open('ShapedDevices.csv', 'w', newline='') as csvfile:
wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC',
'IPv4', 'IPv6', 'Download Min', 'Upload Min', 'Download Max', 'Upload Max', 'Comment'])
for circuit in circuits:
for device in circuit["devices"]:
#Remove brackets and quotes of list so LibreQoS.py can parse it
device["ipv4"] = str(device["ipv4"]).replace('[','').replace(']','').replace("'",'')
device["ipv6"] = str(device["ipv6"]).replace('[','').replace(']','').replace("'",'')
row = [
circuit["id"],
circuit["name"],
device["id"],
device["name"],
circuit["parent"],
device["mac"],
device["ipv4"],
device["ipv6"],
int(circuit["download"] * 0.98),
int(circuit["upload"] * 0.98),
int(circuit["download"] * bandwidthOverheadFactor),
int(circuit["upload"] * bandwidthOverheadFactor),
""
]
wr.writerow(row)
def plotNetworkGraph(self, showClients=False):
# Requires `pip install graphviz` to function.
# You also need to install graphviz on your PC.
# In Ubuntu, apt install graphviz will do it.
# Plots the network graph to a PDF file, allowing
# visual verification that the graph makes sense.
# Could potentially be useful in a future
# web interface.
import importlib.util
if (spec := importlib.util.find_spec('graphviz')) is None:
return
def plotNetworkGraph(self, showClients=False):
# Requires `pip install graphviz` to function.
# You also need to install graphviz on your PC.
# In Ubuntu, apt install graphviz will do it.
# Plots the network graph to a PDF file, allowing
# visual verification that the graph makes sense.
# Could potentially be useful in a future
# web interface.
import importlib.util
if (spec := importlib.util.find_spec('graphviz')) is None:
return
import graphviz
dot = graphviz.Digraph(
'network', comment="Network Graph", engine="fdp")
import graphviz
dot = graphviz.Digraph(
'network', comment="Network Graph", engine="fdp")
for (i, node) in enumerate(self.nodes):
if ((node.type != NodeType.client and node.type != NodeType.device) or showClients):
color = "white"
match node.type:
case NodeType.root: color = "green"
case NodeType.site: color = "red"
case NodeType.ap: color = "blue"
case NodeType.clientWithChildren: color = "magenta"
case NodeType.device: color = "white"
case default: color = "grey"
dot.node("N" + str(i), node.displayName, color=color)
children = self.findChildIndices(i)
for child in children:
if child != i:
if (self.nodes[child].type != NodeType.client and self.nodes[child].type != NodeType.device) or showClients:
dot.edge("N" + str(i), "N" + str(child))
for (i, node) in enumerate(self.nodes):
if ((node.type != NodeType.client and node.type != NodeType.device) or showClients):
color = "white"
match node.type:
case NodeType.root: color = "green"
case NodeType.site: color = "red"
case NodeType.ap: color = "blue"
case NodeType.clientWithChildren: color = "magenta"
case NodeType.device: color = "white"
case default: color = "grey"
dot.node("N" + str(i), node.displayName, color=color)
children = self.findChildIndices(i)
for child in children:
if child != i:
if (self.nodes[child].type != NodeType.client and self.nodes[child].type != NodeType.device) or showClients:
dot.edge("N" + str(i), "N" + str(child))
dot.render("network.pdf")
dot.render("network.pdf")

1
src/integrationSplynx.py
View File

@ -86,6 +86,7 @@ def createShaper():
id=combinedId,
displayName=customerJson["name"],
address=combineAddress(customerJson),
customerName=customerJson["name"],
download=downloadForTariffID[tariff_id],
upload=uploadForTariffID[tariff_id],
)

372
src/integrationUISP.py
View File

@ -7,216 +7,222 @@ from ispConfig import uispSite, uispStrategy
from integrationCommon import isIpv4Permitted, fixSubnet
def uispRequest(target):
# Sends an HTTP request to UISP and returns the
# result in JSON. You only need to specify the
# tail end of the URL, e.g. "sites"
from ispConfig import UISPbaseURL, uispAuthToken
url = UISPbaseURL + "/nms/api/v2.1/" + target
headers = {'accept': 'application/json', 'x-auth-token': uispAuthToken}
r = requests.get(url, headers=headers)
return r.json()
# Sends an HTTP request to UISP and returns the
# result in JSON. You only need to specify the
# tail end of the URL, e.g. "sites"
from ispConfig import UISPbaseURL, uispAuthToken
url = UISPbaseURL + "/nms/api/v2.1/" + target
headers = {'accept': 'application/json', 'x-auth-token': uispAuthToken}
r = requests.get(url, headers=headers)
return r.json()
def buildFlatGraph():
# Builds a high-performance (but lacking in site or AP bandwidth control)
# network.
from integrationCommon import NetworkGraph, NetworkNode, NodeType
from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps
# Builds a high-performance (but lacking in site or AP bandwidth control)
# network.
from integrationCommon import NetworkGraph, NetworkNode, NodeType
from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps
# Load network sites
print("Loading Data from UISP")
sites = uispRequest("sites")
devices = uispRequest("devices?withInterfaces=true&authorized=true")
# Load network sites
print("Loading Data from UISP")
sites = uispRequest("sites")
devices = uispRequest("devices?withInterfaces=true&authorized=true")
# Build a basic network adding every client to the tree
print("Building Flat Topology")
net = NetworkGraph()
# Build a basic network adding every client to the tree
print("Building Flat Topology")
net = NetworkGraph()
for site in sites:
type = site['identification']['type']
if type == "endpoint":
id = site['identification']['id']
address = site['description']['address']
name = site['identification']['name']
type = site['identification']['type']
download = generatedPNDownloadMbps
upload = generatedPNUploadMbps
if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']):
download = int(round(site['qos']['downloadSpeed']/1000000))
upload = int(round(site['qos']['uploadSpeed']/1000000))
for site in sites:
type = site['identification']['type']
if type == "endpoint":
id = site['identification']['id']
address = site['description']['address']
customerName = ''
name = site['identification']['name']
type = site['identification']['type']
download = generatedPNDownloadMbps
upload = generatedPNUploadMbps
if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']):
download = int(round(site['qos']['downloadSpeed']/1000000))
upload = int(round(site['qos']['uploadSpeed']/1000000))
node = NetworkNode(id=id, displayName=name, type=NodeType.client, download=download, upload=upload, address=address)
net.addRawNode(node)
for device in devices:
if device['identification']['site'] is not None and device['identification']['site']['id'] == id:
# The device is at this site, so add it
ipv4 = []
ipv6 = []
node = NetworkNode(id=id, displayName=name, type=NodeType.client, download=download, upload=upload, address=address, customerName=customerName)
net.addRawNode(node)
for device in devices:
if device['identification']['site'] is not None and device['identification']['site']['id'] == id:
# The device is at this site, so add it
ipv4 = []
ipv6 = []
for interface in device["interfaces"]:
for ip in interface["addresses"]:
ip = ip["cidr"]
if isIpv4Permitted(ip):
ip = fixSubnet(ip)
if ip not in ipv4:
ipv4.append(ip)
for interface in device["interfaces"]:
for ip in interface["addresses"]:
ip = ip["cidr"]
if isIpv4Permitted(ip):
ip = fixSubnet(ip)
if ip not in ipv4:
ipv4.append(ip)
# TODO: Figure out Mikrotik IPv6?
mac = device['identification']['mac']
# TODO: Figure out Mikrotik IPv6?
mac = device['identification']['mac']
net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification']
['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac))
net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification']
['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac))
# Finish up
net.prepareTree()
net.plotNetworkGraph(False)
if net.doesNetworkJsonExist():
print("network.json already exists. Leaving in-place.")
else:
net.createNetworkJson()
net.createShapedDevices()
# Finish up
net.prepareTree()
net.plotNetworkGraph(False)
if net.doesNetworkJsonExist():
print("network.json already exists. Leaving in-place.")
else:
net.createNetworkJson()
net.createShapedDevices()
def buildFullGraph():
# Attempts to build a full network graph, incorporating as much of the UISP
# hierarchy as possible.
from integrationCommon import NetworkGraph, NetworkNode, NodeType
from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps
# Attempts to build a full network graph, incorporating as much of the UISP
# hierarchy as possible.
from integrationCommon import NetworkGraph, NetworkNode, NodeType
from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps
# Load network sites
print("Loading Data from UISP")
sites = uispRequest("sites")
devices = uispRequest("devices?withInterfaces=true&authorized=true")
dataLinks = uispRequest("data-links?siteLinksOnly=true")
# Load network sites
print("Loading Data from UISP")
sites = uispRequest("sites")
devices = uispRequest("devices?withInterfaces=true&authorized=true")
dataLinks = uispRequest("data-links?siteLinksOnly=true")
# Do we already have a integrationUISPbandwidths.csv file?
siteBandwidth = {}
if os.path.isfile("integrationUISPbandwidths.csv"):
with open('integrationUISPbandwidths.csv') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
next(csv_reader)
for row in csv_reader:
name, download, upload = row
download = int(download)
upload = int(upload)
siteBandwidth[name] = {"download": download, "upload": upload}
# Do we already have a integrationUISPbandwidths.csv file?
siteBandwidth = {}
if os.path.isfile("integrationUISPbandwidths.csv"):
with open('integrationUISPbandwidths.csv') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
next(csv_reader)
for row in csv_reader:
name, download, upload = row
download = int(download)
upload = int(upload)
siteBandwidth[name] = {"download": download, "upload": upload}
# Find AP capacities from UISP
for device in devices:
if device['identification']['role'] == "ap":
name = device['identification']['name']
if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']:
download = int(device['overview']
['downlinkCapacity'] / 1000000)
upload = int(device['overview']['uplinkCapacity'] / 1000000)
siteBandwidth[device['identification']['name']] = {
"download": download, "upload": upload}
# Find AP capacities from UISP
for device in devices:
if device['identification']['role'] == "ap":
name = device['identification']['name']
if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']:
download = int(device['overview']
['downlinkCapacity'] / 1000000)
upload = int(device['overview']['uplinkCapacity'] / 1000000)
siteBandwidth[device['identification']['name']] = {
"download": download, "upload": upload}
print("Building Topology")
net = NetworkGraph()
# Add all sites and client sites
for site in sites:
id = site['identification']['id']
name = site['identification']['name']
type = site['identification']['type']
download = generatedPNDownloadMbps
upload = generatedPNUploadMbps
address = ""
if site['identification']['parent'] is None:
parent = ""
else:
parent = site['identification']['parent']['id']
match type:
case "site":
nodeType = NodeType.site
if name in siteBandwidth:
# Use the CSV bandwidth values
download = siteBandwidth[name]["download"]
upload = siteBandwidth[name]["upload"]
else:
# Add them just in case
siteBandwidth[name] = {
"download": download, "upload": upload}
case default:
nodeType = NodeType.client
address = site['description']['address']
if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']):
download = int(round(site['qos']['downloadSpeed']/1000000))
upload = int(round(site['qos']['uploadSpeed']/1000000))
print("Building Topology")
net = NetworkGraph()
# Add all sites and client sites
for site in sites:
id = site['identification']['id']
name = site['identification']['name']
type = site['identification']['type']
download = generatedPNDownloadMbps
upload = generatedPNUploadMbps
address = ""
customerName = ""
if site['identification']['parent'] is None:
parent = ""
else:
parent = site['identification']['parent']['id']
match type:
case "site":
nodeType = NodeType.site
if name in siteBandwidth:
# Use the CSV bandwidth values
download = siteBandwidth[name]["download"]
upload = siteBandwidth[name]["upload"]
else:
# Add them just in case
siteBandwidth[name] = {
"download": download, "upload": upload}
case default:
nodeType = NodeType.client
address = site['description']['address']
try:
customerName = site["ucrm"]["client"]["name"]
except:
customerName = ""
if (site['qos']['downloadSpeed']) and (site['qos']['uploadSpeed']):
download = int(round(site['qos']['downloadSpeed']/1000000))
upload = int(round(site['qos']['uploadSpeed']/1000000))
node = NetworkNode(id=id, displayName=name, type=nodeType,
parentId=parent, download=download, upload=upload, address=address)
# If this is the uispSite node, it becomes the root. Otherwise, add it to the
# node soup.
if name == uispSite:
net.replaceRootNote(node)
else:
net.addRawNode(node)
node = NetworkNode(id=id, displayName=name, type=nodeType,
parentId=parent, download=download, upload=upload, address=address, customerName=customerName)
# If this is the uispSite node, it becomes the root. Otherwise, add it to the
# node soup.
if name == uispSite:
net.replaceRootNote(node)
else:
net.addRawNode(node)
for device in devices:
if device['identification']['site'] is not None and device['identification']['site']['id'] == id:
# The device is at this site, so add it
ipv4 = []
ipv6 = []
for device in devices:
if device['identification']['site'] is not None and device['identification']['site']['id'] == id:
# The device is at this site, so add it
ipv4 = []
ipv6 = []
for interface in device["interfaces"]:
for ip in interface["addresses"]:
ip = ip["cidr"]
if isIpv4Permitted(ip):
ip = fixSubnet(ip)
if ip not in ipv4:
ipv4.append(ip)
for interface in device["interfaces"]:
for ip in interface["addresses"]:
ip = ip["cidr"]
if isIpv4Permitted(ip):
ip = fixSubnet(ip)
if ip not in ipv4:
ipv4.append(ip)
# TODO: Figure out Mikrotik IPv6?
mac = device['identification']['mac']
# TODO: Figure out Mikrotik IPv6?
mac = device['identification']['mac']
net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification']
['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac))
net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification']
['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac))
# Now iterate access points, and look for connections to sites
for node in net.nodes:
if node.type == NodeType.device:
for dl in dataLinks:
if dl['from']['device'] is not None and dl['from']['device']['identification']['id'] == node.id:
if dl['to']['site'] is not None and dl['from']['site']['identification']['id'] != dl['to']['site']['identification']['id']:
target = net.findNodeIndexById(
dl['to']['site']['identification']['id'])
if target > -1:
# We found the site
if net.nodes[target].type == NodeType.client or net.nodes[target].type == NodeType.clientWithChildren:
net.nodes[target].parentId = node.id
node.type = NodeType.ap
if node.displayName in siteBandwidth:
# Use the bandwidth numbers from the CSV file
node.uploadMbps = siteBandwidth[node.displayName]["upload"]
node.downloadMbps = siteBandwidth[node.displayName]["download"]
else:
# Add some defaults in case they want to change them
siteBandwidth[node.displayName] = {
"download": generatedPNDownloadMbps, "upload": generatedPNUploadMbps}
# Now iterate access points, and look for connections to sites
for node in net.nodes:
if node.type == NodeType.device:
for dl in dataLinks:
if dl['from']['device'] is not None and dl['from']['device']['identification']['id'] == node.id:
if dl['to']['site'] is not None and dl['from']['site']['identification']['id'] != dl['to']['site']['identification']['id']:
target = net.findNodeIndexById(
dl['to']['site']['identification']['id'])
if target > -1:
# We found the site
if net.nodes[target].type == NodeType.client or net.nodes[target].type == NodeType.clientWithChildren:
net.nodes[target].parentId = node.id
node.type = NodeType.ap
if node.displayName in siteBandwidth:
# Use the bandwidth numbers from the CSV file
node.uploadMbps = siteBandwidth[node.displayName]["upload"]
node.downloadMbps = siteBandwidth[node.displayName]["download"]
else:
# Add some defaults in case they want to change them
siteBandwidth[node.displayName] = {
"download": generatedPNDownloadMbps, "upload": generatedPNUploadMbps}
net.prepareTree()
net.plotNetworkGraph(False)
if net.doesNetworkJsonExist():
print("network.json already exists. Leaving in-place.")
else:
net.createNetworkJson()
net.createShapedDevices()
net.prepareTree()
net.plotNetworkGraph(False)
if net.doesNetworkJsonExist():
print("network.json already exists. Leaving in-place.")
else:
net.createNetworkJson()
net.createShapedDevices()
# Save integrationUISPbandwidths.csv
# (the newLine fixes generating extra blank lines)
with open('integrationUISPbandwidths.csv', 'w', newline='') as csvfile:
wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
wr.writerow(['ParentNode', 'Download Mbps', 'Upload Mbps'])
for device in siteBandwidth:
entry = (
device, siteBandwidth[device]["download"], siteBandwidth[device]["upload"])
wr.writerow(entry)
# Save integrationUISPbandwidths.csv
# (the newLine fixes generating extra blank lines)
with open('integrationUISPbandwidths.csv', 'w', newline='') as csvfile:
wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
wr.writerow(['ParentNode', 'Download Mbps', 'Upload Mbps'])
for device in siteBandwidth:
entry = (
device, siteBandwidth[device]["download"], siteBandwidth[device]["upload"])
wr.writerow(entry)
def importFromUISP():
match uispStrategy:
case "full": buildFullGraph()
case default: buildFlatGraph()
match uispStrategy:
case "full": buildFullGraph()
case default: buildFlatGraph()
if __name__ == '__main__':
importFromUISP()
importFromUISP()

3
src/ispConfig.example.py
View File

@ -60,6 +60,9 @@ influxDBtoken = ""
# NMS/CRM Integration
# Use Customer Name or Address as Circuit Name
circuitNameUseAddress = True
# If a device shows a WAN IP within these subnets, assume they are behind NAT / un-shapable, and ignore them
ignoreSubnets = ['192.168.0.0/16']
allowedSubnets = ['100.64.0.0/10']