mirror of
https://github.com/LibreQoE/LibreQoS.git
synced 2024-11-22 08:16:25 -06:00
Merge pull request #335 from LibreQoE/fix334
UISP Integration Improvements
This commit is contained in:
Robert Chacón
GitHub
commit
24b4609fd5
58
src/csvToNetworkJSON.py
Normal file
58
src/csvToNetworkJSON.py
Normal file
@ -0,0 +1,58 @@
|
||||
from pythonCheck import checkPythonVersion
|
||||
checkPythonVersion()
|
||||
import os
|
||||
import csv
|
||||
import json
|
||||
from ispConfig import uispSite, uispStrategy, overwriteNetworkJSONalways
|
||||
from ispConfig import generatedPNUploadMbps, generatedPNDownloadMbps, upstreamBandwidthCapacityDownloadMbps, upstreamBandwidthCapacityUploadMbps
|
||||
from integrationCommon import NetworkGraph, NetworkNode, NodeType
|
||||
|
||||
def csvToNetworkJSONfile():
|
||||
sites = []
|
||||
with open('manualNetwork.csv') as csv_file:
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
for row in csv_reader:
|
||||
if 'Site Name' in row[0]:
|
||||
header = row
|
||||
else:
|
||||
name, down, up, parent = row
|
||||
site = {'name': name,
|
||||
'download': down,
|
||||
'upload': up,
|
||||
'parent': parent}
|
||||
sites.append(site)
|
||||
|
||||
net = NetworkGraph()
|
||||
idCounter = 1000
|
||||
nameToID = {}
|
||||
for site in sites:
|
||||
site['id'] = idCounter
|
||||
idCounter = idCounter + 1
|
||||
nameToID[site['name']] = site['id']
|
||||
for site in sites:
|
||||
id = site['id']
|
||||
if site['parent'] == '':
|
||||
parentID = None
|
||||
else:
|
||||
parentID = nameToID[site['parent']]
|
||||
name = site['name']
|
||||
parent = site['parent']
|
||||
download = site['download']
|
||||
upload = site['upload']
|
||||
nodeType = NodeType.site
|
||||
node = NetworkNode(id=id, displayName=name, type=nodeType,
|
||||
parentId=parentID, download=download, upload=upload, address=None, customerName=None)
|
||||
net.addRawNode(node)
|
||||
net.prepareTree()
|
||||
net.plotNetworkGraph(False)
|
||||
if net.doesNetworkJsonExist():
|
||||
if overwriteNetworkJSONalways:
|
||||
net.createNetworkJson()
|
||||
else:
|
||||
print("network.json already exists and overwriteNetworkJSONalways set to False. Leaving in-place.")
|
||||
else:
|
||||
net.createNetworkJson()
|
||||
net.createShapedDevices()
|
||||
|
||||
if __name__ == '__main__':
|
||||
csvToNetworkJSONfile()
|
||||
@ -131,11 +131,23 @@ def findApCapacities(devices, siteBandwidth):
|
||||
if device['identification']['role'] == "ap":
|
||||
name = device['identification']['name']
|
||||
if not name in siteBandwidth and device['overview']['downlinkCapacity'] and device['overview']['uplinkCapacity']:
|
||||
safeToUse = True
|
||||
download = int(device['overview']
|
||||
['downlinkCapacity'] / 1000000)
|
||||
upload = int(device['overview']['uplinkCapacity'] / 1000000)
|
||||
siteBandwidth[device['identification']['name']] = {
|
||||
"download": download, "upload": upload}
|
||||
if download < 15:
|
||||
print("WARNING: Device '" + device['identification']['hostname'] + "' has unusually low download capacity (" + str(upload) + " Mbps). Discarding in favor of parent site rates.")
|
||||
safeToUse = False
|
||||
if upload < 15:
|
||||
print("WARNING: Device '" + device['identification']['hostname'] + "' has unusually low upload capacity (" + str(download) + " Mbps). Discarding in favor of parent site rates.")
|
||||
safeToUse = False
|
||||
if device['identification']['model'] == 'WaveAP':
|
||||
if (download < 500) or (upload < 500):
|
||||
download = 2450
|
||||
upload = 2450
|
||||
if safeToUse:
|
||||
siteBandwidth[device['identification']['name']] = {
|
||||
"download": download, "upload": upload}
|
||||
|
||||
def findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps):
|
||||
foundAirFibersBySite = {}
|
||||
@ -147,19 +159,17 @@ def findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps):
|
||||
if device['overview']['downlinkCapacity'] is not None and device['overview']['uplinkCapacity'] is not None:
|
||||
download = int(device['overview']['downlinkCapacity']/ 1000000)
|
||||
upload = int(device['overview']['uplinkCapacity']/ 1000000)
|
||||
else:
|
||||
download = generatedPNDownloadMbps
|
||||
upload = generatedPNUploadMbps
|
||||
# Make sure to use half of reported bandwidth for AF60-LRs
|
||||
if device['identification']['model'] == "AF60-LR":
|
||||
download = int(download / 2)
|
||||
upload = int(download / 2)
|
||||
if device['identification']['site']['id'] in foundAirFibersBySite:
|
||||
if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']):
|
||||
foundAirFibersBySite[device['identification']['site']['id']]['download'] = download
|
||||
foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
|
||||
else:
|
||||
foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload}
|
||||
# Make sure to use half of reported bandwidth for AF60/AF60-LRs
|
||||
if (device['identification']['model'] == "AF60-LR") or (device['identification']['model'] == "AF60"):
|
||||
download = int(download / 2)
|
||||
upload = int(download / 2)
|
||||
if device['identification']['site']['id'] in foundAirFibersBySite:
|
||||
if (download > foundAirFibersBySite[device['identification']['site']['id']]['download']) or (upload > foundAirFibersBySite[device['identification']['site']['id']]['upload']):
|
||||
foundAirFibersBySite[device['identification']['site']['id']]['download'] = download
|
||||
foundAirFibersBySite[device['identification']['site']['id']]['upload'] = upload
|
||||
#print(device['identification']['name'] + ' will override bandwidth for site ' + device['identification']['site']['name'])
|
||||
else:
|
||||
foundAirFibersBySite[device['identification']['site']['id']] = {'download': download, 'upload': upload}
|
||||
return foundAirFibersBySite
|
||||
|
||||
def buildSiteList(sites, dataLinks):
|
||||
@ -240,6 +250,39 @@ def loadRoutingOverrides():
|
||||
#print(overrides)
|
||||
return overrides
|
||||
|
||||
def findNodesBranchedOffPtMP(siteList, dataLinks, sites, rootSite):
|
||||
nodeOffPtMP = {}
|
||||
for site in sites:
|
||||
id = site['identification']['id']
|
||||
name = site['identification']['name']
|
||||
type = site['identification']['type']
|
||||
if id != rootSite['id']:
|
||||
trueParent = findInSiteListById(siteList, id)['parent']
|
||||
if type == 'site':
|
||||
#parent = findInSiteListById(siteList, id)['parent']
|
||||
if ('parent' in site['identification']) and (site['identification']['parent'] is not None):
|
||||
parent = site['identification']['parent']['id']
|
||||
for link in dataLinks:
|
||||
if (link['to']['site'] is not None) and (link['to']['site']['identification'] is not None):
|
||||
if ('identification' in link['to']['site']) and (link['to']['site']['identification'] is not None):
|
||||
if ('identification' in link['from']['site']) and (link['from']['site']['identification'] is not None):
|
||||
# Respect parent defined by topology and overrides
|
||||
if link['from']['site']['identification']['id'] == trueParent:
|
||||
if link['to']['site']['identification']['id'] == id:
|
||||
if link['from']['device']['overview']['wirelessMode'] == 'ap-ptmp':
|
||||
if 'overview' in link['to']['device']:
|
||||
if ('downlinkCapacity' in link['to']['device']['overview']) and ('uplinkCapacity' in link['to']['device']['overview']):
|
||||
if (link['to']['device']['overview']['downlinkCapacity'] is not None) and (link['to']['device']['overview']['uplinkCapacity'] is not None):
|
||||
# Capacity of the PtMP client radio feeding the PoP will be used as the site bandwidth limit
|
||||
download = int(round(link['to']['device']['overview']['downlinkCapacity']/1000000))
|
||||
upload = int(round(link['to']['device']['overview']['uplinkCapacity']/1000000))
|
||||
nodeOffPtMP[id] = { 'parent': link['from']['device']['identification']['id'],
|
||||
'parentName': link['from']['device']['identification']['name'],
|
||||
'download': download,
|
||||
'upload': upload
|
||||
}
|
||||
return nodeOffPtMP
|
||||
|
||||
def buildFullGraph():
|
||||
# Attempts to build a full network graph, incorporating as much of the UISP
|
||||
# hierarchy as possible.
|
||||
@ -253,18 +296,28 @@ def buildFullGraph():
|
||||
dataLinks = uispRequest("data-links?siteLinksOnly=true")
|
||||
|
||||
# Build Site Capacities
|
||||
print("Compiling Site Bandwidths")
|
||||
siteBandwidth = buildSiteBandwidths()
|
||||
print("Finding AP Capacities")
|
||||
findApCapacities(devices, siteBandwidth)
|
||||
foundAirFibersBySite = findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps)
|
||||
|
||||
# Create a list of just network sites
|
||||
print('Creating list of sites')
|
||||
siteList = buildSiteList(sites, dataLinks)
|
||||
rootSite = findInSiteList(siteList, uispSite)
|
||||
print("Finding PtP Capacities")
|
||||
foundAirFibersBySite = findAirfibers(devices, generatedPNDownloadMbps, generatedPNUploadMbps)
|
||||
|
||||
print('Creating list of route overrides')
|
||||
routeOverrides = loadRoutingOverrides()
|
||||
if rootSite is None:
|
||||
print("ERROR: Unable to find root site in UISP")
|
||||
return
|
||||
print('Creating graph')
|
||||
walkGraphOutwards(siteList, rootSite, routeOverrides)
|
||||
|
||||
print("Finding PtMP Capacities")
|
||||
nodeOffPtMP = findNodesBranchedOffPtMP(siteList, dataLinks, sites, rootSite)
|
||||
|
||||
# Debug code: dump the list of site parents
|
||||
# for s in siteList:
|
||||
# if s['parent'] == "":
|
||||
@ -273,25 +326,8 @@ def buildFullGraph():
|
||||
# p = findInSiteListById(siteList, s['parent'])['name']
|
||||
# print(s['name'] + " (" + str(s['cost']) + ") <-- " + p)
|
||||
|
||||
# Find Nodes Connected By PtMP
|
||||
nodeOffPtMP = {}
|
||||
for site in sites:
|
||||
id = site['identification']['id']
|
||||
name = site['identification']['name']
|
||||
type = site['identification']['type']
|
||||
parent = findInSiteListById(siteList, id)['parent']
|
||||
if type == 'site':
|
||||
for link in dataLinks:
|
||||
if link['from']['site'] is not None and link['from']['site']['identification']['id'] == parent:
|
||||
if link['to']['site'] is not None and link['to']['site']['identification']['id'] == id:
|
||||
if link['from']['device'] is not None and link['to']['device'] is not None and link['to']['device']['overview'] is not None and link['to']['device']['overview']['downlinkCapacity'] is not None and link['from']['device']['overview']['wirelessMode'] == 'ap-ptmp':
|
||||
# Capacity of the PtMP client radio feeding the PoP will be used as the site bandwidth limit
|
||||
download = int(round(link['to']['device']['overview']['downlinkCapacity']/1000000))
|
||||
upload = int(round(link['to']['device']['overview']['uplinkCapacity']/1000000))
|
||||
nodeOffPtMP[id] = { 'parent': link['from']['device']['identification']['id'],
|
||||
'download': download,
|
||||
'upload': upload
|
||||
}
|
||||
|
||||
|
||||
print("Building Topology")
|
||||
net = NetworkGraph()
|
||||
# Add all sites and client sites
|
||||
@ -318,15 +354,19 @@ def buildFullGraph():
|
||||
# Use the CSV bandwidth values
|
||||
download = siteBandwidth[name]["download"]
|
||||
upload = siteBandwidth[name]["upload"]
|
||||
elif id in nodeOffPtMP:
|
||||
download = nodeOffPtMP[id]['download']
|
||||
upload = nodeOffPtMP[id]['upload']
|
||||
elif id in foundAirFibersBySite:
|
||||
download = foundAirFibersBySite[id]['download']
|
||||
upload = foundAirFibersBySite[id]['upload']
|
||||
else:
|
||||
# Add them just in case
|
||||
siteBandwidth[name] = {
|
||||
else:
|
||||
# Use limits from foundAirFibersBySite
|
||||
if id in foundAirFibersBySite:
|
||||
download = foundAirFibersBySite[id]['download']
|
||||
upload = foundAirFibersBySite[id]['upload']
|
||||
#print('Site ' + name + ' will use bandwidth from foundAirFibersBySite.')
|
||||
# Use limits from nodeOffPtMP only if they're greater than what is already set
|
||||
if id in nodeOffPtMP:
|
||||
if (nodeOffPtMP[id]['download'] >= download) or (nodeOffPtMP[id]['upload'] >= upload):
|
||||
download = nodeOffPtMP[id]['download']
|
||||
upload = nodeOffPtMP[id]['upload']
|
||||
#print('Site ' + name + ' will use bandwidth from nodeOffPtMP.')
|
||||
siteBandwidth[name] = {
|
||||
"download": download, "upload": upload}
|
||||
case default:
|
||||
nodeType = NodeType.client
|
||||
@ -364,7 +404,7 @@ def buildFullGraph():
|
||||
|
||||
# TODO: Figure out Mikrotik IPv6?
|
||||
mac = device['identification']['mac']
|
||||
|
||||
|
||||
net.addRawNode(NetworkNode(id=device['identification']['id'], displayName=device['identification']
|
||||
['name'], parentId=id, type=NodeType.device, ipv4=ipv4, ipv6=ipv6, mac=mac))
|
||||
|
||||
|
||||
4
src/manualNetwork.template.csv
Normal file
4
src/manualNetwork.template.csv
Normal file
@ -0,0 +1,4 @@
|
||||
Site Name,Download Mbps,Upload Mbps,Parent Site Name
|
||||
SiteA,1000,1000,
|
||||
SiteB,975,975,SiteA
|
||||
SiteC,950,950,SiteB
|
||||
|
Loading…
Reference in New Issue
Block a user