mirror of
https://github.com/LibreQoE/LibreQoS.git
synced 2024-11-29 11:23:49 -06:00
266 lines
11 KiB
Python
266 lines
11 KiB
Python
import requests
|
|
import os
|
|
import csv
|
|
import ipaddress
|
|
from ispConfig import UISPbaseURL, uispAuthToken, shapeRouterOrStation, allowedSubnets, ignoreSubnets, excludeSites, findIPv6usingMikrotik, bandwidthOverheadFactor, exceptionCPEs
|
|
import shutil
|
|
import json
|
|
if findIPv6usingMikrotik == True:
|
|
from mikrotikFindIPv6 import pullMikrotikIPv6
|
|
|
|
knownRouterModels = ['ACB-AC', 'ACB-ISP']
|
|
knownAPmodels = ['LTU-Rocket', 'RP-5AC', 'RP-5AC-Gen2', 'LAP-GPS', 'Wave-AP']
|
|
|
|
def isInAllowedSubnets(inputIP):
|
|
isAllowed = False
|
|
if '/' in inputIP:
|
|
inputIP = inputIP.split('/')[0]
|
|
for subnet in allowedSubnets:
|
|
if (ipaddress.ip_address(inputIP) in ipaddress.ip_network(subnet)):
|
|
isAllowed = True
|
|
return isAllowed
|
|
|
|
def createTree(sites,accessPoints,bandwidthDL,bandwidthUL,siteParentDict,siteIDtoName,sitesWithParents,currentNode):
|
|
currentNodeName = list(currentNode.items())[0][0]
|
|
childrenList = []
|
|
for site in sites:
|
|
try:
|
|
thisOnesParent = siteIDtoName[site['identification']['parent']['id']]
|
|
if thisOnesParent == currentNodeName:
|
|
childrenList.append(site['id'])
|
|
except:
|
|
thisOnesParent = None
|
|
aps = []
|
|
for ap in accessPoints:
|
|
if ap['device']['site'] is None:
|
|
print("Unable to read site information for: " + ap['device']['name'])
|
|
else:
|
|
thisOnesParent = ap['device']['site']['name']
|
|
if thisOnesParent == currentNodeName:
|
|
if ap['device']['model'] in knownAPmodels:
|
|
aps.append(ap['device']['name'])
|
|
apDict = {}
|
|
for ap in aps:
|
|
maxDL = min(bandwidthDL[ap],bandwidthDL[currentNodeName])
|
|
maxUL = min(bandwidthUL[ap],bandwidthUL[currentNodeName])
|
|
apStruct = {
|
|
ap :
|
|
{
|
|
"downloadBandwidthMbps": maxDL,
|
|
"uploadBandwidthMbps": maxUL,
|
|
}
|
|
}
|
|
apDictNew = apDict | apStruct
|
|
apDict = apDictNew
|
|
if bool(apDict):
|
|
currentNode[currentNodeName]['children'] = apDict
|
|
counter = 0
|
|
tempChildren = {}
|
|
for child in childrenList:
|
|
name = siteIDtoName[child]
|
|
maxDL = min(bandwidthDL[name],bandwidthDL[currentNodeName])
|
|
maxUL = min(bandwidthUL[name],bandwidthUL[currentNodeName])
|
|
childStruct = {
|
|
name :
|
|
{
|
|
"downloadBandwidthMbps": maxDL,
|
|
"uploadBandwidthMbps": maxUL,
|
|
}
|
|
}
|
|
childStruct = createTree(sites,accessPoints,bandwidthDL,bandwidthUL,siteParentDict,siteIDtoName,sitesWithParents,childStruct)
|
|
tempChildren = tempChildren | childStruct
|
|
counter += 1
|
|
if tempChildren != {}:
|
|
if 'children' in currentNode[currentNodeName]:
|
|
currentNode[currentNodeName]['children'] = currentNode[currentNodeName]['children'] | tempChildren
|
|
else:
|
|
currentNode[currentNodeName]['children'] = tempChildren
|
|
return currentNode
|
|
|
|
def createNetworkJSON():
|
|
if os.path.isfile("network.json"):
|
|
print("network.json already exists. Leaving in place.")
|
|
else:
|
|
print("Generating network.json")
|
|
bandwidthDL = {}
|
|
bandwidthUL = {}
|
|
url = UISPbaseURL + "/nms/api/v2.1/sites?type=site"
|
|
headers = {'accept':'application/json', 'x-auth-token': uispAuthToken}
|
|
r = requests.get(url, headers=headers)
|
|
sites = r.json()
|
|
url = UISPbaseURL + "/nms/api/v2.1/devices/aps/profiles"
|
|
headers = {'accept':'application/json', 'x-auth-token': uispAuthToken}
|
|
r = requests.get(url, headers=headers)
|
|
apProfiles = r.json()
|
|
listOfTopLevelParentNodes = []
|
|
if os.path.isfile("integrationUISPbandwidths.csv"):
|
|
with open('integrationUISPbandwidths.csv') as csv_file:
|
|
csv_reader = csv.reader(csv_file, delimiter=',')
|
|
next(csv_reader)
|
|
for row in csv_reader:
|
|
name, download, upload = row
|
|
download = int(download)
|
|
upload = int(upload)
|
|
listOfTopLevelParentNodes.append(name)
|
|
bandwidthDL[name] = download
|
|
bandwidthUL[name] = upload
|
|
for ap in apProfiles:
|
|
name = ap['device']['name']
|
|
model = ap['device']['model']
|
|
apID = ap['device']['id']
|
|
if model in knownAPmodels:
|
|
url = UISPbaseURL + "/nms/api/v2.1/devices/airmaxes/" + apID + '?withStations=false'
|
|
headers = {'accept':'application/json', 'x-auth-token': uispAuthToken}
|
|
r = requests.get(url, headers=headers)
|
|
thisAPairmax = r.json()
|
|
downloadCap = int(round(thisAPairmax['overview']['downlinkCapacity']/1000000))
|
|
uploadCap = int(round(thisAPairmax['overview']['uplinkCapacity']/1000000))
|
|
# If operator already included bandwidth definitions for this ParentNode, do not overwrite what they set
|
|
if name not in listOfTopLevelParentNodes:
|
|
print("Found " + name)
|
|
listOfTopLevelParentNodes.append(name)
|
|
bandwidthDL[name] = downloadCap
|
|
bandwidthUL[name] = uploadCap
|
|
for site in sites:
|
|
name = site['identification']['name']
|
|
if name not in excludeSites:
|
|
# If operator already included bandwidth definitions for this ParentNode, do not overwrite what they set
|
|
if name not in listOfTopLevelParentNodes:
|
|
print("Found " + name)
|
|
listOfTopLevelParentNodes.append(name)
|
|
bandwidthDL[name] = 1000
|
|
bandwidthUL[name] = 1000
|
|
with open('integrationUISPbandwidths.csv', 'w') as csvfile:
|
|
wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
|
|
wr.writerow(['ParentNode', 'Download Mbps', 'Upload Mbps'])
|
|
for device in listOfTopLevelParentNodes:
|
|
entry = (device, bandwidthDL[device], bandwidthUL[device])
|
|
wr.writerow(entry)
|
|
url = UISPbaseURL + "/nms/api/v2.1/devices?role=ap"
|
|
headers = {'accept':'application/json', 'x-auth-token': uispAuthToken}
|
|
r = requests.get(url, headers=headers)
|
|
accessPoints = r.json()
|
|
siteIDtoName = {}
|
|
siteParentDict = {}
|
|
sitesWithParents = []
|
|
topLevelSites = []
|
|
for site in sites:
|
|
siteIDtoName[site['id']] = site['identification']['name']
|
|
try:
|
|
siteParentDict[site['id']] = site['identification']['parent']['id']
|
|
sitesWithParents.append(site['id'])
|
|
except:
|
|
siteParentDict[site['id']] = None
|
|
if site['identification']['name'] not in excludeSites:
|
|
topLevelSites.append(site['id'])
|
|
tLname = siteIDtoName[topLevelSites.pop()]
|
|
topLevelNode = {
|
|
tLname :
|
|
{
|
|
"downloadBandwidthMbps": bandwidthDL[tLname],
|
|
"uploadBandwidthMbps": bandwidthUL[tLname],
|
|
}
|
|
}
|
|
tree = createTree(sites,apProfiles, bandwidthDL, bandwidthUL, siteParentDict,siteIDtoName,sitesWithParents,topLevelNode)
|
|
with open('network.json', 'w') as f:
|
|
json.dump(tree, f, indent=4)
|
|
|
|
def createShaper():
|
|
print("Creating ShapedDevices.csv")
|
|
devicesToImport = []
|
|
url = UISPbaseURL + "/nms/api/v2.1/sites?type=site"
|
|
headers = {'accept':'application/json', 'x-auth-token': uispAuthToken}
|
|
r = requests.get(url, headers=headers)
|
|
sites = r.json()
|
|
siteIDtoName = {}
|
|
for site in sites:
|
|
siteIDtoName[site['id']] = site['identification']['name']
|
|
url = UISPbaseURL + "/nms/api/v2.1/sites?type=client&ucrm=true&ucrmDetails=true"
|
|
headers = {'accept':'application/json', 'x-auth-token': uispAuthToken}
|
|
r = requests.get(url, headers=headers)
|
|
clientSites = r.json()
|
|
url = UISPbaseURL + "/nms/api/v2.1/devices"
|
|
headers = {'accept':'application/json', 'x-auth-token': uispAuthToken}
|
|
r = requests.get(url, headers=headers)
|
|
allDevices = r.json()
|
|
ipv4ToIPv6 = {}
|
|
if findIPv6usingMikrotik:
|
|
ipv4ToIPv6 = pullMikrotikIPv6()
|
|
for uispClientSite in clientSites:
|
|
#if (uispClientSite['identification']['status'] == 'active') and (uispClientSite['identification']['suspended'] == False):
|
|
if (uispClientSite['identification']['suspended'] == False):
|
|
foundCPEforThisClientSite = False
|
|
if (uispClientSite['qos']['downloadSpeed']) and (uispClientSite['qos']['uploadSpeed']):
|
|
downloadSpeedMbps = int(round(uispClientSite['qos']['downloadSpeed']/1000000))
|
|
uploadSpeedMbps = int(round(uispClientSite['qos']['uploadSpeed']/1000000))
|
|
address = uispClientSite['description']['address']
|
|
uispClientSiteID = uispClientSite['id']
|
|
|
|
UCRMclientID = uispClientSite['ucrm']['client']['id']
|
|
siteName = uispClientSite['identification']['name']
|
|
AP = 'none'
|
|
thisSiteDevices = []
|
|
#Look for station devices, use those to find AP name
|
|
for device in allDevices:
|
|
if device['identification']['site'] != None:
|
|
if device['identification']['site']['id'] == uispClientSite['id']:
|
|
deviceName = device['identification']['name']
|
|
deviceRole = device['identification']['role']
|
|
deviceModel = device['identification']['model']
|
|
deviceModelName = device['identification']['modelName']
|
|
if (deviceRole == 'station'):
|
|
if device['attributes']['apDevice']:
|
|
AP = device['attributes']['apDevice']['name']
|
|
#Look for router devices, use those as shaped CPE
|
|
for device in allDevices:
|
|
if device['identification']['site'] != None:
|
|
if device['identification']['site']['id'] == uispClientSite['id']:
|
|
deviceModel = device['identification']['model']
|
|
deviceName = device['identification']['name']
|
|
deviceRole = device['identification']['role']
|
|
if device['identification']['mac']:
|
|
deviceMAC = device['identification']['mac'].upper()
|
|
else:
|
|
deviceMAC = ''
|
|
if (deviceRole == 'router') or (deviceModel in knownRouterModels):
|
|
ipv4 = device['ipAddress']
|
|
if '/' in ipv4:
|
|
ipv4 = ipv4.split("/")[0]
|
|
ipv6 = ''
|
|
if ipv4 in ipv4ToIPv6.keys():
|
|
ipv6 = ipv4ToIPv6[ipv4]
|
|
if isInAllowedSubnets(ipv4):
|
|
deviceModel = device['identification']['model']
|
|
deviceModelName = device['identification']['modelName']
|
|
maxSpeedDown = round(bandwidthOverheadFactor*downloadSpeedMbps)
|
|
maxSpeedUp = round(bandwidthOverheadFactor*uploadSpeedMbps)
|
|
minSpeedDown = min(round(maxSpeedDown*.98),maxSpeedDown)
|
|
minSpeedUp = min(round(maxSpeedUp*.98),maxSpeedUp)
|
|
#Customers directly connected to Sites
|
|
if deviceName in exceptionCPEs.keys():
|
|
AP = exceptionCPEs[deviceName]
|
|
if AP == 'none':
|
|
try:
|
|
AP = siteIDtoName[uispClientSite['identification']['parent']['id']]
|
|
except:
|
|
AP = 'none'
|
|
devicesToImport.append((uispClientSiteID, address, '', deviceName, AP, deviceMAC, ipv4, ipv6, str(minSpeedDown), str(minSpeedUp), str(maxSpeedDown),str(maxSpeedUp),''))
|
|
foundCPEforThisClientSite = True
|
|
else:
|
|
print("Failed to import devices from " + uispClientSite['description']['address'] + ". Missing QoS.")
|
|
if foundCPEforThisClientSite != True:
|
|
print("Failed to import devices for " + uispClientSite['description']['address'])
|
|
|
|
with open('ShapedDevices.csv', 'w') as csvfile:
|
|
wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL)
|
|
wr.writerow(['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC', 'IPv4', 'IPv6', 'Download Min', 'Upload Min', 'Download Max', 'Upload Max', 'Comment'])
|
|
for device in devicesToImport:
|
|
wr.writerow(device)
|
|
|
|
def importFromUISP():
|
|
createNetworkJSON()
|
|
createShaper()
|
|
|
|
if __name__ == '__main__':
|
|
importFromUISP()
|