mirror of
https://github.com/LibreQoE/LibreQoS.git
synced 2025-02-25 18:55:32 -06:00
Merge pull request #537 from LibreQoE/splynx-patch-jul28-1.4
Enable Splynx Integration to handle Some Topology Info (v1.4)
This commit is contained in:
Robert Chacón
GitHub
commit
56c00b8c96
@ -1,181 +1,259 @@
|
|||||||
|
|
||||||
from pythonCheck import checkPythonVersion
|
from pythonCheck import checkPythonVersion
|
||||||
checkPythonVersion()
|
checkPythonVersion()
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
import warnings
|
import warnings
|
||||||
from ispConfig import excludeSites, findIPv6usingMikrotik, bandwidthOverheadFactor, exceptionCPEs, splynx_api_key, splynx_api_secret, splynx_api_url
|
from ispConfig import excludeSites, findIPv6usingMikrotik, bandwidthOverheadFactor, exceptionCPEs, splynx_api_key, splynx_api_secret, splynx_api_url
|
||||||
from integrationCommon import isIpv4Permitted
|
from integrationCommon import isIpv4Permitted
|
||||||
import base64
|
import base64
|
||||||
from requests.auth import HTTPBasicAuth
|
from requests.auth import HTTPBasicAuth
|
||||||
if findIPv6usingMikrotik == True:
|
|
||||||
from mikrotikFindIPv6 import pullMikrotikIPv6
|
if findIPv6usingMikrotik:
|
||||||
|
from mikrotikFindIPv6 import pullMikrotikIPv6
|
||||||
|
|
||||||
from integrationCommon import NetworkGraph, NetworkNode, NodeType
|
from integrationCommon import NetworkGraph, NetworkNode, NodeType
|
||||||
|
import os
|
||||||
|
import csv
|
||||||
|
|
||||||
def buildHeaders():
|
def buildHeaders():
|
||||||
credentials = splynx_api_key + ':' + splynx_api_secret
|
"""
|
||||||
credentials = base64.b64encode(credentials.encode()).decode()
|
Build authorization headers for Splynx API requests using API key and secret.
|
||||||
return {'Authorization' : "Basic %s" % credentials}
|
"""
|
||||||
|
credentials = splynx_api_key + ':' + splynx_api_secret
|
||||||
|
credentials = base64.b64encode(credentials.encode()).decode()
|
||||||
|
return {'Authorization': "Basic %s" % credentials}
|
||||||
|
|
||||||
def spylnxRequest(target, headers):
|
def spylnxRequest(target, headers):
|
||||||
# Sends a REST GET request to Spylnx and returns the
|
"""
|
||||||
# result in JSON
|
Send a GET request to the Splynx API and return the JSON response.
|
||||||
url = splynx_api_url + "/api/2.0/" + target
|
"""
|
||||||
r = requests.get(url, headers=headers, timeout=120)
|
url = splynx_api_url + "/api/2.0/" + target
|
||||||
return r.json()
|
r = requests.get(url, headers=headers, timeout=120)
|
||||||
|
return r.json()
|
||||||
|
|
||||||
def getTariffs(headers):
|
def getTariffs(headers):
|
||||||
data = spylnxRequest("admin/tariffs/internet", headers)
|
"""
|
||||||
tariff = []
|
Retrieve tariff data from Splynx API and calculate download/upload speeds for each tariff.
|
||||||
downloadForTariffID = {}
|
"""
|
||||||
uploadForTariffID = {}
|
data = spylnxRequest("admin/tariffs/internet", headers)
|
||||||
for tariff in data:
|
downloadForTariffID = {}
|
||||||
tariffID = tariff['id']
|
uploadForTariffID = {}
|
||||||
speed_download = round((int(tariff['speed_download']) / 1000))
|
for tariff in data:
|
||||||
speed_upload = round((int(tariff['speed_upload']) / 1000))
|
tariffID = tariff['id']
|
||||||
downloadForTariffID[tariffID] = speed_download
|
speed_download = round((int(tariff['speed_download']) / 1000))
|
||||||
uploadForTariffID[tariffID] = speed_upload
|
speed_upload = round((int(tariff['speed_upload']) / 1000))
|
||||||
return (tariff, downloadForTariffID, uploadForTariffID)
|
downloadForTariffID[tariffID] = speed_download
|
||||||
|
uploadForTariffID[tariffID] = speed_upload
|
||||||
|
return (data, downloadForTariffID, uploadForTariffID)
|
||||||
|
|
||||||
|
def buildSiteBandwidths():
|
||||||
|
"""
|
||||||
|
Build a dictionary of site bandwidths by reading data from a CSV file.
|
||||||
|
"""
|
||||||
|
siteBandwidth = {}
|
||||||
|
if os.path.isfile("integrationSplynxBandwidths.csv"):
|
||||||
|
with open('integrationSplynxBandwidths.csv') as csv_file:
|
||||||
|
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||||
|
next(csv_reader)
|
||||||
|
for row in csv_reader:
|
||||||
|
name, download, upload = row
|
||||||
|
download = int(float(download))
|
||||||
|
upload = int(float(upload))
|
||||||
|
siteBandwidth[name] = {"download": download, "upload": upload}
|
||||||
|
return siteBandwidth
|
||||||
|
|
||||||
def getCustomers(headers):
|
def getCustomers(headers):
|
||||||
data = spylnxRequest("admin/customers/customer", headers)
|
"""
|
||||||
#addressForCustomerID = {}
|
Retrieve all customer data from Splynx API.
|
||||||
#customerIDs = []
|
"""
|
||||||
#for customer in data:
|
return spylnxRequest("admin/customers/customer", headers)
|
||||||
# customerIDs.append(customer['id'])
|
|
||||||
# addressForCustomerID[customer['id']] = customer['street_1']
|
def getCustomersOnline(headers):
|
||||||
return data
|
"""
|
||||||
|
Retrieve data of currently online customers from Splynx API.
|
||||||
|
"""
|
||||||
|
return spylnxRequest("admin/customers/customers-online", headers)
|
||||||
|
|
||||||
def getRouters(headers):
|
def getRouters(headers):
|
||||||
data = spylnxRequest("admin/networking/routers", headers)
|
"""
|
||||||
ipForRouter = {}
|
Retrieve router data from Splynx API and build dictionaries for router IPs and names.
|
||||||
for router in data:
|
"""
|
||||||
routerID = router['id']
|
data = spylnxRequest("admin/networking/routers", headers)
|
||||||
ipForRouter[routerID] = router['ip']
|
routerIdList = []
|
||||||
print("Router IPs found: " + str(len(ipForRouter)))
|
ipForRouter = {}
|
||||||
return ipForRouter
|
nameForRouterID = {}
|
||||||
|
for router in data:
|
||||||
|
routerID = router['id']
|
||||||
|
if router['id'] not in routerIdList:
|
||||||
|
routerIdList.append(router['id'])
|
||||||
|
ipForRouter[routerID] = router['ip']
|
||||||
|
nameForRouterID[routerID] = router['title']
|
||||||
|
print("Router IPs found: " + str(len(ipForRouter)))
|
||||||
|
return (ipForRouter, nameForRouterID, routerIdList)
|
||||||
|
|
||||||
|
def getSectors(headers):
|
||||||
|
"""
|
||||||
|
Retrieve sector data from Splynx API and build a dictionary mapping routers to their sectors.
|
||||||
|
"""
|
||||||
|
data = spylnxRequest("admin/networking/routers-sectors", headers)
|
||||||
|
sectorForRouter = {}
|
||||||
|
for sector in data:
|
||||||
|
routerID = sector['router_id']
|
||||||
|
if routerID not in sectorForRouter:
|
||||||
|
newList = []
|
||||||
|
newList.append(sector)
|
||||||
|
sectorForRouter[routerID] = newList
|
||||||
|
else:
|
||||||
|
newList = sectorForRouter[routerID]
|
||||||
|
newList.append(sector)
|
||||||
|
sectorForRouter[routerID] = newList
|
||||||
|
|
||||||
|
print("Router Sectors found: " + str(len(sectorForRouter)))
|
||||||
|
return sectorForRouter
|
||||||
|
|
||||||
def combineAddress(json):
|
def combineAddress(json):
|
||||||
# Combines address fields into a single string
|
"""
|
||||||
# The API docs seem to indicate that there isn't a "state" field?
|
Combine address fields into a single string. If address fields are empty, use ID and name.
|
||||||
if json["street_1"]=="" and json["city"]=="" and json["zip_code"]=="":
|
"""
|
||||||
return str(json["id"]) + "/" + json["name"]
|
if json["street_1"] == "" and json["city"] == "" and json["zip_code"] == "":
|
||||||
else:
|
return str(json["id"]) + "/" + json["name"]
|
||||||
return json["street_1"] + " " + json["city"] + " " + json["zip_code"]
|
else:
|
||||||
|
return json["street_1"] + " " + json["city"] + " " + json["zip_code"]
|
||||||
|
|
||||||
def getAllServices(headers):
|
def getAllServices(headers):
|
||||||
services = spylnxRequest("admin/customers/customer/0/internet-services?main_attributes%5Bstatus%5D=active", headers)
|
"""
|
||||||
return services
|
Retrieve all active internet services from Splynx API.
|
||||||
|
"""
|
||||||
|
return spylnxRequest("admin/customers/customer/0/internet-services?main_attributes%5Bstatus%5D=active", headers)
|
||||||
|
|
||||||
def getAllIPs(headers):
|
def getAllIPs(headers):
|
||||||
ipv4ByCustomerID = {}
|
"""
|
||||||
ipv6ByCustomerID = {}
|
Retrieve all used IPv4 and IPv6 addresses from Splynx API and map them to customer IDs.
|
||||||
allIPv4 = spylnxRequest("admin/networking/ipv4-ip?main_attributes%5Bis_used%5D=1", headers)
|
"""
|
||||||
allIPv6 = spylnxRequest("admin/networking/ipv6-ip", headers)
|
ipv4ByCustomerID = {}
|
||||||
for ipv4 in allIPv4:
|
ipv6ByCustomerID = {}
|
||||||
if ipv4['customer_id'] not in ipv4ByCustomerID:
|
allIPv4 = spylnxRequest("admin/networking/ipv4-ip?main_attributes%5Bis_used%5D=1", headers)
|
||||||
ipv4ByCustomerID[ipv4['customer_id']] = []
|
allIPv6 = spylnxRequest("admin/networking/ipv6-ip", headers)
|
||||||
temp = ipv4ByCustomerID[ipv4['customer_id']]
|
for ipv4 in allIPv4:
|
||||||
temp.append(ipv4['ip'])
|
if ipv4['customer_id'] not in ipv4ByCustomerID:
|
||||||
ipv4ByCustomerID[ipv4['customer_id']] = temp
|
ipv4ByCustomerID[ipv4['customer_id']] = []
|
||||||
for ipv6 in allIPv6:
|
temp = ipv4ByCustomerID[ipv4['customer_id']]
|
||||||
if ipv6['is_used'] == 1:
|
temp.append(ipv4['ip'])
|
||||||
if ipv6['customer_id'] not in ipv6ByCustomerID:
|
ipv4ByCustomerID[ipv4['customer_id']] = temp
|
||||||
ipv6ByCustomerID[ipv6['customer_id']] = []
|
for ipv6 in allIPv6:
|
||||||
temp = ipv6ByCustomerID[ipv6['customer_id']]
|
if ipv6['is_used'] == 1:
|
||||||
temp.append(ipv6['ip'])
|
if ipv6['customer_id'] not in ipv6ByCustomerID:
|
||||||
ipv6ByCustomerID[ipv6['customer_id']] = temp
|
ipv6ByCustomerID[ipv6['customer_id']] = []
|
||||||
return (ipv4ByCustomerID, ipv6ByCustomerID)
|
temp = ipv6ByCustomerID[ipv6['customer_id']]
|
||||||
|
temp.append(ipv6['ip'])
|
||||||
|
ipv6ByCustomerID[ipv6['customer_id']] = temp
|
||||||
|
return (ipv4ByCustomerID, ipv6ByCustomerID)
|
||||||
|
|
||||||
def createShaper():
|
def createShaper():
|
||||||
net = NetworkGraph()
|
"""
|
||||||
|
Main function to fetch data from Splynx, build the network graph, and shape devices.
|
||||||
|
"""
|
||||||
|
net = NetworkGraph()
|
||||||
|
|
||||||
print("Fetching data from Spylnx")
|
print("Fetching data from Spylnx")
|
||||||
headers = buildHeaders()
|
headers = buildHeaders()
|
||||||
tariff, downloadForTariffID, uploadForTariffID = getTariffs(headers)
|
tariff, downloadForTariffID, uploadForTariffID = getTariffs(headers)
|
||||||
customers = getCustomers(headers)
|
customers = getCustomers(headers)
|
||||||
ipForRouter = getRouters(headers)
|
customersOnline = getCustomersOnline(headers)
|
||||||
allServices = getAllServices(headers)
|
ipForRouter, nameForRouterID, routerIdList = getRouters(headers)
|
||||||
ipv4ByCustomerID, ipv6ByCustomerID = getAllIPs(headers)
|
sectorForRouter = getSectors(headers)
|
||||||
|
allServices = getAllServices(headers)
|
||||||
allServicesDict = {}
|
ipv4ByCustomerID, ipv6ByCustomerID = getAllIPs(headers)
|
||||||
for serviceItem in allServices:
|
siteBandwidth = buildSiteBandwidths()
|
||||||
if (serviceItem['status'] == 'active'):
|
|
||||||
if serviceItem["customer_id"] not in allServicesDict:
|
allParentNodes = []
|
||||||
allServicesDict[serviceItem["customer_id"]] = []
|
custIDtoParentNode = {}
|
||||||
temp = allServicesDict[serviceItem["customer_id"]]
|
parentNodeIDCounter = 30000
|
||||||
temp.append(serviceItem)
|
|
||||||
allServicesDict[serviceItem["customer_id"]] = temp
|
# Create nodes for sites and assign bandwidth
|
||||||
|
for customer in customersOnline:
|
||||||
#It's not very clear how a service is meant to handle multiple
|
download = 1000
|
||||||
#devices on a shared tariff. Creating each service as a combined
|
upload = 1000
|
||||||
#entity including the customer, to be on the safe side.
|
nodeName = customer['nas_id'] + "_" + customer['call_to'] + "_" + customer['port']
|
||||||
for customerJson in customers:
|
|
||||||
if customerJson['status'] == 'active':
|
if nodeName not in allParentNodes:
|
||||||
if customerJson['id'] in allServicesDict:
|
if nodeName in siteBandwidth:
|
||||||
servicesForCustomer = allServicesDict[customerJson['id']]
|
download = siteBandwidth[nodeName]["download"]
|
||||||
for service in servicesForCustomer:
|
upload = siteBandwidth[nodeName]["upload"]
|
||||||
combinedId = "c_" + str(customerJson["id"]) + "_s_" + str(service["id"])
|
|
||||||
tariff_id = service['tariff_id']
|
node = NetworkNode(id=parentNodeIDCounter, displayName=nodeName, type=NodeType.site,
|
||||||
customer = NetworkNode(
|
parentId=None, download=download, upload=upload, address=None)
|
||||||
type=NodeType.client,
|
net.addRawNode(node)
|
||||||
id=combinedId,
|
|
||||||
displayName=customerJson["name"],
|
pnEntry = {}
|
||||||
address=combineAddress(customerJson),
|
pnEntry['name'] = nodeName
|
||||||
customerName=customerJson["name"],
|
pnEntry['id'] = parentNodeIDCounter
|
||||||
download=downloadForTariffID[tariff_id],
|
custIDtoParentNode[customer['customer_id']] = pnEntry
|
||||||
upload=uploadForTariffID[tariff_id],
|
|
||||||
)
|
parentNodeIDCounter += 1
|
||||||
net.addRawNode(customer)
|
|
||||||
|
|
||||||
ipv4 = []
|
|
||||||
ipv6 = []
|
|
||||||
routerID = service['router_id']
|
|
||||||
|
|
||||||
# If not "Taking IPv4" (Router will assign IP), then use router's set IP
|
|
||||||
taking_ipv4 = int(service['taking_ipv4'])
|
|
||||||
if taking_ipv4 == 0:
|
|
||||||
if routerID in ipForRouter:
|
|
||||||
ipv4 = [ipForRouter[routerID]]
|
|
||||||
|
|
||||||
elif taking_ipv4 == 1:
|
allServicesDict = {}
|
||||||
ipv4 = [service['ipv4']]
|
for serviceItem in allServices:
|
||||||
if len(ipv4) == 0:
|
if serviceItem['status'] == 'active':
|
||||||
#Only do this if single service for a customer
|
if serviceItem["customer_id"] not in allServicesDict:
|
||||||
if len(servicesForCustomer) == 1:
|
allServicesDict[serviceItem["customer_id"]] = []
|
||||||
if customerJson['id'] in ipv4ByCustomerID:
|
temp = allServicesDict[serviceItem["customer_id"]]
|
||||||
ipv4 = ipv4ByCustomerID[customerJson['id']]
|
temp.append(serviceItem)
|
||||||
|
allServicesDict[serviceItem["customer_id"]] = temp
|
||||||
# If not "Taking IPv6" (Router will assign IP), then use router's set IP
|
|
||||||
if isinstance(service['taking_ipv6'], str):
|
# Create nodes for customers and their devices
|
||||||
taking_ipv6 = int(service['taking_ipv6'])
|
for customerJson in customers:
|
||||||
else:
|
if customerJson['status'] == 'active':
|
||||||
taking_ipv6 = service['taking_ipv6']
|
if customerJson['id'] in allServicesDict:
|
||||||
if taking_ipv6 == 0:
|
servicesForCustomer = allServicesDict[customerJson['id']]
|
||||||
ipv6 = []
|
for service in servicesForCustomer:
|
||||||
elif taking_ipv6 == 1:
|
combinedId = "c_" + str(customerJson["id"]) + "_s_" + str(service["id"])
|
||||||
ipv6 = [service['ipv6']]
|
tariff_id = service['tariff_id']
|
||||||
|
|
||||||
device = NetworkNode(
|
parentID = None
|
||||||
id=combinedId+"_d" + str(service["id"]),
|
if customerJson['id'] in custIDtoParentNode:
|
||||||
displayName=service["id"],
|
parentID = custIDtoParentNode[customerJson['id']]['id']
|
||||||
type=NodeType.device,
|
|
||||||
parentId=combinedId,
|
customer = NetworkNode(
|
||||||
mac=service["mac"],
|
type=NodeType.client,
|
||||||
ipv4=ipv4,
|
id=combinedId,
|
||||||
ipv6=ipv6
|
parentId=parentID,
|
||||||
)
|
displayName=customerJson["name"],
|
||||||
net.addRawNode(device)
|
address=combineAddress(customerJson),
|
||||||
|
customerName=customerJson["name"],
|
||||||
|
download=downloadForTariffID[tariff_id],
|
||||||
|
upload=uploadForTariffID[tariff_id]
|
||||||
|
)
|
||||||
|
net.addRawNode(customer)
|
||||||
|
|
||||||
|
ipv4 = ipv4ByCustomerID.get(customerJson["id"], [])
|
||||||
|
ipv6 = ipv6ByCustomerID.get(customerJson["id"], [])
|
||||||
|
|
||||||
|
device = NetworkNode(
|
||||||
|
id=combinedId + "_d" + str(service["id"]),
|
||||||
|
displayName=service["id"],
|
||||||
|
type=NodeType.device,
|
||||||
|
parentId=combinedId,
|
||||||
|
mac=service["mac"],
|
||||||
|
ipv4=ipv4,
|
||||||
|
ipv6=ipv6
|
||||||
|
)
|
||||||
|
net.addRawNode(device)
|
||||||
|
|
||||||
net.prepareTree()
|
net.prepareTree()
|
||||||
net.plotNetworkGraph(False)
|
net.plotNetworkGraph(False)
|
||||||
if net.doesNetworkJsonExist():
|
if net.doesNetworkJsonExist():
|
||||||
print("network.json already exists. Leaving in-place.")
|
print("network.json already exists. Leaving in-place.")
|
||||||
else:
|
else:
|
||||||
net.createNetworkJson()
|
net.createNetworkJson()
|
||||||
net.createShapedDevices()
|
net.createShapedDevices()
|
||||||
|
|
||||||
def importFromSplynx():
|
def importFromSplynx():
|
||||||
#createNetworkJSON()
|
"""
|
||||||
createShaper()
|
Entry point for the script to initiate the Splynx data import and shaper creation process.
|
||||||
|
"""
|
||||||
|
createShaper()
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
importFromSplynx()
|
importFromSplynx()
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user