mirror of
https://github.com/LibreQoE/LibreQoS.git
synced 2025-02-25 18:55:32 -06:00
Attempt to resolve https://github.com/rchac/LibreQoS/issues/85
This commit is contained in:
Robert Chacón
GitHub
parent
f24267fac6
commit
6964aad285
@ -10,178 +10,176 @@ from ispConfig import interfaceA, interfaceB, influxDBBucket, influxDBOrg, influ
|
|||||||
|
|
||||||
|
|
||||||
def getInterfaceStats(interface):
|
def getInterfaceStats(interface):
|
||||||
command = 'tc -j -s qdisc show dev ' + interface
|
command = 'tc -j -s qdisc show dev ' + interface
|
||||||
jsonAr = json.loads(subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8'))
|
jsonAr = json.loads(subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8'))
|
||||||
jsonDict = {}
|
jsonDict = {}
|
||||||
for element in filter(lambda e: 'parent' in e, jsonAr):
|
for element in filter(lambda e: 'parent' in e, jsonAr):
|
||||||
flowID = ':'.join(map(lambda p: f'0x{p}', element['parent'].split(':')[0:2]))
|
flowID = ':'.join(map(lambda p: f'0x{p}', element['parent'].split(':')[0:2]))
|
||||||
jsonDict[flowID] = element
|
jsonDict[flowID] = element
|
||||||
del jsonAr
|
del jsonAr
|
||||||
return jsonDict
|
return jsonDict
|
||||||
|
|
||||||
|
|
||||||
def getCircuitStats(subscriberCircuits):
|
def chunk_list(l, n):
|
||||||
interfaces = [interfaceA, interfaceB]
|
for i in range(0, len(l), n):
|
||||||
for interface in interfaces:
|
yield l[i:i + n]
|
||||||
tcShowResults = getInterfaceStats(interface)
|
|
||||||
if interface == interfaceA:
|
|
||||||
interfaceAjson = tcShowResults
|
|
||||||
else:
|
|
||||||
interfaceBjson = tcShowResults
|
|
||||||
|
|
||||||
for circuit in subscriberCircuits:
|
|
||||||
if 'timeQueried' in circuit:
|
|
||||||
circuit['priorQueryTime'] = circuit['timeQueried']
|
|
||||||
for interface in interfaces:
|
|
||||||
if interface == interfaceA:
|
|
||||||
jsonVersion = interfaceAjson
|
|
||||||
else:
|
|
||||||
jsonVersion = interfaceBjson
|
|
||||||
|
|
||||||
element = jsonVersion[circuit['qdisc']] if circuit['qdisc'] in jsonVersion else False
|
def getsubscriberCircuitstats(subscriberCircuits):
|
||||||
|
interfaces = [interfaceA, interfaceB]
|
||||||
|
ifaceStats = list(map(getInterfaceStats, interfaces))
|
||||||
|
|
||||||
if element:
|
for circuit in subscriberCircuits:
|
||||||
drops = int(element['drops'])
|
if 'timeQueried' in circuit:
|
||||||
packets = int(element['packets'])
|
circuit['priorQueryTime'] = circuit['timeQueried']
|
||||||
bytesSent = int(element['bytes'])
|
for (interface, stats, dirSuffix) in zip(interfaces, ifaceStats, ['Download', 'Upload']):
|
||||||
if interface == interfaceA:
|
|
||||||
if 'bytesSentDownload' in circuit:
|
element = stats[circuit['qdisc']] if circuit['qdisc'] in stats else False
|
||||||
circuit['priorQueryBytesDownload'] = circuit['bytesSentDownload']
|
|
||||||
circuit['bytesSentDownload'] = bytesSent
|
if element:
|
||||||
else:
|
|
||||||
if 'bytesSentUpload' in circuit:
|
bytesSent = int(element['bytes'])
|
||||||
circuit['priorQueryBytesUpload'] = circuit['bytesSentUpload']
|
drops = int(element['drops'])
|
||||||
circuit['bytesSentUpload'] = bytesSent
|
packets = int(element['packets'])
|
||||||
|
|
||||||
|
if 'bytesSent' + dirSuffix in circuit:
|
||||||
|
circuit['priorQueryBytes' + dirSuffix] = circuit['bytesSent' + dirSuffix]
|
||||||
|
circuit['bytesSent' + dirSuffix] = bytesSent
|
||||||
|
|
||||||
|
if 'dropsSent' + dirSuffix in circuit:
|
||||||
|
circuit['priorDropsSent' + dirSuffix] = circuit['dropsSent' + dirSuffix]
|
||||||
|
circuit['dropsSent' + dirSuffix] = drops
|
||||||
|
|
||||||
|
if 'packetsSent' + dirSuffix in circuit:
|
||||||
|
circuit['priorPacketsSent' + dirSuffix] = circuit['packetsSent' + dirSuffix]
|
||||||
|
circuit['packetsSent' + dirSuffix] = packets
|
||||||
|
|
||||||
|
circuit['timeQueried'] = datetime.now().isoformat()
|
||||||
|
for circuit in subscriberCircuits:
|
||||||
|
circuit['bitsDownloadSinceLastQuery'] = circuit['bitsUploadSinceLastQuery'] = 0
|
||||||
|
if 'priorQueryTime' in circuit:
|
||||||
|
try:
|
||||||
|
bytesDLSinceLastQuery = circuit['bytesSentDownload'] - circuit['priorQueryBytesDownload']
|
||||||
|
bytesULSinceLastQuery = circuit['bytesSentUpload'] - circuit['priorQueryBytesUpload']
|
||||||
|
except:
|
||||||
|
bytesDLSinceLastQuery = bytesULSinceLastQuery = 0
|
||||||
|
|
||||||
|
currentQueryTime = datetime.fromisoformat(circuit['timeQueried'])
|
||||||
|
priorQueryTime = datetime.fromisoformat(circuit['priorQueryTime'])
|
||||||
|
deltaSeconds = (currentQueryTime - priorQueryTime).total_seconds()
|
||||||
|
|
||||||
|
circuit['bitsDownloadSinceLastQuery'] = round(
|
||||||
|
((bytesDLSinceLastQuery * 8) / deltaSeconds)) if deltaSeconds > 0 else 0
|
||||||
|
circuit['bitsUploadSinceLastQuery'] = round(
|
||||||
|
((bytesULSinceLastQuery * 8) / deltaSeconds)) if deltaSeconds > 0 else 0
|
||||||
|
|
||||||
|
return subscriberCircuits
|
||||||
|
|
||||||
circuit['timeQueried'] = datetime.now().isoformat()
|
|
||||||
for circuit in subscriberCircuits:
|
|
||||||
if 'priorQueryTime' in circuit:
|
|
||||||
try:
|
|
||||||
bytesDLSinceLastQuery = circuit['bytesSentDownload'] - circuit['priorQueryBytesDownload']
|
|
||||||
bytesULSinceLastQuery = circuit['bytesSentUpload'] - circuit['priorQueryBytesUpload']
|
|
||||||
except:
|
|
||||||
bytesDLSinceLastQuery = 0
|
|
||||||
bytesULSinceLastQuery = 0
|
|
||||||
currentQueryTime = datetime.fromisoformat(circuit['timeQueried'])
|
|
||||||
priorQueryTime = datetime.fromisoformat(circuit['priorQueryTime'])
|
|
||||||
delta = currentQueryTime - priorQueryTime
|
|
||||||
deltaSeconds = delta.total_seconds()
|
|
||||||
if deltaSeconds > 0:
|
|
||||||
bitsDownload = round((((bytesDLSinceLastQuery*8))/deltaSeconds))
|
|
||||||
bitsUpload = round((((bytesULSinceLastQuery*8))/deltaSeconds))
|
|
||||||
else:
|
|
||||||
bitsDownload = 0
|
|
||||||
bitsUpload = 0
|
|
||||||
circuit['bitsDownloadSinceLastQuery'] = bitsDownload
|
|
||||||
circuit['bitsUploadSinceLastQuery'] = bitsUpload
|
|
||||||
else:
|
|
||||||
circuit['bitsDownloadSinceLastQuery'] = 0
|
|
||||||
circuit['bitsUploadSinceLastQuery'] = 0
|
|
||||||
return (subscriberCircuits)
|
|
||||||
|
|
||||||
def getParentNodeStats(parentNodes, subscriberCircuits):
|
def getParentNodeStats(parentNodes, subscriberCircuits):
|
||||||
for parentNode in parentNodes:
|
for parentNode in parentNodes:
|
||||||
thisNodeBitsDownload = 0
|
thisNodeBitsDownload = 0
|
||||||
thisNodeBitsUpload = 0
|
thisNodeBitsUpload = 0
|
||||||
for circuit in subscriberCircuits:
|
for circuit in subscriberCircuits:
|
||||||
if circuit['ParentNode'] == parentNode['parentNodeName']:
|
if circuit['ParentNode'] == parentNode['parentNodeName']:
|
||||||
thisNodeBitsDownload += circuit['bitsDownloadSinceLastQuery']
|
thisNodeBitsDownload += circuit['bitsDownloadSinceLastQuery']
|
||||||
thisNodeBitsUpload += circuit['bitsUploadSinceLastQuery']
|
thisNodeBitsUpload += circuit['bitsUploadSinceLastQuery']
|
||||||
|
|
||||||
|
parentNode['bitsDownloadSinceLastQuery'] = thisNodeBitsDownload
|
||||||
|
parentNode['bitsUploadSinceLastQuery'] = thisNodeBitsUpload
|
||||||
|
return parentNodes
|
||||||
|
|
||||||
parentNode['bitsDownloadSinceLastQuery'] = thisNodeBitsDownload
|
|
||||||
parentNode['bitsUploadSinceLastQuery'] = thisNodeBitsUpload
|
|
||||||
return parentNodes
|
|
||||||
|
|
||||||
def getParentNodeDict(data, depth, parentNodeNameDict):
|
def getParentNodeDict(data, depth, parentNodeNameDict):
|
||||||
if parentNodeNameDict == None:
|
if parentNodeNameDict == None:
|
||||||
parentNodeNameDict = {}
|
parentNodeNameDict = {}
|
||||||
|
|
||||||
|
for elem in data:
|
||||||
|
if 'children' in data[elem]:
|
||||||
|
for child in data[elem]['children']:
|
||||||
|
parentNodeNameDict[child] = elem
|
||||||
|
tempDict = getParentNodeDict(data[elem]['children'], depth + 1, parentNodeNameDict)
|
||||||
|
parentNodeNameDict = dict(parentNodeNameDict, **tempDict)
|
||||||
|
return parentNodeNameDict
|
||||||
|
|
||||||
for elem in data:
|
|
||||||
if 'children' in data[elem]:
|
|
||||||
for child in data[elem]['children']:
|
|
||||||
parentNodeNameDict[child] = elem
|
|
||||||
tempDict = getParentNodeDict(data[elem]['children'], depth+1, parentNodeNameDict)
|
|
||||||
parentNodeNameDict = dict(parentNodeNameDict, **tempDict)
|
|
||||||
return parentNodeNameDict
|
|
||||||
|
|
||||||
def parentNodeNameDictPull():
|
def parentNodeNameDictPull():
|
||||||
#Load network heirarchy
|
# Load network heirarchy
|
||||||
with open('network.json', 'r') as j:
|
with open('network.json', 'r') as j:
|
||||||
network = json.loads(j.read())
|
network = json.loads(j.read())
|
||||||
parentNodeNameDict = getParentNodeDict(network, 0, None)
|
parentNodeNameDict = getParentNodeDict(network, 0, None)
|
||||||
return parentNodeNameDict
|
return parentNodeNameDict
|
||||||
|
|
||||||
def refreshBandwidthGraphs():
|
def refreshBandwidthGraphs():
|
||||||
startTime = datetime.now()
|
startTime = datetime.now()
|
||||||
with open('statsByParentNode.json', 'r') as j:
|
with open('statsByParentNode.json', 'r') as j:
|
||||||
parentNodes = json.loads(j.read())
|
parentNodes = json.loads(j.read())
|
||||||
|
|
||||||
with open('statsByCircuit.json', 'r') as j:
|
with open('statsByCircuit.json', 'r') as j:
|
||||||
subscriberCircuits = json.loads(j.read())
|
subscriberCircuits = json.loads(j.read())
|
||||||
|
|
||||||
parentNodeNameDict = parentNodeNameDictPull()
|
parentNodeNameDict = parentNodeNameDictPull()
|
||||||
|
|
||||||
print("Retrieving circuit statistics")
|
print("Retrieving circuit statistics")
|
||||||
subscriberCircuits = getCircuitStats(subscriberCircuits)
|
subscriberCircuits = getsubscriberCircuitstats(subscriberCircuits)
|
||||||
print("Computing parent node statistics")
|
print("Computing parent node statistics")
|
||||||
parentNodes = getParentNodeStats(parentNodes, subscriberCircuits)
|
parentNodes = getParentNodeStats(parentNodes, subscriberCircuits)
|
||||||
print("Writing data to InfluxDB")
|
print("Writing data to InfluxDB")
|
||||||
bucket = influxDBBucket
|
client = InfluxDBClient(
|
||||||
org = influxDBOrg
|
url=influxDBurl,
|
||||||
token = influxDBtoken
|
token=influxDBtoken,
|
||||||
url=influxDBurl
|
org=influxDBOrg
|
||||||
client = InfluxDBClient(
|
)
|
||||||
url=url,
|
write_api = client.write_api(write_options=SYNCHRONOUS)
|
||||||
token=token,
|
|
||||||
org=org
|
|
||||||
)
|
|
||||||
write_api = client.write_api(write_options=SYNCHRONOUS)
|
|
||||||
|
|
||||||
queriesToSend = []
|
chunkedsubscriberCircuits = list(chunk_list(subscriberCircuits, 200))
|
||||||
for circuit in subscriberCircuits:
|
|
||||||
bitsDownload = int(circuit['bitsDownloadSinceLastQuery'])
|
|
||||||
bitsUpload = int(circuit['bitsUploadSinceLastQuery'])
|
|
||||||
if (bitsDownload > 0) and (bitsUpload > 0):
|
|
||||||
percentUtilizationDownload = round((bitsDownload / round(circuit['downloadMax']*1000000)),4)
|
|
||||||
percentUtilizationUpload = round((bitsUpload / round(circuit['uploadMax']*1000000)),4)
|
|
||||||
|
|
||||||
p = Point('Bandwidth').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", bitsDownload)
|
queriesToSendCount = 0
|
||||||
queriesToSend.append(p)
|
for chunk in chunkedsubscriberCircuits:
|
||||||
p = Point('Bandwidth').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Upload", bitsUpload)
|
queriesToSend = []
|
||||||
queriesToSend.append(p)
|
for circuit in chunk:
|
||||||
p = Point('Utilization').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", percentUtilizationDownload)
|
bitsDownload = int(circuit['bitsDownloadSinceLastQuery'])
|
||||||
queriesToSend.append(p)
|
bitsUpload = int(circuit['bitsUploadSinceLastQuery'])
|
||||||
p = Point('Utilization').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Upload", percentUtilizationUpload)
|
if (bitsDownload > 0) and (bitsUpload > 0):
|
||||||
queriesToSend.append(p)
|
percentUtilizationDownload = round((bitsDownload / round(circuit['downloadMax'] * 1000000)), 4)
|
||||||
|
percentUtilizationUpload = round((bitsUpload / round(circuit['uploadMax'] * 1000000)), 4)
|
||||||
|
p = Point('Bandwidth').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", bitsDownload).field("Upload", bitsUpload)
|
||||||
|
queriesToSend.append(p)
|
||||||
|
p = Point('Utilization').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload)
|
||||||
|
queriesToSend.append(p)
|
||||||
|
|
||||||
for parentNode in parentNodes:
|
write_api.write(bucket=influxDBBucket, record=queriesToSend)
|
||||||
bitsDownload = int(parentNode['bitsDownloadSinceLastQuery'])
|
# print("Added " + str(len(queriesToSend)) + " points to InfluxDB.")
|
||||||
bitsUpload = int(parentNode['bitsUploadSinceLastQuery'])
|
queriesToSendCount += len(queriesToSend)
|
||||||
if (bitsDownload > 0) and (bitsUpload > 0):
|
|
||||||
percentUtilizationDownload = round((bitsDownload / round(parentNode['downloadMax']*1000000)),4)
|
|
||||||
percentUtilizationUpload = round((bitsUpload / round(parentNode['uploadMax']*1000000)),4)
|
|
||||||
|
|
||||||
p = Point('Bandwidth').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", bitsDownload)
|
queriesToSend = []
|
||||||
queriesToSend.append(p)
|
for parentNode in parentNodes:
|
||||||
p = Point('Bandwidth').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Upload", bitsUpload)
|
bitsDownload = int(parentNode['bitsDownloadSinceLastQuery'])
|
||||||
queriesToSend.append(p)
|
bitsUpload = int(parentNode['bitsUploadSinceLastQuery'])
|
||||||
p = Point('Utilization').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", percentUtilizationDownload)
|
if (bitsDownload > 0) and (bitsUpload > 0):
|
||||||
queriesToSend.append(p)
|
percentUtilizationDownload = round((bitsDownload / round(parentNode['downloadMax'] * 1000000)), 4)
|
||||||
p = Point('Utilization').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Upload", percentUtilizationUpload)
|
percentUtilizationUpload = round((bitsUpload / round(parentNode['uploadMax'] * 1000000)), 4)
|
||||||
queriesToSend.append(p)
|
p = Point('Bandwidth').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", bitsDownload).field("Upload", bitsUpload)
|
||||||
|
queriesToSend.append(p)
|
||||||
|
p = Point('Utilization').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload)
|
||||||
|
queriesToSend.append(p)
|
||||||
|
|
||||||
write_api.write(bucket=bucket, record=queriesToSend)
|
write_api.write(bucket=influxDBBucket, record=queriesToSend)
|
||||||
print("Added " + str(len(queriesToSend)) + " points to InfluxDB.")
|
# print("Added " + str(len(queriesToSend)) + " points to InfluxDB.")
|
||||||
client.close()
|
queriesToSendCount += len(queriesToSend)
|
||||||
|
print("Added " + str(queriesToSendCount) + " points to InfluxDB.")
|
||||||
|
|
||||||
with open('statsByParentNode.json', 'w') as infile:
|
client.close()
|
||||||
json.dump(parentNodes, infile)
|
|
||||||
|
|
||||||
with open('statsByCircuit.json', 'w') as infile:
|
with open('statsByParentNode.json', 'w') as infile:
|
||||||
json.dump(subscriberCircuits, infile)
|
json.dump(parentNodes, infile)
|
||||||
endTime = datetime.now()
|
|
||||||
durationSeconds = round((endTime - startTime).total_seconds(),2)
|
with open('statsByCircuit.json', 'w') as infile:
|
||||||
print("Graphs updated within " + str(durationSeconds) + " seconds.")
|
json.dump(subscriberCircuits, infile)
|
||||||
|
|
||||||
|
endTime = datetime.now()
|
||||||
|
durationSeconds = round((endTime - startTime).total_seconds(), 2)
|
||||||
|
print("Graphs updated within " + str(durationSeconds) + " seconds.")
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
refreshBandwidthGraphs()
|
refreshBandwidthGraphs()
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user