diff --git a/src/graphInfluxDB.py b/src/graphInfluxDB.py index 73965057..7f4e2b78 100644 --- a/src/graphInfluxDB.py +++ b/src/graphInfluxDB.py @@ -12,6 +12,7 @@ from influxdb_client.client.write_api import SYNCHRONOUS from ispConfig import interfaceA, interfaceB, influxDBEnabled, influxDBBucket, influxDBOrg, influxDBtoken, influxDBurl, sqm + def getInterfaceStats(interface): command = 'tc -j -s qdisc show dev ' + interface jsonAr = json.loads(subprocess.run(command.split(' '), stdout=subprocess.PIPE).stdout.decode('utf-8')) @@ -43,15 +44,6 @@ def getCircuitBandwidthStats(subscriberCircuits, tinsStats): #circuit['stats']['priorQuery']['time'] = datetime.now().isoformat() circuit['stats']['currentQuery'] = {} circuit['stats']['sinceLastQuery'] = {} - if 'tinsStats' not in circuit: - circuit['tinsStats'] = {} - if 'currentQuery' in circuit['tinsStats']: - circuit['tinsStats']['priorQuery'] = circuit['tinsStats']['currentQuery'] - circuit['tinsStats']['currentQuery'] = {} - circuit['tinsStats']['sinceLastQuery'] = {} - else: - circuit['tinsStats']['currentQuery'] = {} - circuit['tinsStats']['sinceLastQuery'] = {} #for entry in tinsStats: if 'currentQuery' in tinsStats: @@ -74,16 +66,6 @@ def getCircuitBandwidthStats(subscriberCircuits, tinsStats): } for circuit in subscriberCircuits: - circuit['tinsStats']['currentQuery'] = { 'Bulk': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'BestEffort': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'Video': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'Voice': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - } - circuit['tinsStats']['sinceLastQuery'] = { 'Bulk': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'BestEffort': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'Video': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - 'Voice': {'Download': {'sent_packets': 0.0, 'drops': 0.0}, 'Upload': {'sent_packets': 0.0, 'drops': 0.0}}, - } for (interface, stats, dirSuffix) in zip(interfaces, ifaceStats, ['Download', 'Upload']): element = stats[circuit['classid']] if circuit['classid'] in stats else False @@ -107,24 +89,16 @@ def getCircuitBandwidthStats(subscriberCircuits, tinsStats): trueDrops = ecn_mark + tinDrops - ack_drops if tinCounter == 1: tinsStats['currentQuery']['Bulk'][dirSuffix]['sent_packets'] += sent_packets - circuit['tinsStats']['currentQuery']['Bulk'][dirSuffix]['sent_packets'] += sent_packets tinsStats['currentQuery']['Bulk'][dirSuffix]['drops'] += trueDrops - circuit['tinsStats']['currentQuery']['Bulk'][dirSuffix]['drops'] += trueDrops elif tinCounter == 2: tinsStats['currentQuery']['BestEffort'][dirSuffix]['sent_packets'] += sent_packets - circuit['tinsStats']['currentQuery']['BestEffort'][dirSuffix]['sent_packets'] += sent_packets tinsStats['currentQuery']['BestEffort'][dirSuffix]['drops'] += trueDrops - circuit['tinsStats']['currentQuery']['BestEffort'][dirSuffix]['drops'] += trueDrops elif tinCounter == 3: tinsStats['currentQuery']['Video'][dirSuffix]['sent_packets'] += sent_packets - circuit['tinsStats']['currentQuery']['Video'][dirSuffix]['sent_packets'] += sent_packets tinsStats['currentQuery']['Video'][dirSuffix]['drops'] += trueDrops - circuit['tinsStats']['currentQuery']['Video'][dirSuffix]['drops'] += trueDrops elif tinCounter == 4: tinsStats['currentQuery']['Voice'][dirSuffix]['sent_packets'] += sent_packets - circuit['tinsStats']['currentQuery']['Voice'][dirSuffix]['sent_packets'] += sent_packets tinsStats['currentQuery']['Voice'][dirSuffix]['drops'] += trueDrops - circuit['tinsStats']['currentQuery']['Voice'][dirSuffix]['drops'] += trueDrops tinCounter += 1 circuit['stats']['currentQuery']['bytesSent' + dirSuffix] = bytesSent @@ -140,121 +114,163 @@ def getCircuitBandwidthStats(subscriberCircuits, tinsStats): allPacketsDownload = 0.0 allPacketsUpload = 0.0 for circuit in subscriberCircuits: - circuit['stats']['sinceLastQuery']['bitsDownload'] = circuit['stats']['sinceLastQuery']['bitsUpload'] = 0.0 - circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['sinceLastQuery']['bytesSentUpload'] = 0.0 - circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['sinceLastQuery']['packetDropsUpload'] = 0.0 - circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['sinceLastQuery']['packetsSentUpload'] = 0.0 + circuit['stats']['sinceLastQuery']['bitsDownload'] = circuit['stats']['sinceLastQuery']['bitsUpload'] = None + circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None + circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None + circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None try: if (circuit['stats']['currentQuery']['bytesSentDownload'] - circuit['stats']['priorQuery']['bytesSentDownload']) >= 0.0: circuit['stats']['sinceLastQuery']['bytesSentDownload'] = circuit['stats']['currentQuery']['bytesSentDownload'] - circuit['stats']['priorQuery']['bytesSentDownload'] else: - circuit['stats']['sinceLastQuery']['bytesSentDownload'] = 0.0 + circuit['stats']['sinceLastQuery']['bytesSentDownload'] = None if (circuit['stats']['currentQuery']['bytesSentUpload'] - circuit['stats']['priorQuery']['bytesSentUpload']) >= 0.0: circuit['stats']['sinceLastQuery']['bytesSentUpload'] = circuit['stats']['currentQuery']['bytesSentUpload'] - circuit['stats']['priorQuery']['bytesSentUpload'] else: - circuit['stats']['sinceLastQuery']['bytesSentUpload'] = 0.0 + circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None except: - circuit['stats']['sinceLastQuery']['bytesSentDownload'] = 0.0 - circuit['stats']['sinceLastQuery']['bytesSentUpload'] = 0.0 + circuit['stats']['sinceLastQuery']['bytesSentDownload'] = None + circuit['stats']['sinceLastQuery']['bytesSentUpload'] = None try: if (circuit['stats']['currentQuery']['packetDropsDownload'] - circuit['stats']['priorQuery']['packetDropsDownload']) >= 0.0: circuit['stats']['sinceLastQuery']['packetDropsDownload'] = circuit['stats']['currentQuery']['packetDropsDownload'] - circuit['stats']['priorQuery']['packetDropsDownload'] else: - circuit['stats']['sinceLastQuery']['packetDropsDownload'] = 0.0 + circuit['stats']['sinceLastQuery']['packetDropsDownload'] = None if (circuit['stats']['currentQuery']['packetDropsUpload'] - circuit['stats']['priorQuery']['packetDropsUpload']) >= 0.0: circuit['stats']['sinceLastQuery']['packetDropsUpload'] = circuit['stats']['currentQuery']['packetDropsUpload'] - circuit['stats']['priorQuery']['packetDropsUpload'] else: - circuit['stats']['sinceLastQuery']['packetDropsUpload'] = 0.0 + circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None except: - circuit['stats']['sinceLastQuery']['packetDropsDownload'] = 0.0 - circuit['stats']['sinceLastQuery']['packetDropsUpload'] = 0.0 + circuit['stats']['sinceLastQuery']['packetDropsDownload'] = None + circuit['stats']['sinceLastQuery']['packetDropsUpload'] = None try: if (circuit['stats']['currentQuery']['packetsSentDownload'] - circuit['stats']['priorQuery']['packetsSentDownload']) >= 0.0: circuit['stats']['sinceLastQuery']['packetsSentDownload'] = circuit['stats']['currentQuery']['packetsSentDownload'] - circuit['stats']['priorQuery']['packetsSentDownload'] else: - circuit['stats']['sinceLastQuery']['packetsSentDownload'] = 0.0 + circuit['stats']['sinceLastQuery']['packetsSentDownload'] = None if (circuit['stats']['currentQuery']['packetsSentUpload'] - circuit['stats']['priorQuery']['packetsSentUpload']) >= 0.0: circuit['stats']['sinceLastQuery']['packetsSentUpload'] = circuit['stats']['currentQuery']['packetsSentUpload'] - circuit['stats']['priorQuery']['packetsSentUpload'] else: - circuit['stats']['sinceLastQuery']['packetsSentUpload'] = 0.0 + circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None except: - circuit['stats']['sinceLastQuery']['packetsSentDownload'] = 0.0 - circuit['stats']['sinceLastQuery']['packetsSentUpload'] = 0.0 + circuit['stats']['sinceLastQuery']['packetsSentDownload'] = None + circuit['stats']['sinceLastQuery']['packetsSentUpload'] = None - allPacketsDownload += circuit['stats']['sinceLastQuery']['packetsSentDownload'] - allPacketsUpload += circuit['stats']['sinceLastQuery']['packetsSentUpload'] + if(circuit['stats']['sinceLastQuery']['packetsSentDownload']): + allPacketsDownload += circuit['stats']['sinceLastQuery']['packetsSentDownload'] + if(circuit['stats']['sinceLastQuery']['packetsSentUpload']): + allPacketsUpload += circuit['stats']['sinceLastQuery']['packetsSentUpload'] if 'priorQuery' in circuit['stats']: if 'time' in circuit['stats']['priorQuery']: currentQueryTime = datetime.fromisoformat(circuit['stats']['currentQuery']['time']) priorQueryTime = datetime.fromisoformat(circuit['stats']['priorQuery']['time']) deltaSeconds = (currentQueryTime - priorQueryTime).total_seconds() - if deltaSeconds < 0: - print("Current query time ({}) is before last query time ({}), time sync may be broken. serviceId is {}. Name is: {}.".format(currentQueryTime, priorQueryTime, circuit['circuitID'], circuit['circuitName'])) - circuit['stats']['sinceLastQuery']['bitsDownload'] = round((circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 - circuit['stats']['sinceLastQuery']['bitsUpload'] = round((circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 + if (circuit['stats']['sinceLastQuery']['bytesSentDownload']): + circuit['stats']['sinceLastQuery']['bitsDownload'] = round((circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 + else: + circuit['stats']['sinceLastQuery']['bitsDownload'] = None + if (circuit['stats']['sinceLastQuery']['bytesSentUpload']): + circuit['stats']['sinceLastQuery']['bitsUpload'] = round((circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) / deltaSeconds) if deltaSeconds > 0 else 0 + else: + circuit['stats']['sinceLastQuery']['bitsUpload'] = None + else: - circuit['stats']['sinceLastQuery']['bitsDownload'] = (circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) - circuit['stats']['sinceLastQuery']['bitsUpload'] = (circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) + circuit['stats']['sinceLastQuery']['bitsDownload'] = None + if(circuit['stats']['sinceLastQuery']['bytesSentDownload']): + circuit['stats']['sinceLastQuery']['bitsDownload'] = (circuit['stats']['sinceLastQuery']['bytesSentDownload'] * 8) + circuit['stats']['sinceLastQuery']['bitsUpload'] = None + if(circuit['stats']['sinceLastQuery']['bytesSentUpload']): + circuit['stats']['sinceLastQuery']['bitsUpload'] = (circuit['stats']['sinceLastQuery']['bytesSentUpload'] * 8) + + tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = 0.0 + + tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = 0.0 + tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = 0.0 + tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = 0.0 + + try: + tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] = tinsStats['currentQuery']['Bulk']['Download']['sent_packets'] - tinsStats['priorQuery']['Bulk']['Download']['sent_packets'] + tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] = tinsStats['currentQuery']['BestEffort']['Download']['sent_packets'] - tinsStats['priorQuery']['BestEffort']['Download']['sent_packets'] + tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] = tinsStats['currentQuery']['Video']['Download']['sent_packets'] - tinsStats['priorQuery']['Video']['Download']['sent_packets'] + tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] = tinsStats['currentQuery']['Voice']['Download']['sent_packets'] - tinsStats['priorQuery']['Voice']['Download']['sent_packets'] + tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] = tinsStats['currentQuery']['Bulk']['Upload']['sent_packets'] - tinsStats['priorQuery']['Bulk']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] = tinsStats['currentQuery']['BestEffort']['Upload']['sent_packets'] - tinsStats['priorQuery']['BestEffort']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] = tinsStats['currentQuery']['Video']['Upload']['sent_packets'] - tinsStats['priorQuery']['Video']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] = tinsStats['currentQuery']['Voice']['Upload']['sent_packets'] - tinsStats['priorQuery']['Voice']['Upload']['sent_packets'] + except: + tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] = tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] = tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] = 0.0 + tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] = tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] = 0.0 - # Process Tin Statistics by Circuit - circuit['tinsStats'] = buildTinStats(circuit['tinsStats'], circuit['stats']['sinceLastQuery']['packetsSentDownload'], circuit['stats']['sinceLastQuery']['packetsSentUpload']) + try: + tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] = tinsStats['currentQuery']['Bulk']['Download']['drops'] - tinsStats['priorQuery']['Bulk']['Download']['drops'] + tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] = tinsStats['currentQuery']['BestEffort']['Download']['drops'] - tinsStats['priorQuery']['BestEffort']['Download']['drops'] + tinsStats['sinceLastQuery']['Video']['Download']['drops'] = tinsStats['currentQuery']['Video']['Download']['drops'] - tinsStats['priorQuery']['Video']['Download']['drops'] + tinsStats['sinceLastQuery']['Voice']['Download']['drops'] = tinsStats['currentQuery']['Voice']['Download']['drops'] - tinsStats['priorQuery']['Voice']['Download']['drops'] + tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] = tinsStats['currentQuery']['Bulk']['Upload']['drops'] - tinsStats['priorQuery']['Bulk']['Upload']['drops'] + tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] = tinsStats['currentQuery']['BestEffort']['Upload']['drops'] - tinsStats['priorQuery']['BestEffort']['Upload']['drops'] + tinsStats['sinceLastQuery']['Video']['Upload']['drops'] = tinsStats['currentQuery']['Video']['Upload']['drops'] - tinsStats['priorQuery']['Video']['Upload']['drops'] + tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] = tinsStats['currentQuery']['Voice']['Upload']['drops'] - tinsStats['priorQuery']['Voice']['Upload']['drops'] + except: + tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] = tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Download']['drops'] = tinsStats['sinceLastQuery']['Voice']['Download']['drops'] = 0.0 + tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Upload']['drops'] = tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] = 0.0 - # Process Network-Level Tin Statistics - tinsStats = buildTinStats(tinsStats, allPacketsDownload, allPacketsUpload) + try: + dlPerc = tinsStats['sinceLastQuery']['Bulk']['Download']['drops'] / tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets'] + ulPerc = tinsStats['sinceLastQuery']['Bulk']['Upload']['drops'] / tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) + tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) + + dlPerc = tinsStats['sinceLastQuery']['BestEffort']['Download']['drops'] / tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets'] + ulPerc = tinsStats['sinceLastQuery']['BestEffort']['Upload']['drops'] / tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) + tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) + + dlPerc = tinsStats['sinceLastQuery']['Video']['Download']['drops'] / tinsStats['sinceLastQuery']['Video']['Download']['sent_packets'] + ulPerc = tinsStats['sinceLastQuery']['Video']['Upload']['drops'] / tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) + tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) + + dlPerc = tinsStats['sinceLastQuery']['Voice']['Download']['drops'] / tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets'] + ulPerc = tinsStats['sinceLastQuery']['Voice']['Upload']['drops'] / tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets'] + tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = max(round(dlPerc * 100.0, 3),0.0) + tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = max(round(ulPerc * 100.0, 3),0.0) + except: + tinsStats['sinceLastQuery']['Bulk']['Download']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Bulk']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['BestEffort']['Download']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['BestEffort']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Download']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Video']['Upload']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Voice']['Download']['dropPercentage'] = 0.0 + tinsStats['sinceLastQuery']['Voice']['Upload']['dropPercentage'] = 0.0 + + try: + tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Bulk']['Download']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Bulk']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['BestEffort']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['BestEffort']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Video']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Video']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = min(round((tinsStats['sinceLastQuery']['Voice']['Download']['sent_packets']/allPacketsDownload)*100.0, 3),100.0) + tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = min(round((tinsStats['sinceLastQuery']['Voice']['Upload']['sent_packets']/allPacketsUpload)*100.0, 3),100.0) + except: + # To avoid graphing 0.0 for all categories, which would show unusual graph results upon each queue reload, we just set these to None if the above calculations fail. + tinsStats['sinceLastQuery']['Bulk']['Download']['percentage'] = tinsStats['sinceLastQuery']['Bulk']['Upload']['percentage'] = None + tinsStats['sinceLastQuery']['BestEffort']['Download']['percentage'] = tinsStats['sinceLastQuery']['BestEffort']['Upload']['percentage'] = None + tinsStats['sinceLastQuery']['Video']['Download']['percentage'] = tinsStats['sinceLastQuery']['Video']['Upload']['percentage'] = None + tinsStats['sinceLastQuery']['Voice']['Download']['percentage'] = tinsStats['sinceLastQuery']['Voice']['Upload']['percentage'] = None return subscriberCircuits, tinsStats -def buildTinStats(data, allPacketsDownload, allPacketsUpload): - # data is the dictionary that we want to do the calculations for, usually circuit['tinsStats'] or tinsStats. - allPackets = { - "Download": allPacketsDownload, - "Upload": allPacketsUpload - } - for tinK, tinV in data['sinceLastQuery'].items(): - for directionK, directionV in tinV.items(): - try: - currentQuerySentPackets = data['currentQuery'][tinK][directionK]['sent_packets'] if data['currentQuery'][tinK][directionK]['sent_packets'] > 0 else 0.0 - except KeyError: - currentQuerySentPackets = 0.0 - try: - priorQuerySentPackets = data['priorQuery'][tinK][directionK]['sent_packets'] if data['priorQuery'][tinK][directionK]['sent_packets'] > 0 else 0.0 - except KeyError: - priorQuerySentPackets = 0.0 - try: - data['sinceLastQuery'][tinK][directionK]['sent_packets'] = currentQuerySentPackets - priorQuerySentPackets - except Exception as e: - print("QoE Tins Sent Packet Broken: {}".format(e)) - - try: - currentQueryDrops = data['currentQuery'][tinK][directionK]['drops'] if data['currentQuery'][tinK][directionK]['drops'] > 0 else 0.0 - except KeyError: - currentQueryDrops = 0.0 - try: - priorQueryDrops = data['priorQuery'][tinK][directionK]['drops'] if data['priorQuery'][tinK][directionK]['drops'] > 0 else 0.0 - except KeyError: - priorQueryDrops = 0.0 - try: - data['sinceLastQuery'][tinK][directionK]['drops'] = currentQueryDrops - priorQueryDrops - except Exception as e: - print("QoE Tins Drops Broken: {}".format(e)) - - try: - directionPercentage = directionV['drops'] / directionV['sent_packets'] if directionV['sent_packets'] > 0 else 0.0 - except KeyError: - directionPercentage = 0.0 - try: - data['sinceLastQuery'][tinK][directionK]['dropPercentage'] = max(round(directionPercentage * 100.0, 3), 0.0) - except Exception as e: - print("QoE Tins Drop Percentage Broken: {}".format(e)) - - try: - data['sinceLastQuery'][tinK][directionK]['utilizationPercentage'] = min(round((directionV['sent_packets'] / allPackets[directionK]) * 100.0, 3), 100.0) if allPackets[directionK] > 0 else 0.0 - except Exception as e: - print("QoE Tins Percentage Utilization Broken: {}".format(e)) - - return data def getParentNodeBandwidthStats(parentNodes, subscriberCircuits): for parentNode in parentNodes: @@ -270,14 +286,20 @@ def getParentNodeBandwidthStats(parentNodes, subscriberCircuits): thisParentNodeStats = {'sinceLastQuery': {}} for circuit in subscriberCircuits: if circuit['ParentNode'] == parentNode['parentNodeName']: - thisNodeBitsDownload += circuit['stats']['sinceLastQuery']['bitsDownload'] - thisNodeBitsUpload += circuit['stats']['sinceLastQuery']['bitsUpload'] + if circuit['stats']['sinceLastQuery']['bitsDownload']: + thisNodeBitsDownload += circuit['stats']['sinceLastQuery']['bitsDownload'] + if circuit['stats']['sinceLastQuery']['bitsUpload']: + thisNodeBitsUpload += circuit['stats']['sinceLastQuery']['bitsUpload'] #thisNodeDropsDownload += circuit['packetDropsDownloadSinceLastQuery'] #thisNodeDropsUpload += circuit['packetDropsUploadSinceLastQuery'] - thisNodeDropsTotal += (circuit['stats']['sinceLastQuery']['packetDropsDownload'] + circuit['stats']['sinceLastQuery']['packetDropsUpload']) - packetsSentDownloadAggregate += circuit['stats']['sinceLastQuery']['packetsSentDownload'] - packetsSentUploadAggregate += circuit['stats']['sinceLastQuery']['packetsSentUpload'] - packetsSentTotalAggregate += (circuit['stats']['sinceLastQuery']['packetsSentDownload'] + circuit['stats']['sinceLastQuery']['packetsSentUpload']) + if circuit['stats']['sinceLastQuery']['packetDropsDownload'] and circuit['stats']['sinceLastQuery']['packetDropsUpload']: + thisNodeDropsTotal += (circuit['stats']['sinceLastQuery']['packetDropsDownload'] + circuit['stats']['sinceLastQuery']['packetDropsUpload']) + if circuit['stats']['sinceLastQuery']['packetsSentDownload']: + packetsSentDownloadAggregate += circuit['stats']['sinceLastQuery']['packetsSentDownload'] + if circuit['stats']['sinceLastQuery']['packetsSentUpload']: + packetsSentUploadAggregate += circuit['stats']['sinceLastQuery']['packetsSentUpload'] + if circuit['stats']['sinceLastQuery']['packetsSentDownload'] and circuit['stats']['sinceLastQuery']['packetsSentUpload']: + packetsSentTotalAggregate += (circuit['stats']['sinceLastQuery']['packetsSentDownload'] + circuit['stats']['sinceLastQuery']['packetsSentUpload']) circuitsMatched += 1 if (packetsSentDownloadAggregate > 0) and (packetsSentUploadAggregate > 0): #overloadFactorDownloadSinceLastQuery = float(round((thisNodeDropsDownload/packetsSentDownloadAggregate)*100.0, 3)) @@ -341,15 +363,15 @@ def getCircuitLatencyStats(subscriberCircuits): if classID in tcpLatencyForClassID: circuit['stats']['sinceLastQuery']['tcpLatency'] = tcpLatencyForClassID[classID] else: + # If we can't identify RTT this time around, use most recently recorded RTT + # None by default, change if found in priorQuery circuit['stats']['sinceLastQuery']['tcpLatency'] = None - try: + if 'priorQuery' in circuit['stats']: if circuit['stats']['priorQuery'] != None: if 'priorQuery' in circuit['stats']: if 'tcpLatency' in circuit['stats']['priorQuery']: circuit['stats']['sinceLastQuery']['tcpLatency'] = circuit['stats']['priorQuery']['tcpLatency'] - except: - circuit['stats']['sinceLastQuery']['tcpLatency'] = None - # priorQuery had no latency information, using None instead + return subscriberCircuits @@ -419,36 +441,34 @@ def refreshBandwidthGraphs(): queriesToSendCount = 0 for chunk in chunkedsubscriberCircuits: + seenSomethingBesides0s = False queriesToSend = [] for circuit in chunk: - bitsDownloadMin = float(circuit['minDownload']) * 1000000 - bitsDownloadMax = float(circuit['maxDownload']) * 1000000 - bitsUploadMin = float(circuit['minUpload']) * 1000000 - bitsUploadMax = float(circuit['maxUpload']) * 1000000 - bitsDownload = float(circuit['stats']['sinceLastQuery']['bitsDownload']) - bitsUpload = float(circuit['stats']['sinceLastQuery']['bitsUpload']) - bytesSentDownload = float(circuit['stats']['sinceLastQuery']['bytesSentDownload']) - bytesSentUpload = float(circuit['stats']['sinceLastQuery']['bytesSentUpload']) - percentUtilizationDownload = round((bitsDownload / round(circuit['maxDownload'] * 1000000))*100.0, 1) - percentUtilizationUpload = round((bitsUpload / round(circuit['maxUpload'] * 1000000))*100.0, 1) - p = Point('Bandwidth').tag("CircuitID", circuit['circuitID']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", bitsDownload).field("Upload", bitsUpload).field("Download Minimum", bitsDownloadMin).field("Download Maximum", bitsDownloadMax).field("Upload Minimum", bitsUploadMin).field("Upload Maximum", bitsUploadMax).time(timestamp) + bitsDownloadMin = float(circuit['minDownload']) * 1000000 if circuit['minDownload'] else None + bitsDownloadMax = float(circuit['maxDownload']) * 1000000 if circuit['maxDownload'] else None + bitsUploadMin = float(circuit['minUpload']) * 1000000 if circuit['minUpload'] else None + bitsUploadMax = float(circuit['maxUpload']) * 1000000 if circuit['maxUpload'] else None + bitsDownload = float(circuit['stats']['sinceLastQuery']['bitsDownload']) if circuit['stats']['sinceLastQuery']['bitsDownload'] else None + bitsUpload = float(circuit['stats']['sinceLastQuery']['bitsUpload']) if circuit['stats']['sinceLastQuery']['bitsUpload'] else None + bytesSentDownload = float(circuit['stats']['sinceLastQuery']['bytesSentDownload']) if circuit['stats']['sinceLastQuery']['bytesSentDownload'] else None + bytesSentUpload = float(circuit['stats']['sinceLastQuery']['bytesSentUpload']) if circuit['stats']['sinceLastQuery']['bytesSentUpload'] else None + percentUtilizationDownload = round((bitsDownload / round(circuit['maxDownload'] * 1000000))*100.0, 1) if bitsDownload and circuit['maxDownload'] else None + percentUtilizationUpload = round((bitsUpload / round(circuit['maxUpload'] * 1000000))*100.0, 1) if bitsUpload and circuit['maxUpload'] else None + if bitsDownload and bitsUpload: + if (bitsDownload > 0.0) or (bitsUpload > 0.0): + seenSomethingBesides0s = True + p = Point('Bandwidth').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", bitsDownload).field("Upload", bitsUpload).time(timestamp) queriesToSend.append(p) - p = Point('Utilization').tag("CircuitID", circuit['circuitID']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) + p = Point('Utilization').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) queriesToSend.append(p) - p = Point('BandwidthUsage').tag("CircuitID", circuit['circuitID']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("Download", bytesSentDownload).field("Upload", bytesSentUpload).time(timestamp) - queriesToSend.append(p) - # Parse tins by circuit and parent node and ship to InfluxDB - for tinK, tinV in circuit['tinsStats']['sinceLastQuery'].items(): - for directionK, directionV in tinV.items(): - for metricK, metricV in directionV.items(): - p = Point('Tins By Circuit and Parent Node').tag("CircuitID", circuit['circuitID']).tag("ParentNode", circuit['ParentNode']).tag("Tin", tinK).tag("Direction", directionK).field(metricK, metricV).time(timestamp) - queriesToSend.append(p) - write_api.write(bucket=influxDBBucket, record=queriesToSend) + if seenSomethingBesides0s: + write_api.write(bucket=influxDBBucket, record=queriesToSend) # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") queriesToSendCount += len(queriesToSend) queriesToSend = [] + seenSomethingBesides0s = False for parentNode in parentNodes: bitsDownload = float(parentNode['stats']['sinceLastQuery']['bitsDownload']) bitsUpload = float(parentNode['stats']['sinceLastQuery']['bitsUpload']) @@ -457,6 +477,9 @@ def refreshBandwidthGraphs(): droppedPacketsAllTime += dropsTotal percentUtilizationDownload = round((bitsDownload / round(parentNode['maxDownload'] * 1000000))*100.0, 1) percentUtilizationUpload = round((bitsUpload / round(parentNode['maxUpload'] * 1000000))*100.0, 1) + if bitsDownload and bitsUpload: + if (bitsDownload > 0.0) or (bitsUpload > 0.0): + seenSomethingBesides0s = True p = Point('Bandwidth').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", bitsDownload).field("Upload", bitsUpload).time(timestamp) queriesToSend.append(p) p = Point('Utilization').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Download", percentUtilizationDownload).field("Upload", percentUtilizationUpload).time(timestamp) @@ -464,20 +487,27 @@ def refreshBandwidthGraphs(): p = Point('Overload').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("Overload", overloadFactor).time(timestamp) queriesToSend.append(p) - write_api.write(bucket=influxDBBucket, record=queriesToSend) + if seenSomethingBesides0s: + write_api.write(bucket=influxDBBucket, record=queriesToSend) # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") queriesToSendCount += len(queriesToSend) if 'cake diffserv4' in sqm: + seenSomethingBesides0s = False queriesToSend = [] listOfTins = ['Bulk', 'BestEffort', 'Video', 'Voice'] for tin in listOfTins: p = Point('Tin Drop Percentage').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['dropPercentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['dropPercentage']).time(timestamp) queriesToSend.append(p) - p = Point('Tins Utilization').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['utilizationPercentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['utilizationPercentage']).time(timestamp) - queriesToSend.append(p) + # Check to ensure tin percentage has value (!= None) before graphing. During partial or full reload these will have a value of None. + if (tinsStats['sinceLastQuery'][tin]['Download']['percentage'] != None) and (tinsStats['sinceLastQuery'][tin]['Upload']['percentage'] != None): + if (tinsStats['sinceLastQuery'][tin]['Download']['percentage'] > 0.0) or (tinsStats['sinceLastQuery'][tin]['Upload']['percentage'] > 0.0): + seenSomethingBesides0s = True + p = Point('Tins Assigned').tag("Type", "Tin").tag("Tin", tin).field("Download", tinsStats['sinceLastQuery'][tin]['Download']['percentage']).field("Upload", tinsStats['sinceLastQuery'][tin]['Upload']['percentage']).time(timestamp) + queriesToSend.append(p) - write_api.write(bucket=influxDBBucket, record=queriesToSend) + if seenSomethingBesides0s: + write_api.write(bucket=influxDBBucket, record=queriesToSend) # print("Added " + str(len(queriesToSend)) + " points to InfluxDB.") queriesToSendCount += len(queriesToSend) @@ -545,7 +575,7 @@ def refreshLatencyGraphs(): for circuit in chunk: if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: tcpLatency = float(circuit['stats']['sinceLastQuery']['tcpLatency']) - p = Point('TCP Latency').tag("CircuitID", circuit['circuitID']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("TCP Latency", tcpLatency).time(timestamp) + p = Point('TCP Latency').tag("Circuit", circuit['circuitName']).tag("ParentNode", circuit['ParentNode']).tag("Type", "Circuit").field("TCP Latency", tcpLatency).time(timestamp) queriesToSend.append(p) write_api.write(bucket=influxDBBucket, record=queriesToSend) queriesToSendCount += len(queriesToSend) @@ -565,7 +595,7 @@ def refreshLatencyGraphs(): if circuit['stats']['sinceLastQuery']['tcpLatency'] != None: listOfAllLatencies.append(circuit['stats']['sinceLastQuery']['tcpLatency']) if len(listOfAllLatencies) > 0: - currentNetworkLatency = float(statistics.median(listOfAllLatencies)) + currentNetworkLatency = statistics.median(listOfAllLatencies) p = Point('TCP Latency').tag("Type", "Network").field("TCP Latency", currentNetworkLatency).time(timestamp) write_api.write(bucket=influxDBBucket, record=p) queriesToSendCount += 1