From 4375211432db631a3a60f2bce0eb44d54706b4a4 Mon Sep 17 00:00:00 2001 From: Warren Date: Mon, 16 Jan 2023 09:52:10 +1300 Subject: [PATCH 1/3] fix typo --- src/TESTING-1.4.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TESTING-1.4.md b/src/TESTING-1.4.md index 1a53f779..015df913 100644 --- a/src/TESTING-1.4.md +++ b/src/TESTING-1.4.md @@ -23,7 +23,7 @@ git checkout v1.4-pre-alpha-rust-integration You need to have a few packages from `apt` installed: ``` -apt-get install -y python3-pip clang gcc gcc-multilib llvm libelf-dev git nano graphviz curl screen llvm pkg-config linux-tools-common linux-tools-`uname r` libbpf-dev +apt-get install -y python3-pip clang gcc gcc-multilib llvm libelf-dev git nano graphviz curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev ``` Then you need to install some Python dependencies: From b7b78962a34168a66b2f0082e5877983a2164057 Mon Sep 17 00:00:00 2001 From: Warren Date: Mon, 16 Jan 2023 16:31:18 +1300 Subject: [PATCH 2/3] reforked schedulerAdvanced, plus a few other bits --- src/integrationRestHttp.py | 79 ++++++++++++++++++++++++++++++++++++++ src/ispConfig.example.py | 41 +++++++++++++++----- src/schedulerAdvanced.py | 66 +++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 src/integrationRestHttp.py create mode 100644 src/schedulerAdvanced.py diff --git a/src/integrationRestHttp.py b/src/integrationRestHttp.py new file mode 100644 index 00000000..05d9eacd --- /dev/null +++ b/src/integrationRestHttp.py @@ -0,0 +1,79 @@ +import csv +import os +import shutil +from datetime import datetime + +from requests import get + +from ispConfig import automaticImportRestHttp as restconf +from pydash import objects + +requestsBaseConfig = { + 'verify': True, + 'headers': { + 'accept': 'application/json' + } +} + + +def createShaper(): + + # shutil.copy('Shaper.csv', 'Shaper.csv.bak') + ts = datetime.now().strftime('%Y-%m-%d.%H-%M-%S') + + devicesURL = restconf.get('baseURL') + '/' + restconf.get('devicesURI').strip('/') + + requestConfig = objects.defaults_deep({'params': {}}, restconf.get('requestsConfig'), requestsBaseConfig) + + raw = get(devicesURL, **requestConfig) + + if raw.status_code != 200: + print('Failed to request ' + devicesURL + ', got ' + str(raw.status_code)) + return False + + devicesCsvFP = os.path.dirname(os.path.realpath(__file__)) + '/ShapedDevices.csv' + + with open(devicesCsvFP, 'w') as csvfile: + wr = csv.writer(csvfile, quoting=csv.QUOTE_ALL) + wr.writerow( + ['Circuit ID', 'Circuit Name', 'Device ID', 'Device Name', 'Parent Node', 'MAC', 'IPv4', 'IPv6', + 'Download Min Mbps', 'Upload Min Mbps', 'Download Max Mbps', 'Upload Max Mbps', 'Comment']) + for row in raw.json(): + wr.writerow(row.values()) + + if restconf['logChanges']: + devicesBakFilePath = restconf['logChanges'].rstrip('/') + '/ShapedDevices.' + ts + '.csv' + try: + shutil.copy(devicesCsvFP, devicesBakFilePath) + except: + os.makedirs(restconf['logChanges'], exist_ok=True) + shutil.copy(devicesCsvFP, devicesBakFilePath) + + networkURL = restconf['baseURL'] + '/' + restconf['networkURI'].strip('/') + + raw = get(networkURL, **requestConfig) + + if raw.status_code != 200: + print('Failed to request ' + networkURL + ', got ' + str(raw.status_code)) + return False + + networkJsonFP = os.path.dirname(os.path.realpath(__file__)) + '/network.json' + + with open(networkJsonFP, 'w') as handler: + handler.write(raw.text) + + if restconf['logChanges']: + networkBakFilePath = restconf['logChanges'].rstrip('/') + '/network.' + ts + '.json' + try: + shutil.copy(networkJsonFP, networkBakFilePath) + except: + os.makedirs(restconf['logChanges'], exist_ok=True) + shutil.copy(networkJsonFP, networkBakFilePath) + + +def importFromRestHttp(): + createShaper() + + +if __name__ == '__main__': + importFromRestHttp() diff --git a/src/ispConfig.example.py b/src/ispConfig.example.py index 121c58d3..df75f043 100644 --- a/src/ispConfig.example.py +++ b/src/ispConfig.example.py @@ -23,8 +23,8 @@ interfaceA = 'eth1' # Interface connected to edge router interfaceB = 'eth2' -## WORK IN PROGRESS. Note that interfaceA determines the "stick" interface -## I could only get scanning to work if I issued ethtool -K enp1s0f1 rxvlan off +# WORK IN PROGRESS. Note that interfaceA determines the "stick" interface +# I could only get scanning to work if I issued ethtool -K enp1s0f1 rxvlan off OnAStick = False # VLAN facing the core router StickVlanA = 0 @@ -38,7 +38,8 @@ enableActualShellCommands = True # Add 'sudo' before execution of any shell commands. May be required depending on distribution and environment. runShellCommandsAsSudo = False -# Allows overriding queues / CPU cores used. When set to 0, the max possible queues / CPU cores are utilized. Please leave as 0. +# Allows overriding queues / CPU cores used. When set to 0, the max possible queues / CPU cores are utilized. Please +# leave as 0. queuesAvailableOverride = 0 # Some networks are flat - where there are no Parent Nodes defined in ShapedDevices.csv @@ -83,19 +84,41 @@ uispSite = '' uispStrategy = "full" # List any sites that should not be included, with each site name surrounded by '' and separated by commas excludeSites = [] -# If you use IPv6, this can be used to find associated IPv6 prefixes for your clients' IPv4 addresses, and match them to those devices +# If you use IPv6, this can be used to find associated IPv6 prefixes for your clients' IPv4 addresses, and match them +# to those devices findIPv6usingMikrotik = False -# If you want to provide a safe cushion for speed test results to prevent customer complains, you can set this to 1.15 (15% above plan rate). -# If not, you can leave as 1.0 +# If you want to provide a safe cushion for speed test results to prevent customer complains, you can set this to +# 1.15 (15% above plan rate). If not, you can leave as 1.0 bandwidthOverheadFactor = 1.0 # For edge cases, set the respective ParentNode for these CPEs exceptionCPEs = {} - # 'CPE-SomeLocation1': 'AP-SomeLocation1', - # 'CPE-SomeLocation2': 'AP-SomeLocation2', - #} +# exceptionCPEs = { +# 'CPE-SomeLocation1': 'AP-SomeLocation1', +# 'CPE-SomeLocation2': 'AP-SomeLocation2', +# } # API Auth apiUsername = "testUser" apiPassword = "changeme8343486806" apiHostIP = "127.0.0.1" apiHostPost = 5000 + + +httpRestIntegrationConfig = { + 'enabled': True, + 'baseURL': 'https://domain', + 'networkURI': '/some/path', + 'shaperURI': '/some/path/etc', + 'requestsConfig': { + 'verify': True, # Good for Dev if your dev env doesnt have cert + 'params': { # params for query string ie uri?some-arg=some-value + 'search': 'hold-my-beer' + }, + #'headers': { + # 'Origin': 'SomeHeaderValue', + #}, + }, + # If you want to store a timestamped copy/backup of both network.json and Shaper.csv each time they are updated, + # provide a path + # 'logChanges': '/var/log/libreqos' +} diff --git a/src/schedulerAdvanced.py b/src/schedulerAdvanced.py new file mode 100644 index 00000000..20a46268 --- /dev/null +++ b/src/schedulerAdvanced.py @@ -0,0 +1,66 @@ +import time + +from LibreQoS import refreshShapers, refreshShapersUpdateOnly +from graphInfluxDB import refreshBandwidthGraphs, refreshLatencyGraphs +from ispConfig import influxDBEnabled, automaticImportUISP, automaticImportSplynx, httpRestIntegrationConfig + +if automaticImportUISP: + from integrationUISP import importFromUISP +if automaticImportSplynx: + from integrationSplynx import importFromSplynx +if httpRestIntegrationConfig['enabled']: + from integrationRestHttp import importFromRestHttp + +from apscheduler.schedulers.background import BlockingScheduler + +ads = BlockingScheduler() + + +def importFromCRM(): + if automaticImportUISP: + try: + importFromUISP() + except: + print("Failed to import from UISP") + elif automaticImportSplynx: + try: + importFromSplynx() + except: + print("Failed to import from Splynx") + elif httpRestIntegrationConfig['enabled']: + try: + importFromRestHttp() + except: + print("Failed to import from RestHttp") + + +def importAndShapeFullReload(): + importFromCRM() + refreshShapers() + + +def importAndShapePartialReload(): + importFromCRM() + refreshShapersUpdateOnly() + + +if __name__ == '__main__': + importAndShapeFullReload() + # schedule.every().day.at("04:00").do(importAndShapeFullReload) + ads.add_job(importAndShapeFullReload, 'cron', hour=4) + + # schedule.every(30).minutes.do(importAndShapePartialReload) + ads.add_job(importAndShapePartialReload, 'interval', minutes=30) + + if influxDBEnabled: + # schedule.every(10).seconds.do(refreshBandwidthGraphs) + ads.add_job(refreshBandwidthGraphs, 'interval', seconds=10) + + # schedule.every(30).seconds.do(refreshLatencyGraphs) + ads.add_job(refreshLatencyGraphs, 'interval', seconds=30) + + # while True: + # schedule.run_pending() + # time.sleep(1) + + ads.start() From e4ae7c8cf3eba4c35e32f4650ab4e65552687ac3 Mon Sep 17 00:00:00 2001 From: Warren Date: Mon, 16 Jan 2023 16:49:17 +1300 Subject: [PATCH 3/3] Default http api client back to false --- src/ispConfig.example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ispConfig.example.py b/src/ispConfig.example.py index df75f043..3e3a87e6 100644 --- a/src/ispConfig.example.py +++ b/src/ispConfig.example.py @@ -105,7 +105,7 @@ apiHostPost = 5000 httpRestIntegrationConfig = { - 'enabled': True, + 'enabled': False, 'baseURL': 'https://domain', 'networkURI': '/some/path', 'shaperURI': '/some/path/etc',