From 09707d0dbd35c2d1abd23e3acde8e3009f5f343d Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Fri, 14 Jun 2024 14:54:47 -0500 Subject: [PATCH 01/15] Update the global version string, triggering upgrade prompts. Don't merge this until we're ready. --- src/VERSION_STRING | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VERSION_STRING b/src/VERSION_STRING index 840ca8cb..ecd29bf5 100644 --- a/src/VERSION_STRING +++ b/src/VERSION_STRING @@ -1 +1 @@ -1.4 \ No newline at end of file +1.5-BETA1 \ No newline at end of file From 18344c7cd2c622eca0e122fa23a3c3484150c468 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Wed, 19 Jun 2024 15:20:34 -0500 Subject: [PATCH 02/15] Add support for bin/post_integration_hook.sh in scheduler.py Scheduler.py now checks for the existence of a `bin/post_integration_hook.sh` script. If it exists, it is executed immediately after CRM updating and before LibreQoS is refreshed. This gives an opportuntity to start adding additional chains of calls to work with your shaped devices and network.json after CRM retrieval, and before pushing to live. --- src/scheduler.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/scheduler.py b/src/scheduler.py index 17f33bfc..85041606 100644 --- a/src/scheduler.py +++ b/src/scheduler.py @@ -13,6 +13,7 @@ if automatic_import_sonar(): from integrationSonar import importFromSonar from apscheduler.schedulers.background import BlockingScheduler from apscheduler.executors.pool import ThreadPoolExecutor +import os.path ads = BlockingScheduler(executors={'default': ThreadPoolExecutor(1)}) @@ -40,6 +41,12 @@ def importFromCRM(): except: print("Failed to import from Sonar") + # Post-CRM Hooks + path = get_libreqos_directory() + "/bin/post_integration_hook.sh" + binPath = get_libreqos_directory() + "/bin" + if os.path.isfile(path): + subprocess.Popen(path, cwd=binPath) + def graphHandler(): try: refreshBandwidthGraphs() From 09c3d0c3ffdb6839563ccfb6229124364a33095a Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 27 Jun 2024 15:21:59 -0500 Subject: [PATCH 03/15] Update build_dpkg commands to issue --- src/build_dpkg.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build_dpkg.sh b/src/build_dpkg.sh index 08306870..ad7bfd88 100755 --- a/src/build_dpkg.sh +++ b/src/build_dpkg.sh @@ -63,8 +63,8 @@ echo "pushd /opt/libreqos" >> postinst # - Setup Python dependencies as a post-install task while requirement= read -r line do - echo "python3 -m pip install $line" >> postinst - echo "sudo python3 -m pip install $line" >> postinst + echo "python3 -m pip install $line --break-system-packages" >> postinst + echo "sudo python3 -m pip install $line --break-system-packages" >> postinst done < ../../../../requirements.txt # - Run lqsetup echo "/opt/libreqos/src/bin/lqos_setup" >> postinst From 68abb2b5b422211f798bd91d8e619fca90448112 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 27 Jun 2024 16:14:33 -0500 Subject: [PATCH 04/15] Possible solution to libpython linkage issues for .deb build --- src/build_dpkg.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/build_dpkg.sh b/src/build_dpkg.sh index ad7bfd88..1b3400b7 100755 --- a/src/build_dpkg.sh +++ b/src/build_dpkg.sh @@ -78,6 +78,18 @@ echo "/bin/systemctl start lqosd" >> postinst echo "/bin/systemctl start lqos_node_manager" >> postinst echo "/bin/systemctl start lqos_scheduler" >> postinst echo "popd" >> postinst +# Attempting to fixup versioning issues with libpython. +# This requires that you already have LibreQoS installed. +LINKED_PYTHON=$(ldd /opt/libreqos/src/bin/lqosd | grep libpython | sed -e '/^[^\t]/ d' | sed -e 's/\t//' | sed -e 's/.*=..//' | sed -e 's/ (0.*)//') +echo "if ! test -f $LINKED_PYTHON; then" >> postinst +echo " if test -f /lib/x86_64-linux-gnu/libpython3.12.so.1.0; then" >> postinst +echo " ln -s $LINKED_PYTHON /lib/x86_64-linux-gnu/libpython3.10.so.1.0" >> postinst +echo " fi" >> postinst +echo " if test -f /lib/x86_64-linux-gnu/libpython3.11.so.1.0; then" >> postinst +echo " ln -s $LINKED_PYTHON /lib/x86_64-linux-gnu/libpython3.10.so.1.0" >> postinst +echo " fi" >> postinst +echo "fi" >> postinst +# End of symlink insanity chmod a+x postinst # Uninstall Script From dbd1b6b46dd59cefa3088374611990800e02d4ce Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Thu, 27 Jun 2024 16:22:31 -0500 Subject: [PATCH 05/15] And this time with the parameters the right way around. --- src/build_dpkg.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build_dpkg.sh b/src/build_dpkg.sh index 1b3400b7..508fb509 100755 --- a/src/build_dpkg.sh +++ b/src/build_dpkg.sh @@ -83,10 +83,10 @@ echo "popd" >> postinst LINKED_PYTHON=$(ldd /opt/libreqos/src/bin/lqosd | grep libpython | sed -e '/^[^\t]/ d' | sed -e 's/\t//' | sed -e 's/.*=..//' | sed -e 's/ (0.*)//') echo "if ! test -f $LINKED_PYTHON; then" >> postinst echo " if test -f /lib/x86_64-linux-gnu/libpython3.12.so.1.0; then" >> postinst -echo " ln -s $LINKED_PYTHON /lib/x86_64-linux-gnu/libpython3.10.so.1.0" >> postinst +echo " ln -s /lib/x86_64-linux-gnu/libpython3.12.so.1.0 $LINKED_PYTHON" >> postinst echo " fi" >> postinst echo " if test -f /lib/x86_64-linux-gnu/libpython3.11.so.1.0; then" >> postinst -echo " ln -s $LINKED_PYTHON /lib/x86_64-linux-gnu/libpython3.10.so.1.0" >> postinst +echo " ln -s /lib/x86_64-linux-gnu/libpython3.11.so.1.0 $LINKED_PYTHON" >> postinst echo " fi" >> postinst echo "fi" >> postinst # End of symlink insanity From 28d2eab949eb30b40f14e6a323e0230dda8a15d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20K=C5=82opotek=20-=20INTERDUO?= Date: Fri, 28 Jun 2024 08:26:58 +0200 Subject: [PATCH 06/15] improvement: add rustup command check and suggests installing it if its required --- src/build_rust.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/build_rust.sh b/src/build_rust.sh index 41bab51b..b5008148 100755 --- a/src/build_rust.sh +++ b/src/build_rust.sh @@ -36,6 +36,18 @@ else echo "LLVM/clang found." fi +if ! rustup -V &> /dev/null +then + echo "rustup is not installed." + echo "Visit https://rustup.rs and install Rust from there" + echo "Usually, you can copy the following and follow the on-screen instructions." + echo "Please don't install Rust as root." + echo "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh" + exit 1 +else + echo "rustup found." +fi + # To enable heavy debug mode (slow) #BUILD_FLAGS="" #TARGET=debug From 0adcfd93bbf7ba44ecb18d9c0adc73bfd75aee73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20K=C5=82opotek=20-=20INTERDUO?= Date: Fri, 28 Jun 2024 08:39:43 +0200 Subject: [PATCH 07/15] build_rust.sh - use apt instead of apt-get --- src/build_rust.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/build_rust.sh b/src/build_rust.sh index b5008148..c4f1095f 100755 --- a/src/build_rust.sh +++ b/src/build_rust.sh @@ -13,7 +13,7 @@ if ! bpftool help &> /dev/null then echo "bpftool is not installed." echo "Let's try to install it" - sudo apt-get install linux-tools-common linux-tools-`uname -r` + sudo apt install linux-tools-common linux-tools-`uname -r` else echo "bpftool found." fi @@ -22,7 +22,7 @@ if ! pkg-config --help &> /dev/null then echo "pkg-config is not installed." echo "Let's try to install it" - sudo apt-get install pkg-config + sudo apt install pkg-config else echo "pkg-config found." fi @@ -31,7 +31,7 @@ if ! clang -v &> /dev/null then echo "LLVM/clang is not installed." echo "Let's try to install it" - sudo apt-get install llvm libelf-dev gcc gcc-multilib libbpf-dev + sudo apt install llvm libelf-dev gcc gcc-multilib libbpf-dev else echo "LLVM/clang found." fi From 32a417327a7dcf7c812a7c766d10336b56eab074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20K=C5=82opotek=20-=20INTERDUO?= Date: Fri, 28 Jun 2024 08:40:35 +0200 Subject: [PATCH 08/15] build_rust.sh - libssl-dev is also required during compilation --- src/build_rust.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build_rust.sh b/src/build_rust.sh index c4f1095f..ff1d99ee 100755 --- a/src/build_rust.sh +++ b/src/build_rust.sh @@ -31,7 +31,7 @@ if ! clang -v &> /dev/null then echo "LLVM/clang is not installed." echo "Let's try to install it" - sudo apt install llvm libelf-dev gcc gcc-multilib libbpf-dev + sudo apt install llvm libelf-dev gcc gcc-multilib libbpf-dev libssl-dev else echo "LLVM/clang found." fi From 3ad129f8414994e1a03accbe01335532b65815dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20K=C5=82opotek=20-=20INTERDUO?= Date: Fri, 28 Jun 2024 09:04:14 +0000 Subject: [PATCH 09/15] build_rust.sh - make is required --- src/build_rust.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/build_rust.sh b/src/build_rust.sh index ff1d99ee..d951282b 100755 --- a/src/build_rust.sh +++ b/src/build_rust.sh @@ -18,6 +18,16 @@ else echo "bpftool found." fi +if ! make -v &> /dev/null +then + echo "make is not installed." + echo "Let's try to install it" + sudo apt install make +else + echo "make found." +fi + + if ! pkg-config --help &> /dev/null then echo "pkg-config is not installed." From a031719baf79ed7b13384fe49bec2581fa5b0aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20K=C5=82opotek=20-=20INTERDUO?= Date: Fri, 28 Jun 2024 09:48:20 +0000 Subject: [PATCH 10/15] build_rust.sh - lets simplyfy requirements --- src/build_rust.sh | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/src/build_rust.sh b/src/build_rust.sh index d951282b..109c27f5 100755 --- a/src/build_rust.sh +++ b/src/build_rust.sh @@ -9,42 +9,7 @@ # Don't forget to setup `/etc/lqos.conf` # Check Pre-Requisites -if ! bpftool help &> /dev/null -then - echo "bpftool is not installed." - echo "Let's try to install it" - sudo apt install linux-tools-common linux-tools-`uname -r` -else - echo "bpftool found." -fi - -if ! make -v &> /dev/null -then - echo "make is not installed." - echo "Let's try to install it" - sudo apt install make -else - echo "make found." -fi - - -if ! pkg-config --help &> /dev/null -then - echo "pkg-config is not installed." - echo "Let's try to install it" - sudo apt install pkg-config -else - echo "pkg-config found." -fi - -if ! clang -v &> /dev/null -then - echo "LLVM/clang is not installed." - echo "Let's try to install it" - sudo apt install llvm libelf-dev gcc gcc-multilib libbpf-dev libssl-dev -else - echo "LLVM/clang found." -fi +sudo apt install python3-pip clang gcc gcc-multilib llvm libelf-dev git nano graphviz curl screen llvm pkg-config linux-tools-common linux-tools-`uname -r` libbpf-dev libssl-dev if ! rustup -V &> /dev/null then From 425af46d887fa2a0c5a091b87a71a1221de3eabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20K=C5=82opotek=20-=20INTERDUO?= Date: Wed, 3 Jul 2024 08:28:32 +0200 Subject: [PATCH 11/15] build_dpkg.sh: use VERSION_STRING file instead setting everywhere version mark --- src/build_dpkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build_dpkg.sh b/src/build_dpkg.sh index 508fb509..bef6111c 100755 --- a/src/build_dpkg.sh +++ b/src/build_dpkg.sh @@ -10,7 +10,7 @@ then BUILD_DATE="" fi PACKAGE=libreqos -VERSION=1.5.$BUILD_DATE +VERSION=`cat ./VERSION_STRING`.$BUILD_DATE PKGVERSION=$PACKAGE PKGVERSION+="_" PKGVERSION+=$VERSION From bef5dea9afc6aaa3643109ecdf9b428a7045b6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Chac=C3=B3n?= Date: Thu, 4 Jul 2024 08:59:29 -0600 Subject: [PATCH 12/15] Update integrationSplynx.py --- src/integrationSplynx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integrationSplynx.py b/src/integrationSplynx.py index 35692432..483b8b73 100644 --- a/src/integrationSplynx.py +++ b/src/integrationSplynx.py @@ -20,7 +20,7 @@ def spylnxRequest(target, headers): # Sends a REST GET request to Spylnx and returns the # result in JSON url = splynx_api_url() + "/api/2.0/" + target - r = requests.get(url, headers=headers, timeout=10) + r = requests.get(url, headers=headers, timeout=120) return r.json() def getTariffs(headers): From a5eb4b61e4c3ea6d9efd1cf54b58b3fba9549c46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaros=C5=82aw=20K=C5=82opotek=20-=20INTERDUO?= Date: Tue, 9 Jul 2024 16:09:42 +0200 Subject: [PATCH 13/15] =?UTF-8?q?build=5Fdpkg.sh:=20use=20requirements.txt?= =?UTF-8?q?=20file=20for=20installing=20python=20depend=E2=80=A6=20(#510)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * build_dpkg.sh: use requirements.txt file for installing python dependancies (fix for issue #506) * restore removed --break-system-packages by mistake --- src/build_dpkg.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/build_dpkg.sh b/src/build_dpkg.sh index bef6111c..e0686fcd 100755 --- a/src/build_dpkg.sh +++ b/src/build_dpkg.sh @@ -20,7 +20,7 @@ DEBIAN_DIR=$DPKG_DIR/DEBIAN LQOS_DIR=$DPKG_DIR/opt/libreqos/src ETC_DIR=$DPKG_DIR/etc MOTD_DIR=$DPKG_DIR/etc/update-motd.d -LQOS_FILES="graphInfluxDB.py influxDBdashboardTemplate.json integrationCommon.py integrationPowercode.py integrationRestHttp.py integrationSonar.py integrationSplynx.py integrationUISP.py integrationSonar.py LibreQoS.py lqos.example lqTools.py mikrotikFindIPv6.py network.example.json pythonCheck.py README.md scheduler.py ShapedDevices.example.csv lqos.example" +LQOS_FILES="graphInfluxDB.py influxDBdashboardTemplate.json integrationCommon.py integrationPowercode.py integrationRestHttp.py integrationSonar.py integrationSplynx.py integrationUISP.py integrationSonar.py LibreQoS.py lqos.example lqTools.py mikrotikFindIPv6.py network.example.json pythonCheck.py README.md scheduler.py ShapedDevices.example.csv lqos.example ../requirements.txt" LQOS_BIN_FILES="lqos_scheduler.service.example lqosd.service.example lqos_node_manager.service.example" RUSTPROGS="lqosd lqtop xdp_iphash_to_cpu_cmdline xdp_pping lqos_node_manager lqusers lqos_setup lqos_map_perf uisp_integration lqos_support_tool" @@ -61,11 +61,8 @@ echo "#!/bin/bash" >> postinst echo "# Install Python Dependencies" >> postinst echo "pushd /opt/libreqos" >> postinst # - Setup Python dependencies as a post-install task -while requirement= read -r line -do - echo "python3 -m pip install $line --break-system-packages" >> postinst - echo "sudo python3 -m pip install $line --break-system-packages" >> postinst -done < ../../../../requirements.txt +echo "python3 -m pip install --break-system-packages -r src/requirements.txt" >> postinst +echo "sudo python3 -m pip install --break-system-packages -r src/requirements.txt" >> postinst # - Run lqsetup echo "/opt/libreqos/src/bin/lqos_setup" >> postinst # - Setup the services From 6b94e67faaa9dbac497a688ce348f898264e86a8 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Tue, 9 Jul 2024 09:27:59 -0500 Subject: [PATCH 14/15] ISSUE #511 Replace instant::now/subtraction with a Tokio scheduled ticker, in "skip" mode to handle time overruns. --- src/rust/lqosd/src/throughput_tracker/mod.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/rust/lqosd/src/throughput_tracker/mod.rs b/src/rust/lqosd/src/throughput_tracker/mod.rs index 0e5f81e9..79d29621 100644 --- a/src/rust/lqosd/src/throughput_tracker/mod.rs +++ b/src/rust/lqosd/src/throughput_tracker/mod.rs @@ -73,6 +73,8 @@ async fn throughput_task( false }; + let mut ticker = tokio::time::interval(Duration::from_millis(interval_ms)); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { let start = Instant::now(); @@ -102,13 +104,7 @@ async fn throughput_task( } tokio::spawn(submit_throughput_stats(long_term_stats_tx.clone())); - let elapsed = start.elapsed(); - if elapsed.as_secs_f32() < 1.0 { - let sleep_duration = Duration::from_millis(interval_ms) - start.elapsed(); - tokio::time::sleep(sleep_duration).await; - } else { - log::error!("Throughput monitor thread is running behind. It took {elapsed} to poll the network.", elapsed=elapsed.as_secs_f32()); - } + ticker.tick().await; } } From 8ce08333bae882794aaa6bd1699027098010e6d1 Mon Sep 17 00:00:00 2001 From: Herbert Wolverson Date: Tue, 9 Jul 2024 09:50:39 -0500 Subject: [PATCH 15/15] FIXES #507 - Use VERSION_STRING in setup assistant banner. --- src/rust/lqos_setup/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rust/lqos_setup/src/main.rs b/src/rust/lqos_setup/src/main.rs index 8e81c95e..082d10dc 100644 --- a/src/rust/lqos_setup/src/main.rs +++ b/src/rust/lqos_setup/src/main.rs @@ -166,8 +166,11 @@ fn write_combined_config( std::fs::write("/etc/lqos.conf", raw).unwrap(); } +const VERSION: &str = include_str!("../../../VERSION_STRING"); + fn main() { - println!("{:^80}", "LibreQoS 1.4 Setup Assistant".yellow().on_blue()); + let greeting = format!("LibreQoS {VERSION} Setup Assistant"); + println!("{:^80}", greeting.yellow().on_blue()); println!(); let interfaces = get_available_interfaces(); let mut if_internet: Option = None;