From b5fdba7a72009c52d4e2ffaf0dfe51afa66dc420 Mon Sep 17 00:00:00 2001
From: Stanislav Levin <slev@altlinux.org>
Date: Sat, 8 May 2021 07:46:56 +0300
Subject: [PATCH] azure: Warn about memory issues

The nonzero number of memory/memory+Swap usage hits limits may
indicate the possible env instability(crashes, random failures, etc.).

> memory.failcnt		 # show the number of memory usage hits limits
  memory.memsw.failcnt		 # show the number of memory+Swap hits limits

Reviewed-By: Alexander Bokovoy <abokovoy@redhat.com>
---
 ipatests/azure/scripts/azure-run-tests.sh | 32 ++++++++++++++++-------
 ipatests/azure/templates/test-jobs.yml    | 15 +++++++++++
 2 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/ipatests/azure/scripts/azure-run-tests.sh b/ipatests/azure/scripts/azure-run-tests.sh
index 9be044cc6..47fc06651 100755
--- a/ipatests/azure/scripts/azure-run-tests.sh
+++ b/ipatests/azure/scripts/azure-run-tests.sh
@@ -149,16 +149,28 @@ echo "Report disk usage"
 compose_execute df -h
 
 echo "Report memory statistics"
-files="
-/sys/fs/cgroup/memory/memory.memsw.failcnt
-/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes
-/sys/fs/cgroup/memory/memory.memsw.max_usage_in_bytes
-/sys/fs/cgroup/memory/memory.failcnt
-/sys/fs/cgroup/memory/memory.max_usage_in_bytes
-/sys/fs/cgroup/memory/memory.limit_in_bytes
-/proc/sys/vm/swappiness
-"
-compose_execute head -n 1 $files
+files='/sys/fs/cgroup/memory/memory.memsw.failcnt \
+/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes \
+/sys/fs/cgroup/memory/memory.memsw.max_usage_in_bytes \
+/sys/fs/cgroup/memory/memory.failcnt \
+/sys/fs/cgroup/memory/memory.max_usage_in_bytes \
+/sys/fs/cgroup/memory/memory.limit_in_bytes \
+/proc/sys/vm/swappiness \
+'
+
+MEMORY_STATS_PATH="$project_dir/memory.stats"
+compose_execute $BASH_CMD -eu -c \
+    "for file in $files; do printf '%s=%s\n' \"\$file\" \"\$(head -n 1 \$file)\" ; done" > "$MEMORY_STATS_PATH"
+
+sed -E -n \
+    's/(.*): .*(memory\.(memsw\.)?failcnt)=([0-9]+)/\1 \2 \4/p' \
+    "$MEMORY_STATS_PATH" | \
+tr -d '\r' | \
+while read -r container memtype failcnt; do
+   if [ "$failcnt" -gt 0 ]; then
+      grep "^$container.*memory\..*" "$MEMORY_STATS_PATH" >> "$project_dir/memory.warnings"
+   fi
+done
 
 pushd "$project_dir"
 BUILD_REPOSITORY_LOCALPATH="$BUILD_REPOSITORY_LOCALPATH" \
diff --git a/ipatests/azure/templates/test-jobs.yml b/ipatests/azure/templates/test-jobs.yml
index 51fd81ae2..16b225cb6 100644
--- a/ipatests/azure/templates/test-jobs.yml
+++ b/ipatests/azure/templates/test-jobs.yml
@@ -63,6 +63,20 @@ steps:
   condition: succeededOrFailed()
   displayName: Host's memory statistics
 
+- script: |
+    set -eu
+    function emit_warning() {
+        printf "##vso[task.logissue type=warning]%s\n" "$1"
+    }
+
+    for memory_warn in $(find ${IPA_TESTS_ENV_WORKING_DIR}/*/ -maxdepth 1 -name memory.warnings);
+    do
+        env_name="$(basename $(dirname $memory_warn))"
+        emit_warning "Test env '$env_name' has high memory usage: $(echo '' && cat $memory_warn)"
+    done
+  condition: succeededOrFailed()
+  displayName: Check memory consumption
+
 - script: |
     set -eu
     HOST_JOURNAL=host_journal.log
@@ -147,6 +161,7 @@ steps:
     !*/*.yaml
     !*/*.log
     !*/systemd_boot_logs/*.log
+    !*/memory.stats
     !*.tar.gz
     EOF
     cat "$artifacts_ignore_path"