Azure: Gather coredumps

Applications may crash.
If a crash happens on a remote system during CI run it's sometimes
very hard to understand the reason. The most important means to
analyze such is a stack trace. It's also very important to check
whether there was a core dump or not, even a test passed.

For Docker environment, the core dumps are collected by the host's
systemd-coredump, which knows nothing about such containers (for
now). To build an informative thread stack trace debuginfo packages
should be installed. But they can't be installed on the host OS
(ubuntu), That's why after all the tests completed an additional
container should be up and the host's core dumps and host's journal
should be passed into it.

Even if there weren't enough debuginfo packages at CI-runtime, the
core dump could be analyzed locally later.

Fixes: https://pagure.io/freeipa/issue/8251
Signed-off-by: Stanislav Levin <slev@altlinux.org>
Reviewed-By: Alexander Bokovoy <abokovoy@redhat.com>
Reviewed-By: Rob Crittenden <rcritten@redhat.com>

ipatests/azure/scripts/dump_cores.sh

@@ -0,0 +1,54 @@
+#!/bin/bash -eu
+
+IPA_TESTS_ENV_WORKING_DIR="${IPA_TESTS_REPO_PATH}/ipa_envs"
+COREDUMPS_DIR="${IPA_TESTS_ENV_WORKING_DIR}/${COREDUMPS_SUBDIR}"
+
+since_time="$(cat '/coredumpctl.time.mark' || echo '-1h')"
+debugger="/debugger.sh"
+
+cat > "$debugger" <<EOF
+#!/bin/bash -eux
+
+debug_info="\$@"
+gdb \
+    -ex 'set confirm off' \
+    -ex 'set pagination off' \
+    -ex 'thread apply all bt full' \
+    -ex 'quit' \
+    \$debug_info > "\${CORE_PID}.stacktrace" 2>&1
+EOF
+chmod +x "$debugger"
+
+# make sure coredumpctl installed
+which coredumpctl
+coredumpctl \
+    --no-pager --directory="$HOST_JOURNAL" --since="$since_time" list ||:
+
+rm -rvf "$COREDUMPS_DIR" ||:
+mkdir "$COREDUMPS_DIR"
+cd "$COREDUMPS_DIR"
+
+pids="$(coredumpctl --no-pager --directory="$HOST_JOURNAL" --since="$since_time" -F COREDUMP_PID || echo '')"
+for pid in $pids; do
+    # core dump
+    { coredumpctl \
+        --no-pager \
+        --since="$since_time" \
+        --directory="$HOST_JOURNAL" \
+        -o "${pid}.core" dump "$pid" && \
+      tar -czf "${pid}.core.tar.gz" --remove-files "${pid}.core" ; } ||:
+
+    # stacktrace
+    { CORE_PID="$pid" \
+        coredumpctl \
+        --no-pager \
+        --since="$since_time" \
+        --directory="$HOST_JOURNAL" \
+        --debugger="$debugger" \
+        debug "$pid" && \
+      tar \
+        -czf "${pid}.stacktrace.tar.gz" \
+        --remove-files "${pid}.stacktrace" ; } ||:
+done
+
+chmod a+rw -R "$COREDUMPS_DIR"

ipatests/azure/scripts/install-debuginfo-fedora.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -eu
+
+function install_debuginfo() {
+    dnf makecache ||:
+    dnf install -y \
+        ${IPA_TESTS_REPO_PATH}/dist/rpms_debuginfo/*.rpm \
+        gdb
+
+    dnf debuginfo-install -y \
+        389-ds-base \
+        bind \
+        bind-dyndb-ldap \
+        certmonger \
+        gssproxy \
+        httpd \
+        krb5-server \
+        krb5-workstation \
+        samba \
+        sssd
+}

ipatests/azure/scripts/install-debuginfo.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -eu
+
+function install_debuginfo() { :; }
+
+# override install_debuginfo for the platform specifics
+source "${IPA_TESTS_SCRIPTS}/install-debuginfo-${IPA_PLATFORM}.sh"
+
+install_debuginfo

ipatests/azure/templates/build-fedora.yml

@@ -3,4 +3,8 @@ steps:
     set -e
     echo "Running make target 'rpms'"
     make V=0 rpms LOG_COMPILE='gdb.minimal -return-child-result -ex run -ex "thread apply all bt" -ex "quit" --args'
+    mkdir -p $(builddir)/dist/rpms_debuginfo
+    find $(builddir)/dist/rpms/ -type f \
+        \( -name "*-debuginfo-*.rpm" -o -name '*-debugsource-*.rpm' \) \
+        -exec mv {} $(builddir)/dist/rpms_debuginfo/ \;
   displayName: Build packages

ipatests/azure/templates/test-jobs.yml

@@ -18,11 +18,28 @@ steps:
     sudo apt-get install -y \
         parallel \
         moreutils \
+        systemd-coredump \
         python3-docker
     # ubuntu's one is too old: different API
     python3 -m pip install docker --user
   displayName: Install Host's tests requirements
 
+- script: |
+    set -eu
+    date +'%Y-%m-%d %H:%M:%S' > coredumpctl.time.mark
+    systemd_conf="/etc/systemd/system.conf"
+    sudo sed -i 's/^DumpCore=.*/#&/g' "$systemd_conf"
+    sudo sed -i 's/^DefaultLimitCORE=.*/#&/g' "$systemd_conf"
+    echo -e 'DumpCore=yes\nDefaultLimitCORE=infinity' | \
+        sudo tee -a "$systemd_conf" >/dev/null
+    cat "$systemd_conf"
+    coredump_conf="/etc/systemd/coredump.conf"
+    cat "$coredump_conf"
+    sudo systemctl daemon-reexec
+    # for ns-slapd debugging
+    sudo sysctl -w fs.suid_dumpable=1
+  displayName: Allow coredumps
+
 - template: setup-test-environment.yml
 
 - template: run-test.yml
@@ -33,12 +50,59 @@ steps:
     testRunTitle: $(System.JobIdentifier) results
   condition: succeededOrFailed()
 
+- script: |
+    set -eu
+    # check the host first, containers cores were dumped here
+    COREDUMPS_SUBDIR="coredumps"
+    COREDUMPS_DIR="${IPA_TESTS_ENV_WORKING_DIR}/${COREDUMPS_SUBDIR}"
+    rm -rfv "$COREDUMPS_DIR" ||:
+    mkdir "$COREDUMPS_DIR"
+    since_time="$(cat coredumpctl.time.mark || echo '-1h')"
+    sudo coredumpctl --no-pager --since="$since_time" list ||:
+
+    pids="$(sudo coredumpctl --no-pager --since="$since_time" -F COREDUMP_PID || echo '')"
+    # nothing to dump
+    [ -z "$pids" ] && exit 0
+
+    # continue in container
+    HOST_JOURNAL="/var/log/host_journal"
+    CONTAINER_COREDUMP="dump_cores"
+    docker create --privileged \
+        -v "$(realpath coredumpctl.time.mark)":/coredumpctl.time.mark:ro \
+        -v /var/lib/systemd/coredump:/var/lib/systemd/coredump:ro \
+        -v /var/log/journal:"$HOST_JOURNAL":ro \
+        -v "${BUILD_REPOSITORY_LOCALPATH}":"${IPA_TESTS_REPO_PATH}" \
+        --name "$CONTAINER_COREDUMP" freeipa-azure-builder
+    docker start "$CONTAINER_COREDUMP"
+
+    docker exec -t \
+        --env IPA_TESTS_REPO_PATH="${IPA_TESTS_REPO_PATH}" \
+        --env IPA_TESTS_SCRIPTS="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}" \
+        --env IPA_PLATFORM="${IPA_PLATFORM}" \
+        "$CONTAINER_COREDUMP" \
+        /bin/bash --noprofile --norc -eux \
+            "${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/install-debuginfo.sh"
+
+    docker exec -t \
+        --env IPA_TESTS_REPO_PATH="${IPA_TESTS_REPO_PATH}" \
+        --env COREDUMPS_SUBDIR="$COREDUMPS_SUBDIR" \
+        --env HOST_JOURNAL="$HOST_JOURNAL" \
+        "$CONTAINER_COREDUMP" \
+        /bin/bash --noprofile --norc -eux \
+            "${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/dump_cores.sh"
+    # there should be no crashes
+    exit 1
+  condition: succeededOrFailed()
+  displayName: Check for coredumps
+
 - script: |
     set -e
 
     artifacts_ignore_path="${IPA_TESTS_ENV_WORKING_DIR}/.artifactignore"
     cat > "$artifacts_ignore_path" <<EOF
     **/*
+    !coredumps/*.core.tar.gz
+    !coredumps/*.stacktrace.tar.gz
     !*/logs/**
     !*/*.yml
     !*/*.yaml