freeipa/ipatests/azure/templates/test-jobs.yml
Stanislav Levin 6c2db326f8 azure: coredump: Wait for systemd fully booted
Otherwise, 'Check for coredumps' task fails with:
```
Verifying        : samba-debugsource-2:4.14.4-0.fc34.x86_64             20/20
[Errno 2] No such file or directory: '/var/lib/dnf/rpmdb_lock.pid'
Finishing: Check for coredumps
```

This is due to systemd-tmpfiles(not ready yet).

Reviewed-By: Alexander Bokovoy <abokovoy@redhat.com>
2021-05-25 10:45:49 +03:00

180 lines
5.7 KiB
YAML

steps:
- script: |
set -e
env | sort
displayName: Print Host Enviroment
- script: |
set -e
sudo apt-get update
sudo apt-get install -y \
apparmor-utils \
parallel \
moreutils \
rng-tools \
systemd-coredump \
python3-docker
displayName: Install Host's tests requirements
- script: |
set -e
printf "AppArmor status\n"
sudo aa-status
printf "Disable AppArmor conflicting profiles\n"
sudo aa-disable /etc/apparmor.d/usr.sbin.chronyd
printf "Recheck AppArmor status\n"
sudo aa-status
displayName: Disable AppArmor conflicting profiles on Host
- script: |
set -e
printf "Available entropy: %s\n" $(cat /proc/sys/kernel/random/entropy_avail)
sudo service rng-tools start
sleep 3
printf "Available entropy: %s\n" $(cat /proc/sys/kernel/random/entropy_avail)
displayName: Increase entropy level
- script: |
set -eu
date +'%Y-%m-%d %H:%M:%S' > coredumpctl.time.mark
systemd_conf="/etc/systemd/system.conf"
sudo sed -i 's/^DumpCore=.*/#&/g' "$systemd_conf"
sudo sed -i 's/^DefaultLimitCORE=.*/#&/g' "$systemd_conf"
echo -e 'DumpCore=yes\nDefaultLimitCORE=infinity' | \
sudo tee -a "$systemd_conf" >/dev/null
cat "$systemd_conf"
coredump_conf="/etc/systemd/coredump.conf"
cat "$coredump_conf"
sudo systemctl daemon-reexec
# for ns-slapd debugging
sudo sysctl -w fs.suid_dumpable=1
displayName: Allow coredumps
- template: setup-test-environment.yml
- template: run-test.yml
- script: |
set -eux
free -m
cat /sys/fs/cgroup/memory/memory.memsw.max_usage_in_bytes
cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes
cat /proc/sys/vm/swappiness
condition: succeededOrFailed()
displayName: Host's memory statistics
- script: |
set -eu
function emit_warning() {
printf "##vso[task.logissue type=warning]%s\n" "$1"
}
for memory_warn in $(find ${IPA_TESTS_ENV_WORKING_DIR}/*/ -maxdepth 1 -name memory.warnings);
do
env_name="$(basename $(dirname $memory_warn))"
emit_warning "Test env '$env_name' has high memory usage: $(echo '' && cat $memory_warn)"
done
condition: succeededOrFailed()
displayName: Check memory consumption
- script: |
set -eu
HOST_JOURNAL=host_journal.log
HOST_JOURNAL_PATH="${IPA_TESTS_ENV_WORKING_DIR}/${HOST_JOURNAL}.tar.gz"
sudo journalctl -b | tee "$HOST_JOURNAL"
function emit_warning() {
printf "##vso[task.logissue type=warning]%s\n" "$1"
}
printf "AVC:\n"
grep 'AVC apparmor="DENIED"' "$HOST_JOURNAL" && \
emit_warning "There are Host's AVCs. Please, check the logs."
printf "SECCOMP:\n"
grep ' SECCOMP ' "$HOST_JOURNAL" && \
emit_warning "There are reported SECCOMP syscalls. Please, check the logs."
tar -czf "$HOST_JOURNAL_PATH" "$HOST_JOURNAL"
condition: succeededOrFailed()
displayName: Host's systemd journal
- task: PublishTestResults@2
inputs:
testResultsFiles: 'ipa_envs/*/$(CI_RUNNER_LOGS_DIR)/nosetests.xml'
testRunTitle: $(System.JobIdentifier) results
condition: succeededOrFailed()
- script: |
set -eu
# check the host first, containers cores were dumped here
COREDUMPS_SUBDIR="coredumps"
COREDUMPS_DIR="${IPA_TESTS_ENV_WORKING_DIR}/${COREDUMPS_SUBDIR}"
rm -rfv "$COREDUMPS_DIR" ||:
mkdir "$COREDUMPS_DIR"
since_time="$(cat coredumpctl.time.mark || echo '-1h')"
sudo coredumpctl --no-pager --since="$since_time" list ||:
pids="$(sudo coredumpctl --no-pager --since="$since_time" -F COREDUMP_PID || echo '')"
# nothing to dump
[ -z "$pids" ] && exit 0
# continue in container
HOST_JOURNAL="/var/log/host_journal"
CONTAINER_COREDUMP="dump_cores"
docker create --privileged \
-v "$(realpath coredumpctl.time.mark)":/coredumpctl.time.mark:ro \
-v /var/lib/systemd/coredump:/var/lib/systemd/coredump:ro \
-v /var/log/journal:"$HOST_JOURNAL":ro \
-v "${BUILD_REPOSITORY_LOCALPATH}":"${IPA_TESTS_REPO_PATH}" \
--name "$CONTAINER_COREDUMP" freeipa-azure-builder
docker start "$CONTAINER_COREDUMP"
docker exec -t \
"$CONTAINER_COREDUMP" \
/bin/bash --noprofile --norc -eux \
"${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/wait-for-systemd.sh"
docker exec -t \
--env IPA_TESTS_REPO_PATH="${IPA_TESTS_REPO_PATH}" \
--env IPA_TESTS_SCRIPTS="${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}" \
--env IPA_PLATFORM="${IPA_PLATFORM}" \
"$CONTAINER_COREDUMP" \
/bin/bash --noprofile --norc -eux \
"${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/install-debuginfo.sh"
docker exec -t \
--env IPA_TESTS_REPO_PATH="${IPA_TESTS_REPO_PATH}" \
--env COREDUMPS_SUBDIR="$COREDUMPS_SUBDIR" \
--env HOST_JOURNAL="$HOST_JOURNAL" \
"$CONTAINER_COREDUMP" \
/bin/bash --noprofile --norc -eux \
"${IPA_TESTS_REPO_PATH}/${IPA_TESTS_SCRIPTS}/dump_cores.sh"
# there should be no crashes
exit 1
condition: succeededOrFailed()
displayName: Check for coredumps
- script: |
set -e
artifacts_ignore_path="${IPA_TESTS_ENV_WORKING_DIR}/.artifactignore"
cat > "$artifacts_ignore_path" <<EOF
**/*
!coredumps/*.core.tar.gz
!coredumps/*.stacktrace.tar.gz
!*/logs/**
!*/*.yml
!*/*.yaml
!*/*.log
!*/systemd_boot_logs/*.log
!*/installed_packages/*.log
!*/memory.stats
!*.tar.gz
EOF
cat "$artifacts_ignore_path"
condition: succeededOrFailed()
displayName: Generating artifactignore file
- template: save-test-artifacts.yml
parameters:
logsArtifact: logs-$(System.JobIdentifier)-$(Build.BuildId)-$(System.StageAttempt)-$(System.PhaseAttempt)-$(System.JobPositionInPhase)-$(Agent.OS)-$(Agent.OSArchitecture)