Azure Pipelines: simplify test job definitions

Rewrite templates to make test job declarations simpler and easier to
work with.

A test job template can be instantiated this way:

- template: templates/test-jobs.yml
  parameters:
    jobName: Base
    jobTitle: Base tests
    testsToRun:
    - test_cmdline
    - test_install
    - test_ipaclient
    - test_ipalib
    - test_ipaplatform
    - test_ipapython
    testsToIgnore:
    - test_integration
    - test_webui
    - test_ipapython/test_keyring.py
    taskToRun: run-tests

Both 'testsToRun' and 'testsToIgnore' accept arrays of test matches.
Wildcards also supported:

    ....
    testsToRun:
    - test_xmlrpc/test_hbac*
    ....

'taskToRun' specifies a script ipatests/azure/azure-${taskToRun}.sh that
will be executed in the test environment to actually start tests.

Parameters 'testsToRun' and 'testsToIgnore' define TESTS_TO_{RUN,IGNORE}
variables that will be set in the environment of the test script. These
variables will have entries from the parameters separated by a space.

Reviewed-By: Christian Heimes <cheimes@redhat.com>

ipatests/azure/azure-pipelines.yml

@@ -2,15 +2,7 @@ trigger:
 - master
 
 variables:
-  TEST_RUNNER_IMAGE: freeipa-fedora-builder:latest
-  TEST_RUNNER_CONFIG: .test_runner_azure.yaml
-  PEP8_ERROR_LOG: pycodestyle_errors.log
-  CI_RESULTS_LOG: ci_results_$(System.PullRequest.PullRequestNumber).log
-  CI_BACKLOG_SIZE: 5000
   CI_RUNNER_LOGS_DIR: logs
-  CI_RUNNER_LOG_ARCHIVE: $(System.PullRequest.PullRequestNumber)-job-$(Build.BuildId).tar.gz
-  TRAVIS_EVENT_TYPE: $(Build.Reason)
-  TRAVIS_BRANCH: $(System.PullRequest.TargetBranch)
   localsdir: $(Build.Repository.LocalPath)
   builddir: /__w/1/s
 
@@ -118,49 +110,31 @@ jobs:
         testRunTitle: 'Web UI unit test results'
       condition: succeededOrFailed()
 
-- job: Test
-  dependsOn: Build
-  condition: succeeded()
-  pool:
-    vmImage: 'Ubuntu-16.04'
-  strategy:
-    matrix:
-        Base_tests:
-          TEST_TITLE: 'Base'
-          TASK_TO_RUN: run-tests
-          TESTS_TO_RUN: "test_cmdline
-                test_install
-                test_ipaclient
-                test_ipalib
-                test_ipaplatform
-                test_ipapython
-                test_ipaserver"
-        XMLRPC_tests:
-          TEST_TITLE: 'XMLRPC'
-          TASK_TO_RUN: run-tests
-          TESTS_TO_RUN: "test_xmlrpc"
+- template: templates/test-jobs.yml
+  parameters:
+    jobName: Base
+    jobTitle: Base tests
+    testsToRun:
+    - test_cmdline
+    - test_install
+    - test_ipaclient
+    - test_ipalib
+    - test_ipaplatform
+    - test_ipapython
+    testsToIgnore:
+    - test_integration
+    - test_webui
+    - test_ipapython/test_keyring.py
+    taskToRun: run-tests
 
-  steps:
-    - template: templates/setup-test-environment.yml
-      parameters:
-        pythonVersion: '3.6'
-        architecture: 'x64'
-        logsPath: '$(CI_RUNNER_LOGS_DIR)'
-    - script: |
-        sudo modprobe ip6_tables
-      displayName: Make sure IPv6 firewall support is allowed
-    - template: templates/run-test.yml
-      parameters:
-        containerName: 'freeipa-fedora-builder:latest'
-        logsPath: '$(CI_RUNNER_LOGS_DIR)'
-        testToRun: '$(TASK_TO_RUN)'
-        testParams: '$(TESTS_TO_RUN)'
-    - task: PublishTestResults@2
-      inputs:
-        testResultsFiles: 'nosetests.xml'
-        testRunTitle: 'XMLRPC test results'
-      condition: succeededOrFailed()
-    - template: templates/save-test-artifacts.yml
-      parameters:
-        logsPath: '$(CI_RUNNER_LOGS_DIR)'
-        logsArtifact: '$(TEST_TITLE)-logs-$(CI_RUNNER_LOG_ARCHIVE)'
+- template: templates/test-jobs.yml
+  parameters:
+    jobName: XMLRPC
+    jobTitle: XMLRPC tests
+    testsToRun:
+    - test_xmlrpc
+    testsToIgnore:
+    - test_integration
+    - test_webui
+    - test_ipapython/test_keyring.py
+    taskToRun: run-tests

ipatests/azure/azure-run-tests.sh

@@ -3,34 +3,66 @@ server_realm=EXAMPLE.TEST
 server_domain=example.test
 server_password=Secret123
 
-# Expand list of tests into -k...  -k... -k... .. sequence
-# If remaining string still has { or } characters that shell did not expand, remove them
-tests_to_run=$(eval "eval echo -k{$(echo $TESTS_TO_RUN | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}')
+# Normalize spacing and expand the list afterwards. Remove {} for the single list element case
+tests_to_run=$(eval "echo {$(echo $TESTS_TO_RUN | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}')
+tests_to_ignore=$(eval "echo --ignore\ {$(echo $TESTS_TO_IGNORE | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}')
 
 systemctl --now enable firewalld
-ipa-server-install -U --domain ${server_domain} --realm ${server_realm} -p ${server_password} -a ${server_password} --setup-dns --setup-kra --auto-forwarders
-sed -ri "s/mode = production/mode = development/" /etc/ipa/default.conf
-systemctl restart httpd.service
-firewall-cmd --add-service={freeipa-ldap,freeipa-ldaps,dns}
+echo "Installing FreeIPA master for the domain ${server_domain} and realm ${server_realm}"
+ipa-server-install -U --domain ${server_domain} --realm ${server_realm} \
+                   -p ${server_password} -a ${server_password} \
+                   --setup-dns --setup-kra --auto-forwarders
 
+install_result=$?
 
-cd /freeipa
+tests_result=1
 
-echo ${server_password} | kinit admin && ipa ping
-cp -r /etc/ipa/* ~/.ipa/
-echo ${server_password} > ~/.ipa/.dmpw
-echo 'wait_for_dns=5' >> ~/.ipa/default.conf
-ipa-test-config --help
-ipa-test-task --help
-ipa-run-tests --with-xunit -k-{test_integration,test_webui,test_ipapython/test_keyring.py,test_dns_soa} -v ${tests_to_run}
-grep -n -C5 BytesWarning /var/log/httpd/error_log
+mkdir -p /freeipa/$CI_RUNNER_LOGS_DIR
+cd /freeipa/$CI_RUNNER_LOGS_DIR
+
+if [ "$install_result" -eq 0 ] ; then
+	echo "Run IPA tests"
+	echo "Installation complete. Performance of individual steps:"
+	grep 'service duration:' /var/log/ipaserver-install.log | sed -e 's/DEBUG //g'
+
+	sed -ri "s/mode = production/mode = development/" /etc/ipa/default.conf
+	systemctl restart httpd.service
+	firewall-cmd --add-service={freeipa-ldap,freeipa-ldaps,dns}
+
+	echo ${server_password} | kinit admin && ipa ping
+	mkdir -p ~/.ipa
+	cp -r /etc/ipa/* ~/.ipa/
+	echo ${server_password} > ~/.ipa/.dmpw
+	echo 'wait_for_dns=5' >> ~/.ipa/default.conf
+
+	ipa-test-config --help
+	ipa-test-task --help
+	ipa-run-tests --help
+
+	ipa-run-tests ${tests_to_ignore} --verbose --with-xunit '-k not test_dns_soa' ${tests_to_run}
+	tests_result=$?
+else
+	echo "ipa-server-install failed with code ${save_result}, skip IPA tests"
+fi
+
+echo "Potential Python 3 incompatibilities in the IPA framework:"
+grep -n -C5 BytesWarning /var/log/httpd/error_log || echo "Good, none detected"
+
+echo "State of the directory server instance, httpd databases, PKI CA database:"
+ls -laZ /etc/dirsrv/slapd-*/ /etc/httpd/alias/ /var/lib/ /etc/pki/pki-tomcat/alias/ || true
+ls -laZ /var/lib/ipa/certs/ /var/lib/ipa/passwds/ /var/lib/ipa/private/ || true
+
+echo "Uninstall the server"
 ipa-server-install --uninstall -U
 # second uninstall to verify that --uninstall without installation works
 ipa-server-install --uninstall -U
-firewall-cmd --remove-service={freeipa-ldap,freeipa-ldaps,dns}
 
-mkdir -p /freeipa/logs
-cd /freeipa/logs
+
+if [ "$install_result" -eq 0 ] ; then
+	firewall-cmd --remove-service={freeipa-ldap,freeipa-ldaps,dns}
+fi
+
+echo "Collect the logs"
 journalctl -b --no-pager > systemd_journal.log
 tar --ignore-failed-read -cvf var_log.tar \
     /var/log/dirsrv \
@@ -38,7 +70,9 @@ tar --ignore-failed-read -cvf var_log.tar \
     /var/log/ipa* \
     /var/log/krb5kdc.log \
     /var/log/pki \
+    /var/log/samba \
+    /var/named/data \
     systemd_journal.log
 
-ls -laZ /etc/dirsrv/slapd-*/ /etc/httpd/alias/ /etc/pki/pki-tomcat/alias/ || true
-
+# Final result depends on the exit code of the ipa-run-tests
+test "$tests_result" -eq 0 -a "$install_result" -eq 0

ipatests/azure/templates/run-test.yml

@@ -2,8 +2,9 @@ parameters:
   imageName: 'freeipa-fedora-builder:latest'
   containerName: 'container'
   logsPath: 'logs'
-  testToRun: 'lint'
-  testParams: ''
+  taskToRun: 'run-tests'
+  testsToRun: ''
+  testsToIgnore: ''
 
 steps:
 - script: |
@@ -16,8 +17,10 @@ steps:
     docker inspect $(createContainer.containerName)
   displayName: Start container for running a test
 - script: |
-    docker exec --env TESTS_TO_RUN='${{parameters.testParams}}' \
+    docker exec --env TESTS_TO_RUN="${{ parameters.testsToRun }}" \
+                --env TESTS_TO_IGNORE="${{ parameters.testsToIgnore }}" \
+                --env CI_RUNNER_LOGS_DIR="${{ parameters.logsPath }}" \
                 --privileged -t \
                 $(createContainer.containerName) \
-                /bin/bash --noprofile --norc -x /freeipa/ipatests/azure/azure-${{parameters.testToRun}}.sh
+                /bin/bash --noprofile --norc -x /freeipa/ipatests/azure/azure-${{parameters.taskToRun}}.sh
   displayName: Run test

ipatests/azure/templates/save-test-artifacts.yml

@@ -1,6 +1,6 @@
 parameters:
   logsPath: 'logs'
-  logsArtifact: 'logs-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitrecture)'
+  logsArtifact: ''
 steps:
 - task: PublishPipelineArtifact@0
   displayName: Publish logs

ipatests/azure/templates/setup-test-environment.yml

@@ -1,23 +1,11 @@
-parameters:
-  pythonVersion: '3.6'
-  architecture: 'x64'
-  pythonExec: 'python3'
-  logsPath: 'logs'
-  logsArtifact: 'logs-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitrecture)'
 steps:
 - script: |
     echo '{ "ipv6": true, "fixed-cidr-v6": "2001:db8::/64" }' > docker-daemon.json
     sudo cp docker-daemon.json /etc/docker/daemon.json
     sudo chown root:root /etc/docker/daemon.json
     sudo systemctl restart docker
+    sudo modprobe ip6_tables
   displayName: Configure containerization to allow IPv6 network
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: ${{parameters.pythonVersion}}
-    architecture: ${{parameters.architecture}}
-- script: |
-    ${{parameters.pythonExec}} -m pip install --upgrade pip setuptools pycodestyle
-  displayName: 'Install prerequisites'
 - task: DownloadPipelineArtifact@0
   displayName: Download prebuilt packages
   inputs:

ipatests/azure/templates/test-jobs.yml

@@ -0,0 +1,31 @@
+parameters:
+  jobName: ''
+  jobTitle: ''
+  testsToIgnore: []
+  testsToRun: []
+  taskToRun: ''
+
+jobs:
+- job: ${{ parameters.jobName }}
+  displayName: ${{ parameters.jobTitle }}
+  dependsOn: Build
+  condition: succeeded()
+  pool:
+    vmImage: 'Ubuntu-16.04'
+  steps:
+    - template: setup-test-environment.yml
+    - template: run-test.yml
+      parameters:
+        containerName: 'freeipa-fedora-builder:latest'
+        logsPath: $(CI_RUNNER_LOGS_DIR)
+        taskToRun: ${{ parameters.taskToRun}}
+        testsToRun: ${{ join(' ', parameters.testsToRun ) }}
+        testsToIgnore: ${{ join(' ', parameters.testsToIgnore ) }}
+    - task: PublishTestResults@2
+      inputs:
+        testResultsFiles: $(CI_RUNNER_LOGS_DIR)/nosetests.xml
+        testRunTitle: ${{parameters.jobTitle}} results
+      condition: succeededOrFailed()
+    - template: save-test-artifacts.yml
+      parameters:
+        logsArtifact: logs-${{parameters.jobName}}-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitecture)