Make use of Azure Pipeline slicing

The unit tests execution time within Azure Pipelines(AP) is not
balanced. One test job(Base) takes ~13min, while another(XMLRPC)
~28min. Fortunately, AP supports slicing:

> An agent job can be used to run a suite of tests in parallel. For
example, you can run a large suite of 1000 tests on a single agent.
Or, you can use two agents and run 500 tests on each one in parallel.
To leverage slicing, the tasks in the job should be smart enough to
understand the slice they belong to.

>The step that runs the tests in a job needs to know which test slice
should be run. The variables System.JobPositionInPhase and
System.TotalJobsInPhase can be used for this purpose.

Thus, to support this pytest should know how to split the test suite
into groups(slices). For this, a new internal pytest plugin was added.

About plugin.
- Tests within a slice are grouped by test modules because not all of
the tests within the module are independent from each other.
- Slices are balanced by the number of tests within test module.
- To run some module within its own environment there is a dedicated
slice option (could help with extremely slow tests)

Examples.
- To split `test_cmdline` tests into 2 slices and run the first one:

ipa-run-tests --slices=2 --slice-num=1 test_cmdline

- To split tests into 2 slices, then to move one module out to its own slice
and run the second one:

ipa-run-tests --slices=2 --slice-dedicated=test_cmdline/test_cli.py \
    --slice-num=2 test_cmdline

Fixes: https://pagure.io/freeipa/issue/8008
Signed-off-by: Stanislav Levin <slev@altlinux.org>
Reviewed-By: Alexander Bokovoy <abokovoy@redhat.com>

ipatests/azure/azure-pipelines.yml

@@ -128,8 +128,8 @@ jobs:
 
 - template: templates/test-jobs.yml
   parameters:
-    jobName: Base
-    jobTitle: Base tests
+    jobName: BASE_XMLRPC
+    jobTitle: BASE and XMLRPC tests
     testsToRun:
     - test_cmdline
     - test_install
@@ -138,20 +138,12 @@ jobs:
     - test_ipaplatform
     - test_ipapython
     - test_ipatests_plugins
-    testsToIgnore:
-    - test_integration
-    - test_webui
-    - test_ipapython/test_keyring.py
-    taskToRun: run-tests
-
-- template: templates/test-jobs.yml
-  parameters:
-    jobName: XMLRPC
-    jobTitle: XMLRPC tests
-    testsToRun:
     - test_xmlrpc
     testsToIgnore:
     - test_integration
     - test_webui
     - test_ipapython/test_keyring.py
+    testsToDedicate:
+    - test_xmlrpc/test_dns_plugin.py
     taskToRun: run-tests
+    tasksParallel: 3

ipatests/azure/azure-run-tests.sh

@@ -6,6 +6,9 @@ server_password=Secret123
 # Normalize spacing and expand the list afterwards. Remove {} for the single list element case
 tests_to_run=$(eval "echo {$(echo $TESTS_TO_RUN | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}')
 tests_to_ignore=$(eval "echo --ignore\ {$(echo $TESTS_TO_IGNORE | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}')
+tests_to_dedicate=
+[[ -n "$TESTS_TO_DEDICATE" ]] && \
+tests_to_dedicate=$(eval "echo --slice-dedicated={$(echo $TESTS_TO_DEDICATE | sed -e 's/[ \t]+*/,/g')}" | tr -d '{}')
 
 systemctl --now enable firewalld
 echo "Installing FreeIPA master for the domain ${server_domain} and realm ${server_realm}"
@@ -39,7 +42,11 @@ if [ "$install_result" -eq 0 ] ; then
 	ipa-test-task --help
 	ipa-run-tests --help
 
-	ipa-run-tests ${tests_to_ignore} --verbose --with-xunit '-k not test_dns_soa' ${tests_to_run}
+	ipa-run-tests ${tests_to_ignore} \
+            ${tests_to_dedicate} \
+            --slices=${SYSTEM_TOTALJOBSINPHASE:-1} \
+            --slice-num=${SYSTEM_JOBPOSITIONINPHASE:-1} \
+            --verbose --with-xunit '-k not test_dns_soa' ${tests_to_run}
 	tests_result=$?
 else
 	echo "ipa-server-install failed with code ${save_result}, skip IPA tests"

ipatests/azure/templates/run-test.yml

@@ -5,6 +5,7 @@ parameters:
   taskToRun: 'run-tests'
   testsToRun: ''
   testsToIgnore: ''
+  testsToDedicate: ''
 
 steps:
 - script: |
@@ -22,7 +23,10 @@ steps:
     set -e
     docker exec --env TESTS_TO_RUN="${{ parameters.testsToRun }}" \
                 --env TESTS_TO_IGNORE="${{ parameters.testsToIgnore }}" \
+                --env TESTS_TO_DEDICATE="${{ parameters.testsToDedicate }}" \
                 --env CI_RUNNER_LOGS_DIR="${{ parameters.logsPath }}" \
+                --env SYSTEM_TOTALJOBSINPHASE=$(System.TotalJobsInPhase) \
+                --env SYSTEM_JOBPOSITIONINPHASE=$(System.JobPositionInPhase) \
                 --privileged -t \
                 $(createContainer.containerName) \
                 /bin/bash --noprofile --norc -x /freeipa/ipatests/azure/azure-${{parameters.taskToRun}}.sh

ipatests/azure/templates/test-jobs.yml

@@ -3,7 +3,9 @@ parameters:
   jobTitle: ''
   testsToIgnore: []
   testsToRun: []
+  testsToDedicate: []
   taskToRun: ''
+  tasksParallel: 1
 
 jobs:
 - job: ${{ parameters.jobName }}
@@ -12,6 +14,8 @@ jobs:
   condition: succeeded()
   pool:
     vmImage: 'Ubuntu-16.04'
+  strategy:
+    parallel: ${{ parameters.tasksParallel }}
   steps:
     - template: setup-test-environment.yml
     - template: run-test.yml
@@ -21,6 +25,7 @@ jobs:
         taskToRun: ${{ parameters.taskToRun}}
         testsToRun: ${{ join(' ', parameters.testsToRun ) }}
         testsToIgnore: ${{ join(' ', parameters.testsToIgnore ) }}
+        testsToDedicate: ${{ join(' ', parameters.testsToDedicate ) }}
     - task: PublishTestResults@2
       inputs:
         testResultsFiles: $(CI_RUNNER_LOGS_DIR)/nosetests.xml
@@ -28,4 +33,4 @@ jobs:
       condition: succeededOrFailed()
     - template: save-test-artifacts.yml
       parameters:
-        logsArtifact: logs-${{parameters.jobName}}-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitecture)
+        logsArtifact: logs-${{parameters.jobName}}-$(Build.BuildId)-$(System.JobPositionInPhase)-$(Agent.OS)-$(Agent.OSArchitecture)