Azure: Make it possible to configure distro-specific stuff

This allows to run IPA tests on Azure using any distro.

To achieve this, one has to do:
1) place a platform specific template on 'ipatests/azure/templates/'
and make a soft link from 'ipatests/azure/templates/variables.yml' to
the new template.
2) place a configuration templates on these paths
3) templates have to answer the questions such as:
  a) which Docker image to use to build IPA packages (rpm, deb, etc.)
  b) how to prepare Build environment
  c) how to build IPA packages
  d) how to prepare environment to run Tox tests
  e) how to prepare environment to run WebUI unittests
  f) which base Docker image to use to build the new image to run
     IPA tests within it

Fixes: https://pagure.io/freeipa/issue/8202
Signed-off-by: Stanislav Levin <slev@altlinux.org>
Reviewed-By: Alexander Bokovoy <abokovoy@redhat.com>

BUILD.txt

@@ -20,7 +20,7 @@ TIP: You might need to enable the "nodejs" module:
 # dnf module list nodejs
 # dnf module enable nodejs:<latest version> -y
 
-For more dependencies management tips, you can have a look at ./ipatests/azure/templates/prepare-build.yml
+For more dependencies management tips, you can have a look at ./ipatests/azure/templates/prepare-build-fedora.yml
 
 Building
 --------

ipatests/azure/azure-pipelines.yml

@@ -2,25 +2,19 @@ trigger:
 - master
 
 variables:
-  CI_RUNNER_LOGS_DIR: logs
-  localsdir: $(Build.Repository.LocalPath)
-  builddir: /__w/1/s
-  # Provision script: setup_containers.py requires Python3.6+
-  # Ubuntu-16.04 has Python 3.5.2 on board
-  # https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1604-README.md
-  # Ubuntu-18.04 - 3.6.9
-  # https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md
-  VM_IMAGE: 'Ubuntu-18.04'
+- template: templates/variables-common.yml
+# platform specific variables, links to
+- template: templates/variables.yml
 
 jobs:
 - job: Build
   pool:
     vmImage: $(VM_IMAGE)
   container:
-    image: registry.fedoraproject.org/f31/fedora-toolbox
+    image: $(DOCKER_BUILD_IMAGE)
     options: --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --privileged
   steps:
-    - template: templates/prepare-build.yml
+    - template: templates/${{ variables.PREPARE_BUILD_TEMPLATE }}
     - script: |
         set -e
         echo "Running autoconf generator"
@@ -32,19 +26,15 @@ jobs:
         make V=0 "GIT_BRANCH=$(System.PullRequest.TargetBranch)" fastlint
       displayName: Quick code style check
       condition: eq(variables['Build.Reason'], 'PullRequest')
-    - script: |
-        set -e
-        echo "Running make target 'rpms'"
-        make V=0 rpms LOG_COMPILE='gdb.minimal -return-child-result -ex run -ex "thread apply all bt" -ex "quit" --args'
-      displayName: Build packages
+    - template: templates/${{ variables.BUILD_TEMPLATE }}
     - script: |
         set -e
         mkdir container
         cp -pr dist container/
-        cp ipatests/azure/Dockerfile.build-container container/Dockerfile
+        cp ipatests/azure/$(DOCKER_DOCKERFILE) container/Dockerfile
         cd container
-        docker build -t freeipa-fedora-builder .
-        docker save freeipa-fedora-builder | gzip > '$(builddir)/freeipa-fedora-builder-container.tar.gz'
+        docker build -t freeipa-azure-builder .
+        docker save freeipa-azure-builder | gzip > '$(builddir)/freeipa-azure-builder-container.tar.gz'
       displayName: Create container image for test
     - template: templates/publish-build.yml
 
@@ -52,10 +42,10 @@ jobs:
   pool:
     vmImage: $(VM_IMAGE)
   container:
-    image: registry.fedoraproject.org/f31/fedora-toolbox
+    image: $(DOCKER_BUILD_IMAGE)
     options: --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --privileged
   steps:
-    - template: templates/prepare-build.yml
+    - template: templates/${{ variables.PREPARE_BUILD_TEMPLATE }}
     - script: |
         set -e
         echo "Running autoconf generator"
@@ -71,26 +61,22 @@ jobs:
   pool:
     vmImage: $(VM_IMAGE)
   container:
-    image: registry.fedoraproject.org/f31/fedora-toolbox
+    image: $(DOCKER_BUILD_IMAGE)
     options: --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --privileged
   steps:
-    - template: templates/prepare-build.yml
+    - template: templates/${{ variables.PREPARE_BUILD_TEMPLATE }}
     - task: UsePythonVersion@0
       inputs:
         versionSpec: 3.7
         architecture: x64
-    - script: |
-        set -e
-        sudo dnf -y install nss-tools
-        python3 -m pip install --user --upgrade pip setuptools pycodestyle
-      displayName: 'Install prerequisites'
+    - template: templates/${{ variables.PREPARE_TOX_TEMPLATE }}
     - script: |
         set -e
         echo "Running tox"
         export LANG=en_US.utf8
         export LC_CTYPE=en_US.utf8
         locale
-        tox -e py37,pypi,pylint3
+        $(TOX_COMMAND) -e py37,pypi,pylint3
       displayName: Tox
     - task: PublishTestResults@2
       inputs:
@@ -102,18 +88,15 @@ jobs:
   pool:
     vmImage: $(VM_IMAGE)
   container:
-    image: registry.fedoraproject.org/f31/fedora-toolbox
+    image: $(DOCKER_BUILD_IMAGE)
     options: --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --privileged
   steps:
-    - template: templates/prepare-build.yml
+    - template: templates/${{ variables.PREPARE_BUILD_TEMPLATE }}
     - task: UsePythonVersion@0
       inputs:
         versionSpec: 3.7
         architecture: x64
-    - script: |
-        set -e
-        sudo dnf -y install npm fontconfig
-      displayName: 'Install prerequisites'
+    - template: templates/${{ variables.PREPARE_WEBUI_TEMPLATE }}
     - script: |
         set -e
         echo "Running autoconf generator"

ipatests/azure/templates/build-fedora.yml

@@ -0,0 +1,6 @@
+steps:
+- script: |
+    set -e
+    echo "Running make target 'rpms'"
+    make V=0 rpms LOG_COMPILE='gdb.minimal -return-child-result -ex run -ex "thread apply all bt" -ex "quit" --args'
+  displayName: Build packages

ipatests/azure/templates/prepare-tox-fedora.yml

@@ -0,0 +1,6 @@
+steps:
+- script: |
+    set -e
+    sudo dnf -y install nss-tools
+    python3 -m pip install --user --upgrade pip setuptools pycodestyle
+  displayName: Install Tox prerequisites

ipatests/azure/templates/prepare-webui-fedora.yml

@@ -0,0 +1,5 @@
+steps:
+- script: |
+    set -e
+    sudo dnf -y install npm fontconfig
+  displayName: Install WebUI Unit tests prerequisites

ipatests/azure/templates/publish-build.yml

@@ -8,5 +8,5 @@ steps:
   displayName: Publish container image
   inputs:
     artifactName: 'image-$(Build.BuildId)-$(Agent.OS)-$(Agent.OSArchitecture)'
-    targetPath: $(Agent.BuildDirectory)/s/freeipa-fedora-builder-container.tar.gz
+    targetPath: $(Agent.BuildDirectory)/s/freeipa-azure-builder-container.tar.gz
 

ipatests/azure/templates/setup-test-environment.yml

@@ -1,5 +1,5 @@
 parameters:
-  imageName: 'freeipa-fedora-builder:latest'
+  imageName: 'freeipa-azure-builder:latest'
   topology:
     replicas: 0
     clients: 0
@@ -28,9 +28,9 @@ steps:
     targetPath: $(Agent.BuildDirectory)/s
 - script: |
     set -e
-    docker load --input $(Agent.BuildDirectory)/s/freeipa-fedora-builder-container.tar.gz
+    docker load --input $(Agent.BuildDirectory)/s/freeipa-azure-builder-container.tar.gz
     docker images
-    docker inspect freeipa-fedora-builder:latest
+    docker inspect freeipa-azure-builder:latest
   displayName: Import pre-built container to the engine
 - script: |
     set -ex

ipatests/azure/templates/test-jobs.yml

@@ -19,7 +19,7 @@ steps:
   displayName: Increase entropy level
 - template: setup-test-environment.yml
   parameters:
-    containerName: 'freeipa-fedora-builder:latest'
+    containerName: 'freeipa-azure-builder:latest'
     topology:
       replicas: ${{ parameters.topology.replicas }}
       clients: ${{ parameters.topology.clients }}

ipatests/azure/templates/variables-common.yml

@@ -0,0 +1,10 @@
+variables:
+  CI_RUNNER_LOGS_DIR: logs
+  localsdir: $(Build.Repository.LocalPath)
+  builddir: /__w/1/s
+  # Provision script: setup_containers.py requires Python3.6+
+  # Ubuntu-16.04 has Python 3.5.2 on board
+  # https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1604-REA    DME.md
+  # Ubuntu-18.04 - 3.6.9
+  # https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-REA    DME.md
+  VM_IMAGE: 'Ubuntu-18.04'

ipatests/azure/templates/variables-fedora.yml

@@ -0,0 +1,17 @@
+variables:
+  IPA_PLATFORM: fedora
+  # the Docker public image to build IPA packages (rpms)
+  DOCKER_BUILD_IMAGE: 'registry.fedoraproject.org/f31/fedora-toolbox'
+
+  # the Dockerfile to build Docker image for running IPA tests
+  DOCKER_DOCKERFILE: ${{ format('Dockerfile.build.{0}', variables.IPA_PLATFORM) }}
+
+  # the template to install IPA's buildtime dependencies
+  PREPARE_BUILD_TEMPLATE: ${{ format('prepare-build-{0}.yml', variables.IPA_PLATFORM) }}
+
+  # the template to build IPA packages (rpms)
+  BUILD_TEMPLATE: ${{ format('build-{0}.yml', variables.IPA_PLATFORM) }}
+  PREPARE_TOX_TEMPLATE: ${{ format('prepare-tox-{0}.yml', variables.IPA_PLATFORM) }}
+  PREPARE_WEBUI_TEMPLATE: ${{ format('prepare-webui-{0}.yml', variables.IPA_PLATFORM) }}
+
+  TOX_COMMAND: tox

ipatests/azure/templates/variables.yml

@@ -0,0 +1 @@
+variables-fedora.yml