ipatests: wait while http/ldap/pkinit cert get renew on replica

LDAP/HTTP/PKINIT certificates should be renewd on replica after
moving system date. Test was failing because ipa-cert-fix ran
while these cert was not renewd and it tried to fix it.

This test adds check for replication before calling ipa-cert-fix
on replica.

Fixes: https://pagure.io/freeipa/issue/8815

Signed-off-by: Mohammad Rizwan <myusuf@redhat.com>
Reviewed-By: Florence Blanc-Renaud <flo@redhat.com>
Reviewed-By: Sergey Orlov <sorlov@redhat.com>
Reviewed-By: Rob Crittenden <rcritten@redhat.com>

ipatests/test_integration/test_ipa_cert_fix.py

@@ -5,16 +5,19 @@
 """
 Module provides tests for ipa-cert-fix CLI.
 """
+from cryptography.hazmat.backends import default_backend
+from cryptography import x509
+from datetime import datetime, date
 import pytest
-import re
 import time
 
 import logging
 from ipaplatform.paths import paths
+from ipapython.ipaldap import realm_to_serverid
 from ipatests.pytest_ipa.integration import tasks
 from ipatests.test_integration.base import IntegrationTest
 from ipatests.test_integration.test_caless import CALessBase, ipa_certs_cleanup
-
+from ipatests.test_integration.test_cert import get_certmonger_fs_id
 
 logger = logging.getLogger(__name__)
 
@@ -49,6 +52,49 @@ def check_status(host, cert_count, state, timeout=600):
     return count
 
 
+def needs_resubmit(host, req_id):
+    """Helper method to identify if cert request needs to be resubmitted
+    :param host: the host
+    :param req_id: request id to perform operation for
+
+    Returns True if resubmit needed else False
+    """
+    # check if cert is in monitoring state
+    tasks.wait_for_certmonger_status(
+        host, ('MONITORING'), req_id, timeout=600
+    )
+
+    # check if cert is valid and not expired
+    cmd = host.run_command(
+        'getcert list -i {} | grep expires'.format(req_id)
+    )
+    cert_expiry = cmd.stdout_text.split(' ')
+    cert_expiry = datetime.strptime(cert_expiry[1], '%Y-%m-%d').date()
+    if cert_expiry > date.today():
+        return False
+    else:
+        return True
+
+
+def get_cert_expiry(host, nssdb_path, cert_nick):
+    """Method to get cert expiry date of given certificate
+
+    :param host: the host
+    :param nssdb_path: nssdb path of certificate
+    :param cert_nick: certificate nick name for extracting cert from nssdb
+    """
+    # get initial expiry date to compare later with renewed cert
+    host.run_command([
+        'certutil', '-L', '-a',
+        '-d', nssdb_path,
+        '-n', cert_nick,
+        '-o', '/root/cert.pem'
+    ])
+    data = host.get_file_contents('/root/cert.pem')
+    cert = x509.load_pem_x509_certificate(data, backend=default_backend())
+    return cert.not_valid_after
+
+
 @pytest.fixture
 def expire_cert_critical():
     """
@@ -343,7 +389,19 @@ class TestCertFixReplica(IntegrationTest):
             setup_dns=False, extra_args=['--no-ntp']
         )
 
-    def test_renew_expired_cert_replica(self):
+    @pytest.fixture
+    def expire_certs(self):
+        # move system date to expire certs
+        for host in self.master, self.replicas[0]:
+            tasks.move_date(host, 'stop', '+3years+1days')
+
+        yield
+
+        # move date back on replica and master
+        for host in self.master, self.replicas[0]:
+            tasks.move_date(host, 'start', '-3years-1days')
+
+    def test_renew_expired_cert_replica(self, expire_certs):
         """Test renewal of certificates on replica with ipa-cert-fix
 
         This is to check that ipa-cert-fix renews the certificates
@@ -351,8 +409,6 @@ class TestCertFixReplica(IntegrationTest):
 
         related: https://pagure.io/freeipa/issue/7885
         """
-        tasks.move_date(self.master, 'stop', '+3years+1days')
-
         # wait for cert expiry
         check_status(self.master, 8, "CA_UNREACHABLE")
 
@@ -360,35 +416,95 @@ class TestCertFixReplica(IntegrationTest):
 
         check_status(self.master, 9, "MONITORING")
 
-        # move system date to expire cert on replica
+        # replica operations
-        tasks.move_date(self.replicas[0], 'stop', '+3years+1days')
+        # 'Server-Cert cert-pki-ca' cert will be in CA_UNREACHABLE state
+        cmd = self.replicas[0].run_command(
+            ['getcert', 'list',
+             '-d', paths.PKI_TOMCAT_ALIAS_DIR,
+             '-n', 'Server-Cert cert-pki-ca']
+        )
+        req_id = get_certmonger_fs_id(cmd.stdout_text)
+        tasks.wait_for_certmonger_status(
+            self.replicas[0], ('CA_UNREACHABLE'), req_id, timeout=600
+        )
+        # get initial expiry date to compare later with renewed cert
+        initial_expiry = get_cert_expiry(
+            self.replicas[0],
+            paths.PKI_TOMCAT_ALIAS_DIR,
+            'Server-Cert cert-pki-ca'
+        )
 
-        # RA agent cert will be expired and in CA_UNREACHABLE state
+        # check that HTTP,LDAP,PKINIT are renewed and in MONITORING state
-        check_status(self.replicas[0], 1, "CA_UNREACHABLE")
+        instance = realm_to_serverid(self.master.domain.realm)
+        dirsrv_cert = paths.ETC_DIRSRV_SLAPD_INSTANCE_TEMPLATE % instance
+        for cert in (paths.KDC_CERT, paths.HTTPD_CERT_FILE):
+            cmd = self.replicas[0].run_command(
+                ['getcert', 'list', '-f', cert]
+            )
+            req_id = get_certmonger_fs_id(cmd.stdout_text)
+            tasks.wait_for_certmonger_status(
+                self.replicas[0], ('MONITORING'), req_id, timeout=600
+            )
+
+        cmd = self.replicas[0].run_command(
+            ['getcert', 'list', '-d', dirsrv_cert]
+        )
+        req_id = get_certmonger_fs_id(cmd.stdout_text)
+        tasks.wait_for_certmonger_status(
+            self.replicas[0], ('MONITORING'), req_id, timeout=600
+        )
+
+        # check if replication working fine
+        testuser = 'testuser1'
+        password = 'Secret@123'
+        stdin = (f"{self.master.config.admin_password}\n"
+                 f"{self.master.config.admin_password}\n"
+                 f"{self.master.config.admin_password}\n")
+        self.master.run_command(['kinit', 'admin'], stdin_text=stdin)
+        tasks.user_add(self.master, testuser, password=password)
+        self.replicas[0].run_command(['kinit', 'admin'], stdin_text=stdin)
+        self.replicas[0].run_command(['ipa', 'user-show', testuser])
+
+        # renew shared certificates by resubmitting to certmonger
+        cmd = self.replicas[0].run_command(
+            ['getcert', 'list', '-f', paths.RA_AGENT_PEM]
+        )
+        req_id = get_certmonger_fs_id(cmd.stdout_text)
+        if needs_resubmit(self.replicas[0], req_id):
+            self.replicas[0].run_command(
+                ['getcert', 'resubmit', '-i', req_id]
+            )
+            tasks.wait_for_certmonger_status(
+                self.replicas[0], ('MONITORING'), req_id, timeout=600
+            )
+        for cert_nick in ('auditSigningCert cert-pki-ca',
+                          'ocspSigningCert cert-pki-ca',
+                          'subsystemCert cert-pki-ca'):
+            cmd = self.replicas[0].run_command(
+                ['getcert', 'list',
+                 '-d', paths.PKI_TOMCAT_ALIAS_DIR,
+                 '-n', cert_nick]
+            )
+            req_id = get_certmonger_fs_id(cmd.stdout_text)
+            if needs_resubmit(self.replicas[0], req_id):
+                self.replicas[0].run_command(
+                    ['getcert', 'resubmit', '-i', req_id]
+                )
+                tasks.wait_for_certmonger_status(
+                    self.replicas[0], ('MONITORING'), req_id, timeout=600
+                )
 
-        # renew RA agent cert
         self.replicas[0].run_command(
             ['ipa-cert-fix', '-v'], stdin_text='yes\n'
         )
 
-        # LDAP/HTTP/PKINIT certs will be renewed automaticaly
+        check_status(self.replicas[0], 9, "MONITORING")
-        # after moving date on replica. This 3, 1 CA cert,
-        # 1 RA agent cert. Check for total 5 valid certs.
-        check_status(self.replicas[0], 5, "MONITORING")
 
-        # get the req ids of all certs to renew remaining
+        # Sometimes certmonger takes time to update the cert status
-        # certs by re-submitting it
+        # So check in nssdb instead of relying on getcert command
-        result = self.replicas[0].run_command(['getcert', 'list'])
+        renewed_expiry = get_cert_expiry(
-        req_ids = re.findall(r'\d{14}', result.stdout_text)
+            self.replicas[0],
-
+            paths.PKI_TOMCAT_ALIAS_DIR,
-        # resubmit the certs to renew them
+            'Server-Cert cert-pki-ca'
-        for req_id in req_ids:
+        )
-            self.replicas[0].run_command(
+        assert renewed_expiry > initial_expiry
-                ['getcert', 'resubmit', '-i', req_id]
-            )
-
-        check_status(self.master, 9, "MONITORING")
-
-        # move date back on replica and master
-        tasks.move_date(self.replicas[0], 'start', '-3years-1days')
-        tasks.move_date(self.master, 'start', '-3years-1days')