Test to check if Certmonger tracks certs in between reboots/interruptions and while in "CA_WORKING" state

When a resubmit request is submitted an "invalid cookie"
error message is no longer shown

Earlier an "invlaid cookie" error message was shown when getcert list was called.

The fix allows an empty cookie in dogtag-ipa-ca-renew-agent-submit

Pagure Issue: https://pagure.io/freeipa/issue/8164

Signed-off-by: Sumedh Sidhaye <ssidhaye@redhat.com>

Fixup for test to verify that POLL will not error out on cookie

Author:    Rob Crittenden <rcritten@redhat.com>
Date:      Tue Mar 24 15:30:38 2020 -0400

Fixed review comments

Signed-off-by: Sumedh Sidhaye <ssidhaye@redhat.com>
Reviewed-By: Mohammad Rizwan Yusuf <myusuf@redhat.com>
Reviewed-By: Francois Cami <fcami@redhat.com>
Reviewed-By: Rob Crittenden <rcritten@redhat.com>

ipatests/pytest_ipa/integration/tasks.py

@@ -2135,7 +2135,7 @@ def wait_for_request(host, request_id, timeout=120):
         )
 
         state = result.stdout_text.strip()
-        print("certmonger request is in state %r", state)
+        logger.info("certmonger request is in state %s", state)
         if state in ('CA_REJECTED', 'CA_UNREACHABLE', 'CA_UNCONFIGURED',
                      'NEED_GUIDANCE', 'NEED_CA', 'MONITORING'):
             break
@@ -2146,6 +2146,34 @@ def wait_for_request(host, request_id, timeout=120):
     return state
 
 
+def wait_for_certmonger_status(host, status, request_id, timeout=120):
+    """Aggressively wait for a specific certmonger status.
+
+       This checks the status every second in order to attempt to
+       catch transient states like SUBMITTED. There are no guarantees.
+
+       :param host: the host where the uninstallation takes place
+       :param status: tuple of statuses to look for
+       :param request_id: request_id of request to check status on
+       :param timeout: max time in seconds to wait for the status
+    """
+    for _i in range(0, timeout, 1):
+        result = host.run_command(
+            "getcert list -i %s | grep status: | awk '{ print $2 }'" %
+            request_id
+        )
+
+        state = result.stdout_text.strip()
+        logger.info("certmonger request is in state %s", state)
+        if state in status:
+            break
+        time.sleep(1)
+    else:
+        raise RuntimeError("request timed out")
+
+    return state
+
+
 def wait_for_sssd_domain_status_online(host, timeout=120):
     """Wait up to timeout (in seconds) for sssd domain status to become Online
 

ipatests/test_integration/test_cert.py

@@ -7,8 +7,10 @@ Module provides tests which testing ability of various certificate
 related scenarios.
 """
 import ipaddress
+import pytest
 import re
 
+from ipaplatform.paths import paths
 from cryptography import x509
 from cryptography.x509.oid import ExtensionOID
 from cryptography.hazmat.backends import default_backend
@@ -16,6 +18,40 @@ from cryptography.hazmat.backends import default_backend
 from ipatests.pytest_ipa.integration import tasks
 from ipatests.test_integration.base import IntegrationTest
 
+DEFAULT_RA_AGENT_SUBMITTED_VAL = '19700101000000'
+
+
+def get_certmonger_fs_id(input_str):
+    """Get certmonger FS ID
+    from the `getcert list -f /var/lib/ipa/ra-agent.pem` output
+    command output
+
+    :return request ID string
+    """
+    request_id = re.findall(r'\d+', input_str)
+    return request_id[1]
+
+
+def get_certmonger_request_value(host, requestid, state):
+    """Get certmonger submitted value from
+    /var/lib/certmonger/requests/<timestamp>
+
+    :return submitted timestamp value
+    """
+    result = host.run_command(
+        ['grep', '-rl', 'id={0}'.format(requestid),
+         paths.CERTMONGER_REQUESTS_DIR]
+    )
+    assert result.stdout_text is not None
+    filename = result.stdout_text.strip()
+    request_file = host.get_file_contents(filename, encoding='utf-8')
+    val = None
+    for line in request_file.split('\n'):
+        if line.startswith('%s=' % state):
+            _unused, val = line.partition("=")[::2]
+            break
+    return val
+
 
 class TestInstallMasterClient(IntegrationTest):
     num_clients = 1
@@ -90,3 +126,94 @@ class TestInstallMasterClient(IntegrationTest):
         assert dnsnames == [self.clients[0].hostname]
         ipaddrs = ext.value.get_values_for_type(x509.IPAddress)
         assert ipaddrs == [ipaddress.ip_address(self.clients[0].ip)]
+
+
+class TestCertmongerInterruption(IntegrationTest):
+    num_replicas = 1
+
+    @classmethod
+    def install(cls, mh):
+        tasks.install_master(cls.master)
+        tasks.install_replica(cls.master, cls.replicas[0])
+
+    def test_certmomger_tracks_renewed_certs_during_interruptions(self):
+        """Test that CA renewal handles early CA_WORKING and restarts
+
+        A non-renewal master CA might submit a renewal request before
+        the renewal master actually updating the certs. This is expected.
+        The tracking request will result in CA_WORKING.
+
+        This would trigger a different path within the IPA renewal
+        scripts which differentiate between a SUBMIT (new request) and
+        a POLL (resume request). The script was requiring a cookie
+        value for POLL requests which wasn't available and was
+        erroring out unrecoverably without restarting certmonger.
+
+        Submit a request for renewal early and wait for it to go into
+        CA_WORKING. Resubmit the request to ensure that the request
+        remains in CA_WORKING without reporting any ca_error like
+        Invalid cookie: ''
+
+        Use the submitted value in the certmonger request to validate
+        that the request was resubmitted and not rely on catching
+        the states directly.
+
+        Pagure Issue: https://pagure.io/freeipa/issue/8164
+        """
+        cmd = ['getcert', 'list', '-f', paths.RA_AGENT_PEM]
+        result = self.replicas[0].run_command(cmd)
+
+        # Get Request ID and Submitted Values
+        request_id = get_certmonger_fs_id(result.stdout_text)
+        start_val = get_certmonger_request_value(self.replicas[0],
+                                                 request_id, "submitted")
+
+        # at this point submitted value for RA agent cert should be
+        # 19700101000000 since it has never been submitted for renewal.
+        assert start_val == DEFAULT_RA_AGENT_SUBMITTED_VAL
+
+        cmd = ['getcert', 'resubmit', '-f', paths.RA_AGENT_PEM]
+        self.replicas[0].run_command(cmd)
+
+        tasks.wait_for_certmonger_status(self.replicas[0],
+                                         ('CA_WORKING', 'MONITORING'),
+                                         request_id)
+
+        resubmit_val = get_certmonger_request_value(self.replicas[0],
+                                                    request_id,
+                                                    "submitted")
+
+        if resubmit_val == DEFAULT_RA_AGENT_SUBMITTED_VAL:
+            pytest.fail("Request was not resubmitted")
+
+        ca_error = get_certmonger_request_value(self.replicas[0],
+                                                request_id, "ca_error")
+        state = get_certmonger_request_value(self.replicas[0],
+                                             request_id, "state")
+
+        assert ca_error is None
+        assert state == 'CA_WORKING'
+
+        cmd = ['getcert', 'resubmit', '-f', paths.RA_AGENT_PEM]
+        self.replicas[0].run_command(cmd)
+
+        tasks.wait_for_certmonger_status(self.replicas[0],
+                                         ('CA_WORKING', 'MONITORING'),
+                                         request_id)
+
+        resubmit2_val = get_certmonger_request_value(self.replicas[0],
+                                                     request_id,
+                                                     "submitted")
+
+        if resubmit_val == DEFAULT_RA_AGENT_SUBMITTED_VAL:
+            pytest.fail("Request was not resubmitted")
+
+        assert resubmit2_val > resubmit_val
+
+        ca_error = get_certmonger_request_value(self.replicas[0],
+                                                request_id, "ca_error")
+        state = get_certmonger_request_value(self.replicas[0],
+                                             request_id, "state")
+
+        assert ca_error is None
+        assert state == 'CA_WORKING'