Auto-retry failed certmonger requests

During parallel replica installation, a request sometimes fails with
CA_REJECTED or CA_UNREACHABLE. The error occur when the master is
either busy or some information haven't been replicated yet. Even
a stuck request can be recovered, e.g. when permission and group
information have been replicated.

A new function request_and_retry_cert() automatically resubmits failing
requests until it times out.

Fixes: https://pagure.io/freeipa/issue/7623
Signed-off-by: Christian Heimes <cheimes@redhat.com>
Reviewed-By: Stanislav Laznicka <slaznick@redhat.com>

ipaserver/install/cainstance.py

@@ -926,7 +926,9 @@ class CAInstance(DogtagInstance):
                 profile='caServerCert',
                 pre_command='renew_ra_cert_pre',
                 post_command='renew_ra_cert',
-                storage="FILE")
+                storage="FILE",
+                resubmit_timeout=api.env.replication_wait_timeout
+            )
             self.__set_ra_cert_perms()
 
             self.requestId = str(reqId)

ipaserver/install/certs.py

@@ -658,14 +658,18 @@ class CertDB(object):
     def export_pem_cert(self, nickname, location):
         return self.nssdb.export_pem_cert(nickname, location)
 
-    def request_service_cert(self, nickname, principal, host):
-        certmonger.request_and_wait_for_cert(
+    def request_service_cert(self, nickname, principal, host,
+                             resubmit_timeout=None):
+        if resubmit_timeout is None:
+            resubmit_timeout = api.env.replication_wait_timeout
+        return certmonger.request_and_wait_for_cert(
             certpath=self.secdir,
             storage='NSSDB',
             nickname=nickname,
             principal=principal,
             subject=host,
-            passwd_fname=self.passwd_fname
+            passwd_fname=self.passwd_fname,
+            resubmit_timeout=resubmit_timeout
         )
 
     def is_ipa_issued_cert(self, api, nickname):

ipaserver/install/dsinstance.py

@@ -852,7 +852,8 @@ class DsInstance(service.Service):
                     ca='IPA',
                     profile=dogtag.DEFAULT_PROFILE,
                     dns=[self.fqdn],
-                    post_command=cmd
+                    post_command=cmd,
+                    resubmit_timeout=api.env.replication_wait_timeout
                 )
             finally:
                 if prev_helper is not None:

ipaserver/install/httpinstance.py

@@ -378,7 +378,8 @@ class HTTPInstance(service.Service):
                     dns=[self.fqdn],
                     post_command='restart_httpd',
                     storage='FILE',
-                    passwd_fname=key_passwd_file
+                    passwd_fname=key_passwd_file,
+                    resubmit_timeout=api.env.replication_wait_timeout
                 )
             finally:
                 if prev_helper is not None:

ipaserver/install/krbinstance.py

@@ -456,7 +456,8 @@ class KrbInstance(service.Service):
                 storage='FILE',
                 profile=KDC_PROFILE,
                 post_command='renew_kdc_cert',
-                perms=(0o644, 0o600)
+                perms=(0o644, 0o600),
+                resubmit_timeout=api.env.replication_wait_timeout
             )
         except dbus.DBusException as e:
             # if the certificate is already tracked, ignore the error