Make sure remote hosts have our keys

In complex replication setups a replica may try to obtain CA keys from a
host that is not the master we initially create the keys against.
In this case race conditions may happen due to replication. So we need
to make sure the server we are contacting to get the CA keys has our
keys in LDAP. We do this by waiting to positively fetch our encryption
public key (the last one we create) from the target host LDAP server.

Fixes: https://pagure.io/freeipa/issue/6838

Signed-off-by: Simo Sorce <simo@redhat.com>
Reviewed-By: Stanislav Laznicka <slaznick@redhat.com>
Reviewed-By: Christian Heimes <cheimes@redhat.com>

ipaserver/install/custodiainstance.py

@@ -1,6 +1,6 @@
 # Copyright (C) 2015 FreeIPa Project Contributors, see 'COPYING' for license.
 
-from ipaserver.secrets.kem import IPAKEMKeys
+from ipaserver.secrets.kem import IPAKEMKeys, KEMLdap
 from ipaserver.secrets.client import CustodiaClient
 from ipaplatform.paths import paths
 from ipaplatform.constants import constants
@@ -18,6 +18,7 @@ import shutil
 import os
 import stat
 import tempfile
+import time
 import pwd
 
 
@@ -122,6 +123,27 @@ class CustodiaInstance(SimpleServiceInstance):
         cli = self.__CustodiaClient(server=master_host_name)
         cli.fetch_key('dm/DMHash')
 
+    def __wait_keys(self, host, timeout=300):
+        ldap_uri = 'ldap://%s' % host
+        deadline = int(time.time()) + timeout
+        root_logger.info("Waiting up to {} seconds to see our keys "
+                         "appear on host: {}".format(timeout, host))
+
+        konn = KEMLdap(ldap_uri)
+        saved_e = None
+        while True:
+            try:
+                return konn.check_host_keys(self.fqdn)
+            except Exception as e:
+                # log only once for the same error
+                if not isinstance(e, type(saved_e)):
+                    root_logger.debug(
+                        "Transient error getting keys: '{err}'".format(err=e))
+                    saved_e = e
+                if int(time.time()) > deadline:
+                    raise RuntimeError("Timed out trying to obtain keys.")
+                time.sleep(1)
+
     def __get_keys(self, ca_host, cacerts_file, cacerts_pwd, data):
         # Fecth all needed certs one by one, then combine them in a single
         # p12 file
@@ -129,6 +151,10 @@ class CustodiaInstance(SimpleServiceInstance):
         prefix = data['prefix']
         certlist = data['list']
 
+        # Before we attempt to fetch keys from this host, make sure our public
+        # keys have been replicated there.
+        self.__wait_keys(ca_host)
+
         cli = self.__CustodiaClient(server=ca_host)
 
         # Temporary nssdb

ipaserver/secrets/kem.py

@@ -24,6 +24,7 @@ import ldap
 
 IPA_REL_BASE_DN = 'cn=custodia,cn=ipa,cn=etc'
 IPA_KEYS_QUERY = '(&(ipaKeyUsage={usage:s})(memberPrincipal={princ:s}))'
+IPA_CHECK_QUERY = '(cn=enc/{host:s})'
 RFC5280_USAGE_MAP = {KEY_USAGE_SIG: 'digitalSignature',
                      KEY_USAGE_ENC: 'dataEncipherment'}
 
@@ -78,6 +79,17 @@ class KEMLdap(iSecLdap):
         jwk['use'] = KEY_USAGE_MAP[usage]
         return json_encode(jwk)
 
+    def check_host_keys(self, host):
+        conn = self.connect()
+        scope = ldap.SCOPE_SUBTREE
+
+        ldap_filter = self.build_filter(IPA_CHECK_QUERY, {'host': host})
+        r = conn.search_s(self.keysbase, scope, ldap_filter)
+        if len(r) != 1:
+            raise ValueError("Incorrect number of results (%d) searching for"
+                             "public key for %s" % (len(r), host))
+        return True
+
     def _format_public_key(self, key):
         if isinstance(key, str):
             jwkey = json_decode(key)