DNS Locations: make ipa-ca record generation more robust

__add_ca_records_from_hostname() now skips over DNS exceptions and
retries resolution until timeout of 120 seconds is reached.

Luckily current logic fails safe: In cases where resolution failed for
all the CA servers, the resulting zone object will not contain ipa-ca
record at all and the update logic will skip update for this name.
I.e. the original values in ipa-ca record set will be left in place.

https://fedorahosted.org/freeipa/ticket/2008

Reviewed-By: Martin Basti <mbasti@redhat.com>

ipaserver/dns_data_management.py

@@ -12,12 +12,16 @@ from dns import (
     rdatatype,
     zone,
 )
+from dns.exception import DNSException
 from dns.rdtypes.IN.SRV import SRV
 from dns.rdtypes.ANY.TXT import TXT
 
+from time import sleep, time
+
 from ipalib import errors
 from ipalib.dns import record_name_format
 from ipapython.dnsutil import DNSName, resolve_rrsets
+from ipapython.ipa_log_manager import root_logger
 
 if six.PY3:
     unicode=str
@@ -134,7 +138,22 @@ class IPASystemRecords(object):
     def __add_ca_records_from_hostname(self, zone_obj, hostname):
         assert isinstance(hostname, DNSName) and hostname.is_absolute()
         r_name = DNSName('ipa-ca') + self.domain_abs
-        rrsets = resolve_rrsets(hostname, (rdatatype.A, rdatatype.AAAA))
+        rrsets = []
+        end_time = time() + 120  # timeout in seconds
+        while time() < end_time:
+            try:
+                rrsets = resolve_rrsets(hostname, (rdatatype.A, rdatatype.AAAA))
+            except DNSException:  # logging is done inside resolve_rrsets
+                pass
+            if rrsets:
+                break
+            sleep(5)
+
+        if not rrsets:
+            root_logger.error('unable to resolve host name %s to IP address, '
+                              'ipa-ca DNS record will be incomplete', hostname)
+            return
+
         for rrset in rrsets:
             for rd in rrset:
                 rdataset = zone_obj.get_rdataset(