Wait for Directory Server ports to open

When Directory Server operation is run right after the server restart
the listening ports may not be opened yet. This makes the installation
fail.

This patch fixes this issue by waiting for both secure and insecure
Directory Server ports to open after every restart.

https://fedorahosted.org/freeipa/ticket/1076

ipaserver/install/cainstance.py

@@ -733,18 +733,7 @@ class CAInstance(service.Service):
     def __restart_instance(self):
         try:
             self.restart()
-            # Wait until the dogtag webapp responds
+            installutils.wait_for_open_ports('localhost', 9180, 300)
-            while True:
-                try:
-                    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-                    s.connect(('localhost', 9180))
-                    s.close()
-                    break
-                except socket.error, e:
-                    if e.errno == 111: # Connection refused
-                        time.sleep(1)
-                    else:
-                        raise e
         except Exception:
             # TODO: roll back here?
             logging.critical("Failed to restart the certificate server. See the installation log for details.")

ipaserver/install/dsinstance.py

@@ -412,6 +412,7 @@ class DsInstance(service.Service):
             if not is_ds_running():
                 logging.critical("Failed to restart the directory server. See the installation log for details.")
                 sys.exit(1)
+            installutils.wait_for_open_ports('localhost', [389, 636], 300)
         except SystemExit, e:
             raise e
         except Exception, e:

ipaserver/install/installutils.py

@@ -28,6 +28,7 @@ import sys
 import struct
 import fcntl
 import netaddr
+import time
 
 from ipapython import ipautil
 from ipapython import dnsclient
@@ -389,3 +390,34 @@ def create_keytab(path, principal):
 
     kadmin("ktadd -k " + path + " " + principal)
 
+def wait_for_open_ports(host, ports, timeout=0):
+    """
+    Wait until the specified port(s) on the remote host are open. Timeout
+    in seconds may be specified to limit the wait.
+    """
+    if not isinstance(ports, (tuple, list)):
+        ports = [ports]
+
+    op_timeout = time.time() + timeout
+    ipv6_failover = False
+    
+    for port in ports:
+        while True:
+            try:
+                if ipv6_failover:
+                    s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+                else:
+                    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                s.connect((host, port))
+                s.close()
+                break;
+            except socket.error, e:
+                if e.errno == 111:  # 111: Connection refused
+                    if timeout and time.time() > op_timeout: # timeout exceeded
+                        raise e
+                    time.sleep(1)
+                elif not ipv6_failover: # fallback to IPv6 connection
+                    ipv6_failover = True
+                else:
+                    raise e
+