Centralize timeout for waiting for servers to start.

All service start/restart currently go through ipapython/platform so
move the "wait for service to start" code there as well.

A dictionary of known services and ports to wait on is defined in base.py
This is referenced by the platforms by instance name to determine what
to wait for. For the case of dirsrv if we get that as a plain name
(no specific instance) it is assumed to be the main IPA service.

https://fedorahosted.org/freeipa/ticket/2375
https://fedorahosted.org/freeipa/ticket/2610
This commit is contained in:
Rob Crittenden 2012-05-24 11:23:36 -04:00
parent 6fb802152a
commit e5b6260008
16 changed files with 176 additions and 105 deletions

View File

@ -24,7 +24,8 @@ try:
from ipaserver.install import service, installutils
from ipapython import services as ipaservices
from ipaserver.install.dsinstance import config_dirname, realm_to_serverid
from ipaserver.install.installutils import is_ipa_configured, wait_for_open_ports, wait_for_open_socket, ScriptError
from ipaserver.install.installutils import is_ipa_configured, ScriptError
from ipapython.ipautil import wait_for_open_ports, wait_for_open_socket
from ipapython import sysrestore
from ipapython import config
from ipalib import api, errors
@ -105,22 +106,25 @@ def parse_options():
def emit_err(err):
sys.stderr.write(err + '\n')
def get_config():
def get_config(dirsrv):
base = "cn=%s,cn=masters,cn=ipa,cn=etc,%s" % (api.env.host,
api.env.basedn)
srcfilter = '(ipaConfigString=enabledService)'
attrs = ['cn', 'ipaConfigString']
if not dirsrv.is_running():
raise IpactlError("Failed to get list of services to probe status:\n" +
"Directory Server is stopped", 3)
try:
# systemd services are so fast that we come here before
# Directory Server actually starts listening. Wait for
# the socket/port be really available.
# The start/restart functions already wait for the server to be
# started. What we are doing with this wait is really checking to see
# if the server is listening at all.
lurl = ldapurl.LDAPUrl(api.env.ldap_uri)
if lurl.urlscheme == 'ldapi':
wait_for_open_socket(lurl.hostport, timeout=6)
wait_for_open_socket(lurl.hostport, timeout=api.env.startup_timeout)
else:
(host,port) = lurl.hostport.split(':')
wait_for_open_ports(host, [int(port)], timeout=6)
wait_for_open_ports(host, [int(port)], timeout=api.env.startup_timeout)
con = ldap.initialize(api.env.ldap_uri)
con.sasl_interactive_bind_s('', SASL_EXTERNAL)
res = con.search_st(base,
@ -175,7 +179,7 @@ def ipa_start(options):
svc_list = []
try:
svc_list = get_config()
svc_list = get_config(dirsrv)
except Exception, e:
emit_err("Failed to read data from Directory Service: " + str(e))
emit_err("Shutting down")
@ -219,14 +223,14 @@ def ipa_stop(options):
dirsrv = ipaservices.knownservices.dirsrv
svc_list = []
try:
svc_list = get_config()
svc_list = get_config(dirsrv)
except Exception, e:
# ok if dirsrv died this may fail, so let's try to quickly restart it
# and see if we can get anything. If not throw our hands up and just
# exit
try:
dirsrv.start(capture_output=False)
svc_list = get_config()
svc_list = get_config(dirsrv)
except Exception, e:
emit_err("Failed to read data from Directory Service: " + str(e))
emit_err("Shutting down")
@ -266,7 +270,7 @@ def ipa_restart(options):
svc_list = []
try:
svc_list = get_config()
svc_list = get_config(dirsrv)
except Exception, e:
emit_err("Failed to read data from Directory Service: " + str(e))
emit_err("Shutting down")
@ -318,7 +322,7 @@ def ipa_status(options):
svc_list = []
try:
svc_list = get_config()
svc_list = get_config(dirsrv)
except IpactlError, e:
raise e
except Exception, e:

View File

@ -111,6 +111,8 @@ DEFAULT_CONFIG = (
('xmlrpc_uri', 'http://localhost:8888/ipa/xml'),
('rpc_json_uri', 'http://localhost:8888/ipa/json'),
('ldap_uri', 'ldap://localhost:389'),
# Time to wait for a service to start, in seconds
('startup_timeout', 120),
# Web Application mount points
('mount_ipa', '/ipa/'),

View File

@ -41,6 +41,7 @@ import re
import xmlrpclib
import datetime
import netaddr
import time
from dns import resolver, rdatatype
from dns.exception import DNSException
@ -1010,3 +1011,56 @@ def utf8_encode_values(values):
return map(utf8_encode_value, values)
else:
return utf8_encode_value(values)
def wait_for_open_ports(host, ports, timeout=0):
"""
Wait until the specified port(s) on the remote host are open. Timeout
in seconds may be specified to limit the wait.
"""
if not isinstance(ports, (tuple, list)):
ports = [ports]
root_logger.debug('wait_for_open_ports: %s %s timeout %d' % (host, ports, timeout))
op_timeout = time.time() + timeout
ipv6_failover = False
for port in ports:
while True:
try:
if ipv6_failover:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
else:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
s.close()
break
except socket.error, e:
if e.errno == 111: # 111: Connection refused
if timeout and time.time() > op_timeout: # timeout exceeded
raise e
time.sleep(1)
elif not ipv6_failover: # fallback to IPv6 connection
ipv6_failover = True
else:
raise e
def wait_for_open_socket(socket_name, timeout=0):
"""
Wait until the specified socket on the local host is open. Timeout
in seconds may be specified to limit the wait.
"""
op_timeout = time.time() + timeout
while True:
try:
s = socket.socket(socket.AF_UNIX)
s.connect(socket_name)
s.close()
break
except socket.error, e:
if e.errno in (2,111): # 111: Connection refused, 2: File not found
if timeout and time.time() > op_timeout: # timeout exceeded
raise e
time.sleep(1)
else:
raise e

View File

@ -18,29 +18,42 @@
from ipalib.plugable import MagicDict
# Canonical names of services as IPA wants to see them. As we need to have *some* naming,
# set them as in Red Hat distributions. Actual implementation should make them available
# through knownservices.<name> and take care of remapping internally, if needed
# Canonical names of services as IPA wants to see them. As we need to have
# *some* naming, set them as in Red Hat distributions. Actual implementation
# should make them available through knownservices.<name> and take care of
# re-mapping internally, if needed
wellknownservices = ['certmonger', 'dirsrv', 'httpd', 'ipa', 'krb5kdc',
'messagebus', 'nslcd', 'nscd', 'ntpd', 'portmap',
'rpcbind', 'kadmin', 'sshd', 'autofs', 'rpcgssd',
'rpcidmapd']
# The common ports for these services. This is used to wait for the
# service to become available.
wellknownports = {
'dirsrv@PKI-IPA.service': [7389],
'PKI-IPA': [7389],
'dirsrv': [389], # this is only used if the incoming instance name is blank
'pki-cad': [9180],
}
class AuthConfig(object):
"""
AuthConfig class implements system-independent interface to configure
system authentication resources. In Red Hat systems this is done with
authconfig(8) utility.
AuthConfig class is nothing more than a tool to gather configuration options
and execute their processing. These options then converted by an actual implementation
to series of a system calls to appropriate utilities performing real configuration.
AuthConfig class is nothing more than a tool to gather configuration
options and execute their processing. These options then converted by
an actual implementation to series of a system calls to appropriate
utilities performing real configuration.
IPA *expects* names of AuthConfig's options to follow authconfig(8) naming scheme!
IPA *expects* names of AuthConfig's options to follow authconfig(8)
naming scheme!
Actual implementation should be done in ipapython/platform/<platform>.py by inheriting from
platform.AuthConfig and redefining __build_args() and execute() methods.
Actual implementation should be done in ipapython/platform/<platform>.py
by inheriting from platform.AuthConfig and redefining __build_args()
and execute() methods.
from ipapython.platform import platform
class PlatformAuthConfig(platform.AuthConfig):
@ -53,9 +66,11 @@ class AuthConfig(object):
authconfig = PlatformAuthConfig
....
See ipapython/platform/redhat.py for a sample implementation that uses authconfig(8) as its backend.
See ipapython/platform/redhat.py for a sample implementation that uses
authconfig(8) as its backend.
From IPA code perspective, the authentication configuration should be done with use of ipapython.services.authconfig:
From IPA code perspective, the authentication configuration should be
done with use of ipapython.services.authconfig:
from ipapython import services as ipaservices
auth_config = ipaservices.authconfig()
@ -69,8 +84,8 @@ class AuthConfig(object):
add_parameter("nisdomain","foobar")
auth_config.execute()
If you need to re-use existing AuthConfig instance for multiple runs, make sure to
call 'AuthConfig.reset()' between the runs.
If you need to re-use existing AuthConfig instance for multiple runs,
make sure to call 'AuthConfig.reset()' between the runs.
"""
def __init__(self):
@ -106,21 +121,21 @@ class AuthConfig(object):
class PlatformService(object):
"""
PlatformService abstracts out external process running on the system which is possible
to administer (start, stop, check status, etc).
PlatformService abstracts out external process running on the system
which is possible to administer (start, stop, check status, etc).
"""
def __init__(self, service_name):
self.service_name = service_name
def start(self, instance_name="", capture_output=True):
def start(self, instance_name="", capture_output=True, wait=True):
return
def stop(self, instance_name="", capture_output=True):
return
def restart(self, instance_name="", capture_output=True):
def restart(self, instance_name="", capture_output=True, wait=True):
return
def is_running(self, instance_name=""):
@ -149,8 +164,9 @@ class PlatformService(object):
class KnownServices(MagicDict):
"""
KnownServices is an abstract class factory that should give out instances of well-known
platform services. Actual implementation must create these instances as its own attributes
on first access (or instance creation) and cache them.
KnownServices is an abstract class factory that should give out instances
of well-known platform services. Actual implementation must create these
instances as its own attributes on first access (or instance creation)
and cache them.
"""

View File

@ -98,7 +98,7 @@ class Fedora16DirectoryService(Fedora16Service):
restore_context(dirsrv_systemd)
ipautil.run(["/bin/systemctl", "--system", "daemon-reload"],raiseonerr=False)
def restart(self, instance_name="", capture_output=True):
def restart(self, instance_name="", capture_output=True, wait=True):
if len(instance_name) > 0:
elements = self.service_name.split("@")
srv_etc = os.path.join(self.SYSTEMD_ETC_PATH, self.service_name)
@ -109,7 +109,7 @@ class Fedora16DirectoryService(Fedora16Service):
elif not os.path.samefile(srv_etc, srv_lnk):
os.unlink(srv_lnk)
os.symlink(srv_etc, srv_lnk)
super(Fedora16DirectoryService, self).restart(instance_name, capture_output=capture_output)
super(Fedora16DirectoryService, self).restart(instance_name, capture_output=capture_output, wait=wait)
# Enforce restart of IPA services when we do enable it
# This gets around the fact that after ipa-server-install systemd thinks

View File

@ -26,6 +26,7 @@ import sys
import socket
from ipapython import ipautil
from ipapython.platform import base
from ipalib import api
# All what we allow exporting directly from this module
# Everything else is made available through these symbols when they are
@ -46,14 +47,31 @@ from ipapython.platform import base
__all__ = ['authconfig', 'service', 'knownservices', 'backup_and_replace_hostname', 'restore_context', 'check_selinux_status']
class RedHatService(base.PlatformService):
def __wait_for_open_ports(self, instance_name=""):
"""
If this is a service we need to wait for do so.
"""
ports = None
if instance_name in base.wellknownports:
ports = base.wellknownports[instance_name]
else:
if self.service_name in base.wellknownports:
ports = base.wellknownports[self.service_name]
if ports:
ipautil.wait_for_open_ports('localhost', ports, api.env.startup_timeout)
def stop(self, instance_name="", capture_output=True):
ipautil.run(["/sbin/service", self.service_name, "stop", instance_name], capture_output=capture_output)
def start(self, instance_name="", capture_output=True):
def start(self, instance_name="", capture_output=True, wait=True):
ipautil.run(["/sbin/service", self.service_name, "start", instance_name], capture_output=capture_output)
if wait and self.is_running(instance_name):
self.__wait_for_open_ports(instance_name)
def restart(self, instance_name="", capture_output=True):
def restart(self, instance_name="", capture_output=True, wait=True):
ipautil.run(["/sbin/service", self.service_name, "restart", instance_name], capture_output=capture_output)
if wait and self.is_running(instance_name):
self.__wait_for_open_ports(instance_name)
def is_running(self, instance_name=""):
ret = True

View File

@ -20,6 +20,7 @@
from ipapython import ipautil
from ipapython.platform import base
import sys, os, shutil
from ipalib import api
class SystemdService(base.PlatformService):
SYSTEMD_ETC_PATH = "/etc/systemd/system/"
@ -73,16 +74,34 @@ class SystemdService(base.PlatformService):
return (None,None)
return dict(map(lambda x: splitter(x, separator=separator), text.split("\n")))
def __wait_for_open_ports(self, instance_name=""):
"""
If this is a service we need to wait for do so.
"""
ports = None
if instance_name in base.wellknownports:
ports = base.wellknownports[instance_name]
else:
elements = self.service_name.split("@")
if elements[0] in base.wellknownports:
ports = base.wellknownports[elements[0]]
if ports:
ipautil.wait_for_open_ports('localhost', ports, api.env.startup_timeout)
def stop(self, instance_name="", capture_output=True):
ipautil.run(["/bin/systemctl", "stop", self.service_instance(instance_name)], capture_output=capture_output)
def start(self, instance_name="", capture_output=True):
def start(self, instance_name="", capture_output=True, wait=True):
ipautil.run(["/bin/systemctl", "start", self.service_instance(instance_name)], capture_output=capture_output)
if wait and self.is_running(instance_name):
self.__wait_for_open_ports(self.service_instance(instance_name))
def restart(self, instance_name="", capture_output=True):
def restart(self, instance_name="", capture_output=True, wait=True):
# Restart command is broken before systemd-36-3.fc16
# If you have older systemd version, restart of dependent services will hang systemd indefinetly
ipautil.run(["/bin/systemctl", "restart", self.service_instance(instance_name)], capture_output=capture_output)
if wait and self.is_running(instance_name):
self.__wait_for_open_ports(self.service_instance(instance_name))
def is_running(self, instance_name=""):
ret = True

View File

@ -681,7 +681,6 @@ class CAInstance(service.Service):
def __restart_instance(self):
try:
self.restart(PKI_INSTANCE_NAME)
installutils.wait_for_open_ports('localhost', 9180, 300)
except Exception:
# TODO: roll back here?
root_logger.critical("Failed to restart the certificate server. See the installation log for details.")

View File

@ -416,7 +416,6 @@ class DsInstance(service.Service):
if not is_ds_running(instance):
root_logger.critical("Failed to restart the directory server. See the installation log for details.")
sys.exit(1)
installutils.wait_for_open_ports('localhost', self.open_ports, 300)
except SystemExit, e:
raise e
except Exception, e:
@ -667,7 +666,7 @@ class DsInstance(service.Service):
# (re)start them.
for ds_instance in get_ds_instances():
try:
ipaservices.knownservices.dirsrv.restart(ds_instance)
ipaservices.knownservices.dirsrv.restart(ds_instance, wait=False)
except Exception, e:
root_logger.error('Unable to restart ds instance %s: %s', ds_instance, e)

View File

@ -414,58 +414,6 @@ def create_keytab(path, principal):
kadmin("ktadd -k " + path + " " + principal)
def wait_for_open_ports(host, ports, timeout=0):
"""
Wait until the specified port(s) on the remote host are open. Timeout
in seconds may be specified to limit the wait.
"""
if not isinstance(ports, (tuple, list)):
ports = [ports]
op_timeout = time.time() + timeout
ipv6_failover = False
for port in ports:
while True:
try:
if ipv6_failover:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
else:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
s.close()
break;
except socket.error, e:
if e.errno == 111: # 111: Connection refused
if timeout and time.time() > op_timeout: # timeout exceeded
raise e
time.sleep(1)
elif not ipv6_failover: # fallback to IPv6 connection
ipv6_failover = True
else:
raise e
def wait_for_open_socket(socket_name, timeout=0):
"""
Wait until the specified socket on the local host is open. Timeout
in seconds may be specified to limit the wait.
"""
op_timeout = time.time() + timeout
while True:
try:
s = socket.socket(socket.AF_UNIX)
s.connect(socket_name)
s.close()
break;
except socket.error, e:
if e.errno in (2,111): # 111: Connection refused, 2: File not found
if timeout and time.time() > op_timeout: # timeout exceeded
raise e
time.sleep(1)
else:
raise e
def resolve_host(host_name):
try:
addrinfos = socket.getaddrinfo(host_name, None,

View File

@ -34,6 +34,14 @@ class DSRestart(service.Service):
"""
service.Service.__init__(self, "dirsrv")
def start(self, instance_name="", capture_output=True, wait=True):
"""
During upgrades the server is listening only on the socket so
we don't want to wait on ports. The caller is responsible for
waiting for the socket to be ready.
"""
super(DSRestart, self).start(wait=False)
def create_instance(self):
self.step("stopping directory server", self.stop)
self.step("starting directory server", self.start)

View File

@ -18,11 +18,11 @@
#
import os
from ipaserver.install import installutils
from ipaserver.install.plugins import FIRST, MIDDLE, LAST
from ipaserver.install.plugins import POST_UPDATE
from ipaserver.install.plugins.baseupdate import DSRestart
from ipaserver.install.ldapupdate import LDAPUpdate
from ipapython.ipautil import wait_for_open_socket
from ipalib import api
from ipalib import backend
import ldap as _ldap
@ -161,7 +161,7 @@ class updateclient(backend.Executioner):
if live_run:
self.destroy_context()
dsrestart.create_instance()
installutils.wait_for_open_socket(socket_name)
wait_for_open_socket(socket_name)
self.create_context(dm_password)
else:
self.log.warn("Test mode, skipping restart")

View File

@ -25,7 +25,6 @@ import sys
import ldap
from ipaserver import ipaldap
from ipapython import services as ipaservices
import installutils
from ldap import modlist
from ipalib import api, util, errors
from ipapython import ipautil
@ -92,7 +91,6 @@ def enable_replication_version_checking(hostname, realm, dirman_passwd):
conn.unbind()
serverid = "-".join(realm.split("."))
ipaservices.knownservices.dirsrv.restart(instance_name=serverid)
installutils.wait_for_open_ports('localhost', [389, 636], 300)
else:
conn.unbind()

View File

@ -35,6 +35,8 @@ from ipapython.ipa_log_manager import *
CACERT = "/etc/ipa/ca.crt"
# The service name as stored in cn=masters,cn=ipa,cn=etc. In the tuple
# the first value is the *nix service name, the second the start order.
SERVICE_LIST = {
'KDC':('krb5kdc', 10),
'KPASSWD':('kadmin', 20),
@ -198,11 +200,11 @@ class Service(object):
def stop(self, instance_name="", capture_output=True):
self.service.stop(instance_name, capture_output=capture_output)
def start(self, instance_name="", capture_output=True):
self.service.start(instance_name, capture_output=capture_output)
def start(self, instance_name="", capture_output=True, wait=True):
self.service.start(instance_name, capture_output=capture_output, wait=wait)
def restart(self, instance_name="", capture_output=True):
self.service.restart(instance_name, capture_output=capture_output)
def restart(self, instance_name="", capture_output=True, wait=True):
self.service.restart(instance_name, capture_output=capture_output, wait=wait)
def is_running(self):
return self.service.is_running()

View File

@ -60,6 +60,11 @@ class IPAUpgrade(service.Service):
self.badsyntax = False
self.upgradefailed = False
def start(self, instance_name="", capture_output=True, wait=True):
# Don't wait here because we've turned off port 389. The connection
# we make will wait for the socket.
super(IPAUpgrade, self).start(instance_name, capture_output, wait=False)
def create_instance(self):
self.step("stopping directory server", self.stop)
self.step("saving configuration", self.__save_config)

View File

@ -36,10 +36,9 @@ from ldap.controls import LDAPControl
from ldap.ldapobject import SimpleLDAPObject
from ipapython import ipautil
from ipalib import errors
from ipapython.ipautil import format_netloc
from ipapython.ipautil import format_netloc, wait_for_open_socket, wait_for_open_ports
from ipapython.entity import Entity
from ipaserver.plugins.ldap2 import IPASimpleLDAPObject
from ipaserver.install import installutils
# Global variable to define SASL auth
SASL_AUTH = ldap.sasl.sasl({},'GSSAPI')
@ -337,10 +336,10 @@ class IPAdmin(IPAEntryLDAPObject):
def __wait_for_connection(self, timeout):
lurl = ldapurl.LDAPUrl(self._uri)
if lurl.urlscheme == 'ldapi':
installutils.wait_for_open_socket(lurl.hostport, timeout)
wait_for_open_socket(lurl.hostport, timeout)
else:
(host,port) = lurl.hostport.split(':')
installutils.wait_for_open_ports(host, int(port), timeout)
wait_for_open_ports(host, int(port), timeout)
def __bind_with_wait(self, bind_func, timeout, *args, **kwargs):
try: