Add support for client failover to the ipa command-line.

This adds a new global option to the ipa command, -f/--no-fallback. If this
is included then just the server configured in /etc/ipa/default.conf is used.
Otherwise that is tried first then all servers in DNS with the ldap SRV record
are tried.

Create a new Local() Command class for local-only commands. The help
command is one of these. It shouldn't need a remote connection to execute.

ticket #15

ipa.1

@@ -52,6 +52,9 @@ Don't prompt for any parameters of \fBCOMMAND\fR, even if they are required.
 \fB\-a\fR, \fB\-\-prompt\-all\fR
 Prompt for ALL values (even if optional)
 .TP
+\fB\-f\fR, \fB\-\-no\-fallback\fR
+Don't fall back to other IPA servers if the default doesn't work.
+.TP
 \fB\-v\fR, \fB\-\-verbose\fR
 Produce verbose output. A second \-v displays the XML\-RPC request
 .SH "COMMANDS"
@@ -157,6 +160,16 @@ Only the user with the specified IPA unique ID would match the search criteria.
 .TP
 \fBipa user\-find\fR
 All users would match the search criteria (as there are none).
+.SH "SERVERS"
+The ipa client will determine which server to connect to in this order:
+
+.TP
+1. The server configured in \fB/etc/ipa/default.conf\fR in the \fIxmlrpc_uri\fR directive.
+.TP
+2. An unordered list of servers from the ldap DNS SRV records.
+
+.TP
+If a kerberos error is raised by any of the requests then it will stop processing and display the error message.
 .SH "FILES"
 .TP
 \fB/etc/ipa/default.conf\fR

ipalib/backend.py

@@ -109,7 +109,8 @@ class Executioner(Backend):
         if self.env.in_server:
             self.Backend.ldap2.connect(ccache=ccache)
         else:
-            self.Backend.xmlclient.connect(verbose=(self.env.verbose >= 2))
+            self.Backend.xmlclient.connect(verbose=(self.env.verbose >= 2),
+                fallback=self.env.fallback)
         if client_ip is not None:
             setattr(context, "client_ip", client_ip)
 

ipalib/cli.py

@@ -572,7 +572,7 @@ class textui(backend.Backend):
         self.print_line('')
         return selection
 
-class help(frontend.Command):
+class help(frontend.Local):
     """
     Display help for a command or topic.
     """
@@ -778,12 +778,13 @@ class cli(backend.Executioner):
         if len(argv) == 0:
             self.Command.help()
             return
-        self.create_context()
         (key, argv) = (argv[0], argv[1:])
         name = from_cli(key)
         if name not in self.Command or self.Command[name].INTERNAL:
             raise CommandError(name=key)
         cmd = self.Command[name]
+        if not isinstance(cmd, frontend.Local):
+            self.create_context()
         kw = self.parse(cmd, argv)
         if self.env.interactive:
             self.prompt_interactively(cmd, kw)

ipalib/constants.py

@@ -130,6 +130,7 @@ DEFAULT_CONFIG = (
     # Special CLI:
     ('prompt_all', False),
     ('interactive', True),
+    ('fallback', True),
 
     # Enable certain optional plugins:
     ('enable_ra', False),

ipalib/frontend.py

@@ -930,6 +930,21 @@ class LocalOrRemote(Command):
         return self.execute(*args, **options)
 
 
+class Local(Command):
+    """
+    A command that is explicitly executed locally.
+
+    This is for commands that makes sense to execute only locally
+    such as the help command.
+    """
+
+    def run(self, *args, **options):
+        """
+        Dispatch to forward() onlly.
+        """
+        return self.forward(*args, **options)
+
+
 class Object(HasParam):
     backend = None
     methods = None

ipalib/plugable.py

@@ -455,6 +455,10 @@ class API(DictProxy):
                 dest='interactive',
                 help='Prompt for NO values (even if required)'
             )
+            parser.add_option('-f', '--no-fallback', action='store_false',
+                dest='fallback',
+                help='Only use the server configured in /etc/ipa/default.conf'
+            )
         topics = optparse.OptionGroup(parser, "Available help topics",
                     "ipa help topics")
         cmds = optparse.OptionGroup(parser, "Available commands",
@@ -479,7 +483,8 @@ class API(DictProxy):
                     # --Jason, 2008-10-31
                     pass
                 overrides[str(key.strip())] = value.strip()
-        for key in ('conf', 'debug', 'verbose', 'prompt_all', 'interactive'):
+        for key in ('conf', 'debug', 'verbose', 'prompt_all', 'interactive',
+            'fallback'):
             value = getattr(options, key, None)
             if value is not None:
                 overrides[key] = value

ipalib/plugins/ping.py

@@ -0,0 +1,42 @@
+# Authors:
+#   Rob Crittenden <rcritten@redhat.com>
+#
+# Copyright (C) 2010  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2 only
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+"""
+Ping the remote IPA server
+"""
+
+from ipalib import api
+from ipalib import Command
+from ipalib import output
+
+class ping(Command):
+    """
+    ping a remote server
+    """
+    has_output = (
+        output.summary,
+    )
+
+    def execute(self):
+        """
+        A possible enhancement would be to take an argument and echo it
+        back but a fixed value works for now.
+        """
+        return dict(summary=u'pong')
+
+api.register(ping)

ipalib/rpc.py

@@ -37,13 +37,14 @@ import errno
 from xmlrpclib import Binary, Fault, dumps, loads, ServerProxy, Transport, ProtocolError
 import kerberos
 from ipalib.backend import Connectible
-from ipalib.errors import public_errors, PublicError, UnknownError, NetworkError
+from ipalib.errors import public_errors, PublicError, UnknownError, NetworkError, KerberosError
 from ipalib import errors
 from ipalib.request import context
-from ipapython import ipautil
+from ipapython import ipautil, dnsclient
 import httplib
 from ipapython.nsslib import NSSHTTPS
 from nss.error import NSPRError
+from urllib2 import urlparse
 
 # Some Kerberos error definitions from krb5.h
 KRB5KDC_ERR_S_PRINCIPAL_UNKNOWN = (-1765328377L)
@@ -255,12 +256,70 @@ class xmlclient(Connectible):
         super(xmlclient, self).__init__()
         self.__errors = dict((e.errno, e) for e in public_errors)
 
-    def create_connection(self, ccache=None, verbose=False):
+    def reconstruct_url(self):
-        kw = dict(allow_none=True, encoding='UTF-8')
+        """
-        if self.env.xmlrpc_uri.startswith('https://'):
+        The URL directly isn't stored in the ServerProxy. We can't store
-            kw['transport'] = KerbTransport()
+        it in the connection object itself but we can reconstruct it
-        kw['verbose'] = verbose
+        from the ServerProxy.
-        return ServerProxy(self.env.xmlrpc_uri, **kw)
+        """
+        if not hasattr(self.conn, '_ServerProxy__transport'):
+            return None
+        if isinstance(self.conn._ServerProxy__transport, KerbTransport):
+            scheme = "https"
+        else:
+            scheme = "http"
+        server = '%s://%s%s' % (scheme, self.conn._ServerProxy__host, self.conn._ServerProxy__handler)
+        return server
+
+    def get_url_list(self):
+        """
+        Create a list of urls consisting of the available IPA servers.
+        """
+        # the configured URL defines what we use for the discovered servers
+        (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(self.env.xmlrpc_uri)
+        servers = []
+        name = '_ldap._tcp.%s.' % self.env.domain
+        rs = dnsclient.query(name, dnsclient.DNS_C_IN, dnsclient.DNS_T_SRV)
+        for r in rs:
+            if r.dns_type == dnsclient.DNS_T_SRV:
+                rsrv = r.rdata.server.rstrip('.')
+                servers.append('https://%s%s' % (rsrv, path))
+        servers = list(set(servers))
+        # the list/set conversion won't preserve order so stick in the
+        # local config file version here.
+        servers.insert(0, self.env.xmlrpc_uri)
+        return servers
+
+    def create_connection(self, ccache=None, verbose=False, fallback=True):
+        servers = self.get_url_list()
+        serverproxy = None
+        for server in servers:
+            kw = dict(allow_none=True, encoding='UTF-8')
+            kw['verbose'] = verbose
+            if server.startswith('https://'):
+                kw['transport'] = KerbTransport()
+            self.log.info('trying %s' % server)
+            serverproxy = ServerProxy(server, **kw)
+            if len(servers) == 1 or not fallback:
+                # if we have only 1 server to try then let the main
+                # requester handle any errors
+                return serverproxy
+            try:
+                command = getattr(serverproxy, 'ping')
+                response = command()
+                # We don't care about the response, just that we got one
+                break
+            except KerberosError, krberr:
+                # kerberos error on one server is likely on all
+                raise errors.KerberosError(major=str(krberr), minor='')
+            except Exception, e:
+                if not fallback:
+                    raise e
+                serverproxy = None
+
+        if serverproxy is None:
+            raise NetworkError(uri='any of the configured servers', error=', '.join(servers))
+        return serverproxy
 
     def destroy_connection(self):
         pass
@@ -280,7 +339,8 @@ class xmlclient(Connectible):
             raise ValueError(
                 '%s.forward(): %r not in api.Command' % (self.name, name)
             )
-        self.info('Forwarding %r to server %r', name, self.env.xmlrpc_uri)
+        server = self.reconstruct_url()
+        self.info('Forwarding %r to server %r', name, server)
         command = getattr(self.conn, name)
         params = [args, kw]
         try:
@@ -289,16 +349,16 @@ class xmlclient(Connectible):
         except Fault, e:
             e = decode_fault(e)
             self.debug('Caught fault %d from server %s: %s', e.faultCode,
-                self.env.xmlrpc_uri, e.faultString)
+                server, e.faultString)
             if e.faultCode in self.__errors:
                 error = self.__errors[e.faultCode]
                 raise error(message=e.faultString)
             raise UnknownError(
                 code=e.faultCode,
                 error=e.faultString,
-                server=self.env.xmlrpc_uri,
+                server=server,
             )
         except NSPRError, e:
-            raise NetworkError(uri=self.env.xmlrpc_uri, error=str(e))
+            raise NetworkError(uri=server, error=str(e))
         except ProtocolError, e:
-            raise NetworkError(uri=self.env.xmlrpc_uri, error=e.errmsg)
+            raise NetworkError(uri=server, error=e.errmsg)

ipapython/nsslib.py

@@ -161,6 +161,20 @@ class NSSConnection(httplib.HTTPConnection):
         logging.debug("connect: %s", net_addr)
         self.sock.connect(net_addr)
 
+    def endheaders(self):
+        """
+        Explicitly close the connection if an error is returned after the
+        headers are sent. This will likely mean the initial SSL handshake
+        failed. If this isn't done then the connection is never closed and
+        subsequent NSS activities will fail with a BUSY error.
+        """
+        try:
+            # FIXME: httplib uses old-style classes so super doesn't work
+            httplib.HTTPConnection.endheaders(self)
+        except NSPRError, e:
+            self.close()
+            raise e
+
 class NSSHTTPS(httplib.HTTP):
     # We would like to use HTTP 1.1 not the older HTTP 1.0 but xmlrpclib
     # and httplib do not play well together. httplib when the protocol

tests/test_ipalib/test_rpc.py

@@ -234,7 +234,7 @@ class test_xmlclient(PluginTester):
 
         # Test with an errno the client knows:
         e = raises(errors.RequirementError, o.forward, 'user_add', *args, **kw)
-        assert_equal(e.message, u"'four' is required")
+        assert_equal(e.args[0], u"'four' is required")
 
         # Test with an errno the client doesn't know
         e = raises(errors.UnknownError, o.forward, 'user_add', *args, **kw)

tests/test_xmlrpc/xmlrpc_test.py

@@ -42,7 +42,7 @@ fuzzy_uuid = Fuzzy(
 
 try:
     if not api.Backend.xmlclient.isconnected():
-        api.Backend.xmlclient.connect()
+        api.Backend.xmlclient.connect(fallback=False)
     res = api.Command['user_show'](u'notfound')
 except errors.NetworkError:
     server_available = False
@@ -103,7 +103,7 @@ class XMLRPC_test(object):
                 'Server not available: %r' % api.env.xmlrpc_uri
             )
         if not api.Backend.xmlclient.isconnected():
-            api.Backend.xmlclient.connect()
+            api.Backend.xmlclient.connect(fallback=False)
 
     def tearDown(self):
         """