Framework for admin/install tools, with ipa-ldap-updater

Currently, FreeIPA's install/admin scripts are long pieces of code
that aren't very reusable, importable, or testable.
They have been extended over time with features such as logging and
error handling, but since each tool was extended individually, there
is much inconsistency and code duplication.
This patch starts a framework which the admin tools can use, and
converts ipa-ldap-updater to use the framework.

Common tasks the tools do -- option parsing, validation, logging
setup, error handling -- are represented as methods. Individual
tools can extend, override or reuse the defaults as they see fit.

The ipa-ldap-updater has two modes (normal and --upgrade) that
don't share much functionality. They are represented by separate
classes. Option parsing, and selecting which class to run, happens
before they're instantiated.

All code is moved to importable modules to aid future testing. The
only thing that remains in the ipa-ldap-updater script is a two-line
call to the library.

First part of the work for:
https://fedorahosted.org/freeipa/ticket/2652

ipaserver/install/installutils.py

@@ -32,12 +32,14 @@ import tempfile
 import shutil
 from ConfigParser import SafeConfigParser
 import traceback
+import textwrap
 
 from dns import resolver, rdatatype
 from dns.exception import DNSException
 import ldap
 
-from ipapython import ipautil, sysrestore
+from ipapython import ipautil, sysrestore, admintool
+from ipapython.admintool import ScriptError
 from ipapython.ipa_log_manager import *
 from ipalib.util import validate_hostname
 from ipapython import config
@@ -61,18 +63,6 @@ class HostReverseLookupError(HostLookupError):
 class HostnameLocalhost(HostLookupError):
     pass
 
-
-class ScriptError(StandardError):
-    """An exception that records an error message and a return value
-    """
-    def __init__(self, msg = '', rval = 1):
-        self.msg = msg
-        self.rval = rval
-
-    def __str__(self):
-        return self.msg
-
-
 class ReplicaConfig:
     def __init__(self):
         self.realm_name = ""
@@ -639,65 +629,65 @@ def run_script(main_function, operation_name, log_file_name=None,
             sys.exit(return_value)
 
     except BaseException, error:
-        handle_error(error, log_file_name)
+        message, exitcode = handle_error(error, log_file_name)
+        if message:
+            print >> sys.stderr, message
+        sys.exit(exitcode)
 
 
 def handle_error(error, log_file_name=None):
-    """Handle specific errors"""
+    """Handle specific errors. Returns a message and return code"""
 
     if isinstance(error, SystemExit):
-        sys.exit(error)
+        if isinstance(error.code, int):
+            return None, error.code
+        elif error.code is None:
+            return None, 0
+        else:
+            return str(error), 1
     if isinstance(error, RuntimeError):
-        sys.exit(error)
+        return str(error), 1
     if isinstance(error, KeyboardInterrupt):
-        print >> sys.stderr, "Cancelled."
-        sys.exit(1)
+        return "Cancelled.", 1
 
-    if isinstance(error, ScriptError):
-        if error.msg:
-            print >> sys.stderr, error.msg
-        sys.exit(error.rval)
+    if isinstance(error, admintool.ScriptError):
+        return error.msg, error.rval
 
     if isinstance(error, socket.error):
-        print >> sys.stderr, error
-        sys.exit(1)
+        return error, 1
 
     if isinstance(error, ldap.INVALID_CREDENTIALS):
-        print >> sys.stderr, "Invalid password"
-        sys.exit(1)
+        return "Invalid password", 1
     if isinstance(error, ldap.INSUFFICIENT_ACCESS):
-        print >> sys.stderr, "Insufficient access"
-        sys.exit(1)
+        return "Insufficient access", 1
     if isinstance(error, ldap.LOCAL_ERROR):
-        print >> sys.stderr, error.args[0]['info']
-        sys.exit(1)
+        return error.args[0]['info'], 1
     if isinstance(error, ldap.SERVER_DOWN):
-        print >> sys.stderr, error.args[0]['desc']
-        sys.exit(1)
+        return error.args[0]['desc'], 1
     if isinstance(error, ldap.LDAPError):
-        print >> sys.stderr, 'LDAP error: %s' % type(error).__name__
-        print >> sys.stderr, error.args[0]['info']
-        sys.exit(1)
+        return 'LDAP error: %s\n%s' % (
+            type(error).__name__, error.args[0]['info']), 1
 
     if isinstance(error, config.IPAConfigError):
-        print >> sys.stderr, "An IPA server to update cannot be found. Has one been configured yet?"
-        print >> sys.stderr, "The error was: %s" % error
-        sys.exit(1)
+        message = "An IPA server to update cannot be found. Has one been configured yet?"
+        message += "\nThe error was: %s" % error
+        return message, 1
     if isinstance(error, errors.LDAPError):
-        print >> sys.stderr, "An error occurred while performing operations: %s" % error
-        sys.exit(1)
+        return "An error occurred while performing operations: %s" % error, 1
 
     if isinstance(error, HostnameLocalhost):
-        print >> sys.stderr, "The hostname resolves to the localhost address (127.0.0.1/::1)"
-        print >> sys.stderr, "Please change your /etc/hosts file so that the hostname"
-        print >> sys.stderr, "resolves to the ip address of your network interface."
-        print >> sys.stderr, ""
-        print >> sys.stderr, "Please fix your /etc/hosts file and restart the setup program"
-        sys.exit(1)
+        message = textwrap.dedent("""
+            The hostname resolves to the localhost address (127.0.0.1/::1)
+            Please change your /etc/hosts file so that the hostname
+            resolves to the ip address of your network interface.
+
+            Please fix your /etc/hosts file and restart the setup program
+            """).strip()
+        return message, 1
 
     if log_file_name:
-        print >> sys.stderr, "Unexpected error - see %s for details:" % log_file_name
+        message = "Unexpected error - see %s for details:" % log_file_name
     else:
-        print >> sys.stderr, "Unexpected error"
-    print >> sys.stderr, '%s: %s' % (type(error).__name__, error)
-    sys.exit(1)
+        message = "Unexpected error"
+    message += '\n%s: %s' % (type(error).__name__, error)
+    return message, 1

ipaserver/install/ipa_ldap_updater.py

@@ -0,0 +1,189 @@
+#!/usr/bin/python
+# Authors: Rob Crittenden <rcritten@redhat.com>
+#          Petr Viktorin <pviktori@redhat.com>
+#
+# Copyright (C) 2008  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Documentation can be found at http://freeipa.org/page/LdapUpdate
+
+# TODO
+# save undo files?
+
+import os
+
+import krbV
+
+from ipalib import api
+from ipapython import ipautil, admintool
+from ipaserver.install import installutils
+from ipaserver.install.ldapupdate import LDAPUpdate, UPDATES_DIR
+from ipaserver.install.upgradeinstance import IPAUpgrade
+from ipapython import ipa_log_manager
+
+
+class LDAPUpdater(admintool.AdminTool):
+    command_name = 'ipa-ldap-updater'
+
+    usage = "%prog [options] input_file(s)\n"
+    usage += "%prog [options]\n"
+
+    @classmethod
+    def add_options(cls, parser):
+        super(LDAPUpdater, cls).add_options(parser)
+
+        parser.add_option("-t", "--test", action="store_true", dest="test",
+            default=False,
+            help="Run through the update without changing anything")
+        parser.add_option("-y", dest="password",
+            help="File containing the Directory Manager password")
+        parser.add_option("-l", '--ldapi', action="store_true", dest="ldapi",
+            default=False,
+            help="Connect to the LDAP server using the ldapi socket")
+        parser.add_option("-u", '--upgrade', action="store_true",
+            dest="upgrade", default=False,
+            help="Upgrade an installed server in offline mode")
+        parser.add_option("-p", '--plugins', action="store_true",
+            dest="plugins", default=False,
+            help="Execute update plugins. " +
+                "Always true when applying all update files.")
+        parser.add_option("-W", '--password', action="store_true",
+            dest="ask_password",
+            help="Prompt for the Directory Manager password")
+
+    @classmethod
+    def get_command_class(cls, options, args):
+        if options.upgrade:
+            return LDAPUpdater_Upgrade
+        else:
+            return LDAPUpdater_NonUpgrade
+
+    def validate_options(self):
+        options = self.options
+        super(LDAPUpdater, self).validate_options()
+
+        self.files = self.args
+
+        for filename in self.files:
+            if not os.path.exists(filename):
+                raise admintool.ScriptError("%s: file not found" % filename)
+
+        if os.getegid() == 0:
+            installutils.check_server_configuration()
+        elif not os.path.exists('/etc/ipa/default.conf'):
+            raise admintool.ScriptError(
+                "IPA is not configured on this system.")
+
+        if options.password:
+            pw = ipautil.template_file(options.password, [])
+            self.dirman_password = pw.strip()
+        else:
+            self.dirman_password = None
+
+    def setup_logging(self):
+        ipa_log_manager.standard_logging_setup(self.log_file_name,
+            console_format='%(levelname)s: %(message)s',
+            debug=self.options.debug, filemode='a')
+        ipa_log_manager.log_mgr.get_logger(self, True)
+
+    def run(self):
+        super(LDAPUpdater, self).run()
+
+        api.bootstrap(
+                in_server=True,
+                context='updates',
+                debug=self.options.debug,
+            )
+        api.finalize()
+
+    def handle_error(self, exception):
+        return installutils.handle_error(exception, self.log_file_name)
+
+
+class LDAPUpdater_Upgrade(LDAPUpdater):
+    needs_root = True
+    log_file_name = '/var/log/ipaupgrade.log'
+
+    def validate_options(self):
+        if os.getegid() != 0:
+            raise admintool.ScriptError('Must be root to do an upgrade.', 1)
+
+        super(LDAPUpdater_Upgrade, self).validate_options()
+
+    def run(self):
+        super(LDAPUpdater_Upgrade, self).run()
+        options = self.options
+
+        updates = None
+        realm = krbV.default_context().default_realm
+        upgrade = IPAUpgrade(realm, self.files, live_run=not options.test)
+        upgrade.create_instance()
+        upgradefailed = upgrade.upgradefailed
+
+        if upgrade.badsyntax:
+            raise admintool.ScriptError(
+                'Bad syntax detected in upgrade file(s).', 1)
+        elif upgrade.upgradefailed:
+            raise admintool.ScriptError('IPA upgrade failed.', 1)
+        elif upgrade.modified and options.test:
+            self.info('Update complete, changes to be made, test mode')
+            return 2
+
+
+class LDAPUpdater_NonUpgrade(LDAPUpdater):
+    def validate_options(self):
+        super(LDAPUpdater_NonUpgrade, self).validate_options()
+        options = self.options
+
+        # Only run plugins if no files are given
+        self.run_plugins = not self.files or options.plugins
+
+        # Need root for running plugins
+        if self.run_plugins and os.getegid() != 0:
+            raise admintool.ScriptError('Plugins can only be run as root.', 1)
+
+    def ask_for_options(self):
+        super(LDAPUpdater_NonUpgrade, self).ask_for_options()
+        options = self.options
+        if not self.dirman_password:
+            if options.ask_password or not options.ldapi:
+                password = installutils.read_password("Directory Manager",
+                    confirm=False, validate=False)
+                if password is None:
+                    raise admintool.ScriptError(
+                        "Directory Manager password required")
+                self.dirman_password = password
+
+    def run(self):
+        super(LDAPUpdater_NonUpgrade, self).run()
+        options = self.options
+
+        ld = LDAPUpdate(
+            dm_password=self.dirman_password,
+            sub_dict={},
+            live_run=not options.test,
+            ldapi=options.ldapi,
+            plugins=options.plugins or self.run_plugins)
+
+        if not self.files:
+            self.files = ld.get_all_files(UPDATES_DIR)
+
+        modified = ld.update(self.files)
+
+        if modified and options.test:
+            self.info('Update complete, changes to be made, test mode')
+            return 2

ipaserver/install/ldapupdate.py

@@ -825,7 +825,7 @@ class LDAPUpdate:
                     data = self.read_file(f)
                 except Exception, e:
                     print e
-                    sys.exit(1)
+                    sys.exit(e)
 
                 (all_updates, dn_list) = self.parse_update_file(data, all_updates, dn_list)