Add schema updater based on IPA schema files

The new updater is run as part of `ipa-ldap-updater --upgrade`
and `ipa-ldap-updater --schema` (--schema is a new option).
The --schema-file option to ipa-ldap-updater may be used (multiple
times) to select a non-default set of schema files to update against.

The updater adds an X-ORIGIN tag with the current IPA version to
all elements it adds or modifies.

https://fedorahosted.org/freeipa/ticket/3454

ipapython/ipaldap.py

@@ -1768,6 +1768,12 @@ class LDAPClient(object):
                     if not force_replace:
                         modlist.append((ldap.MOD_DELETE, k, rems))
 
+        # Usually the modlist order does not matter.
+        # However, for schema updates, we want 'attributetypes' before
+        # 'objectclasses'.
+        # A simple sort will ensure this.
+        modlist.sort(key=lambda m: m[1].lower())
+
         return modlist
 
     def update_entry(self, entry, entry_attrs=None):

ipaserver/install/ipa_ldap_updater.py

@@ -30,7 +30,7 @@ import krbV
 
 from ipalib import api
 from ipapython import ipautil, admintool
-from ipaserver.install import installutils
+from ipaserver.install import installutils, dsinstance, schemaupdate
 from ipaserver.install.ldapupdate import LDAPUpdate, UPDATES_DIR
 from ipaserver.install.upgradeinstance import IPAUpgrade
 
@@ -60,6 +60,13 @@ class LDAPUpdater(admintool.AdminTool):
             dest="plugins", default=False,
             help="execute update plugins " +
                 "(implied when no input files are given)")
+        parser.add_option("-s", '--schema', action="store_true",
+            dest="update_schema", default=False,
+            help="update the schema "
+                "(implied when no input files are given)")
+        parser.add_option("-S", '--schema-file', action="append",
+            dest="schema_files",
+            help="custom schema ldif file to use (implies -s)")
         parser.add_option("-W", '--password', action="store_true",
             dest="ask_password",
             help="prompt for the Directory Manager password")
@@ -97,6 +104,12 @@ class LDAPUpdater(admintool.AdminTool):
         else:
             self.dirman_password = None
 
+        if options.schema_files or not self.files:
+            options.update_schema = True
+        if not options.schema_files:
+            options.schema_files = [os.path.join(ipautil.SHARE_DIR, f) for f
+                                    in dsinstance.ALL_SCHEMA_FILES]
+
     def setup_logging(self):
         super(LDAPUpdater, self).setup_logging(log_file_mode='a')
 
@@ -125,7 +138,8 @@ class LDAPUpdater_Upgrade(LDAPUpdater):
 
         updates = None
         realm = krbV.default_context().default_realm
-        upgrade = IPAUpgrade(realm, self.files, live_run=not options.test)
+        upgrade = IPAUpgrade(realm, self.files, live_run=not options.test,
+                             schema_files=options.schema_files)
         upgrade.create_instance()
         upgradefailed = upgrade.upgradefailed
 
@@ -174,6 +188,14 @@ class LDAPUpdater_NonUpgrade(LDAPUpdater):
         super(LDAPUpdater_NonUpgrade, self).run()
         options = self.options
 
+        modified = False
+
+        if options.update_schema:
+            modified = schemaupdate.update_schema(
+                options.schema_files,
+                dm_password=self.dirman_password,
+                live_run=not options.test) or modified
+
         ld = LDAPUpdate(
             dm_password=self.dirman_password,
             sub_dict={},
@@ -184,7 +206,7 @@ class LDAPUpdater_NonUpgrade(LDAPUpdater):
         if not self.files:
             self.files = ld.get_all_files(UPDATES_DIR)
 
-        modified = ld.update(self.files, ordered=True)
+        modified = ld.update(self.files, ordered=True) or modified
 
         if modified and options.test:
             self.log.info('Update complete, changes to be made, test mode')

ipaserver/install/schemaupdate.py

@@ -0,0 +1,137 @@
+# Authors: Petr Viktorin <pviktori@redhat.com>
+#
+# Copyright (C) 2013 Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import pprint
+
+import ldap.schema
+import krbV
+
+import ipapython.version
+from ipapython.ipa_log_manager import log_mgr
+from ipapython.dn import DN
+from ipaserver.install.ldapupdate import connect
+from ipaserver.install import installutils
+
+
+SCHEMA_ELEMENT_CLASSES = {
+    # All schema model classes this tool can modify
+    'objectclasses': ldap.schema.models.ObjectClass,
+    'attributetypes': ldap.schema.models.AttributeType,
+}
+
+ORIGIN = 'IPA v%s' % ipapython.version.VERSION
+
+log = log_mgr.get_logger(__name__)
+
+
+def update_schema(schema_files, ldapi=False, dm_password=None, live_run=True):
+    """Update schema to match the given ldif files
+
+    Schema elements present in the LDIF files but missing from the DS schema
+    are added.
+    Schema elements that differ between LDIF files and DS schema are updated
+    to match the LDIF files. The comparison ignores tags that python-ldap's
+    schema parser does not understand (such as X-ORIGIN).
+    Extra elements present only in the DS schema are left untouched.
+
+    An X-ORIGIN tag containing the current IPA version is added to all new
+    and updated schema elements.
+
+    :param schema_files: List of filenames to update from
+    :param ldapi: if true, use ldapi to connect
+    :param dm_password: directory manager password
+    :live_run: if false, changes will not be applied
+
+    :return:
+        True if modifications were made
+        (or *would be* made, for live_run=false)
+    """
+    conn = connect(ldapi=ldapi, dm_password=dm_password,
+                   realm=krbV.default_context().default_realm,
+                   fqdn=installutils.get_fqdn())
+
+    old_schema = conn.schema
+
+    schema_entry = conn.get_entry(DN(('cn', 'schema')),
+                                  SCHEMA_ELEMENT_CLASSES.keys())
+
+    modified = False
+
+    # The exact representation the DS gives us for each OID
+    # (for debug logging)
+    old_entries_by_oid = {cls(str(attr)).oid: str(attr)
+                          for attrname, cls in SCHEMA_ELEMENT_CLASSES.items()
+                          for attr in schema_entry[attrname]}
+
+    for filename in schema_files:
+        log.info('Processing schema LDIF file %s', filename)
+        dn, new_schema = ldap.schema.subentry.urlfetch(filename)
+
+        for attrname, cls in SCHEMA_ELEMENT_CLASSES.items():
+
+            # Set of all elements of this class, as strings given by the DS
+            new_elements = []
+
+            for oid in new_schema.listall(cls):
+                new_obj = new_schema.get_obj(cls, oid)
+                old_obj = old_schema.get_obj(cls, oid)
+                # Compare python-ldap's sanitized string representations
+                # to see if the value is different
+                # This can give false positives, e.g. with case differences
+                # in case-insensitive names.
+                # But, false positives are harmless (and infrequent)
+                if not old_obj or str(new_obj) != str(old_obj):
+                    # Note: An add will automatically replace any existing
+                    # schema with the same OID. So, we only add.
+                    value = add_x_origin(new_obj)
+                    new_elements.append(value)
+
+                    if old_obj:
+                        old_attr = old_entries_by_oid.get(oid)
+                        log.info('Replace: %s', old_attr)
+                        log.info('   with: %s', value)
+                    else:
+                        log.info('Add: %s', value)
+
+            modified = modified or new_elements
+            schema_entry[attrname].extend(new_elements)
+
+    # FIXME: We should have a better way to display the modlist,
+    # for now display raw output of our internal routine
+    modlist = conn._generate_modlist(schema_entry.dn, schema_entry)
+    log.debug("Complete schema modlist:\n%s", pprint.pformat(modlist))
+
+    if modified and live_run:
+        conn.update_entry(schema_entry)
+    else:
+        log.info('Not updating schema')
+
+    return modified
+
+
+def add_x_origin(element):
+    """Add X-ORIGIN tag to a schema element if it does not already contain one
+    """
+    # Note that python-ldap drops X-ORIGIN when it parses schema elements,
+    # so we need to resort to string manipulation
+    element = str(element)
+    if 'X-ORIGIN' not in element:
+        assert element[-2:] == ' )'
+        element = element[:-1] + "X-ORIGIN '%s' )" % ORIGIN
+    return element

ipaserver/install/upgradeinstance.py

@@ -26,6 +26,7 @@ from ipapython.ipa_log_manager import *
 
 from ipaserver.install import installutils
 from ipaserver.install import dsinstance
+from ipaserver.install import schemaupdate
 from ipaserver.install import ldapupdate
 from ipaserver.install import service
 
@@ -38,7 +39,7 @@ class IPAUpgrade(service.Service):
     listeners and updating over ldapi. This way we know the server is
     quiet.
     """
-    def __init__(self, realm_name, files=[], live_run=True):
+    def __init__(self, realm_name, files=[], live_run=True, schema_files=[]):
         """
         realm_name: kerberos realm name, used to determine DS instance dir
         files: list of update files to process. If none use UPDATEDIR
@@ -60,6 +61,7 @@ class IPAUpgrade(service.Service):
         self.badsyntax = False
         self.upgradefailed = False
         self.serverid = serverid
+        self.schema_files = schema_files
 
     def __start_nowait(self):
         # Don't wait here because we've turned off port 389. The connection
@@ -75,6 +77,8 @@ class IPAUpgrade(service.Service):
         self.step("saving configuration", self.__save_config)
         self.step("disabling listeners", self.__disable_listeners)
         self.step("starting directory server", self.__start_nowait)
+        if self.schema_files:
+            self.step("updating schema", self.__update_schema)
         self.step("upgrading server", self.__upgrade)
         self.step("stopping directory server", self.__stop_instance)
         self.step("restoring configuration", self.__restore_config)
@@ -110,12 +114,18 @@ class IPAUpgrade(service.Service):
         installutils.set_directive(self.filename, 'nsslapd-ldapientrysearchbase',
             None, quotes=False, separator=':')
 
+    def __update_schema(self):
+        self.modified = schemaupdate.update_schema(
+            self.schema_files,
+            dm_password='', ldapi=True, live_run=self.live_run) or self.modified
+
     def __upgrade(self):
         try:
             ld = ldapupdate.LDAPUpdate(dm_password='', ldapi=True, live_run=self.live_run, plugins=True)
             if len(self.files) == 0:
                 self.files = ld.get_all_files(ldapupdate.UPDATES_DIR)
-            self.modified = ld.update(self.files, ordered=True)
+            self.modified = (ld.update(self.files, ordered=True) or
+                             self.modified)
         except ldapupdate.BadSyntax, e:
             root_logger.error('Bad syntax in upgrade %s' % str(e))
             self.modified = False