Apply LDAP update files in blocks of 10, as originally designed.

In order to have control over the order that updates are applied
a numbering system was created for the update files. These values
were not actually used.

The updates were sorted by DN length and in most cases this was
adequate for proper function. The exception was with roles where
in some cases a role was added as a member of a permission before
the role itself was added so the memberOf value was never created.

Now updates are computed and applied in blocks of 10.

https://fedorahosted.org/freeipa/ticket/3377

install/updates/README

@@ -2,7 +2,22 @@ The update files are sorted before being processed because there are
 cases where order matters (such as getting schema added first, creating
 parent entries, etc).
 
-10 - 20: Schema
-20 - 30: FDS Configuration, new indices
-30 - 40: Structual elements of the DIT
-40 - 50: Pre-loaded data
+Updates are applied in blocks of ten so that any entries that are dependant
+on another can be added successfully without having to rely on the length
+of the DN to get the sorting correct.
+
+The file names should use the format #-<description>.update where # conforms
+to this:
+
+10 - 19: Schema
+20 - 29: 389-ds configuration, new indices
+30 - 39: Structual elements of the DIT
+40 - 49: Pre-loaded data
+50 - 59: Cleanup existing data
+60 - 69: AD Trust
+70 - 79: Reserved
+80 - 89: Reserved
+
+These numbers aren't absolute, there may be reasons to put an update
+into one place or another, but by adhereing to the scheme it will be
+easier to find existing updates and know where to put new ones.

ipaserver/install/dsinstance.py

@@ -472,7 +472,7 @@ class DsInstance(service.Service):
     def apply_updates(self):
         ld = ldapupdate.LDAPUpdate(dm_password=self.dm_password, sub_dict=self.sub_dict, plugins=True)
         files = ld.get_all_files(ldapupdate.UPDATES_DIR)
-        ld.update(files)
+        ld.update(files, ordered=True)
 
     def __add_referint_module(self):
         self._ldap_mod("referint-conf.ldif")

ipaserver/install/ipa_ldap_updater.py

@@ -185,7 +185,7 @@ class LDAPUpdater_NonUpgrade(LDAPUpdater):
         if not self.files:
             self.files = ld.get_all_files(UPDATES_DIR)
 
-        modified = ld.update(self.files)
+        modified = ld.update(self.files, ordered=True)
 
         if modified and options.test:
             self.log.info('Update complete, changes to be made, test mode')

ipaserver/install/ldapupdate.py

@@ -33,6 +33,7 @@ import pwd
 import fnmatch
 import csv
 import inspect
+import re
 
 import krbV
 import ldap
@@ -900,21 +901,46 @@ class LDAPUpdate:
         for dn, update in sorted_updates:
             self._delete_record(update)
 
-    def update(self, files):
+    def update(self, files, ordered=False):
         """Execute the update. files is a list of the update files to use.
 
+           If ordered is True then the updates the file must be of the form
+           ##-name.update where ## is an integer between 10 and 89. The
+           changes are applied to LDAP at the end of each value divisible
+           by 10, so after 20, 30, etc.
+
            returns True if anything was changed, otherwise False
         """
 
+        pat = re.compile(r'(\d+)-.*\.update')
         all_updates = {}
+        r = 20
         if self.plugins:
             self.info('PRE_UPDATE')
             updates = api.Backend.updateclient.update(PRE_UPDATE, self.dm_password, self.ldapi, self.live_run)
             self.merge_updates(all_updates, updates)
         try:
             self.create_connection()
+            if ordered and all_updates:
+                # flush out PRE_UPDATE plugin updates before we begin
+                self._run_updates(all_updates)
+                all_updates = {}
 
             for f in files:
+                name = os.path.basename(f)
+                if ordered:
+                    m = pat.match(name)
+                    if not m:
+                        raise RuntimeError("Filename does not match format #-name.update: %s" % f)
+                    index = int(m.group(1))
+                    if index < 10 or index > 90:
+                        raise RuntimeError("Index not legal range: %d" % index)
+
+                    if index >= r:
+                        self._run_updates(all_updates)
+                        all_updates = {}
+                        r += 10
+
                 try:
                     self.info("Parsing update file '%s'" % f)
                     data = self.read_file(f)

ipaserver/install/upgradeinstance.py

@@ -115,7 +115,7 @@ class IPAUpgrade(service.Service):
             ld = ldapupdate.LDAPUpdate(dm_password='', ldapi=True, live_run=self.live_run, plugins=True)
             if len(self.files) == 0:
                 self.files = ld.get_all_files(ldapupdate.UPDATES_DIR)
-            self.modified = ld.update(self.files)
+            self.modified = ld.update(self.files, ordered=True)
         except ldapupdate.BadSyntax, e:
             root_logger.error('Bad syntax in upgrade %s' % str(e))
             self.modified = False