Apply LDAP update files in blocks of 10, as originally designed.

In order to have control over the order that updates are applied
a numbering system was created for the update files. These values
were not actually used.

The updates were sorted by DN length and in most cases this was
adequate for proper function. The exception was with roles where
in some cases a role was added as a member of a permission before
the role itself was added so the memberOf value was never created.

Now updates are computed and applied in blocks of 10.

https://fedorahosted.org/freeipa/ticket/3377

ipaserver/install/dsinstance.py

@@ -472,7 +472,7 @@ class DsInstance(service.Service):
     def apply_updates(self):
         ld = ldapupdate.LDAPUpdate(dm_password=self.dm_password, sub_dict=self.sub_dict, plugins=True)
         files = ld.get_all_files(ldapupdate.UPDATES_DIR)
-        ld.update(files)
+        ld.update(files, ordered=True)
 
     def __add_referint_module(self):
         self._ldap_mod("referint-conf.ldif")

ipaserver/install/ipa_ldap_updater.py

@@ -185,7 +185,7 @@ class LDAPUpdater_NonUpgrade(LDAPUpdater):
         if not self.files:
             self.files = ld.get_all_files(UPDATES_DIR)
 
-        modified = ld.update(self.files)
+        modified = ld.update(self.files, ordered=True)
 
         if modified and options.test:
             self.log.info('Update complete, changes to be made, test mode')

ipaserver/install/ldapupdate.py

@@ -33,6 +33,7 @@ import pwd
 import fnmatch
 import csv
 import inspect
+import re
 
 import krbV
 import ldap
@@ -900,21 +901,46 @@ class LDAPUpdate:
         for dn, update in sorted_updates:
             self._delete_record(update)
 
-    def update(self, files):
+    def update(self, files, ordered=False):
         """Execute the update. files is a list of the update files to use.
 
+           If ordered is True then the updates the file must be of the form
+           ##-name.update where ## is an integer between 10 and 89. The
+           changes are applied to LDAP at the end of each value divisible
+           by 10, so after 20, 30, etc.
+
            returns True if anything was changed, otherwise False
         """
 
+        pat = re.compile(r'(\d+)-.*\.update')
         all_updates = {}
+        r = 20
         if self.plugins:
             self.info('PRE_UPDATE')
             updates = api.Backend.updateclient.update(PRE_UPDATE, self.dm_password, self.ldapi, self.live_run)
             self.merge_updates(all_updates, updates)
         try:
             self.create_connection()
+            if ordered and all_updates:
+                # flush out PRE_UPDATE plugin updates before we begin
+                self._run_updates(all_updates)
+                all_updates = {}
 
             for f in files:
+                name = os.path.basename(f)
+                if ordered:
+                    m = pat.match(name)
+                    if not m:
+                        raise RuntimeError("Filename does not match format #-name.update: %s" % f)
+                    index = int(m.group(1))
+                    if index < 10 or index > 90:
+                        raise RuntimeError("Index not legal range: %d" % index)
+
+                    if index >= r:
+                        self._run_updates(all_updates)
+                        all_updates = {}
+                        r += 10
+
                 try:
                     self.info("Parsing update file '%s'" % f)
                     data = self.read_file(f)

ipaserver/install/upgradeinstance.py

@@ -115,7 +115,7 @@ class IPAUpgrade(service.Service):
             ld = ldapupdate.LDAPUpdate(dm_password='', ldapi=True, live_run=self.live_run, plugins=True)
             if len(self.files) == 0:
                 self.files = ld.get_all_files(ldapupdate.UPDATES_DIR)
-            self.modified = ld.update(self.files)
+            self.modified = ld.update(self.files, ordered=True)
         except ldapupdate.BadSyntax, e:
             root_logger.error('Bad syntax in upgrade %s' % str(e))
             self.modified = False