IntenseWebs/freeipa
3
0
Fork 0
mirror of https://salsa.debian.org/freeipa-team/freeipa.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

LDAPUpdate: Batch index tasks

Browse Source 
The LDAPUpdate framework now keeps record of all changed/added indices
and batches all changed attribute in a single index task. It makes
updates much faster when multiple indices are added or modified.

Signed-off-by: Christian Heimes <cheimes@redhat.com>
Reviewed-By: Rob Crittenden <rcritten@redhat.com>
Reviewed-By: Alexander Bokovoy <abokovoy@redhat.com>
This commit is contained in:
Christian Heimes
2018-12-05 12:57:56 +01:00
parent 18f610caea
commit 0fb87bfe82
1 changed files with 33 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
ipaserver/install/ldapupdate.py
View File

@@ -134,7 +134,14 @@ def safe_output(attr, values):
class LDAPUpdate: class LDAPUpdate:
action_keywords = ["default", "add", "remove", "only", "onlyifexist", "deleteentry", "replace", "addifnew", "addifexist"] action_keywords = [
"default", "add", "remove", "only", "onlyifexist", "deleteentry",
"replace", "addifnew", "addifexist"
]
index_suffix = DN(
('cn', 'index'), ('cn', 'userRoot'), ('cn', 'ldbm database'),
('cn', 'plugins'), ('cn', 'config')
)
def __init__(self, dm_password=None, sub_dict={}, def __init__(self, dm_password=None, sub_dict={},
online=True, ldapi=False): online=True, ldapi=False):
@@ -515,8 +522,8 @@ class LDAPUpdate:
return all_updates return all_updates
def create_index_task(self, attribute): def create_index_task(self, *attributes):
"""Create a task to update an index for an attribute""" """Create a task to update an index for attributes"""
# Sleep a bit to ensure previous operations are complete # Sleep a bit to ensure previous operations are complete
time.sleep(5) time.sleep(5)
@@ -525,7 +532,7 @@ class LDAPUpdate:
# cn_uuid.time is in nanoseconds, but other users of LDAPUpdate expect # cn_uuid.time is in nanoseconds, but other users of LDAPUpdate expect
# seconds in 'TIME' so scale the value down # seconds in 'TIME' so scale the value down
self.sub_dict['TIME'] = int(cn_uuid.time/1e9) self.sub_dict['TIME'] = int(cn_uuid.time/1e9)
cn = "indextask_%s_%s_%s" % (attribute, cn_uuid.time, cn_uuid.clock_seq) cn = "indextask_%s_%s" % (cn_uuid.time, cn_uuid.clock_seq)
dn = DN(('cn', cn), ('cn', 'index'), ('cn', 'tasks'), ('cn', 'config')) dn = DN(('cn', cn), ('cn', 'index'), ('cn', 'tasks'), ('cn', 'config'))
e = self.conn.make_entry( e = self.conn.make_entry(
@@ -533,11 +540,13 @@ class LDAPUpdate:
objectClass=['top', 'extensibleObject'], objectClass=['top', 'extensibleObject'],
cn=[cn], cn=[cn],
nsInstance=['userRoot'], nsInstance=['userRoot'],
nsIndexAttribute=[attribute], nsIndexAttribute=list(attributes),
) )
logger.debug("Creating task to index attribute: %s", attribute) logger.info(
logger.debug("Task id: %s", dn) "Creating task %s to index attributes: %s",
dn, ', '.join(attributes)
)
self.conn.add_entry(e) self.conn.add_entry(e)
@@ -571,8 +580,8 @@ class LDAPUpdate:
time.sleep(1) time.sleep(1)
continue continue
if status.lower().find("finished") > -1: if "finished" in status.lower():
logger.debug("Indexing finished") logger.info("Indexing finished")
break break
logger.debug("Indexing in progress") logger.debug("Indexing in progress")
@@ -792,7 +801,7 @@ class LDAPUpdate:
entry = self._apply_update_disposition(update.get('updates'), entry) entry = self._apply_update_disposition(update.get('updates'), entry)
if entry is None: if entry is None:
# It might be None if it is just deleting an entry # It might be None if it is just deleting an entry
return return None, False
self.print_entity(entry, "Final value after applying updates") self.print_entity(entry, "Final value after applying updates")
@@ -811,7 +820,7 @@ class LDAPUpdate:
# this may not be an error (e.g. entries in NIS container) # this may not be an error (e.g. entries in NIS container)
logger.error("Parent DN of %s may not exist, cannot " logger.error("Parent DN of %s may not exist, cannot "
"create the entry", entry.dn) "create the entry", entry.dn)
return return entry, False
added = True added = True
self.modified = True self.modified = True
except Exception as e: except Exception as e:
@@ -846,12 +855,7 @@ class LDAPUpdate:
if updated: if updated:
self.modified = True self.modified = True
if entry.dn.endswith(DN(('cn', 'index'), ('cn', 'userRoot'), return entry, added or updated
('cn', 'ldbm database'), ('cn', 'plugins'),
('cn', 'config'))) and (added or updated):
taskid = self.create_index_task(entry.single_value['cn'])
self.monitor_index_task(taskid)
return
def _delete_record(self, updates): def _delete_record(self, updates):
""" """
@@ -903,13 +907,24 @@ class LDAPUpdate:
raise RuntimeError("Offline updates are not supported.") raise RuntimeError("Offline updates are not supported.")
def _run_updates(self, all_updates): def _run_updates(self, all_updates):
index_attributes = set()
for update in all_updates: for update in all_updates:
if 'deleteentry' in update: if 'deleteentry' in update:
self._delete_record(update) self._delete_record(update)
elif 'plugin' in update: elif 'plugin' in update:
self._run_update_plugin(update['plugin']) self._run_update_plugin(update['plugin'])
else: else:
self._update_record(update) entry, modified = self._update_record(update)
if modified and entry.dn.endswith(self.index_suffix):
index_attributes.add(entry.single_value['cn'])
if index_attributes:
# The LDAPUpdate framework now keeps record of all changed/added
# indices and batches all changed attribute in a single index
# task. This makes updates much faster when multiple indices are
# added or modified.
task_dn = self.create_index_task(*sorted(index_attributes))
self.monitor_index_task(task_dn)
def update(self, files, ordered=True): def update(self, files, ordered=True):
"""Execute the update. files is a list of the update files to use. """Execute the update. files is a list of the update files to use.