Expand Referential Integrity checks

Many attributes in IPA (e.g. manager, memberuser, managedby, ...)
are used to store DNs of linked objects in IPA (users, hosts, sudo
commands, etc.). However, when the linked objects is deleted or
renamed, the attribute pointing to it stays with the objects and
thus may create a dangling link causing issues in client software
reading the data.

Directory Server has a plugin to enforce referential integrity (RI)
by checking DEL and MODRDN operations and updating affected links.
It was already used for manager and secretary attributes and
should be expanded for the missing attributes to avoid dangling
links.

As a prerequisite, all attributes checked for RI must have pres
and eq indexes to avoid performance issues. Thus, the following
indexes are added:
  * manager (pres index only)
  * secretary (pres index only)
  * memberHost
  * memberUser
  * sourcehost
  * memberservice
  * managedby
  * memberallowcmd
  * memberdenycmd
  * ipasudorunas
  * ipasudorunasgroup

Referential Integrity plugin is updated to enforce RI for all these
attributes. Unit tests covering RI checks for all these attributes
were added as well.

Note: this update will only fix RI on one master as RI plugin does
not check replicated operations.

https://fedorahosted.org/freeipa/ticket/2866

install/share/indices.ldif

@@ -41,6 +41,7 @@ objectClass:nsIndex
 cn:manager
 nsSystemIndex:false
 nsIndexType:eq
+nsIndexType:pres
 
 dn: cn=secretary,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
 changetype: add
@@ -49,6 +50,7 @@ objectClass:nsIndex
 cn:secretary
 nsSystemIndex:false
 nsIndexType:eq
+nsIndexType:pres
 
 dn: cn=displayname,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
 changetype: add
@@ -110,3 +112,83 @@ nsSystemIndex: false
 nsIndexType: eq
 nsIndexType: pres
 
+dn: cn=memberHost,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: memberHost
+ObjectClass: top
+ObjectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+nsIndexType: pres
+
+dn: cn=memberUser,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: memberUser
+ObjectClass: top
+ObjectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+nsIndexType: pres
+
+dn: cn=sourcehost,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: sourcehost
+ObjectClass: top
+ObjectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+nsIndexType: pres
+
+dn: cn=memberservice,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: memberservice
+ObjectClass: top
+ObjectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+nsIndexType: pres
+
+dn: cn=managedby,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: managedby
+ObjectClass: top
+ObjectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+nsIndexType: pres
+
+dn: cn=memberallowcmd,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: memberallowcmd
+ObjectClass: top
+ObjectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+nsIndexType: pres
+
+dn: cn=memberdenycmd,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: memberdenycmd
+ObjectClass: top
+ObjectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+nsIndexType: pres
+
+dn: cn=ipasudorunas,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: ipasudorunas
+ObjectClass: top
+ObjectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+nsIndexType: pres
+
+dn: cn=ipasudorunasgroup,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+changetype: add
+cn: ipasudorunasgroup
+ObjectClass: top
+ObjectClass: nsIndex
+nsSystemIndex: false
+nsIndexType: eq
+nsIndexType: pres

install/share/referint-conf.ldif

@@ -8,4 +8,30 @@ nsslapd-pluginArg7: manager
 -
 add: nsslapd-pluginArg8
 nsslapd-pluginArg8: secretary
-
+-
+add: nsslapd-pluginArg9
+nsslapd-pluginArg9: memberuser
+-
+add: nsslapd-pluginArg10
+nsslapd-pluginArg10: memberhost
+-
+add: nsslapd-pluginArg11
+nsslapd-pluginArg11: sourcehost
+-
+add: nsslapd-pluginArg12
+nsslapd-pluginArg12: memberservice
+-
+add: nsslapd-pluginArg13
+nsslapd-pluginArg13: managedby
+-
+add: nsslapd-pluginArg14
+nsslapd-pluginArg14: memberallowcmd
+-
+add: nsslapd-pluginArg15
+nsslapd-pluginArg15: memberdenycmd
+-
+add: nsslapd-pluginArg16
+nsslapd-pluginArg16: ipasudorunas
+-
+add: nsslapd-pluginArg17
+nsslapd-pluginArg17: ipasudorunasgroup

install/updates/20-indices.update

@@ -26,6 +26,9 @@ default:ObjectClass: nsIndex
 default:nsSystemIndex: false
 default:nsIndexType: eq
 
+dn: cn=memberHost,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+add:nsIndexType: pres
+
 dn: cn=memberUser,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
 default:cn: memberUser
 default:ObjectClass: top
@@ -33,6 +36,9 @@ default:ObjectClass: nsIndex
 default:nsSystemIndex: false
 default:nsIndexType: eq
 
+dn: cn=memberUser,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+only: nsIndexType: eq,pres
+
 dn: cn=fqdn,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
 default:cn: fqdn
 default:ObjectClass: top
@@ -48,3 +54,65 @@ default:ObjectClass: nsIndex
 default:nsSystemIndex: false
 default:nsIndexType: eq
 default:nsIndexType: pres
+
+dn: cn=manager,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+only: nsIndexType: eq,pres
+
+dn: cn=secretary,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+only: nsIndexType: eq,pres
+
+dn: cn=sourcehost,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default:cn: sourcehost
+default:ObjectClass: top
+default:ObjectClass: nsIndex
+default:nsSystemIndex: false
+default:nsIndexType: eq
+default:nsIndexType: pres
+
+dn: cn=memberservice,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default:cn: memberservice
+default:ObjectClass: top
+default:ObjectClass: nsIndex
+default:nsSystemIndex: false
+default:nsIndexType: eq
+default:nsIndexType: pres
+
+dn: cn=managedby,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default:cn: managedby
+default:ObjectClass: top
+default:ObjectClass: nsIndex
+default:nsSystemIndex: false
+default:nsIndexType: eq
+default:nsIndexType: pres
+
+dn: cn=memberallowcmd,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default:cn: memberallowcmd
+default:ObjectClass: top
+default:ObjectClass: nsIndex
+default:nsSystemIndex: false
+default:nsIndexType: eq
+default:nsIndexType: pres
+
+dn: cn=memberdenycmd,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default:cn: memberdenycmd
+default:ObjectClass: top
+default:ObjectClass: nsIndex
+default:nsSystemIndex: false
+default:nsIndexType: eq
+default:nsIndexType: pres
+
+dn: cn=ipasudorunas,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default:cn: ipasudorunas
+default:ObjectClass: top
+default:ObjectClass: nsIndex
+default:nsSystemIndex: false
+default:nsIndexType: eq
+default:nsIndexType: pres
+
+dn: cn=ipasudorunasgroup,cn=index,cn=userRoot,cn=ldbm database,cn=plugins,cn=config
+default:cn: ipasudorunasgroup
+default:ObjectClass: top
+default:ObjectClass: nsIndex
+default:nsSystemIndex: false
+default:nsIndexType: eq
+default:nsIndexType: pres

install/updates/25-referint.update

@@ -0,0 +1,13 @@
+# Expand attributes checked by Referential Integrity plugin
+# pres and eq indexes defined in 20-indices.update must be set for all these
+# attributes
+dn: cn=referential integrity postoperation,cn=plugins,cn=config
+add: nsslapd-pluginArg9: memberuser
+add: nsslapd-pluginArg10: memberhost
+add: nsslapd-pluginArg11: sourcehost
+add: nsslapd-pluginArg12: memberservice
+add: nsslapd-pluginArg13: managedby
+add: nsslapd-pluginArg14: memberallowcmd
+add: nsslapd-pluginArg15: memberdenycmd
+add: nsslapd-pluginArg16: ipasudorunas
+add: nsslapd-pluginArg17: ipasudorunasgroup

install/updates/Makefile.am

@@ -23,6 +23,7 @@ app_DATA =				\
 	20-winsync_index.update		\
 	21-replicas_container.update	\
 	21-ca_renewal_container.update	\
+	25-referint.update		\
 	30-s4u2proxy.update		\
 	40-delegation.update		\
 	40-dns.update			\

ipaserver/install/dsinstance.py

@@ -193,7 +193,6 @@ class DsInstance(service.Service):
         self.step("creating directory server instance", self.__create_instance)
         self.step("adding default schema", self.__add_default_schemas)
         self.step("enabling memberof plugin", self.__add_memberof_module)
-        self.step("enabling referential integrity plugin", self.__add_referint_module)
         self.step("enabling winsync plugin", self.__add_winsync_module)
         self.step("configuring replication version plugin", self.__config_version_module)
         self.step("enabling IPA enrollment plugin", self.__add_enrollment_module)
@@ -204,6 +203,7 @@ class DsInstance(service.Service):
         self.step("enabling entryUSN plugin", self.__enable_entryusn)
         self.step("configuring lockout plugin", self.__config_lockout_module)
         self.step("creating indices", self.__create_indices)
+        self.step("enabling referential integrity plugin", self.__add_referint_module)
         self.step("configuring ssl for ds instance", self.__enable_ssl)
         self.step("configuring certmap.conf", self.__certmap_conf)
         self.step("configure autobind for root", self.__root_autobind)

tests/test_xmlrpc/test_hbac_plugin.py

@@ -547,6 +547,23 @@ class test_hbac(XMLRPC_test):
             accessruletype=u'deny',
         )
 
+    def test_n_hbacrule_links(self):
+        """
+        Test adding various links to HBAC rule
+        """
+        api.Command['hbacrule_add_sourcehost'](
+            self.rule_name, host=self.test_host, hostgroup=self.test_hostgroup
+        )
+        api.Command['hbacrule_add_service'](
+            self.rule_name, hbacsvc=self.test_service
+        )
+
+        entry = api.Command['hbacrule_show'](self.rule_name)['result']
+        assert_attr_equal(entry, 'cn', self.rule_name)
+        assert_attr_equal(entry, 'sourcehost_host', self.test_host)
+        assert_attr_equal(entry, 'sourcehost_hostgroup', self.test_hostgroup)
+        assert_attr_equal(entry, 'memberservice_hbacsvc', self.test_service)
+
     def test_y_hbacrule_zap_testing_data(self):
         """
         Clear data for HBAC plugin testing.
@@ -561,6 +578,16 @@ class test_hbac(XMLRPC_test):
         api.Command['hostgroup_del'](self.test_sourcehostgroup)
         api.Command['hbacsvc_del'](self.test_service)
 
+    def test_k_2_sudorule_referential_integrity(self):
+        """
+        Test that links in HBAC rule were removed by referential integrity plugin
+        """
+        entry = api.Command['hbacrule_show'](self.rule_name)['result']
+        assert_attr_equal(entry, 'cn', self.rule_name)
+        assert 'sourcehost_host' not in entry
+        assert 'sourcehost_hostgroup' not in entry
+        assert 'memberservice_hbacsvc' not in entry
+
     def test_z_hbacrule_del(self):
         """
         Test deleting a HBAC rule using `xmlrpc.hbacrule_del`.

tests/test_xmlrpc/test_host_plugin.py

@@ -783,6 +783,60 @@ class test_host(Declarative):
             ),
         ),
 
+
+        dict(
+            desc='Add managedby_host %r to %r' % (fqdn3, fqdn4),
+            command=('host_add_managedby', [fqdn4], dict(host=fqdn3,),
+            ),
+            expected=dict(
+                completed=1,
+                failed=dict(
+                    managedby = dict(
+                        host=tuple(),
+                    ),
+                ),
+                result=dict(
+                    dn=dn4,
+                    fqdn=[fqdn4],
+                    description=[u'Test host 4'],
+                    l=[u'Undisclosed location 4'],
+                    krbprincipalname=[u'host/%s@%s' % (fqdn4, api.env.realm)],
+                    managedby_host=[fqdn4, fqdn3],
+                ),
+            ),
+        ),
+
+
+        dict(
+            desc='Delete %r' % fqdn3,
+            command=('host_del', [fqdn3], {}),
+            expected=dict(
+                value=fqdn3,
+                summary=u'Deleted host "%s"' % fqdn3,
+                result=dict(failed=u''),
+            ),
+        ),
+
+
+        dict(
+            desc='Retrieve %r to verify that %r is gone from managedBy' % (fqdn4, fqdn3),
+            command=('host_show', [fqdn4], {}),
+            expected=dict(
+                value=fqdn4,
+                summary=None,
+                result=dict(
+                    dn=dn4,
+                    fqdn=[fqdn4],
+                    description=[u'Test host 4'],
+                    l=[u'Undisclosed location 4'],
+                    krbprincipalname=[u'host/%s@%s' % (fqdn4, api.env.realm)],
+                    has_keytab=False,
+                    has_password=False,
+                    managedby_host=[fqdn4],
+                ),
+            ),
+        ),
+
     ]
 
 class test_host_false_pwd_change(XMLRPC_test):

tests/test_xmlrpc/test_sudorule_plugin.py

@@ -674,7 +674,7 @@ class test_sudorule(XMLRPC_test):
             api.Command['sudorule_mod'](self.rule_name, ipasudorunasusercategory=u'')
 
     @raises(errors.MutuallyExclusiveError)
-    def test_j_sudorule_exclusiverunas(self):
+    def test_j_1_sudorule_exclusiverunas(self):
         """
         Test setting ipasudorunasusercategory='all' in an Sudo rule when there are runas users
         """
@@ -684,7 +684,32 @@ class test_sudorule(XMLRPC_test):
         finally:
             api.Command['sudorule_remove_runasuser'](self.rule_name, user=self.test_command)
 
-    def test_k_sudorule_clear_testing_data(self):
+    def test_j_2_sudorule_referential_integrity(self):
+        """
+        Test adding various links to Sudo rule
+        """
+        api.Command['sudorule_add_user'](self.rule_name, user=self.test_user)
+        api.Command['sudorule_add_runasuser'](self.rule_name, user=self.test_runasuser,
+                                              group=self.test_group)
+        api.Command['sudorule_add_runasgroup'](self.rule_name, group=self.test_group)
+        api.Command['sudorule_add_host'](self.rule_name, host=self.test_host)
+        api.Command['sudorule_add_allow_command'](self.rule_name,
+                                                  sudocmd=self.test_command)
+        api.Command['sudorule_add_deny_command'](self.rule_name,
+                                                 sudocmdgroup=self.test_sudodenycmdgroup)
+        entry = api.Command['sudorule_show'](self.rule_name)['result']
+        assert_attr_equal(entry, 'cn', self.rule_name)
+        assert_attr_equal(entry, 'memberuser_user', self.test_user)
+        assert_attr_equal(entry, 'memberallowcmd_sudocmd', self.test_command)
+        assert_attr_equal(entry, 'memberdenycmd_sudocmdgroup',
+            self.test_sudodenycmdgroup)
+        assert_attr_equal(entry, 'memberhost_host', self.test_host)
+        assert_attr_equal(entry, 'ipasudorunas_user', self.test_runasuser)
+        assert_attr_equal(entry, 'ipasudorunas_group', self.test_group)
+        assert_attr_equal(entry, 'ipasudorunasgroup_group', self.test_group)
+
+
+    def test_k_1_sudorule_clear_testing_data(self):
         """
         Clear data for Sudo rule plugin testing.
         """
@@ -697,6 +722,20 @@ class test_sudorule(XMLRPC_test):
         api.Command['sudocmdgroup_del'](self.test_sudoallowcmdgroup)
         api.Command['sudocmdgroup_del'](self.test_sudodenycmdgroup)
 
+    def test_k_2_sudorule_referential_integrity(self):
+        """
+        Test that links in Sudo rule were removed by referential integrity plugin
+        """
+        entry = api.Command['sudorule_show'](self.rule_name)['result']
+        assert_attr_equal(entry, 'cn', self.rule_name)
+        assert 'memberuser_user' not in entry
+        assert 'memberallowcmd_sudocmd' not in entry
+        assert 'memberdenycmd_sudocmdgroup' not in entry
+        assert 'memberhost_host' not in entry
+        assert 'ipasudorunas_user' not in entry
+        assert 'ipasudorunas_group' not in entry
+        assert 'ipasudorunasgroup_group' not in entry
+
     def test_l_sudorule_order(self):
         """
         Test that order uniqueness is maintained

tests/test_xmlrpc/test_user_plugin.py

@@ -64,7 +64,7 @@ def not_upg_check(response):
 class test_user(Declarative):
 
     cleanup_commands = [
-        ('user_del', [user1, user2, renameduser1, admin2], {}),
+        ('user_del', [user1, user2, renameduser1, admin2], {'continue': True}),
         ('group_del', [group1], {}),
     ]
 
@@ -1368,6 +1368,136 @@ class test_user(Declarative):
             ),
         ),
 
+        dict(
+            desc='Set %r as manager of %r' % (user1, user2),
+            command=(
+                'user_mod', [user2], dict(manager=user1)
+            ),
+            expected=dict(
+                result=dict(
+                    givenname=[u'Test'],
+                    homedirectory=[u'/home/tuser2'],
+                    loginshell=[u'/bin/sh'],
+                    sn=[u'User2'],
+                    uid=[user2],
+                    uidnumber=[fuzzy_digits],
+                    gidnumber=[fuzzy_digits],
+                    memberof_group=[group1],
+                    mail=[u'%s@%s' % (user2, api.env.domain)],
+                    nsaccountlock=False,
+                    has_keytab=False,
+                    has_password=False,
+                    manager=[user1],
+                ),
+                summary=u'Modified user "%s"' % user2,
+                value=user2,
+            ),
+        ),
+
+        dict(
+            desc='Rename "%s"' % user1,
+            command=('user_mod', [user1], dict(rename=renameduser1)),
+            expected=dict(
+                result=dict(
+                    givenname=[u'Test'],
+                    homedirectory=[u'/home/tuser1'],
+                    loginshell=[u'/bin/sh'],
+                    sn=[u'User1'],
+                    uid=[renameduser1],
+                    uidnumber=[fuzzy_digits],
+                    gidnumber=[fuzzy_digits],
+                    mail=[u'%s@%s' % (user1, api.env.domain)],
+                    memberof_group=[group1],
+                    nsaccountlock=False,
+                    has_keytab=False,
+                    has_password=False,
+                ),
+                summary=u'Modified user "%s"' % user1,
+                value=user1,
+            ),
+        ),
+
+        dict(
+            desc='Retrieve %r and check that manager is renamed' % user2,
+            command=(
+                'user_show', [user2], {'all': True}
+            ),
+            expected=dict(
+                result=dict(
+                    gecos=[u'Test User2'],
+                    givenname=[u'Test'],
+                    homedirectory=[u'/home/tuser2'],
+                    krbprincipalname=[u'tuser2@' + api.env.realm],
+                    loginshell=[u'/bin/sh'],
+                    objectclass=objectclasses.user_base,
+                    sn=[u'User2'],
+                    uid=[user2],
+                    uidnumber=[fuzzy_digits],
+                    gidnumber=[u'1000'],
+                    displayname=[u'Test User2'],
+                    cn=[u'Test User2'],
+                    mail=[u'%s@%s' % (user2, api.env.domain)],
+                    initials=[u'TU'],
+                    ipauniqueid=[fuzzy_uuid],
+                    krbpwdpolicyreference=[DN(('cn','global_policy'),('cn',api.env.realm),
+                                              ('cn','kerberos'),api.env.basedn)],
+                    memberof_group=[group1],
+                    nsaccountlock=False,
+                    has_keytab=False,
+                    has_password=False,
+                    dn=get_user_dn(user2),
+                    manager=[renameduser1],
+                ),
+                value=user2,
+                summary=None,
+            ),
+        ),
+
+        dict(
+            desc='Delete %r' % renameduser1,
+            command=('user_del', [renameduser1], {}),
+            expected=dict(
+                result=dict(failed=u''),
+                summary=u'Deleted user "%s"' % renameduser1,
+                value=renameduser1,
+            ),
+        ),
+
+        dict(
+            desc='Retrieve %r and check that manager is gone' % user2,
+            command=(
+                'user_show', [user2], {'all': True}
+            ),
+            expected=dict(
+                result=dict(
+                    gecos=[u'Test User2'],
+                    givenname=[u'Test'],
+                    homedirectory=[u'/home/tuser2'],
+                    krbprincipalname=[u'tuser2@' + api.env.realm],
+                    loginshell=[u'/bin/sh'],
+                    objectclass=objectclasses.user_base,
+                    sn=[u'User2'],
+                    uid=[user2],
+                    uidnumber=[fuzzy_digits],
+                    gidnumber=[u'1000'],
+                    displayname=[u'Test User2'],
+                    cn=[u'Test User2'],
+                    mail=[u'%s@%s' % (user2, api.env.domain)],
+                    initials=[u'TU'],
+                    ipauniqueid=[fuzzy_uuid],
+                    krbpwdpolicyreference=[DN(('cn','global_policy'),('cn',api.env.realm),
+                                              ('cn','kerberos'),api.env.basedn)],
+                    memberof_group=[group1],
+                    nsaccountlock=False,
+                    has_keytab=False,
+                    has_password=False,
+                    dn=get_user_dn(user2),
+                ),
+                value=user2,
+                summary=None,
+            ),
+        ),
+
         dict(
             desc='Reset default user group',
             command=(