plugable: Move plugin base class and override logic to API

Each API object now maintains its own view of registered plugins. This change
removes the need to register plugin base classes.

This reverts commit 2db741e847.

https://fedorahosted.org/freeipa/ticket/3090
https://fedorahosted.org/freeipa/ticket/5073

Reviewed-By: Martin Babinsky <mbabinsk@redhat.com>

ipalib/backend.py

@@ -27,10 +27,7 @@ import os
 from errors import PublicError, InternalError, CommandError
 from request import context, Connection, destroy_context
 
-register = plugable.Registry()
 
-
-@register.base()
 class Backend(plugable.Plugin):
     """
     Base class for all backend plugins.

ipalib/frontend.py

@@ -27,7 +27,7 @@ from distutils import version
 from ipapython.version import API_VERSION
 from ipapython.ipa_log_manager import root_logger
 from base import NameSpace
-from plugable import Plugin, Registry, is_production_mode
+from plugable import Plugin, is_production_mode
 from parameters import create_param, Param, Str, Flag, Password
 from output import Output, Entry, ListOfEntries
 from text import _
@@ -40,9 +40,6 @@ from textwrap import wrap
 
 RULE_FLAG = 'validation_rule'
 
-register = Registry()
-
-
 def rule(obj):
     assert not hasattr(obj, RULE_FLAG)
     setattr(obj, RULE_FLAG, True)
@@ -369,7 +366,6 @@ class HasParam(Plugin):
         setattr(self, name, namespace)
 
 
-@register.base()
 class Command(HasParam):
     """
     A public IPA atomic operation.
@@ -1124,7 +1120,6 @@ class Local(Command):
         return self.forward(*args, **options)
 
 
-@register.base()
 class Object(HasParam):
     finalize_early = False
 
@@ -1283,7 +1278,6 @@ class Attribute(Plugin):
         super(Attribute, self)._on_finalize()
 
 
-@register.base()
 class Method(Attribute, Command):
     """
     A command with an associated object.
@@ -1370,7 +1364,6 @@ class Method(Attribute, Command):
             yield param
 
 
-@register.base()
 class Updater(Plugin):
     """
     An LDAP update with an associated object (always update).

ipalib/plugable.py

@@ -35,6 +35,7 @@ import subprocess
 import optparse
 import errors
 import textwrap
+import collections
 
 from config import Env
 import util
@@ -74,94 +75,13 @@ class Registry(object):
     For forward compatibility, make sure that the module-level instance of
     this object is named "register".
     """
-
+    def __call__(self):
-    __allowed = {}
+        def decorator(cls):
-    __registered = set()
+            API.register(cls)
-
+            return cls
-    def base(self):
-        def decorator(base):
-            if not inspect.isclass(base):
-                raise TypeError('plugin base must be a class; got %r' % base)
-
-            if base in self.__allowed:
-                raise errors.PluginDuplicateError(plugin=base)
-
-            self.__allowed[base] = {}
-
-            return base
 
         return decorator
 
-    def __findbases(self, klass):
-        """
-        Iterates through allowed bases that ``klass`` is a subclass of.
-
-        Raises `errors.PluginSubclassError` if ``klass`` is not a subclass of
-        any allowed base.
-
-        :param klass: The plugin class to find bases for.
-        """
-        found = False
-        for (base, sub_d) in self.__allowed.iteritems():
-            if issubclass(klass, base):
-                found = True
-                yield (base, sub_d)
-        if not found:
-            raise errors.PluginSubclassError(
-                plugin=klass, bases=self.__allowed.keys()
-            )
-
-    def __call__(self, override=False):
-        def decorator(klass):
-            if not inspect.isclass(klass):
-                raise TypeError('plugin must be a class; got %r' % klass)
-
-            # Raise DuplicateError if this exact class was already registered:
-            if klass in self.__registered:
-                raise errors.PluginDuplicateError(plugin=klass)
-
-            # Find the base class or raise SubclassError:
-            for (base, sub_d) in self.__findbases(klass):
-                # Check override:
-                if klass.__name__ in sub_d:
-                    if not override:
-                        # Must use override=True to override:
-                        raise errors.PluginOverrideError(
-                            base=base.__name__,
-                            name=klass.__name__,
-                            plugin=klass,
-                        )
-                else:
-                    if override:
-                        # There was nothing already registered to override:
-                        raise errors.PluginMissingOverrideError(
-                            base=base.__name__,
-                            name=klass.__name__,
-                            plugin=klass,
-                        )
-
-                # The plugin is okay, add to sub_d:
-                sub_d[klass.__name__] = klass
-
-            # The plugin is okay, add to __registered:
-            self.__registered.add(klass)
-
-            return klass
-
-        return decorator
-
-    def __base_iter(self, *allowed):
-        for base in allowed:
-            sub_d = self.__allowed[base]
-            subclasses = set(sub_d.itervalues())
-            yield (base, subclasses)
-
-    def iter(self, *allowed):
-        for base in allowed:
-            if base not in self.__allowed:
-                raise TypeError("unknown plugin base %r" % base)
-        return self.__base_iter(*allowed)
-
 
 class SetProxy(ReadOnly):
     """
@@ -441,28 +361,60 @@ class Plugin(ReadOnly):
         )
 
 
-class API(DictProxy):
+class Registrar(collections.Mapping):
     """
-    Dynamic API object through which `Plugin` instances are accessed.
+    Collects plugin classes as they are registered.
+
+    The Registrar does not instantiate plugins... it only implements the
+    override logic and stores the plugins in a namespace per allowed base
+    class.
+
+    The plugins are instantiated when `API.finalize()` is called.
     """
+    def __init__(self):
+        self.__registry = collections.OrderedDict()
 
-    def __init__(self, allowed, packages):
+    def __call__(self, klass, override=False):
-        self.__allowed = allowed
-        self.packages = packages
-        self.__d = dict()
-        self.__done = set()
-        self.__registry = Registry()
-        self.env = Env()
-        super(API, self).__init__(self.__d)
-
-    def register(self, klass, override=False):
         """
         Register the plugin ``klass``.
 
         :param klass: A subclass of `Plugin` to attempt to register.
         :param override: If true, override an already registered plugin.
         """
-        self.__registry(override)(klass)
+        if not inspect.isclass(klass):
+            raise TypeError('plugin must be a class; got %r' % klass)
+
+        # Raise DuplicateError if this exact class was already registered:
+        if klass in self.__registry:
+            raise errors.PluginDuplicateError(plugin=klass)
+
+        # The plugin is okay, add to __registry:
+        self.__registry[klass] = dict(override=override)
+
+    def __getitem__(self, key):
+        return self.__registry[key]
+
+    def __iter__(self):
+        return iter(self.__registry)
+
+    def __len__(self):
+        return len(self.__registry)
+
+
+class API(DictProxy):
+    """
+    Dynamic API object through which `Plugin` instances are accessed.
+    """
+
+    register = Registrar()
+
+    def __init__(self, allowed, packages):
+        self.__plugins = {base: {} for base in allowed}
+        self.packages = packages
+        self.__d = dict()
+        self.__done = set()
+        self.env = Env()
+        super(API, self).__init__(self.__d)
 
     def __doing(self, name):
         if name in self.__done:
@@ -638,6 +590,8 @@ class API(DictProxy):
             return
         for package in self.packages:
             self.import_plugins(package)
+        for klass, kwargs in self.register.iteritems():
+            self.add_plugin(klass, **kwargs)
 
     # FIXME: This method has no unit test
     def import_plugins(self, package):
@@ -686,6 +640,51 @@ class API(DictProxy):
                     self.log.error('could not load plugin module %r\n%s', pyfile, traceback.format_exc())
                 raise
 
+    def add_plugin(self, klass, override=False):
+        """
+        Add the plugin ``klass``.
+
+        :param klass: A subclass of `Plugin` to attempt to add.
+        :param override: If true, override an already added plugin.
+        """
+        if not inspect.isclass(klass):
+            raise TypeError('plugin must be a class; got %r' % klass)
+
+        # Find the base class or raise SubclassError:
+        found = False
+        for (base, sub_d) in self.__plugins.iteritems():
+            if not issubclass(klass, base):
+                continue
+
+            found = True
+
+            # Check override:
+            if klass.__name__ in sub_d:
+                if not override:
+                    # Must use override=True to override:
+                    raise errors.PluginOverrideError(
+                        base=base.__name__,
+                        name=klass.__name__,
+                        plugin=klass,
+                    )
+            else:
+                if override:
+                    # There was nothing already registered to override:
+                    raise errors.PluginMissingOverrideError(
+                        base=base.__name__,
+                        name=klass.__name__,
+                        plugin=klass,
+                    )
+
+            # The plugin is okay, add to sub_d:
+            sub_d[klass.__name__] = klass
+
+        if not found:
+            raise errors.PluginSubclassError(
+                plugin=klass,
+                bases=self.__plugins.keys(),
+            )
+
     def finalize(self):
         """
         Finalize the registration, instantiate the plugins.
@@ -696,56 +695,25 @@ class API(DictProxy):
         self.__doing('finalize')
         self.__do_if_not_done('load_plugins')
 
-        class PluginInstance(object):
-            """
-            Represents a plugin instance.
-            """
-
-            i = 0
-
-            def __init__(self, klass):
-                self.created = self.next()
-                self.klass = klass
-                self.instance = klass()
-                self.bases = []
-
-            @classmethod
-            def next(cls):
-                cls.i += 1
-                return cls.i
-
-        class PluginInfo(ReadOnly):
-            def __init__(self, p):
-                assert isinstance(p, PluginInstance)
-                self.created = p.created
-                self.name = p.klass.__name__
-                self.module = str(p.klass.__module__)
-                self.plugin = '%s.%s' % (self.module, self.name)
-                self.bases = tuple(b.__name__ for b in p.bases)
-                if not is_production_mode(self):
-                    lock(self)
-
-        plugins = {}
-        tofinalize = set()
-        def plugin_iter(base, subclasses):
-            for klass in subclasses:
-                assert issubclass(klass, base)
-                if klass not in plugins:
-                    plugins[klass] = PluginInstance(klass)
-                p = plugins[klass]
-                if not is_production_mode(self):
-                    assert base not in p.bases
-                p.bases.append(base)
-                if klass.finalize_early or not self.env.plugins_on_demand:
-                    tofinalize.add(p)
-                yield p.instance
-
         production_mode = is_production_mode(self)
-        for base, subclasses in self.__registry.iter(*self.__allowed):
+        plugins = {}
+        plugin_info = {}
+
+        for base, sub_d in self.__plugins.iteritems():
             name = base.__name__
-            namespace = NameSpace(
+
-                plugin_iter(base, subclasses)
+            members = []
-            )
+            for klass in sub_d.itervalues():
+                try:
+                    instance = plugins[klass]
+                except KeyError:
+                    instance = plugins[klass] = klass()
+                members.append(instance)
+                plugin_info.setdefault(
+                    '%s.%s' % (klass.__module__, klass.__name__),
+                    []).append(name)
+
+            namespace = NameSpace(members)
             if not production_mode:
                 assert not (
                     name in self.__d or hasattr(self, name)
@@ -753,19 +721,20 @@ class API(DictProxy):
             self.__d[name] = namespace
             object.__setattr__(self, name, namespace)
 
-        for p in plugins.itervalues():
+        for instance in plugins.itervalues():
-            p.instance.set_api(self)
+            instance.set_api(self)
-            if not production_mode:
-                assert p.instance.api is self
 
-        for p in tofinalize:
+        for klass, instance in plugins.iteritems():
-            p.instance.ensure_finalized()
             if not production_mode:
-                assert islocked(p.instance) is True
+                assert instance.api is self
+            if klass.finalize_early or not self.env.plugins_on_demand:
+                instance.ensure_finalized()
+                if not production_mode:
+                    assert islocked(instance)
+
         object.__setattr__(self, '_API__finalized', True)
-        tuple(PluginInfo(p) for p in plugins.itervalues())
         object.__setattr__(self, 'plugins',
-            tuple(PluginInfo(p) for p in plugins.itervalues())
+            tuple((k, tuple(v)) for k, v in plugin_info.iteritems())
         )
 
 

ipaserver/advise/base.py

@@ -19,14 +19,12 @@
 
 import os
 from ipalib import api
-from ipalib.plugable import Plugin, Registry, API
+from ipalib.plugable import Plugin, API
 from ipalib.errors import ValidationError
 from ipapython import admintool
 from textwrap import wrap
 from ipapython.ipa_log_manager import log_mgr
 
-register = Registry()
-
 
 """
 To add configuration instructions for a new use case, define a new class that
@@ -97,7 +95,6 @@ class _AdviceOutput(object):
         self.content.append(line)
 
 
-@register.base()
 class Advice(Plugin):
     """
     Base class for advices, plugins for ipa-advise.

ipatests/test_ipalib/test_plugable.py

@@ -287,9 +287,9 @@ class test_Plugin(ClassChecker):
         assert e.argv == (paths.BIN_FALSE,)
 
 
-def test_Registry():
+def test_Registrar():
     """
-    Test the `ipalib.plugable.Registry` class
+    Test the `ipalib.plugable.Registrar` class
     """
     class Base1(object):
         pass
@@ -304,47 +304,8 @@ def test_Registry():
     class plugin3(Base3):
         pass
 
-    # Test creation of Registry:
+    # Test creation of Registrar:
-    register = plugable.Registry()
+    r = plugable.Registrar()
-    def b(klass):
-        register.base()(klass)
-    def r(klass, override=False):
-        register(override=override)(klass)
-
-    # Check that TypeError is raised trying to register base that isn't
-    # a class:
-    p = Base1()
-    e = raises(TypeError, b, p)
-    assert str(e) == 'plugin base must be a class; got %r' % p
-
-    # Check that base registration works
-    b(Base1)
-    i = tuple(register.iter(Base1))
-    assert len(i) == 1
-    assert i[0][0] is Base1
-    assert not i[0][1]
-
-    # Check that DuplicateError is raised trying to register exact class
-    # again:
-    e = raises(errors.PluginDuplicateError, b, Base1)
-    assert e.plugin is Base1
-
-    # Test that another base can be registered:
-    b(Base2)
-    i = tuple(register.iter(Base2))
-    assert len(i) == 1
-    assert i[0][0] is Base2
-    assert not i[0][1]
-
-    # Test iter:
-    i = tuple(register.iter(Base1, Base2))
-    assert len(i) == 2
-    assert i[0][0] is Base1
-    assert not i[0][1]
-    assert i[1][0] is Base2
-    assert not i[1][1]
-    e = raises(TypeError, register.iter, Base1, Base2, Base3)
-    assert str(e) == 'unknown plugin base %r' % Base3
 
     # Check that TypeError is raised trying to register something that isn't
     # a class:
@@ -352,59 +313,33 @@ def test_Registry():
     e = raises(TypeError, r, p)
     assert str(e) == 'plugin must be a class; got %r' % p
 
-    # Check that SubclassError is raised trying to register a class that is
-    # not a subclass of an allowed base:
-    e = raises(errors.PluginSubclassError, r, plugin3)
-    assert e.plugin is plugin3
-
     # Check that registration works
     r(plugin1)
-    i = tuple(register.iter(Base1))
+    assert len(r) == 1
-    assert len(i) == 1
+    assert plugin1 in r
-    assert i[0][0] is Base1
+    assert r[plugin1] == dict(override=False)
-    assert i[0][1] == {plugin1}
 
     # Check that DuplicateError is raised trying to register exact class
     # again:
     e = raises(errors.PluginDuplicateError, r, plugin1)
     assert e.plugin is plugin1
 
-    # Check that OverrideError is raised trying to register class with same
+    # Check that overriding works
-    # name and same base:
     orig1 = plugin1
     class base1_extended(Base1):
         pass
     class plugin1(base1_extended):  # pylint: disable=function-redefined
         pass
-    e = raises(errors.PluginOverrideError, r, plugin1)
-    assert e.base == 'Base1'
-    assert e.name == 'plugin1'
-    assert e.plugin is plugin1
-
-    # Check that overriding works
     r(plugin1, override=True)
-    i = tuple(register.iter(Base1))
+    assert len(r) == 2
-    assert len(i) == 1
+    assert plugin1 in r
-    assert i[0][0] is Base1
+    assert r[plugin1] == dict(override=True)
-    assert i[0][1] == {plugin1}
-
-    # Check that MissingOverrideError is raised trying to override a name
-    # not yet registerd:
-    e = raises(errors.PluginMissingOverrideError, r, plugin2, override=True)
-    assert e.base == 'Base2'
-    assert e.name == 'plugin2'
-    assert e.plugin is plugin2
 
     # Test that another plugin can be registered:
-    i = tuple(register.iter(Base2))
-    assert len(i) == 1
-    assert i[0][0] is Base2
-    assert not i[0][1]
     r(plugin2)
-    i = tuple(register.iter(Base2))
+    assert len(r) == 3
-    assert len(i) == 1
+    assert plugin2 in r
-    assert i[0][0] is Base2
+    assert r[plugin2] == dict(override=False)
-    assert i[0][1] == {plugin2}
 
     # Setup to test more registration:
     class plugin1a(Base1):
@@ -423,14 +358,6 @@ def test_Registry():
         pass
     r(plugin2b)
 
-    # Again test iter:
-    i = tuple(register.iter(Base1, Base2))
-    assert len(i) == 2
-    assert i[0][0] is Base1
-    assert i[0][1] == {plugin1, plugin1a, plugin1b}
-    assert i[1][0] is Base2
-    assert i[1][1] == {plugin2, plugin2a, plugin2b}
-
 
 class test_API(ClassChecker):
     """
@@ -445,15 +372,11 @@ class test_API(ClassChecker):
         """
         assert issubclass(plugable.API, plugable.ReadOnly)
 
-        register = plugable.Registry()
-
         # Setup the test bases, create the API:
-        @register.base()
         class base0(plugable.Plugin):
             def method(self, n):
                 return n
 
-        @register.base()
         class base1(plugable.Plugin):
             def method(self, n):
                 return n + 1
@@ -461,30 +384,31 @@ class test_API(ClassChecker):
         api = plugable.API([base0, base1], [])
         api.env.mode = 'unit_test'
         api.env.in_tree = True
+        r = api.register
 
-        @register()
         class base0_plugin0(base0):
             pass
+        r(base0_plugin0)
 
-        @register()
         class base0_plugin1(base0):
             pass
+        r(base0_plugin1)
 
-        @register()
         class base0_plugin2(base0):
             pass
+        r(base0_plugin2)
 
-        @register()
         class base1_plugin0(base1):
             pass
+        r(base1_plugin0)
 
-        @register()
         class base1_plugin1(base1):
             pass
+        r(base1_plugin1)
 
-        @register()
         class base1_plugin2(base1):
             pass
+        r(base1_plugin2)
 
         # Test API instance:
         assert api.isdone('bootstrap') is False