plugable: initialize plugins on demand

Use a new API namespace class which does not initialize plugins until they
are accessed.

https://fedorahosted.org/freeipa/ticket/4739

Reviewed-By: David Kupka <dkupka@redhat.com>

ipalib/cli.py

@@ -967,7 +967,7 @@ class show_api(frontend.Command):
                 continue
             for n in member:
                 attr = member[n]
-                if isinstance(attr, plugable.NameSpace) and len(attr) > 0:
+                if isinstance(attr, plugable.APINameSpace) and len(attr) > 0:
                     self.__traverse_namespace(n, attr, lines, tab + 2)
 
 

ipalib/frontend.py

@@ -28,7 +28,7 @@ import six
 from ipapython.version import API_VERSION
 from ipapython.ipa_log_manager import root_logger
 from ipalib.base import NameSpace
-from ipalib.plugable import Plugin
+from ipalib.plugable import Plugin, APINameSpace
 from ipalib.parameters import create_param, Param, Str, Flag
 from ipalib.parameters import Password  # pylint: disable=unused-import
 from ipalib.output import Output, Entry, ListOfEntries
@@ -402,8 +402,6 @@ class Command(HasParam):
     allowed callback types.
     """
 
-    finalize_early = False
-
     takes_options = tuple()
     takes_args = tuple()
     # Create stubs for attributes that are set in _on_finalize()
@@ -1199,8 +1197,6 @@ class Local(Command):
 
 
 class Object(HasParam):
-    finalize_early = False
-
     # Create stubs for attributes that are set in _on_finalize()
     backend = Plugin.finalize_attr('backend')
     methods = Plugin.finalize_attr('methods')
@@ -1261,7 +1257,7 @@ class Object(HasParam):
         if name not in self.api:
             return
         namespace = self.api[name]
-        assert type(namespace) is NameSpace
+        assert type(namespace) is APINameSpace
         for plugin in namespace(): # Equivalent to dict.itervalues()
             if plugin.obj_name == self.name:
                 yield plugin
@@ -1333,8 +1329,6 @@ class Attribute(Plugin):
     In practice the `Attribute` class is not used directly, but rather is
     only the base class for the `Method` class.  Also see the `Object` class.
     """
-    finalize_early = False
-
     @property
     def obj_name(self):
         return self.name.partition('_')[0]

ipalib/plugable.py

@@ -40,7 +40,7 @@ from ipalib import errors
 from ipalib.config import Env
 from ipalib.text import _
 from ipalib.util import classproperty
-from ipalib.base import ReadOnly, NameSpace, lock, islocked
+from ipalib.base import ReadOnly, lock, islocked
 from ipalib.constants import DEFAULT_CONFIG
 from ipapython.ipa_log_manager import (
     log_mgr,
@@ -124,8 +124,6 @@ class Plugin(ReadOnly):
     Base class for all plugins.
     """
 
-    finalize_early = True
-
     def __init__(self, api):
         assert api is not None
         self.__api = api
@@ -268,6 +266,50 @@ class Plugin(ReadOnly):
         )
 
 
+class APINameSpace(collections.Mapping):
+    def __init__(self, api, base):
+        self.__api = api
+        self.__base = base
+        self.__name_seq = None
+        self.__name_set = None
+
+    def __enumerate(self):
+        if self.__name_set is None:
+            self.__name_set = frozenset(
+                name for name, klass in six.iteritems(self.__api._API__plugins)
+                if any(issubclass(b, self.__base) for b in klass.bases))
+
+    def __len__(self):
+        self.__enumerate()
+        return len(self.__name_set)
+
+    def __contains__(self, name):
+        self.__enumerate()
+        return name in self.__name_set
+
+    def __iter__(self):
+        if self.__name_seq is None:
+            self.__enumerate()
+            self.__name_seq = tuple(sorted(self.__name_set))
+        return iter(self.__name_seq)
+
+    def __getitem__(self, name):
+        name = getattr(name, '__name__', name)
+        klass = self.__api._API__plugins[name]
+        if not any(issubclass(b, self.__base) for b in klass.bases):
+            raise KeyError(name)
+        return self.__api._get(name)
+
+    def __call__(self):
+        return six.itervalues(self)
+
+    def __getattr__(self, name):
+        try:
+            return self[name]
+        except KeyError:
+            raise AttributeError(name)
+
+
 class API(ReadOnly):
     """
     Dynamic API object through which `Plugin` instances are accessed.
@@ -276,6 +318,7 @@ class API(ReadOnly):
     def __init__(self):
         super(API, self).__init__()
         self.__plugins = {}
+        self.__instances = {}
         self.__next = {}
         self.__done = set()
         self.env = Env()
@@ -628,33 +671,28 @@ class API(ReadOnly):
         self.__do_if_not_done('load_plugins')
 
         production_mode = self.is_production_mode()
-        plugins = {}
         plugin_info = {}
 
         for base in self.bases:
             name = base.__name__
-            members = []
 
-            for klass in self.__plugins.values():
+            for klass in six.itervalues(self.__plugins):
                 if not any(issubclass(b, base) for b in klass.bases):
                     continue
-                try:
-                    instance = plugins[klass]
-                except KeyError:
-                    instance = plugins[klass] = klass(self)
-                members.append(instance)
                 plugin_info.setdefault(
                     '%s.%s' % (klass.__module__, klass.name),
                     []).append(name)
+                if not self.env.plugins_on_demand:
+                    self._get(klass.name)
 
             if not production_mode:
                 assert not hasattr(self, name)
-            setattr(self, name, NameSpace(members))
+            setattr(self, name, APINameSpace(self, base))
 
-        for klass, instance in plugins.items():
+        for klass, instance in six.iteritems(self.__instances):
             if not production_mode:
                 assert instance.api is self
-            if klass.finalize_early or not self.env.plugins_on_demand:
+            if not self.env.plugins_on_demand:
                 instance.ensure_finalized()
                 if not production_mode:
                     assert islocked(instance)
@@ -665,6 +703,14 @@ class API(ReadOnly):
         if not production_mode:
             lock(self)
 
+    def _get(self, name):
+        klass = self.__plugins[name]
+        try:
+            instance = self.__instances[klass]
+        except KeyError:
+            instance = self.__instances[klass] = klass(self)
+        return instance
+
     def get_plugin_next(self, klass):
         if not callable(klass):
             raise TypeError('plugin must be callable; got %r' % klass)

ipatests/test_ipalib/test_cli.py

@@ -87,7 +87,7 @@ class DummyCommand(object):
 
 class DummyAPI(object):
     def __init__(self, cnt):
-        self.__cmd = plugable.NameSpace(self.__cmd_iter(cnt))
+        self.__cmd = plugable.APINameSpace(self.__cmd_iter(cnt), DummyCommand)
 
     def __get_cmd(self):
         return self.__cmd

ipatests/test_ipalib/test_frontend.py

@@ -283,11 +283,11 @@ class test_Command(ClassChecker):
                 return False
         o = self.cls(api)
         o.finalize()
-        assert type(o.args) is plugable.NameSpace
+        assert type(o.args) is NameSpace
         assert len(o.args) == 0
         args = ('destination', 'source?')
         ns = self.get_instance(args=args).args
-        assert type(ns) is plugable.NameSpace
+        assert type(ns) is NameSpace
         assert len(ns) == len(args)
         assert list(ns) == ['destination', 'source']
         assert type(ns.destination) is parameters.Str
@@ -340,11 +340,11 @@ class test_Command(ClassChecker):
                 return False
         o = self.cls(api)
         o.finalize()
-        assert type(o.options) is plugable.NameSpace
+        assert type(o.options) is NameSpace
         assert len(o.options) == 1
         options = ('target', 'files*')
         ns = self.get_instance(options=options).options
-        assert type(ns) is plugable.NameSpace
+        assert type(ns) is NameSpace
         assert len(ns) == len(options) + 1
         assert list(ns) == ['target', 'files', 'version']
         assert type(ns.target) is parameters.Str
@@ -364,7 +364,7 @@ class test_Command(ClassChecker):
                 return False
         inst = self.cls(api)
         inst.finalize()
-        assert type(inst.output) is plugable.NameSpace
+        assert type(inst.output) is NameSpace
         assert list(inst.output) == ['result']
         assert type(inst.output.result) is output.Output
 
@@ -945,7 +945,7 @@ class test_Object(ClassChecker):
         methods_format = 'method_%d'
 
         class FakeAPI(object):
-            Method = plugable.NameSpace(
+            Method = NameSpace(
                 get_attributes(cnt, methods_format)
             )
             def __contains__(self, key):
@@ -965,7 +965,7 @@ class test_Object(ClassChecker):
         assert read_only(o, 'api') is api
 
         namespace = o.methods
-        assert isinstance(namespace, plugable.NameSpace)
+        assert isinstance(namespace, NameSpace)
         assert len(namespace) == cnt
         f = methods_format
         for i in range(cnt):
@@ -980,13 +980,13 @@ class test_Object(ClassChecker):
         # Test params instance attribute
         o = self.cls(api)
         ns = o.params
-        assert type(ns) is plugable.NameSpace
+        assert type(ns) is NameSpace
         assert len(ns) == 0
         class example(self.cls):
             takes_params = ('banana', 'apple')
         o = example(api)
         ns = o.params
-        assert type(ns) is plugable.NameSpace
+        assert type(ns) is NameSpace
         assert len(ns) == 2, repr(ns)
         assert list(ns) == ['banana', 'apple']
         for p in ns():
@@ -1024,7 +1024,7 @@ class test_Object(ClassChecker):
         assert pk.name == 'three'
         assert pk.primary_key is True
         assert o.params[2] is o.primary_key
-        assert isinstance(o.params_minus_pk, plugable.NameSpace)
+        assert isinstance(o.params_minus_pk, NameSpace)
         assert list(o.params_minus_pk) == ['one', 'two', 'four']
 
         # Test with multiple primary_key: