schema cache: Store and check info for pre-schema servers

Cache CommandError answer to schema command to avoid sending the command
to pre-schema servers every time. This information expires after some
time (1 hour) in order to start using schema as soon as the server is
upgraded.

https://fedorahosted.org/freeipa/ticket/6095

Signed-off-by: Jan Cholasta <jcholast@redhat.com>
Signed-off-by: David Kupka <dkupka@redhat.com>
Reviewed-By: Tomas Krizek <tkrizek@redhat.com>

ipaclient/remote_plugins/__init__.py

@@ -5,7 +5,9 @@
 import collections
 import errno
 import json
+import locale
 import os
+import time
 
 from . import compat
 from . import schema
@@ -23,21 +25,19 @@ class ServerInfo(collections.MutableMapping):
     def __init__(self, api):
         hostname = DNSName(api.env.server).ToASCII()
         self._path = os.path.join(self._DIR, hostname)
+        self._force_check = api.env.force_schema_check
         self._dict = {}
-        self._dirty = False
+
+        # copy-paste from ipalib/rpc.py
+        try:
+            self._language = (
+                 locale.setlocale(locale.LC_ALL, '').split('.')[0].lower()
+            )
+        except locale.Error:
+            self._language = 'en_us'
 
         self._read()
 
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *_exc_info):
-        self.flush()
-
-    def flush(self):
-        if self._dirty:
-            self._write()
-
     def _read(self):
         try:
             with open(self._path, 'r') as sc:
@@ -62,13 +62,10 @@ class ServerInfo(collections.MutableMapping):
         return self._dict[key]
 
     def __setitem__(self, key, value):
-        if key not in self._dict or self._dict[key] != value:
-            self._dirty = True
         self._dict[key] = value
 
     def __delitem__(self, key):
         del self._dict[key]
-        self._dirty = True
 
     def __iter__(self):
         return iter(self._dict)
@@ -76,26 +73,55 @@ class ServerInfo(collections.MutableMapping):
     def __len__(self):
         return len(self._dict)
 
+    def update_validity(self, ttl=None):
+        if ttl is None:
+            ttl = 3600
+        self['expiration'] = time.time() + ttl
+        self['language'] = self._language
+        self._write()
+
+    def is_valid(self):
+        if self._force_check:
+            return False
+
+        try:
+            expiration = self._dict['expiration']
+            language = self._dict['language']
+        except KeyError:
+            # if any of these is missing consider the entry expired
+            return False
+
+        if expiration < time.time():
+            # validity passed
+            return False
+
+        if language != self._language:
+            # language changed since last check
+            return False
+
+        return True
+
 
 def get_package(api):
     if api.env.in_tree:
         from ipaserver import plugins
     else:
-        client = rpcclient(api)
-        client.finalize()
-
         try:
-            server_info = api._server_info
+            plugins = api._remote_plugins
         except AttributeError:
-            server_info = api._server_info = ServerInfo(api)
+            server_info = ServerInfo(api)
 
-        try:
+            client = rpcclient(api)
-            plugins = schema.get_package(api, server_info, client)
+            client.finalize()
-        except schema.NotAvailable:
+
-            plugins = compat.get_package(api, server_info, client)
+            try:
-        finally:
+                plugins = schema.get_package(server_info, client)
-            server_info.flush()
+            except schema.NotAvailable:
-            if client.isconnected():
+                plugins = compat.get_package(server_info, client)
-                client.disconnect()
+            finally:
+                if client.isconnected():
+                    client.disconnect()
+
+            object.__setattr__(api, '_remote_plugins', plugins)
 
     return plugins

ipaclient/remote_plugins/compat.py

@@ -31,10 +31,15 @@ class CompatObject(Object):
     pass
 
 
-def get_package(api, server_info, client):
+def get_package(server_info, client):
     try:
         server_version = server_info['version']
     except KeyError:
+        is_valid = False
+    else:
+        is_valid = server_info.is_valid()
+
+    if not is_valid:
         if not client.isconnected():
             client.connect(verbose=False)
         env = client.forward(u'env', u'api_version', version=u'2.0')
@@ -51,6 +56,8 @@ def get_package(api, server_info, client):
             else:
                 server_version = u'2.0'
         server_info['version'] = server_version
+        server_info.update_validity()
+
     server_version = LooseVersion(server_version)
 
     package_names = {}

ipaclient/remote_plugins/schema.py

@@ -7,10 +7,8 @@ import contextlib
 import errno
 import fcntl
 import json
-import locale
 import os
 import sys
-import time
 import types
 import zipfile
 
@@ -220,7 +218,7 @@ class _SchemaPlugin(object):
 
     def __call__(self, api):
         if self._class is None:
-            schema = api._schema[self.schema_key][self.full_name]
+            schema = self._schema[self.schema_key][self.full_name]
             name, bases, class_dict = self._create_class(api, schema)
             self._class = type(name, bases, class_dict)
 
@@ -361,7 +359,7 @@ class Schema(object):
     namespaces = {'classes', 'commands', 'topics'}
     _DIR = os.path.join(paths.USER_CACHE_PATH, 'ipa', 'schema', FORMAT)
 
-    def __init__(self, api, server_info, client):
+    def __init__(self, client, fingerprint=None):
         self._dict = {}
         self._namespaces = {}
         self._help = None
@@ -371,48 +369,29 @@ class Schema(object):
             self._dict[ns] = {}
             self._namespaces[ns] = _SchemaNameSpace(self, ns)
 
-        # copy-paste from ipalib/rpc.py
+        ttl = None
-        try:
+        read_failed = False
-            self._language = (
-                locale.setlocale(locale.LC_ALL, '').split('.')[0].lower()
-            )
-        except locale.Error:
-            # fallback to default locale
-            self._language = 'en_us'
 
-        try:
+        if fingerprint is not None:
-            self._fingerprint = server_info['fingerprint']
-            self._expiration = server_info['expiration']
-            language = server_info['language']
-        except KeyError:
-            is_known = False
-        else:
-            is_known = (not api.env.force_schema_check and
-                        self._expiration > time.time() and
-                        self._language == language)
-
-        if is_known:
             try:
-                self._read_schema()
+                self._read_schema(fingerprint)
-            except Exception:
+            except Exception as e:
-                pass
+                # Failed to read the schema from cache. There may be a lot of
-            else:
+                # causes and not much we can do about it. Just ensure we will
-                return
+                # ignore the cache and fetch the schema from server.
+                logger.warning("Failed to read schema: {}".format(e))
+                fingerprint = None
+                read_failed = True
 
-        try:
+        if fingerprint is None:
-            self._fetch(client)
+            fingerprint, ttl = self._fetch(client, ignore_cache=read_failed)
-        except NotAvailable:
+            try:
-            raise
+                self._write_schema(fingerprint)
-        except SchemaUpToDate as e:
+            except Exception as e:
-            self._fingerprint = e.fingerprint
+                logger.warning("Failed to write schema: {}".format(e))
-            self._expiration = time.time() + e.ttl
-            self._read_schema()
-        else:
-            self._write_schema()
 
-        server_info['fingerprint'] = self._fingerprint
+        self.fingerprint = fingerprint
-        server_info['expiration'] = self._expiration
+        self.ttl = ttl
-        server_info['language'] = self._language
 
     @contextlib.contextmanager
     def _open(self, filename, mode):
@@ -429,14 +408,16 @@ class Schema(object):
             finally:
                 fcntl.flock(f, fcntl.LOCK_UN)
 
-    def _fetch(self, client):
+    def _fetch(self, client, ignore_cache=False):
         if not client.isconnected():
             client.connect(verbose=False)
 
-        try:
+        fps = []
-            fps = [fsdecode(f) for f in os.listdir(self._DIR)]
+        if not ignore_cache:
-        except EnvironmentError:
+            try:
-            fps = []
+                fps = [fsdecode(f) for f in os.listdir(self._DIR)]
+            except EnvironmentError:
+                pass
 
         kwargs = {u'version': u'2.170'}
         if fps:
@@ -459,12 +440,11 @@ class Schema(object):
             logger.warning("Failed to fetch schema: %s", e)
             raise NotAvailable()
 
-        self._fingerprint = fp
+        return (fp, ttl,)
-        self._expiration = time.time() + ttl
 
-    def _read_schema(self):
+    def _read_schema(self, fingerprint):
         self._file.truncate(0)
-        with self._open(self._fingerprint, 'r') as f:
+        with self._open(fingerprint, 'r') as f:
             self._file.write(f.read())
 
         with zipfile.ZipFile(self._file, 'r') as schema:
@@ -500,13 +480,12 @@ class Schema(object):
 
         return halp
 
-    def _write_schema(self):
+    def _write_schema(self, fingerprint):
         try:
             os.makedirs(self._DIR)
         except EnvironmentError as e:
             if e.errno != errno.EEXIST:
-                logger.warning("Failed to write schema: {}".format(e))
+                raise
-                return
 
         self._file.truncate(0)
         with zipfile.ZipFile(self._file, 'w', zipfile.ZIP_DEFLATED) as schema:
@@ -523,7 +502,7 @@ class Schema(object):
                             json.dumps(self._generate_help(self._dict)))
 
         self._file.seek(0)
-        with self._open(self._fingerprint, 'w') as f:
+        with self._open(fingerprint, 'w') as f:
             f.truncate(0)
             f.write(self._file.read())
 
@@ -550,14 +529,39 @@ class Schema(object):
         return self._help[namespace][member]
 
 
-def get_package(api, server_info, client):
+def get_package(server_info, client):
-    try:
+    NO_FINGERPRINT = object()
-        schema = api._schema
-    except AttributeError:
-        schema = Schema(api, server_info, client)
-        object.__setattr__(api, '_schema', schema)
 
-    fingerprint = str(server_info['fingerprint'])
+    fingerprint = NO_FINGERPRINT
+    if server_info.is_valid():
+        fingerprint = server_info.get('fingerprint', fingerprint)
+
+    if fingerprint is not None:
+        try:
+            try:
+                if fingerprint is NO_FINGERPRINT:
+                    schema = Schema(client)
+                else:
+                    schema = Schema(client, fingerprint)
+            except SchemaUpToDate as e:
+                schema = Schema(client, e.fingerprint)
+        except NotAvailable:
+            fingerprint = None
+            ttl = None
+        except SchemaUpToDate as e:
+            fingerprint = e.fingerprint
+            ttl = e.ttl
+        else:
+            fingerprint = schema.fingerprint
+            ttl = schema.ttl
+
+        server_info['fingerprint'] = fingerprint
+        server_info.update_validity(ttl)
+
+    if fingerprint is None:
+        raise NotAvailable()
+
+    fingerprint = str(fingerprint)
     package_name = '{}${}'.format(__name__, fingerprint)
     package_dir = '{}${}'.format(os.path.splitext(__file__)[0], fingerprint)