IntenseWebs/pgadmin4
3
0
Fork 0
mirror of https://github.com/pgadmin-org/pgadmin4.git synced 2025-02-25 18:55:31 -06:00
Code Issues Packages Projects Releases Wiki Activity

Do not use the sqlite for session handling, as the old implementation is

Browse Source 
optimized to work from multiple threads. It has too many frequent
transaction from multiple threads, and that tends to result into the
'database is locked' error of sqlite.

With the new implemenation, we're using the caching mechanism, which
keep the data in the memory all the time, and saves it on request
completion, and loads it only for the first time. Also, it will storage
the data using pickle, which will be faster than accessing sqlite.

Fixes #1329
This commit is contained in:
Ashesh Vashi 2016-07-08 16:55:53 +05:30
parent 52d1af63a5
commit 61698b7b4d
3 changed files with 234 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
requirements_py2.txt
View File

@ -38,3 +38,4 @@ unittest2==1.1.0
Werkzeug==0.9.6 Werkzeug==0.9.6
WTForms==2.0.2 WTForms==2.0.2
sqlparse==0.1.19 sqlparse==0.1.19
ordereddict; python_version < '2.7'

4
web/pgadmin/__init__.py
View File

@ -22,7 +22,7 @@ from flask_mail import Mail
from flask_security.utils import login_user from flask_security.utils import login_user
from htmlmin.minify import html_minify from htmlmin.minify import html_minify
from pgadmin.utils import PgAdminModule, driver from pgadmin.utils import PgAdminModule, driver
from pgadmin.utils.session import ServerSideSessionInterface from pgadmin.utils.session import create_session_interface
from werkzeug.local import LocalProxy from werkzeug.local import LocalProxy
from werkzeug.utils import find_modules from werkzeug.utils import find_modules
@ -121,7 +121,7 @@ def create_app(app_name=config.APP_NAME):
########################################################################## ##########################################################################
# Setup session management # Setup session management
########################################################################## ##########################################################################
app.session_interface = ServerSideSessionInterface(config.SESSION_DB_PATH) app.session_interface = create_session_interface(app)
########################################################################## ##########################################################################
# Setup logging and log the application startup # Setup logging and log the application startup

392
web/pgadmin/utils/session.py
View File

@ -10,227 +10,265 @@
""" """
Implements the server-side session management. Implements the server-side session management.
Credit/Reference: http://flask.pocoo.org/snippets/86/ Credit/Reference: http://flask.pocoo.org/snippets/109/
Modified to support both Python 2.6+ & Python 3.x Modified to support both Python 2.6+ & Python 3.x
""" """
import errno import base64
import datetime
import hmac
import hashlib
import os import os
import sqlite3 import random
import string
from uuid import uuid4 from uuid import uuid4
try: try:
from cPickle import dumps, loads from cPickle import dump, load
except: except:
from pickle import dumps, loads from pickle import dump, load
try:
from collections import OrderedDict
except:
from ordereddict import OrderedDict
from collections import MutableMapping
from flask import request
from flask.sessions import SessionInterface, SessionMixin from flask.sessions import SessionInterface, SessionMixin
from werkzeug.datastructures import CallbackDict
class SqliteSessionStorage(MutableMapping, SessionMixin): def _calc_hmac(body, secret):
""" return base64.b64encode(hmac.new(secret, body, hashlib.sha1).digest())
A class to store the session as sqlite object.
"""
_create_sql = (
'CREATE TABLE IF NOT EXISTS pg_session '
'('
' key TEXT PRIMARY KEY,'
' val BLOB'
')'
)
_get_sql = 'SELECT val FROM pg_session WHERE key = ?'
_set_sql = 'REPLACE INTO pg_session (key, val) VALUES (?, ?)'
_del_sql = 'DELETE FROM pg_session WHERE key = ?'
_ite_sql = 'SELECT key FROM pg_session'
_len_sql = 'SELECT COUNT(*) FROM pg_session'
def __init__(self, directory, sid, *args, **kwargs): class ManagedSession(CallbackDict, SessionMixin):
"""Initialize the session storage for this particular session. If def __init__(self, initial=None, sid=None, new=False, randval=None, hmac_digest=None):
requires, creates new sqlite database per session (if require). def on_update(self):
""" self.modified = True
self.path = os.path.join(directory, sid)
self.directory = directory CallbackDict.__init__(self, initial, on_update)
self.sid = sid self.sid = sid
self.new = new
self.modified = False self.modified = False
self.conn = None self.randval = randval
self.hmac_digest = hmac_digest
def sign(self, secret):
if not self.hmac_digest:
self.randval = ''.join(random.sample(string.lowercase+string.digits, 20))
self.hmac_digest = _calc_hmac('%s:%s' % (self.sid, self.randval), secret)
class SessionManager(object):
def new_session(self):
'Create a new session'
raise NotImplementedError
def exists(self, sid):
'Does the given session-id exist?'
raise NotImplementedError
def remove(self, sid):
'Remove the session'
raise NotImplementedError
def get(self, sid, digest):
'Retrieve a managed session by session-id, checking the HMAC digest'
raise NotImplementedError
def put(self, session):
'Store a managed session'
raise NotImplementedError
class CachingSessionManager(SessionManager):
def __init__(self, parent, num_to_store):
self.parent = parent
self.num_to_store = num_to_store
self._cache = OrderedDict()
def _normalize(self):
if len(self._cache) > self.num_to_store:
# Flush 20% of the cache
while len(self._cache) > (self.num_to_store * 0.8):
self._cache.popitem(False)
def new_session(self):
session = self.parent.new_session()
self._cache[session.sid] = session
self._normalize()
return session
def remove(self, sid):
self.parent.remove(sid)
if sid in self._cache:
del self._cache[sid]
def exists(self, sid):
if sid in self._cache:
return True
return self.parent.exists(sid)
def get(self, sid, digest):
session = None
if sid in self._cache:
session = self._cache[sid]
if session.hmac_digest != digest:
session = None
# reset order in Dict
del self._cache[sid]
if not session:
session = self.parent.get(sid, digest)
self._cache[sid] = session
self._normalize()
return session
def put(self, session):
self.parent.put(session)
if session.sid in self._cache:
del self._cache[session.sid]
self._cache[session.sid] = session
self._normalize()
class FileBackedSessionManager(SessionManager):
def __init__(self, path, secret):
self.path = path
self.secret = secret
if not os.path.exists(self.path): if not os.path.exists(self.path):
sess_db = os.open(self.path, os.O_CREAT, int("600", 8)) os.makedirs(self.path)
os.close(sess_db)
with self._get_conn() as conn: def exists(self, sid):
conn.execute(self._create_sql) fname = os.path.join(self.path, sid)
self.new = True return os.path.exists(fname)
def __getitem__(self, key): def remove(self, sid):
"""Reads the session data for the particular key from the sqlite fname = os.path.join(self.path, sid)
database. if os.path.exists(fname):
""" os.unlink(fname)
key = dumps(key, 0)
rv = None
with self._get_conn() as conn:
for row in conn.execute(self._get_sql, (key,)):
rv = loads(bytes(row[0]))
break
if rv is None:
raise KeyError('Key not in this session')
return rv
def __setitem__(self, key, value): def new_session(self):
"""Stores the session data for the given key. sid = str(uuid4())
""" fname = os.path.join(self.path, sid)
key = dumps(key, 0)
value = dumps(value, 2)
with self._get_conn() as conn:
conn.execute(self._set_sql, (key, sqlite3.Binary(value)))
self.modified = True
def __delitem__(self, key): while os.path.exists(fname):
"""Removes the session data representing the key from the session. sid = str(uuid4())
""" fname = os.path.join(self.path, sid)
key = dumps(key, 0)
with self._get_conn() as conn:
conn.execute(self._del_sql, (key,))
self.modified = True
def __iter__(self): # touch the file
"""Returns the iterator of the key, value pair stored under this with open(fname, 'w'):
session. pass
"""
with self._get_conn() as conn:
for row in conn.execute(self._ite_sql):
yield loads(dumps(row[0]))
def __len__(self): return ManagedSession(sid=sid)
"""Returns the number of keys stored in this session.
"""
with self._get_conn() as conn:
for row in conn.execute(self._len_sql):
return row[0]
def _get_conn(self): def get(self, sid, digest):
"""Connection object to the sqlite database object. 'Retrieve a managed session by session-id, checking the HMAC digest'
"""
if not self.conn:
self.conn = sqlite3.Connection(self.path)
return self.conn
# These proxy classes are needed in order fname = os.path.join(self.path, sid)
# for this session implementation to work properly. data = None
# That is because sometimes flask will chain method calls hmac_digest = None
# with session'setdefault' calls. randval = None
# Eg: session.setdefault('_flashes', []).append(1)
# With these proxies, the changes made by chained
# method calls will be persisted back to the sqlite
# database.
class CallableAttributeProxy(object):
"""
A proxy class to represent the callable attributes of a object.
"""
def __init__(self, session, key, obj, attr): if os.path.exists(fname):
"""Initialize the proxy instance for the callable attribute. try:
""" with open(fname) as f:
self.session = session randval, hmac_digest, data = load(f)
self.key = key except:
self.obj = obj pass
self.attr = attr
def __call__(self, *args, **kwargs): if not data:
"""Returns the callable attributes for this session. return self.new_session()
"""
rv = self.attr(*args, **kwargs)
self.session[self.key] = self.obj
return rv
class PersistedObjectProxy(object): # This assumes the file is correct, if you really want to
""" # make sure the session is good from the server side, you
A proxy class to represent the persistent object. # can re-calculate the hmac
"""
def __init__(self, session, key, obj): if hmac_digest != digest:
"""Initialize the persitent objects under the session. return self.new_session()
"""
self.session = session
self.key = key
self.obj = obj
def __getattr__(self, name): return ManagedSession(
"""Returns the attribute of the persistent object representing by data, sid=sid, randval=randval, hmac_digest=hmac_digest
the name for this object.
"""
attr = getattr(self.obj, name)
if callable(attr):
return SqliteSessionStorage.CallableAttributeProxy(
self.session, self.key, self.obj, attr
) )
return attr
def setdefault(self, key, value): def put(self, session):
"""Sets the default value for the particular key in the session. 'Store a managed session'
""" if not session.hmac_digest:
session.sign(self.secret)
if key not in self: fname = os.path.join(self.path, session.sid)
self[key] = value with open(fname, 'w') as f:
self.modified = True dump(
(session.randval, session.hmac_digest, dict(session)),
return SqliteSessionStorage.PersistedObjectProxy( f
self, key, self[key]
) )
class ServerSideSessionInterface(SessionInterface): class ManagedSessionInterface(SessionInterface):
""" def __init__(self, manager, skip_paths, cookie_timedelta):
Implements the SessionInterface to support saving/opening session self.manager = manager
as sqlite object. self.skip_paths = skip_paths
""" self.cookie_timedelta = cookie_timedelta
def __init__(self, directory): def get_expiration_time(self, app, session):
"""Initialize the session interface, which uses the sqlite as local if session.permanent:
storage, and works as server side session manager. return app.permanent_session_lifetime
return datetime.datetime.now() + self.cookie_timedelta
It takes directory as parameter, and creates the directory with 700
permission (if not exists).
"""
directory = os.path.abspath(directory)
if not os.path.exists(directory):
os.makedirs(directory, int('700', 8))
self.directory = directory
def open_session(self, app, request): def open_session(self, app, request):
""" cookie_val = request.cookies.get(app.session_cookie_name)
Returns the SqliteSessionStorage object representing this session.
""" if not cookie_val or not '!' in cookie_val:
sid = request.cookies.get(app.session_cookie_name) # Don't bother creating a cookie for static resources
if not sid or len(sid) > 40: for sp in self.skip_paths:
sid = str(uuid4()) if request.path.startswith(sp):
return SqliteSessionStorage(self.directory, sid) return None
return self.manager.new_session()
sid, digest = cookie_val.split('!', 1)
if self.manager.exists(sid):
return self.manager.get(sid, digest)
return self.manager.new_session()
def save_session(self, app, session, response): def save_session(self, app, session, response):
"""
Saves/Detroys the session object.
"""
sid = request.cookies.get(app.session_cookie_name)
domain = self.get_cookie_domain(app) domain = self.get_cookie_domain(app)
if not session: if not session:
try: self.manager.remove(session.sid)
if session is None:
session = SqliteSessionStorage(self.directory, sid)
os.unlink(session.path)
except OSError as ex:
if ex.errno != errno.ENOENT:
raise
if session.modified: if session.modified:
response.delete_cookie( response.delete_cookie(app.session_cookie_name, domain=domain)
app.session_cookie_name,
domain=domain
)
return return
if not session.modified:
# No need to save an unaltered session
# TODO: put logic here to test if the cookie is older than N days,
# if so, update the expiration date
return
self.manager.put(session)
session.modified = False
cookie_exp = self.get_expiration_time(app, session) cookie_exp = self.get_expiration_time(app, session)
response.set_cookie( response.set_cookie(app.session_cookie_name,
app.session_cookie_name, session.sid, '%s!%s' % (session.sid, session.hmac_digest),
expires=cookie_exp, httponly=True, domain=domain expires=cookie_exp, httponly=True, domain=domain)
)
def create_session_interface(app, skip_paths=[]):
return ManagedSessionInterface(
CachingSessionManager(
FileBackedSessionManager(
app.config['SESSION_DB_PATH'],
app.config['SECRET_KEY']
),
1000
), skip_paths,
datetime.timedelta(days=1))