Resolved quite a few file-system encoding/decoding related cases.

In order to resolve the non-ascii characters in path (in user directory,
storage path, etc) on windows, we have converted the path into the
short-path, so that - we don't need to deal with the encoding issues
(specially with Python 2).

We've resolved majority of the issues with this patch.
We still need couple issues to resolve after this in the same area.

TODO
* Add better support for non-ascii characters in the database name on
  windows with Python 3
* Improve the messages created after the background processes by
  different modules (such as Backup, Restore, Import/Export, etc.),
  which does not show short-paths, and xml representable characters for
  non-ascii characters, when found in the database objects, and the file
  PATH.

Fixes #2174, #1797, #2166, #1940

Initial patch by: Surinder Kumar
Reviewed by: Murtuza Zabuawala

web/pgadmin/utils/__init__.py

@@ -131,3 +131,105 @@ class PgAdminModule(Blueprint):
         menu_items = dict((key, sorted(value, key=attrgetter('priority')))
                           for key, value in menu_items.items())
         return menu_items
+
+
+import os
+import sys
+
+IS_PY2 = (sys.version_info[0] == 2)
+IS_WIN = (os.name == 'nt')
+
+sys_encoding = sys.getdefaultencoding()
+if not sys_encoding or sys_encoding == 'ascii':
+    # Fall back to 'utf-8', if we couldn't determine the default encoding,
+    # or 'ascii'.
+    sys_encoding = 'utf-8'
+
+fs_encoding = sys.getfilesystemencoding()
+if not fs_encoding or fs_encoding == 'ascii':
+    # Fall back to 'utf-8', if we couldn't determine the file-system encoding,
+    # or 'ascii'.
+    fs_encoding = 'utf-8'
+
+
+def u(_s, _encoding=sys_encoding):
+    if IS_PY2:
+        if isinstance(_s, str):
+            return unicode(_s, _encoding)
+    return _s
+
+
+def file_quote(_p):
+    if IS_PY2:
+        if isinstance(_p, unicode):
+            return _p.encode(fs_encoding)
+    return _p
+
+
+if IS_WIN:
+    import ctypes
+    from ctypes import wintypes
+
+    if IS_PY2:
+        def env(name):
+            if IS_PY2:
+                # Make sure string argument is unicode
+                name = unicode(name)
+            n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0)
+
+            if n == 0:
+                return None
+
+            buf= ctypes.create_unicode_buffer(u'\0'*n)
+            ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n)
+
+            return buf.value
+    else:
+        def env(name):
+            if name in os.environ:
+                return os.environ[name]
+            return None
+
+    _GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW
+    _GetShortPathNameW.argtypes = [
+        wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD
+    ]
+    _GetShortPathNameW.restype = wintypes.DWORD
+
+    def fs_short_path(_path):
+        """
+        Gets the short path name of a given long path.
+        http://stackoverflow.com/a/23598461/200291
+        """
+        buf_size = len(_path)
+        while True:
+            res = ctypes.create_unicode_buffer(buf_size)
+            needed = _GetShortPathNameW(_path, res, buf_size)
+
+            if buf_size >= needed:
+                return res.value
+            else:
+                buf_size += needed
+
+    def document_dir():
+        CSIDL_PERSONAL = 5  # My Documents
+        SHGFP_TYPE_CURRENT = 0  # Get current, not default value
+
+        buf = ctypes.create_unicode_buffer(wintypes.MAX_PATH)
+        ctypes.windll.shell32.SHGetFolderPathW(
+            None, CSIDL_PERSONAL, None, SHGFP_TYPE_CURRENT, buf
+        )
+
+        return buf.value
+
+else:
+    def env(name):
+        if name in os.environ:
+            return os.environ[name]
+        return None
+
+    def fs_short_path(_path):
+        return _path
+
+    def document_dir():
+        return os.path.realpath(os.path.expanduser(u'~/'))

web/pgadmin/utils/driver/psycopg2/__init__.py

@@ -1491,8 +1491,6 @@ class ServerManager(object):
                 database = self.db
             elif did in self.db_info:
                 database = self.db_info[did]['datname']
-                if hasattr(str, 'decode'):
-                    database = database.decode('utf-8')
             else:
                 maintenance_db_id = u'DB:{0}'.format(self.db)
                 if maintenance_db_id in self.connections:
@@ -1510,9 +1508,6 @@ WHERE db.oid = {0}""".format(did))
                         if status and len(res['rows']) > 0:
                             for row in res['rows']:
                                 self.db_info[did] = row
-                                if hasattr(str, 'decode'):
-                                    self.db_info[did]['datname'] = \
-                                        self.db_info[did]['datname'].decode('utf-8')
                                 database = self.db_info[did]['datname']
 
                         if did not in self.db_info:
@@ -1843,7 +1838,10 @@ class Driver(BaseDriver):
         # Returns in bytes, we need to convert it in string
         if isinstance(res, bytes):
             try:
-                res = res.decode()
+                try:
+                    res = res.decode()
+                except UnicodeDecodeError:
+                    res = res.decode(sys.getfilesystemencoding())
             except UnicodeDecodeError:
                 res = res.decode('utf-8')
 

web/pgadmin/utils/html.py

@@ -10,9 +10,14 @@
 """Utilities for HTML"""
 
 import cgi
+from pgadmin.utils import IS_PY2
 
 
 def safe_str(x):
-    return cgi.escape(x).encode(
-        'ascii', 'xmlcharrefreplace'
-    ).decode()
+    try:
+        x = x.encode('ascii', 'xmlcharrefreplace') if hasattr(x, 'encode') else x
+        if not IS_PY2:
+            x = x.decode('utf-8')
+    except:
+        pass
+    return cgi.escape(x)

web/pgadmin/utils/paths.py

@@ -37,7 +37,11 @@ def get_storage_directory():
     if len(username) == 0 or username[0].isdigit():
         username = 'pga_user_' + username
 
-    storage_dir = os.path.join(storage_dir, username)
+    storage_dir = os.path.join(
+        storage_dir.decode('utf-8') if hasattr(storage_dir, 'decode') \
+            else storage_dir,
+        username
+    )
 
     if not os.path.exists(storage_dir):
         os.makedirs(storage_dir, int('700', 8))

web/pgadmin/utils/preferences.py

@@ -193,7 +193,7 @@ class _Preference(object):
 
         if pref is None:
             pref = UserPrefTable(
-                uid=current_user.id, pid=self.pid, value=str(value)
+                uid=current_user.id, pid=self.pid, value=value
             )
             db.session.add(pref)
         else:

web/pgadmin/utils/session.py

@@ -143,7 +143,10 @@ class CachingSessionManager(SessionManager):
     def put(self, session):
         self.parent.put(session)
         if session.sid in self._cache:
-            del self._cache[session.sid]
+            try:
+                del self._cache[session.sid]
+            except:
+                pass
         self._cache[session.sid] = session
         self._normalize()