Fix various encoding issues with multibyte paths and filenames.

Fixes #1986. Fixes #1940.

web/pgadmin/misc/file_manager/__init__.py

@@ -403,6 +403,9 @@ class Filemanager(object):
         It lists all file and folders within the given
         directory.
         """
+        path = unquote(path)
+        if hasattr(str, 'decode'):
+            path = unquote(path).encode('utf-8').decode('utf-8')
         files = {}
         if (_platform == "win32" and path == '/') and dir is None:
             drives = Filemanager._get_drives()
@@ -430,7 +433,7 @@ class Filemanager(object):
 
         if dir is None:
             dir = ""
-        orig_path = "{0}{1}".format(dir, path)
+        orig_path = u"{0}{1}".format(dir, path)
         user_dir = path
         folders_only = trans_data['folders_only'] if 'folders_only' in \
                                                      trans_data else ''
@@ -450,7 +453,10 @@ class Filemanager(object):
                 if (is_folder_hidden(system_path) or f.startswith('.')):
                     continue
 
-                user_path = os.path.join(os.path.join(user_dir, f))
+                if hasattr(str, 'decode'):
+                    user_path = os.path.join(os.path.join(user_dir, f)).encode('utf-8')
+                else:
+                    user_path = os.path.join(os.path.join(user_dir, f))
                 created = time.ctime(os.path.getctime(system_path))
                 modified = time.ctime(os.path.getmtime(system_path))
                 file_extension = str(splitext(system_path))
@@ -511,11 +517,12 @@ class Filemanager(object):
         Returns a JSON object containing information
         about the given file.
         """
-
         path = unquote(path)
+        if hasattr(str, 'decode'):
+            path = unquote(path).encode('utf-8').decode('utf-8')
         if self.dir is None:
             self.dir = ""
-        orig_path = "{0}{1}".format(self.dir, path)
+        orig_path = u"{0}{1}".format(self.dir, path)
         user_dir = path
         thefile = {
             'Filename': split_path(orig_path)[-1],
@@ -576,7 +583,9 @@ class Filemanager(object):
 
         # extract filename
         oldname = split_path(old)[-1]
-        path = str(old)
+        if hasattr(str, 'decode'):
+            old = old.encode('utf-8').decode('utf-8')
+        path = old
         path = split_path(path)[0]  # extract path
 
         if not path[-1] == '/':
@@ -584,11 +593,13 @@ class Filemanager(object):
 
         # newname = encode_urlpath(new)
         newname = new
+        if hasattr(str, 'decode'):
+            newname = new.encode('utf-8').decode('utf-8')
         newpath = path + newname
 
         # make system old path
-        oldpath_sys = "{0}{1}".format(dir, old)
+        oldpath_sys = u"{0}{1}".format(dir, old)
-        newpath_sys = "{0}{1}".format(dir, newpath)
+        newpath_sys = u"{0}{1}".format(dir, newpath)
 
         error_msg = gettext('Renamed successfully.')
         code = 1
@@ -622,7 +633,8 @@ class Filemanager(object):
             }
 
         dir = self.dir if self.dir is not None else ''
-        orig_path = "{0}{1}".format(dir, path)
+        path = path.encode('utf-8').decode('utf-8') if hasattr(str, 'decode') else path
+        orig_path = u"{0}{1}".format(dir, path)
 
         err_msg = ''
         code = 1
@@ -659,12 +671,17 @@ class Filemanager(object):
         code = 1
         try:
             path = req.form.get('currentpath')
-            orig_path = "{0}{1}".format(dir, path)
+
-            thefile = req.files['newfile']
+            file_obj = req.files['newfile']
-            newName = '{0}{1}'.format(orig_path, thefile.filename)
+            file_name = file_obj.filename
+            if hasattr(str, 'decode'):
+                path = req.form.get('currentpath').encode('utf-8').decode('utf-8')
+                file_name = file_obj.filename.encode('utf-8').decode('utf-8')
+            orig_path = u"{0}{1}".format(dir, path)
+            newName = u'{0}{1}'.format(orig_path, file_name)
 
             with open(newName, 'wb') as f:
-                f.write(thefile.read())
+                f.write(file_obj.read())
             code = 0
         except Exception as e:
             err_msg = "Error: {0}".format(e.strerror)
@@ -684,7 +701,12 @@ class Filemanager(object):
         dir = self.dir if self.dir is not None else ''
         err_msg = ''
         code = 1
+
         name = unquote(name)
+        path = unquote(path)
+        if hasattr(str, 'decode'):
+            name = unquote(name).encode('utf-8')
+            path = unquote(path).encode('utf-8')
         try:
             orig_path = "{0}{1}".format(dir, path)
             newName = '{0}{1}'.format(orig_path, name)
@@ -735,10 +757,19 @@ class Filemanager(object):
 
         dir = self.dir if self.dir is not None else ''
         newName = name
+        if hasattr(str, 'decode'):
+            newName = name.encode('utf-8')
+
         if dir != "":
-            newPath = dir + '/' + path + newName + '/'
+            if hasattr(str, 'decode'):
+                newPath = dir + '/' + path + newName.decode('utf-8') + '/'
+            else:
+                newPath = dir + '/' + path + newName + '/'
         else:
-            newPath = path + newName + '/'
+            if hasattr(str, 'decode'):
+                newPath = path + newName.decode('utf-8') + '/'
+            else:
+                newPath = path + newName + '/'
 
         err_msg = ''
         code = 1
@@ -776,9 +807,13 @@ class Filemanager(object):
             }
 
         dir = self.dir if self.dir is not None else ''
-        orig_path = "{0}{1}".format(dir, path)
+        if hasattr(str, 'decode'):
+            path = path.encode('utf-8')
+            orig_path = u"{0}{1}".format(dir, path.decode('utf-8'))
+        else:
+            orig_path = "{0}{1}".format(dir, path)
         name = path.split('/')[-1]
-        content = open(orig_path, 'r')
+        content = open(orig_path, 'rb')
         resp = Response(content)
         resp.headers['Content-Disposition'] = 'attachment; filename=' + name
         return resp

web/pgadmin/misc/file_manager/templates/file_manager/js/utility.js

@@ -78,47 +78,14 @@ var preg_replace = function(array_pattern, array_pattern_replace, str)  {
   return new_str;
 };
 
-/*
+ //nameFormat (), separate filename from extension
- * cleanString (), on the same model as server side (connector)
- * cleanString
- */
-var cleanString = function(str) {
-  var cleaned = "";
-  var p_search = new Array(
-    "Š", "š", "Đ", "đ", "Ž", "ž", "Č", "č", "Ć", "ć", "À",
-    "Á", "Â", "Ã", "Ä", "Å", "Æ", "Ç", "È", "É", "Ê", "Ë", "Ì", "Í", "Î", "Ï",
-    "Ñ", "Ò", "Ó", "Ô", "Õ", "Ö", "Ő", "Ø", "Ù", "Ú", "Û", "Ü", "Ý", "Þ", "ß",
-    "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì",  "í",
-    "î", "ï", "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "ő", "ø", "ù", "ú", "û", "ü",
-    "ý", "ý", "þ", "ÿ", "Ŕ", "ŕ", " ", "'", "/"
-  );
-  var p_replace = new Array(
-    "S", "s", "Dj", "dj", "Z", "z", "C", "c", "C", "c", "A",
-    "A", "A", "A", "A", "A", "A", "C", "E", "E", "E", "E", "I", "I", "I", "I",
-    "N", "O", "O", "O", "O", "O", "O", "O", "U", "U", "U", "U", "Y", "B", "Ss",
-    "a", "a", "a", "a", "a", "a", "a", "c", "e", "e", "e", "e", "i", "i",
-    "i", "i", "o", "n", "o", "o", "o", "o", "o", "o", "o", "u", "u", "u", "u",
-    "y", "y", "b", "y", "R", "r", "_", "_", ""
-  );
-
-  cleaned = preg_replace(p_search, p_replace, str);
-  cleaned = cleaned.replace(/[^_a-zA-Z0-9]/g, "");
-  cleaned = cleaned.replace(/[_]+/g, "_");
-
-  return cleaned;
-};
-
-/*
- * nameFormat (), separate filename from extension before calling cleanString()
- * nameFormat
- */
 var nameFormat = function(input) {
   var filename = '';
   if (input.lastIndexOf('.') != -1) {
-    filename  = cleanString(input.substr(0, input.lastIndexOf('.')));
+    filename  = input.substr(0, input.lastIndexOf('.'));
     filename += '.' + input.split('.').pop();
   } else {
-    filename = cleanString(input);
+    filename = input;
   }
   return filename;
 };
@@ -316,7 +283,7 @@ var setUploader = function(path) {
       var fname = value;
 
       if (fname != '') {
-        foldername = cleanString(fname);
+        foldername = fname;
         var d = new Date(); // to prevent IE cache issues
         $.getJSON(fileConnector + '?mode=addfolder&path=' + $('.currentpath').val() + '&name=' + foldername, function(resp) {
           var result = resp.data.result;
@@ -366,7 +333,7 @@ var bindToolbar = function(data) {
     $('.fileinfo .delete_item button.btn_yes').unbind().on('click', function() {
       var path;
       if ($('.fileinfo').data('view') == 'grid') {
-        path = $('.fileinfo').find('#contents li.selected .clip span').attr('data-alt');
+        path = decodeURI($('.fileinfo').find('#contents li.selected .clip span').attr('data-alt'));
         if (path.lastIndexOf('/') == path.length - 1) {
           data.Path = path;
           deleteItem(data);
@@ -396,10 +363,10 @@ var bindToolbar = function(data) {
       var path;
       if ($('.fileinfo').data('view') == 'grid') {
         path = $('.fileinfo li.selected').find('.clip span').attr('data-alt');
-        window.open(fileConnector + '?mode=download&path=' + encodeURIComponent(path), '_blank');
+        window.open(fileConnector + '?mode=download&path=' + path, '_blank');
       } else {
         path = $('.fileinfo').find('table#contents tbody tr.selected td:first-child').attr('title');
-        window.open(fileConnector + '?mode=download&path=' + encodeURIComponent(path), '_blank');
+        window.open(fileConnector + '?mode=download&path=' + path, '_blank');
       }
     });
   }
@@ -1101,7 +1068,7 @@ var getFolderInfo = function(path, file_type) {
               config.options.dialog_type == 'create_file' &&
               is_protected == undefined
             ) {
-              $('.create_input input[type="text"]').val(file_name);
+              $('.create_input input[type="text"]').val(decodeURI(file_name));
               $('.file_manager_ok, .file_manager_create').removeClass('disabled');
             }
             getFileInfo(path);

web/pgadmin/tools/sqleditor/__init__.py

@@ -1201,11 +1201,16 @@ def load_file():
         file_data = json.loads(request.data, encoding='utf-8')
 
     file_path = unquote(file_data['file_name'])
+    if hasattr(str, 'decode'):
+        file_path = unquote(file_data['file_name']).encode('utf-8').decode('utf-8')
     # retrieve storage directory path
     storage_manager_path = get_storage_directory()
     if storage_manager_path:
         # generate full path of file
-        file_path = os.path.join(storage_manager_path, file_path.lstrip('/'))
+        file_path = os.path.join(
+            storage_manager_path,
+            file_path.lstrip('/')
+        )
 
     file_data = None
 
@@ -1224,7 +1229,10 @@ def load_file():
             is_binary = is_binary_string(fileObj.read(1024))
             if not is_binary:
                 fileObj.seek(0)
-                file_data = fileObj.read()
+                if hasattr(str, 'decode'):
+                    file_data = fileObj.read().decode('utf-8')
+                else:
+                    file_data = fileObj.read()
             else:
                 return internal_server_error(
                     errormsg=gettext("File type not supported")
@@ -1262,17 +1270,28 @@ def save_file():
 
     # generate full path of file
     file_path = unquote(file_data['file_name'])
+    if hasattr(str, 'decode'):
+        file_path = unquote(
+            file_data['file_name']
+        ).encode('utf-8').decode('utf-8')
     if storage_manager_path is not None:
         file_path = os.path.join(
             storage_manager_path,
-            unquote(file_data['file_name'].lstrip('/'))
+            file_path.lstrip('/')
         )
-    file_content = file_data['file_content']
+
+    if hasattr(str, 'decode'):
+        file_content = file_data['file_content']
+    else:
+        file_content = file_data['file_content'].encode()
 
     # write to file
     try:
-        with open(file_path, 'w') as output_file:
+        with open(file_path, 'wb') as output_file:
-            output_file.write(file_content)
+            if hasattr(str, 'decode'):
+                output_file.write(file_content.encode('utf-8'))
+            else:
+                output_file.write(file_content)
     except IOError as e:
         if e.strerror == 'Permission denied':
             err_msg = "Error: {0}".format(e.strerror)

web/pgadmin/tools/sqleditor/templates/sqleditor/js/sqleditor.js

@@ -2361,7 +2361,7 @@ define(
         setTitle: function(title) {
           _.each(window.top.pgAdmin.Browser.docker.findPanels('frm_datagrid'), function(p) {
             if(p.isVisible()) {
-              p.title(title);
+              p.title(decodeURIComponent(title));
             }
           });
         },
@@ -2401,7 +2401,7 @@ define(
         _select_file_handler: function(e) {
           var self = this,
               data = {
-                'file_name': e
+                'file_name': decodeURI(e)
               };
 
           self.trigger(
@@ -2442,7 +2442,7 @@ define(
         _save_file_handler: function(e) {
           var self = this;
           data = {
-            'file_name': e,
+            'file_name': decodeURI(e),
             'file_content': self.gridView.query_tool_obj.getValue()
           }
           self.trigger(