Handle opening of non-UTF8 compatible files. Fixes #3129

web/pgadmin/tools/sqleditor/__init__.py

@@ -38,6 +38,8 @@ from pgadmin.utils.exception import ConnectionLost
 from pgadmin.utils.sqlautocomplete.autocomplete import SQLAutoComplete
 from pgadmin.tools.sqleditor.utils.query_tool_preferences import \
     RegisterQueryToolPreferences
+from pgadmin.tools.sqleditor.utils.query_tool_fs_utils import \
+    read_file_generator
 
 MODULE_NAME = 'sqleditor'
 
@@ -1360,16 +1362,7 @@ def load_file():
             errormsg=gettext("File type not supported")
         )
 
-    def gen():
-        with codecs.open(file_path, 'r', encoding=enc) as fileObj:
-            while True:
-                # 4MB chunk (4 * 1024 * 1024 Bytes)
-                data = fileObj.read(4194304)
-                if not data:
-                    break
-                yield data
-
-    return Response(gen(), mimetype='text/plain')
+    return Response(read_file_generator(file_path, enc), mimetype='text/plain')
 
 
 @blueprint.route('/save_file/', methods=["PUT", "POST"], endpoint='save_file')

web/pgadmin/tools/sqleditor/utils/query_tool_fs_utils.py

@@ -0,0 +1,53 @@
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2018, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+import codecs
+
+
+def read_file_generator(file, enc):
+    """
+    This will read the content of the file selected by user
+
+    Returns:
+        Content of file
+    """
+    try:
+        with codecs.open(file, 'r', encoding=enc) as fileObj:
+            while True:
+                # 4MB chunk (4 * 1024 * 1024 Bytes)
+                data = fileObj.read(4194304)
+                if not data:
+                    break
+                yield data
+    except UnicodeDecodeError:
+        # This is the closest equivalent Python 3 offers to the permissive
+        # Python 2 text handling model. The latin-1 encoding in Python
+        # implements ISO_8859-1:1987 which maps all possible byte values
+        # to the first 256 Unicode code points, and thus ensures decoding
+        # errors will never occur regardless of the configured error and
+        # handles most of the Windows encodings
+        # handler.
+        # Ref: https://goo.gl/vDhggS
+        with codecs.open(file, 'r', encoding='latin-1') as fileObj:
+            while True:
+                # 4MB chunk (4 * 1024 * 1024 Bytes)
+                data = fileObj.read(4194304)
+                if not data:
+                    break
+                yield data
+    except Exception:
+        # As a last resort we will use the provided encoding and then
+        # ignore the decoding errors
+        with codecs.open(file, 'r', encoding=enc, errors='ignore') as fileObj:
+            while True:
+                # 4MB chunk (4 * 1024 * 1024 Bytes)
+                data = fileObj.read(4194304)
+                if not data:
+                    break
+                yield data

web/pgadmin/tools/sqleditor/utils/tests/test_file_other_encoding.sql

@@ -0,0 +1,2 @@
+/*Copyright <20> 2017/*
+SELECT 1;

web/pgadmin/tools/sqleditor/utils/tests/test_file_utf8_encoding.sql

@@ -0,0 +1,2 @@
+/*Copyright © 2017/*
+SELECT 1;

web/pgadmin/tools/sqleditor/utils/tests/test_query_tool_fs_utils.py

@@ -0,0 +1,45 @@
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2018, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+import os
+from pgadmin.utils.route import BaseTestGenerator
+from pgadmin.tools.sqleditor.utils.query_tool_fs_utils import \
+    read_file_generator
+
+
+class TestReadFileGeneratorForEncoding(BaseTestGenerator):
+    """
+    Check that the start_running_query method works as intended
+    """
+
+    scenarios = [
+        (
+            'When user is trying to load the file with utf-8 encoding',
+            dict(
+                file='test_file_utf8_encoding.sql',
+                encoding='utf-8'
+            )
+        ),
+        (
+            'When user is trying to load the file with other encoding and'
+            ' trying to use utf-8 encoding to read it',
+            dict(
+                file='test_file_other_encoding.sql',
+                encoding='utf-8'
+            )
+        ),
+    ]
+
+    def setUp(self):
+        self.dir_path = os.path.dirname(os.path.realpath(__file__))
+        self.complate_path = os.path.join(self.dir_path, self.file)
+
+    def runTest(self):
+        result = read_file_generator(self.complate_path, self.encoding)
+        # Check if file is read properly by the generator
+        self.assertIn('SELECT 1', next(result))