Handle errors occurring during decoding UTF-8 encoded query result data which contains ascii characters. #4784

This commit is contained in:
Yogesh Mahajan 2023-03-02 11:03:17 +05:30 committed by GitHub
parent 5abb748c3e
commit 740164fbd0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -164,14 +164,9 @@ def register_string_typecasters(connection):
# characters. Here we unescape them using unicode_escape
# and send ahead. When insert update is done, the characters
# are escaped again and sent to the DB.
postgres_encoding, python_encoding = \
get_encoding(connection.info.encoding)
if postgres_encoding != 'UTF-8' and postgres_encoding != 'UTF8':
for typ in (19, 18, 25, 1042, 1043, 0):
if connection:
connection.adapters.register_loader(typ, TextLoaderpgAdmin)
for typ in (19, 18, 25, 1042, 1043, 0):
if connection:
connection.adapters.register_loader(typ, TextLoaderpgAdmin)
def register_binary_typecasters(connection):
@ -222,10 +217,14 @@ class TextLoaderpgAdmin(TextLoader):
postgres_encoding, python_encoding = get_encoding(
self.connection.info.encoding)
if postgres_encoding not in ['SQLASCII', 'SQL_ASCII']:
# In case of errors while decoding data, instead of raising error
# replace errors with empty space.
# Error - utf-8 code'c can not decode byte 0x7f:
# invalid continuation byte
if isinstance(data, memoryview):
return bytes(data).decode(self._encoding)
return bytes(data).decode(self._encoding, errors='replace')
else:
return data.decode(self._encoding)
return data.decode(self._encoding, errors='replace')
else:
# SQL_ASCII Database
try: