Fix the Dashboard Logs tab failing on multi-byte characters split across read chunks. #10511 (#10526)

The Logs tab reads the current log file in fixed-size byte ranges with
pg_read_file(), which raises 'invalid byte sequence' (SQLSTATE 22021)
when a range boundary falls inside a multi-byte character, so Tabular
format failed on logs containing non-ASCII text. The ranges are now read
with pg_read_binary_file(), transported as hex and decoded in pgAdmin
using the server encoding. Tabular reads fetch a few bytes of overlap so
a character split at a page boundary is shown whole on one page, and
genuinely invalid bytes are shown as replacement characters rather than
raising an error. For the EUC family of server encodings a page boundary
can still garble one character, as lead and trail bytes share ranges.

- Keep valid text after an invalid byte near the end of the file, trimming only a character still being written.
- Treat a truncated character in the CJK codecs as incomplete rather than invalid.
- Correct the encoding limitation comment, as GBK and GB18030 are not server encodings.

web/pgadmin/dashboard/__init__.py

@@ -15,6 +15,9 @@ from flask import render_template, Response, g, request
 from flask_babel import gettext
 from pgadmin.user_login_check import pga_login_required
 import json
+
+import psycopg
+
 from pgadmin.utils import PgAdminModule
 from pgadmin.utils.ajax import make_response as ajax_response,\
     internal_server_error, make_json_response, precondition_required
@@ -512,6 +515,108 @@ def log_formats(sid=None):
     )
 
 
+def _decode_log_bytes(raw, python_encoding):
+    """Decode raw log bytes, trimming an incomplete character at the end
+    and replacing any invalid bytes inside the log file itself."""
+    for _ in range(4):
+        try:
+            return raw.decode(python_encoding)
+        except UnicodeDecodeError as e:
+            if (e.start >= len(raw) - 4 and
+                    e.reason in ('unexpected end of data',
+                                 'incomplete multibyte sequence')):
+                # The read may have ended mid-character, e.g. while the
+                # server was writing to the file.
+                raw = raw[:e.start]
+            else:
+                return raw.decode(python_encoding, errors='replace')
+
+    return raw.decode(python_encoding, errors='replace')
+
+
+def decode_log_chunk(data, server_encoding, keep_bytes=None):
+    """Decode a raw log chunk read with pg_read_binary_file.
+
+    The logs are read in fixed-size byte ranges, so a range boundary can
+    split a multi-byte character.  Reading the raw bytes and decoding
+    them here avoids the 'invalid byte sequence' error (SQLSTATE 22021)
+    raised by pg_read_file in that case.  When keep_bytes is set, only
+    characters starting within that many bytes are shown; the caller
+    reads a few bytes of overlap so a character split at the end of the
+    range is still shown whole, and a character split at the start is
+    left to the previous chunk.
+    """
+    if not data:
+        return ''
+
+    raw = bytes.fromhex(data)
+    python_encoding = psycopg._encodings.py_codecs.get(
+        server_encoding.upper().encode(), 'utf-8')
+
+    # A paged chunk can start inside a multi-byte character that the
+    # previous chunk shows in full; drop the leftover bytes here.  Bytes
+    # that cannot be part of such a character are kept, so they are shown
+    # as replacement characters instead of disappearing.
+    #
+    # Recognising a split character by its trail bytes is exact only for
+    # UTF-8 (trail bytes are 0x80-0xBF).  For the EUC family of server
+    # encodings, lead and trail bytes share the same ranges, so a
+    # character split at a page boundary can still produce one garbled
+    # character.  That is a known limitation, accepted in exchange for
+    # avoiding the 22021 error dialog those servers used to get.
+    #
+    # A whole-file read starts at the beginning of the file, so its
+    # leading bytes are kept and replaced by the decoder below if they
+    # are malformed.
+    dropped_lead = 0
+    if keep_bytes is not None:
+        for _ in range(3):
+            if not raw:
+                return ''
+            try:
+                raw.decode(python_encoding)
+                break
+            except UnicodeDecodeError as e:
+                if e.start:
+                    break
+                if (python_encoding == 'utf-8' and
+                        not 0x80 <= raw[0] <= 0xBF):
+                    break
+                raw = raw[1:]
+                dropped_lead += 1
+
+    if keep_bytes is None:
+        return _decode_log_bytes(raw, python_encoding)
+
+    keep = max(keep_bytes - dropped_lead, 0)
+    head = raw[:keep]
+    try:
+        head.decode(python_encoding)
+    except UnicodeDecodeError as e:
+        if e.start < len(head) - 4:
+            # Invalid data within the log file itself.
+            return head.decode(python_encoding, errors='replace')
+        # A character is split at the end of the range; the overlap bytes
+        # may complete it.  Stop at the first extension that decodes, so
+        # nothing beyond the split character is included.
+        for overlap in range(1, 5):
+            try:
+                return raw[:keep + overlap].decode(python_encoding)
+            except UnicodeDecodeError as e2:
+                if e2.start < keep - 4:
+                    break
+        if len(raw) >= keep + 4:
+            # The full overlap was read, so the bytes are malformed
+            # rather than an incomplete character; keep the valid text.
+            return head.decode(python_encoding, errors='replace')
+        # The read reached the end of the file; trim a character the
+        # server may still be writing, but keep any text after an
+        # invalid byte.
+        return _decode_log_bytes(head, python_encoding)
+
+    return head.decode(python_encoding)
+
+
 @blueprint.route('/logs/<log_format>/<disp_format>/<int:sid>', endpoint='logs')
 @blueprint.route('/logs/<log_format>/<disp_format>/<int:sid>/<int:page>',
                  endpoint='get_logs_by_server_id')
@@ -565,6 +670,8 @@ def logs(log_format=None, disp_format=None, sid=None, page=0):
 
     _start = 0
     _end = ON_DEMAND_LOG_COUNT
+    log_text = ''
+    keep_bytes = None
     page = int(page)
     final_cols = []
     if page > 0:
@@ -574,6 +681,12 @@ def logs(log_format=None, disp_format=None, sid=None, page=0):
     if _start < file_stat:
         if disp_format == 'plain':
             _end = file_stat
+            keep_bytes = None
+        else:
+            # Read a few bytes past the chunk so a multi-byte character
+            # split at the end of the chunk can be shown whole.
+            keep_bytes = _end - _start
+            _end = keep_bytes + 4
         sql = render_template(
             "/".join([g.template_path, 'logs.sql']), st=_start, ed=_end,
             log_format=log_format, conn=g.conn
@@ -582,7 +695,10 @@ def logs(log_format=None, disp_format=None, sid=None, page=0):
         if not status:
             return internal_server_error(errormsg=res)
 
-        final_res = res['rows'][0]['pg_read_file'].split('\n')
+        log_row = res['rows'][0]
+        log_text = decode_log_chunk(log_row['log_data'],
+                                    log_row['encoding'], keep_bytes)
+        final_res = log_text.split('\n')
         # Json format
         if log_format == 'jsonlog':
             for f in final_res:
@@ -624,7 +740,7 @@ def logs(log_format=None, disp_format=None, sid=None, page=0):
                         "message": _tmp[1] if len(_tmp) > 1 else ''})
 
     if disp_format == 'plain':
-        final_response = res['rows']
+        final_response = [{'pg_read_file': log_text}]
     else:
         final_response = final_cols
 

web/pgadmin/dashboard/templates/dashboard/sql/default/logs.sql

@@ -1,8 +1,8 @@
 /*pga4dash*/
 {% if log_format != '' %}
-SELECT pg_read_file(pg_current_logfile('{{log_format}}'), {{ st }}, {{ ed }});
+SELECT encode(pg_read_binary_file(pg_current_logfile('{{log_format}}'), {{ st }}, {{ ed }}), 'hex') AS log_data, current_setting('server_encoding') AS encoding;
 {% elif st !='' and ed != '' %}
-SELECT pg_read_file(pg_current_logfile(), {{ st }}, {{ ed }});
+SELECT encode(pg_read_binary_file(pg_current_logfile(), {{ st }}, {{ ed }}), 'hex') AS log_data, current_setting('server_encoding') AS encoding;
 {% else %}
-SELECT pg_read_file(pg_current_logfile());
+SELECT encode(pg_read_binary_file(pg_current_logfile()), 'hex') AS log_data, current_setting('server_encoding') AS encoding;
 {% endif %}

web/pgadmin/dashboard/tests/test_log_decode.py

@@ -0,0 +1,75 @@
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2026, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+"""
+Unit tests for decode_log_chunk(), which decodes dashboard log chunks
+read as raw bytes with pg_read_binary_file() and transported as hex.
+"""
+
+import unittest
+
+from pgadmin.utils.route import BaseTestGenerator
+
+
+class _SkipServerSetUpMixin:
+    """Mixin: skip BaseTestGenerator's Postgres connection — these are
+    pure logic tests."""
+
+    def setUp(self):
+        unittest.TestCase.setUp(self)
+
+
+class DecodeLogChunkTestCase(_SkipServerSetUpMixin, BaseTestGenerator):
+    """decode_log_chunk() handles characters split across page reads and
+    malformed bytes without raising."""
+
+    scenarios = [('default', dict())]
+
+    def runTest(self):
+        from pgadmin.dashboard import decode_log_chunk
+
+        # A multi-byte character split at the start of a page is shown by
+        # the previous page; the leftover trail bytes are dropped here.
+        page = 'é'.encode('utf-8')[1:] + b'INFO'
+        self.assertEqual(
+            decode_log_chunk(page.hex(), 'UTF8', keep_bytes=len(page)),
+            'INFO')
+
+        # A character split at the end of a page is completed from the
+        # overlap bytes read past the page.
+        raw = 'abécd'.encode('utf-8')
+        self.assertEqual(
+            decode_log_chunk(raw[:6].hex(), 'UTF8', keep_bytes=3),
+            'abé')
+
+        # Reassembling pages of a multi-byte log is lossless.
+        text = 'line one польность ünïcødé 日本語\nline two\n'
+        raw = text.encode('utf-8')
+        page_size = 7
+        joined = ''.join(
+            decode_log_chunk(
+                raw[p * page_size:p * page_size + page_size + 4].hex(),
+                'UTF8', keep_bytes=page_size)
+            for p in range((len(raw) + page_size - 1) // page_size))
+        self.assertEqual(joined, text)
+
+        # Malformed leading bytes on a plain whole-file read are kept and
+        # shown as replacement characters, not silently dropped.
+        self.assertEqual(
+            decode_log_chunk((b'\x80\xbfINFO').hex(), 'UTF8'),
+            '��INFO')
+
+        # At the end of the file, a character still being written is
+        # trimmed, but valid text after an invalid byte is kept.
+        self.assertEqual(
+            decode_log_chunk((b'ab\xd1').hex(), 'UTF8', keep_bytes=3),
+            'ab')
+        self.assertEqual(
+            decode_log_chunk((b'ab\xffcd').hex(), 'UTF8', keep_bytes=5),
+            'ab\ufffdcd')