diff --git a/web/pgadmin/dashboard/__init__.py b/web/pgadmin/dashboard/__init__.py index e1c5d9251..3f0537e67 100644 --- a/web/pgadmin/dashboard/__init__.py +++ b/web/pgadmin/dashboard/__init__.py @@ -15,6 +15,9 @@ from flask import render_template, Response, g, request from flask_babel import gettext from pgadmin.user_login_check import pga_login_required import json + +import psycopg + from pgadmin.utils import PgAdminModule from pgadmin.utils.ajax import make_response as ajax_response,\ internal_server_error, make_json_response, precondition_required @@ -512,6 +515,108 @@ def log_formats(sid=None): ) +def _decode_log_bytes(raw, python_encoding): + """Decode raw log bytes, trimming an incomplete character at the end + and replacing any invalid bytes inside the log file itself.""" + for _ in range(4): + try: + return raw.decode(python_encoding) + except UnicodeDecodeError as e: + if (e.start >= len(raw) - 4 and + e.reason in ('unexpected end of data', + 'incomplete multibyte sequence')): + # The read may have ended mid-character, e.g. while the + # server was writing to the file. + raw = raw[:e.start] + else: + return raw.decode(python_encoding, errors='replace') + + return raw.decode(python_encoding, errors='replace') + + +def decode_log_chunk(data, server_encoding, keep_bytes=None): + """Decode a raw log chunk read with pg_read_binary_file. + + The logs are read in fixed-size byte ranges, so a range boundary can + split a multi-byte character. Reading the raw bytes and decoding + them here avoids the 'invalid byte sequence' error (SQLSTATE 22021) + raised by pg_read_file in that case. When keep_bytes is set, only + characters starting within that many bytes are shown; the caller + reads a few bytes of overlap so a character split at the end of the + range is still shown whole, and a character split at the start is + left to the previous chunk. + """ + if not data: + return '' + + raw = bytes.fromhex(data) + python_encoding = psycopg._encodings.py_codecs.get( + server_encoding.upper().encode(), 'utf-8') + + # A paged chunk can start inside a multi-byte character that the + # previous chunk shows in full; drop the leftover bytes here. Bytes + # that cannot be part of such a character are kept, so they are shown + # as replacement characters instead of disappearing. + # + # Recognising a split character by its trail bytes is exact only for + # UTF-8 (trail bytes are 0x80-0xBF). For the EUC family of server + # encodings, lead and trail bytes share the same ranges, so a + # character split at a page boundary can still produce one garbled + # character. That is a known limitation, accepted in exchange for + # avoiding the 22021 error dialog those servers used to get. + # + # A whole-file read starts at the beginning of the file, so its + # leading bytes are kept and replaced by the decoder below if they + # are malformed. + dropped_lead = 0 + if keep_bytes is not None: + for _ in range(3): + if not raw: + return '' + try: + raw.decode(python_encoding) + break + except UnicodeDecodeError as e: + if e.start: + break + if (python_encoding == 'utf-8' and + not 0x80 <= raw[0] <= 0xBF): + break + raw = raw[1:] + dropped_lead += 1 + + if keep_bytes is None: + return _decode_log_bytes(raw, python_encoding) + + keep = max(keep_bytes - dropped_lead, 0) + head = raw[:keep] + try: + head.decode(python_encoding) + except UnicodeDecodeError as e: + if e.start < len(head) - 4: + # Invalid data within the log file itself. + return head.decode(python_encoding, errors='replace') + # A character is split at the end of the range; the overlap bytes + # may complete it. Stop at the first extension that decodes, so + # nothing beyond the split character is included. + for overlap in range(1, 5): + try: + return raw[:keep + overlap].decode(python_encoding) + except UnicodeDecodeError as e2: + if e2.start < keep - 4: + break + if len(raw) >= keep + 4: + # The full overlap was read, so the bytes are malformed + # rather than an incomplete character; keep the valid text. + return head.decode(python_encoding, errors='replace') + # The read reached the end of the file; trim a character the + # server may still be writing, but keep any text after an + # invalid byte. + return _decode_log_bytes(head, python_encoding) + + return head.decode(python_encoding) + + @blueprint.route('/logs///', endpoint='logs') @blueprint.route('/logs////', endpoint='get_logs_by_server_id') @@ -565,6 +670,8 @@ def logs(log_format=None, disp_format=None, sid=None, page=0): _start = 0 _end = ON_DEMAND_LOG_COUNT + log_text = '' + keep_bytes = None page = int(page) final_cols = [] if page > 0: @@ -574,6 +681,12 @@ def logs(log_format=None, disp_format=None, sid=None, page=0): if _start < file_stat: if disp_format == 'plain': _end = file_stat + keep_bytes = None + else: + # Read a few bytes past the chunk so a multi-byte character + # split at the end of the chunk can be shown whole. + keep_bytes = _end - _start + _end = keep_bytes + 4 sql = render_template( "/".join([g.template_path, 'logs.sql']), st=_start, ed=_end, log_format=log_format, conn=g.conn @@ -582,7 +695,10 @@ def logs(log_format=None, disp_format=None, sid=None, page=0): if not status: return internal_server_error(errormsg=res) - final_res = res['rows'][0]['pg_read_file'].split('\n') + log_row = res['rows'][0] + log_text = decode_log_chunk(log_row['log_data'], + log_row['encoding'], keep_bytes) + final_res = log_text.split('\n') # Json format if log_format == 'jsonlog': for f in final_res: @@ -624,7 +740,7 @@ def logs(log_format=None, disp_format=None, sid=None, page=0): "message": _tmp[1] if len(_tmp) > 1 else ''}) if disp_format == 'plain': - final_response = res['rows'] + final_response = [{'pg_read_file': log_text}] else: final_response = final_cols diff --git a/web/pgadmin/dashboard/templates/dashboard/sql/default/logs.sql b/web/pgadmin/dashboard/templates/dashboard/sql/default/logs.sql index 3502491a7..abec7a2ff 100644 --- a/web/pgadmin/dashboard/templates/dashboard/sql/default/logs.sql +++ b/web/pgadmin/dashboard/templates/dashboard/sql/default/logs.sql @@ -1,8 +1,8 @@ /*pga4dash*/ {% if log_format != '' %} -SELECT pg_read_file(pg_current_logfile('{{log_format}}'), {{ st }}, {{ ed }}); +SELECT encode(pg_read_binary_file(pg_current_logfile('{{log_format}}'), {{ st }}, {{ ed }}), 'hex') AS log_data, current_setting('server_encoding') AS encoding; {% elif st !='' and ed != '' %} -SELECT pg_read_file(pg_current_logfile(), {{ st }}, {{ ed }}); +SELECT encode(pg_read_binary_file(pg_current_logfile(), {{ st }}, {{ ed }}), 'hex') AS log_data, current_setting('server_encoding') AS encoding; {% else %} -SELECT pg_read_file(pg_current_logfile()); +SELECT encode(pg_read_binary_file(pg_current_logfile()), 'hex') AS log_data, current_setting('server_encoding') AS encoding; {% endif %} diff --git a/web/pgadmin/dashboard/tests/test_log_decode.py b/web/pgadmin/dashboard/tests/test_log_decode.py new file mode 100644 index 000000000..1fc5797e5 --- /dev/null +++ b/web/pgadmin/dashboard/tests/test_log_decode.py @@ -0,0 +1,75 @@ +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2026, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## + +""" +Unit tests for decode_log_chunk(), which decodes dashboard log chunks +read as raw bytes with pg_read_binary_file() and transported as hex. +""" + +import unittest + +from pgadmin.utils.route import BaseTestGenerator + + +class _SkipServerSetUpMixin: + """Mixin: skip BaseTestGenerator's Postgres connection — these are + pure logic tests.""" + + def setUp(self): + unittest.TestCase.setUp(self) + + +class DecodeLogChunkTestCase(_SkipServerSetUpMixin, BaseTestGenerator): + """decode_log_chunk() handles characters split across page reads and + malformed bytes without raising.""" + + scenarios = [('default', dict())] + + def runTest(self): + from pgadmin.dashboard import decode_log_chunk + + # A multi-byte character split at the start of a page is shown by + # the previous page; the leftover trail bytes are dropped here. + page = 'é'.encode('utf-8')[1:] + b'INFO' + self.assertEqual( + decode_log_chunk(page.hex(), 'UTF8', keep_bytes=len(page)), + 'INFO') + + # A character split at the end of a page is completed from the + # overlap bytes read past the page. + raw = 'abécd'.encode('utf-8') + self.assertEqual( + decode_log_chunk(raw[:6].hex(), 'UTF8', keep_bytes=3), + 'abé') + + # Reassembling pages of a multi-byte log is lossless. + text = 'line one польность ünïcødé 日本語\nline two\n' + raw = text.encode('utf-8') + page_size = 7 + joined = ''.join( + decode_log_chunk( + raw[p * page_size:p * page_size + page_size + 4].hex(), + 'UTF8', keep_bytes=page_size) + for p in range((len(raw) + page_size - 1) // page_size)) + self.assertEqual(joined, text) + + # Malformed leading bytes on a plain whole-file read are kept and + # shown as replacement characters, not silently dropped. + self.assertEqual( + decode_log_chunk((b'\x80\xbfINFO').hex(), 'UTF8'), + '��INFO') + + # At the end of the file, a character still being written is + # trimmed, but valid text after an invalid byte is kept. + self.assertEqual( + decode_log_chunk((b'ab\xd1').hex(), 'UTF8', keep_bytes=3), + 'ab') + self.assertEqual( + decode_log_chunk((b'ab\xffcd').hex(), 'UTF8', keep_bytes=5), + 'ab\ufffdcd')