Added support for viewing Log Based Clusters. #7216

Co-authored-by: Akshay Joshi <akshay.joshi@enterprisedb.com>
This commit is contained in:
Aditya Toshniwal
2024-03-28 12:19:34 +05:30
committed by GitHub
parent 5931162556
commit ace73ebb60
38 changed files with 1264 additions and 159 deletions

View File

@@ -15,6 +15,7 @@ from flask import render_template, request, make_response, jsonify, \
from flask_babel import gettext
from flask_security import current_user, login_required
from psycopg.conninfo import make_conninfo, conninfo_to_dict
from pgadmin.browser.server_groups.servers.types import ServerType
from pgadmin.browser.utils import PGChildNodeView
from pgadmin.utils.ajax import make_json_response, bad_request, forbidden, \
@@ -30,7 +31,8 @@ from pgadmin.utils.driver import get_driver
from pgadmin.utils.master_password import get_crypt_key
from pgadmin.utils.exception import CryptKeyMissing
from pgadmin.tools.schema_diff.node_registry import SchemaDiffRegistry
from pgadmin.browser.server_groups.servers.utils import is_valid_ipaddress
from pgadmin.browser.server_groups.servers.utils import \
is_valid_ipaddress, get_replication_type
from pgadmin.utils.constants import UNAUTH_REQ, MIMETYPE_APP_JS, \
SERVER_CONNECTION_CLOSED
from sqlalchemy import or_
@@ -343,6 +345,9 @@ class ServerModule(sg.ServerGroupPluginModule):
from .tablespaces import blueprint as module
self.submodules.append(module)
from .replica_nodes import blueprint as module
self.submodules.append(module)
super().register(app, options)
# We do not have any preferences for server node.
@@ -469,7 +474,7 @@ class ServerNode(PGChildNodeView):
}],
'check_pgpass': [{'get': 'check_pgpass'}],
'clear_saved_password': [{'put': 'clear_saved_password'}],
'clear_sshtunnel_password': [{'put': 'clear_sshtunnel_password'}]
'clear_sshtunnel_password': [{'put': 'clear_sshtunnel_password'}],
})
SSL_MODES = ['prefer', 'require', 'verify-ca', 'verify-full']
@@ -1247,6 +1252,7 @@ class ServerNode(PGChildNodeView):
connected = False
user = None
manager = None
replication_type = None
if 'connect_now' in data and data['connect_now']:
manager = get_driver(PG_DEFAULT_DRIVER).connection_manager(
@@ -1324,6 +1330,8 @@ class ServerNode(PGChildNodeView):
server.id),
tunnel_password)
replication_type = get_replication_type(conn,
manager.version)
user = manager.user_info
connected = True
@@ -1337,6 +1345,7 @@ class ServerNode(PGChildNodeView):
username=server.username,
user=user,
connected=connected,
replication_type=replication_type,
shared=server.shared,
server_type=manager.server_type
if manager and manager.server_type
@@ -1427,6 +1436,7 @@ class ServerNode(PGChildNodeView):
in_recovery = None
wal_paused = None
errmsg = None
replication_type = None
if connected:
status, result, in_recovery, wal_paused =\
recovery_state(conn, manager.version)
@@ -1436,10 +1446,13 @@ class ServerNode(PGChildNodeView):
manager.release()
errmsg = "{0} : {1}".format(server.name, result)
replication_type = get_replication_type(conn, manager.version)
return make_json_response(
data={
'icon': server_icon_and_background(connected, manager, server),
'connected': connected,
'replication_type': replication_type,
'in_recovery': in_recovery,
'wal_pause': wal_paused,
'server_type': manager.server_type if connected else "pg",
@@ -1709,6 +1722,8 @@ class ServerNode(PGChildNodeView):
_, _, in_recovery, wal_paused =\
recovery_state(conn, manager.version)
replication_type = get_replication_type(conn, manager.version)
return make_json_response(
success=1,
info=gettext("Server connected."),
@@ -1716,6 +1731,7 @@ class ServerNode(PGChildNodeView):
'icon': server_icon_and_background(True, manager, server),
'connected': True,
'server_type': manager.server_type,
'replication_type': replication_type,
'type': manager.server_type,
'version': manager.version,
'db': manager.db,

View File

@@ -0,0 +1,285 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2024, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
"""Implements Replication Nodes for PG/PPAS 9.4 and above"""
from functools import wraps
from pgadmin.browser.server_groups import servers
from flask import render_template
from flask_babel import gettext
from pgadmin.browser.collection import CollectionNodeModule
from pgadmin.browser.utils import PGChildNodeView
from pgadmin.utils.ajax import make_json_response, \
make_response as ajax_response, internal_server_error, gone
from pgadmin.utils.ajax import precondition_required
from pgadmin.utils.driver import get_driver
from config import PG_DEFAULT_DRIVER
from pgadmin.browser.server_groups.servers.utils import get_replication_type
class ReplicationNodesModule(CollectionNodeModule):
"""
class ReplicationNodesModule(CollectionNodeModule)
A module class for Replication Nodes node derived from
CollectionNodeModule.
Methods:
-------
* __init__(*args, **kwargs)
- Method is used to initialize the ReplicationNodesModule and it's
base module.
* backend_supported(manager, **kwargs)
- This function is used to check the database server type and version.
Replication Nodes only supported in PG/PPAS 9.4 and above.
* get_nodes(gid, sid, did)
- Method is used to generate the browser collection node.
* node_inode()
- Method is overridden from its base class to make the node as leaf node.
* script_load()
- Load the module script for Replication Nodes, when any of the server
node is initialized.
"""
_NODE_TYPE = 'replica_nodes'
_COLLECTION_LABEL = gettext("Replica Nodes")
def __init__(self, *args, **kwargs):
"""
Method is used to initialize the ReplicationNodesModule and
it's base module.
Args:
*args:
**kwargs:
"""
super().__init__(*args, **kwargs)
def get_nodes(self, gid, sid):
"""
Method is used to generate the browser collection node
Args:
gid: Server Group ID
sid: Server ID
"""
yield self.generate_browser_collection_node(sid)
@property
def node_inode(self):
"""
Override this property to make the node as leaf node.
Returns: False as this is the leaf node
"""
return False
@property
def script_load(self):
"""
Load the module script for Replication Nodes, when any of the server
node is initialized.
Returns: node type of the server module.
"""
return servers.ServerModule.NODE_TYPE
def backend_supported(self, manager, **kwargs):
"""
Load this module if replication type exists
"""
if super().backend_supported(manager, **kwargs):
conn = manager.connection(sid=kwargs['sid'])
replication_type = get_replication_type(conn, manager.version)
return bool(replication_type)
blueprint = ReplicationNodesModule(__name__)
class ReplicationNodesView(PGChildNodeView):
"""
class ReplicationNodesView(NodeView)
A view class for Replication Nodes node derived from NodeView.
This class is responsible for all the stuff related to view like
showing properties/list of Replication Nodes nodes
Methods:
-------
* __init__(**kwargs)
- Method is used to initialize the ReplicationNodesView,
and it's base view.
* check_precondition()
- This function will behave as a decorator which will checks
database connection before running view, it will also attaches
manager,conn & template_path properties to self
* list()
- This function is used to list all the Replication Nodes within
that collection.
* nodes()
- This function will used to create all the child node within that
collection. Here it will create all the Replication Nodes node.
* properties(gid, sid, did, pid)
- This function will show the properties of the selected node
"""
node_type = blueprint.node_type
BASE_TEMPLATE_PATH = 'replica_nodes/sql/#{0}#'
parent_ids = [
{'type': 'int', 'id': 'gid'},
{'type': 'int', 'id': 'sid'}
]
ids = [
{'type': 'int', 'id': 'pid'}
]
operations = dict({
'obj': [
{'get': 'properties'},
{'get': 'list'}
],
'nodes': [{'get': 'nodes'}, {'get': 'nodes'}],
'replication_slots': [{'get': 'replication_slots'},
{'get': 'replication_slots'}],
})
def __init__(self, **kwargs):
"""
Method is used to initialize the ReplicationNodesView and,
it's base view.
Also initialize all the variables create/used dynamically like conn,
template_path.
Args:
**kwargs:
"""
self.conn = None
self.template_path = None
super().__init__(**kwargs)
def check_precondition(f):
"""
This function will behave as a decorator which will checks
database connection before running view, it will also attaches
manager,conn & template_path properties to self
"""
@wraps(f)
def wrap(*args, **kwargs):
# Here args[0] will hold self & kwargs will hold gid,sid,did
self = args[0]
self.driver = get_driver(PG_DEFAULT_DRIVER)
self.manager = self.driver.connection_manager(kwargs['sid'])
self.conn = self.manager.connection()
if not self.conn.connected():
return precondition_required(
gettext(
"Connection to the server has been lost."
)
)
self.template_path = self.BASE_TEMPLATE_PATH.format(
self.manager.version)
return f(*args, **kwargs)
return wrap
@check_precondition
def list(self, gid, sid):
"""
This function is used to list all the Replication Nodes within
that collection.
Args:
gid: Server Group ID
sid: Server ID
"""
sql = render_template(
"/".join([self.template_path, self._PROPERTIES_SQL]))
status, res = self.conn.execute_dict(sql)
if not status:
return internal_server_error(errormsg=res)
return ajax_response(
response=res['rows'],
status=200
)
@check_precondition
def nodes(self, gid, sid):
"""
This function will used to create all the child node within that
collection. Here it will create all the Replication Nodes node.
Args:
gid: Server Group ID
sid: Server ID
"""
res = []
sql = render_template("/".join([self.template_path, self._NODES_SQL]))
status, result = self.conn.execute_2darray(sql)
if not status:
return internal_server_error(errormsg=result)
for row in result['rows']:
res.append(
self.blueprint.generate_browser_node(
row['pid'],
sid,
row['name'],
icon="icon-replica_nodes"
))
return make_json_response(
data=res,
status=200
)
@check_precondition
def properties(self, gid, sid, pid):
"""
This function will show the properties of the selected node.
Args:
gid: Server Group ID
sid: Server ID
pid: Replication Nodes ID
"""
sql = render_template(
"/".join([self.template_path, self._PROPERTIES_SQL]), pid=pid)
status, res = self.conn.execute_dict(sql)
if not status:
return internal_server_error(errormsg=res)
if len(res['rows']) == 0:
return gone(gettext("""Could not find the Replication Node."""))
return ajax_response(
response=res['rows'][0],
status=200
)
ReplicationNodesView.register_node_view(blueprint)

View File

@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><defs><style>.cls-1,.cls-3{fill:#f2f2f2;}.cls-2{fill:#aaa;}.cls-3{stroke:#aaa;}.cls-3,.cls-5{stroke-miterlimit:1;stroke-width:0.75px;}.cls-4{fill:#7b7b97;}.cls-5{fill:#def4fd;stroke:#7b7b97;}</style></defs><title>server</title><g id="_3" data-name="3"><rect class="cls-1" x="3.08" y="2.6" width="9.85" height="3.1" rx="1.13" ry="1.13"/><path class="cls-2" d="M11.8,3a.75.75,0,0,1,.75.75v.85a.75.75,0,0,1-.75.75H4.2a.75.75,0,0,1-.75-.75V3.73A.75.75,0,0,1,4.2,3h7.6m0-.75H4.2a1.5,1.5,0,0,0-1.5,1.5v.85a1.5,1.5,0,0,0,1.5,1.5h7.6a1.5,1.5,0,0,0,1.5-1.5V3.73a1.5,1.5,0,0,0-1.5-1.5Z"/><line class="cls-3" x1="6.37" y1="4.15" x2="4.27" y2="4.15"/><path class="cls-1" d="M4.2,6.45h7.6a1.13,1.13,0,0,1,1.13,1.13v.85A1.12,1.12,0,0,1,11.8,9.55H4.2A1.12,1.12,0,0,1,3.08,8.42V7.58A1.13,1.13,0,0,1,4.2,6.45Z"/><path class="cls-4" d="M11.8,6.82a.75.75,0,0,1,.75.75v.85a.75.75,0,0,1-.75.75H4.2a.75.75,0,0,1-.75-.75V7.57a.75.75,0,0,1,.75-.75h7.6m0-.75H4.2a1.5,1.5,0,0,0-1.5,1.5v.85a1.5,1.5,0,0,0,1.5,1.5h7.6a1.5,1.5,0,0,0,1.5-1.5V7.57a1.5,1.5,0,0,0-1.5-1.5Z"/><line class="cls-5" x1="6.37" y1="8" x2="4.27" y2="8"/><path class="cls-1" d="M4.2,10.3h7.6a1.12,1.12,0,0,1,1.12,1.12v.85A1.12,1.12,0,0,1,11.8,13.4H4.2a1.13,1.13,0,0,1-1.13-1.13v-.85A1.12,1.12,0,0,1,4.2,10.3Z"/><path class="cls-2" d="M11.8,10.68a.75.75,0,0,1,.75.75v.85a.75.75,0,0,1-.75.75H4.2a.75.75,0,0,1-.75-.75v-.85a.75.75,0,0,1,.75-.75h7.6m0-.75H4.2a1.5,1.5,0,0,0-1.5,1.5v.85a1.5,1.5,0,0,0,1.5,1.5h7.6a1.5,1.5,0,0,0,1.5-1.5v-.85a1.5,1.5,0,0,0-1.5-1.5Z"/><line class="cls-3" x1="6.37" y1="11.85" x2="4.27" y2="11.85"/></g></svg>

After

Width:  |  Height:  |  Size: 1.6 KiB

View File

@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><defs><style>.cls-1,.cls-3{fill:#f2f2f2;}.cls-2{fill:#aaa;}.cls-3{stroke:#aaa;}.cls-3,.cls-5{stroke-miterlimit:1;stroke-width:0.75px;}.cls-4{fill:#7b7b97;}.cls-5{fill:#def4fd;stroke:#7b7b97;}</style></defs><title>server</title><g id="_3" data-name="3"><rect class="cls-1" x="3.08" y="2.6" width="9.85" height="3.1" rx="1.13" ry="1.13"/><path class="cls-2" d="M11.8,3a.75.75,0,0,1,.75.75v.85a.75.75,0,0,1-.75.75H4.2a.75.75,0,0,1-.75-.75V3.73A.75.75,0,0,1,4.2,3h7.6m0-.75H4.2a1.5,1.5,0,0,0-1.5,1.5v.85a1.5,1.5,0,0,0,1.5,1.5h7.6a1.5,1.5,0,0,0,1.5-1.5V3.73a1.5,1.5,0,0,0-1.5-1.5Z"/><line class="cls-3" x1="6.37" y1="4.15" x2="4.27" y2="4.15"/><path class="cls-1" d="M4.2,6.45h7.6a1.13,1.13,0,0,1,1.13,1.13v.85A1.12,1.12,0,0,1,11.8,9.55H4.2A1.12,1.12,0,0,1,3.08,8.42V7.58A1.13,1.13,0,0,1,4.2,6.45Z"/><path class="cls-4" d="M11.8,6.82a.75.75,0,0,1,.75.75v.85a.75.75,0,0,1-.75.75H4.2a.75.75,0,0,1-.75-.75V7.57a.75.75,0,0,1,.75-.75h7.6m0-.75H4.2a1.5,1.5,0,0,0-1.5,1.5v.85a1.5,1.5,0,0,0,1.5,1.5h7.6a1.5,1.5,0,0,0,1.5-1.5V7.57a1.5,1.5,0,0,0-1.5-1.5Z"/><line class="cls-5" x1="6.37" y1="8" x2="4.27" y2="8"/><path class="cls-1" d="M4.2,10.3h7.6a1.12,1.12,0,0,1,1.12,1.12v.85A1.12,1.12,0,0,1,11.8,13.4H4.2a1.13,1.13,0,0,1-1.13-1.13v-.85A1.12,1.12,0,0,1,4.2,10.3Z"/><path class="cls-2" d="M11.8,10.68a.75.75,0,0,1,.75.75v.85a.75.75,0,0,1-.75.75H4.2a.75.75,0,0,1-.75-.75v-.85a.75.75,0,0,1,.75-.75h7.6m0-.75H4.2a1.5,1.5,0,0,0-1.5,1.5v.85a1.5,1.5,0,0,0,1.5,1.5h7.6a1.5,1.5,0,0,0,1.5-1.5v-.85a1.5,1.5,0,0,0-1.5-1.5Z"/><line class="cls-3" x1="6.37" y1="11.85" x2="4.27" y2="11.85"/></g></svg>

After

Width:  |  Height:  |  Size: 1.6 KiB

View File

@@ -0,0 +1,61 @@
/////////////////////////////////////////////////////////////
//
// pgAdmin 4 - PostgreSQL Tools
//
// Copyright (C) 2013 - 2024, The pgAdmin Development Team
// This software is released under the PostgreSQL Licence
//
//////////////////////////////////////////////////////////////
import ReplicaNodeSchema from './replica_node.ui';
define('pgadmin.node.replica_nodes', [
'sources/gettext', 'sources/url_for', 'pgadmin.browser',
'pgadmin.browser.collection',
], function(gettext, url_for, pgBrowser) {
// Extend the browser's collection class for replica nodes collection
if (!pgBrowser.Nodes['coll-replica_nodes']) {
pgBrowser.Nodes['coll-replica_nodes'] =
pgBrowser.Collection.extend({
node: 'replica_nodes',
label: gettext('Replica Nodes'),
type: 'coll-replica_nodes',
columns: ['pid', 'name', 'usename', 'state'],
canEdit: false,
canDrop: false,
canDropCascade: false,
});
}
// Extend the browser's node class for replica nodes node
if (!pgBrowser.Nodes['replica_nodes']) {
pgBrowser.Nodes['replica_nodes'] = pgBrowser.Node.extend({
parent_type: 'server',
type: 'replica_nodes',
epasHelp: false,
sqlAlterHelp: '',
sqlCreateHelp: '',
dialogHelp: url_for('help.static', {'filename': 'replica_nodes_dialog.html'}),
label: gettext('Replica Nodes'),
hasSQL: false,
hasScriptTypes: false,
canDrop: false,
Init: function() {
// Avoid multiple registration of menus
if (this.initialized) {
return;
}
this.initialized = true;
},
getSchema: ()=>{
return new ReplicaNodeSchema();
},
});
}
return pgBrowser.Nodes['coll-replica_nodes'];
});

View File

@@ -0,0 +1,83 @@
/////////////////////////////////////////////////////////////
//
// pgAdmin 4 - PostgreSQL Tools
//
// Copyright (C) 2013 - 2024, The pgAdmin Development Team
// This software is released under the PostgreSQL Licence
//
//////////////////////////////////////////////////////////////
import gettext from 'sources/gettext';
import BaseUISchema from 'sources/SchemaView/base_schema.ui';
export default class ReplicaNodeSchema extends BaseUISchema {
get idAttribute() {
return 'pid';
}
get baseFields() {
return [
{
id: 'pid', label: gettext('PID'), type: 'text', mode:['properties', 'edit'], readonly: true,
},
{
id: 'usename', label: gettext('Username'), type: 'text', mode:['properties', 'edit'], readonly: true,
},
{
id: 'application_name', label: gettext('App Name'), type: 'text', mode:['properties', 'edit'], readonly: true,
},
{
id: 'client_addr', label: gettext('Client Address'), type: 'text', mode:['properties', 'edit'], readonly: true,
},
{
id: 'client_hostname', label: gettext('Client Hostname'), type: 'text', mode:['properties', 'edit'], readonly: true,
},
{
id: 'client_port', label: gettext('Client Port'), type: 'text', mode:['properties', 'edit'], readonly: true,
},
{
id: 'state', label: gettext('State'), type: 'text', mode:['properties', 'edit'], readonly: true,
},
{
id: 'sent_lsn', label: gettext('Sent LSN'), type: 'text', mode:['properties'], readonly: true,
group: gettext('WAL Details')
},
{
id: 'write_lsn', label: gettext('Write LSN'), type: 'text', mode:['properties'], readonly: true,
group: gettext('WAL Details')
},
{
id: 'flush_lsn', label: gettext('Flush LSN'), type: 'text', mode:['properties'], readonly: true,
group: gettext('WAL Details')
},
{
id: 'replay_lsn', label: gettext('Replay LSN'), type: 'text', mode:['properties'], readonly: true,
group: gettext('WAL Details')
},
{
id: 'write_lag', label: gettext('Write Lag'), type: 'text', mode:['properties'], readonly: true,
group: gettext('WAL Details')
},
{
id: 'flush_lag', label: gettext('Flush Lag'), type: 'text', mode:['properties'], readonly: true,
group: gettext('WAL Details')
},
{
id: 'replay_lag', label: gettext('Replay Lag'), type: 'text', mode:['properties'], readonly: true,
group: gettext('WAL Details')
},
{
id: 'slot_name', label: gettext('Slot Name'), type: 'text', mode:['properties', 'edit'], readonly: true,
group: gettext('Replication Slot')
},
{
id: 'slot_type', label: gettext('Slot Type'), type: 'text', mode:['properties', 'edit'], readonly: true,
group: gettext('Replication Slot')
},
{
id: 'active', label: gettext('Active'), type: 'switch', mode:['properties', 'edit'], readonly: true,
group: gettext('Replication Slot')
},
];
}
}

View File

@@ -0,0 +1,2 @@
SELECT count(*)
FROM pg_stat_replication

View File

@@ -0,0 +1,3 @@
SELECT pid, 'Standby ['||COALESCE(client_addr::text, client_hostname,'Socket')||']' as name
FROM pg_stat_replication
ORDER BY pid

View File

@@ -0,0 +1,8 @@
SELECT st.*, 'Standby ['||COALESCE(client_addr::text, client_hostname,'Socket')||']' as name,
sl.slot_name, sl.slot_type, sl.active
FROM pg_stat_replication st JOIN pg_replication_slots sl
ON st.pid = sl.active_pid
{% if pid %}
WHERE st.pid={{pid}}
{% endif %}
ORDER BY st.pid

View File

@@ -0,0 +1,7 @@
SELECT CASE
WHEN (SELECT count(extname) FROM pg_catalog.pg_extension WHERE extname='bdr') > 0
THEN 'pgd'
WHEN (SELECT COUNT(*) FROM pg_stat_replication) > 0
THEN 'log'
ELSE NULL
END as type;

View File

@@ -158,7 +158,9 @@ class ServersConnectTestCase(BaseTestGenerator):
self.manager.connection.connected.side_effect = True
connection_mock_result.execute_dict.side_effect = \
[eval(self.mock_data["return_value"])]
[eval(self.mock_data["return_value"]),
# replication type mock
(True, {'rows': [{'type': None}]})]
response = self.get_server_connection(server_id)
self.assertEqual(response.status_code,

View File

@@ -9,6 +9,8 @@
"""Server helper utilities"""
from ipaddress import ip_address
from werkzeug.exceptions import InternalServerError
from flask import render_template
from pgadmin.utils.crypto import encrypt, decrypt
import config
@@ -277,3 +279,14 @@ def remove_saved_passwords(user_id):
except Exception:
db.session.rollback()
raise
def get_replication_type(conn, sversion):
status, res = conn.execute_dict(render_template(
"/servers/sql/#{0}#/replication_type.sql".format(sversion)
))
if not status:
raise InternalServerError(res)
return res['rows'][0]['type']