diff --git a/docs/en_US/images/server_ssh_tunnel.png b/docs/en_US/images/server_ssh_tunnel.png index c3d3fd6a4..68df1368d 100644 Binary files a/docs/en_US/images/server_ssh_tunnel.png and b/docs/en_US/images/server_ssh_tunnel.png differ diff --git a/docs/en_US/release_notes.rst b/docs/en_US/release_notes.rst index 1b554b757..136bf7df6 100644 --- a/docs/en_US/release_notes.rst +++ b/docs/en_US/release_notes.rst @@ -11,6 +11,7 @@ notes for it. .. toctree:: :maxdepth: 1 + release_notes_8_2 release_notes_8_1 release_notes_8_0 release_notes_7_8 diff --git a/docs/en_US/release_notes_8_2.rst b/docs/en_US/release_notes_8_2.rst new file mode 100644 index 000000000..5b83a7c62 --- /dev/null +++ b/docs/en_US/release_notes_8_2.rst @@ -0,0 +1,32 @@ +*********** +Version 8.2 +*********** + +Release date: 2024-01-11 + +This release contains a number of bug fixes and new features since the release of pgAdmin 4 v8.1. + +Supported Database Servers +************************** +**PostgreSQL**: 12, 13, 14, 15, and 16 + +**EDB Advanced Server**: 12, 13, 14, 15, and 16 + +Bundled PostgreSQL Utilities +**************************** +**psql**, **pg_dump**, **pg_dumpall**, **pg_restore**: 16.0 + + +New features +************ + + | `Issue #5908 `_ - Allow users to convert View/Edit table into a Query tool to enable editing the SQL generated. + | `Issue #7016 `_ - Added keep-alive support for SSH sessions when connecting to a PostgreSQL server via an SSH tunnel. + +Housekeeping +************ + + +Bug fixes +********* + diff --git a/docs/en_US/server_dialog.rst b/docs/en_US/server_dialog.rst index ef770a996..4ec406f66 100644 --- a/docs/en_US/server_dialog.rst +++ b/docs/en_US/server_dialog.rst @@ -181,6 +181,10 @@ not be able to connect directly. password for future use. Use :ref:`Clear SSH Tunnel Password ` to remove the saved password. +* Use the *Keep alive* field to specify interval in seconds defining the period + in which, if no data was sent over the connection, a ‘keepalive’ packet will + be sent (and ignored by the remote host). This can be useful to keep + connections alive over a NAT. You can set to 0 for disable keepalive. Click the *Advanced* tab to continue. diff --git a/web/migrations/versions/ec0f11f9a4e6_.py b/web/migrations/versions/ec0f11f9a4e6_.py new file mode 100644 index 000000000..8799a7b69 --- /dev/null +++ b/web/migrations/versions/ec0f11f9a4e6_.py @@ -0,0 +1,36 @@ +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2023, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## + +""" + +Revision ID: ec0f11f9a4e6 +Revises: 44926ac97232 +Create Date: 2023-12-18 17:09:34.499652 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'ec0f11f9a4e6' +down_revision = '44926ac97232' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('server', sa.Column('tunnel_keep_alive', sa.Integer(), + server_default='0')) + op.add_column('sharedserver', sa.Column('tunnel_keep_alive', sa.Integer(), + server_default='0')) + + +def downgrade(): + # pgAdmin only upgrades, downgrade not implemented. + pass diff --git a/web/pgadmin/browser/server_groups/servers/__init__.py b/web/pgadmin/browser/server_groups/servers/__init__.py index af3605d4d..0e6591da7 100644 --- a/web/pgadmin/browser/server_groups/servers/__init__.py +++ b/web/pgadmin/browser/server_groups/servers/__init__.py @@ -390,6 +390,7 @@ class ServerModule(sg.ServerGroupPluginModule): tunnel_username=None, tunnel_authentication=0, tunnel_identity_file=None, + tunnel_keep_alive=0, shared=True, connection_params=data.connection_params, prepare_threshold=data.prepare_threshold @@ -814,6 +815,7 @@ class ServerNode(PGChildNodeView): 'tunnel_username': 'tunnel_username', 'tunnel_authentication': 'tunnel_authentication', 'tunnel_identity_file': 'tunnel_identity_file', + 'tunnel_keep_alive': 'tunnel_keep_alive', 'shared': 'shared', 'shared_username': 'shared_username', 'kerberos_conn': 'kerberos_conn', @@ -1061,6 +1063,7 @@ class ServerNode(PGChildNodeView): tunnel_port = 22 tunnel_username = None tunnel_authentication = False + tunnel_keep_alive = 0 connection_params = \ self.convert_connection_parameter(server.connection_params) @@ -1070,6 +1073,7 @@ class ServerNode(PGChildNodeView): tunnel_port = server.tunnel_port tunnel_username = server.tunnel_username tunnel_authentication = bool(server.tunnel_authentication) + tunnel_keep_alive = server.tunnel_keep_alive response = { 'id': server.id, @@ -1106,6 +1110,7 @@ class ServerNode(PGChildNodeView): 'tunnel_identity_file': server.tunnel_identity_file if server.tunnel_identity_file else None, 'tunnel_authentication': tunnel_authentication, + 'tunnel_keep_alive': tunnel_keep_alive, 'kerberos_conn': bool(server.kerberos_conn), 'gss_authenticated': manager.gss_authenticated, 'gss_encrypted': manager.gss_encrypted, @@ -1201,6 +1206,7 @@ class ServerNode(PGChildNodeView): tunnel_authentication=1 if data.get('tunnel_authentication', False) else 0, tunnel_identity_file=data.get('tunnel_identity_file', None), + tunnel_keep_alive=data.get('tunnel_keep_alive', 0), shared=data.get('shared', None), shared_username=data.get('shared_username', None), passexec_cmd=data.get('passexec_cmd', None), @@ -2091,6 +2097,7 @@ class ServerNode(PGChildNodeView): "tunnel_username": server.tunnel_username, "tunnel_host": server.tunnel_host, "tunnel_identity_file": server.tunnel_identity_file, + "tunnel_keep_alive": server.tunnel_keep_alive, "errmsg": errmsg, "service": server.service, "prompt_tunnel_password": prompt_tunnel_password, diff --git a/web/pgadmin/browser/server_groups/servers/databases/subscriptions/static/js/subscription.ui.js b/web/pgadmin/browser/server_groups/servers/databases/subscriptions/static/js/subscription.ui.js index 97f626396..003520cb7 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/subscriptions/static/js/subscription.ui.js +++ b/web/pgadmin/browser/server_groups/servers/databases/subscriptions/static/js/subscription.ui.js @@ -494,43 +494,6 @@ export default class SubscriptionSchema extends BaseUISchema{ setError('pub', null); } - if (state.use_ssh_tunnel) { - if(isEmptyString(state.tunnel_host)) { - errmsg = gettext('SSH Tunnel host must be specified.'); - setError('tunnel_host', errmsg); - return true; - } else { - setError('tunnel_host', null); - } - - if(isEmptyString(state.tunnel_port)) { - errmsg = gettext('SSH Tunnel port must be specified.'); - setError('tunnel_port', errmsg); - return true; - } else { - setError('tunnel_port', null); - } - - if(isEmptyString(state.tunnel_username)) { - errmsg = gettext('SSH Tunnel username must be specified.'); - setError('tunnel_username', errmsg); - return true; - } else { - setError('tunnel_username', null); - } - - if (state.tunnel_authentication) { - if(isEmptyString(state.tunnel_identity_file)) { - errmsg = gettext('SSH Tunnel identity file must be specified.'); - setError('tunnel_identity_file', errmsg); - return true; - } else { - setError('tunnel_identity_file', null); - } - } - } - - return false; } diff --git a/web/pgadmin/browser/server_groups/servers/static/js/server.ui.js b/web/pgadmin/browser/server_groups/servers/static/js/server.ui.js index 87eca135f..c96d87c48 100644 --- a/web/pgadmin/browser/server_groups/servers/static/js/server.ui.js +++ b/web/pgadmin/browser/server_groups/servers/static/js/server.ui.js @@ -44,6 +44,7 @@ export default class ServerSchema extends BaseUISchema { tunnel_identity_file: undefined, tunnel_password: undefined, tunnel_authentication: false, + tunnel_keep_alive: 0, save_tunnel_password: false, connection_string: undefined, connection_params: [ @@ -327,6 +328,15 @@ export default class ServerSchema extends BaseUISchema { return (!current_user.allow_save_tunnel_password || !state.use_ssh_tunnel); }, }, + { + id: 'tunnel_keep_alive', label: gettext('Keep alive (seconds)'), + type: 'int', group: gettext('SSH Tunnel'), min: 0, + mode: ['properties', 'edit', 'create'], deps: ['use_ssh_tunnel'], + disabled: function(state) { + return !state.use_ssh_tunnel; + }, + readonly: obj.isConnected, + }, { id: 'db_res', label: gettext('DB restriction'), type: 'select', group: gettext('Advanced'), options: [], @@ -436,6 +446,14 @@ export default class ServerSchema extends BaseUISchema { setError('tunnel_identity_file', null); } } + + if(isEmptyString(state.tunnel_keep_alive)) { + errmsg = gettext('Keep alive must be specified. Specify 0 for no keep alive.'); + setError('tunnel_keep_alive', errmsg); + return true; + } else { + setError('tunnel_keep_alive', null); + } } return false; } diff --git a/web/pgadmin/browser/server_groups/servers/tests/servers_test_data.json b/web/pgadmin/browser/server_groups/servers/tests/servers_test_data.json index 38d818023..3de500b8f 100644 --- a/web/pgadmin/browser/server_groups/servers/tests/servers_test_data.json +++ b/web/pgadmin/browser/server_groups/servers/tests/servers_test_data.json @@ -53,7 +53,8 @@ "tunnel_port": 22, "tunnel_username": "user", "tunnel_authentication": 1, - "tunnel_identity_file": "pkey_rsa" + "tunnel_identity_file": "pkey_rsa", + "tunnel_keep_alive": 5 }, "mocking_required": false, "mock_data": {}, @@ -74,7 +75,8 @@ "tunnel_port": 22, "tunnel_username": "user", "tunnel_authentication": 1, - "tunnel_identity_file": "pkey_rsa" + "tunnel_identity_file": "pkey_rsa", + "tunnel_keep_alive": 0 }, "mocking_required": false, "mock_data": {}, @@ -95,7 +97,8 @@ "tunnel_port": 22, "tunnel_username": "user", "tunnel_authentication": 0, - "tunnel_password": "123456" + "tunnel_password": "123456", + "tunnel_keep_alive": 0 }, "mocking_required": false, "mock_data": {}, @@ -117,7 +120,8 @@ "tunnel_username": "user", "tunnel_authentication": 1, "tunnel_identity_file": "pkey_rsa", - "tunnel_password": "123456" + "tunnel_password": "123456", + "tunnel_keep_alive": 0 }, "mocking_required": false, "mock_data": {}, @@ -574,6 +578,7 @@ "tunnel_authentication": 1, "tunnel_password": "user123", "tunnel_identity_file": "pkey_rsa", + "tunnel_keep_alive": 0, "service": null, "server_info": { "id": 1, @@ -615,6 +620,7 @@ "tunnel_authentication": 1, "tunnel_password": "", "tunnel_identity_file": "pkey_rsa", + "tunnel_keep_alive": 0, "service": null, "server_info": { "id": 1, diff --git a/web/pgadmin/browser/server_groups/servers/tests/test_add_server.py b/web/pgadmin/browser/server_groups/servers/tests/test_add_server.py index 66e7fe653..3cf73a3f1 100644 --- a/web/pgadmin/browser/server_groups/servers/tests/test_add_server.py +++ b/web/pgadmin/browser/server_groups/servers/tests/test_add_server.py @@ -47,6 +47,8 @@ class AddServerTest(BaseTestGenerator): self.server['tunnel_host'] = self.test_data['tunnel_host'] self.server['tunnel_port'] = self.test_data['tunnel_port'] self.server['tunnel_username'] = self.test_data['tunnel_username'] + self.server['tunnel_keep_alive'] = \ + self.test_data['tunnel_keep_alive'] if self.with_password: self.server['tunnel_authentication'] = self.test_data[ diff --git a/web/pgadmin/browser/server_groups/servers/tests/test_check_connect.py b/web/pgadmin/browser/server_groups/servers/tests/test_check_connect.py index 337f03ace..4d693ccec 100644 --- a/web/pgadmin/browser/server_groups/servers/tests/test_check_connect.py +++ b/web/pgadmin/browser/server_groups/servers/tests/test_check_connect.py @@ -30,6 +30,7 @@ class ServersConnectTestCase(BaseTestGenerator): self.server.tunnel_host = '127.0.0.1' self.server.tunnel_port = 22 self.server.tunnel_username = 'user' + self.server.tunnel_keep_alive = 0 if hasattr(self, 'with_password') and self.with_password: self.server.tunnel_authentication = 0 else: diff --git a/web/pgadmin/browser/server_groups/servers/tests/test_check_ssh_mock_connect.py b/web/pgadmin/browser/server_groups/servers/tests/test_check_ssh_mock_connect.py index 7c33ca5ef..e80cacfd0 100644 --- a/web/pgadmin/browser/server_groups/servers/tests/test_check_ssh_mock_connect.py +++ b/web/pgadmin/browser/server_groups/servers/tests/test_check_ssh_mock_connect.py @@ -57,7 +57,8 @@ class ServersSSHConnectTestCase(BaseTestGenerator): def __init__(self, name, id, username, use_ssh_tunnel, tunnel_host, tunnel_port, tunnel_username, tunnel_authentication, - tunnel_identity_file, tunnel_password, service): + tunnel_identity_file, tunnel_password, + tunnel_keep_alive, service): self.name = name self.id = id self.username = username @@ -71,6 +72,7 @@ class ServersSSHConnectTestCase(BaseTestGenerator): self.tunnel_identity_file = \ tunnel_identity_file self.tunnel_password = tunnel_password + self.tunnel_keep_alive = tunnel_keep_alive self.service = service self.shared = None @@ -85,6 +87,7 @@ class ServersSSHConnectTestCase(BaseTestGenerator): self.mock_data['tunnel_authentication'], self.mock_data['tunnel_identity_file'], self.mock_data['tunnel_password'], + self.mock_data['tunnel_keep_alive'], self.mock_data['service'], ) diff --git a/web/pgadmin/browser/server_groups/tests/servers_group_test_data.json b/web/pgadmin/browser/server_groups/tests/servers_group_test_data.json index f0c35b7fb..428fca4fb 100644 --- a/web/pgadmin/browser/server_groups/tests/servers_group_test_data.json +++ b/web/pgadmin/browser/server_groups/tests/servers_group_test_data.json @@ -17,6 +17,7 @@ "tunnel_authentication": 1, "tunnel_password": "user123", "tunnel_identity_file": "pkey_rsa", + "tunnel_keep_alive": 0, "service": null, "fgcolor":"#B6D7A8", "bgcolor": "#0C343D", diff --git a/web/pgadmin/model/__init__.py b/web/pgadmin/model/__init__.py index a44b56bfa..15bcea21a 100644 --- a/web/pgadmin/model/__init__.py +++ b/web/pgadmin/model/__init__.py @@ -33,7 +33,7 @@ import config # ########################################################################## -SCHEMA_VERSION = 38 +SCHEMA_VERSION = 39 ########################################################################## # @@ -201,6 +201,7 @@ class Server(db.Model): ) tunnel_identity_file = db.Column(db.String(64), nullable=True) tunnel_password = db.Column(PgAdminDbBinaryString()) + tunnel_keep_alive = db.Column(db.Integer(), nullable=True) shared = db.Column(db.Boolean(), nullable=False) shared_username = db.Column(db.String(64), nullable=True) kerberos_conn = db.Column(db.Boolean(), nullable=False, default=0) @@ -413,6 +414,7 @@ class SharedServer(db.Model): ) tunnel_identity_file = db.Column(db.String(64), nullable=True) tunnel_password = db.Column(PgAdminDbBinaryString()) + tunnel_keep_alive = db.Column(db.Integer(), nullable=True) shared = db.Column(db.Boolean(), nullable=False) connection_params = db.Column(MutableDict.as_mutable(types.JSON)) prepare_threshold = db.Column(db.Integer(), nullable=True) diff --git a/web/pgadmin/utils/driver/psycopg3/connection.py b/web/pgadmin/utils/driver/psycopg3/connection.py index 3a26834c8..1eda4181b 100644 --- a/web/pgadmin/utils/driver/psycopg3/connection.py +++ b/web/pgadmin/utils/driver/psycopg3/connection.py @@ -1465,7 +1465,7 @@ Failed to reset the connection to the server due to following error: def _wait(self, conn): pass # This function is empty - def _wait_timeout(self, conn): + def _wait_timeout(self, conn, time): pass # This function is empty def poll(self, formatted_exception_msg=False, no_result=False): diff --git a/web/pgadmin/utils/driver/psycopg3/server_manager.py b/web/pgadmin/utils/driver/psycopg3/server_manager.py index 786a94992..0d0be9033 100644 --- a/web/pgadmin/utils/driver/psycopg3/server_manager.py +++ b/web/pgadmin/utils/driver/psycopg3/server_manager.py @@ -30,13 +30,13 @@ from pgadmin.utils.master_password import get_crypt_key from pgadmin.utils.exception import ObjectGone from pgadmin.utils.passexec import PasswordExec from psycopg.conninfo import make_conninfo -import keyring -from pgadmin.utils.constants import KEY_RING_SERVICE_NAME, \ - KEY_RING_USERNAME_FORMAT, KEY_RING_TUNNEL_FORMAT if config.SUPPORT_SSH_TUNNEL: from sshtunnel import SSHTunnelForwarder, BaseSSHTunnelForwarderError +CONN_STRING = 'CONN:{0}' +DB_STRING = 'DB:{0}' + class ServerManager(object): """ @@ -98,6 +98,7 @@ class ServerManager(object): else server.tunnel_authentication self.tunnel_identity_file = server.tunnel_identity_file self.tunnel_password = server.tunnel_password + self.tunnel_keep_alive = server.tunnel_keep_alive else: self.use_ssh_tunnel = 0 self.tunnel_host = None @@ -106,6 +107,7 @@ class ServerManager(object): self.tunnel_authentication = None self.tunnel_identity_file = None self.tunnel_password = None + self.tunnel_keep_alive = 0 self.kerberos_conn = server.kerberos_conn self.gss_authenticated = False @@ -204,7 +206,7 @@ class ServerManager(object): if did is not None and did in self.db_info: self.db_info[did]['datname'] = database else: - conn_str = 'CONN:{0}'.format(conn_id) + conn_str = CONN_STRING.format(conn_id) if did is None: database = self.db elif did in self.db_info: @@ -212,7 +214,7 @@ class ServerManager(object): elif conn_id and conn_str in self.connections: database = self.connections[conn_str].db else: - maintenance_db_id = 'DB:{0}'.format(self.db) + maintenance_db_id = DB_STRING.format(self.db) if maintenance_db_id in self.connections: conn = self.connections[maintenance_db_id] # try to connect maintenance db if not connected @@ -252,8 +254,8 @@ WHERE db.oid = {0}""".format(did)) else: raise ConnectionLost(self.sid, None, None) - my_id = ('CONN:{0}'.format(conn_id)) if conn_id is not None else \ - ('DB:{0}'.format(database)) + my_id = (CONN_STRING.format(conn_id)) if conn_id is not None else \ + (DB_STRING.format(database)) self.pinged = datetime.datetime.now() @@ -321,8 +323,7 @@ WHERE db.oid = {0}""".format(did)) # Check SSH Tunnel needs to be created if self.use_ssh_tunnel == 1 and \ not self.tunnel_created: - status, error = self.create_ssh_tunnel( - data['tunnel_password']) + self.create_ssh_tunnel(data['tunnel_password']) # Check SSH Tunnel is alive or not. self.check_ssh_tunnel_alive() @@ -400,9 +401,7 @@ WHERE db.oid = {0}""".format(did)) # Check SSH Tunnel needs to be created if self.use_ssh_tunnel == 1 and \ not self.tunnel_created: - status, error = self.create_ssh_tunnel( - self.tunnel_password - ) + self.create_ssh_tunnel(self.tunnel_password) # Check SSH Tunnel is alive or not. self.check_ssh_tunnel_alive() @@ -451,9 +450,9 @@ WHERE db.oid = {0}""".format(did)) return True, False, my_id if conn_id is not None: - my_id = 'CONN:{0}'.format(conn_id) + my_id = CONN_STRING.format(conn_id) elif database is not None: - my_id = 'DB:{0}'.format(database) + my_id = DB_STRING.format(database) return False, True, my_id @@ -599,7 +598,8 @@ WHERE db.oid = {0}""".format(did)) ssh_pkey=get_complete_file_path(self.tunnel_identity_file), ssh_private_key_password=tunnel_password, remote_bind_address=(self.host, self.port), - logger=ssh_logger + logger=ssh_logger, + set_keepalive=int(self.tunnel_keep_alive) ) else: self.tunnel_object = SSHTunnelForwarder( @@ -607,7 +607,8 @@ WHERE db.oid = {0}""".format(did)) ssh_username=self.tunnel_username, ssh_password=tunnel_password, remote_bind_address=(self.host, self.port), - logger=ssh_logger + logger=ssh_logger, + set_keepalive=int(self.tunnel_keep_alive) ) # flag tunnel threads in daemon mode to fix hang issue. self.tunnel_object.daemon_forward_servers = True diff --git a/web/regression/javascript/schema_ui_files/server.ui.spec.js b/web/regression/javascript/schema_ui_files/server.ui.spec.js index 9b66ac27e..36374f325 100644 --- a/web/regression/javascript/schema_ui_files/server.ui.spec.js +++ b/web/regression/javascript/schema_ui_files/server.ui.spec.js @@ -73,6 +73,10 @@ describe('ServerSchema', ()=>{ expect(setError).toHaveBeenCalledWith('tunnel_identity_file', 'SSH Tunnel identity file must be specified.'); state.tunnel_identity_file = '/file/path/xyz.pem'; + schemaObj.validate(state, setError); + expect(setError).toHaveBeenCalledWith('tunnel_keep_alive', 'Keep alive must be specified. Specify 0 for no keep alive.'); + + state.tunnel_keep_alive = 0; expect(schemaObj.validate(state, setError)).toBe(false); }); }); diff --git a/web/regression/javascript/schema_ui_files/subscription.ui.spec.js b/web/regression/javascript/schema_ui_files/subscription.ui.spec.js index 10e7550b5..2f2decd9b 100644 --- a/web/regression/javascript/schema_ui_files/subscription.ui.spec.js +++ b/web/regression/javascript/schema_ui_files/subscription.ui.spec.js @@ -86,27 +86,6 @@ describe('SubscriptionSchema', ()=>{ state.port = 5432; schemaObj.validate(state, setError); expect(setError).toHaveBeenCalledWith('pub', 'Publication must be specified.'); - - state.pub = 'testPub'; - state.use_ssh_tunnel = 'Require'; - schemaObj.validate(state, setError); - expect(setError).toHaveBeenCalledWith('tunnel_host', 'SSH Tunnel host must be specified.'); - - state.tunnel_host = 'localhost'; - schemaObj.validate(state, setError); - expect(setError).toHaveBeenCalledWith('tunnel_port', 'SSH Tunnel port must be specified.'); - - state.tunnel_port = 8080; - schemaObj.validate(state, setError); - expect(setError).toHaveBeenCalledWith('tunnel_username', 'SSH Tunnel username must be specified.'); - - state.tunnel_username = 'jasmine'; - state.tunnel_authentication = true; - schemaObj.validate(state, setError); - expect(setError).toHaveBeenCalledWith('tunnel_identity_file', 'SSH Tunnel identity file must be specified.'); - - state.tunnel_identity_file = '/file/path/xyz.pem'; - expect(schemaObj.validate(state, setError)).toBe(false); }); });