Add support for connecting using pg_service.conf files. Fixes #3140

docs/en_US/server_dialog.rst

@@ -32,6 +32,7 @@ Use the fields in the *Connection* tab to configure a connection:
 * Use the *Password* field to provide a password that will be supplied when authenticating with the server.
 * Check the box next to *Save password?* to instruct pgAdmin to save the password for future use.
 * Use the *Role* field to specify the name of a role that has privileges that will be conveyed to the client after authentication with the server. This selection allows you to connect as one role, and then assume the permissions of this specified role after the connection is established. Note that the connecting role must be a member of the role specified.
+* Use the *Service* field to specify the service name. For more information, see `Section 33.16 of the Postgres documentation <https://www.postgresql.org/docs/10/static/libpq-pgservice.html>`_.
 
 Click the *SSL* tab to continue.
 

web/migrations/versions/50aad68f99c2_.py

@@ -0,0 +1,82 @@
+
+"""Added service field option in server table (RM#3140)
+
+Revision ID: 50aad68f99c2
+Revises: 02b9dccdcfcb
+Create Date: 2018-03-07 11:53:57.584280
+
+"""
+from pgadmin.model import db
+
+
+# revision identifiers, used by Alembic.
+revision = '50aad68f99c2'
+down_revision = '02b9dccdcfcb'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+        # To Save previous data
+        db.engine.execute("ALTER TABLE server RENAME TO server_old")
+
+        # With service file some fields won't be mandatory as user can provide
+        # them using service file. Removed NOT NULL constraint from few columns
+        db.engine.execute("""
+            CREATE TABLE server (
+                id	INTEGER NOT NULL,
+                user_id	INTEGER NOT NULL,
+                servergroup_id	INTEGER NOT NULL,
+                name	VARCHAR(128) NOT NULL,
+                host	VARCHAR(128),
+                port	INTEGER NOT NULL CHECK(port >= 1024 AND port <= 65534),
+                maintenance_db	VARCHAR(64),
+                username	VARCHAR(64) NOT NULL,
+                password	VARCHAR(64),
+                role	VARCHAR(64),
+                ssl_mode	VARCHAR(16) NOT NULL CHECK(ssl_mode IN
+                    ( 'allow' , 'prefer' , 'require' , 'disable' ,
+                      'verify-ca' , 'verify-full' )
+                ),
+                comment	VARCHAR(1024),
+                discovery_id	VARCHAR(128),
+                hostaddr	TEXT(1024),
+                db_res	TEXT,
+                passfile	TEXT,
+                sslcert	TEXT,
+                sslkey	TEXT,
+                sslrootcert	TEXT,
+                sslcrl	TEXT,
+                sslcompression	INTEGER DEFAULT 0,
+                bgcolor TEXT(10),
+                fgcolor TEXT(10),
+                PRIMARY KEY(id),
+                FOREIGN KEY(user_id) REFERENCES user(id),
+                FOREIGN KEY(servergroup_id) REFERENCES servergroup(id)
+            )
+        """)
+
+        # Copy old data again into table
+        db.engine.execute("""
+        INSERT INTO server (
+            id,user_id, servergroup_id, name, host, port, maintenance_db,
+            username, ssl_mode, comment, password, role, discovery_id,
+            hostaddr, db_res, passfile, sslcert, sslkey, sslrootcert, sslcrl,
+            bgcolor, fgcolor
+        ) SELECT
+            id,user_id, servergroup_id, name, host, port, maintenance_db,
+            username, ssl_mode, comment, password, role, discovery_id,
+            hostaddr, db_res, passfile, sslcert, sslkey, sslrootcert, sslcrl,
+            bgcolor, fgcolor
+        FROM server_old""")
+
+        # Remove old data
+        db.engine.execute("DROP TABLE server_old")
+
+        # Add column for Service
+        db.engine.execute(
+            'ALTER TABLE server ADD COLUMN service TEXT'
+        )
+
+def downgrade():
+    pass

web/pgadmin/browser/server_groups/servers/__init__.py

@@ -478,7 +478,8 @@ class ServerNode(PGChildNodeView):
             'sslcrl': 'sslcrl',
             'sslcompression': 'sslcompression',
             'bgcolor': 'bgcolor',
-            'fgcolor': 'fgcolor'
+            'fgcolor': 'fgcolor',
+            'service': 'service'
         }
 
         disp_lbl = {
@@ -515,7 +516,7 @@ class ServerNode(PGChildNodeView):
         if connected:
             for arg in (
                     'host', 'hostaddr', 'port', 'db', 'username', 'sslmode',
-                    'role'
+                    'role', 'service'
             ):
                 if arg in data:
                     return forbidden(
@@ -663,7 +664,8 @@ class ServerNode(PGChildNodeView):
                 'sslrootcert': server.sslrootcert if is_ssl else None,
                 'sslcrl': server.sslcrl if is_ssl else None,
                 'sslcompression': True if is_ssl and server.sslcompression
-                else False
+                else False,
+                'service': server.service if server.service else None
             }
         )
 
@@ -672,18 +674,22 @@ class ServerNode(PGChildNodeView):
         """Add a server node to the settings database"""
         required_args = [
             u'name',
-            u'host',
             u'port',
-            u'db',
-            u'username',
             u'sslmode',
-            u'role'
+            u'username'
         ]
 
         data = request.form if request.form else json.loads(
             request.data, encoding='utf-8'
         )
 
+        # Some fields can be provided with service file so they are optional
+        if 'service' in data and not data['service']:
+            required_args.extend([
+                u'host',
+                u'db',
+                u'role'
+            ])
         for arg in required_args:
             if arg not in data:
                 return make_json_response(
@@ -711,29 +717,26 @@ class ServerNode(PGChildNodeView):
         try:
             server = Server(
                 user_id=current_user.id,
-                servergroup_id=data[u'gid'] if u'gid' in data else gid,
-                name=data[u'name'],
-                host=data[u'host'],
-                hostaddr=data[u'hostaddr'] if u'hostaddr' in data else None,
-                port=data[u'port'],
-                maintenance_db=data[u'db'],
-                username=data[u'username'],
-                ssl_mode=data[u'sslmode'],
-                comment=data[u'comment'] if u'comment' in data else None,
-                role=data[u'role'] if u'role' in data else None,
+                servergroup_id=data.get('gid', gid),
+                name=data.get('name'),
+                host=data.get('host', None),
+                hostaddr=data.get('hostaddr', None),
+                port=data.get('port'),
+                maintenance_db=data.get('db', None),
+                username=data.get('username'),
+                ssl_mode=data.get('sslmode'),
+                comment=data.get('comment', None),
+                role=data.get('role', None),
                 db_res=','.join(data[u'db_res'])
-                if u'db_res' in data
-                else None,
-                sslcert=data['sslcert'] if is_ssl else None,
-                sslkey=data['sslkey'] if is_ssl else None,
-                sslrootcert=data['sslrootcert'] if is_ssl else None,
-                sslcrl=data['sslcrl'] if is_ssl else None,
+                if u'db_res' in data else None,
+                sslcert=data.get('sslcert', None),
+                sslkey=data.get('sslkey', None),
+                sslrootcert=data.get('sslrootcert', None),
+                sslcrl=data.get('sslcrl', None),
                 sslcompression=1 if is_ssl and data['sslcompression'] else 0,
-                bgcolor=data['bgcolor'] if u'bgcolor' in data
-                else None,
-                fgcolor=data['fgcolor'] if u'fgcolor' in data
-                else None
-
+                bgcolor=data.get('bgcolor', None),
+                fgcolor=data.get('fgcolor', None),
+                service=data.get('service', None)
             )
             db.session.add(server)
             db.session.commit()
@@ -930,7 +933,7 @@ class ServerNode(PGChildNodeView):
         if 'password' not in data:
             conn_passwd = getattr(conn, 'password', None)
             if conn_passwd is None and server.password is None and \
-                    server.passfile is None:
+                    server.passfile is None and server.service is None:
                 # Return the password template in case password is not
                 # provided, or password has not been saved earlier.
                 return make_json_response(

web/pgadmin/browser/server_groups/servers/static/js/server.js

@@ -665,6 +665,7 @@ define('pgadmin.node.server', [
           sslkey: undefined,
           sslrootcert: undefined,
           sslcrl: undefined,
+          service: undefined,
         },
         // Default values!
         initialize: function(attrs, args) {
@@ -841,12 +842,18 @@ define('pgadmin.node.server', [
             var passfile = m.get('passfile');
             return !_.isUndefined(passfile) && !_.isNull(passfile);
           },
+        },{
+          id: 'service', label: gettext('Service'), type: 'text',
+          mode: ['properties', 'edit', 'create'], disabled: 'isConnected',
+          group: gettext('Connection'),
         }],
         validate: function() {
           var err = {},
             errmsg,
             self = this;
 
+          var service_id = this.get('service');
+
           var check_for_empty = function(id, msg) {
             var v = self.get(id);
             if (
@@ -903,26 +910,41 @@ define('pgadmin.node.server', [
           }
           check_for_empty('name', gettext('Name must be specified.'));
 
-          if (check_for_empty(
-            'host', gettext('Either Host name or Host address must be specified.')
-          ) && check_for_empty('hostaddr', gettext('Either Host name or Host address must be specified.'))){
-            errmsg = errmsg || gettext('Either Host name or Host address must be specified');
+          // If no service id then only check
+          if (
+              _.isUndefined(service_id) || _.isNull(service_id) ||
+              String(service_id).replace(/^\s+|\s+$/g, '') == ''
+            ) {
+            if (check_for_empty(
+              'host', gettext('Either Host name, Address or Service must be specified.')
+            ) && check_for_empty('hostaddr', gettext('Either Host name, Address or Service must be specified.'))){
+              errmsg = errmsg || gettext('Either Host name, Address or Service must be specified.');
+            } else {
+              errmsg = undefined;
+              delete err['host'];
+              delete err['hostaddr'];
+            }
+
+            check_for_empty(
+              'db', gettext('Maintenance database must be specified.')
+            );
+            check_for_valid_ip(
+              'hostaddr', gettext('Host address must be valid IPv4 or IPv6 address.')
+            );
+            check_for_valid_ip(
+              'hostaddr', gettext('Host address must be valid IPv4 or IPv6 address.')
+            );
           } else {
-            errmsg = undefined;
-            delete err['host'];
-            delete err['hostaddr'];
+            _.each(['host', 'hostaddr', 'db'], (item) => {
+              self.errorModel.unset(item);
+            });
           }
 
-          check_for_empty(
-            'db', gettext('Maintenance database must be specified.')
-          );
           check_for_empty(
             'username', gettext('Username must be specified.')
           );
           check_for_empty('port', gettext('Port must be specified.'));
-          check_for_valid_ip(
-            'hostaddr', gettext('Host address must be valid IPv4 or IPv6 address.')
-          );
+
           this.errorModel.set(err);
 
           if (_.size(err)) {

web/pgadmin/browser/server_groups/servers/tests/test_add_server_with_service_id.py

@@ -0,0 +1,47 @@
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2018, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+import json
+
+from pgadmin.utils.route import BaseTestGenerator
+from regression.python_test_utils import test_utils as utils
+
+
+class ServersWithServiceIDAddTestCase(BaseTestGenerator):
+    """ This class will add the servers under default server group. """
+
+    scenarios = [
+        # Fetch the default url for server object
+        (
+            'Default Server Node url', dict(
+                url='/browser/server/obj/'
+            )
+        )
+    ]
+
+    def setUp(self):
+        pass
+
+    def runTest(self):
+        """ This function will add the server under default server group."""
+        url = "{0}{1}/".format(self.url, utils.SERVER_GROUP)
+        # Add service name in the config
+        self.server['service'] = "TestDB"
+        response = self.tester.post(
+            url,
+            data=json.dumps(self.server),
+            content_type='html/json'
+        )
+        self.assertEquals(response.status_code, 200)
+        response_data = json.loads(response.data.decode('utf-8'))
+        self.server_id = response_data['node']['_id']
+
+    def tearDown(self):
+        """This function delete the server from SQLite """
+        utils.delete_server_with_api(self.tester, self.server_id)

web/pgadmin/model/__init__.py

@@ -107,13 +107,13 @@ class Server(db.Model):
         nullable=False
     )
     name = db.Column(db.String(128), nullable=False)
-    host = db.Column(db.String(128), nullable=False)
+    host = db.Column(db.String(128), nullable=True)
     hostaddr = db.Column(db.String(128), nullable=True)
     port = db.Column(
         db.Integer(),
         db.CheckConstraint('port >= 1024 AND port <= 65534'),
         nullable=False)
-    maintenance_db = db.Column(db.String(64), nullable=False)
+    maintenance_db = db.Column(db.String(64), nullable=True)
     username = db.Column(db.String(64), nullable=False)
     password = db.Column(db.String(64), nullable=True)
     role = db.Column(db.String(64), nullable=True)
@@ -144,6 +144,7 @@ class Server(db.Model):
     )
     bgcolor = db.Column(db.Text(10), nullable=True)
     fgcolor = db.Column(db.Text(10), nullable=True)
+    service = db.Column(db.Text(), nullable=True)
 
 
 class ModulePreference(db.Model):

web/pgadmin/utils/driver/psycopg2/server_manager.py

@@ -0,0 +1,333 @@
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2018, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+"""
+Implementation of ServerManager
+"""
+import os
+import datetime
+from flask import current_app, session
+from flask_security import current_user
+from flask_babel import gettext
+
+from pgadmin.utils.crypto import decrypt
+from .connection import Connection
+from pgadmin.model import Server
+
+
+class ServerManager(object):
+    """
+    class ServerManager
+
+    This class contains the information about the given server.
+    And, acts as connection manager for that particular session.
+    """
+
+    def __init__(self, server):
+        self.connections = dict()
+
+        self.update(server)
+
+    def update(self, server):
+        assert (server is not None)
+        assert (isinstance(server, Server))
+
+        self.ver = None
+        self.sversion = None
+        self.server_type = None
+        self.server_cls = None
+        self.password = None
+
+        self.sid = server.id
+        self.host = server.host
+        self.hostaddr = server.hostaddr
+        self.port = server.port
+        self.db = server.maintenance_db
+        self.did = None
+        self.user = server.username
+        self.password = server.password
+        self.role = server.role
+        self.ssl_mode = server.ssl_mode
+        self.pinged = datetime.datetime.now()
+        self.db_info = dict()
+        self.server_types = None
+        self.db_res = server.db_res
+        self.passfile = server.passfile
+        self.sslcert = server.sslcert
+        self.sslkey = server.sslkey
+        self.sslrootcert = server.sslrootcert
+        self.sslcrl = server.sslcrl
+        self.sslcompression = True if server.sslcompression else False
+        self.service = server.service
+
+        for con in self.connections:
+            self.connections[con]._release()
+
+        self.update_session()
+
+        self.connections = dict()
+
+    def as_dict(self):
+        """
+        Returns a dictionary object representing the server manager.
+        """
+        if self.ver is None or len(self.connections) == 0:
+            return None
+
+        res = dict()
+        res['sid'] = self.sid
+        res['ver'] = self.ver
+        res['sversion'] = self.sversion
+        if hasattr(self, 'password') and self.password:
+            # If running under PY2
+            if hasattr(self.password, 'decode'):
+                res['password'] = self.password.decode('utf-8')
+            else:
+                res['password'] = str(self.password)
+        else:
+            res['password'] = self.password
+
+        connections = res['connections'] = dict()
+
+        for conn_id in self.connections:
+            conn = self.connections[conn_id].as_dict()
+
+            if conn is not None:
+                connections[conn_id] = conn
+
+        return res
+
+    def ServerVersion(self):
+        return self.ver
+
+    @property
+    def version(self):
+        return self.sversion
+
+    def MajorVersion(self):
+        if self.sversion is not None:
+            return int(self.sversion / 10000)
+        raise Exception("Information is not available.")
+
+    def MinorVersion(self):
+        if self.sversion:
+            return int(int(self.sversion / 100) % 100)
+        raise Exception("Information is not available.")
+
+    def PatchVersion(self):
+        if self.sversion:
+            return int(int(self.sversion / 100) / 100)
+        raise Exception("Information is not available.")
+
+    def connection(
+            self, database=None, conn_id=None, auto_reconnect=True, did=None,
+            async=None, use_binary_placeholder=False, array_to_string=False
+    ):
+        if database is not None:
+            if hasattr(str, 'decode') and \
+                    not isinstance(database, unicode):
+                database = database.decode('utf-8')
+            if did is not None:
+                if did in self.db_info:
+                    self.db_info[did]['datname'] = database
+        else:
+            if did is None:
+                database = self.db
+            elif did in self.db_info:
+                database = self.db_info[did]['datname']
+            else:
+                maintenance_db_id = u'DB:{0}'.format(self.db)
+                if maintenance_db_id in self.connections:
+                    conn = self.connections[maintenance_db_id]
+                    if conn.connected():
+                        status, res = conn.execute_dict(u"""
+SELECT
+    db.oid as did, db.datname, db.datallowconn,
+    pg_encoding_to_char(db.encoding) AS serverencoding,
+    has_database_privilege(db.oid, 'CREATE') as cancreate, datlastsysoid
+FROM
+    pg_database db
+WHERE db.oid = {0}""".format(did))
+
+                        if status and len(res['rows']) > 0:
+                            for row in res['rows']:
+                                self.db_info[did] = row
+                                database = self.db_info[did]['datname']
+
+                        if did not in self.db_info:
+                            raise Exception(gettext(
+                                "Could not find the specified database."
+                            ))
+
+        if database is None:
+            raise ConnectionLost(self.sid, None, None)
+
+        my_id = (u'CONN:{0}'.format(conn_id)) if conn_id is not None else \
+            (u'DB:{0}'.format(database))
+
+        self.pinged = datetime.datetime.now()
+
+        if my_id in self.connections:
+            return self.connections[my_id]
+        else:
+            if async is None:
+                async = 1 if conn_id is not None else 0
+            else:
+                async = 1 if async is True else 0
+            self.connections[my_id] = Connection(
+                self, my_id, database, auto_reconnect, async,
+                use_binary_placeholder=use_binary_placeholder,
+                array_to_string=array_to_string
+            )
+
+            return self.connections[my_id]
+
+    def _restore(self, data):
+        """
+        Helps restoring to reconnect the auto-connect connections smoothly on
+        reload/restart of the app server..
+        """
+        # restore server version from flask session if flask server was
+        # restarted. As we need server version to resolve sql template paths.
+        from pgadmin.browser.server_groups.servers.types import ServerType
+
+        self.ver = data.get('ver', None)
+        self.sversion = data.get('sversion', None)
+
+        if self.ver and not self.server_type:
+            for st in ServerType.types():
+                if st.instanceOf(self.ver):
+                    self.server_type = st.stype
+                    self.server_cls = st
+                    break
+
+        # Hmm.. we will not honour this request, when I already have
+        # connections
+        if len(self.connections) != 0:
+            return
+
+        # We need to know about the existing server variant supports during
+        # first connection for identifications.
+        self.pinged = datetime.datetime.now()
+        try:
+            if 'password' in data and data['password']:
+                data['password'] = data['password'].encode('utf-8')
+        except Exception as e:
+            current_app.logger.exception(e)
+
+        connections = data['connections']
+        for conn_id in connections:
+            conn_info = connections[conn_id]
+            conn = self.connections[conn_info['conn_id']] = Connection(
+                self, conn_info['conn_id'], conn_info['database'],
+                conn_info['auto_reconnect'], conn_info['async'],
+                use_binary_placeholder=conn_info['use_binary_placeholder'],
+                array_to_string=conn_info['array_to_string']
+            )
+
+            # only try to reconnect if connection was connected previously and
+            # auto_reconnect is true.
+            if conn_info['wasConnected'] and conn_info['auto_reconnect']:
+                try:
+                    conn.connect(
+                        password=data['password'],
+                        server_types=ServerType.types()
+                    )
+                    # This will also update wasConnected flag in connection so
+                    # no need to update the flag manually.
+                except Exception as e:
+                    current_app.logger.exception(e)
+                    self.connections.pop(conn_info['conn_id'])
+
+    def release(self, database=None, conn_id=None, did=None):
+        if did is not None:
+            if did in self.db_info and 'datname' in self.db_info[did]:
+                database = self.db_info[did]['datname']
+                if hasattr(str, 'decode') and \
+                        not isinstance(database, unicode):
+                    database = database.decode('utf-8')
+                if database is None:
+                    return False
+            else:
+                return False
+
+        my_id = (u'CONN:{0}'.format(conn_id)) if conn_id is not None else \
+            (u'DB:{0}'.format(database)) if database is not None else None
+
+        if my_id is not None:
+            if my_id in self.connections:
+                self.connections[my_id]._release()
+                del self.connections[my_id]
+                if did is not None:
+                    del self.db_info[did]
+
+                if len(self.connections) == 0:
+                    self.ver = None
+                    self.sversion = None
+                    self.server_type = None
+                    self.server_cls = None
+                    self.password = None
+
+                self.update_session()
+
+                return True
+            else:
+                return False
+
+        for con in self.connections:
+            self.connections[con]._release()
+
+        self.connections = dict()
+        self.ver = None
+        self.sversion = None
+        self.server_type = None
+        self.server_cls = None
+        self.password = None
+
+        self.update_session()
+
+        return True
+
+    def _update_password(self, passwd):
+        self.password = passwd
+        for conn_id in self.connections:
+            conn = self.connections[conn_id]
+            if conn.conn is not None or conn.wasConnected is True:
+                conn.password = passwd
+
+    def update_session(self):
+        managers = session['__pgsql_server_managers'] \
+            if '__pgsql_server_managers' in session else dict()
+        updated_mgr = self.as_dict()
+
+        if not updated_mgr:
+            if self.sid in managers:
+                managers.pop(self.sid)
+        else:
+            managers[self.sid] = updated_mgr
+        session['__pgsql_server_managers'] = managers
+        session.force_write = True
+
+    def utility(self, operation):
+        """
+        utility(operation)
+
+        Returns: name of the utility which used for the operation
+        """
+        if self.server_cls is not None:
+            return self.server_cls.utility(operation, self.sversion)
+
+        return None
+
+    def export_password_env(self, env):
+        if self.password:
+            password = decrypt(
+                self.password, current_user.password
+            ).decode()
+            os.environ[str(env)] = password