Adding a background process executor, and observer.

We will be using the external utilities like pg_dump, pg_dumpall,
pg_restore in background. pgAdmin 4 can be run as a CGI script, hence -
it is not good idea to run those utility in a controlled environment.
The process executor will run them in background, and we will execute
the process executor in detached mode.

Now that - the process executor runs in detached mode, we need an
observer, which will look at the status of the processes. It also reads
output, and error logs on demand.

Thanks - Surinder for helping in some of the UI changes.

web/config.py

@@ -151,7 +151,7 @@ MAX_SESSION_IDLE_TIME = 60
 
 # The schema version number for the configuration database
 # DO NOT CHANGE UNLESS YOU ARE A PGADMIN DEVELOPER!!
-SETTINGS_SCHEMA_VERSION = 9
+SETTINGS_SCHEMA_VERSION = 10
 
 # The default path to the SQLite database used to store user accounts and
 # settings. This default places the file in the same directory as this

web/pgadmin/browser/templates/browser/js/messages.js

@@ -9,7 +9,7 @@ function(_, S, pgAdmin) {
 
   var messages = pgBrowser.messages = {
     'SERVER_LOST': '{{ _('Connection to the server has been lost.') }}',
-    'CLICK_FOR_DETAILED_MSG': '%s<br><br>' + '{{ _('Click here for details.')|safe }}',
+    'CLICK_FOR_DETAILED_MSG': '{{ _('Click here for details.')|safe }}',
     'GENERAL_CATEGORY': '{{ _("General")|safe }}',
     'SQL_TAB': '{{ _('SQL') }}',
     'SQL_INCOMPLETE': '{{ _('Incomplete definition') }}',
@@ -24,7 +24,7 @@ function(_, S, pgAdmin) {
     'NODE_HAS_NO_STATISTICS': "{{ _("No statistics are available for the selected object.") }}",
     'TRUE': "{{ _("True") }}",
     'FALSE': "{{ _("False") }}",
-    'NOTE_CTRL_LABEL': "{{ _("Note") }}",
+    'NOTE_CTRL_LABEL': "{{ _("Note") }}"
   };
 
 {% for key in current_app.messages.keys() %}

web/pgadmin/misc/bgprocess/__init__.py

@@ -0,0 +1,122 @@
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2016, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+"""
+A blueprint module providing utility functions for the notify the user about
+the long running background-processes.
+"""
+from flask import url_for
+from flask.ext.babel import gettext as _
+from flask.ext.security import login_required
+
+from pgadmin.utils.ajax import make_response, gone, bad_request, success_return
+from pgadmin.utils import PgAdminModule
+
+from .processes import BatchProcess
+
+MODULE_NAME = 'bgprocess'
+
+
+class BGProcessModule(PgAdminModule):
+
+    def get_own_javascripts(self):
+        return [{
+            'name': 'pgadmin.browser.bgprocess',
+            'path': url_for('bgprocess.static', filename='js/bgprocess'),
+            'when': None
+            }]
+
+    def get_own_stylesheets(self):
+        """
+        Returns:
+            list: the stylesheets used by this module.
+        """
+        stylesheets = [
+            url_for('bgprocess.static', filename='css/bgprocess.css')
+            ]
+        return stylesheets
+
+    def get_own_messages(self):
+        """
+        Returns:
+            dict: the i18n messages used by this module
+        """
+        return {
+            'bgprocess.index': url_for("bgprocess.index"),
+            'bgprocess.list': url_for("bgprocess.list"),
+            'seconds': _('seconds'),
+            'started': _('Started'),
+            'START_TIME': _('Start time'),
+            'STATUS': _('Status'),
+            'EXECUTION_TIME': _('Execution time'),
+            'running': _('Running...'),
+            'successfully_finished': _("Successfully Finished!"),
+            'failed_with_exit_code': _("Failed (Exit code: %%s).")
+        }
+
+# Initialise the module
+blueprint = BGProcessModule(
+    MODULE_NAME, __name__, url_prefix='/misc/bgprocess'
+)
+
+
+@blueprint.route('/')
+@login_required
+def index():
+    return bad_request(errormsg=_('User can not call this url directly'))
+
+
+@blueprint.route('/status/<pid>/', methods=['GET'])
+@blueprint.route('/status/<pid>/<int:out>/<int:err>/', methods=['GET'])
+@login_required
+def status(pid, out=-1, err=-1):
+    """
+    Check the status of the process running in background.
+    Sends back the output of stdout/stderr
+    Fetches & sends STDOUT/STDERR logs for the process requested by client
+
+    Args:
+        pid:  Process ID
+        out: position of the last stdout fetched
+        err: position of the last stderr fetched
+
+    Returns:
+        Status of the process and logs (if out, and err not equal to -1)
+    """
+    try:
+        process = BatchProcess(id=pid)
+
+        return make_response(response=process.status(out, err))
+    except LookupError as lerr:
+        return gone(errormsg=str(lerr))
+
+
+@blueprint.route('/list/', methods=['GET'])
+def list():
+    return make_response(response=BatchProcess.list())
+
+
+@blueprint.route('/acknowledge/<pid>/', methods=['PUT'])
+@login_required
+def acknowledge(pid):
+    """
+    User has acknowledge the process
+
+    Args:
+        pid:  Process ID
+
+    Returns:
+        Positive status
+    """
+    try:
+        BatchProcess.acknowledge(pid, True)
+
+        return success_return()
+    except LookupError as lerr:
+        return gone(errormsg=str(lerr))

web/pgadmin/misc/bgprocess/process_executor.py

@@ -0,0 +1,375 @@
+# -*- coding: utf-8 -*-
+
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2016, The pgAdmin Development Team
+# This software is released under the PostgreSQL License
+#
+##########################################################################
+
+"""
+This python script is responsible for executing a process, and logs its output,
+and error in the given output directory.
+
+We will create a detached process, which executes this script.
+
+This script will:
+* Fetch the configuration from the given database.
+* Run the given executable specified in the configuration with the arguments.
+* Create log files for both stdout, and stdout.
+* Update the start time, end time, exit code, etc in the configuration
+  database.
+
+Args:
+  process_id -- Process id
+  db_file -- Database file which holds list of processes to be executed
+  output_directory -- output directory
+"""
+from __future__ import print_function, unicode_literals
+
+# To make print function compatible with python2 & python3
+import sys
+import os
+import argparse
+import sqlite3
+from datetime import datetime
+from subprocess import Popen, PIPE
+from threading import Thread
+import csv
+import pytz
+import codecs
+
+
+# SQLite3 needs all string as UTF-8
+# We need to make string for Python2/3 compatible
+if sys.version_info < (3,):
+    from io import StringIO
+
+    def u(x):
+        return codecs.unicode_escape_decode(x)[0]
+else:
+    from cStringIO import StringIO
+
+    def u(x):
+        return x
+
+
+def usage():
+    """
+    This function will display usage message.
+
+    Args:
+        None
+
+    Returns:
+        Displays help message
+    """
+
+    help_msg = """
+Usage:
+
+executer.py [-h|--help]
+    [-p|--process] Process ID
+    [-d|--db_file] SQLite3 database file path
+"""
+    print(help_msg)
+
+
+def get_current_time(format='%Y-%m-%d %H:%M:%S.%f %z'):
+    return datetime.utcnow().replace(
+        tzinfo=pytz.utc
+    ).strftime(format)
+
+
+class ProcessLogger(Thread):
+    """
+    This class definition is responsible for capturing & logging
+    stdout & stderr messages from subprocess
+
+    Methods:
+    --------
+    * __init__(stream_type, configs)
+     - This method is use to initlize the ProcessLogger class object
+
+    * logging(msg)
+     - This method is use to log messages in sqlite3 database
+
+    * run()
+     - Reads the stdout/stderr for messages and sent them to logger
+    """
+
+    def __init__(self, stream_type, configs):
+        """
+        This method is use to initialize the ProcessLogger class object
+
+        Args:
+            stream_type: Type of STD (std)
+            configs: Process details dict
+
+        Returns:
+            None
+        """
+        Thread.__init__(self)
+        self.configs = configs
+        self.process = None
+        self.stream = None
+        self.logger = codecs.open(
+            os.path.join(
+                configs['output_directory'], stream_type
+            ), 'w', "utf-8"
+        )
+
+    def attach_process_stream(self, process, stream):
+        """
+        This function will attach a process and its stream with this thread.
+
+        Args:
+            process: Process
+            stream: Stream attached with the process
+
+        Returns:
+            None
+        """
+        self.process = process
+        self.stream = stream
+
+    def log(self, msg):
+        """
+        This function will update log file
+
+        Args:
+            msg: message
+
+        Returns:
+            None
+        """
+        # Write into log file
+        if self.logger:
+            if msg:
+                self.logger.write(
+                    str('{0},{1}').format(
+                        get_current_time(format='%Y%m%d%H%M%S%f'), msg
+                    )
+                )
+            return True
+        return False
+
+    def run(self):
+        if self.process and self.stream:
+            while True:
+                nextline = self.stream.readline()
+
+                if nextline:
+                    self.log(nextline)
+                else:
+                    if self.process.poll() is not None:
+                        break
+
+    def release(self):
+        if self.logger:
+            self.logger.close()
+            self.logger = None
+
+
+def read_configs(data):
+    """
+    This reads SQLite3 database and fetches process details
+
+    Args:
+        data - configuration details
+
+    Returns:
+        Process details fetched from database as a dict
+    """
+    if data.db_file is not None and data.process_id is not None:
+        conn = sqlite3.connect(data.db_file)
+        c = conn.cursor()
+        t = (data.process_id,)
+
+        c.execute('SELECT command, arguments FROM process WHERE \
+            exit_code is NULL \
+            AND pid=?', t)
+
+        row = c.fetchone()
+        conn.close()
+
+        if row and len(row) > 1:
+            configs = {
+                'pid': data.process_id,
+                'cmd': row[0],
+                'args': row[1],
+                'output_directory': data.output_directory,
+                'db_file': data.db_file
+            }
+            return configs
+        else:
+            return None
+    else:
+        raise ValueError("Please verify process id and db_file arguments")
+
+
+def update_configs(kwargs):
+    """
+    This function will updates process stats
+
+    Args:
+        kwargs - Process configuration details
+
+    Returns:
+        None
+    """
+    if 'db_file' in kwargs and 'pid' in kwargs:
+        conn = sqlite3.connect(kwargs['db_file'])
+        sql = 'UPDATE process SET '
+        params = list()
+
+        for param in ['start_time', 'end_time', 'exit_code']:
+            if param in kwargs:
+                sql += (',' if len(params) else '') + param + '=? '
+                params.append(kwargs[param])
+
+        if len(params) == 0:
+            return
+
+        sql += 'WHERE pid=?'
+        params.append(kwargs['pid'])
+
+        with conn:
+            c = conn.cursor()
+            c.execute(sql, params)
+            conn.commit()
+
+        # Commit & close cursor
+        conn.close()
+    else:
+        raise ValueError("Please verify pid and db_file arguments")
+
+
+def execute(configs):
+    """
+    This function will execute the background process
+
+    Args:
+        configs: Process configuration details
+
+    Returns:
+        None
+    """
+    if configs is not None:
+        command = [configs['cmd']]
+        args_csv = StringIO(configs['args'])
+        args_reader = csv.reader(args_csv, delimiter=str(','))
+        for args in args_reader:
+            command = command + args
+        args = {
+            'pid': configs['pid'],
+            'db_file': configs['db_file']
+        }
+
+        reload(sys)
+        sys.setdefaultencoding('utf8')
+
+        # Create seprate thread for stdout and stderr
+        process_stdout = ProcessLogger('out', configs)
+        process_stderr = ProcessLogger('err', configs)
+
+        try:
+            # update start_time
+            args.update({
+                'start_time': get_current_time(),
+                'stdout': process_stdout.log,
+                'stderr': process_stderr.log
+            })
+
+            # Update start time
+            update_configs(args)
+
+            process = Popen(
+                command, stdout=PIPE, stderr=PIPE, stdin=PIPE,
+                shell=(os.name == 'nt'), close_fds=(os.name != 'nt')
+            )
+
+            # Attach the stream to the process logger, and start logging.
+            process_stdout.attach_process_stream(process, process.stdout)
+            process_stdout.start()
+            process_stderr.attach_process_stream(process, process.stderr)
+            process_stderr.start()
+
+            # Join both threads together
+            process_stdout.join()
+            process_stderr.join()
+
+            # Child process return code
+            exitCode = process.wait()
+
+            if exitCode is None:
+                exitCode = process.poll()
+
+            args.update({'exit_code': exitCode})
+
+            # Add end_time
+            args.update({'end_time': get_current_time()})
+
+            # Fetch last output, and error from process if it has missed.
+            data = process.communicate()
+            if data:
+                if data[0]:
+                    process_stdout.log(data[0])
+                if data[1]:
+                    process_stderr.log(data[1])
+
+        # If executable not found or invalid arguments passed
+        except OSError as e:
+            if process_stderr:
+                process_stderr.log(e.strerror)
+            else:
+                print("WARNING: ", e.strerror, file=sys.stderr)
+            args.update({'end_time': get_current_time()})
+            args.update({'exit_code': e.errno})
+
+        # Unknown errors
+        except Exception as e:
+            if process_stderr:
+                process_stderr.log(str(e))
+            else:
+                print("WARNING: ", str(e), file=sys.stderr)
+            args.update({'end_time': get_current_time()})
+            args.update({'exit_code': -1})
+        finally:
+            # Update the execution end_time, and exit-code.
+            update_configs(args)
+            if process_stderr:
+                process_stderr.release()
+                process_stderr = None
+            if process_stdout:
+                process_stdout.release()
+                process_stdout = None
+
+    else:
+        raise ValueError("Please verify configs")
+
+
+if __name__ == '__main__':
+    # Read command line arguments
+    parser = argparse.ArgumentParser(
+        description='Process executor for pgAdmin4'
+        )
+    parser.add_argument(
+        '-p', '--process_id', help='Process ID', required=True
+        )
+    parser.add_argument(
+        '-d', '--db_file', help='Configuration Database', required=True
+        )
+    parser.add_argument(
+        '-o', '--output_directory',
+        help='Location where the logs will be created', required=True
+        )
+    args = parser.parse_args()
+
+    # Fetch bakcground process details from SQLite3 database file
+    configs = read_configs(args)
+
+    # Execute the background process
+    execute(configs)

web/pgadmin/misc/bgprocess/processes.py

@@ -0,0 +1,385 @@
+# -*- coding: utf-8 -*-
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2016, The pgAdmin Development Team
+# This software is released under the PostgreSQL License
+#
+##########################################################################
+
+"""
+Introduce a function to run the process executor in detached mode.
+"""
+from __future__ import print_function, unicode_literals
+from abc import ABCMeta, abstractproperty
+import csv
+from datetime import datetime
+from dateutil import parser
+import os
+from pickle import dumps, loads
+import pytz
+from subprocess import Popen, PIPE
+import sys
+import types
+
+from flask.ext.babel import gettext
+from flask.ext.security import current_user
+
+import config
+from pgadmin.model import Process, db
+
+if sys.version_info < (3,):
+    from StringIO import StringIO
+else:
+    from cStringIO import StringIO
+
+
+def get_current_time(format='%Y-%m-%d %H:%M:%S.%f %z'):
+    """
+    Generate the current time string in the given format.
+    """
+    return datetime.utcnow().replace(
+        tzinfo=pytz.utc
+    ).strftime(format)
+
+
+class IProcessDesc(object):
+    __metaclass__ = ABCMeta
+
+    @abstractproperty
+    def message(self):
+        pass
+
+    @abstractproperty
+    def details(self):
+        pass
+
+
+class BatchProcess(object):
+
+    def __init__(self, **kwargs):
+
+        self.id = self.desc = self.cmd = self.args = self.log_dir = \
+            self.stdout = self.stderr = self.stime = self.etime = \
+            self.ecode = None
+
+        if 'id' in kwargs:
+            self._retrieve_process(kwargs['id'])
+        else:
+            self._create_process(kwargs['desc'], kwargs['cmd'], kwargs['args'])
+
+    def _retrieve_process(self, _id):
+        p = Process.query.filter_by(pid=_id, user_id=current_user.id).first()
+
+        if p is None:
+            raise LookupError(gettext(
+                "Couldn't find the process specified by the id!"
+            ))
+
+        # ID
+        self.id = _id
+        # Description
+        self.desc = loads(p.desc)
+        # Status Acknowledged time
+        self.atime = p.acknowledge
+        # Command
+        self.cmd = p.command
+        # Arguments
+        self.args = p.arguments
+        # Log Directory
+        self.log_dir = p.logdir
+        # Standard ouput log file
+        self.stdout = os.path.join(p.logdir, 'out')
+        # Standard error log file
+        self.stderr = os.path.join(p.logdir, 'err')
+        # Start time
+        self.stime = p.start_time
+        # End time
+        self.etime = p.end_time
+        # Exit code
+        self.ecode = p.exit_code
+
+    def _create_process(self, _desc, _cmd, _args):
+        ctime = get_current_time(format='%y%m%d%H%M%S%f')
+        log_dir = os.path.join(
+            config.SESSION_DB_PATH, 'process_logs'
+            )
+
+        def random_number(size):
+            import random
+            import string
+
+            return ''.join(
+                random.choice(
+                    string.ascii_uppercase + string.digits
+                ) for _ in range(size)
+            )
+
+        created = False
+        size = 0
+        id = ctime
+        while not created:
+            try:
+                id += random_number(size)
+                log_dir = os.path.join(log_dir, id)
+                size += 1
+                if not os.path.exists(log_dir):
+                    os.makedirs(log_dir, int('700', 8))
+                    created = True
+            except OSError as oe:
+                import errno
+                if oe.errno != errno.EEXIST:
+                    raise
+
+        # ID
+        self.id = ctime
+        # Description
+        self.desc = _desc
+        # Status Acknowledged time
+        self.atime = None
+        # Command
+        self.cmd = _cmd
+        # Log Directory
+        self.log_dir = log_dir
+        # Standard ouput log file
+        self.stdout = os.path.join(log_dir, 'out')
+        # Standard error log file
+        self.stderr = os.path.join(log_dir, 'err')
+        # Start time
+        self.stime = None
+        # End time
+        self.etime = None
+        # Exit code
+        self.ecode = None
+
+        # Arguments
+        args_csv_io = StringIO()
+        csv_writer = csv.writer(
+            args_csv_io, delimiter=str(','), quoting=csv.QUOTE_MINIMAL
+        )
+        csv_writer.writerow(_args)
+        self.args = args_csv_io.getvalue().strip(str('\r\n'))
+
+        j = Process(
+            pid=int(id), command=_cmd, arguments=self.args, logdir=log_dir,
+            desc=dumps(self.desc), user_id=current_user.id
+        )
+        db.session.add(j)
+        db.session.commit()
+
+    def start(self):
+        if self.stime is not None:
+            if self.etime is None:
+                raise Exception(gettext('Process has already been started!'))
+            raise Exception(gettext(
+                'Process has already been finished, it can not be restared!'
+            ))
+
+        executor = os.path.join(
+            os.path.dirname(__file__), 'process_executor.py'
+        )
+
+        p = None
+        cmd = [
+            sys.executable or 'python',
+            executor,
+            '-p', self.id,
+            '-o', self.log_dir,
+            '-d', config.SQLITE_PATH
+        ]
+
+        if os.name == 'nt':
+            p = Popen(
+                cmd, stdout=None, stderr=None, stdin=None, close_fds=True,
+                shell=False, creationflags=0x00000008
+            )
+        else:
+            def preexec_function():
+                import signal
+                # Detaching from the parent process group
+                os.setpgrp()
+                # Explicitly ignoring signals in the child process
+                signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+            p = Popen(
+                cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, close_fds=True,
+                shell=False, preexec_fn=preexec_function
+            )
+
+        self.ecode = p.poll()
+        if self.ecode is not None:
+            # TODO:: Couldn't start execution
+            pass
+
+
+    def status(self, out=0, err=0):
+        ctime = get_current_time(format='%Y%m%d%H%M%S%f')
+
+        stdout = []
+        stderr = []
+        out_completed = err_completed = False
+        process_output = (out != -1 and err != -1)
+
+        def read_log(logfile, log, pos, ctime, check=True):
+            completed = True
+            lines = 0
+
+            if not os.path.isfile(logfile):
+                return 0
+
+            with open(logfile, 'r') as stream:
+                stream.seek(pos)
+                for line in stream:
+                    logtime = StringIO()
+                    idx = 0
+                    for c in line:
+                        idx += 1
+                        if c == ',':
+                            break
+                        logtime.write(c)
+                    logtime = logtime.getvalue()
+
+                    if check and logtime > ctime:
+                        completed = False
+                        break
+                    if lines == 5120:
+                        ctime = logtime
+                        completed = False
+                        break
+
+                    lines += 1
+                    pos = stream.tell()
+                    log.append([logtime, line[idx:]])
+
+            return pos, completed
+
+        if process_output:
+            out, out_completed = read_log(
+                self.stdout, stdout, out, ctime, True
+            )
+            err, err_completed = read_log(
+                self.stderr, stderr, err, ctime, True
+            )
+
+        j = Process.query.filter_by(
+            pid=self.id, user_id=current_user.id
+        ).first()
+
+        execution_time = None
+
+        if j is not None:
+            self.stime = j.start_time
+            self.etime = j.end_time
+            self.ecode = j.exit_code
+
+            if self.stime is not None:
+                stime = parser.parse(self.stime)
+                etime = parser.parse(self.etime or get_current_time())
+
+                execution_time = (etime - stime).total_seconds()
+
+            if process_output and self.ecode is not None and (
+                len(stdout) + len(stderr) < 3073
+            ):
+                out, out_completed = read_log(
+                    self.stdout, stdout, out, ctime, False
+                )
+                err, err_completed = read_log(
+                    self.stderr, stderr, err, ctime, False
+                )
+        else:
+            out_completed = err_completed = False
+
+        if out == -1 or err == -1:
+            return {
+                'exit_code': self.ecode,
+                'execution_time': execution_time
+            }
+
+        return {
+            'out': {'pos': out, 'lines': stdout, 'done': out_completed},
+            'err': {'pos': err, 'lines': stderr, 'done': err_completed},
+            'exit_code': self.ecode,
+            'execution_time': execution_time
+        }
+
+    @staticmethod
+    def list():
+        processes = Process.query.filter_by(user_id=current_user.id)
+
+        res = []
+        for p in processes:
+            if p.start_time is None or p.acknowledge is not None:
+                continue
+            execution_time = None
+
+            stime = parser.parse(p.start_time)
+            etime = parser.parse(p.end_time or get_current_time())
+
+            execution_time = (etime - stime).total_seconds()
+            desc = loads(p.desc)
+            details = desc
+
+            if not isinstance(desc, types.StringTypes):
+                details = desc.details
+                desc = desc.message
+
+            res.append({
+                'id': p.pid,
+                'desc': desc,
+                'details': details,
+                'stime': (
+                    stime - datetime(
+                        1970, 1, 1, tzinfo=pytz.utc
+                    )
+                ).total_seconds(),
+                'etime': p.end_time,
+                'exit_code': p.exit_code,
+                'acknowledge': p.acknowledge,
+                'execution_time': execution_time
+            })
+
+        return res
+
+    @staticmethod
+    def acknowledge(_pid, _release):
+        p = Process.query.filter_by(
+            user_id=current_user.id, pid=_pid
+        ).first()
+
+        if p is None:
+            raise LookupError(gettext(
+                "Couldn't find the process specified by the id!"
+            ))
+
+        if _release:
+            import shutil
+            shutil.rmtree(p.logdir, True)
+            db.session.delete(p)
+        else:
+            p.acknowledge = get_current_time()
+
+        db.session.commit()
+
+    @staticmethod
+    def release(pid=None):
+        import shutil
+        processes = None
+
+        if pid is not None:
+            processes = Process.query.filter_by(
+                user_id=current_user.id, pid=pid
+            )
+        else:
+            processes = Process.query.filter_by(
+                user_id=current_user.id,
+                acknowledge=None
+            )
+
+        if processes:
+            for p in processes:
+                shutil.rmtree(p.logdir, True)
+
+                db.session.delete(p)
+                db.session.commit()

web/pgadmin/misc/bgprocess/static/css/bgprocess.css

@@ -0,0 +1,153 @@
+.ajs-bg-bgprocess.ajs-visible {
+  padding: 0px !important;
+}
+
+.ajs-bg-bgprocess > .pg-bg-bgprocess {
+  background-color: #31708F;
+  color: #FFFFFF;
+  padding: 0px;
+  border-radius: 5px;
+  text-align: left;
+}
+
+.ajs-bg-bgprocess .col-xs-12 {
+  padding-right: 5px;
+  padding-left: 5px;
+}
+
+.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-notify-header {
+  background-color: #1B4A5A;
+  margin-top: 0px;
+  margin-bottom: 5px;
+  font-weight: 900;
+  padding: 5px;
+  white-space: pre-wrap;
+  border-top-left-radius: 5px;
+  border-top-right-radius: 5px;
+}
+
+.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-notify-body {
+  font-family: monospace;
+}
+
+.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-status {
+  padding: 2px;
+  font-weight: 700;
+  margin: 5px;
+  width: calc(100% - 10px);
+  text-align: center;
+  border-radius: 2px;
+  -moz-border-radius: 2px;
+  -webkit-border-radius: 2px;
+}
+
+.pg-bg-process-logs {
+  width: 100%;
+}
+
+.pg-bg-etime {
+  width: 100%;
+  display: block;
+  font-size: 95%;
+  padding: 5px;
+  font-weight: bold;
+  color: #fff;
+  text-align: center;
+  white-space: nowrap;
+  vertical-align: baseline;
+  border-radius: .25em;
+  background-color: #777;
+}
+
+.pg-bg-click {
+  color: rgb(221, 194, 174);
+  text-decoration: underline;
+  cursor: pointer;
+}
+
+.pg-bg-click:hover {
+  color: darkblue;
+}
+
+.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-status.bg-success,
+.bg-process-status .bg-bgprocess-success {
+  color: green;
+}
+
+.bg-process-status .bg-bgprocess-failed {
+  color: red;
+}
+
+.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-status.bg-failed {
+  color: black;
+  background-color: #E99595;
+}
+
+.pg-panel-content div.bg-process-watcher.col-xs-12 {
+  height: 100%;
+  padding: 0px;
+  WebkitTransition: all 1s;
+  transition: all 1s;
+}
+
+ol.pg-bg-process-logs {
+  padding: 15px 15px 15px 60px;
+  height: 100%;
+  overflow: auto;
+  width: 100%;
+  background: #000;
+}
+
+.pg-bg-res-out, .pg-bg-res-err {
+  background-color: #000;
+  padding-left: 10px;
+  white-space: pre-wrap;
+  font-size: 12px;
+}
+
+.pg-bg-res-out {
+  color: rgb(0, 157, 207);
+}
+
+.pg-bg-res-err {
+  color: rgba(212, 27, 57, 0.81);
+}
+
+.pg-panel-content .bg-process-details {
+  padding: 10px 15px;
+  min-height: 70px;
+  color: #000;
+}
+
+.pg-panel-content .bg-process-stats p{
+  display: inline;
+  padding-left: 5px;
+  margin-bottom: 0;
+  font-size: 13px;
+}
+
+.pg-panel-content .bg-process-footer {
+  border-top: 1px solid #ccc;
+  padding: 10px 15px;
+  position: absolute;
+  bottom: 0;
+  background-color: white;
+}
+
+.pg-panel-content .bg-process-footer p {
+  display: inline;
+  padding-left: 5px;
+  font-size: 13px;
+}
+
+.pg-panel-content .bg-process-footer b, .bg-process-stats span b {
+  color: #285173;
+}
+
+.bg-process-footer .bg-process-status {
+  padding-left: 0;
+}
+
+.bg-process-footer .bg-process-exec-time {
+  padding-right: 0;
+}

web/pgadmin/misc/bgprocess/static/js/bgprocess.js

@@ -0,0 +1,500 @@
+define(
+   ['underscore', 'underscore.string', 'jquery', 'pgadmin.browser', 'alertify', 'pgadmin.browser.messages'],
+function(_, S, $, pgBrowser, alertify, pgMessages) {
+
+  pgBrowser.BackgroundProcessObsorver = pgBrowser.BackgroundProcessObsorver || {};
+
+  if (pgBrowser.BackgroundProcessObsorver.initialized) {
+    return pgBrowser.BackgroundProcessObsorver;
+  }
+
+  var BGProcess = function(info, notify) {
+    var self = this;
+    setTimeout(
+      function() {
+        self.initialize.apply(self, [info, notify]);
+      }, 1
+    );
+  };
+
+  _.extend(
+    BGProcess.prototype, {
+      initialize: function(info, notify) {
+        _.extend(this, {
+          details: false,
+          notify: (_.isUndefined(notify) || notify),
+          curr_status: null,
+          state: 0, // 0: NOT Started, 1: Started, 2: Finished
+          completed: false,
+
+          id: info['id'],
+          desc: null,
+          detailed_desc: null,
+          stime: null,
+          exit_code: null,
+          acknowledge: info['acknowledge'],
+          execution_time: null,
+          out: null,
+          err: null,
+
+          notifier: null,
+          container: null,
+          panel: null,
+          logs: $('<ol></ol>', {class: 'pg-bg-process-logs'})
+        });
+
+        if (this.notify) {
+          pgBrowser.Events && pgBrowser.Events.on(
+            'pgadmin-bgprocess:started:' + this.id,
+            function(process) {
+              if (!process.notifier)
+                process.show.apply(process);
+            }
+          );
+          pgBrowser.Events && pgBrowser.Events.on(
+            'pgadmin-bgprocess:finished:' + this.id,
+            function(process) {
+              if (!process.notifier)
+                process.show.apply(process);
+            }
+          )
+        }
+        var self = this;
+
+        setTimeout(
+          function() {
+            self.update.apply(self, [info]);
+          }, 1
+        );
+      },
+
+      url: function(type) {
+        var base_url = pgMessages['bgprocess.index'],
+            url = base_url;
+
+        switch (type) {
+          case 'status':
+            url = S('%sstatus/%s/').sprintf(base_url, this.id).value();
+            if (this.details) {
+              url = S('%s%s/%s/').sprintf(
+                url, (this.out && this.out.pos) || 0,
+                (this.err && this.err.pos) || 0
+              ).value();
+            }
+            break;
+          case 'info':
+          case 'acknowledge':
+            url = S('%s%s/%s/').sprintf(base_url, type, this.id).value();
+            break;
+        }
+
+        return url;
+      },
+
+      update: function(data) {
+        var self = this,
+            out = [],
+            err = [],
+            idx = 0;
+
+        if ('stime' in data)
+          self.stime = new Date(data.stime);
+
+        if ('execution_time' in data)
+          self.execution_time = parseFloat(data.execution_time);
+
+        if ('desc' in data)
+          self.desc = data.desc;
+
+        if ('details' in data)
+          self.detailed_desc = data.details;
+
+        if ('exit_code' in data)
+          self.exit_code = data.exit_code;
+
+        if ('out' in data) {
+          self.out = data.out && data.out.pos;
+          self.completed = data.out.done;
+
+          if (data.out && data.out.lines) {
+            data.out.lines.sort(function(a, b) { return a[0] < b[0]; });
+            out = data.out.lines;
+          }
+        }
+
+        if ('err' in data) {
+          self.err = data.err && data.err.pos;
+          self.completed = (self.completed && data.err.done);
+
+          if (data.err && data.err.lines) {
+            data.err.lines.sort(function(a, b) { return a[0] < b[0]; });
+            err = data.err.lines;
+          }
+        }
+
+        var io = ie = 0;
+
+        while (io < out.length && ie < err.length &&
+               self.logs[0].children.length < 5120) {
+          if (out[io][0] < err[ie][0]){
+            self.logs.append(
+              $('<li></li>', {class: 'pg-bg-res-out'}).text(out[io++][1])
+            );
+          } else {
+            self.logs.append(
+              $('<li></li>', {class: 'pg-bg-res-err'}).text(err[ie++][1])
+            );
+          }
+        }
+
+        while (io < out.length && self.logs[0].children.length < 5120) {
+          self.logs.append(
+            $('<li></li>', {class: 'pg-bg-res-out'}).text(out[io++][1])
+          );
+        }
+
+        while (ie < err.length && self.logs[0].children.length < 5120) {
+          self.logs.append(
+            $('<li></li>', {class: 'pg-bg-res-err'}).text(err[ie++][1])
+          );
+        }
+
+        if (self.logs[0].children.length >= 5120) {
+          self.completed = true;
+        }
+
+        if (self.stime) {
+          self.curr_status = pgMessages['started'];
+
+          if (self.execution_time >= 2) {
+            self.curr_status = pgMessages['running'];
+          }
+
+          if ('execution_time' in data) {
+            self.execution_time = self.execution_time + ' ' +
+              pgMessages['seconds'];
+          }
+
+          if (!_.isNull(self.exit_code)) {
+            if (self.exit_code == 0) {
+              self.curr_status = pgMessages['successfully_finished'];
+            } else {
+              self.curr_status = S(
+                pgMessages['failed_with_exit_code']
+              ).sprintf(String(self.exit_code)).value();
+            }
+          }
+
+          if (self.state == 0 && self.stime) {
+            self.state = 1;
+            pgBrowser.Events && pgBrowser.Events.trigger(
+              'pgadmin-bgprocess:started:' + self.id, self, self
+            );
+          }
+
+          if (self.state == 1 && !_.isNull(self.exit_code)) {
+            self.state = 2;
+            pgBrowser.Events && pgBrowser.Events.trigger(
+              'pgadmin-bgprocess:finished:' + self.id, self, self
+            );
+          }
+
+          setTimeout(function() {self.show.apply(self)}, 10);
+        }
+
+        if (self.state != 2 || (self.details && !self.completed)) {
+          setTimeout(
+            function() {
+              self.status.apply(self);
+            }, 1000
+          );
+        }
+      },
+
+      status: function() {
+        var self = this;
+
+        $.ajax({
+          typs: 'GET',
+          timeout: 30000,
+          url: self.url('status'),
+          cache: false,
+          async: true,
+          contentType: "application/json",
+          success: function(res) {
+            setTimeout(function() { self.update(res); }, 10);
+          },
+          error: function(res) {
+            // Try after some time
+            setTimeout(function() { self.update(res); }, 10000);
+          }
+        });
+      },
+
+      show: function() {
+        var self = this;
+
+        if (self.notify && !self.details) {
+          if (!self.notifier) {
+            var content = $('<div class="pg-bg-bgprocess row"></div>').append(
+                  $('<div></div>', {
+                    class: "col-xs-12 h3 pg-bg-notify-header"
+                  }).text(
+                    self.desc
+                  )
+                ).append(
+                  $('<div></div>', {class: 'pg-bg-notify-body' }).append(
+                    $('<div></div>', {class: 'pg-bg-start col-xs-12' }).append(
+                      $('<div></div>').text(self.stime.toString())
+                    ).append(
+                      $('<div class="pg-bg-etime"></div>')
+                    )
+                  )
+                ),
+                for_details = $('<div></div>', {
+                  class: "col-xs-12 text-center pg-bg-click"
+                }).text(pgMessages.CLICK_FOR_DETAILED_MSG).appendTo(content),
+                status = $('<div></div>', {
+                  class: "pg-bg-status col-xs-12 " + ((self.exit_code === 0) ?
+                      'bg-success': (self.exit_code == 1) ?
+                      'bg-failed' : '')
+                }).appendTo(content);
+
+            self.container = content;
+            self.notifier = alertify.notify(
+              content.get(0), 'bg-bgprocess', 0, null
+            );
+
+            for_details.on('click', function(ev) {
+              ev = ev || window.event;
+              ev.cancelBubble = true;
+              ev.stopPropagation();
+
+              this.notifier.dismiss();
+              this.notifier = null;
+
+              this.show_detailed_view.apply(this);
+            }.bind(self));
+
+            // Do not close the notifier, when clicked on the container, which
+            // is a default behaviour.
+            content.on('click', function(ev) {
+              ev = ev || window.event;
+              ev.cancelBubble = true;
+              ev.stopPropagation();
+
+              return;
+            });
+          }
+          // TODO:: Formatted execution time
+          self.container.find('.pg-bg-etime').empty().text(
+            String(self.execution_time)
+          );
+          self.container.find('.pg-bg-status').empty().append(
+             self.curr_status
+          )
+        }
+      },
+
+      show_detailed_view: function() {
+        var self = this,
+            panel = this.panel =
+              pgBrowser.BackgroundProcessObsorver.create_panel();
+
+        panel.title('Process Watcher - ' + self.desc);
+        panel.focus();
+
+        var container = panel.$container,
+            status_class = (
+              (self.exit_code === 0) ?
+                'bg-bgprocess-success': (self.exit_code == 1) ?
+                  'bg-bgprocess-failed' : ''
+            ),
+            $logs = container.find('.bg-process-watcher'),
+            $header = container.find('.bg-process-details'),
+            $footer = container.find('.bg-process-footer');
+
+
+        // set logs
+        $logs.html(self.logs);
+
+        // set bgprocess detailed description
+        $header.find('.bg-detailed-desc').html(self.detailed_desc);
+
+        // set bgprocess start time
+        $header.find('.bg-process-stats .bgprocess-start-time').html(self.stime);
+
+        // set status
+        $footer.find('.bg-process-status p').addClass(
+          status_class
+        ).html(
+          self.curr_status
+        );
+
+        // set bgprocess execution time
+        $footer.find('.bg-process-exec-time p').html(self.execution_time);
+
+        self.details = true;
+        setTimeout(
+          function() {
+            self.status.apply(self);
+          }, 1000
+        );
+
+        var resize_log_container = function($logs, $header, $footer) {
+          var h = $header.outerHeight() + $footer.outerHeight();
+          $logs.css('padding-bottom', h);
+        }.bind(panel, $logs, $header, $footer);
+
+        panel.on(wcDocker.EVENT.RESIZED, resize_log_container);
+        panel.on(wcDocker.EVENT.ATTACHED, resize_log_container);
+        panel.on(wcDocker.EVENT.DETACHED, resize_log_container);
+
+        resize_log_container();
+
+        panel.on(wcDocker.EVENT.CLOSED, function(process) {
+          process.panel = null;
+
+          process.details = false;
+          if (process.exit_code != null) {
+            process.acknowledge_server.apply(process);
+          }
+        }.bind(panel, this));
+      },
+
+      acknowledge_server: function() {
+        var self = this;
+        $.ajax({
+          type: 'PUT',
+          timeout: 30000,
+          url: self.url('acknowledge'),
+          cache: false,
+          async: true,
+          contentType: "application/json",
+          success: function(res) {
+            return;
+          },
+          error: function(res) {
+          }
+        });
+      }
+    });
+
+  _.extend(
+    pgBrowser.BackgroundProcessObsorver, {
+      bgprocesses: {},
+      init: function() {
+        var self = this;
+
+        if (self.initialized) {
+          return;
+        }
+        self.initialized = true;
+
+        setTimeout(
+          function() {
+            self.update_process_list.apply(self);
+          }, 1000
+        );
+
+        pgBrowser.Events.on(
+          'pgadmin-bgprocess:created',
+          function() {
+            setTimeout(
+              function() {
+                pgBrowser.BackgroundProcessObsorver.update_process_list();
+              }, 1000
+            );
+          }
+        )
+      },
+
+      update_process_list: function() {
+        var observer = this;
+
+        $.ajax({
+          typs: 'GET',
+          timeout: 30000,
+          url: pgMessages['bgprocess.list'],
+          cache: false,
+          async: true,
+          contentType: "application/json",
+          success: function(res) {
+            if (!res) {
+              // FIXME::
+              // Do you think - we should call the list agains after some
+              // interval?
+              return;
+            }
+            for (idx in res) {
+              var process = res[idx];
+              if ('id' in process) {
+                if (!(process.id in observer.bgprocesses)) {
+                  observer.bgprocesses[process.id] = new BGProcess(process);
+                }
+              }
+            }
+          },
+          error: function(res) {
+            // FIXME:: What to do now?
+          }
+        });
+      },
+
+      create_panel: function() {
+        this.register_panel();
+
+        return pgBrowser.docker.addPanel(
+          'bg_process_watcher',
+          wcDocker.DOCK.FLOAT,
+          null, {
+            w: (screen.width < 700 ?
+                screen.width * 0.95 : screen.width * 0.5),
+                h: (screen.height < 500 ?
+                    screen.height * 0.95 : screen.height * 0.5),
+                    x: (screen.width < 700 ? '2%' : '25%'),
+                    y: (screen.height < 500 ? '2%' : '25%')
+          });
+      },
+
+      register_panel: function() {
+        var w = pgBrowser.docker,
+          panels = w.findPanels('bg_process_watcher');
+
+          if (panels && panels.length >= 1)
+            return;
+
+          var p = new pgBrowser.Panel({
+                name: 'bg_process_watcher',
+                showTitle: true,
+                isCloseable: true,
+                isPrivate: true,
+                content: '<div class="bg-process-details col-xs-12">'+
+                  '<p class="bg-detailed-desc"></p>'+
+                  '<div class="bg-process-stats">'+
+                  '<span><b>' + pgMessages['START_TIME'] + ':</b></span>'+
+                  '<p class="bgprocess-start-time"></p>'+
+                  '</div>'+
+                  '</div>'+
+                  '<div class="bg-process-watcher col-xs-12">'+
+                  '</div>'+
+                  '<div class="bg-process-footer col-xs-12">'+
+                  '<div class="bg-process-status col-xs-6">'+
+                  '<span><b>' + pgMessages['STATUS'] + ':</b></span><p></p>'+
+                  '</div>'+
+                  '<div class="bg-process-exec-time col-xs-6">'+
+                  '<div class="exec-div pull-right">'+
+                  '<span><b>' + pgMessages['EXECUTION_TIME'] + ':</b></span><p></p>'+
+                  '</div>'+
+                  '</div>'+
+                  '</div>',
+                onCreate: function(myPanel, $container) {
+                  $container.addClass('pg-no-overflow');
+                }
+              });
+          p.load(pgBrowser.docker);
+      }
+    });
+
+  return pgBrowser.BackgroundProcessObsorver;
+});

web/pgadmin/misc/sql/__init__.py

@@ -9,10 +9,8 @@
 
 """A blueprint module providing utility functions for the application."""
 
-import datetime
+from flask import url_for
-from flask import session, current_app, url_for
 from pgadmin.utils import PgAdminModule
-import pgadmin.utils.driver as driver
 
 MODULE_NAME = 'sql'
 

web/pgadmin/model/__init__.py

@@ -168,3 +168,22 @@ class DebuggerFunctionArguments(db.Model):
                 nullable=False)
 
     value = db.Column(db.String(), nullable=True)
+
+
+class Process(db.Model):
+    """Define the Process table."""
+    __tablename__ = 'process'
+    pid = db.Column(db.String(), nullable=False, primary_key=True)
+    user_id = db.Column(
+            db.Integer,
+            db.ForeignKey('user.id'),
+            nullable=False
+            )
+    command = db.Column(db.String(), nullable=False)
+    desc = db.Column(db.String(), nullable=False)
+    arguments = db.Column(db.String(), nullable=True)
+    logdir = db.Column(db.String(), nullable=True)
+    start_time = db.Column(db.String(), nullable=True)
+    end_time = db.Column(db.String(), nullable=True)
+    exit_code = db.Column(db.Integer(), nullable=True)
+    acknowledge = db.Column(db.String(), nullable=True)

web/pgadmin/static/js/alertifyjs/pgadmin.defaults.js

@@ -110,7 +110,7 @@ function(alertify, S) {
         if (contentType.indexOf('text/html') == 0) {
           alertify.notify(
               S(
-                window.pgAdmin.Browser.messages.CLICK_FOR_DETAILED_MSG
+                '%s<br><br>' + window.pgAdmin.Browser.messages.CLICK_FOR_DETAILED_MSG
                ).sprintf(promptmsg).value(), type, 0, function() {
                 alertify.pgIframeDialog().show().set({ frameless: false }).set('pg_msg', msg);
               });

web/setup.py

@@ -62,7 +62,7 @@ account:\n""")
 
     # Setup Flask-Security
     user_datastore = SQLAlchemyUserDatastore(db, User, Role)
-    security = Security(app, user_datastore)
+    Security(app, user_datastore)
 
     with app.app_context():
         password = encrypt_password(p1)
@@ -230,6 +230,23 @@ CREATE TABLE IF NOT EXISTS debugger_function_arguments (
     use_default INTEGER NOT NULL CHECK (use_default >= 0 AND use_default <= 1) ,
     value TEXT,
     PRIMARY KEY (server_id, database_id, schema_id, function_id, arg_id)
+    )""")
+
+        if int(version.value) < 10:
+            db.engine.execute("""
+CREATE TABLE process(
+    user_id INTEGER NOT NULL,
+    pid TEXT NOT NULL,
+    desc TEXT NOT NULL,
+    command TEXT NOT NULL,
+    arguments TEXT,
+    start_time TEXT,
+    end_time TEXT,
+    logdir TEXT,
+    exit_code INTEGER,
+    acknowledge TEXT,
+    PRIMARY KEY(pid),
+    FOREIGN KEY(user_id) REFERENCES user (id)
     )""")
 
     # Finally, update the schema version