Use on-demand loading for results in the query tool. Fixes #2137

With a 27420 row query, pgAdmin III runs the query in 5.873s on my laptop. pgAdmin 4 now takes ~1s.

web/pgadmin/utils/driver/abstract.py

@@ -101,6 +101,12 @@ class BaseConnection(object):
       - Implement this method to execute the given query and returns the result
         as an array of dict (column name -> value) format.
 
+    * def async_fetchmany_2darray(records=-1, formatted_exception_msg=False):
+      - Implement this method to retrieve result of asynchronous connection and
+        polling with no_result flag set to True.
+        This returns the result as a 2 dimensional array.
+        If records is -1 then fetchmany will behave as fetchall.
+
     * connected()
       - Implement this method to get the status of the connection. It should
         return True for connected, otherwise False
@@ -133,7 +139,7 @@ class BaseConnection(object):
       - Implement this method to wait for asynchronous connection with timeout.
         This must be a non blocking call.
 
-    * poll(formatted_exception_msg)
+    * poll(formatted_exception_msg, no_result)
       - Implement this method to poll the data of query running on asynchronous
         connection.
 
@@ -179,6 +185,10 @@ class BaseConnection(object):
     def execute_dict(self, query, params=None, formatted_exception_msg=False):
         pass
 
+    @abstractmethod
+    def async_fetchmany_2darray(self, records=-1, formatted_exception_msg=False):
+        pass
+
     @abstractmethod
     def connected(self):
         pass
@@ -208,7 +218,7 @@ class BaseConnection(object):
         pass
 
     @abstractmethod
-    def poll(self, formatted_exception_msg=True):
+    def poll(self, formatted_exception_msg=True, no_result=False):
         pass
 
     @abstractmethod

web/pgadmin/utils/driver/psycopg2/__init__.py

@@ -1079,6 +1079,55 @@ Failed to execute query (execute_void) for the server #{server_id} - {conn_id}
 
         return True, {'columns': columns, 'rows': rows}
 
+    def async_fetchmany_2darray(self, records=2000, formatted_exception_msg=False):
+        """
+        User should poll and check if status is ASYNC_OK before calling this
+        function
+        Args:
+          records: no of records to fetch. use -1 to fetchall.
+          formatted_exception_msg:
+
+        Returns:
+
+        """
+        cur = self.__async_cursor
+        if not cur:
+            return False, gettext(
+                "Cursor could not be found for the async connection."
+            )
+
+        if self.conn.isexecuting():
+            return False, gettext(
+                "Asynchronous query execution/operation underway."
+            )
+
+        if self.row_count > 0:
+            result = []
+            # For DDL operation, we may not have result.
+            #
+            # Because - there is not direct way to differentiate DML and
+            # DDL operations, we need to rely on exception to figure
+            # that out at the moment.
+            try:
+                if records == -1:
+                    res = cur.fetchall()
+                else:
+                    res = cur.fetchmany(records)
+                for row in res:
+                    new_row = []
+                    for col in self.column_info:
+                        new_row.append(row[col['name']])
+                    result.append(new_row)
+            except psycopg2.ProgrammingError as e:
+                result = None
+        else:
+            # User performed operation which dose not produce record/s as
+            # result.
+            # for eg. DDL operations.
+            return True, None
+
+        return True, result
+
     def connected(self):
         if self.conn:
             if not self.conn.closed:
@@ -1226,7 +1275,7 @@ Failed to reset the connection to the server due to following error:
                 "poll() returned %s from _wait_timeout function" % state
             )
 
-    def poll(self, formatted_exception_msg=False):
+    def poll(self, formatted_exception_msg=False, no_result=False):
         """
         This function is a wrapper around connection's poll function.
         It internally uses the _wait_timeout method to poll the
@@ -1236,6 +1285,7 @@ Failed to reset the connection to the server due to following error:
         Args:
             formatted_exception_msg: if True then function return the formatted
                                      exception message, otherwise error string.
+            no_result: If True then only poll status will be returned.
         """
 
         cur = self.__async_cursor
@@ -1291,23 +1341,23 @@ Failed to reset the connection to the server due to following error:
                     pos += 1
 
             self.row_count = cur.rowcount
+            if not no_result:
+                if cur.rowcount > 0:
+                    result = []
+                    # For DDL operation, we may not have result.
+                    #
+                    # Because - there is not direct way to differentiate DML and
+                    # DDL operations, we need to rely on exception to figure
+                    # that out at the moment.
+                    try:
+                        for row in cur:
+                            new_row = []
+                            for col in self.column_info:
+                                new_row.append(row[col['name']])
+                            result.append(new_row)
 
-            if cur.rowcount > 0:
-                result = []
-                # For DDL operation, we may not have result.
-                #
-                # Because - there is not direct way to differentiate DML and
-                # DDL operations, we need to rely on exception to figure that
-                # out at the moment.
-                try:
-                    for row in cur:
-                        new_row = []
-                        for col in self.column_info:
-                            new_row.append(row[col['name']])
-                        result.append(new_row)
-
-                except psycopg2.ProgrammingError:
-                    result = None
+                    except psycopg2.ProgrammingError:
+                        result = None
 
         return status, result