Use on-demand loading for results in the query tool. Fixes #2137

With a 27420 row query, pgAdmin III runs the query in 5.873s on my laptop. pgAdmin 4 now takes ~1s.
This commit is contained in:
Harshal Dhumal
2017-06-27 09:03:04 -04:00
committed by Dave Page
parent 15cb9fc35b
commit c65158312d
28 changed files with 1953 additions and 887 deletions

View File

@@ -27,7 +27,7 @@ from pgadmin.utils.sqlautocomplete.autocomplete import SQLAutoComplete
from pgadmin.misc.file_manager import Filemanager
from config import PG_DEFAULT_DRIVER
from config import PG_DEFAULT_DRIVER, ON_DEMAND_RECORD_COUNT
MODULE_NAME = 'sqleditor'
@@ -82,9 +82,9 @@ class SqlEditorModule(PgAdminModule):
'sqleditor.view_data_start',
'sqleditor.query_tool_start',
'sqleditor.query_tool_preferences',
'sqleditor.get_columns',
'sqleditor.poll',
'sqleditor.fetch_types',
'sqleditor.fetch',
'sqleditor.fetch_all',
'sqleditor.save',
'sqleditor.get_filter',
'sqleditor.apply_filter',
@@ -261,13 +261,32 @@ def start_view_data(trans_id):
# Check the transaction and connection status
status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id)
# get the default connection as current connection which is attached to
# trans id holds the cursor which has query result so we cannot use that
# connection to execute another query otherwise we'll lose query result.
manager = get_driver(PG_DEFAULT_DRIVER).connection_manager(trans_obj.sid)
default_conn = manager.connection(did=trans_obj.did)
# Connect to the Server if not connected.
if not default_conn.connected():
status, msg = default_conn.connect()
if not status:
return make_json_response(
data={'status': status, 'result': u"{}".format(msg)}
)
if status and conn is not None \
and trans_obj is not None and session_obj is not None:
try:
# set fetched row count to 0 as we are executing query again.
trans_obj.update_fetched_row_cnt(0)
session_obj['command_obj'] = pickle.dumps(trans_obj, -1)
# Fetch the sql and primary_keys from the object
sql = trans_obj.get_sql()
pk_names, primary_keys = trans_obj.get_primary_keys()
pk_names, primary_keys = trans_obj.get_primary_keys(default_conn)
# Fetch the applied filter.
filter_applied = trans_obj.is_filter_applied()
@@ -338,6 +357,8 @@ def start_query_tool(trans_id):
# Use pickle.loads function to get the command object
session_obj = grid_data[str(trans_id)]
trans_obj = pickle.loads(session_obj['command_obj'])
# set fetched row count to 0 as we are executing query again.
trans_obj.update_fetched_row_cnt(0)
can_edit = False
can_filter = False
@@ -467,66 +488,6 @@ def preferences(trans_id):
return success_return()
@blueprint.route(
'/columns/<int:trans_id>', methods=["GET"], endpoint='get_columns'
)
@login_required
def get_columns(trans_id):
"""
This method will returns list of columns of last async query.
Args:
trans_id: unique transaction id
"""
columns = dict()
columns_info = None
primary_keys = None
rset = None
status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id)
if status and conn is not None and session_obj is not None:
ver = conn.manager.version
# Get the template path for the column
template_path = 'column/sql/#{0}#'.format(ver)
command_obj = pickle.loads(session_obj['command_obj'])
if hasattr(command_obj, 'obj_id'):
SQL = render_template("/".join([template_path,
'nodes.sql']),
tid=command_obj.obj_id)
# rows with attribute not_null
status, rset = conn.execute_2darray(SQL)
if not status:
return internal_server_error(errormsg=rset)
# Check PK column info is available or not
if 'primary_keys' in session_obj:
primary_keys = session_obj['primary_keys']
# Fetch column information
columns_info = conn.get_column_info()
if columns_info is not None:
for key, col in enumerate(columns_info):
col_type = dict()
col_type['type_code'] = col['type_code']
col_type['type_name'] = None
if rset:
col_type['not_null'] = col['not_null'] = \
rset['rows'][key]['not_null']
col_type['has_default_val'] = col['has_default_val'] = \
rset['rows'][key]['has_default_val']
columns[col['name']] = col_type
# As we changed the transaction object we need to
# restore it and update the session variable.
session_obj['columns_info'] = columns
update_session_grid_transaction(trans_id, session_obj)
return make_json_response(data={'status': True,
'columns': columns_info,
'primary_keys': primary_keys})
@blueprint.route('/poll/<int:trans_id>', methods=["GET"], endpoint='poll')
@login_required
@@ -539,12 +500,21 @@ def poll(trans_id):
"""
result = None
rows_affected = 0
rows_fetched_from = 0
rows_fetched_to = 0
has_more_rows = False
additional_result = []
columns = dict()
columns_info = None
primary_keys = None
types = {}
client_primary_key = None
rset = None
# Check the transaction and connection status
status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id)
if status and conn is not None and session_obj is not None:
status, result = conn.poll(formatted_exception_msg=True)
status, result = conn.poll(formatted_exception_msg=True, no_result=True)
if not status:
return internal_server_error(result)
elif status == ASYNC_OK:
@@ -559,6 +529,80 @@ def poll(trans_id):
if (trans_status == TX_STATUS_INERROR and
trans_obj.auto_rollback):
conn.execute_void("ROLLBACK;")
st, result = conn.async_fetchmany_2darray(ON_DEMAND_RECORD_COUNT)
if st:
if 'primary_keys' in session_obj:
primary_keys = session_obj['primary_keys']
# Fetch column information
columns_info = conn.get_column_info()
client_primary_key = generate_client_primary_key_name(
columns_info
)
session_obj['client_primary_key'] = client_primary_key
if columns_info is not None:
command_obj = pickle.loads(session_obj['command_obj'])
if hasattr(command_obj, 'obj_id'):
# Get the template path for the column
template_path = 'column/sql/#{0}#'.format(
conn.manager.version
)
SQL = render_template("/".join([template_path,
'nodes.sql']),
tid=command_obj.obj_id)
# rows with attribute not_null
colst, rset = conn.execute_2darray(SQL)
if not colst:
return internal_server_error(errormsg=rset)
for key, col in enumerate(columns_info):
col_type = dict()
col_type['type_code'] = col['type_code']
col_type['type_name'] = None
columns[col['name']] = col_type
if rset:
col_type['not_null'] = col['not_null'] = \
rset['rows'][key]['not_null']
col_type['has_default_val'] = \
col['has_default_val'] = \
rset['rows'][key]['has_default_val']
if columns:
st, types = fetch_pg_types(columns, trans_obj)
if not st:
return internal_server_error(types)
for col_info in columns.values():
for col_type in types:
if col_type['oid'] == col_info['type_code']:
col_info['type_name'] = col_type['typname']
session_obj['columns_info'] = columns
# status of async_fetchmany_2darray is True and result is none
# means nothing to fetch
if result and rows_affected > -1:
res_len = len(result)
if res_len == ON_DEMAND_RECORD_COUNT:
has_more_rows = True
if res_len > 0:
rows_fetched_from = trans_obj.get_fetched_row_cnt()
trans_obj.update_fetched_row_cnt(rows_fetched_from + res_len)
rows_fetched_from += 1
rows_fetched_to = trans_obj.get_fetched_row_cnt()
session_obj['command_obj'] = pickle.dumps(trans_obj, -1)
# As we changed the transaction object we need to
# restore it and update the session variable.
update_session_grid_transaction(trans_id, session_obj)
elif status == ASYNC_EXECUTION_ABORTED:
status = 'Cancel'
else:
@@ -599,53 +643,123 @@ def poll(trans_id):
data={
'status': status, 'result': result,
'rows_affected': rows_affected,
'additional_messages': additional_messages
'rows_fetched_from': rows_fetched_from,
'rows_fetched_to': rows_fetched_to,
'additional_messages': additional_messages,
'has_more_rows': has_more_rows,
'colinfo': columns_info,
'primary_keys': primary_keys,
'types': types,
'client_primary_key': client_primary_key
}
)
@blueprint.route(
'/fetch/types/<int:trans_id>', methods=["GET"], endpoint='fetch_types'
)
@blueprint.route('/fetch/<int:trans_id>', methods=["GET"], endpoint='fetch')
@blueprint.route('/fetch/<int:trans_id>/<int:fetch_all>', methods=["GET"], endpoint='fetch_all')
@login_required
def fetch_pg_types(trans_id):
def fetch(trans_id, fetch_all=None):
result = None
has_more_rows = False
rows_fetched_from = 0
rows_fetched_to = 0
fetch_row_cnt = -1 if fetch_all == 1 else ON_DEMAND_RECORD_COUNT
# Check the transaction and connection status
status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id)
if status and conn is not None and session_obj is not None:
status, result = conn.async_fetchmany_2darray(fetch_row_cnt)
if not status:
status = 'Error'
else:
status = 'Success'
res_len = len(result)
if fetch_row_cnt != -1 and res_len == ON_DEMAND_RECORD_COUNT:
has_more_rows = True
if res_len:
rows_fetched_from = trans_obj.get_fetched_row_cnt()
trans_obj.update_fetched_row_cnt(rows_fetched_from + res_len)
rows_fetched_from += 1
rows_fetched_to = trans_obj.get_fetched_row_cnt()
session_obj['command_obj'] = pickle.dumps(trans_obj, -1)
update_session_grid_transaction(trans_id, session_obj)
else:
status = 'NotConnected'
result = error_msg
return make_json_response(
data={
'status': status, 'result': result,
'has_more_rows': has_more_rows,
'rows_fetched_from': rows_fetched_from,
'rows_fetched_to': rows_fetched_to
}
)
def fetch_pg_types(columns_info, trans_obj):
"""
This method is used to fetch the pg types, which is required
to map the data type comes as a result of the query.
Args:
trans_id: unique transaction id
columns_info:
"""
# Check the transaction and connection status
status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id)
if status and conn is not None \
and trans_obj is not None and session_obj is not None:
res = {}
if 'columns_info' in session_obj \
and session_obj['columns_info'] is not None:
# get the default connection as current connection attached to trans id
# holds the cursor which has query result so we cannot use that connection
# to execute another query otherwise we'll lose query result.
oids = [session_obj['columns_info'][col]['type_code'] for col in session_obj['columns_info']]
manager = get_driver(PG_DEFAULT_DRIVER).connection_manager(trans_obj.sid)
default_conn = manager.connection(did=trans_obj.did)
if oids:
status, res = conn.execute_dict(
u"""SELECT oid, format_type(oid,null) as typname FROM pg_type WHERE oid IN %s ORDER BY oid;
# Connect to the Server if not connected.
res = []
if not default_conn.connected():
status, msg = default_conn.connect()
if not status:
return status, msg
oids = [columns_info[col]['type_code'] for col in columns_info]
if oids:
status, res = default_conn.execute_dict(
u"""SELECT oid, format_type(oid,null) as typname FROM pg_type WHERE oid IN %s ORDER BY oid;
""", [tuple(oids)])
if status:
# iterate through pg_types and update the type name in session object
for record in res['rows']:
for col in session_obj['columns_info']:
type_obj = session_obj['columns_info'][col]
if type_obj['type_code'] == record['oid']:
type_obj['type_name'] = record['typname']
if not status:
return False, res
update_session_grid_transaction(trans_id, session_obj)
return status, res['rows']
else:
status = False
res = error_msg
return True, []
return make_json_response(data={'status': status, 'result': res})
def generate_client_primary_key_name(columns_info):
temp_key = '__temp_PK'
if not columns_info:
return temp_key
initial_temp_key_len = len(temp_key)
duplicate = False
suffix = 1
while 1:
for col in columns_info:
if col['name'] == temp_key:
duplicate = True
break
if duplicate:
if initial_temp_key_len == len(temp_key):
temp_key += str(suffix)
suffix += 1
else:
temp_key = temp_key[:-1] + str(suffix)
suffix += 1
duplicate = False
else:
break
return temp_key
@blueprint.route(
@@ -659,7 +773,6 @@ def save(trans_id):
Args:
trans_id: unique transaction id
"""
if request.data:
changed_data = json.loads(request.data, encoding='utf-8')
else:
@@ -669,7 +782,6 @@ def save(trans_id):
status, error_msg, conn, trans_obj, session_obj = check_transaction_status(trans_id)
if status and conn is not None \
and trans_obj is not None and session_obj is not None:
setattr(trans_obj, 'columns_info', session_obj['columns_info'])
# If there is no primary key found then return from the function.
if len(session_obj['primary_keys']) <= 0 or len(changed_data) <= 0:
@@ -680,7 +792,22 @@ def save(trans_id):
}
)
status, res, query_res, _rowid = trans_obj.save(changed_data)
manager = get_driver(PG_DEFAULT_DRIVER).connection_manager(trans_obj.sid)
default_conn = manager.connection(did=trans_obj.did)
# Connect to the Server if not connected.
if not default_conn.connected():
status, msg = default_conn.connect()
if not status:
return make_json_response(
data={'status': status, 'result': u"{}".format(msg)}
)
status, res, query_res, _rowid = trans_obj.save(
changed_data,
session_obj['columns_info'],
session_obj['client_primary_key'],
default_conn)
else:
status = False
res = error_msg

View File

@@ -258,7 +258,21 @@ class SQLFilter(object):
return status, result
class GridCommand(BaseCommand, SQLFilter):
class FetchedRowTracker(object):
"""
Keeps track of fetched row count.
"""
def __init__(self, **kwargs):
self.fetched_rows = 0
def get_fetched_row_cnt(self):
return self.fetched_rows
def update_fetched_row_cnt(self, rows_cnt):
self.fetched_rows = rows_cnt
class GridCommand(BaseCommand, SQLFilter, FetchedRowTracker):
"""
class GridCommand(object)
@@ -290,6 +304,7 @@ class GridCommand(BaseCommand, SQLFilter):
"""
BaseCommand.__init__(self, **kwargs)
SQLFilter.__init__(self, **kwargs)
FetchedRowTracker.__init__(self, **kwargs)
# Save the connection id, command type
self.conn_id = kwargs['conn_id'] if 'conn_id' in kwargs else None
@@ -299,10 +314,10 @@ class GridCommand(BaseCommand, SQLFilter):
if self.cmd_type == VIEW_FIRST_100_ROWS or self.cmd_type == VIEW_LAST_100_ROWS:
self.limit = 100
def get_primary_keys(self):
def get_primary_keys(self, *args, **kwargs):
return None, None
def save(self, changed_data):
def save(self, changed_data, default_conn=None):
return forbidden(errmsg=gettext("Data cannot be saved for the current object."))
def get_limit(self):
@@ -340,14 +355,14 @@ class TableCommand(GridCommand):
# call base class init to fetch the table name
super(TableCommand, self).__init__(**kwargs)
def get_sql(self):
def get_sql(self, default_conn=None):
"""
This method is used to create a proper SQL query
to fetch the data for the specified table
"""
# Fetch the primary keys for the table
pk_names, primary_keys = self.get_primary_keys()
pk_names, primary_keys = self.get_primary_keys(default_conn)
sql_filter = self.get_filter()
@@ -362,13 +377,16 @@ class TableCommand(GridCommand):
return sql
def get_primary_keys(self):
def get_primary_keys(self, default_conn=None):
"""
This function is used to fetch the primary key columns.
"""
driver = get_driver(PG_DEFAULT_DRIVER)
manager = driver.connection_manager(self.sid)
conn = manager.connection(did=self.did, conn_id=self.conn_id)
if default_conn is None:
manager = driver.connection_manager(self.sid)
conn = manager.connection(did=self.did, conn_id=self.conn_id)
else:
conn = default_conn
pk_names = ''
primary_keys = OrderedDict()
@@ -400,7 +418,11 @@ class TableCommand(GridCommand):
def can_filter(self):
return True
def save(self, changed_data):
def save(self,
changed_data,
columns_info,
client_primary_key='__temp_PK',
default_conn=None):
"""
This function is used to save the data into the database.
Depending on condition it will either update or insert the
@@ -408,10 +430,16 @@ class TableCommand(GridCommand):
Args:
changed_data: Contains data to be saved
columns_info:
default_conn:
client_primary_key:
"""
manager = get_driver(PG_DEFAULT_DRIVER).connection_manager(self.sid)
conn = manager.connection(did=self.did, conn_id=self.conn_id)
driver = get_driver(PG_DEFAULT_DRIVER)
if default_conn is None:
manager = driver.connection_manager(self.sid)
conn = manager.connection(did=self.did, conn_id=self.conn_id)
else:
conn = default_conn
status = False
res = None
@@ -421,14 +449,6 @@ class TableCommand(GridCommand):
list_of_sql = []
_rowid = None
# Replace column positions with names
def set_column_names(data):
new_data = {}
for key in data:
new_data[changed_data['columns'][int(key)]['name']] = data[key]
return new_data
if conn.connected():
# Start the transaction
@@ -443,6 +463,20 @@ class TableCommand(GridCommand):
if len(changed_data[of_type]) < 1:
continue
column_type = {}
for each_col in columns_info:
if (
columns_info[each_col]['not_null'] and
not columns_info[each_col][
'has_default_val']
):
column_data[each_col] = None
column_type[each_col] =\
columns_info[each_col]['type_name']
else:
column_type[each_col] = \
columns_info[each_col]['type_name']
# For newly added rows
if of_type == 'added':
@@ -451,37 +485,18 @@ class TableCommand(GridCommand):
# no_default_value, set column to blank, instead
# of not null which is set by default.
column_data = {}
column_type = {}
pk_names, primary_keys = self.get_primary_keys()
for each_col in self.columns_info:
if (
self.columns_info[each_col]['not_null'] and
not self.columns_info[each_col][
'has_default_val']
):
column_data[each_col] = None
column_type[each_col] =\
self.columns_info[each_col]['type_name']
else:
column_type[each_col] = \
self.columns_info[each_col]['type_name']
for each_row in changed_data[of_type]:
data = changed_data[of_type][each_row]['data']
# Remove our unique tracking key
data.pop('__temp_PK', None)
data.pop(client_primary_key, None)
data.pop('is_row_copied', None)
data = set_column_names(data)
data_type = set_column_names(changed_data[of_type][each_row]['data_type'])
list_of_rowid.append(data.get('__temp_PK'))
list_of_rowid.append(data.get(client_primary_key))
# Update columns value and data type
# with columns having not_null=False and has
# no default value
# Update columns value with columns having
# not_null=False and has no default value
column_data.update(data)
column_type.update(data_type)
sql = render_template("/".join([self.sql_path, 'insert.sql']),
data_to_be_saved=column_data,
@@ -497,15 +512,14 @@ class TableCommand(GridCommand):
# For updated rows
elif of_type == 'updated':
for each_row in changed_data[of_type]:
data = set_column_names(changed_data[of_type][each_row]['data'])
pk = set_column_names(changed_data[of_type][each_row]['primary_keys'])
data_type = set_column_names(changed_data[of_type][each_row]['data_type'])
data = changed_data[of_type][each_row]['data']
pk = changed_data[of_type][each_row]['primary_keys']
sql = render_template("/".join([self.sql_path, 'update.sql']),
data_to_be_saved=data,
primary_keys=pk,
object_name=self.object_name,
nsp_name=self.nsp_name,
data_type=data_type)
data_type=column_type)
list_of_sql.append(sql)
list_of_rowid.append(data)
@@ -519,18 +533,19 @@ class TableCommand(GridCommand):
rows_to_delete.append(changed_data[of_type][each_row])
# Fetch the keys for SQL generation
if is_first:
# We need to covert dict_keys to normal list in Python3
# In Python2, it's already a list & We will also fetch column names using index
keys = [
changed_data['columns'][int(k)]['name']
for k in list(changed_data[of_type][each_row].keys())
]
# We need to covert dict_keys to normal list in
# Python3
# In Python2, it's already a list & We will also
# fetch column names using index
keys = list(changed_data[of_type][each_row].keys())
no_of_keys = len(keys)
is_first = False
# Map index with column name for each row
for row in rows_to_delete:
for k, v in row.items():
# Set primary key with label & delete index based mapped key
# Set primary key with label & delete index based
# mapped key
try:
row[changed_data['columns'][int(k)]['name']] = v
except ValueError:
@@ -597,7 +612,7 @@ class ViewCommand(GridCommand):
# call base class init to fetch the table name
super(ViewCommand, self).__init__(**kwargs)
def get_sql(self):
def get_sql(self, default_conn=None):
"""
This method is used to create a proper SQL query
to fetch the data for the specified view
@@ -652,7 +667,7 @@ class ForeignTableCommand(GridCommand):
# call base class init to fetch the table name
super(ForeignTableCommand, self).__init__(**kwargs)
def get_sql(self):
def get_sql(self, default_conn=None):
"""
This method is used to create a proper SQL query
to fetch the data for the specified foreign table
@@ -697,7 +712,7 @@ class CatalogCommand(GridCommand):
# call base class init to fetch the table name
super(CatalogCommand, self).__init__(**kwargs)
def get_sql(self):
def get_sql(self, default_conn=None):
"""
This method is used to create a proper SQL query
to fetch the data for the specified catalog object
@@ -722,7 +737,7 @@ class CatalogCommand(GridCommand):
return True
class QueryToolCommand(BaseCommand):
class QueryToolCommand(BaseCommand, FetchedRowTracker):
"""
class QueryToolCommand(BaseCommand)
@@ -732,13 +747,15 @@ class QueryToolCommand(BaseCommand):
def __init__(self, **kwargs):
# call base class init to fetch the table name
super(QueryToolCommand, self).__init__(**kwargs)
BaseCommand.__init__(self, **kwargs)
FetchedRowTracker.__init__(self, **kwargs)
self.conn_id = None
self.auto_rollback = False
self.auto_commit = True
def get_sql(self):
def get_sql(self, default_conn=None):
return None
def can_edit(self):

View File

@@ -423,7 +423,7 @@ input.editor-checkbox:focus {
/* To highlight all newly inserted rows */
.grid-canvas .new_row {
background: #dff0d7;
background: #dff0d7 !important;
}
/* To highlight all the updated rows */
@@ -433,7 +433,7 @@ input.editor-checkbox:focus {
/* To highlight row at fault */
.grid-canvas .new_row.error, .grid-canvas .updated_row.error {
background: #f2dede;
background: #f2dede !important;
}
/* Disabled row */
@@ -460,6 +460,11 @@ input.editor-checkbox:focus {
background-color: #2C76B4;
}
.slick-cell span[data-cell-type="row-header-selector"] {
display: block;
text-align: right;
}
#datagrid div.slick-header.ui-state-default {
background: #ffffff;
border-bottom: none;
@@ -481,7 +486,9 @@ input.editor-checkbox:focus {
.select-all-icon {
margin-left: 9px;
margin-right: 9px;
vertical-align: bottom;
float: right;
}
.slick-cell, .slick-headerrow-column {