Support EXPLAIN on Greenplum. Fixes #3097

- Extract SQLEditor.execute and SQLEditor._poll into their own files and add test around them
 - Extract SQLEditor backend functions that start executing query to their own files and add tests around it
 - Move the Explain SQL from the front-end and now pass the Explain plan parameters as a JSON object in the start query call.
 - Extract the compile_template_name into a function that can be used by the different places that try to select the version of the template and the server type
This commit is contained in:
Joao Pedro De Almeida Pereira
2018-02-09 11:54:42 +00:00
committed by Dave Page
parent e60a84c44f
commit e16a952753
30 changed files with 3673 additions and 582 deletions

View File

@@ -0,0 +1,8 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################

View File

@@ -0,0 +1,121 @@
#######################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
"""Apply Explain plan wrapper to sql object."""
import sys
from pgadmin.tools.sqleditor.utils import apply_explain_plan_wrapper_if_needed
from pgadmin.utils.route import BaseTestGenerator
if sys.version_info < (3, 3):
from mock import patch, MagicMock
else:
from unittest.mock import patch, MagicMock
class StartRunningQueryTest(BaseTestGenerator):
"""
Check that the apply_explain_plan_weapper_if_needed method works as intended
"""
scenarios = [
('When explain_plan is none, it should return unaltered SQL', dict(
function_input_parameters={
'manager': MagicMock(),
'sql': {
'sql': 'some sql',
'explain_plan': None
}
},
expect_render_template_mock_parameters=None,
expected_return_value='some sql'
)),
('When explain_plan is not present, it should return unaltered SQL', dict(
function_input_parameters={
'manager': MagicMock(),
'sql': {
'sql': 'some sql'
}
},
expect_render_template_mock_parameters=None,
expected_return_value='some sql'
)),
('When explain_plan is present for a Postgres server version 10, it should return SQL with explain plan', dict(
function_input_parameters={
'manager': MagicMock(version=10, server_type='pg'),
'sql': {
'sql': 'some sql',
'explain_plan': {
'format': 'json',
'analyze': False,
'verbose': True,
'buffers': False,
'timing': True
}
}
},
expect_render_template_mock_parameters=dict(
template_name_or_list='sqleditor/sql/#10#/explain_plan.sql',
named_parameters=dict(
format='json',
analyze=False,
verbose=True,
buffers=False,
timing=True
)),
expected_return_value='EXPLAIN (FORMAT JSON, ANALYZE FALSE, VERBOSE TRUE, COSTS FALSE, BUFFERS FALSE, '
'TIMING TRUE) some sql'
)),
('When explain_plan is present for a GreenPlum server version 5, it should return SQL with explain plan', dict(
function_input_parameters={
'manager': MagicMock(version=80323, server_type='gpdb'),
'sql': {
'sql': 'some sql',
'explain_plan': {
'format': 'json',
'analyze': False,
'verbose': True,
'buffers': False,
'timing': True
}
}
},
expect_render_template_mock_parameters=dict(
template_name_or_list='sqleditor/sql/#gpdb#80323#/explain_plan.sql',
named_parameters=dict(
format='json',
analyze=False,
verbose=True,
buffers=False,
timing=True
)),
expected_return_value='EXPLAIN some sql'
))
]
def runTest(self):
with patch('pgadmin.tools.sqleditor.utils.apply_explain_plan_wrapper.render_template') as render_template_mock:
render_template_mock.return_value = self.expected_return_value
result = apply_explain_plan_wrapper_if_needed(**self.function_input_parameters)
self.assertEquals(result, self.expected_return_value)
if self.expect_render_template_mock_parameters:
render_template_mock.assert_called_with(
self.expect_render_template_mock_parameters['template_name_or_list'],
sql=self.function_input_parameters['sql']['sql'],
**self.expect_render_template_mock_parameters['named_parameters']
)
else:
render_template_mock.assert_not_called()

View File

@@ -0,0 +1,445 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
import sys
from flask import Response
import simplejson as json
from pgadmin.tools.sqleditor.utils.start_running_query import StartRunningQuery
from pgadmin.utils.exception import ConnectionLost
from pgadmin.utils.route import BaseTestGenerator
if sys.version_info < (3, 3):
from mock import patch, MagicMock
else:
from unittest.mock import patch, MagicMock
get_driver_exception = Exception('get_driver exception')
class StartRunningQueryTest(BaseTestGenerator):
"""
Check that the start_running_query method works as intended
"""
scenarios = [
('When gridData is not present in the session, it returns an error', dict(
function_parameters=dict(
sql=dict(sql='some sql', explain_plan=None),
trans_id=123,
http_session=dict()
),
pickle_load_return=None,
get_driver_exception=False,
manager_connection_exception=None,
is_connected_to_server=False,
connection_connect_return=None,
execute_async_return_value=None,
is_begin_required=False,
is_rollback_required=False,
apply_explain_plan_wrapper_if_needed_return_value='some sql',
expect_make_json_response_to_have_been_called_with=dict(
success=0,
errormsg='Transaction ID not found in the session.',
info='DATAGRID_TRANSACTION_REQUIRED',
status=404,
),
expect_internal_server_error_to_have_been_called_with=None,
expected_logger_error=None,
expect_execute_void_called_with='some sql',
)),
('When transactionId is not present in the gridData, it returns an error', dict(
function_parameters=dict(
sql=dict(sql='some sql', explain_plan=None),
trans_id=123,
http_session=dict(gridData=dict())
),
pickle_load_return=None,
get_driver_exception=False,
manager_connection_exception=None,
is_connected_to_server=False,
connection_connect_return=None,
execute_async_return_value=None,
is_begin_required=False,
is_rollback_required=False,
apply_explain_plan_wrapper_if_needed_return_value='some sql',
expect_make_json_response_to_have_been_called_with=dict(
success=0,
errormsg='Transaction ID not found in the session.',
info='DATAGRID_TRANSACTION_REQUIRED',
status=404,
),
expect_internal_server_error_to_have_been_called_with=None,
expected_logger_error=None,
expect_execute_void_called_with='some sql',
)),
('When the command information for the transaction cannot be retrieved, it returns an error', dict(
function_parameters=dict(
sql=dict(sql='some sql', explain_plan=None),
trans_id=123,
http_session=dict(gridData={'123': dict(command_obj='')})
),
pickle_load_return=None,
get_driver_exception=False,
manager_connection_exception=None,
is_connected_to_server=False,
connection_connect_return=None,
execute_async_return_value=None,
is_begin_required=False,
is_rollback_required=False,
apply_explain_plan_wrapper_if_needed_return_value='some sql',
expect_make_json_response_to_have_been_called_with=dict(
data=dict(
status=False,
result='Either transaction object or session object not found.',
can_edit=False,
can_filter=False,
info_notifier_timeout=5
)
),
expect_internal_server_error_to_have_been_called_with=None,
expected_logger_error=None,
expect_execute_void_called_with='some sql',
)),
('When exception happens while retrieving the database driver, it returns an error', dict(
function_parameters=dict(
sql=dict(sql='some sql', explain_plan=None),
trans_id=123,
http_session=dict(gridData={'123': dict(command_obj='')})
),
pickle_load_return=MagicMock(conn_id=1, update_fetched_row_cnt=MagicMock()),
get_driver_exception=True,
manager_connection_exception=None,
is_connected_to_server=False,
connection_connect_return=None,
execute_async_return_value=None,
is_begin_required=False,
is_rollback_required=False,
apply_explain_plan_wrapper_if_needed_return_value='some sql',
expect_make_json_response_to_have_been_called_with=None,
expect_internal_server_error_to_have_been_called_with=dict(
errormsg='get_driver exception'
),
expected_logger_error=get_driver_exception,
expect_execute_void_called_with='some sql',
)),
('When ConnectionLost happens while retrieving the database connection, it returns an error', dict(
function_parameters=dict(
sql=dict(sql='some sql', explain_plan=None),
trans_id=123,
http_session=dict(gridData={'123': dict(command_obj='')})
),
pickle_load_return=MagicMock(conn_id=1, update_fetched_row_cnt=MagicMock()),
get_driver_exception=False,
manager_connection_exception=ConnectionLost('1', '2', '3'),
is_connected_to_server=False,
connection_connect_return=None,
execute_async_return_value=None,
is_begin_required=False,
is_rollback_required=False,
apply_explain_plan_wrapper_if_needed_return_value='some sql',
expect_make_json_response_to_have_been_called_with=None,
expect_internal_server_error_to_have_been_called_with=None,
expected_logger_error=None,
expect_execute_void_called_with='some sql',
)),
('When is not connected to the server and fails to connect, it returns an error', dict(
function_parameters=dict(
sql=dict(sql='some sql', explain_plan=None),
trans_id=123,
http_session=dict(gridData={'123': dict(command_obj='')})
),
pickle_load_return=MagicMock(conn_id=1, update_fetched_row_cnt=MagicMock()),
get_driver_exception=False,
manager_connection_exception=None,
is_connected_to_server=False,
connection_connect_return=[False, 'Unable to connect to server'],
execute_async_return_value=None,
is_begin_required=False,
is_rollback_required=False,
apply_explain_plan_wrapper_if_needed_return_value='some sql',
expect_make_json_response_to_have_been_called_with=None,
expect_internal_server_error_to_have_been_called_with=dict(
errormsg='Unable to connect to server'
),
expected_logger_error='Unable to connect to server',
expect_execute_void_called_with='some sql',
)),
('When server is connected and start query async request, it returns an success message', dict(
function_parameters=dict(
sql=dict(sql='some sql', explain_plan=None),
trans_id=123,
http_session=dict(gridData={'123': dict(command_obj='')})
),
pickle_load_return=MagicMock(
conn_id=1,
update_fetched_row_cnt=MagicMock(),
set_connection_id=MagicMock(),
auto_commit=True,
auto_rollback=False,
can_edit=lambda: True,
can_filter=lambda: True
),
get_driver_exception=False,
manager_connection_exception=None,
is_connected_to_server=True,
connection_connect_return=None,
execute_async_return_value=[True, 'async function result output'],
is_begin_required=False,
is_rollback_required=False,
apply_explain_plan_wrapper_if_needed_return_value='some sql',
expect_make_json_response_to_have_been_called_with=dict(
data=dict(
status=True,
result='async function result output',
can_edit=True,
can_filter=True,
info_notifier_timeout=5
)
),
expect_internal_server_error_to_have_been_called_with=None,
expected_logger_error=None,
expect_execute_void_called_with='some sql',
)),
('When server is connected and start query async request and begin is required, '
'it returns an success message', dict(
function_parameters=dict(
sql=dict(sql='some sql', explain_plan=None),
trans_id=123,
http_session=dict(gridData={'123': dict(command_obj='')})
),
pickle_load_return=MagicMock(
conn_id=1,
update_fetched_row_cnt=MagicMock(),
set_connection_id=MagicMock(),
auto_commit=True,
auto_rollback=False,
can_edit=lambda: True,
can_filter=lambda: True
),
get_driver_exception=False,
manager_connection_exception=None,
is_connected_to_server=True,
connection_connect_return=None,
execute_async_return_value=[True, 'async function result output'],
is_begin_required=True,
is_rollback_required=False,
apply_explain_plan_wrapper_if_needed_return_value='some sql',
expect_make_json_response_to_have_been_called_with=dict(
data=dict(
status=True,
result='async function result output',
can_edit=True,
can_filter=True,
info_notifier_timeout=5
)
),
expect_internal_server_error_to_have_been_called_with=None,
expected_logger_error=None,
expect_execute_void_called_with='some sql',
)),
('When server is connected and start query async request and rollback is required, '
'it returns an success message', dict(
function_parameters=dict(
sql=dict(sql='some sql', explain_plan=None),
trans_id=123,
http_session=dict(gridData={'123': dict(command_obj='')})
),
pickle_load_return=MagicMock(
conn_id=1,
update_fetched_row_cnt=MagicMock(),
set_connection_id=MagicMock(),
auto_commit=True,
auto_rollback=False,
can_edit=lambda: True,
can_filter=lambda: True
),
get_driver_exception=False,
manager_connection_exception=None,
is_connected_to_server=True,
connection_connect_return=None,
execute_async_return_value=[True, 'async function result output'],
is_begin_required=False,
is_rollback_required=True,
apply_explain_plan_wrapper_if_needed_return_value='some sql',
expect_make_json_response_to_have_been_called_with=dict(
data=dict(
status=True,
result='async function result output',
can_edit=True,
can_filter=True,
info_notifier_timeout=5
)
),
expect_internal_server_error_to_have_been_called_with=None,
expected_logger_error=None,
expect_execute_void_called_with='some sql',
)),
('When server is connected and start query async request with explain plan wrapper, '
'it returns an success message', dict(
function_parameters=dict(
sql=dict(sql='some sql', explain_plan=None),
trans_id=123,
http_session=dict(gridData={'123': dict(command_obj='')})
),
pickle_load_return=MagicMock(
conn_id=1,
update_fetched_row_cnt=MagicMock(),
set_connection_id=MagicMock(),
auto_commit=True,
auto_rollback=False,
can_edit=lambda: True,
can_filter=lambda: True
),
get_driver_exception=False,
manager_connection_exception=None,
is_connected_to_server=True,
connection_connect_return=None,
execute_async_return_value=[True, 'async function result output'],
is_begin_required=False,
is_rollback_required=True,
apply_explain_plan_wrapper_if_needed_return_value='EXPLAIN PLAN some sql',
expect_make_json_response_to_have_been_called_with=dict(
data=dict(
status=True,
result='async function result output',
can_edit=True,
can_filter=True,
info_notifier_timeout=5
)
),
expect_internal_server_error_to_have_been_called_with=None,
expected_logger_error=None,
expect_execute_void_called_with='EXPLAIN PLAN some sql',
)),
]
@patch('pgadmin.tools.sqleditor.utils.start_running_query.apply_explain_plan_wrapper_if_needed')
@patch('pgadmin.tools.sqleditor.utils.start_running_query.make_json_response')
@patch('pgadmin.tools.sqleditor.utils.start_running_query.pickle')
@patch('pgadmin.tools.sqleditor.utils.start_running_query.get_driver')
@patch('pgadmin.tools.sqleditor.utils.start_running_query.internal_server_error')
@patch('pgadmin.tools.sqleditor.utils.start_running_query.update_session_grid_transaction')
def runTest(self, update_session_grid_transaction_mock,
internal_server_error_mock, get_driver_mock, pickle_mock,
make_json_response_mock, apply_explain_plan_wrapper_if_needed_mock):
"""Check correct function is called to handle to run query."""
self.connection = None
self.loggerMock = MagicMock(error=MagicMock())
expected_response = Response(response=json.dumps({'errormsg': 'some value'}))
make_json_response_mock.return_value = expected_response
if self.expect_internal_server_error_to_have_been_called_with is not None:
internal_server_error_mock.return_value = expected_response
pickle_mock.loads.return_value = self.pickle_load_return
blueprint_mock = MagicMock(info_notifier_timeout=MagicMock(get=lambda: 5))
if self.is_begin_required:
StartRunningQuery.is_begin_required_for_sql_query = MagicMock(return_value=True)
else:
StartRunningQuery.is_begin_required_for_sql_query = MagicMock(return_value=False)
if self.is_rollback_required:
StartRunningQuery.is_rollback_statement_required = MagicMock(return_value=True)
else:
StartRunningQuery.is_rollback_statement_required = MagicMock(return_value=False)
apply_explain_plan_wrapper_if_needed_mock.return_value = self.apply_explain_plan_wrapper_if_needed_return_value
manager = self.__create_manager()
if self.get_driver_exception:
get_driver_mock.side_effect = get_driver_exception
else:
get_driver_mock.return_value = MagicMock(connection_manager=lambda session_id: manager)
try:
result = StartRunningQuery(
blueprint_mock,
self.loggerMock
).execute(
**self.function_parameters
)
if self.manager_connection_exception is not None:
self.fail('Exception: "' + str(self.manager_connection_exception) + '" excepted but not raised')
self.assertEquals(result, expected_response)
except AssertionError:
raise
except Exception as exception:
self.assertEquals(self.manager_connection_exception, exception)
self.__mock_assertions(internal_server_error_mock, make_json_response_mock)
if self.is_connected_to_server:
apply_explain_plan_wrapper_if_needed_mock.assert_called_with(manager, self.function_parameters['sql'])
def __create_manager(self):
self.connection = MagicMock(
connected=lambda: self.is_connected_to_server,
connect=MagicMock(),
execute_async=MagicMock(),
execute_void=MagicMock(),
)
self.connection.connect.return_value = self.connection_connect_return
self.connection.execute_async.return_value = self.execute_async_return_value
if self.manager_connection_exception is None:
manager = MagicMock(
connection=lambda did, conn_id, use_binary_placeholder, array_to_string, auto_reconnect: self.connection
)
else:
manager = MagicMock()
manager.connection.side_effect = self.manager_connection_exception
return manager
def __mock_assertions(self, internal_server_error_mock, make_json_response_mock):
if self.expect_make_json_response_to_have_been_called_with is not None:
make_json_response_mock.assert_called_with(**self.expect_make_json_response_to_have_been_called_with)
else:
make_json_response_mock.assert_not_called()
if self.expect_internal_server_error_to_have_been_called_with is not None:
internal_server_error_mock.assert_called_with(**self.expect_internal_server_error_to_have_been_called_with)
else:
internal_server_error_mock.assert_not_called()
if self.execute_async_return_value is not None:
self.connection.execute_async.assert_called_with(self.expect_execute_void_called_with)
else:
self.connection.execute_async.assert_not_called()
if self.expected_logger_error is not None:
self.loggerMock.error.assert_called_with(self.expected_logger_error)
else:
self.loggerMock.error.assert_not_called()
if self.is_begin_required:
self.connection.execute_void.assert_called_with('BEGIN;')
elif not self.is_rollback_required:
self.connection.execute_void.assert_not_called()
if self.is_rollback_required:
self.connection.execute_void.assert_called_with('ROLLBACK;')
elif not self.is_begin_required:
self.connection.execute_void.assert_not_called()