Support for external tables in GPDB. Fixes

This commit is contained in:
Joao Pedro De Almeida Pereira 2018-03-02 16:49:25 +00:00 committed by Dave Page
parent 92a0bb605d
commit 427314cfdf
28 changed files with 2194 additions and 0 deletions
web

View File

@ -0,0 +1,275 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
"""Implements External Tables Node"""
import os
from functools import wraps
from gettext import gettext
from flask import render_template
from config import PG_DEFAULT_DRIVER
from pgadmin.browser.collection import CollectionNodeModule
from pgadmin.browser.server_groups.servers import databases
from pgadmin.browser.server_groups.servers.databases \
.external_tables.mapping_utils import map_execution_location
from pgadmin.browser.server_groups.servers.databases \
.external_tables.properties import Properties, \
PropertiesTableNotFoundException, PropertiesException
from pgadmin.browser.server_groups.servers.databases \
.external_tables.reverse_engineer_ddl import ReverseEngineerDDL
from pgadmin.browser.utils import PGChildNodeView
from pgadmin.utils.ajax import make_json_response, make_response, \
internal_server_error
from pgadmin.utils.compile_template_name import compile_template_path
from pgadmin.utils.driver import get_driver
class ExternalTablesModule(CollectionNodeModule):
"""
class ExternalTablesModule(CollectionNodeModule)
A module class for External Tables node derived from
CollectionNodeModule.
Methods:
-------
* __init__(*args, **kwargs)
- Method is used to initialize the External Tables module
and it's base module.
* get_nodes(gid, sid, did)
- Method is used to generate the browser collection node.
* script_load()
- Load the module script for External Tables, when any of
the database node is initialized.
"""
NODE_TYPE = 'external_table'
COLLECTION_LABEL = gettext("External Tables")
def __init__(self, *args, **kwargs):
"""
Method is used to initialize the External tables module and
it's base module.
Args:
*args:
**kwargs:
"""
super(ExternalTablesModule, self).__init__(*args, **kwargs)
self.max_ver = 0
def get_nodes(self, gid, sid, did):
yield self.generate_browser_collection_node(did)
@property
def script_load(self):
"""
Load the module script for External tables,
when any of the database node is initialized.
Returns: node type of the database module.
"""
return databases.DatabaseModule.NODE_TYPE
@property
def module_use_template_javascript(self):
"""
Returns whether Jinja2 template is used for generating the javascript
module.
"""
return False
blueprint = ExternalTablesModule(__name__)
class ExternalTablesView(PGChildNodeView):
node_type = blueprint.node_type
parent_ids = [
{'type': 'int', 'id': 'server_group_id'},
{'type': 'int', 'id': 'server_id'},
{'type': 'int', 'id': 'database_id'}
]
ids = [
{'type': 'int', 'id': 'external_table_id'}
]
operations = dict({
'obj': [
{'get': 'properties'}
],
'nodes': [{'get': 'node'}, {'get': 'nodes'}],
'sql': [{'get': 'sql'}],
'children': [{'get': 'children'}]
})
def check_precondition(function_wrapped):
"""
This function will behave as a decorator which will checks
database connection before running view, it will also attaches
manager,conn & template_path properties to self
"""
@wraps(function_wrapped)
def wrap(*args, **kwargs):
# Here args[0] will hold self & kwargs will hold gid,sid,did
self = args[0]
self.manager = get_driver(PG_DEFAULT_DRIVER).connection_manager(
kwargs['server_id']
)
self.connection = self.manager.connection(
did=kwargs['database_id']
)
self.sql_template_path = compile_template_path(
'sql/',
self.manager.server_type,
self.manager.sversion
)
return function_wrapped(*args, **kwargs)
return wrap
def __init__(self, *args, **kwargs):
super(ExternalTablesView, self).__init__(*args, **kwargs)
self.connection = None
self.manager = None
self.sql_template_path = None
@check_precondition
def nodes(self, server_group_id, server_id, database_id):
"""
This function will used to create all the child node within that
collection.
Here it will create all the foreign data wrapper node.
Args:
server_group_id: Server Group ID
server_id: Server ID
database_id: Database ID
"""
sql_statement = render_template(
os.path.join(self.sql_template_path, 'list.sql')
)
result = self.get_external_tables(database_id, sql_statement)
if type(result) is not list:
return result
return make_json_response(
data=result,
status=200
)
@check_precondition
def node(self, server_group_id, server_id, database_id, external_table_id):
"""
This function will used to create all the child node within that
collection.
Here it will create all the foreign data wrapper node.
Args:
server_group_id: Server Group ID
server_id: Server ID
database_id: Database ID
external_table_id: External Table ID
"""
sql_statement = render_template(
template_name_or_list=os.path.join(
self.sql_template_path,
'node.sql'
),
external_table_id=external_table_id
)
result = self.get_external_tables(database_id, sql_statement)
if type(result) is not list:
return result
if len(result) == 0:
return make_json_response(
data=gettext('Could not find the external table.'),
status=404
)
return make_json_response(
data=result[0],
status=200
)
@check_precondition
def sql(self, server_group_id, server_id, database_id, external_table_id):
"""
This function will used to create all the child node within that
collection.
Here it will create all the foreign data wrapper node.
Args:
server_group_id: Server Group ID
server_id: Server ID
database_id: Database ID
external_table_id: External Table ID
"""
sql = ReverseEngineerDDL(self.sql_template_path,
render_template,
self.connection, server_group_id, server_id,
database_id).execute(external_table_id)
return make_response(
sql.strip('\n')
)
@check_precondition
def properties(self, server_group_id, server_id, database_id,
external_table_id):
try:
response = Properties(render_template, self.connection,
self.sql_template_path).retrieve(
external_table_id)
return make_response(
response=response,
status=200)
except PropertiesTableNotFoundException:
return make_json_response(
data=gettext('Could not find the external table.'),
status=404
)
except PropertiesException as exception:
return exception.response_object
def children(self, **kwargs):
return make_json_response(data=[])
def get_external_tables(self, database_id, sql_statement):
status, external_tables = self.connection \
.execute_2darray(sql_statement)
if not status:
return internal_server_error(errormsg=external_tables)
icon_css_class = 'icon-external_table'
result = []
for external_table in external_tables['rows']:
result.append(self.blueprint.generate_browser_node(
external_table['oid'],
database_id,
external_table['name'],
inode=False,
icon=icon_css_class
))
return result
ExternalTablesView.register_node_view(blueprint)

View File

@ -0,0 +1,4 @@
class GetAllNodes:
def execute(self):
pass

View File

@ -0,0 +1,165 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
import re
def map_column_from_database(column_information):
return {
'name': column_information['name'],
'type': column_information['cltype']
}
def map_table_information_from_database(table_information):
format_type = map_format_type(table_information['fmttype'])
return {
'uris': sql_array_notation_to_array(table_information['urilocation']),
'isWeb': is_web_table(
table_information['urilocation'],
table_information['command']
),
'executionLocation': map_execution_location(
table_information['execlocation']),
'formatType': format_type,
'formatOptions': format_options(format_type,
table_information['fmtopts']),
'command': table_information['command'],
'rejectLimit': table_information['rejectlimit'],
'rejectLimitType': table_information['rejectlimittype'],
'errorTableName': table_information['errtblname'],
'erroToFile': table_information['errortofile'],
'pgEncodingToChar': table_information['pg_encoding_to_char'],
'writable': table_information['writable'],
'options': table_information['options'],
'distribution': table_information['distribution'],
'name': table_information['name'],
'namespace': table_information['namespace']
}
def map_execution_location(execution_location):
stripped_execution_location = execution_location[0].lstrip('{').rstrip('}')
if stripped_execution_location.startswith('HOST:'):
return {
'type': 'host',
'value': stripped_execution_location.replace('HOST:', '').strip()
}
elif stripped_execution_location == 'PER_HOST':
return {'type': 'per_host', 'value': None}
elif stripped_execution_location == "MASTER_ONLY":
return {'type': 'master_only', 'value': None}
elif stripped_execution_location == "ALL_SEGMENTS":
return {'type': 'all_segments', 'value': None}
elif stripped_execution_location.startswith("SEGMENT_ID:"):
return {
'type': 'segment',
'value': stripped_execution_location.replace('SEGMENT_ID:', '')
.strip()
}
elif stripped_execution_location.startswith("TOTAL_SEGS:"):
return {
'type': 'segments',
'value': stripped_execution_location.replace('TOTAL_SEGS:', '')
.strip()
}
def map_format_type(format_type):
if format_type == 'b':
return 'custom'
elif format_type == 'a':
return 'avro'
elif format_type == 't':
return 'text'
elif format_type == 'p':
return 'parquet'
else:
return 'csv'
def is_web_table(uris, command):
if uris is None and command is None:
return False
if command is not None:
return True
return re.search('^https?:\\/\\/',
sql_array_notation_to_array(uris)[0]) is not None
def format_options(format_type, options):
if options is None:
return None
if len(options) == 0:
return options
result_options = tokenize_options(options)
all_keys = list(result_options.keys())
all_keys.sort()
if format_type not in ['csv', 'text']:
return ','.join([
'%s = %s' % (key, result_options[key]) for key in all_keys
])
else:
return ' '.join([
'%s %s' % (key, result_options[key]) for key in all_keys
])
def sql_array_notation_to_array(sql_result):
if sql_result is None:
return None
if sql_result[0] == '{':
return sql_result[1:-1].split(',')
return sql_result
def tokenize_options(options):
in_key = True
in_value = False
current_key = ''
current_value = ''
tokens = {}
for index in range(0, len(options)):
if is_end_of_key(in_key, options, index, current_key):
in_key = False
elif is_not_end_of_key(in_key, index, options):
current_key += options[index]
elif is_start_of_value(in_value, index, options):
in_value = True
current_value = ''
elif is_end_of_value(in_value, index, options):
in_value = False
in_key = True
tokens[current_key] = '$$' + current_value + '$$'
current_key = ''
current_value = ''
elif in_value:
current_value += options[index]
return tokens
def found_apostrophe_inside_value(in_value, index, options):
return in_value and options[index] == '\''
def is_end_of_value(in_value, index, options):
return in_value and options[index] == '\'' and (
index == (len(options) - 1) or options[index + 1] == ' ')
def is_start_of_value(in_value, index, options):
return not in_value and options[index] == '\''
def is_not_end_of_key(in_key, index, options):
return in_key and options[index] != ' '
def is_end_of_key(in_key, options, index, current_key):
return in_key and options[index] == ' ' and len(current_key) > 0

View File

@ -0,0 +1,78 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
from gettext import gettext
from os import path
from pgadmin.browser.server_groups.servers.databases.external_tables import \
map_execution_location
from pgadmin.utils.ajax import internal_server_error
class PropertiesException(Exception):
def __init__(self, response_object, *args):
super(PropertiesException, self).__init__(*args)
self.response_object = response_object
class PropertiesTableNotFoundException(PropertiesException):
def __init__(self, *args):
super(PropertiesException, self).__init__(None, *args)
class Properties:
def __init__(self, render_template, db_connection, sql_template_path):
self.render_template = render_template
self.db_connection = db_connection
self.sql_template_path = sql_template_path
def retrieve(self, table_oid):
table_information_sql = self.render_template(
template_name_or_list=path.join(self.sql_template_path,
'get_table_information.sql'),
table_oid=table_oid
)
(status, table_information_results) = \
self.db_connection.execute_2darray(table_information_sql)
if not status:
raise PropertiesException(
internal_server_error(table_information_results))
if len(table_information_results['rows']) != 1:
raise PropertiesTableNotFoundException()
table_information_result = table_information_results['rows'][0]
execute_on = map_execution_location(
table_information_result['execlocation'])
execute_on_text = self.translate_execute_on_text(execute_on)
response = dict(
name=table_information_result['name'],
type=gettext('readable' if not table_information_result[
'writable'] else 'writable'),
format_type=table_information_result['pg_encoding_to_char'],
format_options=table_information_result['fmtopts'],
external_options=table_information_result['options'],
command=table_information_result['command'],
execute_on=execute_on_text,
)
return response
@staticmethod
def translate_execute_on_text(execute_on):
if execute_on['type'] == 'host':
return 'host %s' % execute_on['value']
elif execute_on['type'] == 'per_host':
return 'per host'
elif execute_on['type'] == 'master_only':
return 'master segment'
elif execute_on['type'] == 'all_segments':
return 'all segments'
elif execute_on['type'] == 'segment':
return '%s segment' % execute_on['value']
elif execute_on['type'] == 'segments':
return '%d segments' % execute_on['value']

View File

@ -0,0 +1,69 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
from os import path
from pgadmin.browser.server_groups.servers.databases\
.external_tables.mapping_utils import \
map_column_from_database, map_table_information_from_database
class ReverseEngineerDDLException(Exception):
pass
class ReverseEngineerDDL:
def __init__(self, sql_template_paths,
render_template,
database_connection,
server_group_id, server_id, database_id):
self.sql_template_path = sql_template_paths
self.render_template = render_template
self.database_connection = database_connection
def execute(self, table_oid):
reverse_engineer_data = self.table_information(table_oid)
reverse_engineer_data['columns'] = self.find_columns(table_oid)
return self.render_template(
template_name_or_list=path.join(self.sql_template_path,
'create.sql'),
table=reverse_engineer_data
)
def find_columns(self, table_oid):
columns_sql = self.render_template(
template_name_or_list=path.join(self.sql_template_path,
'get_columns.sql'),
table_oid=table_oid
)
(status, column_result) = \
self.database_connection.execute_2darray(columns_sql)
if not status:
raise ReverseEngineerDDLException(column_result)
return list(map(map_column_from_database, column_result['rows']))
def table_information(self, table_oid):
table_information_sql = self.render_template(
template_name_or_list=path.join(self.sql_template_path,
'get_table_information.sql'),
table_oid=table_oid
)
(status, table_information_result) = \
self.database_connection.execute_2darray(table_information_sql)
if not status:
raise ReverseEngineerDDLException(table_information_result)
elif 'rows' not in table_information_result.keys() or len(
table_information_result['rows']
) == 0:
raise ReverseEngineerDDLException('Table not found')
return map_table_information_from_database(
table_information_result['rows'][0])

View File

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><defs><style>.cls-1{fill:#34495e;stroke:#ac9230;stroke-linecap:round;stroke-linejoin:round;}.cls-2{fill:#f2f2f2;}.cls-3{fill:#ac9230;}</style></defs><title>coll-foreign_data_wrapper</title><g id="_2" data-name="2"><line class="cls-1" x1="11.5" y1="4.25" x2="13.5" y2="2.25"/><line class="cls-1" x1="11.5" y1="2.25" x2="13.5" y2="2.25"/><line class="cls-1" x1="13.5" y1="4.25" x2="13.5" y2="2.25"/></g><g id="_3" data-name="3"><path class="cls-2" d="M12.82,6.25a.87.87,0,0,0,.18-.5c0-1.1-2.46-2-5.5-2s-5.5.9-5.5,2a.87.87,0,0,0,.18.5.78.78,0,0,0,0,1,.78.78,0,0,0,0,1,.78.78,0,0,0,0,1,.78.78,0,0,0,0,1,.78.78,0,0,0,0,1,.87.87,0,0,0-.18.5c0,1.1,2.46,2,5.5,2s5.5-.9,5.5-2a.87.87,0,0,0-.18-.5.78.78,0,0,0,0-1,.78.78,0,0,0,0-1,.78.78,0,0,0,0-1,.78.78,0,0,0,0-1,.78.78,0,0,0,0-1Z"/><ellipse class="cls-3" cx="7.5" cy="5.75" rx="5.5" ry="2"/><path class="cls-3" d="M7.5,12.75c-2.56,0-4.71-.64-5.32-1.5a.87.87,0,0,0-.18.5c0,1.1,2.46,2,5.5,2s5.5-.9,5.5-2a.87.87,0,0,0-.18-.5C12.21,12.11,10.07,12.75,7.5,12.75Z"/><path class="cls-3" d="M7.5,8.75c-2.56,0-4.71-.64-5.32-1.5a.87.87,0,0,0-.18.5c0,1.1,2.46,2,5.5,2s5.5-.9,5.5-2a.87.87,0,0,0-.18-.5C12.21,8.11,10.07,8.75,7.5,8.75Z"/><path class="cls-3" d="M7.5,10.75c-2.56,0-4.71-.64-5.32-1.5a.87.87,0,0,0-.18.5c0,1.1,2.46,2,5.5,2s5.5-.9,5.5-2a.87.87,0,0,0-.18-.5C12.21,10.11,10.07,10.75,7.5,10.75Z"/></g></svg>

After

(image error) Size: 1.4 KiB

View File

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><defs><style>.cls-1{fill:#f2f2f2;}.cls-2{fill:#2195e7;}.cls-3{fill:none;stroke:#c1cbd5;stroke-linejoin:round;}.cls-3,.cls-4{stroke-width:0.75px;}.cls-4{fill:#def4fd;stroke:#2195e7;stroke-miterlimit:1;}</style></defs><title>coll-table</title><g id="_2" data-name="2"><rect class="cls-1" x="2.42" y="2.66" width="8.88" height="8.25" rx="0.52" ry="0.52"/><path class="cls-2" d="M10.77,3a.15.15,0,0,1,.15.15v7.2a.15.15,0,0,1-.15.15H2.94a.15.15,0,0,1-.15-.15V3.19A.15.15,0,0,1,2.94,3h7.83m0-.75H2.94a.9.9,0,0,0-.9.9v7.2a.9.9,0,0,0,.9.9h7.83a.9.9,0,0,0,.9-.9V3.19a.9.9,0,0,0-.9-.9Z"/><path class="cls-1" d="M4.9,4.63h8.16a.53.53,0,0,1,.53.53v7.66a.52.52,0,0,1-.52.52H4.9a.52.52,0,0,1-.52-.52V5.16A.52.52,0,0,1,4.9,4.63Z"/><path class="cls-2" d="M13.06,5a.15.15,0,0,1,.15.15v7.66a.15.15,0,0,1-.15.15H4.9a.15.15,0,0,1-.15-.15V5.16A.15.15,0,0,1,4.9,5h8.16m0-.75H4.9a.9.9,0,0,0-.9.9v7.66a.9.9,0,0,0,.9.9h8.16a.9.9,0,0,0,.9-.9V5.16a.9.9,0,0,0-.9-.9Z"/><line class="cls-3" x1="4.76" y1="10.42" x2="13.19" y2="10.42"/><line class="cls-3" x1="8.9" y1="8.08" x2="8.89" y2="12.95"/><line class="cls-4" x1="8.9" y1="5.01" x2="8.89" y2="7.56"/><line class="cls-4" x1="13.19" y1="7.73" x2="4.76" y2="7.73"/></g></svg>

After

(image error) Size: 1.2 KiB

View File

@ -0,0 +1,60 @@
{% if table.rejectLimitType == 'r' %}
{% set rejectionLimit = 'ROWS' %}
{% else %}
{% set rejectionLimit = 'PERCENT' %}
{% endif %}
CREATE {% if table.writable %}WRITABLE {% endif %}EXTERNAL {% if table.isWeb %}WEB {% endif %}TABLE {{conn|qtIdent(table.namespace, table.name)}}{% if table.columns and table.columns|length > 0 %}(
{% for c in table.columns %}
{% if c.name and c.type -%}
{% if loop.index != 1 %},
{% endif %}
{{conn|qtIdent(c.name)}} {{c.type}}
{%- endif %}
{% endfor %}
)
{% else %}
()
{% endif %}
{% if table.command and table.command|length > 0 %}
EXECUTE $pgAdmin${{ table.command }}'$pgAdmin$
{% else %}
LOCATION (
{% for uri in table.uris %}
{% if loop.index != 1 -%},
{% endif %}
'{{uri}}'
{%- endfor %}
)
{% endif %}
{% if not table.writable and table.executionLocation %}
{% if table.executionLocation.type == 'host' %}
ON HOST {{ table.executionLocation.value }}
{% elif table.executionLocation.type == 'per_host' %}
ON HOST
{% elif table.executionLocation.type == 'master_only' %}
ON MASTER
{% elif table.executionLocation.type == 'all_segments' %}
ON ALL
{% elif table.executionLocation.type == 'segment' %}
ON SEGMENT {{ table.executionLocation.value }}
{% elif table.executionLocation.type == 'segments' %}
ON {{ table.executionLocation.value }}
{% endif %}
{% endif %}
FORMAT '{{ table.formatType }}' ({{ table.formatOptions }})
{% if table.options and table.options|length > 0 %}
OPTIONS (
{{ table.options }}
)
{% endif %}
ENCODING '{{ table.pgEncodingToChar }}'
{% if table.rejectLimit and table.rejectLimit|length > 0 %}
{% if table.errorTableName and table.errorTableName|length > 0 %}
LOG ERRORS {% endif %}SEGMENT REJECT LIMIT {{ table.rejectLimit }} {{ rejectionLimit }}
{% endif %}
{% if table.writable and table.distribution %}
DISTRIBUTED BY ({% for attrnum in table.distribution %}{% if loop.index != 1 %}, {% endif %}{{ table.columns[attrnum-1].name }}{% endfor %});
{% elif table.writable %}
DISTRIBUTED RANDOMLY;
{% else %};
{% endif %}

View File

@ -0,0 +1,12 @@
SELECT
a.attname AS name, format_type(a.atttypid, NULL) AS cltype,
quote_ident(n.nspname)||'.'||quote_ident(c.relname) as inheritedfrom,
c.oid as inheritedid
FROM
pg_class c
JOIN
pg_namespace n ON c.relnamespace=n.oid
JOIN
pg_attribute a ON a.attrelid = c.oid AND NOT a.attisdropped AND a.attnum > 0
WHERE
c.oid = {{ table_oid }}::OID

View File

@ -0,0 +1,22 @@
SELECT x.urilocation, x.execlocation, x.fmttype, x.fmtopts, x.command,
x.rejectlimit, x.rejectlimittype,
(SELECT relname
FROM pg_catalog.pg_class
WHERE Oid=x.fmterrtbl) AS errtblname,
x.fmterrtbl = x.reloid AS errortofile ,
pg_catalog.pg_encoding_to_char(x.encoding),
x.writable,
array_to_string(ARRAY(
SELECT pg_catalog.quote_ident(option_name) || ' ' ||
pg_catalog.quote_literal(option_value)
FROM pg_options_to_table(x.options)
ORDER BY option_name
), E',\n ') AS options,
gdp.attrnums AS distribution,
c.relname AS name,
nsp.nspname AS namespace
FROM pg_catalog.pg_exttable x,
pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace nsp ON nsp.oid = c.relnamespace
LEFT JOIN gp_distribution_policy gdp ON gdp.localoid = c.oid
WHERE x.reloid = c.oid AND c.oid = {{ table_oid }}::oid;

View File

@ -0,0 +1,6 @@
SELECT pg_class.oid, relname as name
FROM pg_class
LEFT JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace::oid
WHERE relkind = 'r'
AND relstorage = 'x'
AND pg_namespace.nspname not like 'gp_toolkit';

View File

@ -0,0 +1,5 @@
SELECT pg_class.oid, relname as name
FROM pg_class
WHERE relkind = 'r'
AND relstorage = 'x'
AND pg_class.oid = {{ external_table_id }}::oid;

View File

@ -0,0 +1,99 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
import sys
import six
from pgadmin.browser.server_groups.servers\
.databases.external_tables import ExternalTablesModule
from pgadmin.utils.route import BaseTestGenerator
if sys.version_info < (3, 3):
from mock import MagicMock, Mock
else:
from unittest.mock import MagicMock, Mock
class TestExternalTablesModule(BaseTestGenerator):
scenarios = [
('#BackendSupported When access the on a Postgresql Database, '
'it returns false',
dict(
test_type='backend-support',
manager=dict(
server_type='pg',
sversion=90100
),
expected_result=False,
)),
('#BackendSupported When access the on a GreenPlum Database, '
'it returns true',
dict(
test_type='backend-support',
manager=dict(
server_type='gpdb',
sversion=82303
),
expected_result=True
)),
('#get_nodes when trying to retrieve the node, '
'it should return true',
dict(
test_type='get-nodes',
function_parameters=dict(
gid=10,
sid=11,
did=12,
),
expected_generate_browser_collection_node_called_with=12
)),
('#get_module_use_template_javascript when checking if need to '
'generate javascript from template, '
'it should return false',
dict(
test_type='template-javascript',
expected_result=False
))
]
def runTest(self):
if self.test_type == 'backend-support':
self.__test_backend_support()
elif self.test_type == 'get-nodes':
self.__test_get_nodes()
elif self.test_type == 'template-javascript':
self.__test_template_javascript()
def __test_backend_support(self):
manager = MagicMock()
manager.sversion = self.manager['sversion']
manager.server_type = self.manager['server_type']
module = ExternalTablesModule('something')
self.assertEquals(
self.expected_result,
module.BackendSupported(manager)
)
def __test_get_nodes(self):
module = ExternalTablesModule('something')
module.generate_browser_collection_node = Mock()
result = module.get_nodes(**self.function_parameters)
six.next(result)
module.generate_browser_collection_node.assert_called_with(
self.expected_generate_browser_collection_node_called_with
)
def __test_template_javascript(self):
module = ExternalTablesModule('something')
self.assertEquals(
self.expected_result,
module.module_use_template_javascript)

View File

@ -0,0 +1,428 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
import sys
from pgadmin.browser.server_groups.servers.databases.external_tables import \
ExternalTablesView
from pgadmin.utils.route import BaseTestGenerator
if sys.version_info < (3, 3):
from mock import MagicMock, patch
else:
from unittest.mock import MagicMock, patch
class TestExternalTablesView(BaseTestGenerator):
scenarios = [
('#check_precondition When executing any http call, '
'it saves stores the connection and the manager in the class object',
dict(
test_type='check-precondition',
function_parameters=dict(
server_group_id=0,
server_id=1,
database_id=2,
),
manager=MagicMock(),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(True, dict(rows=[])),
expected_manager_connection_to_be_called_with=dict(
did=2
),
)),
('#nodes When retrieving the children of external tables, '
'it return no child '
'and status 200',
dict(
test_type='children',
function_parameters=dict(
server_group_id=0,
server_id=1,
database_id=2,
),
manager=MagicMock(server_type='gpdb', sversion=80323),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(True, dict(rows=[])),
expected_make_json_response_called_with=dict(data=[]),
)),
('#nodes When retrieving the nodes '
'and the database does not have external tables, '
'it return no child nodes '
'and status 200',
dict(
test_type='nodes',
function_parameters=dict(
server_group_id=0,
server_id=1,
database_id=2,
),
manager=MagicMock(server_type='gpdb', sversion=80323),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(True, dict(rows=[])),
expect_render_template_called_with='sql/#gpdb#80323#/list.sql',
expected_make_json_response_called_with=dict(
data=[],
status=200
),
)),
('#nodes When retrieving the nodes '
'and an error happens while executing the query, '
'it return an internal server error '
'and status 500',
dict(
test_type='nodes',
function_parameters=dict(
server_group_id=0,
server_id=1,
database_id=2,
),
manager=MagicMock(server_type='gpdb', sversion=80323),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(False, 'Some error message'),
expect_render_template_called_with='sql/#gpdb#80323#/list.sql',
expected_internal_server_error_called_with=dict(
errormsg='Some error message'
),
)),
('#nodes When retrieving the nodes '
'and the database has 2 external tables, '
'it return 2 child nodes '
'and status 200',
dict(
test_type='nodes',
function_parameters=dict(
server_group_id=0,
server_id=1,
database_id=2,
),
manager=MagicMock(server_type='gpdb', sversion=80323),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(True, dict(
rows=[
dict(
oid='oid1',
name='table_one'
),
dict(
oid='oid2',
name='table_two'
),
]
)),
expect_render_template_called_with='sql/#gpdb#80323#/list.sql',
expected_make_json_response_called_with=dict(
data=[
{
'id': "external_table/oid1",
'label': 'table_one',
'icon': 'icon-external_table',
'inode': False,
'_type': 'external_table',
'_id': 'oid1',
'_pid': 2,
'module': 'pgadmin.node.external_table'
},
{
'id': "external_table/oid2",
'label': 'table_two',
'icon': 'icon-external_table',
'inode': False,
'_type': 'external_table',
'_id': 'oid2',
'_pid': 2,
'module': 'pgadmin.node.external_table'
}
],
status=200
),
)),
('#node When retrieving the information about 1 external table '
'and an error happens while executing the query, '
'it return an internal server error '
'and status 500',
dict(
test_type='node',
function_parameters=dict(
server_group_id=0,
server_id=1,
database_id=2,
external_table_id=11
),
manager=MagicMock(server_type='gpdb', sversion=80323),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(False, 'Some error message'),
expect_render_template_called_with=dict(
template_name_or_list='sql/#gpdb#80323#/node.sql',
external_table_id=11
),
expected_internal_server_error_called_with=dict(
errormsg='Some error message'
),
)),
('#node When retrieving the information about 1 external table '
'and table does not exist, '
'it return an error message '
'and status 404',
dict(
test_type='node',
function_parameters=dict(
server_group_id=0,
server_id=1,
database_id=2,
external_table_id=11
),
manager=MagicMock(server_type='gpdb', sversion=80323),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(True, dict(rows=[])),
expect_render_template_called_with=dict(
template_name_or_list='sql/#gpdb#80323#/node.sql',
external_table_id=11
),
expected_make_json_response_called_with=dict(
data='Could not find the external table.',
status=404
),
)),
('#nodes When retrieving the information about 1 external table '
'and the table exists, '
'it return external node information '
'and status 200',
dict(
test_type='node',
function_parameters=dict(
server_group_id=0,
server_id=1,
database_id=2,
external_table_id=11
),
manager=MagicMock(server_type='gpdb', sversion=80323),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(True, dict(
rows=[
dict(
oid='oid1',
name='table_one'
),
dict(
oid='oid2',
name='table_two'
),
]
)),
expect_render_template_called_with=dict(
template_name_or_list='sql/#gpdb#80323#/node.sql',
external_table_id=11
),
expected_make_json_response_called_with=dict(
data={
'id': "external_table/oid1",
'label': 'table_one',
'icon': 'icon-external_table',
'inode': False,
'_type': 'external_table',
'_id': 'oid1',
'_pid': 2,
'module': 'pgadmin.node.external_table'
},
status=200
),
)),
('#properties When retrieving the properties of a external table '
'and the table exists, '
'it return the properties '
'and status 200',
dict(
test_type='properties',
function_parameters=dict(
server_group_id=0,
server_id=1,
database_id=2,
external_table_id=11
),
manager=MagicMock(server_type='gpdb', sversion=80323),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(True, dict(
rows=[dict(
urilocation='{http://someurl.com}',
execlocation=['ALL_SEGMENTS'],
fmttype='a',
fmtopts='delimiter \',\' null \'\' '
'escape \'"\' quote \'"\'',
command=None,
rejectlimit=None,
rejectlimittype=None,
errtblname=None,
errortofile=None,
pg_encoding_to_char='UTF8',
writable=False,
options=None,
distribution=None,
name='some_table',
namespace='public'
)]
)),
expect_render_template_called_with=dict(
template_name_or_list='sql/#gpdb#80323#/'
'get_table_information.sql',
table_oid=11
),
expected_make_response_called_with=dict(
response=dict(
name="some_table",
type='readable',
format_type='UTF8',
format_options='delimiter \',\' null \'\' '
'escape \'"\' quote \'"\'',
external_options=None,
command=None,
execute_on='all segments',
),
status=200
),
)),
]
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.get_driver')
def runTest(self, get_driver_mock):
self.__before_all(get_driver_mock)
if self.test_type == 'check-precondition':
self.__test_backend_support()
elif self.test_type == 'nodes':
self.__test_nodes()
elif self.test_type == 'node':
self.__test_node()
elif self.test_type == 'children':
self.__test_children()
elif self.test_type == 'properties':
self.__test_properties()
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.make_json_response')
def __test_children(self, make_json_response_mock):
self.manager.connection = MagicMock(return_value=self.connection)
external_tables_view = ExternalTablesView(cmd='')
external_tables_view.children(**self.function_parameters)
make_json_response_mock.assert_called_with(
**self.expected_make_json_response_called_with
)
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.render_template')
def __test_backend_support(self, _):
self.manager.connection = MagicMock(return_value=self.connection)
external_tables_view = ExternalTablesView(cmd='')
external_tables_view.nodes(**self.function_parameters)
self.manager.connection.assert_called_with(
**self.expected_manager_connection_to_be_called_with
)
self.assertEquals(self.manager, external_tables_view.manager)
self.assertEquals(self.connection, external_tables_view.connection)
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.render_template')
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.make_json_response')
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.internal_server_error')
def __test_nodes(self, internal_server_error_mock,
make_json_response_mock, render_template_mock):
external_tables_view = ExternalTablesView(cmd='')
external_tables_view.nodes(**self.function_parameters)
if hasattr(self, 'expected_internal_server_error_called_with'):
internal_server_error_mock.assert_called_with(
**self.expected_internal_server_error_called_with
)
else:
internal_server_error_mock.assert_not_called()
if hasattr(self, 'expected_make_json_response_called_with'):
make_json_response_mock.assert_called_with(
**self.expected_make_json_response_called_with
)
else:
make_json_response_mock.assert_not_called()
render_template_mock.assert_called_with(
self.expect_render_template_called_with
)
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.render_template')
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.make_json_response')
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.internal_server_error')
def __test_node(self, internal_server_error_mock,
make_json_response_mock, render_template_mock):
external_tables_view = ExternalTablesView(cmd='')
external_tables_view.node(**self.function_parameters)
if hasattr(self, 'expected_internal_server_error_called_with'):
internal_server_error_mock.assert_called_with(
**self.expected_internal_server_error_called_with
)
else:
internal_server_error_mock.assert_not_called()
if hasattr(self, 'expected_make_json_response_called_with'):
make_json_response_mock.assert_called_with(
**self.expected_make_json_response_called_with
)
else:
make_json_response_mock.assert_not_called()
render_template_mock.assert_called_with(
**self.expect_render_template_called_with
)
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.render_template')
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.make_response')
@patch('pgadmin.browser.server_groups.servers.databases.external_tables'
'.internal_server_error')
def __test_properties(self, internal_server_error_mock,
make_response_mock, render_template_mock):
external_tables_view = ExternalTablesView(cmd='')
external_tables_view.properties(**self.function_parameters)
if hasattr(self, 'expected_internal_server_error_called_with'):
internal_server_error_mock.assert_called_with(
**self.expected_internal_server_error_called_with
)
else:
internal_server_error_mock.assert_not_called()
if hasattr(self, 'expected_make_response_called_with'):
make_response_mock.assert_called_with(
**self.expected_make_response_called_with
)
else:
make_response_mock.assert_not_called()
render_template_mock.assert_called_with(
**self.expect_render_template_called_with
)
def __before_all(self, get_driver_mock):
self.connection.execute_2darray.return_value = \
self.execute_2darray_return_value
self.manager.connection = MagicMock(return_value=self.connection)
get_driver_mock.return_value = MagicMock(
connection_manager=MagicMock(return_value=self.manager)
)

View File

@ -0,0 +1,375 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
from pgadmin.browser.server_groups.servers.databases \
.external_tables.mapping_utils import \
map_column_from_database, map_table_information_from_database, \
is_web_table, format_options, map_execution_location, map_format_type
from pgadmin.utils.route import BaseTestGenerator
class TestMappingUtils(BaseTestGenerator):
scenarios = [
('#map_column_from_database When retrieving columns from table, '
'it returns only the name and type',
dict(
test_type='map_column_from_database',
function_arguments=dict(column_information=dict(
name='some name',
cltype='some type',
other_column='some other column'
)),
expected_result=dict(name='some name', type='some type')
)),
('#map_table_information_from_database When retrieving information '
'from web table, '
'it returns all fields',
dict(
test_type='map_table_information_from_database',
function_arguments=dict(table_information=dict(
urilocation='{http://someurl.com}',
execlocation=['ALL_SEGMENTS'],
fmttype='b',
fmtopts='delimiter \',\' null \'\' escape \'"\' quote \'"\'',
command=None,
rejectlimit=None,
rejectlimittype=None,
errtblname=None,
errortofile=None,
pg_encoding_to_char='UTF8',
writable=False,
options=None,
distribution=None,
name='some_table_name',
namespace='some_name_space'
)),
expected_result=dict(
uris=['http://someurl.com'],
isWeb=True,
executionLocation=dict(type='all_segments', value=None),
formatType='custom',
formatOptions='delimiter = $$,$$,escape = $$"$$,'
'null = $$$$,quote = $$"$$',
command=None,
rejectLimit=None,
rejectLimitType=None,
errorTableName=None,
erroToFile=None,
pgEncodingToChar='UTF8',
writable=False,
options=None,
distribution=None,
name='some_table_name',
namespace='some_name_space'
)
)),
('#map_table_information_from_database When retrieving information '
'from a web table using command instead of URIs, '
'it returns all fields',
dict(
test_type='map_table_information_from_database',
function_arguments=dict(table_information=dict(
urilocation=None,
execlocation=['ALL_SEGMENTS'],
fmttype='b',
fmtopts='delimiter \',\' null \'\' escape \'"\' quote \'"\'',
command='cat /tmp/places || echo \'error\'',
rejectlimit=None,
rejectlimittype=None,
errtblname=None,
errortofile=None,
pg_encoding_to_char='UTF8',
writable=False,
options=None,
distribution=None,
name='some_table_name',
namespace='some_name_space'
)),
expected_result=dict(
uris=None,
isWeb=True,
executionLocation=dict(type='all_segments', value=None),
formatType='custom',
formatOptions='delimiter = $$,$$,escape = $$"$$,'
'null = $$$$,quote = $$"$$',
command='cat /tmp/places || echo \'error\'',
rejectLimit=None,
rejectLimitType=None,
errorTableName=None,
erroToFile=None,
pgEncodingToChar='UTF8',
writable=False,
options=None,
distribution=None,
name='some_table_name',
namespace='some_name_space'
)
)),
('#map_table_information_from_database When retrieving information '
'from a none web table, '
'it returns all fields',
dict(
test_type='map_table_information_from_database',
function_arguments=dict(table_information=dict(
urilocation='{gpfdist://filehost:8081/*.csv}',
execlocation=['ALL_SEGMENTS'],
fmttype='b',
fmtopts='delimiter \',\' null \'\' escape \'"\' quote \'"\'',
command=None,
rejectlimit=None,
rejectlimittype=None,
errtblname=None,
errortofile=None,
pg_encoding_to_char='UTF8',
writable=False,
options=None,
distribution=None,
name='some_table_name',
namespace='some_name_space'
)),
expected_result=dict(
uris=['gpfdist://filehost:8081/*.csv'],
isWeb=False,
executionLocation=dict(type='all_segments', value=None),
formatType='custom',
formatOptions='delimiter = $$,$$,escape = $$"$$,'
'null = $$$$,quote = $$"$$',
command=None,
rejectLimit=None,
rejectLimitType=None,
errorTableName=None,
erroToFile=None,
pgEncodingToChar='UTF8',
writable=False,
options=None,
distribution=None,
name='some_table_name',
namespace='some_name_space'
)
)),
('#is_web_table When url starts with http '
'and command is None '
'it returns true',
dict(
test_type='is_web_table',
function_arguments=dict(
uris='{http://someurl.com}',
command=None
),
expected_result=True
)),
('#is_web_table When url starts with https '
'and command is None, '
'it returns true',
dict(
test_type='is_web_table',
function_arguments=dict(
uris='{https://someurl.com}',
command=None
),
expected_result=True
)),
('#is_web_table When url starts with s3 '
'and command is None'
'it returns false',
dict(
test_type='is_web_table',
function_arguments=dict(uris='{s3://someurl.com}', command=None),
expected_result=False
)),
('#is_web_table When url is None '
'and command is not None'
'it returns false',
dict(
test_type='is_web_table',
function_arguments=dict(uris=None, command='Some command'),
expected_result=True
)),
('#map_execution_location When value is "HOST: 1.1.1.1", '
'it returns {type: "host", value: "1.1.1.1"}',
dict(
test_type='map_execution_location',
function_arguments=dict(execution_location=['HOST: 1.1.1.1']),
expected_result=dict(type='host', value='1.1.1.1')
)),
('#map_execution_location When value is "PER_HOST", '
'it returns {type: "per_host", value: None}',
dict(
test_type='map_execution_location',
function_arguments=dict(execution_location=['PER_HOST']),
expected_result=dict(type='per_host', value=None)
)),
('#map_execution_location When value is "MASTER_ONLY", '
'it returns {type: "master_only", value: None}',
dict(
test_type='map_execution_location',
function_arguments=dict(execution_location=['MASTER_ONLY']),
expected_result=dict(type='master_only', value=None)
)),
('#map_execution_location When value is "SEGMENT_ID: 1234", '
'it returns {type: "segment", value: "1234"}',
dict(
test_type='map_execution_location',
function_arguments=dict(execution_location=['SEGMENT_ID: 1234']),
expected_result=dict(type='segment', value='1234')
)),
('#map_execution_location When value is "TOTAL_SEGS: 4", '
'it returns {type: "segments", value: "4"}',
dict(
test_type='map_execution_location',
function_arguments=dict(execution_location=['TOTAL_SEGS: 4']),
expected_result=dict(type='segments', value='4')
)),
('#map_execution_location When value is "{ALL_SEGMENTS}", '
'it returns {type: "all_segments", value: None}',
dict(
test_type='map_execution_location',
function_arguments=dict(execution_location=['ALL_SEGMENTS']),
expected_result=dict(type='all_segments', value=None)
)),
('#map_format_type When value is "c", '
'it returns csv',
dict(
test_type='map_format_type',
function_arguments=dict(format_type='c'),
expected_result='csv'
)),
('#map_format_type When value is "something strange", '
'it returns csv',
dict(
test_type='map_format_type',
function_arguments=dict(format_type='something strange'),
expected_result='csv'
)),
('#map_format_type When value is "b", '
'it returns custom',
dict(
test_type='map_format_type',
function_arguments=dict(format_type='b'),
expected_result='custom'
)),
('#map_format_type When value is "t", '
'it returns text',
dict(
test_type='map_format_type',
function_arguments=dict(format_type='t'),
expected_result='text'
)),
('#map_format_type When value is "a", '
'it returns avro',
dict(
test_type='map_format_type',
function_arguments=dict(format_type='a'),
expected_result='avro'
)),
('#map_format_type When value is "p", '
'it returns parquet',
dict(
test_type='map_format_type',
function_arguments=dict(format_type='p'),
expected_result='parquet'
)),
('#format_options passing None, '
'it returns None',
dict(
test_type='format_options',
function_arguments=dict(format_type='avro', options=None),
expected_result=None
)),
('#format_options passing empty string, '
'it returns empty string',
dict(
test_type='format_options',
function_arguments=dict(format_type='parquet', options=''),
expected_result=''
)),
('#format_options passing "formatter \'fixedwidth_in\' null \' \'", '
'it returns "formatter = $$fixedwidth_in$$,null = $$ $$"',
dict(
test_type='format_options',
function_arguments=dict(format_type='custom',
options='formatter \'fixedwidth_in\' '
'null \' \''),
expected_result='formatter = $$fixedwidth_in$$,null = $$ $$'
)),
('#format_options passing '
'"formatter \'fixedwidth_in\' comma \'\'\' null \' \'", '
'it returns '
'"formatter = $$fixedwidth_in$$,comma = $$\'$$,null = $$ $$"',
dict(
test_type='format_options',
function_arguments=dict(format_type='custom',
options='formatter \'fixedwidth_in\' '
'comma \'\'\' null \' \''),
expected_result='comma = $$\'$$,formatter = $$fixedwidth_in$$,'
'null = $$ $$'
)),
('#format_options passing '
'"formatter \'fixedwidth_in\' null \' \' preserve_blanks '
'\'on\' comma \'\\\'\'", '
'it returns '
'"formatter = $$fixedwidth_in$$,null = $$ $$,preserve_blanks = '
'$$on$$,comma = $$\'$$"',
dict(
test_type='format_options',
function_arguments=dict(format_type='custom',
options='formatter \'fixedwidth_in\' '
'null \' \' '
'preserve_blanks \'on\' '
'comma \'\'\''),
expected_result='comma = $$\'$$,formatter = $$fixedwidth_in$$,'
'null = $$ $$,'
'preserve_blanks = $$on$$'
)),
('#format_options When format type is text '
'it returns escaped string',
dict(
test_type='format_options',
function_arguments=dict(format_type='text',
options='something \'strange\' '
'other \'\'\''),
expected_result='other $$\'$$ '
'something $$strange$$'
)),
('#format_options When format type is csv '
'it returns escaped string',
dict(
test_type='format_options',
function_arguments=dict(format_type='csv',
options='something \'strange\' '
'other \'\'\''),
expected_result='other $$\'$$ '
'something $$strange$$'
))
]
def runTest(self):
result = None
if self.test_type == 'map_column_from_database':
result = map_column_from_database(**self.function_arguments)
elif self.test_type == 'map_table_information_from_database':
result = map_table_information_from_database(
**self.function_arguments)
elif self.test_type == 'map_execution_location':
result = map_execution_location(**self.function_arguments)
elif self.test_type == 'map_format_type':
result = map_format_type(**self.function_arguments)
elif self.test_type == 'is_web_table':
result = is_web_table(**self.function_arguments)
elif self.test_type == 'format_options':
result = format_options(**self.function_arguments)
self.assertEqual(result, self.expected_result)

View File

@ -0,0 +1,156 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
import sys
from pgadmin.browser.server_groups.servers.databases \
.external_tables import Properties
from pgadmin.browser.server_groups.servers.databases.external_tables \
.properties import PropertiesException, PropertiesTableNotFoundException
from pgadmin.utils.route import BaseTestGenerator
if sys.version_info < (3, 3):
from mock import MagicMock, patch
else:
from unittest.mock import MagicMock, patch
class TestExternalTablesView(BaseTestGenerator):
scenarios = [
('#properties When retrieving the properties of a external table '
'and the table exists, '
'it return the properties ',
dict(
test_type='properties',
function_parameters=dict(
table_oid=11
),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(True, dict(
rows=[dict(
urilocation='{http://someurl.com}',
execlocation=['ALL_SEGMENTS'],
fmttype='a',
fmtopts='delimiter \',\' null \'\' '
'escape \'"\' quote \'"\'',
command=None,
rejectlimit=None,
rejectlimittype=None,
errtblname=None,
errortofile=None,
pg_encoding_to_char='UTF8',
writable=False,
options=None,
distribution=None,
name='some_table',
namespace='public'
)]
)),
expect_render_template_called_with=dict(
template_name_or_list='some/sql/location/'
'get_table_information.sql',
table_oid=11
),
expected_result=dict(
name="some_table",
type='readable',
format_type='UTF8',
format_options='delimiter \',\' null \'\' '
'escape \'"\' quote \'"\'',
external_options=None,
command=None,
execute_on='all segments',
),
)),
('#properties When retrieving the properties of a external table '
'and a SQL error happens, '
'it raises exception with the error message',
dict(
test_type='properties',
function_parameters=dict(
table_oid=11
),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(False, 'Some error'),
expect_render_template_called_with=dict(
template_name_or_list='some/sql/location/'
'get_table_information.sql',
table_oid=11
),
expected_raise_exception=PropertiesException,
expected_internal_server_error_called_with=['Some error']
)),
('#properties When retrieving the properties of a external table '
'and table is not found, '
'it raises exception ',
dict(
test_type='properties',
function_parameters=dict(
table_oid=11
),
connection=MagicMock(execute_2darray=MagicMock()),
execute_2darray_return_value=(True, dict(rows=[])),
expect_render_template_called_with=dict(
template_name_or_list='some/sql/location/'
'get_table_information.sql',
table_oid=11
),
expected_raise_exception=PropertiesTableNotFoundException
)),
]
def runTest(self):
self.connection.execute_2darray.return_value = \
self.execute_2darray_return_value
self.__test_properties()
@patch('pgadmin.browser.server_groups.servers.databases'
'.external_tables.properties.internal_server_error')
def __test_properties(self, internal_server_error_mock):
self.maxDiff = None
render_template_mock = MagicMock()
external_tables_view = Properties(
render_template_mock,
self.connection,
'some/sql/location/'
)
result = None
try:
result = external_tables_view.retrieve(**self.function_parameters)
if hasattr(self, 'expected_raise_exception'):
self.fail('No exception was raised')
except PropertiesException as exception:
if hasattr(self, 'expected_raise_exception'):
if type(exception) is self.expected_raise_exception:
if hasattr(self,
'expected_internal_server_error_called_with'):
internal_server_error_mock.assert_called_with(
*self.expected_internal_server_error_called_with
)
else:
internal_server_error_mock.assert_not_called()
else:
self.fail('Wrong exception type: ' + str(exception))
else:
raise exception
if hasattr(self, 'expected_result'):
self.assertEqual(result, self.expected_result)
render_template_mock.assert_called_with(
**self.expect_render_template_called_with
)

View File

@ -0,0 +1,261 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2018, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
import sys
from pgadmin.browser.server_groups.servers.databases \
.external_tables.reverse_engineer_ddl import \
ReverseEngineerDDL, ReverseEngineerDDLException
from pgadmin.utils.route import BaseTestGenerator
if sys.version_info < (3, 3):
from mock import MagicMock
else:
from unittest.mock import MagicMock
class TestReverseEngineerDDL(BaseTestGenerator):
scenarios = [
('#execute When retriving the DDL for the creation of external '
'tables, '
'it retrieves information of the columns and the tables '
'and generate the SQL to create the table',
dict(
test_type='execute',
function_parameters=dict(table_oid=14),
find_columns_return_value=dict(somevalue='value'),
table_information_return_value=dict(someother='bamm'),
expect_find_columns_called_with=14,
expect_table_information_called_with=14,
expect_render_template_called_with=dict(
template_name_or_list='sql/#gpdb#80323#/create.sql',
table=dict(
someother='bamm',
columns=dict(somevalue='value')
)
)
)),
('#find_columns When an external table exists, '
'and have 3 columns, '
'it returns a list with 1 object that as the table name to inherit '
'from',
dict(
test_type='find_columns',
function_parameters={'table_oid': 123},
execute_2darray_return_value=(True, dict(rows=[
{
'name': 'column_1',
'cltype': 'text',
'inheritedFrom': 'other_table',
'inheritedid': '1234',
}, {
'name': 'column_2',
'cltype': 'int',
'inheritedFrom': 'other_table',
'inheritedid': '1234',
}, {
'name': 'column_3',
'cltype': 'numeric',
'inheritedFrom': 'other_table',
'inheritedid': '1234',
}
])),
expect_render_template_called_with=dict(
template_name_or_list='sql/#gpdb#80323#/get_columns.sql',
table_oid=123
),
expected_result=[
{
'name': 'column_1',
'type': 'text'
},
{
'name': 'column_2',
'type': 'int'
},
{
'name': 'column_3',
'type': 'numeric'
},
],
)),
('#find_columns When error happens while retrieving '
'column information, '
'it raise an exception',
dict(
test_type='find_columns',
function_parameters={'table_oid': 123},
execute_2darray_return_value=(False, 'Some error message'),
expect_render_template_called_with=dict(
template_name_or_list='sql/#gpdb#80323#/get_columns.sql',
table_oid=123
),
expected_exception=ReverseEngineerDDLException(
'Some error message'),
)
),
('#table_information When error happens while retrieving '
'table generic information, '
'it raise an exception',
dict(
test_type='table_information',
function_parameters={'table_oid': 123},
execute_2darray_return_value=(False, 'Some error message'),
expect_render_template_called_with=dict(
template_name_or_list='sql/#gpdb#80323#/'
'get_table_information.sql',
table_oid=123
),
expected_exception=ReverseEngineerDDLException(
'Some error message'),
)
),
('#table_information When cannot find the table, '
'it raise an exception',
dict(
test_type='table_information',
function_parameters={'table_oid': 123},
execute_2darray_return_value=(True, {'rows': []}),
expect_render_template_called_with=dict(
template_name_or_list='sql/#gpdb#80323#/'
'get_table_information.sql',
table_oid=123
),
expected_exception=ReverseEngineerDDLException(
'Table not found'),
)),
('#table_information When retrieving generic information '
'about a Web table, '
'it returns the table information',
dict(
test_type='table_information',
function_parameters={'table_oid': 123},
execute_2darray_return_value=(True, dict(rows=[
{
'urilocation': '{http://someurl.com}',
'execlocation': ['ALL_SEGMENTS'],
'fmttype': 'a',
'fmtopts': 'delimiter \',\' null \'\' '
'escape \'"\' quote \'"\'',
'command': None,
'rejectlimit': None,
'rejectlimittype': None,
'errtblname': None,
'errortofile': None,
'pg_encoding_to_char': 'UTF8',
'writable': False,
'options': None,
'distribution': None,
'name': 'some_table',
'namespace': 'public'
}
])),
expect_render_template_called_with=dict(
template_name_or_list='sql/#gpdb#80323#/'
'get_table_information.sql',
table_oid=123
),
expected_result={
'uris': ['http://someurl.com'],
'isWeb': True,
'executionLocation': dict(type='all_segments', value=None),
'formatType': 'avro',
'formatOptions': 'delimiter = $$,$$,escape = $$"$$,'
'null = $$$$,quote = $$"$$',
'command': None,
'rejectLimit': None,
'rejectLimitType': None,
'errorTableName': None,
'erroToFile': None,
'pgEncodingToChar': 'UTF8',
'writable': False,
'options': None,
'distribution': None,
'name': 'some_table',
'namespace': 'public'
},
)),
]
def __init__(self, *args, **kwargs):
super(TestReverseEngineerDDL, self).__init__(*args, **kwargs)
self.connection = None
self.subject = None
self.render_template_mock = None
def runTest(self):
self.render_template_mock = MagicMock()
self.connection = MagicMock(execute_2darray=MagicMock())
if hasattr(self, 'execute_2darray_return_value'):
self.connection.execute_2darray.return_value = \
self.execute_2darray_return_value
self.subject = ReverseEngineerDDL(
'sql/#gpdb#80323#/',
self.render_template_mock,
self.connection,
1, 2, 3)
if self.test_type == 'find_columns':
self.__test_find_columns()
elif self.test_type == 'table_information':
self.__test_table_information()
elif self.test_type == 'execute':
self.__test_execute()
def __test_find_columns(self):
if hasattr(self, 'expected_exception'):
try:
self.subject.find_columns(**self.function_parameters)
self.fail('Exception not raise')
except ReverseEngineerDDLException as exception:
self.assertEqual(str(exception),
str(self.expected_exception))
else:
result = self.subject.find_columns(**self.function_parameters)
self.assertEqual(self.expected_result, result)
self.render_template_mock.assert_called_with(
**self.expect_render_template_called_with
)
def __test_table_information(self):
if hasattr(self, 'expected_exception'):
try:
self.subject.table_information(**self.function_parameters)
self.fail('Exception not raise')
except ReverseEngineerDDLException as exception:
self.assertEqual(str(exception),
str(self.expected_exception))
else:
result = self.subject.table_information(**self.function_parameters)
self.assertEqual(self.expected_result, result)
self.render_template_mock.assert_called_with(
**self.expect_render_template_called_with
)
def __test_execute(self):
self.subject.find_columns = MagicMock(
return_value=self.find_columns_return_value)
self.subject.table_information = MagicMock(
return_value=self.table_information_return_value)
self.subject.execute(**self.function_parameters)
self.subject.find_columns.assert_called_with(
self.expect_find_columns_called_with)
self.subject.table_information.assert_called_with(
self.expect_table_information_called_with)
self.render_template_mock.assert_called_with(
**self.expect_render_template_called_with)

View File

@ -4,6 +4,7 @@ SELECT rel.oid, rel.relname AS name,
FROM pg_class rel
WHERE rel.relkind IN ('r','s','t') AND rel.relnamespace = {{ scid }}::oid
AND rel.relname NOT IN (SELECT partitiontablename FROM pg_partitions)
AND rel.oid NOT IN (SELECT reloid from pg_exttable)
{% if tid %}
AND rel.oid = {{tid}}::OID
{% endif %}

View File

@ -1,5 +1,6 @@
define('bundled_browser',[
'pgadmin.browser',
'sources/browser/server_groups/servers/databases/external_tables/index',
], function(pgBrowser) {
pgBrowser.init();
});

View File

@ -0,0 +1,88 @@
/////////////////////////////////////////////////////////////
//
// pgAdmin 4 - PostgreSQL Tools
//
// Copyright (C) 2013 - 2018, The pgAdmin Development Team
// This software is released under the PostgreSQL Licence
//
//////////////////////////////////////////////////////////////
export function initialize(pgBrowser, gettext) {
if (!pgBrowser.Nodes['coll-external_table']) {
pgBrowser.Nodes['coll-external_table'] =
pgBrowser.Collection.extend({
node: 'external_table',
label: gettext('External Tables'),
type: 'coll-external_tables',
columns: ['name', 'fdwowner', 'description'],
});
}
if (!pgBrowser.Nodes['external_table']) {
pgBrowser.Nodes['external_table'] = pgBrowser.Node.extend({
parent_type: 'database',
type: 'external_table',
label: gettext('External Table'),
collection_type: 'coll-external_table',
hasSQL: true,
model: pgBrowser.Node.Model.extend({
defaults: {
name: undefined,
type: undefined,
encoding: undefined,
format_type: undefined,
format_option: undefined,
external_options: undefined,
command: undefined,
execute_on: undefined,
},
schema: [
{
id: 'name',
label: gettext('Name'),
type: 'text',
mode: ['properties'],
}, {
id: 'type',
label: gettext('Type'),
type: 'text',
mode: ['properties'],
}, {
id: 'encoding',
label: gettext('Encoding'),
type: 'text',
mode: ['properties'],
}, {
id: 'format_type',
label: gettext('Format Type'),
type: 'text',
mode: ['properties'],
}, {
id: 'format_option',
label: gettext('Format Optionos'),
type: 'text',
mode: ['properties'],
}, {
id: 'external_options',
label: gettext('External Options'),
type: 'text',
mode: ['properties'],
}, {
id: 'command',
label: gettext('Command'),
type: 'text',
mode: ['properties'],
}, {
id: 'execute_on',
label: gettext('Execute on'),
type: 'text',
mode: ['properties'],
},
],
}),
});
}
return pgBrowser;
}

View File

@ -0,0 +1,18 @@
/////////////////////////////////////////////////////////////
//
// pgAdmin 4 - PostgreSQL Tools
//
// Copyright (C) 2013 - 2018, The pgAdmin Development Team
// This software is released under the PostgreSQL Licence
//
//////////////////////////////////////////////////////////////
import pgBrowser from 'top/browser/static/js/browser';
import gettext from 'sources/gettext';
import {initialize} from './external_tables';
let pgBrowserOut = initialize(pgBrowser, gettext);
module.exports = {
pgBrowser: pgBrowserOut,
};

View File

@ -438,6 +438,7 @@ def poll(trans_id):
conn.execute_void("ROLLBACK;")
st, result = conn.async_fetchmany_2darray(ON_DEMAND_RECORD_COUNT)
if st:
if 'primary_keys' in session_obj:
primary_keys = session_obj['primary_keys']

View File

@ -0,0 +1,56 @@
/////////////////////////////////////////////////////////////
//
// pgAdmin 4 - PostgreSQL Tools
//
// Copyright (C) 2013 - 2018, The pgAdmin Development Team
// This software is released under the PostgreSQL Licence
//
//////////////////////////////////////////////////////////////
import {initialize} from 'sources/browser/server_groups/servers/databases/external_tables/external_tables';
describe('when external tables is loaded', () => {
let pgBrowser;
let gettext;
let result;
beforeEach(() => {
pgBrowser = {
Nodes: {},
};
pgBrowser.Collection = jasmine.createSpyObj('Collection', ['extend']);
pgBrowser.Node = jasmine.createSpyObj('Node', ['extend', 'Model']);
pgBrowser.Node.Model = jasmine.createSpyObj('Model', ['extend']);
pgBrowser.Collection.extend.and.returnValue('extended object');
pgBrowser.Node.extend.and.returnValue('extended node object');
gettext = jasmine.createSpy('gettext').and.callFake((text) => text);
});
describe('when external tables is already defined', () => {
beforeEach(() => {
pgBrowser.Nodes['coll-external_table'] = {};
result = initialize(pgBrowser, gettext);
});
it('does not reinitialize it', () => {
expect(pgBrowser.Collection.extend).not.toHaveBeenCalled();
});
it('returns the not updated version of pgBrowser', () => {
expect(result).toBe(pgBrowser);
});
});
describe('when external tables is not defined', () => {
beforeEach(() => {
result = initialize(pgBrowser, gettext);
});
it('initializes "coll-external_tables"', () => {
expect(pgBrowser.Collection.extend).toHaveBeenCalled();
});
it('returns the updated version of pgBrowser', () => {
expect(result.Nodes['coll-external_table']).not.toBeUndefined();
});
});
});

View File

@ -146,12 +146,21 @@ module.exports = {
presets: ['es2015', 'react'],
},
},
}, {
test: /external_table.*\.js/,
use: {
loader: 'babel-loader',
options: {
presets: ['es2015'],
},
},
}, {
// Transforms the code in a way that it works in the webpack environment.
// It uses imports-loader internally to load dependency. Its
// configuration is specified in webpack.shim.js
// Ref: https://www.npmjs.com/package/shim-loader
test: /\.js/,
exclude: [/external_table/],
loader: 'shim-loader',
query: webpackShimConfig,
include: path.join(__dirname, '/pgadmin/browser'),

View File

@ -120,6 +120,7 @@ var webpackShimConfig = {
// Map module id to file path used in 'define(['baseurl', 'misc']). It is
// used by webpack while creating bundle
resolveAlias: {
'top': path.join(__dirname, './pgadmin'),
'bundled_codemirror': path.join(__dirname, './pgadmin/static/bundle/codemirror'),
'bundled_browser': path.join(__dirname, './pgadmin/static/bundle/browser'),
'sources': path.join(__dirname, './pgadmin/static/js'),
@ -213,6 +214,8 @@ var webpackShimConfig = {
'pgadmin.node.catalog_object': path.join(__dirname, './pgadmin/browser/server_groups/servers/databases/schemas/catalog_objects/static/js/catalog_object'),
'pgadmin.dashboard': path.join(__dirname, './pgadmin/dashboard/static/js/dashboard'),
'pgadmin.node.foreign_data_wrapper': path.join(__dirname, './pgadmin/browser/server_groups/servers/databases/foreign_data_wrappers/static/js/foreign_data_wrapper'),
'pgadmin.node.external_table': path.join(__dirname, './pgadmin/static/js/browser/server_groups/servers/databases/external_tables/index'),
'pgadmin.node.external_tables': path.join(__dirname, './pgadmin/static/js/browser/server_groups/servers/databases/external_tables/index'),
'pgadmin.node.foreign_key': path.join(__dirname, './pgadmin/browser/server_groups/servers/databases/schemas/tables/constraints/foreign_key/static/js/foreign_key'),
'pgadmin.browser.server.variable': path.join(__dirname, './pgadmin/browser/server_groups/servers/static/js/variable'),
'pgadmin.tools.grant_wizard': path.join(__dirname, './pgadmin/tools/grant_wizard/static/js/grant_wizard'),