Added support for importing and exporting the table data.

This commit is contained in:
Neel Patel 2016-05-21 15:43:40 +05:30 committed by Ashesh Vashi
parent 2c6ca7d82c
commit da28dc8507
4 changed files with 785 additions and 3 deletions

View File

@ -72,10 +72,12 @@
.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-status.bg-success,
.bg-process-status .bg-bgprocess-success {
color: green;
font-weight: bold;
}
.bg-process-status .bg-bgprocess-failed {
color: red;
font-weight: bold;
}
.ajs-bg-bgprocess > .pg-bg-bgprocess > .pg-bg-status.bg-failed {
@ -83,6 +85,11 @@
background-color: #E99595;
}
.pg-bg-cmd {
color: #4156CC;
font-style: italic;
}
.pg-panel-content div.bg-process-watcher.col-xs-12 {
height: 100%;
padding: 0px 0px 0px 0px;
@ -96,17 +103,17 @@ ol.pg-bg-process-logs {
height: 100%;
overflow: auto;
width: 100%;
background: rgb(255, 239, 217);
font-style: italic;
font-size: small;
}
.pg-bg-res-out, .pg-bg-res-err {
background-color: rgb(255, 239, 217);
padding-left: 10px;
white-space: pre-wrap;
}
.pg-bg-res-out {
color: rgb(0, 157, 207);
color: rgb(12, 116, 149);
}
.pg-bg-res-err {

View File

@ -0,0 +1,286 @@
##########################################################################
#
# pgAdmin 4 - PostgreSQL Tools
#
# Copyright (C) 2013 - 2016, The pgAdmin Development Team
# This software is released under the PostgreSQL Licence
#
##########################################################################
"""A blueprint module implementing the import and export functionality"""
import json
import os
from flask import url_for, Response, render_template, request, current_app
from flask.ext.babel import gettext as _
from flask.ext.security import login_required, current_user
from config import PG_DEFAULT_DRIVER
from pgadmin.utils import PgAdminModule, get_storage_directory, html
from pgadmin.utils.ajax import make_json_response, bad_request
from pgadmin.model import Server
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
MODULE_NAME = 'import_export'
class ImportExportModule(PgAdminModule):
"""
class ImportExportModule(PgAdminModule)
A module class for import which is derived from PgAdminModule.
Methods:
-------
* get_own_javascripts(self)
- Method is used to load the required javascript files for import module
"""
LABEL = _('Import/Export')
def get_own_javascripts(self):
scripts = list()
for name, script in [
['pgadmin.tools.import_export', 'js/import_export']
]:
scripts.append({
'name': name,
'path': url_for('import_export.index') + script,
'when': None
})
return scripts
blueprint = ImportExportModule(MODULE_NAME, __name__)
class Message(IProcessDesc):
"""
Message(IProcessDesc)
Defines the message shown for the Message operation.
"""
def __init__(self, _sid, _schema, _tbl, _database, _storage):
self.sid = _sid
self.schema = _schema
self.table = _tbl
self.database = _database
self.storage = _storage
@property
def message(self):
# Fetch the server details like hostname, port, roles etc
s = Server.query.filter_by(
id=self.sid, user_id=current_user.id
).first()
return _(
"Copying table data - '{0}.{1}' on database '{2}' and server ({3}{4})..."
).format(
self.schema, self.table, self.database, s.host, s.port
)
def details(self, cmd, args):
# Fetch the server details like hostname, port, roles etc
s = Server.query.filter_by(
id=self.sid, user_id=current_user.id
).first()
res = '<div class="h5">'
res += html.safe_str(
_(
"Copying table data '{0}.{1}' on database '{2}' for the server - '{3}'"
).format(
self.schema, self.table, self.database,
"{0} ({1}:{2})".format(s.name, s.host, s.port)
)
)
res += '</div><div class="h5">'
res += html.safe_str(
_("Running command:")
)
res += '</b><br><span class="pg-bg-cmd">'
res += html.safe_str(cmd)
replace_next = False
def cmdArg(x):
if x:
x = x.replace('\\', '\\\\')
x = x.replace('"', '\\"')
x = x.replace('""', '\\"')
return ' "' + html.safe_str(x) + '"'
return ''
for arg in args:
if arg and len(arg) >= 2 and arg[:2] == '--':
if arg == '--command':
replace_next = True
res += ' ' + arg
elif replace_next:
if self.storage:
arg = arg.replace(self.storage, '<STORAGE_DIR>')
res += ' "' + html.safe_str(arg) + '"'
else:
res += cmdArg(arg)
res += '</span></div>'
return res
@blueprint.route("/")
@login_required
def index():
return bad_request(errormsg=_("This URL can not be called directly!"))
@blueprint.route("/js/import_export.js")
@login_required
def script():
"""render the import/export javascript file"""
return Response(
response=render_template("import_export/js/import_export.js", _=_),
status=200,
mimetype="application/javascript"
)
@blueprint.route('/create_job/<int:sid>', methods=['POST'])
@login_required
def create_import_export_job(sid):
"""
Args:
sid: Server ID
Creates a new job for import and export table data functionality
Returns:
None
"""
if request.form:
# Convert ImmutableDict to dict
data = dict(request.form)
data = json.loads(data['data'][0])
else:
data = json.loads(request.data.decode())
# Fetch the server details like hostname, port, roles etc
server = Server.query.filter_by(
id=sid).first()
if server is None:
return make_json_response(
success=0,
errormsg=_("Couldn't find the given server")
)
# To fetch MetaData for the server
from pgadmin.utils.driver import get_driver
driver = get_driver(PG_DEFAULT_DRIVER)
manager = driver.connection_manager(server.id)
conn = manager.connection()
connected = conn.connected()
if not connected:
return make_json_response(
success=0,
errormsg=_("Please connect to the server first...")
)
# Get the utility path from the connection manager
utility = manager.utility('sql')
# Get the storage path from preference
storage_dir = get_storage_directory()
if 'filename' in data:
if os.name == 'nt':
data['filename'] = data['filename'].replace('/', '\\')
if storage_dir:
storage_dir = storage_dir.replace('/', '\\')
data['filename'] = data['filename'].replace('\\', '\\\\')
data['filename'] = os.path.join(storage_dir, data['filename'])
else:
data['filename'] = os.path.join(storage_dir, data['filename'])
else:
return make_json_response(
data={'status': False, 'info': 'Please specify a valid file'}
)
cols = None
icols = None
if data['icolumns']:
ignore_cols = data['icolumns']
# format the ignore column list required as per copy command
# requirement
if ignore_cols and len(ignore_cols) > 0:
for col in ignore_cols:
if icols:
icols += ', '
else:
icols = '('
icols += driver.qtIdent(conn, col)
icols += ')'
# format the column import/export list required as per copy command
# requirement
if data['columns']:
columns = data['columns']
if columns and len(columns) > 0:
for col in columns:
if cols:
cols += ', '
else:
cols = '('
cols += driver.qtIdent(conn, col)
cols += ')'
# Create the COPY FROM/TO from template
query = render_template(
'import_export/sql/cmd.sql',
conn=conn,
data=data,
columns=cols,
ignore_column_list=icols
)
args = [
'--host', server.host, '--port', str(server.port),
'--username', server.username, '--dbname',
driver.qtIdent(conn, data['database']),
'--command', query
]
try:
p = BatchProcess(
desc=Message(
sid,
data['schema'],
data['table'],
data['database'],
storage_dir
),
cmd=utility, args=args
)
manager.export_password_env(p.id)
p.start()
jid = p.id
except Exception as e:
current_app.logger.exception(e)
return make_json_response(
status=410,
success=0,
errormsg=str(e)
)
# Return response
return make_json_response(
data={'job_id': jid, 'success': 1}
)

View File

@ -0,0 +1,488 @@
define(
['jquery', 'underscore', 'underscore.string', 'alertify', 'pgadmin',
'pgadmin.browser', 'backbone', 'backgrid', 'backform',
'pgadmin.backform', 'pgadmin.backgrid', 'pgadmin.browser.node.ui'],
function($, _, S, Alertify, pgAdmin, pgBrowser, Backbone, Backgrid, Backform) {
pgAdmin = pgAdmin || window.pgAdmin || {};
var pgTools = pgAdmin.Tools = pgAdmin.Tools || {};
// Return back, this has been called more than once
if (pgAdmin.Tools.import_utility)
return pgAdmin.Tools.import_utility;
// Main model for Import/Export functionality
var ImportExportModel = Backbone.Model.extend({
defaults: {
is_import: false, /* false for Export */
filename: undefined,
format: 'csv',
encoding: undefined,
oid: undefined,
header: undefined,
delimiter: ';',
quote: '\"',
escape: '\'',
null_string: undefined,
columns: null,
icolumns: [],
database: undefined,
schema: undefined,
table: undefined
},
schema: [{
id: 'is_import', label:'{{ _('Import/Export') }}', cell: 'switch',
type: 'switch', group: '{{ _('Options')}}',
options: {
'onText': '{{ _('Import') }}', 'offText': '{{ _('Export') }}',
'onColor': 'success', 'offColor': 'primary'
}
}, {
type: 'nested', control: 'fieldset', label: '{{ _('File Info') }}',
group: '{{ _('Options') }}',
schema:[{ /* select file control for import */
id: 'filename', label: '{{ _('Filename')}}', deps: ['is_import'],
type: 'text', control: Backform.FileControl, group: '{{ _('File Info')}}',
dialog_type: 'select_file', supp_types: ['csv', 'txt', '*'],
visible: 'importing'
}, { /* create file control for export */
id: 'filename', label: '{{ _('Filename')}}', deps: ['is_import'],
type: 'text', control: Backform.FileControl, group: '{{ _('File Info')}}',
dialog_type: 'create_file', supp_types: ['csv', 'txt', '*'],
visible: 'exporting'
}, {
id: 'format', label: '{{ _("Format") }}', cell: 'string',
control: 'select2', group: '{{ _('File Info')}}',
options:[
{'label': 'binary', 'value': 'binary'}, {'label': 'csv', 'value': 'csv'}, {'label': 'text', 'value': 'text'},
],
disabled: 'isDisabled', select2: {allowClear: false, width: "100%" },
}, {
id: 'encoding', label: '{{ _("Encoding") }}', cell: 'string',
control: 'node-ajax-options', node: 'database', url: 'get_encodings', first_empty: true,
group: '{{ _('File Info')}}'
}]
},{
id: 'columns', label: '{{ _("Columns to import") }}', cell: 'string',
deps: ['is_import'], type: 'array', first_empty: false,
control: Backform.NodeListByNameControl.extend({
// By default, all the import columns should be selected
initialize: function() {
Backform.NodeListByNameControl.prototype.initialize.apply(this, arguments);
var self = this,
options = self.field.get('options'),
op_vals = [];
if (_.isFunction(options)) {
try {
var all_cols = options.apply(self);
for(idx in all_cols) {
op_vals.push((all_cols[idx])['value']);
}
} catch(e) {
// Do nothing
options = [];
}
} else {
for (idx in options) {
op_vals.push((options[idx])['value']);
}
}
self.model.set('columns',op_vals);
}
}),
transform: function(rows) {
var self = this,
node = self.field.get('schema_node'),
res = [];
_.each(rows, function(r) {
var l = (_.isFunction(node['node_label']) ?
(node['node_label']).apply(node, [r, self.model, self]) :
r.label),
image = (_.isFunction(node['node_image']) ?
(node['node_image']).apply(
node, [r, self.model, self]
) :
(node['node_image'] || ('icon-' + node.type)));
res.push({
'value': r.label,
'image': image,
'label': l
});
});
return res;
},
node: 'column', url: 'nodes', group: '{{ _('Columns')}}',
select2: {
multiple: true, allowClear: false,
placeholder: '{{ _('Columns for importing...') }}',
first_empty: false
}, visible: 'importing',
helpMessage:
'{{ _('An optional list of columns to be copied. If no column list is specified, all columns of the table will be copied.') }}'
}, {
id: 'columns', label: '{{ _("Columns to export") }}', cell: 'string',
deps: ['is_import'], type: 'array',
control: 'node-list-by-name', first_empty: false,
node: 'column', url: 'nodes', group: '{{ _('Columns')}}',
select2: {
multiple: true, allowClear: true,
placeholder: '{{ _('Colums for exporting...') }}'
}, visible: 'exporting',
transform: function(rows) {
var self = this,
node = self.field.get('schema_node'),
res = [];
_.each(rows, function(r) {
var l = (_.isFunction(node['node_label']) ?
(node['node_label']).apply(node, [r, self.model, self]) :
r.label),
image = (_.isFunction(node['node_image']) ?
(node['node_image']).apply(
node, [r, self.model, self]
) :
(node['node_image'] || ('icon-' + node.type)));
res.push({
'value': r.label,
'image': image,
'label': l
});
});
return res;
},
helpMessage:
'{{ _('An optional list of columns to be copied. If no column list is specified, all columns of the table will be copied.') }}'
}, {
id: 'null_string', label: '{{ _("NULL Strings") }}', cell: 'string',
type: 'text', group: '{{ _('Columns')}}', disabled: 'isDisabled',
deps: ['format'],
helpMessage:
"{{ _("Specifies the string that represents a null value. The default is \\\\N (backslash-N) in text format, and an unquoted empty string in CSV format. You might prefer an empty string even in text format for cases where you don't want to distinguish nulls from empty strings. This option is not allowed when using binary format.") }}"
}, {
id: 'icolumns', label: '{{ _("Not null columns") }}', cell: 'string',
control: 'node-list-by-name', node: 'column',
group: '{{ _('Columns')}}', deps: ['format', 'is_import'], disabled: 'isDisabled',
type: 'array', first_empty: false,
select2: {
multiple: true, allowClear: true, first_empty: true,
placeholder: '{{ _('Not null columns...') }}'
},
helpMessage:
'{{ _('Do not match the specified columns values against the null string. In the default case where the null string is empty, this means that empty values will be read as zero-length strings rather than nulls, even when they are not quoted. This option is allowed only in import, and only when using CSV format.') }}'
}, {
type: 'nested', control: 'fieldset', label: '{{ _('Miscellaneous') }}',
group: '{{ _('Options') }}',
schema:[{
id: 'oid', label:'{{ _('OID') }}', cell: 'string',
type: 'switch', group: '{{ _('Miscellaneous') }}'
},{
id: 'header', label:'{{ _('Header') }}', cell: 'string',
type: 'switch', group: '{{ _('Miscellaneous') }}', deps: ['format'], disabled: 'isDisabled'
},{
id: 'delimiter', label:'{{ _('Delimiter') }}', cell: 'string', first_empty: true,
type: 'text', control: 'node-ajax-options', group: '{{ _('Miscellaneous') }}', disabled: 'isDisabled',
deps: ['format'],
options:[
{'label': ';', 'value': ';'},
{'label': ',', 'value': ','},
{'label': '|', 'value': '|'},
{'label': '[tab]', 'value': '[tab]'}
],
select2: {
tags: true,
allowClear: false,
width: "100%",
placeholder: '{{ _('Select from list...') }}'
}, helpMessage:
'{{ _('Specifies the character that separates columns within each row (line) of the file. The default is a tab character in text format, a comma in CSV format. This must be a single one-byte character. This option is not allowed when using binary format.') }}'
},
{
id: 'quote', label:'{{ _('Quote') }}', cell: 'string', first_empty: true, deps: ['format'],
type: 'text', control: 'node-ajax-options', group: '{{ _('Miscellaneous') }}', disabled: 'isDisabled',
options:[
{'label': '\"', 'value': '\"'},
{'label': '\'', 'value': '\''},
],
select2: {
tags: true,
allowClear: false,
width: "100%",
placeholder: '{{ _('Select from list...') }}'
}, helpMessage:
'{{ _('Specifies the quoting character to be used when a data value is quoted. The default is double-quote. This must be a single one-byte character. This option is allowed only when using CSV format.') }}'
},
{
id: 'escape', label:'{{ _('Escape') }}', cell: 'string', first_empty: true, deps: ['format'],
type: 'text', control: 'node-ajax-options', group: '{{ _('Miscellaneous') }}', disabled: 'isDisabled',
options:[
{'label': '\"', 'value': '\"'},
{'label': '\'', 'value': '\''},
],
select2: {
tags: true,
allowClear: false,
width: "100%",
placeholder: '{{ _('Select from list...') }}'
}, helpMessage:
'{{ _('Specifies the character that should appear before a data character that matches the QUOTE value. The default is the same as the QUOTE value (so that the quoting character is doubled if it appears in the data). This must be a single one-byte character. This option is allowed only when using CSV format.') }}'
}]
}
],
// Enable/Disable the items based on the user file format selection
isDisabled: function(m) {
name = this.name;
switch(name) {
case 'quote':
case 'escape':
case 'header':
return (m.get('format') != 'csv')
case 'icolumns':
return (m.get('format') != 'csv' || !m.get('is_import'));
case 'null_string':
case 'delimiter':
return (m.get('format') == 'binary');
default:
return false;
}
},
importing: function(m) {
return m.get('is_import');
},
exporting: function(m) {
return !(m.importing.apply(this, arguments));
}
});
pgTools.import_utility = {
init: function() {
// We do not want to initialize the module multiple times.
if (this.initialized)
return;
this.initialized = true;
/**
Enable/disable import menu in tools based on node selected
Import menu will be enabled only when user select table node.
*/
menu_enabled = function(itemData, item, data) {
var t = pgBrowser.tree, i = item, d = itemData;
var parent_item = t.hasParent(i) ? t.parent(i): null,
parent_data = parent_item ? t.itemData(parent_item) : null;
if(!_.isUndefined(d) && !_.isNull(d) && !_.isNull(parent_data))
return (
(_.indexOf(['table'], d._type) !== -1 &&
parent_data._type != 'catalog') ? true: false
);
else
return false;
};
// Initialize the context menu to display the import options when user open the context menu for table
pgBrowser.add_menus([{
name: 'import', node: 'table', module: this,
applies: ['tools', 'context'], callback: 'callback_import_export',
category: 'import', priority: 10, label: '{{ _('Import/Export...') }}',
icon: 'fa fa-shopping-cart', enable: menu_enabled
}]);
},
/*
Open the dialog for the import functionality
*/
callback_import_export: function(args, item) {
var self = this;
var input = args || {},
t = pgBrowser.tree,
i = item || t.selected(),
d = i && i.length == 1 ? t.itemData(i) : undefined,
node = d && pgBrowser.Nodes[d._type];
if (!d)
return;
var objName = d.label;
var treeInfo = node.getTreeNodeHierarchy.apply(node, [i]);
if (!Alertify.ImportDialog) {
Alertify.dialog('ImportDialog', function factory() {
return {
main: function(title, node, item, data) {
this.set('title', title);
this.setting('pg_node', node);
this.setting('pg_item', item);
this.setting('pg_item_data', data);
},
setup: function() {
return {
buttons:[{
text: "{{ _('OK') }}", key: 27, disable: true,
className:
"btn btn-primary fa fa-lg fa-save pg-alertify-button"
}, {
text: "{{ _('Cancel') }}", key: 27,
className:
"btn btn-danger fa fa-lg fa-times pg-alertify-button"
}],
options: {modal: true}
};
},
settings: {
pg_node: null,
pg_item: null,
pg_item_data: null
},
// Callback functions when click on the buttons of the Alertify dialogs
callback: function(e) {
if (e.button.text === "{{ _('OK') }}") {
var n = this.settings['pg_node'],
i = this.settings['pg_item'],
treeInfo = n.getTreeNodeHierarchy.apply(n, [i])
this.view.model.set({
'database': treeInfo.database.label,
'schema': treeInfo.schema.label,
'table': treeInfo.table.label
});
var self = this,
baseUrl = "{{ url_for('import_export.index') }}" +
"create_job/" + treeInfo.server._id,
args = this.view.model.toJSON();
$.ajax({
url: baseUrl,
method: 'POST',
data:{ 'data': JSON.stringify(args) },
success: function(res) {
if (res.success) {
Alertify.message('{{ _('Background process for taking import/export has been created!') }}', 1);
pgBrowser.Events.trigger('pgadmin-bgprocess:created', self);
}
},
error: function(xhr, status, error) {
try {
var err = $.parseJSON(xhr.responseText);
Alertify.alert(
'{{ _('Import failed...') }}',
err.errormsg
);
} catch (e) {}
}
});
}
},
hooks: {
onclose: function() {
if (this.view) {
this.view.remove({data: true, internal: true, silent: true});
}
},
// triggered when a dialog option gets update.
onupdate: function(option,oldValue, newValue) {
switch(option){
case 'resizable':
if(newValue){
this.elements.content.removeAttribute('style');
} else {
this.elements.content.style.minHeight = 'inherit';
}
break;
}
}
},
prepare: function() {
// Main import module container
var self = this;
// Disable OK button until user provides valid Filename
this.__internal.buttons[0].element.disabled = true;
var $container = $("<div class='import_dlg'></div>"),
n = this.settings.pg_node,
i = this.settings.pg_item,
treeInfo = n.getTreeNodeHierarchy.apply(n, [i]),
newModel = new ImportExportModel ({}, {
node_info: treeInfo
}),
fields = Backform.generateViewSchema(
treeInfo, newModel, 'create', node, treeInfo.server, true
),
view = this.view = new Backform.Dialog({
el: $container, model: newModel, schema: fields
});
$(this.elements.body.childNodes[0]).addClass(
'alertify_tools_dialog_properties obj_properties'
);
view.render();
this.elements.content.appendChild($container.get(0));
// Listen to model & if filename is provided then enable OK button
// For the 'Quote', 'escape' and 'delimiter' only one character is allowed to enter
this.view.model.on('change', function() {
if (!_.isUndefined(this.get('filename')) && this.get('filename') !== '') {
this.errorModel.clear();
if (!_.isUndefined(this.get('delimiter')) && this.get('delimiter') !== '' &&
(this.get('delimiter').length == 1 || this.get('delimiter') == '[tab]')) {
this.errorModel.clear();
if (!_.isUndefined(this.get('quote')) && this.get('quote') !== '' &&
this.get('quote').length == 1) {
this.errorModel.clear();
if (!_.isUndefined(this.get('escape')) && this.get('escape') !== '' &&
this.get('escape').length == 1) {
this.errorModel.clear();
self.__internal.buttons[0].element.disabled = false;
} else {
self.__internal.buttons[0].element.disabled = true;
this.errorModel.set('escape', '{{ _('Escape should contain only one character') }}')
}
} else {
self.__internal.buttons[0].element.disabled = true;
this.errorModel.set('quote', '{{ _('Quote should contain only one character') }}')
}
} else {
self.__internal.buttons[0].element.disabled = true;
this.errorModel.set('delimiter', '{{ _('Delimiter should contain only one character') }}')
}
} else {
self.__internal.buttons[0].element.disabled = true;
this.errorModel.set('filename', '{{ _('Please provide filename') }}')
}
});
// Give the dialog initial height & width
this.elements.dialog.style.minHeight = '80%';
this.elements.dialog.style.minWidth = '70%';
}
};
});
}
// Open the Alertify dialog for the import/export module
Alertify.ImportDialog(
S(
"{{ _("Import/Export data - table '%%s'") }}"
).sprintf(treeInfo.table.label).value(), node, i, d
).set('resizable',true).resizeTo('70%','80%');
}
};
return pgAdmin.Tools.import_utility;
});

View File

@ -0,0 +1 @@
\copy {{ conn|qtIdent(data.schema, data.table) }} {% if columns %} {{ columns }} {% endif %} {% if data.is_import %}FROM{% else %}TO{% endif %} {{ data.filename|qtLiteral }} {% if data.oid %} OIDS {% endif %}{% if data.delimiter and data.format != 'binary' and data.delimiter == '[tab]' %} DELIMITER E'\\t' {% elif data.format != 'binary' and data.delimiter %} DELIMITER {{ data.delimiter|qtLiteral }}{% endif %}{% if data.format == 'csv' %} CSV {% endif %} {% if data.header %} HEADER {% endif %}{% if data.encoding %} ENCODING {{ data.encoding|qtLiteral }}{% endif %}{% if data.format == 'csv' and data.quote %} QUOTE {{ data.quote|qtLiteral }}{% endif %}{% if data.format != 'binary' and data.null_string %} NULL {{ data.null_string|qtLiteral }}{% endif %}{% if data.format == 'csv' and data.escape %} ESCAPE {{ data.escape|qtLiteral }}{% endif %}{% if data.format == 'csv' and data.is_import and ignore_column_list %} FORCE_NOT_NULL {{ ignore_column_list }} {% endif %};