mirror of
https://github.com/pgadmin-org/pgadmin4.git
synced 2026-08-07 11:45:07 -05:00
Resolved quite a few file-system encoding/decoding related cases.
In order to resolve the non-ascii characters in path (in user directory, storage path, etc) on windows, we have converted the path into the short-path, so that - we don't need to deal with the encoding issues (specially with Python 2). We've resolved majority of the issues with this patch. We still need couple issues to resolve after this in the same area. TODO * Add better support for non-ascii characters in the database name on windows with Python 3 * Improve the messages created after the background processes by different modules (such as Backup, Restore, Import/Export, etc.), which does not show short-paths, and xml representable characters for non-ascii characters, when found in the database objects, and the file PATH. Fixes #2174, #1797, #2166, #1940 Initial patch by: Surinder Kumar Reviewed by: Murtuza Zabuawala
This commit is contained in:
Ashesh Vashi
@@ -9,6 +9,7 @@
|
||||
|
||||
"""Implements Backup Utility"""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
import simplejson as json
|
||||
import os
|
||||
|
||||
@@ -17,7 +18,8 @@ from flask import render_template, request, current_app, \
|
||||
from flask_babel import gettext as _
|
||||
from flask_security import login_required, current_user
|
||||
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html, \
|
||||
fs_short_path, document_dir
|
||||
from pgadmin.utils.ajax import make_json_response, bad_request
|
||||
|
||||
from config import PG_DEFAULT_DRIVER
|
||||
@@ -79,14 +81,28 @@ class BackupMessage(IProcessDesc):
|
||||
Defines the message shown for the backup operation.
|
||||
"""
|
||||
|
||||
def __init__(self, _type, _sid, _bfile, **kwargs):
|
||||
def __init__(self, _type, _sid, _bfile, *_args, **_kwargs):
|
||||
self.backup_type = _type
|
||||
self.sid = _sid
|
||||
self.bfile = _bfile
|
||||
self.database = None
|
||||
self.database = _kwargs['database'] if 'database' in _kwargs else None
|
||||
self.cmd = ''
|
||||
|
||||
if 'database' in kwargs:
|
||||
self.database = kwargs['database']
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
# x = html.safe_str(x)
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
|
||||
return ' "' + x + '"'
|
||||
return ''
|
||||
|
||||
for arg in _args:
|
||||
if arg and len(arg) >= 2 and arg[:2] == '--':
|
||||
self.cmd += ' ' + arg
|
||||
else:
|
||||
self.cmd += cmdArg(arg)
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
@@ -123,65 +139,32 @@ class BackupMessage(IProcessDesc):
|
||||
res = '<div class="h5">'
|
||||
|
||||
if self.backup_type == BACKUP.OBJECT:
|
||||
res += html.safe_str(
|
||||
_(
|
||||
"Backing up an object on the server '{0}' from database '{1}'..."
|
||||
).format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port),
|
||||
self.database
|
||||
)
|
||||
res += _(
|
||||
"Backing up an object on the server '{0}' from database '{1}'..."
|
||||
).format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port),
|
||||
self.database
|
||||
)
|
||||
elif self.backup_type == BACKUP.GLOBALS:
|
||||
res += html.safe_str(
|
||||
_("Backing up the global objects on the server '{0}'").format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
res += _("Backing up the global objects on the server '{0}'").format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
elif self.backup_type == BACKUP.SERVER:
|
||||
res += html.safe_str(
|
||||
_("Backing up the server '{0}'").format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
res += _("Backing up the server '{0}'").format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
else:
|
||||
# It should never reach here.
|
||||
res += "Backup"
|
||||
|
||||
res += '</div><div class="h5">'
|
||||
res += html.safe_str(
|
||||
_("Running command:")
|
||||
)
|
||||
res += _("Running command:")
|
||||
res += '</b><br><span class="pg-bg-cmd enable-selection">'
|
||||
res += html.safe_str(cmd)
|
||||
|
||||
replace_next = False
|
||||
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
|
||||
return ' "' + html.safe_str(x) + '"'
|
||||
|
||||
return ''
|
||||
|
||||
for arg in args:
|
||||
if arg and len(arg) >= 2 and arg[:2] == '--':
|
||||
res += ' ' + arg
|
||||
elif replace_next:
|
||||
res += ' "' + html.safe_str(
|
||||
self.bfile
|
||||
) + '"'
|
||||
else:
|
||||
if arg == '--file':
|
||||
replace_next = True
|
||||
res += cmdArg(arg)
|
||||
res += self.cmd
|
||||
res += '</span></div>'
|
||||
|
||||
return res
|
||||
|
||||
|
||||
@blueprint.route("/")
|
||||
@login_required
|
||||
def index():
|
||||
@@ -201,7 +184,7 @@ def script():
|
||||
)
|
||||
|
||||
|
||||
def filename_with_file_manager_path(file):
|
||||
def filename_with_file_manager_path(_file):
|
||||
"""
|
||||
Args:
|
||||
file: File name returned from client file manager
|
||||
@@ -213,9 +196,15 @@ def filename_with_file_manager_path(file):
|
||||
storage_dir = get_storage_directory()
|
||||
|
||||
if storage_dir:
|
||||
return os.path.join(storage_dir, file.lstrip('/'))
|
||||
_file = os.path.join(storage_dir, _file.lstrip(u'/').lstrip(u'\\'))
|
||||
elif not os.path.isabs(_file):
|
||||
_file = os.path.join(document_dir(), _file)
|
||||
|
||||
return file
|
||||
# Touch the file to get the short path of the file on windows.
|
||||
with open(_file, 'a'):
|
||||
pass
|
||||
|
||||
return fs_short_path(_file)
|
||||
|
||||
|
||||
@blueprint.route('/create_job/<int:sid>', methods=['POST'])
|
||||
@@ -292,7 +281,11 @@ def create_backup_job(sid):
|
||||
p = BatchProcess(
|
||||
desc=BackupMessage(
|
||||
BACKUP.SERVER if data['type'] != 'global' else BACKUP.GLOBALS,
|
||||
sid, data['file']
|
||||
sid,
|
||||
data['file'].encode('utf-8') if hasattr(
|
||||
data['file'], 'encode'
|
||||
) else data['file'],
|
||||
*args
|
||||
),
|
||||
cmd=utility, args=args
|
||||
)
|
||||
@@ -440,9 +433,15 @@ def create_backup_objects_job(sid):
|
||||
try:
|
||||
p = BatchProcess(
|
||||
desc=BackupMessage(
|
||||
BACKUP.OBJECT, sid, data['file'], database=data['database']
|
||||
BACKUP.OBJECT, sid,
|
||||
data['file'].encode('utf-8') if hasattr(
|
||||
data['file'], 'encode'
|
||||
) else data['file'],
|
||||
*args,
|
||||
database=data['database']
|
||||
),
|
||||
cmd=utility, args=args)
|
||||
cmd=utility, args=args
|
||||
)
|
||||
manager.export_password_env(p.id)
|
||||
p.start()
|
||||
jid = p.id
|
||||
|
||||
@@ -16,7 +16,8 @@ from flask import url_for, Response, render_template, request, current_app
|
||||
from flask_babel import gettext as _
|
||||
from flask_security import login_required, current_user
|
||||
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html, \
|
||||
fs_short_path, document_dir, IS_WIN
|
||||
from pgadmin.utils.ajax import make_json_response, bad_request
|
||||
|
||||
from config import PG_DEFAULT_DRIVER
|
||||
@@ -56,19 +57,47 @@ class ImportExportModule(PgAdminModule):
|
||||
blueprint = ImportExportModule(MODULE_NAME, __name__)
|
||||
|
||||
|
||||
class Message(IProcessDesc):
|
||||
class IEMessage(IProcessDesc):
|
||||
"""
|
||||
Message(IProcessDesc)
|
||||
IEMessage(IProcessDesc)
|
||||
|
||||
Defines the message shown for the Message operation.
|
||||
Defines the message shown for the import/export operation.
|
||||
"""
|
||||
|
||||
def __init__(self, _sid, _schema, _tbl, _database, _storage):
|
||||
def __init__(self, _sid, _schema, _tbl, _database, _storage, *_args):
|
||||
self.sid = _sid
|
||||
self.schema = _schema
|
||||
self.table = _tbl
|
||||
self.database = _database
|
||||
self.storage = _storage
|
||||
self._cmd = ''
|
||||
|
||||
if _storage:
|
||||
_storage = _storage.replace('\\', '/')
|
||||
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
x = html.safe_str(x)
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
|
||||
return ' "' + x + '"'
|
||||
return ''
|
||||
|
||||
replace_next = False
|
||||
for arg in _args:
|
||||
if arg and len(arg) >= 2 and arg[:2] == '--':
|
||||
if arg == '--command':
|
||||
replace_next = True
|
||||
self._cmd += ' ' + arg
|
||||
elif replace_next:
|
||||
arg = cmdArg(arg)
|
||||
if _storage is not None:
|
||||
arg = arg.replace(_storage, '<STORAGE_DIR>')
|
||||
self._cmd += ' "' + arg + '"'
|
||||
else:
|
||||
self._cmd+= cmdArg(arg)
|
||||
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
@@ -90,45 +119,17 @@ class Message(IProcessDesc):
|
||||
).first()
|
||||
|
||||
res = '<div class="h5">'
|
||||
res += html.safe_str(
|
||||
_(
|
||||
"Copying table data '{0}.{1}' on database '{2}' for the server - '{3}'"
|
||||
).format(
|
||||
self.schema, self.table, self.database,
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
res += _(
|
||||
"Copying table data '{0}.{1}' on database '{2}' for the server - '{3}'"
|
||||
).format(
|
||||
self.schema, self.table, self.database,
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
|
||||
res += '</div><div class="h5">'
|
||||
res += html.safe_str(
|
||||
_("Running command:")
|
||||
)
|
||||
res += _("Running command:")
|
||||
res += '</b><br><span class="pg-bg-cmd enable-selection">'
|
||||
res += html.safe_str(cmd)
|
||||
|
||||
replace_next = False
|
||||
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
|
||||
return ' "' + html.safe_str(x) + '"'
|
||||
|
||||
return ''
|
||||
|
||||
for arg in args:
|
||||
if arg and len(arg) >= 2 and arg[:2] == '--':
|
||||
if arg == '--command':
|
||||
replace_next = True
|
||||
res += ' ' + arg
|
||||
elif replace_next:
|
||||
if self.storage:
|
||||
arg = arg.replace(self.storage, '<STORAGE_DIR>')
|
||||
res += ' "' + html.safe_str(arg) + '"'
|
||||
else:
|
||||
res += cmdArg(arg)
|
||||
res += self._cmd
|
||||
res += '</span></div>'
|
||||
|
||||
return res
|
||||
@@ -151,6 +152,33 @@ def script():
|
||||
)
|
||||
|
||||
|
||||
def filename_with_file_manager_path(_file, _present=False):
|
||||
"""
|
||||
Args:
|
||||
file: File name returned from client file manager
|
||||
|
||||
Returns:
|
||||
Filename to use for backup with full path taken from preference
|
||||
"""
|
||||
# Set file manager directory from preference
|
||||
storage_dir = get_storage_directory()
|
||||
|
||||
if storage_dir:
|
||||
_file = os.path.join(storage_dir, _file.lstrip(u'/').lstrip(u'\\'))
|
||||
elif not os.path.isabs(_file):
|
||||
_file = os.path.join(document_dir(), _file)
|
||||
|
||||
if not _present:
|
||||
# Touch the file to get the short path of the file on windows.
|
||||
with open(_file, 'a'):
|
||||
pass
|
||||
else:
|
||||
if not os.path.isfile(_file):
|
||||
return None
|
||||
|
||||
return fs_short_path(_file)
|
||||
|
||||
|
||||
@blueprint.route('/create_job/<int:sid>', methods=['POST'])
|
||||
@login_required
|
||||
def create_import_export_job(sid):
|
||||
@@ -175,10 +203,8 @@ def create_import_export_job(sid):
|
||||
id=sid).first()
|
||||
|
||||
if server is None:
|
||||
return make_json_response(
|
||||
success=0,
|
||||
errormsg=_("Couldn't find the given server")
|
||||
)
|
||||
return bad_request(errormsg=_("Couldn't find the given server"))
|
||||
|
||||
|
||||
# To fetch MetaData for the server
|
||||
from pgadmin.utils.driver import get_driver
|
||||
@@ -188,10 +214,7 @@ def create_import_export_job(sid):
|
||||
connected = conn.connected()
|
||||
|
||||
if not connected:
|
||||
return make_json_response(
|
||||
success=0,
|
||||
errormsg=_("Please connect to the server first...")
|
||||
)
|
||||
return bad_request(errormsg=_("Please connect to the server first..."))
|
||||
|
||||
# Get the utility path from the connection manager
|
||||
utility = manager.utility('sql')
|
||||
@@ -200,21 +223,16 @@ def create_import_export_job(sid):
|
||||
storage_dir = get_storage_directory()
|
||||
|
||||
if 'filename' in data:
|
||||
if os.name == 'nt':
|
||||
data['filename'] = data['filename'].replace('/', '\\')
|
||||
if storage_dir:
|
||||
storage_dir = storage_dir.replace('/', '\\')
|
||||
data['filename'] = data['filename'].replace('\\', '\\\\')
|
||||
data['filename'] = os.path.join(storage_dir, data['filename'].lstrip('/'))
|
||||
elif storage_dir:
|
||||
data['filename'] = os.path.join(storage_dir, data['filename'].lstrip('/'))
|
||||
else:
|
||||
data['filename'] = data['filename']
|
||||
_file = filename_with_file_manager_path(data['filename'], data['is_import'])
|
||||
if not _file:
|
||||
return bad_request(errormsg=_('Please specify a valid file'))
|
||||
|
||||
if IS_WIN:
|
||||
_file = _file.replace('\\', '/')
|
||||
|
||||
data['filename'] = _file
|
||||
else:
|
||||
return make_json_response(
|
||||
data={'status': False, 'info': 'Please specify a valid file'}
|
||||
)
|
||||
return bad_request(errormsg=_('Please specify a valid file'))
|
||||
|
||||
cols = None
|
||||
icols = None
|
||||
@@ -255,33 +273,32 @@ def create_import_export_job(sid):
|
||||
ignore_column_list=icols
|
||||
)
|
||||
|
||||
args = [
|
||||
'--host', server.host, '--port', str(server.port),
|
||||
'--username', server.username, '--dbname', data['database'],
|
||||
'--command', query
|
||||
]
|
||||
args = ['--command', query]
|
||||
|
||||
try:
|
||||
p = BatchProcess(
|
||||
desc=Message(
|
||||
desc=IEMessage(
|
||||
sid,
|
||||
data['schema'],
|
||||
data['table'],
|
||||
data['database'],
|
||||
storage_dir
|
||||
storage_dir,
|
||||
utility, *args
|
||||
),
|
||||
cmd=utility, args=args
|
||||
)
|
||||
manager.export_password_env(p.id)
|
||||
p.start()
|
||||
def export_pg_env(env):
|
||||
env['PGHOST'] = server.host
|
||||
env['PGPORT'] = str(server.port)
|
||||
env['PGUSER'] = server.username
|
||||
env['PGDATABASE'] = data['database']
|
||||
|
||||
p.start(export_pg_env)
|
||||
jid = p.id
|
||||
except Exception as e:
|
||||
current_app.logger.exception(e)
|
||||
return make_json_response(
|
||||
status=410,
|
||||
success=0,
|
||||
errormsg=str(e)
|
||||
)
|
||||
return bad_request(errormsg=str(e))
|
||||
|
||||
# Return response
|
||||
return make_json_response(
|
||||
|
||||
@@ -17,7 +17,8 @@ from flask import render_template, request, current_app, \
|
||||
from flask_babel import gettext as _
|
||||
from flask_security import login_required, current_user
|
||||
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html, \
|
||||
fs_short_path, document_dir
|
||||
from pgadmin.utils.ajax import make_json_response, bad_request
|
||||
|
||||
from config import PG_DEFAULT_DRIVER
|
||||
@@ -58,9 +59,26 @@ blueprint = RestoreModule(
|
||||
|
||||
|
||||
class RestoreMessage(IProcessDesc):
|
||||
def __init__(self, _sid, _bfile):
|
||||
def __init__(self, _sid, _bfile, *_args):
|
||||
self.sid = _sid
|
||||
self.bfile = _bfile
|
||||
self.cmd = ''
|
||||
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
x = html.safe_str(x)
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
return ' "' + x + '"'
|
||||
return ''
|
||||
|
||||
for arg in _args:
|
||||
if arg and len(arg) >= 2 and arg[:2] == '--':
|
||||
self.cmd += ' ' + arg
|
||||
else:
|
||||
self.cmd += cmdArg(arg)
|
||||
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
@@ -95,30 +113,7 @@ class RestoreMessage(IProcessDesc):
|
||||
)
|
||||
res += '</b><br><span class="pg-bg-cmd enable-selection">'
|
||||
res += html.safe_str(cmd)
|
||||
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
|
||||
return ' "' + html.safe_str(x) + '"'
|
||||
|
||||
return ''
|
||||
|
||||
idx = 0
|
||||
no_args = len(args)
|
||||
for arg in args:
|
||||
if idx < no_args - 1:
|
||||
if arg[:2] == '--':
|
||||
res += ' ' + arg
|
||||
else:
|
||||
res += cmdArg(arg)
|
||||
idx += 1
|
||||
|
||||
if no_args > 1:
|
||||
res += ' "' + html.safe_str(arg) + '"'
|
||||
|
||||
res += self.cmd
|
||||
res += '</span></div>'
|
||||
|
||||
return res
|
||||
@@ -143,7 +138,7 @@ def script():
|
||||
)
|
||||
|
||||
|
||||
def filename_with_file_manager_path(file):
|
||||
def filename_with_file_manager_path(_file):
|
||||
"""
|
||||
Args:
|
||||
file: File name returned from client file manager
|
||||
@@ -155,9 +150,14 @@ def filename_with_file_manager_path(file):
|
||||
storage_dir = get_storage_directory()
|
||||
|
||||
if storage_dir:
|
||||
return os.path.join(storage_dir, file.lstrip('/'))
|
||||
_file = os.path.join(storage_dir, _file.lstrip(u'/').lstrip(u'\\'))
|
||||
elif not os.path.isabs(_file):
|
||||
_file = os.path.join(document_dir(), _file)
|
||||
|
||||
return file
|
||||
if not os.path.isfile(_file):
|
||||
return None
|
||||
|
||||
return fs_short_path(_file)
|
||||
|
||||
|
||||
@blueprint.route('/create_job/<int:sid>', methods=['POST'])
|
||||
@@ -179,7 +179,13 @@ def create_restore_job(sid):
|
||||
else:
|
||||
data = json.loads(request.data, encoding='utf-8')
|
||||
|
||||
backup_file = filename_with_file_manager_path(data['file'])
|
||||
_file = filename_with_file_manager_path(data['file'])
|
||||
|
||||
if _file is None:
|
||||
return make_json_response(
|
||||
success=0,
|
||||
errormsg=_("File couldn't be found!")
|
||||
)
|
||||
|
||||
# Fetch the server details like hostname, port, roles etc
|
||||
server = Server.query.filter_by(
|
||||
@@ -261,7 +267,7 @@ def create_restore_job(sid):
|
||||
return False
|
||||
|
||||
args.extend([
|
||||
'--host', server.host, '--port', server.port,
|
||||
'--host', server.host, '--port', str(server.port),
|
||||
'--username', server.username, '--no-password'
|
||||
])
|
||||
|
||||
@@ -300,11 +306,17 @@ def create_restore_job(sid):
|
||||
set_multiple('trigger_funcs', '--function')
|
||||
set_multiple('indexes', '--index')
|
||||
|
||||
args.append(backup_file)
|
||||
args.append(fs_short_path(_file))
|
||||
|
||||
try:
|
||||
p = BatchProcess(
|
||||
desc=RestoreMessage(sid, data['file']),
|
||||
desc=RestoreMessage(
|
||||
sid,
|
||||
data['file'].encode('utf-8') if hasattr(
|
||||
data['file'], 'encode'
|
||||
) else data['file'],
|
||||
*args
|
||||
),
|
||||
cmd=utility, args=args
|
||||
)
|
||||
manager.export_password_env(p.id)
|
||||
|
||||
Reference in New Issue
Block a user