mirror of
https://github.com/pgadmin-org/pgadmin4.git
synced 2024-11-22 08:46:39 -06:00
Resolved quite a few file-system encoding/decoding related cases.
In order to resolve the non-ascii characters in path (in user directory, storage path, etc) on windows, we have converted the path into the short-path, so that - we don't need to deal with the encoding issues (specially with Python 2). We've resolved majority of the issues with this patch. We still need couple issues to resolve after this in the same area. TODO * Add better support for non-ascii characters in the database name on windows with Python 3 * Improve the messages created after the background processes by different modules (such as Backup, Restore, Import/Export, etc.), which does not show short-paths, and xml representable characters for non-ascii characters, when found in the database objects, and the file PATH. Fixes #2174, #1797, #2166, #1940 Initial patch by: Surinder Kumar Reviewed by: Murtuza Zabuawala
This commit is contained in:
Ashesh Vashi
parent
063177155e
commit
f2fc1ceba8
@ -12,7 +12,7 @@
|
||||
##########################################################################
|
||||
|
||||
import os
|
||||
from logging import *
|
||||
from pgadmin.utils import env, IS_PY2, IS_WIN, fs_short_path
|
||||
|
||||
##########################################################################
|
||||
# Application settings
|
||||
@ -81,12 +81,17 @@ MODULE_BLACKLIST = ['test']
|
||||
# List of treeview browser nodes to skip when dynamically loading
|
||||
NODE_BLACKLIST = []
|
||||
|
||||
|
||||
# Data directory for storage of config settings etc. This shouldn't normally
|
||||
# need to be changed - it's here as various other settings depend on it.
|
||||
if os.name == 'nt':
|
||||
DATA_DIR = os.path.realpath(os.getenv('APPDATA') + "/pgAdmin")
|
||||
if IS_WIN:
|
||||
# Use the short path on windows
|
||||
DATA_DIR = os.path.realpath(
|
||||
os.path.join(fs_short_path(env('APPDATA')), u"pgAdmin")
|
||||
)
|
||||
else:
|
||||
DATA_DIR = os.path.realpath(os.path.expanduser('~/.pgadmin/'))
|
||||
DATA_DIR = os.path.realpath(os.path.expanduser(u'~/.pgadmin/'))
|
||||
|
||||
|
||||
##########################################################################
|
||||
# Log settings
|
||||
@ -95,6 +100,9 @@ else:
|
||||
# Debug mode?
|
||||
DEBUG = False
|
||||
|
||||
|
||||
import logging
|
||||
|
||||
# Application log level - one of:
|
||||
# CRITICAL 50
|
||||
# ERROR 40
|
||||
@ -103,18 +111,16 @@ DEBUG = False
|
||||
# INFO 20
|
||||
# DEBUG 10
|
||||
# NOTSET 0
|
||||
CONSOLE_LOG_LEVEL = WARNING
|
||||
FILE_LOG_LEVEL = WARNING
|
||||
CONSOLE_LOG_LEVEL = logging.WARNING
|
||||
FILE_LOG_LEVEL = logging.WARNING
|
||||
|
||||
# Log format.
|
||||
CONSOLE_LOG_FORMAT = '%(asctime)s: %(levelname)s\t%(name)s:\t%(message)s'
|
||||
FILE_LOG_FORMAT = '%(asctime)s: %(levelname)s\t%(name)s:\t%(message)s'
|
||||
|
||||
# Log file name
|
||||
LOG_FILE = os.path.join(
|
||||
DATA_DIR,
|
||||
'pgadmin4.log'
|
||||
)
|
||||
LOG_FILE = os.path.join(DATA_DIR, 'pgadmin4.log')
|
||||
|
||||
|
||||
##########################################################################
|
||||
# Server settings
|
||||
@ -176,11 +182,8 @@ MAX_SESSION_IDLE_TIME = 60
|
||||
# The default path to the SQLite database used to store user accounts and
|
||||
# settings. This default places the file in the same directory as this
|
||||
# config file, but generates an absolute path for use througout the app.
|
||||
SQLITE_PATH = os.environ.get("SQLITE_PATH") or \
|
||||
os.path.join(
|
||||
DATA_DIR,
|
||||
'pgadmin4.db'
|
||||
)
|
||||
SQLITE_PATH = env('SQLITE_PATH') or os.path.join(DATA_DIR, 'pgadmin4.db')
|
||||
|
||||
# SQLITE_TIMEOUT will define how long to wait before throwing the error -
|
||||
# OperationError due to database lock. On slower system, you may need to change
|
||||
# this to some higher value.
|
||||
@ -207,10 +210,7 @@ SQLITE_TIMEOUT = 500
|
||||
# SESSION_DB_PATH = '/run/shm/pgAdmin4_session'
|
||||
#
|
||||
##########################################################################
|
||||
SESSION_DB_PATH = os.path.join(
|
||||
DATA_DIR,
|
||||
'sessions'
|
||||
)
|
||||
SESSION_DB_PATH = os.path.join(DATA_DIR, 'sessions')
|
||||
|
||||
SESSION_COOKIE_NAME = 'pga4_session'
|
||||
|
||||
@ -262,10 +262,8 @@ UPGRADE_CHECK_URL = 'https://www.pgadmin.org/versions.json'
|
||||
# 2. Set path manually like
|
||||
# STORAGE_DIR = "/path/to/directory/"
|
||||
##########################################################################
|
||||
STORAGE_DIR = os.path.join(
|
||||
DATA_DIR,
|
||||
'storage'
|
||||
)
|
||||
STORAGE_DIR = os.path.join(DATA_DIR, 'storage')
|
||||
|
||||
|
||||
##########################################################################
|
||||
# Default locations for binary utilities (pg_dump, pg_restore etc)
|
||||
@ -296,6 +294,7 @@ TESTING_MODE = False
|
||||
# The default path for SQLite database for testing
|
||||
TEST_SQLITE_PATH = os.path.join(DATA_DIR, 'test_pgadmin4.db')
|
||||
|
||||
|
||||
##########################################################################
|
||||
# Allows flask application to response to the each request asynchronously
|
||||
##########################################################################
|
||||
|
||||
@ -34,9 +34,11 @@ config.SETTINGS_SCHEMA_VERSION = SCHEMA_VERSION
|
||||
|
||||
# Check if the database exists. If it does not, create it.
|
||||
if not os.path.isfile(config.SQLITE_PATH):
|
||||
setupfile = os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
||||
'setup.py')
|
||||
exec (open(setupfile).read())
|
||||
from pgadmin.utils import u, fs_encoding, file_quote
|
||||
setupfile = os.path.join(
|
||||
os.path.dirname(os.path.realpath(u(__file__, fs_encoding))), u'setup.py'
|
||||
)
|
||||
exec(open(file_quote(setupfile), 'r').read())
|
||||
|
||||
##########################################################################
|
||||
# Server starup
|
||||
|
||||
@ -30,8 +30,6 @@ from werkzeug.local import LocalProxy
|
||||
from werkzeug.utils import find_modules
|
||||
|
||||
from pgadmin.model import db, Role, Server, ServerGroup, User, Version, Keys
|
||||
# Configuration settings
|
||||
import config
|
||||
|
||||
# If script is running under python3, it will not have the xrange function
|
||||
# defined
|
||||
@ -129,7 +127,13 @@ def _find_blueprint():
|
||||
current_blueprint = LocalProxy(_find_blueprint)
|
||||
|
||||
|
||||
def create_app(app_name=config.APP_NAME):
|
||||
def create_app(app_name=None):
|
||||
|
||||
# Configuration settings
|
||||
import config
|
||||
if not app_name:
|
||||
app_name = config.APP_NAME
|
||||
|
||||
"""Create the Flask application, startup logging and dynamically load
|
||||
additional modules (blueprints) that are found in this directory."""
|
||||
app = PgAdmin(__name__, static_url_path='/static')
|
||||
@ -193,8 +197,8 @@ def create_app(app_name=config.APP_NAME):
|
||||
# Setup authentication
|
||||
##########################################################################
|
||||
|
||||
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///{0}?timeout={1}'.format(
|
||||
config.SQLITE_PATH.replace('\\', '/'),
|
||||
app.config['SQLALCHEMY_DATABASE_URI'] = u'sqlite:///{0}?timeout={1}'.format(
|
||||
config.SQLITE_PATH.replace(u'\\', u'/'),
|
||||
getattr(config, 'SQLITE_TIMEOUT', 500)
|
||||
)
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ It also depends on the following environment variable for proper execution.
|
||||
PROCID - Process-id
|
||||
OUTDIR - Output directory
|
||||
"""
|
||||
from __future__ import print_function, unicode_literals
|
||||
from __future__ import print_function
|
||||
|
||||
# To make print function compatible with python2 & python3
|
||||
import sys
|
||||
@ -37,27 +37,31 @@ import os
|
||||
from datetime import datetime, timedelta, tzinfo
|
||||
from subprocess import Popen, PIPE
|
||||
from threading import Thread
|
||||
import codecs
|
||||
import signal
|
||||
|
||||
def log(msg):
|
||||
if 'OUTDIR' not in os.environ:
|
||||
return
|
||||
_IS_WIN = (os.name == 'nt')
|
||||
_IS_PY2 = (sys.version_info[0] == 2)
|
||||
_ZERO = timedelta(0)
|
||||
_sys_encoding = None
|
||||
_fs_encoding = None
|
||||
_u = None
|
||||
_out_dir = None
|
||||
_log_file = None
|
||||
|
||||
with open(
|
||||
os.path.join(os.environ['OUTDIR'], ('log_%s' % os.getpid())), 'a'
|
||||
) as fp:
|
||||
fp.write(('INFO:: %s\n' % str(msg)))
|
||||
if _IS_PY2:
|
||||
def _log(msg):
|
||||
with open(_log_file, 'a') as fp:
|
||||
fp.write(('INFO:: %s\n' % str(msg)))
|
||||
else:
|
||||
def _log(msg):
|
||||
with open(_log_file, 'a') as fp:
|
||||
fp.write(('INFO:: %s\n' % msg.encode('ascii', 'xmlcharrefreplace')))
|
||||
|
||||
|
||||
def log_exception():
|
||||
if 'OUTDIR' not in os.environ:
|
||||
return
|
||||
def _log_exception():
|
||||
type_, value_, traceback_ = info=sys.exc_info()
|
||||
|
||||
with open(
|
||||
os.path.join(os.environ['OUTDIR'], ('log_%s' % os.getpid())), 'a'
|
||||
) as fp:
|
||||
with open(_log_file, 'ab') as fp:
|
||||
from traceback import format_exception
|
||||
res = ''.join(
|
||||
format_exception(type_, value_, traceback_)
|
||||
@ -67,11 +71,6 @@ def log_exception():
|
||||
return res
|
||||
|
||||
|
||||
IS_WIN = (os.name == 'nt')
|
||||
|
||||
ZERO = timedelta(0)
|
||||
default_encoding = sys.getdefaultencoding() or "utf-8"
|
||||
|
||||
# Copied the 'UTC' class from the 'pytz' package to allow to run this script
|
||||
# without any external dependent library, and can be used with any python
|
||||
# version.
|
||||
@ -83,8 +82,8 @@ class UTC(tzinfo):
|
||||
"""
|
||||
zone = "UTC"
|
||||
|
||||
_utcoffset = ZERO
|
||||
_dst = ZERO
|
||||
_utcoffset = _ZERO
|
||||
_dst = _ZERO
|
||||
_tzname = zone
|
||||
|
||||
def fromutc(self, dt):
|
||||
@ -93,13 +92,13 @@ class UTC(tzinfo):
|
||||
return super(UTC.__class__, self).fromutc(dt)
|
||||
|
||||
def utcoffset(self, dt):
|
||||
return ZERO
|
||||
return _ZERO
|
||||
|
||||
def tzname(self, dt):
|
||||
return "UTC"
|
||||
|
||||
def dst(self, dt):
|
||||
return ZERO
|
||||
return _ZERO
|
||||
|
||||
def localize(self, dt, is_dst=False):
|
||||
'''Convert naive time to local time'''
|
||||
@ -155,15 +154,12 @@ class ProcessLogger(Thread):
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
import codecs
|
||||
|
||||
Thread.__init__(self)
|
||||
self.process = None
|
||||
self.stream = None
|
||||
self.encoding = default_encoding
|
||||
self.logger = codecs.open(
|
||||
os.path.join(
|
||||
os.environ['OUTDIR'], stream_type
|
||||
), 'w', self.encoding, "ignore"
|
||||
)
|
||||
self.logger = open(os.path.join(_out_dir, stream_type), 'wb')
|
||||
|
||||
def attach_process_stream(self, process, stream):
|
||||
"""
|
||||
@ -179,27 +175,52 @@ class ProcessLogger(Thread):
|
||||
self.process = process
|
||||
self.stream = stream
|
||||
|
||||
def log(self, msg):
|
||||
"""
|
||||
This function will update log file
|
||||
if not _IS_PY2:
|
||||
def log(self, msg):
|
||||
"""
|
||||
This function will update log file
|
||||
|
||||
Args:
|
||||
msg: message
|
||||
Args:
|
||||
msg: message
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
# Write into log file
|
||||
if self.logger:
|
||||
if msg:
|
||||
self.logger.write(
|
||||
str('{0},{1}').format(
|
||||
get_current_time(format='%y%m%d%H%M%S%f'),
|
||||
msg.decode(self.encoding, 'replace')
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
# Write into log file
|
||||
if self.logger:
|
||||
if msg:
|
||||
self.logger.write(get_current_time(format='%y%m%d%H%M%S%f').encode('utf-8'))
|
||||
self.logger.write(b',')
|
||||
self.logger.write(msg.lstrip(b'\r\n' if _IS_WIN else b'\n'))
|
||||
self.logger.write(os.linesep.encode('utf-8'))
|
||||
|
||||
return True
|
||||
return False
|
||||
else:
|
||||
def log(self, msg):
|
||||
"""
|
||||
This function will update log file
|
||||
|
||||
Args:
|
||||
msg: message
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
# Write into log file
|
||||
if self.logger:
|
||||
if msg:
|
||||
self.logger.write(
|
||||
b'{0},{1}{2}'.format(
|
||||
get_current_time(
|
||||
format='%y%m%d%H%M%S%f'
|
||||
),
|
||||
msg.lstrip(b'\r\n' if _IS_WIN else b'\n'), os.linesep
|
||||
)
|
||||
)
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
return True
|
||||
return False
|
||||
|
||||
def run(self):
|
||||
if self.process and self.stream:
|
||||
@ -230,14 +251,14 @@ def update_status(**kw):
|
||||
"""
|
||||
import json
|
||||
|
||||
if os.environ['OUTDIR']:
|
||||
if _out_dir:
|
||||
status = dict(
|
||||
(k, v) for k, v in kw.items() if k in [
|
||||
'start_time', 'end_time', 'exit_code', 'pid'
|
||||
]
|
||||
)
|
||||
log('Updating the status:\n{0}'.format(json.dumps(status)))
|
||||
with open(os.path.join(os.environ['OUTDIR'], 'status'), 'w') as fp:
|
||||
_log('Updating the status:\n{0}'.format(json.dumps(status)))
|
||||
with open(os.path.join(_out_dir, 'status'), 'w') as fp:
|
||||
json.dump(status, fp)
|
||||
else:
|
||||
raise ValueError("Please verify pid and db_file arguments.")
|
||||
@ -252,7 +273,7 @@ def execute():
|
||||
"""
|
||||
command = sys.argv[1:]
|
||||
args = dict()
|
||||
log('Initialize the process execution: {0}'.format(command))
|
||||
_log('Initialize the process execution: {0}'.format(command))
|
||||
|
||||
# Create seprate thread for stdout and stderr
|
||||
process_stdout = ProcessLogger('out')
|
||||
@ -270,25 +291,28 @@ def execute():
|
||||
|
||||
# Update start time
|
||||
update_status(**args)
|
||||
log('Status updated...')
|
||||
_log('Status updated...')
|
||||
|
||||
if 'PROCID' in os.environ and os.environ['PROCID'] in os.environ:
|
||||
os.environ['PGPASSWORD'] = os.environ[os.environ['PROCID']]
|
||||
|
||||
kwargs = dict()
|
||||
kwargs['close_fds'] = False
|
||||
kwargs['shell'] = True if IS_WIN else False
|
||||
kwargs['shell'] = True if _IS_WIN else False
|
||||
|
||||
# We need environment variables & values in string
|
||||
log('Converting the environment variable in the bytes format...')
|
||||
kwargs['env'] = convert_environment_variables(os.environ.copy())
|
||||
if _IS_PY2:
|
||||
_log('Converting the environment variable in the bytes format...')
|
||||
kwargs['env'] = convert_environment_variables(os.environ.copy())
|
||||
else:
|
||||
kwargs['env'] = os.environ.copy()
|
||||
|
||||
log('Starting the command execution...')
|
||||
_log('Starting the command execution...')
|
||||
process = Popen(
|
||||
command, stdout=PIPE, stderr=PIPE, stdin=None, **kwargs
|
||||
)
|
||||
|
||||
log('Attaching the loggers to stdout, and stderr...')
|
||||
_log('Attaching the loggers to stdout, and stderr...')
|
||||
# Attach the stream to the process logger, and start logging.
|
||||
process_stdout.attach_process_stream(process, process.stdout)
|
||||
process_stdout.start()
|
||||
@ -299,14 +323,14 @@ def execute():
|
||||
process_stdout.join()
|
||||
process_stderr.join()
|
||||
|
||||
log('Waiting for the process to finish...')
|
||||
_log('Waiting for the process to finish...')
|
||||
# Child process return code
|
||||
exitCode = process.wait()
|
||||
|
||||
if exitCode is None:
|
||||
exitCode = process.poll()
|
||||
|
||||
log('Process exited with code: {0}'.format(exitCode))
|
||||
_log('Process exited with code: {0}'.format(exitCode))
|
||||
args.update({'exit_code': exitCode})
|
||||
|
||||
# Add end_time
|
||||
@ -322,7 +346,7 @@ def execute():
|
||||
|
||||
# If executable not found or invalid arguments passed
|
||||
except OSError:
|
||||
info = log_exception()
|
||||
info = _log_exception()
|
||||
args.update({'exit_code': 500})
|
||||
if process_stderr:
|
||||
process_stderr.log(info)
|
||||
@ -333,7 +357,7 @@ def execute():
|
||||
|
||||
# Unknown errors
|
||||
except Exception:
|
||||
info = log_exception()
|
||||
info = _log_exception()
|
||||
args.update({'exit_code': 501})
|
||||
if process_stderr:
|
||||
process_stderr.log(info)
|
||||
@ -344,12 +368,12 @@ def execute():
|
||||
finally:
|
||||
# Update the execution end_time, and exit-code.
|
||||
update_status(**args)
|
||||
log('Exiting the process executor...')
|
||||
_log('Exiting the process executor...')
|
||||
if process_stderr:
|
||||
process_stderr.release()
|
||||
if process_stdout:
|
||||
process_stdout.release()
|
||||
log('Bye!')
|
||||
_log('Bye!')
|
||||
|
||||
|
||||
# Let's ignore all the signal comming to us.
|
||||
@ -368,27 +392,50 @@ def convert_environment_variables(env):
|
||||
for key, value in env.items():
|
||||
try:
|
||||
if not isinstance(key, str):
|
||||
key = key.encode(default_encoding)
|
||||
if not isinstance(value, str):
|
||||
value = value.encode(default_encoding)
|
||||
key = key.encode(_sys_encoding)
|
||||
if not isinstance(value, str):
|
||||
value = value.encode(_sys_encoding)
|
||||
temp_env[key] = value
|
||||
except Exception as e:
|
||||
log_exception()
|
||||
_log_exception()
|
||||
return temp_env
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
log('Starting the process executor...')
|
||||
|
||||
_sys_encoding = sys.getdefaultencoding()
|
||||
if not _sys_encoding or _sys_encoding == 'ascii':
|
||||
# Fall back to 'utf-8', if we couldn't determine the default encoding,
|
||||
# or 'ascii'.
|
||||
_sys_encoding = 'utf-8'
|
||||
|
||||
_fs_encoding = sys.getfilesystemencoding()
|
||||
if not _fs_encoding or _fs_encoding == 'ascii':
|
||||
# Fall back to 'utf-8', if we couldn't determine the file-system encoding,
|
||||
# or 'ascii'.
|
||||
_fs_encoding = 'utf-8'
|
||||
|
||||
def u(_s, _encoding=_sys_encoding):
|
||||
if _IS_PY2:
|
||||
if isinstance(_s, str):
|
||||
return unicode(_s, _encoding)
|
||||
return _s
|
||||
_u = u
|
||||
|
||||
_out_dir = u(os.environ['OUTDIR'])
|
||||
_log_file = os.path.join(_out_dir, ('log_%s' % os.getpid()))
|
||||
|
||||
_log('Starting the process executor...')
|
||||
|
||||
# Ignore any signals
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
log('Disabled the SIGINT, SIGTERM signals...')
|
||||
_log('Disabled the SIGINT, SIGTERM signals...')
|
||||
|
||||
if IS_WIN:
|
||||
log('Disable the SIGBREAKM signal (windows)...')
|
||||
if _IS_WIN:
|
||||
_log('Disable the SIGBREAKM signal (windows)...')
|
||||
signal.signal(signal.SIGBREAK, signal_handler)
|
||||
log('Disabled the SIGBREAKM signal (windows)...')
|
||||
_log('Disabled the SIGBREAKM signal (windows)...')
|
||||
|
||||
# For windows:
|
||||
# We would run the process_executor in the detached mode again to make
|
||||
@ -396,11 +443,14 @@ if __name__ == '__main__':
|
||||
# depending on the status of the web-server.
|
||||
if 'PGA_BGP_FOREGROUND' in os.environ and \
|
||||
os.environ['PGA_BGP_FOREGROUND'] == "1":
|
||||
log('[CHILD] Start process execution...')
|
||||
log('Executing the command now from the detached child...')
|
||||
_log('[CHILD] Start process execution...')
|
||||
# This is a child process running as the daemon process.
|
||||
# Let's do the job assing to it.
|
||||
execute()
|
||||
# Let's do the job assigning to it.
|
||||
try:
|
||||
_log('Executing the command now from the detached child...')
|
||||
execute()
|
||||
except:
|
||||
_log_exception()
|
||||
else:
|
||||
from subprocess import CREATE_NEW_PROCESS_GROUP
|
||||
DETACHED_PROCESS = 0x00000008
|
||||
@ -414,11 +464,11 @@ if __name__ == '__main__':
|
||||
env['PGA_BGP_FOREGROUND'] = "1"
|
||||
|
||||
# We need environment variables & values in string
|
||||
log('[PARENT] Converting the environment variable in the bytes format...')
|
||||
_log('[PARENT] Converting the environment variable in the bytes format...')
|
||||
try:
|
||||
env = convert_environment_variables(env)
|
||||
except Exception as e:
|
||||
log_exception()
|
||||
_log_exception()
|
||||
|
||||
kwargs = {
|
||||
'stdin': stdin.fileno(),
|
||||
@ -426,33 +476,33 @@ if __name__ == '__main__':
|
||||
'stderr': stderr.fileno(),
|
||||
'creationflags': CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS,
|
||||
'close_fds': False,
|
||||
'cwd': os.environ['OUTDIR'],
|
||||
'cwd': _out_dir,
|
||||
'env': env
|
||||
}
|
||||
|
||||
cmd = [sys.executable]
|
||||
cmd.extend(sys.argv)
|
||||
|
||||
log('[PARENT] Command executings: {0}'.format(cmd))
|
||||
_log('[PARENT] Command executings: {0}'.format(cmd))
|
||||
|
||||
p = Popen(cmd, **kwargs)
|
||||
|
||||
exitCode = p.poll()
|
||||
|
||||
if exitCode is not None:
|
||||
log(
|
||||
_log(
|
||||
'[PARENT] Child exited with exit-code#{0}...'.format(
|
||||
exitCode
|
||||
)
|
||||
)
|
||||
else:
|
||||
log('[PARENT] Started the child with PID#{0}'.format(p.pid))
|
||||
_log('[PARENT] Started the child with PID#{0}'.format(p.pid))
|
||||
|
||||
# Question: Should we wait for sometime?
|
||||
# Answer: Looks the case...
|
||||
from time import sleep
|
||||
sleep(2)
|
||||
log('[PARENT] Exiting...')
|
||||
_log('[PARENT] Exiting...')
|
||||
sys.exit(0)
|
||||
else:
|
||||
r, w = os.pipe()
|
||||
@ -461,35 +511,36 @@ if __name__ == '__main__':
|
||||
# We will fork the process, and run the child process as daemon, and
|
||||
# let it do the job.
|
||||
if os.fork() == 0:
|
||||
log('[CHILD] Forked the child process...')
|
||||
_log('[CHILD] Forked the child process...')
|
||||
# Hmm... So - I need to do the job now...
|
||||
try:
|
||||
os.close(r)
|
||||
|
||||
log('[CHILD] Make the child process leader...')
|
||||
_log('[CHILD] Make the child process leader...')
|
||||
# Let me be the process leader first.
|
||||
os.setsid()
|
||||
os.umask(0)
|
||||
|
||||
log('[CHILD] Make the child process leader...')
|
||||
_log('[CHILD] Make the child process leader...')
|
||||
w = os.fdopen(w, 'w')
|
||||
# Let me inform my parent - I will do the job, do not worry
|
||||
# now, and die peacefully.
|
||||
log('[CHILD] Inform parent about successful child forking...')
|
||||
_log('[CHILD] Inform parent about successful child forking...')
|
||||
w.write('1')
|
||||
w.close()
|
||||
|
||||
log('[CHILD] Start executing the background process...')
|
||||
_log('[CHILD] Start executing the background process...')
|
||||
execute()
|
||||
except Exception:
|
||||
_log_exception()
|
||||
sys.exit(1)
|
||||
else:
|
||||
os.close(w)
|
||||
r = os.fdopen(r)
|
||||
# I do not care, what the child send.
|
||||
r.read()
|
||||
log('[PARENT] Got message from the child...')
|
||||
_log('[PARENT] Got message from the child...')
|
||||
r.close()
|
||||
|
||||
log('[PARENT] Exiting...')
|
||||
_log('[PARENT] Exiting...')
|
||||
sys.exit(0)
|
||||
|
||||
@ -11,8 +11,6 @@
|
||||
"""
|
||||
Introduce a function to run the process executor in detached mode.
|
||||
"""
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import csv
|
||||
import os
|
||||
import sys
|
||||
@ -21,6 +19,12 @@ from datetime import datetime
|
||||
from pickle import dumps, loads
|
||||
from subprocess import Popen
|
||||
|
||||
from pgadmin.utils import IS_PY2, u, file_quote, fs_encoding
|
||||
|
||||
if IS_PY2:
|
||||
from StringIO import StringIO
|
||||
else:
|
||||
from io import StringIO
|
||||
import pytz
|
||||
from dateutil import parser
|
||||
from flask import current_app
|
||||
@ -30,11 +34,6 @@ from flask_security import current_user
|
||||
import config
|
||||
from pgadmin.model import Process, db
|
||||
|
||||
if sys.version_info < (3,):
|
||||
from StringIO import StringIO
|
||||
else:
|
||||
from io import StringIO
|
||||
|
||||
|
||||
def get_current_time(format='%Y-%m-%d %H:%M:%S.%f %z'):
|
||||
"""
|
||||
@ -159,17 +158,23 @@ class BatchProcess(object):
|
||||
csv_writer = csv.writer(
|
||||
args_csv_io, delimiter=str(','), quoting=csv.QUOTE_MINIMAL
|
||||
)
|
||||
csv_writer.writerow(_args)
|
||||
if sys.version_info.major == 2:
|
||||
csv_writer.writerow([a.encode('utf-8') if isinstance(a, unicode) else a for a in _args])
|
||||
else:
|
||||
csv_writer.writerow(_args)
|
||||
|
||||
args_val = args_csv_io.getvalue().strip(str('\r\n'))
|
||||
|
||||
j = Process(
|
||||
pid=int(id), command=_cmd,
|
||||
arguments=args_csv_io.getvalue().strip(str('\r\n')),
|
||||
arguments=args_val.decode('utf-8', 'replace') if IS_PY2 and hasattr(args_val, 'decode') \
|
||||
else args_val,
|
||||
logdir=log_dir, desc=dumps(self.desc), user_id=current_user.id
|
||||
)
|
||||
db.session.add(j)
|
||||
db.session.commit()
|
||||
|
||||
def start(self):
|
||||
def start(self, cb=None):
|
||||
|
||||
def which(program, paths):
|
||||
def is_exe(fpath):
|
||||
@ -178,9 +183,9 @@ class BatchProcess(object):
|
||||
for path in paths:
|
||||
if not os.path.isdir(path):
|
||||
continue
|
||||
exe_file = os.path.join(path, program)
|
||||
exe_file = os.path.join(u(path, fs_encoding), program)
|
||||
if is_exe(exe_file):
|
||||
return exe_file
|
||||
return file_quote(exe_file)
|
||||
return None
|
||||
|
||||
def convert_environment_variables(env):
|
||||
@ -191,6 +196,8 @@ class BatchProcess(object):
|
||||
:return: Encoded environment variable as string
|
||||
"""
|
||||
encoding = sys.getdefaultencoding()
|
||||
if encoding is None or encoding == 'ascii':
|
||||
encoding = 'utf-8'
|
||||
temp_env = dict()
|
||||
for key, value in env.items():
|
||||
if not isinstance(key, str):
|
||||
@ -207,22 +214,22 @@ class BatchProcess(object):
|
||||
_('The process has already finished and can not be restarted.')
|
||||
)
|
||||
|
||||
executor = os.path.join(
|
||||
os.path.dirname(__file__), 'process_executor.py'
|
||||
)
|
||||
executor = file_quote(os.path.join(
|
||||
os.path.dirname(u(__file__)), u'process_executor.py'
|
||||
))
|
||||
paths = sys.path[:]
|
||||
interpreter = None
|
||||
|
||||
if os.name == 'nt':
|
||||
paths.insert(0, os.path.join(sys.prefix, 'Scripts'))
|
||||
paths.insert(0, os.path.join(sys.prefix))
|
||||
paths.insert(0, os.path.join(u(sys.prefix), u'Scripts'))
|
||||
paths.insert(0, u(sys.prefix))
|
||||
|
||||
interpreter = which('pythonw.exe', paths)
|
||||
interpreter = which(u'pythonw.exe', paths)
|
||||
if interpreter is None:
|
||||
interpreter = which('python.exe', paths)
|
||||
interpreter = which(u'python.exe', paths)
|
||||
else:
|
||||
paths.insert(0, os.path.join(sys.prefix, 'bin'))
|
||||
interpreter = which('python', paths)
|
||||
paths.insert(0, os.path.join(u(sys.prefix), u'bin'))
|
||||
interpreter = which(u'python', paths)
|
||||
|
||||
p = None
|
||||
cmd = [
|
||||
@ -231,15 +238,21 @@ class BatchProcess(object):
|
||||
]
|
||||
cmd.extend(self.args)
|
||||
|
||||
command = []
|
||||
for c in cmd:
|
||||
command.append(str(c))
|
||||
if os.name == 'nt' and IS_PY2:
|
||||
command = []
|
||||
for c in cmd:
|
||||
command.append(c.encode('utf-8') if isinstance(c, unicode) else str(c))
|
||||
|
||||
current_app.logger.info(
|
||||
"Executing the process executor with the arguments: %s",
|
||||
' '.join(command)
|
||||
)
|
||||
cmd = command
|
||||
current_app.logger.info(
|
||||
u"Executing the process executor with the arguments: %s",
|
||||
''.join(command)
|
||||
)
|
||||
|
||||
cmd = command
|
||||
else:
|
||||
current_app.logger.info(
|
||||
u"Executing the process executor with the arguments: %s", str(cmd)
|
||||
)
|
||||
|
||||
# Make a copy of environment, and add new variables to support
|
||||
env = os.environ.copy()
|
||||
@ -247,8 +260,12 @@ class BatchProcess(object):
|
||||
env['OUTDIR'] = self.log_dir
|
||||
env['PGA_BGP_FOREGROUND'] = "1"
|
||||
|
||||
# We need environment variables & values in string
|
||||
env = convert_environment_variables(env)
|
||||
if cb is not None:
|
||||
cb(env)
|
||||
|
||||
if IS_PY2:
|
||||
# We need environment variables & values in string
|
||||
env = convert_environment_variables(env)
|
||||
|
||||
if os.name == 'nt':
|
||||
DETACHED_PROCESS = 0x00000008
|
||||
@ -325,12 +342,16 @@ class BatchProcess(object):
|
||||
line = f.readline()
|
||||
line = line.decode(enc, 'replace')
|
||||
r = c.split(line)
|
||||
if len(r) < 3:
|
||||
# ignore this line
|
||||
pos = f.tell()
|
||||
continue
|
||||
if r[1] > ctime:
|
||||
completed = False
|
||||
break
|
||||
log.append([r[1], r[2]])
|
||||
pos = f.tell()
|
||||
if idx == 1024:
|
||||
if idx >= 1024:
|
||||
completed = False
|
||||
break
|
||||
if pos == eofs:
|
||||
@ -453,7 +474,10 @@ class BatchProcess(object):
|
||||
|
||||
if isinstance(desc, IProcessDesc):
|
||||
args = []
|
||||
args_csv = StringIO(p.arguments)
|
||||
args_csv = StringIO(
|
||||
p.arguments.encode('utf-8') \
|
||||
if hasattr(p.arguments, 'decode') else p.arguments
|
||||
)
|
||||
args_reader = csv.reader(args_csv, delimiter=str(','))
|
||||
for arg in args_reader:
|
||||
args = args + arg
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
|
||||
"""Implements Backup Utility"""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
import simplejson as json
|
||||
import os
|
||||
|
||||
@ -17,7 +18,8 @@ from flask import render_template, request, current_app, \
|
||||
from flask_babel import gettext as _
|
||||
from flask_security import login_required, current_user
|
||||
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html, \
|
||||
fs_short_path, document_dir
|
||||
from pgadmin.utils.ajax import make_json_response, bad_request
|
||||
|
||||
from config import PG_DEFAULT_DRIVER
|
||||
@ -79,14 +81,28 @@ class BackupMessage(IProcessDesc):
|
||||
Defines the message shown for the backup operation.
|
||||
"""
|
||||
|
||||
def __init__(self, _type, _sid, _bfile, **kwargs):
|
||||
def __init__(self, _type, _sid, _bfile, *_args, **_kwargs):
|
||||
self.backup_type = _type
|
||||
self.sid = _sid
|
||||
self.bfile = _bfile
|
||||
self.database = None
|
||||
self.database = _kwargs['database'] if 'database' in _kwargs else None
|
||||
self.cmd = ''
|
||||
|
||||
if 'database' in kwargs:
|
||||
self.database = kwargs['database']
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
# x = html.safe_str(x)
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
|
||||
return ' "' + x + '"'
|
||||
return ''
|
||||
|
||||
for arg in _args:
|
||||
if arg and len(arg) >= 2 and arg[:2] == '--':
|
||||
self.cmd += ' ' + arg
|
||||
else:
|
||||
self.cmd += cmdArg(arg)
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
@ -123,65 +139,32 @@ class BackupMessage(IProcessDesc):
|
||||
res = '<div class="h5">'
|
||||
|
||||
if self.backup_type == BACKUP.OBJECT:
|
||||
res += html.safe_str(
|
||||
_(
|
||||
"Backing up an object on the server '{0}' from database '{1}'..."
|
||||
).format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port),
|
||||
self.database
|
||||
)
|
||||
res += _(
|
||||
"Backing up an object on the server '{0}' from database '{1}'..."
|
||||
).format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port),
|
||||
self.database
|
||||
)
|
||||
elif self.backup_type == BACKUP.GLOBALS:
|
||||
res += html.safe_str(
|
||||
_("Backing up the global objects on the server '{0}'").format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
res += _("Backing up the global objects on the server '{0}'").format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
elif self.backup_type == BACKUP.SERVER:
|
||||
res += html.safe_str(
|
||||
_("Backing up the server '{0}'").format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
res += _("Backing up the server '{0}'").format(
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
else:
|
||||
# It should never reach here.
|
||||
res += "Backup"
|
||||
|
||||
res += '</div><div class="h5">'
|
||||
res += html.safe_str(
|
||||
_("Running command:")
|
||||
)
|
||||
res += _("Running command:")
|
||||
res += '</b><br><span class="pg-bg-cmd enable-selection">'
|
||||
res += html.safe_str(cmd)
|
||||
|
||||
replace_next = False
|
||||
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
|
||||
return ' "' + html.safe_str(x) + '"'
|
||||
|
||||
return ''
|
||||
|
||||
for arg in args:
|
||||
if arg and len(arg) >= 2 and arg[:2] == '--':
|
||||
res += ' ' + arg
|
||||
elif replace_next:
|
||||
res += ' "' + html.safe_str(
|
||||
self.bfile
|
||||
) + '"'
|
||||
else:
|
||||
if arg == '--file':
|
||||
replace_next = True
|
||||
res += cmdArg(arg)
|
||||
res += self.cmd
|
||||
res += '</span></div>'
|
||||
|
||||
return res
|
||||
|
||||
|
||||
@blueprint.route("/")
|
||||
@login_required
|
||||
def index():
|
||||
@ -201,7 +184,7 @@ def script():
|
||||
)
|
||||
|
||||
|
||||
def filename_with_file_manager_path(file):
|
||||
def filename_with_file_manager_path(_file):
|
||||
"""
|
||||
Args:
|
||||
file: File name returned from client file manager
|
||||
@ -213,9 +196,15 @@ def filename_with_file_manager_path(file):
|
||||
storage_dir = get_storage_directory()
|
||||
|
||||
if storage_dir:
|
||||
return os.path.join(storage_dir, file.lstrip('/'))
|
||||
_file = os.path.join(storage_dir, _file.lstrip(u'/').lstrip(u'\\'))
|
||||
elif not os.path.isabs(_file):
|
||||
_file = os.path.join(document_dir(), _file)
|
||||
|
||||
return file
|
||||
# Touch the file to get the short path of the file on windows.
|
||||
with open(_file, 'a'):
|
||||
pass
|
||||
|
||||
return fs_short_path(_file)
|
||||
|
||||
|
||||
@blueprint.route('/create_job/<int:sid>', methods=['POST'])
|
||||
@ -292,7 +281,11 @@ def create_backup_job(sid):
|
||||
p = BatchProcess(
|
||||
desc=BackupMessage(
|
||||
BACKUP.SERVER if data['type'] != 'global' else BACKUP.GLOBALS,
|
||||
sid, data['file']
|
||||
sid,
|
||||
data['file'].encode('utf-8') if hasattr(
|
||||
data['file'], 'encode'
|
||||
) else data['file'],
|
||||
*args
|
||||
),
|
||||
cmd=utility, args=args
|
||||
)
|
||||
@ -440,9 +433,15 @@ def create_backup_objects_job(sid):
|
||||
try:
|
||||
p = BatchProcess(
|
||||
desc=BackupMessage(
|
||||
BACKUP.OBJECT, sid, data['file'], database=data['database']
|
||||
BACKUP.OBJECT, sid,
|
||||
data['file'].encode('utf-8') if hasattr(
|
||||
data['file'], 'encode'
|
||||
) else data['file'],
|
||||
*args,
|
||||
database=data['database']
|
||||
),
|
||||
cmd=utility, args=args)
|
||||
cmd=utility, args=args
|
||||
)
|
||||
manager.export_password_env(p.id)
|
||||
p.start()
|
||||
jid = p.id
|
||||
|
||||
@ -16,7 +16,8 @@ from flask import url_for, Response, render_template, request, current_app
|
||||
from flask_babel import gettext as _
|
||||
from flask_security import login_required, current_user
|
||||
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html, \
|
||||
fs_short_path, document_dir, IS_WIN
|
||||
from pgadmin.utils.ajax import make_json_response, bad_request
|
||||
|
||||
from config import PG_DEFAULT_DRIVER
|
||||
@ -56,19 +57,47 @@ class ImportExportModule(PgAdminModule):
|
||||
blueprint = ImportExportModule(MODULE_NAME, __name__)
|
||||
|
||||
|
||||
class Message(IProcessDesc):
|
||||
class IEMessage(IProcessDesc):
|
||||
"""
|
||||
Message(IProcessDesc)
|
||||
IEMessage(IProcessDesc)
|
||||
|
||||
Defines the message shown for the Message operation.
|
||||
Defines the message shown for the import/export operation.
|
||||
"""
|
||||
|
||||
def __init__(self, _sid, _schema, _tbl, _database, _storage):
|
||||
def __init__(self, _sid, _schema, _tbl, _database, _storage, *_args):
|
||||
self.sid = _sid
|
||||
self.schema = _schema
|
||||
self.table = _tbl
|
||||
self.database = _database
|
||||
self.storage = _storage
|
||||
self._cmd = ''
|
||||
|
||||
if _storage:
|
||||
_storage = _storage.replace('\\', '/')
|
||||
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
x = html.safe_str(x)
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
|
||||
return ' "' + x + '"'
|
||||
return ''
|
||||
|
||||
replace_next = False
|
||||
for arg in _args:
|
||||
if arg and len(arg) >= 2 and arg[:2] == '--':
|
||||
if arg == '--command':
|
||||
replace_next = True
|
||||
self._cmd += ' ' + arg
|
||||
elif replace_next:
|
||||
arg = cmdArg(arg)
|
||||
if _storage is not None:
|
||||
arg = arg.replace(_storage, '<STORAGE_DIR>')
|
||||
self._cmd += ' "' + arg + '"'
|
||||
else:
|
||||
self._cmd+= cmdArg(arg)
|
||||
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
@ -90,45 +119,17 @@ class Message(IProcessDesc):
|
||||
).first()
|
||||
|
||||
res = '<div class="h5">'
|
||||
res += html.safe_str(
|
||||
_(
|
||||
"Copying table data '{0}.{1}' on database '{2}' for the server - '{3}'"
|
||||
).format(
|
||||
self.schema, self.table, self.database,
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
res += _(
|
||||
"Copying table data '{0}.{1}' on database '{2}' for the server - '{3}'"
|
||||
).format(
|
||||
self.schema, self.table, self.database,
|
||||
"{0} ({1}:{2})".format(s.name, s.host, s.port)
|
||||
)
|
||||
|
||||
res += '</div><div class="h5">'
|
||||
res += html.safe_str(
|
||||
_("Running command:")
|
||||
)
|
||||
res += _("Running command:")
|
||||
res += '</b><br><span class="pg-bg-cmd enable-selection">'
|
||||
res += html.safe_str(cmd)
|
||||
|
||||
replace_next = False
|
||||
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
|
||||
return ' "' + html.safe_str(x) + '"'
|
||||
|
||||
return ''
|
||||
|
||||
for arg in args:
|
||||
if arg and len(arg) >= 2 and arg[:2] == '--':
|
||||
if arg == '--command':
|
||||
replace_next = True
|
||||
res += ' ' + arg
|
||||
elif replace_next:
|
||||
if self.storage:
|
||||
arg = arg.replace(self.storage, '<STORAGE_DIR>')
|
||||
res += ' "' + html.safe_str(arg) + '"'
|
||||
else:
|
||||
res += cmdArg(arg)
|
||||
res += self._cmd
|
||||
res += '</span></div>'
|
||||
|
||||
return res
|
||||
@ -151,6 +152,33 @@ def script():
|
||||
)
|
||||
|
||||
|
||||
def filename_with_file_manager_path(_file, _present=False):
|
||||
"""
|
||||
Args:
|
||||
file: File name returned from client file manager
|
||||
|
||||
Returns:
|
||||
Filename to use for backup with full path taken from preference
|
||||
"""
|
||||
# Set file manager directory from preference
|
||||
storage_dir = get_storage_directory()
|
||||
|
||||
if storage_dir:
|
||||
_file = os.path.join(storage_dir, _file.lstrip(u'/').lstrip(u'\\'))
|
||||
elif not os.path.isabs(_file):
|
||||
_file = os.path.join(document_dir(), _file)
|
||||
|
||||
if not _present:
|
||||
# Touch the file to get the short path of the file on windows.
|
||||
with open(_file, 'a'):
|
||||
pass
|
||||
else:
|
||||
if not os.path.isfile(_file):
|
||||
return None
|
||||
|
||||
return fs_short_path(_file)
|
||||
|
||||
|
||||
@blueprint.route('/create_job/<int:sid>', methods=['POST'])
|
||||
@login_required
|
||||
def create_import_export_job(sid):
|
||||
@ -175,10 +203,8 @@ def create_import_export_job(sid):
|
||||
id=sid).first()
|
||||
|
||||
if server is None:
|
||||
return make_json_response(
|
||||
success=0,
|
||||
errormsg=_("Couldn't find the given server")
|
||||
)
|
||||
return bad_request(errormsg=_("Couldn't find the given server"))
|
||||
|
||||
|
||||
# To fetch MetaData for the server
|
||||
from pgadmin.utils.driver import get_driver
|
||||
@ -188,10 +214,7 @@ def create_import_export_job(sid):
|
||||
connected = conn.connected()
|
||||
|
||||
if not connected:
|
||||
return make_json_response(
|
||||
success=0,
|
||||
errormsg=_("Please connect to the server first...")
|
||||
)
|
||||
return bad_request(errormsg=_("Please connect to the server first..."))
|
||||
|
||||
# Get the utility path from the connection manager
|
||||
utility = manager.utility('sql')
|
||||
@ -200,21 +223,16 @@ def create_import_export_job(sid):
|
||||
storage_dir = get_storage_directory()
|
||||
|
||||
if 'filename' in data:
|
||||
if os.name == 'nt':
|
||||
data['filename'] = data['filename'].replace('/', '\\')
|
||||
if storage_dir:
|
||||
storage_dir = storage_dir.replace('/', '\\')
|
||||
data['filename'] = data['filename'].replace('\\', '\\\\')
|
||||
data['filename'] = os.path.join(storage_dir, data['filename'].lstrip('/'))
|
||||
elif storage_dir:
|
||||
data['filename'] = os.path.join(storage_dir, data['filename'].lstrip('/'))
|
||||
else:
|
||||
data['filename'] = data['filename']
|
||||
_file = filename_with_file_manager_path(data['filename'], data['is_import'])
|
||||
if not _file:
|
||||
return bad_request(errormsg=_('Please specify a valid file'))
|
||||
|
||||
if IS_WIN:
|
||||
_file = _file.replace('\\', '/')
|
||||
|
||||
data['filename'] = _file
|
||||
else:
|
||||
return make_json_response(
|
||||
data={'status': False, 'info': 'Please specify a valid file'}
|
||||
)
|
||||
return bad_request(errormsg=_('Please specify a valid file'))
|
||||
|
||||
cols = None
|
||||
icols = None
|
||||
@ -255,33 +273,32 @@ def create_import_export_job(sid):
|
||||
ignore_column_list=icols
|
||||
)
|
||||
|
||||
args = [
|
||||
'--host', server.host, '--port', str(server.port),
|
||||
'--username', server.username, '--dbname', data['database'],
|
||||
'--command', query
|
||||
]
|
||||
args = ['--command', query]
|
||||
|
||||
try:
|
||||
p = BatchProcess(
|
||||
desc=Message(
|
||||
desc=IEMessage(
|
||||
sid,
|
||||
data['schema'],
|
||||
data['table'],
|
||||
data['database'],
|
||||
storage_dir
|
||||
storage_dir,
|
||||
utility, *args
|
||||
),
|
||||
cmd=utility, args=args
|
||||
)
|
||||
manager.export_password_env(p.id)
|
||||
p.start()
|
||||
def export_pg_env(env):
|
||||
env['PGHOST'] = server.host
|
||||
env['PGPORT'] = str(server.port)
|
||||
env['PGUSER'] = server.username
|
||||
env['PGDATABASE'] = data['database']
|
||||
|
||||
p.start(export_pg_env)
|
||||
jid = p.id
|
||||
except Exception as e:
|
||||
current_app.logger.exception(e)
|
||||
return make_json_response(
|
||||
status=410,
|
||||
success=0,
|
||||
errormsg=str(e)
|
||||
)
|
||||
return bad_request(errormsg=str(e))
|
||||
|
||||
# Return response
|
||||
return make_json_response(
|
||||
|
||||
@ -17,7 +17,8 @@ from flask import render_template, request, current_app, \
|
||||
from flask_babel import gettext as _
|
||||
from flask_security import login_required, current_user
|
||||
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html
|
||||
from pgadmin.utils import PgAdminModule, get_storage_directory, html, \
|
||||
fs_short_path, document_dir
|
||||
from pgadmin.utils.ajax import make_json_response, bad_request
|
||||
|
||||
from config import PG_DEFAULT_DRIVER
|
||||
@ -58,9 +59,26 @@ blueprint = RestoreModule(
|
||||
|
||||
|
||||
class RestoreMessage(IProcessDesc):
|
||||
def __init__(self, _sid, _bfile):
|
||||
def __init__(self, _sid, _bfile, *_args):
|
||||
self.sid = _sid
|
||||
self.bfile = _bfile
|
||||
self.cmd = ''
|
||||
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
x = html.safe_str(x)
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
return ' "' + x + '"'
|
||||
return ''
|
||||
|
||||
for arg in _args:
|
||||
if arg and len(arg) >= 2 and arg[:2] == '--':
|
||||
self.cmd += ' ' + arg
|
||||
else:
|
||||
self.cmd += cmdArg(arg)
|
||||
|
||||
|
||||
@property
|
||||
def message(self):
|
||||
@ -95,30 +113,7 @@ class RestoreMessage(IProcessDesc):
|
||||
)
|
||||
res += '</b><br><span class="pg-bg-cmd enable-selection">'
|
||||
res += html.safe_str(cmd)
|
||||
|
||||
def cmdArg(x):
|
||||
if x:
|
||||
x = x.replace('\\', '\\\\')
|
||||
x = x.replace('"', '\\"')
|
||||
x = x.replace('""', '\\"')
|
||||
|
||||
return ' "' + html.safe_str(x) + '"'
|
||||
|
||||
return ''
|
||||
|
||||
idx = 0
|
||||
no_args = len(args)
|
||||
for arg in args:
|
||||
if idx < no_args - 1:
|
||||
if arg[:2] == '--':
|
||||
res += ' ' + arg
|
||||
else:
|
||||
res += cmdArg(arg)
|
||||
idx += 1
|
||||
|
||||
if no_args > 1:
|
||||
res += ' "' + html.safe_str(arg) + '"'
|
||||
|
||||
res += self.cmd
|
||||
res += '</span></div>'
|
||||
|
||||
return res
|
||||
@ -143,7 +138,7 @@ def script():
|
||||
)
|
||||
|
||||
|
||||
def filename_with_file_manager_path(file):
|
||||
def filename_with_file_manager_path(_file):
|
||||
"""
|
||||
Args:
|
||||
file: File name returned from client file manager
|
||||
@ -155,9 +150,14 @@ def filename_with_file_manager_path(file):
|
||||
storage_dir = get_storage_directory()
|
||||
|
||||
if storage_dir:
|
||||
return os.path.join(storage_dir, file.lstrip('/'))
|
||||
_file = os.path.join(storage_dir, _file.lstrip(u'/').lstrip(u'\\'))
|
||||
elif not os.path.isabs(_file):
|
||||
_file = os.path.join(document_dir(), _file)
|
||||
|
||||
return file
|
||||
if not os.path.isfile(_file):
|
||||
return None
|
||||
|
||||
return fs_short_path(_file)
|
||||
|
||||
|
||||
@blueprint.route('/create_job/<int:sid>', methods=['POST'])
|
||||
@ -179,7 +179,13 @@ def create_restore_job(sid):
|
||||
else:
|
||||
data = json.loads(request.data, encoding='utf-8')
|
||||
|
||||
backup_file = filename_with_file_manager_path(data['file'])
|
||||
_file = filename_with_file_manager_path(data['file'])
|
||||
|
||||
if _file is None:
|
||||
return make_json_response(
|
||||
success=0,
|
||||
errormsg=_("File couldn't be found!")
|
||||
)
|
||||
|
||||
# Fetch the server details like hostname, port, roles etc
|
||||
server = Server.query.filter_by(
|
||||
@ -261,7 +267,7 @@ def create_restore_job(sid):
|
||||
return False
|
||||
|
||||
args.extend([
|
||||
'--host', server.host, '--port', server.port,
|
||||
'--host', server.host, '--port', str(server.port),
|
||||
'--username', server.username, '--no-password'
|
||||
])
|
||||
|
||||
@ -300,11 +306,17 @@ def create_restore_job(sid):
|
||||
set_multiple('trigger_funcs', '--function')
|
||||
set_multiple('indexes', '--index')
|
||||
|
||||
args.append(backup_file)
|
||||
args.append(fs_short_path(_file))
|
||||
|
||||
try:
|
||||
p = BatchProcess(
|
||||
desc=RestoreMessage(sid, data['file']),
|
||||
desc=RestoreMessage(
|
||||
sid,
|
||||
data['file'].encode('utf-8') if hasattr(
|
||||
data['file'], 'encode'
|
||||
) else data['file'],
|
||||
*args
|
||||
),
|
||||
cmd=utility, args=args
|
||||
)
|
||||
manager.export_password_env(p.id)
|
||||
|
||||
@ -131,3 +131,105 @@ class PgAdminModule(Blueprint):
|
||||
menu_items = dict((key, sorted(value, key=attrgetter('priority')))
|
||||
for key, value in menu_items.items())
|
||||
return menu_items
|
||||
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
IS_PY2 = (sys.version_info[0] == 2)
|
||||
IS_WIN = (os.name == 'nt')
|
||||
|
||||
sys_encoding = sys.getdefaultencoding()
|
||||
if not sys_encoding or sys_encoding == 'ascii':
|
||||
# Fall back to 'utf-8', if we couldn't determine the default encoding,
|
||||
# or 'ascii'.
|
||||
sys_encoding = 'utf-8'
|
||||
|
||||
fs_encoding = sys.getfilesystemencoding()
|
||||
if not fs_encoding or fs_encoding == 'ascii':
|
||||
# Fall back to 'utf-8', if we couldn't determine the file-system encoding,
|
||||
# or 'ascii'.
|
||||
fs_encoding = 'utf-8'
|
||||
|
||||
|
||||
def u(_s, _encoding=sys_encoding):
|
||||
if IS_PY2:
|
||||
if isinstance(_s, str):
|
||||
return unicode(_s, _encoding)
|
||||
return _s
|
||||
|
||||
|
||||
def file_quote(_p):
|
||||
if IS_PY2:
|
||||
if isinstance(_p, unicode):
|
||||
return _p.encode(fs_encoding)
|
||||
return _p
|
||||
|
||||
|
||||
if IS_WIN:
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
if IS_PY2:
|
||||
def env(name):
|
||||
if IS_PY2:
|
||||
# Make sure string argument is unicode
|
||||
name = unicode(name)
|
||||
n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0)
|
||||
|
||||
if n == 0:
|
||||
return None
|
||||
|
||||
buf= ctypes.create_unicode_buffer(u'\0'*n)
|
||||
ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n)
|
||||
|
||||
return buf.value
|
||||
else:
|
||||
def env(name):
|
||||
if name in os.environ:
|
||||
return os.environ[name]
|
||||
return None
|
||||
|
||||
_GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW
|
||||
_GetShortPathNameW.argtypes = [
|
||||
wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD
|
||||
]
|
||||
_GetShortPathNameW.restype = wintypes.DWORD
|
||||
|
||||
def fs_short_path(_path):
|
||||
"""
|
||||
Gets the short path name of a given long path.
|
||||
http://stackoverflow.com/a/23598461/200291
|
||||
"""
|
||||
buf_size = len(_path)
|
||||
while True:
|
||||
res = ctypes.create_unicode_buffer(buf_size)
|
||||
needed = _GetShortPathNameW(_path, res, buf_size)
|
||||
|
||||
if buf_size >= needed:
|
||||
return res.value
|
||||
else:
|
||||
buf_size += needed
|
||||
|
||||
def document_dir():
|
||||
CSIDL_PERSONAL = 5 # My Documents
|
||||
SHGFP_TYPE_CURRENT = 0 # Get current, not default value
|
||||
|
||||
buf = ctypes.create_unicode_buffer(wintypes.MAX_PATH)
|
||||
ctypes.windll.shell32.SHGetFolderPathW(
|
||||
None, CSIDL_PERSONAL, None, SHGFP_TYPE_CURRENT, buf
|
||||
)
|
||||
|
||||
return buf.value
|
||||
|
||||
else:
|
||||
def env(name):
|
||||
if name in os.environ:
|
||||
return os.environ[name]
|
||||
return None
|
||||
|
||||
def fs_short_path(_path):
|
||||
return _path
|
||||
|
||||
def document_dir():
|
||||
return os.path.realpath(os.path.expanduser(u'~/'))
|
||||
|
||||
@ -1491,8 +1491,6 @@ class ServerManager(object):
|
||||
database = self.db
|
||||
elif did in self.db_info:
|
||||
database = self.db_info[did]['datname']
|
||||
if hasattr(str, 'decode'):
|
||||
database = database.decode('utf-8')
|
||||
else:
|
||||
maintenance_db_id = u'DB:{0}'.format(self.db)
|
||||
if maintenance_db_id in self.connections:
|
||||
@ -1510,9 +1508,6 @@ WHERE db.oid = {0}""".format(did))
|
||||
if status and len(res['rows']) > 0:
|
||||
for row in res['rows']:
|
||||
self.db_info[did] = row
|
||||
if hasattr(str, 'decode'):
|
||||
self.db_info[did]['datname'] = \
|
||||
self.db_info[did]['datname'].decode('utf-8')
|
||||
database = self.db_info[did]['datname']
|
||||
|
||||
if did not in self.db_info:
|
||||
@ -1843,7 +1838,10 @@ class Driver(BaseDriver):
|
||||
# Returns in bytes, we need to convert it in string
|
||||
if isinstance(res, bytes):
|
||||
try:
|
||||
res = res.decode()
|
||||
try:
|
||||
res = res.decode()
|
||||
except UnicodeDecodeError:
|
||||
res = res.decode(sys.getfilesystemencoding())
|
||||
except UnicodeDecodeError:
|
||||
res = res.decode('utf-8')
|
||||
|
||||
|
||||
@ -10,9 +10,14 @@
|
||||
"""Utilities for HTML"""
|
||||
|
||||
import cgi
|
||||
from pgadmin.utils import IS_PY2
|
||||
|
||||
|
||||
def safe_str(x):
|
||||
return cgi.escape(x).encode(
|
||||
'ascii', 'xmlcharrefreplace'
|
||||
).decode()
|
||||
try:
|
||||
x = x.encode('ascii', 'xmlcharrefreplace') if hasattr(x, 'encode') else x
|
||||
if not IS_PY2:
|
||||
x = x.decode('utf-8')
|
||||
except:
|
||||
pass
|
||||
return cgi.escape(x)
|
||||
|
||||
@ -37,7 +37,11 @@ def get_storage_directory():
|
||||
if len(username) == 0 or username[0].isdigit():
|
||||
username = 'pga_user_' + username
|
||||
|
||||
storage_dir = os.path.join(storage_dir, username)
|
||||
storage_dir = os.path.join(
|
||||
storage_dir.decode('utf-8') if hasattr(storage_dir, 'decode') \
|
||||
else storage_dir,
|
||||
username
|
||||
)
|
||||
|
||||
if not os.path.exists(storage_dir):
|
||||
os.makedirs(storage_dir, int('700', 8))
|
||||
|
||||
@ -193,7 +193,7 @@ class _Preference(object):
|
||||
|
||||
if pref is None:
|
||||
pref = UserPrefTable(
|
||||
uid=current_user.id, pid=self.pid, value=str(value)
|
||||
uid=current_user.id, pid=self.pid, value=value
|
||||
)
|
||||
db.session.add(pref)
|
||||
else:
|
||||
|
||||
@ -143,7 +143,10 @@ class CachingSessionManager(SessionManager):
|
||||
def put(self, session):
|
||||
self.parent.put(session)
|
||||
if session.sid in self._cache:
|
||||
del self._cache[session.sid]
|
||||
try:
|
||||
del self._cache[session.sid]
|
||||
except:
|
||||
pass
|
||||
self._cache[session.sid] = session
|
||||
self._normalize()
|
||||
|
||||
|
||||
71
web/setup.py
71
web/setup.py
@ -22,14 +22,20 @@ from flask import Flask
|
||||
from flask_security import Security, SQLAlchemyUserDatastore
|
||||
from flask_security.utils import encrypt_password
|
||||
|
||||
from pgadmin.model import db, Role, User, Server, \
|
||||
ServerGroup, Version, Keys
|
||||
# We need to include the root directory in sys.path to ensure that we can
|
||||
# find everything we need when running in the standalone runtime.
|
||||
root = os.path.dirname(os.path.realpath(__file__))
|
||||
if sys.path[0] != root:
|
||||
sys.path.insert(0, root)
|
||||
|
||||
|
||||
# Configuration settings
|
||||
import config
|
||||
|
||||
# Get the config database schema version. We store this in pgadmin.model
|
||||
# as it turns out that putting it in the config files isn't a great idea
|
||||
from pgadmin.model import SCHEMA_VERSION
|
||||
from pgadmin.model import db, Role, User, Server, ServerGroup, Version, Keys, \
|
||||
SCHEMA_VERSION
|
||||
from pgadmin.utils.versioned_template_loader import VersionedTemplateLoader
|
||||
|
||||
config.SETTINGS_SCHEMA_VERSION = SCHEMA_VERSION
|
||||
@ -44,7 +50,7 @@ def do_setup(app):
|
||||
"""Create a new settings database from scratch"""
|
||||
|
||||
if config.SERVER_MODE is False:
|
||||
print("NOTE: Configuring authentication for DESKTOP mode.")
|
||||
print(u"NOTE: Configuring authentication for DESKTOP mode.")
|
||||
email = config.DESKTOP_USER
|
||||
p1 = ''.join([
|
||||
random.choice(string.ascii_letters + string.digits)
|
||||
@ -52,7 +58,7 @@ def do_setup(app):
|
||||
])
|
||||
|
||||
else:
|
||||
print("NOTE: Configuring authentication for SERVER mode.\n")
|
||||
print(u"NOTE: Configuring authentication for SERVER mode.\n")
|
||||
|
||||
if all(value in os.environ for value in
|
||||
['PGADMIN_SETUP_EMAIL', 'PGADMIN_SETUP_PASSWORD']):
|
||||
@ -64,9 +70,11 @@ def do_setup(app):
|
||||
p1 = os.environ['PGADMIN_SETUP_PASSWORD']
|
||||
else:
|
||||
# Prompt the user for their default username and password.
|
||||
print("""
|
||||
Enter the email address and password to use for the initial pgAdmin user \
|
||||
account:\n""")
|
||||
print(
|
||||
u"Enter the email address and password to use for the initial "
|
||||
u"pgAdmin user account:\n"
|
||||
)
|
||||
|
||||
email_filter = re.compile(
|
||||
"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9]"
|
||||
"(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9]"
|
||||
@ -74,7 +82,7 @@ def do_setup(app):
|
||||
|
||||
email = input("Email address: ")
|
||||
while email == '' or not email_filter.match(email):
|
||||
print('Invalid email address. Please try again.')
|
||||
print(u'Invalid email address. Please try again.')
|
||||
email = input("Email address: ")
|
||||
|
||||
def pprompt():
|
||||
@ -83,10 +91,11 @@ def do_setup(app):
|
||||
p1, p2 = pprompt()
|
||||
while p1 != p2 or len(p1) < 6:
|
||||
if p1 != p2:
|
||||
print('Passwords do not match. Please try again.')
|
||||
print(u'Passwords do not match. Please try again.')
|
||||
else:
|
||||
print(
|
||||
'Password must be at least 6 characters. Please try again.')
|
||||
u'Password must be at least 6 characters. Please try again.'
|
||||
)
|
||||
p1, p2 = pprompt()
|
||||
|
||||
# Setup Flask-Security
|
||||
@ -134,9 +143,9 @@ def do_setup(app):
|
||||
db.session.commit()
|
||||
|
||||
# Done!
|
||||
print("")
|
||||
print(u"")
|
||||
print(
|
||||
"The configuration database has been created at {0}".format(
|
||||
u"The configuration database has been created at {0}".format(
|
||||
config.SQLITE_PATH
|
||||
)
|
||||
)
|
||||
@ -154,19 +163,19 @@ def do_upgrade(app, datastore, version):
|
||||
|
||||
# Pre-flight checks
|
||||
if int(version.value) > int(config.SETTINGS_SCHEMA_VERSION):
|
||||
print("""
|
||||
print(u"""
|
||||
The database schema version is {0}, whilst the version required by the \
|
||||
software is {1}.
|
||||
Exiting...""".format(version.value, config.SETTINGS_SCHEMA_VERSION))
|
||||
sys.exit(1)
|
||||
elif int(version.value) == int(config.SETTINGS_SCHEMA_VERSION):
|
||||
print("""
|
||||
print(u"""
|
||||
The database schema version is {0} as required.
|
||||
Exiting...""".format(version.value))
|
||||
sys.exit(1)
|
||||
|
||||
app.logger.info(
|
||||
"NOTE: Upgrading database schema from version %d to %d." %
|
||||
u"NOTE: Upgrading database schema from version %d to %d." %
|
||||
(version.value, config.SETTINGS_SCHEMA_VERSION)
|
||||
)
|
||||
|
||||
@ -394,19 +403,24 @@ if __name__ == '__main__':
|
||||
'sqlite:///' + config.SQLITE_PATH.replace('\\', '/')
|
||||
db.init_app(app)
|
||||
|
||||
print("pgAdmin 4 - Application Initialisation")
|
||||
print("======================================\n")
|
||||
print(u"pgAdmin 4 - Application Initialisation")
|
||||
print(u"======================================\n")
|
||||
|
||||
from pgadmin.utils import u, fs_encoding, file_quote
|
||||
|
||||
local_config = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)),
|
||||
'config_local.py'
|
||||
os.path.dirname(os.path.realpath(u(__file__, fs_encoding))),
|
||||
u'config_local.py'
|
||||
)
|
||||
|
||||
# Check if the database exists. If it does, tell the user and exit.
|
||||
if os.path.isfile(config.SQLITE_PATH):
|
||||
print("""
|
||||
The configuration database '%s' already exists.
|
||||
Entering upgrade mode...""" % config.SQLITE_PATH)
|
||||
print(
|
||||
u"The configuration database '{0}' already exists.".format(
|
||||
config.SQLITE_PATH
|
||||
)
|
||||
)
|
||||
print(u"Entering upgrade mode...")
|
||||
|
||||
# Setup Flask-Security
|
||||
user_datastore = SQLAlchemyUserDatastore(db, User, Role)
|
||||
@ -417,18 +431,18 @@ Entering upgrade mode...""" % config.SQLITE_PATH)
|
||||
|
||||
# Pre-flight checks
|
||||
if int(version.value) > int(config.SETTINGS_SCHEMA_VERSION):
|
||||
print("""
|
||||
print(u"""
|
||||
The database schema version is %d, whilst the version required by the \
|
||||
software is %d.
|
||||
Exiting...""" % (version.value, config.SETTINGS_SCHEMA_VERSION))
|
||||
sys.exit(1)
|
||||
elif int(version.value) == int(config.SETTINGS_SCHEMA_VERSION):
|
||||
print("""
|
||||
print(u"""
|
||||
The database schema version is %d as required.
|
||||
Exiting...""" % (version.value))
|
||||
sys.exit(1)
|
||||
|
||||
print("NOTE: Upgrading database schema from version %d to %d." % (
|
||||
print(u"NOTE: Upgrading database schema from version %d to %d." % (
|
||||
version.value, config.SETTINGS_SCHEMA_VERSION
|
||||
))
|
||||
do_upgrade(app, user_datastore, version)
|
||||
@ -441,12 +455,15 @@ Exiting...""" % (version.value))
|
||||
app.config.from_object(config)
|
||||
|
||||
directory = os.path.dirname(config.SQLITE_PATH)
|
||||
|
||||
if not os.path.exists(directory):
|
||||
os.makedirs(directory, int('700', 8))
|
||||
|
||||
db_file = os.open(config.SQLITE_PATH, os.O_CREAT, int('600', 8))
|
||||
os.close(db_file)
|
||||
|
||||
print("""
|
||||
print(u"""
|
||||
The configuration database - '{0}' does not exist.
|
||||
Entering initial setup mode...""".format(config.SQLITE_PATH))
|
||||
|
||||
do_setup(app)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user