IntenseWebs/pgadmin4
3
0
Fork 0
mirror of https://github.com/pgadmin-org/pgadmin4.git synced 2025-02-25 18:55:31 -06:00
Code Issues Packages Projects Releases Wiki Activity

Resolved quite a few file-system encoding/decoding related cases.

Browse Source 
In order to resolve the non-ascii characters in path (in user directory,
storage path, etc) on windows, we have converted the path into the
short-path, so that - we don't need to deal with the encoding issues
(specially with Python 2).

We've resolved majority of the issues with this patch.
We still need couple issues to resolve after this in the same area.

TODO
* Add better support for non-ascii characters in the database name on
  windows with Python 3
* Improve the messages created after the background processes by
  different modules (such as Backup, Restore, Import/Export, etc.),
  which does not show short-paths, and xml representable characters for
  non-ascii characters, when found in the database objects, and the file
  PATH.

Fixes #2174, #1797, #2166, #1940

Initial patch by: Surinder Kumar
Reviewed by: Murtuza Zabuawala
This commit is contained in:
Ashesh Vashi 2017-03-07 15:30:57 +05:30
parent 063177155e
commit f2fc1ceba8
15 changed files with 594 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
web/config.py
View File

@ -12,7 +12,7 @@
########################################################################## ##########################################################################
import os import os
from logging import * from pgadmin.utils import env, IS_PY2, IS_WIN, fs_short_path
########################################################################## ##########################################################################
# Application settings # Application settings
@ -81,12 +81,17 @@ MODULE_BLACKLIST = ['test']
# List of treeview browser nodes to skip when dynamically loading # List of treeview browser nodes to skip when dynamically loading
NODE_BLACKLIST = [] NODE_BLACKLIST = []
# Data directory for storage of config settings etc. This shouldn't normally # Data directory for storage of config settings etc. This shouldn't normally
# need to be changed - it's here as various other settings depend on it. # need to be changed - it's here as various other settings depend on it.
if os.name == 'nt': if IS_WIN:
DATA_DIR = os.path.realpath(os.getenv('APPDATA') + "/pgAdmin") # Use the short path on windows
DATA_DIR = os.path.realpath(
os.path.join(fs_short_path(env('APPDATA')), u"pgAdmin")
)
else: else:
DATA_DIR = os.path.realpath(os.path.expanduser('~/.pgadmin/')) DATA_DIR = os.path.realpath(os.path.expanduser(u'~/.pgadmin/'))
########################################################################## ##########################################################################
# Log settings # Log settings
@ -95,6 +100,9 @@ else:
# Debug mode? # Debug mode?
DEBUG = False DEBUG = False
import logging
# Application log level - one of: # Application log level - one of:
# CRITICAL 50 # CRITICAL 50
# ERROR 40 # ERROR 40
@ -103,18 +111,16 @@ DEBUG = False
# INFO 20 # INFO 20
# DEBUG 10 # DEBUG 10
# NOTSET 0 # NOTSET 0
CONSOLE_LOG_LEVEL = WARNING CONSOLE_LOG_LEVEL = logging.WARNING
FILE_LOG_LEVEL = WARNING FILE_LOG_LEVEL = logging.WARNING
# Log format. # Log format.
CONSOLE_LOG_FORMAT = '%(asctime)s: %(levelname)s\t%(name)s:\t%(message)s' CONSOLE_LOG_FORMAT = '%(asctime)s: %(levelname)s\t%(name)s:\t%(message)s'
FILE_LOG_FORMAT = '%(asctime)s: %(levelname)s\t%(name)s:\t%(message)s' FILE_LOG_FORMAT = '%(asctime)s: %(levelname)s\t%(name)s:\t%(message)s'
# Log file name # Log file name
LOG_FILE = os.path.join( LOG_FILE = os.path.join(DATA_DIR, 'pgadmin4.log')
DATA_DIR,
'pgadmin4.log'
)
########################################################################## ##########################################################################
# Server settings # Server settings
@ -176,11 +182,8 @@ MAX_SESSION_IDLE_TIME = 60
# The default path to the SQLite database used to store user accounts and # The default path to the SQLite database used to store user accounts and
# settings. This default places the file in the same directory as this # settings. This default places the file in the same directory as this
# config file, but generates an absolute path for use througout the app. # config file, but generates an absolute path for use througout the app.
SQLITE_PATH = os.environ.get("SQLITE_PATH") or \ SQLITE_PATH = env('SQLITE_PATH') or os.path.join(DATA_DIR, 'pgadmin4.db')
os.path.join(
DATA_DIR,
'pgadmin4.db'
)
# SQLITE_TIMEOUT will define how long to wait before throwing the error - # SQLITE_TIMEOUT will define how long to wait before throwing the error -
# OperationError due to database lock. On slower system, you may need to change # OperationError due to database lock. On slower system, you may need to change
# this to some higher value. # this to some higher value.
@ -207,10 +210,7 @@ SQLITE_TIMEOUT = 500
# SESSION_DB_PATH = '/run/shm/pgAdmin4_session' # SESSION_DB_PATH = '/run/shm/pgAdmin4_session'
# #
########################################################################## ##########################################################################
SESSION_DB_PATH = os.path.join( SESSION_DB_PATH = os.path.join(DATA_DIR, 'sessions')
DATA_DIR,
'sessions'
)
SESSION_COOKIE_NAME = 'pga4_session' SESSION_COOKIE_NAME = 'pga4_session'
@ -262,10 +262,8 @@ UPGRADE_CHECK_URL = 'https://www.pgadmin.org/versions.json'
# 2. Set path manually like # 2. Set path manually like
# STORAGE_DIR = "/path/to/directory/" # STORAGE_DIR = "/path/to/directory/"
########################################################################## ##########################################################################
STORAGE_DIR = os.path.join( STORAGE_DIR = os.path.join(DATA_DIR, 'storage')
DATA_DIR,
'storage'
)
########################################################################## ##########################################################################
# Default locations for binary utilities (pg_dump, pg_restore etc) # Default locations for binary utilities (pg_dump, pg_restore etc)
@ -296,6 +294,7 @@ TESTING_MODE = False
# The default path for SQLite database for testing # The default path for SQLite database for testing
TEST_SQLITE_PATH = os.path.join(DATA_DIR, 'test_pgadmin4.db') TEST_SQLITE_PATH = os.path.join(DATA_DIR, 'test_pgadmin4.db')
########################################################################## ##########################################################################
# Allows flask application to response to the each request asynchronously # Allows flask application to response to the each request asynchronously
########################################################################## ##########################################################################

8
web/pgAdmin4.py
View File

@ -34,9 +34,11 @@ config.SETTINGS_SCHEMA_VERSION = SCHEMA_VERSION
# Check if the database exists. If it does not, create it. # Check if the database exists. If it does not, create it.
if not os.path.isfile(config.SQLITE_PATH): if not os.path.isfile(config.SQLITE_PATH):
setupfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), from pgadmin.utils import u, fs_encoding, file_quote
'setup.py') setupfile = os.path.join(
exec (open(setupfile).read()) os.path.dirname(os.path.realpath(u(__file__, fs_encoding))), u'setup.py'
)
exec(open(file_quote(setupfile), 'r').read())
########################################################################## ##########################################################################
# Server starup # Server starup

14
web/pgadmin/__init__.py
View File

@ -30,8 +30,6 @@ from werkzeug.local import LocalProxy
from werkzeug.utils import find_modules from werkzeug.utils import find_modules
from pgadmin.model import db, Role, Server, ServerGroup, User, Version, Keys from pgadmin.model import db, Role, Server, ServerGroup, User, Version, Keys
# Configuration settings
import config
# If script is running under python3, it will not have the xrange function # If script is running under python3, it will not have the xrange function
# defined # defined
@ -129,7 +127,13 @@ def _find_blueprint():
current_blueprint = LocalProxy(_find_blueprint) current_blueprint = LocalProxy(_find_blueprint)
def create_app(app_name=config.APP_NAME): def create_app(app_name=None):
# Configuration settings
import config
if not app_name:
app_name = config.APP_NAME
"""Create the Flask application, startup logging and dynamically load """Create the Flask application, startup logging and dynamically load
additional modules (blueprints) that are found in this directory.""" additional modules (blueprints) that are found in this directory."""
app = PgAdmin(__name__, static_url_path='/static') app = PgAdmin(__name__, static_url_path='/static')
@ -193,8 +197,8 @@ def create_app(app_name=config.APP_NAME):
# Setup authentication # Setup authentication
########################################################################## ##########################################################################
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///{0}?timeout={1}'.format( app.config['SQLALCHEMY_DATABASE_URI'] = u'sqlite:///{0}?timeout={1}'.format(
config.SQLITE_PATH.replace('\\', '/'), config.SQLITE_PATH.replace(u'\\', u'/'),
getattr(config, 'SQLITE_TIMEOUT', 500) getattr(config, 'SQLITE_TIMEOUT', 500)
) )

233
web/pgadmin/misc/bgprocess/process_executor.py
View File

@ -29,7 +29,7 @@ It also depends on the following environment variable for proper execution.
PROCID - Process-id PROCID - Process-id
OUTDIR - Output directory OUTDIR - Output directory
""" """
from __future__ import print_function, unicode_literals from __future__ import print_function
# To make print function compatible with python2 & python3 # To make print function compatible with python2 & python3
import sys import sys
@ -37,27 +37,31 @@ import os
from datetime import datetime, timedelta, tzinfo from datetime import datetime, timedelta, tzinfo
from subprocess import Popen, PIPE from subprocess import Popen, PIPE
from threading import Thread from threading import Thread
import codecs
import signal import signal
def log(msg): _IS_WIN = (os.name == 'nt')
if 'OUTDIR' not in os.environ: _IS_PY2 = (sys.version_info[0] == 2)
return _ZERO = timedelta(0)
_sys_encoding = None
_fs_encoding = None
_u = None
_out_dir = None
_log_file = None
with open( if _IS_PY2:
os.path.join(os.environ['OUTDIR'], ('log_%s' % os.getpid())), 'a' def _log(msg):
) as fp: with open(_log_file, 'a') as fp:
fp.write(('INFO:: %s\n' % str(msg))) fp.write(('INFO:: %s\n' % str(msg)))
else:
def _log(msg):
with open(_log_file, 'a') as fp:
fp.write(('INFO:: %s\n' % msg.encode('ascii', 'xmlcharrefreplace')))
def log_exception(): def _log_exception():
if 'OUTDIR' not in os.environ:
return
type_, value_, traceback_ = info=sys.exc_info() type_, value_, traceback_ = info=sys.exc_info()
with open( with open(_log_file, 'ab') as fp:
os.path.join(os.environ['OUTDIR'], ('log_%s' % os.getpid())), 'a'
) as fp:
from traceback import format_exception from traceback import format_exception
res = ''.join( res = ''.join(
format_exception(type_, value_, traceback_) format_exception(type_, value_, traceback_)
@ -67,11 +71,6 @@ def log_exception():
return res return res
IS_WIN = (os.name == 'nt')
ZERO = timedelta(0)
default_encoding = sys.getdefaultencoding() or "utf-8"
# Copied the 'UTC' class from the 'pytz' package to allow to run this script # Copied the 'UTC' class from the 'pytz' package to allow to run this script
# without any external dependent library, and can be used with any python # without any external dependent library, and can be used with any python
# version. # version.
@ -83,8 +82,8 @@ class UTC(tzinfo):
""" """
zone = "UTC" zone = "UTC"
_utcoffset = ZERO _utcoffset = _ZERO
_dst = ZERO _dst = _ZERO
_tzname = zone _tzname = zone
def fromutc(self, dt): def fromutc(self, dt):
@ -93,13 +92,13 @@ class UTC(tzinfo):
return super(UTC.__class__, self).fromutc(dt) return super(UTC.__class__, self).fromutc(dt)
def utcoffset(self, dt): def utcoffset(self, dt):
return ZERO return _ZERO
def tzname(self, dt): def tzname(self, dt):
return "UTC" return "UTC"
def dst(self, dt): def dst(self, dt):
return ZERO return _ZERO
def localize(self, dt, is_dst=False): def localize(self, dt, is_dst=False):
'''Convert naive time to local time''' '''Convert naive time to local time'''
@ -155,15 +154,12 @@ class ProcessLogger(Thread):
Returns: Returns:
None None
""" """
import codecs
Thread.__init__(self) Thread.__init__(self)
self.process = None self.process = None
self.stream = None self.stream = None
self.encoding = default_encoding self.logger = open(os.path.join(_out_dir, stream_type), 'wb')
self.logger = codecs.open(
os.path.join(
os.environ['OUTDIR'], stream_type
), 'w', self.encoding, "ignore"
)
def attach_process_stream(self, process, stream): def attach_process_stream(self, process, stream):
""" """
@ -179,27 +175,52 @@ class ProcessLogger(Thread):
self.process = process self.process = process
self.stream = stream self.stream = stream
def log(self, msg): if not _IS_PY2:
""" def log(self, msg):
This function will update log file """
This function will update log file
Args: Args:
msg: message msg: message
Returns: Returns:
None None
""" """
# Write into log file # Write into log file
if self.logger: if self.logger:
if msg: if msg:
self.logger.write( self.logger.write(get_current_time(format='%y%m%d%H%M%S%f').encode('utf-8'))
str('{0},{1}').format( self.logger.write(b',')
get_current_time(format='%y%m%d%H%M%S%f'), self.logger.write(msg.lstrip(b'\r\n' if _IS_WIN else b'\n'))
msg.decode(self.encoding, 'replace') self.logger.write(os.linesep.encode('utf-8'))
return True
return False
else:
def log(self, msg):
"""
This function will update log file
Args:
msg: message
Returns:
None
"""
# Write into log file
if self.logger:
if msg:
self.logger.write(
b'{0},{1}{2}'.format(
get_current_time(
format='%y%m%d%H%M%S%f'
),
msg.lstrip(b'\r\n' if _IS_WIN else b'\n'), os.linesep
)
) )
)
return True return True
return False return False
def run(self): def run(self):
if self.process and self.stream: if self.process and self.stream:
@ -230,14 +251,14 @@ def update_status(**kw):
""" """
import json import json
if os.environ['OUTDIR']: if _out_dir:
status = dict( status = dict(
(k, v) for k, v in kw.items() if k in [ (k, v) for k, v in kw.items() if k in [
'start_time', 'end_time', 'exit_code', 'pid' 'start_time', 'end_time', 'exit_code', 'pid'
] ]
) )
log('Updating the status:\n{0}'.format(json.dumps(status))) _log('Updating the status:\n{0}'.format(json.dumps(status)))
with open(os.path.join(os.environ['OUTDIR'], 'status'), 'w') as fp: with open(os.path.join(_out_dir, 'status'), 'w') as fp:
json.dump(status, fp) json.dump(status, fp)
else: else:
raise ValueError("Please verify pid and db_file arguments.") raise ValueError("Please verify pid and db_file arguments.")
@ -252,7 +273,7 @@ def execute():
""" """
command = sys.argv[1:] command = sys.argv[1:]
args = dict() args = dict()
log('Initialize the process execution: {0}'.format(command)) _log('Initialize the process execution: {0}'.format(command))
# Create seprate thread for stdout and stderr # Create seprate thread for stdout and stderr
process_stdout = ProcessLogger('out') process_stdout = ProcessLogger('out')
@ -270,25 +291,28 @@ def execute():
# Update start time # Update start time
update_status(**args) update_status(**args)
log('Status updated...') _log('Status updated...')
if 'PROCID' in os.environ and os.environ['PROCID'] in os.environ: if 'PROCID' in os.environ and os.environ['PROCID'] in os.environ:
os.environ['PGPASSWORD'] = os.environ[os.environ['PROCID']] os.environ['PGPASSWORD'] = os.environ[os.environ['PROCID']]
kwargs = dict() kwargs = dict()
kwargs['close_fds'] = False kwargs['close_fds'] = False
kwargs['shell'] = True if IS_WIN else False kwargs['shell'] = True if _IS_WIN else False
# We need environment variables & values in string # We need environment variables & values in string
log('Converting the environment variable in the bytes format...') if _IS_PY2:
kwargs['env'] = convert_environment_variables(os.environ.copy()) _log('Converting the environment variable in the bytes format...')
kwargs['env'] = convert_environment_variables(os.environ.copy())
else:
kwargs['env'] = os.environ.copy()
log('Starting the command execution...') _log('Starting the command execution...')
process = Popen( process = Popen(
command, stdout=PIPE, stderr=PIPE, stdin=None, **kwargs command, stdout=PIPE, stderr=PIPE, stdin=None, **kwargs
) )
log('Attaching the loggers to stdout, and stderr...') _log('Attaching the loggers to stdout, and stderr...')
# Attach the stream to the process logger, and start logging. # Attach the stream to the process logger, and start logging.
process_stdout.attach_process_stream(process, process.stdout) process_stdout.attach_process_stream(process, process.stdout)
process_stdout.start() process_stdout.start()
@ -299,14 +323,14 @@ def execute():
process_stdout.join() process_stdout.join()
process_stderr.join() process_stderr.join()
log('Waiting for the process to finish...') _log('Waiting for the process to finish...')
# Child process return code # Child process return code
exitCode = process.wait() exitCode = process.wait()
if exitCode is None: if exitCode is None:
exitCode = process.poll() exitCode = process.poll()
log('Process exited with code: {0}'.format(exitCode)) _log('Process exited with code: {0}'.format(exitCode))
args.update({'exit_code': exitCode}) args.update({'exit_code': exitCode})
# Add end_time # Add end_time
@ -322,7 +346,7 @@ def execute():
# If executable not found or invalid arguments passed # If executable not found or invalid arguments passed
except OSError: except OSError:
info = log_exception() info = _log_exception()
args.update({'exit_code': 500}) args.update({'exit_code': 500})
if process_stderr: if process_stderr:
process_stderr.log(info) process_stderr.log(info)
@ -333,7 +357,7 @@ def execute():
# Unknown errors # Unknown errors
except Exception: except Exception:
info = log_exception() info = _log_exception()
args.update({'exit_code': 501}) args.update({'exit_code': 501})
if process_stderr: if process_stderr:
process_stderr.log(info) process_stderr.log(info)
@ -344,12 +368,12 @@ def execute():
finally: finally:
# Update the execution end_time, and exit-code. # Update the execution end_time, and exit-code.
update_status(**args) update_status(**args)
log('Exiting the process executor...') _log('Exiting the process executor...')
if process_stderr: if process_stderr:
process_stderr.release() process_stderr.release()
if process_stdout: if process_stdout:
process_stdout.release() process_stdout.release()
log('Bye!') _log('Bye!')
# Let's ignore all the signal comming to us. # Let's ignore all the signal comming to us.
@ -368,27 +392,50 @@ def convert_environment_variables(env):
for key, value in env.items(): for key, value in env.items():
try: try:
if not isinstance(key, str): if not isinstance(key, str):
key = key.encode(default_encoding) key = key.encode(_sys_encoding)
if not isinstance(value, str): if not isinstance(value, str):
value = value.encode(default_encoding) value = value.encode(_sys_encoding)
temp_env[key] = value temp_env[key] = value
except Exception as e: except Exception as e:
log_exception() _log_exception()
return temp_env return temp_env
if __name__ == '__main__': if __name__ == '__main__':
log('Starting the process executor...')
_sys_encoding = sys.getdefaultencoding()
if not _sys_encoding or _sys_encoding == 'ascii':
# Fall back to 'utf-8', if we couldn't determine the default encoding,
# or 'ascii'.
_sys_encoding = 'utf-8'
_fs_encoding = sys.getfilesystemencoding()
if not _fs_encoding or _fs_encoding == 'ascii':
# Fall back to 'utf-8', if we couldn't determine the file-system encoding,
# or 'ascii'.
_fs_encoding = 'utf-8'
def u(_s, _encoding=_sys_encoding):
if _IS_PY2:
if isinstance(_s, str):
return unicode(_s, _encoding)
return _s
_u = u
_out_dir = u(os.environ['OUTDIR'])
_log_file = os.path.join(_out_dir, ('log_%s' % os.getpid()))
_log('Starting the process executor...')
# Ignore any signals # Ignore any signals
signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGTERM, signal_handler)
log('Disabled the SIGINT, SIGTERM signals...') _log('Disabled the SIGINT, SIGTERM signals...')
if IS_WIN: if _IS_WIN:
log('Disable the SIGBREAKM signal (windows)...') _log('Disable the SIGBREAKM signal (windows)...')
signal.signal(signal.SIGBREAK, signal_handler) signal.signal(signal.SIGBREAK, signal_handler)
log('Disabled the SIGBREAKM signal (windows)...') _log('Disabled the SIGBREAKM signal (windows)...')
# For windows: # For windows:
# We would run the process_executor in the detached mode again to make # We would run the process_executor in the detached mode again to make
@ -396,11 +443,14 @@ if __name__ == '__main__':
# depending on the status of the web-server. # depending on the status of the web-server.
if 'PGA_BGP_FOREGROUND' in os.environ and \ if 'PGA_BGP_FOREGROUND' in os.environ and \
os.environ['PGA_BGP_FOREGROUND'] == "1": os.environ['PGA_BGP_FOREGROUND'] == "1":
log('[CHILD] Start process execution...') _log('[CHILD] Start process execution...')
log('Executing the command now from the detached child...')
# This is a child process running as the daemon process. # This is a child process running as the daemon process.
# Let's do the job assing to it. # Let's do the job assigning to it.
execute() try:
_log('Executing the command now from the detached child...')
execute()
except:
_log_exception()
else: else:
from subprocess import CREATE_NEW_PROCESS_GROUP from subprocess import CREATE_NEW_PROCESS_GROUP
DETACHED_PROCESS = 0x00000008 DETACHED_PROCESS = 0x00000008
@ -414,11 +464,11 @@ if __name__ == '__main__':
env['PGA_BGP_FOREGROUND'] = "1" env['PGA_BGP_FOREGROUND'] = "1"
# We need environment variables & values in string # We need environment variables & values in string
log('[PARENT] Converting the environment variable in the bytes format...') _log('[PARENT] Converting the environment variable in the bytes format...')
try: try:
env = convert_environment_variables(env) env = convert_environment_variables(env)
except Exception as e: except Exception as e:
log_exception() _log_exception()
kwargs = { kwargs = {
'stdin': stdin.fileno(), 'stdin': stdin.fileno(),
@ -426,33 +476,33 @@ if __name__ == '__main__':
'stderr': stderr.fileno(), 'stderr': stderr.fileno(),
'creationflags': CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS, 'creationflags': CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS,
'close_fds': False, 'close_fds': False,
'cwd': os.environ['OUTDIR'], 'cwd': _out_dir,
'env': env 'env': env
} }
cmd = [sys.executable] cmd = [sys.executable]
cmd.extend(sys.argv) cmd.extend(sys.argv)
log('[PARENT] Command executings: {0}'.format(cmd)) _log('[PARENT] Command executings: {0}'.format(cmd))
p = Popen(cmd, **kwargs) p = Popen(cmd, **kwargs)
exitCode = p.poll() exitCode = p.poll()
if exitCode is not None: if exitCode is not None:
log( _log(
'[PARENT] Child exited with exit-code#{0}...'.format( '[PARENT] Child exited with exit-code#{0}...'.format(
exitCode exitCode
) )
) )
else: else:
log('[PARENT] Started the child with PID#{0}'.format(p.pid)) _log('[PARENT] Started the child with PID#{0}'.format(p.pid))
# Question: Should we wait for sometime? # Question: Should we wait for sometime?
# Answer: Looks the case... # Answer: Looks the case...
from time import sleep from time import sleep
sleep(2) sleep(2)
log('[PARENT] Exiting...') _log('[PARENT] Exiting...')
sys.exit(0) sys.exit(0)
else: else:
r, w = os.pipe() r, w = os.pipe()
@ -461,35 +511,36 @@ if __name__ == '__main__':
# We will fork the process, and run the child process as daemon, and # We will fork the process, and run the child process as daemon, and
# let it do the job. # let it do the job.
if os.fork() == 0: if os.fork() == 0:
log('[CHILD] Forked the child process...') _log('[CHILD] Forked the child process...')
# Hmm... So - I need to do the job now... # Hmm... So - I need to do the job now...
try: try:
os.close(r) os.close(r)
log('[CHILD] Make the child process leader...') _log('[CHILD] Make the child process leader...')
# Let me be the process leader first. # Let me be the process leader first.
os.setsid() os.setsid()
os.umask(0) os.umask(0)
log('[CHILD] Make the child process leader...') _log('[CHILD] Make the child process leader...')
w = os.fdopen(w, 'w') w = os.fdopen(w, 'w')
# Let me inform my parent - I will do the job, do not worry # Let me inform my parent - I will do the job, do not worry
# now, and die peacefully. # now, and die peacefully.
log('[CHILD] Inform parent about successful child forking...') _log('[CHILD] Inform parent about successful child forking...')
w.write('1') w.write('1')
w.close() w.close()
log('[CHILD] Start executing the background process...') _log('[CHILD] Start executing the background process...')
execute() execute()
except Exception: except Exception:
_log_exception()
sys.exit(1) sys.exit(1)
else: else:
os.close(w) os.close(w)
r = os.fdopen(r) r = os.fdopen(r)
# I do not care, what the child send. # I do not care, what the child send.
r.read() r.read()
log('[PARENT] Got message from the child...') _log('[PARENT] Got message from the child...')
r.close() r.close()
log('[PARENT] Exiting...') _log('[PARENT] Exiting...')
sys.exit(0) sys.exit(0)

90
web/pgadmin/misc/bgprocess/processes.py
View File

@ -11,8 +11,6 @@
""" """
Introduce a function to run the process executor in detached mode. Introduce a function to run the process executor in detached mode.
""" """
from __future__ import print_function, unicode_literals
import csv import csv
import os import os
import sys import sys
@ -21,6 +19,12 @@ from datetime import datetime
from pickle import dumps, loads from pickle import dumps, loads
from subprocess import Popen from subprocess import Popen
from pgadmin.utils import IS_PY2, u, file_quote, fs_encoding
if IS_PY2:
from StringIO import StringIO
else:
from io import StringIO
import pytz import pytz
from dateutil import parser from dateutil import parser
from flask import current_app from flask import current_app
@ -30,11 +34,6 @@ from flask_security import current_user
import config import config
from pgadmin.model import Process, db from pgadmin.model import Process, db
if sys.version_info < (3,):
from StringIO import StringIO
else:
from io import StringIO
def get_current_time(format='%Y-%m-%d %H:%M:%S.%f %z'): def get_current_time(format='%Y-%m-%d %H:%M:%S.%f %z'):
""" """
@ -159,17 +158,23 @@ class BatchProcess(object):
csv_writer = csv.writer( csv_writer = csv.writer(
args_csv_io, delimiter=str(','), quoting=csv.QUOTE_MINIMAL args_csv_io, delimiter=str(','), quoting=csv.QUOTE_MINIMAL
) )
csv_writer.writerow(_args) if sys.version_info.major == 2:
csv_writer.writerow([a.encode('utf-8') if isinstance(a, unicode) else a for a in _args])
else:
csv_writer.writerow(_args)
args_val = args_csv_io.getvalue().strip(str('\r\n'))
j = Process( j = Process(
pid=int(id), command=_cmd, pid=int(id), command=_cmd,
arguments=args_csv_io.getvalue().strip(str('\r\n')), arguments=args_val.decode('utf-8', 'replace') if IS_PY2 and hasattr(args_val, 'decode') \
else args_val,
logdir=log_dir, desc=dumps(self.desc), user_id=current_user.id logdir=log_dir, desc=dumps(self.desc), user_id=current_user.id
) )
db.session.add(j) db.session.add(j)
db.session.commit() db.session.commit()
def start(self): def start(self, cb=None):
def which(program, paths): def which(program, paths):
def is_exe(fpath): def is_exe(fpath):
@ -178,9 +183,9 @@ class BatchProcess(object):
for path in paths: for path in paths:
if not os.path.isdir(path): if not os.path.isdir(path):
continue continue
exe_file = os.path.join(path, program) exe_file = os.path.join(u(path, fs_encoding), program)
if is_exe(exe_file): if is_exe(exe_file):
return exe_file return file_quote(exe_file)
return None return None
def convert_environment_variables(env): def convert_environment_variables(env):
@ -191,6 +196,8 @@ class BatchProcess(object):
:return: Encoded environment variable as string :return: Encoded environment variable as string
""" """
encoding = sys.getdefaultencoding() encoding = sys.getdefaultencoding()
if encoding is None or encoding == 'ascii':
encoding = 'utf-8'
temp_env = dict() temp_env = dict()
for key, value in env.items(): for key, value in env.items():
if not isinstance(key, str): if not isinstance(key, str):
@ -207,22 +214,22 @@ class BatchProcess(object):
_('The process has already finished and can not be restarted.') _('The process has already finished and can not be restarted.')
) )
executor = os.path.join( executor = file_quote(os.path.join(
os.path.dirname(__file__), 'process_executor.py' os.path.dirname(u(__file__)), u'process_executor.py'
) ))
paths = sys.path[:] paths = sys.path[:]
interpreter = None interpreter = None
if os.name == 'nt': if os.name == 'nt':
paths.insert(0, os.path.join(sys.prefix, 'Scripts')) paths.insert(0, os.path.join(u(sys.prefix), u'Scripts'))
paths.insert(0, os.path.join(sys.prefix)) paths.insert(0, u(sys.prefix))
interpreter = which('pythonw.exe', paths) interpreter = which(u'pythonw.exe', paths)
if interpreter is None: if interpreter is None:
interpreter = which('python.exe', paths) interpreter = which(u'python.exe', paths)
else: else:
paths.insert(0, os.path.join(sys.prefix, 'bin')) paths.insert(0, os.path.join(u(sys.prefix), u'bin'))
interpreter = which('python', paths) interpreter = which(u'python', paths)
p = None p = None
cmd = [ cmd = [
@ -231,15 +238,21 @@ class BatchProcess(object):
] ]
cmd.extend(self.args) cmd.extend(self.args)
command = [] if os.name == 'nt' and IS_PY2:
for c in cmd: command = []
command.append(str(c)) for c in cmd:
command.append(c.encode('utf-8') if isinstance(c, unicode) else str(c))
current_app.logger.info( current_app.logger.info(
"Executing the process executor with the arguments: %s", u"Executing the process executor with the arguments: %s",
' '.join(command) ''.join(command)
) )
cmd = command
cmd = command
else:
current_app.logger.info(
u"Executing the process executor with the arguments: %s", str(cmd)
)
# Make a copy of environment, and add new variables to support # Make a copy of environment, and add new variables to support
env = os.environ.copy() env = os.environ.copy()
@ -247,8 +260,12 @@ class BatchProcess(object):
env['OUTDIR'] = self.log_dir env['OUTDIR'] = self.log_dir
env['PGA_BGP_FOREGROUND'] = "1" env['PGA_BGP_FOREGROUND'] = "1"
# We need environment variables & values in string if cb is not None:
env = convert_environment_variables(env) cb(env)
if IS_PY2:
# We need environment variables & values in string
env = convert_environment_variables(env)
if os.name == 'nt': if os.name == 'nt':
DETACHED_PROCESS = 0x00000008 DETACHED_PROCESS = 0x00000008
@ -325,12 +342,16 @@ class BatchProcess(object):
line = f.readline() line = f.readline()
line = line.decode(enc, 'replace') line = line.decode(enc, 'replace')
r = c.split(line) r = c.split(line)
if len(r) < 3:
# ignore this line
pos = f.tell()
continue
if r[1] > ctime: if r[1] > ctime:
completed = False completed = False
break break
log.append([r[1], r[2]]) log.append([r[1], r[2]])
pos = f.tell() pos = f.tell()
if idx == 1024: if idx >= 1024:
completed = False completed = False
break break
if pos == eofs: if pos == eofs:
@ -453,7 +474,10 @@ class BatchProcess(object):
if isinstance(desc, IProcessDesc): if isinstance(desc, IProcessDesc):
args = [] args = []
args_csv = StringIO(p.arguments) args_csv = StringIO(
p.arguments.encode('utf-8') \
if hasattr(p.arguments, 'decode') else p.arguments
)
args_reader = csv.reader(args_csv, delimiter=str(',')) args_reader = csv.reader(args_csv, delimiter=str(','))
for arg in args_reader: for arg in args_reader:
args = args + arg args = args + arg

109
web/pgadmin/tools/backup/__init__.py
View File

@ -9,6 +9,7 @@
"""Implements Backup Utility""" """Implements Backup Utility"""
from __future__ import unicode_literals
import simplejson as json import simplejson as json
import os import os
@ -17,7 +18,8 @@ from flask import render_template, request, current_app, \
from flask_babel import gettext as _ from flask_babel import gettext as _
from flask_security import login_required, current_user from flask_security import login_required, current_user
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
from pgadmin.utils import PgAdminModule, get_storage_directory, html from pgadmin.utils import PgAdminModule, get_storage_directory, html, \
fs_short_path, document_dir
from pgadmin.utils.ajax import make_json_response, bad_request from pgadmin.utils.ajax import make_json_response, bad_request
from config import PG_DEFAULT_DRIVER from config import PG_DEFAULT_DRIVER
@ -79,14 +81,28 @@ class BackupMessage(IProcessDesc):
Defines the message shown for the backup operation. Defines the message shown for the backup operation.
""" """
def __init__(self, _type, _sid, _bfile, **kwargs): def __init__(self, _type, _sid, _bfile, *_args, **_kwargs):
self.backup_type = _type self.backup_type = _type
self.sid = _sid self.sid = _sid
self.bfile = _bfile self.bfile = _bfile
self.database = None self.database = _kwargs['database'] if 'database' in _kwargs else None
self.cmd = ''
if 'database' in kwargs: def cmdArg(x):
self.database = kwargs['database'] if x:
# x = html.safe_str(x)
x = x.replace('\\', '\\\\')
x = x.replace('"', '\\"')
x = x.replace('""', '\\"')
return ' "' + x + '"'
return ''
for arg in _args:
if arg and len(arg) >= 2 and arg[:2] == '--':
self.cmd += ' ' + arg
else:
self.cmd += cmdArg(arg)
@property @property
def message(self): def message(self):
@ -123,65 +139,32 @@ class BackupMessage(IProcessDesc):
res = '<div class="h5">' res = '<div class="h5">'
if self.backup_type == BACKUP.OBJECT: if self.backup_type == BACKUP.OBJECT:
res += html.safe_str( res += _(
_( "Backing up an object on the server '{0}' from database '{1}'..."
"Backing up an object on the server '{0}' from database '{1}'..." ).format(
).format( "{0} ({1}:{2})".format(s.name, s.host, s.port),
"{0} ({1}:{2})".format(s.name, s.host, s.port), self.database
self.database
)
) )
elif self.backup_type == BACKUP.GLOBALS: elif self.backup_type == BACKUP.GLOBALS:
res += html.safe_str( res += _("Backing up the global objects on the server '{0}'").format(
_("Backing up the global objects on the server '{0}'").format( "{0} ({1}:{2})".format(s.name, s.host, s.port)
"{0} ({1}:{2})".format(s.name, s.host, s.port)
)
) )
elif self.backup_type == BACKUP.SERVER: elif self.backup_type == BACKUP.SERVER:
res += html.safe_str( res += _("Backing up the server '{0}'").format(
_("Backing up the server '{0}'").format( "{0} ({1}:{2})".format(s.name, s.host, s.port)
"{0} ({1}:{2})".format(s.name, s.host, s.port)
)
) )
else: else:
# It should never reach here. # It should never reach here.
res += "Backup" res += "Backup"
res += '</div><div class="h5">' res += '</div><div class="h5">'
res += html.safe_str( res += _("Running command:")
_("Running command:")
)
res += '</b><br><span class="pg-bg-cmd enable-selection">' res += '</b><br><span class="pg-bg-cmd enable-selection">'
res += html.safe_str(cmd) res += self.cmd
replace_next = False
def cmdArg(x):
if x:
x = x.replace('\\', '\\\\')
x = x.replace('"', '\\"')
x = x.replace('""', '\\"')
return ' "' + html.safe_str(x) + '"'
return ''
for arg in args:
if arg and len(arg) >= 2 and arg[:2] == '--':
res += ' ' + arg
elif replace_next:
res += ' "' + html.safe_str(
self.bfile
) + '"'
else:
if arg == '--file':
replace_next = True
res += cmdArg(arg)
res += '</span></div>' res += '</span></div>'
return res return res
@blueprint.route("/") @blueprint.route("/")
@login_required @login_required
def index(): def index():
@ -201,7 +184,7 @@ def script():
) )
def filename_with_file_manager_path(file): def filename_with_file_manager_path(_file):
""" """
Args: Args:
file: File name returned from client file manager file: File name returned from client file manager
@ -213,9 +196,15 @@ def filename_with_file_manager_path(file):
storage_dir = get_storage_directory() storage_dir = get_storage_directory()
if storage_dir: if storage_dir:
return os.path.join(storage_dir, file.lstrip('/')) _file = os.path.join(storage_dir, _file.lstrip(u'/').lstrip(u'\\'))
elif not os.path.isabs(_file):
_file = os.path.join(document_dir(), _file)
return file # Touch the file to get the short path of the file on windows.
with open(_file, 'a'):
pass
return fs_short_path(_file)
@blueprint.route('/create_job/<int:sid>', methods=['POST']) @blueprint.route('/create_job/<int:sid>', methods=['POST'])
@ -292,7 +281,11 @@ def create_backup_job(sid):
p = BatchProcess( p = BatchProcess(
desc=BackupMessage( desc=BackupMessage(
BACKUP.SERVER if data['type'] != 'global' else BACKUP.GLOBALS, BACKUP.SERVER if data['type'] != 'global' else BACKUP.GLOBALS,
sid, data['file'] sid,
data['file'].encode('utf-8') if hasattr(
data['file'], 'encode'
) else data['file'],
*args
), ),
cmd=utility, args=args cmd=utility, args=args
) )
@ -440,9 +433,15 @@ def create_backup_objects_job(sid):
try: try:
p = BatchProcess( p = BatchProcess(
desc=BackupMessage( desc=BackupMessage(
BACKUP.OBJECT, sid, data['file'], database=data['database'] BACKUP.OBJECT, sid,
data['file'].encode('utf-8') if hasattr(
data['file'], 'encode'
) else data['file'],
*args,
database=data['database']
), ),
cmd=utility, args=args) cmd=utility, args=args
)
manager.export_password_env(p.id) manager.export_password_env(p.id)
p.start() p.start()
jid = p.id jid = p.id

167
web/pgadmin/tools/import_export/__init__.py
View File

@ -16,7 +16,8 @@ from flask import url_for, Response, render_template, request, current_app
from flask_babel import gettext as _ from flask_babel import gettext as _
from flask_security import login_required, current_user from flask_security import login_required, current_user
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
from pgadmin.utils import PgAdminModule, get_storage_directory, html from pgadmin.utils import PgAdminModule, get_storage_directory, html, \
fs_short_path, document_dir, IS_WIN
from pgadmin.utils.ajax import make_json_response, bad_request from pgadmin.utils.ajax import make_json_response, bad_request
from config import PG_DEFAULT_DRIVER from config import PG_DEFAULT_DRIVER
@ -56,19 +57,47 @@ class ImportExportModule(PgAdminModule):
blueprint = ImportExportModule(MODULE_NAME, __name__) blueprint = ImportExportModule(MODULE_NAME, __name__)
class Message(IProcessDesc): class IEMessage(IProcessDesc):
""" """
Message(IProcessDesc) IEMessage(IProcessDesc)
Defines the message shown for the Message operation. Defines the message shown for the import/export operation.
""" """
def __init__(self, _sid, _schema, _tbl, _database, _storage): def __init__(self, _sid, _schema, _tbl, _database, _storage, *_args):
self.sid = _sid self.sid = _sid
self.schema = _schema self.schema = _schema
self.table = _tbl self.table = _tbl
self.database = _database self.database = _database
self.storage = _storage self._cmd = ''
if _storage:
_storage = _storage.replace('\\', '/')
def cmdArg(x):
if x:
x = html.safe_str(x)
x = x.replace('\\', '\\\\')
x = x.replace('"', '\\"')
x = x.replace('""', '\\"')
return ' "' + x + '"'
return ''
replace_next = False
for arg in _args:
if arg and len(arg) >= 2 and arg[:2] == '--':
if arg == '--command':
replace_next = True
self._cmd += ' ' + arg
elif replace_next:
arg = cmdArg(arg)
if _storage is not None:
arg = arg.replace(_storage, '<STORAGE_DIR>')
self._cmd += ' "' + arg + '"'
else:
self._cmd+= cmdArg(arg)
@property @property
def message(self): def message(self):
@ -90,45 +119,17 @@ class Message(IProcessDesc):
).first() ).first()
res = '<div class="h5">' res = '<div class="h5">'
res += html.safe_str( res += _(
_( "Copying table data '{0}.{1}' on database '{2}' for the server - '{3}'"
"Copying table data '{0}.{1}' on database '{2}' for the server - '{3}'" ).format(
).format( self.schema, self.table, self.database,
self.schema, self.table, self.database, "{0} ({1}:{2})".format(s.name, s.host, s.port)
"{0} ({1}:{2})".format(s.name, s.host, s.port)
)
) )
res += '</div><div class="h5">' res += '</div><div class="h5">'
res += html.safe_str( res += _("Running command:")
_("Running command:")
)
res += '</b><br><span class="pg-bg-cmd enable-selection">' res += '</b><br><span class="pg-bg-cmd enable-selection">'
res += html.safe_str(cmd) res += self._cmd
replace_next = False
def cmdArg(x):
if x:
x = x.replace('\\', '\\\\')
x = x.replace('"', '\\"')
x = x.replace('""', '\\"')
return ' "' + html.safe_str(x) + '"'
return ''
for arg in args:
if arg and len(arg) >= 2 and arg[:2] == '--':
if arg == '--command':
replace_next = True
res += ' ' + arg
elif replace_next:
if self.storage:
arg = arg.replace(self.storage, '<STORAGE_DIR>')
res += ' "' + html.safe_str(arg) + '"'
else:
res += cmdArg(arg)
res += '</span></div>' res += '</span></div>'
return res return res
@ -151,6 +152,33 @@ def script():
) )
def filename_with_file_manager_path(_file, _present=False):
"""
Args:
file: File name returned from client file manager
Returns:
Filename to use for backup with full path taken from preference
"""
# Set file manager directory from preference
storage_dir = get_storage_directory()
if storage_dir:
_file = os.path.join(storage_dir, _file.lstrip(u'/').lstrip(u'\\'))
elif not os.path.isabs(_file):
_file = os.path.join(document_dir(), _file)
if not _present:
# Touch the file to get the short path of the file on windows.
with open(_file, 'a'):
pass
else:
if not os.path.isfile(_file):
return None
return fs_short_path(_file)
@blueprint.route('/create_job/<int:sid>', methods=['POST']) @blueprint.route('/create_job/<int:sid>', methods=['POST'])
@login_required @login_required
def create_import_export_job(sid): def create_import_export_job(sid):
@ -175,10 +203,8 @@ def create_import_export_job(sid):
id=sid).first() id=sid).first()
if server is None: if server is None:
return make_json_response( return bad_request(errormsg=_("Couldn't find the given server"))
success=0,
errormsg=_("Couldn't find the given server")
)
# To fetch MetaData for the server # To fetch MetaData for the server
from pgadmin.utils.driver import get_driver from pgadmin.utils.driver import get_driver
@ -188,10 +214,7 @@ def create_import_export_job(sid):
connected = conn.connected() connected = conn.connected()
if not connected: if not connected:
return make_json_response( return bad_request(errormsg=_("Please connect to the server first..."))
success=0,
errormsg=_("Please connect to the server first...")
)
# Get the utility path from the connection manager # Get the utility path from the connection manager
utility = manager.utility('sql') utility = manager.utility('sql')
@ -200,21 +223,16 @@ def create_import_export_job(sid):
storage_dir = get_storage_directory() storage_dir = get_storage_directory()
if 'filename' in data: if 'filename' in data:
if os.name == 'nt': _file = filename_with_file_manager_path(data['filename'], data['is_import'])
data['filename'] = data['filename'].replace('/', '\\') if not _file:
if storage_dir: return bad_request(errormsg=_('Please specify a valid file'))
storage_dir = storage_dir.replace('/', '\\')
data['filename'] = data['filename'].replace('\\', '\\\\')
data['filename'] = os.path.join(storage_dir, data['filename'].lstrip('/'))
elif storage_dir:
data['filename'] = os.path.join(storage_dir, data['filename'].lstrip('/'))
else:
data['filename'] = data['filename']
if IS_WIN:
_file = _file.replace('\\', '/')
data['filename'] = _file
else: else:
return make_json_response( return bad_request(errormsg=_('Please specify a valid file'))
data={'status': False, 'info': 'Please specify a valid file'}
)
cols = None cols = None
icols = None icols = None
@ -255,33 +273,32 @@ def create_import_export_job(sid):
ignore_column_list=icols ignore_column_list=icols
) )
args = [ args = ['--command', query]
'--host', server.host, '--port', str(server.port),
'--username', server.username, '--dbname', data['database'],
'--command', query
]
try: try:
p = BatchProcess( p = BatchProcess(
desc=Message( desc=IEMessage(
sid, sid,
data['schema'], data['schema'],
data['table'], data['table'],
data['database'], data['database'],
storage_dir storage_dir,
utility, *args
), ),
cmd=utility, args=args cmd=utility, args=args
) )
manager.export_password_env(p.id) manager.export_password_env(p.id)
p.start() def export_pg_env(env):
env['PGHOST'] = server.host
env['PGPORT'] = str(server.port)
env['PGUSER'] = server.username
env['PGDATABASE'] = data['database']
p.start(export_pg_env)
jid = p.id jid = p.id
except Exception as e: except Exception as e:
current_app.logger.exception(e) current_app.logger.exception(e)
return make_json_response( return bad_request(errormsg=str(e))
status=410,
success=0,
errormsg=str(e)
)
# Return response # Return response
return make_json_response( return make_json_response(

78
web/pgadmin/tools/restore/__init__.py
View File

@ -17,7 +17,8 @@ from flask import render_template, request, current_app, \
from flask_babel import gettext as _ from flask_babel import gettext as _
from flask_security import login_required, current_user from flask_security import login_required, current_user
from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc from pgadmin.misc.bgprocess.processes import BatchProcess, IProcessDesc
from pgadmin.utils import PgAdminModule, get_storage_directory, html from pgadmin.utils import PgAdminModule, get_storage_directory, html, \
fs_short_path, document_dir
from pgadmin.utils.ajax import make_json_response, bad_request from pgadmin.utils.ajax import make_json_response, bad_request
from config import PG_DEFAULT_DRIVER from config import PG_DEFAULT_DRIVER
@ -58,9 +59,26 @@ blueprint = RestoreModule(
class RestoreMessage(IProcessDesc): class RestoreMessage(IProcessDesc):
def __init__(self, _sid, _bfile): def __init__(self, _sid, _bfile, *_args):
self.sid = _sid self.sid = _sid
self.bfile = _bfile self.bfile = _bfile
self.cmd = ''
def cmdArg(x):
if x:
x = html.safe_str(x)
x = x.replace('\\', '\\\\')
x = x.replace('"', '\\"')
x = x.replace('""', '\\"')
return ' "' + x + '"'
return ''
for arg in _args:
if arg and len(arg) >= 2 and arg[:2] == '--':
self.cmd += ' ' + arg
else:
self.cmd += cmdArg(arg)
@property @property
def message(self): def message(self):
@ -95,30 +113,7 @@ class RestoreMessage(IProcessDesc):
) )
res += '</b><br><span class="pg-bg-cmd enable-selection">' res += '</b><br><span class="pg-bg-cmd enable-selection">'
res += html.safe_str(cmd) res += html.safe_str(cmd)
res += self.cmd
def cmdArg(x):
if x:
x = x.replace('\\', '\\\\')
x = x.replace('"', '\\"')
x = x.replace('""', '\\"')
return ' "' + html.safe_str(x) + '"'
return ''
idx = 0
no_args = len(args)
for arg in args:
if idx < no_args - 1:
if arg[:2] == '--':
res += ' ' + arg
else:
res += cmdArg(arg)
idx += 1
if no_args > 1:
res += ' "' + html.safe_str(arg) + '"'
res += '</span></div>' res += '</span></div>'
return res return res
@ -143,7 +138,7 @@ def script():
) )
def filename_with_file_manager_path(file): def filename_with_file_manager_path(_file):
""" """
Args: Args:
file: File name returned from client file manager file: File name returned from client file manager
@ -155,9 +150,14 @@ def filename_with_file_manager_path(file):
storage_dir = get_storage_directory() storage_dir = get_storage_directory()
if storage_dir: if storage_dir:
return os.path.join(storage_dir, file.lstrip('/')) _file = os.path.join(storage_dir, _file.lstrip(u'/').lstrip(u'\\'))
elif not os.path.isabs(_file):
_file = os.path.join(document_dir(), _file)
return file if not os.path.isfile(_file):
return None
return fs_short_path(_file)
@blueprint.route('/create_job/<int:sid>', methods=['POST']) @blueprint.route('/create_job/<int:sid>', methods=['POST'])
@ -179,7 +179,13 @@ def create_restore_job(sid):
else: else:
data = json.loads(request.data, encoding='utf-8') data = json.loads(request.data, encoding='utf-8')
backup_file = filename_with_file_manager_path(data['file']) _file = filename_with_file_manager_path(data['file'])
if _file is None:
return make_json_response(
success=0,
errormsg=_("File couldn't be found!")
)
# Fetch the server details like hostname, port, roles etc # Fetch the server details like hostname, port, roles etc
server = Server.query.filter_by( server = Server.query.filter_by(
@ -261,7 +267,7 @@ def create_restore_job(sid):
return False return False
args.extend([ args.extend([
'--host', server.host, '--port', server.port, '--host', server.host, '--port', str(server.port),
'--username', server.username, '--no-password' '--username', server.username, '--no-password'
]) ])
@ -300,11 +306,17 @@ def create_restore_job(sid):
set_multiple('trigger_funcs', '--function') set_multiple('trigger_funcs', '--function')
set_multiple('indexes', '--index') set_multiple('indexes', '--index')
args.append(backup_file) args.append(fs_short_path(_file))
try: try:
p = BatchProcess( p = BatchProcess(
desc=RestoreMessage(sid, data['file']), desc=RestoreMessage(
sid,
data['file'].encode('utf-8') if hasattr(
data['file'], 'encode'
) else data['file'],
*args
),
cmd=utility, args=args cmd=utility, args=args
) )
manager.export_password_env(p.id) manager.export_password_env(p.id)

102
web/pgadmin/utils/__init__.py
View File

@ -131,3 +131,105 @@ class PgAdminModule(Blueprint):
menu_items = dict((key, sorted(value, key=attrgetter('priority'))) menu_items = dict((key, sorted(value, key=attrgetter('priority')))
for key, value in menu_items.items()) for key, value in menu_items.items())
return menu_items return menu_items
import os
import sys
IS_PY2 = (sys.version_info[0] == 2)
IS_WIN = (os.name == 'nt')
sys_encoding = sys.getdefaultencoding()
if not sys_encoding or sys_encoding == 'ascii':
# Fall back to 'utf-8', if we couldn't determine the default encoding,
# or 'ascii'.
sys_encoding = 'utf-8'
fs_encoding = sys.getfilesystemencoding()
if not fs_encoding or fs_encoding == 'ascii':
# Fall back to 'utf-8', if we couldn't determine the file-system encoding,
# or 'ascii'.
fs_encoding = 'utf-8'
def u(_s, _encoding=sys_encoding):
if IS_PY2:
if isinstance(_s, str):
return unicode(_s, _encoding)
return _s
def file_quote(_p):
if IS_PY2:
if isinstance(_p, unicode):
return _p.encode(fs_encoding)
return _p
if IS_WIN:
import ctypes
from ctypes import wintypes
if IS_PY2:
def env(name):
if IS_PY2:
# Make sure string argument is unicode
name = unicode(name)
n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0)
if n == 0:
return None
buf= ctypes.create_unicode_buffer(u'\0'*n)
ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n)
return buf.value
else:
def env(name):
if name in os.environ:
return os.environ[name]
return None
_GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW
_GetShortPathNameW.argtypes = [
wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD
]
_GetShortPathNameW.restype = wintypes.DWORD
def fs_short_path(_path):
"""
Gets the short path name of a given long path.
http://stackoverflow.com/a/23598461/200291
"""
buf_size = len(_path)
while True:
res = ctypes.create_unicode_buffer(buf_size)
needed = _GetShortPathNameW(_path, res, buf_size)
if buf_size >= needed:
return res.value
else:
buf_size += needed
def document_dir():
CSIDL_PERSONAL = 5 # My Documents
SHGFP_TYPE_CURRENT = 0 # Get current, not default value
buf = ctypes.create_unicode_buffer(wintypes.MAX_PATH)
ctypes.windll.shell32.SHGetFolderPathW(
None, CSIDL_PERSONAL, None, SHGFP_TYPE_CURRENT, buf
)
return buf.value
else:
def env(name):
if name in os.environ:
return os.environ[name]
return None
def fs_short_path(_path):
return _path
def document_dir():
return os.path.realpath(os.path.expanduser(u'~/'))

10
web/pgadmin/utils/driver/psycopg2/__init__.py
View File

@ -1491,8 +1491,6 @@ class ServerManager(object):
database = self.db database = self.db
elif did in self.db_info: elif did in self.db_info:
database = self.db_info[did]['datname'] database = self.db_info[did]['datname']
if hasattr(str, 'decode'):
database = database.decode('utf-8')
else: else:
maintenance_db_id = u'DB:{0}'.format(self.db) maintenance_db_id = u'DB:{0}'.format(self.db)
if maintenance_db_id in self.connections: if maintenance_db_id in self.connections:
@ -1510,9 +1508,6 @@ WHERE db.oid = {0}""".format(did))
if status and len(res['rows']) > 0: if status and len(res['rows']) > 0:
for row in res['rows']: for row in res['rows']:
self.db_info[did] = row self.db_info[did] = row
if hasattr(str, 'decode'):
self.db_info[did]['datname'] = \
self.db_info[did]['datname'].decode('utf-8')
database = self.db_info[did]['datname'] database = self.db_info[did]['datname']
if did not in self.db_info: if did not in self.db_info:
@ -1843,7 +1838,10 @@ class Driver(BaseDriver):
# Returns in bytes, we need to convert it in string # Returns in bytes, we need to convert it in string
if isinstance(res, bytes): if isinstance(res, bytes):
try: try:
res = res.decode() try:
res = res.decode()
except UnicodeDecodeError:
res = res.decode(sys.getfilesystemencoding())
except UnicodeDecodeError: except UnicodeDecodeError:
res = res.decode('utf-8') res = res.decode('utf-8')

11
web/pgadmin/utils/html.py
View File

@ -10,9 +10,14 @@
"""Utilities for HTML""" """Utilities for HTML"""
import cgi import cgi
from pgadmin.utils import IS_PY2
def safe_str(x): def safe_str(x):
return cgi.escape(x).encode( try:
'ascii', 'xmlcharrefreplace' x = x.encode('ascii', 'xmlcharrefreplace') if hasattr(x, 'encode') else x
).decode() if not IS_PY2:
x = x.decode('utf-8')
except:
pass
return cgi.escape(x)

6
web/pgadmin/utils/paths.py
View File

@ -37,7 +37,11 @@ def get_storage_directory():
if len(username) == 0 or username[0].isdigit(): if len(username) == 0 or username[0].isdigit():
username = 'pga_user_' + username username = 'pga_user_' + username
storage_dir = os.path.join(storage_dir, username) storage_dir = os.path.join(
storage_dir.decode('utf-8') if hasattr(storage_dir, 'decode') \
else storage_dir,
username
)
if not os.path.exists(storage_dir): if not os.path.exists(storage_dir):
os.makedirs(storage_dir, int('700', 8)) os.makedirs(storage_dir, int('700', 8))

2
web/pgadmin/utils/preferences.py
View File

@ -193,7 +193,7 @@ class _Preference(object):
if pref is None: if pref is None:
pref = UserPrefTable( pref = UserPrefTable(
uid=current_user.id, pid=self.pid, value=str(value) uid=current_user.id, pid=self.pid, value=value
) )
db.session.add(pref) db.session.add(pref)
else: else:

5
web/pgadmin/utils/session.py
View File

@ -143,7 +143,10 @@ class CachingSessionManager(SessionManager):
def put(self, session): def put(self, session):
self.parent.put(session) self.parent.put(session)
if session.sid in self._cache: if session.sid in self._cache:
del self._cache[session.sid] try:
del self._cache[session.sid]
except:
pass
self._cache[session.sid] = session self._cache[session.sid] = session
self._normalize() self._normalize()

71
web/setup.py
View File

@ -22,14 +22,20 @@ from flask import Flask
from flask_security import Security, SQLAlchemyUserDatastore from flask_security import Security, SQLAlchemyUserDatastore
from flask_security.utils import encrypt_password from flask_security.utils import encrypt_password
from pgadmin.model import db, Role, User, Server, \ # We need to include the root directory in sys.path to ensure that we can
ServerGroup, Version, Keys # find everything we need when running in the standalone runtime.
root = os.path.dirname(os.path.realpath(__file__))
if sys.path[0] != root:
sys.path.insert(0, root)
# Configuration settings # Configuration settings
import config import config
# Get the config database schema version. We store this in pgadmin.model # Get the config database schema version. We store this in pgadmin.model
# as it turns out that putting it in the config files isn't a great idea # as it turns out that putting it in the config files isn't a great idea
from pgadmin.model import SCHEMA_VERSION from pgadmin.model import db, Role, User, Server, ServerGroup, Version, Keys, \
SCHEMA_VERSION
from pgadmin.utils.versioned_template_loader import VersionedTemplateLoader from pgadmin.utils.versioned_template_loader import VersionedTemplateLoader
config.SETTINGS_SCHEMA_VERSION = SCHEMA_VERSION config.SETTINGS_SCHEMA_VERSION = SCHEMA_VERSION
@ -44,7 +50,7 @@ def do_setup(app):
"""Create a new settings database from scratch""" """Create a new settings database from scratch"""
if config.SERVER_MODE is False: if config.SERVER_MODE is False:
print("NOTE: Configuring authentication for DESKTOP mode.") print(u"NOTE: Configuring authentication for DESKTOP mode.")
email = config.DESKTOP_USER email = config.DESKTOP_USER
p1 = ''.join([ p1 = ''.join([
random.choice(string.ascii_letters + string.digits) random.choice(string.ascii_letters + string.digits)
@ -52,7 +58,7 @@ def do_setup(app):
]) ])
else: else:
print("NOTE: Configuring authentication for SERVER mode.\n") print(u"NOTE: Configuring authentication for SERVER mode.\n")
if all(value in os.environ for value in if all(value in os.environ for value in
['PGADMIN_SETUP_EMAIL', 'PGADMIN_SETUP_PASSWORD']): ['PGADMIN_SETUP_EMAIL', 'PGADMIN_SETUP_PASSWORD']):
@ -64,9 +70,11 @@ def do_setup(app):
p1 = os.environ['PGADMIN_SETUP_PASSWORD'] p1 = os.environ['PGADMIN_SETUP_PASSWORD']
else: else:
# Prompt the user for their default username and password. # Prompt the user for their default username and password.
print(""" print(
Enter the email address and password to use for the initial pgAdmin user \ u"Enter the email address and password to use for the initial "
account:\n""") u"pgAdmin user account:\n"
)
email_filter = re.compile( email_filter = re.compile(
"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9]" "^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9]"
"(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9]" "(?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9]"
@ -74,7 +82,7 @@ def do_setup(app):
email = input("Email address: ") email = input("Email address: ")
while email == '' or not email_filter.match(email): while email == '' or not email_filter.match(email):
print('Invalid email address. Please try again.') print(u'Invalid email address. Please try again.')
email = input("Email address: ") email = input("Email address: ")
def pprompt(): def pprompt():
@ -83,10 +91,11 @@ def do_setup(app):
p1, p2 = pprompt() p1, p2 = pprompt()
while p1 != p2 or len(p1) < 6: while p1 != p2 or len(p1) < 6:
if p1 != p2: if p1 != p2:
print('Passwords do not match. Please try again.') print(u'Passwords do not match. Please try again.')
else: else:
print( print(
'Password must be at least 6 characters. Please try again.') u'Password must be at least 6 characters. Please try again.'
)
p1, p2 = pprompt() p1, p2 = pprompt()
# Setup Flask-Security # Setup Flask-Security
@ -134,9 +143,9 @@ def do_setup(app):
db.session.commit() db.session.commit()
# Done! # Done!
print("") print(u"")
print( print(
"The configuration database has been created at {0}".format( u"The configuration database has been created at {0}".format(
config.SQLITE_PATH config.SQLITE_PATH
) )
) )
@ -154,19 +163,19 @@ def do_upgrade(app, datastore, version):
# Pre-flight checks # Pre-flight checks
if int(version.value) > int(config.SETTINGS_SCHEMA_VERSION): if int(version.value) > int(config.SETTINGS_SCHEMA_VERSION):
print(""" print(u"""
The database schema version is {0}, whilst the version required by the \ The database schema version is {0}, whilst the version required by the \
software is {1}. software is {1}.
Exiting...""".format(version.value, config.SETTINGS_SCHEMA_VERSION)) Exiting...""".format(version.value, config.SETTINGS_SCHEMA_VERSION))
sys.exit(1) sys.exit(1)
elif int(version.value) == int(config.SETTINGS_SCHEMA_VERSION): elif int(version.value) == int(config.SETTINGS_SCHEMA_VERSION):
print(""" print(u"""
The database schema version is {0} as required. The database schema version is {0} as required.
Exiting...""".format(version.value)) Exiting...""".format(version.value))
sys.exit(1) sys.exit(1)
app.logger.info( app.logger.info(
"NOTE: Upgrading database schema from version %d to %d." % u"NOTE: Upgrading database schema from version %d to %d." %
(version.value, config.SETTINGS_SCHEMA_VERSION) (version.value, config.SETTINGS_SCHEMA_VERSION)
) )
@ -394,19 +403,24 @@ if __name__ == '__main__':
'sqlite:///' + config.SQLITE_PATH.replace('\\', '/') 'sqlite:///' + config.SQLITE_PATH.replace('\\', '/')
db.init_app(app) db.init_app(app)
print("pgAdmin 4 - Application Initialisation") print(u"pgAdmin 4 - Application Initialisation")
print("======================================\n") print(u"======================================\n")
from pgadmin.utils import u, fs_encoding, file_quote
local_config = os.path.join( local_config = os.path.join(
os.path.dirname(os.path.realpath(__file__)), os.path.dirname(os.path.realpath(u(__file__, fs_encoding))),
'config_local.py' u'config_local.py'
) )
# Check if the database exists. If it does, tell the user and exit. # Check if the database exists. If it does, tell the user and exit.
if os.path.isfile(config.SQLITE_PATH): if os.path.isfile(config.SQLITE_PATH):
print(""" print(
The configuration database '%s' already exists. u"The configuration database '{0}' already exists.".format(
Entering upgrade mode...""" % config.SQLITE_PATH) config.SQLITE_PATH
)
)
print(u"Entering upgrade mode...")
# Setup Flask-Security # Setup Flask-Security
user_datastore = SQLAlchemyUserDatastore(db, User, Role) user_datastore = SQLAlchemyUserDatastore(db, User, Role)
@ -417,18 +431,18 @@ Entering upgrade mode...""" % config.SQLITE_PATH)
# Pre-flight checks # Pre-flight checks
if int(version.value) > int(config.SETTINGS_SCHEMA_VERSION): if int(version.value) > int(config.SETTINGS_SCHEMA_VERSION):
print(""" print(u"""
The database schema version is %d, whilst the version required by the \ The database schema version is %d, whilst the version required by the \
software is %d. software is %d.
Exiting...""" % (version.value, config.SETTINGS_SCHEMA_VERSION)) Exiting...""" % (version.value, config.SETTINGS_SCHEMA_VERSION))
sys.exit(1) sys.exit(1)
elif int(version.value) == int(config.SETTINGS_SCHEMA_VERSION): elif int(version.value) == int(config.SETTINGS_SCHEMA_VERSION):
print(""" print(u"""
The database schema version is %d as required. The database schema version is %d as required.
Exiting...""" % (version.value)) Exiting...""" % (version.value))
sys.exit(1) sys.exit(1)
print("NOTE: Upgrading database schema from version %d to %d." % ( print(u"NOTE: Upgrading database schema from version %d to %d." % (
version.value, config.SETTINGS_SCHEMA_VERSION version.value, config.SETTINGS_SCHEMA_VERSION
)) ))
do_upgrade(app, user_datastore, version) do_upgrade(app, user_datastore, version)
@ -441,12 +455,15 @@ Exiting...""" % (version.value))
app.config.from_object(config) app.config.from_object(config)
directory = os.path.dirname(config.SQLITE_PATH) directory = os.path.dirname(config.SQLITE_PATH)
if not os.path.exists(directory): if not os.path.exists(directory):
os.makedirs(directory, int('700', 8)) os.makedirs(directory, int('700', 8))
db_file = os.open(config.SQLITE_PATH, os.O_CREAT, int('600', 8)) db_file = os.open(config.SQLITE_PATH, os.O_CREAT, int('600', 8))
os.close(db_file) os.close(db_file)
print(""" print(u"""
The configuration database - '{0}' does not exist. The configuration database - '{0}' does not exist.
Entering initial setup mode...""".format(config.SQLITE_PATH)) Entering initial setup mode...""".format(config.SQLITE_PATH))
do_setup(app) do_setup(app)