Added support for Azure PostgreSQL deployment in server mode. Fixes #7522

This commit is contained in:
Yogesh Mahajan 2022-07-06 11:43:49 +05:30 committed by Akshay Joshi
parent 64e6700228
commit 59f5c0d955
12 changed files with 267 additions and 134 deletions

View File

@ -10,8 +10,6 @@ To deploy a PostgreSQL server on the Azure cloud, follow the below steps.
:alt: Cloud Deployment
:align: center
**Note:** This feature is currently available in Desktop mode only.
Once you launch the tool, select the Azure PostgreSQL option.
Click on the *Next* button to proceed further.

View File

@ -23,4 +23,5 @@ Bug fixes
| `Issue #7517 <https://redmine.postgresql.org/issues/7517>`_ - Enable the start debugging button once execution is completed.
| `Issue #7518 <https://redmine.postgresql.org/issues/7518>`_ - Ensure that dashboard graph API is not called after the panel has been closed.
| `Issue #7523 <https://redmine.postgresql.org/issues/7517>`_ - Fixed typo error for Statistics on the table header.
| `Issue #7522 <https://redmine.postgresql.org/issues/7522>`_ - Added support for Azure PostgreSQL deployment in server mode.
| `Issue #7523 <https://redmine.postgresql.org/issues/7523>`_ - Fixed typo error for Statistics on the table header.

View File

@ -8,12 +8,11 @@
##########################################################################
""" Azure PostgreSQL provider """
from azure.mgmt.rdbms.postgresql_flexibleservers import \
PostgreSQLManagementClient
from azure.mgmt.rdbms.postgresql_flexibleservers.models import Sku, SkuTier, \
CreateMode, Storage, Server, FirewallRule, HighAvailability
from azure.identity import AzureCliCredential, InteractiveBrowserCredential, \
from azure.identity import AzureCliCredential, DeviceCodeCredential, \
AuthenticationRecord
from azure.mgmt.resource import ResourceManagementClient
from azure.core.exceptions import ResourceNotFoundError
@ -37,12 +36,13 @@ class AzureProvider(AbsProvider):
self._client_secret = None
self._subscription_id = None
self._default_region = None
self._use_interactive_browser_credential = False
self._interactive_browser_credential = False
self._available_capabilities = None
self._credentials = None
self._authentication_record_json = None
self._cli_credentials = None
self.azure_cred_cache_name = None
self._azure_cred_cache_name = None
self._azure_cred_cache_location = None
# Get the credentials
if 'AUTHENTICATION_RECORD_JSON' in os.environ:
@ -56,7 +56,7 @@ class AzureProvider(AbsProvider):
self._tenant_id = os.environ['AZURE_TENANT_ID']
if 'AUTH_TYPE' in os.environ:
self._use_interactive_browser_credential = False \
self._interactive_browser_credential = False \
if os.environ['AUTH_TYPE'] == 'azure_cli_credential' else True
if 'AZURE_DATABASE_PASSWORD' in os.environ:
@ -150,49 +150,48 @@ class AzureProvider(AbsProvider):
##########################################################################
def _get_azure_credentials(self):
try:
if self._use_interactive_browser_credential:
if self._authentication_record_json is None:
_credentials = self._azure_interactive_browser_credential()
_auth_record_ = _credentials.authenticate()
self._authentication_record_json = \
_auth_record_.serialize()
else:
deserialized_auth_record = AuthenticationRecord.\
deserialize(self._authentication_record_json)
_credentials = \
self._azure_interactive_browser_credential(
deserialized_auth_record)
if self._interactive_browser_credential:
_credentials = self._azure_interactive_auth()
else:
if self._cli_credentials is None:
self._cli_credentials = AzureCliCredential()
_credentials = self._cli_credentials
_credentials = self._azure_cli_auth()
except Exception as e:
return False, str(e)
return True, _credentials
def _azure_interactive_browser_credential(
self, deserialized_auth_record=None):
if deserialized_auth_record:
_credential = InteractiveBrowserCredential(
def _azure_cli_auth(self):
if self._cli_credentials is None:
self._cli_credentials = AzureCliCredential()
return self._cli_credentials
def _azure_interactive_auth(self):
if self._authentication_record_json is None:
_interactive_credential = DeviceCodeCredential(
tenant_id=self._tenant_id,
timeout=180,
_cache=load_persistent_cache(
TokenCachePersistenceOptions(
name=self._azure_cred_cache_name,
allow_unencrypted_storage=True,
cache_location=self._azure_cred_cache_location)),
authentication_record=deserialized_auth_record)
else:
_credential = InteractiveBrowserCredential(
tenant_id=self._tenant_id,
timeout=180,
_cache=load_persistent_cache(
TokenCachePersistenceOptions(
name=self._azure_cred_cache_name,
allow_unencrypted_storage=True,
cache_location=self._azure_cred_cache_location))
prompt_callback=None,
_cache=load_persistent_cache(TokenCachePersistenceOptions(
name=self._azure_cred_cache_name,
allow_unencrypted_storage=True,
cache_location=self._azure_cred_cache_location)
)
)
return _credential
_auth_record = _interactive_credential.authenticate()
self._authentication_record_json = _auth_record.serialize()
else:
deserialized_auth_record = AuthenticationRecord.deserialize(
self._authentication_record_json)
_interactive_credential = DeviceCodeCredential(
tenant_id=self._tenant_id,
timeout=180,
prompt_callback=None,
_cache=load_persistent_cache(TokenCachePersistenceOptions(
name=self._azure_cred_cache_name,
allow_unencrypted_storage=True,
cache_location=self._azure_cred_cache_location)
),
authentication_record=deserialized_auth_record
)
return _interactive_credential
def _get_azure_client(self, type):
""" Create/cache/return an Azure client object """

View File

@ -507,6 +507,7 @@ class BatchProcess(object):
err = 0
cloud_server_id = 0
cloud_instance = ''
pid = self.id
enc = sys.getdefaultencoding()
if enc == 'ascii':
@ -519,7 +520,7 @@ class BatchProcess(object):
self.stderr, stderr, err, ctime, _process.exit_code, enc
)
from pgadmin.misc.cloud import update_server
from pgadmin.misc.cloud import update_server, clear_cloud_session
if out_completed and not _process.exit_code:
for value in stdout:
if 'instance' in value[1] and value[1] != '':
@ -530,12 +531,16 @@ class BatchProcess(object):
'instance' in cloud_instance:
cloud_instance['instance']['sid'] = cloud_server_id
cloud_instance['instance']['status'] = True
cloud_instance['instance']['pid'] = pid
return update_server(cloud_instance)
elif err_completed and _process.exit_code > 0:
cloud_instance = {'instance': {}}
cloud_instance['instance']['sid'] = _process.server_id
cloud_instance['instance']['status'] = False
cloud_instance['instance']['pid'] = pid
return update_server(cloud_instance)
else:
clear_cloud_session(pid)
return True, {}
def status(self, out=0, err=0):

View File

@ -134,11 +134,7 @@ def deploy_on_cloud():
elif data['cloud'] == 'biganimal':
status, resp = deploy_on_biganimal(data)
elif data['cloud'] == 'azure':
if config.SERVER_MODE:
status = False
resp = gettext('Invalid Operation for Server mode.')
else:
status, resp = deploy_on_azure(data)
status, resp = deploy_on_azure(data)
else:
status = False
resp = gettext('No cloud implementation.')
@ -172,6 +168,7 @@ def deploy_on_cloud():
def update_server(data):
"""Update Server."""
server_data = data
pid = data['instance']['pid']
server = Server.query.filter_by(
user_id=current_user.id,
id=server_data['instance']['sid']
@ -204,16 +201,16 @@ def update_server(data):
_server['status'] = False
else:
_server['status'] = True
clear_cloud_session()
clear_cloud_session(pid)
return True, _server
def clear_cloud_session():
def clear_cloud_session(pid=None):
"""Clear cloud sessions."""
clear_aws_session()
clear_biganimal_session()
clear_azure_session()
clear_azure_session(pid)
@blueprint.route(

View File

@ -8,9 +8,8 @@
# ##########################################################################
# Azure implementation
import random
import config
import random
from pgadmin.misc.cloud.utils import _create_server, CloudProcessDesc
from pgadmin.misc.bgprocess.processes import BatchProcess
from pgadmin import make_json_response
@ -26,7 +25,7 @@ import os
from azure.mgmt.rdbms.postgresql_flexibleservers import \
PostgreSQLManagementClient
from azure.identity import AzureCliCredential, InteractiveBrowserCredential,\
from azure.identity import AzureCliCredential, DeviceCodeCredential,\
AuthenticationRecord
from azure.mgmt.resource import ResourceManagementClient
from azure.mgmt.subscription import SubscriptionClient
@ -57,7 +56,8 @@ class AzurePostgresqlModule(PgAdminModule):
'azure.db_versions',
'azure.instance_types',
'azure.availability_zones',
'azure.storage_types']
'azure.storage_types',
'azure.get_azure_verification_codes']
blueprint = AzurePostgresqlModule(MODULE_NAME, __name__,
@ -101,6 +101,20 @@ def verify_credentials():
return make_json_response(success=status, errormsg=error)
@blueprint.route('/get_azure_verification_codes/',
methods=['GET'], endpoint='get_azure_verification_codes')
@login_required
def get_azure_verification_codes():
"""Get azure code for authentication."""
azure_auth_code = None
status = False
if 'azure' in session and 'azure_auth_code' in session['azure']:
azure_auth_code = session['azure']['azure_auth_code']
status = True
return make_json_response(success=status,
data=azure_auth_code)
@blueprint.route('/check_cluster_name_availability/',
methods=['GET'], endpoint='check_cluster_name_availability')
@login_required
@ -237,8 +251,7 @@ class Azure:
self._clients = {}
self._tenant_id = tenant_id
self._session_token = session_token
self._use_interactive_browser_credential = \
interactive_browser_credential
self._use_interactive_credential = interactive_browser_credential
self.authentication_record_json = None
self._cli_credentials = None
self._credentials = None
@ -246,72 +259,74 @@ class Azure:
self.subscription_id = None
self._availability_zone = None
self._available_capabilities_list = []
self.cache_name = current_user.username + "_msal.cache"
self.azure_cache_name = current_user.username \
+ str(random.randint(1, 9999)) + "_msal.cache"
self.azure_cache_location = config.AZURE_CREDENTIAL_CACHE_DIR + '/'
##########################################################################
# Azure Helper functions
##########################################################################
def validate_azure_credentials(self):
"""
Validates azure credentials
:return: True if valid credentials else false
"""
status, identity = self._get_azure_credentials()
session['azure']['azure_cache_file_name'] = self.cache_name
session['azure']['azure_cache_file_name'] = self.azure_cache_name
error = ''
if not status:
error = identity
return status, error
def _get_azure_credentials(self):
"""
Gets azure credentials depending on
self._use_interactive_browser_credential
:return:
"""
try:
if self._use_interactive_browser_credential:
if self.authentication_record_json is None:
_credentials = self._azure_interactive_browser_credential()
_auth_record_ = _credentials.authenticate()
self.authentication_record_json = _auth_record_.serialize()
else:
deserialized_auth_record = AuthenticationRecord. \
deserialize(self.authentication_record_json)
_credentials = \
self._azure_interactive_browser_credential(
deserialized_auth_record)
if self._use_interactive_credential:
_credentials = self._azure_interactive_auth()
else:
if self._cli_credentials is None:
self._cli_credentials = AzureCliCredential()
self.list_subscriptions()
_credentials = self._cli_credentials
_credentials = self._azure_cli_auth()
except Exception as e:
return False, str(e)
return True, _credentials
def _azure_interactive_browser_credential(
self, deserialized_auth_record=None):
if deserialized_auth_record:
_credential = InteractiveBrowserCredential(
tenant_id=self._tenant_id,
timeout=180,
_cache=load_persistent_cache(
TokenCachePersistenceOptions(
name=self.cache_name,
allow_unencrypted_storage=True)),
authentication_record=deserialized_auth_record)
else:
_credential = InteractiveBrowserCredential(
def _azure_cli_auth(self):
if self._cli_credentials is None:
self._cli_credentials = AzureCliCredential()
self.list_subscriptions()
return self._cli_credentials
@staticmethod
def _azure_interactive_auth_prompt_callback(
verification_uri, user_code, expires_at):
azure_auth_code = {'verification_uri': verification_uri,
'user_code': user_code,
'expires_at': expires_at}
session['azure']['azure_auth_code'] = azure_auth_code
def _azure_interactive_auth(self):
if self.authentication_record_json is None:
_interactive_credential = DeviceCodeCredential(
tenant_id=self._tenant_id,
timeout=180,
prompt_callback=self._azure_interactive_auth_prompt_callback,
_cache=load_persistent_cache(TokenCachePersistenceOptions(
name=self.cache_name,
allow_unencrypted_storage=True))
name=self.azure_cache_name, allow_unencrypted_storage=True)
)
)
return _credential
_auth_record = _interactive_credential.authenticate()
self.authentication_record_json = _auth_record.serialize()
else:
deserialized_auth_record = AuthenticationRecord.deserialize(
self.authentication_record_json)
_interactive_credential = DeviceCodeCredential(
tenant_id=self._tenant_id,
timeout=180,
prompt_callback=self._azure_interactive_auth_prompt_callback,
_cache=load_persistent_cache(TokenCachePersistenceOptions(
name=self.azure_cache_name, allow_unencrypted_storage=True)
),
authentication_record=deserialized_auth_record
)
return _interactive_credential
def _get_azure_client(self, type):
""" Create/cache/return an Azure client object """
@ -684,7 +699,7 @@ def deploy_on_azure(data):
azure = session['azure']['azure_obj']
env['AZURE_SUBSCRIPTION_ID'] = azure.subscription_id
env['AUTH_TYPE'] = data['secret']['auth_type']
env['AZURE_CRED_CACHE_NAME'] = azure.cache_name
env['AZURE_CRED_CACHE_NAME'] = azure.azure_cache_name
env['AZURE_CRED_CACHE_LOCATION'] = azure.azure_cache_location
if azure.authentication_record_json is not None:
env['AUTHENTICATION_RECORD_JSON'] = \
@ -698,6 +713,14 @@ def deploy_on_azure(data):
p.set_env_variables(None, env=env)
p.update_server_id(p.id, sid)
p.start()
# add pid: cache file dict in session['azure_cache_files_list']
if 'azure_cache_files_list' in session and \
session['azure_cache_files_list'] is not None:
session['azure_cache_files_list'][p.id] = azure.azure_cache_name
else:
session['azure_cache_files_list'] = {p.id: azure.azure_cache_name}
return True, {'label': _label, 'sid': sid}
except Exception as e:
current_app.logger.exception(e)
@ -706,12 +729,30 @@ def deploy_on_azure(data):
del session['azure']['azure_obj']
def clear_azure_session():
def clear_azure_session(pid=None):
"""Clear session data."""
cache_file_to_delete = None
if 'azure_cache_files_list' in session and \
pid in session['azure_cache_files_list']:
cache_file_to_delete = session['azure_cache_files_list'][pid]
delete_azure_cache(cache_file_to_delete)
del session['azure_cache_files_list'][pid]
if 'azure' in session:
file_name = session['azure']['azure_cache_file_name']
file = config.AZURE_CREDENTIAL_CACHE_DIR + '/' + file_name
# Delete cache file if exists
if os.path.exists(file):
os.remove(file)
if cache_file_to_delete is None and \
'azure_cache_file_name' in session['azure']:
cache_file_to_delete = session['azure']['azure_cache_file_name']
delete_azure_cache(cache_file_to_delete)
session.pop('azure')
def delete_azure_cache(file_name):
"""
Delete specified file from azure cache directory
:param file_name:
:return:
"""
file = config.AZURE_CREDENTIAL_CACHE_DIR + '/' + file_name
# Delete cache file if exists
if os.path.exists(file):
os.remove(file)

View File

@ -325,10 +325,9 @@ export default function CloudWizard({ nodeInfo, nodeData }) {
setErrMsg([]);
});
let cloud_providers = [{label: 'Amazon RDS', value: 'rds', icon: <AWSIcon className={classes.icon} />}, {label: 'EDB BigAnimal', value: 'biganimal', icon: <BigAnimalIcon className={classes.icon} />}];
if (pgAdmin.server_mode == 'False'){
cloud_providers.push({'label': 'Azure PostgreSQL', value: 'azure', icon: <AzureIcon className={classes.icon} /> });
}
let cloud_providers = [{label: 'Amazon RDS', value: 'rds', icon: <AWSIcon className={classes.icon} />},
{label: 'EDB BigAnimal', value: 'biganimal', icon: <BigAnimalIcon className={classes.icon} />},
{'label': 'Azure PostgreSQL', value: 'azure', icon: <AzureIcon className={classes.icon} /> }];
return (
<CloudWizardEventsContext.Provider value={eventBus.current}>
@ -367,7 +366,7 @@ export default function CloudWizard({ nodeInfo, nodeData }) {
</Box>}
</Box>
{cloudProvider == 'rds' && <AwsCredentials cloudProvider={cloudProvider} nodeInfo={nodeInfo} nodeData={nodeData} setCloudDBCred={setCloudDBCred}/>}
<Box>
<Box flexGrow={1}>
{cloudProvider == 'azure' && <AzureCredentials cloudProvider={cloudProvider} nodeInfo={nodeInfo} nodeData={nodeData} setAzureCredData={setAzureCredData}/>}
</Box>
<FormFooterMessage type={errMsg[0]} message={errMsg[1]} onClose={onErrClose} />

View File

@ -53,7 +53,28 @@ export function AzureCredentials(props) {
.catch((error) => {
_eventBus.fireEvent('SET_ERROR_MESSAGE_FOR_CLOUD_WIZARD',[MESSAGE_TYPE.ERROR, gettext(`Error while verification Microsoft Azure: ${error.response.data.errormsg}`)]);
reject(false);
});});
});
});
},
getAuthCode:()=>{
let _url_get_azure_verification_codes = url_for('azure.get_azure_verification_codes');
const axiosApi = getApiInstance();
return new Promise((resolve, reject)=>{
const interval = setInterval(()=>{
axiosApi.get(_url_get_azure_verification_codes)
.then((res)=>{
if (res.data.success){
clearInterval(interval);
window.open(res.data.data.verification_uri, 'azure_authentication');
resolve(res);
}
})
.catch((error)=>{
clearInterval(interval);
reject(error);
});
}, 1000);
});
}
});
setCloudDBCredInstance(azureCloudDBCredSchema);

View File

@ -21,6 +21,9 @@ class AzureCredSchema extends BaseUISchema {
auth_type: 'interactive_browser_credential',
azure_tenant_id: '',
azure_subscription_id: '',
is_authenticating: false,
is_authenticated: false,
auth_code: '',
...initValues,
});
@ -97,30 +100,83 @@ class AzureCredSchema extends BaseUISchema {
id: 'auth_btn',
mode: ['create'],
deps: ['auth_type', 'azure_tenant_id'],
type: 'button',
btnName: gettext('Click here to authenticate yourself to Microsoft Azure'),
type: (state) => {
return {
type: 'button',
onClick: () => {
obj.fieldOptions.authenticateAzure(state.auth_type, state.azure_tenant_id).then((res)=>{state._disabled_auth_btn= res;});
},
};
},
helpMessage: gettext(
'After clicking the button above you will be redirected to the Microsoft Azure authentication page in a new browser tab if the Interactive Browser option is selected.'
),
depChange: (state, source)=> {
if(source[0] == 'auth_type' || source[0] == 'azure_tenant_id'){
state._disabled_auth_btn = false;
if(source == 'auth_type' || source == 'azure_tenant_id'){
return {is_authenticated: false, auth_code: ''};
}
if(source == 'auth_btn') {
return {is_authenticating: true};
}
},
deferredDepChange: (state, source)=>{
return new Promise((resolve, reject)=>{
/* button clicked */
if(source == 'auth_btn') {
obj.fieldOptions.authenticateAzure(state.auth_type, state.azure_tenant_id)
.then(()=>{
resolve(()=>({
is_authenticated: true,
is_authenticating: false,
auth_code: ''
}));
})
.catch((err)=>{
reject(err);
});
}
});
},
disabled: (state)=> {
if(state.auth_type == 'interactive_browser_credential' && state.azure_tenant_id == ''){
return true;
}
return state._disabled_auth_btn;
return state.is_authenticating || state.is_authenticated;
},
},
{
id: 'is_authenticating',
visible: false,
type: '',
deps:['auth_btn'],
deferredDepChange: (state, source)=>{
return new Promise((resolve, reject)=>{
if(source == 'auth_btn' && state.auth_type == 'interactive_browser_credential' && state.is_authenticating ) {
obj.fieldOptions.getAuthCode()
.then((res)=>{
resolve(()=>{
return {
is_authenticating: false,
auth_code: res.data.data.user_code,
};
});
})
.catch((err)=>{
reject(err);
});
}
});
},
},
{
id: 'auth_code',
mode: ['create'],
deps: ['auth_btn'],
type: (state)=>({
type: 'note',
text: `To complete the authenticatation, use a web browser to open the page https://microsoft.com/devicelogin and enter the code : <strong>${state.auth_code}</strong>`,
}),
visible: (state)=>{
return Boolean(state.auth_code);
},
controlProps: {
raw: true,
}
}
];
}
}

View File

@ -224,10 +224,22 @@ export const MappedFormControl = (props) => {
newProps.type = typeProps;
}
let origOnClick = newProps.onClick;
newProps.onClick = ()=>{
origOnClick?.();
/* Consider on click as change for button.
Just increase state val by 1 to inform the deps and self depChange */
newProps.onChange?.((newProps.state[props.id]||0)+1);
};
/* Filter out garbage props if any using ALLOWED_PROPS_FIELD */
return <MappedFormControlBase {..._.pick(newProps, _.union(ALLOWED_PROPS_FIELD_COMMON, ALLOWED_PROPS_FIELD_FORM))} />;
};
MappedFormControl.propTypes = {
id: PropTypes.oneOfType([PropTypes.string, PropTypes.number]).isRequired
};
export const MappedCellControl = (props) => {
let newProps = { ...props };
let cellProps = evalFunc(null, newProps.cell, newProps.row);

View File

@ -494,20 +494,22 @@ function SchemaDialogView({
useEffect(()=>{
if(sessData.__deferred__?.length > 0) {
let items = sessData.__deferred__;
sessDispatch({
type: SCHEMA_STATE_ACTIONS.CLEAR_DEFERRED_QUEUE,
});
let item = sessData.__deferred__[0];
item.promise.then((resFunc)=>{
sessDispatch({
type: SCHEMA_STATE_ACTIONS.DEFERRED_DEPCHANGE,
path: item.action.path,
depChange: item.action.depChange,
listener: {
...item.listener,
callback: resFunc,
},
items.forEach((item)=>{
item.promise.then((resFunc)=>{
sessDispatch({
type: SCHEMA_STATE_ACTIONS.DEFERRED_DEPCHANGE,
path: item.action.path,
depChange: item.action.depChange,
listener: {
...item.listener,
callback: resFunc,
},
});
});
});
}

View File

@ -1118,12 +1118,13 @@ PlainString.propTypes = {
value: PropTypes.any,
};
export function FormNote({ text, className }) {
export function FormNote({ text, className, controlProps }) {
const classes = useStyles();
/* If raw, then remove the styles and icon */
return (
<Box className={className}>
<Paper elevation={0} className={classes.noteRoot}>
<Box paddingRight="0.25rem"><DescriptionIcon fontSize="small" /></Box>
<Paper elevation={0} className={controlProps?.raw ? '' : classes.noteRoot}>
{!controlProps?.raw && <Box paddingRight="0.25rem"><DescriptionIcon fontSize="small" /></Box>}
<Box>{HTMLReactParse(text || '')}</Box>
</Paper>
</Box>
@ -1132,6 +1133,7 @@ export function FormNote({ text, className }) {
FormNote.propTypes = {
text: PropTypes.string,
className: CustomPropTypes.className,
controlProps: PropTypes.object,
};
const useStylesFormFooter = makeStyles((theme) => ({